Lib/test/test_parser.py

   1 import parser
   2 import os
   3 import unittest
   4 import sys
   5 from test import test_support
   6
   7 #
   8 #  First, we test that we can generate trees from valid source fragments,
   9 #  and that these valid trees are indeed allowed by the tree-loading side
  10 #  of the parser module.
  11 #
  12
  13 class RoundtripLegalSyntaxTestCase(unittest.TestCase):
  14
  15     def roundtrip(self, f, s):
  16         st1 = f(s)
  17         t = st1.totuple()
  18         try:
  19             st2 = parser.sequence2st(t)
  20         except parser.ParserError, why:
  21             self.fail("could not roundtrip %r: %s" % (s, why))
  22
  23         self.assertEquals(t, st2.totuple(),
  24                           "could not re-generate syntax tree")
  25
  26     def check_expr(self, s):
  27         self.roundtrip(parser.expr, s)
  28
  29     def test_flags_passed(self):
  30         # The unicode literals flags has to be passed from the paser to AST
  31         # generation.
  32         suite = parser.suite("from __future__ import unicode_literals; x = ''")
  33         code = suite.compile()
  34         scope = {}
  35         exec code in scope
  36         self.assertTrue(isinstance(scope["x"], unicode))
  37
  38     def check_suite(self, s):
  39         self.roundtrip(parser.suite, s)
  40
  41     def test_yield_statement(self):
  42         self.check_suite("def f(): yield 1")
  43         self.check_suite("def f(): yield")
  44         self.check_suite("def f(): x += yield")
  45         self.check_suite("def f(): x = yield 1")
  46         self.check_suite("def f(): x = y = yield 1")
  47         self.check_suite("def f(): x = yield")
  48         self.check_suite("def f(): x = y = yield")
  49         self.check_suite("def f(): 1 + (yield)*2")
  50         self.check_suite("def f(): (yield 1)*2")
  51         self.check_suite("def f(): return; yield 1")
  52         self.check_suite("def f(): yield 1; return")
  53         self.check_suite("def f():\n"
  54                          "    for x in range(30):\n"
  55                          "        yield x\n")
  56         self.check_suite("def f():\n"
  57                          "    if (yield):\n"
  58                          "        yield x\n")
  59
  60     def test_expressions(self):
  61         self.check_expr("foo(1)")
  62         self.check_expr("[1, 2, 3]")
  63         self.check_expr("[x**3 for x in range(20)]")
  64         self.check_expr("[x**3 for x in range(20) if x % 3]")
  65         self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]")
  66         self.check_expr("list(x**3 for x in range(20))")
  67         self.check_expr("list(x**3 for x in range(20) if x % 3)")
  68         self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)")
  69         self.check_expr("foo(*args)")
  70         self.check_expr("foo(*args, **kw)")
  71         self.check_expr("foo(**kw)")
  72         self.check_expr("foo(key=value)")
  73         self.check_expr("foo(key=value, *args)")
  74         self.check_expr("foo(key=value, *args, **kw)")
  75         self.check_expr("foo(key=value, **kw)")
  76         self.check_expr("foo(a, b, c, *args)")
  77         self.check_expr("foo(a, b, c, *args, **kw)")
  78         self.check_expr("foo(a, b, c, **kw)")
  79         self.check_expr("foo(a, *args, keyword=23)")
  80         self.check_expr("foo + bar")
  81         self.check_expr("foo - bar")
  82         self.check_expr("foo * bar")
  83         self.check_expr("foo / bar")
  84         self.check_expr("foo // bar")
  85         self.check_expr("lambda: 0")
  86         self.check_expr("lambda x: 0")
  87         self.check_expr("lambda *y: 0")
  88         self.check_expr("lambda *y, **z: 0")
  89         self.check_expr("lambda **z: 0")
  90         self.check_expr("lambda x, y: 0")
  91         self.check_expr("lambda foo=bar: 0")
  92         self.check_expr("lambda foo=bar, spaz=nifty+spit: 0")
  93         self.check_expr("lambda foo=bar, **z: 0")
  94         self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0")
  95         self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0")
  96         self.check_expr("lambda x, *y, **z: 0")
  97         self.check_expr("(x for x in range(10))")
  98         self.check_expr("foo(x for x in range(10))")
  99
 100     def test_print(self):
 101         self.check_suite("print")
 102         self.check_suite("print 1")
 103         self.check_suite("print 1,")
 104         self.check_suite("print >>fp")
 105         self.check_suite("print >>fp, 1")
 106         self.check_suite("print >>fp, 1,")
 107
 108     def test_simple_expression(self):
 109         # expr_stmt
 110         self.check_suite("a")
 111
 112     def test_simple_assignments(self):
 113         self.check_suite("a = b")
 114         self.check_suite("a = b = c = d = e")
 115
 116     def test_simple_augmented_assignments(self):
 117         self.check_suite("a += b")
 118         self.check_suite("a -= b")
 119         self.check_suite("a *= b")
 120         self.check_suite("a /= b")
 121         self.check_suite("a //= b")
 122         self.check_suite("a %= b")
 123         self.check_suite("a &= b")
 124         self.check_suite("a |= b")
 125         self.check_suite("a ^= b")
 126         self.check_suite("a <<= b")
 127         self.check_suite("a >>= b")
 128         self.check_suite("a **= b")
 129
 130     def test_function_defs(self):
 131         self.check_suite("def f(): pass")
 132         self.check_suite("def f(*args): pass")
 133         self.check_suite("def f(*args, **kw): pass")
 134         self.check_suite("def f(**kw): pass")
 135         self.check_suite("def f(foo=bar): pass")
 136         self.check_suite("def f(foo=bar, *args): pass")
 137         self.check_suite("def f(foo=bar, *args, **kw): pass")
 138         self.check_suite("def f(foo=bar, **kw): pass")
 139
 140         self.check_suite("def f(a, b): pass")
 141         self.check_suite("def f(a, b, *args): pass")
 142         self.check_suite("def f(a, b, *args, **kw): pass")
 143         self.check_suite("def f(a, b, **kw): pass")
 144         self.check_suite("def f(a, b, foo=bar): pass")
 145         self.check_suite("def f(a, b, foo=bar, *args): pass")
 146         self.check_suite("def f(a, b, foo=bar, *args, **kw): pass")
 147         self.check_suite("def f(a, b, foo=bar, **kw): pass")
 148
 149         self.check_suite("@staticmethod\n"
 150                          "def f(): pass")
 151         self.check_suite("@staticmethod\n"
 152                          "@funcattrs(x, y)\n"
 153                          "def f(): pass")
 154         self.check_suite("@funcattrs()\n"
 155                          "def f(): pass")
 156
 157     def test_class_defs(self):
 158         self.check_suite("class foo():pass")
 159
 160     def test_import_from_statement(self):
 161         self.check_suite("from sys.path import *")
 162         self.check_suite("from sys.path import dirname")
 163         self.check_suite("from sys.path import (dirname)")
 164         self.check_suite("from sys.path import (dirname,)")
 165         self.check_suite("from sys.path import dirname as my_dirname")
 166         self.check_suite("from sys.path import (dirname as my_dirname)")
 167         self.check_suite("from sys.path import (dirname as my_dirname,)")
 168         self.check_suite("from sys.path import dirname, basename")
 169         self.check_suite("from sys.path import (dirname, basename)")
 170         self.check_suite("from sys.path import (dirname, basename,)")
 171         self.check_suite(
 172             "from sys.path import dirname as my_dirname, basename")
 173         self.check_suite(
 174             "from sys.path import (dirname as my_dirname, basename)")
 175         self.check_suite(
 176             "from sys.path import (dirname as my_dirname, basename,)")
 177         self.check_suite(
 178             "from sys.path import dirname, basename as my_basename")
 179         self.check_suite(
 180             "from sys.path import (dirname, basename as my_basename)")
 181         self.check_suite(
 182             "from sys.path import (dirname, basename as my_basename,)")
 183         self.check_suite("from .bogus import x")
 184
 185     def test_basic_import_statement(self):
 186         self.check_suite("import sys")
 187         self.check_suite("import sys as system")
 188         self.check_suite("import sys, math")
 189         self.check_suite("import sys as system, math")
 190         self.check_suite("import sys, math as my_math")
 191
 192     def test_pep263(self):
 193         self.check_suite("# -*- coding: iso-8859-1 -*-\n"
 194                          "pass\n")
 195
 196     def test_assert(self):
 197         self.check_suite("assert alo < ahi and blo < bhi\n")
 198
 199     def test_with(self):
 200         self.check_suite("with open('x'): pass\n")
 201         self.check_suite("with open('x') as f: pass\n")
 202         self.check_suite("with open('x') as f, open('y') as g: pass\n")
 203
 204     def test_try_stmt(self):
 205         self.check_suite("try: pass\nexcept: pass\n")
 206         self.check_suite("try: pass\nfinally: pass\n")
 207         self.check_suite("try: pass\nexcept A: pass\nfinally: pass\n")
 208         self.check_suite("try: pass\nexcept A: pass\nexcept: pass\n"
 209                          "finally: pass\n")
 210         self.check_suite("try: pass\nexcept: pass\nelse: pass\n")
 211         self.check_suite("try: pass\nexcept: pass\nelse: pass\n"
 212                          "finally: pass\n")
 213
 214     def test_position(self):
 215         # An absolutely minimal test of position information.  Better
 216         # tests would be a big project.
 217         code = "def f(x):\n    return x + 1\n"
 218         st1 = parser.suite(code)
 219         st2 = st1.totuple(line_info=1, col_info=1)
 220
 221         def walk(tree):
 222             node_type = tree[0]
 223             next = tree[1]
 224             if isinstance(next, tuple):
 225                 for elt in tree[1:]:
 226                     for x in walk(elt):
 227                         yield x
 228             else:
 229                 yield tree
 230
 231         terminals = list(walk(st2))
 232         self.assertEqual([
 233             (1, 'def', 1, 0),
 234             (1, 'f', 1, 4),
 235             (7, '(', 1, 5),
 236             (1, 'x', 1, 6),
 237             (8, ')', 1, 7),
 238             (11, ':', 1, 8),
 239             (4, '', 1, 9),
 240             (5, '', 2, -1),
 241             (1, 'return', 2, 4),
 242             (1, 'x', 2, 11),
 243             (14, '+', 2, 13),
 244             (2, '1', 2, 15),
 245             (4, '', 2, 16),
 246             (6, '', 2, -1),
 247             (4, '', 2, -1),
 248             (0, '', 2, -1)],
 249                          terminals)
 250
 251
 252 #
 253 #  Second, we take *invalid* trees and make sure we get ParserError
 254 #  rejections for them.
 255 #
 256
 257 class IllegalSyntaxTestCase(unittest.TestCase):
 258
 259     def check_bad_tree(self, tree, label):
 260         try:
 261             parser.sequence2st(tree)
 262         except parser.ParserError:
 263             pass
 264         else:
 265             self.fail("did not detect invalid tree for %r" % label)
 266
 267     def test_junk(self):
 268         # not even remotely valid:
 269         self.check_bad_tree((1, 2, 3), "<junk>")
 270
 271     def test_illegal_yield_1(self):
 272         # Illegal yield statement: def f(): return 1; yield 1
 273         tree = \
 274         (257,
 275          (264,
 276           (285,
 277            (259,
 278             (1, 'def'),
 279             (1, 'f'),
 280             (260, (7, '('), (8, ')')),
 281             (11, ':'),
 282             (291,
 283              (4, ''),
 284              (5, ''),
 285              (264,
 286               (265,
 287                (266,
 288                 (272,
 289                  (275,
 290                   (1, 'return'),
 291                   (313,
 292                    (292,
 293                     (293,
 294                      (294,
 295                       (295,
 296                        (297,
 297                         (298,
 298                          (299,
 299                           (300,
 300                            (301,
 301                             (302, (303, (304, (305, (2, '1')))))))))))))))))),
 302                (264,
 303                 (265,
 304                  (266,
 305                   (272,
 306                    (276,
 307                     (1, 'yield'),
 308                     (313,
 309                      (292,
 310                       (293,
 311                        (294,
 312                         (295,
 313                          (297,
 314                           (298,
 315                            (299,
 316                             (300,
 317                              (301,
 318                               (302,
 319                                (303, (304, (305, (2, '1')))))))))))))))))),
 320                  (4, ''))),
 321                (6, ''))))),
 322            (4, ''),
 323            (0, ''))))
 324         self.check_bad_tree(tree, "def f():\n  return 1\n  yield 1")
 325
 326     def test_illegal_yield_2(self):
 327         # Illegal return in generator: def f(): return 1; yield 1
 328         tree = \
 329         (257,
 330          (264,
 331           (265,
 332            (266,
 333             (278,
 334              (1, 'from'),
 335              (281, (1, '__future__')),
 336              (1, 'import'),
 337              (279, (1, 'generators')))),
 338            (4, ''))),
 339          (264,
 340           (285,
 341            (259,
 342             (1, 'def'),
 343             (1, 'f'),
 344             (260, (7, '('), (8, ')')),
 345             (11, ':'),
 346             (291,
 347              (4, ''),
 348              (5, ''),
 349              (264,
 350               (265,
 351                (266,
 352                 (272,
 353                  (275,
 354                   (1, 'return'),
 355                   (313,
 356                    (292,
 357                     (293,
 358                      (294,
 359                       (295,
 360                        (297,
 361                         (298,
 362                          (299,
 363                           (300,
 364                            (301,
 365                             (302, (303, (304, (305, (2, '1')))))))))))))))))),
 366                (264,
 367                 (265,
 368                  (266,
 369                   (272,
 370                    (276,
 371                     (1, 'yield'),
 372                     (313,
 373                      (292,
 374                       (293,
 375                        (294,
 376                         (295,
 377                          (297,
 378                           (298,
 379                            (299,
 380                             (300,
 381                              (301,
 382                               (302,
 383                                (303, (304, (305, (2, '1')))))))))))))))))),
 384                  (4, ''))),
 385                (6, ''))))),
 386            (4, ''),
 387            (0, ''))))
 388         self.check_bad_tree(tree, "def f():\n  return 1\n  yield 1")
 389
 390     def test_print_chevron_comma(self):
 391         # Illegal input: print >>fp,
 392         tree = \
 393         (257,
 394          (264,
 395           (265,
 396            (266,
 397             (268,
 398              (1, 'print'),
 399              (35, '>>'),
 400              (290,
 401               (291,
 402                (292,
 403                 (293,
 404                  (295,
 405                   (296,
 406                    (297,
 407                     (298, (299, (300, (301, (302, (303, (1, 'fp')))))))))))))),
 408              (12, ','))),
 409            (4, ''))),
 410          (0, ''))
 411         self.check_bad_tree(tree, "print >>fp,")
 412
 413     def test_a_comma_comma_c(self):
 414         # Illegal input: a,,c
 415         tree = \
 416         (258,
 417          (311,
 418           (290,
 419            (291,
 420             (292,
 421              (293,
 422               (295,
 423                (296,
 424                 (297,
 425                  (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))),
 426           (12, ','),
 427           (12, ','),
 428           (290,
 429            (291,
 430             (292,
 431              (293,
 432               (295,
 433                (296,
 434                 (297,
 435                  (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))),
 436          (4, ''),
 437          (0, ''))
 438         self.check_bad_tree(tree, "a,,c")
 439
 440     def test_illegal_operator(self):
 441         # Illegal input: a $= b
 442         tree = \
 443         (257,
 444          (264,
 445           (265,
 446            (266,
 447             (267,
 448              (312,
 449               (291,
 450                (292,
 451                 (293,
 452                  (294,
 453                   (296,
 454                    (297,
 455                     (298,
 456                      (299,
 457                       (300, (301, (302, (303, (304, (1, 'a'))))))))))))))),
 458              (268, (37, '$=')),
 459              (312,
 460               (291,
 461                (292,
 462                 (293,
 463                  (294,
 464                   (296,
 465                    (297,
 466                     (298,
 467                      (299,
 468                       (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))),
 469            (4, ''))),
 470          (0, ''))
 471         self.check_bad_tree(tree, "a $= b")
 472
 473     def test_malformed_global(self):
 474         #doesn't have global keyword in ast
 475         tree = (257,
 476                 (264,
 477                  (265,
 478                   (266,
 479                    (282, (1, 'foo'))), (4, ''))),
 480                 (4, ''),
 481                 (0, ''))
 482         self.check_bad_tree(tree, "malformed global ast")
 483
 484
 485 class CompileTestCase(unittest.TestCase):
 486
 487     # These tests are very minimal. :-(
 488
 489     def test_compile_expr(self):
 490         st = parser.expr('2 + 3')
 491         code = parser.compilest(st)
 492         self.assertEquals(eval(code), 5)
 493
 494     def test_compile_suite(self):
 495         st = parser.suite('x = 2; y = x + 3')
 496         code = parser.compilest(st)
 497         globs = {}
 498         exec code in globs
 499         self.assertEquals(globs['y'], 5)
 500
 501     def test_compile_error(self):
 502         st = parser.suite('1 = 3 + 4')
 503         self.assertRaises(SyntaxError, parser.compilest, st)
 504
 505     def test_compile_badunicode(self):
 506         st = parser.suite('a = u"\U12345678"')
 507         self.assertRaises(SyntaxError, parser.compilest, st)
 508         st = parser.suite('a = u"\u1"')
 509         self.assertRaises(SyntaxError, parser.compilest, st)
 510
 511 class ParserStackLimitTestCase(unittest.TestCase):
 512     """try to push the parser to/over it's limits.
 513     see http://bugs.python.org/issue1881 for a discussion
 514     """
 515     def _nested_expression(self, level):
 516         return "["*level+"]"*level
 517
 518     def test_deeply_nested_list(self):
 519         e = self._nested_expression(99)
 520         st = parser.expr(e)
 521         st.compile()
 522
 523     def test_trigger_memory_error(self):
 524         e = self._nested_expression(100)
 525         print >>sys.stderr, "Expecting 's_push: parser stack overflow' in next line"
 526         self.assertRaises(MemoryError, parser.expr, e)
 527
 528 def test_main():
 529     test_support.run_unittest(
 530         RoundtripLegalSyntaxTestCase,
 531         IllegalSyntaxTestCase,
 532         CompileTestCase,
 533         ParserStackLimitTestCase,
 534     )
 535
 536
 537 if __name__ == "__main__":
 538     test_main()