Lib/test/test_parser.py

   1 import parser
   2 import unittest
   3 import sys
   4 from test import test_support
   5
   6 #
   7 #  First, we test that we can generate trees from valid source fragments,
   8 #  and that these valid trees are indeed allowed by the tree-loading side
   9 #  of the parser module.
  10 #
  11
  12 class RoundtripLegalSyntaxTestCase(unittest.TestCase):
  13
  14     def roundtrip(self, f, s):
  15         st1 = f(s)
  16         t = st1.totuple()
  17         try:
  18             st2 = parser.sequence2st(t)
  19         except parser.ParserError, why:
  20             self.fail("could not roundtrip %r: %s" % (s, why))
  21
  22         self.assertEquals(t, st2.totuple(),
  23                           "could not re-generate syntax tree")
  24
  25     def check_expr(self, s):
  26         self.roundtrip(parser.expr, s)
  27
  28     def test_flags_passed(self):
  29         # The unicode literals flags has to be passed from the paser to AST
  30         # generation.
  31         suite = parser.suite("from __future__ import unicode_literals; x = ''")
  32         code = suite.compile()
  33         scope = {}
  34         exec code in scope
  35         self.assertIsInstance(scope["x"], unicode)
  36
  37     def check_suite(self, s):
  38         self.roundtrip(parser.suite, s)
  39
  40     def test_yield_statement(self):
  41         self.check_suite("def f(): yield 1")
  42         self.check_suite("def f(): yield")
  43         self.check_suite("def f(): x += yield")
  44         self.check_suite("def f(): x = yield 1")
  45         self.check_suite("def f(): x = y = yield 1")
  46         self.check_suite("def f(): x = yield")
  47         self.check_suite("def f(): x = y = yield")
  48         self.check_suite("def f(): 1 + (yield)*2")
  49         self.check_suite("def f(): (yield 1)*2")
  50         self.check_suite("def f(): return; yield 1")
  51         self.check_suite("def f(): yield 1; return")
  52         self.check_suite("def f():\n"
  53                          "    for x in range(30):\n"
  54                          "        yield x\n")
  55         self.check_suite("def f():\n"
  56                          "    if (yield):\n"
  57                          "        yield x\n")
  58
  59     def test_expressions(self):
  60         self.check_expr("foo(1)")
  61         self.check_expr("{1:1}")
  62         self.check_expr("{1:1, 2:2, 3:3}")
  63         self.check_expr("{1:1, 2:2, 3:3,}")
  64         self.check_expr("{1}")
  65         self.check_expr("{1, 2, 3}")
  66         self.check_expr("{1, 2, 3,}")
  67         self.check_expr("[]")
  68         self.check_expr("[1]")
  69         self.check_expr("[1, 2, 3]")
  70         self.check_expr("[1, 2, 3,]")
  71         self.check_expr("()")
  72         self.check_expr("(1,)")
  73         self.check_expr("(1, 2, 3)")
  74         self.check_expr("(1, 2, 3,)")
  75         self.check_expr("[x**3 for x in range(20)]")
  76         self.check_expr("[x**3 for x in range(20) if x % 3]")
  77         self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]")
  78         self.check_expr("[x+y for x in range(30) for y in range(20) if x % 2 if y % 3]")
  79         #self.check_expr("[x for x in lambda: True, lambda: False if x()]")
  80         self.check_expr("list(x**3 for x in range(20))")
  81         self.check_expr("list(x**3 for x in range(20) if x % 3)")
  82         self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)")
  83         self.check_expr("list(x+y for x in range(30) for y in range(20) if x % 2 if y % 3)")
  84         self.check_expr("{x**3 for x in range(30)}")
  85         self.check_expr("{x**3 for x in range(30) if x % 3}")
  86         self.check_expr("{x**3 for x in range(30) if x % 2 if x % 3}")
  87         self.check_expr("{x+y for x in range(30) for y in range(20) if x % 2 if y % 3}")
  88         self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30))}")
  89         self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30)) if x % 3}")
  90         self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30)) if x % 3 if y % 3}")
  91         self.check_expr("{x:y for x in range(30) for y in range(20) if x % 2 if y % 3}")
  92         self.check_expr("foo(*args)")
  93         self.check_expr("foo(*args, **kw)")
  94         self.check_expr("foo(**kw)")
  95         self.check_expr("foo(key=value)")
  96         self.check_expr("foo(key=value, *args)")
  97         self.check_expr("foo(key=value, *args, **kw)")
  98         self.check_expr("foo(key=value, **kw)")
  99         self.check_expr("foo(a, b, c, *args)")
 100         self.check_expr("foo(a, b, c, *args, **kw)")
 101         self.check_expr("foo(a, b, c, **kw)")
 102         self.check_expr("foo(a, *args, keyword=23)")
 103         self.check_expr("foo + bar")
 104         self.check_expr("foo - bar")
 105         self.check_expr("foo * bar")
 106         self.check_expr("foo / bar")
 107         self.check_expr("foo // bar")
 108         self.check_expr("lambda: 0")
 109         self.check_expr("lambda x: 0")
 110         self.check_expr("lambda *y: 0")
 111         self.check_expr("lambda *y, **z: 0")
 112         self.check_expr("lambda **z: 0")
 113         self.check_expr("lambda x, y: 0")
 114         self.check_expr("lambda foo=bar: 0")
 115         self.check_expr("lambda foo=bar, spaz=nifty+spit: 0")
 116         self.check_expr("lambda foo=bar, **z: 0")
 117         self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0")
 118         self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0")
 119         self.check_expr("lambda x, *y, **z: 0")
 120         self.check_expr("lambda x: 5 if x else 2")
 121         self.check_expr("(x for x in range(10))")
 122         self.check_expr("foo(x for x in range(10))")
 123
 124     def test_print(self):
 125         self.check_suite("print")
 126         self.check_suite("print 1")
 127         self.check_suite("print 1,")
 128         self.check_suite("print >>fp")
 129         self.check_suite("print >>fp, 1")
 130         self.check_suite("print >>fp, 1,")
 131
 132     def test_simple_expression(self):
 133         # expr_stmt
 134         self.check_suite("a")
 135
 136     def test_simple_assignments(self):
 137         self.check_suite("a = b")
 138         self.check_suite("a = b = c = d = e")
 139
 140     def test_simple_augmented_assignments(self):
 141         self.check_suite("a += b")
 142         self.check_suite("a -= b")
 143         self.check_suite("a *= b")
 144         self.check_suite("a /= b")
 145         self.check_suite("a //= b")
 146         self.check_suite("a %= b")
 147         self.check_suite("a &= b")
 148         self.check_suite("a |= b")
 149         self.check_suite("a ^= b")
 150         self.check_suite("a <<= b")
 151         self.check_suite("a >>= b")
 152         self.check_suite("a **= b")
 153
 154     def test_function_defs(self):
 155         self.check_suite("def f(): pass")
 156         self.check_suite("def f(*args): pass")
 157         self.check_suite("def f(*args, **kw): pass")
 158         self.check_suite("def f(**kw): pass")
 159         self.check_suite("def f(foo=bar): pass")
 160         self.check_suite("def f(foo=bar, *args): pass")
 161         self.check_suite("def f(foo=bar, *args, **kw): pass")
 162         self.check_suite("def f(foo=bar, **kw): pass")
 163
 164         self.check_suite("def f(a, b): pass")
 165         self.check_suite("def f(a, b, *args): pass")
 166         self.check_suite("def f(a, b, *args, **kw): pass")
 167         self.check_suite("def f(a, b, **kw): pass")
 168         self.check_suite("def f(a, b, foo=bar): pass")
 169         self.check_suite("def f(a, b, foo=bar, *args): pass")
 170         self.check_suite("def f(a, b, foo=bar, *args, **kw): pass")
 171         self.check_suite("def f(a, b, foo=bar, **kw): pass")
 172
 173         self.check_suite("@staticmethod\n"
 174                          "def f(): pass")
 175         self.check_suite("@staticmethod\n"
 176                          "@funcattrs(x, y)\n"
 177                          "def f(): pass")
 178         self.check_suite("@funcattrs()\n"
 179                          "def f(): pass")
 180
 181     def test_class_defs(self):
 182         self.check_suite("class foo():pass")
 183
 184     def test_import_from_statement(self):
 185         self.check_suite("from sys.path import *")
 186         self.check_suite("from sys.path import dirname")
 187         self.check_suite("from sys.path import (dirname)")
 188         self.check_suite("from sys.path import (dirname,)")
 189         self.check_suite("from sys.path import dirname as my_dirname")
 190         self.check_suite("from sys.path import (dirname as my_dirname)")
 191         self.check_suite("from sys.path import (dirname as my_dirname,)")
 192         self.check_suite("from sys.path import dirname, basename")
 193         self.check_suite("from sys.path import (dirname, basename)")
 194         self.check_suite("from sys.path import (dirname, basename,)")
 195         self.check_suite(
 196             "from sys.path import dirname as my_dirname, basename")
 197         self.check_suite(
 198             "from sys.path import (dirname as my_dirname, basename)")
 199         self.check_suite(
 200             "from sys.path import (dirname as my_dirname, basename,)")
 201         self.check_suite(
 202             "from sys.path import dirname, basename as my_basename")
 203         self.check_suite(
 204             "from sys.path import (dirname, basename as my_basename)")
 205         self.check_suite(
 206             "from sys.path import (dirname, basename as my_basename,)")
 207         self.check_suite("from .bogus import x")
 208
 209     def test_basic_import_statement(self):
 210         self.check_suite("import sys")
 211         self.check_suite("import sys as system")
 212         self.check_suite("import sys, math")
 213         self.check_suite("import sys as system, math")
 214         self.check_suite("import sys, math as my_math")
 215
 216     def test_pep263(self):
 217         self.check_suite("# -*- coding: iso-8859-1 -*-\n"
 218                          "pass\n")
 219
 220     def test_assert(self):
 221         self.check_suite("assert alo < ahi and blo < bhi\n")
 222
 223     def test_with(self):
 224         self.check_suite("with open('x'): pass\n")
 225         self.check_suite("with open('x') as f: pass\n")
 226         self.check_suite("with open('x') as f, open('y') as g: pass\n")
 227
 228     def test_try_stmt(self):
 229         self.check_suite("try: pass\nexcept: pass\n")
 230         self.check_suite("try: pass\nfinally: pass\n")
 231         self.check_suite("try: pass\nexcept A: pass\nfinally: pass\n")
 232         self.check_suite("try: pass\nexcept A: pass\nexcept: pass\n"
 233                          "finally: pass\n")
 234         self.check_suite("try: pass\nexcept: pass\nelse: pass\n")
 235         self.check_suite("try: pass\nexcept: pass\nelse: pass\n"
 236                          "finally: pass\n")
 237
 238     def test_except_clause(self):
 239         self.check_suite("try: pass\nexcept: pass\n")
 240         self.check_suite("try: pass\nexcept A: pass\n")
 241         self.check_suite("try: pass\nexcept A, e: pass\n")
 242         self.check_suite("try: pass\nexcept A as e: pass\n")
 243
 244     def test_position(self):
 245         # An absolutely minimal test of position information.  Better
 246         # tests would be a big project.
 247         code = "def f(x):\n    return x + 1"
 248         st1 = parser.suite(code)
 249         st2 = st1.totuple(line_info=1, col_info=1)
 250
 251         def walk(tree):
 252             node_type = tree[0]
 253             next = tree[1]
 254             if isinstance(next, tuple):
 255                 for elt in tree[1:]:
 256                     for x in walk(elt):
 257                         yield x
 258             else:
 259                 yield tree
 260
 261         terminals = list(walk(st2))
 262         self.assertEqual([
 263             (1, 'def', 1, 0),
 264             (1, 'f', 1, 4),
 265             (7, '(', 1, 5),
 266             (1, 'x', 1, 6),
 267             (8, ')', 1, 7),
 268             (11, ':', 1, 8),
 269             (4, '', 1, 9),
 270             (5, '', 2, -1),
 271             (1, 'return', 2, 4),
 272             (1, 'x', 2, 11),
 273             (14, '+', 2, 13),
 274             (2, '1', 2, 15),
 275             (4, '', 2, 16),
 276             (6, '', 2, -1),
 277             (4, '', 2, -1),
 278             (0, '', 2, -1)],
 279                          terminals)
 280
 281
 282 #
 283 #  Second, we take *invalid* trees and make sure we get ParserError
 284 #  rejections for them.
 285 #
 286
 287 class IllegalSyntaxTestCase(unittest.TestCase):
 288
 289     def check_bad_tree(self, tree, label):
 290         try:
 291             parser.sequence2st(tree)
 292         except parser.ParserError:
 293             pass
 294         else:
 295             self.fail("did not detect invalid tree for %r" % label)
 296
 297     def test_junk(self):
 298         # not even remotely valid:
 299         self.check_bad_tree((1, 2, 3), "<junk>")
 300
 301     def test_illegal_yield_1(self):
 302         # Illegal yield statement: def f(): return 1; yield 1
 303         tree = \
 304         (257,
 305          (264,
 306           (285,
 307            (259,
 308             (1, 'def'),
 309             (1, 'f'),
 310             (260, (7, '('), (8, ')')),
 311             (11, ':'),
 312             (291,
 313              (4, ''),
 314              (5, ''),
 315              (264,
 316               (265,
 317                (266,
 318                 (272,
 319                  (275,
 320                   (1, 'return'),
 321                   (313,
 322                    (292,
 323                     (293,
 324                      (294,
 325                       (295,
 326                        (297,
 327                         (298,
 328                          (299,
 329                           (300,
 330                            (301,
 331                             (302, (303, (304, (305, (2, '1')))))))))))))))))),
 332                (264,
 333                 (265,
 334                  (266,
 335                   (272,
 336                    (276,
 337                     (1, 'yield'),
 338                     (313,
 339                      (292,
 340                       (293,
 341                        (294,
 342                         (295,
 343                          (297,
 344                           (298,
 345                            (299,
 346                             (300,
 347                              (301,
 348                               (302,
 349                                (303, (304, (305, (2, '1')))))))))))))))))),
 350                  (4, ''))),
 351                (6, ''))))),
 352            (4, ''),
 353            (0, ''))))
 354         self.check_bad_tree(tree, "def f():\n  return 1\n  yield 1")
 355
 356     def test_illegal_yield_2(self):
 357         # Illegal return in generator: def f(): return 1; yield 1
 358         tree = \
 359         (257,
 360          (264,
 361           (265,
 362            (266,
 363             (278,
 364              (1, 'from'),
 365              (281, (1, '__future__')),
 366              (1, 'import'),
 367              (279, (1, 'generators')))),
 368            (4, ''))),
 369          (264,
 370           (285,
 371            (259,
 372             (1, 'def'),
 373             (1, 'f'),
 374             (260, (7, '('), (8, ')')),
 375             (11, ':'),
 376             (291,
 377              (4, ''),
 378              (5, ''),
 379              (264,
 380               (265,
 381                (266,
 382                 (272,
 383                  (275,
 384                   (1, 'return'),
 385                   (313,
 386                    (292,
 387                     (293,
 388                      (294,
 389                       (295,
 390                        (297,
 391                         (298,
 392                          (299,
 393                           (300,
 394                            (301,
 395                             (302, (303, (304, (305, (2, '1')))))))))))))))))),
 396                (264,
 397                 (265,
 398                  (266,
 399                   (272,
 400                    (276,
 401                     (1, 'yield'),
 402                     (313,
 403                      (292,
 404                       (293,
 405                        (294,
 406                         (295,
 407                          (297,
 408                           (298,
 409                            (299,
 410                             (300,
 411                              (301,
 412                               (302,
 413                                (303, (304, (305, (2, '1')))))))))))))))))),
 414                  (4, ''))),
 415                (6, ''))))),
 416            (4, ''),
 417            (0, ''))))
 418         self.check_bad_tree(tree, "def f():\n  return 1\n  yield 1")
 419
 420     def test_print_chevron_comma(self):
 421         # Illegal input: print >>fp,
 422         tree = \
 423         (257,
 424          (264,
 425           (265,
 426            (266,
 427             (268,
 428              (1, 'print'),
 429              (35, '>>'),
 430              (290,
 431               (291,
 432                (292,
 433                 (293,
 434                  (295,
 435                   (296,
 436                    (297,
 437                     (298, (299, (300, (301, (302, (303, (1, 'fp')))))))))))))),
 438              (12, ','))),
 439            (4, ''))),
 440          (0, ''))
 441         self.check_bad_tree(tree, "print >>fp,")
 442
 443     def test_a_comma_comma_c(self):
 444         # Illegal input: a,,c
 445         tree = \
 446         (258,
 447          (311,
 448           (290,
 449            (291,
 450             (292,
 451              (293,
 452               (295,
 453                (296,
 454                 (297,
 455                  (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))),
 456           (12, ','),
 457           (12, ','),
 458           (290,
 459            (291,
 460             (292,
 461              (293,
 462               (295,
 463                (296,
 464                 (297,
 465                  (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))),
 466          (4, ''),
 467          (0, ''))
 468         self.check_bad_tree(tree, "a,,c")
 469
 470     def test_illegal_operator(self):
 471         # Illegal input: a $= b
 472         tree = \
 473         (257,
 474          (264,
 475           (265,
 476            (266,
 477             (267,
 478              (312,
 479               (291,
 480                (292,
 481                 (293,
 482                  (294,
 483                   (296,
 484                    (297,
 485                     (298,
 486                      (299,
 487                       (300, (301, (302, (303, (304, (1, 'a'))))))))))))))),
 488              (268, (37, '$=')),
 489              (312,
 490               (291,
 491                (292,
 492                 (293,
 493                  (294,
 494                   (296,
 495                    (297,
 496                     (298,
 497                      (299,
 498                       (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))),
 499            (4, ''))),
 500          (0, ''))
 501         self.check_bad_tree(tree, "a $= b")
 502
 503     def test_malformed_global(self):
 504         #doesn't have global keyword in ast
 505         tree = (257,
 506                 (264,
 507                  (265,
 508                   (266,
 509                    (282, (1, 'foo'))), (4, ''))),
 510                 (4, ''),
 511                 (0, ''))
 512         self.check_bad_tree(tree, "malformed global ast")
 513
 514
 515 class CompileTestCase(unittest.TestCase):
 516
 517     # These tests are very minimal. :-(
 518
 519     def test_compile_expr(self):
 520         st = parser.expr('2 + 3')
 521         code = parser.compilest(st)
 522         self.assertEquals(eval(code), 5)
 523
 524     def test_compile_suite(self):
 525         st = parser.suite('x = 2; y = x + 3')
 526         code = parser.compilest(st)
 527         globs = {}
 528         exec code in globs
 529         self.assertEquals(globs['y'], 5)
 530
 531     def test_compile_error(self):
 532         st = parser.suite('1 = 3 + 4')
 533         self.assertRaises(SyntaxError, parser.compilest, st)
 534
 535     def test_compile_badunicode(self):
 536         st = parser.suite('a = u"\U12345678"')
 537         self.assertRaises(SyntaxError, parser.compilest, st)
 538         st = parser.suite('a = u"\u1"')
 539         self.assertRaises(SyntaxError, parser.compilest, st)
 540
 541 class ParserStackLimitTestCase(unittest.TestCase):
 542     """try to push the parser to/over it's limits.
 543     see http://bugs.python.org/issue1881 for a discussion
 544     """
 545     def _nested_expression(self, level):
 546         return "["*level+"]"*level
 547
 548     def test_deeply_nested_list(self):
 549         e = self._nested_expression(99)
 550         st = parser.expr(e)
 551         st.compile()
 552
 553     def test_trigger_memory_error(self):
 554         e = self._nested_expression(100)
 555         print >>sys.stderr, "Expecting 's_push: parser stack overflow' in next line"
 556         self.assertRaises(MemoryError, parser.expr, e)
 557
 558 def test_main():
 559     test_support.run_unittest(
 560         RoundtripLegalSyntaxTestCase,
 561         IllegalSyntaxTestCase,
 562         CompileTestCase,
 563         ParserStackLimitTestCase,
 564     )
 565
 566
 567 if __name__ == "__main__":
 568     test_main()