Lib/lib2to3/tests/test_parser.py

   1 """Test suite for 2to3's parser and grammar files.
   2
   3 This is the place to add tests for changes to 2to3's grammar, such as those
   4 merging the grammars for Python 2 and 3. In addition to specific tests for
   5 parts of the grammar we've changed, we also make sure we can parse the
   6 test_grammar.py files from both Python 2 and Python 3.
   7 """
   8
   9 # Testing imports
  10 from . import support
  11 from .support import driver, test_dir
  12
  13 # Python imports
  14 import os
  15
  16 # Local imports
  17 from lib2to3.pgen2 import tokenize
  18 from ..pgen2.parse import ParseError
  19
  20
  21 class GrammarTest(support.TestCase):
  22     def validate(self, code):
  23         support.parse_string(code)
  24
  25     def invalid_syntax(self, code):
  26         try:
  27             self.validate(code)
  28         except ParseError:
  29             pass
  30         else:
  31             raise AssertionError("Syntax shouldn't have been valid")
  32
  33
  34 class TestRaiseChanges(GrammarTest):
  35     def test_2x_style_1(self):
  36         self.validate("raise")
  37
  38     def test_2x_style_2(self):
  39         self.validate("raise E, V")
  40
  41     def test_2x_style_3(self):
  42         self.validate("raise E, V, T")
  43
  44     def test_2x_style_invalid_1(self):
  45         self.invalid_syntax("raise E, V, T, Z")
  46
  47     def test_3x_style(self):
  48         self.validate("raise E1 from E2")
  49
  50     def test_3x_style_invalid_1(self):
  51         self.invalid_syntax("raise E, V from E1")
  52
  53     def test_3x_style_invalid_2(self):
  54         self.invalid_syntax("raise E from E1, E2")
  55
  56     def test_3x_style_invalid_3(self):
  57         self.invalid_syntax("raise from E1, E2")
  58
  59     def test_3x_style_invalid_4(self):
  60         self.invalid_syntax("raise E from")
  61
  62
  63 # Adapated from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef
  64 class TestFunctionAnnotations(GrammarTest):
  65     def test_1(self):
  66         self.validate("""def f(x) -> list: pass""")
  67
  68     def test_2(self):
  69         self.validate("""def f(x:int): pass""")
  70
  71     def test_3(self):
  72         self.validate("""def f(*x:str): pass""")
  73
  74     def test_4(self):
  75         self.validate("""def f(**x:float): pass""")
  76
  77     def test_5(self):
  78         self.validate("""def f(x, y:1+2): pass""")
  79
  80     def test_6(self):
  81         self.validate("""def f(a, (b:1, c:2, d)): pass""")
  82
  83     def test_7(self):
  84         self.validate("""def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass""")
  85
  86     def test_8(self):
  87         s = """def f(a, (b:1, c:2, d), e:3=4, f=5,
  88                         *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass"""
  89         self.validate(s)
  90
  91
  92 class TestExcept(GrammarTest):
  93     def test_new(self):
  94         s = """
  95             try:
  96                 x
  97             except E as N:
  98                 y"""
  99         self.validate(s)
 100
 101     def test_old(self):
 102         s = """
 103             try:
 104                 x
 105             except E, N:
 106                 y"""
 107         self.validate(s)
 108
 109
 110 # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms
 111 class TestSetLiteral(GrammarTest):
 112     def test_1(self):
 113         self.validate("""x = {'one'}""")
 114
 115     def test_2(self):
 116         self.validate("""x = {'one', 1,}""")
 117
 118     def test_3(self):
 119         self.validate("""x = {'one', 'two', 'three'}""")
 120
 121     def test_4(self):
 122         self.validate("""x = {2, 3, 4,}""")
 123
 124
 125 class TestNumericLiterals(GrammarTest):
 126     def test_new_octal_notation(self):
 127         self.validate("""0o7777777777777""")
 128         self.invalid_syntax("""0o7324528887""")
 129
 130     def test_new_binary_notation(self):
 131         self.validate("""0b101010""")
 132         self.invalid_syntax("""0b0101021""")
 133
 134
 135 class TestClassDef(GrammarTest):
 136     def test_new_syntax(self):
 137         self.validate("class B(t=7): pass")
 138         self.validate("class B(t, *args): pass")
 139         self.validate("class B(t, **kwargs): pass")
 140         self.validate("class B(t, *args, **kwargs): pass")
 141         self.validate("class B(t, y=9, *args, **kwargs): pass")
 142
 143
 144 class TestParserIdempotency(support.TestCase):
 145
 146     """A cut-down version of pytree_idempotency.py."""
 147
 148     def test_all_project_files(self):
 149         for filepath in support.all_project_files():
 150             with open(filepath, "rb") as fp:
 151                 encoding = tokenize.detect_encoding(fp.readline)[0]
 152                 fp.seek(0)
 153                 source = fp.read()
 154                 if encoding:
 155                     source = source.decode(encoding)
 156             tree = driver.parse_string(source)
 157             new = str(tree)
 158             if encoding:
 159                 new = new.encode(encoding)
 160             if diff(filepath, new):
 161                 self.fail("Idempotency failed: %s" % filepath)
 162
 163     def test_extended_unpacking(self):
 164         driver.parse_string("a, *b, c = x\n")
 165         driver.parse_string("[*a, b] = x\n")
 166         driver.parse_string("(z, *y, w) = m\n")
 167         driver.parse_string("for *z, m in d: pass\n")
 168
 169 class TestLiterals(GrammarTest):
 170
 171     def validate(self, s):
 172         driver.parse_string(support.dedent(s) + "\n\n")
 173
 174     def test_multiline_bytes_literals(self):
 175         s = """
 176             md5test(b"\xaa" * 80,
 177                     (b"Test Using Larger Than Block-Size Key "
 178                      b"and Larger Than One Block-Size Data"),
 179                     "6f630fad67cda0ee1fb1f562db3aa53e")
 180             """
 181         self.validate(s)
 182
 183     def test_multiline_bytes_tripquote_literals(self):
 184         s = '''
 185             b"""
 186             <?xml version="1.0" encoding="UTF-8"?>
 187             <!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN">
 188             """
 189             '''
 190         self.validate(s)
 191
 192     def test_multiline_str_literals(self):
 193         s = """
 194             md5test("\xaa" * 80,
 195                     ("Test Using Larger Than Block-Size Key "
 196                      "and Larger Than One Block-Size Data"),
 197                     "6f630fad67cda0ee1fb1f562db3aa53e")
 198             """
 199         self.validate(s)
 200
 201
 202 def diff(fn, result):
 203     f = open("@", "wb")
 204     try:
 205         f.write(result)
 206     finally:
 207         f.close()
 208     try:
 209         return os.system("diff -u %s @" % fn)
 210     finally:
 211         os.remove("@")