ยปCore Development>Code coverage>Lib/lib2to3/tests/test_parser.py

Python code coverage for Lib/lib2to3/tests/test_parser.py

#countcontent
1n/a"""Test suite for 2to3's parser and grammar files.
2n/a
3n/aThis is the place to add tests for changes to 2to3's grammar, such as those
4n/amerging the grammars for Python 2 and 3. In addition to specific tests for
5n/aparts of the grammar we've changed, we also make sure we can parse the
6n/atest_grammar.py files from both Python 2 and Python 3.
7n/a"""
8n/a
9n/a# Testing imports
10n/afrom . import support
11n/afrom .support import driver
12n/afrom test.support import verbose
13n/a
14n/a# Python imports
15n/aimport os
16n/aimport shutil
17n/aimport subprocess
18n/aimport sys
19n/aimport tempfile
20n/aimport unittest
21n/aimport warnings
22n/a
23n/a# Local imports
24n/afrom lib2to3.pgen2 import driver as pgen2_driver
25n/afrom lib2to3.pgen2 import tokenize
26n/afrom ..pgen2.parse import ParseError
27n/afrom lib2to3.pygram import python_symbols as syms
28n/a
29n/a
30n/aclass TestDriver(support.TestCase):
31n/a
32n/a def test_formfeed(self):
33n/a s = """print 1\n\x0Cprint 2\n"""
34n/a t = driver.parse_string(s)
35n/a self.assertEqual(t.children[0].children[0].type, syms.print_stmt)
36n/a self.assertEqual(t.children[1].children[0].type, syms.print_stmt)
37n/a
38n/a
39n/aclass TestPgen2Caching(support.TestCase):
40n/a def test_load_grammar_from_txt_file(self):
41n/a pgen2_driver.load_grammar(support.grammar_path, save=False, force=True)
42n/a
43n/a def test_load_grammar_from_pickle(self):
44n/a # Make a copy of the grammar file in a temp directory we are
45n/a # guaranteed to be able to write to.
46n/a tmpdir = tempfile.mkdtemp()
47n/a try:
48n/a grammar_copy = os.path.join(
49n/a tmpdir, os.path.basename(support.grammar_path))
50n/a shutil.copy(support.grammar_path, grammar_copy)
51n/a pickle_name = pgen2_driver._generate_pickle_name(grammar_copy)
52n/a
53n/a pgen2_driver.load_grammar(grammar_copy, save=True, force=True)
54n/a self.assertTrue(os.path.exists(pickle_name))
55n/a
56n/a os.unlink(grammar_copy) # Only the pickle remains...
57n/a pgen2_driver.load_grammar(grammar_copy, save=False, force=False)
58n/a finally:
59n/a shutil.rmtree(tmpdir)
60n/a
61n/a @unittest.skipIf(sys.executable is None, 'sys.executable required')
62n/a def test_load_grammar_from_subprocess(self):
63n/a tmpdir = tempfile.mkdtemp()
64n/a tmpsubdir = os.path.join(tmpdir, 'subdir')
65n/a try:
66n/a os.mkdir(tmpsubdir)
67n/a grammar_base = os.path.basename(support.grammar_path)
68n/a grammar_copy = os.path.join(tmpdir, grammar_base)
69n/a grammar_sub_copy = os.path.join(tmpsubdir, grammar_base)
70n/a shutil.copy(support.grammar_path, grammar_copy)
71n/a shutil.copy(support.grammar_path, grammar_sub_copy)
72n/a pickle_name = pgen2_driver._generate_pickle_name(grammar_copy)
73n/a pickle_sub_name = pgen2_driver._generate_pickle_name(
74n/a grammar_sub_copy)
75n/a self.assertNotEqual(pickle_name, pickle_sub_name)
76n/a
77n/a # Generate a pickle file from this process.
78n/a pgen2_driver.load_grammar(grammar_copy, save=True, force=True)
79n/a self.assertTrue(os.path.exists(pickle_name))
80n/a
81n/a # Generate a new pickle file in a subprocess with a most likely
82n/a # different hash randomization seed.
83n/a sub_env = dict(os.environ)
84n/a sub_env['PYTHONHASHSEED'] = 'random'
85n/a subprocess.check_call(
86n/a [sys.executable, '-c', """
87n/afrom lib2to3.pgen2 import driver as pgen2_driver
88n/apgen2_driver.load_grammar(%r, save=True, force=True)
89n/a """ % (grammar_sub_copy,)],
90n/a env=sub_env)
91n/a self.assertTrue(os.path.exists(pickle_sub_name))
92n/a
93n/a with open(pickle_name, 'rb') as pickle_f_1, \
94n/a open(pickle_sub_name, 'rb') as pickle_f_2:
95n/a self.assertEqual(
96n/a pickle_f_1.read(), pickle_f_2.read(),
97n/a msg='Grammar caches generated using different hash seeds'
98n/a ' were not identical.')
99n/a finally:
100n/a shutil.rmtree(tmpdir)
101n/a
102n/a
103n/a
104n/aclass GrammarTest(support.TestCase):
105n/a def validate(self, code):
106n/a support.parse_string(code)
107n/a
108n/a def invalid_syntax(self, code):
109n/a try:
110n/a self.validate(code)
111n/a except ParseError:
112n/a pass
113n/a else:
114n/a raise AssertionError("Syntax shouldn't have been valid")
115n/a
116n/a
117n/aclass TestMatrixMultiplication(GrammarTest):
118n/a def test_matrix_multiplication_operator(self):
119n/a self.validate("a @ b")
120n/a self.validate("a @= b")
121n/a
122n/a
123n/aclass TestYieldFrom(GrammarTest):
124n/a def test_yield_from(self):
125n/a self.validate("yield from x")
126n/a self.validate("(yield from x) + y")
127n/a self.invalid_syntax("yield from")
128n/a
129n/a
130n/aclass TestAsyncAwait(GrammarTest):
131n/a def test_await_expr(self):
132n/a self.validate("""async def foo():
133n/a await x
134n/a """)
135n/a
136n/a self.validate("""async def foo():
137n/a [i async for i in b]
138n/a """)
139n/a
140n/a self.validate("""async def foo():
141n/a {i for i in b
142n/a async for i in a if await i
143n/a for b in i}
144n/a """)
145n/a
146n/a self.validate("""async def foo():
147n/a [await i for i in b if await c]
148n/a """)
149n/a
150n/a self.validate("""async def foo():
151n/a [ i for i in b if c]
152n/a """)
153n/a
154n/a self.validate("""async def foo():
155n/a
156n/a def foo(): pass
157n/a
158n/a def foo(): pass
159n/a
160n/a await x
161n/a """)
162n/a
163n/a self.validate("""async def foo(): return await a""")
164n/a
165n/a self.validate("""def foo():
166n/a def foo(): pass
167n/a async def foo(): await x
168n/a """)
169n/a
170n/a self.invalid_syntax("await x")
171n/a self.invalid_syntax("""def foo():
172n/a await x""")
173n/a
174n/a self.invalid_syntax("""def foo():
175n/a def foo(): pass
176n/a async def foo(): pass
177n/a await x
178n/a """)
179n/a
180n/a def test_async_var(self):
181n/a self.validate("""async = 1""")
182n/a self.validate("""await = 1""")
183n/a self.validate("""def async(): pass""")
184n/a
185n/a def test_async_with(self):
186n/a self.validate("""async def foo():
187n/a async for a in b: pass""")
188n/a
189n/a self.invalid_syntax("""def foo():
190n/a async for a in b: pass""")
191n/a
192n/a def test_async_for(self):
193n/a self.validate("""async def foo():
194n/a async with a: pass""")
195n/a
196n/a self.invalid_syntax("""def foo():
197n/a async with a: pass""")
198n/a
199n/a
200n/aclass TestRaiseChanges(GrammarTest):
201n/a def test_2x_style_1(self):
202n/a self.validate("raise")
203n/a
204n/a def test_2x_style_2(self):
205n/a self.validate("raise E, V")
206n/a
207n/a def test_2x_style_3(self):
208n/a self.validate("raise E, V, T")
209n/a
210n/a def test_2x_style_invalid_1(self):
211n/a self.invalid_syntax("raise E, V, T, Z")
212n/a
213n/a def test_3x_style(self):
214n/a self.validate("raise E1 from E2")
215n/a
216n/a def test_3x_style_invalid_1(self):
217n/a self.invalid_syntax("raise E, V from E1")
218n/a
219n/a def test_3x_style_invalid_2(self):
220n/a self.invalid_syntax("raise E from E1, E2")
221n/a
222n/a def test_3x_style_invalid_3(self):
223n/a self.invalid_syntax("raise from E1, E2")
224n/a
225n/a def test_3x_style_invalid_4(self):
226n/a self.invalid_syntax("raise E from")
227n/a
228n/a
229n/a# Modelled after Lib/test/test_grammar.py:TokenTests.test_funcdef issue2292
230n/a# and Lib/test/text_parser.py test_list_displays, test_set_displays,
231n/a# test_dict_displays, test_argument_unpacking, ... changes.
232n/aclass TestUnpackingGeneralizations(GrammarTest):
233n/a def test_mid_positional_star(self):
234n/a self.validate("""func(1, *(2, 3), 4)""")
235n/a
236n/a def test_double_star_dict_literal(self):
237n/a self.validate("""func(**{'eggs':'scrambled', 'spam':'fried'})""")
238n/a
239n/a def test_double_star_dict_literal_after_keywords(self):
240n/a self.validate("""func(spam='fried', **{'eggs':'scrambled'})""")
241n/a
242n/a def test_list_display(self):
243n/a self.validate("""[*{2}, 3, *[4]]""")
244n/a
245n/a def test_set_display(self):
246n/a self.validate("""{*{2}, 3, *[4]}""")
247n/a
248n/a def test_dict_display_1(self):
249n/a self.validate("""{**{}}""")
250n/a
251n/a def test_dict_display_2(self):
252n/a self.validate("""{**{}, 3:4, **{5:6, 7:8}}""")
253n/a
254n/a def test_argument_unpacking_1(self):
255n/a self.validate("""f(a, *b, *c, d)""")
256n/a
257n/a def test_argument_unpacking_2(self):
258n/a self.validate("""f(**a, **b)""")
259n/a
260n/a def test_argument_unpacking_3(self):
261n/a self.validate("""f(2, *a, *b, **b, **c, **d)""")
262n/a
263n/a
264n/a# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef
265n/aclass TestFunctionAnnotations(GrammarTest):
266n/a def test_1(self):
267n/a self.validate("""def f(x) -> list: pass""")
268n/a
269n/a def test_2(self):
270n/a self.validate("""def f(x:int): pass""")
271n/a
272n/a def test_3(self):
273n/a self.validate("""def f(*x:str): pass""")
274n/a
275n/a def test_4(self):
276n/a self.validate("""def f(**x:float): pass""")
277n/a
278n/a def test_5(self):
279n/a self.validate("""def f(x, y:1+2): pass""")
280n/a
281n/a def test_6(self):
282n/a self.validate("""def f(a, (b:1, c:2, d)): pass""")
283n/a
284n/a def test_7(self):
285n/a self.validate("""def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass""")
286n/a
287n/a def test_8(self):
288n/a s = """def f(a, (b:1, c:2, d), e:3=4, f=5,
289n/a *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass"""
290n/a self.validate(s)
291n/a
292n/a
293n/a# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot
294n/aclass TestVarAnnotations(GrammarTest):
295n/a def test_1(self):
296n/a self.validate("var1: int = 5")
297n/a
298n/a def test_2(self):
299n/a self.validate("var2: [int, str]")
300n/a
301n/a def test_3(self):
302n/a self.validate("def f():\n"
303n/a " st: str = 'Hello'\n"
304n/a " a.b: int = (1, 2)\n"
305n/a " return st\n")
306n/a
307n/a def test_4(self):
308n/a self.validate("def fbad():\n"
309n/a " x: int\n"
310n/a " print(x)\n")
311n/a
312n/a def test_5(self):
313n/a self.validate("class C:\n"
314n/a " x: int\n"
315n/a " s: str = 'attr'\n"
316n/a " z = 2\n"
317n/a " def __init__(self, x):\n"
318n/a " self.x: int = x\n")
319n/a
320n/a def test_6(self):
321n/a self.validate("lst: List[int] = []")
322n/a
323n/aclass TestExcept(GrammarTest):
324n/a def test_new(self):
325n/a s = """
326n/a try:
327n/a x
328n/a except E as N:
329n/a y"""
330n/a self.validate(s)
331n/a
332n/a def test_old(self):
333n/a s = """
334n/a try:
335n/a x
336n/a except E, N:
337n/a y"""
338n/a self.validate(s)
339n/a
340n/a
341n/a# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms
342n/aclass TestSetLiteral(GrammarTest):
343n/a def test_1(self):
344n/a self.validate("""x = {'one'}""")
345n/a
346n/a def test_2(self):
347n/a self.validate("""x = {'one', 1,}""")
348n/a
349n/a def test_3(self):
350n/a self.validate("""x = {'one', 'two', 'three'}""")
351n/a
352n/a def test_4(self):
353n/a self.validate("""x = {2, 3, 4,}""")
354n/a
355n/a
356n/aclass TestNumericLiterals(GrammarTest):
357n/a def test_new_octal_notation(self):
358n/a self.validate("""0o7777777777777""")
359n/a self.invalid_syntax("""0o7324528887""")
360n/a
361n/a def test_new_binary_notation(self):
362n/a self.validate("""0b101010""")
363n/a self.invalid_syntax("""0b0101021""")
364n/a
365n/a
366n/aclass TestClassDef(GrammarTest):
367n/a def test_new_syntax(self):
368n/a self.validate("class B(t=7): pass")
369n/a self.validate("class B(t, *args): pass")
370n/a self.validate("class B(t, **kwargs): pass")
371n/a self.validate("class B(t, *args, **kwargs): pass")
372n/a self.validate("class B(t, y=9, *args, **kwargs): pass")
373n/a
374n/a
375n/aclass TestParserIdempotency(support.TestCase):
376n/a
377n/a """A cut-down version of pytree_idempotency.py."""
378n/a
379n/a # Issue 13125
380n/a @unittest.expectedFailure
381n/a def test_all_project_files(self):
382n/a for filepath in support.all_project_files():
383n/a with open(filepath, "rb") as fp:
384n/a encoding = tokenize.detect_encoding(fp.readline)[0]
385n/a self.assertIsNotNone(encoding,
386n/a "can't detect encoding for %s" % filepath)
387n/a with open(filepath, "r", encoding=encoding) as fp:
388n/a source = fp.read()
389n/a try:
390n/a tree = driver.parse_string(source)
391n/a except ParseError as err:
392n/a if verbose > 0:
393n/a warnings.warn('ParseError on file %s (%s)' % (filepath, err))
394n/a continue
395n/a new = str(tree)
396n/a x = diff(filepath, new)
397n/a if x:
398n/a self.fail("Idempotency failed: %s" % filepath)
399n/a
400n/a def test_extended_unpacking(self):
401n/a driver.parse_string("a, *b, c = x\n")
402n/a driver.parse_string("[*a, b] = x\n")
403n/a driver.parse_string("(z, *y, w) = m\n")
404n/a driver.parse_string("for *z, m in d: pass\n")
405n/a
406n/a
407n/aclass TestLiterals(GrammarTest):
408n/a
409n/a def validate(self, s):
410n/a driver.parse_string(support.dedent(s) + "\n\n")
411n/a
412n/a def test_multiline_bytes_literals(self):
413n/a s = """
414n/a md5test(b"\xaa" * 80,
415n/a (b"Test Using Larger Than Block-Size Key "
416n/a b"and Larger Than One Block-Size Data"),
417n/a "6f630fad67cda0ee1fb1f562db3aa53e")
418n/a """
419n/a self.validate(s)
420n/a
421n/a def test_multiline_bytes_tripquote_literals(self):
422n/a s = '''
423n/a b"""
424n/a <?xml version="1.0" encoding="UTF-8"?>
425n/a <!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN">
426n/a """
427n/a '''
428n/a self.validate(s)
429n/a
430n/a def test_multiline_str_literals(self):
431n/a s = """
432n/a md5test("\xaa" * 80,
433n/a ("Test Using Larger Than Block-Size Key "
434n/a "and Larger Than One Block-Size Data"),
435n/a "6f630fad67cda0ee1fb1f562db3aa53e")
436n/a """
437n/a self.validate(s)
438n/a
439n/a
440n/adef diff(fn, result):
441n/a try:
442n/a with open('@', 'w') as f:
443n/a f.write(str(result))
444n/a fn = fn.replace('"', '\\"')
445n/a return subprocess.call(['diff', '-u', fn, '@'], stdout=(subprocess.DEVNULL if verbose < 1 else None))
446n/a finally:
447n/a try:
448n/a os.remove("@")
449n/a except OSError:
450n/a pass