Python code coverage for Lib/lib2to3/tests/test_parser.py

#	count	content
1	n/a	"""Test suite for 2to3's parser and grammar files.
2	n/a
3	n/a	This is the place to add tests for changes to 2to3's grammar, such as those
4	n/a	merging the grammars for Python 2 and 3. In addition to specific tests for
5	n/a	parts of the grammar we've changed, we also make sure we can parse the
6	n/a	test_grammar.py files from both Python 2 and Python 3.
7	n/a	"""
8	n/a
9	n/a	# Testing imports
10	n/a	from . import support
11	n/a	from .support import driver
12	n/a	from test.support import verbose
13	n/a
14	n/a	# Python imports
15	n/a	import os
16	n/a	import shutil
17	n/a	import subprocess
18	n/a	import sys
19	n/a	import tempfile
20	n/a	import unittest
21	n/a	import warnings
22	n/a
23	n/a	# Local imports
24	n/a	from lib2to3.pgen2 import driver as pgen2_driver
25	n/a	from lib2to3.pgen2 import tokenize
26	n/a	from ..pgen2.parse import ParseError
27	n/a	from lib2to3.pygram import python_symbols as syms
28	n/a
29	n/a
30	n/a	class TestDriver(support.TestCase):
31	n/a
32	n/a	def test_formfeed(self):
33	n/a	s = """print 1\n\x0Cprint 2\n"""
34	n/a	t = driver.parse_string(s)
35	n/a	self.assertEqual(t.children[0].children[0].type, syms.print_stmt)
36	n/a	self.assertEqual(t.children[1].children[0].type, syms.print_stmt)
37	n/a
38	n/a
39	n/a	class TestPgen2Caching(support.TestCase):
40	n/a	def test_load_grammar_from_txt_file(self):
41	n/a	pgen2_driver.load_grammar(support.grammar_path, save=False, force=True)
42	n/a
43	n/a	def test_load_grammar_from_pickle(self):
44	n/a	# Make a copy of the grammar file in a temp directory we are
45	n/a	# guaranteed to be able to write to.
46	n/a	tmpdir = tempfile.mkdtemp()
47	n/a	try:
48	n/a	grammar_copy = os.path.join(
49	n/a	tmpdir, os.path.basename(support.grammar_path))
50	n/a	shutil.copy(support.grammar_path, grammar_copy)
51	n/a	pickle_name = pgen2_driver._generate_pickle_name(grammar_copy)
52	n/a
53	n/a	pgen2_driver.load_grammar(grammar_copy, save=True, force=True)
54	n/a	self.assertTrue(os.path.exists(pickle_name))
55	n/a
56	n/a	os.unlink(grammar_copy) # Only the pickle remains...
57	n/a	pgen2_driver.load_grammar(grammar_copy, save=False, force=False)
58	n/a	finally:
59	n/a	shutil.rmtree(tmpdir)
60	n/a
61	n/a	@unittest.skipIf(sys.executable is None, 'sys.executable required')
62	n/a	def test_load_grammar_from_subprocess(self):
63	n/a	tmpdir = tempfile.mkdtemp()
64	n/a	tmpsubdir = os.path.join(tmpdir, 'subdir')
65	n/a	try:
66	n/a	os.mkdir(tmpsubdir)
67	n/a	grammar_base = os.path.basename(support.grammar_path)
68	n/a	grammar_copy = os.path.join(tmpdir, grammar_base)
69	n/a	grammar_sub_copy = os.path.join(tmpsubdir, grammar_base)
70	n/a	shutil.copy(support.grammar_path, grammar_copy)
71	n/a	shutil.copy(support.grammar_path, grammar_sub_copy)
72	n/a	pickle_name = pgen2_driver._generate_pickle_name(grammar_copy)
73	n/a	pickle_sub_name = pgen2_driver._generate_pickle_name(
74	n/a	grammar_sub_copy)
75	n/a	self.assertNotEqual(pickle_name, pickle_sub_name)
76	n/a
77	n/a	# Generate a pickle file from this process.
78	n/a	pgen2_driver.load_grammar(grammar_copy, save=True, force=True)
79	n/a	self.assertTrue(os.path.exists(pickle_name))
80	n/a
81	n/a	# Generate a new pickle file in a subprocess with a most likely
82	n/a	# different hash randomization seed.
83	n/a	sub_env = dict(os.environ)
84	n/a	sub_env['PYTHONHASHSEED'] = 'random'
85	n/a	subprocess.check_call(
86	n/a	[sys.executable, '-c', """
87	n/a	from lib2to3.pgen2 import driver as pgen2_driver
88	n/a	pgen2_driver.load_grammar(%r, save=True, force=True)
89	n/a	""" % (grammar_sub_copy,)],
90	n/a	env=sub_env)
91	n/a	self.assertTrue(os.path.exists(pickle_sub_name))
92	n/a
93	n/a	with open(pickle_name, 'rb') as pickle_f_1, \
94	n/a	open(pickle_sub_name, 'rb') as pickle_f_2:
95	n/a	self.assertEqual(
96	n/a	pickle_f_1.read(), pickle_f_2.read(),
97	n/a	msg='Grammar caches generated using different hash seeds'
98	n/a	' were not identical.')
99	n/a	finally:
100	n/a	shutil.rmtree(tmpdir)
101	n/a
102	n/a
103	n/a
104	n/a	class GrammarTest(support.TestCase):
105	n/a	def validate(self, code):
106	n/a	support.parse_string(code)
107	n/a
108	n/a	def invalid_syntax(self, code):
109	n/a	try:
110	n/a	self.validate(code)
111	n/a	except ParseError:
112	n/a	pass
113	n/a	else:
114	n/a	raise AssertionError("Syntax shouldn't have been valid")
115	n/a
116	n/a
117	n/a	class TestMatrixMultiplication(GrammarTest):
118	n/a	def test_matrix_multiplication_operator(self):
119	n/a	self.validate("a @ b")
120	n/a	self.validate("a @= b")
121	n/a
122	n/a
123	n/a	class TestYieldFrom(GrammarTest):
124	n/a	def test_yield_from(self):
125	n/a	self.validate("yield from x")
126	n/a	self.validate("(yield from x) + y")
127	n/a	self.invalid_syntax("yield from")
128	n/a
129	n/a
130	n/a	class TestAsyncAwait(GrammarTest):
131	n/a	def test_await_expr(self):
132	n/a	self.validate("""async def foo():
133	n/a	await x
134	n/a	""")
135	n/a
136	n/a	self.validate("""async def foo():
137	n/a	[i async for i in b]
138	n/a	""")
139	n/a
140	n/a	self.validate("""async def foo():
141	n/a	{i for i in b
142	n/a	async for i in a if await i
143	n/a	for b in i}
144	n/a	""")
145	n/a
146	n/a	self.validate("""async def foo():
147	n/a	[await i for i in b if await c]
148	n/a	""")
149	n/a
150	n/a	self.validate("""async def foo():
151	n/a	[ i for i in b if c]
152	n/a	""")
153	n/a
154	n/a	self.validate("""async def foo():
155	n/a
156	n/a	def foo(): pass
157	n/a
158	n/a	def foo(): pass
159	n/a
160	n/a	await x
161	n/a	""")
162	n/a
163	n/a	self.validate("""async def foo(): return await a""")
164	n/a
165	n/a	self.validate("""def foo():
166	n/a	def foo(): pass
167	n/a	async def foo(): await x
168	n/a	""")
169	n/a
170	n/a	self.invalid_syntax("await x")
171	n/a	self.invalid_syntax("""def foo():
172	n/a	await x""")
173	n/a
174	n/a	self.invalid_syntax("""def foo():
175	n/a	def foo(): pass
176	n/a	async def foo(): pass
177	n/a	await x
178	n/a	""")
179	n/a
180	n/a	def test_async_var(self):
181	n/a	self.validate("""async = 1""")
182	n/a	self.validate("""await = 1""")
183	n/a	self.validate("""def async(): pass""")
184	n/a
185	n/a	def test_async_with(self):
186	n/a	self.validate("""async def foo():
187	n/a	async for a in b: pass""")
188	n/a
189	n/a	self.invalid_syntax("""def foo():
190	n/a	async for a in b: pass""")
191	n/a
192	n/a	def test_async_for(self):
193	n/a	self.validate("""async def foo():
194	n/a	async with a: pass""")
195	n/a
196	n/a	self.invalid_syntax("""def foo():
197	n/a	async with a: pass""")
198	n/a
199	n/a
200	n/a	class TestRaiseChanges(GrammarTest):
201	n/a	def test_2x_style_1(self):
202	n/a	self.validate("raise")
203	n/a
204	n/a	def test_2x_style_2(self):
205	n/a	self.validate("raise E, V")
206	n/a
207	n/a	def test_2x_style_3(self):
208	n/a	self.validate("raise E, V, T")
209	n/a
210	n/a	def test_2x_style_invalid_1(self):
211	n/a	self.invalid_syntax("raise E, V, T, Z")
212	n/a
213	n/a	def test_3x_style(self):
214	n/a	self.validate("raise E1 from E2")
215	n/a
216	n/a	def test_3x_style_invalid_1(self):
217	n/a	self.invalid_syntax("raise E, V from E1")
218	n/a
219	n/a	def test_3x_style_invalid_2(self):
220	n/a	self.invalid_syntax("raise E from E1, E2")
221	n/a
222	n/a	def test_3x_style_invalid_3(self):
223	n/a	self.invalid_syntax("raise from E1, E2")
224	n/a
225	n/a	def test_3x_style_invalid_4(self):
226	n/a	self.invalid_syntax("raise E from")
227	n/a
228	n/a
229	n/a	# Modelled after Lib/test/test_grammar.py:TokenTests.test_funcdef issue2292
230	n/a	# and Lib/test/text_parser.py test_list_displays, test_set_displays,
231	n/a	# test_dict_displays, test_argument_unpacking, ... changes.
232	n/a	class TestUnpackingGeneralizations(GrammarTest):
233	n/a	def test_mid_positional_star(self):
234	n/a	self.validate("""func(1, *(2, 3), 4)""")
235	n/a
236	n/a	def test_double_star_dict_literal(self):
237	n/a	self.validate("""func(**{'eggs':'scrambled', 'spam':'fried'})""")
238	n/a
239	n/a	def test_double_star_dict_literal_after_keywords(self):
240	n/a	self.validate("""func(spam='fried', **{'eggs':'scrambled'})""")
241	n/a
242	n/a	def test_list_display(self):
243	n/a	self.validate("""[{2}, 3, [4]]""")
244	n/a
245	n/a	def test_set_display(self):
246	n/a	self.validate("""{{2}, 3, [4]}""")
247	n/a
248	n/a	def test_dict_display_1(self):
249	n/a	self.validate("""{**{}}""")
250	n/a
251	n/a	def test_dict_display_2(self):
252	n/a	self.validate("""{{}, 3:4, {5:6, 7:8}}""")
253	n/a
254	n/a	def test_argument_unpacking_1(self):
255	n/a	self.validate("""f(a, b, c, d)""")
256	n/a
257	n/a	def test_argument_unpacking_2(self):
258	n/a	self.validate("""f(a, b)""")
259	n/a
260	n/a	def test_argument_unpacking_3(self):
261	n/a	self.validate("""f(2, a, b, b, c, **d)""")
262	n/a
263	n/a
264	n/a	# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef
265	n/a	class TestFunctionAnnotations(GrammarTest):
266	n/a	def test_1(self):
267	n/a	self.validate("""def f(x) -> list: pass""")
268	n/a
269	n/a	def test_2(self):
270	n/a	self.validate("""def f(x:int): pass""")
271	n/a
272	n/a	def test_3(self):
273	n/a	self.validate("""def f(*x:str): pass""")
274	n/a
275	n/a	def test_4(self):
276	n/a	self.validate("""def f(**x:float): pass""")
277	n/a
278	n/a	def test_5(self):
279	n/a	self.validate("""def f(x, y:1+2): pass""")
280	n/a
281	n/a	def test_6(self):
282	n/a	self.validate("""def f(a, (b:1, c:2, d)): pass""")
283	n/a
284	n/a	def test_7(self):
285	n/a	self.validate("""def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass""")
286	n/a
287	n/a	def test_8(self):
288	n/a	s = """def f(a, (b:1, c:2, d), e:3=4, f=5,
289	n/a	g:6, h:7, i=8, j:9=10, *k:11) -> 12: pass"""
290	n/a	self.validate(s)
291	n/a
292	n/a
293	n/a	# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot
294	n/a	class TestVarAnnotations(GrammarTest):
295	n/a	def test_1(self):
296	n/a	self.validate("var1: int = 5")
297	n/a
298	n/a	def test_2(self):
299	n/a	self.validate("var2: [int, str]")
300	n/a
301	n/a	def test_3(self):
302	n/a	self.validate("def f():\n"
303	n/a	" st: str = 'Hello'\n"
304	n/a	" a.b: int = (1, 2)\n"
305	n/a	" return st\n")
306	n/a
307	n/a	def test_4(self):
308	n/a	self.validate("def fbad():\n"
309	n/a	" x: int\n"
310	n/a	" print(x)\n")
311	n/a
312	n/a	def test_5(self):
313	n/a	self.validate("class C:\n"
314	n/a	" x: int\n"
315	n/a	" s: str = 'attr'\n"
316	n/a	" z = 2\n"
317	n/a	" def __init__(self, x):\n"
318	n/a	" self.x: int = x\n")
319	n/a
320	n/a	def test_6(self):
321	n/a	self.validate("lst: List[int] = []")
322	n/a
323	n/a	class TestExcept(GrammarTest):
324	n/a	def test_new(self):
325	n/a	s = """
326	n/a	try:
327	n/a	x
328	n/a	except E as N:
329	n/a	y"""
330	n/a	self.validate(s)
331	n/a
332	n/a	def test_old(self):
333	n/a	s = """
334	n/a	try:
335	n/a	x
336	n/a	except E, N:
337	n/a	y"""
338	n/a	self.validate(s)
339	n/a
340	n/a
341	n/a	# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms
342	n/a	class TestSetLiteral(GrammarTest):
343	n/a	def test_1(self):
344	n/a	self.validate("""x = {'one'}""")
345	n/a
346	n/a	def test_2(self):
347	n/a	self.validate("""x = {'one', 1,}""")
348	n/a
349	n/a	def test_3(self):
350	n/a	self.validate("""x = {'one', 'two', 'three'}""")
351	n/a
352	n/a	def test_4(self):
353	n/a	self.validate("""x = {2, 3, 4,}""")
354	n/a
355	n/a
356	n/a	class TestNumericLiterals(GrammarTest):
357	n/a	def test_new_octal_notation(self):
358	n/a	self.validate("""0o7777777777777""")
359	n/a	self.invalid_syntax("""0o7324528887""")
360	n/a
361	n/a	def test_new_binary_notation(self):
362	n/a	self.validate("""0b101010""")
363	n/a	self.invalid_syntax("""0b0101021""")
364	n/a
365	n/a
366	n/a	class TestClassDef(GrammarTest):
367	n/a	def test_new_syntax(self):
368	n/a	self.validate("class B(t=7): pass")
369	n/a	self.validate("class B(t, *args): pass")
370	n/a	self.validate("class B(t, **kwargs): pass")
371	n/a	self.validate("class B(t, args, *kwargs): pass")
372	n/a	self.validate("class B(t, y=9, args, *kwargs): pass")
373	n/a
374	n/a
375	n/a	class TestParserIdempotency(support.TestCase):
376	n/a
377	n/a	"""A cut-down version of pytree_idempotency.py."""
378	n/a
379	n/a	# Issue 13125
380	n/a	@unittest.expectedFailure
381	n/a	def test_all_project_files(self):
382	n/a	for filepath in support.all_project_files():
383	n/a	with open(filepath, "rb") as fp:
384	n/a	encoding = tokenize.detect_encoding(fp.readline)[0]
385	n/a	self.assertIsNotNone(encoding,
386	n/a	"can't detect encoding for %s" % filepath)
387	n/a	with open(filepath, "r", encoding=encoding) as fp:
388	n/a	source = fp.read()
389	n/a	try:
390	n/a	tree = driver.parse_string(source)
391	n/a	except ParseError as err:
392	n/a	if verbose > 0:
393	n/a	warnings.warn('ParseError on file %s (%s)' % (filepath, err))
394	n/a	continue
395	n/a	new = str(tree)
396	n/a	x = diff(filepath, new)
397	n/a	if x:
398	n/a	self.fail("Idempotency failed: %s" % filepath)
399	n/a
400	n/a	def test_extended_unpacking(self):
401	n/a	driver.parse_string("a, *b, c = x\n")
402	n/a	driver.parse_string("[*a, b] = x\n")
403	n/a	driver.parse_string("(z, *y, w) = m\n")
404	n/a	driver.parse_string("for *z, m in d: pass\n")
405	n/a
406	n/a
407	n/a	class TestLiterals(GrammarTest):
408	n/a
409	n/a	def validate(self, s):
410	n/a	driver.parse_string(support.dedent(s) + "\n\n")
411	n/a
412	n/a	def test_multiline_bytes_literals(self):
413	n/a	s = """
414	n/a	md5test(b"\xaa" * 80,
415	n/a	(b"Test Using Larger Than Block-Size Key "
416	n/a	b"and Larger Than One Block-Size Data"),
417	n/a	"6f630fad67cda0ee1fb1f562db3aa53e")
418	n/a	"""
419	n/a	self.validate(s)
420	n/a
421	n/a	def test_multiline_bytes_tripquote_literals(self):
422	n/a	s = '''
423	n/a	b"""
424	n/a	<?xml version="1.0" encoding="UTF-8"?>
425	n/a	<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN">
426	n/a	"""
427	n/a	'''
428	n/a	self.validate(s)
429	n/a
430	n/a	def test_multiline_str_literals(self):
431	n/a	s = """
432	n/a	md5test("\xaa" * 80,
433	n/a	("Test Using Larger Than Block-Size Key "
434	n/a	"and Larger Than One Block-Size Data"),
435	n/a	"6f630fad67cda0ee1fb1f562db3aa53e")
436	n/a	"""
437	n/a	self.validate(s)
438	n/a
439	n/a
440	n/a	def diff(fn, result):
441	n/a	try:
442	n/a	with open('@', 'w') as f:
443	n/a	f.write(str(result))
444	n/a	fn = fn.replace('"', '\\"')
445	n/a	return subprocess.call(['diff', '-u', fn, '@'], stdout=(subprocess.DEVNULL if verbose < 1 else None))
446	n/a	finally:
447	n/a	try:
448	n/a	os.remove("@")
449	n/a	except OSError:
450	n/a	pass