Python code coverage for Lib/test/test_string_literals.py

#	count	content
1	n/a	r"""Test correct treatment of various string literals by the parser.
2	n/a
3	n/a	There are four types of string literals:
4	n/a
5	n/a	'abc' -- normal str
6	n/a	r'abc' -- raw str
7	n/a	b'xyz' -- normal bytes
8	n/a	br'xyz' \| rb'xyz' -- raw bytes
9	n/a
10	n/a	The difference between normal and raw strings is of course that in a
11	n/a	raw string, \ escapes (while still used to determine the end of the
12	n/a	literal) are not interpreted, so that r'\x00' contains four
13	n/a	characters: a backslash, an x, and two zeros; while '\x00' contains a
14	n/a	single character (code point zero).
15	n/a
16	n/a	The tricky thing is what should happen when non-ASCII bytes are used
17	n/a	inside literals. For bytes literals, this is considered illegal. But
18	n/a	for str literals, those bytes are supposed to be decoded using the
19	n/a	encoding declared for the file (UTF-8 by default).
20	n/a
21	n/a	We have to test this with various file encodings. We also test it with
22	n/a	exec()/eval(), which uses a different code path.
23	n/a
24	n/a	This file is really about correct treatment of encodings and
25	n/a	backslashes. It doesn't concern itself with issues like single
26	n/a	vs. double quotes or singly- vs. triply-quoted strings: that's dealt
27	n/a	with elsewhere (I assume).
28	n/a	"""
29	n/a
30	n/a	import os
31	n/a	import sys
32	n/a	import shutil
33	n/a	import tempfile
34	n/a	import warnings
35	n/a	import unittest
36	n/a
37	n/a
38	n/a	TEMPLATE = r"""# coding: %s
39	n/a	a = 'x'
40	n/a	assert ord(a) == 120
41	n/a	b = '\x01'
42	n/a	assert ord(b) == 1
43	n/a	c = r'\x01'
44	n/a	assert list(map(ord, c)) == [92, 120, 48, 49]
45	n/a	d = '\x81'
46	n/a	assert ord(d) == 0x81
47	n/a	e = r'\x81'
48	n/a	assert list(map(ord, e)) == [92, 120, 56, 49]
49	n/a	f = '\u1881'
50	n/a	assert ord(f) == 0x1881
51	n/a	g = r'\u1881'
52	n/a	assert list(map(ord, g)) == [92, 117, 49, 56, 56, 49]
53	n/a	h = '\U0001d120'
54	n/a	assert ord(h) == 0x1d120
55	n/a	i = r'\U0001d120'
56	n/a	assert list(map(ord, i)) == [92, 85, 48, 48, 48, 49, 100, 49, 50, 48]
57	n/a	"""
58	n/a
59	n/a
60	n/a	def byte(i):
61	n/a	return bytes([i])
62	n/a
63	n/a
64	n/a	class TestLiterals(unittest.TestCase):
65	n/a
66	n/a	def setUp(self):
67	n/a	self.save_path = sys.path[:]
68	n/a	self.tmpdir = tempfile.mkdtemp()
69	n/a	sys.path.insert(0, self.tmpdir)
70	n/a
71	n/a	def tearDown(self):
72	n/a	sys.path[:] = self.save_path
73	n/a	shutil.rmtree(self.tmpdir, ignore_errors=True)
74	n/a
75	n/a	def test_template(self):
76	n/a	# Check that the template doesn't contain any non-printables
77	n/a	# except for \n.
78	n/a	for c in TEMPLATE:
79	n/a	assert c == '\n' or ' ' <= c <= '~', repr(c)
80	n/a
81	n/a	def test_eval_str_normal(self):
82	n/a	self.assertEqual(eval(""" 'x' """), 'x')
83	n/a	self.assertEqual(eval(r""" '\x01' """), chr(1))
84	n/a	self.assertEqual(eval(""" '\x01' """), chr(1))
85	n/a	self.assertEqual(eval(r""" '\x81' """), chr(0x81))
86	n/a	self.assertEqual(eval(""" '\x81' """), chr(0x81))
87	n/a	self.assertEqual(eval(r""" '\u1881' """), chr(0x1881))
88	n/a	self.assertEqual(eval(""" '\u1881' """), chr(0x1881))
89	n/a	self.assertEqual(eval(r""" '\U0001d120' """), chr(0x1d120))
90	n/a	self.assertEqual(eval(""" '\U0001d120' """), chr(0x1d120))
91	n/a
92	n/a	def test_eval_str_incomplete(self):
93	n/a	self.assertRaises(SyntaxError, eval, r""" '\x' """)
94	n/a	self.assertRaises(SyntaxError, eval, r""" '\x0' """)
95	n/a	self.assertRaises(SyntaxError, eval, r""" '\u' """)
96	n/a	self.assertRaises(SyntaxError, eval, r""" '\u0' """)
97	n/a	self.assertRaises(SyntaxError, eval, r""" '\u00' """)
98	n/a	self.assertRaises(SyntaxError, eval, r""" '\u000' """)
99	n/a	self.assertRaises(SyntaxError, eval, r""" '\U' """)
100	n/a	self.assertRaises(SyntaxError, eval, r""" '\U0' """)
101	n/a	self.assertRaises(SyntaxError, eval, r""" '\U00' """)
102	n/a	self.assertRaises(SyntaxError, eval, r""" '\U000' """)
103	n/a	self.assertRaises(SyntaxError, eval, r""" '\U0000' """)
104	n/a	self.assertRaises(SyntaxError, eval, r""" '\U00000' """)
105	n/a	self.assertRaises(SyntaxError, eval, r""" '\U000000' """)
106	n/a	self.assertRaises(SyntaxError, eval, r""" '\U0000000' """)
107	n/a
108	n/a	def test_eval_str_invalid_escape(self):
109	n/a	for b in range(1, 128):
110	n/a	if b in b"""\n\r"'01234567NU\\abfnrtuvx""":
111	n/a	continue
112	n/a	with self.assertWarns(DeprecationWarning):
113	n/a	self.assertEqual(eval(r"'\%c'" % b), '\\' + chr(b))
114	n/a
115	n/a	with warnings.catch_warnings(record=True) as w:
116	n/a	warnings.simplefilter('always', category=DeprecationWarning)
117	n/a	eval("'''\n\\z'''")
118	n/a	self.assertEqual(len(w), 1)
119	n/a	self.assertEqual(w[0].filename, '<string>')
120	n/a	self.assertEqual(w[0].lineno, 2)
121	n/a
122	n/a	with warnings.catch_warnings(record=True) as w:
123	n/a	warnings.simplefilter('error', category=DeprecationWarning)
124	n/a	with self.assertRaises(SyntaxError) as cm:
125	n/a	eval("'''\n\\z'''")
126	n/a	exc = cm.exception
127	n/a	self.assertEqual(w, [])
128	n/a	self.assertEqual(exc.filename, '<string>')
129	n/a	self.assertEqual(exc.lineno, 2)
130	n/a
131	n/a	def test_eval_str_raw(self):
132	n/a	self.assertEqual(eval(""" r'x' """), 'x')
133	n/a	self.assertEqual(eval(r""" r'\x01' """), '\\' + 'x01')
134	n/a	self.assertEqual(eval(""" r'\x01' """), chr(1))
135	n/a	self.assertEqual(eval(r""" r'\x81' """), '\\' + 'x81')
136	n/a	self.assertEqual(eval(""" r'\x81' """), chr(0x81))
137	n/a	self.assertEqual(eval(r""" r'\u1881' """), '\\' + 'u1881')
138	n/a	self.assertEqual(eval(""" r'\u1881' """), chr(0x1881))
139	n/a	self.assertEqual(eval(r""" r'\U0001d120' """), '\\' + 'U0001d120')
140	n/a	self.assertEqual(eval(""" r'\U0001d120' """), chr(0x1d120))
141	n/a
142	n/a	def test_eval_bytes_normal(self):
143	n/a	self.assertEqual(eval(""" b'x' """), b'x')
144	n/a	self.assertEqual(eval(r""" b'\x01' """), byte(1))
145	n/a	self.assertEqual(eval(""" b'\x01' """), byte(1))
146	n/a	self.assertEqual(eval(r""" b'\x81' """), byte(0x81))
147	n/a	self.assertRaises(SyntaxError, eval, """ b'\x81' """)
148	n/a	self.assertEqual(eval(r""" br'\u1881' """), b'\\' + b'u1881')
149	n/a	self.assertRaises(SyntaxError, eval, """ b'\u1881' """)
150	n/a	self.assertEqual(eval(r""" br'\U0001d120' """), b'\\' + b'U0001d120')
151	n/a	self.assertRaises(SyntaxError, eval, """ b'\U0001d120' """)
152	n/a
153	n/a	def test_eval_bytes_incomplete(self):
154	n/a	self.assertRaises(SyntaxError, eval, r""" b'\x' """)
155	n/a	self.assertRaises(SyntaxError, eval, r""" b'\x0' """)
156	n/a
157	n/a	def test_eval_bytes_invalid_escape(self):
158	n/a	for b in range(1, 128):
159	n/a	if b in b"""\n\r"'01234567\\abfnrtvx""":
160	n/a	continue
161	n/a	with self.assertWarns(DeprecationWarning):
162	n/a	self.assertEqual(eval(r"b'\%c'" % b), b'\\' + bytes([b]))
163	n/a
164	n/a	with warnings.catch_warnings(record=True) as w:
165	n/a	warnings.simplefilter('always', category=DeprecationWarning)
166	n/a	eval("b'''\n\\z'''")
167	n/a	self.assertEqual(len(w), 1)
168	n/a	self.assertEqual(w[0].filename, '<string>')
169	n/a	self.assertEqual(w[0].lineno, 2)
170	n/a
171	n/a	with warnings.catch_warnings(record=True) as w:
172	n/a	warnings.simplefilter('error', category=DeprecationWarning)
173	n/a	with self.assertRaises(SyntaxError) as cm:
174	n/a	eval("b'''\n\\z'''")
175	n/a	exc = cm.exception
176	n/a	self.assertEqual(w, [])
177	n/a	self.assertEqual(exc.filename, '<string>')
178	n/a	self.assertEqual(exc.lineno, 2)
179	n/a
180	n/a	def test_eval_bytes_raw(self):
181	n/a	self.assertEqual(eval(""" br'x' """), b'x')
182	n/a	self.assertEqual(eval(""" rb'x' """), b'x')
183	n/a	self.assertEqual(eval(r""" br'\x01' """), b'\\' + b'x01')
184	n/a	self.assertEqual(eval(r""" rb'\x01' """), b'\\' + b'x01')
185	n/a	self.assertEqual(eval(""" br'\x01' """), byte(1))
186	n/a	self.assertEqual(eval(""" rb'\x01' """), byte(1))
187	n/a	self.assertEqual(eval(r""" br'\x81' """), b"\\" + b"x81")
188	n/a	self.assertEqual(eval(r""" rb'\x81' """), b"\\" + b"x81")
189	n/a	self.assertRaises(SyntaxError, eval, """ br'\x81' """)
190	n/a	self.assertRaises(SyntaxError, eval, """ rb'\x81' """)
191	n/a	self.assertEqual(eval(r""" br'\u1881' """), b"\\" + b"u1881")
192	n/a	self.assertEqual(eval(r""" rb'\u1881' """), b"\\" + b"u1881")
193	n/a	self.assertRaises(SyntaxError, eval, """ br'\u1881' """)
194	n/a	self.assertRaises(SyntaxError, eval, """ rb'\u1881' """)
195	n/a	self.assertEqual(eval(r""" br'\U0001d120' """), b"\\" + b"U0001d120")
196	n/a	self.assertEqual(eval(r""" rb'\U0001d120' """), b"\\" + b"U0001d120")
197	n/a	self.assertRaises(SyntaxError, eval, """ br'\U0001d120' """)
198	n/a	self.assertRaises(SyntaxError, eval, """ rb'\U0001d120' """)
199	n/a	self.assertRaises(SyntaxError, eval, """ bb'' """)
200	n/a	self.assertRaises(SyntaxError, eval, """ rr'' """)
201	n/a	self.assertRaises(SyntaxError, eval, """ brr'' """)
202	n/a	self.assertRaises(SyntaxError, eval, """ bbr'' """)
203	n/a	self.assertRaises(SyntaxError, eval, """ rrb'' """)
204	n/a	self.assertRaises(SyntaxError, eval, """ rbb'' """)
205	n/a
206	n/a	def test_eval_str_u(self):
207	n/a	self.assertEqual(eval(""" u'x' """), 'x')
208	n/a	self.assertEqual(eval(""" U'\u00e4' """), 'Ã¤')
209	n/a	self.assertEqual(eval(""" u'\N{LATIN SMALL LETTER A WITH DIAERESIS}' """), 'Ã¤')
210	n/a	self.assertRaises(SyntaxError, eval, """ ur'' """)
211	n/a	self.assertRaises(SyntaxError, eval, """ ru'' """)
212	n/a	self.assertRaises(SyntaxError, eval, """ bu'' """)
213	n/a	self.assertRaises(SyntaxError, eval, """ ub'' """)
214	n/a
215	n/a	def check_encoding(self, encoding, extra=""):
216	n/a	modname = "xx_" + encoding.replace("-", "_")
217	n/a	fn = os.path.join(self.tmpdir, modname + ".py")
218	n/a	f = open(fn, "w", encoding=encoding)
219	n/a	try:
220	n/a	f.write(TEMPLATE % encoding)
221	n/a	f.write(extra)
222	n/a	finally:
223	n/a	f.close()
224	n/a	__import__(modname)
225	n/a	del sys.modules[modname]
226	n/a
227	n/a	def test_file_utf_8(self):
228	n/a	extra = "z = '\u1234'; assert ord(z) == 0x1234\n"
229	n/a	self.check_encoding("utf-8", extra)
230	n/a
231	n/a	def test_file_utf_8_error(self):
232	n/a	extra = "b'\x80'\n"
233	n/a	self.assertRaises(SyntaxError, self.check_encoding, "utf-8", extra)
234	n/a
235	n/a	def test_file_utf8(self):
236	n/a	self.check_encoding("utf-8")
237	n/a
238	n/a	def test_file_iso_8859_1(self):
239	n/a	self.check_encoding("iso-8859-1")
240	n/a
241	n/a	def test_file_latin_1(self):
242	n/a	self.check_encoding("latin-1")
243	n/a
244	n/a	def test_file_latin9(self):
245	n/a	self.check_encoding("latin9")
246	n/a
247	n/a
248	n/a	if __name__ == "__main__":
249	n/a	unittest.main()