Python code coverage for Lib/test/test_shlex.py

#	count	content
1	n/a	import io
2	n/a	import shlex
3	n/a	import string
4	n/a	import unittest
5	n/a
6	n/a
7	n/a
8	n/a	# The original test data set was from shellwords, by Hartmut Goebel.
9	n/a
10	n/a	data = r"""x\|x\|
11	n/a	foo bar\|foo\|bar\|
12	n/a	foo bar\|foo\|bar\|
13	n/a	foo bar \|foo\|bar\|
14	n/a	foo bar bla fasel\|foo\|bar\|bla\|fasel\|
15	n/a	x y z xxxx\|x\|y\|z\|xxxx\|
16	n/a	\x bar\|\\|x\|bar\|
17	n/a	\ x bar\|\\|x\|bar\|
18	n/a	\ bar\|\\|bar\|
19	n/a	foo \x bar\|foo\|\\|x\|bar\|
20	n/a	foo \ x bar\|foo\|\\|x\|bar\|
21	n/a	foo \ bar\|foo\|\\|bar\|
22	n/a	foo "bar" bla\|foo\|"bar"\|bla\|
23	n/a	"foo" "bar" "bla"\|"foo"\|"bar"\|"bla"\|
24	n/a	"foo" bar "bla"\|"foo"\|bar\|"bla"\|
25	n/a	"foo" bar bla\|"foo"\|bar\|bla\|
26	n/a	foo 'bar' bla\|foo\|'bar'\|bla\|
27	n/a	'foo' 'bar' 'bla'\|'foo'\|'bar'\|'bla'\|
28	n/a	'foo' bar 'bla'\|'foo'\|bar\|'bla'\|
29	n/a	'foo' bar bla\|'foo'\|bar\|bla\|
30	n/a	blurb foo"bar"bar"fasel" baz\|blurb\|foo"bar"bar"fasel"\|baz\|
31	n/a	blurb foo'bar'bar'fasel' baz\|blurb\|foo'bar'bar'fasel'\|baz\|
32	n/a	""\|""\|
33	n/a	''\|''\|
34	n/a	foo "" bar\|foo\|""\|bar\|
35	n/a	foo '' bar\|foo\|''\|bar\|
36	n/a	foo "" "" "" bar\|foo\|""\|""\|""\|bar\|
37	n/a	foo '' '' '' bar\|foo\|''\|''\|''\|bar\|
38	n/a	\""\|\\|""\|
39	n/a	"\"\|"\"\|
40	n/a	"foo\ bar"\|"foo\ bar"\|
41	n/a	"foo\\ bar"\|"foo\\ bar"\|
42	n/a	"foo\\ bar\"\|"foo\\ bar\"\|
43	n/a	"foo\\" bar\""\|"foo\\"\|bar\|\\|""\|
44	n/a	"foo\\ bar\" dfadf"\|"foo\\ bar\"\|dfadf"\|
45	n/a	"foo\\\ bar\" dfadf"\|"foo\\\ bar\"\|dfadf"\|
46	n/a	"foo\\\x bar\" dfadf"\|"foo\\\x bar\"\|dfadf"\|
47	n/a	"foo\x bar\" dfadf"\|"foo\x bar\"\|dfadf"\|
48	n/a	\''\|\\|''\|
49	n/a	'foo\ bar'\|'foo\ bar'\|
50	n/a	'foo\\ bar'\|'foo\\ bar'\|
51	n/a	"foo\\\x bar\" df'a\ 'df'\|"foo\\\x bar\"\|df'a\|\\|'df'\|
52	n/a	\"foo"\|\\|"foo"\|
53	n/a	\"foo"\x\|\\|"foo"\|\\|x\|
54	n/a	"foo\x"\|"foo\x"\|
55	n/a	"foo\ "\|"foo\ "\|
56	n/a	foo\ xx\|foo\|\\|xx\|
57	n/a	foo\ x\x\|foo\|\\|x\|\\|x\|
58	n/a	foo\ x\x\""\|foo\|\\|x\|\\|x\|\\|""\|
59	n/a	"foo\ x\x"\|"foo\ x\x"\|
60	n/a	"foo\ x\x\\"\|"foo\ x\x\\"\|
61	n/a	"foo\ x\x\\""foobar"\|"foo\ x\x\\"\|"foobar"\|
62	n/a	"foo\ x\x\\"\''"foobar"\|"foo\ x\x\\"\|\\|''\|"foobar"\|
63	n/a	"foo\ x\x\\"\'"fo'obar"\|"foo\ x\x\\"\|\\|'"fo'\|obar"\|
64	n/a	"foo\ x\x\\"\'"fo'obar" 'don'\''t'\|"foo\ x\x\\"\|\\|'"fo'\|obar"\|'don'\|\\|''\|t'\|
65	n/a	'foo\ bar'\|'foo\ bar'\|
66	n/a	'foo\\ bar'\|'foo\\ bar'\|
67	n/a	foo\ bar\|foo\|\\|bar\|
68	n/a	foo#bar\nbaz\|foobaz\|
69	n/a	:-) ;-)\|:\|-\|)\|;\|-\|)\|
70	n/a	Ã¡Ã©ÃÃ³Ãº\|Ã¡\|Ã©\|Ã\|Ã³\|Ãº\|
71	n/a	"""
72	n/a
73	n/a	posix_data = r"""x\|x\|
74	n/a	foo bar\|foo\|bar\|
75	n/a	foo bar\|foo\|bar\|
76	n/a	foo bar \|foo\|bar\|
77	n/a	foo bar bla fasel\|foo\|bar\|bla\|fasel\|
78	n/a	x y z xxxx\|x\|y\|z\|xxxx\|
79	n/a	\x bar\|x\|bar\|
80	n/a	\ x bar\| x\|bar\|
81	n/a	\ bar\| bar\|
82	n/a	foo \x bar\|foo\|x\|bar\|
83	n/a	foo \ x bar\|foo\| x\|bar\|
84	n/a	foo \ bar\|foo\| bar\|
85	n/a	foo "bar" bla\|foo\|bar\|bla\|
86	n/a	"foo" "bar" "bla"\|foo\|bar\|bla\|
87	n/a	"foo" bar "bla"\|foo\|bar\|bla\|
88	n/a	"foo" bar bla\|foo\|bar\|bla\|
89	n/a	foo 'bar' bla\|foo\|bar\|bla\|
90	n/a	'foo' 'bar' 'bla'\|foo\|bar\|bla\|
91	n/a	'foo' bar 'bla'\|foo\|bar\|bla\|
92	n/a	'foo' bar bla\|foo\|bar\|bla\|
93	n/a	blurb foo"bar"bar"fasel" baz\|blurb\|foobarbarfasel\|baz\|
94	n/a	blurb foo'bar'bar'fasel' baz\|blurb\|foobarbarfasel\|baz\|
95	n/a	""\|\|
96	n/a	''\|\|
97	n/a	foo "" bar\|foo\|\|bar\|
98	n/a	foo '' bar\|foo\|\|bar\|
99	n/a	foo "" "" "" bar\|foo\|\|\|\|bar\|
100	n/a	foo '' '' '' bar\|foo\|\|\|\|bar\|
101	n/a	\"\|"\|
102	n/a	"\""\|"\|
103	n/a	"foo\ bar"\|foo\ bar\|
104	n/a	"foo\\ bar"\|foo\ bar\|
105	n/a	"foo\\ bar\""\|foo\ bar"\|
106	n/a	"foo\\" bar\"\|foo\\|bar"\|
107	n/a	"foo\\ bar\" dfadf"\|foo\ bar" dfadf\|
108	n/a	"foo\\\ bar\" dfadf"\|foo\\ bar" dfadf\|
109	n/a	"foo\\\x bar\" dfadf"\|foo\\x bar" dfadf\|
110	n/a	"foo\x bar\" dfadf"\|foo\x bar" dfadf\|
111	n/a	\'\|'\|
112	n/a	'foo\ bar'\|foo\ bar\|
113	n/a	'foo\\ bar'\|foo\\ bar\|
114	n/a	"foo\\\x bar\" df'a\ 'df"\|foo\\x bar" df'a\ 'df\|
115	n/a	\"foo\|"foo\|
116	n/a	\"foo\x\|"foox\|
117	n/a	"foo\x"\|foo\x\|
118	n/a	"foo\ "\|foo\ \|
119	n/a	foo\ xx\|foo xx\|
120	n/a	foo\ x\x\|foo xx\|
121	n/a	foo\ x\x\"\|foo xx"\|
122	n/a	"foo\ x\x"\|foo\ x\x\|
123	n/a	"foo\ x\x\\"\|foo\ x\x\\|
124	n/a	"foo\ x\x\\""foobar"\|foo\ x\x\foobar\|
125	n/a	"foo\ x\x\\"\'"foobar"\|foo\ x\x\'foobar\|
126	n/a	"foo\ x\x\\"\'"fo'obar"\|foo\ x\x\'fo'obar\|
127	n/a	"foo\ x\x\\"\'"fo'obar" 'don'\''t'\|foo\ x\x\'fo'obar\|don't\|
128	n/a	"foo\ x\x\\"\'"fo'obar" 'don'\''t' \\\|foo\ x\x\'fo'obar\|don't\|\\|
129	n/a	'foo\ bar'\|foo\ bar\|
130	n/a	'foo\\ bar'\|foo\\ bar\|
131	n/a	foo\ bar\|foo bar\|
132	n/a	foo#bar\nbaz\|foo\|baz\|
133	n/a	:-) ;-)\|:-)\|;-)\|
134	n/a	Ã¡Ã©ÃÃ³Ãº\|Ã¡Ã©ÃÃ³Ãº\|
135	n/a	"""
136	n/a
137	n/a	class ShlexTest(unittest.TestCase):
138	n/a	def setUp(self):
139	n/a	self.data = [x.split("\|")[:-1]
140	n/a	for x in data.splitlines()]
141	n/a	self.posix_data = [x.split("\|")[:-1]
142	n/a	for x in posix_data.splitlines()]
143	n/a	for item in self.data:
144	n/a	item[0] = item[0].replace(r"\n", "\n")
145	n/a	for item in self.posix_data:
146	n/a	item[0] = item[0].replace(r"\n", "\n")
147	n/a
148	n/a	def splitTest(self, data, comments):
149	n/a	for i in range(len(data)):
150	n/a	l = shlex.split(data[i][0], comments=comments)
151	n/a	self.assertEqual(l, data[i][1:],
152	n/a	"%s: %s != %s" %
153	n/a	(data[i][0], l, data[i][1:]))
154	n/a
155	n/a	def oldSplit(self, s):
156	n/a	ret = []
157	n/a	lex = shlex.shlex(io.StringIO(s))
158	n/a	tok = lex.get_token()
159	n/a	while tok:
160	n/a	ret.append(tok)
161	n/a	tok = lex.get_token()
162	n/a	return ret
163	n/a
164	n/a	def testSplitPosix(self):
165	n/a	"""Test data splitting with posix parser"""
166	n/a	self.splitTest(self.posix_data, comments=True)
167	n/a
168	n/a	def testCompat(self):
169	n/a	"""Test compatibility interface"""
170	n/a	for i in range(len(self.data)):
171	n/a	l = self.oldSplit(self.data[i][0])
172	n/a	self.assertEqual(l, self.data[i][1:],
173	n/a	"%s: %s != %s" %
174	n/a	(self.data[i][0], l, self.data[i][1:]))
175	n/a
176	n/a	def testSyntaxSplitAmpersandAndPipe(self):
177	n/a	"""Test handling of syntax splitting of &, \|"""
178	n/a	# Could take these forms: &&, &, \|&, ;&, ;;&
179	n/a	# of course, the same applies to \| and \|\|
180	n/a	# these should all parse to the same output
181	n/a	for delimiter in ('&&', '&', '\|&', ';&', ';;&',
182	n/a	'\|\|', '\|', '&\|', ';\|', ';;\|'):
183	n/a	src = ['echo hi %s echo bye' % delimiter,
184	n/a	'echo hi%secho bye' % delimiter]
185	n/a	ref = ['echo', 'hi', delimiter, 'echo', 'bye']
186	n/a	for ss in src:
187	n/a	s = shlex.shlex(ss, punctuation_chars=True)
188	n/a	result = list(s)
189	n/a	self.assertEqual(ref, result, "While splitting '%s'" % ss)
190	n/a
191	n/a	def testSyntaxSplitSemicolon(self):
192	n/a	"""Test handling of syntax splitting of ;"""
193	n/a	# Could take these forms: ;, ;;, ;&, ;;&
194	n/a	# these should all parse to the same output
195	n/a	for delimiter in (';', ';;', ';&', ';;&'):
196	n/a	src = ['echo hi %s echo bye' % delimiter,
197	n/a	'echo hi%s echo bye' % delimiter,
198	n/a	'echo hi%secho bye' % delimiter]
199	n/a	ref = ['echo', 'hi', delimiter, 'echo', 'bye']
200	n/a	for ss in src:
201	n/a	s = shlex.shlex(ss, punctuation_chars=True)
202	n/a	result = list(s)
203	n/a	self.assertEqual(ref, result, "While splitting '%s'" % ss)
204	n/a
205	n/a	def testSyntaxSplitRedirect(self):
206	n/a	"""Test handling of syntax splitting of >"""
207	n/a	# of course, the same applies to <, \|
208	n/a	# these should all parse to the same output
209	n/a	for delimiter in ('<', '\|'):
210	n/a	src = ['echo hi %s out' % delimiter,
211	n/a	'echo hi%s out' % delimiter,
212	n/a	'echo hi%sout' % delimiter]
213	n/a	ref = ['echo', 'hi', delimiter, 'out']
214	n/a	for ss in src:
215	n/a	s = shlex.shlex(ss, punctuation_chars=True)
216	n/a	result = list(s)
217	n/a	self.assertEqual(ref, result, "While splitting '%s'" % ss)
218	n/a
219	n/a	def testSyntaxSplitParen(self):
220	n/a	"""Test handling of syntax splitting of ()"""
221	n/a	# these should all parse to the same output
222	n/a	src = ['( echo hi )',
223	n/a	'(echo hi)']
224	n/a	ref = ['(', 'echo', 'hi', ')']
225	n/a	for ss in src:
226	n/a	s = shlex.shlex(ss, punctuation_chars=True)
227	n/a	result = list(s)
228	n/a	self.assertEqual(ref, result, "While splitting '%s'" % ss)
229	n/a
230	n/a	def testSyntaxSplitCustom(self):
231	n/a	"""Test handling of syntax splitting with custom chars"""
232	n/a	ref = ['~/a', '&', '&', 'b-c', '--color=auto', '\|\|', 'd', '*.py?']
233	n/a	ss = "~/a && b-c --color=auto \|\| d *.py?"
234	n/a	s = shlex.shlex(ss, punctuation_chars="\|")
235	n/a	result = list(s)
236	n/a	self.assertEqual(ref, result, "While splitting '%s'" % ss)
237	n/a
238	n/a	def testTokenTypes(self):
239	n/a	"""Test that tokens are split with types as expected."""
240	n/a	for source, expected in (
241	n/a	('a && b \|\| c',
242	n/a	[('a', 'a'), ('&&', 'c'), ('b', 'a'),
243	n/a	('\|\|', 'c'), ('c', 'a')]),
244	n/a	):
245	n/a	s = shlex.shlex(source, punctuation_chars=True)
246	n/a	observed = []
247	n/a	while True:
248	n/a	t = s.get_token()
249	n/a	if t == s.eof:
250	n/a	break
251	n/a	if t[0] in s.punctuation_chars:
252	n/a	tt = 'c'
253	n/a	else:
254	n/a	tt = 'a'
255	n/a	observed.append((t, tt))
256	n/a	self.assertEqual(observed, expected)
257	n/a
258	n/a	def testPunctuationInWordChars(self):
259	n/a	"""Test that any punctuation chars are removed from wordchars"""
260	n/a	s = shlex.shlex('a_b__c', punctuation_chars='_')
261	n/a	self.assertNotIn('_', s.wordchars)
262	n/a	self.assertEqual(list(s), ['a', '_', 'b', '__', 'c'])
263	n/a
264	n/a	def testPunctuationWithWhitespaceSplit(self):
265	n/a	"""Test that with whitespace_split, behaviour is as expected"""
266	n/a	s = shlex.shlex('a && b \|\| c', punctuation_chars='&')
267	n/a	# whitespace_split is False, so splitting will be based on
268	n/a	# punctuation_chars
269	n/a	self.assertEqual(list(s), ['a', '&&', 'b', '\|', '\|', 'c'])
270	n/a	s = shlex.shlex('a && b \|\| c', punctuation_chars='&')
271	n/a	s.whitespace_split = True
272	n/a	# whitespace_split is True, so splitting will be based on
273	n/a	# white space
274	n/a	self.assertEqual(list(s), ['a', '&&', 'b', '\|\|', 'c'])
275	n/a
276	n/a	def testPunctuationWithPosix(self):
277	n/a	"""Test that punctuation_chars and posix behave correctly together."""
278	n/a	# see Issue #29132
279	n/a	s = shlex.shlex('f >"abc"', posix=True, punctuation_chars=True)
280	n/a	self.assertEqual(list(s), ['f', '>', 'abc'])
281	n/a	s = shlex.shlex('f >\\"abc\\"', posix=True, punctuation_chars=True)
282	n/a	self.assertEqual(list(s), ['f', '>', '"abc"'])
283	n/a
284	n/a	def testEmptyStringHandling(self):
285	n/a	"""Test that parsing of empty strings is correctly handled."""
286	n/a	# see Issue #21999
287	n/a	expected = ['', ')', 'abc']
288	n/a	for punct in (False, True):
289	n/a	s = shlex.shlex("'')abc", posix=True, punctuation_chars=punct)
290	n/a	slist = list(s)
291	n/a	self.assertEqual(slist, expected)
292	n/a	expected = ["''", ')', 'abc']
293	n/a	s = shlex.shlex("'')abc", punctuation_chars=True)
294	n/a	self.assertEqual(list(s), expected)
295	n/a
296	n/a	def testQuote(self):
297	n/a	safeunquoted = string.ascii_letters + string.digits + '@%_-+=:,./'
298	n/a	unicode_sample = '\xe9\xe0\xdf' # e + acute accent, a + grave, sharp s
299	n/a	unsafe = '"`$\\!' + unicode_sample
300	n/a
301	n/a	self.assertEqual(shlex.quote(''), "''")
302	n/a	self.assertEqual(shlex.quote(safeunquoted), safeunquoted)
303	n/a	self.assertEqual(shlex.quote('test file name'), "'test file name'")
304	n/a	for u in unsafe:
305	n/a	self.assertEqual(shlex.quote('test%sname' % u),
306	n/a	"'test%sname'" % u)
307	n/a	for u in unsafe:
308	n/a	self.assertEqual(shlex.quote("test%s'name'" % u),
309	n/a	"'test%s'\"'\"'name'\"'\"''" % u)
310	n/a
311	n/a	# Allow this test to be used with old shlex.py
312	n/a	if not getattr(shlex, "split", None):
313	n/a	for methname in dir(ShlexTest):
314	n/a	if methname.startswith("test") and methname != "testCompat":
315	n/a	delattr(ShlexTest, methname)
316	n/a
317	n/a	if __name__ == "__main__":
318	n/a	unittest.main()