»Core Development>Code coverage>Lib/test/test_shlex.py

Python code coverage for Lib/test/test_shlex.py

1n/aimport io
2n/aimport shlex
3n/aimport string
4n/aimport unittest
8n/a# The original test data set was from shellwords, by Hartmut Goebel.
10n/adata = r"""x|x|
11n/afoo bar|foo|bar|
12n/a foo bar|foo|bar|
13n/a foo bar |foo|bar|
14n/afoo bar bla fasel|foo|bar|bla|fasel|
15n/ax y z xxxx|x|y|z|xxxx|
16n/a\x bar|\|x|bar|
17n/a\ x bar|\|x|bar|
18n/a\ bar|\|bar|
19n/afoo \x bar|foo|\|x|bar|
20n/afoo \ x bar|foo|\|x|bar|
21n/afoo \ bar|foo|\|bar|
22n/afoo "bar" bla|foo|"bar"|bla|
23n/a"foo" "bar" "bla"|"foo"|"bar"|"bla"|
24n/a"foo" bar "bla"|"foo"|bar|"bla"|
25n/a"foo" bar bla|"foo"|bar|bla|
26n/afoo 'bar' bla|foo|'bar'|bla|
27n/a'foo' 'bar' 'bla'|'foo'|'bar'|'bla'|
28n/a'foo' bar 'bla'|'foo'|bar|'bla'|
29n/a'foo' bar bla|'foo'|bar|bla|
30n/ablurb foo"bar"bar"fasel" baz|blurb|foo"bar"bar"fasel"|baz|
31n/ablurb foo'bar'bar'fasel' baz|blurb|foo'bar'bar'fasel'|baz|
34n/afoo "" bar|foo|""|bar|
35n/afoo '' bar|foo|''|bar|
36n/afoo "" "" "" bar|foo|""|""|""|bar|
37n/afoo '' '' '' bar|foo|''|''|''|bar|
40n/a"foo\ bar"|"foo\ bar"|
41n/a"foo\\ bar"|"foo\\ bar"|
42n/a"foo\\ bar\"|"foo\\ bar\"|
43n/a"foo\\" bar\""|"foo\\"|bar|\|""|
44n/a"foo\\ bar\" dfadf"|"foo\\ bar\"|dfadf"|
45n/a"foo\\\ bar\" dfadf"|"foo\\\ bar\"|dfadf"|
46n/a"foo\\\x bar\" dfadf"|"foo\\\x bar\"|dfadf"|
47n/a"foo\x bar\" dfadf"|"foo\x bar\"|dfadf"|
49n/a'foo\ bar'|'foo\ bar'|
50n/a'foo\\ bar'|'foo\\ bar'|
51n/a"foo\\\x bar\" df'a\ 'df'|"foo\\\x bar\"|df'a|\|'df'|
55n/a"foo\ "|"foo\ "|
56n/afoo\ xx|foo|\|xx|
57n/afoo\ x\x|foo|\|x|\|x|
58n/afoo\ x\x\""|foo|\|x|\|x|\|""|
59n/a"foo\ x\x"|"foo\ x\x"|
60n/a"foo\ x\x\\"|"foo\ x\x\\"|
61n/a"foo\ x\x\\""foobar"|"foo\ x\x\\"|"foobar"|
62n/a"foo\ x\x\\"\''"foobar"|"foo\ x\x\\"|\|''|"foobar"|
63n/a"foo\ x\x\\"\'"fo'obar"|"foo\ x\x\\"|\|'"fo'|obar"|
64n/a"foo\ x\x\\"\'"fo'obar" 'don'\''t'|"foo\ x\x\\"|\|'"fo'|obar"|'don'|\|''|t'|
65n/a'foo\ bar'|'foo\ bar'|
66n/a'foo\\ bar'|'foo\\ bar'|
67n/afoo\ bar|foo|\|bar|
69n/a:-) ;-)|:|-|)|;|-|)|
73n/aposix_data = r"""x|x|
74n/afoo bar|foo|bar|
75n/a foo bar|foo|bar|
76n/a foo bar |foo|bar|
77n/afoo bar bla fasel|foo|bar|bla|fasel|
78n/ax y z xxxx|x|y|z|xxxx|
79n/a\x bar|x|bar|
80n/a\ x bar| x|bar|
81n/a\ bar| bar|
82n/afoo \x bar|foo|x|bar|
83n/afoo \ x bar|foo| x|bar|
84n/afoo \ bar|foo| bar|
85n/afoo "bar" bla|foo|bar|bla|
86n/a"foo" "bar" "bla"|foo|bar|bla|
87n/a"foo" bar "bla"|foo|bar|bla|
88n/a"foo" bar bla|foo|bar|bla|
89n/afoo 'bar' bla|foo|bar|bla|
90n/a'foo' 'bar' 'bla'|foo|bar|bla|
91n/a'foo' bar 'bla'|foo|bar|bla|
92n/a'foo' bar bla|foo|bar|bla|
93n/ablurb foo"bar"bar"fasel" baz|blurb|foobarbarfasel|baz|
94n/ablurb foo'bar'bar'fasel' baz|blurb|foobarbarfasel|baz|
97n/afoo "" bar|foo||bar|
98n/afoo '' bar|foo||bar|
99n/afoo "" "" "" bar|foo||||bar|
100n/afoo '' '' '' bar|foo||||bar|
103n/a"foo\ bar"|foo\ bar|
104n/a"foo\\ bar"|foo\ bar|
105n/a"foo\\ bar\""|foo\ bar"|
106n/a"foo\\" bar\"|foo\|bar"|
107n/a"foo\\ bar\" dfadf"|foo\ bar" dfadf|
108n/a"foo\\\ bar\" dfadf"|foo\\ bar" dfadf|
109n/a"foo\\\x bar\" dfadf"|foo\\x bar" dfadf|
110n/a"foo\x bar\" dfadf"|foo\x bar" dfadf|
112n/a'foo\ bar'|foo\ bar|
113n/a'foo\\ bar'|foo\\ bar|
114n/a"foo\\\x bar\" df'a\ 'df"|foo\\x bar" df'a\ 'df|
118n/a"foo\ "|foo\ |
119n/afoo\ xx|foo xx|
120n/afoo\ x\x|foo xx|
121n/afoo\ x\x\"|foo xx"|
122n/a"foo\ x\x"|foo\ x\x|
123n/a"foo\ x\x\\"|foo\ x\x\|
124n/a"foo\ x\x\\""foobar"|foo\ x\x\foobar|
125n/a"foo\ x\x\\"\'"foobar"|foo\ x\x\'foobar|
126n/a"foo\ x\x\\"\'"fo'obar"|foo\ x\x\'fo'obar|
127n/a"foo\ x\x\\"\'"fo'obar" 'don'\''t'|foo\ x\x\'fo'obar|don't|
128n/a"foo\ x\x\\"\'"fo'obar" 'don'\''t' \\|foo\ x\x\'fo'obar|don't|\|
129n/a'foo\ bar'|foo\ bar|
130n/a'foo\\ bar'|foo\\ bar|
131n/afoo\ bar|foo bar|
133n/a:-) ;-)|:-)|;-)|
137n/aclass ShlexTest(unittest.TestCase):
138n/a def setUp(self):
139n/a self.data = [x.split("|")[:-1]
140n/a for x in data.splitlines()]
141n/a self.posix_data = [x.split("|")[:-1]
142n/a for x in posix_data.splitlines()]
143n/a for item in self.data:
144n/a item[0] = item[0].replace(r"\n", "\n")
145n/a for item in self.posix_data:
146n/a item[0] = item[0].replace(r"\n", "\n")
148n/a def splitTest(self, data, comments):
149n/a for i in range(len(data)):
150n/a l = shlex.split(data[i][0], comments=comments)
151n/a self.assertEqual(l, data[i][1:],
152n/a "%s: %s != %s" %
153n/a (data[i][0], l, data[i][1:]))
155n/a def oldSplit(self, s):
156n/a ret = []
157n/a lex = shlex.shlex(io.StringIO(s))
158n/a tok = lex.get_token()
159n/a while tok:
160n/a ret.append(tok)
161n/a tok = lex.get_token()
162n/a return ret
164n/a def testSplitPosix(self):
165n/a """Test data splitting with posix parser"""
166n/a self.splitTest(self.posix_data, comments=True)
168n/a def testCompat(self):
169n/a """Test compatibility interface"""
170n/a for i in range(len(self.data)):
171n/a l = self.oldSplit(self.data[i][0])
172n/a self.assertEqual(l, self.data[i][1:],
173n/a "%s: %s != %s" %
174n/a (self.data[i][0], l, self.data[i][1:]))
176n/a def testSyntaxSplitAmpersandAndPipe(self):
177n/a """Test handling of syntax splitting of &, |"""
178n/a # Could take these forms: &&, &, |&, ;&, ;;&
179n/a # of course, the same applies to | and ||
180n/a # these should all parse to the same output
181n/a for delimiter in ('&&', '&', '|&', ';&', ';;&',
182n/a '||', '|', '&|', ';|', ';;|'):
183n/a src = ['echo hi %s echo bye' % delimiter,
184n/a 'echo hi%secho bye' % delimiter]
185n/a ref = ['echo', 'hi', delimiter, 'echo', 'bye']
186n/a for ss in src:
187n/a s = shlex.shlex(ss, punctuation_chars=True)
188n/a result = list(s)
189n/a self.assertEqual(ref, result, "While splitting '%s'" % ss)
191n/a def testSyntaxSplitSemicolon(self):
192n/a """Test handling of syntax splitting of ;"""
193n/a # Could take these forms: ;, ;;, ;&, ;;&
194n/a # these should all parse to the same output
195n/a for delimiter in (';', ';;', ';&', ';;&'):
196n/a src = ['echo hi %s echo bye' % delimiter,
197n/a 'echo hi%s echo bye' % delimiter,
198n/a 'echo hi%secho bye' % delimiter]
199n/a ref = ['echo', 'hi', delimiter, 'echo', 'bye']
200n/a for ss in src:
201n/a s = shlex.shlex(ss, punctuation_chars=True)
202n/a result = list(s)
203n/a self.assertEqual(ref, result, "While splitting '%s'" % ss)
205n/a def testSyntaxSplitRedirect(self):
206n/a """Test handling of syntax splitting of >"""
207n/a # of course, the same applies to <, |
208n/a # these should all parse to the same output
209n/a for delimiter in ('<', '|'):
210n/a src = ['echo hi %s out' % delimiter,
211n/a 'echo hi%s out' % delimiter,
212n/a 'echo hi%sout' % delimiter]
213n/a ref = ['echo', 'hi', delimiter, 'out']
214n/a for ss in src:
215n/a s = shlex.shlex(ss, punctuation_chars=True)
216n/a result = list(s)
217n/a self.assertEqual(ref, result, "While splitting '%s'" % ss)
219n/a def testSyntaxSplitParen(self):
220n/a """Test handling of syntax splitting of ()"""
221n/a # these should all parse to the same output
222n/a src = ['( echo hi )',
223n/a '(echo hi)']
224n/a ref = ['(', 'echo', 'hi', ')']
225n/a for ss in src:
226n/a s = shlex.shlex(ss, punctuation_chars=True)
227n/a result = list(s)
228n/a self.assertEqual(ref, result, "While splitting '%s'" % ss)
230n/a def testSyntaxSplitCustom(self):
231n/a """Test handling of syntax splitting with custom chars"""
232n/a ref = ['~/a', '&', '&', 'b-c', '--color=auto', '||', 'd', '*.py?']
233n/a ss = "~/a && b-c --color=auto || d *.py?"
234n/a s = shlex.shlex(ss, punctuation_chars="|")
235n/a result = list(s)
236n/a self.assertEqual(ref, result, "While splitting '%s'" % ss)
238n/a def testTokenTypes(self):
239n/a """Test that tokens are split with types as expected."""
240n/a for source, expected in (
241n/a ('a && b || c',
242n/a [('a', 'a'), ('&&', 'c'), ('b', 'a'),
243n/a ('||', 'c'), ('c', 'a')]),
244n/a ):
245n/a s = shlex.shlex(source, punctuation_chars=True)
246n/a observed = []
247n/a while True:
248n/a t = s.get_token()
249n/a if t == s.eof:
250n/a break
251n/a if t[0] in s.punctuation_chars:
252n/a tt = 'c'
253n/a else:
254n/a tt = 'a'
255n/a observed.append((t, tt))
256n/a self.assertEqual(observed, expected)
258n/a def testPunctuationInWordChars(self):
259n/a """Test that any punctuation chars are removed from wordchars"""
260n/a s = shlex.shlex('a_b__c', punctuation_chars='_')
261n/a self.assertNotIn('_', s.wordchars)
262n/a self.assertEqual(list(s), ['a', '_', 'b', '__', 'c'])
264n/a def testPunctuationWithWhitespaceSplit(self):
265n/a """Test that with whitespace_split, behaviour is as expected"""
266n/a s = shlex.shlex('a && b || c', punctuation_chars='&')
267n/a # whitespace_split is False, so splitting will be based on
268n/a # punctuation_chars
269n/a self.assertEqual(list(s), ['a', '&&', 'b', '|', '|', 'c'])
270n/a s = shlex.shlex('a && b || c', punctuation_chars='&')
271n/a s.whitespace_split = True
272n/a # whitespace_split is True, so splitting will be based on
273n/a # white space
274n/a self.assertEqual(list(s), ['a', '&&', 'b', '||', 'c'])
276n/a def testPunctuationWithPosix(self):
277n/a """Test that punctuation_chars and posix behave correctly together."""
278n/a # see Issue #29132
279n/a s = shlex.shlex('f >"abc"', posix=True, punctuation_chars=True)
280n/a self.assertEqual(list(s), ['f', '>', 'abc'])
281n/a s = shlex.shlex('f >\\"abc\\"', posix=True, punctuation_chars=True)
282n/a self.assertEqual(list(s), ['f', '>', '"abc"'])
284n/a def testEmptyStringHandling(self):
285n/a """Test that parsing of empty strings is correctly handled."""
286n/a # see Issue #21999
287n/a expected = ['', ')', 'abc']
288n/a for punct in (False, True):
289n/a s = shlex.shlex("'')abc", posix=True, punctuation_chars=punct)
290n/a slist = list(s)
291n/a self.assertEqual(slist, expected)
292n/a expected = ["''", ')', 'abc']
293n/a s = shlex.shlex("'')abc", punctuation_chars=True)
294n/a self.assertEqual(list(s), expected)
296n/a def testQuote(self):
297n/a safeunquoted = string.ascii_letters + string.digits + '@%_-+=:,./'
298n/a unicode_sample = '\xe9\xe0\xdf' # e + acute accent, a + grave, sharp s
299n/a unsafe = '"`$\\!' + unicode_sample
301n/a self.assertEqual(shlex.quote(''), "''")
302n/a self.assertEqual(shlex.quote(safeunquoted), safeunquoted)
303n/a self.assertEqual(shlex.quote('test file name'), "'test file name'")
304n/a for u in unsafe:
305n/a self.assertEqual(shlex.quote('test%sname' % u),
306n/a "'test%sname'" % u)
307n/a for u in unsafe:
308n/a self.assertEqual(shlex.quote("test%s'name'" % u),
309n/a "'test%s'\"'\"'name'\"'\"''" % u)
311n/a# Allow this test to be used with old shlex.py
312n/aif not getattr(shlex, "split", None):
313n/a for methname in dir(ShlexTest):
314n/a if methname.startswith("test") and methname != "testCompat":
315n/a delattr(ShlexTest, methname)
317n/aif __name__ == "__main__":
318n/a unittest.main()