Python code coverage for Lib/shlex.py

#	count	content
1	n/a	"""A lexical analyzer class for simple shell-like syntaxes."""
2	n/a
3	n/a	# Module and documentation by Eric S. Raymond, 21 Dec 1998
4	n/a	# Input stacking and error message cleanup added by ESR, March 2000
5	n/a	# push_source() and pop_source() made explicit by ESR, January 2001.
6	n/a	# Posix compliance, split(), string arguments, and
7	n/a	# iterator interface by Gustavo Niemeyer, April 2003.
8	n/a	# changes to tokenize more like Posix shells by Vinay Sajip, July 2016.
9	n/a
10	n/a	import os
11	n/a	import re
12	n/a	import sys
13	n/a	from collections import deque
14	n/a
15	n/a	from io import StringIO
16	n/a
17	n/a	__all__ = ["shlex", "split", "quote"]
18	n/a
19	n/a	class shlex:
20	n/a	"A lexical analyzer class for simple shell-like syntaxes."
21	n/a	def __init__(self, instream=None, infile=None, posix=False,
22	n/a	punctuation_chars=False):
23	n/a	if isinstance(instream, str):
24	n/a	instream = StringIO(instream)
25	n/a	if instream is not None:
26	n/a	self.instream = instream
27	n/a	self.infile = infile
28	n/a	else:
29	n/a	self.instream = sys.stdin
30	n/a	self.infile = None
31	n/a	self.posix = posix
32	n/a	if posix:
33	n/a	self.eof = None
34	n/a	else:
35	n/a	self.eof = ''
36	n/a	self.commenters = '#'
37	n/a	self.wordchars = ('abcdfeghijklmnopqrstuvwxyz'
38	n/a	'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')
39	n/a	if self.posix:
40	n/a	self.wordchars += ('ÃŸÃ Ã¡Ã¢Ã£Ã¤Ã¥Ã¦Ã§Ã¨Ã©ÃªÃ«Ã¬ÃÃ®Ã¯Ã°Ã±Ã²Ã³Ã´ÃµÃ¶Ã¸Ã¹ÃºÃ»Ã¼Ã½Ã¾Ã¿'
41	n/a	'Ã€ÃÃ‚ÃƒÃ„ÃÃ†Ã‡ÃˆÃ‰ÃŠÃ‹ÃŒÃÃŽÃÃÃ‘Ã’Ã“Ã”Ã•Ã–Ã˜Ã™ÃšÃ›ÃœÃÃž')
42	n/a	self.whitespace = ' \t\r\n'
43	n/a	self.whitespace_split = False
44	n/a	self.quotes = '\'"'
45	n/a	self.escape = '\\'
46	n/a	self.escapedquotes = '"'
47	n/a	self.state = ' '
48	n/a	self.pushback = deque()
49	n/a	self.lineno = 1
50	n/a	self.debug = 0
51	n/a	self.token = ''
52	n/a	self.filestack = deque()
53	n/a	self.source = None
54	n/a	if not punctuation_chars:
55	n/a	punctuation_chars = ''
56	n/a	elif punctuation_chars is True:
57	n/a	punctuation_chars = '();<>\|&'
58	n/a	self.punctuation_chars = punctuation_chars
59	n/a	if punctuation_chars:
60	n/a	# _pushback_chars is a push back queue used by lookahead logic
61	n/a	self._pushback_chars = deque()
62	n/a	# these chars added because allowed in file names, args, wildcards
63	n/a	self.wordchars += '~-./*?='
64	n/a	#remove any punctuation chars from wordchars
65	n/a	t = self.wordchars.maketrans(dict.fromkeys(punctuation_chars))
66	n/a	self.wordchars = self.wordchars.translate(t)
67	n/a
68	n/a	def push_token(self, tok):
69	n/a	"Push a token onto the stack popped by the get_token method"
70	n/a	if self.debug >= 1:
71	n/a	print("shlex: pushing token " + repr(tok))
72	n/a	self.pushback.appendleft(tok)
73	n/a
74	n/a	def push_source(self, newstream, newfile=None):
75	n/a	"Push an input source onto the lexer's input source stack."
76	n/a	if isinstance(newstream, str):
77	n/a	newstream = StringIO(newstream)
78	n/a	self.filestack.appendleft((self.infile, self.instream, self.lineno))
79	n/a	self.infile = newfile
80	n/a	self.instream = newstream
81	n/a	self.lineno = 1
82	n/a	if self.debug:
83	n/a	if newfile is not None:
84	n/a	print('shlex: pushing to file %s' % (self.infile,))
85	n/a	else:
86	n/a	print('shlex: pushing to stream %s' % (self.instream,))
87	n/a
88	n/a	def pop_source(self):
89	n/a	"Pop the input source stack."
90	n/a	self.instream.close()
91	n/a	(self.infile, self.instream, self.lineno) = self.filestack.popleft()
92	n/a	if self.debug:
93	n/a	print('shlex: popping to %s, line %d' \
94	n/a	% (self.instream, self.lineno))
95	n/a	self.state = ' '
96	n/a
97	n/a	def get_token(self):
98	n/a	"Get a token from the input stream (or from stack if it's nonempty)"
99	n/a	if self.pushback:
100	n/a	tok = self.pushback.popleft()
101	n/a	if self.debug >= 1:
102	n/a	print("shlex: popping token " + repr(tok))
103	n/a	return tok
104	n/a	# No pushback. Get a token.
105	n/a	raw = self.read_token()
106	n/a	# Handle inclusions
107	n/a	if self.source is not None:
108	n/a	while raw == self.source:
109	n/a	spec = self.sourcehook(self.read_token())
110	n/a	if spec:
111	n/a	(newfile, newstream) = spec
112	n/a	self.push_source(newstream, newfile)
113	n/a	raw = self.get_token()
114	n/a	# Maybe we got EOF instead?
115	n/a	while raw == self.eof:
116	n/a	if not self.filestack:
117	n/a	return self.eof
118	n/a	else:
119	n/a	self.pop_source()
120	n/a	raw = self.get_token()
121	n/a	# Neither inclusion nor EOF
122	n/a	if self.debug >= 1:
123	n/a	if raw != self.eof:
124	n/a	print("shlex: token=" + repr(raw))
125	n/a	else:
126	n/a	print("shlex: token=EOF")
127	n/a	return raw
128	n/a
129	n/a	def read_token(self):
130	n/a	quoted = False
131	n/a	escapedstate = ' '
132	n/a	while True:
133	n/a	if self.punctuation_chars and self._pushback_chars:
134	n/a	nextchar = self._pushback_chars.pop()
135	n/a	else:
136	n/a	nextchar = self.instream.read(1)
137	n/a	if nextchar == '\n':
138	n/a	self.lineno += 1
139	n/a	if self.debug >= 3:
140	n/a	print("shlex: in state %r I see character: %r" % (self.state,
141	n/a	nextchar))
142	n/a	if self.state is None:
143	n/a	self.token = '' # past end of file
144	n/a	break
145	n/a	elif self.state == ' ':
146	n/a	if not nextchar:
147	n/a	self.state = None # end of file
148	n/a	break
149	n/a	elif nextchar in self.whitespace:
150	n/a	if self.debug >= 2:
151	n/a	print("shlex: I see whitespace in whitespace state")
152	n/a	if self.token or (self.posix and quoted):
153	n/a	break # emit current token
154	n/a	else:
155	n/a	continue
156	n/a	elif nextchar in self.commenters:
157	n/a	self.instream.readline()
158	n/a	self.lineno += 1
159	n/a	elif self.posix and nextchar in self.escape:
160	n/a	escapedstate = 'a'
161	n/a	self.state = nextchar
162	n/a	elif nextchar in self.wordchars:
163	n/a	self.token = nextchar
164	n/a	self.state = 'a'
165	n/a	elif nextchar in self.punctuation_chars:
166	n/a	self.token = nextchar
167	n/a	self.state = 'c'
168	n/a	elif nextchar in self.quotes:
169	n/a	if not self.posix:
170	n/a	self.token = nextchar
171	n/a	self.state = nextchar
172	n/a	elif self.whitespace_split:
173	n/a	self.token = nextchar
174	n/a	self.state = 'a'
175	n/a	else:
176	n/a	self.token = nextchar
177	n/a	if self.token or (self.posix and quoted):
178	n/a	break # emit current token
179	n/a	else:
180	n/a	continue
181	n/a	elif self.state in self.quotes:
182	n/a	quoted = True
183	n/a	if not nextchar: # end of file
184	n/a	if self.debug >= 2:
185	n/a	print("shlex: I see EOF in quotes state")
186	n/a	# XXX what error should be raised here?
187	n/a	raise ValueError("No closing quotation")
188	n/a	if nextchar == self.state:
189	n/a	if not self.posix:
190	n/a	self.token += nextchar
191	n/a	self.state = ' '
192	n/a	break
193	n/a	else:
194	n/a	self.state = 'a'
195	n/a	elif (self.posix and nextchar in self.escape and self.state
196	n/a	in self.escapedquotes):
197	n/a	escapedstate = self.state
198	n/a	self.state = nextchar
199	n/a	else:
200	n/a	self.token += nextchar
201	n/a	elif self.state in self.escape:
202	n/a	if not nextchar: # end of file
203	n/a	if self.debug >= 2:
204	n/a	print("shlex: I see EOF in escape state")
205	n/a	# XXX what error should be raised here?
206	n/a	raise ValueError("No escaped character")
207	n/a	# In posix shells, only the quote itself or the escape
208	n/a	# character may be escaped within quotes.
209	n/a	if (escapedstate in self.quotes and
210	n/a	nextchar != self.state and nextchar != escapedstate):
211	n/a	self.token += self.state
212	n/a	self.token += nextchar
213	n/a	self.state = escapedstate
214	n/a	elif self.state in ('a', 'c'):
215	n/a	if not nextchar:
216	n/a	self.state = None # end of file
217	n/a	break
218	n/a	elif nextchar in self.whitespace:
219	n/a	if self.debug >= 2:
220	n/a	print("shlex: I see whitespace in word state")
221	n/a	self.state = ' '
222	n/a	if self.token or (self.posix and quoted):
223	n/a	break # emit current token
224	n/a	else:
225	n/a	continue
226	n/a	elif nextchar in self.commenters:
227	n/a	self.instream.readline()
228	n/a	self.lineno += 1
229	n/a	if self.posix:
230	n/a	self.state = ' '
231	n/a	if self.token or (self.posix and quoted):
232	n/a	break # emit current token
233	n/a	else:
234	n/a	continue
235	n/a	elif self.state == 'c':
236	n/a	if nextchar in self.punctuation_chars:
237	n/a	self.token += nextchar
238	n/a	else:
239	n/a	if nextchar not in self.whitespace:
240	n/a	self._pushback_chars.append(nextchar)
241	n/a	self.state = ' '
242	n/a	break
243	n/a	elif self.posix and nextchar in self.quotes:
244	n/a	self.state = nextchar
245	n/a	elif self.posix and nextchar in self.escape:
246	n/a	escapedstate = 'a'
247	n/a	self.state = nextchar
248	n/a	elif (nextchar in self.wordchars or nextchar in self.quotes
249	n/a	or self.whitespace_split):
250	n/a	self.token += nextchar
251	n/a	else:
252	n/a	if self.punctuation_chars:
253	n/a	self._pushback_chars.append(nextchar)
254	n/a	else:
255	n/a	self.pushback.appendleft(nextchar)
256	n/a	if self.debug >= 2:
257	n/a	print("shlex: I see punctuation in word state")
258	n/a	self.state = ' '
259	n/a	if self.token or (self.posix and quoted):
260	n/a	break # emit current token
261	n/a	else:
262	n/a	continue
263	n/a	result = self.token
264	n/a	self.token = ''
265	n/a	if self.posix and not quoted and result == '':
266	n/a	result = None
267	n/a	if self.debug > 1:
268	n/a	if result:
269	n/a	print("shlex: raw token=" + repr(result))
270	n/a	else:
271	n/a	print("shlex: raw token=EOF")
272	n/a	return result
273	n/a
274	n/a	def sourcehook(self, newfile):
275	n/a	"Hook called on a filename to be sourced."
276	n/a	if newfile[0] == '"':
277	n/a	newfile = newfile[1:-1]
278	n/a	# This implements cpp-like semantics for relative-path inclusion.
279	n/a	if isinstance(self.infile, str) and not os.path.isabs(newfile):
280	n/a	newfile = os.path.join(os.path.dirname(self.infile), newfile)
281	n/a	return (newfile, open(newfile, "r"))
282	n/a
283	n/a	def error_leader(self, infile=None, lineno=None):
284	n/a	"Emit a C-compiler-like, Emacs-friendly error-message leader."
285	n/a	if infile is None:
286	n/a	infile = self.infile
287	n/a	if lineno is None:
288	n/a	lineno = self.lineno
289	n/a	return "\"%s\", line %d: " % (infile, lineno)
290	n/a
291	n/a	def __iter__(self):
292	n/a	return self
293	n/a
294	n/a	def __next__(self):
295	n/a	token = self.get_token()
296	n/a	if token == self.eof:
297	n/a	raise StopIteration
298	n/a	return token
299	n/a
300	n/a	def split(s, comments=False, posix=True):
301	n/a	lex = shlex(s, posix=posix)
302	n/a	lex.whitespace_split = True
303	n/a	if not comments:
304	n/a	lex.commenters = ''
305	n/a	return list(lex)
306	n/a
307	n/a
308	n/a	_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search
309	n/a
310	n/a	def quote(s):
311	n/a	"""Return a shell-escaped version of the string s."""
312	n/a	if not s:
313	n/a	return "''"
314	n/a	if _find_unsafe(s) is None:
315	n/a	return s
316	n/a
317	n/a	# use single quotes, and put single quotes into double quotes
318	n/a	# the string $'b is then quoted as '$'"'"'b'
319	n/a	return "'" + s.replace("'", "'\"'\"'") + "'"
320	n/a
321	n/a
322	n/a	def _print_tokens(lexer):
323	n/a	while 1:
324	n/a	tt = lexer.get_token()
325	n/a	if not tt:
326	n/a	break
327	n/a	print("Token: " + repr(tt))
328	n/a
329	n/a	if __name__ == '__main__':
330	n/a	if len(sys.argv) == 1:
331	n/a	_print_tokens(shlex())
332	n/a	else:
333	n/a	fn = sys.argv[1]
334	n/a	with open(fn) as f:
335	n/a	_print_tokens(shlex(f, fn))