Python code coverage for Tools/scripts/pindent.py

#	count	content
1	n/a	#! /usr/bin/env python
2	n/a
3	n/a	# This file contains a class and a main program that perform three
4	n/a	# related (though complimentary) formatting operations on Python
5	n/a	# programs. When called as "pindent -c", it takes a valid Python
6	n/a	# program as input and outputs a version augmented with block-closing
7	n/a	# comments. When called as "pindent -d", it assumes its input is a
8	n/a	# Python program with block-closing comments and outputs a commentless
9	n/a	# version. When called as "pindent -r" it assumes its input is a
10	n/a	# Python program with block-closing comments but with its indentation
11	n/a	# messed up, and outputs a properly indented version.
12	n/a
13	n/a	# A "block-closing comment" is a comment of the form '# end <keyword>'
14	n/a	# where <keyword> is the keyword that opened the block. If the
15	n/a	# opening keyword is 'def' or 'class', the function or class name may
16	n/a	# be repeated in the block-closing comment as well. Here is an
17	n/a	# example of a program fully augmented with block-closing comments:
18	n/a
19	n/a	# def foobar(a, b):
20	n/a	# if a == b:
21	n/a	# a = a+1
22	n/a	# elif a < b:
23	n/a	# b = b-1
24	n/a	# if b > a: a = a-1
25	n/a	# # end if
26	n/a	# else:
27	n/a	# print 'oops!'
28	n/a	# # end if
29	n/a	# # end def foobar
30	n/a
31	n/a	# Note that only the last part of an if...elif...else... block needs a
32	n/a	# block-closing comment; the same is true for other compound
33	n/a	# statements (e.g. try...except). Also note that "short-form" blocks
34	n/a	# like the second 'if' in the example must be closed as well;
35	n/a	# otherwise the 'else' in the example would be ambiguous (remember
36	n/a	# that indentation is not significant when interpreting block-closing
37	n/a	# comments).
38	n/a
39	n/a	# The operations are idempotent (i.e. applied to their own output
40	n/a	# they yield an identical result). Running first "pindent -c" and
41	n/a	# then "pindent -r" on a valid Python program produces a program that
42	n/a	# is semantically identical to the input (though its indentation may
43	n/a	# be different). Running "pindent -e" on that output produces a
44	n/a	# program that only differs from the original in indentation.
45	n/a
46	n/a	# Other options:
47	n/a	# -s stepsize: set the indentation step size (default 8)
48	n/a	# -t tabsize : set the number of spaces a tab character is worth (default 8)
49	n/a	# -e : expand TABs into spaces
50	n/a	# file ... : input file(s) (default standard input)
51	n/a	# The results always go to standard output
52	n/a
53	n/a	# Caveats:
54	n/a	# - comments ending in a backslash will be mistaken for continued lines
55	n/a	# - continuations using backslash are always left unchanged
56	n/a	# - continuations inside parentheses are not extra indented by -r
57	n/a	# but must be indented for -c to work correctly (this breaks
58	n/a	# idempotency!)
59	n/a	# - continued lines inside triple-quoted strings are totally garbled
60	n/a
61	n/a	# Secret feature:
62	n/a	# - On input, a block may also be closed with an "end statement" --
63	n/a	# this is a block-closing comment without the '#' sign.
64	n/a
65	n/a	# Possible improvements:
66	n/a	# - check syntax based on transitions in 'next' table
67	n/a	# - better error reporting
68	n/a	# - better error recovery
69	n/a	# - check identifier after class/def
70	n/a
71	n/a	# The following wishes need a more complete tokenization of the source:
72	n/a	# - Don't get fooled by comments ending in backslash
73	n/a	# - reindent continuation lines indicated by backslash
74	n/a	# - handle continuation lines inside parentheses/braces/brackets
75	n/a	# - handle triple quoted strings spanning lines
76	n/a	# - realign comments
77	n/a	# - optionally do much more thorough reformatting, a la C indent
78	n/a
79	n/a	# Defaults
80	n/a	STEPSIZE = 8
81	n/a	TABSIZE = 8
82	n/a	EXPANDTABS = 0
83	n/a
84	n/a	import re
85	n/a	import sys
86	n/a
87	n/a	next = {}
88	n/a	next['if'] = next['elif'] = 'elif', 'else', 'end'
89	n/a	next['while'] = next['for'] = 'else', 'end'
90	n/a	next['try'] = 'except', 'finally'
91	n/a	next['except'] = 'except', 'else', 'finally', 'end'
92	n/a	next['else'] = next['finally'] = next['def'] = next['class'] = 'end'
93	n/a	next['end'] = ()
94	n/a	start = 'if', 'while', 'for', 'try', 'with', 'def', 'class'
95	n/a
96	n/a	class PythonIndenter:
97	n/a
98	n/a	def __init__(self, fpi = sys.stdin, fpo = sys.stdout,
99	n/a	indentsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
100	n/a	self.fpi = fpi
101	n/a	self.fpo = fpo
102	n/a	self.indentsize = indentsize
103	n/a	self.tabsize = tabsize
104	n/a	self.lineno = 0
105	n/a	self.expandtabs = expandtabs
106	n/a	self._write = fpo.write
107	n/a	self.kwprog = re.compile(
108	n/a	r'^\s*(?P<kw>[a-z]+)'
109	n/a	r'(\s+(?P<id>[a-zA-Z_]\w*))?'
110	n/a	r'[^\w]')
111	n/a	self.endprog = re.compile(
112	n/a	r'^\s#?\send\s+(?P<kw>[a-z]+)'
113	n/a	r'(\s+(?P<id>[a-zA-Z_]\w*))?'
114	n/a	r'[^\w]')
115	n/a	self.wsprog = re.compile(r'^[ \t]*')
116	n/a	# end def __init__
117	n/a
118	n/a	def write(self, line):
119	n/a	if self.expandtabs:
120	n/a	self._write(line.expandtabs(self.tabsize))
121	n/a	else:
122	n/a	self._write(line)
123	n/a	# end if
124	n/a	# end def write
125	n/a
126	n/a	def readline(self):
127	n/a	line = self.fpi.readline()
128	n/a	if line: self.lineno = self.lineno + 1
129	n/a	# end if
130	n/a	return line
131	n/a	# end def readline
132	n/a
133	n/a	def error(self, fmt, *args):
134	n/a	if args: fmt = fmt % args
135	n/a	# end if
136	n/a	sys.stderr.write('Error at line %d: %s\n' % (self.lineno, fmt))
137	n/a	self.write('### %s ###\n' % fmt)
138	n/a	# end def error
139	n/a
140	n/a	def getline(self):
141	n/a	line = self.readline()
142	n/a	while line[-2:] == '\\\n':
143	n/a	line2 = self.readline()
144	n/a	if not line2: break
145	n/a	# end if
146	n/a	line = line + line2
147	n/a	# end while
148	n/a	return line
149	n/a	# end def getline
150	n/a
151	n/a	def putline(self, line, indent = None):
152	n/a	if indent is None:
153	n/a	self.write(line)
154	n/a	return
155	n/a	# end if
156	n/a	tabs, spaces = divmod(indent*self.indentsize, self.tabsize)
157	n/a	i = 0
158	n/a	m = self.wsprog.match(line)
159	n/a	if m: i = m.end()
160	n/a	# end if
161	n/a	self.write('\t'tabs + ' 'spaces + line[i:])
162	n/a	# end def putline
163	n/a
164	n/a	def reformat(self):
165	n/a	stack = []
166	n/a	while 1:
167	n/a	line = self.getline()
168	n/a	if not line: break # EOF
169	n/a	# end if
170	n/a	m = self.endprog.match(line)
171	n/a	if m:
172	n/a	kw = 'end'
173	n/a	kw2 = m.group('kw')
174	n/a	if not stack:
175	n/a	self.error('unexpected end')
176	n/a	elif stack[-1][0] != kw2:
177	n/a	self.error('unmatched end')
178	n/a	# end if
179	n/a	del stack[-1:]
180	n/a	self.putline(line, len(stack))
181	n/a	continue
182	n/a	# end if
183	n/a	m = self.kwprog.match(line)
184	n/a	if m:
185	n/a	kw = m.group('kw')
186	n/a	if kw in start:
187	n/a	self.putline(line, len(stack))
188	n/a	stack.append((kw, kw))
189	n/a	continue
190	n/a	# end if
191	n/a	if next.has_key(kw) and stack:
192	n/a	self.putline(line, len(stack)-1)
193	n/a	kwa, kwb = stack[-1]
194	n/a	stack[-1] = kwa, kw
195	n/a	continue
196	n/a	# end if
197	n/a	# end if
198	n/a	self.putline(line, len(stack))
199	n/a	# end while
200	n/a	if stack:
201	n/a	self.error('unterminated keywords')
202	n/a	for kwa, kwb in stack:
203	n/a	self.write('\t%s\n' % kwa)
204	n/a	# end for
205	n/a	# end if
206	n/a	# end def reformat
207	n/a
208	n/a	def delete(self):
209	n/a	begin_counter = 0
210	n/a	end_counter = 0
211	n/a	while 1:
212	n/a	line = self.getline()
213	n/a	if not line: break # EOF
214	n/a	# end if
215	n/a	m = self.endprog.match(line)
216	n/a	if m:
217	n/a	end_counter = end_counter + 1
218	n/a	continue
219	n/a	# end if
220	n/a	m = self.kwprog.match(line)
221	n/a	if m:
222	n/a	kw = m.group('kw')
223	n/a	if kw in start:
224	n/a	begin_counter = begin_counter + 1
225	n/a	# end if
226	n/a	# end if
227	n/a	self.putline(line)
228	n/a	# end while
229	n/a	if begin_counter - end_counter < 0:
230	n/a	sys.stderr.write('Warning: input contained more end tags than expected\n')
231	n/a	elif begin_counter - end_counter > 0:
232	n/a	sys.stderr.write('Warning: input contained less end tags than expected\n')
233	n/a	# end if
234	n/a	# end def delete
235	n/a
236	n/a	def complete(self):
237	n/a	self.indentsize = 1
238	n/a	stack = []
239	n/a	todo = []
240	n/a	thisid = ''
241	n/a	current, firstkw, lastkw, topid = 0, '', '', ''
242	n/a	while 1:
243	n/a	line = self.getline()
244	n/a	i = 0
245	n/a	m = self.wsprog.match(line)
246	n/a	if m: i = m.end()
247	n/a	# end if
248	n/a	m = self.endprog.match(line)
249	n/a	if m:
250	n/a	thiskw = 'end'
251	n/a	endkw = m.group('kw')
252	n/a	thisid = m.group('id')
253	n/a	else:
254	n/a	m = self.kwprog.match(line)
255	n/a	if m:
256	n/a	thiskw = m.group('kw')
257	n/a	if not next.has_key(thiskw):
258	n/a	thiskw = ''
259	n/a	# end if
260	n/a	if thiskw in ('def', 'class'):
261	n/a	thisid = m.group('id')
262	n/a	else:
263	n/a	thisid = ''
264	n/a	# end if
265	n/a	elif line[i:i+1] in ('\n', '#'):
266	n/a	todo.append(line)
267	n/a	continue
268	n/a	else:
269	n/a	thiskw = ''
270	n/a	# end if
271	n/a	# end if
272	n/a	indent = len(line[:i].expandtabs(self.tabsize))
273	n/a	while indent < current:
274	n/a	if firstkw:
275	n/a	if topid:
276	n/a	s = '# end %s %s\n' % (
277	n/a	firstkw, topid)
278	n/a	else:
279	n/a	s = '# end %s\n' % firstkw
280	n/a	# end if
281	n/a	self.putline(s, current)
282	n/a	firstkw = lastkw = ''
283	n/a	# end if
284	n/a	current, firstkw, lastkw, topid = stack[-1]
285	n/a	del stack[-1]
286	n/a	# end while
287	n/a	if indent == current and firstkw:
288	n/a	if thiskw == 'end':
289	n/a	if endkw != firstkw:
290	n/a	self.error('mismatched end')
291	n/a	# end if
292	n/a	firstkw = lastkw = ''
293	n/a	elif not thiskw or thiskw in start:
294	n/a	if topid:
295	n/a	s = '# end %s %s\n' % (
296	n/a	firstkw, topid)
297	n/a	else:
298	n/a	s = '# end %s\n' % firstkw
299	n/a	# end if
300	n/a	self.putline(s, current)
301	n/a	firstkw = lastkw = topid = ''
302	n/a	# end if
303	n/a	# end if
304	n/a	if indent > current:
305	n/a	stack.append((current, firstkw, lastkw, topid))
306	n/a	if thiskw and thiskw not in start:
307	n/a	# error
308	n/a	thiskw = ''
309	n/a	# end if
310	n/a	current, firstkw, lastkw, topid = \
311	n/a	indent, thiskw, thiskw, thisid
312	n/a	# end if
313	n/a	if thiskw:
314	n/a	if thiskw in start:
315	n/a	firstkw = lastkw = thiskw
316	n/a	topid = thisid
317	n/a	else:
318	n/a	lastkw = thiskw
319	n/a	# end if
320	n/a	# end if
321	n/a	for l in todo: self.write(l)
322	n/a	# end for
323	n/a	todo = []
324	n/a	if not line: break
325	n/a	# end if
326	n/a	self.write(line)
327	n/a	# end while
328	n/a	# end def complete
329	n/a
330	n/a	# end class PythonIndenter
331	n/a
332	n/a	# Simplified user interface
333	n/a	# - xxx_filter(input, output): read and write file objects
334	n/a	# - xxx_string(s): take and return string object
335	n/a	# - xxx_file(filename): process file in place, return true iff changed
336	n/a
337	n/a	def complete_filter(input = sys.stdin, output = sys.stdout,
338	n/a	stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
339	n/a	pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
340	n/a	pi.complete()
341	n/a	# end def complete_filter
342	n/a
343	n/a	def delete_filter(input= sys.stdin, output = sys.stdout,
344	n/a	stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
345	n/a	pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
346	n/a	pi.delete()
347	n/a	# end def delete_filter
348	n/a
349	n/a	def reformat_filter(input = sys.stdin, output = sys.stdout,
350	n/a	stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
351	n/a	pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
352	n/a	pi.reformat()
353	n/a	# end def reformat_filter
354	n/a
355	n/a	class StringReader:
356	n/a	def __init__(self, buf):
357	n/a	self.buf = buf
358	n/a	self.pos = 0
359	n/a	self.len = len(self.buf)
360	n/a	# end def __init__
361	n/a	def read(self, n = 0):
362	n/a	if n <= 0:
363	n/a	n = self.len - self.pos
364	n/a	else:
365	n/a	n = min(n, self.len - self.pos)
366	n/a	# end if
367	n/a	r = self.buf[self.pos : self.pos + n]
368	n/a	self.pos = self.pos + n
369	n/a	return r
370	n/a	# end def read
371	n/a	def readline(self):
372	n/a	i = self.buf.find('\n', self.pos)
373	n/a	return self.read(i + 1 - self.pos)
374	n/a	# end def readline
375	n/a	def readlines(self):
376	n/a	lines = []
377	n/a	line = self.readline()
378	n/a	while line:
379	n/a	lines.append(line)
380	n/a	line = self.readline()
381	n/a	# end while
382	n/a	return lines
383	n/a	# end def readlines
384	n/a	# seek/tell etc. are left as an exercise for the reader
385	n/a	# end class StringReader
386	n/a
387	n/a	class StringWriter:
388	n/a	def __init__(self):
389	n/a	self.buf = ''
390	n/a	# end def __init__
391	n/a	def write(self, s):
392	n/a	self.buf = self.buf + s
393	n/a	# end def write
394	n/a	def getvalue(self):
395	n/a	return self.buf
396	n/a	# end def getvalue
397	n/a	# end class StringWriter
398	n/a
399	n/a	def complete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
400	n/a	input = StringReader(source)
401	n/a	output = StringWriter()
402	n/a	pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
403	n/a	pi.complete()
404	n/a	return output.getvalue()
405	n/a	# end def complete_string
406	n/a
407	n/a	def delete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
408	n/a	input = StringReader(source)
409	n/a	output = StringWriter()
410	n/a	pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
411	n/a	pi.delete()
412	n/a	return output.getvalue()
413	n/a	# end def delete_string
414	n/a
415	n/a	def reformat_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
416	n/a	input = StringReader(source)
417	n/a	output = StringWriter()
418	n/a	pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
419	n/a	pi.reformat()
420	n/a	return output.getvalue()
421	n/a	# end def reformat_string
422	n/a
423	n/a	def complete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
424	n/a	source = open(filename, 'r').read()
425	n/a	result = complete_string(source, stepsize, tabsize, expandtabs)
426	n/a	if source == result: return 0
427	n/a	# end if
428	n/a	import os
429	n/a	try: os.rename(filename, filename + '~')
430	n/a	except os.error: pass
431	n/a	# end try
432	n/a	f = open(filename, 'w')
433	n/a	f.write(result)
434	n/a	f.close()
435	n/a	return 1
436	n/a	# end def complete_file
437	n/a
438	n/a	def delete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
439	n/a	source = open(filename, 'r').read()
440	n/a	result = delete_string(source, stepsize, tabsize, expandtabs)
441	n/a	if source == result: return 0
442	n/a	# end if
443	n/a	import os
444	n/a	try: os.rename(filename, filename + '~')
445	n/a	except os.error: pass
446	n/a	# end try
447	n/a	f = open(filename, 'w')
448	n/a	f.write(result)
449	n/a	f.close()
450	n/a	return 1
451	n/a	# end def delete_file
452	n/a
453	n/a	def reformat_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
454	n/a	source = open(filename, 'r').read()
455	n/a	result = reformat_string(source, stepsize, tabsize, expandtabs)
456	n/a	if source == result: return 0
457	n/a	# end if
458	n/a	import os
459	n/a	try: os.rename(filename, filename + '~')
460	n/a	except os.error: pass
461	n/a	# end try
462	n/a	f = open(filename, 'w')
463	n/a	f.write(result)
464	n/a	f.close()
465	n/a	return 1
466	n/a	# end def reformat_file
467	n/a
468	n/a	# Test program when called as a script
469	n/a
470	n/a	usage = """
471	n/a	usage: pindent (-c\|-d\|-r) [-s stepsize] [-t tabsize] [-e] [file] ...
472	n/a	-c : complete a correctly indented program (add #end directives)
473	n/a	-d : delete #end directives
474	n/a	-r : reformat a completed program (use #end directives)
475	n/a	-s stepsize: indentation step (default %(STEPSIZE)d)
476	n/a	-t tabsize : the worth in spaces of a tab (default %(TABSIZE)d)
477	n/a	-e : expand TABs into spaces (defailt OFF)
478	n/a	[file] ... : files are changed in place, with backups in file~
479	n/a	If no files are specified or a single - is given,
480	n/a	the program acts as a filter (reads stdin, writes stdout).
481	n/a	""" % vars()
482	n/a
483	n/a	def error_both(op1, op2):
484	n/a	sys.stderr.write('Error: You can not specify both '+op1+' and -'+op2[0]+' at the same time\n')
485	n/a	sys.stderr.write(usage)
486	n/a	sys.exit(2)
487	n/a	# end def error_both
488	n/a
489	n/a	def test():
490	n/a	import getopt
491	n/a	try:
492	n/a	opts, args = getopt.getopt(sys.argv[1:], 'cdrs:t:e')
493	n/a	except getopt.error, msg:
494	n/a	sys.stderr.write('Error: %s\n' % msg)
495	n/a	sys.stderr.write(usage)
496	n/a	sys.exit(2)
497	n/a	# end try
498	n/a	action = None
499	n/a	stepsize = STEPSIZE
500	n/a	tabsize = TABSIZE
501	n/a	expandtabs = EXPANDTABS
502	n/a	for o, a in opts:
503	n/a	if o == '-c':
504	n/a	if action: error_both(o, action)
505	n/a	# end if
506	n/a	action = 'complete'
507	n/a	elif o == '-d':
508	n/a	if action: error_both(o, action)
509	n/a	# end if
510	n/a	action = 'delete'
511	n/a	elif o == '-r':
512	n/a	if action: error_both(o, action)
513	n/a	# end if
514	n/a	action = 'reformat'
515	n/a	elif o == '-s':
516	n/a	stepsize = int(a)
517	n/a	elif o == '-t':
518	n/a	tabsize = int(a)
519	n/a	elif o == '-e':
520	n/a	expandtabs = 1
521	n/a	# end if
522	n/a	# end for
523	n/a	if not action:
524	n/a	sys.stderr.write(
525	n/a	'You must specify -c(omplete), -d(elete) or -r(eformat)\n')
526	n/a	sys.stderr.write(usage)
527	n/a	sys.exit(2)
528	n/a	# end if
529	n/a	if not args or args == ['-']:
530	n/a	action = eval(action + '_filter')
531	n/a	action(sys.stdin, sys.stdout, stepsize, tabsize, expandtabs)
532	n/a	else:
533	n/a	action = eval(action + '_file')
534	n/a	for filename in args:
535	n/a	action(filename, stepsize, tabsize, expandtabs)
536	n/a	# end for
537	n/a	# end if
538	n/a	# end def test
539	n/a
540	n/a	if __name__ == '__main__':
541	n/a	test()
542	n/a	# end if