Python code coverage for Lib/pipes.py

#	count	content
1	n/a	"""Conversion pipeline templates.
2	n/a
3	n/a	The problem:
4	n/a	------------
5	n/a
6	n/a	Suppose you have some data that you want to convert to another format,
7	n/a	such as from GIF image format to PPM image format. Maybe the
8	n/a	conversion involves several steps (e.g. piping it through compress or
9	n/a	uuencode). Some of the conversion steps may require that their input
10	n/a	is a disk file, others may be able to read standard input; similar for
11	n/a	their output. The input to the entire conversion may also be read
12	n/a	from a disk file or from an open file, and similar for its output.
13	n/a
14	n/a	The module lets you construct a pipeline template by sticking one or
15	n/a	more conversion steps together. It will take care of creating and
16	n/a	removing temporary files if they are necessary to hold intermediate
17	n/a	data. You can then use the template to do conversions from many
18	n/a	different sources to many different destinations. The temporary
19	n/a	file names used are different each time the template is used.
20	n/a
21	n/a	The templates are objects so you can create templates for many
22	n/a	different conversion steps and store them in a dictionary, for
23	n/a	instance.
24	n/a
25	n/a
26	n/a	Directions:
27	n/a	-----------
28	n/a
29	n/a	To create a template:
30	n/a	t = Template()
31	n/a
32	n/a	To add a conversion step to a template:
33	n/a	t.append(command, kind)
34	n/a	where kind is a string of two characters: the first is '-' if the
35	n/a	command reads its standard input or 'f' if it requires a file; the
36	n/a	second likewise for the output. The command must be valid /bin/sh
37	n/a	syntax. If input or output files are required, they are passed as
38	n/a	$IN and $OUT; otherwise, it must be possible to use the command in
39	n/a	a pipeline.
40	n/a
41	n/a	To add a conversion step at the beginning:
42	n/a	t.prepend(command, kind)
43	n/a
44	n/a	To convert a file to another file using a template:
45	n/a	sts = t.copy(infile, outfile)
46	n/a	If infile or outfile are the empty string, standard input is read or
47	n/a	standard output is written, respectively. The return value is the
48	n/a	exit status of the conversion pipeline.
49	n/a
50	n/a	To open a file for reading or writing through a conversion pipeline:
51	n/a	fp = t.open(file, mode)
52	n/a	where mode is 'r' to read the file, or 'w' to write it -- just like
53	n/a	for the built-in function open() or for os.popen().
54	n/a
55	n/a	To create a new template object initialized to a given one:
56	n/a	t2 = t.clone()
57	n/a	""" # '
58	n/a
59	n/a
60	n/a	import re
61	n/a	import os
62	n/a	import tempfile
63	n/a	# we import the quote function rather than the module for backward compat
64	n/a	# (quote used to be an undocumented but used function in pipes)
65	n/a	from shlex import quote
66	n/a
67	n/a	__all__ = ["Template"]
68	n/a
69	n/a	# Conversion step kinds
70	n/a
71	n/a	FILEIN_FILEOUT = 'ff' # Must read & write real files
72	n/a	STDIN_FILEOUT = '-f' # Must write a real file
73	n/a	FILEIN_STDOUT = 'f-' # Must read a real file
74	n/a	STDIN_STDOUT = '--' # Normal pipeline element
75	n/a	SOURCE = '.-' # Must be first, writes stdout
76	n/a	SINK = '-.' # Must be last, reads stdin
77	n/a
78	n/a	stepkinds = [FILEIN_FILEOUT, STDIN_FILEOUT, FILEIN_STDOUT, STDIN_STDOUT, \
79	n/a	SOURCE, SINK]
80	n/a
81	n/a
82	n/a	class Template:
83	n/a	"""Class representing a pipeline template."""
84	n/a
85	n/a	def __init__(self):
86	n/a	"""Template() returns a fresh pipeline template."""
87	n/a	self.debugging = 0
88	n/a	self.reset()
89	n/a
90	n/a	def __repr__(self):
91	n/a	"""t.__repr__() implements repr(t)."""
92	n/a	return '<Template instance, steps=%r>' % (self.steps,)
93	n/a
94	n/a	def reset(self):
95	n/a	"""t.reset() restores a pipeline template to its initial state."""
96	n/a	self.steps = []
97	n/a
98	n/a	def clone(self):
99	n/a	"""t.clone() returns a new pipeline template with identical
100	n/a	initial state as the current one."""
101	n/a	t = Template()
102	n/a	t.steps = self.steps[:]
103	n/a	t.debugging = self.debugging
104	n/a	return t
105	n/a
106	n/a	def debug(self, flag):
107	n/a	"""t.debug(flag) turns debugging on or off."""
108	n/a	self.debugging = flag
109	n/a
110	n/a	def append(self, cmd, kind):
111	n/a	"""t.append(cmd, kind) adds a new step at the end."""
112	n/a	if type(cmd) is not type(''):
113	n/a	raise TypeError('Template.append: cmd must be a string')
114	n/a	if kind not in stepkinds:
115	n/a	raise ValueError('Template.append: bad kind %r' % (kind,))
116	n/a	if kind == SOURCE:
117	n/a	raise ValueError('Template.append: SOURCE can only be prepended')
118	n/a	if self.steps and self.steps[-1][1] == SINK:
119	n/a	raise ValueError('Template.append: already ends with SINK')
120	n/a	if kind[0] == 'f' and not re.search(r'\$IN\b', cmd):
121	n/a	raise ValueError('Template.append: missing $IN in cmd')
122	n/a	if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd):
123	n/a	raise ValueError('Template.append: missing $OUT in cmd')
124	n/a	self.steps.append((cmd, kind))
125	n/a
126	n/a	def prepend(self, cmd, kind):
127	n/a	"""t.prepend(cmd, kind) adds a new step at the front."""
128	n/a	if type(cmd) is not type(''):
129	n/a	raise TypeError('Template.prepend: cmd must be a string')
130	n/a	if kind not in stepkinds:
131	n/a	raise ValueError('Template.prepend: bad kind %r' % (kind,))
132	n/a	if kind == SINK:
133	n/a	raise ValueError('Template.prepend: SINK can only be appended')
134	n/a	if self.steps and self.steps[0][1] == SOURCE:
135	n/a	raise ValueError('Template.prepend: already begins with SOURCE')
136	n/a	if kind[0] == 'f' and not re.search(r'\$IN\b', cmd):
137	n/a	raise ValueError('Template.prepend: missing $IN in cmd')
138	n/a	if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd):
139	n/a	raise ValueError('Template.prepend: missing $OUT in cmd')
140	n/a	self.steps.insert(0, (cmd, kind))
141	n/a
142	n/a	def open(self, file, rw):
143	n/a	"""t.open(file, rw) returns a pipe or file object open for
144	n/a	reading or writing; the file is the other end of the pipeline."""
145	n/a	if rw == 'r':
146	n/a	return self.open_r(file)
147	n/a	if rw == 'w':
148	n/a	return self.open_w(file)
149	n/a	raise ValueError('Template.open: rw must be \'r\' or \'w\', not %r'
150	n/a	% (rw,))
151	n/a
152	n/a	def open_r(self, file):
153	n/a	"""t.open_r(file) and t.open_w(file) implement
154	n/a	t.open(file, 'r') and t.open(file, 'w') respectively."""
155	n/a	if not self.steps:
156	n/a	return open(file, 'r')
157	n/a	if self.steps[-1][1] == SINK:
158	n/a	raise ValueError('Template.open_r: pipeline ends width SINK')
159	n/a	cmd = self.makepipeline(file, '')
160	n/a	return os.popen(cmd, 'r')
161	n/a
162	n/a	def open_w(self, file):
163	n/a	if not self.steps:
164	n/a	return open(file, 'w')
165	n/a	if self.steps[0][1] == SOURCE:
166	n/a	raise ValueError('Template.open_w: pipeline begins with SOURCE')
167	n/a	cmd = self.makepipeline('', file)
168	n/a	return os.popen(cmd, 'w')
169	n/a
170	n/a	def copy(self, infile, outfile):
171	n/a	return os.system(self.makepipeline(infile, outfile))
172	n/a
173	n/a	def makepipeline(self, infile, outfile):
174	n/a	cmd = makepipeline(infile, self.steps, outfile)
175	n/a	if self.debugging:
176	n/a	print(cmd)
177	n/a	cmd = 'set -x; ' + cmd
178	n/a	return cmd
179	n/a
180	n/a
181	n/a	def makepipeline(infile, steps, outfile):
182	n/a	# Build a list with for each command:
183	n/a	# [input filename or '', command string, kind, output filename or '']
184	n/a
185	n/a	list = []
186	n/a	for cmd, kind in steps:
187	n/a	list.append(['', cmd, kind, ''])
188	n/a	#
189	n/a	# Make sure there is at least one step
190	n/a	#
191	n/a	if not list:
192	n/a	list.append(['', 'cat', '--', ''])
193	n/a	#
194	n/a	# Take care of the input and output ends
195	n/a	#
196	n/a	[cmd, kind] = list[0][1:3]
197	n/a	if kind[0] == 'f' and not infile:
198	n/a	list.insert(0, ['', 'cat', '--', ''])
199	n/a	list[0][0] = infile
200	n/a	#
201	n/a	[cmd, kind] = list[-1][1:3]
202	n/a	if kind[1] == 'f' and not outfile:
203	n/a	list.append(['', 'cat', '--', ''])
204	n/a	list[-1][-1] = outfile
205	n/a	#
206	n/a	# Invent temporary files to connect stages that need files
207	n/a	#
208	n/a	garbage = []
209	n/a	for i in range(1, len(list)):
210	n/a	lkind = list[i-1][2]
211	n/a	rkind = list[i][2]
212	n/a	if lkind[1] == 'f' or rkind[0] == 'f':
213	n/a	(fd, temp) = tempfile.mkstemp()
214	n/a	os.close(fd)
215	n/a	garbage.append(temp)
216	n/a	list[i-1][-1] = list[i][0] = temp
217	n/a	#
218	n/a	for item in list:
219	n/a	[inf, cmd, kind, outf] = item
220	n/a	if kind[1] == 'f':
221	n/a	cmd = 'OUT=' + quote(outf) + '; ' + cmd
222	n/a	if kind[0] == 'f':
223	n/a	cmd = 'IN=' + quote(inf) + '; ' + cmd
224	n/a	if kind[0] == '-' and inf:
225	n/a	cmd = cmd + ' <' + quote(inf)
226	n/a	if kind[1] == '-' and outf:
227	n/a	cmd = cmd + ' >' + quote(outf)
228	n/a	item[1] = cmd
229	n/a	#
230	n/a	cmdlist = list[0][1]
231	n/a	for item in list[1:]:
232	n/a	[cmd, kind] = item[1:3]
233	n/a	if item[0] == '':
234	n/a	if 'f' in kind:
235	n/a	cmd = '{ ' + cmd + '; }'
236	n/a	cmdlist = cmdlist + ' \|\n' + cmd
237	n/a	else:
238	n/a	cmdlist = cmdlist + '\n' + cmd
239	n/a	#
240	n/a	if garbage:
241	n/a	rmcmd = 'rm -f'
242	n/a	for file in garbage:
243	n/a	rmcmd = rmcmd + ' ' + quote(file)
244	n/a	trapcmd = 'trap ' + quote(rmcmd + '; exit') + ' 1 2 3 13 14 15'
245	n/a	cmdlist = trapcmd + '\n' + cmdlist + '\n' + rmcmd
246	n/a	#
247	n/a	return cmdlist