ยปCore Development>Code coverage>Lib/pipes.py

Python code coverage for Lib/pipes.py

#countcontent
1n/a"""Conversion pipeline templates.
2n/a
3n/aThe problem:
4n/a------------
5n/a
6n/aSuppose you have some data that you want to convert to another format,
7n/asuch as from GIF image format to PPM image format. Maybe the
8n/aconversion involves several steps (e.g. piping it through compress or
9n/auuencode). Some of the conversion steps may require that their input
10n/ais a disk file, others may be able to read standard input; similar for
11n/atheir output. The input to the entire conversion may also be read
12n/afrom a disk file or from an open file, and similar for its output.
13n/a
14n/aThe module lets you construct a pipeline template by sticking one or
15n/amore conversion steps together. It will take care of creating and
16n/aremoving temporary files if they are necessary to hold intermediate
17n/adata. You can then use the template to do conversions from many
18n/adifferent sources to many different destinations. The temporary
19n/afile names used are different each time the template is used.
20n/a
21n/aThe templates are objects so you can create templates for many
22n/adifferent conversion steps and store them in a dictionary, for
23n/ainstance.
24n/a
25n/a
26n/aDirections:
27n/a-----------
28n/a
29n/aTo create a template:
30n/a t = Template()
31n/a
32n/aTo add a conversion step to a template:
33n/a t.append(command, kind)
34n/awhere kind is a string of two characters: the first is '-' if the
35n/acommand reads its standard input or 'f' if it requires a file; the
36n/asecond likewise for the output. The command must be valid /bin/sh
37n/asyntax. If input or output files are required, they are passed as
38n/a$IN and $OUT; otherwise, it must be possible to use the command in
39n/aa pipeline.
40n/a
41n/aTo add a conversion step at the beginning:
42n/a t.prepend(command, kind)
43n/a
44n/aTo convert a file to another file using a template:
45n/a sts = t.copy(infile, outfile)
46n/aIf infile or outfile are the empty string, standard input is read or
47n/astandard output is written, respectively. The return value is the
48n/aexit status of the conversion pipeline.
49n/a
50n/aTo open a file for reading or writing through a conversion pipeline:
51n/a fp = t.open(file, mode)
52n/awhere mode is 'r' to read the file, or 'w' to write it -- just like
53n/afor the built-in function open() or for os.popen().
54n/a
55n/aTo create a new template object initialized to a given one:
56n/a t2 = t.clone()
57n/a""" # '
58n/a
59n/a
60n/aimport re
61n/aimport os
62n/aimport tempfile
63n/a# we import the quote function rather than the module for backward compat
64n/a# (quote used to be an undocumented but used function in pipes)
65n/afrom shlex import quote
66n/a
67n/a__all__ = ["Template"]
68n/a
69n/a# Conversion step kinds
70n/a
71n/aFILEIN_FILEOUT = 'ff' # Must read & write real files
72n/aSTDIN_FILEOUT = '-f' # Must write a real file
73n/aFILEIN_STDOUT = 'f-' # Must read a real file
74n/aSTDIN_STDOUT = '--' # Normal pipeline element
75n/aSOURCE = '.-' # Must be first, writes stdout
76n/aSINK = '-.' # Must be last, reads stdin
77n/a
78n/astepkinds = [FILEIN_FILEOUT, STDIN_FILEOUT, FILEIN_STDOUT, STDIN_STDOUT, \
79n/a SOURCE, SINK]
80n/a
81n/a
82n/aclass Template:
83n/a """Class representing a pipeline template."""
84n/a
85n/a def __init__(self):
86n/a """Template() returns a fresh pipeline template."""
87n/a self.debugging = 0
88n/a self.reset()
89n/a
90n/a def __repr__(self):
91n/a """t.__repr__() implements repr(t)."""
92n/a return '<Template instance, steps=%r>' % (self.steps,)
93n/a
94n/a def reset(self):
95n/a """t.reset() restores a pipeline template to its initial state."""
96n/a self.steps = []
97n/a
98n/a def clone(self):
99n/a """t.clone() returns a new pipeline template with identical
100n/a initial state as the current one."""
101n/a t = Template()
102n/a t.steps = self.steps[:]
103n/a t.debugging = self.debugging
104n/a return t
105n/a
106n/a def debug(self, flag):
107n/a """t.debug(flag) turns debugging on or off."""
108n/a self.debugging = flag
109n/a
110n/a def append(self, cmd, kind):
111n/a """t.append(cmd, kind) adds a new step at the end."""
112n/a if type(cmd) is not type(''):
113n/a raise TypeError('Template.append: cmd must be a string')
114n/a if kind not in stepkinds:
115n/a raise ValueError('Template.append: bad kind %r' % (kind,))
116n/a if kind == SOURCE:
117n/a raise ValueError('Template.append: SOURCE can only be prepended')
118n/a if self.steps and self.steps[-1][1] == SINK:
119n/a raise ValueError('Template.append: already ends with SINK')
120n/a if kind[0] == 'f' and not re.search(r'\$IN\b', cmd):
121n/a raise ValueError('Template.append: missing $IN in cmd')
122n/a if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd):
123n/a raise ValueError('Template.append: missing $OUT in cmd')
124n/a self.steps.append((cmd, kind))
125n/a
126n/a def prepend(self, cmd, kind):
127n/a """t.prepend(cmd, kind) adds a new step at the front."""
128n/a if type(cmd) is not type(''):
129n/a raise TypeError('Template.prepend: cmd must be a string')
130n/a if kind not in stepkinds:
131n/a raise ValueError('Template.prepend: bad kind %r' % (kind,))
132n/a if kind == SINK:
133n/a raise ValueError('Template.prepend: SINK can only be appended')
134n/a if self.steps and self.steps[0][1] == SOURCE:
135n/a raise ValueError('Template.prepend: already begins with SOURCE')
136n/a if kind[0] == 'f' and not re.search(r'\$IN\b', cmd):
137n/a raise ValueError('Template.prepend: missing $IN in cmd')
138n/a if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd):
139n/a raise ValueError('Template.prepend: missing $OUT in cmd')
140n/a self.steps.insert(0, (cmd, kind))
141n/a
142n/a def open(self, file, rw):
143n/a """t.open(file, rw) returns a pipe or file object open for
144n/a reading or writing; the file is the other end of the pipeline."""
145n/a if rw == 'r':
146n/a return self.open_r(file)
147n/a if rw == 'w':
148n/a return self.open_w(file)
149n/a raise ValueError('Template.open: rw must be \'r\' or \'w\', not %r'
150n/a % (rw,))
151n/a
152n/a def open_r(self, file):
153n/a """t.open_r(file) and t.open_w(file) implement
154n/a t.open(file, 'r') and t.open(file, 'w') respectively."""
155n/a if not self.steps:
156n/a return open(file, 'r')
157n/a if self.steps[-1][1] == SINK:
158n/a raise ValueError('Template.open_r: pipeline ends width SINK')
159n/a cmd = self.makepipeline(file, '')
160n/a return os.popen(cmd, 'r')
161n/a
162n/a def open_w(self, file):
163n/a if not self.steps:
164n/a return open(file, 'w')
165n/a if self.steps[0][1] == SOURCE:
166n/a raise ValueError('Template.open_w: pipeline begins with SOURCE')
167n/a cmd = self.makepipeline('', file)
168n/a return os.popen(cmd, 'w')
169n/a
170n/a def copy(self, infile, outfile):
171n/a return os.system(self.makepipeline(infile, outfile))
172n/a
173n/a def makepipeline(self, infile, outfile):
174n/a cmd = makepipeline(infile, self.steps, outfile)
175n/a if self.debugging:
176n/a print(cmd)
177n/a cmd = 'set -x; ' + cmd
178n/a return cmd
179n/a
180n/a
181n/adef makepipeline(infile, steps, outfile):
182n/a # Build a list with for each command:
183n/a # [input filename or '', command string, kind, output filename or '']
184n/a
185n/a list = []
186n/a for cmd, kind in steps:
187n/a list.append(['', cmd, kind, ''])
188n/a #
189n/a # Make sure there is at least one step
190n/a #
191n/a if not list:
192n/a list.append(['', 'cat', '--', ''])
193n/a #
194n/a # Take care of the input and output ends
195n/a #
196n/a [cmd, kind] = list[0][1:3]
197n/a if kind[0] == 'f' and not infile:
198n/a list.insert(0, ['', 'cat', '--', ''])
199n/a list[0][0] = infile
200n/a #
201n/a [cmd, kind] = list[-1][1:3]
202n/a if kind[1] == 'f' and not outfile:
203n/a list.append(['', 'cat', '--', ''])
204n/a list[-1][-1] = outfile
205n/a #
206n/a # Invent temporary files to connect stages that need files
207n/a #
208n/a garbage = []
209n/a for i in range(1, len(list)):
210n/a lkind = list[i-1][2]
211n/a rkind = list[i][2]
212n/a if lkind[1] == 'f' or rkind[0] == 'f':
213n/a (fd, temp) = tempfile.mkstemp()
214n/a os.close(fd)
215n/a garbage.append(temp)
216n/a list[i-1][-1] = list[i][0] = temp
217n/a #
218n/a for item in list:
219n/a [inf, cmd, kind, outf] = item
220n/a if kind[1] == 'f':
221n/a cmd = 'OUT=' + quote(outf) + '; ' + cmd
222n/a if kind[0] == 'f':
223n/a cmd = 'IN=' + quote(inf) + '; ' + cmd
224n/a if kind[0] == '-' and inf:
225n/a cmd = cmd + ' <' + quote(inf)
226n/a if kind[1] == '-' and outf:
227n/a cmd = cmd + ' >' + quote(outf)
228n/a item[1] = cmd
229n/a #
230n/a cmdlist = list[0][1]
231n/a for item in list[1:]:
232n/a [cmd, kind] = item[1:3]
233n/a if item[0] == '':
234n/a if 'f' in kind:
235n/a cmd = '{ ' + cmd + '; }'
236n/a cmdlist = cmdlist + ' |\n' + cmd
237n/a else:
238n/a cmdlist = cmdlist + '\n' + cmd
239n/a #
240n/a if garbage:
241n/a rmcmd = 'rm -f'
242n/a for file in garbage:
243n/a rmcmd = rmcmd + ' ' + quote(file)
244n/a trapcmd = 'trap ' + quote(rmcmd + '; exit') + ' 1 2 3 13 14 15'
245n/a cmdlist = trapcmd + '\n' + cmdlist + '\n' + rmcmd
246n/a #
247n/a return cmdlist