Python code coverage for Lib/distutils/filelist.py

#	count	content
1	n/a	"""distutils.filelist
2	n/a
3	n/a	Provides the FileList class, used for poking about the filesystem
4	n/a	and building lists of files.
5	n/a	"""
6	n/a
7	n/a	import os, re
8	n/a	import fnmatch
9	n/a	import functools
10	n/a	from distutils.util import convert_path
11	n/a	from distutils.errors import DistutilsTemplateError, DistutilsInternalError
12	n/a	from distutils import log
13	n/a
14	n/a	class FileList:
15	n/a	"""A list of files built by on exploring the filesystem and filtered by
16	n/a	applying various patterns to what we find there.
17	n/a
18	n/a	Instance attributes:
19	n/a	dir
20	n/a	directory from which files will be taken -- only used if
21	n/a	'allfiles' not supplied to constructor
22	n/a	files
23	n/a	list of filenames currently being built/filtered/manipulated
24	n/a	allfiles
25	n/a	complete list of files under consideration (ie. without any
26	n/a	filtering applied)
27	n/a	"""
28	n/a
29	n/a	def __init__(self, warn=None, debug_print=None):
30	n/a	# ignore argument to FileList, but keep them for backwards
31	n/a	# compatibility
32	n/a	self.allfiles = None
33	n/a	self.files = []
34	n/a
35	n/a	def set_allfiles(self, allfiles):
36	n/a	self.allfiles = allfiles
37	n/a
38	n/a	def findall(self, dir=os.curdir):
39	n/a	self.allfiles = findall(dir)
40	n/a
41	n/a	def debug_print(self, msg):
42	n/a	"""Print 'msg' to stdout if the global DEBUG (taken from the
43	n/a	DISTUTILS_DEBUG environment variable) flag is true.
44	n/a	"""
45	n/a	from distutils.debug import DEBUG
46	n/a	if DEBUG:
47	n/a	print(msg)
48	n/a
49	n/a	# -- List-like methods ---------------------------------------------
50	n/a
51	n/a	def append(self, item):
52	n/a	self.files.append(item)
53	n/a
54	n/a	def extend(self, items):
55	n/a	self.files.extend(items)
56	n/a
57	n/a	def sort(self):
58	n/a	# Not a strict lexical sort!
59	n/a	sortable_files = sorted(map(os.path.split, self.files))
60	n/a	self.files = []
61	n/a	for sort_tuple in sortable_files:
62	n/a	self.files.append(os.path.join(*sort_tuple))
63	n/a
64	n/a
65	n/a	# -- Other miscellaneous utility methods ---------------------------
66	n/a
67	n/a	def remove_duplicates(self):
68	n/a	# Assumes list has been sorted!
69	n/a	for i in range(len(self.files) - 1, 0, -1):
70	n/a	if self.files[i] == self.files[i - 1]:
71	n/a	del self.files[i]
72	n/a
73	n/a
74	n/a	# -- "File template" methods ---------------------------------------
75	n/a
76	n/a	def _parse_template_line(self, line):
77	n/a	words = line.split()
78	n/a	action = words[0]
79	n/a
80	n/a	patterns = dir = dir_pattern = None
81	n/a
82	n/a	if action in ('include', 'exclude',
83	n/a	'global-include', 'global-exclude'):
84	n/a	if len(words) < 2:
85	n/a	raise DistutilsTemplateError(
86	n/a	"'%s' expects <pattern1> <pattern2> ..." % action)
87	n/a	patterns = [convert_path(w) for w in words[1:]]
88	n/a	elif action in ('recursive-include', 'recursive-exclude'):
89	n/a	if len(words) < 3:
90	n/a	raise DistutilsTemplateError(
91	n/a	"'%s' expects <dir> <pattern1> <pattern2> ..." % action)
92	n/a	dir = convert_path(words[1])
93	n/a	patterns = [convert_path(w) for w in words[2:]]
94	n/a	elif action in ('graft', 'prune'):
95	n/a	if len(words) != 2:
96	n/a	raise DistutilsTemplateError(
97	n/a	"'%s' expects a single <dir_pattern>" % action)
98	n/a	dir_pattern = convert_path(words[1])
99	n/a	else:
100	n/a	raise DistutilsTemplateError("unknown action '%s'" % action)
101	n/a
102	n/a	return (action, patterns, dir, dir_pattern)
103	n/a
104	n/a	def process_template_line(self, line):
105	n/a	# Parse the line: split it up, make sure the right number of words
106	n/a	# is there, and return the relevant words. 'action' is always
107	n/a	# defined: it's the first word of the line. Which of the other
108	n/a	# three are defined depends on the action; it'll be either
109	n/a	# patterns, (dir and patterns), or (dir_pattern).
110	n/a	(action, patterns, dir, dir_pattern) = self._parse_template_line(line)
111	n/a
112	n/a	# OK, now we know that the action is valid and we have the
113	n/a	# right number of words on the line for that action -- so we
114	n/a	# can proceed with minimal error-checking.
115	n/a	if action == 'include':
116	n/a	self.debug_print("include " + ' '.join(patterns))
117	n/a	for pattern in patterns:
118	n/a	if not self.include_pattern(pattern, anchor=1):
119	n/a	log.warn("warning: no files found matching '%s'",
120	n/a	pattern)
121	n/a
122	n/a	elif action == 'exclude':
123	n/a	self.debug_print("exclude " + ' '.join(patterns))
124	n/a	for pattern in patterns:
125	n/a	if not self.exclude_pattern(pattern, anchor=1):
126	n/a	log.warn(("warning: no previously-included files "
127	n/a	"found matching '%s'"), pattern)
128	n/a
129	n/a	elif action == 'global-include':
130	n/a	self.debug_print("global-include " + ' '.join(patterns))
131	n/a	for pattern in patterns:
132	n/a	if not self.include_pattern(pattern, anchor=0):
133	n/a	log.warn(("warning: no files found matching '%s' "
134	n/a	"anywhere in distribution"), pattern)
135	n/a
136	n/a	elif action == 'global-exclude':
137	n/a	self.debug_print("global-exclude " + ' '.join(patterns))
138	n/a	for pattern in patterns:
139	n/a	if not self.exclude_pattern(pattern, anchor=0):
140	n/a	log.warn(("warning: no previously-included files matching "
141	n/a	"'%s' found anywhere in distribution"),
142	n/a	pattern)
143	n/a
144	n/a	elif action == 'recursive-include':
145	n/a	self.debug_print("recursive-include %s %s" %
146	n/a	(dir, ' '.join(patterns)))
147	n/a	for pattern in patterns:
148	n/a	if not self.include_pattern(pattern, prefix=dir):
149	n/a	log.warn(("warning: no files found matching '%s' "
150	n/a	"under directory '%s'"),
151	n/a	pattern, dir)
152	n/a
153	n/a	elif action == 'recursive-exclude':
154	n/a	self.debug_print("recursive-exclude %s %s" %
155	n/a	(dir, ' '.join(patterns)))
156	n/a	for pattern in patterns:
157	n/a	if not self.exclude_pattern(pattern, prefix=dir):
158	n/a	log.warn(("warning: no previously-included files matching "
159	n/a	"'%s' found under directory '%s'"),
160	n/a	pattern, dir)
161	n/a
162	n/a	elif action == 'graft':
163	n/a	self.debug_print("graft " + dir_pattern)
164	n/a	if not self.include_pattern(None, prefix=dir_pattern):
165	n/a	log.warn("warning: no directories found matching '%s'",
166	n/a	dir_pattern)
167	n/a
168	n/a	elif action == 'prune':
169	n/a	self.debug_print("prune " + dir_pattern)
170	n/a	if not self.exclude_pattern(None, prefix=dir_pattern):
171	n/a	log.warn(("no previously-included directories found "
172	n/a	"matching '%s'"), dir_pattern)
173	n/a	else:
174	n/a	raise DistutilsInternalError(
175	n/a	"this cannot happen: invalid action '%s'" % action)
176	n/a
177	n/a
178	n/a	# -- Filtering/selection methods -----------------------------------
179	n/a
180	n/a	def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0):
181	n/a	"""Select strings (presumably filenames) from 'self.files' that
182	n/a	match 'pattern', a Unix-style wildcard (glob) pattern. Patterns
183	n/a	are not quite the same as implemented by the 'fnmatch' module: '*'
184	n/a	and '?' match non-special characters, where "special" is platform-
185	n/a	dependent: slash on Unix; colon, slash, and backslash on
186	n/a	DOS/Windows; and colon on Mac OS.
187	n/a
188	n/a	If 'anchor' is true (the default), then the pattern match is more
189	n/a	stringent: "*.py" will match "foo.py" but not "foo/bar.py". If
190	n/a	'anchor' is false, both of these will match.
191	n/a
192	n/a	If 'prefix' is supplied, then only filenames starting with 'prefix'
193	n/a	(itself a pattern) and ending with 'pattern', with anything in between
194	n/a	them, will match. 'anchor' is ignored in this case.
195	n/a
196	n/a	If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and
197	n/a	'pattern' is assumed to be either a string containing a regex or a
198	n/a	regex object -- no translation is done, the regex is just compiled
199	n/a	and used as-is.
200	n/a
201	n/a	Selected strings will be added to self.files.
202	n/a
203	n/a	Return True if files are found, False otherwise.
204	n/a	"""
205	n/a	# XXX docstring lying about what the special chars are?
206	n/a	files_found = False
207	n/a	pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)
208	n/a	self.debug_print("include_pattern: applying regex r'%s'" %
209	n/a	pattern_re.pattern)
210	n/a
211	n/a	# delayed loading of allfiles list
212	n/a	if self.allfiles is None:
213	n/a	self.findall()
214	n/a
215	n/a	for name in self.allfiles:
216	n/a	if pattern_re.search(name):
217	n/a	self.debug_print(" adding " + name)
218	n/a	self.files.append(name)
219	n/a	files_found = True
220	n/a	return files_found
221	n/a
222	n/a
223	n/a	def exclude_pattern (self, pattern,
224	n/a	anchor=1, prefix=None, is_regex=0):
225	n/a	"""Remove strings (presumably filenames) from 'files' that match
226	n/a	'pattern'. Other parameters are the same as for
227	n/a	'include_pattern()', above.
228	n/a	The list 'self.files' is modified in place.
229	n/a	Return True if files are found, False otherwise.
230	n/a	"""
231	n/a	files_found = False
232	n/a	pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)
233	n/a	self.debug_print("exclude_pattern: applying regex r'%s'" %
234	n/a	pattern_re.pattern)
235	n/a	for i in range(len(self.files)-1, -1, -1):
236	n/a	if pattern_re.search(self.files[i]):
237	n/a	self.debug_print(" removing " + self.files[i])
238	n/a	del self.files[i]
239	n/a	files_found = True
240	n/a	return files_found
241	n/a
242	n/a
243	n/a	# ----------------------------------------------------------------------
244	n/a	# Utility functions
245	n/a
246	n/a	def _find_all_simple(path):
247	n/a	"""
248	n/a	Find all files under 'path'
249	n/a	"""
250	n/a	results = (
251	n/a	os.path.join(base, file)
252	n/a	for base, dirs, files in os.walk(path, followlinks=True)
253	n/a	for file in files
254	n/a	)
255	n/a	return filter(os.path.isfile, results)
256	n/a
257	n/a
258	n/a	def findall(dir=os.curdir):
259	n/a	"""
260	n/a	Find all files under 'dir' and return the list of full filenames.
261	n/a	Unless dir is '.', return full filenames with dir prepended.
262	n/a	"""
263	n/a	files = _find_all_simple(dir)
264	n/a	if dir == os.curdir:
265	n/a	make_rel = functools.partial(os.path.relpath, start=dir)
266	n/a	files = map(make_rel, files)
267	n/a	return list(files)
268	n/a
269	n/a
270	n/a	def glob_to_re(pattern):
271	n/a	"""Translate a shell-like glob pattern to a regular expression; return
272	n/a	a string containing the regex. Differs from 'fnmatch.translate()' in
273	n/a	that '*' does not match "special characters" (which are
274	n/a	platform-specific).
275	n/a	"""
276	n/a	pattern_re = fnmatch.translate(pattern)
277	n/a
278	n/a	# '?' and '' in the glob pattern become '.' and '.' in the RE, which
279	n/a	# IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix,
280	n/a	# and by extension they shouldn't match such "special characters" under
281	n/a	# any OS. So change all non-escaped dots in the RE to match any
282	n/a	# character except the special characters (currently: just os.sep).
283	n/a	sep = os.sep
284	n/a	if os.sep == '\\':
285	n/a	# we're using a regex to manipulate a regex, so we need
286	n/a	# to escape the backslash twice
287	n/a	sep = r'\\\\'
288	n/a	escaped = r'\1[^%s]' % sep
289	n/a	pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', escaped, pattern_re)
290	n/a	return pattern_re
291	n/a
292	n/a
293	n/a	def translate_pattern(pattern, anchor=1, prefix=None, is_regex=0):
294	n/a	"""Translate a shell-like wildcard pattern to a compiled regular
295	n/a	expression. Return the compiled regex. If 'is_regex' true,
296	n/a	then 'pattern' is directly compiled to a regex (if it's a string)
297	n/a	or just returned as-is (assumes it's a regex object).
298	n/a	"""
299	n/a	if is_regex:
300	n/a	if isinstance(pattern, str):
301	n/a	return re.compile(pattern)
302	n/a	else:
303	n/a	return pattern
304	n/a
305	n/a	# ditch start and end characters
306	n/a	start, _, end = glob_to_re('_').partition('_')
307	n/a
308	n/a	if pattern:
309	n/a	pattern_re = glob_to_re(pattern)
310	n/a	assert pattern_re.startswith(start) and pattern_re.endswith(end)
311	n/a	else:
312	n/a	pattern_re = ''
313	n/a
314	n/a	if prefix is not None:
315	n/a	prefix_re = glob_to_re(prefix)
316	n/a	assert prefix_re.startswith(start) and prefix_re.endswith(end)
317	n/a	prefix_re = prefix_re[len(start): len(prefix_re) - len(end)]
318	n/a	sep = os.sep
319	n/a	if os.sep == '\\':
320	n/a	sep = r'\\'
321	n/a	pattern_re = pattern_re[len(start): len(pattern_re) - len(end)]
322	n/a	pattern_re = r'%s\A%s%s.*%s%s' % (start, prefix_re, sep, pattern_re, end)
323	n/a	else: # no prefix -- respect anchor flag
324	n/a	if anchor:
325	n/a	pattern_re = r'%s\A%s' % (start, pattern_re[len(start):])
326	n/a
327	n/a	return re.compile(pattern_re)