Python code coverage for Lib/packaging/manifest.py

#	count	content
1	n/a	"""Class representing the list of files in a distribution.
2	n/a
3	n/a	The Manifest class can be used to:
4	n/a
5	n/a	- read or write a MANIFEST file
6	n/a	- read a template file and find out the file list
7	n/a	"""
8	n/a	# XXX todo: document + add tests
9	n/a	import re
10	n/a	import os
11	n/a	import fnmatch
12	n/a
13	n/a	from packaging import logger
14	n/a	from packaging.util import write_file, convert_path
15	n/a	from packaging.errors import (PackagingTemplateError,
16	n/a	PackagingInternalError)
17	n/a
18	n/a	__all__ = ['Manifest']
19	n/a
20	n/a	# a \ followed by some spaces + EOL
21	n/a	_COLLAPSE_PATTERN = re.compile('\\\w*\n', re.M)
22	n/a	_COMMENTED_LINE = re.compile('#.*?(?=\n)\|\n(?=$)', re.M \| re.S)
23	n/a
24	n/a
25	n/a	class Manifest(object):
26	n/a	"""A list of files built by on exploring the filesystem and filtered by
27	n/a	applying various patterns to what we find there.
28	n/a	"""
29	n/a
30	n/a	def __init__(self):
31	n/a	self.allfiles = None
32	n/a	self.files = []
33	n/a
34	n/a	#
35	n/a	# Public API
36	n/a	#
37	n/a
38	n/a	def findall(self, dir=os.curdir):
39	n/a	self.allfiles = _findall(dir)
40	n/a
41	n/a	def append(self, item):
42	n/a	self.files.append(item)
43	n/a
44	n/a	def extend(self, items):
45	n/a	self.files.extend(items)
46	n/a
47	n/a	def sort(self):
48	n/a	# Not a strict lexical sort!
49	n/a	self.files = [os.path.join(*path_tuple) for path_tuple in
50	n/a	sorted(os.path.split(path) for path in self.files)]
51	n/a
52	n/a	def clear(self):
53	n/a	"""Clear all collected files."""
54	n/a	self.files = []
55	n/a	if self.allfiles is not None:
56	n/a	self.allfiles = []
57	n/a
58	n/a	def remove_duplicates(self):
59	n/a	# Assumes list has been sorted!
60	n/a	for i in range(len(self.files) - 1, 0, -1):
61	n/a	if self.files[i] == self.files[i - 1]:
62	n/a	del self.files[i]
63	n/a
64	n/a	def read_template(self, path_or_file):
65	n/a	"""Read and parse a manifest template file.
66	n/a	'path' can be a path or a file-like object.
67	n/a
68	n/a	Updates the list accordingly.
69	n/a	"""
70	n/a	if isinstance(path_or_file, str):
71	n/a	f = open(path_or_file)
72	n/a	else:
73	n/a	f = path_or_file
74	n/a
75	n/a	try:
76	n/a	content = f.read()
77	n/a	# first, let's unwrap collapsed lines
78	n/a	content = _COLLAPSE_PATTERN.sub('', content)
79	n/a	# next, let's remove commented lines and empty lines
80	n/a	content = _COMMENTED_LINE.sub('', content)
81	n/a
82	n/a	# now we have our cleaned up lines
83	n/a	lines = [line.strip() for line in content.split('\n')]
84	n/a	finally:
85	n/a	f.close()
86	n/a
87	n/a	for line in lines:
88	n/a	if line == '':
89	n/a	continue
90	n/a	try:
91	n/a	self._process_template_line(line)
92	n/a	except PackagingTemplateError as msg:
93	n/a	logger.warning("%s, %s", path_or_file, msg)
94	n/a
95	n/a	def write(self, path):
96	n/a	"""Write the file list in 'self.filelist' (presumably as filled in
97	n/a	by 'add_defaults()' and 'read_template()') to the manifest file
98	n/a	named by 'self.manifest'.
99	n/a	"""
100	n/a	if os.path.isfile(path):
101	n/a	with open(path) as fp:
102	n/a	first_line = fp.readline()
103	n/a
104	n/a	if first_line != '# file GENERATED by packaging, do NOT edit\n':
105	n/a	logger.info("not writing to manually maintained "
106	n/a	"manifest file %r", path)
107	n/a	return
108	n/a
109	n/a	self.sort()
110	n/a	self.remove_duplicates()
111	n/a	content = self.files[:]
112	n/a	content.insert(0, '# file GENERATED by packaging, do NOT edit')
113	n/a	logger.info("writing manifest file %r", path)
114	n/a	write_file(path, content)
115	n/a
116	n/a	def read(self, path):
117	n/a	"""Read the manifest file (named by 'self.manifest') and use it to
118	n/a	fill in 'self.filelist', the list of files to include in the source
119	n/a	distribution.
120	n/a	"""
121	n/a	logger.info("reading manifest file %r", path)
122	n/a	with open(path) as manifest:
123	n/a	for line in manifest.readlines():
124	n/a	self.append(line)
125	n/a
126	n/a	def exclude_pattern(self, pattern, anchor=True, prefix=None,
127	n/a	is_regex=False):
128	n/a	"""Remove strings (presumably filenames) from 'files' that match
129	n/a	'pattern'.
130	n/a
131	n/a	Other parameters are the same as for 'include_pattern()', above.
132	n/a	The list 'self.files' is modified in place. Return True if files are
133	n/a	found.
134	n/a	"""
135	n/a	files_found = False
136	n/a	pattern_re = _translate_pattern(pattern, anchor, prefix, is_regex)
137	n/a	for i in range(len(self.files) - 1, -1, -1):
138	n/a	if pattern_re.search(self.files[i]):
139	n/a	del self.files[i]
140	n/a	files_found = True
141	n/a
142	n/a	return files_found
143	n/a
144	n/a	#
145	n/a	# Private API
146	n/a	#
147	n/a
148	n/a	def _parse_template_line(self, line):
149	n/a	words = line.split()
150	n/a	if len(words) == 1 and words[0] not in (
151	n/a	'include', 'exclude', 'global-include', 'global-exclude',
152	n/a	'recursive-include', 'recursive-exclude', 'graft', 'prune'):
153	n/a	# no action given, let's use the default 'include'
154	n/a	words.insert(0, 'include')
155	n/a
156	n/a	action = words[0]
157	n/a	patterns = dir = dir_pattern = None
158	n/a
159	n/a	if action in ('include', 'exclude',
160	n/a	'global-include', 'global-exclude'):
161	n/a	if len(words) < 2:
162	n/a	raise PackagingTemplateError(
163	n/a	"%r expects <pattern1> <pattern2> ..." % action)
164	n/a
165	n/a	patterns = [convert_path(word) for word in words[1:]]
166	n/a
167	n/a	elif action in ('recursive-include', 'recursive-exclude'):
168	n/a	if len(words) < 3:
169	n/a	raise PackagingTemplateError(
170	n/a	"%r expects <dir> <pattern1> <pattern2> ..." % action)
171	n/a
172	n/a	dir = convert_path(words[1])
173	n/a	patterns = [convert_path(word) for word in words[2:]]
174	n/a
175	n/a	elif action in ('graft', 'prune'):
176	n/a	if len(words) != 2:
177	n/a	raise PackagingTemplateError(
178	n/a	"%r expects a single <dir_pattern>" % action)
179	n/a
180	n/a	dir_pattern = convert_path(words[1])
181	n/a
182	n/a	else:
183	n/a	raise PackagingTemplateError("unknown action %r" % action)
184	n/a
185	n/a	return action, patterns, dir, dir_pattern
186	n/a
187	n/a	def _process_template_line(self, line):
188	n/a	# Parse the line: split it up, make sure the right number of words
189	n/a	# is there, and return the relevant words. 'action' is always
190	n/a	# defined: it's the first word of the line. Which of the other
191	n/a	# three are defined depends on the action; it'll be either
192	n/a	# patterns, (dir and patterns), or (dir_pattern).
193	n/a	action, patterns, dir, dir_pattern = self._parse_template_line(line)
194	n/a
195	n/a	# OK, now we know that the action is valid and we have the
196	n/a	# right number of words on the line for that action -- so we
197	n/a	# can proceed with minimal error-checking.
198	n/a	if action == 'include':
199	n/a	for pattern in patterns:
200	n/a	if not self._include_pattern(pattern, anchor=True):
201	n/a	logger.warning("no files found matching %r", pattern)
202	n/a
203	n/a	elif action == 'exclude':
204	n/a	for pattern in patterns:
205	n/a	if not self.exclude_pattern(pattern, anchor=True):
206	n/a	logger.warning("no previously-included files "
207	n/a	"found matching %r", pattern)
208	n/a
209	n/a	elif action == 'global-include':
210	n/a	for pattern in patterns:
211	n/a	if not self._include_pattern(pattern, anchor=False):
212	n/a	logger.warning("no files found matching %r "
213	n/a	"anywhere in distribution", pattern)
214	n/a
215	n/a	elif action == 'global-exclude':
216	n/a	for pattern in patterns:
217	n/a	if not self.exclude_pattern(pattern, anchor=False):
218	n/a	logger.warning("no previously-included files "
219	n/a	"matching %r found anywhere in "
220	n/a	"distribution", pattern)
221	n/a
222	n/a	elif action == 'recursive-include':
223	n/a	for pattern in patterns:
224	n/a	if not self._include_pattern(pattern, prefix=dir):
225	n/a	logger.warning("no files found matching %r "
226	n/a	"under directory %r", pattern, dir)
227	n/a
228	n/a	elif action == 'recursive-exclude':
229	n/a	for pattern in patterns:
230	n/a	if not self.exclude_pattern(pattern, prefix=dir):
231	n/a	logger.warning("no previously-included files "
232	n/a	"matching %r found under directory %r",
233	n/a	pattern, dir)
234	n/a
235	n/a	elif action == 'graft':
236	n/a	if not self._include_pattern(None, prefix=dir_pattern):
237	n/a	logger.warning("no directories found matching %r",
238	n/a	dir_pattern)
239	n/a
240	n/a	elif action == 'prune':
241	n/a	if not self.exclude_pattern(None, prefix=dir_pattern):
242	n/a	logger.warning("no previously-included directories found "
243	n/a	"matching %r", dir_pattern)
244	n/a	else:
245	n/a	raise PackagingInternalError(
246	n/a	"this cannot happen: invalid action %r" % action)
247	n/a
248	n/a	def _include_pattern(self, pattern, anchor=True, prefix=None,
249	n/a	is_regex=False):
250	n/a	"""Select strings (presumably filenames) from 'self.files' that
251	n/a	match 'pattern', a Unix-style wildcard (glob) pattern.
252	n/a
253	n/a	Patterns are not quite the same as implemented by the 'fnmatch'
254	n/a	module: '*' and '?' match non-special characters, where "special"
255	n/a	is platform-dependent: slash on Unix; colon, slash, and backslash on
256	n/a	DOS/Windows; and colon on Mac OS.
257	n/a
258	n/a	If 'anchor' is true (the default), then the pattern match is more
259	n/a	stringent: "*.py" will match "foo.py" but not "foo/bar.py". If
260	n/a	'anchor' is false, both of these will match.
261	n/a
262	n/a	If 'prefix' is supplied, then only filenames starting with 'prefix'
263	n/a	(itself a pattern) and ending with 'pattern', with anything in between
264	n/a	them, will match. 'anchor' is ignored in this case.
265	n/a
266	n/a	If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and
267	n/a	'pattern' is assumed to be either a string containing a regex or a
268	n/a	regex object -- no translation is done, the regex is just compiled
269	n/a	and used as-is.
270	n/a
271	n/a	Selected strings will be added to self.files.
272	n/a
273	n/a	Return True if files are found.
274	n/a	"""
275	n/a	# XXX docstring lying about what the special chars are?
276	n/a	files_found = False
277	n/a	pattern_re = _translate_pattern(pattern, anchor, prefix, is_regex)
278	n/a
279	n/a	# delayed loading of allfiles list
280	n/a	if self.allfiles is None:
281	n/a	self.findall()
282	n/a
283	n/a	for name in self.allfiles:
284	n/a	if pattern_re.search(name):
285	n/a	self.files.append(name)
286	n/a	files_found = True
287	n/a
288	n/a	return files_found
289	n/a
290	n/a
291	n/a	#
292	n/a	# Utility functions
293	n/a	#
294	n/a	def _findall(dir=os.curdir):
295	n/a	"""Find all files under 'dir' and return the list of full filenames
296	n/a	(relative to 'dir').
297	n/a	"""
298	n/a	from stat import S_ISREG, S_ISDIR, S_ISLNK
299	n/a
300	n/a	list = []
301	n/a	stack = [dir]
302	n/a	pop = stack.pop
303	n/a	push = stack.append
304	n/a
305	n/a	while stack:
306	n/a	dir = pop()
307	n/a	names = os.listdir(dir)
308	n/a
309	n/a	for name in names:
310	n/a	if dir != os.curdir: # avoid the dreaded "./" syndrome
311	n/a	fullname = os.path.join(dir, name)
312	n/a	else:
313	n/a	fullname = name
314	n/a
315	n/a	# Avoid excess stat calls -- just one will do, thank you!
316	n/a	stat = os.stat(fullname)
317	n/a	mode = stat.st_mode
318	n/a	if S_ISREG(mode):
319	n/a	list.append(fullname)
320	n/a	elif S_ISDIR(mode) and not S_ISLNK(mode):
321	n/a	push(fullname)
322	n/a
323	n/a	return list
324	n/a
325	n/a
326	n/a	def _glob_to_re(pattern):
327	n/a	"""Translate a shell-like glob pattern to a regular expression.
328	n/a
329	n/a	Return a string containing the regex. Differs from
330	n/a	'fnmatch.translate()' in that '*' does not match "special characters"
331	n/a	(which are platform-specific).
332	n/a	"""
333	n/a	pattern_re = fnmatch.translate(pattern)
334	n/a
335	n/a	# '?' and '' in the glob pattern become '.' and '.' in the RE, which
336	n/a	# IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix,
337	n/a	# and by extension they shouldn't match such "special characters" under
338	n/a	# any OS. So change all non-escaped dots in the RE to match any
339	n/a	# character except the special characters (currently: just os.sep).
340	n/a	sep = os.sep
341	n/a	if os.sep == '\\':
342	n/a	# we're using a regex to manipulate a regex, so we need
343	n/a	# to escape the backslash twice
344	n/a	sep = r'\\\\'
345	n/a	escaped = r'\1[^%s]' % sep
346	n/a	pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', escaped, pattern_re)
347	n/a	return pattern_re
348	n/a
349	n/a
350	n/a	def _translate_pattern(pattern, anchor=True, prefix=None, is_regex=False):
351	n/a	"""Translate a shell-like wildcard pattern to a compiled regular
352	n/a	expression.
353	n/a
354	n/a	Return the compiled regex. If 'is_regex' true,
355	n/a	then 'pattern' is directly compiled to a regex (if it's a string)
356	n/a	or just returned as-is (assumes it's a regex object).
357	n/a	"""
358	n/a	if is_regex:
359	n/a	if isinstance(pattern, str):
360	n/a	return re.compile(pattern)
361	n/a	else:
362	n/a	return pattern
363	n/a
364	n/a	if pattern:
365	n/a	pattern_re = _glob_to_re(pattern)
366	n/a	else:
367	n/a	pattern_re = ''
368	n/a
369	n/a	if prefix is not None:
370	n/a	# ditch end of pattern character
371	n/a	empty_pattern = _glob_to_re('')
372	n/a	prefix_re = _glob_to_re(prefix)[:-len(empty_pattern)]
373	n/a	sep = os.sep
374	n/a	if os.sep == '\\':
375	n/a	sep = r'\\'
376	n/a	pattern_re = "^" + sep.join((prefix_re, ".*" + pattern_re))
377	n/a	else: # no prefix -- respect anchor flag
378	n/a	if anchor:
379	n/a	pattern_re = "^" + pattern_re
380	n/a
381	n/a	return re.compile(pattern_re)