Python code coverage for Lib/packaging/util.py

#	count	content
1	n/a	"""Miscellaneous utility functions."""
2	n/a
3	n/a	import os
4	n/a	import re
5	n/a	import csv
6	n/a	import imp
7	n/a	import sys
8	n/a	import errno
9	n/a	import codecs
10	n/a	import shutil
11	n/a	import string
12	n/a	import hashlib
13	n/a	import posixpath
14	n/a	import subprocess
15	n/a	import sysconfig
16	n/a	from glob import iglob as std_iglob
17	n/a	from fnmatch import fnmatchcase
18	n/a	from inspect import getsource
19	n/a	from configparser import RawConfigParser
20	n/a
21	n/a	from packaging import logger
22	n/a	from packaging.errors import (PackagingPlatformError, PackagingFileError,
23	n/a	PackagingExecError, InstallationException,
24	n/a	PackagingInternalError)
25	n/a
26	n/a	__all__ = [
27	n/a	# file dependencies
28	n/a	'newer', 'newer_group',
29	n/a	# helpers for commands (dry-run system)
30	n/a	'execute', 'write_file',
31	n/a	# spawning programs
32	n/a	'find_executable', 'spawn',
33	n/a	# path manipulation
34	n/a	'convert_path', 'change_root',
35	n/a	# 2to3 conversion
36	n/a	'Mixin2to3', 'run_2to3',
37	n/a	# packaging compatibility helpers
38	n/a	'cfg_to_args', 'generate_setup_py',
39	n/a	'egginfo_to_distinfo',
40	n/a	'get_install_method',
41	n/a	# misc
42	n/a	'ask', 'check_environ', 'encode_multipart', 'resolve_name',
43	n/a	# querying for information TODO move to sysconfig
44	n/a	'get_compiler_versions', 'get_platform', 'set_platform',
45	n/a	# configuration TODO move to packaging.config
46	n/a	'get_pypirc_path', 'read_pypirc', 'generate_pypirc',
47	n/a	'strtobool', 'split_multiline',
48	n/a	]
49	n/a
50	n/a	_PLATFORM = None
51	n/a	_DEFAULT_INSTALLER = 'packaging'
52	n/a
53	n/a
54	n/a	def newer(source, target):
55	n/a	"""Tell if the target is newer than the source.
56	n/a
57	n/a	Returns true if 'source' exists and is more recently modified than
58	n/a	'target', or if 'source' exists and 'target' doesn't.
59	n/a
60	n/a	Returns false if both exist and 'target' is the same age or younger
61	n/a	than 'source'. Raise PackagingFileError if 'source' does not exist.
62	n/a
63	n/a	Note that this test is not very accurate: files created in the same second
64	n/a	will have the same "age".
65	n/a	"""
66	n/a	if not os.path.exists(source):
67	n/a	raise PackagingFileError("file '%s' does not exist" %
68	n/a	os.path.abspath(source))
69	n/a	if not os.path.exists(target):
70	n/a	return True
71	n/a
72	n/a	return os.stat(source).st_mtime > os.stat(target).st_mtime
73	n/a
74	n/a
75	n/a	def get_platform():
76	n/a	"""Return a string that identifies the current platform.
77	n/a
78	n/a	By default, will return the value returned by sysconfig.get_platform(),
79	n/a	but it can be changed by calling set_platform().
80	n/a	"""
81	n/a	global _PLATFORM
82	n/a	if _PLATFORM is None:
83	n/a	_PLATFORM = sysconfig.get_platform()
84	n/a	return _PLATFORM
85	n/a
86	n/a
87	n/a	def set_platform(identifier):
88	n/a	"""Set the platform string identifier returned by get_platform().
89	n/a
90	n/a	Note that this change doesn't impact the value returned by
91	n/a	sysconfig.get_platform(); it is local to packaging.
92	n/a	"""
93	n/a	global _PLATFORM
94	n/a	_PLATFORM = identifier
95	n/a
96	n/a
97	n/a	def convert_path(pathname):
98	n/a	"""Return 'pathname' as a name that will work on the native filesystem.
99	n/a
100	n/a	The path is split on '/' and put back together again using the current
101	n/a	directory separator. Needed because filenames in the setup script are
102	n/a	always supplied in Unix style, and have to be converted to the local
103	n/a	convention before we can actually use them in the filesystem. Raises
104	n/a	ValueError on non-Unix-ish systems if 'pathname' either starts or
105	n/a	ends with a slash.
106	n/a	"""
107	n/a	if os.sep == '/':
108	n/a	return pathname
109	n/a	if not pathname:
110	n/a	return pathname
111	n/a	if pathname[0] == '/':
112	n/a	raise ValueError("path '%s' cannot be absolute" % pathname)
113	n/a	if pathname[-1] == '/':
114	n/a	raise ValueError("path '%s' cannot end with '/'" % pathname)
115	n/a
116	n/a	paths = pathname.split('/')
117	n/a	while os.curdir in paths:
118	n/a	paths.remove(os.curdir)
119	n/a	if not paths:
120	n/a	return os.curdir
121	n/a	return os.path.join(*paths)
122	n/a
123	n/a
124	n/a	def change_root(new_root, pathname):
125	n/a	"""Return 'pathname' with 'new_root' prepended.
126	n/a
127	n/a	If 'pathname' is relative, this is equivalent to
128	n/a	os.path.join(new_root,pathname). Otherwise, it requires making 'pathname'
129	n/a	relative and then joining the two, which is tricky on DOS/Windows.
130	n/a	"""
131	n/a	if os.name == 'posix':
132	n/a	if not os.path.isabs(pathname):
133	n/a	return os.path.join(new_root, pathname)
134	n/a	else:
135	n/a	return os.path.join(new_root, pathname[1:])
136	n/a
137	n/a	elif os.name == 'nt':
138	n/a	drive, path = os.path.splitdrive(pathname)
139	n/a	if path[0] == '\\':
140	n/a	path = path[1:]
141	n/a	return os.path.join(new_root, path)
142	n/a
143	n/a	elif os.name == 'os2':
144	n/a	drive, path = os.path.splitdrive(pathname)
145	n/a	if path[0] == os.sep:
146	n/a	path = path[1:]
147	n/a	return os.path.join(new_root, path)
148	n/a
149	n/a	else:
150	n/a	raise PackagingPlatformError("nothing known about "
151	n/a	"platform '%s'" % os.name)
152	n/a
153	n/a	_environ_checked = False
154	n/a
155	n/a
156	n/a	def check_environ():
157	n/a	"""Ensure that 'os.environ' has all the environment variables needed.
158	n/a
159	n/a	We guarantee that users can use in config files, command-line options,
160	n/a	etc. Currently this includes:
161	n/a	HOME - user's home directory (Unix only)
162	n/a	PLAT - description of the current platform, including hardware
163	n/a	and OS (see 'get_platform()')
164	n/a	"""
165	n/a	global _environ_checked
166	n/a	if _environ_checked:
167	n/a	return
168	n/a
169	n/a	if os.name == 'posix' and 'HOME' not in os.environ:
170	n/a	import pwd
171	n/a	os.environ['HOME'] = pwd.getpwuid(os.getuid())[5]
172	n/a
173	n/a	if 'PLAT' not in os.environ:
174	n/a	os.environ['PLAT'] = sysconfig.get_platform()
175	n/a
176	n/a	_environ_checked = True
177	n/a
178	n/a
179	n/a	# Needed by 'split_quoted()'
180	n/a	_wordchars_re = _squote_re = _dquote_re = None
181	n/a
182	n/a
183	n/a	def _init_regex():
184	n/a	global _wordchars_re, _squote_re, _dquote_re
185	n/a	_wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace)
186	n/a	_squote_re = re.compile(r"'(?:[^'\\]\|\\.)*'")
187	n/a	_dquote_re = re.compile(r'"(?:[^"\\]\|\\.)*"')
188	n/a
189	n/a
190	n/a	# TODO replace with shlex.split after testing
191	n/a
192	n/a	def split_quoted(s):
193	n/a	"""Split a string up according to Unix shell-like rules for quotes and
194	n/a	backslashes.
195	n/a
196	n/a	In short: words are delimited by spaces, as long as those
197	n/a	spaces are not escaped by a backslash, or inside a quoted string.
198	n/a	Single and double quotes are equivalent, and the quote characters can
199	n/a	be backslash-escaped. The backslash is stripped from any two-character
200	n/a	escape sequence, leaving only the escaped character. The quote
201	n/a	characters are stripped from any quoted string. Returns a list of
202	n/a	words.
203	n/a	"""
204	n/a	# This is a nice algorithm for splitting up a single string, since it
205	n/a	# doesn't require character-by-character examination. It was a little
206	n/a	# bit of a brain-bender to get it working right, though...
207	n/a	if _wordchars_re is None:
208	n/a	_init_regex()
209	n/a
210	n/a	s = s.strip()
211	n/a	words = []
212	n/a	pos = 0
213	n/a
214	n/a	while s:
215	n/a	m = _wordchars_re.match(s, pos)
216	n/a	end = m.end()
217	n/a	if end == len(s):
218	n/a	words.append(s[:end])
219	n/a	break
220	n/a
221	n/a	if s[end] in string.whitespace: # unescaped, unquoted whitespace: now
222	n/a	words.append(s[:end]) # we definitely have a word delimiter
223	n/a	s = s[end:].lstrip()
224	n/a	pos = 0
225	n/a
226	n/a	elif s[end] == '\\': # preserve whatever is being escaped;
227	n/a	# will become part of the current word
228	n/a	s = s[:end] + s[end + 1:]
229	n/a	pos = end + 1
230	n/a
231	n/a	else:
232	n/a	if s[end] == "'": # slurp singly-quoted string
233	n/a	m = _squote_re.match(s, end)
234	n/a	elif s[end] == '"': # slurp doubly-quoted string
235	n/a	m = _dquote_re.match(s, end)
236	n/a	else:
237	n/a	raise RuntimeError("this can't happen "
238	n/a	"(bad char '%c')" % s[end])
239	n/a
240	n/a	if m is None:
241	n/a	raise ValueError("bad string (mismatched %s quotes?)" % s[end])
242	n/a
243	n/a	beg, end = m.span()
244	n/a	s = s[:beg] + s[beg + 1:end - 1] + s[end:]
245	n/a	pos = m.end() - 2
246	n/a
247	n/a	if pos >= len(s):
248	n/a	words.append(s)
249	n/a	break
250	n/a
251	n/a	return words
252	n/a
253	n/a
254	n/a	def split_multiline(value):
255	n/a	"""Split a multiline string into a list, excluding blank lines."""
256	n/a
257	n/a	return [element for element in
258	n/a	(line.strip() for line in value.split('\n'))
259	n/a	if element]
260	n/a
261	n/a
262	n/a	def execute(func, args, msg=None, dry_run=False):
263	n/a	"""Perform some action that affects the outside world.
264	n/a
265	n/a	Some actions (e.g. writing to the filesystem) are special because
266	n/a	they are disabled by the 'dry_run' flag. This method takes care of all
267	n/a	that bureaucracy for you; all you have to do is supply the
268	n/a	function to call and an argument tuple for it (to embody the
269	n/a	"external action" being performed), and an optional message to
270	n/a	print.
271	n/a	"""
272	n/a	if msg is None:
273	n/a	msg = "%s%r" % (func.__name__, args)
274	n/a	if msg[-2:] == ',)': # correct for singleton tuple
275	n/a	msg = msg[0:-2] + ')'
276	n/a
277	n/a	logger.info(msg)
278	n/a	if not dry_run:
279	n/a	func(*args)
280	n/a
281	n/a
282	n/a	def strtobool(val):
283	n/a	"""Convert a string representation of truth to a boolean.
284	n/a
285	n/a	True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
286	n/a	are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if
287	n/a	'val' is anything else.
288	n/a	"""
289	n/a	val = val.lower()
290	n/a	if val in ('y', 'yes', 't', 'true', 'on', '1'):
291	n/a	return True
292	n/a	elif val in ('n', 'no', 'f', 'false', 'off', '0'):
293	n/a	return False
294	n/a	else:
295	n/a	raise ValueError("invalid truth value %r" % (val,))
296	n/a
297	n/a
298	n/a	def byte_compile(py_files, optimize=0, force=False, prefix=None,
299	n/a	base_dir=None, dry_run=False, direct=None):
300	n/a	"""Byte-compile a collection of Python source files to either .pyc
301	n/a	or .pyo files in a __pycache__ subdirectory.
302	n/a
303	n/a	'py_files' is a list of files to compile; any files that don't end in
304	n/a	".py" are silently skipped. 'optimize' must be one of the following:
305	n/a	0 - don't optimize (generate .pyc)
306	n/a	1 - normal optimization (like "python -O")
307	n/a	2 - extra optimization (like "python -OO")
308	n/a	This function is independent from the running Python's -O or -B options;
309	n/a	it is fully controlled by the parameters passed in.
310	n/a
311	n/a	If 'force' is true, all files are recompiled regardless of
312	n/a	timestamps.
313	n/a
314	n/a	The source filename encoded in each bytecode file defaults to the
315	n/a	filenames listed in 'py_files'; you can modify these with 'prefix' and
316	n/a	'basedir'. 'prefix' is a string that will be stripped off of each
317	n/a	source filename, and 'base_dir' is a directory name that will be
318	n/a	prepended (after 'prefix' is stripped). You can supply either or both
319	n/a	(or neither) of 'prefix' and 'base_dir', as you wish.
320	n/a
321	n/a	If 'dry_run' is true, doesn't actually do anything that would
322	n/a	affect the filesystem.
323	n/a
324	n/a	Byte-compilation is either done directly in this interpreter process
325	n/a	with the standard py_compile module, or indirectly by writing a
326	n/a	temporary script and executing it. Normally, you should let
327	n/a	'byte_compile()' figure out to use direct compilation or not (see
328	n/a	the source for details). The 'direct' flag is used by the script
329	n/a	generated in indirect mode; unless you know what you're doing, leave
330	n/a	it set to None.
331	n/a	"""
332	n/a	# FIXME use compileall + remove direct/indirect shenanigans
333	n/a
334	n/a	# First, if the caller didn't force us into direct or indirect mode,
335	n/a	# figure out which mode we should be in. We take a conservative
336	n/a	# approach: choose direct mode only if the current interpreter is
337	n/a	# in debug mode and optimize is 0. If we're not in debug mode (-O
338	n/a	# or -OO), we don't know which level of optimization this
339	n/a	# interpreter is running with, so we can't do direct
340	n/a	# byte-compilation and be certain that it's the right thing. Thus,
341	n/a	# always compile indirectly if the current interpreter is in either
342	n/a	# optimize mode, or if either optimization level was requested by
343	n/a	# the caller.
344	n/a	if direct is None:
345	n/a	direct = (__debug__ and optimize == 0)
346	n/a
347	n/a	# "Indirect" byte-compilation: write a temporary script and then
348	n/a	# run it with the appropriate flags.
349	n/a	if not direct:
350	n/a	from tempfile import mkstemp
351	n/a	# XXX use something better than mkstemp
352	n/a	script_fd, script_name = mkstemp(".py")
353	n/a	os.close(script_fd)
354	n/a	script_fd = None
355	n/a	logger.info("writing byte-compilation script '%s'", script_name)
356	n/a	if not dry_run:
357	n/a	if script_fd is not None:
358	n/a	script = os.fdopen(script_fd, "w", encoding='utf-8')
359	n/a	else:
360	n/a	script = open(script_name, "w", encoding='utf-8')
361	n/a
362	n/a	with script:
363	n/a	script.write("""\
364	n/a	from packaging.util import byte_compile
365	n/a	files = [
366	n/a	""")
367	n/a
368	n/a	# XXX would be nice to write absolute filenames, just for
369	n/a	# safety's sake (script should be more robust in the face of
370	n/a	# chdir'ing before running it). But this requires abspath'ing
371	n/a	# 'prefix' as well, and that breaks the hack in build_lib's
372	n/a	# 'byte_compile()' method that carefully tacks on a trailing
373	n/a	# slash (os.sep really) to make sure the prefix here is "just
374	n/a	# right". This whole prefix business is rather delicate -- the
375	n/a	# problem is that it's really a directory, but I'm treating it
376	n/a	# as a dumb string, so trailing slashes and so forth matter.
377	n/a
378	n/a	#py_files = map(os.path.abspath, py_files)
379	n/a	#if prefix:
380	n/a	# prefix = os.path.abspath(prefix)
381	n/a
382	n/a	script.write(",\n".join(map(repr, py_files)) + "]\n")
383	n/a	script.write("""
384	n/a	byte_compile(files, optimize=%r, force=%r,
385	n/a	prefix=%r, base_dir=%r,
386	n/a	dry_run=False,
387	n/a	direct=True)
388	n/a	""" % (optimize, force, prefix, base_dir))
389	n/a
390	n/a	cmd = [sys.executable, script_name]
391	n/a
392	n/a	env = os.environ.copy()
393	n/a	env['PYTHONPATH'] = os.path.pathsep.join(sys.path)
394	n/a	try:
395	n/a	spawn(cmd, env=env)
396	n/a	finally:
397	n/a	execute(os.remove, (script_name,), "removing %s" % script_name,
398	n/a	dry_run=dry_run)
399	n/a
400	n/a	# "Direct" byte-compilation: use the py_compile module to compile
401	n/a	# right here, right now. Note that the script generated in indirect
402	n/a	# mode simply calls 'byte_compile()' in direct mode, a weird sort of
403	n/a	# cross-process recursion. Hey, it works!
404	n/a	else:
405	n/a	from py_compile import compile
406	n/a
407	n/a	for file in py_files:
408	n/a	if file[-3:] != ".py":
409	n/a	# This lets us be lazy and not filter filenames in
410	n/a	# the "install_lib" command.
411	n/a	continue
412	n/a
413	n/a	# Terminology from the py_compile module:
414	n/a	# cfile - byte-compiled file
415	n/a	# dfile - purported source filename (same as 'file' by default)
416	n/a	# The second argument to cache_from_source forces the extension to
417	n/a	# be .pyc (if true) or .pyo (if false); without it, the extension
418	n/a	# would depend on the calling Python's -O option
419	n/a	cfile = imp.cache_from_source(file, not optimize)
420	n/a	dfile = file
421	n/a
422	n/a	if prefix:
423	n/a	if file[:len(prefix)] != prefix:
424	n/a	raise ValueError("invalid prefix: filename %r doesn't "
425	n/a	"start with %r" % (file, prefix))
426	n/a	dfile = dfile[len(prefix):]
427	n/a	if base_dir:
428	n/a	dfile = os.path.join(base_dir, dfile)
429	n/a
430	n/a	cfile_base = os.path.basename(cfile)
431	n/a	if direct:
432	n/a	if force or newer(file, cfile):
433	n/a	logger.info("byte-compiling %s to %s", file, cfile_base)
434	n/a	if not dry_run:
435	n/a	compile(file, cfile, dfile)
436	n/a	else:
437	n/a	logger.debug("skipping byte-compilation of %s to %s",
438	n/a	file, cfile_base)
439	n/a
440	n/a
441	n/a	_RE_VERSION = re.compile('(\d+\.\d+(\.\d+)*)')
442	n/a	_MAC_OS_X_LD_VERSION = re.compile('^@$#$PROGRAM:ld '
443	n/a	'PROJECT:ld64-((\d+)(\.\d+)*)')
444	n/a
445	n/a
446	n/a	def _find_ld_version():
447	n/a	"""Find the ld version. The version scheme differs under Mac OS X."""
448	n/a	if sys.platform == 'darwin':
449	n/a	return _find_exe_version('ld -v', _MAC_OS_X_LD_VERSION)
450	n/a	else:
451	n/a	return _find_exe_version('ld -v')
452	n/a
453	n/a
454	n/a	def _find_exe_version(cmd, pattern=_RE_VERSION):
455	n/a	"""Find the version of an executable by running `cmd` in the shell.
456	n/a
457	n/a	`pattern` is a compiled regular expression. If not provided, defaults
458	n/a	to _RE_VERSION. If the command is not found, or the output does not
459	n/a	match the mattern, returns None.
460	n/a	"""
461	n/a	from subprocess import Popen, PIPE
462	n/a	executable = cmd.split()[0]
463	n/a	if find_executable(executable) is None:
464	n/a	return None
465	n/a	pipe = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
466	n/a	try:
467	n/a	stdout, stderr = pipe.communicate()
468	n/a	finally:
469	n/a	pipe.stdout.close()
470	n/a	pipe.stderr.close()
471	n/a	# some commands like ld under MacOS X, will give the
472	n/a	# output in the stderr, rather than stdout.
473	n/a	if stdout != '':
474	n/a	out_string = stdout
475	n/a	else:
476	n/a	out_string = stderr
477	n/a
478	n/a	result = pattern.search(out_string)
479	n/a	if result is None:
480	n/a	return None
481	n/a	return result.group(1)
482	n/a
483	n/a
484	n/a	def get_compiler_versions():
485	n/a	"""Return a tuple providing the versions of gcc, ld and dllwrap
486	n/a
487	n/a	For each command, if a command is not found, None is returned.
488	n/a	Otherwise a string with the version is returned.
489	n/a	"""
490	n/a	gcc = _find_exe_version('gcc -dumpversion')
491	n/a	ld = _find_ld_version()
492	n/a	dllwrap = _find_exe_version('dllwrap --version')
493	n/a	return gcc, ld, dllwrap
494	n/a
495	n/a
496	n/a	def newer_group(sources, target, missing='error'):
497	n/a	"""Return true if 'target' is out-of-date with respect to any file
498	n/a	listed in 'sources'.
499	n/a
500	n/a	In other words, if 'target' exists and is newer
501	n/a	than every file in 'sources', return false; otherwise return true.
502	n/a	'missing' controls what we do when a source file is missing; the
503	n/a	default ("error") is to blow up with an OSError from inside 'stat()';
504	n/a	if it is "ignore", we silently drop any missing source files; if it is
505	n/a	"newer", any missing source files make us assume that 'target' is
506	n/a	out-of-date (this is handy in "dry-run" mode: it'll make you pretend to
507	n/a	carry out commands that wouldn't work because inputs are missing, but
508	n/a	that doesn't matter because you're not actually going to run the
509	n/a	commands).
510	n/a	"""
511	n/a	# If the target doesn't even exist, then it's definitely out-of-date.
512	n/a	if not os.path.exists(target):
513	n/a	return True
514	n/a
515	n/a	# Otherwise we have to find out the hard way: if any source file
516	n/a	# is more recent than 'target', then 'target' is out-of-date and
517	n/a	# we can immediately return true. If we fall through to the end
518	n/a	# of the loop, then 'target' is up-to-date and we return false.
519	n/a	target_mtime = os.stat(target).st_mtime
520	n/a
521	n/a	for source in sources:
522	n/a	if not os.path.exists(source):
523	n/a	if missing == 'error': # blow up when we stat() the file
524	n/a	pass
525	n/a	elif missing == 'ignore': # missing source dropped from
526	n/a	continue # target's dependency list
527	n/a	elif missing == 'newer': # missing source means target is
528	n/a	return True # out-of-date
529	n/a
530	n/a	if os.stat(source).st_mtime > target_mtime:
531	n/a	return True
532	n/a
533	n/a	return False
534	n/a
535	n/a
536	n/a	def write_file(filename, contents):
537	n/a	"""Create filename and write contents to it.
538	n/a
539	n/a	contents is a sequence of strings without line terminators.
540	n/a
541	n/a	This functions is not intended to replace the usual with open + write
542	n/a	idiom in all cases, only with Command.execute, which runs depending on
543	n/a	the dry_run argument and also logs its arguments).
544	n/a	"""
545	n/a	with open(filename, "w") as f:
546	n/a	for line in contents:
547	n/a	f.write(line + "\n")
548	n/a
549	n/a
550	n/a	def _is_package(path):
551	n/a	return os.path.isdir(path) and os.path.isfile(
552	n/a	os.path.join(path, '__init__.py'))
553	n/a
554	n/a
555	n/a	# Code taken from the pip project
556	n/a	def _is_archive_file(name):
557	n/a	archives = ('.zip', '.tar.gz', '.tar.bz2', '.tgz', '.tar')
558	n/a	ext = splitext(name)[1].lower()
559	n/a	return ext in archives
560	n/a
561	n/a
562	n/a	def _under(path, root):
563	n/a	# XXX use os.path
564	n/a	path = path.split(os.sep)
565	n/a	root = root.split(os.sep)
566	n/a	if len(root) > len(path):
567	n/a	return False
568	n/a	for pos, part in enumerate(root):
569	n/a	if path[pos] != part:
570	n/a	return False
571	n/a	return True
572	n/a
573	n/a
574	n/a	def _package_name(root_path, path):
575	n/a	# Return a dotted package name, given a subpath
576	n/a	if not _under(path, root_path):
577	n/a	raise ValueError('"%s" is not a subpath of "%s"' % (path, root_path))
578	n/a	return path[len(root_path) + 1:].replace(os.sep, '.')
579	n/a
580	n/a
581	n/a	def find_packages(paths=(os.curdir,), exclude=()):
582	n/a	"""Return a list all Python packages found recursively within
583	n/a	directories 'paths'
584	n/a
585	n/a	'paths' should be supplied as a sequence of "cross-platform"
586	n/a	(i.e. URL-style) path; it will be converted to the appropriate local
587	n/a	path syntax.
588	n/a
589	n/a	'exclude' is a sequence of package names to exclude; '*' can be used as
590	n/a	a wildcard in the names, such that 'foo.*' will exclude all subpackages
591	n/a	of 'foo' (but not 'foo' itself).
592	n/a	"""
593	n/a	packages = []
594	n/a	discarded = []
595	n/a
596	n/a	def _discarded(path):
597	n/a	for discard in discarded:
598	n/a	if _under(path, discard):
599	n/a	return True
600	n/a	return False
601	n/a
602	n/a	for path in paths:
603	n/a	path = convert_path(path)
604	n/a	for root, dirs, files in os.walk(path):
605	n/a	for dir_ in dirs:
606	n/a	fullpath = os.path.join(root, dir_)
607	n/a	if _discarded(fullpath):
608	n/a	continue
609	n/a	# we work only with Python packages
610	n/a	if not _is_package(fullpath):
611	n/a	discarded.append(fullpath)
612	n/a	continue
613	n/a	# see if it's excluded
614	n/a	excluded = False
615	n/a	package_name = _package_name(path, fullpath)
616	n/a	for pattern in exclude:
617	n/a	if fnmatchcase(package_name, pattern):
618	n/a	excluded = True
619	n/a	break
620	n/a	if excluded:
621	n/a	continue
622	n/a
623	n/a	# adding it to the list
624	n/a	packages.append(package_name)
625	n/a	return packages
626	n/a
627	n/a
628	n/a	def resolve_name(name):
629	n/a	"""Resolve a name like ``module.object`` to an object and return it.
630	n/a
631	n/a	This functions supports packages and attributes without depth limitation:
632	n/a	``package.package.module.class.class.function.attr`` is valid input.
633	n/a	However, looking up builtins is not directly supported: use
634	n/a	``builtins.name``.
635	n/a
636	n/a	Raises ImportError if importing the module fails or if one requested
637	n/a	attribute is not found.
638	n/a	"""
639	n/a	if '.' not in name:
640	n/a	# shortcut
641	n/a	__import__(name)
642	n/a	return sys.modules[name]
643	n/a
644	n/a	# FIXME clean up this code!
645	n/a	parts = name.split('.')
646	n/a	cursor = len(parts)
647	n/a	module_name = parts[:cursor]
648	n/a	ret = ''
649	n/a
650	n/a	while cursor > 0:
651	n/a	try:
652	n/a	ret = __import__('.'.join(module_name))
653	n/a	break
654	n/a	except ImportError:
655	n/a	cursor -= 1
656	n/a	module_name = parts[:cursor]
657	n/a
658	n/a	if ret == '':
659	n/a	raise ImportError(parts[0])
660	n/a
661	n/a	for part in parts[1:]:
662	n/a	try:
663	n/a	ret = getattr(ret, part)
664	n/a	except AttributeError as exc:
665	n/a	raise ImportError(exc)
666	n/a
667	n/a	return ret
668	n/a
669	n/a
670	n/a	def splitext(path):
671	n/a	"""Like os.path.splitext, but take off .tar too"""
672	n/a	base, ext = posixpath.splitext(path)
673	n/a	if base.lower().endswith('.tar'):
674	n/a	ext = base[-4:] + ext
675	n/a	base = base[:-4]
676	n/a	return base, ext
677	n/a
678	n/a
679	n/a	if sys.platform == 'darwin':
680	n/a	_cfg_target = None
681	n/a	_cfg_target_split = None
682	n/a
683	n/a
684	n/a	def spawn(cmd, search_path=True, dry_run=False, env=None):
685	n/a	"""Run another program specified as a command list 'cmd' in a new process.
686	n/a
687	n/a	'cmd' is just the argument list for the new process, ie.
688	n/a	cmd[0] is the program to run and cmd[1:] are the rest of its arguments.
689	n/a	There is no way to run a program with a name different from that of its
690	n/a	executable.
691	n/a
692	n/a	If 'search_path' is true (the default), the system's executable
693	n/a	search path will be used to find the program; otherwise, cmd[0]
694	n/a	must be the exact path to the executable. If 'dry_run' is true,
695	n/a	the command will not actually be run.
696	n/a
697	n/a	If 'env' is given, it's a environment dictionary used for the execution
698	n/a	environment.
699	n/a
700	n/a	Raise PackagingExecError if running the program fails in any way; just
701	n/a	return on success.
702	n/a	"""
703	n/a	logger.debug('spawn: running %r', cmd)
704	n/a	if dry_run:
705	n/a	logger.debug('dry run, no process actually spawned')
706	n/a	return
707	n/a	if sys.platform == 'darwin':
708	n/a	global _cfg_target, _cfg_target_split
709	n/a	if _cfg_target is None:
710	n/a	_cfg_target = sysconfig.get_config_var(
711	n/a	'MACOSX_DEPLOYMENT_TARGET') or ''
712	n/a	if _cfg_target:
713	n/a	_cfg_target_split = [int(x) for x in _cfg_target.split('.')]
714	n/a	if _cfg_target:
715	n/a	# ensure that the deployment target of build process is not less
716	n/a	# than that used when the interpreter was built. This ensures
717	n/a	# extension modules are built with correct compatibility values
718	n/a	env = env or os.environ
719	n/a	cur_target = env.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target)
720	n/a	if _cfg_target_split > [int(x) for x in cur_target.split('.')]:
721	n/a	my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: '
722	n/a	'now "%s" but "%s" during configure'
723	n/a	% (cur_target, _cfg_target))
724	n/a	raise PackagingPlatformError(my_msg)
725	n/a	env = dict(env, MACOSX_DEPLOYMENT_TARGET=cur_target)
726	n/a
727	n/a	exit_status = subprocess.call(cmd, env=env)
728	n/a	if exit_status != 0:
729	n/a	msg = "command %r failed with exit status %d"
730	n/a	raise PackagingExecError(msg % (cmd, exit_status))
731	n/a
732	n/a
733	n/a	def find_executable(executable, path=None):
734	n/a	"""Try to find 'executable' in the directories listed in 'path'.
735	n/a
736	n/a	path is a string listing directories separated by 'os.pathsep' and
737	n/a	defaults to os.environ['PATH']. Returns the complete filename or None
738	n/a	if not found.
739	n/a	"""
740	n/a	if path is None:
741	n/a	path = os.environ['PATH']
742	n/a	paths = path.split(os.pathsep)
743	n/a	base, ext = os.path.splitext(executable)
744	n/a
745	n/a	if (sys.platform == 'win32' or os.name == 'os2') and (ext != '.exe'):
746	n/a	executable = executable + '.exe'
747	n/a
748	n/a	if not os.path.isfile(executable):
749	n/a	for p in paths:
750	n/a	f = os.path.join(p, executable)
751	n/a	if os.path.isfile(f):
752	n/a	# the file exists, we have a shot at spawn working
753	n/a	return f
754	n/a	return None
755	n/a	else:
756	n/a	return executable
757	n/a
758	n/a
759	n/a	DEFAULT_REPOSITORY = 'http://pypi.python.org/pypi'
760	n/a	DEFAULT_REALM = 'pypi'
761	n/a	DEFAULT_PYPIRC = """\
762	n/a	[distutils]
763	n/a	index-servers =
764	n/a	pypi
765	n/a
766	n/a	[pypi]
767	n/a	username:%s
768	n/a	password:%s
769	n/a	"""
770	n/a
771	n/a
772	n/a	def get_pypirc_path():
773	n/a	"""Return path to pypirc config file."""
774	n/a	return os.path.join(os.path.expanduser('~'), '.pypirc')
775	n/a
776	n/a
777	n/a	def generate_pypirc(username, password):
778	n/a	"""Create a default .pypirc file."""
779	n/a	rc = get_pypirc_path()
780	n/a	with open(rc, 'w') as f:
781	n/a	f.write(DEFAULT_PYPIRC % (username, password))
782	n/a	try:
783	n/a	os.chmod(rc, 0o600)
784	n/a	except OSError:
785	n/a	# should do something better here
786	n/a	pass
787	n/a
788	n/a
789	n/a	def read_pypirc(repository=DEFAULT_REPOSITORY, realm=DEFAULT_REALM):
790	n/a	"""Read the .pypirc file."""
791	n/a	rc = get_pypirc_path()
792	n/a	if os.path.exists(rc):
793	n/a	config = RawConfigParser()
794	n/a	config.read(rc)
795	n/a	sections = config.sections()
796	n/a	if 'distutils' in sections:
797	n/a	# let's get the list of servers
798	n/a	index_servers = config.get('distutils', 'index-servers')
799	n/a	_servers = [server.strip() for server in
800	n/a	index_servers.split('\n')
801	n/a	if server.strip() != '']
802	n/a	if _servers == []:
803	n/a	# nothing set, let's try to get the default pypi
804	n/a	if 'pypi' in sections:
805	n/a	_servers = ['pypi']
806	n/a	else:
807	n/a	# the file is not properly defined, returning
808	n/a	# an empty dict
809	n/a	return {}
810	n/a	for server in _servers:
811	n/a	current = {'server': server}
812	n/a	current['username'] = config.get(server, 'username')
813	n/a
814	n/a	# optional params
815	n/a	for key, default in (('repository', DEFAULT_REPOSITORY),
816	n/a	('realm', DEFAULT_REALM),
817	n/a	('password', None)):
818	n/a	if config.has_option(server, key):
819	n/a	current[key] = config.get(server, key)
820	n/a	else:
821	n/a	current[key] = default
822	n/a	if (current['server'] == repository or
823	n/a	current['repository'] == repository):
824	n/a	return current
825	n/a	elif 'server-login' in sections:
826	n/a	# old format
827	n/a	server = 'server-login'
828	n/a	if config.has_option(server, 'repository'):
829	n/a	repository = config.get(server, 'repository')
830	n/a	else:
831	n/a	repository = DEFAULT_REPOSITORY
832	n/a
833	n/a	return {'username': config.get(server, 'username'),
834	n/a	'password': config.get(server, 'password'),
835	n/a	'repository': repository,
836	n/a	'server': server,
837	n/a	'realm': DEFAULT_REALM}
838	n/a
839	n/a	return {}
840	n/a
841	n/a
842	n/a	# utility functions for 2to3 support
843	n/a
844	n/a	def run_2to3(files, doctests_only=False, fixer_names=None,
845	n/a	options=None, explicit=None):
846	n/a	""" Wrapper function around the refactor() class which
847	n/a	performs the conversions on a list of python files.
848	n/a	Invoke 2to3 on a list of Python files. The files should all come
849	n/a	from the build area, as the modification is done in-place."""
850	n/a
851	n/a	#if not files:
852	n/a	# return
853	n/a
854	n/a	# Make this class local, to delay import of 2to3
855	n/a	from lib2to3.refactor import get_fixers_from_package, RefactoringTool
856	n/a	fixers = get_fixers_from_package('lib2to3.fixes')
857	n/a
858	n/a	if fixer_names:
859	n/a	for fixername in fixer_names:
860	n/a	fixers.extend(get_fixers_from_package(fixername))
861	n/a	r = RefactoringTool(fixers, options=options)
862	n/a	r.refactor(files, write=True, doctests_only=doctests_only)
863	n/a
864	n/a
865	n/a	class Mixin2to3:
866	n/a	""" Wrapper class for commands that run 2to3.
867	n/a	To configure 2to3, setup scripts may either change
868	n/a	the class variables, or inherit from this class
869	n/a	to override how 2to3 is invoked.
870	n/a	"""
871	n/a	# list of fixers to run; defaults to all implicit from lib2to3.fixers
872	n/a	fixer_names = None
873	n/a	# dict of options
874	n/a	options = None
875	n/a	# list of extra fixers to invoke
876	n/a	explicit = None
877	n/a	# TODO need a better way to add just one fixer from a package
878	n/a	# TODO need a way to exclude individual fixers
879	n/a
880	n/a	def run_2to3(self, files, doctests_only=False):
881	n/a	""" Issues a call to util.run_2to3. """
882	n/a	return run_2to3(files, doctests_only, self.fixer_names,
883	n/a	self.options, self.explicit)
884	n/a
885	n/a	# TODO provide initialize/finalize_options
886	n/a
887	n/a
888	n/a	RICH_GLOB = re.compile(r'\{([^}]*)\}')
889	n/a	_CHECK_RECURSIVE_GLOB = re.compile(r'[^/\\,{]\\\|\\[^/\\,}]')
890	n/a	_CHECK_MISMATCH_SET = re.compile(r'^[^{]\}\|\{[^}]$')
891	n/a
892	n/a
893	n/a	def iglob(path_glob):
894	n/a	"""Extended globbing function that supports ** and {opt1,opt2,opt3}."""
895	n/a	if _CHECK_RECURSIVE_GLOB.search(path_glob):
896	n/a	msg = """invalid glob %r: recursive glob "**" must be used alone"""
897	n/a	raise ValueError(msg % path_glob)
898	n/a	if _CHECK_MISMATCH_SET.search(path_glob):
899	n/a	msg = """invalid glob %r: mismatching set marker '{' or '}'"""
900	n/a	raise ValueError(msg % path_glob)
901	n/a	return _iglob(path_glob)
902	n/a
903	n/a
904	n/a	def _iglob(path_glob):
905	n/a	rich_path_glob = RICH_GLOB.split(path_glob, 1)
906	n/a	if len(rich_path_glob) > 1:
907	n/a	assert len(rich_path_glob) == 3, rich_path_glob
908	n/a	prefix, set, suffix = rich_path_glob
909	n/a	for item in set.split(','):
910	n/a	for path in _iglob(''.join((prefix, item, suffix))):
911	n/a	yield path
912	n/a	else:
913	n/a	if '**' not in path_glob:
914	n/a	for item in std_iglob(path_glob):
915	n/a	yield item
916	n/a	else:
917	n/a	prefix, radical = path_glob.split('**', 1)
918	n/a	if prefix == '':
919	n/a	prefix = '.'
920	n/a	if radical == '':
921	n/a	radical = '*'
922	n/a	else:
923	n/a	# we support both
924	n/a	radical = radical.lstrip('/')
925	n/a	radical = radical.lstrip('\\')
926	n/a	for path, dir, files in os.walk(prefix):
927	n/a	path = os.path.normpath(path)
928	n/a	for file in _iglob(os.path.join(path, radical)):
929	n/a	yield file
930	n/a
931	n/a
932	n/a	# HOWTO change cfg_to_args
933	n/a	#
934	n/a	# This function has two major constraints: It is copied by inspect.getsource
935	n/a	# in generate_setup_py; it is used in generated setup.py which may be run by
936	n/a	# any Python version supported by distutils2 (2.4-3.3).
937	n/a	#
938	n/a	# * Keep objects like D1_D2_SETUP_ARGS static, i.e. in the function body
939	n/a	# instead of global.
940	n/a	# * If you use a function from another module, update the imports in
941	n/a	# SETUP_TEMPLATE. Use only modules, classes and functions compatible with
942	n/a	# all versions: codecs.open instead of open, RawConfigParser.readfp instead
943	n/a	# of read, standard exceptions instead of Packaging*Error, etc.
944	n/a	# * If you use a function from this module, update the template and
945	n/a	# generate_setup_py.
946	n/a	#
947	n/a	# test_util tests this function and the generated setup.py, but does not test
948	n/a	# that it's compatible with all Python versions.
949	n/a
950	n/a	def cfg_to_args(path='setup.cfg'):
951	n/a	"""Compatibility helper to use setup.cfg in setup.py.
952	n/a
953	n/a	This functions uses an existing setup.cfg to generate a dictionnary of
954	n/a	keywords that can be used by distutils.core.setup(**kwargs). It is used
955	n/a	by generate_setup_py.
956	n/a
957	n/a	file is the path to the setup.cfg file. If it doesn't exist,
958	n/a	PackagingFileError is raised.
959	n/a	"""
960	n/a
961	n/a	# XXX ** == needs testing
962	n/a	D1_D2_SETUP_ARGS = {"name": ("metadata",),
963	n/a	"version": ("metadata",),
964	n/a	"author": ("metadata",),
965	n/a	"author_email": ("metadata",),
966	n/a	"maintainer": ("metadata",),
967	n/a	"maintainer_email": ("metadata",),
968	n/a	"url": ("metadata", "home_page"),
969	n/a	"description": ("metadata", "summary"),
970	n/a	"long_description": ("metadata", "description"),
971	n/a	"download-url": ("metadata",),
972	n/a	"classifiers": ("metadata", "classifier"),
973	n/a	"platforms": ("metadata", "platform"), # **
974	n/a	"license": ("metadata",),
975	n/a	"requires": ("metadata", "requires_dist"),
976	n/a	"provides": ("metadata", "provides_dist"), # **
977	n/a	"obsoletes": ("metadata", "obsoletes_dist"), # **
978	n/a	"package_dir": ("files", 'packages_root'),
979	n/a	"packages": ("files",),
980	n/a	"scripts": ("files",),
981	n/a	"py_modules": ("files", "modules"), # **
982	n/a	}
983	n/a
984	n/a	MULTI_FIELDS = ("classifiers",
985	n/a	"platforms",
986	n/a	"requires",
987	n/a	"provides",
988	n/a	"obsoletes",
989	n/a	"packages",
990	n/a	"scripts",
991	n/a	"py_modules")
992	n/a
993	n/a	def has_get_option(config, section, option):
994	n/a	if config.has_option(section, option):
995	n/a	return config.get(section, option)
996	n/a	elif config.has_option(section, option.replace('_', '-')):
997	n/a	return config.get(section, option.replace('_', '-'))
998	n/a	else:
999	n/a	return False
1000	n/a
1001	n/a	# The real code starts here
1002	n/a	config = RawConfigParser()
1003	n/a	f = codecs.open(path, encoding='utf-8')
1004	n/a	try:
1005	n/a	config.readfp(f)
1006	n/a	finally:
1007	n/a	f.close()
1008	n/a
1009	n/a	kwargs = {}
1010	n/a	for arg in D1_D2_SETUP_ARGS:
1011	n/a	if len(D1_D2_SETUP_ARGS[arg]) == 2:
1012	n/a	# The distutils field name is different than packaging's
1013	n/a	section, option = D1_D2_SETUP_ARGS[arg]
1014	n/a
1015	n/a	else:
1016	n/a	# The distutils field name is the same thant packaging's
1017	n/a	section = D1_D2_SETUP_ARGS[arg][0]
1018	n/a	option = arg
1019	n/a
1020	n/a	in_cfg_value = has_get_option(config, section, option)
1021	n/a	if not in_cfg_value:
1022	n/a	# There is no such option in the setup.cfg
1023	n/a	if arg == 'long_description':
1024	n/a	filenames = has_get_option(config, section, 'description-file')
1025	n/a	if filenames:
1026	n/a	filenames = split_multiline(filenames)
1027	n/a	in_cfg_value = []
1028	n/a	for filename in filenames:
1029	n/a	fp = codecs.open(filename, encoding='utf-8')
1030	n/a	try:
1031	n/a	in_cfg_value.append(fp.read())
1032	n/a	finally:
1033	n/a	fp.close()
1034	n/a	in_cfg_value = '\n\n'.join(in_cfg_value)
1035	n/a	else:
1036	n/a	continue
1037	n/a
1038	n/a	if arg == 'package_dir' and in_cfg_value:
1039	n/a	in_cfg_value = {'': in_cfg_value}
1040	n/a
1041	n/a	if arg in MULTI_FIELDS:
1042	n/a	# support multiline options
1043	n/a	in_cfg_value = split_multiline(in_cfg_value)
1044	n/a
1045	n/a	kwargs[arg] = in_cfg_value
1046	n/a
1047	n/a	return kwargs
1048	n/a
1049	n/a
1050	n/a	SETUP_TEMPLATE = """\
1051	n/a	# This script was automatically generated by packaging
1052	n/a	import codecs
1053	n/a	from distutils.core import setup
1054	n/a	try:
1055	n/a	from ConfigParser import RawConfigParser
1056	n/a	except ImportError:
1057	n/a	from configparser import RawConfigParser
1058	n/a
1059	n/a
1060	n/a	%(split_multiline)s
1061	n/a
1062	n/a	%(cfg_to_args)s
1063	n/a
1064	n/a	setup(**cfg_to_args())
1065	n/a	"""
1066	n/a
1067	n/a
1068	n/a	def generate_setup_py():
1069	n/a	"""Generate a distutils compatible setup.py using an existing setup.cfg.
1070	n/a
1071	n/a	Raises a PackagingFileError when a setup.py already exists.
1072	n/a	"""
1073	n/a	if os.path.exists("setup.py"):
1074	n/a	raise PackagingFileError("a setup.py file already exists")
1075	n/a
1076	n/a	source = SETUP_TEMPLATE % {'split_multiline': getsource(split_multiline),
1077	n/a	'cfg_to_args': getsource(cfg_to_args)}
1078	n/a	with open("setup.py", "w", encoding='utf-8') as fp:
1079	n/a	fp.write(source)
1080	n/a
1081	n/a
1082	n/a	# Taken from the pip project
1083	n/a	# https://github.com/pypa/pip/blob/master/pip/util.py
1084	n/a	def ask(message, options):
1085	n/a	"""Prompt the user with message; options contains allowed responses."""
1086	n/a	while True:
1087	n/a	response = input(message)
1088	n/a	response = response.strip().lower()
1089	n/a	if response not in options:
1090	n/a	print('invalid response:', repr(response))
1091	n/a	print('choose one of', ', '.join(repr(o) for o in options))
1092	n/a	else:
1093	n/a	return response
1094	n/a
1095	n/a
1096	n/a	def _parse_record_file(record_file):
1097	n/a	distinfo, extra_metadata, installed = ({}, [], [])
1098	n/a	with open(record_file, 'r') as rfile:
1099	n/a	for path in rfile:
1100	n/a	path = path.strip()
1101	n/a	if path.endswith('egg-info') and os.path.isfile(path):
1102	n/a	distinfo_dir = path.replace('egg-info', 'dist-info')
1103	n/a	metadata = path
1104	n/a	egginfo = path
1105	n/a	elif path.endswith('egg-info') and os.path.isdir(path):
1106	n/a	distinfo_dir = path.replace('egg-info', 'dist-info')
1107	n/a	egginfo = path
1108	n/a	for metadata_file in os.listdir(path):
1109	n/a	metadata_fpath = os.path.join(path, metadata_file)
1110	n/a	if metadata_file == 'PKG-INFO':
1111	n/a	metadata = metadata_fpath
1112	n/a	else:
1113	n/a	extra_metadata.append(metadata_fpath)
1114	n/a	elif 'egg-info' in path and os.path.isfile(path):
1115	n/a	# skip extra metadata files
1116	n/a	continue
1117	n/a	else:
1118	n/a	installed.append(path)
1119	n/a
1120	n/a	distinfo['egginfo'] = egginfo
1121	n/a	distinfo['metadata'] = metadata
1122	n/a	distinfo['distinfo_dir'] = distinfo_dir
1123	n/a	distinfo['installer_path'] = os.path.join(distinfo_dir, 'INSTALLER')
1124	n/a	distinfo['metadata_path'] = os.path.join(distinfo_dir, 'METADATA')
1125	n/a	distinfo['record_path'] = os.path.join(distinfo_dir, 'RECORD')
1126	n/a	distinfo['requested_path'] = os.path.join(distinfo_dir, 'REQUESTED')
1127	n/a	installed.extend([distinfo['installer_path'], distinfo['metadata_path']])
1128	n/a	distinfo['installed'] = installed
1129	n/a	distinfo['extra_metadata'] = extra_metadata
1130	n/a	return distinfo
1131	n/a
1132	n/a
1133	n/a	def _write_record_file(record_path, installed_files):
1134	n/a	with open(record_path, 'w', encoding='utf-8') as f:
1135	n/a	writer = csv.writer(f, delimiter=',', lineterminator=os.linesep,
1136	n/a	quotechar='"')
1137	n/a
1138	n/a	for fpath in installed_files:
1139	n/a	if fpath.endswith('.pyc') or fpath.endswith('.pyo'):
1140	n/a	# do not put size and md5 hash, as in PEP-376
1141	n/a	writer.writerow((fpath, '', ''))
1142	n/a	else:
1143	n/a	hash = hashlib.md5()
1144	n/a	with open(fpath, 'rb') as fp:
1145	n/a	hash.update(fp.read())
1146	n/a	md5sum = hash.hexdigest()
1147	n/a	size = os.path.getsize(fpath)
1148	n/a	writer.writerow((fpath, md5sum, size))
1149	n/a
1150	n/a	# add the RECORD file itself
1151	n/a	writer.writerow((record_path, '', ''))
1152	n/a	return record_path
1153	n/a
1154	n/a
1155	n/a	def egginfo_to_distinfo(record_file, installer=_DEFAULT_INSTALLER,
1156	n/a	requested=False, remove_egginfo=False):
1157	n/a	"""Create files and directories required for PEP 376
1158	n/a
1159	n/a	:param record_file: path to RECORD file as produced by setup.py --record
1160	n/a	:param installer: installer name
1161	n/a	:param requested: True if not installed as a dependency
1162	n/a	:param remove_egginfo: delete egginfo dir?
1163	n/a	"""
1164	n/a	distinfo = _parse_record_file(record_file)
1165	n/a	distinfo_dir = distinfo['distinfo_dir']
1166	n/a	if os.path.isdir(distinfo_dir) and not os.path.islink(distinfo_dir):
1167	n/a	shutil.rmtree(distinfo_dir)
1168	n/a	elif os.path.exists(distinfo_dir):
1169	n/a	os.unlink(distinfo_dir)
1170	n/a
1171	n/a	os.makedirs(distinfo_dir)
1172	n/a
1173	n/a	# copy setuptools extra metadata files
1174	n/a	if distinfo['extra_metadata']:
1175	n/a	for path in distinfo['extra_metadata']:
1176	n/a	shutil.copy2(path, distinfo_dir)
1177	n/a	new_path = path.replace('egg-info', 'dist-info')
1178	n/a	distinfo['installed'].append(new_path)
1179	n/a
1180	n/a	metadata_path = distinfo['metadata_path']
1181	n/a	logger.info('creating %s', metadata_path)
1182	n/a	shutil.copy2(distinfo['metadata'], metadata_path)
1183	n/a
1184	n/a	installer_path = distinfo['installer_path']
1185	n/a	logger.info('creating %s', installer_path)
1186	n/a	with open(installer_path, 'w') as f:
1187	n/a	f.write(installer)
1188	n/a
1189	n/a	if requested:
1190	n/a	requested_path = distinfo['requested_path']
1191	n/a	logger.info('creating %s', requested_path)
1192	n/a	open(requested_path, 'wb').close()
1193	n/a	distinfo['installed'].append(requested_path)
1194	n/a
1195	n/a	record_path = distinfo['record_path']
1196	n/a	logger.info('creating %s', record_path)
1197	n/a	_write_record_file(record_path, distinfo['installed'])
1198	n/a
1199	n/a	if remove_egginfo:
1200	n/a	egginfo = distinfo['egginfo']
1201	n/a	logger.info('removing %s', egginfo)
1202	n/a	if os.path.isfile(egginfo):
1203	n/a	os.remove(egginfo)
1204	n/a	else:
1205	n/a	shutil.rmtree(egginfo)
1206	n/a
1207	n/a
1208	n/a	def _has_egg_info(srcdir):
1209	n/a	if os.path.isdir(srcdir):
1210	n/a	for item in os.listdir(srcdir):
1211	n/a	full_path = os.path.join(srcdir, item)
1212	n/a	if item.endswith('.egg-info') and os.path.isdir(full_path):
1213	n/a	logger.debug("Found egg-info directory.")
1214	n/a	return True
1215	n/a	logger.debug("No egg-info directory found.")
1216	n/a	return False
1217	n/a
1218	n/a
1219	n/a	def _has_setuptools_text(setup_py):
1220	n/a	return _has_text(setup_py, 'setuptools')
1221	n/a
1222	n/a
1223	n/a	def _has_distutils_text(setup_py):
1224	n/a	return _has_text(setup_py, 'distutils')
1225	n/a
1226	n/a
1227	n/a	def _has_text(setup_py, installer):
1228	n/a	installer_pattern = re.compile('import {0}\|from {0}'.format(installer))
1229	n/a	with open(setup_py, 'r', encoding='utf-8') as setup:
1230	n/a	for line in setup:
1231	n/a	if re.search(installer_pattern, line):
1232	n/a	logger.debug("Found %s text in setup.py.", installer)
1233	n/a	return True
1234	n/a	logger.debug("No %s text found in setup.py.", installer)
1235	n/a	return False
1236	n/a
1237	n/a
1238	n/a	def _has_required_metadata(setup_cfg):
1239	n/a	config = RawConfigParser()
1240	n/a	config.read([setup_cfg], encoding='utf8')
1241	n/a	return (config.has_section('metadata') and
1242	n/a	'name' in config.options('metadata') and
1243	n/a	'version' in config.options('metadata'))
1244	n/a
1245	n/a
1246	n/a	def _has_pkg_info(srcdir):
1247	n/a	pkg_info = os.path.join(srcdir, 'PKG-INFO')
1248	n/a	has_pkg_info = os.path.isfile(pkg_info)
1249	n/a	if has_pkg_info:
1250	n/a	logger.debug("PKG-INFO file found.")
1251	n/a	else:
1252	n/a	logger.debug("No PKG-INFO file found.")
1253	n/a	return has_pkg_info
1254	n/a
1255	n/a
1256	n/a	def _has_setup_py(srcdir):
1257	n/a	setup_py = os.path.join(srcdir, 'setup.py')
1258	n/a	if os.path.isfile(setup_py):
1259	n/a	logger.debug('setup.py file found.')
1260	n/a	return True
1261	n/a	return False
1262	n/a
1263	n/a
1264	n/a	def _has_setup_cfg(srcdir):
1265	n/a	setup_cfg = os.path.join(srcdir, 'setup.cfg')
1266	n/a	if os.path.isfile(setup_cfg):
1267	n/a	logger.debug('setup.cfg file found.')
1268	n/a	return True
1269	n/a	logger.debug("No setup.cfg file found.")
1270	n/a	return False
1271	n/a
1272	n/a
1273	n/a	def is_setuptools(path):
1274	n/a	"""Check if the project is based on setuptools.
1275	n/a
1276	n/a	:param path: path to source directory containing a setup.py script.
1277	n/a
1278	n/a	Return True if the project requires setuptools to install, else False.
1279	n/a	"""
1280	n/a	srcdir = os.path.abspath(path)
1281	n/a	setup_py = os.path.join(srcdir, 'setup.py')
1282	n/a
1283	n/a	return _has_setup_py(srcdir) and (_has_egg_info(srcdir) or
1284	n/a	_has_setuptools_text(setup_py))
1285	n/a
1286	n/a
1287	n/a	def is_distutils(path):
1288	n/a	"""Check if the project is based on distutils.
1289	n/a
1290	n/a	:param path: path to source directory containing a setup.py script.
1291	n/a
1292	n/a	Return True if the project requires distutils to install, else False.
1293	n/a	"""
1294	n/a	srcdir = os.path.abspath(path)
1295	n/a	setup_py = os.path.join(srcdir, 'setup.py')
1296	n/a
1297	n/a	return _has_setup_py(srcdir) and (_has_pkg_info(srcdir) or
1298	n/a	_has_distutils_text(setup_py))
1299	n/a
1300	n/a
1301	n/a	def is_packaging(path):
1302	n/a	"""Check if the project is based on packaging
1303	n/a
1304	n/a	:param path: path to source directory containing a setup.cfg file.
1305	n/a
1306	n/a	Return True if the project has a valid setup.cfg, else False.
1307	n/a	"""
1308	n/a	srcdir = os.path.abspath(path)
1309	n/a	setup_cfg = os.path.join(srcdir, 'setup.cfg')
1310	n/a
1311	n/a	return _has_setup_cfg(srcdir) and _has_required_metadata(setup_cfg)
1312	n/a
1313	n/a
1314	n/a	def get_install_method(path):
1315	n/a	"""Check if the project is based on packaging, setuptools, or distutils
1316	n/a
1317	n/a	:param path: path to source directory containing a setup.cfg file,
1318	n/a	or setup.py.
1319	n/a
1320	n/a	Returns a string representing the best install method to use.
1321	n/a	"""
1322	n/a	if is_packaging(path):
1323	n/a	return "packaging"
1324	n/a	elif is_setuptools(path):
1325	n/a	return "setuptools"
1326	n/a	elif is_distutils(path):
1327	n/a	return "distutils"
1328	n/a	else:
1329	n/a	raise InstallationException('Cannot detect install method')
1330	n/a
1331	n/a
1332	n/a	# XXX to be replaced by shutil.copytree
1333	n/a	def copy_tree(src, dst, preserve_mode=True, preserve_times=True,
1334	n/a	preserve_symlinks=False, update=False, dry_run=False):
1335	n/a	# FIXME use of this function is why we get spurious logging message on
1336	n/a	# stdout when tests run; kill and replace by shutil!
1337	n/a	from distutils.file_util import copy_file
1338	n/a
1339	n/a	if not dry_run and not os.path.isdir(src):
1340	n/a	raise PackagingFileError(
1341	n/a	"cannot copy tree '%s': not a directory" % src)
1342	n/a	try:
1343	n/a	names = os.listdir(src)
1344	n/a	except os.error as e:
1345	n/a	errstr = e[1]
1346	n/a	if dry_run:
1347	n/a	names = []
1348	n/a	else:
1349	n/a	raise PackagingFileError(
1350	n/a	"error listing files in '%s': %s" % (src, errstr))
1351	n/a
1352	n/a	if not dry_run:
1353	n/a	_mkpath(dst)
1354	n/a
1355	n/a	outputs = []
1356	n/a
1357	n/a	for n in names:
1358	n/a	src_name = os.path.join(src, n)
1359	n/a	dst_name = os.path.join(dst, n)
1360	n/a
1361	n/a	if preserve_symlinks and os.path.islink(src_name):
1362	n/a	link_dest = os.readlink(src_name)
1363	n/a	logger.info("linking %s -> %s", dst_name, link_dest)
1364	n/a	if not dry_run:
1365	n/a	os.symlink(link_dest, dst_name)
1366	n/a	outputs.append(dst_name)
1367	n/a
1368	n/a	elif os.path.isdir(src_name):
1369	n/a	outputs.extend(
1370	n/a	copy_tree(src_name, dst_name, preserve_mode,
1371	n/a	preserve_times, preserve_symlinks, update,
1372	n/a	dry_run=dry_run))
1373	n/a	else:
1374	n/a	copy_file(src_name, dst_name, preserve_mode,
1375	n/a	preserve_times, update, dry_run=dry_run)
1376	n/a	outputs.append(dst_name)
1377	n/a
1378	n/a	return outputs
1379	n/a
1380	n/a	# cache for by mkpath() -- in addition to cheapening redundant calls,
1381	n/a	# eliminates redundant "creating /foo/bar/baz" messages in dry-run mode
1382	n/a	_path_created = set()
1383	n/a
1384	n/a
1385	n/a	# I don't use os.makedirs because a) it's new to Python 1.5.2, and
1386	n/a	# b) it blows up if the directory already exists (I want to silently
1387	n/a	# succeed in that case).
1388	n/a	def _mkpath(name, mode=0o777, dry_run=False):
1389	n/a	# Detect a common bug -- name is None
1390	n/a	if not isinstance(name, str):
1391	n/a	raise PackagingInternalError(
1392	n/a	"mkpath: 'name' must be a string (got %r)" % (name,))
1393	n/a
1394	n/a	# XXX what's the better way to handle verbosity? print as we create
1395	n/a	# each directory in the path (the current behaviour), or only announce
1396	n/a	# the creation of the whole path? (quite easy to do the latter since
1397	n/a	# we're not using a recursive algorithm)
1398	n/a
1399	n/a	name = os.path.normpath(name)
1400	n/a	created_dirs = []
1401	n/a	if os.path.isdir(name) or name == '':
1402	n/a	return created_dirs
1403	n/a	if os.path.abspath(name) in _path_created:
1404	n/a	return created_dirs
1405	n/a
1406	n/a	head, tail = os.path.split(name)
1407	n/a	tails = [tail] # stack of lone dirs to create
1408	n/a
1409	n/a	while head and tail and not os.path.isdir(head):
1410	n/a	head, tail = os.path.split(head)
1411	n/a	tails.insert(0, tail) # push next higher dir onto stack
1412	n/a
1413	n/a	# now 'head' contains the deepest directory that already exists
1414	n/a	# (that is, the child of 'head' in 'name' is the highest directory
1415	n/a	# that does not exist)
1416	n/a	for d in tails:
1417	n/a	head = os.path.join(head, d)
1418	n/a	abs_head = os.path.abspath(head)
1419	n/a
1420	n/a	if abs_head in _path_created:
1421	n/a	continue
1422	n/a
1423	n/a	logger.info("creating %s", head)
1424	n/a	if not dry_run:
1425	n/a	try:
1426	n/a	os.mkdir(head, mode)
1427	n/a	except OSError as exc:
1428	n/a	if not (exc.errno == errno.EEXIST and os.path.isdir(head)):
1429	n/a	raise PackagingFileError(
1430	n/a	"could not create '%s': %s" % (head, exc.args[-1]))
1431	n/a	created_dirs.append(head)
1432	n/a
1433	n/a	_path_created.add(abs_head)
1434	n/a	return created_dirs
1435	n/a
1436	n/a
1437	n/a	def encode_multipart(fields, files, boundary=None):
1438	n/a	"""Prepare a multipart HTTP request.
1439	n/a
1440	n/a	fields is a sequence of (name: str, value: str) elements for regular
1441	n/a	form fields, files is a sequence of (name: str, filename: str, value:
1442	n/a	bytes) elements for data to be uploaded as files.
1443	n/a
1444	n/a	Returns (content_type: bytes, body: bytes) ready for http.client.HTTP.
1445	n/a	"""
1446	n/a	# Taken from http://code.activestate.com/recipes/146306
1447	n/a
1448	n/a	if boundary is None:
1449	n/a	boundary = b'--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
1450	n/a	elif not isinstance(boundary, bytes):
1451	n/a	raise TypeError('boundary must be bytes, not %r' % type(boundary))
1452	n/a
1453	n/a	l = []
1454	n/a	for key, values in fields:
1455	n/a	# handle multiple entries for the same name
1456	n/a	if not isinstance(values, (tuple, list)):
1457	n/a	values = [values]
1458	n/a
1459	n/a	for value in values:
1460	n/a	l.extend((
1461	n/a	b'--' + boundary,
1462	n/a	('Content-Disposition: form-data; name="%s"' %
1463	n/a	key).encode('utf-8'),
1464	n/a	b'',
1465	n/a	value.encode('utf-8')))
1466	n/a
1467	n/a	for key, filename, value in files:
1468	n/a	l.extend((
1469	n/a	b'--' + boundary,
1470	n/a	('Content-Disposition: form-data; name="%s"; filename="%s"' %
1471	n/a	(key, filename)).encode('utf-8'),
1472	n/a	b'',
1473	n/a	value))
1474	n/a
1475	n/a	l.append(b'--' + boundary + b'--')
1476	n/a	l.append(b'')
1477	n/a
1478	n/a	body = b'\r\n'.join(l)
1479	n/a	content_type = b'multipart/form-data; boundary=' + boundary
1480	n/a	return content_type, body