Python code coverage for Lib/packaging/install.py

#	count	content
1	n/a	"""Building blocks for installers.
2	n/a
3	n/a	When used as a script, this module installs a release thanks to info
4	n/a	obtained from an index (e.g. PyPI), with dependencies.
5	n/a
6	n/a	This is a higher-level module built on packaging.database and
7	n/a	packaging.pypi.
8	n/a	"""
9	n/a	import os
10	n/a	import sys
11	n/a	import stat
12	n/a	import errno
13	n/a	import shutil
14	n/a	import logging
15	n/a	import tempfile
16	n/a	from sysconfig import get_config_var, get_path, is_python_build
17	n/a
18	n/a	from packaging import logger
19	n/a	from packaging.dist import Distribution
20	n/a	from packaging.util import (_is_archive_file, ask, get_install_method,
21	n/a	egginfo_to_distinfo)
22	n/a	from packaging.pypi import wrapper
23	n/a	from packaging.version import get_version_predicate
24	n/a	from packaging.database import get_distributions, get_distribution
25	n/a	from packaging.depgraph import generate_graph
26	n/a
27	n/a	from packaging.errors import (PackagingError, InstallationException,
28	n/a	InstallationConflict, CCompilerError)
29	n/a	from packaging.pypi.errors import ProjectNotFound, ReleaseNotFound
30	n/a	from packaging import database
31	n/a
32	n/a
33	n/a	__all__ = ['install_dists', 'install_from_infos', 'get_infos', 'remove',
34	n/a	'install', 'install_local_project']
35	n/a
36	n/a
37	n/a	def _move_files(files, destination):
38	n/a	"""Move the list of files in the destination folder, keeping the same
39	n/a	structure.
40	n/a
41	n/a	Return a list of tuple (old, new) emplacement of files
42	n/a
43	n/a	:param files: a list of files to move.
44	n/a	:param destination: the destination directory to put on the files.
45	n/a	"""
46	n/a
47	n/a	for old in files:
48	n/a	filename = os.path.split(old)[-1]
49	n/a	new = os.path.join(destination, filename)
50	n/a	# try to make the paths.
51	n/a	try:
52	n/a	os.makedirs(os.path.dirname(new))
53	n/a	except OSError as e:
54	n/a	if e.errno != errno.EEXIST:
55	n/a	raise
56	n/a	os.rename(old, new)
57	n/a	yield old, new
58	n/a
59	n/a
60	n/a	def _run_distutils_install(path):
61	n/a	# backward compat: using setuptools or plain-distutils
62	n/a	cmd = '%s setup.py install --record=%s'
63	n/a	record_file = os.path.join(path, 'RECORD')
64	n/a	os.system(cmd % (sys.executable, record_file))
65	n/a	if not os.path.exists(record_file):
66	n/a	raise ValueError('failed to install')
67	n/a	else:
68	n/a	egginfo_to_distinfo(record_file, remove_egginfo=True)
69	n/a
70	n/a
71	n/a	def _run_setuptools_install(path):
72	n/a	cmd = '%s setup.py install --record=%s --single-version-externally-managed'
73	n/a	record_file = os.path.join(path, 'RECORD')
74	n/a
75	n/a	os.system(cmd % (sys.executable, record_file))
76	n/a	if not os.path.exists(record_file):
77	n/a	raise ValueError('failed to install')
78	n/a	else:
79	n/a	egginfo_to_distinfo(record_file, remove_egginfo=True)
80	n/a
81	n/a
82	n/a	def _run_packaging_install(path):
83	n/a	# XXX check for a valid setup.cfg?
84	n/a	dist = Distribution()
85	n/a	dist.parse_config_files()
86	n/a	try:
87	n/a	dist.run_command('install_dist')
88	n/a	name = dist.metadata['Name']
89	n/a	return database.get_distribution(name) is not None
90	n/a	except (IOError, os.error, PackagingError, CCompilerError) as msg:
91	n/a	raise ValueError("Failed to install, " + str(msg))
92	n/a
93	n/a
94	n/a	def _install_dist(dist, path):
95	n/a	"""Install a distribution into a path.
96	n/a
97	n/a	This:
98	n/a
99	n/a	* unpack the distribution
100	n/a	* copy the files in "path"
101	n/a	* determine if the distribution is packaging or distutils1.
102	n/a	"""
103	n/a	where = dist.unpack()
104	n/a
105	n/a	if where is None:
106	n/a	raise ValueError('Cannot locate the unpacked archive')
107	n/a
108	n/a	return _run_install_from_archive(where)
109	n/a
110	n/a
111	n/a	def install_local_project(path):
112	n/a	"""Install a distribution from a source directory.
113	n/a
114	n/a	If the source directory contains a setup.py install using distutils1.
115	n/a	If a setup.cfg is found, install using the install_dist command.
116	n/a
117	n/a	Returns True on success, False on Failure.
118	n/a	"""
119	n/a	path = os.path.abspath(path)
120	n/a	if os.path.isdir(path):
121	n/a	logger.info('Installing from source directory: %r', path)
122	n/a	return _run_install_from_dir(path)
123	n/a	elif _is_archive_file(path):
124	n/a	logger.info('Installing from archive: %r', path)
125	n/a	_unpacked_dir = tempfile.mkdtemp()
126	n/a	try:
127	n/a	shutil.unpack_archive(path, _unpacked_dir)
128	n/a	return _run_install_from_archive(_unpacked_dir)
129	n/a	finally:
130	n/a	shutil.rmtree(_unpacked_dir)
131	n/a	else:
132	n/a	logger.warning('No project to install.')
133	n/a	return False
134	n/a
135	n/a
136	n/a	def _run_install_from_archive(source_dir):
137	n/a	# XXX need a better way
138	n/a	for item in os.listdir(source_dir):
139	n/a	fullpath = os.path.join(source_dir, item)
140	n/a	if os.path.isdir(fullpath):
141	n/a	source_dir = fullpath
142	n/a	break
143	n/a	return _run_install_from_dir(source_dir)
144	n/a
145	n/a
146	n/a	install_methods = {
147	n/a	'packaging': _run_packaging_install,
148	n/a	'setuptools': _run_setuptools_install,
149	n/a	'distutils': _run_distutils_install}
150	n/a
151	n/a
152	n/a	def _run_install_from_dir(source_dir):
153	n/a	old_dir = os.getcwd()
154	n/a	os.chdir(source_dir)
155	n/a	install_method = get_install_method(source_dir)
156	n/a	func = install_methods[install_method]
157	n/a	try:
158	n/a	func = install_methods[install_method]
159	n/a	try:
160	n/a	func(source_dir)
161	n/a	return True
162	n/a	except ValueError as err:
163	n/a	# failed to install
164	n/a	logger.info(str(err))
165	n/a	return False
166	n/a	finally:
167	n/a	os.chdir(old_dir)
168	n/a
169	n/a
170	n/a	def install_dists(dists, path, paths=None):
171	n/a	"""Install all distributions provided in dists, with the given prefix.
172	n/a
173	n/a	If an error occurs while installing one of the distributions, uninstall all
174	n/a	the installed distribution (in the context if this function).
175	n/a
176	n/a	Return a list of installed dists.
177	n/a
178	n/a	:param dists: distributions to install
179	n/a	:param path: base path to install distribution in
180	n/a	:param paths: list of paths (defaults to sys.path) to look for info
181	n/a	"""
182	n/a
183	n/a	installed_dists = []
184	n/a	for dist in dists:
185	n/a	logger.info('Installing %r %s...', dist.name, dist.version)
186	n/a	try:
187	n/a	_install_dist(dist, path)
188	n/a	installed_dists.append(dist)
189	n/a	except Exception as e:
190	n/a	logger.info('Failed: %s', e)
191	n/a
192	n/a	# reverting
193	n/a	for installed_dist in installed_dists:
194	n/a	logger.info('Reverting %r', installed_dist)
195	n/a	remove(installed_dist.name, paths)
196	n/a	raise e
197	n/a	return installed_dists
198	n/a
199	n/a
200	n/a	def install_from_infos(install_path=None, install=[], remove=[], conflicts=[],
201	n/a	paths=None):
202	n/a	"""Install and remove the given distributions.
203	n/a
204	n/a	The function signature is made to be compatible with the one of get_infos.
205	n/a	The aim of this script is to povide a way to install/remove what's asked,
206	n/a	and to rollback if needed.
207	n/a
208	n/a	So, it's not possible to be in an inconsistant state, it could be either
209	n/a	installed, either uninstalled, not half-installed.
210	n/a
211	n/a	The process follow those steps:
212	n/a
213	n/a	1. Move all distributions that will be removed in a temporary location
214	n/a	2. Install all the distributions that will be installed in a temp. loc.
215	n/a	3. If the installation fails, rollback (eg. move back) those
216	n/a	distributions, or remove what have been installed.
217	n/a	4. Else, move the distributions to the right locations, and remove for
218	n/a	real the distributions thats need to be removed.
219	n/a
220	n/a	:param install_path: the installation path where we want to install the
221	n/a	distributions.
222	n/a	:param install: list of distributions that will be installed; install_path
223	n/a	must be provided if this list is not empty.
224	n/a	:param remove: list of distributions that will be removed.
225	n/a	:param conflicts: list of conflicting distributions, eg. that will be in
226	n/a	conflict once the install and remove distribution will be
227	n/a	processed.
228	n/a	:param paths: list of paths (defaults to sys.path) to look for info
229	n/a	"""
230	n/a	# first of all, if we have conflicts, stop here.
231	n/a	if conflicts:
232	n/a	raise InstallationConflict(conflicts)
233	n/a
234	n/a	if install and not install_path:
235	n/a	raise ValueError("Distributions are to be installed but `install_path`"
236	n/a	" is not provided.")
237	n/a
238	n/a	# before removing the files, we will start by moving them away
239	n/a	# then, if any error occurs, we could replace them in the good place.
240	n/a	temp_files = {} # contains lists of {dist: (old, new)} paths
241	n/a	temp_dir = None
242	n/a	if remove:
243	n/a	temp_dir = tempfile.mkdtemp()
244	n/a	for dist in remove:
245	n/a	files = dist.list_installed_files()
246	n/a	temp_files[dist] = _move_files(files, temp_dir)
247	n/a	try:
248	n/a	if install:
249	n/a	install_dists(install, install_path, paths)
250	n/a	except:
251	n/a	# if an error occurs, put back the files in the right place.
252	n/a	for files in temp_files.values():
253	n/a	for old, new in files:
254	n/a	shutil.move(new, old)
255	n/a	if temp_dir:
256	n/a	shutil.rmtree(temp_dir)
257	n/a	# now re-raising
258	n/a	raise
259	n/a
260	n/a	# we can remove them for good
261	n/a	for files in temp_files.values():
262	n/a	for old, new in files:
263	n/a	os.remove(new)
264	n/a	if temp_dir:
265	n/a	shutil.rmtree(temp_dir)
266	n/a
267	n/a
268	n/a	def _get_setuptools_deps(release):
269	n/a	# NotImplementedError
270	n/a	pass
271	n/a
272	n/a
273	n/a	def get_infos(requirements, index=None, installed=None, prefer_final=True):
274	n/a	"""Return the informations on what's going to be installed and upgraded.
275	n/a
276	n/a	:param requirements: is a string containing the requirements for this
277	n/a	project (for instance "FooBar 1.1" or "BarBaz (<1.2)")
278	n/a	:param index: If an index is specified, use this one, otherwise, use
279	n/a	:class index.ClientWrapper: to get project metadatas.
280	n/a	:param installed: a list of already installed distributions.
281	n/a	:param prefer_final: when picking up the releases, prefer a "final" one
282	n/a	over a beta/alpha/etc one.
283	n/a
284	n/a	The results are returned in a dict, containing all the operations
285	n/a	needed to install the given requirements::
286	n/a
287	n/a	>>> get_install_info("FooBar (<=1.2)")
288	n/a	{'install': [<FooBar 1.1>], 'remove': [], 'conflict': []}
289	n/a
290	n/a	Conflict contains all the conflicting distributions, if there is a
291	n/a	conflict.
292	n/a	"""
293	n/a	# this function does several things:
294	n/a	# 1. get a release specified by the requirements
295	n/a	# 2. gather its metadata, using setuptools compatibility if needed
296	n/a	# 3. compare this tree with what is currently installed on the system,
297	n/a	# return the requirements of what is missing
298	n/a	# 4. do that recursively and merge back the results
299	n/a	# 5. return a dict containing information about what is needed to install
300	n/a	# or remove
301	n/a
302	n/a	if not installed:
303	n/a	logger.debug('Reading installed distributions')
304	n/a	installed = list(get_distributions(use_egg_info=True))
305	n/a
306	n/a	infos = {'install': [], 'remove': [], 'conflict': []}
307	n/a	# Is a compatible version of the project already installed ?
308	n/a	predicate = get_version_predicate(requirements)
309	n/a	found = False
310	n/a
311	n/a	# check that the project isn't already installed
312	n/a	for installed_project in installed:
313	n/a	# is it a compatible project ?
314	n/a	if predicate.name.lower() != installed_project.name.lower():
315	n/a	continue
316	n/a	found = True
317	n/a	logger.info('Found %r %s', installed_project.name,
318	n/a	installed_project.version)
319	n/a
320	n/a	# if we already have something installed, check it matches the
321	n/a	# requirements
322	n/a	if predicate.match(installed_project.version):
323	n/a	return infos
324	n/a	break
325	n/a
326	n/a	if not found:
327	n/a	logger.debug('Project not installed')
328	n/a
329	n/a	if not index:
330	n/a	index = wrapper.ClientWrapper()
331	n/a
332	n/a	if not installed:
333	n/a	installed = get_distributions(use_egg_info=True)
334	n/a
335	n/a	# Get all the releases that match the requirements
336	n/a	try:
337	n/a	release = index.get_release(requirements)
338	n/a	except (ReleaseNotFound, ProjectNotFound):
339	n/a	raise InstallationException('Release not found: %r' % requirements)
340	n/a
341	n/a	if release is None:
342	n/a	logger.info('Could not find a matching project')
343	n/a	return infos
344	n/a
345	n/a	metadata = release.fetch_metadata()
346	n/a
347	n/a	# we need to build setuptools deps if any
348	n/a	if 'requires_dist' not in metadata:
349	n/a	metadata['requires_dist'] = _get_setuptools_deps(release)
350	n/a
351	n/a	# build the dependency graph with local and required dependencies
352	n/a	dists = list(installed)
353	n/a	dists.append(release)
354	n/a	depgraph = generate_graph(dists)
355	n/a
356	n/a	# Get what the missing deps are
357	n/a	dists = depgraph.missing[release]
358	n/a	if dists:
359	n/a	logger.info("Missing dependencies found, retrieving metadata")
360	n/a	# we have missing deps
361	n/a	for dist in dists:
362	n/a	_update_infos(infos, get_infos(dist, index, installed))
363	n/a
364	n/a	# Fill in the infos
365	n/a	existing = [d for d in installed if d.name == release.name]
366	n/a	if existing:
367	n/a	infos['remove'].append(existing[0])
368	n/a	infos['conflict'].extend(depgraph.reverse_list[existing[0]])
369	n/a	infos['install'].append(release)
370	n/a	return infos
371	n/a
372	n/a
373	n/a	def _update_infos(infos, new_infos):
374	n/a	"""extends the lists contained in the `info` dict with those contained
375	n/a	in the `new_info` one
376	n/a	"""
377	n/a	for key, value in infos.items():
378	n/a	if key in new_infos:
379	n/a	infos[key].extend(new_infos[key])
380	n/a
381	n/a
382	n/a	def remove(project_name, paths=None, auto_confirm=True):
383	n/a	"""Removes a single project from the installation.
384	n/a
385	n/a	Returns True on success
386	n/a	"""
387	n/a	dist = get_distribution(project_name, use_egg_info=True, paths=paths)
388	n/a	if dist is None:
389	n/a	raise PackagingError('Distribution %r not found' % project_name)
390	n/a	files = dist.list_installed_files(local=True)
391	n/a	rmdirs = []
392	n/a	rmfiles = []
393	n/a	tmp = tempfile.mkdtemp(prefix=project_name + '-uninstall')
394	n/a
395	n/a	def _move_file(source, target):
396	n/a	try:
397	n/a	os.rename(source, target)
398	n/a	except OSError as err:
399	n/a	return err
400	n/a	return None
401	n/a
402	n/a	success = True
403	n/a	error = None
404	n/a	try:
405	n/a	for file_, md5, size in files:
406	n/a	if os.path.isfile(file_):
407	n/a	dirname, filename = os.path.split(file_)
408	n/a	tmpfile = os.path.join(tmp, filename)
409	n/a	try:
410	n/a	error = _move_file(file_, tmpfile)
411	n/a	if error is not None:
412	n/a	success = False
413	n/a	break
414	n/a	finally:
415	n/a	if not os.path.isfile(file_):
416	n/a	os.rename(tmpfile, file_)
417	n/a	if file_ not in rmfiles:
418	n/a	rmfiles.append(file_)
419	n/a	if dirname not in rmdirs:
420	n/a	rmdirs.append(dirname)
421	n/a	finally:
422	n/a	shutil.rmtree(tmp)
423	n/a
424	n/a	if not success:
425	n/a	logger.info('%r cannot be removed.', project_name)
426	n/a	logger.info('Error: %s', error)
427	n/a	return False
428	n/a
429	n/a	logger.info('Removing %r: ', project_name)
430	n/a
431	n/a	for file_ in rmfiles:
432	n/a	logger.info(' %s', file_)
433	n/a
434	n/a	# Taken from the pip project
435	n/a	if auto_confirm:
436	n/a	response = 'y'
437	n/a	else:
438	n/a	response = ask('Proceed (y/n)? ', ('y', 'n'))
439	n/a
440	n/a	if response == 'y':
441	n/a	file_count = 0
442	n/a	for file_ in rmfiles:
443	n/a	os.remove(file_)
444	n/a	file_count += 1
445	n/a
446	n/a	dir_count = 0
447	n/a	for dirname in rmdirs:
448	n/a	if not os.path.exists(dirname):
449	n/a	# could
450	n/a	continue
451	n/a
452	n/a	files_count = 0
453	n/a	for root, dir, files in os.walk(dirname):
454	n/a	files_count += len(files)
455	n/a
456	n/a	if files_count > 0:
457	n/a	# XXX Warning
458	n/a	continue
459	n/a
460	n/a	# empty dirs with only empty dirs
461	n/a	if os.stat(dirname).st_mode & stat.S_IWUSR:
462	n/a	# XXX Add a callable in shutil.rmtree to count
463	n/a	# the number of deleted elements
464	n/a	shutil.rmtree(dirname)
465	n/a	dir_count += 1
466	n/a
467	n/a	# removing the top path
468	n/a	# XXX count it ?
469	n/a	if os.path.exists(dist.path):
470	n/a	shutil.rmtree(dist.path)
471	n/a
472	n/a	logger.info('Success: removed %d files and %d dirs',
473	n/a	file_count, dir_count)
474	n/a
475	n/a	return True
476	n/a
477	n/a
478	n/a	def install(project):
479	n/a	"""Installs a project.
480	n/a
481	n/a	Returns True on success, False on failure
482	n/a	"""
483	n/a	if is_python_build():
484	n/a	# Python would try to install into the site-packages directory under
485	n/a	# $PREFIX, but when running from an uninstalled code checkout we don't
486	n/a	# want to create directories under the installation root
487	n/a	message = ('installing third-party projects from an uninstalled '
488	n/a	'Python is not supported')
489	n/a	logger.error(message)
490	n/a	return False
491	n/a
492	n/a	logger.info('Checking the installation location...')
493	n/a	purelib_path = get_path('purelib')
494	n/a
495	n/a	# trying to write a file there
496	n/a	try:
497	n/a	with tempfile.NamedTemporaryFile(suffix=project,
498	n/a	dir=purelib_path) as testfile:
499	n/a	testfile.write(b'test')
500	n/a	except OSError:
501	n/a	# FIXME this should check the errno, or be removed altogether (race
502	n/a	# condition: the directory permissions could be changed between here
503	n/a	# and the actual install)
504	n/a	logger.info('Unable to write in "%s". Do you have the permissions ?'
505	n/a	% purelib_path)
506	n/a	return False
507	n/a
508	n/a	logger.info('Getting information about %r...', project)
509	n/a	try:
510	n/a	info = get_infos(project)
511	n/a	except InstallationException:
512	n/a	logger.info('Cound not find %r', project)
513	n/a	return False
514	n/a
515	n/a	if info['install'] == []:
516	n/a	logger.info('Nothing to install')
517	n/a	return False
518	n/a
519	n/a	install_path = get_config_var('base')
520	n/a	try:
521	n/a	install_from_infos(install_path,
522	n/a	info['install'], info['remove'], info['conflict'])
523	n/a
524	n/a	except InstallationConflict as e:
525	n/a	if logger.isEnabledFor(logging.INFO):
526	n/a	projects = ('%r %s' % (p.name, p.version) for p in e.args[0])
527	n/a	logger.info('%r conflicts with %s', project, ','.join(projects))
528	n/a
529	n/a	return True