Python code coverage for Lib/packaging/pypi/dist.py

#	count	content
1	n/a	"""Classes representing releases and distributions retrieved from indexes.
2	n/a
3	n/a	A project (= unique name) can have several releases (= versions) and
4	n/a	each release can have several distributions (= sdist and bdists).
5	n/a
6	n/a	Release objects contain metadata-related information (see PEP 376);
7	n/a	distribution objects contain download-related information.
8	n/a	"""
9	n/a
10	n/a	import re
11	n/a	import hashlib
12	n/a	import tempfile
13	n/a	import urllib.request
14	n/a	import urllib.parse
15	n/a	import urllib.error
16	n/a	import urllib.parse
17	n/a	from shutil import unpack_archive
18	n/a
19	n/a	from packaging.errors import IrrationalVersionError
20	n/a	from packaging.version import (suggest_normalized_version, NormalizedVersion,
21	n/a	get_version_predicate)
22	n/a	from packaging.metadata import Metadata
23	n/a	from packaging.pypi.errors import (HashDoesNotMatch, UnsupportedHashName,
24	n/a	CantParseArchiveName)
25	n/a
26	n/a
27	n/a	__all__ = ['ReleaseInfo', 'DistInfo', 'ReleasesList', 'get_infos_from_url']
28	n/a
29	n/a	EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz .egg".split()
30	n/a	MD5_HASH = re.compile(r'^.*#md5=([a-f0-9]+)$')
31	n/a	DIST_TYPES = ['bdist', 'sdist']
32	n/a
33	n/a
34	n/a	class IndexReference:
35	n/a	"""Mixin used to store the index reference"""
36	n/a	def set_index(self, index=None):
37	n/a	self._index = index
38	n/a
39	n/a
40	n/a	class ReleaseInfo(IndexReference):
41	n/a	"""Represent a release of a project (a project with a specific version).
42	n/a	The release contain the _metadata informations related to this specific
43	n/a	version, and is also a container for distribution related informations.
44	n/a
45	n/a	See the DistInfo class for more information about distributions.
46	n/a	"""
47	n/a
48	n/a	def __init__(self, name, version, metadata=None, hidden=False,
49	n/a	index=None, **kwargs):
50	n/a	"""
51	n/a	:param name: the name of the distribution
52	n/a	:param version: the version of the distribution
53	n/a	:param metadata: the metadata fields of the release.
54	n/a	:type metadata: dict
55	n/a	:param kwargs: optional arguments for a new distribution.
56	n/a	"""
57	n/a	self.set_index(index)
58	n/a	self.name = name
59	n/a	self._version = None
60	n/a	self.version = version
61	n/a	if metadata:
62	n/a	self.metadata = Metadata(mapping=metadata)
63	n/a	else:
64	n/a	self.metadata = None
65	n/a	self.dists = {}
66	n/a	self.hidden = hidden
67	n/a
68	n/a	if 'dist_type' in kwargs:
69	n/a	dist_type = kwargs.pop('dist_type')
70	n/a	self.add_distribution(dist_type, **kwargs)
71	n/a
72	n/a	def set_version(self, version):
73	n/a	try:
74	n/a	self._version = NormalizedVersion(version)
75	n/a	except IrrationalVersionError:
76	n/a	suggestion = suggest_normalized_version(version)
77	n/a	if suggestion:
78	n/a	self.version = suggestion
79	n/a	else:
80	n/a	raise IrrationalVersionError(version)
81	n/a
82	n/a	def get_version(self):
83	n/a	return self._version
84	n/a
85	n/a	version = property(get_version, set_version)
86	n/a
87	n/a	def fetch_metadata(self):
88	n/a	"""If the metadata is not set, use the indexes to get it"""
89	n/a	if not self.metadata:
90	n/a	self._index.get_metadata(self.name, str(self.version))
91	n/a	return self.metadata
92	n/a
93	n/a	@property
94	n/a	def is_final(self):
95	n/a	"""proxy to version.is_final"""
96	n/a	return self.version.is_final
97	n/a
98	n/a	def fetch_distributions(self):
99	n/a	if self.dists is None:
100	n/a	self._index.get_distributions(self.name, str(self.version))
101	n/a	if self.dists is None:
102	n/a	self.dists = {}
103	n/a	return self.dists
104	n/a
105	n/a	def add_distribution(self, dist_type='sdist', python_version=None,
106	n/a	**params):
107	n/a	"""Add distribution informations to this release.
108	n/a	If distribution information is already set for this distribution type,
109	n/a	add the given url paths to the distribution. This can be useful while
110	n/a	some of them fails to download.
111	n/a
112	n/a	:param dist_type: the distribution type (eg. "sdist", "bdist", etc.)
113	n/a	:param params: the fields to be passed to the distribution object
114	n/a	(see the :class:DistInfo constructor).
115	n/a	"""
116	n/a	if dist_type not in DIST_TYPES:
117	n/a	raise ValueError(dist_type)
118	n/a	if dist_type in self.dists:
119	n/a	self.dists[dist_type].add_url(**params)
120	n/a	else:
121	n/a	self.dists[dist_type] = DistInfo(self, dist_type,
122	n/a	index=self._index, **params)
123	n/a	if python_version:
124	n/a	self.dists[dist_type].python_version = python_version
125	n/a
126	n/a	def get_distribution(self, dist_type=None, prefer_source=True):
127	n/a	"""Return a distribution.
128	n/a
129	n/a	If dist_type is set, find first for this distribution type, and just
130	n/a	act as an alias of __get_item__.
131	n/a
132	n/a	If prefer_source is True, search first for source distribution, and if
133	n/a	not return one existing distribution.
134	n/a	"""
135	n/a	if len(self.dists) == 0:
136	n/a	raise LookupError
137	n/a	if dist_type:
138	n/a	return self[dist_type]
139	n/a	if prefer_source:
140	n/a	if "sdist" in self.dists:
141	n/a	dist = self["sdist"]
142	n/a	else:
143	n/a	dist = next(self.dists.values())
144	n/a	return dist
145	n/a
146	n/a	def unpack(self, path=None, prefer_source=True):
147	n/a	"""Unpack the distribution to the given path.
148	n/a
149	n/a	If not destination is given, creates a temporary location.
150	n/a
151	n/a	Returns the location of the extracted files (root).
152	n/a	"""
153	n/a	return self.get_distribution(prefer_source=prefer_source)\
154	n/a	.unpack(path=path)
155	n/a
156	n/a	def download(self, temp_path=None, prefer_source=True):
157	n/a	"""Download the distribution, using the requirements.
158	n/a
159	n/a	If more than one distribution match the requirements, use the last
160	n/a	version.
161	n/a	Download the distribution, and put it in the temp_path. If no temp_path
162	n/a	is given, creates and return one.
163	n/a
164	n/a	Returns the complete absolute path to the downloaded archive.
165	n/a	"""
166	n/a	return self.get_distribution(prefer_source=prefer_source)\
167	n/a	.download(path=temp_path)
168	n/a
169	n/a	def set_metadata(self, metadata):
170	n/a	if not self.metadata:
171	n/a	self.metadata = Metadata()
172	n/a	self.metadata.update(metadata)
173	n/a
174	n/a	def __getitem__(self, item):
175	n/a	"""distributions are available using release["sdist"]"""
176	n/a	return self.dists[item]
177	n/a
178	n/a	def _check_is_comparable(self, other):
179	n/a	if not isinstance(other, ReleaseInfo):
180	n/a	raise TypeError("cannot compare %s and %s"
181	n/a	% (type(self).__name__, type(other).__name__))
182	n/a	elif self.name != other.name:
183	n/a	raise TypeError("cannot compare %s and %s"
184	n/a	% (self.name, other.name))
185	n/a
186	n/a	def __repr__(self):
187	n/a	return "<%s %s>" % (self.name, self.version)
188	n/a
189	n/a	def __eq__(self, other):
190	n/a	self._check_is_comparable(other)
191	n/a	return self.version == other.version
192	n/a
193	n/a	def __lt__(self, other):
194	n/a	self._check_is_comparable(other)
195	n/a	return self.version < other.version
196	n/a
197	n/a	def __ne__(self, other):
198	n/a	return not self.__eq__(other)
199	n/a
200	n/a	def __gt__(self, other):
201	n/a	return not (self.__lt__(other) or self.__eq__(other))
202	n/a
203	n/a	def __le__(self, other):
204	n/a	return self.__eq__(other) or self.__lt__(other)
205	n/a
206	n/a	def __ge__(self, other):
207	n/a	return self.__eq__(other) or self.__gt__(other)
208	n/a
209	n/a	# See http://docs.python.org/reference/datamodel#object.__hash__
210	n/a	__hash__ = object.__hash__
211	n/a
212	n/a
213	n/a	class DistInfo(IndexReference):
214	n/a	"""Represents a distribution retrieved from an index (sdist, bdist, ...)
215	n/a	"""
216	n/a
217	n/a	def __init__(self, release, dist_type=None, url=None, hashname=None,
218	n/a	hashval=None, is_external=True, python_version=None,
219	n/a	index=None):
220	n/a	"""Create a new instance of DistInfo.
221	n/a
222	n/a	:param release: a DistInfo class is relative to a release.
223	n/a	:param dist_type: the type of the dist (eg. source, bin-*, etc.)
224	n/a	:param url: URL where we found this distribution
225	n/a	:param hashname: the name of the hash we want to use. Refer to the
226	n/a	hashlib.new documentation for more information.
227	n/a	:param hashval: the hash value.
228	n/a	:param is_external: we need to know if the provided url comes from
229	n/a	an index browsing, or from an external resource.
230	n/a
231	n/a	"""
232	n/a	self.set_index(index)
233	n/a	self.release = release
234	n/a	self.dist_type = dist_type
235	n/a	self.python_version = python_version
236	n/a	self._unpacked_dir = None
237	n/a	# set the downloaded path to None by default. The goal here
238	n/a	# is to not download distributions multiple times
239	n/a	self.downloaded_location = None
240	n/a	# We store urls in dict, because we need to have a bit more infos
241	n/a	# than the simple URL. It will be used later to find the good url to
242	n/a	# use.
243	n/a	# We have two _url* attributes: _url and urls. urls contains a list
244	n/a	# of dict for the different urls, and _url contains the choosen url, in
245	n/a	# order to dont make the selection process multiple times.
246	n/a	self.urls = []
247	n/a	self._url = None
248	n/a	self.add_url(url, hashname, hashval, is_external)
249	n/a
250	n/a	def add_url(self, url=None, hashname=None, hashval=None, is_external=True):
251	n/a	"""Add a new url to the list of urls"""
252	n/a	if hashname is not None:
253	n/a	try:
254	n/a	hashlib.new(hashname)
255	n/a	except ValueError:
256	n/a	raise UnsupportedHashName(hashname)
257	n/a	if url not in [u['url'] for u in self.urls]:
258	n/a	self.urls.append({
259	n/a	'url': url,
260	n/a	'hashname': hashname,
261	n/a	'hashval': hashval,
262	n/a	'is_external': is_external,
263	n/a	})
264	n/a	# reset the url selection process
265	n/a	self._url = None
266	n/a
267	n/a	@property
268	n/a	def url(self):
269	n/a	"""Pick up the right url for the list of urls in self.urls"""
270	n/a	# We return internal urls over externals.
271	n/a	# If there is more than one internal or external, return the first
272	n/a	# one.
273	n/a	if self._url is None:
274	n/a	if len(self.urls) > 1:
275	n/a	internals_urls = [u for u in self.urls \
276	n/a	if u['is_external'] == False]
277	n/a	if len(internals_urls) >= 1:
278	n/a	self._url = internals_urls[0]
279	n/a	if self._url is None:
280	n/a	self._url = self.urls[0]
281	n/a	return self._url
282	n/a
283	n/a	@property
284	n/a	def is_source(self):
285	n/a	"""return if the distribution is a source one or not"""
286	n/a	return self.dist_type == 'sdist'
287	n/a
288	n/a	def download(self, path=None):
289	n/a	"""Download the distribution to a path, and return it.
290	n/a
291	n/a	If the path is given in path, use this, otherwise, generates a new one
292	n/a	Return the download location.
293	n/a	"""
294	n/a	if path is None:
295	n/a	path = tempfile.mkdtemp()
296	n/a
297	n/a	# if we do not have downloaded it yet, do it.
298	n/a	if self.downloaded_location is None:
299	n/a	url = self.url['url']
300	n/a	archive_name = urllib.parse.urlparse(url)[2].split('/')[-1]
301	n/a	filename, headers = urllib.request.urlretrieve(url,
302	n/a	path + "/" + archive_name)
303	n/a	self.downloaded_location = filename
304	n/a	self._check_md5(filename)
305	n/a	return self.downloaded_location
306	n/a
307	n/a	def unpack(self, path=None):
308	n/a	"""Unpack the distribution to the given path.
309	n/a
310	n/a	If not destination is given, creates a temporary location.
311	n/a
312	n/a	Returns the location of the extracted files (root).
313	n/a	"""
314	n/a	if not self._unpacked_dir:
315	n/a	if path is None:
316	n/a	path = tempfile.mkdtemp()
317	n/a
318	n/a	filename = self.download(path)
319	n/a	unpack_archive(filename, path)
320	n/a	self._unpacked_dir = path
321	n/a
322	n/a	return path
323	n/a
324	n/a	def _check_md5(self, filename):
325	n/a	"""Check that the md5 checksum of the given file matches the one in
326	n/a	url param"""
327	n/a	hashname = self.url['hashname']
328	n/a	expected_hashval = self.url['hashval']
329	n/a	if None not in (expected_hashval, hashname):
330	n/a	with open(filename, 'rb') as f:
331	n/a	hashval = hashlib.new(hashname)
332	n/a	hashval.update(f.read())
333	n/a
334	n/a	if hashval.hexdigest() != expected_hashval:
335	n/a	raise HashDoesNotMatch("got %s instead of %s"
336	n/a	% (hashval.hexdigest(), expected_hashval))
337	n/a
338	n/a	def __repr__(self):
339	n/a	if self.release is None:
340	n/a	return "<? ? %s>" % self.dist_type
341	n/a
342	n/a	return "<%s %s %s>" % (
343	n/a	self.release.name, self.release.version, self.dist_type or "")
344	n/a
345	n/a
346	n/a	class ReleasesList(IndexReference):
347	n/a	"""A container of Release.
348	n/a
349	n/a	Provides useful methods and facilities to sort and filter releases.
350	n/a	"""
351	n/a	def __init__(self, name, releases=None, contains_hidden=False, index=None):
352	n/a	self.set_index(index)
353	n/a	self.releases = []
354	n/a	self.name = name
355	n/a	self.contains_hidden = contains_hidden
356	n/a	if releases:
357	n/a	self.add_releases(releases)
358	n/a
359	n/a	def fetch_releases(self):
360	n/a	self._index.get_releases(self.name)
361	n/a	return self.releases
362	n/a
363	n/a	def filter(self, predicate):
364	n/a	"""Filter and return a subset of releases matching the given predicate.
365	n/a	"""
366	n/a	return ReleasesList(self.name, [release for release in self.releases
367	n/a	if predicate.match(release.version)],
368	n/a	index=self._index)
369	n/a
370	n/a	def get_last(self, requirements, prefer_final=None):
371	n/a	"""Return the "last" release, that satisfy the given predicates.
372	n/a
373	n/a	"last" is defined by the version number of the releases, you also could
374	n/a	set prefer_final parameter to True or False to change the order results
375	n/a	"""
376	n/a	predicate = get_version_predicate(requirements)
377	n/a	releases = self.filter(predicate)
378	n/a	if len(releases) == 0:
379	n/a	return None
380	n/a	releases.sort_releases(prefer_final, reverse=True)
381	n/a	return releases[0]
382	n/a
383	n/a	def add_releases(self, releases):
384	n/a	"""Add releases in the release list.
385	n/a
386	n/a	:param: releases is a list of ReleaseInfo objects.
387	n/a	"""
388	n/a	for r in releases:
389	n/a	self.add_release(release=r)
390	n/a
391	n/a	def add_release(self, version=None, dist_type='sdist', release=None,
392	n/a	**dist_args):
393	n/a	"""Add a release to the list.
394	n/a
395	n/a	The release can be passed in the `release` parameter, and in this case,
396	n/a	it will be crawled to extract the useful informations if necessary, or
397	n/a	the release informations can be directly passed in the `version` and
398	n/a	`dist_type` arguments.
399	n/a
400	n/a	Other keywords arguments can be provided, and will be forwarded to the
401	n/a	distribution creation (eg. the arguments of the DistInfo constructor).
402	n/a	"""
403	n/a	if release:
404	n/a	if release.name.lower() != self.name.lower():
405	n/a	raise ValueError("%s is not the same project as %s" %
406	n/a	(release.name, self.name))
407	n/a	version = str(release.version)
408	n/a
409	n/a	if version not in self.get_versions():
410	n/a	# append only if not already exists
411	n/a	self.releases.append(release)
412	n/a	for dist in release.dists.values():
413	n/a	for url in dist.urls:
414	n/a	self.add_release(version, dist.dist_type, **url)
415	n/a	else:
416	n/a	matches = [r for r in self.releases
417	n/a	if str(r.version) == version and r.name == self.name]
418	n/a	if not matches:
419	n/a	release = ReleaseInfo(self.name, version, index=self._index)
420	n/a	self.releases.append(release)
421	n/a	else:
422	n/a	release = matches[0]
423	n/a
424	n/a	release.add_distribution(dist_type=dist_type, **dist_args)
425	n/a
426	n/a	def sort_releases(self, prefer_final=False, reverse=True, args, *kwargs):
427	n/a	"""Sort the results with the given properties.
428	n/a
429	n/a	The `prefer_final` argument can be used to specify if final
430	n/a	distributions (eg. not dev, beta or alpha) would be preferred or not.
431	n/a
432	n/a	Results can be inverted by using `reverse`.
433	n/a
434	n/a	Any other parameter provided will be forwarded to the sorted call. You
435	n/a	cannot redefine the key argument of "sorted" here, as it is used
436	n/a	internally to sort the releases.
437	n/a	"""
438	n/a
439	n/a	sort_by = []
440	n/a	if prefer_final:
441	n/a	sort_by.append("is_final")
442	n/a	sort_by.append("version")
443	n/a
444	n/a	self.releases.sort(
445	n/a	key=lambda i: tuple(getattr(i, arg) for arg in sort_by),
446	n/a	reverse=reverse, args, *kwargs)
447	n/a
448	n/a	def get_release(self, version):
449	n/a	"""Return a release from its version."""
450	n/a	matches = [r for r in self.releases if str(r.version) == version]
451	n/a	if len(matches) != 1:
452	n/a	raise KeyError(version)
453	n/a	return matches[0]
454	n/a
455	n/a	def get_versions(self):
456	n/a	"""Return a list of releases versions contained"""
457	n/a	return [str(r.version) for r in self.releases]
458	n/a
459	n/a	def __getitem__(self, key):
460	n/a	return self.releases[key]
461	n/a
462	n/a	def __len__(self):
463	n/a	return len(self.releases)
464	n/a
465	n/a	def __repr__(self):
466	n/a	string = 'Project "%s"' % self.name
467	n/a	if self.get_versions():
468	n/a	string += ' versions: %s' % ', '.join(self.get_versions())
469	n/a	return '<%s>' % string
470	n/a
471	n/a
472	n/a	def get_infos_from_url(url, probable_dist_name=None, is_external=True):
473	n/a	"""Get useful informations from an URL.
474	n/a
475	n/a	Return a dict of (name, version, url, hashtype, hash, is_external)
476	n/a
477	n/a	:param url: complete url of the distribution
478	n/a	:param probable_dist_name: A probable name of the project.
479	n/a	:param is_external: Tell if the url commes from an index or from
480	n/a	an external URL.
481	n/a	"""
482	n/a	# if the url contains a md5 hash, get it.
483	n/a	md5_hash = None
484	n/a	match = MD5_HASH.match(url)
485	n/a	if match is not None:
486	n/a	md5_hash = match.group(1)
487	n/a	# remove the hash
488	n/a	url = url.replace("#md5=%s" % md5_hash, "")
489	n/a
490	n/a	# parse the archive name to find dist name and version
491	n/a	archive_name = urllib.parse.urlparse(url)[2].split('/')[-1]
492	n/a	extension_matched = False
493	n/a	# remove the extension from the name
494	n/a	for ext in EXTENSIONS:
495	n/a	if archive_name.endswith(ext):
496	n/a	archive_name = archive_name[:-len(ext)]
497	n/a	extension_matched = True
498	n/a
499	n/a	name, version = split_archive_name(archive_name)
500	n/a	if extension_matched is True:
501	n/a	return {'name': name,
502	n/a	'version': version,
503	n/a	'url': url,
504	n/a	'hashname': "md5",
505	n/a	'hashval': md5_hash,
506	n/a	'is_external': is_external,
507	n/a	'dist_type': 'sdist'}
508	n/a
509	n/a
510	n/a	def split_archive_name(archive_name, probable_name=None):
511	n/a	"""Split an archive name into two parts: name and version.
512	n/a
513	n/a	Return the tuple (name, version)
514	n/a	"""
515	n/a	# Try to determine wich part is the name and wich is the version using the
516	n/a	# "-" separator. Take the larger part to be the version number then reduce
517	n/a	# if this not works.
518	n/a	def eager_split(str, maxsplit=2):
519	n/a	# split using the "-" separator
520	n/a	splits = str.rsplit("-", maxsplit)
521	n/a	name = splits[0]
522	n/a	version = "-".join(splits[1:])
523	n/a	if version.startswith("-"):
524	n/a	version = version[1:]
525	n/a	if suggest_normalized_version(version) is None and maxsplit >= 0:
526	n/a	# we dont get a good version number: recurse !
527	n/a	return eager_split(str, maxsplit - 1)
528	n/a	else:
529	n/a	return name, version
530	n/a	if probable_name is not None:
531	n/a	probable_name = probable_name.lower()
532	n/a	name = None
533	n/a	if probable_name is not None and probable_name in archive_name:
534	n/a	# we get the name from probable_name, if given.
535	n/a	name = probable_name
536	n/a	version = archive_name.lstrip(name)
537	n/a	else:
538	n/a	name, version = eager_split(archive_name)
539	n/a
540	n/a	version = suggest_normalized_version(version)
541	n/a	if version is not None and name != "":
542	n/a	return name.lower(), version
543	n/a	else:
544	n/a	raise CantParseArchiveName(archive_name)