Python code coverage for Lib/packaging/pypi/simple.py

#	count	content
1	n/a	"""Spider using the screen-scraping "simple" PyPI API.
2	n/a
3	n/a	This module contains the class Crawler, a simple spider that
4	n/a	can be used to find and retrieve distributions from a project index
5	n/a	(like the Python Package Index), using its so-called simple API (see
6	n/a	reference implementation available at http://pypi.python.org/simple/).
7	n/a	"""
8	n/a
9	n/a	import http.client
10	n/a	import re
11	n/a	import socket
12	n/a	import sys
13	n/a	import urllib.request
14	n/a	import urllib.parse
15	n/a	import urllib.error
16	n/a	import os
17	n/a
18	n/a	from fnmatch import translate
19	n/a	from functools import wraps
20	n/a	from packaging import logger
21	n/a	from packaging.metadata import Metadata
22	n/a	from packaging.version import get_version_predicate
23	n/a	from packaging import __version__ as packaging_version
24	n/a	from packaging.pypi.base import BaseClient
25	n/a	from packaging.pypi.dist import (ReleasesList, EXTENSIONS,
26	n/a	get_infos_from_url, MD5_HASH)
27	n/a	from packaging.pypi.errors import (PackagingPyPIError, DownloadError,
28	n/a	UnableToDownload, CantParseArchiveName,
29	n/a	ReleaseNotFound, ProjectNotFound)
30	n/a	from packaging.pypi.mirrors import get_mirrors
31	n/a
32	n/a	__all__ = ['Crawler', 'DEFAULT_SIMPLE_INDEX_URL']
33	n/a
34	n/a	# -- Constants -----------------------------------------------
35	n/a	DEFAULT_SIMPLE_INDEX_URL = "http://a.pypi.python.org/simple/"
36	n/a	DEFAULT_HOSTS = ("*",)
37	n/a	SOCKET_TIMEOUT = 15
38	n/a	USER_AGENT = "Python-urllib/%s.%s packaging/%s" % (
39	n/a	sys.version_info[0], sys.version_info[1], packaging_version)
40	n/a
41	n/a	# -- Regexps -------------------------------------------------
42	n/a	EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$')
43	n/a	HREF = re.compile("""href\\s=\\s['"]?([^'"> ]+)""", re.I)
44	n/a	URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match
45	n/a
46	n/a	# This pattern matches a character entity reference (a decimal numeric
47	n/a	# references, a hexadecimal numeric reference, or a named reference).
48	n/a	ENTITY_SUB = re.compile(r'&(#(\d+\|x[\da-fA-F]+)\|[\w.:-]+);?').sub
49	n/a	REL = re.compile("""<([^>]\srel\s=\s['"]?([^'">]+)[^>])>""", re.I)
50	n/a
51	n/a
52	n/a	def socket_timeout(timeout=SOCKET_TIMEOUT):
53	n/a	"""Decorator to add a socket timeout when requesting pages on PyPI.
54	n/a	"""
55	n/a	def wrapper(func):
56	n/a	@wraps(func)
57	n/a	def wrapped(self, args, *kwargs):
58	n/a	old_timeout = socket.getdefaulttimeout()
59	n/a	if hasattr(self, "_timeout"):
60	n/a	timeout = self._timeout
61	n/a	socket.setdefaulttimeout(timeout)
62	n/a	try:
63	n/a	return func(self, args, *kwargs)
64	n/a	finally:
65	n/a	socket.setdefaulttimeout(old_timeout)
66	n/a	return wrapped
67	n/a	return wrapper
68	n/a
69	n/a
70	n/a	def with_mirror_support():
71	n/a	"""Decorator that makes the mirroring support easier"""
72	n/a	def wrapper(func):
73	n/a	@wraps(func)
74	n/a	def wrapped(self, args, *kwargs):
75	n/a	try:
76	n/a	return func(self, args, *kwargs)
77	n/a	except DownloadError:
78	n/a	# if an error occurs, try with the next index_url
79	n/a	if self._mirrors_tries >= self._mirrors_max_tries:
80	n/a	try:
81	n/a	self._switch_to_next_mirror()
82	n/a	except KeyError:
83	n/a	raise UnableToDownload("Tried all mirrors")
84	n/a	else:
85	n/a	self._mirrors_tries += 1
86	n/a	self._projects.clear()
87	n/a	return wrapped(self, args, *kwargs)
88	n/a	return wrapped
89	n/a	return wrapper
90	n/a
91	n/a
92	n/a	class Crawler(BaseClient):
93	n/a	"""Provides useful tools to request the Python Package Index simple API.
94	n/a
95	n/a	You can specify both mirrors and mirrors_url, but mirrors_url will only be
96	n/a	used if mirrors is set to None.
97	n/a
98	n/a	:param index_url: the url of the simple index to search on.
99	n/a	:param prefer_final: if the version is not mentioned, and the last
100	n/a	version is not a "final" one (alpha, beta, etc.),
101	n/a	pick up the last final version.
102	n/a	:param prefer_source: if the distribution type is not mentioned, pick up
103	n/a	the source one if available.
104	n/a	:param follow_externals: tell if following external links is needed or
105	n/a	not. Default is False.
106	n/a	:param hosts: a list of hosts allowed to be processed while using
107	n/a	follow_externals=True. Default behavior is to follow all
108	n/a	hosts.
109	n/a	:param follow_externals: tell if following external links is needed or
110	n/a	not. Default is False.
111	n/a	:param mirrors_url: the url to look on for DNS records giving mirror
112	n/a	addresses.
113	n/a	:param mirrors: a list of mirrors (see PEP 381).
114	n/a	:param timeout: time in seconds to consider a url has timeouted.
115	n/a	:param mirrors_max_tries": number of times to try requesting informations
116	n/a	on mirrors before switching.
117	n/a	"""
118	n/a
119	n/a	def __init__(self, index_url=DEFAULT_SIMPLE_INDEX_URL, prefer_final=False,
120	n/a	prefer_source=True, hosts=DEFAULT_HOSTS,
121	n/a	follow_externals=False, mirrors_url=None, mirrors=None,
122	n/a	timeout=SOCKET_TIMEOUT, mirrors_max_tries=0):
123	n/a	super(Crawler, self).__init__(prefer_final, prefer_source)
124	n/a	self.follow_externals = follow_externals
125	n/a
126	n/a	# mirroring attributes.
127	n/a	parsed = urllib.parse.urlparse(index_url)
128	n/a	self.scheme = parsed[0]
129	n/a	if self.scheme == 'file':
130	n/a	ender = os.path.sep
131	n/a	else:
132	n/a	ender = '/'
133	n/a	if not index_url.endswith(ender):
134	n/a	index_url += ender
135	n/a	# if no mirrors are defined, use the method described in PEP 381.
136	n/a	if mirrors is None:
137	n/a	mirrors = get_mirrors(mirrors_url)
138	n/a	self._mirrors = set(mirrors)
139	n/a	self._mirrors_used = set()
140	n/a	self.index_url = index_url
141	n/a	self._mirrors_max_tries = mirrors_max_tries
142	n/a	self._mirrors_tries = 0
143	n/a	self._timeout = timeout
144	n/a
145	n/a	# create a regexp to match all given hosts
146	n/a	self._allowed_hosts = re.compile('\|'.join(map(translate, hosts))).match
147	n/a
148	n/a	# we keep an index of pages we have processed, in order to avoid
149	n/a	# scanning them multple time (eg. if there is multiple pages pointing
150	n/a	# on one)
151	n/a	self._processed_urls = []
152	n/a	self._projects = {}
153	n/a
154	n/a	@with_mirror_support()
155	n/a	def search_projects(self, name=None, **kwargs):
156	n/a	"""Search the index for projects containing the given name.
157	n/a
158	n/a	Return a list of names.
159	n/a	"""
160	n/a	if '*' in name:
161	n/a	name.replace('', '.')
162	n/a	else:
163	n/a	name = "%s%s%s" % ('.?', name, '.?')
164	n/a	name = name.replace('', '[^<]') # avoid matching end tag
165	n/a	pattern = ('<a[^>]*>(%s)</a>' % name).encode('utf-8')
166	n/a	projectname = re.compile(pattern, re.I)
167	n/a	matching_projects = []
168	n/a
169	n/a	with self._open_url(self.index_url) as index:
170	n/a	index_content = index.read()
171	n/a
172	n/a	for match in projectname.finditer(index_content):
173	n/a	project_name = match.group(1).decode('utf-8')
174	n/a	matching_projects.append(self._get_project(project_name))
175	n/a	return matching_projects
176	n/a
177	n/a	def get_releases(self, requirements, prefer_final=None,
178	n/a	force_update=False):
179	n/a	"""Search for releases and return a ReleasesList object containing
180	n/a	the results.
181	n/a	"""
182	n/a	predicate = get_version_predicate(requirements)
183	n/a	if predicate.name.lower() in self._projects and not force_update:
184	n/a	return self._projects.get(predicate.name.lower())
185	n/a	prefer_final = self._get_prefer_final(prefer_final)
186	n/a	logger.debug('Reading info on PyPI about %s', predicate.name)
187	n/a	self._process_index_page(predicate.name)
188	n/a
189	n/a	if predicate.name.lower() not in self._projects:
190	n/a	raise ProjectNotFound
191	n/a
192	n/a	releases = self._projects.get(predicate.name.lower())
193	n/a	releases.sort_releases(prefer_final=prefer_final)
194	n/a	return releases
195	n/a
196	n/a	def get_release(self, requirements, prefer_final=None):
197	n/a	"""Return only one release that fulfill the given requirements"""
198	n/a	predicate = get_version_predicate(requirements)
199	n/a	release = self.get_releases(predicate, prefer_final)\
200	n/a	.get_last(predicate)
201	n/a	if not release:
202	n/a	raise ReleaseNotFound("No release matches the given criterias")
203	n/a	return release
204	n/a
205	n/a	def get_distributions(self, project_name, version):
206	n/a	"""Return the distributions found on the index for the specific given
207	n/a	release"""
208	n/a	# as the default behavior of get_release is to return a release
209	n/a	# containing the distributions, just alias it.
210	n/a	return self.get_release("%s (%s)" % (project_name, version))
211	n/a
212	n/a	def get_metadata(self, project_name, version):
213	n/a	"""Return the metadatas from the simple index.
214	n/a
215	n/a	Currently, download one archive, extract it and use the PKG-INFO file.
216	n/a	"""
217	n/a	release = self.get_distributions(project_name, version)
218	n/a	if not release.metadata:
219	n/a	location = release.get_distribution().unpack()
220	n/a	pkg_info = os.path.join(location, 'PKG-INFO')
221	n/a	release.metadata = Metadata(pkg_info)
222	n/a	return release
223	n/a
224	n/a	def _switch_to_next_mirror(self):
225	n/a	"""Switch to the next mirror (eg. point self.index_url to the next
226	n/a	mirror url.
227	n/a
228	n/a	Raise a KeyError if all mirrors have been tried.
229	n/a	"""
230	n/a	self._mirrors_used.add(self.index_url)
231	n/a	index_url = self._mirrors.pop()
232	n/a	# XXX use urllib.parse for a real check of missing scheme part
233	n/a	if not index_url.startswith(("http://", "https://", "file://")):
234	n/a	index_url = "http://%s" % index_url
235	n/a
236	n/a	if not index_url.endswith("/simple"):
237	n/a	index_url = "%s/simple/" % index_url
238	n/a
239	n/a	self.index_url = index_url
240	n/a
241	n/a	def _is_browsable(self, url):
242	n/a	"""Tell if the given URL can be browsed or not.
243	n/a
244	n/a	It uses the follow_externals and the hosts list to tell if the given
245	n/a	url is browsable or not.
246	n/a	"""
247	n/a	# if _index_url is contained in the given URL, we are browsing the
248	n/a	# index, and it's always "browsable".
249	n/a	# local files are always considered browable resources
250	n/a	if self.index_url in url or urllib.parse.urlparse(url)[0] == "file":
251	n/a	return True
252	n/a	elif self.follow_externals:
253	n/a	if self._allowed_hosts(urllib.parse.urlparse(url)[1]): # 1 is netloc
254	n/a	return True
255	n/a	else:
256	n/a	return False
257	n/a	return False
258	n/a
259	n/a	def _is_distribution(self, link):
260	n/a	"""Tell if the given URL matches to a distribution name or not.
261	n/a	"""
262	n/a	#XXX find a better way to check that links are distributions
263	n/a	# Using a regexp ?
264	n/a	for ext in EXTENSIONS:
265	n/a	if ext in link:
266	n/a	return True
267	n/a	return False
268	n/a
269	n/a	def _register_release(self, release=None, release_info={}):
270	n/a	"""Register a new release.
271	n/a
272	n/a	Both a release or a dict of release_info can be provided, the preferred
273	n/a	way (eg. the quicker) is the dict one.
274	n/a
275	n/a	Return the list of existing releases for the given project.
276	n/a	"""
277	n/a	# Check if the project already has a list of releases (refering to
278	n/a	# the project name). If not, create a new release list.
279	n/a	# Then, add the release to the list.
280	n/a	if release:
281	n/a	name = release.name
282	n/a	else:
283	n/a	name = release_info['name']
284	n/a	if name.lower() not in self._projects:
285	n/a	self._projects[name.lower()] = ReleasesList(name, index=self._index)
286	n/a
287	n/a	if release:
288	n/a	self._projects[name.lower()].add_release(release=release)
289	n/a	else:
290	n/a	name = release_info.pop('name')
291	n/a	version = release_info.pop('version')
292	n/a	dist_type = release_info.pop('dist_type')
293	n/a	self._projects[name.lower()].add_release(version, dist_type,
294	n/a	**release_info)
295	n/a	return self._projects[name.lower()]
296	n/a
297	n/a	def _process_url(self, url, project_name=None, follow_links=True):
298	n/a	"""Process an url and search for distributions packages.
299	n/a
300	n/a	For each URL found, if it's a download, creates a PyPIdistribution
301	n/a	object. If it's a homepage and we can follow links, process it too.
302	n/a
303	n/a	:param url: the url to process
304	n/a	:param project_name: the project name we are searching for.
305	n/a	:param follow_links: Do not want to follow links more than from one
306	n/a	level. This parameter tells if we want to follow
307	n/a	the links we find (eg. run recursively this
308	n/a	method on it)
309	n/a	"""
310	n/a	with self._open_url(url) as f:
311	n/a	base_url = f.url
312	n/a	if url not in self._processed_urls:
313	n/a	self._processed_urls.append(url)
314	n/a	link_matcher = self._get_link_matcher(url)
315	n/a	for link, is_download in link_matcher(f.read().decode(), base_url):
316	n/a	if link not in self._processed_urls:
317	n/a	if self._is_distribution(link) or is_download:
318	n/a	self._processed_urls.append(link)
319	n/a	# it's a distribution, so create a dist object
320	n/a	try:
321	n/a	infos = get_infos_from_url(link, project_name,
322	n/a	is_external=self.index_url not in url)
323	n/a	except CantParseArchiveName as e:
324	n/a	logger.warning(
325	n/a	"version has not been parsed: %s", e)
326	n/a	else:
327	n/a	self._register_release(release_info=infos)
328	n/a	else:
329	n/a	if self._is_browsable(link) and follow_links:
330	n/a	self._process_url(link, project_name,
331	n/a	follow_links=False)
332	n/a
333	n/a	def _get_link_matcher(self, url):
334	n/a	"""Returns the right link matcher function of the given url
335	n/a	"""
336	n/a	if self.index_url in url:
337	n/a	return self._simple_link_matcher
338	n/a	else:
339	n/a	return self._default_link_matcher
340	n/a
341	n/a	def _get_full_url(self, url, base_url):
342	n/a	return urllib.parse.urljoin(base_url, self._htmldecode(url))
343	n/a
344	n/a	def _simple_link_matcher(self, content, base_url):
345	n/a	"""Yield all links with a rel="download" or rel="homepage".
346	n/a
347	n/a	This matches the simple index requirements for matching links.
348	n/a	If follow_externals is set to False, dont yeld the external
349	n/a	urls.
350	n/a
351	n/a	:param content: the content of the page we want to parse
352	n/a	:param base_url: the url of this page.
353	n/a	"""
354	n/a	for match in HREF.finditer(content):
355	n/a	url = self._get_full_url(match.group(1), base_url)
356	n/a	if MD5_HASH.match(url):
357	n/a	yield (url, True)
358	n/a
359	n/a	for match in REL.finditer(content):
360	n/a	# search for rel links.
361	n/a	tag, rel = match.groups()
362	n/a	rels = [s.strip() for s in rel.lower().split(',')]
363	n/a	if 'homepage' in rels or 'download' in rels:
364	n/a	for match in HREF.finditer(tag):
365	n/a	url = self._get_full_url(match.group(1), base_url)
366	n/a	if 'download' in rels or self._is_browsable(url):
367	n/a	# yield a list of (url, is_download)
368	n/a	yield (url, 'download' in rels)
369	n/a
370	n/a	def _default_link_matcher(self, content, base_url):
371	n/a	"""Yield all links found on the page.
372	n/a	"""
373	n/a	for match in HREF.finditer(content):
374	n/a	url = self._get_full_url(match.group(1), base_url)
375	n/a	if self._is_browsable(url):
376	n/a	yield (url, False)
377	n/a
378	n/a	@with_mirror_support()
379	n/a	def _process_index_page(self, name):
380	n/a	"""Find and process a PyPI page for the given project name.
381	n/a
382	n/a	:param name: the name of the project to find the page
383	n/a	"""
384	n/a	# Browse and index the content of the given PyPI page.
385	n/a	if self.scheme == 'file':
386	n/a	ender = os.path.sep
387	n/a	else:
388	n/a	ender = '/'
389	n/a	url = self.index_url + name + ender
390	n/a	self._process_url(url, name)
391	n/a
392	n/a	@socket_timeout()
393	n/a	def _open_url(self, url):
394	n/a	"""Open a urllib2 request, handling HTTP authentication, and local
395	n/a	files support.
396	n/a
397	n/a	"""
398	n/a	scheme, netloc, path, params, query, frag = urllib.parse.urlparse(url)
399	n/a
400	n/a	# authentication stuff
401	n/a	if scheme in ('http', 'https'):
402	n/a	auth, host = urllib.parse.splituser(netloc)
403	n/a	else:
404	n/a	auth = None
405	n/a
406	n/a	# add index.html automatically for filesystem paths
407	n/a	if scheme == 'file':
408	n/a	if url.endswith(os.path.sep):
409	n/a	url += "index.html"
410	n/a
411	n/a	# add authorization headers if auth is provided
412	n/a	if auth:
413	n/a	auth = "Basic " + \
414	n/a	urllib.parse.unquote(auth).encode('base64').strip()
415	n/a	new_url = urllib.parse.urlunparse((
416	n/a	scheme, host, path, params, query, frag))
417	n/a	request = urllib.request.Request(new_url)
418	n/a	request.add_header("Authorization", auth)
419	n/a	else:
420	n/a	request = urllib.request.Request(url)
421	n/a	request.add_header('User-Agent', USER_AGENT)
422	n/a	try:
423	n/a	fp = urllib.request.urlopen(request)
424	n/a	except (ValueError, http.client.InvalidURL) as v:
425	n/a	msg = ' '.join([str(arg) for arg in v.args])
426	n/a	raise PackagingPyPIError('%s %s' % (url, msg))
427	n/a	except urllib.error.HTTPError as v:
428	n/a	return v
429	n/a	except urllib.error.URLError as v:
430	n/a	raise DownloadError("Download error for %s: %s" % (url, v.reason))
431	n/a	except http.client.BadStatusLine as v:
432	n/a	raise DownloadError('%s returned a bad status line. '
433	n/a	'The server might be down, %s' % (url, v.line))
434	n/a	except http.client.HTTPException as v:
435	n/a	raise DownloadError("Download error for %s: %s" % (url, v))
436	n/a	except socket.timeout:
437	n/a	raise DownloadError("The server timeouted")
438	n/a
439	n/a	if auth:
440	n/a	# Put authentication info back into request URL if same host,
441	n/a	# so that links found on the page will work
442	n/a	s2, h2, path2, param2, query2, frag2 = \
443	n/a	urllib.parse.urlparse(fp.url)
444	n/a	if s2 == scheme and h2 == host:
445	n/a	fp.url = urllib.parse.urlunparse(
446	n/a	(s2, netloc, path2, param2, query2, frag2))
447	n/a	return fp
448	n/a
449	n/a	def _decode_entity(self, match):
450	n/a	what = match.group(1)
451	n/a	if what.startswith('#x'):
452	n/a	what = int(what[2:], 16)
453	n/a	elif what.startswith('#'):
454	n/a	what = int(what[1:])
455	n/a	else:
456	n/a	from html.entities import name2codepoint
457	n/a	what = name2codepoint.get(what, match.group(0))
458	n/a	return chr(what)
459	n/a
460	n/a	def _htmldecode(self, text):
461	n/a	"""Decode HTML entities in the given text."""
462	n/a	return ENTITY_SUB(self._decode_entity, text)