Python code coverage for Lib/urllib.py

#	count	content
1	n/a	"""Open an arbitrary URL.
2	n/a
3	n/a	See the following document for more info on URLs:
4	n/a	"Names and Addresses, URIs, URLs, URNs, URCs", at
5	n/a	http://www.w3.org/pub/WWW/Addressing/Overview.html
6	n/a
7	n/a	See also the HTTP spec (from which the error codes are derived):
8	n/a	"HTTP - Hypertext Transfer Protocol", at
9	n/a	http://www.w3.org/pub/WWW/Protocols/
10	n/a
11	n/a	Related standards and specs:
12	n/a	- RFC1808: the "relative URL" spec. (authoritative status)
13	n/a	- RFC1738 - the "URL standard". (authoritative status)
14	n/a	- RFC1630 - the "URI spec". (informational status)
15	n/a
16	n/a	The object returned by URLopener().open(file) will differ per
17	n/a	protocol. All you know is that is has methods read(), readline(),
18	n/a	readlines(), fileno(), close() and info(). The read*(), fileno()
19	n/a	and close() methods work like those of open files.
20	n/a	The info() method returns a mimetools.Message object which can be
21	n/a	used to query various info about the object, if available.
22	n/a	(mimetools.Message objects are queried with the getheader() method.)
23	1	"""
24	n/a
25	1	import string
26	1	import socket
27	1	import os
28	1	import time
29	1	import sys
30	1	from urlparse import urljoin as basejoin
31	n/a
32	1	__all__ = ["urlopen", "URLopener", "FancyURLopener", "urlretrieve",
33	1	"urlcleanup", "quote", "quote_plus", "unquote", "unquote_plus",
34	1	"urlencode", "url2pathname", "pathname2url", "splittag",
35	1	"localhost", "thishost", "ftperrors", "basejoin", "unwrap",
36	1	"splittype", "splithost", "splituser", "splitpasswd", "splitport",
37	1	"splitnport", "splitquery", "splitattr", "splitvalue",
38	1	"getproxies"]
39	n/a
40	1	__version__ = '1.17' # XXX This version is not always updated :-(
41	n/a
42	1	MAXFTPCACHE = 10 # Trim the ftp cache beyond this size
43	n/a
44	n/a	# Helper for non-unix systems
45	1	if os.name == 'nt':
46	0	from nturl2path import url2pathname, pathname2url
47	1	elif os.name == 'riscos':
48	0	from rourl2path import url2pathname, pathname2url
49	n/a	else:
50	1	def url2pathname(pathname):
51	n/a	"""OS-specific conversion from a relative URL of the 'file' scheme
52	n/a	to a file system path; not recommended for general use."""
53	35	return unquote(pathname)
54	n/a
55	1	def pathname2url(pathname):
56	n/a	"""OS-specific conversion from a file system path to a relative URL
57	n/a	of the 'file' scheme; not recommended for general use."""
58	10	return quote(pathname)
59	n/a
60	n/a	# This really consists of two pieces:
61	n/a	# (1) a class which handles opening of all sorts of URLs
62	n/a	# (plus assorted utilities etc.)
63	n/a	# (2) a set of functions for parsing URLs
64	n/a	# XXX Should these be separated out into different modules?
65	n/a
66	n/a
67	n/a	# Shortcut for basic usage
68	1	_urlopener = None
69	1	def urlopen(url, data=None, proxies=None):
70	n/a	"""Create a file-like object for the specified URL to read from."""
71	21	from warnings import warnpy3k
72	21	warnpy3k("urllib.urlopen() has been removed in Python 3.0 in "
73	21	"favor of urllib2.urlopen()", stacklevel=2)
74	n/a
75	n/a	global _urlopener
76	21	if proxies is not None:
77	0	opener = FancyURLopener(proxies=proxies)
78	21	elif not _urlopener:
79	1	opener = FancyURLopener()
80	1	_urlopener = opener
81	n/a	else:
82	20	opener = _urlopener
83	21	if data is None:
84	21	return opener.open(url)
85	n/a	else:
86	0	return opener.open(url, data)
87	1	def urlretrieve(url, filename=None, reporthook=None, data=None):
88	n/a	global _urlopener
89	10	if not _urlopener:
90	0	_urlopener = FancyURLopener()
91	10	return _urlopener.retrieve(url, filename, reporthook, data)
92	1	def urlcleanup():
93	0	if _urlopener:
94	0	_urlopener.cleanup()
95	0	_safe_quoters.clear()
96	0	ftpcache.clear()
97	n/a
98	n/a	# check for SSL
99	1	try:
100	1	import ssl
101	0	except:
102	0	_have_ssl = False
103	n/a	else:
104	1	_have_ssl = True
105	n/a
106	n/a	# exception raised when downloaded size does not match content-length
107	2	class ContentTooShortError(IOError):
108	1	def __init__(self, message, content):
109	0	IOError.__init__(self, message)
110	0	self.content = content
111	n/a
112	1	ftpcache = {}
113	2	class URLopener:
114	n/a	"""Class to open URLs.
115	n/a	This is a class rather than just a subroutine because we may need
116	n/a	more than one set of global protocol-specific options.
117	n/a	Note -- this is a base class for those who don't want the
118	n/a	automatic handling of errors type 302 (relocated) and 401
119	1	(authorization needed)."""
120	n/a
121	1	__tempfiles = None
122	n/a
123	1	version = "Python-urllib/%s" % __version__
124	n/a
125	n/a	# Constructor
126	1	def __init__(self, proxies=None, **x509):
127	6	if proxies is None:
128	6	proxies = getproxies()
129	6	assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
130	6	self.proxies = proxies
131	6	self.key_file = x509.get('key_file')
132	6	self.cert_file = x509.get('cert_file')
133	6	self.addheaders = [('User-Agent', self.version)]
134	6	self.__tempfiles = []
135	6	self.__unlink = os.unlink # See cleanup()
136	6	self.tempcache = None
137	n/a	# Undocumented feature: if you assign {} to tempcache,
138	n/a	# it is used to cache files retrieved with
139	n/a	# self.retrieve(). This is not enabled by default
140	n/a	# since it does not work for changing documents (and I
141	n/a	# haven't got the logic to check expiration headers
142	n/a	# yet).
143	6	self.ftpcache = ftpcache
144	n/a	# Undocumented feature: you can use a different
145	n/a	# ftp cache by assigning to the .ftpcache member;
146	n/a	# in case you want logically independent URL openers
147	n/a	# XXX This is not threadsafe. Bah.
148	n/a
149	1	def __del__(self):
150	5	self.close()
151	n/a
152	1	def close(self):
153	5	self.cleanup()
154	n/a
155	1	def cleanup(self):
156	n/a	# This code sometimes runs when the rest of this module
157	n/a	# has already been deleted, so it can't use any globals
158	n/a	# or import anything.
159	5	if self.__tempfiles:
160	0	for file in self.__tempfiles:
161	0	try:
162	0	self.__unlink(file)
163	0	except OSError:
164	0	pass
165	0	del self.__tempfiles[:]
166	5	if self.tempcache:
167	0	self.tempcache.clear()
168	n/a
169	1	def addheader(self, *args):
170	n/a	"""Add a header to be used by the HTTP interface only
171	n/a	e.g. u.addheader('Accept', 'sound/basic')"""
172	0	self.addheaders.append(args)
173	n/a
174	n/a	# External interface
175	1	def open(self, fullurl, data=None):
176	n/a	"""Use URLopener().open(file) instead of open(file, 'r')."""
177	35	fullurl = unwrap(toBytes(fullurl))
178	n/a	# percent encode url, fixing lame server errors for e.g, like space
179	n/a	# within url paths.
180	35	fullurl = quote(fullurl, safe="%/:=&?~#+!$,;'@()*[]\|")
181	35	if self.tempcache and fullurl in self.tempcache:
182	0	filename, headers = self.tempcache[fullurl]
183	0	fp = open(filename, 'rb')
184	0	return addinfourl(fp, headers, fullurl)
185	35	urltype, url = splittype(fullurl)
186	35	if not urltype:
187	0	urltype = 'file'
188	35	if urltype in self.proxies:
189	0	proxy = self.proxies[urltype]
190	0	urltype, proxyhost = splittype(proxy)
191	0	host, selector = splithost(proxyhost)
192	0	url = (host, fullurl) # Signal special case to open_*()
193	n/a	else:
194	35	proxy = None
195	35	name = 'open_' + urltype
196	35	self.type = urltype
197	35	name = name.replace('-', '_')
198	35	if not hasattr(self, name):
199	0	if proxy:
200	0	return self.open_unknown_proxy(proxy, fullurl, data)
201	n/a	else:
202	0	return self.open_unknown(fullurl, data)
203	35	try:
204	35	if data is None:
205	35	return getattr(self, name)(url)
206	n/a	else:
207	0	return getattr(self, name)(url, data)
208	3	except socket.error, msg:
209	1	raise IOError, ('socket error', msg), sys.exc_info()[2]
210	n/a
211	1	def open_unknown(self, fullurl, data=None):
212	n/a	"""Overridable interface to open unknown URL type."""
213	0	type, url = splittype(fullurl)
214	0	raise IOError, ('url error', 'unknown url type', type)
215	n/a
216	1	def open_unknown_proxy(self, proxy, fullurl, data=None):
217	n/a	"""Overridable interface to open unknown URL type."""
218	0	type, url = splittype(fullurl)
219	0	raise IOError, ('url error', 'invalid proxy for %s' % type, proxy)
220	n/a
221	n/a	# External interface
222	1	def retrieve(self, url, filename=None, reporthook=None, data=None):
223	n/a	"""retrieve(url) returns (filename, headers) for a local object
224	n/a	or (tempfilename, headers) for a remote object."""
225	10	url = unwrap(toBytes(url))
226	10	if self.tempcache and url in self.tempcache:
227	0	return self.tempcache[url]
228	10	type, url1 = splittype(url)
229	10	if filename is None and (not type or type == 'file'):
230	1	try:
231	1	fp = self.open_local_file(url1)
232	1	hdrs = fp.info()
233	1	fp.close()
234	1	return url2pathname(splithost(url1)[1]), hdrs
235	0	except IOError:
236	0	pass
237	9	fp = self.open(url, data)
238	9	try:
239	9	headers = fp.info()
240	9	if filename:
241	6	tfp = open(filename, 'wb')
242	n/a	else:
243	3	import tempfile
244	3	garbage, path = splittype(url)
245	3	garbage, path = splithost(path or "")
246	3	path, garbage = splitquery(path or "")
247	3	path, garbage = splitattr(path or "")
248	3	suffix = os.path.splitext(path)[1]
249	3	(fd, filename) = tempfile.mkstemp(suffix)
250	3	self.__tempfiles.append(filename)
251	3	tfp = os.fdopen(fd, 'wb')
252	9	try:
253	9	result = filename, headers
254	9	if self.tempcache is not None:
255	0	self.tempcache[url] = result
256	9	bs = 1024*8
257	9	size = -1
258	9	read = 0
259	9	blocknum = 0
260	9	if reporthook:
261	4	if "content-length" in headers:
262	4	size = int(headers["Content-Length"])
263	4	reporthook(blocknum, bs, size)
264	9	while 1:
265	34	block = fp.read(bs)
266	34	if block == "":
267	9	break
268	25	read += len(block)
269	25	tfp.write(block)
270	25	blocknum += 1
271	25	if reporthook:
272	4	reporthook(blocknum, bs, size)
273	n/a	finally:
274	9	tfp.close()
275	n/a	finally:
276	9	fp.close()
277	n/a
278	n/a	# raise exception if actual size does not match content-length header
279	9	if size >= 0 and read < size:
280	0	raise ContentTooShortError("retrieval incomplete: got only %i out "
281	0	"of %i bytes" % (read, size), result)
282	n/a
283	9	return result
284	n/a
285	n/a	# Each method named open_<type> knows how to open that type of URL
286	n/a
287	1	def open_http(self, url, data=None):
288	n/a	"""Use HTTP protocol."""
289	17	import httplib
290	17	user_passwd = None
291	17	proxy_passwd= None
292	17	if isinstance(url, str):
293	17	host, selector = splithost(url)
294	17	if host:
295	17	user_passwd, host = splituser(host)
296	17	host = unquote(host)
297	17	realhost = host
298	n/a	else:
299	0	host, selector = url
300	n/a	# check whether the proxy contains authorization information
301	0	proxy_passwd, host = splituser(host)
302	n/a	# now we proceed with the url we want to obtain
303	0	urltype, rest = splittype(selector)
304	0	url = rest
305	0	user_passwd = None
306	0	if urltype.lower() != 'http':
307	0	realhost = None
308	n/a	else:
309	0	realhost, rest = splithost(rest)
310	0	if realhost:
311	0	user_passwd, realhost = splituser(realhost)
312	0	if user_passwd:
313	0	selector = "%s://%s%s" % (urltype, realhost, rest)
314	0	if proxy_bypass(realhost):
315	0	host = realhost
316	n/a
317	n/a	#print "proxy via http:", host, selector
318	17	if not host: raise IOError, ('http error', 'no host given')
319	n/a
320	17	if proxy_passwd:
321	0	import base64
322	0	proxy_auth = base64.b64encode(proxy_passwd).strip()
323	n/a	else:
324	17	proxy_auth = None
325	n/a
326	17	if user_passwd:
327	0	import base64
328	0	auth = base64.b64encode(user_passwd).strip()
329	n/a	else:
330	17	auth = None
331	17	h = httplib.HTTP(host)
332	17	if data is not None:
333	0	h.putrequest('POST', selector)
334	0	h.putheader('Content-Type', 'application/x-www-form-urlencoded')
335	0	h.putheader('Content-Length', '%d' % len(data))
336	n/a	else:
337	17	h.putrequest('GET', selector)
338	17	if proxy_auth: h.putheader('Proxy-Authorization', 'Basic %s' % proxy_auth)
339	17	if auth: h.putheader('Authorization', 'Basic %s' % auth)
340	17	if realhost: h.putheader('Host', realhost)
341	34	for args in self.addheaders: h.putheader(*args)
342	17	h.endheaders(data)
343	16	errcode, errmsg, headers = h.getreply()
344	16	fp = h.getfile()
345	16	if errcode == -1:
346	1	if fp: fp.close()
347	n/a	# something went wrong with the HTTP status line
348	1	raise IOError, ('http protocol error', 0,
349	1	'got a bad status line', None)
350	n/a	# According to RFC 2616, "2xx" code indicates that the client's
351	n/a	# request was successfully received, understood, and accepted.
352	15	if (200 <= errcode < 300):
353	12	return addinfourl(fp, headers, "http:" + url, errcode)
354	n/a	else:
355	3	if data is None:
356	3	return self.http_error(url, fp, errcode, errmsg, headers)
357	n/a	else:
358	0	return self.http_error(url, fp, errcode, errmsg, headers, data)
359	n/a
360	1	def http_error(self, url, fp, errcode, errmsg, headers, data=None):
361	n/a	"""Handle http errors.
362	n/a	Derived class can override this, or provide specific handlers
363	n/a	named http_error_DDD where DDD is the 3-digit error code."""
364	n/a	# First check if there's a specific handler for this error
365	3	name = 'http_error_%d' % errcode
366	3	if hasattr(self, name):
367	2	method = getattr(self, name)
368	2	if data is None:
369	2	result = method(url, fp, errcode, errmsg, headers)
370	n/a	else:
371	0	result = method(url, fp, errcode, errmsg, headers, data)
372	1	if result: return result
373	2	return self.http_error_default(url, fp, errcode, errmsg, headers)
374	n/a
375	1	def http_error_default(self, url, fp, errcode, errmsg, headers):
376	n/a	"""Default error handler: close the connection and raise IOError."""
377	1	void = fp.read()
378	1	fp.close()
379	1	raise IOError, ('http error', errcode, errmsg, headers)
380	n/a
381	1	if _have_ssl:
382	1	def open_https(self, url, data=None):
383	n/a	"""Use HTTPS protocol."""
384	n/a
385	1	import httplib
386	1	user_passwd = None
387	1	proxy_passwd = None
388	1	if isinstance(url, str):
389	1	host, selector = splithost(url)
390	1	if host:
391	1	user_passwd, host = splituser(host)
392	1	host = unquote(host)
393	1	realhost = host
394	n/a	else:
395	0	host, selector = url
396	n/a	# here, we determine, whether the proxy contains authorization information
397	0	proxy_passwd, host = splituser(host)
398	0	urltype, rest = splittype(selector)
399	0	url = rest
400	0	user_passwd = None
401	0	if urltype.lower() != 'https':
402	0	realhost = None
403	n/a	else:
404	0	realhost, rest = splithost(rest)
405	0	if realhost:
406	0	user_passwd, realhost = splituser(realhost)
407	0	if user_passwd:
408	0	selector = "%s://%s%s" % (urltype, realhost, rest)
409	n/a	#print "proxy via https:", host, selector
410	1	if not host: raise IOError, ('https error', 'no host given')
411	1	if proxy_passwd:
412	0	import base64
413	0	proxy_auth = base64.b64encode(proxy_passwd).strip()
414	n/a	else:
415	1	proxy_auth = None
416	1	if user_passwd:
417	0	import base64
418	0	auth = base64.b64encode(user_passwd).strip()
419	n/a	else:
420	1	auth = None
421	1	h = httplib.HTTPS(host, 0,
422	1	key_file=self.key_file,
423	1	cert_file=self.cert_file)
424	1	if data is not None:
425	0	h.putrequest('POST', selector)
426	0	h.putheader('Content-Type',
427	0	'application/x-www-form-urlencoded')
428	0	h.putheader('Content-Length', '%d' % len(data))
429	n/a	else:
430	1	h.putrequest('GET', selector)
431	1	if proxy_auth: h.putheader('Proxy-Authorization', 'Basic %s' % proxy_auth)
432	1	if auth: h.putheader('Authorization', 'Basic %s' % auth)
433	1	if realhost: h.putheader('Host', realhost)
434	2	for args in self.addheaders: h.putheader(*args)
435	1	h.endheaders(data)
436	1	errcode, errmsg, headers = h.getreply()
437	1	fp = h.getfile()
438	1	if errcode == -1:
439	0	if fp: fp.close()
440	n/a	# something went wrong with the HTTP status line
441	0	raise IOError, ('http protocol error', 0,
442	0	'got a bad status line', None)
443	n/a	# According to RFC 2616, "2xx" code indicates that the client's
444	n/a	# request was successfully received, understood, and accepted.
445	1	if (200 <= errcode < 300):
446	1	return addinfourl(fp, headers, "https:" + url, errcode)
447	n/a	else:
448	0	if data is None:
449	0	return self.http_error(url, fp, errcode, errmsg, headers)
450	n/a	else:
451	0	return self.http_error(url, fp, errcode, errmsg, headers,
452	0	data)
453	n/a
454	1	def open_file(self, url):
455	n/a	"""Use local file or FTP depending on form of URL."""
456	15	if not isinstance(url, str):
457	0	raise IOError, ('file error', 'proxy support for file protocol currently not implemented')
458	15	if url[:2] == '//' and url[2:3] != '/' and url[2:12].lower() != 'localhost/':
459	0	return self.open_ftp(url)
460	n/a	else:
461	15	return self.open_local_file(url)
462	n/a
463	1	def open_local_file(self, url):
464	n/a	"""Use local file."""
465	16	import mimetypes, mimetools, email.utils
466	16	try:
467	16	from cStringIO import StringIO
468	0	except ImportError:
469	0	from StringIO import StringIO
470	16	host, file = splithost(url)
471	16	localname = url2pathname(file)
472	16	try:
473	16	stats = os.stat(localname)
474	0	except OSError, e:
475	0	raise IOError(e.errno, e.strerror, e.filename)
476	16	size = stats.st_size
477	16	modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
478	16	mtype = mimetypes.guess_type(url)[0]
479	16	headers = mimetools.Message(StringIO(
480	16	'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' %
481	16	(mtype or 'text/plain', size, modified)))
482	16	if not host:
483	16	urlfile = file
484	16	if file[:1] == '/':
485	5	urlfile = 'file://' + file
486	16	return addinfourl(open(localname, 'rb'),
487	16	headers, urlfile)
488	0	host, port = splitport(host)
489	0	if not port \
490	0	and socket.gethostbyname(host) in (localhost(), thishost()):
491	0	urlfile = file
492	0	if file[:1] == '/':
493	0	urlfile = 'file://' + file
494	0	return addinfourl(open(localname, 'rb'),
495	0	headers, urlfile)
496	0	raise IOError, ('local file error', 'not on local host')
497	n/a
498	1	def open_ftp(self, url):
499	n/a	"""Use FTP protocol."""
500	0	if not isinstance(url, str):
501	0	raise IOError, ('ftp error', 'proxy support for ftp protocol currently not implemented')
502	0	import mimetypes, mimetools
503	0	try:
504	0	from cStringIO import StringIO
505	0	except ImportError:
506	0	from StringIO import StringIO
507	0	host, path = splithost(url)
508	0	if not host: raise IOError, ('ftp error', 'no host given')
509	0	host, port = splitport(host)
510	0	user, host = splituser(host)
511	0	if user: user, passwd = splitpasswd(user)
512	0	else: passwd = None
513	0	host = unquote(host)
514	0	user = unquote(user or '')
515	0	passwd = unquote(passwd or '')
516	0	host = socket.gethostbyname(host)
517	0	if not port:
518	0	import ftplib
519	0	port = ftplib.FTP_PORT
520	n/a	else:
521	0	port = int(port)
522	0	path, attrs = splitattr(path)
523	0	path = unquote(path)
524	0	dirs = path.split('/')
525	0	dirs, file = dirs[:-1], dirs[-1]
526	0	if dirs and not dirs[0]: dirs = dirs[1:]
527	0	if dirs and not dirs[0]: dirs[0] = '/'
528	0	key = user, host, port, '/'.join(dirs)
529	n/a	# XXX thread unsafe!
530	0	if len(self.ftpcache) > MAXFTPCACHE:
531	n/a	# Prune the cache, rather arbitrarily
532	0	for k in self.ftpcache.keys():
533	0	if k != key:
534	0	v = self.ftpcache[k]
535	0	del self.ftpcache[k]
536	0	v.close()
537	0	try:
538	0	if not key in self.ftpcache:
539	n/a	self.ftpcache[key] = \
540	0	ftpwrapper(user, passwd, host, port, dirs)
541	0	if not file: type = 'D'
542	0	else: type = 'I'
543	0	for attr in attrs:
544	0	attr, value = splitvalue(attr)
545	0	if attr.lower() == 'type' and \
546	0	value in ('a', 'A', 'i', 'I', 'd', 'D'):
547	0	type = value.upper()
548	0	(fp, retrlen) = self.ftpcache[key].retrfile(file, type)
549	0	mtype = mimetypes.guess_type("ftp:" + url)[0]
550	0	headers = ""
551	0	if mtype:
552	0	headers += "Content-Type: %s\n" % mtype
553	0	if retrlen is not None and retrlen >= 0:
554	0	headers += "Content-Length: %d\n" % retrlen
555	0	headers = mimetools.Message(StringIO(headers))
556	0	return addinfourl(fp, headers, "ftp:" + url)
557	0	except ftperrors(), msg:
558	0	raise IOError, ('ftp error', msg), sys.exc_info()[2]
559	n/a
560	1	def open_data(self, url, data=None):
561	n/a	"""Use "data" URL."""
562	0	if not isinstance(url, str):
563	0	raise IOError, ('data error', 'proxy support for data protocol currently not implemented')
564	n/a	# ignore POSTed data
565	n/a	#
566	n/a	# syntax of data URLs:
567	n/a	# dataurl := "data:" [ mediatype ] [ ";base64" ] "," data
568	n/a	# mediatype := [ type "/" subtype ] *( ";" parameter )
569	n/a	# data := *urlchar
570	n/a	# parameter := attribute "=" value
571	0	import mimetools
572	0	try:
573	0	from cStringIO import StringIO
574	0	except ImportError:
575	0	from StringIO import StringIO
576	0	try:
577	0	[type, data] = url.split(',', 1)
578	0	except ValueError:
579	0	raise IOError, ('data error', 'bad data URL')
580	0	if not type:
581	0	type = 'text/plain;charset=US-ASCII'
582	0	semi = type.rfind(';')
583	0	if semi >= 0 and '=' not in type[semi:]:
584	0	encoding = type[semi+1:]
585	0	type = type[:semi]
586	n/a	else:
587	0	encoding = ''
588	0	msg = []
589	0	msg.append('Date: %s'%time.strftime('%a, %d %b %Y %H:%M:%S GMT',
590	0	time.gmtime(time.time())))
591	0	msg.append('Content-type: %s' % type)
592	0	if encoding == 'base64':
593	0	import base64
594	0	data = base64.decodestring(data)
595	n/a	else:
596	0	data = unquote(data)
597	0	msg.append('Content-Length: %d' % len(data))
598	0	msg.append('')
599	0	msg.append(data)
600	0	msg = '\n'.join(msg)
601	0	f = StringIO(msg)
602	0	headers = mimetools.Message(f, 0)
603	n/a	#f.fileno = None # needed for addinfourl
604	0	return addinfourl(f, headers, url)
605	n/a
606	n/a
607	2	class FancyURLopener(URLopener):
608	1	"""Derived class with handlers for errors we can handle (perhaps)."""
609	n/a
610	1	def __init__(self, args, *kwargs):
611	4	URLopener.__init__(self, args, *kwargs)
612	4	self.auth_cache = {}
613	4	self.tries = 0
614	4	self.maxtries = 10
615	n/a
616	1	def http_error_default(self, url, fp, errcode, errmsg, headers):
617	n/a	"""Default error handling -- don't raise an exception."""
618	2	return addinfourl(fp, headers, "http:" + url, errcode)
619	n/a
620	1	def http_error_302(self, url, fp, errcode, errmsg, headers, data=None):
621	n/a	"""Error 302 -- relocated (temporarily)."""
622	0	self.tries += 1
623	0	if self.maxtries and self.tries >= self.maxtries:
624	0	if hasattr(self, "http_error_500"):
625	0	meth = self.http_error_500
626	n/a	else:
627	0	meth = self.http_error_default
628	0	self.tries = 0
629	0	return meth(url, fp, 500,
630	0	"Internal Server Error: Redirect Recursion", headers)
631	0	result = self.redirect_internal(url, fp, errcode, errmsg, headers,
632	0	data)
633	0	self.tries = 0
634	0	return result
635	n/a
636	1	def redirect_internal(self, url, fp, errcode, errmsg, headers, data):
637	0	if 'location' in headers:
638	0	newurl = headers['location']
639	0	elif 'uri' in headers:
640	0	newurl = headers['uri']
641	n/a	else:
642	0	return
643	0	void = fp.read()
644	0	fp.close()
645	n/a	# In case the server sent a relative URL, join with original:
646	0	newurl = basejoin(self.type + ":" + url, newurl)
647	0	return self.open(newurl)
648	n/a
649	1	def http_error_301(self, url, fp, errcode, errmsg, headers, data=None):
650	n/a	"""Error 301 -- also relocated (permanently)."""
651	0	return self.http_error_302(url, fp, errcode, errmsg, headers, data)
652	n/a
653	1	def http_error_303(self, url, fp, errcode, errmsg, headers, data=None):
654	n/a	"""Error 303 -- also relocated (essentially identical to 302)."""
655	0	return self.http_error_302(url, fp, errcode, errmsg, headers, data)
656	n/a
657	1	def http_error_307(self, url, fp, errcode, errmsg, headers, data=None):
658	n/a	"""Error 307 -- relocated, but turn POST into error."""
659	0	if data is None:
660	0	return self.http_error_302(url, fp, errcode, errmsg, headers, data)
661	n/a	else:
662	0	return self.http_error_default(url, fp, errcode, errmsg, headers)
663	n/a
664	1	def http_error_401(self, url, fp, errcode, errmsg, headers, data=None):
665	n/a	"""Error 401 -- authentication required.
666	n/a	This function supports Basic authentication only."""
667	2	if not 'www-authenticate' in headers:
668	1	URLopener.http_error_default(self, url, fp,
669	1	errcode, errmsg, headers)
670	1	stuff = headers['www-authenticate']
671	1	import re
672	1	match = re.match('[ \t]([^ \t]+)[ \t]+realm="([^"])"', stuff)
673	1	if not match:
674	0	URLopener.http_error_default(self, url, fp,
675	0	errcode, errmsg, headers)
676	1	scheme, realm = match.groups()
677	1	if scheme.lower() != 'basic':
678	0	URLopener.http_error_default(self, url, fp,
679	0	errcode, errmsg, headers)
680	1	name = 'retry_' + self.type + '_basic_auth'
681	1	if data is None:
682	1	return getattr(self,name)(url, realm)
683	n/a	else:
684	0	return getattr(self,name)(url, realm, data)
685	n/a
686	1	def http_error_407(self, url, fp, errcode, errmsg, headers, data=None):
687	n/a	"""Error 407 -- proxy authentication required.
688	n/a	This function supports Basic authentication only."""
689	0	if not 'proxy-authenticate' in headers:
690	0	URLopener.http_error_default(self, url, fp,
691	0	errcode, errmsg, headers)
692	0	stuff = headers['proxy-authenticate']
693	0	import re
694	0	match = re.match('[ \t]([^ \t]+)[ \t]+realm="([^"])"', stuff)
695	0	if not match:
696	0	URLopener.http_error_default(self, url, fp,
697	0	errcode, errmsg, headers)
698	0	scheme, realm = match.groups()
699	0	if scheme.lower() != 'basic':
700	0	URLopener.http_error_default(self, url, fp,
701	0	errcode, errmsg, headers)
702	0	name = 'retry_proxy_' + self.type + '_basic_auth'
703	0	if data is None:
704	0	return getattr(self,name)(url, realm)
705	n/a	else:
706	0	return getattr(self,name)(url, realm, data)
707	n/a
708	1	def retry_proxy_http_basic_auth(self, url, realm, data=None):
709	0	host, selector = splithost(url)
710	0	newurl = 'http://' + host + selector
711	0	proxy = self.proxies['http']
712	0	urltype, proxyhost = splittype(proxy)
713	0	proxyhost, proxyselector = splithost(proxyhost)
714	0	i = proxyhost.find('@') + 1
715	0	proxyhost = proxyhost[i:]
716	0	user, passwd = self.get_user_passwd(proxyhost, realm, i)
717	0	if not (user or passwd): return None
718	0	proxyhost = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + proxyhost
719	0	self.proxies['http'] = 'http://' + proxyhost + proxyselector
720	0	if data is None:
721	0	return self.open(newurl)
722	n/a	else:
723	0	return self.open(newurl, data)
724	n/a
725	1	def retry_proxy_https_basic_auth(self, url, realm, data=None):
726	0	host, selector = splithost(url)
727	0	newurl = 'https://' + host + selector
728	0	proxy = self.proxies['https']
729	0	urltype, proxyhost = splittype(proxy)
730	0	proxyhost, proxyselector = splithost(proxyhost)
731	0	i = proxyhost.find('@') + 1
732	0	proxyhost = proxyhost[i:]
733	0	user, passwd = self.get_user_passwd(proxyhost, realm, i)
734	0	if not (user or passwd): return None
735	0	proxyhost = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + proxyhost
736	0	self.proxies['https'] = 'https://' + proxyhost + proxyselector
737	0	if data is None:
738	0	return self.open(newurl)
739	n/a	else:
740	0	return self.open(newurl, data)
741	n/a
742	1	def retry_http_basic_auth(self, url, realm, data=None):
743	1	host, selector = splithost(url)
744	1	i = host.find('@') + 1
745	1	host = host[i:]
746	1	user, passwd = self.get_user_passwd(host, realm, i)
747	1	if not (user or passwd): return None
748	0	host = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + host
749	0	newurl = 'http://' + host + selector
750	0	if data is None:
751	0	return self.open(newurl)
752	n/a	else:
753	0	return self.open(newurl, data)
754	n/a
755	1	def retry_https_basic_auth(self, url, realm, data=None):
756	0	host, selector = splithost(url)
757	0	i = host.find('@') + 1
758	0	host = host[i:]
759	0	user, passwd = self.get_user_passwd(host, realm, i)
760	0	if not (user or passwd): return None
761	0	host = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + host
762	0	newurl = 'https://' + host + selector
763	0	if data is None:
764	0	return self.open(newurl)
765	n/a	else:
766	0	return self.open(newurl, data)
767	n/a
768	1	def get_user_passwd(self, host, realm, clear_cache=0):
769	1	key = realm + '@' + host.lower()
770	1	if key in self.auth_cache:
771	0	if clear_cache:
772	0	del self.auth_cache[key]
773	n/a	else:
774	0	return self.auth_cache[key]
775	1	user, passwd = self.prompt_user_passwd(host, realm)
776	1	if user or passwd: self.auth_cache[key] = (user, passwd)
777	1	return user, passwd
778	n/a
779	1	def prompt_user_passwd(self, host, realm):
780	n/a	"""Override this in a GUI environment!"""
781	0	import getpass
782	0	try:
783	0	user = raw_input("Enter username for %s at %s: " % (realm,
784	0	host))
785	0	passwd = getpass.getpass("Enter password for %s in %s at %s: " %
786	0	(user, realm, host))
787	0	return user, passwd
788	0	except KeyboardInterrupt:
789	0	print
790	0	return None, None
791	n/a
792	n/a
793	n/a	# Utility functions
794	n/a
795	1	_localhost = None
796	1	def localhost():
797	n/a	"""Return the IP address of the magic hostname 'localhost'."""
798	n/a	global _localhost
799	0	if _localhost is None:
800	0	_localhost = socket.gethostbyname('localhost')
801	0	return _localhost
802	n/a
803	1	_thishost = None
804	1	def thishost():
805	n/a	"""Return the IP address of the current host."""
806	n/a	global _thishost
807	0	if _thishost is None:
808	0	_thishost = socket.gethostbyname(socket.gethostname())
809	0	return _thishost
810	n/a
811	1	_ftperrors = None
812	1	def ftperrors():
813	n/a	"""Return the set of errors raised by the FTP class."""
814	n/a	global _ftperrors
815	0	if _ftperrors is None:
816	0	import ftplib
817	0	_ftperrors = ftplib.all_errors
818	0	return _ftperrors
819	n/a
820	1	_noheaders = None
821	1	def noheaders():
822	n/a	"""Return an empty mimetools.Message object."""
823	n/a	global _noheaders
824	0	if _noheaders is None:
825	0	import mimetools
826	0	try:
827	0	from cStringIO import StringIO
828	0	except ImportError:
829	0	from StringIO import StringIO
830	0	_noheaders = mimetools.Message(StringIO(), 0)
831	0	_noheaders.fp.close() # Recycle file descriptor
832	0	return _noheaders
833	n/a
834	n/a
835	n/a	# Utility classes
836	n/a
837	2	class ftpwrapper:
838	1	"""Class used by open_ftp() for cache of open FTP connections."""
839	n/a
840	n/a	def __init__(self, user, passwd, host, port, dirs,
841	1	timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
842	6	self.user = user
843	6	self.passwd = passwd
844	6	self.host = host
845	6	self.port = port
846	6	self.dirs = dirs
847	6	self.timeout = timeout
848	6	self.init()
849	n/a
850	1	def init(self):
851	6	import ftplib
852	6	self.busy = 0
853	6	self.ftp = ftplib.FTP()
854	6	self.ftp.connect(self.host, self.port, self.timeout)
855	6	self.ftp.login(self.user, self.passwd)
856	21	for dir in self.dirs:
857	15	self.ftp.cwd(dir)
858	n/a
859	1	def retrfile(self, file, type):
860	9	import ftplib
861	9	self.endtransfer()
862	9	if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1
863	5	else: cmd = 'TYPE ' + type; isdir = 0
864	9	try:
865	9	self.ftp.voidcmd(cmd)
866	0	except ftplib.all_errors:
867	0	self.init()
868	0	self.ftp.voidcmd(cmd)
869	9	conn = None
870	9	if file and not isdir:
871	n/a	# Try to retrieve as a file
872	5	try:
873	5	cmd = 'RETR ' + file
874	5	conn = self.ftp.ntransfercmd(cmd)
875	3	except ftplib.error_perm, reason:
876	3	if str(reason)[:3] != '550':
877	0	raise IOError, ('ftp error', reason), sys.exc_info()[2]
878	9	if not conn:
879	n/a	# Set transfer mode to ASCII!
880	7	self.ftp.voidcmd('TYPE A')
881	n/a	# Try a directory listing. Verify that directory exists.
882	7	if file:
883	3	pwd = self.ftp.pwd()
884	3	try:
885	3	try:
886	3	self.ftp.cwd(file)
887	3	except ftplib.error_perm, reason:
888	3	raise IOError, ('ftp error', reason), sys.exc_info()[2]
889	n/a	finally:
890	3	self.ftp.cwd(pwd)
891	0	cmd = 'LIST ' + file
892	n/a	else:
893	4	cmd = 'LIST'
894	4	conn = self.ftp.ntransfercmd(cmd)
895	6	self.busy = 1
896	n/a	# Pass back both a suitably decorated object and a retrieval length
897	6	return (addclosehook(conn[0].makefile('rb'),
898	6	self.endtransfer), conn[1])
899	1	def endtransfer(self):
900	12	if not self.busy:
901	10	return
902	2	self.busy = 0
903	2	try:
904	2	self.ftp.voidresp()
905	0	except ftperrors():
906	0	pass
907	n/a
908	1	def close(self):
909	1	self.endtransfer()
910	1	try:
911	1	self.ftp.close()
912	0	except ftperrors():
913	0	pass
914	n/a
915	2	class addbase:
916	1	"""Base class for addinfo and addclosehook."""
917	n/a
918	1	def __init__(self, fp):
919	100	self.fp = fp
920	100	self.read = self.fp.read
921	100	self.readline = self.fp.readline
922	100	if hasattr(self.fp, "readlines"): self.readlines = self.fp.readlines
923	100	if hasattr(self.fp, "fileno"):
924	93	self.fileno = self.fp.fileno
925	n/a	else:
926	7	self.fileno = lambda: None
927	100	if hasattr(self.fp, "__iter__"):
928	97	self.__iter__ = self.fp.__iter__
929	97	if hasattr(self.fp, "next"):
930	97	self.next = self.fp.next
931	n/a
932	1	def __repr__(self):
933	0	return '<%s at %r whose fp = %r>' % (self.__class__.__name__,
934	0	id(self), self.fp)
935	n/a
936	1	def close(self):
937	63	self.read = None
938	63	self.readline = None
939	63	self.readlines = None
940	63	self.fileno = None
941	63	if self.fp: self.fp.close()
942	63	self.fp = None
943	n/a
944	2	class addclosehook(addbase):
945	1	"""Class to add a close hook to an open file."""
946	n/a
947	1	def __init__(self, fp, closehook, *hookargs):
948	6	addbase.__init__(self, fp)
949	6	self.closehook = closehook
950	6	self.hookargs = hookargs
951	n/a
952	1	def close(self):
953	2	addbase.close(self)
954	2	if self.closehook:
955	2	self.closehook(*self.hookargs)
956	2	self.closehook = None
957	2	self.hookargs = None
958	n/a
959	2	class addinfo(addbase):
960	1	"""class to add an info() method to an open file."""
961	n/a
962	1	def __init__(self, fp, headers):
963	0	addbase.__init__(self, fp)
964	0	self.headers = headers
965	n/a
966	1	def info(self):
967	0	return self.headers
968	n/a
969	2	class addinfourl(addbase):
970	1	"""class to add info() and geturl() methods to an open file."""
971	n/a
972	1	def __init__(self, fp, headers, url, code=None):
973	94	addbase.__init__(self, fp)
974	94	self.headers = headers
975	94	self.url = url
976	94	self.code = code
977	n/a
978	1	def info(self):
979	63	return self.headers
980	n/a
981	1	def getcode(self):
982	3	return self.code
983	n/a
984	1	def geturl(self):
985	10	return self.url
986	n/a
987	n/a
988	n/a	# Utilities to parse URLs (most of these return None for missing parts):
989	n/a	# unwrap('<URL:type://host/path>') --> 'type://host/path'
990	n/a	# splittype('type:opaquestring') --> 'type', 'opaquestring'
991	n/a	# splithost('//host[:port]/path') --> 'host[:port]', '/path'
992	n/a	# splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'
993	n/a	# splitpasswd('user:passwd') -> 'user', 'passwd'
994	n/a	# splitport('host:port') --> 'host', 'port'
995	n/a	# splitquery('/path?query') --> '/path', 'query'
996	n/a	# splittag('/path#tag') --> '/path', 'tag'
997	n/a	# splitattr('/path;attr1=value1;attr2=value2;...') ->
998	n/a	# '/path', ['attr1=value1', 'attr2=value2', ...]
999	n/a	# splitvalue('attr=value') --> 'attr', 'value'
1000	n/a	# unquote('abc%20def') -> 'abc def'
1001	n/a	# quote('abc def') -> 'abc%20def')
1002	n/a
1003	1	try:
1004	1	unicode
1005	0	except NameError:
1006	0	def _is_unicode(x):
1007	0	return 0
1008	n/a	else:
1009	1	def _is_unicode(x):
1010	46	return isinstance(x, unicode)
1011	n/a
1012	1	def toBytes(url):
1013	n/a	"""toBytes(u"URL") --> 'URL'."""
1014	n/a	# Most URL schemes require ASCII. If that changes, the conversion
1015	n/a	# can be relaxed
1016	45	if _is_unicode(url):
1017	0	try:
1018	0	url = url.encode("ASCII")
1019	0	except UnicodeError:
1020	0	raise UnicodeError("URL " + repr(url) +
1021	0	" contains non-ASCII characters")
1022	45	return url
1023	n/a
1024	1	def unwrap(url):
1025	n/a	"""unwrap('<URL:type://host/path>') --> 'type://host/path'."""
1026	395	url = url.strip()
1027	395	if url[:1] == '<' and url[-1:] == '>':
1028	0	url = url[1:-1].strip()
1029	395	if url[:4] == 'URL:': url = url[4:].strip()
1030	395	return url
1031	n/a
1032	1	_typeprog = None
1033	1	def splittype(url):
1034	n/a	"""splittype('type:opaquestring') --> 'type', 'opaquestring'."""
1035	n/a	global _typeprog
1036	286	if _typeprog is None:
1037	1	import re
1038	1	_typeprog = re.compile('^([^/:]+):')
1039	n/a
1040	286	match = _typeprog.match(url)
1041	286	if match:
1042	250	scheme = match.group(1)
1043	250	return scheme.lower(), url[len(scheme) + 1:]
1044	36	return None, url
1045	n/a
1046	1	_hostprog = None
1047	1	def splithost(url):
1048	n/a	"""splithost('//host[:port]/path') --> 'host[:port]', '/path'."""
1049	n/a	global _hostprog
1050	181	if _hostprog is None:
1051	1	import re
1052	1	_hostprog = re.compile('^//([^/?])(.)$')
1053	n/a
1054	181	match = _hostprog.match(url)
1055	181	if match: return match.group(1, 2)
1056	14	return None, url
1057	n/a
1058	1	_userprog = None
1059	1	def splituser(host):
1060	n/a	"""splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""
1061	n/a	global _userprog
1062	85	if _userprog is None:
1063	1	import re
1064	1	_userprog = re.compile('^(.)@(.)$')
1065	n/a
1066	85	match = _userprog.match(host)
1067	85	if match: return map(unquote, match.group(1, 2))
1068	79	return None, host
1069	n/a
1070	1	_passwdprog = None
1071	1	def splitpasswd(user):
1072	n/a	"""splitpasswd('user:passwd') -> 'user', 'passwd'."""
1073	n/a	global _passwdprog
1074	13	if _passwdprog is None:
1075	1	import re
1076	1	_passwdprog = re.compile('^([^:]):(.)$',re.S)
1077	n/a
1078	13	match = _passwdprog.match(user)
1079	13	if match: return match.group(1, 2)
1080	0	return user, None
1081	n/a
1082	n/a	# splittag('/path#tag') --> '/path', 'tag'
1083	1	_portprog = None
1084	1	def splitport(host):
1085	n/a	"""splitport('host:port') --> 'host', 'port'."""
1086	n/a	global _portprog
1087	125	if _portprog is None:
1088	1	import re
1089	1	_portprog = re.compile('^(.*):([0-9]+)$')
1090	n/a
1091	125	match = _portprog.match(host)
1092	125	if match: return match.group(1, 2)
1093	89	return host, None
1094	n/a
1095	1	_nportprog = None
1096	1	def splitnport(host, defport=-1):
1097	n/a	"""Split host and port, returning numeric port.
1098	n/a	Return given default port if no ':' found; defaults to -1.
1099	n/a	Return numerical port if a valid number are found after ':'.
1100	n/a	Return None if ':' but not a valid number."""
1101	n/a	global _nportprog
1102	0	if _nportprog is None:
1103	0	import re
1104	0	_nportprog = re.compile('^(.):(.)$')
1105	n/a
1106	0	match = _nportprog.match(host)
1107	0	if match:
1108	0	host, port = match.group(1, 2)
1109	0	try:
1110	0	if not port: raise ValueError, "no digits"
1111	0	nport = int(port)
1112	0	except ValueError:
1113	0	nport = None
1114	0	return host, nport
1115	0	return host, defport
1116	n/a
1117	1	_queryprog = None
1118	1	def splitquery(url):
1119	n/a	"""splitquery('/path?query') --> '/path', 'query'."""
1120	n/a	global _queryprog
1121	3	if _queryprog is None:
1122	1	import re
1123	1	_queryprog = re.compile('^(.)\?([^?])$')
1124	n/a
1125	3	match = _queryprog.match(url)
1126	3	if match: return match.group(1, 2)
1127	3	return url, None
1128	n/a
1129	1	_tagprog = None
1130	1	def splittag(url):
1131	n/a	"""splittag('/path#tag') --> '/path', 'tag'."""
1132	n/a	global _tagprog
1133	0	if _tagprog is None:
1134	0	import re
1135	0	_tagprog = re.compile('^(.)#([^#])$')
1136	n/a
1137	0	match = _tagprog.match(url)
1138	0	if match: return match.group(1, 2)
1139	0	return url, None
1140	n/a
1141	1	def splitattr(url):
1142	n/a	"""splitattr('/path;attr1=value1;attr2=value2;...') ->
1143	n/a	'/path', ['attr1=value1', 'attr2=value2', ...]."""
1144	15	words = url.split(';')
1145	15	return words[0], words[1:]
1146	n/a
1147	1	_valueprog = None
1148	1	def splitvalue(attr):
1149	n/a	"""splitvalue('attr=value') --> 'attr', 'value'."""
1150	n/a	global _valueprog
1151	1	if _valueprog is None:
1152	1	import re
1153	1	_valueprog = re.compile('^([^=])=(.)$')
1154	n/a
1155	1	match = _valueprog.match(attr)
1156	1	if match: return match.group(1, 2)
1157	0	return attr, None
1158	n/a
1159	n/a	# urlparse contains a duplicate of this method to avoid a circular import. If
1160	n/a	# you update this method, also update the copy in urlparse. This code
1161	n/a	# duplication does not exist in Python3.
1162	n/a
1163	1	_hexdig = '0123456789ABCDEFabcdef'
1164	508	_hextochr = dict((a + b, chr(int(a + b, 16)))
1165	1	for a in _hexdig for b in _hexdig)
1166	n/a
1167	1	def unquote(s):
1168	n/a	"""unquote('abc%20def') -> 'abc def'."""
1169	775	res = s.split('%')
1170	n/a	# fastpath
1171	775	if len(res) == 1:
1172	423	return s
1173	352	s = res[0]
1174	1272	for item in res[1:]:
1175	920	try:
1176	920	s += _hextochr[item[:2]] + item[2:]
1177	5	except KeyError:
1178	3	s += '%' + item
1179	2	except UnicodeDecodeError:
1180	2	s += unichr(int(item[:2], 16)) + item[2:]
1181	352	return s
1182	n/a
1183	1	def unquote_plus(s):
1184	n/a	"""unquote('%7e/abc+def') -> '~/abc def'"""
1185	130	s = s.replace('+', ' ')
1186	130	return unquote(s)
1187	n/a
1188	1	always_safe = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
1189	n/a	'abcdefghijklmnopqrstuvwxyz'
1190	n/a	'0123456789' '_.-')
1191	1	_safe_map = {}
1192	257	for i, c in zip(xrange(256), str(bytearray(xrange(256)))):
1193	256	_safe_map[c] = c if (i < 128 and c in always_safe) else '%{:02X}'.format(i)
1194	1	_safe_quoters = {}
1195	n/a
1196	1	def quote(s, safe='/'):
1197	n/a	"""quote('abc def') -> 'abc%20def'
1198	n/a
1199	n/a	Each part of a URL, e.g. the path info, the query, etc., has a
1200	n/a	different set of reserved characters that must be quoted.
1201	n/a
1202	n/a	RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists
1203	n/a	the following reserved characters.
1204	n/a
1205	n/a	reserved = ";" \| "/" \| "?" \| ":" \| "@" \| "&" \| "=" \| "+" \|
1206	n/a	"$" \| ","
1207	n/a
1208	n/a	Each of these characters is reserved in some component of a URL,
1209	n/a	but not necessarily in all of them.
1210	n/a
1211	n/a	By default, the quote function is intended for quoting the path
1212	n/a	section of a URL. Thus, it will not encode '/'. This character
1213	n/a	is reserved, but in typical usage the quote function is being
1214	n/a	called on a path where the existing slash characters are used as
1215	n/a	reserved characters.
1216	n/a	"""
1217	n/a	# fastpath
1218	590	if not s:
1219	23	return s
1220	567	cachekey = (safe, always_safe)
1221	567	try:
1222	567	(quoter, safe) = _safe_quoters[cachekey]
1223	7	except KeyError:
1224	7	safe_map = _safe_map.copy()
1225	50	safe_map.update([(c, c) for c in safe])
1226	7	quoter = safe_map.__getitem__
1227	7	safe = always_safe + safe
1228	7	_safe_quoters[cachekey] = (quoter, safe)
1229	567	if not s.rstrip(safe):
1230	421	return s
1231	146	return ''.join(map(quoter, s))
1232	n/a
1233	1	def quote_plus(s, safe=''):
1234	n/a	"""Quote the query fragment of a URL; replacing ' ' with '+'"""
1235	81	if ' ' in s:
1236	8	s = quote(s, safe + ' ')
1237	8	return s.replace(' ', '+')
1238	73	return quote(s, safe)
1239	n/a
1240	1	def urlencode(query, doseq=0):
1241	n/a	"""Encode a sequence of two-element tuples or dictionary into a URL query string.
1242	n/a
1243	n/a	If any values in the query arg are sequences and doseq is true, each
1244	n/a	sequence element is converted to a separate parameter.
1245	n/a
1246	n/a	If the query arg is a sequence of two-element tuples, the order of the
1247	n/a	parameters in the output will match the order of parameters in the
1248	n/a	input.
1249	n/a	"""
1250	n/a
1251	7	if hasattr(query,"items"):
1252	n/a	# mapping objects
1253	6	query = query.items()
1254	n/a	else:
1255	n/a	# it's a bother at times that strings and string-like objects are
1256	n/a	# sequences...
1257	1	try:
1258	n/a	# non-sequence items should not work with len()
1259	n/a	# non-empty strings will fail this
1260	1	if len(query) and not isinstance(query[0], tuple):
1261	0	raise TypeError
1262	n/a	# zero-length sequences of all types will get here and succeed,
1263	n/a	# but that's a minor nit - since the original implementation
1264	n/a	# allowed empty dicts that type of behavior probably should be
1265	n/a	# preserved for consistency
1266	0	except TypeError:
1267	0	ty,va,tb = sys.exc_info()
1268	0	raise TypeError, "not a valid non-string sequence or mapping object", tb
1269	n/a
1270	7	l = []
1271	7	if not doseq:
1272	n/a	# preserve old behavior
1273	18	for k, v in query:
1274	12	k = quote_plus(str(k))
1275	12	v = quote_plus(str(v))
1276	12	l.append(k + '=' + v)
1277	n/a	else:
1278	2	for k, v in query:
1279	1	k = quote_plus(str(k))
1280	1	if isinstance(v, str):
1281	0	v = quote_plus(v)
1282	0	l.append(k + '=' + v)
1283	1	elif _is_unicode(v):
1284	n/a	# is there a reasonable way to convert to ASCII?
1285	n/a	# encode generates a string, but "replace" or "ignore"
1286	n/a	# lose information and "strict" can raise UnicodeError
1287	0	v = quote_plus(v.encode("ASCII","replace"))
1288	0	l.append(k + '=' + v)
1289	n/a	else:
1290	1	try:
1291	n/a	# is this a sufficient test for sequence-ness?
1292	1	len(v)
1293	0	except TypeError:
1294	n/a	# not a sequence
1295	0	v = quote_plus(str(v))
1296	0	l.append(k + '=' + v)
1297	n/a	else:
1298	n/a	# loop over the sequence
1299	4	for elt in v:
1300	3	l.append(k + '=' + quote_plus(str(elt)))
1301	7	return '&'.join(l)
1302	n/a
1303	n/a	# Proxy handling
1304	1	def getproxies_environment():
1305	n/a	"""Return a dictionary of scheme -> proxy server URL mappings.
1306	n/a
1307	n/a	Scan the environment for variables named <scheme>_proxy;
1308	n/a	this seems to be the standard convention. If you need a
1309	n/a	different way, you can pass a proxies dictionary to the
1310	n/a	[Fancy]URLopener constructor.
1311	n/a
1312	n/a	"""
1313	17	proxies = {}
1314	630	for name, value in os.environ.items():
1315	613	name = name.lower()
1316	613	if value and name[-6:] == '_proxy':
1317	1	proxies[name[:-6]] = value
1318	17	return proxies
1319	n/a
1320	1	def proxy_bypass_environment(host):
1321	n/a	"""Test if proxies should not be used for a particular host.
1322	n/a
1323	n/a	Checks the environment for a variable named no_proxy, which should
1324	n/a	be a list of DNS suffixes separated by commas, or '*' for all hosts.
1325	n/a	"""
1326	17	no_proxy = os.environ.get('no_proxy', '') or os.environ.get('NO_PROXY', '')
1327	n/a	# '*' is special case for always bypass
1328	17	if no_proxy == '*':
1329	0	return 1
1330	n/a	# strip port off host
1331	17	hostonly, port = splitport(host)
1332	n/a	# check if the host ends with any of the DNS suffixes
1333	33	for name in no_proxy.split(','):
1334	17	if name and (hostonly.endswith(name) or host.endswith(name)):
1335	1	return 1
1336	n/a	# otherwise, don't bypass
1337	16	return 0
1338	n/a
1339	n/a
1340	1	if sys.platform == 'darwin':
1341	0	from _scproxy import _get_proxy_settings, _get_proxies
1342	n/a
1343	0	def proxy_bypass_macosx_sysconf(host):
1344	n/a	"""
1345	n/a	Return True iff this host shouldn't be accessed using a proxy
1346	n/a
1347	n/a	This function uses the MacOSX framework SystemConfiguration
1348	n/a	to fetch the proxy information.
1349	n/a	"""
1350	0	import re
1351	0	import socket
1352	0	from fnmatch import fnmatch
1353	n/a
1354	0	hostonly, port = splitport(host)
1355	n/a
1356	0	def ip2num(ipAddr):
1357	0	parts = ipAddr.split('.')
1358	0	parts = map(int, parts)
1359	0	if len(parts) != 4:
1360	0	parts = (parts + [0, 0, 0, 0])[:4]
1361	0	return (parts[0] << 24) \| (parts[1] << 16) \| (parts[2] << 8) \| parts[3]
1362	n/a
1363	0	proxy_settings = _get_proxy_settings()
1364	n/a
1365	n/a	# Check for simple host names:
1366	0	if '.' not in host:
1367	0	if proxy_settings['exclude_simple']:
1368	0	return True
1369	n/a
1370	0	hostIP = None
1371	n/a
1372	0	for value in proxy_settings.get('exceptions', ()):
1373	n/a	# Items in the list are strings like these: *.local, 169.254/16
1374	0	if not value: continue
1375	n/a
1376	0	m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value)
1377	0	if m is not None:
1378	0	if hostIP is None:
1379	0	try:
1380	0	hostIP = socket.gethostbyname(hostonly)
1381	0	hostIP = ip2num(hostIP)
1382	0	except socket.error:
1383	0	continue
1384	n/a
1385	0	base = ip2num(m.group(1))
1386	0	mask = m.group(2)
1387	0	if mask is None:
1388	0	mask = 8 * (m.group(1).count('.') + 1)
1389	n/a
1390	n/a	else:
1391	0	mask = int(mask[1:])
1392	0	mask = 32 - mask
1393	n/a
1394	0	if (hostIP >> mask) == (base >> mask):
1395	0	return True
1396	n/a
1397	0	elif fnmatch(host, value):
1398	0	return True
1399	n/a
1400	0	return False
1401	n/a
1402	0	def getproxies_macosx_sysconf():
1403	n/a	"""Return a dictionary of scheme -> proxy server URL mappings.
1404	n/a
1405	n/a	This function uses the MacOSX framework SystemConfiguration
1406	n/a	to fetch the proxy information.
1407	n/a	"""
1408	0	return _get_proxies()
1409	n/a
1410	0	def proxy_bypass(host):
1411	0	if getproxies_environment():
1412	0	return proxy_bypass_environment(host)
1413	n/a	else:
1414	0	return proxy_bypass_macosx_sysconf(host)
1415	n/a
1416	0	def getproxies():
1417	0	return getproxies_environment() or getproxies_macosx_sysconf()
1418	n/a
1419	1	elif os.name == 'nt':
1420	0	def getproxies_registry():
1421	n/a	"""Return a dictionary of scheme -> proxy server URL mappings.
1422	n/a
1423	n/a	Win32 uses the registry to store proxies.
1424	n/a
1425	n/a	"""
1426	0	proxies = {}
1427	0	try:
1428	0	import _winreg
1429	0	except ImportError:
1430	n/a	# Std module, so should be around - but you never know!
1431	0	return proxies
1432	0	try:
1433	0	internetSettings = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER,
1434	0	r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
1435	0	proxyEnable = _winreg.QueryValueEx(internetSettings,
1436	0	'ProxyEnable')[0]
1437	0	if proxyEnable:
1438	n/a	# Returned as Unicode but problems if not converted to ASCII
1439	0	proxyServer = str(_winreg.QueryValueEx(internetSettings,
1440	0	'ProxyServer')[0])
1441	0	if '=' in proxyServer:
1442	n/a	# Per-protocol settings
1443	0	for p in proxyServer.split(';'):
1444	0	protocol, address = p.split('=', 1)
1445	n/a	# See if address has a type:// prefix
1446	0	import re
1447	0	if not re.match('^([^/:]+)://', address):
1448	0	address = '%s://%s' % (protocol, address)
1449	0	proxies[protocol] = address
1450	n/a	else:
1451	n/a	# Use one setting for all protocols
1452	0	if proxyServer[:5] == 'http:':
1453	0	proxies['http'] = proxyServer
1454	n/a	else:
1455	0	proxies['http'] = 'http://%s' % proxyServer
1456	0	proxies['ftp'] = 'ftp://%s' % proxyServer
1457	0	internetSettings.Close()
1458	0	except (WindowsError, ValueError, TypeError):
1459	n/a	# Either registry key not found etc, or the value in an
1460	n/a	# unexpected format.
1461	n/a	# proxies already set up to be empty so nothing to do
1462	0	pass
1463	0	return proxies
1464	n/a
1465	0	def getproxies():
1466	n/a	"""Return a dictionary of scheme -> proxy server URL mappings.
1467	n/a
1468	n/a	Returns settings gathered from the environment, if specified,
1469	n/a	or the registry.
1470	n/a
1471	n/a	"""
1472	0	return getproxies_environment() or getproxies_registry()
1473	n/a
1474	0	def proxy_bypass_registry(host):
1475	0	try:
1476	0	import _winreg
1477	0	import re
1478	0	except ImportError:
1479	n/a	# Std modules, so should be around - but you never know!
1480	0	return 0
1481	0	try:
1482	0	internetSettings = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER,
1483	0	r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
1484	0	proxyEnable = _winreg.QueryValueEx(internetSettings,
1485	0	'ProxyEnable')[0]
1486	0	proxyOverride = str(_winreg.QueryValueEx(internetSettings,
1487	0	'ProxyOverride')[0])
1488	n/a	# ^^^^ Returned as Unicode but problems if not converted to ASCII
1489	0	except WindowsError:
1490	0	return 0
1491	0	if not proxyEnable or not proxyOverride:
1492	0	return 0
1493	n/a	# try to make a host list from name and IP address.
1494	0	rawHost, port = splitport(host)
1495	0	host = [rawHost]
1496	0	try:
1497	0	addr = socket.gethostbyname(rawHost)
1498	0	if addr != rawHost:
1499	0	host.append(addr)
1500	0	except socket.error:
1501	0	pass
1502	0	try:
1503	0	fqdn = socket.getfqdn(rawHost)
1504	0	if fqdn != rawHost:
1505	0	host.append(fqdn)
1506	0	except socket.error:
1507	0	pass
1508	n/a	# make a check value list from the registry entry: replace the
1509	n/a	# '<local>' string by the localhost entry and the corresponding
1510	n/a	# canonical entry.
1511	0	proxyOverride = proxyOverride.split(';')
1512	n/a	# now check if we match one of the registry values.
1513	0	for test in proxyOverride:
1514	0	if test == '<local>':
1515	0	if '.' not in rawHost:
1516	0	return 1
1517	0	test = test.replace(".", r"\.") # mask dots
1518	0	test = test.replace("", r".") # change glob sequence
1519	0	test = test.replace("?", r".") # change glob char
1520	0	for val in host:
1521	n/a	# print "%s <--> %s" %( test, val )
1522	0	if re.match(test, val, re.I):
1523	0	return 1
1524	0	return 0
1525	n/a
1526	0	def proxy_bypass(host):
1527	n/a	"""Return a dictionary of scheme -> proxy server URL mappings.
1528	n/a
1529	n/a	Returns settings gathered from the environment, if specified,
1530	n/a	or the registry.
1531	n/a
1532	n/a	"""
1533	0	if getproxies_environment():
1534	0	return proxy_bypass_environment(host)
1535	n/a	else:
1536	0	return proxy_bypass_registry(host)
1537	n/a
1538	n/a	else:
1539	n/a	# By default use environment variables
1540	1	getproxies = getproxies_environment
1541	1	proxy_bypass = proxy_bypass_environment
1542	n/a
1543	n/a	# Test and time quote() and unquote()
1544	1	def test1():
1545	0	s = ''
1546	0	for i in range(256): s = s + chr(i)
1547	0	s = s*4
1548	0	t0 = time.time()
1549	0	qs = quote(s)
1550	0	uqs = unquote(qs)
1551	0	t1 = time.time()
1552	0	if uqs != s:
1553	0	print 'Wrong!'
1554	0	print repr(s)
1555	0	print repr(qs)
1556	0	print repr(uqs)
1557	0	print round(t1 - t0, 3), 'sec'
1558	n/a
1559	n/a
1560	1	def reporthook(blocknum, blocksize, totalsize):
1561	n/a	# Report during remote transfers
1562	0	print "Block number: %d, Block size: %d, Total size: %d" % (
1563	0	blocknum, blocksize, totalsize)
1564	n/a
1565	n/a	# Test program
1566	1	def test(args=[]):
1567	0	if not args:
1568	n/a	args = [
1569	0	'/etc/passwd',
1570	0	'file:/etc/passwd',
1571	0	'file://localhost/etc/passwd',
1572	0	'ftp://ftp.gnu.org/pub/README',
1573	0	'http://www.python.org/index.html',
1574	n/a	]
1575	0	if hasattr(URLopener, "open_https"):
1576	0	args.append('https://synergy.as.cmu.edu/~geek/')
1577	0	try:
1578	0	for url in args:
1579	0	print '-'10, url, '-'10
1580	0	fn, h = urlretrieve(url, None, reporthook)
1581	0	print fn
1582	0	if h:
1583	0	print '======'
1584	0	for k in h.keys(): print k + ':', h[k]
1585	0	print '======'
1586	0	with open(fn, 'rb') as fp:
1587	0	data = fp.read()
1588	0	if '\r' in data:
1589	0	table = string.maketrans("", "")
1590	0	data = data.translate(table, "\r")
1591	0	print data
1592	0	fn, h = None, None
1593	0	print '-'*40
1594	n/a	finally:
1595	0	urlcleanup()
1596	n/a
1597	1	def main():
1598	0	import getopt, sys
1599	0	try:
1600	0	opts, args = getopt.getopt(sys.argv[1:], "th")
1601	0	except getopt.error, msg:
1602	0	print msg
1603	0	print "Use -h for help"
1604	0	return
1605	0	t = 0
1606	0	for o, a in opts:
1607	0	if o == '-t':
1608	0	t = t + 1
1609	0	if o == '-h':
1610	0	print "Usage: python urllib.py [-t] [url ...]"
1611	0	print "-t runs self-test;",
1612	0	print "otherwise, contents of urls are printed"
1613	0	return
1614	0	if t:
1615	0	if t > 1:
1616	0	test1()
1617	0	test(args)
1618	n/a	else:
1619	0	if not args:
1620	0	print "Use -h for help"
1621	0	for url in args:
1622	0	print urlopen(url).read(),
1623	n/a
1624	n/a	# Run test program when run as a script
1625	1	if __name__ == '__main__':
1626	0	main()