Python code coverage for Lib/test/test_urllib2net.py

#	count	content
1	n/a	import unittest
2	n/a	from test import support
3	n/a	from test.test_urllib2 import sanepathname2url
4	n/a
5	n/a	import os
6	n/a	import socket
7	n/a	import urllib.error
8	n/a	import urllib.request
9	n/a	import sys
10	n/a
11	n/a	support.requires("network")
12	n/a
13	n/a	TIMEOUT = 60 # seconds
14	n/a
15	n/a
16	n/a	def _retry_thrice(func, exc, args, *kwargs):
17	n/a	for i in range(3):
18	n/a	try:
19	n/a	return func(args, *kwargs)
20	n/a	except exc as e:
21	n/a	last_exc = e
22	n/a	continue
23	n/a	raise last_exc
24	n/a
25	n/a	def _wrap_with_retry_thrice(func, exc):
26	n/a	def wrapped(args, *kwargs):
27	n/a	return _retry_thrice(func, exc, args, *kwargs)
28	n/a	return wrapped
29	n/a
30	n/a	# Connecting to remote hosts is flaky. Make it more robust by retrying
31	n/a	# the connection several times.
32	n/a	_urlopen_with_retry = _wrap_with_retry_thrice(urllib.request.urlopen,
33	n/a	urllib.error.URLError)
34	n/a
35	n/a
36	n/a	class AuthTests(unittest.TestCase):
37	n/a	"""Tests urllib2 authentication features."""
38	n/a
39	n/a	## Disabled at the moment since there is no page under python.org which
40	n/a	## could be used to HTTP authentication.
41	n/a	#
42	n/a	# def test_basic_auth(self):
43	n/a	# import http.client
44	n/a	#
45	n/a	# test_url = "http://www.python.org/test/test_urllib2/basic_auth"
46	n/a	# test_hostport = "www.python.org"
47	n/a	# test_realm = 'Test Realm'
48	n/a	# test_user = 'test.test_urllib2net'
49	n/a	# test_password = 'blah'
50	n/a	#
51	n/a	# # failure
52	n/a	# try:
53	n/a	# _urlopen_with_retry(test_url)
54	n/a	# except urllib2.HTTPError, exc:
55	n/a	# self.assertEqual(exc.code, 401)
56	n/a	# else:
57	n/a	# self.fail("urlopen() should have failed with 401")
58	n/a	#
59	n/a	# # success
60	n/a	# auth_handler = urllib2.HTTPBasicAuthHandler()
61	n/a	# auth_handler.add_password(test_realm, test_hostport,
62	n/a	# test_user, test_password)
63	n/a	# opener = urllib2.build_opener(auth_handler)
64	n/a	# f = opener.open('http://localhost/')
65	n/a	# response = _urlopen_with_retry("http://www.python.org/")
66	n/a	#
67	n/a	# # The 'userinfo' URL component is deprecated by RFC 3986 for security
68	n/a	# # reasons, let's not implement it! (it's already implemented for proxy
69	n/a	# # specification strings (that is, URLs or authorities specifying a
70	n/a	# # proxy), so we must keep that)
71	n/a	# self.assertRaises(http.client.InvalidURL,
72	n/a	# urllib2.urlopen, "http://evil:thing@example.com")
73	n/a
74	n/a
75	n/a	class CloseSocketTest(unittest.TestCase):
76	n/a
77	n/a	def test_close(self):
78	n/a	# calling .close() on urllib2's response objects should close the
79	n/a	# underlying socket
80	n/a	url = "http://www.example.com/"
81	n/a	with support.transient_internet(url):
82	n/a	response = _urlopen_with_retry(url)
83	n/a	sock = response.fp
84	n/a	self.assertFalse(sock.closed)
85	n/a	response.close()
86	n/a	self.assertTrue(sock.closed)
87	n/a
88	n/a	class OtherNetworkTests(unittest.TestCase):
89	n/a	def setUp(self):
90	n/a	if 0: # for debugging
91	n/a	import logging
92	n/a	logger = logging.getLogger("test_urllib2net")
93	n/a	logger.addHandler(logging.StreamHandler())
94	n/a
95	n/a	# XXX The rest of these tests aren't very good -- they don't check much.
96	n/a	# They do sometimes catch some major disasters, though.
97	n/a
98	n/a	def test_ftp(self):
99	n/a	urls = [
100	n/a	'ftp://ftp.debian.org/debian/README',
101	n/a	('ftp://ftp.debian.org/debian/non-existent-file',
102	n/a	None, urllib.error.URLError),
103	n/a	]
104	n/a	self._test_urls(urls, self._extra_handlers())
105	n/a
106	n/a	def test_file(self):
107	n/a	TESTFN = support.TESTFN
108	n/a	f = open(TESTFN, 'w')
109	n/a	try:
110	n/a	f.write('hi there\n')
111	n/a	f.close()
112	n/a	urls = [
113	n/a	'file:' + sanepathname2url(os.path.abspath(TESTFN)),
114	n/a	('file:///nonsensename/etc/passwd', None,
115	n/a	urllib.error.URLError),
116	n/a	]
117	n/a	self._test_urls(urls, self._extra_handlers(), retry=True)
118	n/a	finally:
119	n/a	os.remove(TESTFN)
120	n/a
121	n/a	self.assertRaises(ValueError, urllib.request.urlopen,'./relative_path/to/file')
122	n/a
123	n/a	# XXX Following test depends on machine configurations that are internal
124	n/a	# to CNRI. Need to set up a public server with the right authentication
125	n/a	# configuration for test purposes.
126	n/a
127	n/a	## def test_cnri(self):
128	n/a	## if socket.gethostname() == 'bitdiddle':
129	n/a	## localhost = 'bitdiddle.cnri.reston.va.us'
130	n/a	## elif socket.gethostname() == 'bitdiddle.concentric.net':
131	n/a	## localhost = 'localhost'
132	n/a	## else:
133	n/a	## localhost = None
134	n/a	## if localhost is not None:
135	n/a	## urls = [
136	n/a	## 'file://%s/etc/passwd' % localhost,
137	n/a	## 'http://%s/simple/' % localhost,
138	n/a	## 'http://%s/digest/' % localhost,
139	n/a	## 'http://%s/not/found.h' % localhost,
140	n/a	## ]
141	n/a
142	n/a	## bauth = HTTPBasicAuthHandler()
143	n/a	## bauth.add_password('basic_test_realm', localhost, 'jhylton',
144	n/a	## 'password')
145	n/a	## dauth = HTTPDigestAuthHandler()
146	n/a	## dauth.add_password('digest_test_realm', localhost, 'jhylton',
147	n/a	## 'password')
148	n/a
149	n/a	## self._test_urls(urls, self._extra_handlers()+[bauth, dauth])
150	n/a
151	n/a	def test_urlwithfrag(self):
152	n/a	urlwith_frag = "http://www.pythontest.net/index.html#frag"
153	n/a	with support.transient_internet(urlwith_frag):
154	n/a	req = urllib.request.Request(urlwith_frag)
155	n/a	res = urllib.request.urlopen(req)
156	n/a	self.assertEqual(res.geturl(),
157	n/a	"http://www.pythontest.net/index.html#frag")
158	n/a
159	n/a	def test_redirect_url_withfrag(self):
160	n/a	redirect_url_with_frag = "http://www.pythontest.net/redir/with_frag/"
161	n/a	with support.transient_internet(redirect_url_with_frag):
162	n/a	req = urllib.request.Request(redirect_url_with_frag)
163	n/a	res = urllib.request.urlopen(req)
164	n/a	self.assertEqual(res.geturl(),
165	n/a	"http://www.pythontest.net/elsewhere/#frag")
166	n/a
167	n/a	def test_custom_headers(self):
168	n/a	url = "http://www.example.com"
169	n/a	with support.transient_internet(url):
170	n/a	opener = urllib.request.build_opener()
171	n/a	request = urllib.request.Request(url)
172	n/a	self.assertFalse(request.header_items())
173	n/a	opener.open(request)
174	n/a	self.assertTrue(request.header_items())
175	n/a	self.assertTrue(request.has_header('User-agent'))
176	n/a	request.add_header('User-Agent','Test-Agent')
177	n/a	opener.open(request)
178	n/a	self.assertEqual(request.get_header('User-agent'),'Test-Agent')
179	n/a
180	n/a	def test_sites_no_connection_close(self):
181	n/a	# Some sites do not send Connection: close header.
182	n/a	# Verify that those work properly. (#issue12576)
183	n/a
184	n/a	URL = 'http://www.imdb.com' # mangles Connection:close
185	n/a
186	n/a	with support.transient_internet(URL):
187	n/a	try:
188	n/a	with urllib.request.urlopen(URL) as res:
189	n/a	pass
190	n/a	except ValueError as e:
191	n/a	self.fail("urlopen failed for site not sending \
192	n/a	Connection:close")
193	n/a	else:
194	n/a	self.assertTrue(res)
195	n/a
196	n/a	req = urllib.request.urlopen(URL)
197	n/a	res = req.read()
198	n/a	self.assertTrue(res)
199	n/a
200	n/a	def _test_urls(self, urls, handlers, retry=True):
201	n/a	import time
202	n/a	import logging
203	n/a	debug = logging.getLogger("test_urllib2").debug
204	n/a
205	n/a	urlopen = urllib.request.build_opener(*handlers).open
206	n/a	if retry:
207	n/a	urlopen = _wrap_with_retry_thrice(urlopen, urllib.error.URLError)
208	n/a
209	n/a	for url in urls:
210	n/a	with self.subTest(url=url):
211	n/a	if isinstance(url, tuple):
212	n/a	url, req, expected_err = url
213	n/a	else:
214	n/a	req = expected_err = None
215	n/a
216	n/a	with support.transient_internet(url):
217	n/a	try:
218	n/a	f = urlopen(url, req, TIMEOUT)
219	n/a	# urllib.error.URLError is a subclass of OSError
220	n/a	except OSError as err:
221	n/a	if expected_err:
222	n/a	msg = ("Didn't get expected error(s) %s for %s %s, got %s: %s" %
223	n/a	(expected_err, url, req, type(err), err))
224	n/a	self.assertIsInstance(err, expected_err, msg)
225	n/a	else:
226	n/a	raise
227	n/a	else:
228	n/a	try:
229	n/a	with support.time_out, \
230	n/a	support.socket_peer_reset, \
231	n/a	support.ioerror_peer_reset:
232	n/a	buf = f.read()
233	n/a	debug("read %d bytes" % len(buf))
234	n/a	except socket.timeout:
235	n/a	print("<timeout: %s>" % url, file=sys.stderr)
236	n/a	f.close()
237	n/a	time.sleep(0.1)
238	n/a
239	n/a	def _extra_handlers(self):
240	n/a	handlers = []
241	n/a
242	n/a	cfh = urllib.request.CacheFTPHandler()
243	n/a	self.addCleanup(cfh.clear_cache)
244	n/a	cfh.setTimeout(1)
245	n/a	handlers.append(cfh)
246	n/a
247	n/a	return handlers
248	n/a
249	n/a
250	n/a	class TimeoutTest(unittest.TestCase):
251	n/a	def test_http_basic(self):
252	n/a	self.assertIsNone(socket.getdefaulttimeout())
253	n/a	url = "http://www.example.com"
254	n/a	with support.transient_internet(url, timeout=None):
255	n/a	u = _urlopen_with_retry(url)
256	n/a	self.addCleanup(u.close)
257	n/a	self.assertIsNone(u.fp.raw._sock.gettimeout())
258	n/a
259	n/a	def test_http_default_timeout(self):
260	n/a	self.assertIsNone(socket.getdefaulttimeout())
261	n/a	url = "http://www.example.com"
262	n/a	with support.transient_internet(url):
263	n/a	socket.setdefaulttimeout(60)
264	n/a	try:
265	n/a	u = _urlopen_with_retry(url)
266	n/a	self.addCleanup(u.close)
267	n/a	finally:
268	n/a	socket.setdefaulttimeout(None)
269	n/a	self.assertEqual(u.fp.raw._sock.gettimeout(), 60)
270	n/a
271	n/a	def test_http_no_timeout(self):
272	n/a	self.assertIsNone(socket.getdefaulttimeout())
273	n/a	url = "http://www.example.com"
274	n/a	with support.transient_internet(url):
275	n/a	socket.setdefaulttimeout(60)
276	n/a	try:
277	n/a	u = _urlopen_with_retry(url, timeout=None)
278	n/a	self.addCleanup(u.close)
279	n/a	finally:
280	n/a	socket.setdefaulttimeout(None)
281	n/a	self.assertIsNone(u.fp.raw._sock.gettimeout())
282	n/a
283	n/a	def test_http_timeout(self):
284	n/a	url = "http://www.example.com"
285	n/a	with support.transient_internet(url):
286	n/a	u = _urlopen_with_retry(url, timeout=120)
287	n/a	self.addCleanup(u.close)
288	n/a	self.assertEqual(u.fp.raw._sock.gettimeout(), 120)
289	n/a
290	n/a	FTP_HOST = 'ftp://ftp.debian.org/debian/'
291	n/a
292	n/a	def test_ftp_basic(self):
293	n/a	self.assertIsNone(socket.getdefaulttimeout())
294	n/a	with support.transient_internet(self.FTP_HOST, timeout=None):
295	n/a	u = _urlopen_with_retry(self.FTP_HOST)
296	n/a	self.addCleanup(u.close)
297	n/a	self.assertIsNone(u.fp.fp.raw._sock.gettimeout())
298	n/a
299	n/a	def test_ftp_default_timeout(self):
300	n/a	self.assertIsNone(socket.getdefaulttimeout())
301	n/a	with support.transient_internet(self.FTP_HOST):
302	n/a	socket.setdefaulttimeout(60)
303	n/a	try:
304	n/a	u = _urlopen_with_retry(self.FTP_HOST)
305	n/a	self.addCleanup(u.close)
306	n/a	finally:
307	n/a	socket.setdefaulttimeout(None)
308	n/a	self.assertEqual(u.fp.fp.raw._sock.gettimeout(), 60)
309	n/a
310	n/a	def test_ftp_no_timeout(self):
311	n/a	self.assertIsNone(socket.getdefaulttimeout())
312	n/a	with support.transient_internet(self.FTP_HOST):
313	n/a	socket.setdefaulttimeout(60)
314	n/a	try:
315	n/a	u = _urlopen_with_retry(self.FTP_HOST, timeout=None)
316	n/a	self.addCleanup(u.close)
317	n/a	finally:
318	n/a	socket.setdefaulttimeout(None)
319	n/a	self.assertIsNone(u.fp.fp.raw._sock.gettimeout())
320	n/a
321	n/a	def test_ftp_timeout(self):
322	n/a	with support.transient_internet(self.FTP_HOST):
323	n/a	u = _urlopen_with_retry(self.FTP_HOST, timeout=60)
324	n/a	self.addCleanup(u.close)
325	n/a	self.assertEqual(u.fp.fp.raw._sock.gettimeout(), 60)
326	n/a
327	n/a
328	n/a	if __name__ == "__main__":
329	n/a	unittest.main()