Python code coverage for Lib/test/test_urllib2.py

#	count	content
1	n/a	import unittest
2	n/a	from test import support
3	n/a	from test import test_urllib
4	n/a
5	n/a	import os
6	n/a	import io
7	n/a	import socket
8	n/a	import array
9	n/a	import sys
10	n/a	import tempfile
11	n/a	import subprocess
12	n/a
13	n/a	import urllib.request
14	n/a	# The proxy bypass method imported below has logic specific to the OSX
15	n/a	# proxy config data structure but is testable on all platforms.
16	n/a	from urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler,
17	n/a	HTTPPasswordMgrWithPriorAuth, _parse_proxy,
18	n/a	_proxy_bypass_macosx_sysconf,
19	n/a	AbstractDigestAuthHandler)
20	n/a	from urllib.parse import urlparse
21	n/a	import urllib.error
22	n/a	import http.client
23	n/a
24	n/a	# XXX
25	n/a	# Request
26	n/a	# CacheFTPHandler (hard to write)
27	n/a	# parse_keqv_list, parse_http_list, HTTPDigestAuthHandler
28	n/a
29	n/a
30	n/a	class TrivialTests(unittest.TestCase):
31	n/a
32	n/a	def test___all__(self):
33	n/a	# Verify which names are exposed
34	n/a	for module in 'request', 'response', 'parse', 'error', 'robotparser':
35	n/a	context = {}
36	n/a	exec('from urllib.%s import *' % module, context)
37	n/a	del context['__builtins__']
38	n/a	if module == 'request' and os.name == 'nt':
39	n/a	u, p = context.pop('url2pathname'), context.pop('pathname2url')
40	n/a	self.assertEqual(u.__module__, 'nturl2path')
41	n/a	self.assertEqual(p.__module__, 'nturl2path')
42	n/a	for k, v in context.items():
43	n/a	self.assertEqual(v.__module__, 'urllib.%s' % module,
44	n/a	"%r is exposed in 'urllib.%s' but defined in %r" %
45	n/a	(k, module, v.__module__))
46	n/a
47	n/a	def test_trivial(self):
48	n/a	# A couple trivial tests
49	n/a
50	n/a	self.assertRaises(ValueError, urllib.request.urlopen, 'bogus url')
51	n/a
52	n/a	# XXX Name hacking to get this to work on Windows.
53	n/a	fname = os.path.abspath(urllib.request.__file__).replace(os.sep, '/')
54	n/a
55	n/a	if os.name == 'nt':
56	n/a	file_url = "file:///%s" % fname
57	n/a	else:
58	n/a	file_url = "file://%s" % fname
59	n/a
60	n/a	f = urllib.request.urlopen(file_url)
61	n/a
62	n/a	f.read()
63	n/a	f.close()
64	n/a
65	n/a	def test_parse_http_list(self):
66	n/a	tests = [
67	n/a	('a,b,c', ['a', 'b', 'c']),
68	n/a	('path"o,l"og"i"cal, example', ['path"o,l"og"i"cal', 'example']),
69	n/a	('a, b, "c", "d", "e,f", g, h',
70	n/a	['a', 'b', '"c"', '"d"', '"e,f"', 'g', 'h']),
71	n/a	('a="b\\"c", d="e\\,f", g="h\\\\i"',
72	n/a	['a="b"c"', 'd="e,f"', 'g="h\\i"'])]
73	n/a	for string, list in tests:
74	n/a	self.assertEqual(urllib.request.parse_http_list(string), list)
75	n/a
76	n/a	def test_URLError_reasonstr(self):
77	n/a	err = urllib.error.URLError('reason')
78	n/a	self.assertIn(err.reason, str(err))
79	n/a
80	n/a
81	n/a	class RequestHdrsTests(unittest.TestCase):
82	n/a
83	n/a	def test_request_headers_dict(self):
84	n/a	"""
85	n/a	The Request.headers dictionary is not a documented interface. It
86	n/a	should stay that way, because the complete set of headers are only
87	n/a	accessible through the .get_header(), .has_header(), .header_items()
88	n/a	interface. However, .headers pre-dates those methods, and so real code
89	n/a	will be using the dictionary.
90	n/a
91	n/a	The introduction in 2.4 of those methods was a mistake for the same
92	n/a	reason: code that previously saw all (urllib2 user)-provided headers in
93	n/a	.headers now sees only a subset.
94	n/a
95	n/a	"""
96	n/a	url = "http://example.com"
97	n/a	self.assertEqual(Request(url,
98	n/a	headers={"Spam-eggs": "blah"}
99	n/a	).headers["Spam-eggs"], "blah")
100	n/a	self.assertEqual(Request(url,
101	n/a	headers={"spam-EggS": "blah"}
102	n/a	).headers["Spam-eggs"], "blah")
103	n/a
104	n/a	def test_request_headers_methods(self):
105	n/a	"""
106	n/a	Note the case normalization of header names here, to
107	n/a	.capitalize()-case. This should be preserved for
108	n/a	backwards-compatibility. (In the HTTP case, normalization to
109	n/a	.title()-case is done by urllib2 before sending headers to
110	n/a	http.client).
111	n/a
112	n/a	Note that e.g. r.has_header("spam-EggS") is currently False, and
113	n/a	r.get_header("spam-EggS") returns None, but that could be changed in
114	n/a	future.
115	n/a
116	n/a	Method r.remove_header should remove items both from r.headers and
117	n/a	r.unredirected_hdrs dictionaries
118	n/a	"""
119	n/a	url = "http://example.com"
120	n/a	req = Request(url, headers={"Spam-eggs": "blah"})
121	n/a	self.assertTrue(req.has_header("Spam-eggs"))
122	n/a	self.assertEqual(req.header_items(), [('Spam-eggs', 'blah')])
123	n/a
124	n/a	req.add_header("Foo-Bar", "baz")
125	n/a	self.assertEqual(sorted(req.header_items()),
126	n/a	[('Foo-bar', 'baz'), ('Spam-eggs', 'blah')])
127	n/a	self.assertFalse(req.has_header("Not-there"))
128	n/a	self.assertIsNone(req.get_header("Not-there"))
129	n/a	self.assertEqual(req.get_header("Not-there", "default"), "default")
130	n/a
131	n/a	req.remove_header("Spam-eggs")
132	n/a	self.assertFalse(req.has_header("Spam-eggs"))
133	n/a
134	n/a	req.add_unredirected_header("Unredirected-spam", "Eggs")
135	n/a	self.assertTrue(req.has_header("Unredirected-spam"))
136	n/a
137	n/a	req.remove_header("Unredirected-spam")
138	n/a	self.assertFalse(req.has_header("Unredirected-spam"))
139	n/a
140	n/a	def test_password_manager(self):
141	n/a	mgr = urllib.request.HTTPPasswordMgr()
142	n/a	add = mgr.add_password
143	n/a	find_user_pass = mgr.find_user_password
144	n/a	add("Some Realm", "http://example.com/", "joe", "password")
145	n/a	add("Some Realm", "http://example.com/ni", "ni", "ni")
146	n/a	add("c", "http://example.com/foo", "foo", "ni")
147	n/a	add("c", "http://example.com/bar", "bar", "nini")
148	n/a	add("b", "http://example.com/", "first", "blah")
149	n/a	add("b", "http://example.com/", "second", "spam")
150	n/a	add("a", "http://example.com", "1", "a")
151	n/a	add("Some Realm", "http://c.example.com:3128", "3", "c")
152	n/a	add("Some Realm", "d.example.com", "4", "d")
153	n/a	add("Some Realm", "e.example.com:3128", "5", "e")
154	n/a
155	n/a	self.assertEqual(find_user_pass("Some Realm", "example.com"),
156	n/a	('joe', 'password'))
157	n/a
158	n/a	#self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"),
159	n/a	# ('ni', 'ni'))
160	n/a
161	n/a	self.assertEqual(find_user_pass("Some Realm", "http://example.com"),
162	n/a	('joe', 'password'))
163	n/a	self.assertEqual(find_user_pass("Some Realm", "http://example.com/"),
164	n/a	('joe', 'password'))
165	n/a	self.assertEqual(
166	n/a	find_user_pass("Some Realm", "http://example.com/spam"),
167	n/a	('joe', 'password'))
168	n/a	self.assertEqual(
169	n/a	find_user_pass("Some Realm", "http://example.com/spam/spam"),
170	n/a	('joe', 'password'))
171	n/a	self.assertEqual(find_user_pass("c", "http://example.com/foo"),
172	n/a	('foo', 'ni'))
173	n/a	self.assertEqual(find_user_pass("c", "http://example.com/bar"),
174	n/a	('bar', 'nini'))
175	n/a	self.assertEqual(find_user_pass("b", "http://example.com/"),
176	n/a	('second', 'spam'))
177	n/a
178	n/a	# No special relationship between a.example.com and example.com:
179	n/a
180	n/a	self.assertEqual(find_user_pass("a", "http://example.com/"),
181	n/a	('1', 'a'))
182	n/a	self.assertEqual(find_user_pass("a", "http://a.example.com/"),
183	n/a	(None, None))
184	n/a
185	n/a	# Ports:
186	n/a
187	n/a	self.assertEqual(find_user_pass("Some Realm", "c.example.com"),
188	n/a	(None, None))
189	n/a	self.assertEqual(find_user_pass("Some Realm", "c.example.com:3128"),
190	n/a	('3', 'c'))
191	n/a	self.assertEqual(
192	n/a	find_user_pass("Some Realm", "http://c.example.com:3128"),
193	n/a	('3', 'c'))
194	n/a	self.assertEqual(find_user_pass("Some Realm", "d.example.com"),
195	n/a	('4', 'd'))
196	n/a	self.assertEqual(find_user_pass("Some Realm", "e.example.com:3128"),
197	n/a	('5', 'e'))
198	n/a
199	n/a	def test_password_manager_default_port(self):
200	n/a	"""
201	n/a	The point to note here is that we can't guess the default port if
202	n/a	there's no scheme. This applies to both add_password and
203	n/a	find_user_password.
204	n/a	"""
205	n/a	mgr = urllib.request.HTTPPasswordMgr()
206	n/a	add = mgr.add_password
207	n/a	find_user_pass = mgr.find_user_password
208	n/a	add("f", "http://g.example.com:80", "10", "j")
209	n/a	add("g", "http://h.example.com", "11", "k")
210	n/a	add("h", "i.example.com:80", "12", "l")
211	n/a	add("i", "j.example.com", "13", "m")
212	n/a	self.assertEqual(find_user_pass("f", "g.example.com:100"),
213	n/a	(None, None))
214	n/a	self.assertEqual(find_user_pass("f", "g.example.com:80"),
215	n/a	('10', 'j'))
216	n/a	self.assertEqual(find_user_pass("f", "g.example.com"),
217	n/a	(None, None))
218	n/a	self.assertEqual(find_user_pass("f", "http://g.example.com:100"),
219	n/a	(None, None))
220	n/a	self.assertEqual(find_user_pass("f", "http://g.example.com:80"),
221	n/a	('10', 'j'))
222	n/a	self.assertEqual(find_user_pass("f", "http://g.example.com"),
223	n/a	('10', 'j'))
224	n/a	self.assertEqual(find_user_pass("g", "h.example.com"), ('11', 'k'))
225	n/a	self.assertEqual(find_user_pass("g", "h.example.com:80"), ('11', 'k'))
226	n/a	self.assertEqual(find_user_pass("g", "http://h.example.com:80"),
227	n/a	('11', 'k'))
228	n/a	self.assertEqual(find_user_pass("h", "i.example.com"), (None, None))
229	n/a	self.assertEqual(find_user_pass("h", "i.example.com:80"), ('12', 'l'))
230	n/a	self.assertEqual(find_user_pass("h", "http://i.example.com:80"),
231	n/a	('12', 'l'))
232	n/a	self.assertEqual(find_user_pass("i", "j.example.com"), ('13', 'm'))
233	n/a	self.assertEqual(find_user_pass("i", "j.example.com:80"),
234	n/a	(None, None))
235	n/a	self.assertEqual(find_user_pass("i", "http://j.example.com"),
236	n/a	('13', 'm'))
237	n/a	self.assertEqual(find_user_pass("i", "http://j.example.com:80"),
238	n/a	(None, None))
239	n/a
240	n/a
241	n/a	class MockOpener:
242	n/a	addheaders = []
243	n/a
244	n/a	def open(self, req, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
245	n/a	self.req, self.data, self.timeout = req, data, timeout
246	n/a
247	n/a	def error(self, proto, *args):
248	n/a	self.proto, self.args = proto, args
249	n/a
250	n/a
251	n/a	class MockFile:
252	n/a	def read(self, count=None):
253	n/a	pass
254	n/a
255	n/a	def readline(self, count=None):
256	n/a	pass
257	n/a
258	n/a	def close(self):
259	n/a	pass
260	n/a
261	n/a
262	n/a	class MockHeaders(dict):
263	n/a	def getheaders(self, name):
264	n/a	return list(self.values())
265	n/a
266	n/a
267	n/a	class MockResponse(io.StringIO):
268	n/a	def __init__(self, code, msg, headers, data, url=None):
269	n/a	io.StringIO.__init__(self, data)
270	n/a	self.code, self.msg, self.headers, self.url = code, msg, headers, url
271	n/a
272	n/a	def info(self):
273	n/a	return self.headers
274	n/a
275	n/a	def geturl(self):
276	n/a	return self.url
277	n/a
278	n/a
279	n/a	class MockCookieJar:
280	n/a	def add_cookie_header(self, request):
281	n/a	self.ach_req = request
282	n/a
283	n/a	def extract_cookies(self, response, request):
284	n/a	self.ec_req, self.ec_r = request, response
285	n/a
286	n/a
287	n/a	class FakeMethod:
288	n/a	def __init__(self, meth_name, action, handle):
289	n/a	self.meth_name = meth_name
290	n/a	self.handle = handle
291	n/a	self.action = action
292	n/a
293	n/a	def __call__(self, *args):
294	n/a	return self.handle(self.meth_name, self.action, *args)
295	n/a
296	n/a
297	n/a	class MockHTTPResponse(io.IOBase):
298	n/a	def __init__(self, fp, msg, status, reason):
299	n/a	self.fp = fp
300	n/a	self.msg = msg
301	n/a	self.status = status
302	n/a	self.reason = reason
303	n/a	self.code = 200
304	n/a
305	n/a	def read(self):
306	n/a	return ''
307	n/a
308	n/a	def info(self):
309	n/a	return {}
310	n/a
311	n/a	def geturl(self):
312	n/a	return self.url
313	n/a
314	n/a
315	n/a	class MockHTTPClass:
316	n/a	def __init__(self):
317	n/a	self.level = 0
318	n/a	self.req_headers = []
319	n/a	self.data = None
320	n/a	self.raise_on_endheaders = False
321	n/a	self.sock = None
322	n/a	self._tunnel_headers = {}
323	n/a
324	n/a	def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
325	n/a	self.host = host
326	n/a	self.timeout = timeout
327	n/a	return self
328	n/a
329	n/a	def set_debuglevel(self, level):
330	n/a	self.level = level
331	n/a
332	n/a	def set_tunnel(self, host, port=None, headers=None):
333	n/a	self._tunnel_host = host
334	n/a	self._tunnel_port = port
335	n/a	if headers:
336	n/a	self._tunnel_headers = headers
337	n/a	else:
338	n/a	self._tunnel_headers.clear()
339	n/a
340	n/a	def request(self, method, url, body=None, headers=None, *,
341	n/a	encode_chunked=False):
342	n/a	self.method = method
343	n/a	self.selector = url
344	n/a	if headers is not None:
345	n/a	self.req_headers += headers.items()
346	n/a	self.req_headers.sort()
347	n/a	if body:
348	n/a	self.data = body
349	n/a	self.encode_chunked = encode_chunked
350	n/a	if self.raise_on_endheaders:
351	n/a	raise OSError()
352	n/a
353	n/a	def getresponse(self):
354	n/a	return MockHTTPResponse(MockFile(), {}, 200, "OK")
355	n/a
356	n/a	def close(self):
357	n/a	pass
358	n/a
359	n/a
360	n/a	class MockHandler:
361	n/a	# useful for testing handler machinery
362	n/a	# see add_ordered_mock_handlers() docstring
363	n/a	handler_order = 500
364	n/a
365	n/a	def __init__(self, methods):
366	n/a	self._define_methods(methods)
367	n/a
368	n/a	def _define_methods(self, methods):
369	n/a	for spec in methods:
370	n/a	if len(spec) == 2:
371	n/a	name, action = spec
372	n/a	else:
373	n/a	name, action = spec, None
374	n/a	meth = FakeMethod(name, action, self.handle)
375	n/a	setattr(self.__class__, name, meth)
376	n/a
377	n/a	def handle(self, fn_name, action, args, *kwds):
378	n/a	self.parent.calls.append((self, fn_name, args, kwds))
379	n/a	if action is None:
380	n/a	return None
381	n/a	elif action == "return self":
382	n/a	return self
383	n/a	elif action == "return response":
384	n/a	res = MockResponse(200, "OK", {}, "")
385	n/a	return res
386	n/a	elif action == "return request":
387	n/a	return Request("http://blah/")
388	n/a	elif action.startswith("error"):
389	n/a	code = action[action.rfind(" ")+1:]
390	n/a	try:
391	n/a	code = int(code)
392	n/a	except ValueError:
393	n/a	pass
394	n/a	res = MockResponse(200, "OK", {}, "")
395	n/a	return self.parent.error("http", args[0], res, code, "", {})
396	n/a	elif action == "raise":
397	n/a	raise urllib.error.URLError("blah")
398	n/a	assert False
399	n/a
400	n/a	def close(self):
401	n/a	pass
402	n/a
403	n/a	def add_parent(self, parent):
404	n/a	self.parent = parent
405	n/a	self.parent.calls = []
406	n/a
407	n/a	def __lt__(self, other):
408	n/a	if not hasattr(other, "handler_order"):
409	n/a	# No handler_order, leave in original order. Yuck.
410	n/a	return True
411	n/a	return self.handler_order < other.handler_order
412	n/a
413	n/a
414	n/a	def add_ordered_mock_handlers(opener, meth_spec):
415	n/a	"""Create MockHandlers and add them to an OpenerDirector.
416	n/a
417	n/a	meth_spec: list of lists of tuples and strings defining methods to define
418	n/a	on handlers. eg:
419	n/a
420	n/a	[["http_error", "ftp_open"], ["http_open"]]
421	n/a
422	n/a	defines methods .http_error() and .ftp_open() on one handler, and
423	n/a	.http_open() on another. These methods just record their arguments and
424	n/a	return None. Using a tuple instead of a string causes the method to
425	n/a	perform some action (see MockHandler.handle()), eg:
426	n/a
427	n/a	[["http_error"], [("http_open", "return request")]]
428	n/a
429	n/a	defines .http_error() on one handler (which simply returns None), and
430	n/a	.http_open() on another handler, which returns a Request object.
431	n/a
432	n/a	"""
433	n/a	handlers = []
434	n/a	count = 0
435	n/a	for meths in meth_spec:
436	n/a	class MockHandlerSubclass(MockHandler):
437	n/a	pass
438	n/a
439	n/a	h = MockHandlerSubclass(meths)
440	n/a	h.handler_order += count
441	n/a	h.add_parent(opener)
442	n/a	count = count + 1
443	n/a	handlers.append(h)
444	n/a	opener.add_handler(h)
445	n/a	return handlers
446	n/a
447	n/a
448	n/a	def build_test_opener(*handler_instances):
449	n/a	opener = OpenerDirector()
450	n/a	for h in handler_instances:
451	n/a	opener.add_handler(h)
452	n/a	return opener
453	n/a
454	n/a
455	n/a	class MockHTTPHandler(urllib.request.BaseHandler):
456	n/a	# useful for testing redirections and auth
457	n/a	# sends supplied headers and code as first response
458	n/a	# sends 200 OK as second response
459	n/a	def __init__(self, code, headers):
460	n/a	self.code = code
461	n/a	self.headers = headers
462	n/a	self.reset()
463	n/a
464	n/a	def reset(self):
465	n/a	self._count = 0
466	n/a	self.requests = []
467	n/a
468	n/a	def http_open(self, req):
469	n/a	import email, copy
470	n/a	self.requests.append(copy.deepcopy(req))
471	n/a	if self._count == 0:
472	n/a	self._count = self._count + 1
473	n/a	name = http.client.responses[self.code]
474	n/a	msg = email.message_from_string(self.headers)
475	n/a	return self.parent.error(
476	n/a	"http", req, MockFile(), self.code, name, msg)
477	n/a	else:
478	n/a	self.req = req
479	n/a	msg = email.message_from_string("\r\n\r\n")
480	n/a	return MockResponse(200, "OK", msg, "", req.get_full_url())
481	n/a
482	n/a
483	n/a	class MockHTTPSHandler(urllib.request.AbstractHTTPHandler):
484	n/a	# Useful for testing the Proxy-Authorization request by verifying the
485	n/a	# properties of httpcon
486	n/a
487	n/a	def __init__(self, debuglevel=0):
488	n/a	urllib.request.AbstractHTTPHandler.__init__(self, debuglevel=debuglevel)
489	n/a	self.httpconn = MockHTTPClass()
490	n/a
491	n/a	def https_open(self, req):
492	n/a	return self.do_open(self.httpconn, req)
493	n/a
494	n/a
495	n/a	class MockHTTPHandlerCheckAuth(urllib.request.BaseHandler):
496	n/a	# useful for testing auth
497	n/a	# sends supplied code response
498	n/a	# checks if auth header is specified in request
499	n/a	def __init__(self, code):
500	n/a	self.code = code
501	n/a	self.has_auth_header = False
502	n/a
503	n/a	def reset(self):
504	n/a	self.has_auth_header = False
505	n/a
506	n/a	def http_open(self, req):
507	n/a	if req.has_header('Authorization'):
508	n/a	self.has_auth_header = True
509	n/a	name = http.client.responses[self.code]
510	n/a	return MockResponse(self.code, name, MockFile(), "", req.get_full_url())
511	n/a
512	n/a
513	n/a
514	n/a	class MockPasswordManager:
515	n/a	def add_password(self, realm, uri, user, password):
516	n/a	self.realm = realm
517	n/a	self.url = uri
518	n/a	self.user = user
519	n/a	self.password = password
520	n/a
521	n/a	def find_user_password(self, realm, authuri):
522	n/a	self.target_realm = realm
523	n/a	self.target_url = authuri
524	n/a	return self.user, self.password
525	n/a
526	n/a
527	n/a	class OpenerDirectorTests(unittest.TestCase):
528	n/a
529	n/a	def test_add_non_handler(self):
530	n/a	class NonHandler(object):
531	n/a	pass
532	n/a	self.assertRaises(TypeError,
533	n/a	OpenerDirector().add_handler, NonHandler())
534	n/a
535	n/a	def test_badly_named_methods(self):
536	n/a	# test work-around for three methods that accidentally follow the
537	n/a	# naming conventions for handler methods
538	n/a	# (_open() / _request() / *_response())
539	n/a
540	n/a	# These used to call the accidentally-named methods, causing a
541	n/a	# TypeError in real code; here, returning self from these mock
542	n/a	# methods would either cause no exception, or AttributeError.
543	n/a
544	n/a	from urllib.error import URLError
545	n/a
546	n/a	o = OpenerDirector()
547	n/a	meth_spec = [
548	n/a	[("do_open", "return self"), ("proxy_open", "return self")],
549	n/a	[("redirect_request", "return self")],
550	n/a	]
551	n/a	add_ordered_mock_handlers(o, meth_spec)
552	n/a	o.add_handler(urllib.request.UnknownHandler())
553	n/a	for scheme in "do", "proxy", "redirect":
554	n/a	self.assertRaises(URLError, o.open, scheme+"://example.com/")
555	n/a
556	n/a	def test_handled(self):
557	n/a	# handler returning non-None means no more handlers will be called
558	n/a	o = OpenerDirector()
559	n/a	meth_spec = [
560	n/a	["http_open", "ftp_open", "http_error_302"],
561	n/a	["ftp_open"],
562	n/a	[("http_open", "return self")],
563	n/a	[("http_open", "return self")],
564	n/a	]
565	n/a	handlers = add_ordered_mock_handlers(o, meth_spec)
566	n/a
567	n/a	req = Request("http://example.com/")
568	n/a	r = o.open(req)
569	n/a	# Second .http_open() gets called, third doesn't, since second returned
570	n/a	# non-None. Handlers without .http_open() never get any methods called
571	n/a	# on them.
572	n/a	# In fact, second mock handler defining .http_open() returns self
573	n/a	# (instead of response), which becomes the OpenerDirector's return
574	n/a	# value.
575	n/a	self.assertEqual(r, handlers[2])
576	n/a	calls = [(handlers[0], "http_open"), (handlers[2], "http_open")]
577	n/a	for expected, got in zip(calls, o.calls):
578	n/a	handler, name, args, kwds = got
579	n/a	self.assertEqual((handler, name), expected)
580	n/a	self.assertEqual(args, (req,))
581	n/a
582	n/a	def test_handler_order(self):
583	n/a	o = OpenerDirector()
584	n/a	handlers = []
585	n/a	for meths, handler_order in [([("http_open", "return self")], 500),
586	n/a	(["http_open"], 0)]:
587	n/a	class MockHandlerSubclass(MockHandler):
588	n/a	pass
589	n/a
590	n/a	h = MockHandlerSubclass(meths)
591	n/a	h.handler_order = handler_order
592	n/a	handlers.append(h)
593	n/a	o.add_handler(h)
594	n/a
595	n/a	o.open("http://example.com/")
596	n/a	# handlers called in reverse order, thanks to their sort order
597	n/a	self.assertEqual(o.calls[0][0], handlers[1])
598	n/a	self.assertEqual(o.calls[1][0], handlers[0])
599	n/a
600	n/a	def test_raise(self):
601	n/a	# raising URLError stops processing of request
602	n/a	o = OpenerDirector()
603	n/a	meth_spec = [
604	n/a	[("http_open", "raise")],
605	n/a	[("http_open", "return self")],
606	n/a	]
607	n/a	handlers = add_ordered_mock_handlers(o, meth_spec)
608	n/a
609	n/a	req = Request("http://example.com/")
610	n/a	self.assertRaises(urllib.error.URLError, o.open, req)
611	n/a	self.assertEqual(o.calls, [(handlers[0], "http_open", (req,), {})])
612	n/a
613	n/a	def test_http_error(self):
614	n/a	# XXX http_error_default
615	n/a	# http errors are a special case
616	n/a	o = OpenerDirector()
617	n/a	meth_spec = [
618	n/a	[("http_open", "error 302")],
619	n/a	[("http_error_400", "raise"), "http_open"],
620	n/a	[("http_error_302", "return response"), "http_error_303",
621	n/a	"http_error"],
622	n/a	[("http_error_302")],
623	n/a	]
624	n/a	handlers = add_ordered_mock_handlers(o, meth_spec)
625	n/a
626	n/a	class Unknown:
627	n/a	def __eq__(self, other):
628	n/a	return True
629	n/a
630	n/a	req = Request("http://example.com/")
631	n/a	o.open(req)
632	n/a	assert len(o.calls) == 2
633	n/a	calls = [(handlers[0], "http_open", (req,)),
634	n/a	(handlers[2], "http_error_302",
635	n/a	(req, Unknown(), 302, "", {}))]
636	n/a	for expected, got in zip(calls, o.calls):
637	n/a	handler, method_name, args = expected
638	n/a	self.assertEqual((handler, method_name), got[:2])
639	n/a	self.assertEqual(args, got[2])
640	n/a
641	n/a	def test_processors(self):
642	n/a	# _request / _response methods get called appropriately
643	n/a	o = OpenerDirector()
644	n/a	meth_spec = [
645	n/a	[("http_request", "return request"),
646	n/a	("http_response", "return response")],
647	n/a	[("http_request", "return request"),
648	n/a	("http_response", "return response")],
649	n/a	]
650	n/a	handlers = add_ordered_mock_handlers(o, meth_spec)
651	n/a
652	n/a	req = Request("http://example.com/")
653	n/a	o.open(req)
654	n/a	# processor methods are called on all handlers that define them,
655	n/a	# not just the first handler that handles the request
656	n/a	calls = [
657	n/a	(handlers[0], "http_request"), (handlers[1], "http_request"),
658	n/a	(handlers[0], "http_response"), (handlers[1], "http_response")]
659	n/a
660	n/a	for i, (handler, name, args, kwds) in enumerate(o.calls):
661	n/a	if i < 2:
662	n/a	# *_request
663	n/a	self.assertEqual((handler, name), calls[i])
664	n/a	self.assertEqual(len(args), 1)
665	n/a	self.assertIsInstance(args[0], Request)
666	n/a	else:
667	n/a	# *_response
668	n/a	self.assertEqual((handler, name), calls[i])
669	n/a	self.assertEqual(len(args), 2)
670	n/a	self.assertIsInstance(args[0], Request)
671	n/a	# response from opener.open is None, because there's no
672	n/a	# handler that defines http_open to handle it
673	n/a	if args[1] is not None:
674	n/a	self.assertIsInstance(args[1], MockResponse)
675	n/a
676	n/a
677	n/a	def sanepathname2url(path):
678	n/a	try:
679	n/a	path.encode("utf-8")
680	n/a	except UnicodeEncodeError:
681	n/a	raise unittest.SkipTest("path is not encodable to utf8")
682	n/a	urlpath = urllib.request.pathname2url(path)
683	n/a	if os.name == "nt" and urlpath.startswith("///"):
684	n/a	urlpath = urlpath[2:]
685	n/a	# XXX don't ask me about the mac...
686	n/a	return urlpath
687	n/a
688	n/a
689	n/a	class HandlerTests(unittest.TestCase):
690	n/a
691	n/a	def test_ftp(self):
692	n/a	class MockFTPWrapper:
693	n/a	def __init__(self, data):
694	n/a	self.data = data
695	n/a
696	n/a	def retrfile(self, filename, filetype):
697	n/a	self.filename, self.filetype = filename, filetype
698	n/a	return io.StringIO(self.data), len(self.data)
699	n/a
700	n/a	def close(self):
701	n/a	pass
702	n/a
703	n/a	class NullFTPHandler(urllib.request.FTPHandler):
704	n/a	def __init__(self, data):
705	n/a	self.data = data
706	n/a
707	n/a	def connect_ftp(self, user, passwd, host, port, dirs,
708	n/a	timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
709	n/a	self.user, self.passwd = user, passwd
710	n/a	self.host, self.port = host, port
711	n/a	self.dirs = dirs
712	n/a	self.ftpwrapper = MockFTPWrapper(self.data)
713	n/a	return self.ftpwrapper
714	n/a
715	n/a	import ftplib
716	n/a	data = "rheum rhaponicum"
717	n/a	h = NullFTPHandler(data)
718	n/a	h.parent = MockOpener()
719	n/a
720	n/a	for url, host, port, user, passwd, type_, dirs, filename, mimetype in [
721	n/a	("ftp://localhost/foo/bar/baz.html",
722	n/a	"localhost", ftplib.FTP_PORT, "", "", "I",
723	n/a	["foo", "bar"], "baz.html", "text/html"),
724	n/a	("ftp://parrot@localhost/foo/bar/baz.html",
725	n/a	"localhost", ftplib.FTP_PORT, "parrot", "", "I",
726	n/a	["foo", "bar"], "baz.html", "text/html"),
727	n/a	("ftp://%25parrot@localhost/foo/bar/baz.html",
728	n/a	"localhost", ftplib.FTP_PORT, "%parrot", "", "I",
729	n/a	["foo", "bar"], "baz.html", "text/html"),
730	n/a	("ftp://%2542parrot@localhost/foo/bar/baz.html",
731	n/a	"localhost", ftplib.FTP_PORT, "%42parrot", "", "I",
732	n/a	["foo", "bar"], "baz.html", "text/html"),
733	n/a	("ftp://localhost:80/foo/bar/",
734	n/a	"localhost", 80, "", "", "D",
735	n/a	["foo", "bar"], "", None),
736	n/a	("ftp://localhost/baz.gif;type=a",
737	n/a	"localhost", ftplib.FTP_PORT, "", "", "A",
738	n/a	[], "baz.gif", None), # XXX really this should guess image/gif
739	n/a	]:
740	n/a	req = Request(url)
741	n/a	req.timeout = None
742	n/a	r = h.ftp_open(req)
743	n/a	# ftp authentication not yet implemented by FTPHandler
744	n/a	self.assertEqual(h.user, user)
745	n/a	self.assertEqual(h.passwd, passwd)
746	n/a	self.assertEqual(h.host, socket.gethostbyname(host))
747	n/a	self.assertEqual(h.port, port)
748	n/a	self.assertEqual(h.dirs, dirs)
749	n/a	self.assertEqual(h.ftpwrapper.filename, filename)
750	n/a	self.assertEqual(h.ftpwrapper.filetype, type_)
751	n/a	headers = r.info()
752	n/a	self.assertEqual(headers.get("Content-type"), mimetype)
753	n/a	self.assertEqual(int(headers["Content-length"]), len(data))
754	n/a
755	n/a	def test_file(self):
756	n/a	import email.utils
757	n/a	h = urllib.request.FileHandler()
758	n/a	o = h.parent = MockOpener()
759	n/a
760	n/a	TESTFN = support.TESTFN
761	n/a	urlpath = sanepathname2url(os.path.abspath(TESTFN))
762	n/a	towrite = b"hello, world\n"
763	n/a	urls = [
764	n/a	"file://localhost%s" % urlpath,
765	n/a	"file://%s" % urlpath,
766	n/a	"file://%s%s" % (socket.gethostbyname('localhost'), urlpath),
767	n/a	]
768	n/a	try:
769	n/a	localaddr = socket.gethostbyname(socket.gethostname())
770	n/a	except socket.gaierror:
771	n/a	localaddr = ''
772	n/a	if localaddr:
773	n/a	urls.append("file://%s%s" % (localaddr, urlpath))
774	n/a
775	n/a	for url in urls:
776	n/a	f = open(TESTFN, "wb")
777	n/a	try:
778	n/a	try:
779	n/a	f.write(towrite)
780	n/a	finally:
781	n/a	f.close()
782	n/a
783	n/a	r = h.file_open(Request(url))
784	n/a	try:
785	n/a	data = r.read()
786	n/a	headers = r.info()
787	n/a	respurl = r.geturl()
788	n/a	finally:
789	n/a	r.close()
790	n/a	stats = os.stat(TESTFN)
791	n/a	modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
792	n/a	finally:
793	n/a	os.remove(TESTFN)
794	n/a	self.assertEqual(data, towrite)
795	n/a	self.assertEqual(headers["Content-type"], "text/plain")
796	n/a	self.assertEqual(headers["Content-length"], "13")
797	n/a	self.assertEqual(headers["Last-modified"], modified)
798	n/a	self.assertEqual(respurl, url)
799	n/a
800	n/a	for url in [
801	n/a	"file://localhost:80%s" % urlpath,
802	n/a	"file:///file_does_not_exist.txt",
803	n/a	"file://not-a-local-host.com//dir/file.txt",
804	n/a	"file://%s:80%s/%s" % (socket.gethostbyname('localhost'),
805	n/a	os.getcwd(), TESTFN),
806	n/a	"file://somerandomhost.ontheinternet.com%s/%s" %
807	n/a	(os.getcwd(), TESTFN),
808	n/a	]:
809	n/a	try:
810	n/a	f = open(TESTFN, "wb")
811	n/a	try:
812	n/a	f.write(towrite)
813	n/a	finally:
814	n/a	f.close()
815	n/a
816	n/a	self.assertRaises(urllib.error.URLError,
817	n/a	h.file_open, Request(url))
818	n/a	finally:
819	n/a	os.remove(TESTFN)
820	n/a
821	n/a	h = urllib.request.FileHandler()
822	n/a	o = h.parent = MockOpener()
823	n/a	# XXXX why does // mean ftp (and /// mean not ftp!), and where
824	n/a	# is file: scheme specified? I think this is really a bug, and
825	n/a	# what was intended was to distinguish between URLs like:
826	n/a	# file:/blah.txt (a file)
827	n/a	# file://localhost/blah.txt (a file)
828	n/a	# file:///blah.txt (a file)
829	n/a	# file://ftp.example.com/blah.txt (an ftp URL)
830	n/a	for url, ftp in [
831	n/a	("file://ftp.example.com//foo.txt", False),
832	n/a	("file://ftp.example.com///foo.txt", False),
833	n/a	# XXXX bug: fails with OSError, should be URLError
834	n/a	("file://ftp.example.com/foo.txt", False),
835	n/a	("file://somehost//foo/something.txt", False),
836	n/a	("file://localhost//foo/something.txt", False),
837	n/a	]:
838	n/a	req = Request(url)
839	n/a	try:
840	n/a	h.file_open(req)
841	n/a	# XXXX remove OSError when bug fixed
842	n/a	except (urllib.error.URLError, OSError):
843	n/a	self.assertFalse(ftp)
844	n/a	else:
845	n/a	self.assertIs(o.req, req)
846	n/a	self.assertEqual(req.type, "ftp")
847	n/a	self.assertEqual(req.type == "ftp", ftp)
848	n/a
849	n/a	def test_http(self):
850	n/a
851	n/a	h = urllib.request.AbstractHTTPHandler()
852	n/a	o = h.parent = MockOpener()
853	n/a
854	n/a	url = "http://example.com/"
855	n/a	for method, data in [("GET", None), ("POST", b"blah")]:
856	n/a	req = Request(url, data, {"Foo": "bar"})
857	n/a	req.timeout = None
858	n/a	req.add_unredirected_header("Spam", "eggs")
859	n/a	http = MockHTTPClass()
860	n/a	r = h.do_open(http, req)
861	n/a
862	n/a	# result attributes
863	n/a	r.read; r.readline # wrapped MockFile methods
864	n/a	r.info; r.geturl # addinfourl methods
865	n/a	r.code, r.msg == 200, "OK" # added from MockHTTPClass.getreply()
866	n/a	hdrs = r.info()
867	n/a	hdrs.get; hdrs.__contains__ # r.info() gives dict from .getreply()
868	n/a	self.assertEqual(r.geturl(), url)
869	n/a
870	n/a	self.assertEqual(http.host, "example.com")
871	n/a	self.assertEqual(http.level, 0)
872	n/a	self.assertEqual(http.method, method)
873	n/a	self.assertEqual(http.selector, "/")
874	n/a	self.assertEqual(http.req_headers,
875	n/a	[("Connection", "close"),
876	n/a	("Foo", "bar"), ("Spam", "eggs")])
877	n/a	self.assertEqual(http.data, data)
878	n/a
879	n/a	# check OSError converted to URLError
880	n/a	http.raise_on_endheaders = True
881	n/a	self.assertRaises(urllib.error.URLError, h.do_open, http, req)
882	n/a
883	n/a	# Check for TypeError on POST data which is str.
884	n/a	req = Request("http://example.com/","badpost")
885	n/a	self.assertRaises(TypeError, h.do_request_, req)
886	n/a
887	n/a	# check adding of standard headers
888	n/a	o.addheaders = [("Spam", "eggs")]
889	n/a	for data in b"", None: # POST, GET
890	n/a	req = Request("http://example.com/", data)
891	n/a	r = MockResponse(200, "OK", {}, "")
892	n/a	newreq = h.do_request_(req)
893	n/a	if data is None: # GET
894	n/a	self.assertNotIn("Content-length", req.unredirected_hdrs)
895	n/a	self.assertNotIn("Content-type", req.unredirected_hdrs)
896	n/a	else: # POST
897	n/a	self.assertEqual(req.unredirected_hdrs["Content-length"], "0")
898	n/a	self.assertEqual(req.unredirected_hdrs["Content-type"],
899	n/a	"application/x-www-form-urlencoded")
900	n/a	# XXX the details of Host could be better tested
901	n/a	self.assertEqual(req.unredirected_hdrs["Host"], "example.com")
902	n/a	self.assertEqual(req.unredirected_hdrs["Spam"], "eggs")
903	n/a
904	n/a	# don't clobber existing headers
905	n/a	req.add_unredirected_header("Content-length", "foo")
906	n/a	req.add_unredirected_header("Content-type", "bar")
907	n/a	req.add_unredirected_header("Host", "baz")
908	n/a	req.add_unredirected_header("Spam", "foo")
909	n/a	newreq = h.do_request_(req)
910	n/a	self.assertEqual(req.unredirected_hdrs["Content-length"], "foo")
911	n/a	self.assertEqual(req.unredirected_hdrs["Content-type"], "bar")
912	n/a	self.assertEqual(req.unredirected_hdrs["Host"], "baz")
913	n/a	self.assertEqual(req.unredirected_hdrs["Spam"], "foo")
914	n/a
915	n/a	def test_http_body_file(self):
916	n/a	# A regular file - chunked encoding is used unless Content Length is
917	n/a	# already set.
918	n/a
919	n/a	h = urllib.request.AbstractHTTPHandler()
920	n/a	o = h.parent = MockOpener()
921	n/a
922	n/a	file_obj = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
923	n/a	file_path = file_obj.name
924	n/a	file_obj.close()
925	n/a	self.addCleanup(os.unlink, file_path)
926	n/a
927	n/a	with open(file_path, "rb") as f:
928	n/a	req = Request("http://example.com/", f, {})
929	n/a	newreq = h.do_request_(req)
930	n/a	te = newreq.get_header('Transfer-encoding')
931	n/a	self.assertEqual(te, "chunked")
932	n/a	self.assertFalse(newreq.has_header('Content-length'))
933	n/a
934	n/a	with open(file_path, "rb") as f:
935	n/a	req = Request("http://example.com/", f, {"Content-Length": 30})
936	n/a	newreq = h.do_request_(req)
937	n/a	self.assertEqual(int(newreq.get_header('Content-length')), 30)
938	n/a	self.assertFalse(newreq.has_header("Transfer-encoding"))
939	n/a
940	n/a	def test_http_body_fileobj(self):
941	n/a	# A file object - chunked encoding is used
942	n/a	# unless Content Length is already set.
943	n/a	# (Note that there are some subtle differences to a regular
944	n/a	# file, that is why we are testing both cases.)
945	n/a
946	n/a	h = urllib.request.AbstractHTTPHandler()
947	n/a	o = h.parent = MockOpener()
948	n/a	file_obj = io.BytesIO()
949	n/a
950	n/a	req = Request("http://example.com/", file_obj, {})
951	n/a	newreq = h.do_request_(req)
952	n/a	self.assertEqual(newreq.get_header('Transfer-encoding'), 'chunked')
953	n/a	self.assertFalse(newreq.has_header('Content-length'))
954	n/a
955	n/a	headers = {"Content-Length": 30}
956	n/a	req = Request("http://example.com/", file_obj, headers)
957	n/a	newreq = h.do_request_(req)
958	n/a	self.assertEqual(int(newreq.get_header('Content-length')), 30)
959	n/a	self.assertFalse(newreq.has_header("Transfer-encoding"))
960	n/a
961	n/a	file_obj.close()
962	n/a
963	n/a	def test_http_body_pipe(self):
964	n/a	# A file reading from a pipe.
965	n/a	# A pipe cannot be seek'ed. There is no way to determine the
966	n/a	# content length up front. Thus, do_request_() should fall
967	n/a	# back to Transfer-encoding chunked.
968	n/a
969	n/a	h = urllib.request.AbstractHTTPHandler()
970	n/a	o = h.parent = MockOpener()
971	n/a
972	n/a	cmd = [sys.executable, "-c", r"pass"]
973	n/a	for headers in {}, {"Content-Length": 30}:
974	n/a	with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc:
975	n/a	req = Request("http://example.com/", proc.stdout, headers)
976	n/a	newreq = h.do_request_(req)
977	n/a	if not headers:
978	n/a	self.assertEqual(newreq.get_header('Content-length'), None)
979	n/a	self.assertEqual(newreq.get_header('Transfer-encoding'),
980	n/a	'chunked')
981	n/a	else:
982	n/a	self.assertEqual(int(newreq.get_header('Content-length')),
983	n/a	30)
984	n/a
985	n/a	def test_http_body_iterable(self):
986	n/a	# Generic iterable. There is no way to determine the content
987	n/a	# length up front. Fall back to Transfer-encoding chunked.
988	n/a
989	n/a	h = urllib.request.AbstractHTTPHandler()
990	n/a	o = h.parent = MockOpener()
991	n/a
992	n/a	def iterable_body():
993	n/a	yield b"one"
994	n/a
995	n/a	for headers in {}, {"Content-Length": 11}:
996	n/a	req = Request("http://example.com/", iterable_body(), headers)
997	n/a	newreq = h.do_request_(req)
998	n/a	if not headers:
999	n/a	self.assertEqual(newreq.get_header('Content-length'), None)
1000	n/a	self.assertEqual(newreq.get_header('Transfer-encoding'),
1001	n/a	'chunked')
1002	n/a	else:
1003	n/a	self.assertEqual(int(newreq.get_header('Content-length')), 11)
1004	n/a
1005	n/a	def test_http_body_empty_seq(self):
1006	n/a	# Zero-length iterable body should be treated like any other iterable
1007	n/a	h = urllib.request.AbstractHTTPHandler()
1008	n/a	h.parent = MockOpener()
1009	n/a	req = h.do_request_(Request("http://example.com/", ()))
1010	n/a	self.assertEqual(req.get_header("Transfer-encoding"), "chunked")
1011	n/a	self.assertFalse(req.has_header("Content-length"))
1012	n/a
1013	n/a	def test_http_body_array(self):
1014	n/a	# array.array Iterable - Content Length is calculated
1015	n/a
1016	n/a	h = urllib.request.AbstractHTTPHandler()
1017	n/a	o = h.parent = MockOpener()
1018	n/a
1019	n/a	iterable_array = array.array("I",[1,2,3,4])
1020	n/a
1021	n/a	for headers in {}, {"Content-Length": 16}:
1022	n/a	req = Request("http://example.com/", iterable_array, headers)
1023	n/a	newreq = h.do_request_(req)
1024	n/a	self.assertEqual(int(newreq.get_header('Content-length')),16)
1025	n/a
1026	n/a	def test_http_handler_debuglevel(self):
1027	n/a	o = OpenerDirector()
1028	n/a	h = MockHTTPSHandler(debuglevel=1)
1029	n/a	o.add_handler(h)
1030	n/a	o.open("https://www.example.com")
1031	n/a	self.assertEqual(h._debuglevel, 1)
1032	n/a
1033	n/a	def test_http_doubleslash(self):
1034	n/a	# Checks the presence of any unnecessary double slash in url does not
1035	n/a	# break anything. Previously, a double slash directly after the host
1036	n/a	# could cause incorrect parsing.
1037	n/a	h = urllib.request.AbstractHTTPHandler()
1038	n/a	h.parent = MockOpener()
1039	n/a
1040	n/a	data = b""
1041	n/a	ds_urls = [
1042	n/a	"http://example.com/foo/bar/baz.html",
1043	n/a	"http://example.com//foo/bar/baz.html",
1044	n/a	"http://example.com/foo//bar/baz.html",
1045	n/a	"http://example.com/foo/bar//baz.html"
1046	n/a	]
1047	n/a
1048	n/a	for ds_url in ds_urls:
1049	n/a	ds_req = Request(ds_url, data)
1050	n/a
1051	n/a	# Check whether host is determined correctly if there is no proxy
1052	n/a	np_ds_req = h.do_request_(ds_req)
1053	n/a	self.assertEqual(np_ds_req.unredirected_hdrs["Host"], "example.com")
1054	n/a
1055	n/a	# Check whether host is determined correctly if there is a proxy
1056	n/a	ds_req.set_proxy("someproxy:3128", None)
1057	n/a	p_ds_req = h.do_request_(ds_req)
1058	n/a	self.assertEqual(p_ds_req.unredirected_hdrs["Host"], "example.com")
1059	n/a
1060	n/a	def test_full_url_setter(self):
1061	n/a	# Checks to ensure that components are set correctly after setting the
1062	n/a	# full_url of a Request object
1063	n/a
1064	n/a	urls = [
1065	n/a	'http://example.com?foo=bar#baz',
1066	n/a	'http://example.com?foo=bar&spam=eggs#bash',
1067	n/a	'http://example.com',
1068	n/a	]
1069	n/a
1070	n/a	# testing a reusable request instance, but the url parameter is
1071	n/a	# required, so just use a dummy one to instantiate
1072	n/a	r = Request('http://example.com')
1073	n/a	for url in urls:
1074	n/a	r.full_url = url
1075	n/a	parsed = urlparse(url)
1076	n/a
1077	n/a	self.assertEqual(r.get_full_url(), url)
1078	n/a	# full_url setter uses splittag to split into components.
1079	n/a	# splittag sets the fragment as None while urlparse sets it to ''
1080	n/a	self.assertEqual(r.fragment or '', parsed.fragment)
1081	n/a	self.assertEqual(urlparse(r.get_full_url()).query, parsed.query)
1082	n/a
1083	n/a	def test_full_url_deleter(self):
1084	n/a	r = Request('http://www.example.com')
1085	n/a	del r.full_url
1086	n/a	self.assertIsNone(r.full_url)
1087	n/a	self.assertIsNone(r.fragment)
1088	n/a	self.assertEqual(r.selector, '')
1089	n/a
1090	n/a	def test_fixpath_in_weirdurls(self):
1091	n/a	# Issue4493: urllib2 to supply '/' when to urls where path does not
1092	n/a	# start with'/'
1093	n/a
1094	n/a	h = urllib.request.AbstractHTTPHandler()
1095	n/a	h.parent = MockOpener()
1096	n/a
1097	n/a	weird_url = 'http://www.python.org?getspam'
1098	n/a	req = Request(weird_url)
1099	n/a	newreq = h.do_request_(req)
1100	n/a	self.assertEqual(newreq.host, 'www.python.org')
1101	n/a	self.assertEqual(newreq.selector, '/?getspam')
1102	n/a
1103	n/a	url_without_path = 'http://www.python.org'
1104	n/a	req = Request(url_without_path)
1105	n/a	newreq = h.do_request_(req)
1106	n/a	self.assertEqual(newreq.host, 'www.python.org')
1107	n/a	self.assertEqual(newreq.selector, '')
1108	n/a
1109	n/a	def test_errors(self):
1110	n/a	h = urllib.request.HTTPErrorProcessor()
1111	n/a	o = h.parent = MockOpener()
1112	n/a
1113	n/a	url = "http://example.com/"
1114	n/a	req = Request(url)
1115	n/a	# all 2xx are passed through
1116	n/a	r = MockResponse(200, "OK", {}, "", url)
1117	n/a	newr = h.http_response(req, r)
1118	n/a	self.assertIs(r, newr)
1119	n/a	self.assertFalse(hasattr(o, "proto")) # o.error not called
1120	n/a	r = MockResponse(202, "Accepted", {}, "", url)
1121	n/a	newr = h.http_response(req, r)
1122	n/a	self.assertIs(r, newr)
1123	n/a	self.assertFalse(hasattr(o, "proto")) # o.error not called
1124	n/a	r = MockResponse(206, "Partial content", {}, "", url)
1125	n/a	newr = h.http_response(req, r)
1126	n/a	self.assertIs(r, newr)
1127	n/a	self.assertFalse(hasattr(o, "proto")) # o.error not called
1128	n/a	# anything else calls o.error (and MockOpener returns None, here)
1129	n/a	r = MockResponse(502, "Bad gateway", {}, "", url)
1130	n/a	self.assertIsNone(h.http_response(req, r))
1131	n/a	self.assertEqual(o.proto, "http") # o.error called
1132	n/a	self.assertEqual(o.args, (req, r, 502, "Bad gateway", {}))
1133	n/a
1134	n/a	def test_cookies(self):
1135	n/a	cj = MockCookieJar()
1136	n/a	h = urllib.request.HTTPCookieProcessor(cj)
1137	n/a	h.parent = MockOpener()
1138	n/a
1139	n/a	req = Request("http://example.com/")
1140	n/a	r = MockResponse(200, "OK", {}, "")
1141	n/a	newreq = h.http_request(req)
1142	n/a	self.assertIs(cj.ach_req, req)
1143	n/a	self.assertIs(cj.ach_req, newreq)
1144	n/a	self.assertEqual(req.origin_req_host, "example.com")
1145	n/a	self.assertFalse(req.unverifiable)
1146	n/a	newr = h.http_response(req, r)
1147	n/a	self.assertIs(cj.ec_req, req)
1148	n/a	self.assertIs(cj.ec_r, r)
1149	n/a	self.assertIs(r, newr)
1150	n/a
1151	n/a	def test_redirect(self):
1152	n/a	from_url = "http://example.com/a.html"
1153	n/a	to_url = "http://example.com/b.html"
1154	n/a	h = urllib.request.HTTPRedirectHandler()
1155	n/a	o = h.parent = MockOpener()
1156	n/a
1157	n/a	# ordinary redirect behaviour
1158	n/a	for code in 301, 302, 303, 307:
1159	n/a	for data in None, "blah\nblah\n":
1160	n/a	method = getattr(h, "http_error_%s" % code)
1161	n/a	req = Request(from_url, data)
1162	n/a	req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1163	n/a	req.add_header("Nonsense", "viking=withhold")
1164	n/a	if data is not None:
1165	n/a	req.add_header("Content-Length", str(len(data)))
1166	n/a	req.add_unredirected_header("Spam", "spam")
1167	n/a	try:
1168	n/a	method(req, MockFile(), code, "Blah",
1169	n/a	MockHeaders({"location": to_url}))
1170	n/a	except urllib.error.HTTPError:
1171	n/a	# 307 in response to POST requires user OK
1172	n/a	self.assertEqual(code, 307)
1173	n/a	self.assertIsNotNone(data)
1174	n/a	self.assertEqual(o.req.get_full_url(), to_url)
1175	n/a	try:
1176	n/a	self.assertEqual(o.req.get_method(), "GET")
1177	n/a	except AttributeError:
1178	n/a	self.assertFalse(o.req.data)
1179	n/a
1180	n/a	# now it's a GET, there should not be headers regarding content
1181	n/a	# (possibly dragged from before being a POST)
1182	n/a	headers = [x.lower() for x in o.req.headers]
1183	n/a	self.assertNotIn("content-length", headers)
1184	n/a	self.assertNotIn("content-type", headers)
1185	n/a
1186	n/a	self.assertEqual(o.req.headers["Nonsense"],
1187	n/a	"viking=withhold")
1188	n/a	self.assertNotIn("Spam", o.req.headers)
1189	n/a	self.assertNotIn("Spam", o.req.unredirected_hdrs)
1190	n/a
1191	n/a	# loop detection
1192	n/a	req = Request(from_url)
1193	n/a	req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1194	n/a
1195	n/a	def redirect(h, req, url=to_url):
1196	n/a	h.http_error_302(req, MockFile(), 302, "Blah",
1197	n/a	MockHeaders({"location": url}))
1198	n/a	# Note that the original request shares the same record of
1199	n/a	# redirections with the sub-requests caused by the redirections.
1200	n/a
1201	n/a	# detect infinite loop redirect of a URL to itself
1202	n/a	req = Request(from_url, origin_req_host="example.com")
1203	n/a	count = 0
1204	n/a	req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1205	n/a	try:
1206	n/a	while 1:
1207	n/a	redirect(h, req, "http://example.com/")
1208	n/a	count = count + 1
1209	n/a	except urllib.error.HTTPError:
1210	n/a	# don't stop until max_repeats, because cookies may introduce state
1211	n/a	self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_repeats)
1212	n/a
1213	n/a	# detect endless non-repeating chain of redirects
1214	n/a	req = Request(from_url, origin_req_host="example.com")
1215	n/a	count = 0
1216	n/a	req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1217	n/a	try:
1218	n/a	while 1:
1219	n/a	redirect(h, req, "http://example.com/%d" % count)
1220	n/a	count = count + 1
1221	n/a	except urllib.error.HTTPError:
1222	n/a	self.assertEqual(count,
1223	n/a	urllib.request.HTTPRedirectHandler.max_redirections)
1224	n/a
1225	n/a	def test_invalid_redirect(self):
1226	n/a	from_url = "http://example.com/a.html"
1227	n/a	valid_schemes = ['http','https','ftp']
1228	n/a	invalid_schemes = ['file','imap','ldap']
1229	n/a	schemeless_url = "example.com/b.html"
1230	n/a	h = urllib.request.HTTPRedirectHandler()
1231	n/a	o = h.parent = MockOpener()
1232	n/a	req = Request(from_url)
1233	n/a	req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1234	n/a
1235	n/a	for scheme in invalid_schemes:
1236	n/a	invalid_url = scheme + '://' + schemeless_url
1237	n/a	self.assertRaises(urllib.error.HTTPError, h.http_error_302,
1238	n/a	req, MockFile(), 302, "Security Loophole",
1239	n/a	MockHeaders({"location": invalid_url}))
1240	n/a
1241	n/a	for scheme in valid_schemes:
1242	n/a	valid_url = scheme + '://' + schemeless_url
1243	n/a	h.http_error_302(req, MockFile(), 302, "That's fine",
1244	n/a	MockHeaders({"location": valid_url}))
1245	n/a	self.assertEqual(o.req.get_full_url(), valid_url)
1246	n/a
1247	n/a	def test_relative_redirect(self):
1248	n/a	from_url = "http://example.com/a.html"
1249	n/a	relative_url = "/b.html"
1250	n/a	h = urllib.request.HTTPRedirectHandler()
1251	n/a	o = h.parent = MockOpener()
1252	n/a	req = Request(from_url)
1253	n/a	req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1254	n/a
1255	n/a	valid_url = urllib.parse.urljoin(from_url,relative_url)
1256	n/a	h.http_error_302(req, MockFile(), 302, "That's fine",
1257	n/a	MockHeaders({"location": valid_url}))
1258	n/a	self.assertEqual(o.req.get_full_url(), valid_url)
1259	n/a
1260	n/a	def test_cookie_redirect(self):
1261	n/a	# cookies shouldn't leak into redirected requests
1262	n/a	from http.cookiejar import CookieJar
1263	n/a	from test.test_http_cookiejar import interact_netscape
1264	n/a
1265	n/a	cj = CookieJar()
1266	n/a	interact_netscape(cj, "http://www.example.com/", "spam=eggs")
1267	n/a	hh = MockHTTPHandler(302, "Location: http://www.cracker.com/\r\n\r\n")
1268	n/a	hdeh = urllib.request.HTTPDefaultErrorHandler()
1269	n/a	hrh = urllib.request.HTTPRedirectHandler()
1270	n/a	cp = urllib.request.HTTPCookieProcessor(cj)
1271	n/a	o = build_test_opener(hh, hdeh, hrh, cp)
1272	n/a	o.open("http://www.example.com/")
1273	n/a	self.assertFalse(hh.req.has_header("Cookie"))
1274	n/a
1275	n/a	def test_redirect_fragment(self):
1276	n/a	redirected_url = 'http://www.example.com/index.html#OK\r\n\r\n'
1277	n/a	hh = MockHTTPHandler(302, 'Location: ' + redirected_url)
1278	n/a	hdeh = urllib.request.HTTPDefaultErrorHandler()
1279	n/a	hrh = urllib.request.HTTPRedirectHandler()
1280	n/a	o = build_test_opener(hh, hdeh, hrh)
1281	n/a	fp = o.open('http://www.example.com')
1282	n/a	self.assertEqual(fp.geturl(), redirected_url.strip())
1283	n/a
1284	n/a	def test_redirect_no_path(self):
1285	n/a	# Issue 14132: Relative redirect strips original path
1286	n/a	real_class = http.client.HTTPConnection
1287	n/a	response1 = b"HTTP/1.1 302 Found\r\nLocation: ?query\r\n\r\n"
1288	n/a	http.client.HTTPConnection = test_urllib.fakehttp(response1)
1289	n/a	self.addCleanup(setattr, http.client, "HTTPConnection", real_class)
1290	n/a	urls = iter(("/path", "/path?query"))
1291	n/a	def request(conn, method, url, pos, *kw):
1292	n/a	self.assertEqual(url, next(urls))
1293	n/a	real_class.request(conn, method, url, pos, *kw)
1294	n/a	# Change response for subsequent connection
1295	n/a	conn.__class__.fakedata = b"HTTP/1.1 200 OK\r\n\r\nHello!"
1296	n/a	http.client.HTTPConnection.request = request
1297	n/a	fp = urllib.request.urlopen("http://python.org/path")
1298	n/a	self.assertEqual(fp.geturl(), "http://python.org/path?query")
1299	n/a
1300	n/a	def test_redirect_encoding(self):
1301	n/a	# Some characters in the redirect target may need special handling,
1302	n/a	# but most ASCII characters should be treated as already encoded
1303	n/a	class Handler(urllib.request.HTTPHandler):
1304	n/a	def http_open(self, req):
1305	n/a	result = self.do_open(self.connection, req)
1306	n/a	self.last_buf = self.connection.buf
1307	n/a	# Set up a normal response for the next request
1308	n/a	self.connection = test_urllib.fakehttp(
1309	n/a	b'HTTP/1.1 200 OK\r\n'
1310	n/a	b'Content-Length: 3\r\n'
1311	n/a	b'\r\n'
1312	n/a	b'123'
1313	n/a	)
1314	n/a	return result
1315	n/a	handler = Handler()
1316	n/a	opener = urllib.request.build_opener(handler)
1317	n/a	tests = (
1318	n/a	(b'/p\xC3\xA5-dansk/', b'/p%C3%A5-dansk/'),
1319	n/a	(b'/spaced%20path/', b'/spaced%20path/'),
1320	n/a	(b'/spaced path/', b'/spaced%20path/'),
1321	n/a	(b'/?p\xC3\xA5-dansk', b'/?p%C3%A5-dansk'),
1322	n/a	)
1323	n/a	for [location, result] in tests:
1324	n/a	with self.subTest(repr(location)):
1325	n/a	handler.connection = test_urllib.fakehttp(
1326	n/a	b'HTTP/1.1 302 Redirect\r\n'
1327	n/a	b'Location: ' + location + b'\r\n'
1328	n/a	b'\r\n'
1329	n/a	)
1330	n/a	response = opener.open('http://example.com/')
1331	n/a	expected = b'GET ' + result + b' '
1332	n/a	request = handler.last_buf
1333	n/a	self.assertTrue(request.startswith(expected), repr(request))
1334	n/a
1335	n/a	def test_proxy(self):
1336	n/a	o = OpenerDirector()
1337	n/a	ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128"))
1338	n/a	o.add_handler(ph)
1339	n/a	meth_spec = [
1340	n/a	[("http_open", "return response")]
1341	n/a	]
1342	n/a	handlers = add_ordered_mock_handlers(o, meth_spec)
1343	n/a
1344	n/a	req = Request("http://acme.example.com/")
1345	n/a	self.assertEqual(req.host, "acme.example.com")
1346	n/a	o.open(req)
1347	n/a	self.assertEqual(req.host, "proxy.example.com:3128")
1348	n/a
1349	n/a	self.assertEqual([(handlers[0], "http_open")],
1350	n/a	[tup[0:2] for tup in o.calls])
1351	n/a
1352	n/a	def test_proxy_no_proxy(self):
1353	n/a	os.environ['no_proxy'] = 'python.org'
1354	n/a	o = OpenerDirector()
1355	n/a	ph = urllib.request.ProxyHandler(dict(http="proxy.example.com"))
1356	n/a	o.add_handler(ph)
1357	n/a	req = Request("http://www.perl.org/")
1358	n/a	self.assertEqual(req.host, "www.perl.org")
1359	n/a	o.open(req)
1360	n/a	self.assertEqual(req.host, "proxy.example.com")
1361	n/a	req = Request("http://www.python.org")
1362	n/a	self.assertEqual(req.host, "www.python.org")
1363	n/a	o.open(req)
1364	n/a	self.assertEqual(req.host, "www.python.org")
1365	n/a	del os.environ['no_proxy']
1366	n/a
1367	n/a	def test_proxy_no_proxy_all(self):
1368	n/a	os.environ['no_proxy'] = '*'
1369	n/a	o = OpenerDirector()
1370	n/a	ph = urllib.request.ProxyHandler(dict(http="proxy.example.com"))
1371	n/a	o.add_handler(ph)
1372	n/a	req = Request("http://www.python.org")
1373	n/a	self.assertEqual(req.host, "www.python.org")
1374	n/a	o.open(req)
1375	n/a	self.assertEqual(req.host, "www.python.org")
1376	n/a	del os.environ['no_proxy']
1377	n/a
1378	n/a	def test_proxy_https(self):
1379	n/a	o = OpenerDirector()
1380	n/a	ph = urllib.request.ProxyHandler(dict(https="proxy.example.com:3128"))
1381	n/a	o.add_handler(ph)
1382	n/a	meth_spec = [
1383	n/a	[("https_open", "return response")]
1384	n/a	]
1385	n/a	handlers = add_ordered_mock_handlers(o, meth_spec)
1386	n/a
1387	n/a	req = Request("https://www.example.com/")
1388	n/a	self.assertEqual(req.host, "www.example.com")
1389	n/a	o.open(req)
1390	n/a	self.assertEqual(req.host, "proxy.example.com:3128")
1391	n/a	self.assertEqual([(handlers[0], "https_open")],
1392	n/a	[tup[0:2] for tup in o.calls])
1393	n/a
1394	n/a	def test_proxy_https_proxy_authorization(self):
1395	n/a	o = OpenerDirector()
1396	n/a	ph = urllib.request.ProxyHandler(dict(https='proxy.example.com:3128'))
1397	n/a	o.add_handler(ph)
1398	n/a	https_handler = MockHTTPSHandler()
1399	n/a	o.add_handler(https_handler)
1400	n/a	req = Request("https://www.example.com/")
1401	n/a	req.add_header("Proxy-Authorization", "FooBar")
1402	n/a	req.add_header("User-Agent", "Grail")
1403	n/a	self.assertEqual(req.host, "www.example.com")
1404	n/a	self.assertIsNone(req._tunnel_host)
1405	n/a	o.open(req)
1406	n/a	# Verify Proxy-Authorization gets tunneled to request.
1407	n/a	# httpsconn req_headers do not have the Proxy-Authorization header but
1408	n/a	# the req will have.
1409	n/a	self.assertNotIn(("Proxy-Authorization", "FooBar"),
1410	n/a	https_handler.httpconn.req_headers)
1411	n/a	self.assertIn(("User-Agent", "Grail"),
1412	n/a	https_handler.httpconn.req_headers)
1413	n/a	self.assertIsNotNone(req._tunnel_host)
1414	n/a	self.assertEqual(req.host, "proxy.example.com:3128")
1415	n/a	self.assertEqual(req.get_header("Proxy-authorization"), "FooBar")
1416	n/a
1417	n/a	# TODO: This should be only for OSX
1418	n/a	@unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX")
1419	n/a	def test_osx_proxy_bypass(self):
1420	n/a	bypass = {
1421	n/a	'exclude_simple': False,
1422	n/a	'exceptions': ['foo.bar', '*.bar.com', '127.0.0.1', '10.10',
1423	n/a	'10.0/16']
1424	n/a	}
1425	n/a	# Check hosts that should trigger the proxy bypass
1426	n/a	for host in ('foo.bar', 'www.bar.com', '127.0.0.1', '10.10.0.1',
1427	n/a	'10.0.0.1'):
1428	n/a	self.assertTrue(_proxy_bypass_macosx_sysconf(host, bypass),
1429	n/a	'expected bypass of %s to be True' % host)
1430	n/a	# Check hosts that should not trigger the proxy bypass
1431	n/a	for host in ('abc.foo.bar', 'bar.com', '127.0.0.2', '10.11.0.1',
1432	n/a	'notinbypass'):
1433	n/a	self.assertFalse(_proxy_bypass_macosx_sysconf(host, bypass),
1434	n/a	'expected bypass of %s to be False' % host)
1435	n/a
1436	n/a	# Check the exclude_simple flag
1437	n/a	bypass = {'exclude_simple': True, 'exceptions': []}
1438	n/a	self.assertTrue(_proxy_bypass_macosx_sysconf('test', bypass))
1439	n/a
1440	n/a	def test_basic_auth(self, quote_char='"'):
1441	n/a	opener = OpenerDirector()
1442	n/a	password_manager = MockPasswordManager()
1443	n/a	auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
1444	n/a	realm = "ACME Widget Store"
1445	n/a	http_handler = MockHTTPHandler(
1446	n/a	401, 'WWW-Authenticate: Basic realm=%s%s%s\r\n\r\n' %
1447	n/a	(quote_char, realm, quote_char))
1448	n/a	opener.add_handler(auth_handler)
1449	n/a	opener.add_handler(http_handler)
1450	n/a	self._test_basic_auth(opener, auth_handler, "Authorization",
1451	n/a	realm, http_handler, password_manager,
1452	n/a	"http://acme.example.com/protected",
1453	n/a	"http://acme.example.com/protected",
1454	n/a	)
1455	n/a
1456	n/a	def test_basic_auth_with_single_quoted_realm(self):
1457	n/a	self.test_basic_auth(quote_char="'")
1458	n/a
1459	n/a	def test_basic_auth_with_unquoted_realm(self):
1460	n/a	opener = OpenerDirector()
1461	n/a	password_manager = MockPasswordManager()
1462	n/a	auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
1463	n/a	realm = "ACME Widget Store"
1464	n/a	http_handler = MockHTTPHandler(
1465	n/a	401, 'WWW-Authenticate: Basic realm=%s\r\n\r\n' % realm)
1466	n/a	opener.add_handler(auth_handler)
1467	n/a	opener.add_handler(http_handler)
1468	n/a	with self.assertWarns(UserWarning):
1469	n/a	self._test_basic_auth(opener, auth_handler, "Authorization",
1470	n/a	realm, http_handler, password_manager,
1471	n/a	"http://acme.example.com/protected",
1472	n/a	"http://acme.example.com/protected",
1473	n/a	)
1474	n/a
1475	n/a	def test_proxy_basic_auth(self):
1476	n/a	opener = OpenerDirector()
1477	n/a	ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128"))
1478	n/a	opener.add_handler(ph)
1479	n/a	password_manager = MockPasswordManager()
1480	n/a	auth_handler = urllib.request.ProxyBasicAuthHandler(password_manager)
1481	n/a	realm = "ACME Networks"
1482	n/a	http_handler = MockHTTPHandler(
1483	n/a	407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm)
1484	n/a	opener.add_handler(auth_handler)
1485	n/a	opener.add_handler(http_handler)
1486	n/a	self._test_basic_auth(opener, auth_handler, "Proxy-authorization",
1487	n/a	realm, http_handler, password_manager,
1488	n/a	"http://acme.example.com:3128/protected",
1489	n/a	"proxy.example.com:3128",
1490	n/a	)
1491	n/a
1492	n/a	def test_basic_and_digest_auth_handlers(self):
1493	n/a	# HTTPDigestAuthHandler raised an exception if it couldn't handle a 40*
1494	n/a	# response (http://python.org/sf/1479302), where it should instead
1495	n/a	# return None to allow another handler (especially
1496	n/a	# HTTPBasicAuthHandler) to handle the response.
1497	n/a
1498	n/a	# Also (http://python.org/sf/14797027, RFC 2617 section 1.2), we must
1499	n/a	# try digest first (since it's the strongest auth scheme), so we record
1500	n/a	# order of calls here to check digest comes first:
1501	n/a	class RecordingOpenerDirector(OpenerDirector):
1502	n/a	def __init__(self):
1503	n/a	OpenerDirector.__init__(self)
1504	n/a	self.recorded = []
1505	n/a
1506	n/a	def record(self, info):
1507	n/a	self.recorded.append(info)
1508	n/a
1509	n/a	class TestDigestAuthHandler(urllib.request.HTTPDigestAuthHandler):
1510	n/a	def http_error_401(self, args, *kwds):
1511	n/a	self.parent.record("digest")
1512	n/a	urllib.request.HTTPDigestAuthHandler.http_error_401(self,
1513	n/a	args, *kwds)
1514	n/a
1515	n/a	class TestBasicAuthHandler(urllib.request.HTTPBasicAuthHandler):
1516	n/a	def http_error_401(self, args, *kwds):
1517	n/a	self.parent.record("basic")
1518	n/a	urllib.request.HTTPBasicAuthHandler.http_error_401(self,
1519	n/a	args, *kwds)
1520	n/a
1521	n/a	opener = RecordingOpenerDirector()
1522	n/a	password_manager = MockPasswordManager()
1523	n/a	digest_handler = TestDigestAuthHandler(password_manager)
1524	n/a	basic_handler = TestBasicAuthHandler(password_manager)
1525	n/a	realm = "ACME Networks"
1526	n/a	http_handler = MockHTTPHandler(
1527	n/a	401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % realm)
1528	n/a	opener.add_handler(basic_handler)
1529	n/a	opener.add_handler(digest_handler)
1530	n/a	opener.add_handler(http_handler)
1531	n/a
1532	n/a	# check basic auth isn't blocked by digest handler failing
1533	n/a	self._test_basic_auth(opener, basic_handler, "Authorization",
1534	n/a	realm, http_handler, password_manager,
1535	n/a	"http://acme.example.com/protected",
1536	n/a	"http://acme.example.com/protected",
1537	n/a	)
1538	n/a	# check digest was tried before basic (twice, because
1539	n/a	# _test_basic_auth called .open() twice)
1540	n/a	self.assertEqual(opener.recorded, ["digest", "basic"]*2)
1541	n/a
1542	n/a	def test_unsupported_auth_digest_handler(self):
1543	n/a	opener = OpenerDirector()
1544	n/a	# While using DigestAuthHandler
1545	n/a	digest_auth_handler = urllib.request.HTTPDigestAuthHandler(None)
1546	n/a	http_handler = MockHTTPHandler(
1547	n/a	401, 'WWW-Authenticate: Kerberos\r\n\r\n')
1548	n/a	opener.add_handler(digest_auth_handler)
1549	n/a	opener.add_handler(http_handler)
1550	n/a	self.assertRaises(ValueError, opener.open, "http://www.example.com")
1551	n/a
1552	n/a	def test_unsupported_auth_basic_handler(self):
1553	n/a	# While using BasicAuthHandler
1554	n/a	opener = OpenerDirector()
1555	n/a	basic_auth_handler = urllib.request.HTTPBasicAuthHandler(None)
1556	n/a	http_handler = MockHTTPHandler(
1557	n/a	401, 'WWW-Authenticate: NTLM\r\n\r\n')
1558	n/a	opener.add_handler(basic_auth_handler)
1559	n/a	opener.add_handler(http_handler)
1560	n/a	self.assertRaises(ValueError, opener.open, "http://www.example.com")
1561	n/a
1562	n/a	def _test_basic_auth(self, opener, auth_handler, auth_header,
1563	n/a	realm, http_handler, password_manager,
1564	n/a	request_url, protected_url):
1565	n/a	import base64
1566	n/a	user, password = "wile", "coyote"
1567	n/a
1568	n/a	# .add_password() fed through to password manager
1569	n/a	auth_handler.add_password(realm, request_url, user, password)
1570	n/a	self.assertEqual(realm, password_manager.realm)
1571	n/a	self.assertEqual(request_url, password_manager.url)
1572	n/a	self.assertEqual(user, password_manager.user)
1573	n/a	self.assertEqual(password, password_manager.password)
1574	n/a
1575	n/a	opener.open(request_url)
1576	n/a
1577	n/a	# should have asked the password manager for the username/password
1578	n/a	self.assertEqual(password_manager.target_realm, realm)
1579	n/a	self.assertEqual(password_manager.target_url, protected_url)
1580	n/a
1581	n/a	# expect one request without authorization, then one with
1582	n/a	self.assertEqual(len(http_handler.requests), 2)
1583	n/a	self.assertFalse(http_handler.requests[0].has_header(auth_header))
1584	n/a	userpass = bytes('%s:%s' % (user, password), "ascii")
1585	n/a	auth_hdr_value = ('Basic ' +
1586	n/a	base64.encodebytes(userpass).strip().decode())
1587	n/a	self.assertEqual(http_handler.requests[1].get_header(auth_header),
1588	n/a	auth_hdr_value)
1589	n/a	self.assertEqual(http_handler.requests[1].unredirected_hdrs[auth_header],
1590	n/a	auth_hdr_value)
1591	n/a	# if the password manager can't find a password, the handler won't
1592	n/a	# handle the HTTP auth error
1593	n/a	password_manager.user = password_manager.password = None
1594	n/a	http_handler.reset()
1595	n/a	opener.open(request_url)
1596	n/a	self.assertEqual(len(http_handler.requests), 1)
1597	n/a	self.assertFalse(http_handler.requests[0].has_header(auth_header))
1598	n/a
1599	n/a	def test_basic_prior_auth_auto_send(self):
1600	n/a	# Assume already authenticated if is_authenticated=True
1601	n/a	# for APIs like Github that don't return 401
1602	n/a
1603	n/a	user, password = "wile", "coyote"
1604	n/a	request_url = "http://acme.example.com/protected"
1605	n/a
1606	n/a	http_handler = MockHTTPHandlerCheckAuth(200)
1607	n/a
1608	n/a	pwd_manager = HTTPPasswordMgrWithPriorAuth()
1609	n/a	auth_prior_handler = HTTPBasicAuthHandler(pwd_manager)
1610	n/a	auth_prior_handler.add_password(
1611	n/a	None, request_url, user, password, is_authenticated=True)
1612	n/a
1613	n/a	is_auth = pwd_manager.is_authenticated(request_url)
1614	n/a	self.assertTrue(is_auth)
1615	n/a
1616	n/a	opener = OpenerDirector()
1617	n/a	opener.add_handler(auth_prior_handler)
1618	n/a	opener.add_handler(http_handler)
1619	n/a
1620	n/a	opener.open(request_url)
1621	n/a
1622	n/a	# expect request to be sent with auth header
1623	n/a	self.assertTrue(http_handler.has_auth_header)
1624	n/a
1625	n/a	def test_basic_prior_auth_send_after_first_success(self):
1626	n/a	# Auto send auth header after authentication is successful once
1627	n/a
1628	n/a	user, password = 'wile', 'coyote'
1629	n/a	request_url = 'http://acme.example.com/protected'
1630	n/a	realm = 'ACME'
1631	n/a
1632	n/a	pwd_manager = HTTPPasswordMgrWithPriorAuth()
1633	n/a	auth_prior_handler = HTTPBasicAuthHandler(pwd_manager)
1634	n/a	auth_prior_handler.add_password(realm, request_url, user, password)
1635	n/a
1636	n/a	is_auth = pwd_manager.is_authenticated(request_url)
1637	n/a	self.assertFalse(is_auth)
1638	n/a
1639	n/a	opener = OpenerDirector()
1640	n/a	opener.add_handler(auth_prior_handler)
1641	n/a
1642	n/a	http_handler = MockHTTPHandler(
1643	n/a	401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % None)
1644	n/a	opener.add_handler(http_handler)
1645	n/a
1646	n/a	opener.open(request_url)
1647	n/a
1648	n/a	is_auth = pwd_manager.is_authenticated(request_url)
1649	n/a	self.assertTrue(is_auth)
1650	n/a
1651	n/a	http_handler = MockHTTPHandlerCheckAuth(200)
1652	n/a	self.assertFalse(http_handler.has_auth_header)
1653	n/a
1654	n/a	opener = OpenerDirector()
1655	n/a	opener.add_handler(auth_prior_handler)
1656	n/a	opener.add_handler(http_handler)
1657	n/a
1658	n/a	# After getting 200 from MockHTTPHandler
1659	n/a	# Next request sends header in the first request
1660	n/a	opener.open(request_url)
1661	n/a
1662	n/a	# expect request to be sent with auth header
1663	n/a	self.assertTrue(http_handler.has_auth_header)
1664	n/a
1665	n/a	def test_http_closed(self):
1666	n/a	"""Test the connection is cleaned up when the response is closed"""
1667	n/a	for (transfer, data) in (
1668	n/a	("Connection: close", b"data"),
1669	n/a	("Transfer-Encoding: chunked", b"4\r\ndata\r\n0\r\n\r\n"),
1670	n/a	("Content-Length: 4", b"data"),
1671	n/a	):
1672	n/a	header = "HTTP/1.1 200 OK\r\n{}\r\n\r\n".format(transfer)
1673	n/a	conn = test_urllib.fakehttp(header.encode() + data)
1674	n/a	handler = urllib.request.AbstractHTTPHandler()
1675	n/a	req = Request("http://dummy/")
1676	n/a	req.timeout = None
1677	n/a	with handler.do_open(conn, req) as resp:
1678	n/a	resp.read()
1679	n/a	self.assertTrue(conn.fakesock.closed,
1680	n/a	"Connection not closed with {!r}".format(transfer))
1681	n/a
1682	n/a	def test_invalid_closed(self):
1683	n/a	"""Test the connection is cleaned up after an invalid response"""
1684	n/a	conn = test_urllib.fakehttp(b"")
1685	n/a	handler = urllib.request.AbstractHTTPHandler()
1686	n/a	req = Request("http://dummy/")
1687	n/a	req.timeout = None
1688	n/a	with self.assertRaises(http.client.BadStatusLine):
1689	n/a	handler.do_open(conn, req)
1690	n/a	self.assertTrue(conn.fakesock.closed, "Connection not closed")
1691	n/a
1692	n/a
1693	n/a
1694	n/a	class MiscTests(unittest.TestCase):
1695	n/a
1696	n/a	def opener_has_handler(self, opener, handler_class):
1697	n/a	self.assertTrue(any(h.__class__ == handler_class
1698	n/a	for h in opener.handlers))
1699	n/a
1700	n/a	def test_build_opener(self):
1701	n/a	class MyHTTPHandler(urllib.request.HTTPHandler):
1702	n/a	pass
1703	n/a
1704	n/a	class FooHandler(urllib.request.BaseHandler):
1705	n/a	def foo_open(self):
1706	n/a	pass
1707	n/a
1708	n/a	class BarHandler(urllib.request.BaseHandler):
1709	n/a	def bar_open(self):
1710	n/a	pass
1711	n/a
1712	n/a	build_opener = urllib.request.build_opener
1713	n/a
1714	n/a	o = build_opener(FooHandler, BarHandler)
1715	n/a	self.opener_has_handler(o, FooHandler)
1716	n/a	self.opener_has_handler(o, BarHandler)
1717	n/a
1718	n/a	# can take a mix of classes and instances
1719	n/a	o = build_opener(FooHandler, BarHandler())
1720	n/a	self.opener_has_handler(o, FooHandler)
1721	n/a	self.opener_has_handler(o, BarHandler)
1722	n/a
1723	n/a	# subclasses of default handlers override default handlers
1724	n/a	o = build_opener(MyHTTPHandler)
1725	n/a	self.opener_has_handler(o, MyHTTPHandler)
1726	n/a
1727	n/a	# a particular case of overriding: default handlers can be passed
1728	n/a	# in explicitly
1729	n/a	o = build_opener()
1730	n/a	self.opener_has_handler(o, urllib.request.HTTPHandler)
1731	n/a	o = build_opener(urllib.request.HTTPHandler)
1732	n/a	self.opener_has_handler(o, urllib.request.HTTPHandler)
1733	n/a	o = build_opener(urllib.request.HTTPHandler())
1734	n/a	self.opener_has_handler(o, urllib.request.HTTPHandler)
1735	n/a
1736	n/a	# Issue2670: multiple handlers sharing the same base class
1737	n/a	class MyOtherHTTPHandler(urllib.request.HTTPHandler):
1738	n/a	pass
1739	n/a
1740	n/a	o = build_opener(MyHTTPHandler, MyOtherHTTPHandler)
1741	n/a	self.opener_has_handler(o, MyHTTPHandler)
1742	n/a	self.opener_has_handler(o, MyOtherHTTPHandler)
1743	n/a
1744	n/a	@unittest.skipUnless(support.is_resource_enabled('network'),
1745	n/a	'test requires network access')
1746	n/a	def test_issue16464(self):
1747	n/a	with support.transient_internet("http://www.example.com/"):
1748	n/a	opener = urllib.request.build_opener()
1749	n/a	request = urllib.request.Request("http://www.example.com/")
1750	n/a	self.assertEqual(None, request.data)
1751	n/a
1752	n/a	opener.open(request, "1".encode("us-ascii"))
1753	n/a	self.assertEqual(b"1", request.data)
1754	n/a	self.assertEqual("1", request.get_header("Content-length"))
1755	n/a
1756	n/a	opener.open(request, "1234567890".encode("us-ascii"))
1757	n/a	self.assertEqual(b"1234567890", request.data)
1758	n/a	self.assertEqual("10", request.get_header("Content-length"))
1759	n/a
1760	n/a	def test_HTTPError_interface(self):
1761	n/a	"""
1762	n/a	Issue 13211 reveals that HTTPError didn't implement the URLError
1763	n/a	interface even though HTTPError is a subclass of URLError.
1764	n/a	"""
1765	n/a	msg = 'something bad happened'
1766	n/a	url = code = fp = None
1767	n/a	hdrs = 'Content-Length: 42'
1768	n/a	err = urllib.error.HTTPError(url, code, msg, hdrs, fp)
1769	n/a	self.assertTrue(hasattr(err, 'reason'))
1770	n/a	self.assertEqual(err.reason, 'something bad happened')
1771	n/a	self.assertTrue(hasattr(err, 'headers'))
1772	n/a	self.assertEqual(err.headers, 'Content-Length: 42')
1773	n/a	expected_errmsg = 'HTTP Error %s: %s' % (err.code, err.msg)
1774	n/a	self.assertEqual(str(err), expected_errmsg)
1775	n/a	expected_errmsg = '<HTTPError %s: %r>' % (err.code, err.msg)
1776	n/a	self.assertEqual(repr(err), expected_errmsg)
1777	n/a
1778	n/a	def test_parse_proxy(self):
1779	n/a	parse_proxy_test_cases = [
1780	n/a	('proxy.example.com',
1781	n/a	(None, None, None, 'proxy.example.com')),
1782	n/a	('proxy.example.com:3128',
1783	n/a	(None, None, None, 'proxy.example.com:3128')),
1784	n/a	('proxy.example.com', (None, None, None, 'proxy.example.com')),
1785	n/a	('proxy.example.com:3128',
1786	n/a	(None, None, None, 'proxy.example.com:3128')),
1787	n/a	# The authority component may optionally include userinfo
1788	n/a	# (assumed to be # username:password):
1789	n/a	('joe:password@proxy.example.com',
1790	n/a	(None, 'joe', 'password', 'proxy.example.com')),
1791	n/a	('joe:password@proxy.example.com:3128',
1792	n/a	(None, 'joe', 'password', 'proxy.example.com:3128')),
1793	n/a	#Examples with URLS
1794	n/a	('http://proxy.example.com/',
1795	n/a	('http', None, None, 'proxy.example.com')),
1796	n/a	('http://proxy.example.com:3128/',
1797	n/a	('http', None, None, 'proxy.example.com:3128')),
1798	n/a	('http://joe:password@proxy.example.com/',
1799	n/a	('http', 'joe', 'password', 'proxy.example.com')),
1800	n/a	('http://joe:password@proxy.example.com:3128',
1801	n/a	('http', 'joe', 'password', 'proxy.example.com:3128')),
1802	n/a	# Everything after the authority is ignored
1803	n/a	('ftp://joe:password@proxy.example.com/rubbish:3128',
1804	n/a	('ftp', 'joe', 'password', 'proxy.example.com')),
1805	n/a	# Test for no trailing '/' case
1806	n/a	('http://joe:password@proxy.example.com',
1807	n/a	('http', 'joe', 'password', 'proxy.example.com'))
1808	n/a	]
1809	n/a
1810	n/a	for tc, expected in parse_proxy_test_cases:
1811	n/a	self.assertEqual(_parse_proxy(tc), expected)
1812	n/a
1813	n/a	self.assertRaises(ValueError, _parse_proxy, 'file:/ftp.example.com'),
1814	n/a
1815	n/a	def test_unsupported_algorithm(self):
1816	n/a	handler = AbstractDigestAuthHandler()
1817	n/a	with self.assertRaises(ValueError) as exc:
1818	n/a	handler.get_algorithm_impls('invalid')
1819	n/a	self.assertEqual(
1820	n/a	str(exc.exception),
1821	n/a	"Unsupported digest authentication algorithm 'invalid'"
1822	n/a	)
1823	n/a
1824	n/a
1825	n/a	class RequestTests(unittest.TestCase):
1826	n/a	class PutRequest(Request):
1827	n/a	method = 'PUT'
1828	n/a
1829	n/a	def setUp(self):
1830	n/a	self.get = Request("http://www.python.org/~jeremy/")
1831	n/a	self.post = Request("http://www.python.org/~jeremy/",
1832	n/a	"data",
1833	n/a	headers={"X-Test": "test"})
1834	n/a	self.head = Request("http://www.python.org/~jeremy/", method='HEAD')
1835	n/a	self.put = self.PutRequest("http://www.python.org/~jeremy/")
1836	n/a	self.force_post = self.PutRequest("http://www.python.org/~jeremy/",
1837	n/a	method="POST")
1838	n/a
1839	n/a	def test_method(self):
1840	n/a	self.assertEqual("POST", self.post.get_method())
1841	n/a	self.assertEqual("GET", self.get.get_method())
1842	n/a	self.assertEqual("HEAD", self.head.get_method())
1843	n/a	self.assertEqual("PUT", self.put.get_method())
1844	n/a	self.assertEqual("POST", self.force_post.get_method())
1845	n/a
1846	n/a	def test_data(self):
1847	n/a	self.assertFalse(self.get.data)
1848	n/a	self.assertEqual("GET", self.get.get_method())
1849	n/a	self.get.data = "spam"
1850	n/a	self.assertTrue(self.get.data)
1851	n/a	self.assertEqual("POST", self.get.get_method())
1852	n/a
1853	n/a	# issue 16464
1854	n/a	# if we change data we need to remove content-length header
1855	n/a	# (cause it's most probably calculated for previous value)
1856	n/a	def test_setting_data_should_remove_content_length(self):
1857	n/a	self.assertNotIn("Content-length", self.get.unredirected_hdrs)
1858	n/a	self.get.add_unredirected_header("Content-length", 42)
1859	n/a	self.assertEqual(42, self.get.unredirected_hdrs["Content-length"])
1860	n/a	self.get.data = "spam"
1861	n/a	self.assertNotIn("Content-length", self.get.unredirected_hdrs)
1862	n/a
1863	n/a	# issue 17485 same for deleting data.
1864	n/a	def test_deleting_data_should_remove_content_length(self):
1865	n/a	self.assertNotIn("Content-length", self.get.unredirected_hdrs)
1866	n/a	self.get.data = 'foo'
1867	n/a	self.get.add_unredirected_header("Content-length", 3)
1868	n/a	self.assertEqual(3, self.get.unredirected_hdrs["Content-length"])
1869	n/a	del self.get.data
1870	n/a	self.assertNotIn("Content-length", self.get.unredirected_hdrs)
1871	n/a
1872	n/a	def test_get_full_url(self):
1873	n/a	self.assertEqual("http://www.python.org/~jeremy/",
1874	n/a	self.get.get_full_url())
1875	n/a
1876	n/a	def test_selector(self):
1877	n/a	self.assertEqual("/~jeremy/", self.get.selector)
1878	n/a	req = Request("http://www.python.org/")
1879	n/a	self.assertEqual("/", req.selector)
1880	n/a
1881	n/a	def test_get_type(self):
1882	n/a	self.assertEqual("http", self.get.type)
1883	n/a
1884	n/a	def test_get_host(self):
1885	n/a	self.assertEqual("www.python.org", self.get.host)
1886	n/a
1887	n/a	def test_get_host_unquote(self):
1888	n/a	req = Request("http://www.%70ython.org/")
1889	n/a	self.assertEqual("www.python.org", req.host)
1890	n/a
1891	n/a	def test_proxy(self):
1892	n/a	self.assertFalse(self.get.has_proxy())
1893	n/a	self.get.set_proxy("www.perl.org", "http")
1894	n/a	self.assertTrue(self.get.has_proxy())
1895	n/a	self.assertEqual("www.python.org", self.get.origin_req_host)
1896	n/a	self.assertEqual("www.perl.org", self.get.host)
1897	n/a
1898	n/a	def test_wrapped_url(self):
1899	n/a	req = Request("<URL:http://www.python.org>")
1900	n/a	self.assertEqual("www.python.org", req.host)
1901	n/a
1902	n/a	def test_url_fragment(self):
1903	n/a	req = Request("http://www.python.org/?qs=query#fragment=true")
1904	n/a	self.assertEqual("/?qs=query", req.selector)
1905	n/a	req = Request("http://www.python.org/#fun=true")
1906	n/a	self.assertEqual("/", req.selector)
1907	n/a
1908	n/a	# Issue 11703: geturl() omits fragment in the original URL.
1909	n/a	url = 'http://docs.python.org/library/urllib2.html#OK'
1910	n/a	req = Request(url)
1911	n/a	self.assertEqual(req.get_full_url(), url)
1912	n/a
1913	n/a	def test_url_fullurl_get_full_url(self):
1914	n/a	urls = ['http://docs.python.org',
1915	n/a	'http://docs.python.org/library/urllib2.html#OK',
1916	n/a	'http://www.python.org/?qs=query#fragment=true']
1917	n/a	for url in urls:
1918	n/a	req = Request(url)
1919	n/a	self.assertEqual(req.get_full_url(), req.full_url)
1920	n/a
1921	n/a
1922	n/a	if __name__ == "__main__":
1923	n/a	unittest.main()