ยปCore Development>Code coverage>Lib/test/test_urllib2.py

Python code coverage for Lib/test/test_urllib2.py

#countcontent
1n/aimport unittest
2n/afrom test import support
3n/afrom test import test_urllib
4n/a
5n/aimport os
6n/aimport io
7n/aimport socket
8n/aimport array
9n/aimport sys
10n/aimport tempfile
11n/aimport subprocess
12n/a
13n/aimport urllib.request
14n/a# The proxy bypass method imported below has logic specific to the OSX
15n/a# proxy config data structure but is testable on all platforms.
16n/afrom urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler,
17n/a HTTPPasswordMgrWithPriorAuth, _parse_proxy,
18n/a _proxy_bypass_macosx_sysconf,
19n/a AbstractDigestAuthHandler)
20n/afrom urllib.parse import urlparse
21n/aimport urllib.error
22n/aimport http.client
23n/a
24n/a# XXX
25n/a# Request
26n/a# CacheFTPHandler (hard to write)
27n/a# parse_keqv_list, parse_http_list, HTTPDigestAuthHandler
28n/a
29n/a
30n/aclass TrivialTests(unittest.TestCase):
31n/a
32n/a def test___all__(self):
33n/a # Verify which names are exposed
34n/a for module in 'request', 'response', 'parse', 'error', 'robotparser':
35n/a context = {}
36n/a exec('from urllib.%s import *' % module, context)
37n/a del context['__builtins__']
38n/a if module == 'request' and os.name == 'nt':
39n/a u, p = context.pop('url2pathname'), context.pop('pathname2url')
40n/a self.assertEqual(u.__module__, 'nturl2path')
41n/a self.assertEqual(p.__module__, 'nturl2path')
42n/a for k, v in context.items():
43n/a self.assertEqual(v.__module__, 'urllib.%s' % module,
44n/a "%r is exposed in 'urllib.%s' but defined in %r" %
45n/a (k, module, v.__module__))
46n/a
47n/a def test_trivial(self):
48n/a # A couple trivial tests
49n/a
50n/a self.assertRaises(ValueError, urllib.request.urlopen, 'bogus url')
51n/a
52n/a # XXX Name hacking to get this to work on Windows.
53n/a fname = os.path.abspath(urllib.request.__file__).replace(os.sep, '/')
54n/a
55n/a if os.name == 'nt':
56n/a file_url = "file:///%s" % fname
57n/a else:
58n/a file_url = "file://%s" % fname
59n/a
60n/a f = urllib.request.urlopen(file_url)
61n/a
62n/a f.read()
63n/a f.close()
64n/a
65n/a def test_parse_http_list(self):
66n/a tests = [
67n/a ('a,b,c', ['a', 'b', 'c']),
68n/a ('path"o,l"og"i"cal, example', ['path"o,l"og"i"cal', 'example']),
69n/a ('a, b, "c", "d", "e,f", g, h',
70n/a ['a', 'b', '"c"', '"d"', '"e,f"', 'g', 'h']),
71n/a ('a="b\\"c", d="e\\,f", g="h\\\\i"',
72n/a ['a="b"c"', 'd="e,f"', 'g="h\\i"'])]
73n/a for string, list in tests:
74n/a self.assertEqual(urllib.request.parse_http_list(string), list)
75n/a
76n/a def test_URLError_reasonstr(self):
77n/a err = urllib.error.URLError('reason')
78n/a self.assertIn(err.reason, str(err))
79n/a
80n/a
81n/aclass RequestHdrsTests(unittest.TestCase):
82n/a
83n/a def test_request_headers_dict(self):
84n/a """
85n/a The Request.headers dictionary is not a documented interface. It
86n/a should stay that way, because the complete set of headers are only
87n/a accessible through the .get_header(), .has_header(), .header_items()
88n/a interface. However, .headers pre-dates those methods, and so real code
89n/a will be using the dictionary.
90n/a
91n/a The introduction in 2.4 of those methods was a mistake for the same
92n/a reason: code that previously saw all (urllib2 user)-provided headers in
93n/a .headers now sees only a subset.
94n/a
95n/a """
96n/a url = "http://example.com"
97n/a self.assertEqual(Request(url,
98n/a headers={"Spam-eggs": "blah"}
99n/a ).headers["Spam-eggs"], "blah")
100n/a self.assertEqual(Request(url,
101n/a headers={"spam-EggS": "blah"}
102n/a ).headers["Spam-eggs"], "blah")
103n/a
104n/a def test_request_headers_methods(self):
105n/a """
106n/a Note the case normalization of header names here, to
107n/a .capitalize()-case. This should be preserved for
108n/a backwards-compatibility. (In the HTTP case, normalization to
109n/a .title()-case is done by urllib2 before sending headers to
110n/a http.client).
111n/a
112n/a Note that e.g. r.has_header("spam-EggS") is currently False, and
113n/a r.get_header("spam-EggS") returns None, but that could be changed in
114n/a future.
115n/a
116n/a Method r.remove_header should remove items both from r.headers and
117n/a r.unredirected_hdrs dictionaries
118n/a """
119n/a url = "http://example.com"
120n/a req = Request(url, headers={"Spam-eggs": "blah"})
121n/a self.assertTrue(req.has_header("Spam-eggs"))
122n/a self.assertEqual(req.header_items(), [('Spam-eggs', 'blah')])
123n/a
124n/a req.add_header("Foo-Bar", "baz")
125n/a self.assertEqual(sorted(req.header_items()),
126n/a [('Foo-bar', 'baz'), ('Spam-eggs', 'blah')])
127n/a self.assertFalse(req.has_header("Not-there"))
128n/a self.assertIsNone(req.get_header("Not-there"))
129n/a self.assertEqual(req.get_header("Not-there", "default"), "default")
130n/a
131n/a req.remove_header("Spam-eggs")
132n/a self.assertFalse(req.has_header("Spam-eggs"))
133n/a
134n/a req.add_unredirected_header("Unredirected-spam", "Eggs")
135n/a self.assertTrue(req.has_header("Unredirected-spam"))
136n/a
137n/a req.remove_header("Unredirected-spam")
138n/a self.assertFalse(req.has_header("Unredirected-spam"))
139n/a
140n/a def test_password_manager(self):
141n/a mgr = urllib.request.HTTPPasswordMgr()
142n/a add = mgr.add_password
143n/a find_user_pass = mgr.find_user_password
144n/a add("Some Realm", "http://example.com/", "joe", "password")
145n/a add("Some Realm", "http://example.com/ni", "ni", "ni")
146n/a add("c", "http://example.com/foo", "foo", "ni")
147n/a add("c", "http://example.com/bar", "bar", "nini")
148n/a add("b", "http://example.com/", "first", "blah")
149n/a add("b", "http://example.com/", "second", "spam")
150n/a add("a", "http://example.com", "1", "a")
151n/a add("Some Realm", "http://c.example.com:3128", "3", "c")
152n/a add("Some Realm", "d.example.com", "4", "d")
153n/a add("Some Realm", "e.example.com:3128", "5", "e")
154n/a
155n/a self.assertEqual(find_user_pass("Some Realm", "example.com"),
156n/a ('joe', 'password'))
157n/a
158n/a #self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"),
159n/a # ('ni', 'ni'))
160n/a
161n/a self.assertEqual(find_user_pass("Some Realm", "http://example.com"),
162n/a ('joe', 'password'))
163n/a self.assertEqual(find_user_pass("Some Realm", "http://example.com/"),
164n/a ('joe', 'password'))
165n/a self.assertEqual(
166n/a find_user_pass("Some Realm", "http://example.com/spam"),
167n/a ('joe', 'password'))
168n/a self.assertEqual(
169n/a find_user_pass("Some Realm", "http://example.com/spam/spam"),
170n/a ('joe', 'password'))
171n/a self.assertEqual(find_user_pass("c", "http://example.com/foo"),
172n/a ('foo', 'ni'))
173n/a self.assertEqual(find_user_pass("c", "http://example.com/bar"),
174n/a ('bar', 'nini'))
175n/a self.assertEqual(find_user_pass("b", "http://example.com/"),
176n/a ('second', 'spam'))
177n/a
178n/a # No special relationship between a.example.com and example.com:
179n/a
180n/a self.assertEqual(find_user_pass("a", "http://example.com/"),
181n/a ('1', 'a'))
182n/a self.assertEqual(find_user_pass("a", "http://a.example.com/"),
183n/a (None, None))
184n/a
185n/a # Ports:
186n/a
187n/a self.assertEqual(find_user_pass("Some Realm", "c.example.com"),
188n/a (None, None))
189n/a self.assertEqual(find_user_pass("Some Realm", "c.example.com:3128"),
190n/a ('3', 'c'))
191n/a self.assertEqual(
192n/a find_user_pass("Some Realm", "http://c.example.com:3128"),
193n/a ('3', 'c'))
194n/a self.assertEqual(find_user_pass("Some Realm", "d.example.com"),
195n/a ('4', 'd'))
196n/a self.assertEqual(find_user_pass("Some Realm", "e.example.com:3128"),
197n/a ('5', 'e'))
198n/a
199n/a def test_password_manager_default_port(self):
200n/a """
201n/a The point to note here is that we can't guess the default port if
202n/a there's no scheme. This applies to both add_password and
203n/a find_user_password.
204n/a """
205n/a mgr = urllib.request.HTTPPasswordMgr()
206n/a add = mgr.add_password
207n/a find_user_pass = mgr.find_user_password
208n/a add("f", "http://g.example.com:80", "10", "j")
209n/a add("g", "http://h.example.com", "11", "k")
210n/a add("h", "i.example.com:80", "12", "l")
211n/a add("i", "j.example.com", "13", "m")
212n/a self.assertEqual(find_user_pass("f", "g.example.com:100"),
213n/a (None, None))
214n/a self.assertEqual(find_user_pass("f", "g.example.com:80"),
215n/a ('10', 'j'))
216n/a self.assertEqual(find_user_pass("f", "g.example.com"),
217n/a (None, None))
218n/a self.assertEqual(find_user_pass("f", "http://g.example.com:100"),
219n/a (None, None))
220n/a self.assertEqual(find_user_pass("f", "http://g.example.com:80"),
221n/a ('10', 'j'))
222n/a self.assertEqual(find_user_pass("f", "http://g.example.com"),
223n/a ('10', 'j'))
224n/a self.assertEqual(find_user_pass("g", "h.example.com"), ('11', 'k'))
225n/a self.assertEqual(find_user_pass("g", "h.example.com:80"), ('11', 'k'))
226n/a self.assertEqual(find_user_pass("g", "http://h.example.com:80"),
227n/a ('11', 'k'))
228n/a self.assertEqual(find_user_pass("h", "i.example.com"), (None, None))
229n/a self.assertEqual(find_user_pass("h", "i.example.com:80"), ('12', 'l'))
230n/a self.assertEqual(find_user_pass("h", "http://i.example.com:80"),
231n/a ('12', 'l'))
232n/a self.assertEqual(find_user_pass("i", "j.example.com"), ('13', 'm'))
233n/a self.assertEqual(find_user_pass("i", "j.example.com:80"),
234n/a (None, None))
235n/a self.assertEqual(find_user_pass("i", "http://j.example.com"),
236n/a ('13', 'm'))
237n/a self.assertEqual(find_user_pass("i", "http://j.example.com:80"),
238n/a (None, None))
239n/a
240n/a
241n/aclass MockOpener:
242n/a addheaders = []
243n/a
244n/a def open(self, req, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
245n/a self.req, self.data, self.timeout = req, data, timeout
246n/a
247n/a def error(self, proto, *args):
248n/a self.proto, self.args = proto, args
249n/a
250n/a
251n/aclass MockFile:
252n/a def read(self, count=None):
253n/a pass
254n/a
255n/a def readline(self, count=None):
256n/a pass
257n/a
258n/a def close(self):
259n/a pass
260n/a
261n/a
262n/aclass MockHeaders(dict):
263n/a def getheaders(self, name):
264n/a return list(self.values())
265n/a
266n/a
267n/aclass MockResponse(io.StringIO):
268n/a def __init__(self, code, msg, headers, data, url=None):
269n/a io.StringIO.__init__(self, data)
270n/a self.code, self.msg, self.headers, self.url = code, msg, headers, url
271n/a
272n/a def info(self):
273n/a return self.headers
274n/a
275n/a def geturl(self):
276n/a return self.url
277n/a
278n/a
279n/aclass MockCookieJar:
280n/a def add_cookie_header(self, request):
281n/a self.ach_req = request
282n/a
283n/a def extract_cookies(self, response, request):
284n/a self.ec_req, self.ec_r = request, response
285n/a
286n/a
287n/aclass FakeMethod:
288n/a def __init__(self, meth_name, action, handle):
289n/a self.meth_name = meth_name
290n/a self.handle = handle
291n/a self.action = action
292n/a
293n/a def __call__(self, *args):
294n/a return self.handle(self.meth_name, self.action, *args)
295n/a
296n/a
297n/aclass MockHTTPResponse(io.IOBase):
298n/a def __init__(self, fp, msg, status, reason):
299n/a self.fp = fp
300n/a self.msg = msg
301n/a self.status = status
302n/a self.reason = reason
303n/a self.code = 200
304n/a
305n/a def read(self):
306n/a return ''
307n/a
308n/a def info(self):
309n/a return {}
310n/a
311n/a def geturl(self):
312n/a return self.url
313n/a
314n/a
315n/aclass MockHTTPClass:
316n/a def __init__(self):
317n/a self.level = 0
318n/a self.req_headers = []
319n/a self.data = None
320n/a self.raise_on_endheaders = False
321n/a self.sock = None
322n/a self._tunnel_headers = {}
323n/a
324n/a def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
325n/a self.host = host
326n/a self.timeout = timeout
327n/a return self
328n/a
329n/a def set_debuglevel(self, level):
330n/a self.level = level
331n/a
332n/a def set_tunnel(self, host, port=None, headers=None):
333n/a self._tunnel_host = host
334n/a self._tunnel_port = port
335n/a if headers:
336n/a self._tunnel_headers = headers
337n/a else:
338n/a self._tunnel_headers.clear()
339n/a
340n/a def request(self, method, url, body=None, headers=None, *,
341n/a encode_chunked=False):
342n/a self.method = method
343n/a self.selector = url
344n/a if headers is not None:
345n/a self.req_headers += headers.items()
346n/a self.req_headers.sort()
347n/a if body:
348n/a self.data = body
349n/a self.encode_chunked = encode_chunked
350n/a if self.raise_on_endheaders:
351n/a raise OSError()
352n/a
353n/a def getresponse(self):
354n/a return MockHTTPResponse(MockFile(), {}, 200, "OK")
355n/a
356n/a def close(self):
357n/a pass
358n/a
359n/a
360n/aclass MockHandler:
361n/a # useful for testing handler machinery
362n/a # see add_ordered_mock_handlers() docstring
363n/a handler_order = 500
364n/a
365n/a def __init__(self, methods):
366n/a self._define_methods(methods)
367n/a
368n/a def _define_methods(self, methods):
369n/a for spec in methods:
370n/a if len(spec) == 2:
371n/a name, action = spec
372n/a else:
373n/a name, action = spec, None
374n/a meth = FakeMethod(name, action, self.handle)
375n/a setattr(self.__class__, name, meth)
376n/a
377n/a def handle(self, fn_name, action, *args, **kwds):
378n/a self.parent.calls.append((self, fn_name, args, kwds))
379n/a if action is None:
380n/a return None
381n/a elif action == "return self":
382n/a return self
383n/a elif action == "return response":
384n/a res = MockResponse(200, "OK", {}, "")
385n/a return res
386n/a elif action == "return request":
387n/a return Request("http://blah/")
388n/a elif action.startswith("error"):
389n/a code = action[action.rfind(" ")+1:]
390n/a try:
391n/a code = int(code)
392n/a except ValueError:
393n/a pass
394n/a res = MockResponse(200, "OK", {}, "")
395n/a return self.parent.error("http", args[0], res, code, "", {})
396n/a elif action == "raise":
397n/a raise urllib.error.URLError("blah")
398n/a assert False
399n/a
400n/a def close(self):
401n/a pass
402n/a
403n/a def add_parent(self, parent):
404n/a self.parent = parent
405n/a self.parent.calls = []
406n/a
407n/a def __lt__(self, other):
408n/a if not hasattr(other, "handler_order"):
409n/a # No handler_order, leave in original order. Yuck.
410n/a return True
411n/a return self.handler_order < other.handler_order
412n/a
413n/a
414n/adef add_ordered_mock_handlers(opener, meth_spec):
415n/a """Create MockHandlers and add them to an OpenerDirector.
416n/a
417n/a meth_spec: list of lists of tuples and strings defining methods to define
418n/a on handlers. eg:
419n/a
420n/a [["http_error", "ftp_open"], ["http_open"]]
421n/a
422n/a defines methods .http_error() and .ftp_open() on one handler, and
423n/a .http_open() on another. These methods just record their arguments and
424n/a return None. Using a tuple instead of a string causes the method to
425n/a perform some action (see MockHandler.handle()), eg:
426n/a
427n/a [["http_error"], [("http_open", "return request")]]
428n/a
429n/a defines .http_error() on one handler (which simply returns None), and
430n/a .http_open() on another handler, which returns a Request object.
431n/a
432n/a """
433n/a handlers = []
434n/a count = 0
435n/a for meths in meth_spec:
436n/a class MockHandlerSubclass(MockHandler):
437n/a pass
438n/a
439n/a h = MockHandlerSubclass(meths)
440n/a h.handler_order += count
441n/a h.add_parent(opener)
442n/a count = count + 1
443n/a handlers.append(h)
444n/a opener.add_handler(h)
445n/a return handlers
446n/a
447n/a
448n/adef build_test_opener(*handler_instances):
449n/a opener = OpenerDirector()
450n/a for h in handler_instances:
451n/a opener.add_handler(h)
452n/a return opener
453n/a
454n/a
455n/aclass MockHTTPHandler(urllib.request.BaseHandler):
456n/a # useful for testing redirections and auth
457n/a # sends supplied headers and code as first response
458n/a # sends 200 OK as second response
459n/a def __init__(self, code, headers):
460n/a self.code = code
461n/a self.headers = headers
462n/a self.reset()
463n/a
464n/a def reset(self):
465n/a self._count = 0
466n/a self.requests = []
467n/a
468n/a def http_open(self, req):
469n/a import email, copy
470n/a self.requests.append(copy.deepcopy(req))
471n/a if self._count == 0:
472n/a self._count = self._count + 1
473n/a name = http.client.responses[self.code]
474n/a msg = email.message_from_string(self.headers)
475n/a return self.parent.error(
476n/a "http", req, MockFile(), self.code, name, msg)
477n/a else:
478n/a self.req = req
479n/a msg = email.message_from_string("\r\n\r\n")
480n/a return MockResponse(200, "OK", msg, "", req.get_full_url())
481n/a
482n/a
483n/aclass MockHTTPSHandler(urllib.request.AbstractHTTPHandler):
484n/a # Useful for testing the Proxy-Authorization request by verifying the
485n/a # properties of httpcon
486n/a
487n/a def __init__(self, debuglevel=0):
488n/a urllib.request.AbstractHTTPHandler.__init__(self, debuglevel=debuglevel)
489n/a self.httpconn = MockHTTPClass()
490n/a
491n/a def https_open(self, req):
492n/a return self.do_open(self.httpconn, req)
493n/a
494n/a
495n/aclass MockHTTPHandlerCheckAuth(urllib.request.BaseHandler):
496n/a # useful for testing auth
497n/a # sends supplied code response
498n/a # checks if auth header is specified in request
499n/a def __init__(self, code):
500n/a self.code = code
501n/a self.has_auth_header = False
502n/a
503n/a def reset(self):
504n/a self.has_auth_header = False
505n/a
506n/a def http_open(self, req):
507n/a if req.has_header('Authorization'):
508n/a self.has_auth_header = True
509n/a name = http.client.responses[self.code]
510n/a return MockResponse(self.code, name, MockFile(), "", req.get_full_url())
511n/a
512n/a
513n/a
514n/aclass MockPasswordManager:
515n/a def add_password(self, realm, uri, user, password):
516n/a self.realm = realm
517n/a self.url = uri
518n/a self.user = user
519n/a self.password = password
520n/a
521n/a def find_user_password(self, realm, authuri):
522n/a self.target_realm = realm
523n/a self.target_url = authuri
524n/a return self.user, self.password
525n/a
526n/a
527n/aclass OpenerDirectorTests(unittest.TestCase):
528n/a
529n/a def test_add_non_handler(self):
530n/a class NonHandler(object):
531n/a pass
532n/a self.assertRaises(TypeError,
533n/a OpenerDirector().add_handler, NonHandler())
534n/a
535n/a def test_badly_named_methods(self):
536n/a # test work-around for three methods that accidentally follow the
537n/a # naming conventions for handler methods
538n/a # (*_open() / *_request() / *_response())
539n/a
540n/a # These used to call the accidentally-named methods, causing a
541n/a # TypeError in real code; here, returning self from these mock
542n/a # methods would either cause no exception, or AttributeError.
543n/a
544n/a from urllib.error import URLError
545n/a
546n/a o = OpenerDirector()
547n/a meth_spec = [
548n/a [("do_open", "return self"), ("proxy_open", "return self")],
549n/a [("redirect_request", "return self")],
550n/a ]
551n/a add_ordered_mock_handlers(o, meth_spec)
552n/a o.add_handler(urllib.request.UnknownHandler())
553n/a for scheme in "do", "proxy", "redirect":
554n/a self.assertRaises(URLError, o.open, scheme+"://example.com/")
555n/a
556n/a def test_handled(self):
557n/a # handler returning non-None means no more handlers will be called
558n/a o = OpenerDirector()
559n/a meth_spec = [
560n/a ["http_open", "ftp_open", "http_error_302"],
561n/a ["ftp_open"],
562n/a [("http_open", "return self")],
563n/a [("http_open", "return self")],
564n/a ]
565n/a handlers = add_ordered_mock_handlers(o, meth_spec)
566n/a
567n/a req = Request("http://example.com/")
568n/a r = o.open(req)
569n/a # Second .http_open() gets called, third doesn't, since second returned
570n/a # non-None. Handlers without .http_open() never get any methods called
571n/a # on them.
572n/a # In fact, second mock handler defining .http_open() returns self
573n/a # (instead of response), which becomes the OpenerDirector's return
574n/a # value.
575n/a self.assertEqual(r, handlers[2])
576n/a calls = [(handlers[0], "http_open"), (handlers[2], "http_open")]
577n/a for expected, got in zip(calls, o.calls):
578n/a handler, name, args, kwds = got
579n/a self.assertEqual((handler, name), expected)
580n/a self.assertEqual(args, (req,))
581n/a
582n/a def test_handler_order(self):
583n/a o = OpenerDirector()
584n/a handlers = []
585n/a for meths, handler_order in [([("http_open", "return self")], 500),
586n/a (["http_open"], 0)]:
587n/a class MockHandlerSubclass(MockHandler):
588n/a pass
589n/a
590n/a h = MockHandlerSubclass(meths)
591n/a h.handler_order = handler_order
592n/a handlers.append(h)
593n/a o.add_handler(h)
594n/a
595n/a o.open("http://example.com/")
596n/a # handlers called in reverse order, thanks to their sort order
597n/a self.assertEqual(o.calls[0][0], handlers[1])
598n/a self.assertEqual(o.calls[1][0], handlers[0])
599n/a
600n/a def test_raise(self):
601n/a # raising URLError stops processing of request
602n/a o = OpenerDirector()
603n/a meth_spec = [
604n/a [("http_open", "raise")],
605n/a [("http_open", "return self")],
606n/a ]
607n/a handlers = add_ordered_mock_handlers(o, meth_spec)
608n/a
609n/a req = Request("http://example.com/")
610n/a self.assertRaises(urllib.error.URLError, o.open, req)
611n/a self.assertEqual(o.calls, [(handlers[0], "http_open", (req,), {})])
612n/a
613n/a def test_http_error(self):
614n/a # XXX http_error_default
615n/a # http errors are a special case
616n/a o = OpenerDirector()
617n/a meth_spec = [
618n/a [("http_open", "error 302")],
619n/a [("http_error_400", "raise"), "http_open"],
620n/a [("http_error_302", "return response"), "http_error_303",
621n/a "http_error"],
622n/a [("http_error_302")],
623n/a ]
624n/a handlers = add_ordered_mock_handlers(o, meth_spec)
625n/a
626n/a class Unknown:
627n/a def __eq__(self, other):
628n/a return True
629n/a
630n/a req = Request("http://example.com/")
631n/a o.open(req)
632n/a assert len(o.calls) == 2
633n/a calls = [(handlers[0], "http_open", (req,)),
634n/a (handlers[2], "http_error_302",
635n/a (req, Unknown(), 302, "", {}))]
636n/a for expected, got in zip(calls, o.calls):
637n/a handler, method_name, args = expected
638n/a self.assertEqual((handler, method_name), got[:2])
639n/a self.assertEqual(args, got[2])
640n/a
641n/a def test_processors(self):
642n/a # *_request / *_response methods get called appropriately
643n/a o = OpenerDirector()
644n/a meth_spec = [
645n/a [("http_request", "return request"),
646n/a ("http_response", "return response")],
647n/a [("http_request", "return request"),
648n/a ("http_response", "return response")],
649n/a ]
650n/a handlers = add_ordered_mock_handlers(o, meth_spec)
651n/a
652n/a req = Request("http://example.com/")
653n/a o.open(req)
654n/a # processor methods are called on *all* handlers that define them,
655n/a # not just the first handler that handles the request
656n/a calls = [
657n/a (handlers[0], "http_request"), (handlers[1], "http_request"),
658n/a (handlers[0], "http_response"), (handlers[1], "http_response")]
659n/a
660n/a for i, (handler, name, args, kwds) in enumerate(o.calls):
661n/a if i < 2:
662n/a # *_request
663n/a self.assertEqual((handler, name), calls[i])
664n/a self.assertEqual(len(args), 1)
665n/a self.assertIsInstance(args[0], Request)
666n/a else:
667n/a # *_response
668n/a self.assertEqual((handler, name), calls[i])
669n/a self.assertEqual(len(args), 2)
670n/a self.assertIsInstance(args[0], Request)
671n/a # response from opener.open is None, because there's no
672n/a # handler that defines http_open to handle it
673n/a if args[1] is not None:
674n/a self.assertIsInstance(args[1], MockResponse)
675n/a
676n/a
677n/adef sanepathname2url(path):
678n/a try:
679n/a path.encode("utf-8")
680n/a except UnicodeEncodeError:
681n/a raise unittest.SkipTest("path is not encodable to utf8")
682n/a urlpath = urllib.request.pathname2url(path)
683n/a if os.name == "nt" and urlpath.startswith("///"):
684n/a urlpath = urlpath[2:]
685n/a # XXX don't ask me about the mac...
686n/a return urlpath
687n/a
688n/a
689n/aclass HandlerTests(unittest.TestCase):
690n/a
691n/a def test_ftp(self):
692n/a class MockFTPWrapper:
693n/a def __init__(self, data):
694n/a self.data = data
695n/a
696n/a def retrfile(self, filename, filetype):
697n/a self.filename, self.filetype = filename, filetype
698n/a return io.StringIO(self.data), len(self.data)
699n/a
700n/a def close(self):
701n/a pass
702n/a
703n/a class NullFTPHandler(urllib.request.FTPHandler):
704n/a def __init__(self, data):
705n/a self.data = data
706n/a
707n/a def connect_ftp(self, user, passwd, host, port, dirs,
708n/a timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
709n/a self.user, self.passwd = user, passwd
710n/a self.host, self.port = host, port
711n/a self.dirs = dirs
712n/a self.ftpwrapper = MockFTPWrapper(self.data)
713n/a return self.ftpwrapper
714n/a
715n/a import ftplib
716n/a data = "rheum rhaponicum"
717n/a h = NullFTPHandler(data)
718n/a h.parent = MockOpener()
719n/a
720n/a for url, host, port, user, passwd, type_, dirs, filename, mimetype in [
721n/a ("ftp://localhost/foo/bar/baz.html",
722n/a "localhost", ftplib.FTP_PORT, "", "", "I",
723n/a ["foo", "bar"], "baz.html", "text/html"),
724n/a ("ftp://parrot@localhost/foo/bar/baz.html",
725n/a "localhost", ftplib.FTP_PORT, "parrot", "", "I",
726n/a ["foo", "bar"], "baz.html", "text/html"),
727n/a ("ftp://%25parrot@localhost/foo/bar/baz.html",
728n/a "localhost", ftplib.FTP_PORT, "%parrot", "", "I",
729n/a ["foo", "bar"], "baz.html", "text/html"),
730n/a ("ftp://%2542parrot@localhost/foo/bar/baz.html",
731n/a "localhost", ftplib.FTP_PORT, "%42parrot", "", "I",
732n/a ["foo", "bar"], "baz.html", "text/html"),
733n/a ("ftp://localhost:80/foo/bar/",
734n/a "localhost", 80, "", "", "D",
735n/a ["foo", "bar"], "", None),
736n/a ("ftp://localhost/baz.gif;type=a",
737n/a "localhost", ftplib.FTP_PORT, "", "", "A",
738n/a [], "baz.gif", None), # XXX really this should guess image/gif
739n/a ]:
740n/a req = Request(url)
741n/a req.timeout = None
742n/a r = h.ftp_open(req)
743n/a # ftp authentication not yet implemented by FTPHandler
744n/a self.assertEqual(h.user, user)
745n/a self.assertEqual(h.passwd, passwd)
746n/a self.assertEqual(h.host, socket.gethostbyname(host))
747n/a self.assertEqual(h.port, port)
748n/a self.assertEqual(h.dirs, dirs)
749n/a self.assertEqual(h.ftpwrapper.filename, filename)
750n/a self.assertEqual(h.ftpwrapper.filetype, type_)
751n/a headers = r.info()
752n/a self.assertEqual(headers.get("Content-type"), mimetype)
753n/a self.assertEqual(int(headers["Content-length"]), len(data))
754n/a
755n/a def test_file(self):
756n/a import email.utils
757n/a h = urllib.request.FileHandler()
758n/a o = h.parent = MockOpener()
759n/a
760n/a TESTFN = support.TESTFN
761n/a urlpath = sanepathname2url(os.path.abspath(TESTFN))
762n/a towrite = b"hello, world\n"
763n/a urls = [
764n/a "file://localhost%s" % urlpath,
765n/a "file://%s" % urlpath,
766n/a "file://%s%s" % (socket.gethostbyname('localhost'), urlpath),
767n/a ]
768n/a try:
769n/a localaddr = socket.gethostbyname(socket.gethostname())
770n/a except socket.gaierror:
771n/a localaddr = ''
772n/a if localaddr:
773n/a urls.append("file://%s%s" % (localaddr, urlpath))
774n/a
775n/a for url in urls:
776n/a f = open(TESTFN, "wb")
777n/a try:
778n/a try:
779n/a f.write(towrite)
780n/a finally:
781n/a f.close()
782n/a
783n/a r = h.file_open(Request(url))
784n/a try:
785n/a data = r.read()
786n/a headers = r.info()
787n/a respurl = r.geturl()
788n/a finally:
789n/a r.close()
790n/a stats = os.stat(TESTFN)
791n/a modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
792n/a finally:
793n/a os.remove(TESTFN)
794n/a self.assertEqual(data, towrite)
795n/a self.assertEqual(headers["Content-type"], "text/plain")
796n/a self.assertEqual(headers["Content-length"], "13")
797n/a self.assertEqual(headers["Last-modified"], modified)
798n/a self.assertEqual(respurl, url)
799n/a
800n/a for url in [
801n/a "file://localhost:80%s" % urlpath,
802n/a "file:///file_does_not_exist.txt",
803n/a "file://not-a-local-host.com//dir/file.txt",
804n/a "file://%s:80%s/%s" % (socket.gethostbyname('localhost'),
805n/a os.getcwd(), TESTFN),
806n/a "file://somerandomhost.ontheinternet.com%s/%s" %
807n/a (os.getcwd(), TESTFN),
808n/a ]:
809n/a try:
810n/a f = open(TESTFN, "wb")
811n/a try:
812n/a f.write(towrite)
813n/a finally:
814n/a f.close()
815n/a
816n/a self.assertRaises(urllib.error.URLError,
817n/a h.file_open, Request(url))
818n/a finally:
819n/a os.remove(TESTFN)
820n/a
821n/a h = urllib.request.FileHandler()
822n/a o = h.parent = MockOpener()
823n/a # XXXX why does // mean ftp (and /// mean not ftp!), and where
824n/a # is file: scheme specified? I think this is really a bug, and
825n/a # what was intended was to distinguish between URLs like:
826n/a # file:/blah.txt (a file)
827n/a # file://localhost/blah.txt (a file)
828n/a # file:///blah.txt (a file)
829n/a # file://ftp.example.com/blah.txt (an ftp URL)
830n/a for url, ftp in [
831n/a ("file://ftp.example.com//foo.txt", False),
832n/a ("file://ftp.example.com///foo.txt", False),
833n/a# XXXX bug: fails with OSError, should be URLError
834n/a ("file://ftp.example.com/foo.txt", False),
835n/a ("file://somehost//foo/something.txt", False),
836n/a ("file://localhost//foo/something.txt", False),
837n/a ]:
838n/a req = Request(url)
839n/a try:
840n/a h.file_open(req)
841n/a # XXXX remove OSError when bug fixed
842n/a except (urllib.error.URLError, OSError):
843n/a self.assertFalse(ftp)
844n/a else:
845n/a self.assertIs(o.req, req)
846n/a self.assertEqual(req.type, "ftp")
847n/a self.assertEqual(req.type == "ftp", ftp)
848n/a
849n/a def test_http(self):
850n/a
851n/a h = urllib.request.AbstractHTTPHandler()
852n/a o = h.parent = MockOpener()
853n/a
854n/a url = "http://example.com/"
855n/a for method, data in [("GET", None), ("POST", b"blah")]:
856n/a req = Request(url, data, {"Foo": "bar"})
857n/a req.timeout = None
858n/a req.add_unredirected_header("Spam", "eggs")
859n/a http = MockHTTPClass()
860n/a r = h.do_open(http, req)
861n/a
862n/a # result attributes
863n/a r.read; r.readline # wrapped MockFile methods
864n/a r.info; r.geturl # addinfourl methods
865n/a r.code, r.msg == 200, "OK" # added from MockHTTPClass.getreply()
866n/a hdrs = r.info()
867n/a hdrs.get; hdrs.__contains__ # r.info() gives dict from .getreply()
868n/a self.assertEqual(r.geturl(), url)
869n/a
870n/a self.assertEqual(http.host, "example.com")
871n/a self.assertEqual(http.level, 0)
872n/a self.assertEqual(http.method, method)
873n/a self.assertEqual(http.selector, "/")
874n/a self.assertEqual(http.req_headers,
875n/a [("Connection", "close"),
876n/a ("Foo", "bar"), ("Spam", "eggs")])
877n/a self.assertEqual(http.data, data)
878n/a
879n/a # check OSError converted to URLError
880n/a http.raise_on_endheaders = True
881n/a self.assertRaises(urllib.error.URLError, h.do_open, http, req)
882n/a
883n/a # Check for TypeError on POST data which is str.
884n/a req = Request("http://example.com/","badpost")
885n/a self.assertRaises(TypeError, h.do_request_, req)
886n/a
887n/a # check adding of standard headers
888n/a o.addheaders = [("Spam", "eggs")]
889n/a for data in b"", None: # POST, GET
890n/a req = Request("http://example.com/", data)
891n/a r = MockResponse(200, "OK", {}, "")
892n/a newreq = h.do_request_(req)
893n/a if data is None: # GET
894n/a self.assertNotIn("Content-length", req.unredirected_hdrs)
895n/a self.assertNotIn("Content-type", req.unredirected_hdrs)
896n/a else: # POST
897n/a self.assertEqual(req.unredirected_hdrs["Content-length"], "0")
898n/a self.assertEqual(req.unredirected_hdrs["Content-type"],
899n/a "application/x-www-form-urlencoded")
900n/a # XXX the details of Host could be better tested
901n/a self.assertEqual(req.unredirected_hdrs["Host"], "example.com")
902n/a self.assertEqual(req.unredirected_hdrs["Spam"], "eggs")
903n/a
904n/a # don't clobber existing headers
905n/a req.add_unredirected_header("Content-length", "foo")
906n/a req.add_unredirected_header("Content-type", "bar")
907n/a req.add_unredirected_header("Host", "baz")
908n/a req.add_unredirected_header("Spam", "foo")
909n/a newreq = h.do_request_(req)
910n/a self.assertEqual(req.unredirected_hdrs["Content-length"], "foo")
911n/a self.assertEqual(req.unredirected_hdrs["Content-type"], "bar")
912n/a self.assertEqual(req.unredirected_hdrs["Host"], "baz")
913n/a self.assertEqual(req.unredirected_hdrs["Spam"], "foo")
914n/a
915n/a def test_http_body_file(self):
916n/a # A regular file - chunked encoding is used unless Content Length is
917n/a # already set.
918n/a
919n/a h = urllib.request.AbstractHTTPHandler()
920n/a o = h.parent = MockOpener()
921n/a
922n/a file_obj = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
923n/a file_path = file_obj.name
924n/a file_obj.close()
925n/a self.addCleanup(os.unlink, file_path)
926n/a
927n/a with open(file_path, "rb") as f:
928n/a req = Request("http://example.com/", f, {})
929n/a newreq = h.do_request_(req)
930n/a te = newreq.get_header('Transfer-encoding')
931n/a self.assertEqual(te, "chunked")
932n/a self.assertFalse(newreq.has_header('Content-length'))
933n/a
934n/a with open(file_path, "rb") as f:
935n/a req = Request("http://example.com/", f, {"Content-Length": 30})
936n/a newreq = h.do_request_(req)
937n/a self.assertEqual(int(newreq.get_header('Content-length')), 30)
938n/a self.assertFalse(newreq.has_header("Transfer-encoding"))
939n/a
940n/a def test_http_body_fileobj(self):
941n/a # A file object - chunked encoding is used
942n/a # unless Content Length is already set.
943n/a # (Note that there are some subtle differences to a regular
944n/a # file, that is why we are testing both cases.)
945n/a
946n/a h = urllib.request.AbstractHTTPHandler()
947n/a o = h.parent = MockOpener()
948n/a file_obj = io.BytesIO()
949n/a
950n/a req = Request("http://example.com/", file_obj, {})
951n/a newreq = h.do_request_(req)
952n/a self.assertEqual(newreq.get_header('Transfer-encoding'), 'chunked')
953n/a self.assertFalse(newreq.has_header('Content-length'))
954n/a
955n/a headers = {"Content-Length": 30}
956n/a req = Request("http://example.com/", file_obj, headers)
957n/a newreq = h.do_request_(req)
958n/a self.assertEqual(int(newreq.get_header('Content-length')), 30)
959n/a self.assertFalse(newreq.has_header("Transfer-encoding"))
960n/a
961n/a file_obj.close()
962n/a
963n/a def test_http_body_pipe(self):
964n/a # A file reading from a pipe.
965n/a # A pipe cannot be seek'ed. There is no way to determine the
966n/a # content length up front. Thus, do_request_() should fall
967n/a # back to Transfer-encoding chunked.
968n/a
969n/a h = urllib.request.AbstractHTTPHandler()
970n/a o = h.parent = MockOpener()
971n/a
972n/a cmd = [sys.executable, "-c", r"pass"]
973n/a for headers in {}, {"Content-Length": 30}:
974n/a with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc:
975n/a req = Request("http://example.com/", proc.stdout, headers)
976n/a newreq = h.do_request_(req)
977n/a if not headers:
978n/a self.assertEqual(newreq.get_header('Content-length'), None)
979n/a self.assertEqual(newreq.get_header('Transfer-encoding'),
980n/a 'chunked')
981n/a else:
982n/a self.assertEqual(int(newreq.get_header('Content-length')),
983n/a 30)
984n/a
985n/a def test_http_body_iterable(self):
986n/a # Generic iterable. There is no way to determine the content
987n/a # length up front. Fall back to Transfer-encoding chunked.
988n/a
989n/a h = urllib.request.AbstractHTTPHandler()
990n/a o = h.parent = MockOpener()
991n/a
992n/a def iterable_body():
993n/a yield b"one"
994n/a
995n/a for headers in {}, {"Content-Length": 11}:
996n/a req = Request("http://example.com/", iterable_body(), headers)
997n/a newreq = h.do_request_(req)
998n/a if not headers:
999n/a self.assertEqual(newreq.get_header('Content-length'), None)
1000n/a self.assertEqual(newreq.get_header('Transfer-encoding'),
1001n/a 'chunked')
1002n/a else:
1003n/a self.assertEqual(int(newreq.get_header('Content-length')), 11)
1004n/a
1005n/a def test_http_body_empty_seq(self):
1006n/a # Zero-length iterable body should be treated like any other iterable
1007n/a h = urllib.request.AbstractHTTPHandler()
1008n/a h.parent = MockOpener()
1009n/a req = h.do_request_(Request("http://example.com/", ()))
1010n/a self.assertEqual(req.get_header("Transfer-encoding"), "chunked")
1011n/a self.assertFalse(req.has_header("Content-length"))
1012n/a
1013n/a def test_http_body_array(self):
1014n/a # array.array Iterable - Content Length is calculated
1015n/a
1016n/a h = urllib.request.AbstractHTTPHandler()
1017n/a o = h.parent = MockOpener()
1018n/a
1019n/a iterable_array = array.array("I",[1,2,3,4])
1020n/a
1021n/a for headers in {}, {"Content-Length": 16}:
1022n/a req = Request("http://example.com/", iterable_array, headers)
1023n/a newreq = h.do_request_(req)
1024n/a self.assertEqual(int(newreq.get_header('Content-length')),16)
1025n/a
1026n/a def test_http_handler_debuglevel(self):
1027n/a o = OpenerDirector()
1028n/a h = MockHTTPSHandler(debuglevel=1)
1029n/a o.add_handler(h)
1030n/a o.open("https://www.example.com")
1031n/a self.assertEqual(h._debuglevel, 1)
1032n/a
1033n/a def test_http_doubleslash(self):
1034n/a # Checks the presence of any unnecessary double slash in url does not
1035n/a # break anything. Previously, a double slash directly after the host
1036n/a # could cause incorrect parsing.
1037n/a h = urllib.request.AbstractHTTPHandler()
1038n/a h.parent = MockOpener()
1039n/a
1040n/a data = b""
1041n/a ds_urls = [
1042n/a "http://example.com/foo/bar/baz.html",
1043n/a "http://example.com//foo/bar/baz.html",
1044n/a "http://example.com/foo//bar/baz.html",
1045n/a "http://example.com/foo/bar//baz.html"
1046n/a ]
1047n/a
1048n/a for ds_url in ds_urls:
1049n/a ds_req = Request(ds_url, data)
1050n/a
1051n/a # Check whether host is determined correctly if there is no proxy
1052n/a np_ds_req = h.do_request_(ds_req)
1053n/a self.assertEqual(np_ds_req.unredirected_hdrs["Host"], "example.com")
1054n/a
1055n/a # Check whether host is determined correctly if there is a proxy
1056n/a ds_req.set_proxy("someproxy:3128", None)
1057n/a p_ds_req = h.do_request_(ds_req)
1058n/a self.assertEqual(p_ds_req.unredirected_hdrs["Host"], "example.com")
1059n/a
1060n/a def test_full_url_setter(self):
1061n/a # Checks to ensure that components are set correctly after setting the
1062n/a # full_url of a Request object
1063n/a
1064n/a urls = [
1065n/a 'http://example.com?foo=bar#baz',
1066n/a 'http://example.com?foo=bar&spam=eggs#bash',
1067n/a 'http://example.com',
1068n/a ]
1069n/a
1070n/a # testing a reusable request instance, but the url parameter is
1071n/a # required, so just use a dummy one to instantiate
1072n/a r = Request('http://example.com')
1073n/a for url in urls:
1074n/a r.full_url = url
1075n/a parsed = urlparse(url)
1076n/a
1077n/a self.assertEqual(r.get_full_url(), url)
1078n/a # full_url setter uses splittag to split into components.
1079n/a # splittag sets the fragment as None while urlparse sets it to ''
1080n/a self.assertEqual(r.fragment or '', parsed.fragment)
1081n/a self.assertEqual(urlparse(r.get_full_url()).query, parsed.query)
1082n/a
1083n/a def test_full_url_deleter(self):
1084n/a r = Request('http://www.example.com')
1085n/a del r.full_url
1086n/a self.assertIsNone(r.full_url)
1087n/a self.assertIsNone(r.fragment)
1088n/a self.assertEqual(r.selector, '')
1089n/a
1090n/a def test_fixpath_in_weirdurls(self):
1091n/a # Issue4493: urllib2 to supply '/' when to urls where path does not
1092n/a # start with'/'
1093n/a
1094n/a h = urllib.request.AbstractHTTPHandler()
1095n/a h.parent = MockOpener()
1096n/a
1097n/a weird_url = 'http://www.python.org?getspam'
1098n/a req = Request(weird_url)
1099n/a newreq = h.do_request_(req)
1100n/a self.assertEqual(newreq.host, 'www.python.org')
1101n/a self.assertEqual(newreq.selector, '/?getspam')
1102n/a
1103n/a url_without_path = 'http://www.python.org'
1104n/a req = Request(url_without_path)
1105n/a newreq = h.do_request_(req)
1106n/a self.assertEqual(newreq.host, 'www.python.org')
1107n/a self.assertEqual(newreq.selector, '')
1108n/a
1109n/a def test_errors(self):
1110n/a h = urllib.request.HTTPErrorProcessor()
1111n/a o = h.parent = MockOpener()
1112n/a
1113n/a url = "http://example.com/"
1114n/a req = Request(url)
1115n/a # all 2xx are passed through
1116n/a r = MockResponse(200, "OK", {}, "", url)
1117n/a newr = h.http_response(req, r)
1118n/a self.assertIs(r, newr)
1119n/a self.assertFalse(hasattr(o, "proto")) # o.error not called
1120n/a r = MockResponse(202, "Accepted", {}, "", url)
1121n/a newr = h.http_response(req, r)
1122n/a self.assertIs(r, newr)
1123n/a self.assertFalse(hasattr(o, "proto")) # o.error not called
1124n/a r = MockResponse(206, "Partial content", {}, "", url)
1125n/a newr = h.http_response(req, r)
1126n/a self.assertIs(r, newr)
1127n/a self.assertFalse(hasattr(o, "proto")) # o.error not called
1128n/a # anything else calls o.error (and MockOpener returns None, here)
1129n/a r = MockResponse(502, "Bad gateway", {}, "", url)
1130n/a self.assertIsNone(h.http_response(req, r))
1131n/a self.assertEqual(o.proto, "http") # o.error called
1132n/a self.assertEqual(o.args, (req, r, 502, "Bad gateway", {}))
1133n/a
1134n/a def test_cookies(self):
1135n/a cj = MockCookieJar()
1136n/a h = urllib.request.HTTPCookieProcessor(cj)
1137n/a h.parent = MockOpener()
1138n/a
1139n/a req = Request("http://example.com/")
1140n/a r = MockResponse(200, "OK", {}, "")
1141n/a newreq = h.http_request(req)
1142n/a self.assertIs(cj.ach_req, req)
1143n/a self.assertIs(cj.ach_req, newreq)
1144n/a self.assertEqual(req.origin_req_host, "example.com")
1145n/a self.assertFalse(req.unverifiable)
1146n/a newr = h.http_response(req, r)
1147n/a self.assertIs(cj.ec_req, req)
1148n/a self.assertIs(cj.ec_r, r)
1149n/a self.assertIs(r, newr)
1150n/a
1151n/a def test_redirect(self):
1152n/a from_url = "http://example.com/a.html"
1153n/a to_url = "http://example.com/b.html"
1154n/a h = urllib.request.HTTPRedirectHandler()
1155n/a o = h.parent = MockOpener()
1156n/a
1157n/a # ordinary redirect behaviour
1158n/a for code in 301, 302, 303, 307:
1159n/a for data in None, "blah\nblah\n":
1160n/a method = getattr(h, "http_error_%s" % code)
1161n/a req = Request(from_url, data)
1162n/a req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1163n/a req.add_header("Nonsense", "viking=withhold")
1164n/a if data is not None:
1165n/a req.add_header("Content-Length", str(len(data)))
1166n/a req.add_unredirected_header("Spam", "spam")
1167n/a try:
1168n/a method(req, MockFile(), code, "Blah",
1169n/a MockHeaders({"location": to_url}))
1170n/a except urllib.error.HTTPError:
1171n/a # 307 in response to POST requires user OK
1172n/a self.assertEqual(code, 307)
1173n/a self.assertIsNotNone(data)
1174n/a self.assertEqual(o.req.get_full_url(), to_url)
1175n/a try:
1176n/a self.assertEqual(o.req.get_method(), "GET")
1177n/a except AttributeError:
1178n/a self.assertFalse(o.req.data)
1179n/a
1180n/a # now it's a GET, there should not be headers regarding content
1181n/a # (possibly dragged from before being a POST)
1182n/a headers = [x.lower() for x in o.req.headers]
1183n/a self.assertNotIn("content-length", headers)
1184n/a self.assertNotIn("content-type", headers)
1185n/a
1186n/a self.assertEqual(o.req.headers["Nonsense"],
1187n/a "viking=withhold")
1188n/a self.assertNotIn("Spam", o.req.headers)
1189n/a self.assertNotIn("Spam", o.req.unredirected_hdrs)
1190n/a
1191n/a # loop detection
1192n/a req = Request(from_url)
1193n/a req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1194n/a
1195n/a def redirect(h, req, url=to_url):
1196n/a h.http_error_302(req, MockFile(), 302, "Blah",
1197n/a MockHeaders({"location": url}))
1198n/a # Note that the *original* request shares the same record of
1199n/a # redirections with the sub-requests caused by the redirections.
1200n/a
1201n/a # detect infinite loop redirect of a URL to itself
1202n/a req = Request(from_url, origin_req_host="example.com")
1203n/a count = 0
1204n/a req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1205n/a try:
1206n/a while 1:
1207n/a redirect(h, req, "http://example.com/")
1208n/a count = count + 1
1209n/a except urllib.error.HTTPError:
1210n/a # don't stop until max_repeats, because cookies may introduce state
1211n/a self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_repeats)
1212n/a
1213n/a # detect endless non-repeating chain of redirects
1214n/a req = Request(from_url, origin_req_host="example.com")
1215n/a count = 0
1216n/a req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1217n/a try:
1218n/a while 1:
1219n/a redirect(h, req, "http://example.com/%d" % count)
1220n/a count = count + 1
1221n/a except urllib.error.HTTPError:
1222n/a self.assertEqual(count,
1223n/a urllib.request.HTTPRedirectHandler.max_redirections)
1224n/a
1225n/a def test_invalid_redirect(self):
1226n/a from_url = "http://example.com/a.html"
1227n/a valid_schemes = ['http','https','ftp']
1228n/a invalid_schemes = ['file','imap','ldap']
1229n/a schemeless_url = "example.com/b.html"
1230n/a h = urllib.request.HTTPRedirectHandler()
1231n/a o = h.parent = MockOpener()
1232n/a req = Request(from_url)
1233n/a req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1234n/a
1235n/a for scheme in invalid_schemes:
1236n/a invalid_url = scheme + '://' + schemeless_url
1237n/a self.assertRaises(urllib.error.HTTPError, h.http_error_302,
1238n/a req, MockFile(), 302, "Security Loophole",
1239n/a MockHeaders({"location": invalid_url}))
1240n/a
1241n/a for scheme in valid_schemes:
1242n/a valid_url = scheme + '://' + schemeless_url
1243n/a h.http_error_302(req, MockFile(), 302, "That's fine",
1244n/a MockHeaders({"location": valid_url}))
1245n/a self.assertEqual(o.req.get_full_url(), valid_url)
1246n/a
1247n/a def test_relative_redirect(self):
1248n/a from_url = "http://example.com/a.html"
1249n/a relative_url = "/b.html"
1250n/a h = urllib.request.HTTPRedirectHandler()
1251n/a o = h.parent = MockOpener()
1252n/a req = Request(from_url)
1253n/a req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1254n/a
1255n/a valid_url = urllib.parse.urljoin(from_url,relative_url)
1256n/a h.http_error_302(req, MockFile(), 302, "That's fine",
1257n/a MockHeaders({"location": valid_url}))
1258n/a self.assertEqual(o.req.get_full_url(), valid_url)
1259n/a
1260n/a def test_cookie_redirect(self):
1261n/a # cookies shouldn't leak into redirected requests
1262n/a from http.cookiejar import CookieJar
1263n/a from test.test_http_cookiejar import interact_netscape
1264n/a
1265n/a cj = CookieJar()
1266n/a interact_netscape(cj, "http://www.example.com/", "spam=eggs")
1267n/a hh = MockHTTPHandler(302, "Location: http://www.cracker.com/\r\n\r\n")
1268n/a hdeh = urllib.request.HTTPDefaultErrorHandler()
1269n/a hrh = urllib.request.HTTPRedirectHandler()
1270n/a cp = urllib.request.HTTPCookieProcessor(cj)
1271n/a o = build_test_opener(hh, hdeh, hrh, cp)
1272n/a o.open("http://www.example.com/")
1273n/a self.assertFalse(hh.req.has_header("Cookie"))
1274n/a
1275n/a def test_redirect_fragment(self):
1276n/a redirected_url = 'http://www.example.com/index.html#OK\r\n\r\n'
1277n/a hh = MockHTTPHandler(302, 'Location: ' + redirected_url)
1278n/a hdeh = urllib.request.HTTPDefaultErrorHandler()
1279n/a hrh = urllib.request.HTTPRedirectHandler()
1280n/a o = build_test_opener(hh, hdeh, hrh)
1281n/a fp = o.open('http://www.example.com')
1282n/a self.assertEqual(fp.geturl(), redirected_url.strip())
1283n/a
1284n/a def test_redirect_no_path(self):
1285n/a # Issue 14132: Relative redirect strips original path
1286n/a real_class = http.client.HTTPConnection
1287n/a response1 = b"HTTP/1.1 302 Found\r\nLocation: ?query\r\n\r\n"
1288n/a http.client.HTTPConnection = test_urllib.fakehttp(response1)
1289n/a self.addCleanup(setattr, http.client, "HTTPConnection", real_class)
1290n/a urls = iter(("/path", "/path?query"))
1291n/a def request(conn, method, url, *pos, **kw):
1292n/a self.assertEqual(url, next(urls))
1293n/a real_class.request(conn, method, url, *pos, **kw)
1294n/a # Change response for subsequent connection
1295n/a conn.__class__.fakedata = b"HTTP/1.1 200 OK\r\n\r\nHello!"
1296n/a http.client.HTTPConnection.request = request
1297n/a fp = urllib.request.urlopen("http://python.org/path")
1298n/a self.assertEqual(fp.geturl(), "http://python.org/path?query")
1299n/a
1300n/a def test_redirect_encoding(self):
1301n/a # Some characters in the redirect target may need special handling,
1302n/a # but most ASCII characters should be treated as already encoded
1303n/a class Handler(urllib.request.HTTPHandler):
1304n/a def http_open(self, req):
1305n/a result = self.do_open(self.connection, req)
1306n/a self.last_buf = self.connection.buf
1307n/a # Set up a normal response for the next request
1308n/a self.connection = test_urllib.fakehttp(
1309n/a b'HTTP/1.1 200 OK\r\n'
1310n/a b'Content-Length: 3\r\n'
1311n/a b'\r\n'
1312n/a b'123'
1313n/a )
1314n/a return result
1315n/a handler = Handler()
1316n/a opener = urllib.request.build_opener(handler)
1317n/a tests = (
1318n/a (b'/p\xC3\xA5-dansk/', b'/p%C3%A5-dansk/'),
1319n/a (b'/spaced%20path/', b'/spaced%20path/'),
1320n/a (b'/spaced path/', b'/spaced%20path/'),
1321n/a (b'/?p\xC3\xA5-dansk', b'/?p%C3%A5-dansk'),
1322n/a )
1323n/a for [location, result] in tests:
1324n/a with self.subTest(repr(location)):
1325n/a handler.connection = test_urllib.fakehttp(
1326n/a b'HTTP/1.1 302 Redirect\r\n'
1327n/a b'Location: ' + location + b'\r\n'
1328n/a b'\r\n'
1329n/a )
1330n/a response = opener.open('http://example.com/')
1331n/a expected = b'GET ' + result + b' '
1332n/a request = handler.last_buf
1333n/a self.assertTrue(request.startswith(expected), repr(request))
1334n/a
1335n/a def test_proxy(self):
1336n/a o = OpenerDirector()
1337n/a ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128"))
1338n/a o.add_handler(ph)
1339n/a meth_spec = [
1340n/a [("http_open", "return response")]
1341n/a ]
1342n/a handlers = add_ordered_mock_handlers(o, meth_spec)
1343n/a
1344n/a req = Request("http://acme.example.com/")
1345n/a self.assertEqual(req.host, "acme.example.com")
1346n/a o.open(req)
1347n/a self.assertEqual(req.host, "proxy.example.com:3128")
1348n/a
1349n/a self.assertEqual([(handlers[0], "http_open")],
1350n/a [tup[0:2] for tup in o.calls])
1351n/a
1352n/a def test_proxy_no_proxy(self):
1353n/a os.environ['no_proxy'] = 'python.org'
1354n/a o = OpenerDirector()
1355n/a ph = urllib.request.ProxyHandler(dict(http="proxy.example.com"))
1356n/a o.add_handler(ph)
1357n/a req = Request("http://www.perl.org/")
1358n/a self.assertEqual(req.host, "www.perl.org")
1359n/a o.open(req)
1360n/a self.assertEqual(req.host, "proxy.example.com")
1361n/a req = Request("http://www.python.org")
1362n/a self.assertEqual(req.host, "www.python.org")
1363n/a o.open(req)
1364n/a self.assertEqual(req.host, "www.python.org")
1365n/a del os.environ['no_proxy']
1366n/a
1367n/a def test_proxy_no_proxy_all(self):
1368n/a os.environ['no_proxy'] = '*'
1369n/a o = OpenerDirector()
1370n/a ph = urllib.request.ProxyHandler(dict(http="proxy.example.com"))
1371n/a o.add_handler(ph)
1372n/a req = Request("http://www.python.org")
1373n/a self.assertEqual(req.host, "www.python.org")
1374n/a o.open(req)
1375n/a self.assertEqual(req.host, "www.python.org")
1376n/a del os.environ['no_proxy']
1377n/a
1378n/a def test_proxy_https(self):
1379n/a o = OpenerDirector()
1380n/a ph = urllib.request.ProxyHandler(dict(https="proxy.example.com:3128"))
1381n/a o.add_handler(ph)
1382n/a meth_spec = [
1383n/a [("https_open", "return response")]
1384n/a ]
1385n/a handlers = add_ordered_mock_handlers(o, meth_spec)
1386n/a
1387n/a req = Request("https://www.example.com/")
1388n/a self.assertEqual(req.host, "www.example.com")
1389n/a o.open(req)
1390n/a self.assertEqual(req.host, "proxy.example.com:3128")
1391n/a self.assertEqual([(handlers[0], "https_open")],
1392n/a [tup[0:2] for tup in o.calls])
1393n/a
1394n/a def test_proxy_https_proxy_authorization(self):
1395n/a o = OpenerDirector()
1396n/a ph = urllib.request.ProxyHandler(dict(https='proxy.example.com:3128'))
1397n/a o.add_handler(ph)
1398n/a https_handler = MockHTTPSHandler()
1399n/a o.add_handler(https_handler)
1400n/a req = Request("https://www.example.com/")
1401n/a req.add_header("Proxy-Authorization", "FooBar")
1402n/a req.add_header("User-Agent", "Grail")
1403n/a self.assertEqual(req.host, "www.example.com")
1404n/a self.assertIsNone(req._tunnel_host)
1405n/a o.open(req)
1406n/a # Verify Proxy-Authorization gets tunneled to request.
1407n/a # httpsconn req_headers do not have the Proxy-Authorization header but
1408n/a # the req will have.
1409n/a self.assertNotIn(("Proxy-Authorization", "FooBar"),
1410n/a https_handler.httpconn.req_headers)
1411n/a self.assertIn(("User-Agent", "Grail"),
1412n/a https_handler.httpconn.req_headers)
1413n/a self.assertIsNotNone(req._tunnel_host)
1414n/a self.assertEqual(req.host, "proxy.example.com:3128")
1415n/a self.assertEqual(req.get_header("Proxy-authorization"), "FooBar")
1416n/a
1417n/a # TODO: This should be only for OSX
1418n/a @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX")
1419n/a def test_osx_proxy_bypass(self):
1420n/a bypass = {
1421n/a 'exclude_simple': False,
1422n/a 'exceptions': ['foo.bar', '*.bar.com', '127.0.0.1', '10.10',
1423n/a '10.0/16']
1424n/a }
1425n/a # Check hosts that should trigger the proxy bypass
1426n/a for host in ('foo.bar', 'www.bar.com', '127.0.0.1', '10.10.0.1',
1427n/a '10.0.0.1'):
1428n/a self.assertTrue(_proxy_bypass_macosx_sysconf(host, bypass),
1429n/a 'expected bypass of %s to be True' % host)
1430n/a # Check hosts that should not trigger the proxy bypass
1431n/a for host in ('abc.foo.bar', 'bar.com', '127.0.0.2', '10.11.0.1',
1432n/a 'notinbypass'):
1433n/a self.assertFalse(_proxy_bypass_macosx_sysconf(host, bypass),
1434n/a 'expected bypass of %s to be False' % host)
1435n/a
1436n/a # Check the exclude_simple flag
1437n/a bypass = {'exclude_simple': True, 'exceptions': []}
1438n/a self.assertTrue(_proxy_bypass_macosx_sysconf('test', bypass))
1439n/a
1440n/a def test_basic_auth(self, quote_char='"'):
1441n/a opener = OpenerDirector()
1442n/a password_manager = MockPasswordManager()
1443n/a auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
1444n/a realm = "ACME Widget Store"
1445n/a http_handler = MockHTTPHandler(
1446n/a 401, 'WWW-Authenticate: Basic realm=%s%s%s\r\n\r\n' %
1447n/a (quote_char, realm, quote_char))
1448n/a opener.add_handler(auth_handler)
1449n/a opener.add_handler(http_handler)
1450n/a self._test_basic_auth(opener, auth_handler, "Authorization",
1451n/a realm, http_handler, password_manager,
1452n/a "http://acme.example.com/protected",
1453n/a "http://acme.example.com/protected",
1454n/a )
1455n/a
1456n/a def test_basic_auth_with_single_quoted_realm(self):
1457n/a self.test_basic_auth(quote_char="'")
1458n/a
1459n/a def test_basic_auth_with_unquoted_realm(self):
1460n/a opener = OpenerDirector()
1461n/a password_manager = MockPasswordManager()
1462n/a auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
1463n/a realm = "ACME Widget Store"
1464n/a http_handler = MockHTTPHandler(
1465n/a 401, 'WWW-Authenticate: Basic realm=%s\r\n\r\n' % realm)
1466n/a opener.add_handler(auth_handler)
1467n/a opener.add_handler(http_handler)
1468n/a with self.assertWarns(UserWarning):
1469n/a self._test_basic_auth(opener, auth_handler, "Authorization",
1470n/a realm, http_handler, password_manager,
1471n/a "http://acme.example.com/protected",
1472n/a "http://acme.example.com/protected",
1473n/a )
1474n/a
1475n/a def test_proxy_basic_auth(self):
1476n/a opener = OpenerDirector()
1477n/a ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128"))
1478n/a opener.add_handler(ph)
1479n/a password_manager = MockPasswordManager()
1480n/a auth_handler = urllib.request.ProxyBasicAuthHandler(password_manager)
1481n/a realm = "ACME Networks"
1482n/a http_handler = MockHTTPHandler(
1483n/a 407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm)
1484n/a opener.add_handler(auth_handler)
1485n/a opener.add_handler(http_handler)
1486n/a self._test_basic_auth(opener, auth_handler, "Proxy-authorization",
1487n/a realm, http_handler, password_manager,
1488n/a "http://acme.example.com:3128/protected",
1489n/a "proxy.example.com:3128",
1490n/a )
1491n/a
1492n/a def test_basic_and_digest_auth_handlers(self):
1493n/a # HTTPDigestAuthHandler raised an exception if it couldn't handle a 40*
1494n/a # response (http://python.org/sf/1479302), where it should instead
1495n/a # return None to allow another handler (especially
1496n/a # HTTPBasicAuthHandler) to handle the response.
1497n/a
1498n/a # Also (http://python.org/sf/14797027, RFC 2617 section 1.2), we must
1499n/a # try digest first (since it's the strongest auth scheme), so we record
1500n/a # order of calls here to check digest comes first:
1501n/a class RecordingOpenerDirector(OpenerDirector):
1502n/a def __init__(self):
1503n/a OpenerDirector.__init__(self)
1504n/a self.recorded = []
1505n/a
1506n/a def record(self, info):
1507n/a self.recorded.append(info)
1508n/a
1509n/a class TestDigestAuthHandler(urllib.request.HTTPDigestAuthHandler):
1510n/a def http_error_401(self, *args, **kwds):
1511n/a self.parent.record("digest")
1512n/a urllib.request.HTTPDigestAuthHandler.http_error_401(self,
1513n/a *args, **kwds)
1514n/a
1515n/a class TestBasicAuthHandler(urllib.request.HTTPBasicAuthHandler):
1516n/a def http_error_401(self, *args, **kwds):
1517n/a self.parent.record("basic")
1518n/a urllib.request.HTTPBasicAuthHandler.http_error_401(self,
1519n/a *args, **kwds)
1520n/a
1521n/a opener = RecordingOpenerDirector()
1522n/a password_manager = MockPasswordManager()
1523n/a digest_handler = TestDigestAuthHandler(password_manager)
1524n/a basic_handler = TestBasicAuthHandler(password_manager)
1525n/a realm = "ACME Networks"
1526n/a http_handler = MockHTTPHandler(
1527n/a 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % realm)
1528n/a opener.add_handler(basic_handler)
1529n/a opener.add_handler(digest_handler)
1530n/a opener.add_handler(http_handler)
1531n/a
1532n/a # check basic auth isn't blocked by digest handler failing
1533n/a self._test_basic_auth(opener, basic_handler, "Authorization",
1534n/a realm, http_handler, password_manager,
1535n/a "http://acme.example.com/protected",
1536n/a "http://acme.example.com/protected",
1537n/a )
1538n/a # check digest was tried before basic (twice, because
1539n/a # _test_basic_auth called .open() twice)
1540n/a self.assertEqual(opener.recorded, ["digest", "basic"]*2)
1541n/a
1542n/a def test_unsupported_auth_digest_handler(self):
1543n/a opener = OpenerDirector()
1544n/a # While using DigestAuthHandler
1545n/a digest_auth_handler = urllib.request.HTTPDigestAuthHandler(None)
1546n/a http_handler = MockHTTPHandler(
1547n/a 401, 'WWW-Authenticate: Kerberos\r\n\r\n')
1548n/a opener.add_handler(digest_auth_handler)
1549n/a opener.add_handler(http_handler)
1550n/a self.assertRaises(ValueError, opener.open, "http://www.example.com")
1551n/a
1552n/a def test_unsupported_auth_basic_handler(self):
1553n/a # While using BasicAuthHandler
1554n/a opener = OpenerDirector()
1555n/a basic_auth_handler = urllib.request.HTTPBasicAuthHandler(None)
1556n/a http_handler = MockHTTPHandler(
1557n/a 401, 'WWW-Authenticate: NTLM\r\n\r\n')
1558n/a opener.add_handler(basic_auth_handler)
1559n/a opener.add_handler(http_handler)
1560n/a self.assertRaises(ValueError, opener.open, "http://www.example.com")
1561n/a
1562n/a def _test_basic_auth(self, opener, auth_handler, auth_header,
1563n/a realm, http_handler, password_manager,
1564n/a request_url, protected_url):
1565n/a import base64
1566n/a user, password = "wile", "coyote"
1567n/a
1568n/a # .add_password() fed through to password manager
1569n/a auth_handler.add_password(realm, request_url, user, password)
1570n/a self.assertEqual(realm, password_manager.realm)
1571n/a self.assertEqual(request_url, password_manager.url)
1572n/a self.assertEqual(user, password_manager.user)
1573n/a self.assertEqual(password, password_manager.password)
1574n/a
1575n/a opener.open(request_url)
1576n/a
1577n/a # should have asked the password manager for the username/password
1578n/a self.assertEqual(password_manager.target_realm, realm)
1579n/a self.assertEqual(password_manager.target_url, protected_url)
1580n/a
1581n/a # expect one request without authorization, then one with
1582n/a self.assertEqual(len(http_handler.requests), 2)
1583n/a self.assertFalse(http_handler.requests[0].has_header(auth_header))
1584n/a userpass = bytes('%s:%s' % (user, password), "ascii")
1585n/a auth_hdr_value = ('Basic ' +
1586n/a base64.encodebytes(userpass).strip().decode())
1587n/a self.assertEqual(http_handler.requests[1].get_header(auth_header),
1588n/a auth_hdr_value)
1589n/a self.assertEqual(http_handler.requests[1].unredirected_hdrs[auth_header],
1590n/a auth_hdr_value)
1591n/a # if the password manager can't find a password, the handler won't
1592n/a # handle the HTTP auth error
1593n/a password_manager.user = password_manager.password = None
1594n/a http_handler.reset()
1595n/a opener.open(request_url)
1596n/a self.assertEqual(len(http_handler.requests), 1)
1597n/a self.assertFalse(http_handler.requests[0].has_header(auth_header))
1598n/a
1599n/a def test_basic_prior_auth_auto_send(self):
1600n/a # Assume already authenticated if is_authenticated=True
1601n/a # for APIs like Github that don't return 401
1602n/a
1603n/a user, password = "wile", "coyote"
1604n/a request_url = "http://acme.example.com/protected"
1605n/a
1606n/a http_handler = MockHTTPHandlerCheckAuth(200)
1607n/a
1608n/a pwd_manager = HTTPPasswordMgrWithPriorAuth()
1609n/a auth_prior_handler = HTTPBasicAuthHandler(pwd_manager)
1610n/a auth_prior_handler.add_password(
1611n/a None, request_url, user, password, is_authenticated=True)
1612n/a
1613n/a is_auth = pwd_manager.is_authenticated(request_url)
1614n/a self.assertTrue(is_auth)
1615n/a
1616n/a opener = OpenerDirector()
1617n/a opener.add_handler(auth_prior_handler)
1618n/a opener.add_handler(http_handler)
1619n/a
1620n/a opener.open(request_url)
1621n/a
1622n/a # expect request to be sent with auth header
1623n/a self.assertTrue(http_handler.has_auth_header)
1624n/a
1625n/a def test_basic_prior_auth_send_after_first_success(self):
1626n/a # Auto send auth header after authentication is successful once
1627n/a
1628n/a user, password = 'wile', 'coyote'
1629n/a request_url = 'http://acme.example.com/protected'
1630n/a realm = 'ACME'
1631n/a
1632n/a pwd_manager = HTTPPasswordMgrWithPriorAuth()
1633n/a auth_prior_handler = HTTPBasicAuthHandler(pwd_manager)
1634n/a auth_prior_handler.add_password(realm, request_url, user, password)
1635n/a
1636n/a is_auth = pwd_manager.is_authenticated(request_url)
1637n/a self.assertFalse(is_auth)
1638n/a
1639n/a opener = OpenerDirector()
1640n/a opener.add_handler(auth_prior_handler)
1641n/a
1642n/a http_handler = MockHTTPHandler(
1643n/a 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % None)
1644n/a opener.add_handler(http_handler)
1645n/a
1646n/a opener.open(request_url)
1647n/a
1648n/a is_auth = pwd_manager.is_authenticated(request_url)
1649n/a self.assertTrue(is_auth)
1650n/a
1651n/a http_handler = MockHTTPHandlerCheckAuth(200)
1652n/a self.assertFalse(http_handler.has_auth_header)
1653n/a
1654n/a opener = OpenerDirector()
1655n/a opener.add_handler(auth_prior_handler)
1656n/a opener.add_handler(http_handler)
1657n/a
1658n/a # After getting 200 from MockHTTPHandler
1659n/a # Next request sends header in the first request
1660n/a opener.open(request_url)
1661n/a
1662n/a # expect request to be sent with auth header
1663n/a self.assertTrue(http_handler.has_auth_header)
1664n/a
1665n/a def test_http_closed(self):
1666n/a """Test the connection is cleaned up when the response is closed"""
1667n/a for (transfer, data) in (
1668n/a ("Connection: close", b"data"),
1669n/a ("Transfer-Encoding: chunked", b"4\r\ndata\r\n0\r\n\r\n"),
1670n/a ("Content-Length: 4", b"data"),
1671n/a ):
1672n/a header = "HTTP/1.1 200 OK\r\n{}\r\n\r\n".format(transfer)
1673n/a conn = test_urllib.fakehttp(header.encode() + data)
1674n/a handler = urllib.request.AbstractHTTPHandler()
1675n/a req = Request("http://dummy/")
1676n/a req.timeout = None
1677n/a with handler.do_open(conn, req) as resp:
1678n/a resp.read()
1679n/a self.assertTrue(conn.fakesock.closed,
1680n/a "Connection not closed with {!r}".format(transfer))
1681n/a
1682n/a def test_invalid_closed(self):
1683n/a """Test the connection is cleaned up after an invalid response"""
1684n/a conn = test_urllib.fakehttp(b"")
1685n/a handler = urllib.request.AbstractHTTPHandler()
1686n/a req = Request("http://dummy/")
1687n/a req.timeout = None
1688n/a with self.assertRaises(http.client.BadStatusLine):
1689n/a handler.do_open(conn, req)
1690n/a self.assertTrue(conn.fakesock.closed, "Connection not closed")
1691n/a
1692n/a
1693n/a
1694n/aclass MiscTests(unittest.TestCase):
1695n/a
1696n/a def opener_has_handler(self, opener, handler_class):
1697n/a self.assertTrue(any(h.__class__ == handler_class
1698n/a for h in opener.handlers))
1699n/a
1700n/a def test_build_opener(self):
1701n/a class MyHTTPHandler(urllib.request.HTTPHandler):
1702n/a pass
1703n/a
1704n/a class FooHandler(urllib.request.BaseHandler):
1705n/a def foo_open(self):
1706n/a pass
1707n/a
1708n/a class BarHandler(urllib.request.BaseHandler):
1709n/a def bar_open(self):
1710n/a pass
1711n/a
1712n/a build_opener = urllib.request.build_opener
1713n/a
1714n/a o = build_opener(FooHandler, BarHandler)
1715n/a self.opener_has_handler(o, FooHandler)
1716n/a self.opener_has_handler(o, BarHandler)
1717n/a
1718n/a # can take a mix of classes and instances
1719n/a o = build_opener(FooHandler, BarHandler())
1720n/a self.opener_has_handler(o, FooHandler)
1721n/a self.opener_has_handler(o, BarHandler)
1722n/a
1723n/a # subclasses of default handlers override default handlers
1724n/a o = build_opener(MyHTTPHandler)
1725n/a self.opener_has_handler(o, MyHTTPHandler)
1726n/a
1727n/a # a particular case of overriding: default handlers can be passed
1728n/a # in explicitly
1729n/a o = build_opener()
1730n/a self.opener_has_handler(o, urllib.request.HTTPHandler)
1731n/a o = build_opener(urllib.request.HTTPHandler)
1732n/a self.opener_has_handler(o, urllib.request.HTTPHandler)
1733n/a o = build_opener(urllib.request.HTTPHandler())
1734n/a self.opener_has_handler(o, urllib.request.HTTPHandler)
1735n/a
1736n/a # Issue2670: multiple handlers sharing the same base class
1737n/a class MyOtherHTTPHandler(urllib.request.HTTPHandler):
1738n/a pass
1739n/a
1740n/a o = build_opener(MyHTTPHandler, MyOtherHTTPHandler)
1741n/a self.opener_has_handler(o, MyHTTPHandler)
1742n/a self.opener_has_handler(o, MyOtherHTTPHandler)
1743n/a
1744n/a @unittest.skipUnless(support.is_resource_enabled('network'),
1745n/a 'test requires network access')
1746n/a def test_issue16464(self):
1747n/a with support.transient_internet("http://www.example.com/"):
1748n/a opener = urllib.request.build_opener()
1749n/a request = urllib.request.Request("http://www.example.com/")
1750n/a self.assertEqual(None, request.data)
1751n/a
1752n/a opener.open(request, "1".encode("us-ascii"))
1753n/a self.assertEqual(b"1", request.data)
1754n/a self.assertEqual("1", request.get_header("Content-length"))
1755n/a
1756n/a opener.open(request, "1234567890".encode("us-ascii"))
1757n/a self.assertEqual(b"1234567890", request.data)
1758n/a self.assertEqual("10", request.get_header("Content-length"))
1759n/a
1760n/a def test_HTTPError_interface(self):
1761n/a """
1762n/a Issue 13211 reveals that HTTPError didn't implement the URLError
1763n/a interface even though HTTPError is a subclass of URLError.
1764n/a """
1765n/a msg = 'something bad happened'
1766n/a url = code = fp = None
1767n/a hdrs = 'Content-Length: 42'
1768n/a err = urllib.error.HTTPError(url, code, msg, hdrs, fp)
1769n/a self.assertTrue(hasattr(err, 'reason'))
1770n/a self.assertEqual(err.reason, 'something bad happened')
1771n/a self.assertTrue(hasattr(err, 'headers'))
1772n/a self.assertEqual(err.headers, 'Content-Length: 42')
1773n/a expected_errmsg = 'HTTP Error %s: %s' % (err.code, err.msg)
1774n/a self.assertEqual(str(err), expected_errmsg)
1775n/a expected_errmsg = '<HTTPError %s: %r>' % (err.code, err.msg)
1776n/a self.assertEqual(repr(err), expected_errmsg)
1777n/a
1778n/a def test_parse_proxy(self):
1779n/a parse_proxy_test_cases = [
1780n/a ('proxy.example.com',
1781n/a (None, None, None, 'proxy.example.com')),
1782n/a ('proxy.example.com:3128',
1783n/a (None, None, None, 'proxy.example.com:3128')),
1784n/a ('proxy.example.com', (None, None, None, 'proxy.example.com')),
1785n/a ('proxy.example.com:3128',
1786n/a (None, None, None, 'proxy.example.com:3128')),
1787n/a # The authority component may optionally include userinfo
1788n/a # (assumed to be # username:password):
1789n/a ('joe:password@proxy.example.com',
1790n/a (None, 'joe', 'password', 'proxy.example.com')),
1791n/a ('joe:password@proxy.example.com:3128',
1792n/a (None, 'joe', 'password', 'proxy.example.com:3128')),
1793n/a #Examples with URLS
1794n/a ('http://proxy.example.com/',
1795n/a ('http', None, None, 'proxy.example.com')),
1796n/a ('http://proxy.example.com:3128/',
1797n/a ('http', None, None, 'proxy.example.com:3128')),
1798n/a ('http://joe:password@proxy.example.com/',
1799n/a ('http', 'joe', 'password', 'proxy.example.com')),
1800n/a ('http://joe:password@proxy.example.com:3128',
1801n/a ('http', 'joe', 'password', 'proxy.example.com:3128')),
1802n/a # Everything after the authority is ignored
1803n/a ('ftp://joe:password@proxy.example.com/rubbish:3128',
1804n/a ('ftp', 'joe', 'password', 'proxy.example.com')),
1805n/a # Test for no trailing '/' case
1806n/a ('http://joe:password@proxy.example.com',
1807n/a ('http', 'joe', 'password', 'proxy.example.com'))
1808n/a ]
1809n/a
1810n/a for tc, expected in parse_proxy_test_cases:
1811n/a self.assertEqual(_parse_proxy(tc), expected)
1812n/a
1813n/a self.assertRaises(ValueError, _parse_proxy, 'file:/ftp.example.com'),
1814n/a
1815n/a def test_unsupported_algorithm(self):
1816n/a handler = AbstractDigestAuthHandler()
1817n/a with self.assertRaises(ValueError) as exc:
1818n/a handler.get_algorithm_impls('invalid')
1819n/a self.assertEqual(
1820n/a str(exc.exception),
1821n/a "Unsupported digest authentication algorithm 'invalid'"
1822n/a )
1823n/a
1824n/a
1825n/aclass RequestTests(unittest.TestCase):
1826n/a class PutRequest(Request):
1827n/a method = 'PUT'
1828n/a
1829n/a def setUp(self):
1830n/a self.get = Request("http://www.python.org/~jeremy/")
1831n/a self.post = Request("http://www.python.org/~jeremy/",
1832n/a "data",
1833n/a headers={"X-Test": "test"})
1834n/a self.head = Request("http://www.python.org/~jeremy/", method='HEAD')
1835n/a self.put = self.PutRequest("http://www.python.org/~jeremy/")
1836n/a self.force_post = self.PutRequest("http://www.python.org/~jeremy/",
1837n/a method="POST")
1838n/a
1839n/a def test_method(self):
1840n/a self.assertEqual("POST", self.post.get_method())
1841n/a self.assertEqual("GET", self.get.get_method())
1842n/a self.assertEqual("HEAD", self.head.get_method())
1843n/a self.assertEqual("PUT", self.put.get_method())
1844n/a self.assertEqual("POST", self.force_post.get_method())
1845n/a
1846n/a def test_data(self):
1847n/a self.assertFalse(self.get.data)
1848n/a self.assertEqual("GET", self.get.get_method())
1849n/a self.get.data = "spam"
1850n/a self.assertTrue(self.get.data)
1851n/a self.assertEqual("POST", self.get.get_method())
1852n/a
1853n/a # issue 16464
1854n/a # if we change data we need to remove content-length header
1855n/a # (cause it's most probably calculated for previous value)
1856n/a def test_setting_data_should_remove_content_length(self):
1857n/a self.assertNotIn("Content-length", self.get.unredirected_hdrs)
1858n/a self.get.add_unredirected_header("Content-length", 42)
1859n/a self.assertEqual(42, self.get.unredirected_hdrs["Content-length"])
1860n/a self.get.data = "spam"
1861n/a self.assertNotIn("Content-length", self.get.unredirected_hdrs)
1862n/a
1863n/a # issue 17485 same for deleting data.
1864n/a def test_deleting_data_should_remove_content_length(self):
1865n/a self.assertNotIn("Content-length", self.get.unredirected_hdrs)
1866n/a self.get.data = 'foo'
1867n/a self.get.add_unredirected_header("Content-length", 3)
1868n/a self.assertEqual(3, self.get.unredirected_hdrs["Content-length"])
1869n/a del self.get.data
1870n/a self.assertNotIn("Content-length", self.get.unredirected_hdrs)
1871n/a
1872n/a def test_get_full_url(self):
1873n/a self.assertEqual("http://www.python.org/~jeremy/",
1874n/a self.get.get_full_url())
1875n/a
1876n/a def test_selector(self):
1877n/a self.assertEqual("/~jeremy/", self.get.selector)
1878n/a req = Request("http://www.python.org/")
1879n/a self.assertEqual("/", req.selector)
1880n/a
1881n/a def test_get_type(self):
1882n/a self.assertEqual("http", self.get.type)
1883n/a
1884n/a def test_get_host(self):
1885n/a self.assertEqual("www.python.org", self.get.host)
1886n/a
1887n/a def test_get_host_unquote(self):
1888n/a req = Request("http://www.%70ython.org/")
1889n/a self.assertEqual("www.python.org", req.host)
1890n/a
1891n/a def test_proxy(self):
1892n/a self.assertFalse(self.get.has_proxy())
1893n/a self.get.set_proxy("www.perl.org", "http")
1894n/a self.assertTrue(self.get.has_proxy())
1895n/a self.assertEqual("www.python.org", self.get.origin_req_host)
1896n/a self.assertEqual("www.perl.org", self.get.host)
1897n/a
1898n/a def test_wrapped_url(self):
1899n/a req = Request("<URL:http://www.python.org>")
1900n/a self.assertEqual("www.python.org", req.host)
1901n/a
1902n/a def test_url_fragment(self):
1903n/a req = Request("http://www.python.org/?qs=query#fragment=true")
1904n/a self.assertEqual("/?qs=query", req.selector)
1905n/a req = Request("http://www.python.org/#fun=true")
1906n/a self.assertEqual("/", req.selector)
1907n/a
1908n/a # Issue 11703: geturl() omits fragment in the original URL.
1909n/a url = 'http://docs.python.org/library/urllib2.html#OK'
1910n/a req = Request(url)
1911n/a self.assertEqual(req.get_full_url(), url)
1912n/a
1913n/a def test_url_fullurl_get_full_url(self):
1914n/a urls = ['http://docs.python.org',
1915n/a 'http://docs.python.org/library/urllib2.html#OK',
1916n/a 'http://www.python.org/?qs=query#fragment=true']
1917n/a for url in urls:
1918n/a req = Request(url)
1919n/a self.assertEqual(req.get_full_url(), req.full_url)
1920n/a
1921n/a
1922n/aif __name__ == "__main__":
1923n/a unittest.main()