Python code coverage for Lib/http/cookiejar.py

#	count	content
1	n/a	r"""HTTP cookie handling for web clients.
2	n/a
3	n/a	This module has (now fairly distant) origins in Gisle Aas' Perl module
4	n/a	HTTP::Cookies, from the libwww-perl library.
5	n/a
6	n/a	Docstrings, comments and debug strings in this code refer to the
7	n/a	attributes of the HTTP cookie system as cookie-attributes, to distinguish
8	n/a	them clearly from Python attributes.
9	n/a
10	n/a	Class diagram (note that BSDDBCookieJar and the MSIE* classes are not
11	n/a	distributed with the Python standard library, but are available from
12	n/a	http://wwwsearch.sf.net/):
13	n/a
14	n/a	CookieJar____
15	n/a	/ \ \
16	n/a	FileCookieJar \ \
17	n/a	/ \| \ \ \
18	n/a	MozillaCookieJar \| LWPCookieJar \ \
19	n/a	\| \| \
20	n/a	\| ---MSIEBase \| \
21	n/a	\| / \| \| \
22	n/a	\| / MSIEDBCookieJar BSDDBCookieJar
23	n/a	\|/
24	n/a	MSIECookieJar
25	n/a
26	n/a	"""
27	n/a
28	n/a	__all__ = ['Cookie', 'CookieJar', 'CookiePolicy', 'DefaultCookiePolicy',
29	n/a	'FileCookieJar', 'LWPCookieJar', 'LoadError', 'MozillaCookieJar']
30	n/a
31	n/a	import copy
32	n/a	import datetime
33	n/a	import re
34	n/a	import time
35	n/a	import urllib.parse, urllib.request
36	n/a	try:
37	n/a	import threading as _threading
38	n/a	except ImportError:
39	n/a	import dummy_threading as _threading
40	n/a	import http.client # only for the default HTTP port
41	n/a	from calendar import timegm
42	n/a
43	n/a	debug = False # set to True to enable debugging via the logging module
44	n/a	logger = None
45	n/a
46	n/a	def _debug(*args):
47	n/a	if not debug:
48	n/a	return
49	n/a	global logger
50	n/a	if not logger:
51	n/a	import logging
52	n/a	logger = logging.getLogger("http.cookiejar")
53	n/a	return logger.debug(*args)
54	n/a
55	n/a
56	n/a	DEFAULT_HTTP_PORT = str(http.client.HTTP_PORT)
57	n/a	MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar "
58	n/a	"instance initialised with one)")
59	n/a
60	n/a	def _warn_unhandled_exception():
61	n/a	# There are a few catch-all except: statements in this module, for
62	n/a	# catching input that's bad in unexpected ways. Warn if any
63	n/a	# exceptions are caught there.
64	n/a	import io, warnings, traceback
65	n/a	f = io.StringIO()
66	n/a	traceback.print_exc(None, f)
67	n/a	msg = f.getvalue()
68	n/a	warnings.warn("http.cookiejar bug!\n%s" % msg, stacklevel=2)
69	n/a
70	n/a
71	n/a	# Date/time conversion
72	n/a	# -----------------------------------------------------------------------------
73	n/a
74	n/a	EPOCH_YEAR = 1970
75	n/a	def _timegm(tt):
76	n/a	year, month, mday, hour, min, sec = tt[:6]
77	n/a	if ((year >= EPOCH_YEAR) and (1 <= month <= 12) and (1 <= mday <= 31) and
78	n/a	(0 <= hour <= 24) and (0 <= min <= 59) and (0 <= sec <= 61)):
79	n/a	return timegm(tt)
80	n/a	else:
81	n/a	return None
82	n/a
83	n/a	DAYS = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
84	n/a	MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
85	n/a	"Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
86	n/a	MONTHS_LOWER = []
87	n/a	for month in MONTHS: MONTHS_LOWER.append(month.lower())
88	n/a
89	n/a	def time2isoz(t=None):
90	n/a	"""Return a string representing time in seconds since epoch, t.
91	n/a
92	n/a	If the function is called without an argument, it will use the current
93	n/a	time.
94	n/a
95	n/a	The format of the returned string is like "YYYY-MM-DD hh:mm:ssZ",
96	n/a	representing Universal Time (UTC, aka GMT). An example of this format is:
97	n/a
98	n/a	1994-11-24 08:49:37Z
99	n/a
100	n/a	"""
101	n/a	if t is None:
102	n/a	dt = datetime.datetime.utcnow()
103	n/a	else:
104	n/a	dt = datetime.datetime.utcfromtimestamp(t)
105	n/a	return "%04d-%02d-%02d %02d:%02d:%02dZ" % (
106	n/a	dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
107	n/a
108	n/a	def time2netscape(t=None):
109	n/a	"""Return a string representing time in seconds since epoch, t.
110	n/a
111	n/a	If the function is called without an argument, it will use the current
112	n/a	time.
113	n/a
114	n/a	The format of the returned string is like this:
115	n/a
116	n/a	Wed, DD-Mon-YYYY HH:MM:SS GMT
117	n/a
118	n/a	"""
119	n/a	if t is None:
120	n/a	dt = datetime.datetime.utcnow()
121	n/a	else:
122	n/a	dt = datetime.datetime.utcfromtimestamp(t)
123	n/a	return "%s, %02d-%s-%04d %02d:%02d:%02d GMT" % (
124	n/a	DAYS[dt.weekday()], dt.day, MONTHS[dt.month-1],
125	n/a	dt.year, dt.hour, dt.minute, dt.second)
126	n/a
127	n/a
128	n/a	UTC_ZONES = {"GMT": None, "UTC": None, "UT": None, "Z": None}
129	n/a
130	n/a	TIMEZONE_RE = re.compile(r"^([-+])?(\d\d?):?(\d\d)?$", re.ASCII)
131	n/a	def offset_from_tz_string(tz):
132	n/a	offset = None
133	n/a	if tz in UTC_ZONES:
134	n/a	offset = 0
135	n/a	else:
136	n/a	m = TIMEZONE_RE.search(tz)
137	n/a	if m:
138	n/a	offset = 3600 * int(m.group(2))
139	n/a	if m.group(3):
140	n/a	offset = offset + 60 * int(m.group(3))
141	n/a	if m.group(1) == '-':
142	n/a	offset = -offset
143	n/a	return offset
144	n/a
145	n/a	def _str2time(day, mon, yr, hr, min, sec, tz):
146	n/a	yr = int(yr)
147	n/a	if yr > datetime.MAXYEAR:
148	n/a	return None
149	n/a
150	n/a	# translate month name to number
151	n/a	# month numbers start with 1 (January)
152	n/a	try:
153	n/a	mon = MONTHS_LOWER.index(mon.lower())+1
154	n/a	except ValueError:
155	n/a	# maybe it's already a number
156	n/a	try:
157	n/a	imon = int(mon)
158	n/a	except ValueError:
159	n/a	return None
160	n/a	if 1 <= imon <= 12:
161	n/a	mon = imon
162	n/a	else:
163	n/a	return None
164	n/a
165	n/a	# make sure clock elements are defined
166	n/a	if hr is None: hr = 0
167	n/a	if min is None: min = 0
168	n/a	if sec is None: sec = 0
169	n/a
170	n/a	day = int(day)
171	n/a	hr = int(hr)
172	n/a	min = int(min)
173	n/a	sec = int(sec)
174	n/a
175	n/a	if yr < 1000:
176	n/a	# find "obvious" year
177	n/a	cur_yr = time.localtime(time.time())[0]
178	n/a	m = cur_yr % 100
179	n/a	tmp = yr
180	n/a	yr = yr + cur_yr - m
181	n/a	m = m - tmp
182	n/a	if abs(m) > 50:
183	n/a	if m > 0: yr = yr + 100
184	n/a	else: yr = yr - 100
185	n/a
186	n/a	# convert UTC time tuple to seconds since epoch (not timezone-adjusted)
187	n/a	t = _timegm((yr, mon, day, hr, min, sec, tz))
188	n/a
189	n/a	if t is not None:
190	n/a	# adjust time using timezone string, to get absolute time since epoch
191	n/a	if tz is None:
192	n/a	tz = "UTC"
193	n/a	tz = tz.upper()
194	n/a	offset = offset_from_tz_string(tz)
195	n/a	if offset is None:
196	n/a	return None
197	n/a	t = t - offset
198	n/a
199	n/a	return t
200	n/a
201	n/a	STRICT_DATE_RE = re.compile(
202	n/a	r"^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) "
203	n/a	r"(\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$", re.ASCII)
204	n/a	WEEKDAY_RE = re.compile(
205	n/a	r"^(?:Sun\|Mon\|Tue\|Wed\|Thu\|Fri\|Sat)[a-z],?\s", re.I \| re.ASCII)
206	n/a	LOOSE_HTTP_DATE_RE = re.compile(
207	n/a	r"""^
208	n/a	(\d\d?) # day
209	n/a	(?:\s+\|[-\/])
210	n/a	(\w+) # month
211	n/a	(?:\s+\|[-\/])
212	n/a	(\d+) # year
213	n/a	(?:
214	n/a	(?:\s+\|:) # separator before clock
215	n/a	(\d\d?):(\d\d) # hour:min
216	n/a	(?::(\d\d))? # optional seconds
217	n/a	)? # optional clock
218	n/a	\s*
219	n/a	([-+]?\d{2,4}\|(?![APap][Mm]\b)[A-Za-z]+)? # timezone
220	n/a	\s*
221	n/a	(?:$\w+$)? # ASCII representation of timezone in parens.
222	n/a	\s*$""", re.X \| re.ASCII)
223	n/a	def http2time(text):
224	n/a	"""Returns time in seconds since epoch of time represented by a string.
225	n/a
226	n/a	Return value is an integer.
227	n/a
228	n/a	None is returned if the format of str is unrecognized, the time is outside
229	n/a	the representable range, or the timezone string is not recognized. If the
230	n/a	string contains no timezone, UTC is assumed.
231	n/a
232	n/a	The timezone in the string may be numerical (like "-0800" or "+0100") or a
233	n/a	string timezone (like "UTC", "GMT", "BST" or "EST"). Currently, only the
234	n/a	timezone strings equivalent to UTC (zero offset) are known to the function.
235	n/a
236	n/a	The function loosely parses the following formats:
237	n/a
238	n/a	Wed, 09 Feb 1994 22:23:32 GMT -- HTTP format
239	n/a	Tuesday, 08-Feb-94 14:15:29 GMT -- old rfc850 HTTP format
240	n/a	Tuesday, 08-Feb-1994 14:15:29 GMT -- broken rfc850 HTTP format
241	n/a	09 Feb 1994 22:23:32 GMT -- HTTP format (no weekday)
242	n/a	08-Feb-94 14:15:29 GMT -- rfc850 format (no weekday)
243	n/a	08-Feb-1994 14:15:29 GMT -- broken rfc850 format (no weekday)
244	n/a
245	n/a	The parser ignores leading and trailing whitespace. The time may be
246	n/a	absent.
247	n/a
248	n/a	If the year is given with only 2 digits, the function will select the
249	n/a	century that makes the year closest to the current date.
250	n/a
251	n/a	"""
252	n/a	# fast exit for strictly conforming string
253	n/a	m = STRICT_DATE_RE.search(text)
254	n/a	if m:
255	n/a	g = m.groups()
256	n/a	mon = MONTHS_LOWER.index(g[1].lower()) + 1
257	n/a	tt = (int(g[2]), mon, int(g[0]),
258	n/a	int(g[3]), int(g[4]), float(g[5]))
259	n/a	return _timegm(tt)
260	n/a
261	n/a	# No, we need some messy parsing...
262	n/a
263	n/a	# clean up
264	n/a	text = text.lstrip()
265	n/a	text = WEEKDAY_RE.sub("", text, 1) # Useless weekday
266	n/a
267	n/a	# tz is time zone specifier string
268	n/a	day, mon, yr, hr, min, sec, tz = [None]*7
269	n/a
270	n/a	# loose regexp parse
271	n/a	m = LOOSE_HTTP_DATE_RE.search(text)
272	n/a	if m is not None:
273	n/a	day, mon, yr, hr, min, sec, tz = m.groups()
274	n/a	else:
275	n/a	return None # bad format
276	n/a
277	n/a	return _str2time(day, mon, yr, hr, min, sec, tz)
278	n/a
279	n/a	ISO_DATE_RE = re.compile(
280	n/a	r"""^
281	n/a	(\d{4}) # year
282	n/a	[-\/]?
283	n/a	(\d\d?) # numerical month
284	n/a	[-\/]?
285	n/a	(\d\d?) # day
286	n/a	(?:
287	n/a	(?:\s+\|[-:Tt]) # separator before clock
288	n/a	(\d\d?):?(\d\d) # hour:min
289	n/a	(?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional)
290	n/a	)? # optional clock
291	n/a	\s*
292	n/a	([-+]?\d\d?:?(:?\d\d)?
293	n/a	\|Z\|z)? # timezone (Z is "zero meridian", i.e. GMT)
294	n/a	\s*$""", re.X \| re. ASCII)
295	n/a	def iso2time(text):
296	n/a	"""
297	n/a	As for http2time, but parses the ISO 8601 formats:
298	n/a
299	n/a	1994-02-03 14:15:29 -0100 -- ISO 8601 format
300	n/a	1994-02-03 14:15:29 -- zone is optional
301	n/a	1994-02-03 -- only date
302	n/a	1994-02-03T14:15:29 -- Use T as separator
303	n/a	19940203T141529Z -- ISO 8601 compact format
304	n/a	19940203 -- only date
305	n/a
306	n/a	"""
307	n/a	# clean up
308	n/a	text = text.lstrip()
309	n/a
310	n/a	# tz is time zone specifier string
311	n/a	day, mon, yr, hr, min, sec, tz = [None]*7
312	n/a
313	n/a	# loose regexp parse
314	n/a	m = ISO_DATE_RE.search(text)
315	n/a	if m is not None:
316	n/a	# XXX there's an extra bit of the timezone I'm ignoring here: is
317	n/a	# this the right thing to do?
318	n/a	yr, mon, day, hr, min, sec, tz, _ = m.groups()
319	n/a	else:
320	n/a	return None # bad format
321	n/a
322	n/a	return _str2time(day, mon, yr, hr, min, sec, tz)
323	n/a
324	n/a
325	n/a	# Header parsing
326	n/a	# -----------------------------------------------------------------------------
327	n/a
328	n/a	def unmatched(match):
329	n/a	"""Return unmatched part of re.Match object."""
330	n/a	start, end = match.span(0)
331	n/a	return match.string[:start]+match.string[end:]
332	n/a
333	n/a	HEADER_TOKEN_RE = re.compile(r"^\s*([^=\s;,]+)")
334	n/a	HEADER_QUOTED_VALUE_RE = re.compile(r"^\s=\s\"([^\"\\](?:\\.[^\"\\])*)\"")
335	n/a	HEADER_VALUE_RE = re.compile(r"^\s=\s([^\s;,]*)")
336	n/a	HEADER_ESCAPE_RE = re.compile(r"\\(.)")
337	n/a	def split_header_words(header_values):
338	n/a	r"""Parse header values into a list of lists containing key,value pairs.
339	n/a
340	n/a	The function knows how to deal with ",", ";" and "=" as well as quoted
341	n/a	values after "=". A list of space separated tokens are parsed as if they
342	n/a	were separated by ";".
343	n/a
344	n/a	If the header_values passed as argument contains multiple values, then they
345	n/a	are treated as if they were a single value separated by comma ",".
346	n/a
347	n/a	This means that this function is useful for parsing header fields that
348	n/a	follow this syntax (BNF as from the HTTP/1.1 specification, but we relax
349	n/a	the requirement for tokens).
350	n/a
351	n/a	headers = #header
352	n/a	header = (token \| parameter) *( [";"] (token \| parameter))
353	n/a
354	n/a	token = 1*<any CHAR except CTLs or separators>
355	n/a	separators = "(" \| ")" \| "<" \| ">" \| "@"
356	n/a	\| "," \| ";" \| ":" \| "\" \| <">
357	n/a	\| "/" \| "[" \| "]" \| "?" \| "="
358	n/a	\| "{" \| "}" \| SP \| HT
359	n/a
360	n/a	quoted-string = ( <"> *(qdtext \| quoted-pair ) <"> )
361	n/a	qdtext = <any TEXT except <">>
362	n/a	quoted-pair = "\" CHAR
363	n/a
364	n/a	parameter = attribute "=" value
365	n/a	attribute = token
366	n/a	value = token \| quoted-string
367	n/a
368	n/a	Each header is represented by a list of key/value pairs. The value for a
369	n/a	simple token (not part of a parameter) is None. Syntactically incorrect
370	n/a	headers will not necessarily be parsed as you would want.
371	n/a
372	n/a	This is easier to describe with some examples:
373	n/a
374	n/a	>>> split_header_words(['foo="bar"; port="80,81"; discard, bar=baz'])
375	n/a	[[('foo', 'bar'), ('port', '80,81'), ('discard', None)], [('bar', 'baz')]]
376	n/a	>>> split_header_words(['text/html; charset="iso-8859-1"'])
377	n/a	[[('text/html', None), ('charset', 'iso-8859-1')]]
378	n/a	>>> split_header_words([r'Basic realm="\"foo\bar\""'])
379	n/a	[[('Basic', None), ('realm', '"foobar"')]]
380	n/a
381	n/a	"""
382	n/a	assert not isinstance(header_values, str)
383	n/a	result = []
384	n/a	for text in header_values:
385	n/a	orig_text = text
386	n/a	pairs = []
387	n/a	while text:
388	n/a	m = HEADER_TOKEN_RE.search(text)
389	n/a	if m:
390	n/a	text = unmatched(m)
391	n/a	name = m.group(1)
392	n/a	m = HEADER_QUOTED_VALUE_RE.search(text)
393	n/a	if m: # quoted value
394	n/a	text = unmatched(m)
395	n/a	value = m.group(1)
396	n/a	value = HEADER_ESCAPE_RE.sub(r"\1", value)
397	n/a	else:
398	n/a	m = HEADER_VALUE_RE.search(text)
399	n/a	if m: # unquoted value
400	n/a	text = unmatched(m)
401	n/a	value = m.group(1)
402	n/a	value = value.rstrip()
403	n/a	else:
404	n/a	# no value, a lone token
405	n/a	value = None
406	n/a	pairs.append((name, value))
407	n/a	elif text.lstrip().startswith(","):
408	n/a	# concatenated headers, as per RFC 2616 section 4.2
409	n/a	text = text.lstrip()[1:]
410	n/a	if pairs: result.append(pairs)
411	n/a	pairs = []
412	n/a	else:
413	n/a	# skip junk
414	n/a	non_junk, nr_junk_chars = re.subn(r"^[=\s;]*", "", text)
415	n/a	assert nr_junk_chars > 0, (
416	n/a	"split_header_words bug: '%s', '%s', %s" %
417	n/a	(orig_text, text, pairs))
418	n/a	text = non_junk
419	n/a	if pairs: result.append(pairs)
420	n/a	return result
421	n/a
422	n/a	HEADER_JOIN_ESCAPE_RE = re.compile(r"([\"\\])")
423	n/a	def join_header_words(lists):
424	n/a	"""Do the inverse (almost) of the conversion done by split_header_words.
425	n/a
426	n/a	Takes a list of lists of (key, value) pairs and produces a single header
427	n/a	value. Attribute values are quoted if needed.
428	n/a
429	n/a	>>> join_header_words([[("text/plain", None), ("charset", "iso-8859-1")]])
430	n/a	'text/plain; charset="iso-8859-1"'
431	n/a	>>> join_header_words([[("text/plain", None)], [("charset", "iso-8859-1")]])
432	n/a	'text/plain, charset="iso-8859-1"'
433	n/a
434	n/a	"""
435	n/a	headers = []
436	n/a	for pairs in lists:
437	n/a	attr = []
438	n/a	for k, v in pairs:
439	n/a	if v is not None:
440	n/a	if not re.search(r"^\w+$", v):
441	n/a	v = HEADER_JOIN_ESCAPE_RE.sub(r"\\\1", v) # escape " and \
442	n/a	v = '"%s"' % v
443	n/a	k = "%s=%s" % (k, v)
444	n/a	attr.append(k)
445	n/a	if attr: headers.append("; ".join(attr))
446	n/a	return ", ".join(headers)
447	n/a
448	n/a	def strip_quotes(text):
449	n/a	if text.startswith('"'):
450	n/a	text = text[1:]
451	n/a	if text.endswith('"'):
452	n/a	text = text[:-1]
453	n/a	return text
454	n/a
455	n/a	def parse_ns_headers(ns_headers):
456	n/a	"""Ad-hoc parser for Netscape protocol cookie-attributes.
457	n/a
458	n/a	The old Netscape cookie format for Set-Cookie can for instance contain
459	n/a	an unquoted "," in the expires field, so we have to use this ad-hoc
460	n/a	parser instead of split_header_words.
461	n/a
462	n/a	XXX This may not make the best possible effort to parse all the crap
463	n/a	that Netscape Cookie headers contain. Ronald Tschalar's HTTPClient
464	n/a	parser is probably better, so could do worse than following that if
465	n/a	this ever gives any trouble.
466	n/a
467	n/a	Currently, this is also used for parsing RFC 2109 cookies.
468	n/a
469	n/a	"""
470	n/a	known_attrs = ("expires", "domain", "path", "secure",
471	n/a	# RFC 2109 attrs (may turn up in Netscape cookies, too)
472	n/a	"version", "port", "max-age")
473	n/a
474	n/a	result = []
475	n/a	for ns_header in ns_headers:
476	n/a	pairs = []
477	n/a	version_set = False
478	n/a
479	n/a	# XXX: The following does not strictly adhere to RFCs in that empty
480	n/a	# names and values are legal (the former will only appear once and will
481	n/a	# be overwritten if multiple occurrences are present). This is
482	n/a	# mostly to deal with backwards compatibility.
483	n/a	for ii, param in enumerate(ns_header.split(';')):
484	n/a	param = param.strip()
485	n/a
486	n/a	key, sep, val = param.partition('=')
487	n/a	key = key.strip()
488	n/a
489	n/a	if not key:
490	n/a	if ii == 0:
491	n/a	break
492	n/a	else:
493	n/a	continue
494	n/a
495	n/a	# allow for a distinction between present and empty and missing
496	n/a	# altogether
497	n/a	val = val.strip() if sep else None
498	n/a
499	n/a	if ii != 0:
500	n/a	lc = key.lower()
501	n/a	if lc in known_attrs:
502	n/a	key = lc
503	n/a
504	n/a	if key == "version":
505	n/a	# This is an RFC 2109 cookie.
506	n/a	if val is not None:
507	n/a	val = strip_quotes(val)
508	n/a	version_set = True
509	n/a	elif key == "expires":
510	n/a	# convert expires date to seconds since epoch
511	n/a	if val is not None:
512	n/a	val = http2time(strip_quotes(val)) # None if invalid
513	n/a	pairs.append((key, val))
514	n/a
515	n/a	if pairs:
516	n/a	if not version_set:
517	n/a	pairs.append(("version", "0"))
518	n/a	result.append(pairs)
519	n/a
520	n/a	return result
521	n/a
522	n/a
523	n/a	IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
524	n/a	def is_HDN(text):
525	n/a	"""Return True if text is a host domain name."""
526	n/a	# XXX
527	n/a	# This may well be wrong. Which RFC is HDN defined in, if any (for
528	n/a	# the purposes of RFC 2965)?
529	n/a	# For the current implementation, what about IPv6? Remember to look
530	n/a	# at other uses of IPV4_RE also, if change this.
531	n/a	if IPV4_RE.search(text):
532	n/a	return False
533	n/a	if text == "":
534	n/a	return False
535	n/a	if text[0] == "." or text[-1] == ".":
536	n/a	return False
537	n/a	return True
538	n/a
539	n/a	def domain_match(A, B):
540	n/a	"""Return True if domain A domain-matches domain B, according to RFC 2965.
541	n/a
542	n/a	A and B may be host domain names or IP addresses.
543	n/a
544	n/a	RFC 2965, section 1:
545	n/a
546	n/a	Host names can be specified either as an IP address or a HDN string.
547	n/a	Sometimes we compare one host name with another. (Such comparisons SHALL
548	n/a	be case-insensitive.) Host A's name domain-matches host B's if
549	n/a
550	n/a	* their host name strings string-compare equal; or
551	n/a
552	n/a	* A is a HDN string and has the form NB, where N is a non-empty
553	n/a	name string, B has the form .B', and B' is a HDN string. (So,
554	n/a	x.y.com domain-matches .Y.com but not Y.com.)
555	n/a
556	n/a	Note that domain-match is not a commutative operation: a.b.c.com
557	n/a	domain-matches .c.com, but not the reverse.
558	n/a
559	n/a	"""
560	n/a	# Note that, if A or B are IP addresses, the only relevant part of the
561	n/a	# definition of the domain-match algorithm is the direct string-compare.
562	n/a	A = A.lower()
563	n/a	B = B.lower()
564	n/a	if A == B:
565	n/a	return True
566	n/a	if not is_HDN(A):
567	n/a	return False
568	n/a	i = A.rfind(B)
569	n/a	if i == -1 or i == 0:
570	n/a	# A does not have form NB, or N is the empty string
571	n/a	return False
572	n/a	if not B.startswith("."):
573	n/a	return False
574	n/a	if not is_HDN(B[1:]):
575	n/a	return False
576	n/a	return True
577	n/a
578	n/a	def liberal_is_HDN(text):
579	n/a	"""Return True if text is a sort-of-like a host domain name.
580	n/a
581	n/a	For accepting/blocking domains.
582	n/a
583	n/a	"""
584	n/a	if IPV4_RE.search(text):
585	n/a	return False
586	n/a	return True
587	n/a
588	n/a	def user_domain_match(A, B):
589	n/a	"""For blocking/accepting domains.
590	n/a
591	n/a	A and B may be host domain names or IP addresses.
592	n/a
593	n/a	"""
594	n/a	A = A.lower()
595	n/a	B = B.lower()
596	n/a	if not (liberal_is_HDN(A) and liberal_is_HDN(B)):
597	n/a	if A == B:
598	n/a	# equal IP addresses
599	n/a	return True
600	n/a	return False
601	n/a	initial_dot = B.startswith(".")
602	n/a	if initial_dot and A.endswith(B):
603	n/a	return True
604	n/a	if not initial_dot and A == B:
605	n/a	return True
606	n/a	return False
607	n/a
608	n/a	cut_port_re = re.compile(r":\d+$", re.ASCII)
609	n/a	def request_host(request):
610	n/a	"""Return request-host, as defined by RFC 2965.
611	n/a
612	n/a	Variation from RFC: returned value is lowercased, for convenient
613	n/a	comparison.
614	n/a
615	n/a	"""
616	n/a	url = request.get_full_url()
617	n/a	host = urllib.parse.urlparse(url)[1]
618	n/a	if host == "":
619	n/a	host = request.get_header("Host", "")
620	n/a
621	n/a	# remove port, if present
622	n/a	host = cut_port_re.sub("", host, 1)
623	n/a	return host.lower()
624	n/a
625	n/a	def eff_request_host(request):
626	n/a	"""Return a tuple (request-host, effective request-host name).
627	n/a
628	n/a	As defined by RFC 2965, except both are lowercased.
629	n/a
630	n/a	"""
631	n/a	erhn = req_host = request_host(request)
632	n/a	if req_host.find(".") == -1 and not IPV4_RE.search(req_host):
633	n/a	erhn = req_host + ".local"
634	n/a	return req_host, erhn
635	n/a
636	n/a	def request_path(request):
637	n/a	"""Path component of request-URI, as defined by RFC 2965."""
638	n/a	url = request.get_full_url()
639	n/a	parts = urllib.parse.urlsplit(url)
640	n/a	path = escape_path(parts.path)
641	n/a	if not path.startswith("/"):
642	n/a	# fix bad RFC 2396 absoluteURI
643	n/a	path = "/" + path
644	n/a	return path
645	n/a
646	n/a	def request_port(request):
647	n/a	host = request.host
648	n/a	i = host.find(':')
649	n/a	if i >= 0:
650	n/a	port = host[i+1:]
651	n/a	try:
652	n/a	int(port)
653	n/a	except ValueError:
654	n/a	_debug("nonnumeric port: '%s'", port)
655	n/a	return None
656	n/a	else:
657	n/a	port = DEFAULT_HTTP_PORT
658	n/a	return port
659	n/a
660	n/a	# Characters in addition to A-Z, a-z, 0-9, '_', '.', and '-' that don't
661	n/a	# need to be escaped to form a valid HTTP URL (RFCs 2396 and 1738).
662	n/a	HTTP_PATH_SAFE = "%/;:@&=+$,!~*'()"
663	n/a	ESCAPED_CHAR_RE = re.compile(r"%([0-9a-fA-F][0-9a-fA-F])")
664	n/a	def uppercase_escaped_char(match):
665	n/a	return "%%%s" % match.group(1).upper()
666	n/a	def escape_path(path):
667	n/a	"""Escape any invalid characters in HTTP URL, and uppercase all escapes."""
668	n/a	# There's no knowing what character encoding was used to create URLs
669	n/a	# containing %-escapes, but since we have to pick one to escape invalid
670	n/a	# path characters, we pick UTF-8, as recommended in the HTML 4.0
671	n/a	# specification:
672	n/a	# http://www.w3.org/TR/REC-html40/appendix/notes.html#h-B.2.1
673	n/a	# And here, kind of: draft-fielding-uri-rfc2396bis-03
674	n/a	# (And in draft IRI specification: draft-duerst-iri-05)
675	n/a	# (And here, for new URI schemes: RFC 2718)
676	n/a	path = urllib.parse.quote(path, HTTP_PATH_SAFE)
677	n/a	path = ESCAPED_CHAR_RE.sub(uppercase_escaped_char, path)
678	n/a	return path
679	n/a
680	n/a	def reach(h):
681	n/a	"""Return reach of host h, as defined by RFC 2965, section 1.
682	n/a
683	n/a	The reach R of a host name H is defined as follows:
684	n/a
685	n/a	* If
686	n/a
687	n/a	- H is the host domain name of a host; and,
688	n/a
689	n/a	- H has the form A.B; and
690	n/a
691	n/a	- A has no embedded (that is, interior) dots; and
692	n/a
693	n/a	- B has at least one embedded dot, or B is the string "local".
694	n/a	then the reach of H is .B.
695	n/a
696	n/a	* Otherwise, the reach of H is H.
697	n/a
698	n/a	>>> reach("www.acme.com")
699	n/a	'.acme.com'
700	n/a	>>> reach("acme.com")
701	n/a	'acme.com'
702	n/a	>>> reach("acme.local")
703	n/a	'.local'
704	n/a
705	n/a	"""
706	n/a	i = h.find(".")
707	n/a	if i >= 0:
708	n/a	#a = h[:i] # this line is only here to show what a is
709	n/a	b = h[i+1:]
710	n/a	i = b.find(".")
711	n/a	if is_HDN(h) and (i >= 0 or b == "local"):
712	n/a	return "."+b
713	n/a	return h
714	n/a
715	n/a	def is_third_party(request):
716	n/a	"""
717	n/a
718	n/a	RFC 2965, section 3.3.6:
719	n/a
720	n/a	An unverifiable transaction is to a third-party host if its request-
721	n/a	host U does not domain-match the reach R of the request-host O in the
722	n/a	origin transaction.
723	n/a
724	n/a	"""
725	n/a	req_host = request_host(request)
726	n/a	if not domain_match(req_host, reach(request.origin_req_host)):
727	n/a	return True
728	n/a	else:
729	n/a	return False
730	n/a
731	n/a
732	n/a	class Cookie:
733	n/a	"""HTTP Cookie.
734	n/a
735	n/a	This class represents both Netscape and RFC 2965 cookies.
736	n/a
737	n/a	This is deliberately a very simple class. It just holds attributes. It's
738	n/a	possible to construct Cookie instances that don't comply with the cookie
739	n/a	standards. CookieJar.make_cookies is the factory function for Cookie
740	n/a	objects -- it deals with cookie parsing, supplying defaults, and
741	n/a	normalising to the representation used in this class. CookiePolicy is
742	n/a	responsible for checking them to see whether they should be accepted from
743	n/a	and returned to the server.
744	n/a
745	n/a	Note that the port may be present in the headers, but unspecified ("Port"
746	n/a	rather than"Port=80", for example); if this is the case, port is None.
747	n/a
748	n/a	"""
749	n/a
750	n/a	def __init__(self, version, name, value,
751	n/a	port, port_specified,
752	n/a	domain, domain_specified, domain_initial_dot,
753	n/a	path, path_specified,
754	n/a	secure,
755	n/a	expires,
756	n/a	discard,
757	n/a	comment,
758	n/a	comment_url,
759	n/a	rest,
760	n/a	rfc2109=False,
761	n/a	):
762	n/a
763	n/a	if version is not None: version = int(version)
764	n/a	if expires is not None: expires = int(float(expires))
765	n/a	if port is None and port_specified is True:
766	n/a	raise ValueError("if port is None, port_specified must be false")
767	n/a
768	n/a	self.version = version
769	n/a	self.name = name
770	n/a	self.value = value
771	n/a	self.port = port
772	n/a	self.port_specified = port_specified
773	n/a	# normalise case, as per RFC 2965 section 3.3.3
774	n/a	self.domain = domain.lower()
775	n/a	self.domain_specified = domain_specified
776	n/a	# Sigh. We need to know whether the domain given in the
777	n/a	# cookie-attribute had an initial dot, in order to follow RFC 2965
778	n/a	# (as clarified in draft errata). Needed for the returned $Domain
779	n/a	# value.
780	n/a	self.domain_initial_dot = domain_initial_dot
781	n/a	self.path = path
782	n/a	self.path_specified = path_specified
783	n/a	self.secure = secure
784	n/a	self.expires = expires
785	n/a	self.discard = discard
786	n/a	self.comment = comment
787	n/a	self.comment_url = comment_url
788	n/a	self.rfc2109 = rfc2109
789	n/a
790	n/a	self._rest = copy.copy(rest)
791	n/a
792	n/a	def has_nonstandard_attr(self, name):
793	n/a	return name in self._rest
794	n/a	def get_nonstandard_attr(self, name, default=None):
795	n/a	return self._rest.get(name, default)
796	n/a	def set_nonstandard_attr(self, name, value):
797	n/a	self._rest[name] = value
798	n/a
799	n/a	def is_expired(self, now=None):
800	n/a	if now is None: now = time.time()
801	n/a	if (self.expires is not None) and (self.expires <= now):
802	n/a	return True
803	n/a	return False
804	n/a
805	n/a	def __str__(self):
806	n/a	if self.port is None: p = ""
807	n/a	else: p = ":"+self.port
808	n/a	limit = self.domain + p + self.path
809	n/a	if self.value is not None:
810	n/a	namevalue = "%s=%s" % (self.name, self.value)
811	n/a	else:
812	n/a	namevalue = self.name
813	n/a	return "<Cookie %s for %s>" % (namevalue, limit)
814	n/a
815	n/a	def __repr__(self):
816	n/a	args = []
817	n/a	for name in ("version", "name", "value",
818	n/a	"port", "port_specified",
819	n/a	"domain", "domain_specified", "domain_initial_dot",
820	n/a	"path", "path_specified",
821	n/a	"secure", "expires", "discard", "comment", "comment_url",
822	n/a	):
823	n/a	attr = getattr(self, name)
824	n/a	args.append("%s=%s" % (name, repr(attr)))
825	n/a	args.append("rest=%s" % repr(self._rest))
826	n/a	args.append("rfc2109=%s" % repr(self.rfc2109))
827	n/a	return "%s(%s)" % (self.__class__.__name__, ", ".join(args))
828	n/a
829	n/a
830	n/a	class CookiePolicy:
831	n/a	"""Defines which cookies get accepted from and returned to server.
832	n/a
833	n/a	May also modify cookies, though this is probably a bad idea.
834	n/a
835	n/a	The subclass DefaultCookiePolicy defines the standard rules for Netscape
836	n/a	and RFC 2965 cookies -- override that if you want a customized policy.
837	n/a
838	n/a	"""
839	n/a	def set_ok(self, cookie, request):
840	n/a	"""Return true if (and only if) cookie should be accepted from server.
841	n/a
842	n/a	Currently, pre-expired cookies never get this far -- the CookieJar
843	n/a	class deletes such cookies itself.
844	n/a
845	n/a	"""
846	n/a	raise NotImplementedError()
847	n/a
848	n/a	def return_ok(self, cookie, request):
849	n/a	"""Return true if (and only if) cookie should be returned to server."""
850	n/a	raise NotImplementedError()
851	n/a
852	n/a	def domain_return_ok(self, domain, request):
853	n/a	"""Return false if cookies should not be returned, given cookie domain.
854	n/a	"""
855	n/a	return True
856	n/a
857	n/a	def path_return_ok(self, path, request):
858	n/a	"""Return false if cookies should not be returned, given cookie path.
859	n/a	"""
860	n/a	return True
861	n/a
862	n/a
863	n/a	class DefaultCookiePolicy(CookiePolicy):
864	n/a	"""Implements the standard rules for accepting and returning cookies."""
865	n/a
866	n/a	DomainStrictNoDots = 1
867	n/a	DomainStrictNonDomain = 2
868	n/a	DomainRFC2965Match = 4
869	n/a
870	n/a	DomainLiberal = 0
871	n/a	DomainStrict = DomainStrictNoDots\|DomainStrictNonDomain
872	n/a
873	n/a	def __init__(self,
874	n/a	blocked_domains=None, allowed_domains=None,
875	n/a	netscape=True, rfc2965=False,
876	n/a	rfc2109_as_netscape=None,
877	n/a	hide_cookie2=False,
878	n/a	strict_domain=False,
879	n/a	strict_rfc2965_unverifiable=True,
880	n/a	strict_ns_unverifiable=False,
881	n/a	strict_ns_domain=DomainLiberal,
882	n/a	strict_ns_set_initial_dollar=False,
883	n/a	strict_ns_set_path=False,
884	n/a	):
885	n/a	"""Constructor arguments should be passed as keyword arguments only."""
886	n/a	self.netscape = netscape
887	n/a	self.rfc2965 = rfc2965
888	n/a	self.rfc2109_as_netscape = rfc2109_as_netscape
889	n/a	self.hide_cookie2 = hide_cookie2
890	n/a	self.strict_domain = strict_domain
891	n/a	self.strict_rfc2965_unverifiable = strict_rfc2965_unverifiable
892	n/a	self.strict_ns_unverifiable = strict_ns_unverifiable
893	n/a	self.strict_ns_domain = strict_ns_domain
894	n/a	self.strict_ns_set_initial_dollar = strict_ns_set_initial_dollar
895	n/a	self.strict_ns_set_path = strict_ns_set_path
896	n/a
897	n/a	if blocked_domains is not None:
898	n/a	self._blocked_domains = tuple(blocked_domains)
899	n/a	else:
900	n/a	self._blocked_domains = ()
901	n/a
902	n/a	if allowed_domains is not None:
903	n/a	allowed_domains = tuple(allowed_domains)
904	n/a	self._allowed_domains = allowed_domains
905	n/a
906	n/a	def blocked_domains(self):
907	n/a	"""Return the sequence of blocked domains (as a tuple)."""
908	n/a	return self._blocked_domains
909	n/a	def set_blocked_domains(self, blocked_domains):
910	n/a	"""Set the sequence of blocked domains."""
911	n/a	self._blocked_domains = tuple(blocked_domains)
912	n/a
913	n/a	def is_blocked(self, domain):
914	n/a	for blocked_domain in self._blocked_domains:
915	n/a	if user_domain_match(domain, blocked_domain):
916	n/a	return True
917	n/a	return False
918	n/a
919	n/a	def allowed_domains(self):
920	n/a	"""Return None, or the sequence of allowed domains (as a tuple)."""
921	n/a	return self._allowed_domains
922	n/a	def set_allowed_domains(self, allowed_domains):
923	n/a	"""Set the sequence of allowed domains, or None."""
924	n/a	if allowed_domains is not None:
925	n/a	allowed_domains = tuple(allowed_domains)
926	n/a	self._allowed_domains = allowed_domains
927	n/a
928	n/a	def is_not_allowed(self, domain):
929	n/a	if self._allowed_domains is None:
930	n/a	return False
931	n/a	for allowed_domain in self._allowed_domains:
932	n/a	if user_domain_match(domain, allowed_domain):
933	n/a	return False
934	n/a	return True
935	n/a
936	n/a	def set_ok(self, cookie, request):
937	n/a	"""
938	n/a	If you override .set_ok(), be sure to call this method. If it returns
939	n/a	false, so should your subclass (assuming your subclass wants to be more
940	n/a	strict about which cookies to accept).
941	n/a
942	n/a	"""
943	n/a	_debug(" - checking cookie %s=%s", cookie.name, cookie.value)
944	n/a
945	n/a	assert cookie.name is not None
946	n/a
947	n/a	for n in "version", "verifiability", "name", "path", "domain", "port":
948	n/a	fn_name = "set_ok_"+n
949	n/a	fn = getattr(self, fn_name)
950	n/a	if not fn(cookie, request):
951	n/a	return False
952	n/a
953	n/a	return True
954	n/a
955	n/a	def set_ok_version(self, cookie, request):
956	n/a	if cookie.version is None:
957	n/a	# Version is always set to 0 by parse_ns_headers if it's a Netscape
958	n/a	# cookie, so this must be an invalid RFC 2965 cookie.
959	n/a	_debug(" Set-Cookie2 without version attribute (%s=%s)",
960	n/a	cookie.name, cookie.value)
961	n/a	return False
962	n/a	if cookie.version > 0 and not self.rfc2965:
963	n/a	_debug(" RFC 2965 cookies are switched off")
964	n/a	return False
965	n/a	elif cookie.version == 0 and not self.netscape:
966	n/a	_debug(" Netscape cookies are switched off")
967	n/a	return False
968	n/a	return True
969	n/a
970	n/a	def set_ok_verifiability(self, cookie, request):
971	n/a	if request.unverifiable and is_third_party(request):
972	n/a	if cookie.version > 0 and self.strict_rfc2965_unverifiable:
973	n/a	_debug(" third-party RFC 2965 cookie during "
974	n/a	"unverifiable transaction")
975	n/a	return False
976	n/a	elif cookie.version == 0 and self.strict_ns_unverifiable:
977	n/a	_debug(" third-party Netscape cookie during "
978	n/a	"unverifiable transaction")
979	n/a	return False
980	n/a	return True
981	n/a
982	n/a	def set_ok_name(self, cookie, request):
983	n/a	# Try and stop servers setting V0 cookies designed to hack other
984	n/a	# servers that know both V0 and V1 protocols.
985	n/a	if (cookie.version == 0 and self.strict_ns_set_initial_dollar and
986	n/a	cookie.name.startswith("$")):
987	n/a	_debug(" illegal name (starts with '$'): '%s'", cookie.name)
988	n/a	return False
989	n/a	return True
990	n/a
991	n/a	def set_ok_path(self, cookie, request):
992	n/a	if cookie.path_specified:
993	n/a	req_path = request_path(request)
994	n/a	if ((cookie.version > 0 or
995	n/a	(cookie.version == 0 and self.strict_ns_set_path)) and
996	n/a	not req_path.startswith(cookie.path)):
997	n/a	_debug(" path attribute %s is not a prefix of request "
998	n/a	"path %s", cookie.path, req_path)
999	n/a	return False
1000	n/a	return True
1001	n/a
1002	n/a	def set_ok_domain(self, cookie, request):
1003	n/a	if self.is_blocked(cookie.domain):
1004	n/a	_debug(" domain %s is in user block-list", cookie.domain)
1005	n/a	return False
1006	n/a	if self.is_not_allowed(cookie.domain):
1007	n/a	_debug(" domain %s is not in user allow-list", cookie.domain)
1008	n/a	return False
1009	n/a	if cookie.domain_specified:
1010	n/a	req_host, erhn = eff_request_host(request)
1011	n/a	domain = cookie.domain
1012	n/a	if self.strict_domain and (domain.count(".") >= 2):
1013	n/a	# XXX This should probably be compared with the Konqueror
1014	n/a	# (kcookiejar.cpp) and Mozilla implementations, but it's a
1015	n/a	# losing battle.
1016	n/a	i = domain.rfind(".")
1017	n/a	j = domain.rfind(".", 0, i)
1018	n/a	if j == 0: # domain like .foo.bar
1019	n/a	tld = domain[i+1:]
1020	n/a	sld = domain[j+1:i]
1021	n/a	if sld.lower() in ("co", "ac", "com", "edu", "org", "net",
1022	n/a	"gov", "mil", "int", "aero", "biz", "cat", "coop",
1023	n/a	"info", "jobs", "mobi", "museum", "name", "pro",
1024	n/a	"travel", "eu") and len(tld) == 2:
1025	n/a	# domain like .co.uk
1026	n/a	_debug(" country-code second level domain %s", domain)
1027	n/a	return False
1028	n/a	if domain.startswith("."):
1029	n/a	undotted_domain = domain[1:]
1030	n/a	else:
1031	n/a	undotted_domain = domain
1032	n/a	embedded_dots = (undotted_domain.find(".") >= 0)
1033	n/a	if not embedded_dots and domain != ".local":
1034	n/a	_debug(" non-local domain %s contains no embedded dot",
1035	n/a	domain)
1036	n/a	return False
1037	n/a	if cookie.version == 0:
1038	n/a	if (not erhn.endswith(domain) and
1039	n/a	(not erhn.startswith(".") and
1040	n/a	not ("."+erhn).endswith(domain))):
1041	n/a	_debug(" effective request-host %s (even with added "
1042	n/a	"initial dot) does not end with %s",
1043	n/a	erhn, domain)
1044	n/a	return False
1045	n/a	if (cookie.version > 0 or
1046	n/a	(self.strict_ns_domain & self.DomainRFC2965Match)):
1047	n/a	if not domain_match(erhn, domain):
1048	n/a	_debug(" effective request-host %s does not domain-match "
1049	n/a	"%s", erhn, domain)
1050	n/a	return False
1051	n/a	if (cookie.version > 0 or
1052	n/a	(self.strict_ns_domain & self.DomainStrictNoDots)):
1053	n/a	host_prefix = req_host[:-len(domain)]
1054	n/a	if (host_prefix.find(".") >= 0 and
1055	n/a	not IPV4_RE.search(req_host)):
1056	n/a	_debug(" host prefix %s for domain %s contains a dot",
1057	n/a	host_prefix, domain)
1058	n/a	return False
1059	n/a	return True
1060	n/a
1061	n/a	def set_ok_port(self, cookie, request):
1062	n/a	if cookie.port_specified:
1063	n/a	req_port = request_port(request)
1064	n/a	if req_port is None:
1065	n/a	req_port = "80"
1066	n/a	else:
1067	n/a	req_port = str(req_port)
1068	n/a	for p in cookie.port.split(","):
1069	n/a	try:
1070	n/a	int(p)
1071	n/a	except ValueError:
1072	n/a	_debug(" bad port %s (not numeric)", p)
1073	n/a	return False
1074	n/a	if p == req_port:
1075	n/a	break
1076	n/a	else:
1077	n/a	_debug(" request port (%s) not found in %s",
1078	n/a	req_port, cookie.port)
1079	n/a	return False
1080	n/a	return True
1081	n/a
1082	n/a	def return_ok(self, cookie, request):
1083	n/a	"""
1084	n/a	If you override .return_ok(), be sure to call this method. If it
1085	n/a	returns false, so should your subclass (assuming your subclass wants to
1086	n/a	be more strict about which cookies to return).
1087	n/a
1088	n/a	"""
1089	n/a	# Path has already been checked by .path_return_ok(), and domain
1090	n/a	# blocking done by .domain_return_ok().
1091	n/a	_debug(" - checking cookie %s=%s", cookie.name, cookie.value)
1092	n/a
1093	n/a	for n in "version", "verifiability", "secure", "expires", "port", "domain":
1094	n/a	fn_name = "return_ok_"+n
1095	n/a	fn = getattr(self, fn_name)
1096	n/a	if not fn(cookie, request):
1097	n/a	return False
1098	n/a	return True
1099	n/a
1100	n/a	def return_ok_version(self, cookie, request):
1101	n/a	if cookie.version > 0 and not self.rfc2965:
1102	n/a	_debug(" RFC 2965 cookies are switched off")
1103	n/a	return False
1104	n/a	elif cookie.version == 0 and not self.netscape:
1105	n/a	_debug(" Netscape cookies are switched off")
1106	n/a	return False
1107	n/a	return True
1108	n/a
1109	n/a	def return_ok_verifiability(self, cookie, request):
1110	n/a	if request.unverifiable and is_third_party(request):
1111	n/a	if cookie.version > 0 and self.strict_rfc2965_unverifiable:
1112	n/a	_debug(" third-party RFC 2965 cookie during unverifiable "
1113	n/a	"transaction")
1114	n/a	return False
1115	n/a	elif cookie.version == 0 and self.strict_ns_unverifiable:
1116	n/a	_debug(" third-party Netscape cookie during unverifiable "
1117	n/a	"transaction")
1118	n/a	return False
1119	n/a	return True
1120	n/a
1121	n/a	def return_ok_secure(self, cookie, request):
1122	n/a	if cookie.secure and request.type != "https":
1123	n/a	_debug(" secure cookie with non-secure request")
1124	n/a	return False
1125	n/a	return True
1126	n/a
1127	n/a	def return_ok_expires(self, cookie, request):
1128	n/a	if cookie.is_expired(self._now):
1129	n/a	_debug(" cookie expired")
1130	n/a	return False
1131	n/a	return True
1132	n/a
1133	n/a	def return_ok_port(self, cookie, request):
1134	n/a	if cookie.port:
1135	n/a	req_port = request_port(request)
1136	n/a	if req_port is None:
1137	n/a	req_port = "80"
1138	n/a	for p in cookie.port.split(","):
1139	n/a	if p == req_port:
1140	n/a	break
1141	n/a	else:
1142	n/a	_debug(" request port %s does not match cookie port %s",
1143	n/a	req_port, cookie.port)
1144	n/a	return False
1145	n/a	return True
1146	n/a
1147	n/a	def return_ok_domain(self, cookie, request):
1148	n/a	req_host, erhn = eff_request_host(request)
1149	n/a	domain = cookie.domain
1150	n/a
1151	n/a	# strict check of non-domain cookies: Mozilla does this, MSIE5 doesn't
1152	n/a	if (cookie.version == 0 and
1153	n/a	(self.strict_ns_domain & self.DomainStrictNonDomain) and
1154	n/a	not cookie.domain_specified and domain != erhn):
1155	n/a	_debug(" cookie with unspecified domain does not string-compare "
1156	n/a	"equal to request domain")
1157	n/a	return False
1158	n/a
1159	n/a	if cookie.version > 0 and not domain_match(erhn, domain):
1160	n/a	_debug(" effective request-host name %s does not domain-match "
1161	n/a	"RFC 2965 cookie domain %s", erhn, domain)
1162	n/a	return False
1163	n/a	if cookie.version == 0 and not ("."+erhn).endswith(domain):
1164	n/a	_debug(" request-host %s does not match Netscape cookie domain "
1165	n/a	"%s", req_host, domain)
1166	n/a	return False
1167	n/a	return True
1168	n/a
1169	n/a	def domain_return_ok(self, domain, request):
1170	n/a	# Liberal check of. This is here as an optimization to avoid
1171	n/a	# having to load lots of MSIE cookie files unless necessary.
1172	n/a	req_host, erhn = eff_request_host(request)
1173	n/a	if not req_host.startswith("."):
1174	n/a	req_host = "."+req_host
1175	n/a	if not erhn.startswith("."):
1176	n/a	erhn = "."+erhn
1177	n/a	if not (req_host.endswith(domain) or erhn.endswith(domain)):
1178	n/a	#_debug(" request domain %s does not match cookie domain %s",
1179	n/a	# req_host, domain)
1180	n/a	return False
1181	n/a
1182	n/a	if self.is_blocked(domain):
1183	n/a	_debug(" domain %s is in user block-list", domain)
1184	n/a	return False
1185	n/a	if self.is_not_allowed(domain):
1186	n/a	_debug(" domain %s is not in user allow-list", domain)
1187	n/a	return False
1188	n/a
1189	n/a	return True
1190	n/a
1191	n/a	def path_return_ok(self, path, request):
1192	n/a	_debug("- checking cookie path=%s", path)
1193	n/a	req_path = request_path(request)
1194	n/a	if not req_path.startswith(path):
1195	n/a	_debug(" %s does not path-match %s", req_path, path)
1196	n/a	return False
1197	n/a	return True
1198	n/a
1199	n/a
1200	n/a	def vals_sorted_by_key(adict):
1201	n/a	keys = sorted(adict.keys())
1202	n/a	return map(adict.get, keys)
1203	n/a
1204	n/a	def deepvalues(mapping):
1205	n/a	"""Iterates over nested mapping, depth-first, in sorted order by key."""
1206	n/a	values = vals_sorted_by_key(mapping)
1207	n/a	for obj in values:
1208	n/a	mapping = False
1209	n/a	try:
1210	n/a	obj.items
1211	n/a	except AttributeError:
1212	n/a	pass
1213	n/a	else:
1214	n/a	mapping = True
1215	n/a	yield from deepvalues(obj)
1216	n/a	if not mapping:
1217	n/a	yield obj
1218	n/a
1219	n/a
1220	n/a	# Used as second parameter to dict.get() method, to distinguish absent
1221	n/a	# dict key from one with a None value.
1222	n/a	class Absent: pass
1223	n/a
1224	n/a	class CookieJar:
1225	n/a	"""Collection of HTTP cookies.
1226	n/a
1227	n/a	You may not need to know about this class: try
1228	n/a	urllib.request.build_opener(HTTPCookieProcessor).open(url).
1229	n/a	"""
1230	n/a
1231	n/a	non_word_re = re.compile(r"\W")
1232	n/a	quote_re = re.compile(r"([\"\\])")
1233	n/a	strict_domain_re = re.compile(r"\.?[^.]*")
1234	n/a	domain_re = re.compile(r"[^.]*")
1235	n/a	dots_re = re.compile(r"^\.+")
1236	n/a
1237	n/a	magic_re = re.compile(r"^\#LWP-Cookies-(\d+\.\d+)", re.ASCII)
1238	n/a
1239	n/a	def __init__(self, policy=None):
1240	n/a	if policy is None:
1241	n/a	policy = DefaultCookiePolicy()
1242	n/a	self._policy = policy
1243	n/a
1244	n/a	self._cookies_lock = _threading.RLock()
1245	n/a	self._cookies = {}
1246	n/a
1247	n/a	def set_policy(self, policy):
1248	n/a	self._policy = policy
1249	n/a
1250	n/a	def _cookies_for_domain(self, domain, request):
1251	n/a	cookies = []
1252	n/a	if not self._policy.domain_return_ok(domain, request):
1253	n/a	return []
1254	n/a	_debug("Checking %s for cookies to return", domain)
1255	n/a	cookies_by_path = self._cookies[domain]
1256	n/a	for path in cookies_by_path.keys():
1257	n/a	if not self._policy.path_return_ok(path, request):
1258	n/a	continue
1259	n/a	cookies_by_name = cookies_by_path[path]
1260	n/a	for cookie in cookies_by_name.values():
1261	n/a	if not self._policy.return_ok(cookie, request):
1262	n/a	_debug(" not returning cookie")
1263	n/a	continue
1264	n/a	_debug(" it's a match")
1265	n/a	cookies.append(cookie)
1266	n/a	return cookies
1267	n/a
1268	n/a	def _cookies_for_request(self, request):
1269	n/a	"""Return a list of cookies to be returned to server."""
1270	n/a	cookies = []
1271	n/a	for domain in self._cookies.keys():
1272	n/a	cookies.extend(self._cookies_for_domain(domain, request))
1273	n/a	return cookies
1274	n/a
1275	n/a	def _cookie_attrs(self, cookies):
1276	n/a	"""Return a list of cookie-attributes to be returned to server.
1277	n/a
1278	n/a	like ['foo="bar"; $Path="/"', ...]
1279	n/a
1280	n/a	The $Version attribute is also added when appropriate (currently only
1281	n/a	once per request).
1282	n/a
1283	n/a	"""
1284	n/a	# add cookies in order of most specific (ie. longest) path first
1285	n/a	cookies.sort(key=lambda a: len(a.path), reverse=True)
1286	n/a
1287	n/a	version_set = False
1288	n/a
1289	n/a	attrs = []
1290	n/a	for cookie in cookies:
1291	n/a	# set version of Cookie header
1292	n/a	# XXX
1293	n/a	# What should it be if multiple matching Set-Cookie headers have
1294	n/a	# different versions themselves?
1295	n/a	# Answer: there is no answer; was supposed to be settled by
1296	n/a	# RFC 2965 errata, but that may never appear...
1297	n/a	version = cookie.version
1298	n/a	if not version_set:
1299	n/a	version_set = True
1300	n/a	if version > 0:
1301	n/a	attrs.append("$Version=%s" % version)
1302	n/a
1303	n/a	# quote cookie value if necessary
1304	n/a	# (not for Netscape protocol, which already has any quotes
1305	n/a	# intact, due to the poorly-specified Netscape Cookie: syntax)
1306	n/a	if ((cookie.value is not None) and
1307	n/a	self.non_word_re.search(cookie.value) and version > 0):
1308	n/a	value = self.quote_re.sub(r"\\\1", cookie.value)
1309	n/a	else:
1310	n/a	value = cookie.value
1311	n/a
1312	n/a	# add cookie-attributes to be returned in Cookie header
1313	n/a	if cookie.value is None:
1314	n/a	attrs.append(cookie.name)
1315	n/a	else:
1316	n/a	attrs.append("%s=%s" % (cookie.name, value))
1317	n/a	if version > 0:
1318	n/a	if cookie.path_specified:
1319	n/a	attrs.append('$Path="%s"' % cookie.path)
1320	n/a	if cookie.domain.startswith("."):
1321	n/a	domain = cookie.domain
1322	n/a	if (not cookie.domain_initial_dot and
1323	n/a	domain.startswith(".")):
1324	n/a	domain = domain[1:]
1325	n/a	attrs.append('$Domain="%s"' % domain)
1326	n/a	if cookie.port is not None:
1327	n/a	p = "$Port"
1328	n/a	if cookie.port_specified:
1329	n/a	p = p + ('="%s"' % cookie.port)
1330	n/a	attrs.append(p)
1331	n/a
1332	n/a	return attrs
1333	n/a
1334	n/a	def add_cookie_header(self, request):
1335	n/a	"""Add correct Cookie: header to request (urllib.request.Request object).
1336	n/a
1337	n/a	The Cookie2 header is also added unless policy.hide_cookie2 is true.
1338	n/a
1339	n/a	"""
1340	n/a	_debug("add_cookie_header")
1341	n/a	self._cookies_lock.acquire()
1342	n/a	try:
1343	n/a
1344	n/a	self._policy._now = self._now = int(time.time())
1345	n/a
1346	n/a	cookies = self._cookies_for_request(request)
1347	n/a
1348	n/a	attrs = self._cookie_attrs(cookies)
1349	n/a	if attrs:
1350	n/a	if not request.has_header("Cookie"):
1351	n/a	request.add_unredirected_header(
1352	n/a	"Cookie", "; ".join(attrs))
1353	n/a
1354	n/a	# if necessary, advertise that we know RFC 2965
1355	n/a	if (self._policy.rfc2965 and not self._policy.hide_cookie2 and
1356	n/a	not request.has_header("Cookie2")):
1357	n/a	for cookie in cookies:
1358	n/a	if cookie.version != 1:
1359	n/a	request.add_unredirected_header("Cookie2", '$Version="1"')
1360	n/a	break
1361	n/a
1362	n/a	finally:
1363	n/a	self._cookies_lock.release()
1364	n/a
1365	n/a	self.clear_expired_cookies()
1366	n/a
1367	n/a	def _normalized_cookie_tuples(self, attrs_set):
1368	n/a	"""Return list of tuples containing normalised cookie information.
1369	n/a
1370	n/a	attrs_set is the list of lists of key,value pairs extracted from
1371	n/a	the Set-Cookie or Set-Cookie2 headers.
1372	n/a
1373	n/a	Tuples are name, value, standard, rest, where name and value are the
1374	n/a	cookie name and value, standard is a dictionary containing the standard
1375	n/a	cookie-attributes (discard, secure, version, expires or max-age,
1376	n/a	domain, path and port) and rest is a dictionary containing the rest of
1377	n/a	the cookie-attributes.
1378	n/a
1379	n/a	"""
1380	n/a	cookie_tuples = []
1381	n/a
1382	n/a	boolean_attrs = "discard", "secure"
1383	n/a	value_attrs = ("version",
1384	n/a	"expires", "max-age",
1385	n/a	"domain", "path", "port",
1386	n/a	"comment", "commenturl")
1387	n/a
1388	n/a	for cookie_attrs in attrs_set:
1389	n/a	name, value = cookie_attrs[0]
1390	n/a
1391	n/a	# Build dictionary of standard cookie-attributes (standard) and
1392	n/a	# dictionary of other cookie-attributes (rest).
1393	n/a
1394	n/a	# Note: expiry time is normalised to seconds since epoch. V0
1395	n/a	# cookies should have the Expires cookie-attribute, and V1 cookies
1396	n/a	# should have Max-Age, but since V1 includes RFC 2109 cookies (and
1397	n/a	# since V0 cookies may be a mish-mash of Netscape and RFC 2109), we
1398	n/a	# accept either (but prefer Max-Age).
1399	n/a	max_age_set = False
1400	n/a
1401	n/a	bad_cookie = False
1402	n/a
1403	n/a	standard = {}
1404	n/a	rest = {}
1405	n/a	for k, v in cookie_attrs[1:]:
1406	n/a	lc = k.lower()
1407	n/a	# don't lose case distinction for unknown fields
1408	n/a	if lc in value_attrs or lc in boolean_attrs:
1409	n/a	k = lc
1410	n/a	if k in boolean_attrs and v is None:
1411	n/a	# boolean cookie-attribute is present, but has no value
1412	n/a	# (like "discard", rather than "port=80")
1413	n/a	v = True
1414	n/a	if k in standard:
1415	n/a	# only first value is significant
1416	n/a	continue
1417	n/a	if k == "domain":
1418	n/a	if v is None:
1419	n/a	_debug(" missing value for domain attribute")
1420	n/a	bad_cookie = True
1421	n/a	break
1422	n/a	# RFC 2965 section 3.3.3
1423	n/a	v = v.lower()
1424	n/a	if k == "expires":
1425	n/a	if max_age_set:
1426	n/a	# Prefer max-age to expires (like Mozilla)
1427	n/a	continue
1428	n/a	if v is None:
1429	n/a	_debug(" missing or invalid value for expires "
1430	n/a	"attribute: treating as session cookie")
1431	n/a	continue
1432	n/a	if k == "max-age":
1433	n/a	max_age_set = True
1434	n/a	try:
1435	n/a	v = int(v)
1436	n/a	except ValueError:
1437	n/a	_debug(" missing or invalid (non-numeric) value for "
1438	n/a	"max-age attribute")
1439	n/a	bad_cookie = True
1440	n/a	break
1441	n/a	# convert RFC 2965 Max-Age to seconds since epoch
1442	n/a	# XXX Strictly you're supposed to follow RFC 2616
1443	n/a	# age-calculation rules. Remember that zero Max-Age
1444	n/a	# is a request to discard (old and new) cookie, though.
1445	n/a	k = "expires"
1446	n/a	v = self._now + v
1447	n/a	if (k in value_attrs) or (k in boolean_attrs):
1448	n/a	if (v is None and
1449	n/a	k not in ("port", "comment", "commenturl")):
1450	n/a	_debug(" missing value for %s attribute" % k)
1451	n/a	bad_cookie = True
1452	n/a	break
1453	n/a	standard[k] = v
1454	n/a	else:
1455	n/a	rest[k] = v
1456	n/a
1457	n/a	if bad_cookie:
1458	n/a	continue
1459	n/a
1460	n/a	cookie_tuples.append((name, value, standard, rest))
1461	n/a
1462	n/a	return cookie_tuples
1463	n/a
1464	n/a	def _cookie_from_cookie_tuple(self, tup, request):
1465	n/a	# standard is dict of standard cookie-attributes, rest is dict of the
1466	n/a	# rest of them
1467	n/a	name, value, standard, rest = tup
1468	n/a
1469	n/a	domain = standard.get("domain", Absent)
1470	n/a	path = standard.get("path", Absent)
1471	n/a	port = standard.get("port", Absent)
1472	n/a	expires = standard.get("expires", Absent)
1473	n/a
1474	n/a	# set the easy defaults
1475	n/a	version = standard.get("version", None)
1476	n/a	if version is not None:
1477	n/a	try:
1478	n/a	version = int(version)
1479	n/a	except ValueError:
1480	n/a	return None # invalid version, ignore cookie
1481	n/a	secure = standard.get("secure", False)
1482	n/a	# (discard is also set if expires is Absent)
1483	n/a	discard = standard.get("discard", False)
1484	n/a	comment = standard.get("comment", None)
1485	n/a	comment_url = standard.get("commenturl", None)
1486	n/a
1487	n/a	# set default path
1488	n/a	if path is not Absent and path != "":
1489	n/a	path_specified = True
1490	n/a	path = escape_path(path)
1491	n/a	else:
1492	n/a	path_specified = False
1493	n/a	path = request_path(request)
1494	n/a	i = path.rfind("/")
1495	n/a	if i != -1:
1496	n/a	if version == 0:
1497	n/a	# Netscape spec parts company from reality here
1498	n/a	path = path[:i]
1499	n/a	else:
1500	n/a	path = path[:i+1]
1501	n/a	if len(path) == 0: path = "/"
1502	n/a
1503	n/a	# set default domain
1504	n/a	domain_specified = domain is not Absent
1505	n/a	# but first we have to remember whether it starts with a dot
1506	n/a	domain_initial_dot = False
1507	n/a	if domain_specified:
1508	n/a	domain_initial_dot = bool(domain.startswith("."))
1509	n/a	if domain is Absent:
1510	n/a	req_host, erhn = eff_request_host(request)
1511	n/a	domain = erhn
1512	n/a	elif not domain.startswith("."):
1513	n/a	domain = "."+domain
1514	n/a
1515	n/a	# set default port
1516	n/a	port_specified = False
1517	n/a	if port is not Absent:
1518	n/a	if port is None:
1519	n/a	# Port attr present, but has no value: default to request port.
1520	n/a	# Cookie should then only be sent back on that port.
1521	n/a	port = request_port(request)
1522	n/a	else:
1523	n/a	port_specified = True
1524	n/a	port = re.sub(r"\s+", "", port)
1525	n/a	else:
1526	n/a	# No port attr present. Cookie can be sent back on any port.
1527	n/a	port = None
1528	n/a
1529	n/a	# set default expires and discard
1530	n/a	if expires is Absent:
1531	n/a	expires = None
1532	n/a	discard = True
1533	n/a	elif expires <= self._now:
1534	n/a	# Expiry date in past is request to delete cookie. This can't be
1535	n/a	# in DefaultCookiePolicy, because can't delete cookies there.
1536	n/a	try:
1537	n/a	self.clear(domain, path, name)
1538	n/a	except KeyError:
1539	n/a	pass
1540	n/a	_debug("Expiring cookie, domain='%s', path='%s', name='%s'",
1541	n/a	domain, path, name)
1542	n/a	return None
1543	n/a
1544	n/a	return Cookie(version,
1545	n/a	name, value,
1546	n/a	port, port_specified,
1547	n/a	domain, domain_specified, domain_initial_dot,
1548	n/a	path, path_specified,
1549	n/a	secure,
1550	n/a	expires,
1551	n/a	discard,
1552	n/a	comment,
1553	n/a	comment_url,
1554	n/a	rest)
1555	n/a
1556	n/a	def _cookies_from_attrs_set(self, attrs_set, request):
1557	n/a	cookie_tuples = self._normalized_cookie_tuples(attrs_set)
1558	n/a
1559	n/a	cookies = []
1560	n/a	for tup in cookie_tuples:
1561	n/a	cookie = self._cookie_from_cookie_tuple(tup, request)
1562	n/a	if cookie: cookies.append(cookie)
1563	n/a	return cookies
1564	n/a
1565	n/a	def _process_rfc2109_cookies(self, cookies):
1566	n/a	rfc2109_as_ns = getattr(self._policy, 'rfc2109_as_netscape', None)
1567	n/a	if rfc2109_as_ns is None:
1568	n/a	rfc2109_as_ns = not self._policy.rfc2965
1569	n/a	for cookie in cookies:
1570	n/a	if cookie.version == 1:
1571	n/a	cookie.rfc2109 = True
1572	n/a	if rfc2109_as_ns:
1573	n/a	# treat 2109 cookies as Netscape cookies rather than
1574	n/a	# as RFC2965 cookies
1575	n/a	cookie.version = 0
1576	n/a
1577	n/a	def make_cookies(self, response, request):
1578	n/a	"""Return sequence of Cookie objects extracted from response object."""
1579	n/a	# get cookie-attributes for RFC 2965 and Netscape protocols
1580	n/a	headers = response.info()
1581	n/a	rfc2965_hdrs = headers.get_all("Set-Cookie2", [])
1582	n/a	ns_hdrs = headers.get_all("Set-Cookie", [])
1583	n/a
1584	n/a	rfc2965 = self._policy.rfc2965
1585	n/a	netscape = self._policy.netscape
1586	n/a
1587	n/a	if ((not rfc2965_hdrs and not ns_hdrs) or
1588	n/a	(not ns_hdrs and not rfc2965) or
1589	n/a	(not rfc2965_hdrs and not netscape) or
1590	n/a	(not netscape and not rfc2965)):
1591	n/a	return [] # no relevant cookie headers: quick exit
1592	n/a
1593	n/a	try:
1594	n/a	cookies = self._cookies_from_attrs_set(
1595	n/a	split_header_words(rfc2965_hdrs), request)
1596	n/a	except Exception:
1597	n/a	_warn_unhandled_exception()
1598	n/a	cookies = []
1599	n/a
1600	n/a	if ns_hdrs and netscape:
1601	n/a	try:
1602	n/a	# RFC 2109 and Netscape cookies
1603	n/a	ns_cookies = self._cookies_from_attrs_set(
1604	n/a	parse_ns_headers(ns_hdrs), request)
1605	n/a	except Exception:
1606	n/a	_warn_unhandled_exception()
1607	n/a	ns_cookies = []
1608	n/a	self._process_rfc2109_cookies(ns_cookies)
1609	n/a
1610	n/a	# Look for Netscape cookies (from Set-Cookie headers) that match
1611	n/a	# corresponding RFC 2965 cookies (from Set-Cookie2 headers).
1612	n/a	# For each match, keep the RFC 2965 cookie and ignore the Netscape
1613	n/a	# cookie (RFC 2965 section 9.1). Actually, RFC 2109 cookies are
1614	n/a	# bundled in with the Netscape cookies for this purpose, which is
1615	n/a	# reasonable behaviour.
1616	n/a	if rfc2965:
1617	n/a	lookup = {}
1618	n/a	for cookie in cookies:
1619	n/a	lookup[(cookie.domain, cookie.path, cookie.name)] = None
1620	n/a
1621	n/a	def no_matching_rfc2965(ns_cookie, lookup=lookup):
1622	n/a	key = ns_cookie.domain, ns_cookie.path, ns_cookie.name
1623	n/a	return key not in lookup
1624	n/a	ns_cookies = filter(no_matching_rfc2965, ns_cookies)
1625	n/a
1626	n/a	if ns_cookies:
1627	n/a	cookies.extend(ns_cookies)
1628	n/a
1629	n/a	return cookies
1630	n/a
1631	n/a	def set_cookie_if_ok(self, cookie, request):
1632	n/a	"""Set a cookie if policy says it's OK to do so."""
1633	n/a	self._cookies_lock.acquire()
1634	n/a	try:
1635	n/a	self._policy._now = self._now = int(time.time())
1636	n/a
1637	n/a	if self._policy.set_ok(cookie, request):
1638	n/a	self.set_cookie(cookie)
1639	n/a
1640	n/a
1641	n/a	finally:
1642	n/a	self._cookies_lock.release()
1643	n/a
1644	n/a	def set_cookie(self, cookie):
1645	n/a	"""Set a cookie, without checking whether or not it should be set."""
1646	n/a	c = self._cookies
1647	n/a	self._cookies_lock.acquire()
1648	n/a	try:
1649	n/a	if cookie.domain not in c: c[cookie.domain] = {}
1650	n/a	c2 = c[cookie.domain]
1651	n/a	if cookie.path not in c2: c2[cookie.path] = {}
1652	n/a	c3 = c2[cookie.path]
1653	n/a	c3[cookie.name] = cookie
1654	n/a	finally:
1655	n/a	self._cookies_lock.release()
1656	n/a
1657	n/a	def extract_cookies(self, response, request):
1658	n/a	"""Extract cookies from response, where allowable given the request."""
1659	n/a	_debug("extract_cookies: %s", response.info())
1660	n/a	self._cookies_lock.acquire()
1661	n/a	try:
1662	n/a	self._policy._now = self._now = int(time.time())
1663	n/a
1664	n/a	for cookie in self.make_cookies(response, request):
1665	n/a	if self._policy.set_ok(cookie, request):
1666	n/a	_debug(" setting cookie: %s", cookie)
1667	n/a	self.set_cookie(cookie)
1668	n/a	finally:
1669	n/a	self._cookies_lock.release()
1670	n/a
1671	n/a	def clear(self, domain=None, path=None, name=None):
1672	n/a	"""Clear some cookies.
1673	n/a
1674	n/a	Invoking this method without arguments will clear all cookies. If
1675	n/a	given a single argument, only cookies belonging to that domain will be
1676	n/a	removed. If given two arguments, cookies belonging to the specified
1677	n/a	path within that domain are removed. If given three arguments, then
1678	n/a	the cookie with the specified name, path and domain is removed.
1679	n/a
1680	n/a	Raises KeyError if no matching cookie exists.
1681	n/a
1682	n/a	"""
1683	n/a	if name is not None:
1684	n/a	if (domain is None) or (path is None):
1685	n/a	raise ValueError(
1686	n/a	"domain and path must be given to remove a cookie by name")
1687	n/a	del self._cookies[domain][path][name]
1688	n/a	elif path is not None:
1689	n/a	if domain is None:
1690	n/a	raise ValueError(
1691	n/a	"domain must be given to remove cookies by path")
1692	n/a	del self._cookies[domain][path]
1693	n/a	elif domain is not None:
1694	n/a	del self._cookies[domain]
1695	n/a	else:
1696	n/a	self._cookies = {}
1697	n/a
1698	n/a	def clear_session_cookies(self):
1699	n/a	"""Discard all session cookies.
1700	n/a
1701	n/a	Note that the .save() method won't save session cookies anyway, unless
1702	n/a	you ask otherwise by passing a true ignore_discard argument.
1703	n/a
1704	n/a	"""
1705	n/a	self._cookies_lock.acquire()
1706	n/a	try:
1707	n/a	for cookie in self:
1708	n/a	if cookie.discard:
1709	n/a	self.clear(cookie.domain, cookie.path, cookie.name)
1710	n/a	finally:
1711	n/a	self._cookies_lock.release()
1712	n/a
1713	n/a	def clear_expired_cookies(self):
1714	n/a	"""Discard all expired cookies.
1715	n/a
1716	n/a	You probably don't need to call this method: expired cookies are never
1717	n/a	sent back to the server (provided you're using DefaultCookiePolicy),
1718	n/a	this method is called by CookieJar itself every so often, and the
1719	n/a	.save() method won't save expired cookies anyway (unless you ask
1720	n/a	otherwise by passing a true ignore_expires argument).
1721	n/a
1722	n/a	"""
1723	n/a	self._cookies_lock.acquire()
1724	n/a	try:
1725	n/a	now = time.time()
1726	n/a	for cookie in self:
1727	n/a	if cookie.is_expired(now):
1728	n/a	self.clear(cookie.domain, cookie.path, cookie.name)
1729	n/a	finally:
1730	n/a	self._cookies_lock.release()
1731	n/a
1732	n/a	def __iter__(self):
1733	n/a	return deepvalues(self._cookies)
1734	n/a
1735	n/a	def __len__(self):
1736	n/a	"""Return number of contained cookies."""
1737	n/a	i = 0
1738	n/a	for cookie in self: i = i + 1
1739	n/a	return i
1740	n/a
1741	n/a	def __repr__(self):
1742	n/a	r = []
1743	n/a	for cookie in self: r.append(repr(cookie))
1744	n/a	return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
1745	n/a
1746	n/a	def __str__(self):
1747	n/a	r = []
1748	n/a	for cookie in self: r.append(str(cookie))
1749	n/a	return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
1750	n/a
1751	n/a
1752	n/a	# derives from OSError for backwards-compatibility with Python 2.4.0
1753	n/a	class LoadError(OSError): pass
1754	n/a
1755	n/a	class FileCookieJar(CookieJar):
1756	n/a	"""CookieJar that can be loaded from and saved to a file."""
1757	n/a
1758	n/a	def __init__(self, filename=None, delayload=False, policy=None):
1759	n/a	"""
1760	n/a	Cookies are NOT loaded from the named file until either the .load() or
1761	n/a	.revert() method is called.
1762	n/a
1763	n/a	"""
1764	n/a	CookieJar.__init__(self, policy)
1765	n/a	if filename is not None:
1766	n/a	try:
1767	n/a	filename+""
1768	n/a	except:
1769	n/a	raise ValueError("filename must be string-like")
1770	n/a	self.filename = filename
1771	n/a	self.delayload = bool(delayload)
1772	n/a
1773	n/a	def save(self, filename=None, ignore_discard=False, ignore_expires=False):
1774	n/a	"""Save cookies to a file."""
1775	n/a	raise NotImplementedError()
1776	n/a
1777	n/a	def load(self, filename=None, ignore_discard=False, ignore_expires=False):
1778	n/a	"""Load cookies from a file."""
1779	n/a	if filename is None:
1780	n/a	if self.filename is not None: filename = self.filename
1781	n/a	else: raise ValueError(MISSING_FILENAME_TEXT)
1782	n/a
1783	n/a	with open(filename) as f:
1784	n/a	self._really_load(f, filename, ignore_discard, ignore_expires)
1785	n/a
1786	n/a	def revert(self, filename=None,
1787	n/a	ignore_discard=False, ignore_expires=False):
1788	n/a	"""Clear all cookies and reload cookies from a saved file.
1789	n/a
1790	n/a	Raises LoadError (or OSError) if reversion is not successful; the
1791	n/a	object's state will not be altered if this happens.
1792	n/a
1793	n/a	"""
1794	n/a	if filename is None:
1795	n/a	if self.filename is not None: filename = self.filename
1796	n/a	else: raise ValueError(MISSING_FILENAME_TEXT)
1797	n/a
1798	n/a	self._cookies_lock.acquire()
1799	n/a	try:
1800	n/a
1801	n/a	old_state = copy.deepcopy(self._cookies)
1802	n/a	self._cookies = {}
1803	n/a	try:
1804	n/a	self.load(filename, ignore_discard, ignore_expires)
1805	n/a	except OSError:
1806	n/a	self._cookies = old_state
1807	n/a	raise
1808	n/a
1809	n/a	finally:
1810	n/a	self._cookies_lock.release()
1811	n/a
1812	n/a
1813	n/a	def lwp_cookie_str(cookie):
1814	n/a	"""Return string representation of Cookie in the LWP cookie file format.
1815	n/a
1816	n/a	Actually, the format is extended a bit -- see module docstring.
1817	n/a
1818	n/a	"""
1819	n/a	h = [(cookie.name, cookie.value),
1820	n/a	("path", cookie.path),
1821	n/a	("domain", cookie.domain)]
1822	n/a	if cookie.port is not None: h.append(("port", cookie.port))
1823	n/a	if cookie.path_specified: h.append(("path_spec", None))
1824	n/a	if cookie.port_specified: h.append(("port_spec", None))
1825	n/a	if cookie.domain_initial_dot: h.append(("domain_dot", None))
1826	n/a	if cookie.secure: h.append(("secure", None))
1827	n/a	if cookie.expires: h.append(("expires",
1828	n/a	time2isoz(float(cookie.expires))))
1829	n/a	if cookie.discard: h.append(("discard", None))
1830	n/a	if cookie.comment: h.append(("comment", cookie.comment))
1831	n/a	if cookie.comment_url: h.append(("commenturl", cookie.comment_url))
1832	n/a
1833	n/a	keys = sorted(cookie._rest.keys())
1834	n/a	for k in keys:
1835	n/a	h.append((k, str(cookie._rest[k])))
1836	n/a
1837	n/a	h.append(("version", str(cookie.version)))
1838	n/a
1839	n/a	return join_header_words([h])
1840	n/a
1841	n/a	class LWPCookieJar(FileCookieJar):
1842	n/a	"""
1843	n/a	The LWPCookieJar saves a sequence of "Set-Cookie3" lines.
1844	n/a	"Set-Cookie3" is the format used by the libwww-perl library, not known
1845	n/a	to be compatible with any browser, but which is easy to read and
1846	n/a	doesn't lose information about RFC 2965 cookies.
1847	n/a
1848	n/a	Additional methods
1849	n/a
1850	n/a	as_lwp_str(ignore_discard=True, ignore_expired=True)
1851	n/a
1852	n/a	"""
1853	n/a
1854	n/a	def as_lwp_str(self, ignore_discard=True, ignore_expires=True):
1855	n/a	"""Return cookies as a string of "\\n"-separated "Set-Cookie3" headers.
1856	n/a
1857	n/a	ignore_discard and ignore_expires: see docstring for FileCookieJar.save
1858	n/a
1859	n/a	"""
1860	n/a	now = time.time()
1861	n/a	r = []
1862	n/a	for cookie in self:
1863	n/a	if not ignore_discard and cookie.discard:
1864	n/a	continue
1865	n/a	if not ignore_expires and cookie.is_expired(now):
1866	n/a	continue
1867	n/a	r.append("Set-Cookie3: %s" % lwp_cookie_str(cookie))
1868	n/a	return "\n".join(r+[""])
1869	n/a
1870	n/a	def save(self, filename=None, ignore_discard=False, ignore_expires=False):
1871	n/a	if filename is None:
1872	n/a	if self.filename is not None: filename = self.filename
1873	n/a	else: raise ValueError(MISSING_FILENAME_TEXT)
1874	n/a
1875	n/a	with open(filename, "w") as f:
1876	n/a	# There really isn't an LWP Cookies 2.0 format, but this indicates
1877	n/a	# that there is extra information in here (domain_dot and
1878	n/a	# port_spec) while still being compatible with libwww-perl, I hope.
1879	n/a	f.write("#LWP-Cookies-2.0\n")
1880	n/a	f.write(self.as_lwp_str(ignore_discard, ignore_expires))
1881	n/a
1882	n/a	def _really_load(self, f, filename, ignore_discard, ignore_expires):
1883	n/a	magic = f.readline()
1884	n/a	if not self.magic_re.search(magic):
1885	n/a	msg = ("%r does not look like a Set-Cookie3 (LWP) format "
1886	n/a	"file" % filename)
1887	n/a	raise LoadError(msg)
1888	n/a
1889	n/a	now = time.time()
1890	n/a
1891	n/a	header = "Set-Cookie3:"
1892	n/a	boolean_attrs = ("port_spec", "path_spec", "domain_dot",
1893	n/a	"secure", "discard")
1894	n/a	value_attrs = ("version",
1895	n/a	"port", "path", "domain",
1896	n/a	"expires",
1897	n/a	"comment", "commenturl")
1898	n/a
1899	n/a	try:
1900	n/a	while 1:
1901	n/a	line = f.readline()
1902	n/a	if line == "": break
1903	n/a	if not line.startswith(header):
1904	n/a	continue
1905	n/a	line = line[len(header):].strip()
1906	n/a
1907	n/a	for data in split_header_words([line]):
1908	n/a	name, value = data[0]
1909	n/a	standard = {}
1910	n/a	rest = {}
1911	n/a	for k in boolean_attrs:
1912	n/a	standard[k] = False
1913	n/a	for k, v in data[1:]:
1914	n/a	if k is not None:
1915	n/a	lc = k.lower()
1916	n/a	else:
1917	n/a	lc = None
1918	n/a	# don't lose case distinction for unknown fields
1919	n/a	if (lc in value_attrs) or (lc in boolean_attrs):
1920	n/a	k = lc
1921	n/a	if k in boolean_attrs:
1922	n/a	if v is None: v = True
1923	n/a	standard[k] = v
1924	n/a	elif k in value_attrs:
1925	n/a	standard[k] = v
1926	n/a	else:
1927	n/a	rest[k] = v
1928	n/a
1929	n/a	h = standard.get
1930	n/a	expires = h("expires")
1931	n/a	discard = h("discard")
1932	n/a	if expires is not None:
1933	n/a	expires = iso2time(expires)
1934	n/a	if expires is None:
1935	n/a	discard = True
1936	n/a	domain = h("domain")
1937	n/a	domain_specified = domain.startswith(".")
1938	n/a	c = Cookie(h("version"), name, value,
1939	n/a	h("port"), h("port_spec"),
1940	n/a	domain, domain_specified, h("domain_dot"),
1941	n/a	h("path"), h("path_spec"),
1942	n/a	h("secure"),
1943	n/a	expires,
1944	n/a	discard,
1945	n/a	h("comment"),
1946	n/a	h("commenturl"),
1947	n/a	rest)
1948	n/a	if not ignore_discard and c.discard:
1949	n/a	continue
1950	n/a	if not ignore_expires and c.is_expired(now):
1951	n/a	continue
1952	n/a	self.set_cookie(c)
1953	n/a	except OSError:
1954	n/a	raise
1955	n/a	except Exception:
1956	n/a	_warn_unhandled_exception()
1957	n/a	raise LoadError("invalid Set-Cookie3 format file %r: %r" %
1958	n/a	(filename, line))
1959	n/a
1960	n/a
1961	n/a	class MozillaCookieJar(FileCookieJar):
1962	n/a	"""
1963	n/a
1964	n/a	WARNING: you may want to backup your browser's cookies file if you use
1965	n/a	this class to save cookies. I think it works, but there have been
1966	n/a	bugs in the past!
1967	n/a
1968	n/a	This class differs from CookieJar only in the format it uses to save and
1969	n/a	load cookies to and from a file. This class uses the Mozilla/Netscape
1970	n/a	`cookies.txt' format. lynx uses this file format, too.
1971	n/a
1972	n/a	Don't expect cookies saved while the browser is running to be noticed by
1973	n/a	the browser (in fact, Mozilla on unix will overwrite your saved cookies if
1974	n/a	you change them on disk while it's running; on Windows, you probably can't
1975	n/a	save at all while the browser is running).
1976	n/a
1977	n/a	Note that the Mozilla/Netscape format will downgrade RFC2965 cookies to
1978	n/a	Netscape cookies on saving.
1979	n/a
1980	n/a	In particular, the cookie version and port number information is lost,
1981	n/a	together with information about whether or not Path, Port and Discard were
1982	n/a	specified by the Set-Cookie2 (or Set-Cookie) header, and whether or not the
1983	n/a	domain as set in the HTTP header started with a dot (yes, I'm aware some
1984	n/a	domains in Netscape files start with a dot and some don't -- trust me, you
1985	n/a	really don't want to know any more about this).
1986	n/a
1987	n/a	Note that though Mozilla and Netscape use the same format, they use
1988	n/a	slightly different headers. The class saves cookies using the Netscape
1989	n/a	header by default (Mozilla can cope with that).
1990	n/a
1991	n/a	"""
1992	n/a	magic_re = re.compile("#( Netscape)? HTTP Cookie File")
1993	n/a	header = """\
1994	n/a	# Netscape HTTP Cookie File
1995	n/a	# http://curl.haxx.se/rfc/cookie_spec.html
1996	n/a	# This is a generated file! Do not edit.
1997	n/a
1998	n/a	"""
1999	n/a
2000	n/a	def _really_load(self, f, filename, ignore_discard, ignore_expires):
2001	n/a	now = time.time()
2002	n/a
2003	n/a	magic = f.readline()
2004	n/a	if not self.magic_re.search(magic):
2005	n/a	raise LoadError(
2006	n/a	"%r does not look like a Netscape format cookies file" %
2007	n/a	filename)
2008	n/a
2009	n/a	try:
2010	n/a	while 1:
2011	n/a	line = f.readline()
2012	n/a	if line == "": break
2013	n/a
2014	n/a	# last field may be absent, so keep any trailing tab
2015	n/a	if line.endswith("\n"): line = line[:-1]
2016	n/a
2017	n/a	# skip comments and blank lines XXX what is $ for?
2018	n/a	if (line.strip().startswith(("#", "$")) or
2019	n/a	line.strip() == ""):
2020	n/a	continue
2021	n/a
2022	n/a	domain, domain_specified, path, secure, expires, name, value = \
2023	n/a	line.split("\t")
2024	n/a	secure = (secure == "TRUE")
2025	n/a	domain_specified = (domain_specified == "TRUE")
2026	n/a	if name == "":
2027	n/a	# cookies.txt regards 'Set-Cookie: foo' as a cookie
2028	n/a	# with no name, whereas http.cookiejar regards it as a
2029	n/a	# cookie with no value.
2030	n/a	name = value
2031	n/a	value = None
2032	n/a
2033	n/a	initial_dot = domain.startswith(".")
2034	n/a	assert domain_specified == initial_dot
2035	n/a
2036	n/a	discard = False
2037	n/a	if expires == "":
2038	n/a	expires = None
2039	n/a	discard = True
2040	n/a
2041	n/a	# assume path_specified is false
2042	n/a	c = Cookie(0, name, value,
2043	n/a	None, False,
2044	n/a	domain, domain_specified, initial_dot,
2045	n/a	path, False,
2046	n/a	secure,
2047	n/a	expires,
2048	n/a	discard,
2049	n/a	None,
2050	n/a	None,
2051	n/a	{})
2052	n/a	if not ignore_discard and c.discard:
2053	n/a	continue
2054	n/a	if not ignore_expires and c.is_expired(now):
2055	n/a	continue
2056	n/a	self.set_cookie(c)
2057	n/a
2058	n/a	except OSError:
2059	n/a	raise
2060	n/a	except Exception:
2061	n/a	_warn_unhandled_exception()
2062	n/a	raise LoadError("invalid Netscape format cookies file %r: %r" %
2063	n/a	(filename, line))
2064	n/a
2065	n/a	def save(self, filename=None, ignore_discard=False, ignore_expires=False):
2066	n/a	if filename is None:
2067	n/a	if self.filename is not None: filename = self.filename
2068	n/a	else: raise ValueError(MISSING_FILENAME_TEXT)
2069	n/a
2070	n/a	with open(filename, "w") as f:
2071	n/a	f.write(self.header)
2072	n/a	now = time.time()
2073	n/a	for cookie in self:
2074	n/a	if not ignore_discard and cookie.discard:
2075	n/a	continue
2076	n/a	if not ignore_expires and cookie.is_expired(now):
2077	n/a	continue
2078	n/a	if cookie.secure: secure = "TRUE"
2079	n/a	else: secure = "FALSE"
2080	n/a	if cookie.domain.startswith("."): initial_dot = "TRUE"
2081	n/a	else: initial_dot = "FALSE"
2082	n/a	if cookie.expires is not None:
2083	n/a	expires = str(cookie.expires)
2084	n/a	else:
2085	n/a	expires = ""
2086	n/a	if cookie.value is None:
2087	n/a	# cookies.txt regards 'Set-Cookie: foo' as a cookie
2088	n/a	# with no name, whereas http.cookiejar regards it as a
2089	n/a	# cookie with no value.
2090	n/a	name = ""
2091	n/a	value = cookie.name
2092	n/a	else:
2093	n/a	name = cookie.name
2094	n/a	value = cookie.value
2095	n/a	f.write(
2096	n/a	"\t".join([cookie.domain, initial_dot, cookie.path,
2097	n/a	secure, expires, name, value])+
2098	n/a	"\n")