Python code coverage for Lib/HTMLParser.py

#	count	content
1	1	"""A parser for HTML and XHTML."""
2	n/a
3	n/a	# This file is based on sgmllib.py, but the API is slightly different.
4	n/a
5	n/a	# XXX There should be a way to distinguish between PCDATA (parsed
6	n/a	# character data -- the normal case), RCDATA (replaceable character
7	n/a	# data -- only char and entity references and end tags are special)
8	n/a	# and CDATA (character data -- only end tags are special).
9	n/a
10	n/a
11	1	import markupbase
12	1	import re
13	n/a
14	n/a	# Regular expressions used for parsing
15	n/a
16	1	interesting_normal = re.compile('[&<]')
17	1	interesting_cdata = re.compile(r'<(/\|\Z)')
18	1	incomplete = re.compile('&[a-zA-Z#]')
19	n/a
20	1	entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
21	1	charref = re.compile('&#(?:[0-9]+\|[xX][0-9a-fA-F]+)[^0-9a-fA-F]')
22	n/a
23	1	starttagopen = re.compile('<[a-zA-Z]')
24	1	piclose = re.compile('>')
25	1	commentclose = re.compile(r'--\s*>')
26	1	tagfind = re.compile('[a-zA-Z][-.a-zA-Z0-9:_]*')
27	1	attrfind = re.compile(
28	1	r'\s([a-zA-Z_][-.:a-zA-Z_0-9])(\s=\s'
29	n/a	r'(\'[^\']\'\|"[^"]"\|[-a-zA-Z0-9./,:;+%?!&$_#=~@]))?')
30	n/a
31	1	locatestarttagend = re.compile(r"""
32	n/a	<[a-zA-Z][-.a-zA-Z0-9:_]* # tag name
33	n/a	(?:\s+ # whitespace before attribute name
34	n/a	(?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name
35	n/a	(?:\s=\s # value indicator
36	n/a	(?:'[^']*' # LITA-enclosed value
37	n/a	\|\"[^\"]*\" # LIT-enclosed value
38	n/a	\|[^'\">\s]+ # bare value
39	n/a	)
40	n/a	)?
41	n/a	)
42	n/a	)*
43	n/a	\s* # trailing whitespace
44	1	""", re.VERBOSE)
45	1	endendtag = re.compile('>')
46	1	endtagfind = re.compile('</\s([a-zA-Z][-.a-zA-Z0-9:_])\s*>')
47	n/a
48	n/a
49	2	class HTMLParseError(Exception):
50	1	"""Exception raised for all parse errors."""
51	n/a
52	1	def __init__(self, msg, position=(None, None)):
53	16	assert msg
54	16	self.msg = msg
55	16	self.lineno = position[0]
56	16	self.offset = position[1]
57	n/a
58	1	def __str__(self):
59	0	result = self.msg
60	0	if self.lineno is not None:
61	0	result = result + ", at line %d" % self.lineno
62	0	if self.offset is not None:
63	0	result = result + ", column %d" % (self.offset + 1)
64	0	return result
65	n/a
66	n/a
67	2	class HTMLParser(markupbase.ParserBase):
68	n/a	"""Find tags and other markup and call handler functions.
69	n/a
70	n/a	Usage:
71	n/a	p = HTMLParser()
72	n/a	p.feed(data)
73	n/a	...
74	n/a	p.close()
75	n/a
76	n/a	Start tags are handled by calling self.handle_starttag() or
77	n/a	self.handle_startendtag(); end tags by self.handle_endtag(). The
78	n/a	data between tags is passed from the parser to the derived class
79	n/a	by calling self.handle_data() with the data as argument (the data
80	n/a	may be split up in arbitrary chunks). Entity references are
81	n/a	passed by calling self.handle_entityref() with the entity
82	n/a	reference as the argument. Numeric character references are
83	n/a	passed to self.handle_charref() with the string containing the
84	n/a	reference as the argument.
85	1	"""
86	n/a
87	1	CDATA_CONTENT_ELEMENTS = ("script", "style")
88	n/a
89	n/a
90	1	def __init__(self):
91	n/a	"""Initialize and reset this instance."""
92	67	self.reset()
93	n/a
94	1	def reset(self):
95	n/a	"""Reset this instance. Loses all unprocessed data."""
96	67	self.rawdata = ''
97	67	self.lasttag = '???'
98	67	self.interesting = interesting_normal
99	67	markupbase.ParserBase.reset(self)
100	n/a
101	1	def feed(self, data):
102	n/a	"""Feed data to the parser.
103	n/a
104	n/a	Call this as often as you want, with as little or as much text
105	n/a	as you want (may include '\n').
106	n/a	"""
107	1322	self.rawdata = self.rawdata + data
108	1322	self.goahead(0)
109	n/a
110	1	def close(self):
111	n/a	"""Handle any buffered data."""
112	60	self.goahead(1)
113	n/a
114	1	def error(self, message):
115	16	raise HTMLParseError(message, self.getpos())
116	n/a
117	1	__starttag_text = None
118	n/a
119	1	def get_starttag_text(self):
120	n/a	"""Return full source of start tag: '<...>'."""
121	1	return self.__starttag_text
122	n/a
123	1	def set_cdata_mode(self):
124	2	self.interesting = interesting_cdata
125	n/a
126	1	def clear_cdata_mode(self):
127	8	self.interesting = interesting_normal
128	n/a
129	n/a	# Internal -- handle data as far as reasonable. May leave state
130	n/a	# and data to be processed by a subsequent call. If 'end' is
131	n/a	# true, force handling all data as if followed by EOF marker.
132	1	def goahead(self, end):
133	1382	rawdata = self.rawdata
134	1382	i = 0
135	1382	n = len(rawdata)
136	1454	while i < n:
137	1336	match = self.interesting.search(rawdata, i) # < or &
138	1336	if match:
139	1175	j = match.start()
140	n/a	else:
141	161	j = n
142	1336	if i < j: self.handle_data(rawdata[i:j])
143	1336	i = self.updatepos(i, j)
144	1336	if i == n: break
145	1175	startswith = rawdata.startswith
146	1175	if startswith('<', i):
147	1131	if starttagopen.match(rawdata, i): # < + letter
148	446	k = self.parse_starttag(i)
149	685	elif startswith("</", i):
150	44	k = self.parse_endtag(i)
151	641	elif startswith("<!--", i):
152	100	k = self.parse_comment(i)
153	541	elif startswith("<?", i):
154	50	k = self.parse_pi(i)
155	491	elif startswith("<!", i):
156	442	k = self.parse_declaration(i)
157	49	elif (i + 1) < n:
158	3	self.handle_data("<")
159	3	k = i + 1
160	n/a	else:
161	46	break
162	1078	if k < 0:
163	1012	if end:
164	9	self.error("EOF in middle of construct")
165	1003	break
166	66	i = self.updatepos(i, k)
167	44	elif startswith("&#", i):
168	16	match = charref.match(rawdata, i)
169	16	if match:
170	2	name = match.group()[2:-1]
171	2	self.handle_charref(name)
172	2	k = match.end()
173	2	if not startswith(';', k-1):
174	0	k = k - 1
175	2	i = self.updatepos(i, k)
176	2	continue
177	n/a	else:
178	14	if ";" in rawdata[i:]: #bail by consuming &#
179	1	self.handle_data(rawdata[0:2])
180	1	i = self.updatepos(i, 2)
181	14	break
182	28	elif startswith('&', i):
183	28	match = entityref.match(rawdata, i)
184	28	if match:
185	2	name = match.group(1)
186	2	self.handle_entityref(name)
187	2	k = match.end()
188	2	if not startswith(';', k-1):
189	1	k = k - 1
190	2	i = self.updatepos(i, k)
191	2	continue
192	26	match = incomplete.match(rawdata, i)
193	26	if match:
194	n/a	# match.group() will contain at least 2 chars
195	15	if end and match.group() == rawdata[i:]:
196	0	self.error("EOF in middle of entity or char ref")
197	n/a	# incomplete
198	15	break
199	11	elif (i + 1) < n:
200	n/a	# not the end of the buffer, and can't be confused
201	n/a	# with some other construct
202	2	self.handle_data("&")
203	2	i = self.updatepos(i, i + 1)
204	n/a	else:
205	9	break
206	n/a	else:
207	0	assert 0, "interesting.search() lied"
208	n/a	# end while
209	1366	if end and i < n:
210	1	self.handle_data(rawdata[i:n])
211	1	i = self.updatepos(i, n)
212	1366	self.rawdata = rawdata[i:]
213	n/a
214	n/a	# Internal -- parse processing instr, return end or -1 if not terminated
215	1	def parse_pi(self, i):
216	50	rawdata = self.rawdata
217	50	assert rawdata[i:i+2] == '<?', 'unexpected call to parse_pi()'
218	50	match = piclose.search(rawdata, i+2) # >
219	50	if not match:
220	48	return -1
221	2	j = match.start()
222	2	self.handle_pi(rawdata[i+2: j])
223	2	j = match.end()
224	2	return j
225	n/a
226	n/a	# Internal -- handle starttag, return end or -1 if not terminated
227	1	def parse_starttag(self, i):
228	446	self.__starttag_text = None
229	446	endpos = self.check_for_whole_start_tag(i)
230	444	if endpos < 0:
231	407	return endpos
232	37	rawdata = self.rawdata
233	37	self.__starttag_text = rawdata[i:endpos]
234	n/a
235	n/a	# Now parse the data between i+1 and j into a tag and attrs
236	37	attrs = []
237	37	match = tagfind.match(rawdata, i+1)
238	37	assert match, 'unexpected call to parse_starttag()'
239	37	k = match.end()
240	37	self.lasttag = tag = rawdata[i+1:k].lower()
241	n/a
242	84	while k < endpos:
243	84	m = attrfind.match(rawdata, k)
244	84	if not m:
245	37	break
246	47	attrname, rest, attrvalue = m.group(1, 2, 3)
247	47	if not rest:
248	5	attrvalue = None
249	42	elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
250	16	attrvalue[:1] == '"' == attrvalue[-1:]:
251	33	attrvalue = attrvalue[1:-1]
252	33	attrvalue = self.unescape(attrvalue)
253	47	attrs.append((attrname.lower(), attrvalue))
254	47	k = m.end()
255	n/a
256	37	end = rawdata[k:endpos].strip()
257	37	if end not in (">", "/>"):
258	0	lineno, offset = self.getpos()
259	0	if "\n" in self.__starttag_text:
260	0	lineno = lineno + self.__starttag_text.count("\n")
261	0	offset = len(self.__starttag_text) \
262	0	- self.__starttag_text.rfind("\n")
263	n/a	else:
264	0	offset = offset + len(self.__starttag_text)
265	0	self.error("junk characters in start tag: %r"
266	0	% (rawdata[k:endpos][:20],))
267	37	if end.endswith('/>'):
268	n/a	# XHTML-style empty tag: <span attr="value" />
269	2	self.handle_startendtag(tag, attrs)
270	n/a	else:
271	35	self.handle_starttag(tag, attrs)
272	35	if tag in self.CDATA_CONTENT_ELEMENTS:
273	2	self.set_cdata_mode()
274	37	return endpos
275	n/a
276	n/a	# Internal -- check to see if we have a complete starttag; return end
277	n/a	# or -1 if incomplete.
278	1	def check_for_whole_start_tag(self, i):
279	446	rawdata = self.rawdata
280	446	m = locatestarttagend.match(rawdata, i)
281	446	if m:
282	446	j = m.end()
283	446	next = rawdata[j:j+1]
284	446	if next == ">":
285	35	return j + 1
286	411	if next == "/":
287	4	if rawdata.startswith("/>", j):
288	2	return j + 2
289	2	if rawdata.startswith("/", j):
290	n/a	# buffer boundary
291	2	return -1
292	n/a	# else bogus input
293	0	self.updatepos(i, j + 1)
294	0	self.error("malformed empty start tag")
295	407	if next == "":
296	n/a	# end of input
297	236	return -1
298	171	if next in ("abcdefghijklmnopqrstuvwxyz=/"
299	n/a	"ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
300	n/a	# end of input in or before attribute value, or we have the
301	n/a	# '/' from a '/>' ending
302	169	return -1
303	2	self.updatepos(i, j)
304	2	self.error("malformed start tag")
305	0	raise AssertionError("we should not get here!")
306	n/a
307	n/a	# Internal -- parse endtag, return end or -1 if incomplete
308	1	def parse_endtag(self, i):
309	44	rawdata = self.rawdata
310	44	assert rawdata[i:i+2] == "</", "unexpected call to parse_endtag"
311	44	match = endendtag.search(rawdata, i+1) # >
312	44	if not match:
313	33	return -1
314	11	j = match.end()
315	11	match = endtagfind.match(rawdata, i) # </ + tag + >
316	11	if not match:
317	3	self.error("bad end tag: %r" % (rawdata[i:j],))
318	8	tag = match.group(1)
319	8	self.handle_endtag(tag.lower())
320	8	self.clear_cdata_mode()
321	8	return j
322	n/a
323	n/a	# Overridable -- finish processing of start+end tag: <tag.../>
324	1	def handle_startendtag(self, tag, attrs):
325	0	self.handle_starttag(tag, attrs)
326	0	self.handle_endtag(tag)
327	n/a
328	n/a	# Overridable -- handle start tag
329	1	def handle_starttag(self, tag, attrs):
330	0	pass
331	n/a
332	n/a	# Overridable -- handle end tag
333	1	def handle_endtag(self, tag):
334	0	pass
335	n/a
336	n/a	# Overridable -- handle character reference
337	1	def handle_charref(self, name):
338	0	pass
339	n/a
340	n/a	# Overridable -- handle entity reference
341	1	def handle_entityref(self, name):
342	0	pass
343	n/a
344	n/a	# Overridable -- handle data
345	1	def handle_data(self, data):
346	0	pass
347	n/a
348	n/a	# Overridable -- handle comment
349	1	def handle_comment(self, data):
350	0	pass
351	n/a
352	n/a	# Overridable -- handle declaration
353	1	def handle_decl(self, decl):
354	0	pass
355	n/a
356	n/a	# Overridable -- handle processing instruction
357	1	def handle_pi(self, data):
358	0	pass
359	n/a
360	1	def unknown_decl(self, data):
361	0	self.error("unknown declaration: %r" % (data,))
362	n/a
363	n/a	# Internal -- helper to remove special character quoting
364	1	entitydefs = None
365	1	def unescape(self, s):
366	33	if '&' not in s:
367	31	return s
368	2	def replaceEntities(s):
369	9	s = s.groups()[0]
370	9	if s[0] == "#":
371	2	s = s[1:]
372	2	if s[0] in ['x','X']:
373	1	c = int(s[1:], 16)
374	n/a	else:
375	1	c = int(s)
376	2	return unichr(c)
377	n/a	else:
378	n/a	# Cannot use name2codepoint directly, because HTMLParser supports apos,
379	n/a	# which is not part of HTML 4
380	7	import htmlentitydefs
381	7	if HTMLParser.entitydefs is None:
382	1	entitydefs = HTMLParser.entitydefs = {'apos':u"'"}
383	253	for k, v in htmlentitydefs.name2codepoint.iteritems():
384	252	entitydefs[k] = unichr(v)
385	7	try:
386	7	return self.entitydefs[s]
387	0	except KeyError:
388	0	return '&'+s+';'
389	n/a
390	2	return re.sub(r"&(#?[xX]?(?:[0-9a-fA-F]+\|\w{1,8}));", replaceEntities, s)