Python code coverage for Lib/test/test_htmlparser.py

#	count	content
1	n/a	"""Tests for HTMLParser.py."""
2	n/a
3	n/a	import html.parser
4	n/a	import pprint
5	n/a	import unittest
6	n/a
7	n/a
8	n/a	class EventCollector(html.parser.HTMLParser):
9	n/a
10	n/a	def __init__(self, args, *kw):
11	n/a	self.events = []
12	n/a	self.append = self.events.append
13	n/a	html.parser.HTMLParser.__init__(self, args, *kw)
14	n/a
15	n/a	def get_events(self):
16	n/a	# Normalize the list of events so that buffer artefacts don't
17	n/a	# separate runs of contiguous characters.
18	n/a	L = []
19	n/a	prevtype = None
20	n/a	for event in self.events:
21	n/a	type = event[0]
22	n/a	if type == prevtype == "data":
23	n/a	L[-1] = ("data", L[-1][1] + event[1])
24	n/a	else:
25	n/a	L.append(event)
26	n/a	prevtype = type
27	n/a	self.events = L
28	n/a	return L
29	n/a
30	n/a	# structure markup
31	n/a
32	n/a	def handle_starttag(self, tag, attrs):
33	n/a	self.append(("starttag", tag, attrs))
34	n/a
35	n/a	def handle_startendtag(self, tag, attrs):
36	n/a	self.append(("startendtag", tag, attrs))
37	n/a
38	n/a	def handle_endtag(self, tag):
39	n/a	self.append(("endtag", tag))
40	n/a
41	n/a	# all other markup
42	n/a
43	n/a	def handle_comment(self, data):
44	n/a	self.append(("comment", data))
45	n/a
46	n/a	def handle_charref(self, data):
47	n/a	self.append(("charref", data))
48	n/a
49	n/a	def handle_data(self, data):
50	n/a	self.append(("data", data))
51	n/a
52	n/a	def handle_decl(self, data):
53	n/a	self.append(("decl", data))
54	n/a
55	n/a	def handle_entityref(self, data):
56	n/a	self.append(("entityref", data))
57	n/a
58	n/a	def handle_pi(self, data):
59	n/a	self.append(("pi", data))
60	n/a
61	n/a	def unknown_decl(self, decl):
62	n/a	self.append(("unknown decl", decl))
63	n/a
64	n/a
65	n/a	class EventCollectorExtra(EventCollector):
66	n/a
67	n/a	def handle_starttag(self, tag, attrs):
68	n/a	EventCollector.handle_starttag(self, tag, attrs)
69	n/a	self.append(("starttag_text", self.get_starttag_text()))
70	n/a
71	n/a
72	n/a	class EventCollectorCharrefs(EventCollector):
73	n/a
74	n/a	def handle_charref(self, data):
75	n/a	self.fail('This should never be called with convert_charrefs=True')
76	n/a
77	n/a	def handle_entityref(self, data):
78	n/a	self.fail('This should never be called with convert_charrefs=True')
79	n/a
80	n/a
81	n/a	class TestCaseBase(unittest.TestCase):
82	n/a
83	n/a	def get_collector(self):
84	n/a	return EventCollector(convert_charrefs=False)
85	n/a
86	n/a	def _run_check(self, source, expected_events, collector=None):
87	n/a	if collector is None:
88	n/a	collector = self.get_collector()
89	n/a	parser = collector
90	n/a	for s in source:
91	n/a	parser.feed(s)
92	n/a	parser.close()
93	n/a	events = parser.get_events()
94	n/a	if events != expected_events:
95	n/a	self.fail("received events did not match expected events" +
96	n/a	"\nSource:\n" + repr(source) +
97	n/a	"\nExpected:\n" + pprint.pformat(expected_events) +
98	n/a	"\nReceived:\n" + pprint.pformat(events))
99	n/a
100	n/a	def _run_check_extra(self, source, events):
101	n/a	self._run_check(source, events,
102	n/a	EventCollectorExtra(convert_charrefs=False))
103	n/a
104	n/a
105	n/a	class HTMLParserTestCase(TestCaseBase):
106	n/a
107	n/a	def test_processing_instruction_only(self):
108	n/a	self._run_check("<?processing instruction>", [
109	n/a	("pi", "processing instruction"),
110	n/a	])
111	n/a	self._run_check("<?processing instruction ?>", [
112	n/a	("pi", "processing instruction ?"),
113	n/a	])
114	n/a
115	n/a	def test_simple_html(self):
116	n/a	self._run_check("""
117	n/a	<!DOCTYPE html PUBLIC 'foo'>
118	n/a	<HTML>&entity;
119	n/a	<!--comment1a
120	n/a	-></foo><bar><<?pi?></foo<bar
121	n/a	comment1b-->
122	n/a	<Img sRc='Bar' isMAP>sample
123	n/a	text
124	n/a	“
125	n/a	<!--comment2a-- --comment2b-->
126	n/a	</Html>
127	n/a	""", [
128	n/a	("data", "\n"),
129	n/a	("decl", "DOCTYPE html PUBLIC 'foo'"),
130	n/a	("data", "\n"),
131	n/a	("starttag", "html", []),
132	n/a	("entityref", "entity"),
133	n/a	("charref", "32"),
134	n/a	("data", "\n"),
135	n/a	("comment", "comment1a\n-></foo><bar><<?pi?></foo<bar\ncomment1b"),
136	n/a	("data", "\n"),
137	n/a	("starttag", "img", [("src", "Bar"), ("ismap", None)]),
138	n/a	("data", "sample\ntext\n"),
139	n/a	("charref", "x201C"),
140	n/a	("data", "\n"),
141	n/a	("comment", "comment2a-- --comment2b"),
142	n/a	("data", "\n"),
143	n/a	("endtag", "html"),
144	n/a	("data", "\n"),
145	n/a	])
146	n/a
147	n/a	def test_malformatted_charref(self):
148	n/a	self._run_check("<p>&#bad;</p>", [
149	n/a	("starttag", "p", []),
150	n/a	("data", "&#bad;"),
151	n/a	("endtag", "p"),
152	n/a	])
153	n/a	# add the [] as a workaround to avoid buffering (see #20288)
154	n/a	self._run_check(["<div>&#bad;</div>"], [
155	n/a	("starttag", "div", []),
156	n/a	("data", "&#bad;"),
157	n/a	("endtag", "div"),
158	n/a	])
159	n/a
160	n/a	def test_unclosed_entityref(self):
161	n/a	self._run_check("&entityref foo", [
162	n/a	("entityref", "entityref"),
163	n/a	("data", " foo"),
164	n/a	])
165	n/a
166	n/a	def test_bad_nesting(self):
167	n/a	# Strangely, this is supposed to test that overlapping
168	n/a	# elements are allowed. HTMLParser is more geared toward
169	n/a	# lexing the input that parsing the structure.
170	n/a	self._run_check("<a><b></a></b>", [
171	n/a	("starttag", "a", []),
172	n/a	("starttag", "b", []),
173	n/a	("endtag", "a"),
174	n/a	("endtag", "b"),
175	n/a	])
176	n/a
177	n/a	def test_bare_ampersands(self):
178	n/a	self._run_check("this text & contains & ampersands &", [
179	n/a	("data", "this text & contains & ampersands &"),
180	n/a	])
181	n/a
182	n/a	def test_bare_pointy_brackets(self):
183	n/a	self._run_check("this < text > contains < bare>pointy< brackets", [
184	n/a	("data", "this < text > contains < bare>pointy< brackets"),
185	n/a	])
186	n/a
187	n/a	def test_starttag_end_boundary(self):
188	n/a	self._run_check("""<a b='<'>""", [("starttag", "a", [("b", "<")])])
189	n/a	self._run_check("""<a b='>'>""", [("starttag", "a", [("b", ">")])])
190	n/a
191	n/a	def test_buffer_artefacts(self):
192	n/a	output = [("starttag", "a", [("b", "<")])]
193	n/a	self._run_check(["<a b='<'>"], output)
194	n/a	self._run_check(["<a ", "b='<'>"], output)
195	n/a	self._run_check(["<a b", "='<'>"], output)
196	n/a	self._run_check(["<a b=", "'<'>"], output)
197	n/a	self._run_check(["<a b='<", "'>"], output)
198	n/a	self._run_check(["<a b='<'", ">"], output)
199	n/a
200	n/a	output = [("starttag", "a", [("b", ">")])]
201	n/a	self._run_check(["<a b='>'>"], output)
202	n/a	self._run_check(["<a ", "b='>'>"], output)
203	n/a	self._run_check(["<a b", "='>'>"], output)
204	n/a	self._run_check(["<a b=", "'>'>"], output)
205	n/a	self._run_check(["<a b='>", "'>"], output)
206	n/a	self._run_check(["<a b='>'", ">"], output)
207	n/a
208	n/a	output = [("comment", "abc")]
209	n/a	self._run_check(["", "<!--abc-->"], output)
210	n/a	self._run_check(["<", "!--abc-->"], output)
211	n/a	self._run_check(["<!", "--abc-->"], output)
212	n/a	self._run_check(["<!-", "-abc-->"], output)
213	n/a	self._run_check(["<!--", "abc-->"], output)
214	n/a	self._run_check(["<!--a", "bc-->"], output)
215	n/a	self._run_check(["<!--ab", "c-->"], output)
216	n/a	self._run_check(["<!--abc", "-->"], output)
217	n/a	self._run_check(["<!--abc-", "->"], output)
218	n/a	self._run_check(["<!--abc--", ">"], output)
219	n/a	self._run_check(["<!--abc-->", ""], output)
220	n/a
221	n/a	def test_valid_doctypes(self):
222	n/a	# from http://www.w3.org/QA/2002/04/valid-dtd-list.html
223	n/a	dtds = ['HTML', # HTML5 doctype
224	n/a	('HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" '
225	n/a	'"http://www.w3.org/TR/html4/strict.dtd"'),
226	n/a	('HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" '
227	n/a	'"http://www.w3.org/TR/html4/loose.dtd"'),
228	n/a	('html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" '
229	n/a	'"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"'),
230	n/a	('html PUBLIC "-//W3C//DTD XHTML 1.0 Frameset//EN" '
231	n/a	'"http://www.w3.org/TR/xhtml1/DTD/xhtml1-frameset.dtd"'),
232	n/a	('math PUBLIC "-//W3C//DTD MathML 2.0//EN" '
233	n/a	'"http://www.w3.org/Math/DTD/mathml2/mathml2.dtd"'),
234	n/a	('html PUBLIC "-//W3C//DTD '
235	n/a	'XHTML 1.1 plus MathML 2.0 plus SVG 1.1//EN" '
236	n/a	'"http://www.w3.org/2002/04/xhtml-math-svg/xhtml-math-svg.dtd"'),
237	n/a	('svg PUBLIC "-//W3C//DTD SVG 1.1//EN" '
238	n/a	'"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"'),
239	n/a	'html PUBLIC "-//IETF//DTD HTML 2.0//EN"',
240	n/a	'html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN"']
241	n/a	for dtd in dtds:
242	n/a	self._run_check("<!DOCTYPE %s>" % dtd,
243	n/a	[('decl', 'DOCTYPE ' + dtd)])
244	n/a
245	n/a	def test_startendtag(self):
246	n/a	self._run_check("<p/>", [
247	n/a	("startendtag", "p", []),
248	n/a	])
249	n/a	self._run_check("<p></p>", [
250	n/a	("starttag", "p", []),
251	n/a	("endtag", "p"),
252	n/a	])
253	n/a	self._run_check("<p><img src='foo' /></p>", [
254	n/a	("starttag", "p", []),
255	n/a	("startendtag", "img", [("src", "foo")]),
256	n/a	("endtag", "p"),
257	n/a	])
258	n/a
259	n/a	def test_get_starttag_text(self):
260	n/a	s = """<foo:bar \n one="1"\ttwo=2 >"""
261	n/a	self._run_check_extra(s, [
262	n/a	("starttag", "foo:bar", [("one", "1"), ("two", "2")]),
263	n/a	("starttag_text", s)])
264	n/a
265	n/a	def test_cdata_content(self):
266	n/a	contents = [
267	n/a	'<!-- not a comment --> &not-an-entity-ref;',
268	n/a	"<not a='start tag'>",
269	n/a	'<a href="" /> <p> <span></span>',
270	n/a	'foo = "</scr" + "ipt>";',
271	n/a	'foo = "</SCRIPT" + ">";',
272	n/a	'foo = <\n/script> ',
273	n/a	'<!-- document.write("</scr" + "ipt>"); -->',
274	n/a	('\n//<![CDATA[\n'
275	n/a	'document.write(\'<s\'+\'cript type="text/javascript" '
276	n/a	'src="http://www.example.org/r=\'+new '
277	n/a	'Date().getTime()+\'"><\\/s\'+\'cript>\');\n//]]>'),
278	n/a	'\n<!-- //\nvar foo = 3.14;\n// -->\n',
279	n/a	'foo = "</sty" + "le>";',
280	n/a	'<!-- \u2603 -->',
281	n/a	# these two should be invalid according to the HTML 5 spec,
282	n/a	# section 8.1.2.2
283	n/a	#'foo = </\nscript>',
284	n/a	#'foo = </ script>',
285	n/a	]
286	n/a	elements = ['script', 'style', 'SCRIPT', 'STYLE', 'Script', 'Style']
287	n/a	for content in contents:
288	n/a	for element in elements:
289	n/a	element_lower = element.lower()
290	n/a	s = '<{element}>{content}</{element}>'.format(element=element,
291	n/a	content=content)
292	n/a	self._run_check(s, [("starttag", element_lower, []),
293	n/a	("data", content),
294	n/a	("endtag", element_lower)])
295	n/a
296	n/a	def test_cdata_with_closing_tags(self):
297	n/a	# see issue #13358
298	n/a	# make sure that HTMLParser calls handle_data only once for each CDATA.
299	n/a	# The normal event collector normalizes the events in get_events,
300	n/a	# so we override it to return the original list of events.
301	n/a	class Collector(EventCollector):
302	n/a	def get_events(self):
303	n/a	return self.events
304	n/a
305	n/a	content = """<!-- not a comment --> &not-an-entity-ref;
306	n/a	<a href="" /> </p><p> <span></span></style>
307	n/a	'</script' + '>'"""
308	n/a	for element in [' script', 'script ', ' script ',
309	n/a	'\nscript', 'script\n', '\nscript\n']:
310	n/a	element_lower = element.lower().strip()
311	n/a	s = '<script>{content}</{element}>'.format(element=element,
312	n/a	content=content)
313	n/a	self._run_check(s, [("starttag", element_lower, []),
314	n/a	("data", content),
315	n/a	("endtag", element_lower)],
316	n/a	collector=Collector(convert_charrefs=False))
317	n/a
318	n/a	def test_comments(self):
319	n/a	html = ("<!-- I'm a valid comment -->"
320	n/a	'<!--me too!-->'
321	n/a	'<!------>'
322	n/a	'<!---->'
323	n/a	'<!----I have many hyphens---->'
324	n/a	'<!-- I have a > in the middle -->'
325	n/a	'<!-- and I have -- in the middle! -->')
326	n/a	expected = [('comment', " I'm a valid comment "),
327	n/a	('comment', 'me too!'),
328	n/a	('comment', '--'),
329	n/a	('comment', ''),
330	n/a	('comment', '--I have many hyphens--'),
331	n/a	('comment', ' I have a > in the middle '),
332	n/a	('comment', ' and I have -- in the middle! ')]
333	n/a	self._run_check(html, expected)
334	n/a
335	n/a	def test_condcoms(self):
336	n/a	html = ('<!--[if IE & !(lte IE 8)]>aren\'t<![endif]-->'
337	n/a	'<!--[if IE 8]>condcoms<![endif]-->'
338	n/a	'<!--[if lte IE 7]>pretty?<![endif]-->')
339	n/a	expected = [('comment', "[if IE & !(lte IE 8)]>aren't<![endif]"),
340	n/a	('comment', '[if IE 8]>condcoms<![endif]'),
341	n/a	('comment', '[if lte IE 7]>pretty?<![endif]')]
342	n/a	self._run_check(html, expected)
343	n/a
344	n/a	def test_convert_charrefs(self):
345	n/a	# default value for convert_charrefs is now True
346	n/a	collector = lambda: EventCollectorCharrefs()
347	n/a	self.assertTrue(collector().convert_charrefs)
348	n/a	charrefs = ['"', '"', '"', '&quot', '&#34', '&#x22']
349	n/a	# check charrefs in the middle of the text/attributes
350	n/a	expected = [('starttag', 'a', [('href', 'foo"zar')]),
351	n/a	('data', 'a"z'), ('endtag', 'a')]
352	n/a	for charref in charrefs:
353	n/a	self._run_check('<a href="foo{0}zar">a{0}z</a>'.format(charref),
354	n/a	expected, collector=collector())
355	n/a	# check charrefs at the beginning/end of the text/attributes
356	n/a	expected = [('data', '"'),
357	n/a	('starttag', 'a', [('x', '"'), ('y', '"X'), ('z', 'X"')]),
358	n/a	('data', '"'), ('endtag', 'a'), ('data', '"')]
359	n/a	for charref in charrefs:
360	n/a	self._run_check('{0}<a x="{0}" y="{0}X" z="X{0}">'
361	n/a	'{0}</a>{0}'.format(charref),
362	n/a	expected, collector=collector())
363	n/a	# check charrefs in <script>/<style> elements
364	n/a	for charref in charrefs:
365	n/a	text = 'X'.join([charref]*3)
366	n/a	expected = [('data', '"'),
367	n/a	('starttag', 'script', []), ('data', text),
368	n/a	('endtag', 'script'), ('data', '"'),
369	n/a	('starttag', 'style', []), ('data', text),
370	n/a	('endtag', 'style'), ('data', '"')]
371	n/a	self._run_check('{1}<script>{0}</script>{1}'
372	n/a	'<style>{0}</style>{1}'.format(text, charref),
373	n/a	expected, collector=collector())
374	n/a	# check truncated charrefs at the end of the file
375	n/a	html = '&quo &# &#x'
376	n/a	for x in range(1, len(html)):
377	n/a	self._run_check(html[:x], [('data', html[:x])],
378	n/a	collector=collector())
379	n/a	# check a string with no charrefs
380	n/a	self._run_check('no charrefs here', [('data', 'no charrefs here')],
381	n/a	collector=collector())
382	n/a
383	n/a	# the remaining tests were for the "tolerant" parser (which is now
384	n/a	# the default), and check various kind of broken markup
385	n/a	def test_tolerant_parsing(self):
386	n/a	self._run_check('<html <html>te>>xt&a<<bc</a></html>\n'
387	n/a	'<img src="URL><//img></html</html>', [
388	n/a	('starttag', 'html', [('<html', None)]),
389	n/a	('data', 'te>>xt'),
390	n/a	('entityref', 'a'),
391	n/a	('data', '<'),
392	n/a	('starttag', 'bc<', [('a', None)]),
393	n/a	('endtag', 'html'),
394	n/a	('data', '\n<img src="URL>'),
395	n/a	('comment', '/img'),
396	n/a	('endtag', 'html<')])
397	n/a
398	n/a	def test_starttag_junk_chars(self):
399	n/a	self._run_check("</>", [])
400	n/a	self._run_check("</$>", [('comment', '$')])
401	n/a	self._run_check("</", [('data', '</')])
402	n/a	self._run_check("</a", [('data', '</a')])
403	n/a	self._run_check("<a<a>", [('starttag', 'a<a', [])])
404	n/a	self._run_check("</a<a>", [('endtag', 'a<a')])
405	n/a	self._run_check("<!", [('data', '<!')])
406	n/a	self._run_check("<a", [('data', '<a')])
407	n/a	self._run_check("<a foo='bar'", [('data', "<a foo='bar'")])
408	n/a	self._run_check("<a foo='bar", [('data', "<a foo='bar")])
409	n/a	self._run_check("<a foo='>'", [('data', "<a foo='>'")])
410	n/a	self._run_check("<a foo='>", [('data', "<a foo='>")])
411	n/a	self._run_check("<a$>", [('starttag', 'a$', [])])
412	n/a	self._run_check("<a$b>", [('starttag', 'a$b', [])])
413	n/a	self._run_check("<a$b/>", [('startendtag', 'a$b', [])])
414	n/a	self._run_check("<a$b >", [('starttag', 'a$b', [])])
415	n/a	self._run_check("<a$b />", [('startendtag', 'a$b', [])])
416	n/a
417	n/a	def test_slashes_in_starttag(self):
418	n/a	self._run_check('<a foo="var"/>', [('startendtag', 'a', [('foo', 'var')])])
419	n/a	html = ('<img width=902 height=250px '
420	n/a	'src="/sites/default/files/images/homepage/foo.jpg" '
421	n/a	'/what am I doing here/ />')
422	n/a	expected = [(
423	n/a	'startendtag', 'img',
424	n/a	[('width', '902'), ('height', '250px'),
425	n/a	('src', '/sites/default/files/images/homepage/foo.jpg'),
426	n/a	('*what', None), ('am', None), ('i', None),
427	n/a	('doing', None), ('here*', None)]
428	n/a	)]
429	n/a	self._run_check(html, expected)
430	n/a	html = ('<a / /foo/ / /=/ / /bar/ / />'
431	n/a	'<a / /foo/ / /=/ / /bar/ / >')
432	n/a	expected = [
433	n/a	('startendtag', 'a', [('foo', None), ('=', None), ('bar', None)]),
434	n/a	('starttag', 'a', [('foo', None), ('=', None), ('bar', None)])
435	n/a	]
436	n/a	self._run_check(html, expected)
437	n/a	#see issue #14538
438	n/a	html = ('<meta><meta / ><meta // ><meta / / >'
439	n/a	'<meta/><meta /><meta //><meta//>')
440	n/a	expected = [
441	n/a	('starttag', 'meta', []), ('starttag', 'meta', []),
442	n/a	('starttag', 'meta', []), ('starttag', 'meta', []),
443	n/a	('startendtag', 'meta', []), ('startendtag', 'meta', []),
444	n/a	('startendtag', 'meta', []), ('startendtag', 'meta', []),
445	n/a	]
446	n/a	self._run_check(html, expected)
447	n/a
448	n/a	def test_declaration_junk_chars(self):
449	n/a	self._run_check("<!DOCTYPE foo $ >", [('decl', 'DOCTYPE foo $ ')])
450	n/a
451	n/a	def test_illegal_declarations(self):
452	n/a	self._run_check('<!spacer type="block" height="25">',
453	n/a	[('comment', 'spacer type="block" height="25"')])
454	n/a
455	n/a	def test_with_unquoted_attributes(self):
456	n/a	# see #12008
457	n/a	html = ("<html><body bgcolor=d0ca90 text='181008'>"
458	n/a	"<table cellspacing=0 cellpadding=1 width=100% ><tr>"
459	n/a	"<td align=left><font size=-1>"
460	n/a	"- <a href=/rabota/><span class=en> software-and-i</span></a>"
461	n/a	"- <a href='/1/'><span class=en> library</span></a></table>")
462	n/a	expected = [
463	n/a	('starttag', 'html', []),
464	n/a	('starttag', 'body', [('bgcolor', 'd0ca90'), ('text', '181008')]),
465	n/a	('starttag', 'table',
466	n/a	[('cellspacing', '0'), ('cellpadding', '1'), ('width', '100%')]),
467	n/a	('starttag', 'tr', []),
468	n/a	('starttag', 'td', [('align', 'left')]),
469	n/a	('starttag', 'font', [('size', '-1')]),
470	n/a	('data', '- '), ('starttag', 'a', [('href', '/rabota/')]),
471	n/a	('starttag', 'span', [('class', 'en')]), ('data', ' software-and-i'),
472	n/a	('endtag', 'span'), ('endtag', 'a'),
473	n/a	('data', '- '), ('starttag', 'a', [('href', '/1/')]),
474	n/a	('starttag', 'span', [('class', 'en')]), ('data', ' library'),
475	n/a	('endtag', 'span'), ('endtag', 'a'), ('endtag', 'table')
476	n/a	]
477	n/a	self._run_check(html, expected)
478	n/a
479	n/a	def test_comma_between_attributes(self):
480	n/a	self._run_check('<form action="/xxx.php?a=1&b=2&amp", '
481	n/a	'method="post">', [
482	n/a	('starttag', 'form',
483	n/a	[('action', '/xxx.php?a=1&b=2&'),
484	n/a	(',', None), ('method', 'post')])])
485	n/a
486	n/a	def test_weird_chars_in_unquoted_attribute_values(self):
487	n/a	self._run_check('<form action=bogus\|&#()value>', [
488	n/a	('starttag', 'form',
489	n/a	[('action', 'bogus\|&#()value')])])
490	n/a
491	n/a	def test_invalid_end_tags(self):
492	n/a	# A collection of broken end tags. <br> is used as separator.
493	n/a	# see http://www.w3.org/TR/html5/tokenization.html#end-tag-open-state
494	n/a	# and #13993
495	n/a	html = ('<br></label</p><br></div end tmAd-leaderBoard><br></<h4><br>'
496	n/a	'</li class="unit"><br></li\r\n\t\t\t\t\t\t</ul><br></><br>')
497	n/a	expected = [('starttag', 'br', []),
498	n/a	# < is part of the name, / is discarded, p is an attribute
499	n/a	('endtag', 'label<'),
500	n/a	('starttag', 'br', []),
501	n/a	# text and attributes are discarded
502	n/a	('endtag', 'div'),
503	n/a	('starttag', 'br', []),
504	n/a	# comment because the first char after </ is not a-zA-Z
505	n/a	('comment', '<h4'),
506	n/a	('starttag', 'br', []),
507	n/a	# attributes are discarded
508	n/a	('endtag', 'li'),
509	n/a	('starttag', 'br', []),
510	n/a	# everything till ul (included) is discarded
511	n/a	('endtag', 'li'),
512	n/a	('starttag', 'br', []),
513	n/a	# </> is ignored
514	n/a	('starttag', 'br', [])]
515	n/a	self._run_check(html, expected)
516	n/a
517	n/a	def test_broken_invalid_end_tag(self):
518	n/a	# This is technically wrong (the "> shouldn't be included in the 'data')
519	n/a	# but is probably not worth fixing it (in addition to all the cases of
520	n/a	# the previous test, it would require a full attribute parsing).
521	n/a	# see #13993
522	n/a	html = '<b>This</b attr=">"> confuses the parser'
523	n/a	expected = [('starttag', 'b', []),
524	n/a	('data', 'This'),
525	n/a	('endtag', 'b'),
526	n/a	('data', '"> confuses the parser')]
527	n/a	self._run_check(html, expected)
528	n/a
529	n/a	def test_correct_detection_of_start_tags(self):
530	n/a	# see #13273
531	n/a	html = ('<div style="" ><b>The <a href="some_url">rain</a> '
532	n/a	'<br /> in <span>Spain</span></b></div>')
533	n/a	expected = [
534	n/a	('starttag', 'div', [('style', '')]),
535	n/a	('starttag', 'b', []),
536	n/a	('data', 'The '),
537	n/a	('starttag', 'a', [('href', 'some_url')]),
538	n/a	('data', 'rain'),
539	n/a	('endtag', 'a'),
540	n/a	('data', ' '),
541	n/a	('startendtag', 'br', []),
542	n/a	('data', ' in '),
543	n/a	('starttag', 'span', []),
544	n/a	('data', 'Spain'),
545	n/a	('endtag', 'span'),
546	n/a	('endtag', 'b'),
547	n/a	('endtag', 'div')
548	n/a	]
549	n/a	self._run_check(html, expected)
550	n/a
551	n/a	html = '<div style="", foo = "bar" ><b>The <a href="some_url">rain</a>'
552	n/a	expected = [
553	n/a	('starttag', 'div', [('style', ''), (',', None), ('foo', 'bar')]),
554	n/a	('starttag', 'b', []),
555	n/a	('data', 'The '),
556	n/a	('starttag', 'a', [('href', 'some_url')]),
557	n/a	('data', 'rain'),
558	n/a	('endtag', 'a'),
559	n/a	]
560	n/a	self._run_check(html, expected)
561	n/a
562	n/a	def test_EOF_in_charref(self):
563	n/a	# see #17802
564	n/a	# This test checks that the UnboundLocalError reported in the issue
565	n/a	# is not raised, however I'm not sure the returned values are correct.
566	n/a	# Maybe HTMLParser should use self.unescape for these
567	n/a	data = [
568	n/a	('a&', [('data', 'a&')]),
569	n/a	('a&b', [('data', 'ab')]),
570	n/a	('a&b ', [('data', 'a'), ('entityref', 'b'), ('data', ' ')]),
571	n/a	('a&b;', [('data', 'a'), ('entityref', 'b')]),
572	n/a	]
573	n/a	for html, expected in data:
574	n/a	self._run_check(html, expected)
575	n/a
576	n/a	def test_unescape_method(self):
577	n/a	from html import unescape
578	n/a	p = self.get_collector()
579	n/a	with self.assertWarns(DeprecationWarning):
580	n/a	s = '"""&quot&#34&#x22&#bad;'
581	n/a	self.assertEqual(p.unescape(s), unescape(s))
582	n/a
583	n/a	def test_broken_comments(self):
584	n/a	html = ('<! not really a comment >'
585	n/a	'<! not a comment either -->'
586	n/a	'<! -- close enough -->'
587	n/a	'<!><!<-- this was an empty comment>'
588	n/a	'<!!! another bogus comment !!!>')
589	n/a	expected = [
590	n/a	('comment', ' not really a comment '),
591	n/a	('comment', ' not a comment either --'),
592	n/a	('comment', ' -- close enough --'),
593	n/a	('comment', ''),
594	n/a	('comment', '<-- this was an empty comment'),
595	n/a	('comment', '!! another bogus comment !!!'),
596	n/a	]
597	n/a	self._run_check(html, expected)
598	n/a
599	n/a	def test_broken_condcoms(self):
600	n/a	# these condcoms are missing the '--' after '<!' and before the '>'
601	n/a	html = ('<![if !(IE)]>broken condcom<![endif]>'
602	n/a	'<![if ! IE]><link href="favicon.tiff"/><![endif]>'
603	n/a	'<![if !IE 6]><img src="firefox.png" /><![endif]>'
604	n/a	'<![if !ie 6]><b>foo</b><![endif]>'
605	n/a	'<![if (!IE)\|(lt IE 9)]><img src="mammoth.bmp" /><![endif]>')
606	n/a	# According to the HTML5 specs sections "8.2.4.44 Bogus comment state"
607	n/a	# and "8.2.4.45 Markup declaration open state", comment tokens should
608	n/a	# be emitted instead of 'unknown decl', but calling unknown_decl
609	n/a	# provides more flexibility.
610	n/a	# See also Lib/_markupbase.py:parse_declaration
611	n/a	expected = [
612	n/a	('unknown decl', 'if !(IE)'),
613	n/a	('data', 'broken condcom'),
614	n/a	('unknown decl', 'endif'),
615	n/a	('unknown decl', 'if ! IE'),
616	n/a	('startendtag', 'link', [('href', 'favicon.tiff')]),
617	n/a	('unknown decl', 'endif'),
618	n/a	('unknown decl', 'if !IE 6'),
619	n/a	('startendtag', 'img', [('src', 'firefox.png')]),
620	n/a	('unknown decl', 'endif'),
621	n/a	('unknown decl', 'if !ie 6'),
622	n/a	('starttag', 'b', []),
623	n/a	('data', 'foo'),
624	n/a	('endtag', 'b'),
625	n/a	('unknown decl', 'endif'),
626	n/a	('unknown decl', 'if (!IE)\|(lt IE 9)'),
627	n/a	('startendtag', 'img', [('src', 'mammoth.bmp')]),
628	n/a	('unknown decl', 'endif')
629	n/a	]
630	n/a	self._run_check(html, expected)
631	n/a
632	n/a	def test_convert_charrefs_dropped_text(self):
633	n/a	# #23144: make sure that all the events are triggered when
634	n/a	# convert_charrefs is True, even if we don't call .close()
635	n/a	parser = EventCollector(convert_charrefs=True)
636	n/a	# before the fix, bar & baz was missing
637	n/a	parser.feed("foo <a>link</a> bar & baz")
638	n/a	self.assertEqual(
639	n/a	parser.get_events(),
640	n/a	[('data', 'foo '), ('starttag', 'a', []), ('data', 'link'),
641	n/a	('endtag', 'a'), ('data', ' bar & baz')]
642	n/a	)
643	n/a
644	n/a
645	n/a	class AttributesTestCase(TestCaseBase):
646	n/a
647	n/a	def test_attr_syntax(self):
648	n/a	output = [
649	n/a	("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", None)])
650	n/a	]
651	n/a	self._run_check("""<a b='v' c="v" d=v e>""", output)
652	n/a	self._run_check("""<a b = 'v' c = "v" d = v e>""", output)
653	n/a	self._run_check("""<a\nb\n=\n'v'\nc\n=\n"v"\nd\n=\nv\ne>""", output)
654	n/a	self._run_check("""<a\tb\t=\t'v'\tc\t=\t"v"\td\t=\tv\te>""", output)
655	n/a
656	n/a	def test_attr_values(self):
657	n/a	self._run_check("""<a b='xxx\n\txxx' c="yyy\t\nyyy" d='\txyz\n'>""",
658	n/a	[("starttag", "a", [("b", "xxx\n\txxx"),
659	n/a	("c", "yyy\t\nyyy"),
660	n/a	("d", "\txyz\n")])])
661	n/a	self._run_check("""<a b='' c="">""",
662	n/a	[("starttag", "a", [("b", ""), ("c", "")])])
663	n/a	# Regression test for SF patch #669683.
664	n/a	self._run_check("<e a=rgb(1,2,3)>",
665	n/a	[("starttag", "e", [("a", "rgb(1,2,3)")])])
666	n/a	# Regression test for SF bug #921657.
667	n/a	self._run_check(
668	n/a	"<a href=mailto:xyz@example.com>",
669	n/a	[("starttag", "a", [("href", "mailto:xyz@example.com")])])
670	n/a
671	n/a	def test_attr_nonascii(self):
672	n/a	# see issue 7311
673	n/a	self._run_check(
674	n/a	"<img src=/foo/bar.png alt=\u4e2d\u6587>",
675	n/a	[("starttag", "img", [("src", "/foo/bar.png"),
676	n/a	("alt", "\u4e2d\u6587")])])
677	n/a	self._run_check(
678	n/a	"<a title='\u30c6\u30b9\u30c8' href='\u30c6\u30b9\u30c8.html'>",
679	n/a	[("starttag", "a", [("title", "\u30c6\u30b9\u30c8"),
680	n/a	("href", "\u30c6\u30b9\u30c8.html")])])
681	n/a	self._run_check(
682	n/a	'<a title="\u30c6\u30b9\u30c8" href="\u30c6\u30b9\u30c8.html">',
683	n/a	[("starttag", "a", [("title", "\u30c6\u30b9\u30c8"),
684	n/a	("href", "\u30c6\u30b9\u30c8.html")])])
685	n/a
686	n/a	def test_attr_entity_replacement(self):
687	n/a	self._run_check(
688	n/a	"<a b='&><"''>",
689	n/a	[("starttag", "a", [("b", "&><\"'")])])
690	n/a
691	n/a	def test_attr_funky_names(self):
692	n/a	self._run_check(
693	n/a	"<a a.b='v' c:d=v e-f=v>",
694	n/a	[("starttag", "a", [("a.b", "v"), ("c:d", "v"), ("e-f", "v")])])
695	n/a
696	n/a	def test_entityrefs_in_attributes(self):
697	n/a	self._run_check(
698	n/a	"<html foo='€&aa&unsupported;'>",
699	n/a	[("starttag", "html", [("foo", "\u20AC&aa&unsupported;")])])
700	n/a
701	n/a
702	n/a	def test_attr_funky_names2(self):
703	n/a	self._run_check(
704	n/a	r"<a $><b $=%><c \=/>",
705	n/a	[("starttag", "a", [("$", None)]),
706	n/a	("starttag", "b", [("$", "%")]),
707	n/a	("starttag", "c", [("\\", "/")])])
708	n/a
709	n/a	def test_entities_in_attribute_value(self):
710	n/a	# see #1200313
711	n/a	for entity in ['&', '&', '&', '&']:
712	n/a	self._run_check('<a href="%s">' % entity,
713	n/a	[("starttag", "a", [("href", "&")])])
714	n/a	self._run_check("<a href='%s'>" % entity,
715	n/a	[("starttag", "a", [("href", "&")])])
716	n/a	self._run_check("<a href=%s>" % entity,
717	n/a	[("starttag", "a", [("href", "&")])])
718	n/a
719	n/a	def test_malformed_attributes(self):
720	n/a	# see #13357
721	n/a	html = (
722	n/a	"<a href=test'style='color:red;bad1'>test - bad1</a>"
723	n/a	"<a href=test'+style='color:red;ba2'>test - bad2</a>"
724	n/a	"<a href=test' style='color:red;bad3'>test - bad3</a>"
725	n/a	"<a href = test' style='color:red;bad4' >test - bad4</a>"
726	n/a	)
727	n/a	expected = [
728	n/a	('starttag', 'a', [('href', "test'style='color:red;bad1'")]),
729	n/a	('data', 'test - bad1'), ('endtag', 'a'),
730	n/a	('starttag', 'a', [('href', "test'+style='color:red;ba2'")]),
731	n/a	('data', 'test - bad2'), ('endtag', 'a'),
732	n/a	('starttag', 'a', [('href', "test'\xa0style='color:red;bad3'")]),
733	n/a	('data', 'test - bad3'), ('endtag', 'a'),
734	n/a	('starttag', 'a', [('href', "test'\xa0style='color:red;bad4'")]),
735	n/a	('data', 'test - bad4'), ('endtag', 'a')
736	n/a	]
737	n/a	self._run_check(html, expected)
738	n/a
739	n/a	def test_malformed_adjacent_attributes(self):
740	n/a	# see #12629
741	n/a	self._run_check('<x><y z=""o"" /></x>',
742	n/a	[('starttag', 'x', []),
743	n/a	('startendtag', 'y', [('z', ''), ('o""', None)]),
744	n/a	('endtag', 'x')])
745	n/a	self._run_check('<x><y z="""" /></x>',
746	n/a	[('starttag', 'x', []),
747	n/a	('startendtag', 'y', [('z', ''), ('""', None)]),
748	n/a	('endtag', 'x')])
749	n/a
750	n/a	# see #755670 for the following 3 tests
751	n/a	def test_adjacent_attributes(self):
752	n/a	self._run_check('<a width="100%"cellspacing=0>',
753	n/a	[("starttag", "a",
754	n/a	[("width", "100%"), ("cellspacing","0")])])
755	n/a
756	n/a	self._run_check('<a id="foo"class="bar">',
757	n/a	[("starttag", "a",
758	n/a	[("id", "foo"), ("class","bar")])])
759	n/a
760	n/a	def test_missing_attribute_value(self):
761	n/a	self._run_check('<a v=>',
762	n/a	[("starttag", "a", [("v", "")])])
763	n/a
764	n/a	def test_javascript_attribute_value(self):
765	n/a	self._run_check("<a href=javascript:popup('/popup/help.html')>",
766	n/a	[("starttag", "a",
767	n/a	[("href", "javascript:popup('/popup/help.html')")])])
768	n/a
769	n/a	def test_end_tag_in_attribute_value(self):
770	n/a	# see #1745761
771	n/a	self._run_check("<a href='http://www.example.org/\">;'>spam</a>",
772	n/a	[("starttag", "a",
773	n/a	[("href", "http://www.example.org/\">;")]),
774	n/a	("data", "spam"), ("endtag", "a")])
775	n/a
776	n/a
777	n/a	if __name__ == "__main__":
778	n/a	unittest.main()