Python code coverage for Lib/test/test_codecs.py

#	count	content
1	n/a	import codecs
2	n/a	import contextlib
3	n/a	import io
4	n/a	import locale
5	n/a	import sys
6	n/a	import unittest
7	n/a	import encodings
8	n/a
9	n/a	from test import support
10	n/a
11	n/a	try:
12	n/a	import ctypes
13	n/a	except ImportError:
14	n/a	ctypes = None
15	n/a	SIZEOF_WCHAR_T = -1
16	n/a	else:
17	n/a	SIZEOF_WCHAR_T = ctypes.sizeof(ctypes.c_wchar)
18	n/a
19	n/a	def coding_checker(self, coder):
20	n/a	def check(input, expect):
21	n/a	self.assertEqual(coder(input), (expect, len(input)))
22	n/a	return check
23	n/a
24	n/a
25	n/a	class Queue(object):
26	n/a	"""
27	n/a	queue: write bytes at one end, read bytes from the other end
28	n/a	"""
29	n/a	def __init__(self, buffer):
30	n/a	self._buffer = buffer
31	n/a
32	n/a	def write(self, chars):
33	n/a	self._buffer += chars
34	n/a
35	n/a	def read(self, size=-1):
36	n/a	if size<0:
37	n/a	s = self._buffer
38	n/a	self._buffer = self._buffer[:0] # make empty
39	n/a	return s
40	n/a	else:
41	n/a	s = self._buffer[:size]
42	n/a	self._buffer = self._buffer[size:]
43	n/a	return s
44	n/a
45	n/a
46	n/a	class MixInCheckStateHandling:
47	n/a	def check_state_handling_decode(self, encoding, u, s):
48	n/a	for i in range(len(s)+1):
49	n/a	d = codecs.getincrementaldecoder(encoding)()
50	n/a	part1 = d.decode(s[:i])
51	n/a	state = d.getstate()
52	n/a	self.assertIsInstance(state[1], int)
53	n/a	# Check that the condition stated in the documentation for
54	n/a	# IncrementalDecoder.getstate() holds
55	n/a	if not state[1]:
56	n/a	# reset decoder to the default state without anything buffered
57	n/a	d.setstate((state[0][:0], 0))
58	n/a	# Feeding the previous input may not produce any output
59	n/a	self.assertTrue(not d.decode(state[0]))
60	n/a	# The decoder must return to the same state
61	n/a	self.assertEqual(state, d.getstate())
62	n/a	# Create a new decoder and set it to the state
63	n/a	# we extracted from the old one
64	n/a	d = codecs.getincrementaldecoder(encoding)()
65	n/a	d.setstate(state)
66	n/a	part2 = d.decode(s[i:], True)
67	n/a	self.assertEqual(u, part1+part2)
68	n/a
69	n/a	def check_state_handling_encode(self, encoding, u, s):
70	n/a	for i in range(len(u)+1):
71	n/a	d = codecs.getincrementalencoder(encoding)()
72	n/a	part1 = d.encode(u[:i])
73	n/a	state = d.getstate()
74	n/a	d = codecs.getincrementalencoder(encoding)()
75	n/a	d.setstate(state)
76	n/a	part2 = d.encode(u[i:], True)
77	n/a	self.assertEqual(s, part1+part2)
78	n/a
79	n/a
80	n/a	class ReadTest(MixInCheckStateHandling):
81	n/a	def check_partial(self, input, partialresults):
82	n/a	# get a StreamReader for the encoding and feed the bytestring version
83	n/a	# of input to the reader byte by byte. Read everything available from
84	n/a	# the StreamReader and check that the results equal the appropriate
85	n/a	# entries from partialresults.
86	n/a	q = Queue(b"")
87	n/a	r = codecs.getreader(self.encoding)(q)
88	n/a	result = ""
89	n/a	for (c, partialresult) in zip(input.encode(self.encoding), partialresults):
90	n/a	q.write(bytes([c]))
91	n/a	result += r.read()
92	n/a	self.assertEqual(result, partialresult)
93	n/a	# check that there's nothing left in the buffers
94	n/a	self.assertEqual(r.read(), "")
95	n/a	self.assertEqual(r.bytebuffer, b"")
96	n/a
97	n/a	# do the check again, this time using an incremental decoder
98	n/a	d = codecs.getincrementaldecoder(self.encoding)()
99	n/a	result = ""
100	n/a	for (c, partialresult) in zip(input.encode(self.encoding), partialresults):
101	n/a	result += d.decode(bytes([c]))
102	n/a	self.assertEqual(result, partialresult)
103	n/a	# check that there's nothing left in the buffers
104	n/a	self.assertEqual(d.decode(b"", True), "")
105	n/a	self.assertEqual(d.buffer, b"")
106	n/a
107	n/a	# Check whether the reset method works properly
108	n/a	d.reset()
109	n/a	result = ""
110	n/a	for (c, partialresult) in zip(input.encode(self.encoding), partialresults):
111	n/a	result += d.decode(bytes([c]))
112	n/a	self.assertEqual(result, partialresult)
113	n/a	# check that there's nothing left in the buffers
114	n/a	self.assertEqual(d.decode(b"", True), "")
115	n/a	self.assertEqual(d.buffer, b"")
116	n/a
117	n/a	# check iterdecode()
118	n/a	encoded = input.encode(self.encoding)
119	n/a	self.assertEqual(
120	n/a	input,
121	n/a	"".join(codecs.iterdecode([bytes([c]) for c in encoded], self.encoding))
122	n/a	)
123	n/a
124	n/a	def test_readline(self):
125	n/a	def getreader(input):
126	n/a	stream = io.BytesIO(input.encode(self.encoding))
127	n/a	return codecs.getreader(self.encoding)(stream)
128	n/a
129	n/a	def readalllines(input, keepends=True, size=None):
130	n/a	reader = getreader(input)
131	n/a	lines = []
132	n/a	while True:
133	n/a	line = reader.readline(size=size, keepends=keepends)
134	n/a	if not line:
135	n/a	break
136	n/a	lines.append(line)
137	n/a	return "\|".join(lines)
138	n/a
139	n/a	s = "foo\nbar\r\nbaz\rspam\u2028eggs"
140	n/a	sexpected = "foo\n\|bar\r\n\|baz\r\|spam\u2028\|eggs"
141	n/a	sexpectednoends = "foo\|bar\|baz\|spam\|eggs"
142	n/a	self.assertEqual(readalllines(s, True), sexpected)
143	n/a	self.assertEqual(readalllines(s, False), sexpectednoends)
144	n/a	self.assertEqual(readalllines(s, True, 10), sexpected)
145	n/a	self.assertEqual(readalllines(s, False, 10), sexpectednoends)
146	n/a
147	n/a	lineends = ("\n", "\r\n", "\r", "\u2028")
148	n/a	# Test long lines (multiple calls to read() in readline())
149	n/a	vw = []
150	n/a	vwo = []
151	n/a	for (i, lineend) in enumerate(lineends):
152	n/a	vw.append((i200+200)"\u3042" + lineend)
153	n/a	vwo.append((i200+200)"\u3042")
154	n/a	self.assertEqual(readalllines("".join(vw), True), "\|".join(vw))
155	n/a	self.assertEqual(readalllines("".join(vw), False), "\|".join(vwo))
156	n/a
157	n/a	# Test lines where the first read might end with \r, so the
158	n/a	# reader has to look ahead whether this is a lone \r or a \r\n
159	n/a	for size in range(80):
160	n/a	for lineend in lineends:
161	n/a	s = 10(size"a" + lineend + "xxx\n")
162	n/a	reader = getreader(s)
163	n/a	for i in range(10):
164	n/a	self.assertEqual(
165	n/a	reader.readline(keepends=True),
166	n/a	size*"a" + lineend,
167	n/a	)
168	n/a	self.assertEqual(
169	n/a	reader.readline(keepends=True),
170	n/a	"xxx\n",
171	n/a	)
172	n/a	reader = getreader(s)
173	n/a	for i in range(10):
174	n/a	self.assertEqual(
175	n/a	reader.readline(keepends=False),
176	n/a	size*"a",
177	n/a	)
178	n/a	self.assertEqual(
179	n/a	reader.readline(keepends=False),
180	n/a	"xxx",
181	n/a	)
182	n/a
183	n/a	def test_mixed_readline_and_read(self):
184	n/a	lines = ["Humpty Dumpty sat on a wall,\n",
185	n/a	"Humpty Dumpty had a great fall.\r\n",
186	n/a	"All the king's horses and all the king's men\r",
187	n/a	"Couldn't put Humpty together again."]
188	n/a	data = ''.join(lines)
189	n/a	def getreader():
190	n/a	stream = io.BytesIO(data.encode(self.encoding))
191	n/a	return codecs.getreader(self.encoding)(stream)
192	n/a
193	n/a	# Issue #8260: Test readline() followed by read()
194	n/a	f = getreader()
195	n/a	self.assertEqual(f.readline(), lines[0])
196	n/a	self.assertEqual(f.read(), ''.join(lines[1:]))
197	n/a	self.assertEqual(f.read(), '')
198	n/a
199	n/a	# Issue #16636: Test readline() followed by readlines()
200	n/a	f = getreader()
201	n/a	self.assertEqual(f.readline(), lines[0])
202	n/a	self.assertEqual(f.readlines(), lines[1:])
203	n/a	self.assertEqual(f.read(), '')
204	n/a
205	n/a	# Test read() followed by read()
206	n/a	f = getreader()
207	n/a	self.assertEqual(f.read(size=40, chars=5), data[:5])
208	n/a	self.assertEqual(f.read(), data[5:])
209	n/a	self.assertEqual(f.read(), '')
210	n/a
211	n/a	# Issue #12446: Test read() followed by readlines()
212	n/a	f = getreader()
213	n/a	self.assertEqual(f.read(size=40, chars=5), data[:5])
214	n/a	self.assertEqual(f.readlines(), [lines[0][5:]] + lines[1:])
215	n/a	self.assertEqual(f.read(), '')
216	n/a
217	n/a	def test_bug1175396(self):
218	n/a	s = [
219	n/a	'<%!--===================================================\r\n',
220	n/a	' BLOG index page: show recent articles,\r\n',
221	n/a	' today\'s articles, or articles of a specific date.\r\n',
222	n/a	'========================================================--%>\r\n',
223	n/a	'<%@inputencoding="ISO-8859-1"%>\r\n',
224	n/a	'<%@pagetemplate=TEMPLATE.y%>\r\n',
225	n/a	'<%@import=import frog.util, frog%>\r\n',
226	n/a	'<%@import=import frog.objects%>\r\n',
227	n/a	'<%@import=from frog.storageerrors import StorageError%>\r\n',
228	n/a	'<%\r\n',
229	n/a	'\r\n',
230	n/a	'import logging\r\n',
231	n/a	'log=logging.getLogger("Snakelets.logger")\r\n',
232	n/a	'\r\n',
233	n/a	'\r\n',
234	n/a	'user=self.SessionCtx.user\r\n',
235	n/a	'storageEngine=self.SessionCtx.storageEngine\r\n',
236	n/a	'\r\n',
237	n/a	'\r\n',
238	n/a	'def readArticlesFromDate(date, count=None):\r\n',
239	n/a	' entryids=storageEngine.listBlogEntries(date)\r\n',
240	n/a	' entryids.reverse() # descending\r\n',
241	n/a	' if count:\r\n',
242	n/a	' entryids=entryids[:count]\r\n',
243	n/a	' try:\r\n',
244	n/a	' return [ frog.objects.BlogEntry.load(storageEngine, date, Id) for Id in entryids ]\r\n',
245	n/a	' except StorageError,x:\r\n',
246	n/a	' log.error("Error loading articles: "+str(x))\r\n',
247	n/a	' self.abort("cannot load articles")\r\n',
248	n/a	'\r\n',
249	n/a	'showdate=None\r\n',
250	n/a	'\r\n',
251	n/a	'arg=self.Request.getArg()\r\n',
252	n/a	'if arg=="today":\r\n',
253	n/a	' #-------------------- TODAY\'S ARTICLES\r\n',
254	n/a	' self.write("<h2>Today\'s articles</h2>")\r\n',
255	n/a	' showdate = frog.util.isodatestr() \r\n',
256	n/a	' entries = readArticlesFromDate(showdate)\r\n',
257	n/a	'elif arg=="active":\r\n',
258	n/a	' #-------------------- ACTIVE ARTICLES redirect\r\n',
259	n/a	' self.Yredirect("active.y")\r\n',
260	n/a	'elif arg=="login":\r\n',
261	n/a	' #-------------------- LOGIN PAGE redirect\r\n',
262	n/a	' self.Yredirect("login.y")\r\n',
263	n/a	'elif arg=="date":\r\n',
264	n/a	' #-------------------- ARTICLES OF A SPECIFIC DATE\r\n',
265	n/a	' showdate = self.Request.getParameter("date")\r\n',
266	n/a	' self.write("<h2>Articles written on %s</h2>"% frog.util.mediumdatestr(showdate))\r\n',
267	n/a	' entries = readArticlesFromDate(showdate)\r\n',
268	n/a	'else:\r\n',
269	n/a	' #-------------------- RECENT ARTICLES\r\n',
270	n/a	' self.write("<h2>Recent articles</h2>")\r\n',
271	n/a	' dates=storageEngine.listBlogEntryDates()\r\n',
272	n/a	' if dates:\r\n',
273	n/a	' entries=[]\r\n',
274	n/a	' SHOWAMOUNT=10\r\n',
275	n/a	' for showdate in dates:\r\n',
276	n/a	' entries.extend( readArticlesFromDate(showdate, SHOWAMOUNT-len(entries)) )\r\n',
277	n/a	' if len(entries)>=SHOWAMOUNT:\r\n',
278	n/a	' break\r\n',
279	n/a	' \r\n',
280	n/a	]
281	n/a	stream = io.BytesIO("".join(s).encode(self.encoding))
282	n/a	reader = codecs.getreader(self.encoding)(stream)
283	n/a	for (i, line) in enumerate(reader):
284	n/a	self.assertEqual(line, s[i])
285	n/a
286	n/a	def test_readlinequeue(self):
287	n/a	q = Queue(b"")
288	n/a	writer = codecs.getwriter(self.encoding)(q)
289	n/a	reader = codecs.getreader(self.encoding)(q)
290	n/a
291	n/a	# No lineends
292	n/a	writer.write("foo\r")
293	n/a	self.assertEqual(reader.readline(keepends=False), "foo")
294	n/a	writer.write("\nbar\r")
295	n/a	self.assertEqual(reader.readline(keepends=False), "")
296	n/a	self.assertEqual(reader.readline(keepends=False), "bar")
297	n/a	writer.write("baz")
298	n/a	self.assertEqual(reader.readline(keepends=False), "baz")
299	n/a	self.assertEqual(reader.readline(keepends=False), "")
300	n/a
301	n/a	# Lineends
302	n/a	writer.write("foo\r")
303	n/a	self.assertEqual(reader.readline(keepends=True), "foo\r")
304	n/a	writer.write("\nbar\r")
305	n/a	self.assertEqual(reader.readline(keepends=True), "\n")
306	n/a	self.assertEqual(reader.readline(keepends=True), "bar\r")
307	n/a	writer.write("baz")
308	n/a	self.assertEqual(reader.readline(keepends=True), "baz")
309	n/a	self.assertEqual(reader.readline(keepends=True), "")
310	n/a	writer.write("foo\r\n")
311	n/a	self.assertEqual(reader.readline(keepends=True), "foo\r\n")
312	n/a
313	n/a	def test_bug1098990_a(self):
314	n/a	s1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\r\n"
315	n/a	s2 = "offending line: ladfj askldfj klasdj fskla dfzaskdj fasklfj laskd fjasklfzzzzaa%whereisthis!!!\r\n"
316	n/a	s3 = "next line.\r\n"
317	n/a
318	n/a	s = (s1+s2+s3).encode(self.encoding)
319	n/a	stream = io.BytesIO(s)
320	n/a	reader = codecs.getreader(self.encoding)(stream)
321	n/a	self.assertEqual(reader.readline(), s1)
322	n/a	self.assertEqual(reader.readline(), s2)
323	n/a	self.assertEqual(reader.readline(), s3)
324	n/a	self.assertEqual(reader.readline(), "")
325	n/a
326	n/a	def test_bug1098990_b(self):
327	n/a	s1 = "aaaaaaaaaaaaaaaaaaaaaaaa\r\n"
328	n/a	s2 = "bbbbbbbbbbbbbbbbbbbbbbbb\r\n"
329	n/a	s3 = "stillokay:bbbbxx\r\n"
330	n/a	s4 = "broken!!!!badbad\r\n"
331	n/a	s5 = "againokay.\r\n"
332	n/a
333	n/a	s = (s1+s2+s3+s4+s5).encode(self.encoding)
334	n/a	stream = io.BytesIO(s)
335	n/a	reader = codecs.getreader(self.encoding)(stream)
336	n/a	self.assertEqual(reader.readline(), s1)
337	n/a	self.assertEqual(reader.readline(), s2)
338	n/a	self.assertEqual(reader.readline(), s3)
339	n/a	self.assertEqual(reader.readline(), s4)
340	n/a	self.assertEqual(reader.readline(), s5)
341	n/a	self.assertEqual(reader.readline(), "")
342	n/a
343	n/a	ill_formed_sequence_replace = "\ufffd"
344	n/a
345	n/a	def test_lone_surrogates(self):
346	n/a	self.assertRaises(UnicodeEncodeError, "\ud800".encode, self.encoding)
347	n/a	self.assertEqual("[\uDC80]".encode(self.encoding, "backslashreplace"),
348	n/a	"[\\udc80]".encode(self.encoding))
349	n/a	self.assertEqual("[\uDC80]".encode(self.encoding, "namereplace"),
350	n/a	"[\\udc80]".encode(self.encoding))
351	n/a	self.assertEqual("[\uDC80]".encode(self.encoding, "xmlcharrefreplace"),
352	n/a	"[&#56448;]".encode(self.encoding))
353	n/a	self.assertEqual("[\uDC80]".encode(self.encoding, "ignore"),
354	n/a	"[]".encode(self.encoding))
355	n/a	self.assertEqual("[\uDC80]".encode(self.encoding, "replace"),
356	n/a	"[?]".encode(self.encoding))
357	n/a
358	n/a	# sequential surrogate characters
359	n/a	self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "ignore"),
360	n/a	"[]".encode(self.encoding))
361	n/a	self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "replace"),
362	n/a	"[??]".encode(self.encoding))
363	n/a
364	n/a	bom = "".encode(self.encoding)
365	n/a	for before, after in [("\U00010fff", "A"), ("[", "]"),
366	n/a	("A", "\U00010fff")]:
367	n/a	before_sequence = before.encode(self.encoding)[len(bom):]
368	n/a	after_sequence = after.encode(self.encoding)[len(bom):]
369	n/a	test_string = before + "\uDC80" + after
370	n/a	test_sequence = (bom + before_sequence +
371	n/a	self.ill_formed_sequence + after_sequence)
372	n/a	self.assertRaises(UnicodeDecodeError, test_sequence.decode,
373	n/a	self.encoding)
374	n/a	self.assertEqual(test_string.encode(self.encoding,
375	n/a	"surrogatepass"),
376	n/a	test_sequence)
377	n/a	self.assertEqual(test_sequence.decode(self.encoding,
378	n/a	"surrogatepass"),
379	n/a	test_string)
380	n/a	self.assertEqual(test_sequence.decode(self.encoding, "ignore"),
381	n/a	before + after)
382	n/a	self.assertEqual(test_sequence.decode(self.encoding, "replace"),
383	n/a	before + self.ill_formed_sequence_replace + after)
384	n/a	backslashreplace = ''.join('\\x%02x' % b
385	n/a	for b in self.ill_formed_sequence)
386	n/a	self.assertEqual(test_sequence.decode(self.encoding, "backslashreplace"),
387	n/a	before + backslashreplace + after)
388	n/a
389	n/a
390	n/a	class UTF32Test(ReadTest, unittest.TestCase):
391	n/a	encoding = "utf-32"
392	n/a	if sys.byteorder == 'little':
393	n/a	ill_formed_sequence = b"\x80\xdc\x00\x00"
394	n/a	else:
395	n/a	ill_formed_sequence = b"\x00\x00\xdc\x80"
396	n/a
397	n/a	spamle = (b'\xff\xfe\x00\x00'
398	n/a	b's\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m\x00\x00\x00'
399	n/a	b's\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m\x00\x00\x00')
400	n/a	spambe = (b'\x00\x00\xfe\xff'
401	n/a	b'\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m'
402	n/a	b'\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m')
403	n/a
404	n/a	def test_only_one_bom(self):
405	n/a	_,_,reader,writer = codecs.lookup(self.encoding)
406	n/a	# encode some stream
407	n/a	s = io.BytesIO()
408	n/a	f = writer(s)
409	n/a	f.write("spam")
410	n/a	f.write("spam")
411	n/a	d = s.getvalue()
412	n/a	# check whether there is exactly one BOM in it
413	n/a	self.assertTrue(d == self.spamle or d == self.spambe)
414	n/a	# try to read it back
415	n/a	s = io.BytesIO(d)
416	n/a	f = reader(s)
417	n/a	self.assertEqual(f.read(), "spamspam")
418	n/a
419	n/a	def test_badbom(self):
420	n/a	s = io.BytesIO(4*b"\xff")
421	n/a	f = codecs.getreader(self.encoding)(s)
422	n/a	self.assertRaises(UnicodeError, f.read)
423	n/a
424	n/a	s = io.BytesIO(8*b"\xff")
425	n/a	f = codecs.getreader(self.encoding)(s)
426	n/a	self.assertRaises(UnicodeError, f.read)
427	n/a
428	n/a	def test_partial(self):
429	n/a	self.check_partial(
430	n/a	"\x00\xff\u0100\uffff\U00010000",
431	n/a	[
432	n/a	"", # first byte of BOM read
433	n/a	"", # second byte of BOM read
434	n/a	"", # third byte of BOM read
435	n/a	"", # fourth byte of BOM read => byteorder known
436	n/a	"",
437	n/a	"",
438	n/a	"",
439	n/a	"\x00",
440	n/a	"\x00",
441	n/a	"\x00",
442	n/a	"\x00",
443	n/a	"\x00\xff",
444	n/a	"\x00\xff",
445	n/a	"\x00\xff",
446	n/a	"\x00\xff",
447	n/a	"\x00\xff\u0100",
448	n/a	"\x00\xff\u0100",
449	n/a	"\x00\xff\u0100",
450	n/a	"\x00\xff\u0100",
451	n/a	"\x00\xff\u0100\uffff",
452	n/a	"\x00\xff\u0100\uffff",
453	n/a	"\x00\xff\u0100\uffff",
454	n/a	"\x00\xff\u0100\uffff",
455	n/a	"\x00\xff\u0100\uffff\U00010000",
456	n/a	]
457	n/a	)
458	n/a
459	n/a	def test_handlers(self):
460	n/a	self.assertEqual(('\ufffd', 1),
461	n/a	codecs.utf_32_decode(b'\x01', 'replace', True))
462	n/a	self.assertEqual(('', 1),
463	n/a	codecs.utf_32_decode(b'\x01', 'ignore', True))
464	n/a
465	n/a	def test_errors(self):
466	n/a	self.assertRaises(UnicodeDecodeError, codecs.utf_32_decode,
467	n/a	b"\xff", "strict", True)
468	n/a
469	n/a	def test_decoder_state(self):
470	n/a	self.check_state_handling_decode(self.encoding,
471	n/a	"spamspam", self.spamle)
472	n/a	self.check_state_handling_decode(self.encoding,
473	n/a	"spamspam", self.spambe)
474	n/a
475	n/a	def test_issue8941(self):
476	n/a	# Issue #8941: insufficient result allocation when decoding into
477	n/a	# surrogate pairs on UCS-2 builds.
478	n/a	encoded_le = b'\xff\xfe\x00\x00' + b'\x00\x00\x01\x00' * 1024
479	n/a	self.assertEqual('\U00010000' * 1024,
480	n/a	codecs.utf_32_decode(encoded_le)[0])
481	n/a	encoded_be = b'\x00\x00\xfe\xff' + b'\x00\x01\x00\x00' * 1024
482	n/a	self.assertEqual('\U00010000' * 1024,
483	n/a	codecs.utf_32_decode(encoded_be)[0])
484	n/a
485	n/a
486	n/a	class UTF32LETest(ReadTest, unittest.TestCase):
487	n/a	encoding = "utf-32-le"
488	n/a	ill_formed_sequence = b"\x80\xdc\x00\x00"
489	n/a
490	n/a	def test_partial(self):
491	n/a	self.check_partial(
492	n/a	"\x00\xff\u0100\uffff\U00010000",
493	n/a	[
494	n/a	"",
495	n/a	"",
496	n/a	"",
497	n/a	"\x00",
498	n/a	"\x00",
499	n/a	"\x00",
500	n/a	"\x00",
501	n/a	"\x00\xff",
502	n/a	"\x00\xff",
503	n/a	"\x00\xff",
504	n/a	"\x00\xff",
505	n/a	"\x00\xff\u0100",
506	n/a	"\x00\xff\u0100",
507	n/a	"\x00\xff\u0100",
508	n/a	"\x00\xff\u0100",
509	n/a	"\x00\xff\u0100\uffff",
510	n/a	"\x00\xff\u0100\uffff",
511	n/a	"\x00\xff\u0100\uffff",
512	n/a	"\x00\xff\u0100\uffff",
513	n/a	"\x00\xff\u0100\uffff\U00010000",
514	n/a	]
515	n/a	)
516	n/a
517	n/a	def test_simple(self):
518	n/a	self.assertEqual("\U00010203".encode(self.encoding), b"\x03\x02\x01\x00")
519	n/a
520	n/a	def test_errors(self):
521	n/a	self.assertRaises(UnicodeDecodeError, codecs.utf_32_le_decode,
522	n/a	b"\xff", "strict", True)
523	n/a
524	n/a	def test_issue8941(self):
525	n/a	# Issue #8941: insufficient result allocation when decoding into
526	n/a	# surrogate pairs on UCS-2 builds.
527	n/a	encoded = b'\x00\x00\x01\x00' * 1024
528	n/a	self.assertEqual('\U00010000' * 1024,
529	n/a	codecs.utf_32_le_decode(encoded)[0])
530	n/a
531	n/a
532	n/a	class UTF32BETest(ReadTest, unittest.TestCase):
533	n/a	encoding = "utf-32-be"
534	n/a	ill_formed_sequence = b"\x00\x00\xdc\x80"
535	n/a
536	n/a	def test_partial(self):
537	n/a	self.check_partial(
538	n/a	"\x00\xff\u0100\uffff\U00010000",
539	n/a	[
540	n/a	"",
541	n/a	"",
542	n/a	"",
543	n/a	"\x00",
544	n/a	"\x00",
545	n/a	"\x00",
546	n/a	"\x00",
547	n/a	"\x00\xff",
548	n/a	"\x00\xff",
549	n/a	"\x00\xff",
550	n/a	"\x00\xff",
551	n/a	"\x00\xff\u0100",
552	n/a	"\x00\xff\u0100",
553	n/a	"\x00\xff\u0100",
554	n/a	"\x00\xff\u0100",
555	n/a	"\x00\xff\u0100\uffff",
556	n/a	"\x00\xff\u0100\uffff",
557	n/a	"\x00\xff\u0100\uffff",
558	n/a	"\x00\xff\u0100\uffff",
559	n/a	"\x00\xff\u0100\uffff\U00010000",
560	n/a	]
561	n/a	)
562	n/a
563	n/a	def test_simple(self):
564	n/a	self.assertEqual("\U00010203".encode(self.encoding), b"\x00\x01\x02\x03")
565	n/a
566	n/a	def test_errors(self):
567	n/a	self.assertRaises(UnicodeDecodeError, codecs.utf_32_be_decode,
568	n/a	b"\xff", "strict", True)
569	n/a
570	n/a	def test_issue8941(self):
571	n/a	# Issue #8941: insufficient result allocation when decoding into
572	n/a	# surrogate pairs on UCS-2 builds.
573	n/a	encoded = b'\x00\x01\x00\x00' * 1024
574	n/a	self.assertEqual('\U00010000' * 1024,
575	n/a	codecs.utf_32_be_decode(encoded)[0])
576	n/a
577	n/a
578	n/a	class UTF16Test(ReadTest, unittest.TestCase):
579	n/a	encoding = "utf-16"
580	n/a	if sys.byteorder == 'little':
581	n/a	ill_formed_sequence = b"\x80\xdc"
582	n/a	else:
583	n/a	ill_formed_sequence = b"\xdc\x80"
584	n/a
585	n/a	spamle = b'\xff\xfes\x00p\x00a\x00m\x00s\x00p\x00a\x00m\x00'
586	n/a	spambe = b'\xfe\xff\x00s\x00p\x00a\x00m\x00s\x00p\x00a\x00m'
587	n/a
588	n/a	def test_only_one_bom(self):
589	n/a	_,_,reader,writer = codecs.lookup(self.encoding)
590	n/a	# encode some stream
591	n/a	s = io.BytesIO()
592	n/a	f = writer(s)
593	n/a	f.write("spam")
594	n/a	f.write("spam")
595	n/a	d = s.getvalue()
596	n/a	# check whether there is exactly one BOM in it
597	n/a	self.assertTrue(d == self.spamle or d == self.spambe)
598	n/a	# try to read it back
599	n/a	s = io.BytesIO(d)
600	n/a	f = reader(s)
601	n/a	self.assertEqual(f.read(), "spamspam")
602	n/a
603	n/a	def test_badbom(self):
604	n/a	s = io.BytesIO(b"\xff\xff")
605	n/a	f = codecs.getreader(self.encoding)(s)
606	n/a	self.assertRaises(UnicodeError, f.read)
607	n/a
608	n/a	s = io.BytesIO(b"\xff\xff\xff\xff")
609	n/a	f = codecs.getreader(self.encoding)(s)
610	n/a	self.assertRaises(UnicodeError, f.read)
611	n/a
612	n/a	def test_partial(self):
613	n/a	self.check_partial(
614	n/a	"\x00\xff\u0100\uffff\U00010000",
615	n/a	[
616	n/a	"", # first byte of BOM read
617	n/a	"", # second byte of BOM read => byteorder known
618	n/a	"",
619	n/a	"\x00",
620	n/a	"\x00",
621	n/a	"\x00\xff",
622	n/a	"\x00\xff",
623	n/a	"\x00\xff\u0100",
624	n/a	"\x00\xff\u0100",
625	n/a	"\x00\xff\u0100\uffff",
626	n/a	"\x00\xff\u0100\uffff",
627	n/a	"\x00\xff\u0100\uffff",
628	n/a	"\x00\xff\u0100\uffff",
629	n/a	"\x00\xff\u0100\uffff\U00010000",
630	n/a	]
631	n/a	)
632	n/a
633	n/a	def test_handlers(self):
634	n/a	self.assertEqual(('\ufffd', 1),
635	n/a	codecs.utf_16_decode(b'\x01', 'replace', True))
636	n/a	self.assertEqual(('', 1),
637	n/a	codecs.utf_16_decode(b'\x01', 'ignore', True))
638	n/a
639	n/a	def test_errors(self):
640	n/a	self.assertRaises(UnicodeDecodeError, codecs.utf_16_decode,
641	n/a	b"\xff", "strict", True)
642	n/a
643	n/a	def test_decoder_state(self):
644	n/a	self.check_state_handling_decode(self.encoding,
645	n/a	"spamspam", self.spamle)
646	n/a	self.check_state_handling_decode(self.encoding,
647	n/a	"spamspam", self.spambe)
648	n/a
649	n/a	def test_bug691291(self):
650	n/a	# Files are always opened in binary mode, even if no binary mode was
651	n/a	# specified. This means that no automatic conversion of '\n' is done
652	n/a	# on reading and writing.
653	n/a	s1 = 'Hello\r\nworld\r\n'
654	n/a
655	n/a	s = s1.encode(self.encoding)
656	n/a	self.addCleanup(support.unlink, support.TESTFN)
657	n/a	with open(support.TESTFN, 'wb') as fp:
658	n/a	fp.write(s)
659	n/a	with support.check_warnings(('', DeprecationWarning)):
660	n/a	reader = codecs.open(support.TESTFN, 'U', encoding=self.encoding)
661	n/a	with reader:
662	n/a	self.assertEqual(reader.read(), s1)
663	n/a
664	n/a	class UTF16LETest(ReadTest, unittest.TestCase):
665	n/a	encoding = "utf-16-le"
666	n/a	ill_formed_sequence = b"\x80\xdc"
667	n/a
668	n/a	def test_partial(self):
669	n/a	self.check_partial(
670	n/a	"\x00\xff\u0100\uffff\U00010000",
671	n/a	[
672	n/a	"",
673	n/a	"\x00",
674	n/a	"\x00",
675	n/a	"\x00\xff",
676	n/a	"\x00\xff",
677	n/a	"\x00\xff\u0100",
678	n/a	"\x00\xff\u0100",
679	n/a	"\x00\xff\u0100\uffff",
680	n/a	"\x00\xff\u0100\uffff",
681	n/a	"\x00\xff\u0100\uffff",
682	n/a	"\x00\xff\u0100\uffff",
683	n/a	"\x00\xff\u0100\uffff\U00010000",
684	n/a	]
685	n/a	)
686	n/a
687	n/a	def test_errors(self):
688	n/a	tests = [
689	n/a	(b'\xff', '\ufffd'),
690	n/a	(b'A\x00Z', 'A\ufffd'),
691	n/a	(b'A\x00B\x00C\x00D\x00Z', 'ABCD\ufffd'),
692	n/a	(b'\x00\xd8', '\ufffd'),
693	n/a	(b'\x00\xd8A', '\ufffd'),
694	n/a	(b'\x00\xd8A\x00', '\ufffdA'),
695	n/a	(b'\x00\xdcA\x00', '\ufffdA'),
696	n/a	]
697	n/a	for raw, expected in tests:
698	n/a	self.assertRaises(UnicodeDecodeError, codecs.utf_16_le_decode,
699	n/a	raw, 'strict', True)
700	n/a	self.assertEqual(raw.decode('utf-16le', 'replace'), expected)
701	n/a
702	n/a	def test_nonbmp(self):
703	n/a	self.assertEqual("\U00010203".encode(self.encoding),
704	n/a	b'\x00\xd8\x03\xde')
705	n/a	self.assertEqual(b'\x00\xd8\x03\xde'.decode(self.encoding),
706	n/a	"\U00010203")
707	n/a
708	n/a	class UTF16BETest(ReadTest, unittest.TestCase):
709	n/a	encoding = "utf-16-be"
710	n/a	ill_formed_sequence = b"\xdc\x80"
711	n/a
712	n/a	def test_partial(self):
713	n/a	self.check_partial(
714	n/a	"\x00\xff\u0100\uffff\U00010000",
715	n/a	[
716	n/a	"",
717	n/a	"\x00",
718	n/a	"\x00",
719	n/a	"\x00\xff",
720	n/a	"\x00\xff",
721	n/a	"\x00\xff\u0100",
722	n/a	"\x00\xff\u0100",
723	n/a	"\x00\xff\u0100\uffff",
724	n/a	"\x00\xff\u0100\uffff",
725	n/a	"\x00\xff\u0100\uffff",
726	n/a	"\x00\xff\u0100\uffff",
727	n/a	"\x00\xff\u0100\uffff\U00010000",
728	n/a	]
729	n/a	)
730	n/a
731	n/a	def test_errors(self):
732	n/a	tests = [
733	n/a	(b'\xff', '\ufffd'),
734	n/a	(b'\x00A\xff', 'A\ufffd'),
735	n/a	(b'\x00A\x00B\x00C\x00DZ', 'ABCD\ufffd'),
736	n/a	(b'\xd8\x00', '\ufffd'),
737	n/a	(b'\xd8\x00\xdc', '\ufffd'),
738	n/a	(b'\xd8\x00\x00A', '\ufffdA'),
739	n/a	(b'\xdc\x00\x00A', '\ufffdA'),
740	n/a	]
741	n/a	for raw, expected in tests:
742	n/a	self.assertRaises(UnicodeDecodeError, codecs.utf_16_be_decode,
743	n/a	raw, 'strict', True)
744	n/a	self.assertEqual(raw.decode('utf-16be', 'replace'), expected)
745	n/a
746	n/a	def test_nonbmp(self):
747	n/a	self.assertEqual("\U00010203".encode(self.encoding),
748	n/a	b'\xd8\x00\xde\x03')
749	n/a	self.assertEqual(b'\xd8\x00\xde\x03'.decode(self.encoding),
750	n/a	"\U00010203")
751	n/a
752	n/a	class UTF8Test(ReadTest, unittest.TestCase):
753	n/a	encoding = "utf-8"
754	n/a	ill_formed_sequence = b"\xed\xb2\x80"
755	n/a	ill_formed_sequence_replace = "\ufffd" * 3
756	n/a	BOM = b''
757	n/a
758	n/a	def test_partial(self):
759	n/a	self.check_partial(
760	n/a	"\x00\xff\u07ff\u0800\uffff\U00010000",
761	n/a	[
762	n/a	"\x00",
763	n/a	"\x00",
764	n/a	"\x00\xff",
765	n/a	"\x00\xff",
766	n/a	"\x00\xff\u07ff",
767	n/a	"\x00\xff\u07ff",
768	n/a	"\x00\xff\u07ff",
769	n/a	"\x00\xff\u07ff\u0800",
770	n/a	"\x00\xff\u07ff\u0800",
771	n/a	"\x00\xff\u07ff\u0800",
772	n/a	"\x00\xff\u07ff\u0800\uffff",
773	n/a	"\x00\xff\u07ff\u0800\uffff",
774	n/a	"\x00\xff\u07ff\u0800\uffff",
775	n/a	"\x00\xff\u07ff\u0800\uffff",
776	n/a	"\x00\xff\u07ff\u0800\uffff\U00010000",
777	n/a	]
778	n/a	)
779	n/a
780	n/a	def test_decoder_state(self):
781	n/a	u = "\x00\x7f\x80\xff\u0100\u07ff\u0800\uffff\U0010ffff"
782	n/a	self.check_state_handling_decode(self.encoding,
783	n/a	u, u.encode(self.encoding))
784	n/a
785	n/a	def test_decode_error(self):
786	n/a	for data, error_handler, expected in (
787	n/a	(b'[\x80\xff]', 'ignore', '[]'),
788	n/a	(b'[\x80\xff]', 'replace', '[\ufffd\ufffd]'),
789	n/a	(b'[\x80\xff]', 'surrogateescape', '[\udc80\udcff]'),
790	n/a	(b'[\x80\xff]', 'backslashreplace', '[\\x80\\xff]'),
791	n/a	):
792	n/a	with self.subTest(data=data, error_handler=error_handler,
793	n/a	expected=expected):
794	n/a	self.assertEqual(data.decode(self.encoding, error_handler),
795	n/a	expected)
796	n/a
797	n/a	def test_lone_surrogates(self):
798	n/a	super().test_lone_surrogates()
799	n/a	# not sure if this is making sense for
800	n/a	# UTF-16 and UTF-32
801	n/a	self.assertEqual("[\uDC80]".encode(self.encoding, "surrogateescape"),
802	n/a	self.BOM + b'[\x80]')
803	n/a
804	n/a	with self.assertRaises(UnicodeEncodeError) as cm:
805	n/a	"[\uDC80\uD800\uDFFF]".encode(self.encoding, "surrogateescape")
806	n/a	exc = cm.exception
807	n/a	self.assertEqual(exc.object[exc.start:exc.end], '\uD800\uDFFF')
808	n/a
809	n/a	def test_surrogatepass_handler(self):
810	n/a	self.assertEqual("abc\ud800def".encode(self.encoding, "surrogatepass"),
811	n/a	self.BOM + b"abc\xed\xa0\x80def")
812	n/a	self.assertEqual("\U00010fff\uD800".encode(self.encoding, "surrogatepass"),
813	n/a	self.BOM + b"\xf0\x90\xbf\xbf\xed\xa0\x80")
814	n/a	self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "surrogatepass"),
815	n/a	self.BOM + b'[\xed\xa0\x80\xed\xb2\x80]')
816	n/a
817	n/a	self.assertEqual(b"abc\xed\xa0\x80def".decode(self.encoding, "surrogatepass"),
818	n/a	"abc\ud800def")
819	n/a	self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode(self.encoding, "surrogatepass"),
820	n/a	"\U00010fff\uD800")
821	n/a
822	n/a	self.assertTrue(codecs.lookup_error("surrogatepass"))
823	n/a	with self.assertRaises(UnicodeDecodeError):
824	n/a	b"abc\xed\xa0".decode(self.encoding, "surrogatepass")
825	n/a	with self.assertRaises(UnicodeDecodeError):
826	n/a	b"abc\xed\xa0z".decode(self.encoding, "surrogatepass")
827	n/a
828	n/a
829	n/a	@unittest.skipUnless(sys.platform == 'win32',
830	n/a	'cp65001 is a Windows-only codec')
831	n/a	class CP65001Test(ReadTest, unittest.TestCase):
832	n/a	encoding = "cp65001"
833	n/a
834	n/a	def test_encode(self):
835	n/a	tests = [
836	n/a	('abc', 'strict', b'abc'),
837	n/a	('\xe9\u20ac', 'strict', b'\xc3\xa9\xe2\x82\xac'),
838	n/a	('\U0010ffff', 'strict', b'\xf4\x8f\xbf\xbf'),
839	n/a	('\udc80', 'strict', None),
840	n/a	('\udc80', 'ignore', b''),
841	n/a	('\udc80', 'replace', b'?'),
842	n/a	('\udc80', 'backslashreplace', b'\\udc80'),
843	n/a	('\udc80', 'namereplace', b'\\udc80'),
844	n/a	('\udc80', 'surrogatepass', b'\xed\xb2\x80'),
845	n/a	]
846	n/a	for text, errors, expected in tests:
847	n/a	if expected is not None:
848	n/a	try:
849	n/a	encoded = text.encode('cp65001', errors)
850	n/a	except UnicodeEncodeError as err:
851	n/a	self.fail('Unable to encode %a to cp65001 with '
852	n/a	'errors=%r: %s' % (text, errors, err))
853	n/a	self.assertEqual(encoded, expected,
854	n/a	'%a.encode("cp65001", %r)=%a != %a'
855	n/a	% (text, errors, encoded, expected))
856	n/a	else:
857	n/a	self.assertRaises(UnicodeEncodeError,
858	n/a	text.encode, "cp65001", errors)
859	n/a
860	n/a	def test_decode(self):
861	n/a	tests = [
862	n/a	(b'abc', 'strict', 'abc'),
863	n/a	(b'\xc3\xa9\xe2\x82\xac', 'strict', '\xe9\u20ac'),
864	n/a	(b'\xf4\x8f\xbf\xbf', 'strict', '\U0010ffff'),
865	n/a	(b'\xef\xbf\xbd', 'strict', '\ufffd'),
866	n/a	(b'[\xc3\xa9]', 'strict', '[\xe9]'),
867	n/a	# invalid bytes
868	n/a	(b'[\xff]', 'strict', None),
869	n/a	(b'[\xff]', 'ignore', '[]'),
870	n/a	(b'[\xff]', 'replace', '[\ufffd]'),
871	n/a	(b'[\xff]', 'surrogateescape', '[\udcff]'),
872	n/a	(b'[\xed\xb2\x80]', 'strict', None),
873	n/a	(b'[\xed\xb2\x80]', 'ignore', '[]'),
874	n/a	(b'[\xed\xb2\x80]', 'replace', '[\ufffd\ufffd\ufffd]'),
875	n/a	]
876	n/a	for raw, errors, expected in tests:
877	n/a	if expected is not None:
878	n/a	try:
879	n/a	decoded = raw.decode('cp65001', errors)
880	n/a	except UnicodeDecodeError as err:
881	n/a	self.fail('Unable to decode %a from cp65001 with '
882	n/a	'errors=%r: %s' % (raw, errors, err))
883	n/a	self.assertEqual(decoded, expected,
884	n/a	'%a.decode("cp65001", %r)=%a != %a'
885	n/a	% (raw, errors, decoded, expected))
886	n/a	else:
887	n/a	self.assertRaises(UnicodeDecodeError,
888	n/a	raw.decode, 'cp65001', errors)
889	n/a
890	n/a	def test_lone_surrogates(self):
891	n/a	self.assertRaises(UnicodeEncodeError, "\ud800".encode, "cp65001")
892	n/a	self.assertRaises(UnicodeDecodeError, b"\xed\xa0\x80".decode, "cp65001")
893	n/a	self.assertEqual("[\uDC80]".encode("cp65001", "backslashreplace"),
894	n/a	b'[\\udc80]')
895	n/a	self.assertEqual("[\uDC80]".encode("cp65001", "namereplace"),
896	n/a	b'[\\udc80]')
897	n/a	self.assertEqual("[\uDC80]".encode("cp65001", "xmlcharrefreplace"),
898	n/a	b'[&#56448;]')
899	n/a	self.assertEqual("[\uDC80]".encode("cp65001", "surrogateescape"),
900	n/a	b'[\x80]')
901	n/a	self.assertEqual("[\uDC80]".encode("cp65001", "ignore"),
902	n/a	b'[]')
903	n/a	self.assertEqual("[\uDC80]".encode("cp65001", "replace"),
904	n/a	b'[?]')
905	n/a
906	n/a	def test_surrogatepass_handler(self):
907	n/a	self.assertEqual("abc\ud800def".encode("cp65001", "surrogatepass"),
908	n/a	b"abc\xed\xa0\x80def")
909	n/a	self.assertEqual(b"abc\xed\xa0\x80def".decode("cp65001", "surrogatepass"),
910	n/a	"abc\ud800def")
911	n/a	self.assertEqual("\U00010fff\uD800".encode("cp65001", "surrogatepass"),
912	n/a	b"\xf0\x90\xbf\xbf\xed\xa0\x80")
913	n/a	self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode("cp65001", "surrogatepass"),
914	n/a	"\U00010fff\uD800")
915	n/a	self.assertTrue(codecs.lookup_error("surrogatepass"))
916	n/a
917	n/a
918	n/a	class UTF7Test(ReadTest, unittest.TestCase):
919	n/a	encoding = "utf-7"
920	n/a
921	n/a	def test_ascii(self):
922	n/a	# Set D (directly encoded characters)
923	n/a	set_d = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
924	n/a	'abcdefghijklmnopqrstuvwxyz'
925	n/a	'0123456789'
926	n/a	'\'(),-./:?')
927	n/a	self.assertEqual(set_d.encode(self.encoding), set_d.encode('ascii'))
928	n/a	self.assertEqual(set_d.encode('ascii').decode(self.encoding), set_d)
929	n/a	# Set O (optional direct characters)
930	n/a	set_o = ' !"#$%&*;<=>@[]^_`{\|}'
931	n/a	self.assertEqual(set_o.encode(self.encoding), set_o.encode('ascii'))
932	n/a	self.assertEqual(set_o.encode('ascii').decode(self.encoding), set_o)
933	n/a	# +
934	n/a	self.assertEqual('a+b'.encode(self.encoding), b'a+-b')
935	n/a	self.assertEqual(b'a+-b'.decode(self.encoding), 'a+b')
936	n/a	# White spaces
937	n/a	ws = ' \t\n\r'
938	n/a	self.assertEqual(ws.encode(self.encoding), ws.encode('ascii'))
939	n/a	self.assertEqual(ws.encode('ascii').decode(self.encoding), ws)
940	n/a	# Other ASCII characters
941	n/a	other_ascii = ''.join(sorted(set(bytes(range(0x80)).decode()) -
942	n/a	set(set_d + set_o + '+' + ws)))
943	n/a	self.assertEqual(other_ascii.encode(self.encoding),
944	n/a	b'+AAAAAQACAAMABAAFAAYABwAIAAsADAAOAA8AEAARABIAEwAU'
945	n/a	b'ABUAFgAXABgAGQAaABsAHAAdAB4AHwBcAH4Afw-')
946	n/a
947	n/a	def test_partial(self):
948	n/a	self.check_partial(
949	n/a	'a+-b\x00c\x80d\u0100e\U00010000f',
950	n/a	[
951	n/a	'a',
952	n/a	'a',
953	n/a	'a+',
954	n/a	'a+-',
955	n/a	'a+-b',
956	n/a	'a+-b',
957	n/a	'a+-b',
958	n/a	'a+-b',
959	n/a	'a+-b',
960	n/a	'a+-b\x00',
961	n/a	'a+-b\x00c',
962	n/a	'a+-b\x00c',
963	n/a	'a+-b\x00c',
964	n/a	'a+-b\x00c',
965	n/a	'a+-b\x00c',
966	n/a	'a+-b\x00c\x80',
967	n/a	'a+-b\x00c\x80d',
968	n/a	'a+-b\x00c\x80d',
969	n/a	'a+-b\x00c\x80d',
970	n/a	'a+-b\x00c\x80d',
971	n/a	'a+-b\x00c\x80d',
972	n/a	'a+-b\x00c\x80d\u0100',
973	n/a	'a+-b\x00c\x80d\u0100e',
974	n/a	'a+-b\x00c\x80d\u0100e',
975	n/a	'a+-b\x00c\x80d\u0100e',
976	n/a	'a+-b\x00c\x80d\u0100e',
977	n/a	'a+-b\x00c\x80d\u0100e',
978	n/a	'a+-b\x00c\x80d\u0100e',
979	n/a	'a+-b\x00c\x80d\u0100e',
980	n/a	'a+-b\x00c\x80d\u0100e',
981	n/a	'a+-b\x00c\x80d\u0100e\U00010000',
982	n/a	'a+-b\x00c\x80d\u0100e\U00010000f',
983	n/a	]
984	n/a	)
985	n/a
986	n/a	def test_errors(self):
987	n/a	tests = [
988	n/a	(b'\xffb', '\ufffdb'),
989	n/a	(b'a\xffb', 'a\ufffdb'),
990	n/a	(b'a\xff\xffb', 'a\ufffd\ufffdb'),
991	n/a	(b'a+IK', 'a\ufffd'),
992	n/a	(b'a+IK-b', 'a\ufffdb'),
993	n/a	(b'a+IK,b', 'a\ufffdb'),
994	n/a	(b'a+IKx', 'a\u20ac\ufffd'),
995	n/a	(b'a+IKx-b', 'a\u20ac\ufffdb'),
996	n/a	(b'a+IKwgr', 'a\u20ac\ufffd'),
997	n/a	(b'a+IKwgr-b', 'a\u20ac\ufffdb'),
998	n/a	(b'a+IKwgr,', 'a\u20ac\ufffd'),
999	n/a	(b'a+IKwgr,-b', 'a\u20ac\ufffd-b'),
1000	n/a	(b'a+IKwgrB', 'a\u20ac\u20ac\ufffd'),
1001	n/a	(b'a+IKwgrB-b', 'a\u20ac\u20ac\ufffdb'),
1002	n/a	(b'a+/,+IKw-b', 'a\ufffd\u20acb'),
1003	n/a	(b'a+//,+IKw-b', 'a\ufffd\u20acb'),
1004	n/a	(b'a+///,+IKw-b', 'a\uffff\ufffd\u20acb'),
1005	n/a	(b'a+////,+IKw-b', 'a\uffff\ufffd\u20acb'),
1006	n/a	(b'a+IKw-b\xff', 'a\u20acb\ufffd'),
1007	n/a	(b'a+IKw\xffb', 'a\u20ac\ufffdb'),
1008	n/a	]
1009	n/a	for raw, expected in tests:
1010	n/a	with self.subTest(raw=raw):
1011	n/a	self.assertRaises(UnicodeDecodeError, codecs.utf_7_decode,
1012	n/a	raw, 'strict', True)
1013	n/a	self.assertEqual(raw.decode('utf-7', 'replace'), expected)
1014	n/a
1015	n/a	def test_nonbmp(self):
1016	n/a	self.assertEqual('\U000104A0'.encode(self.encoding), b'+2AHcoA-')
1017	n/a	self.assertEqual('\ud801\udca0'.encode(self.encoding), b'+2AHcoA-')
1018	n/a	self.assertEqual(b'+2AHcoA-'.decode(self.encoding), '\U000104A0')
1019	n/a	self.assertEqual(b'+2AHcoA'.decode(self.encoding), '\U000104A0')
1020	n/a	self.assertEqual('\u20ac\U000104A0'.encode(self.encoding), b'+IKzYAdyg-')
1021	n/a	self.assertEqual(b'+IKzYAdyg-'.decode(self.encoding), '\u20ac\U000104A0')
1022	n/a	self.assertEqual(b'+IKzYAdyg'.decode(self.encoding), '\u20ac\U000104A0')
1023	n/a	self.assertEqual('\u20ac\u20ac\U000104A0'.encode(self.encoding),
1024	n/a	b'+IKwgrNgB3KA-')
1025	n/a	self.assertEqual(b'+IKwgrNgB3KA-'.decode(self.encoding),
1026	n/a	'\u20ac\u20ac\U000104A0')
1027	n/a	self.assertEqual(b'+IKwgrNgB3KA'.decode(self.encoding),
1028	n/a	'\u20ac\u20ac\U000104A0')
1029	n/a
1030	n/a	def test_lone_surrogates(self):
1031	n/a	tests = [
1032	n/a	(b'a+2AE-b', 'a\ud801b'),
1033	n/a	(b'a+2AE\xffb', 'a\ufffdb'),
1034	n/a	(b'a+2AE', 'a\ufffd'),
1035	n/a	(b'a+2AEA-b', 'a\ufffdb'),
1036	n/a	(b'a+2AH-b', 'a\ufffdb'),
1037	n/a	(b'a+IKzYAQ-b', 'a\u20ac\ud801b'),
1038	n/a	(b'a+IKzYAQ\xffb', 'a\u20ac\ufffdb'),
1039	n/a	(b'a+IKzYAQA-b', 'a\u20ac\ufffdb'),
1040	n/a	(b'a+IKzYAd-b', 'a\u20ac\ufffdb'),
1041	n/a	(b'a+IKwgrNgB-b', 'a\u20ac\u20ac\ud801b'),
1042	n/a	(b'a+IKwgrNgB\xffb', 'a\u20ac\u20ac\ufffdb'),
1043	n/a	(b'a+IKwgrNgB', 'a\u20ac\u20ac\ufffd'),
1044	n/a	(b'a+IKwgrNgBA-b', 'a\u20ac\u20ac\ufffdb'),
1045	n/a	]
1046	n/a	for raw, expected in tests:
1047	n/a	with self.subTest(raw=raw):
1048	n/a	self.assertEqual(raw.decode('utf-7', 'replace'), expected)
1049	n/a
1050	n/a
1051	n/a	class UTF16ExTest(unittest.TestCase):
1052	n/a
1053	n/a	def test_errors(self):
1054	n/a	self.assertRaises(UnicodeDecodeError, codecs.utf_16_ex_decode, b"\xff", "strict", 0, True)
1055	n/a
1056	n/a	def test_bad_args(self):
1057	n/a	self.assertRaises(TypeError, codecs.utf_16_ex_decode)
1058	n/a
1059	n/a	class ReadBufferTest(unittest.TestCase):
1060	n/a
1061	n/a	def test_array(self):
1062	n/a	import array
1063	n/a	self.assertEqual(
1064	n/a	codecs.readbuffer_encode(array.array("b", b"spam")),
1065	n/a	(b"spam", 4)
1066	n/a	)
1067	n/a
1068	n/a	def test_empty(self):
1069	n/a	self.assertEqual(codecs.readbuffer_encode(""), (b"", 0))
1070	n/a
1071	n/a	def test_bad_args(self):
1072	n/a	self.assertRaises(TypeError, codecs.readbuffer_encode)
1073	n/a	self.assertRaises(TypeError, codecs.readbuffer_encode, 42)
1074	n/a
1075	n/a	class UTF8SigTest(UTF8Test, unittest.TestCase):
1076	n/a	encoding = "utf-8-sig"
1077	n/a	BOM = codecs.BOM_UTF8
1078	n/a
1079	n/a	def test_partial(self):
1080	n/a	self.check_partial(
1081	n/a	"\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
1082	n/a	[
1083	n/a	"",
1084	n/a	"",
1085	n/a	"", # First BOM has been read and skipped
1086	n/a	"",
1087	n/a	"",
1088	n/a	"\ufeff", # Second BOM has been read and emitted
1089	n/a	"\ufeff\x00", # "\x00" read and emitted
1090	n/a	"\ufeff\x00", # First byte of encoded "\xff" read
1091	n/a	"\ufeff\x00\xff", # Second byte of encoded "\xff" read
1092	n/a	"\ufeff\x00\xff", # First byte of encoded "\u07ff" read
1093	n/a	"\ufeff\x00\xff\u07ff", # Second byte of encoded "\u07ff" read
1094	n/a	"\ufeff\x00\xff\u07ff",
1095	n/a	"\ufeff\x00\xff\u07ff",
1096	n/a	"\ufeff\x00\xff\u07ff\u0800",
1097	n/a	"\ufeff\x00\xff\u07ff\u0800",
1098	n/a	"\ufeff\x00\xff\u07ff\u0800",
1099	n/a	"\ufeff\x00\xff\u07ff\u0800\uffff",
1100	n/a	"\ufeff\x00\xff\u07ff\u0800\uffff",
1101	n/a	"\ufeff\x00\xff\u07ff\u0800\uffff",
1102	n/a	"\ufeff\x00\xff\u07ff\u0800\uffff",
1103	n/a	"\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
1104	n/a	]
1105	n/a	)
1106	n/a
1107	n/a	def test_bug1601501(self):
1108	n/a	# SF bug #1601501: check that the codec works with a buffer
1109	n/a	self.assertEqual(str(b"\xef\xbb\xbf", "utf-8-sig"), "")
1110	n/a
1111	n/a	def test_bom(self):
1112	n/a	d = codecs.getincrementaldecoder("utf-8-sig")()
1113	n/a	s = "spam"
1114	n/a	self.assertEqual(d.decode(s.encode("utf-8-sig")), s)
1115	n/a
1116	n/a	def test_stream_bom(self):
1117	n/a	unistring = "ABC\u00A1\u2200XYZ"
1118	n/a	bytestring = codecs.BOM_UTF8 + b"ABC\xC2\xA1\xE2\x88\x80XYZ"
1119	n/a
1120	n/a	reader = codecs.getreader("utf-8-sig")
1121	n/a	for sizehint in [None] + list(range(1, 11)) + \
1122	n/a	[64, 128, 256, 512, 1024]:
1123	n/a	istream = reader(io.BytesIO(bytestring))
1124	n/a	ostream = io.StringIO()
1125	n/a	while 1:
1126	n/a	if sizehint is not None:
1127	n/a	data = istream.read(sizehint)
1128	n/a	else:
1129	n/a	data = istream.read()
1130	n/a
1131	n/a	if not data:
1132	n/a	break
1133	n/a	ostream.write(data)
1134	n/a
1135	n/a	got = ostream.getvalue()
1136	n/a	self.assertEqual(got, unistring)
1137	n/a
1138	n/a	def test_stream_bare(self):
1139	n/a	unistring = "ABC\u00A1\u2200XYZ"
1140	n/a	bytestring = b"ABC\xC2\xA1\xE2\x88\x80XYZ"
1141	n/a
1142	n/a	reader = codecs.getreader("utf-8-sig")
1143	n/a	for sizehint in [None] + list(range(1, 11)) + \
1144	n/a	[64, 128, 256, 512, 1024]:
1145	n/a	istream = reader(io.BytesIO(bytestring))
1146	n/a	ostream = io.StringIO()
1147	n/a	while 1:
1148	n/a	if sizehint is not None:
1149	n/a	data = istream.read(sizehint)
1150	n/a	else:
1151	n/a	data = istream.read()
1152	n/a
1153	n/a	if not data:
1154	n/a	break
1155	n/a	ostream.write(data)
1156	n/a
1157	n/a	got = ostream.getvalue()
1158	n/a	self.assertEqual(got, unistring)
1159	n/a
1160	n/a	class EscapeDecodeTest(unittest.TestCase):
1161	n/a	def test_empty(self):
1162	n/a	self.assertEqual(codecs.escape_decode(b""), (b"", 0))
1163	n/a	self.assertEqual(codecs.escape_decode(bytearray()), (b"", 0))
1164	n/a
1165	n/a	def test_raw(self):
1166	n/a	decode = codecs.escape_decode
1167	n/a	for b in range(256):
1168	n/a	b = bytes([b])
1169	n/a	if b != b'\\':
1170	n/a	self.assertEqual(decode(b + b'0'), (b + b'0', 2))
1171	n/a
1172	n/a	def test_escape(self):
1173	n/a	decode = codecs.escape_decode
1174	n/a	check = coding_checker(self, decode)
1175	n/a	check(b"[\\\n]", b"[]")
1176	n/a	check(br'[\"]', b'["]')
1177	n/a	check(br"[\']", b"[']")
1178	n/a	check(br"[\\]", b"[\\]")
1179	n/a	check(br"[\a]", b"[\x07]")
1180	n/a	check(br"[\b]", b"[\x08]")
1181	n/a	check(br"[\t]", b"[\x09]")
1182	n/a	check(br"[\n]", b"[\x0a]")
1183	n/a	check(br"[\v]", b"[\x0b]")
1184	n/a	check(br"[\f]", b"[\x0c]")
1185	n/a	check(br"[\r]", b"[\x0d]")
1186	n/a	check(br"[\7]", b"[\x07]")
1187	n/a	check(br"[\78]", b"[\x078]")
1188	n/a	check(br"[\41]", b"[!]")
1189	n/a	check(br"[\418]", b"[!8]")
1190	n/a	check(br"[\101]", b"[A]")
1191	n/a	check(br"[\1010]", b"[A0]")
1192	n/a	check(br"[\501]", b"[A]")
1193	n/a	check(br"[\x41]", b"[A]")
1194	n/a	check(br"[\x410]", b"[A0]")
1195	n/a	for i in range(97, 123):
1196	n/a	b = bytes([i])
1197	n/a	if b not in b'abfnrtvx':
1198	n/a	with self.assertWarns(DeprecationWarning):
1199	n/a	check(b"\\" + b, b"\\" + b)
1200	n/a	with self.assertWarns(DeprecationWarning):
1201	n/a	check(b"\\" + b.upper(), b"\\" + b.upper())
1202	n/a	with self.assertWarns(DeprecationWarning):
1203	n/a	check(br"\8", b"\\8")
1204	n/a	with self.assertWarns(DeprecationWarning):
1205	n/a	check(br"\9", b"\\9")
1206	n/a
1207	n/a	def test_errors(self):
1208	n/a	decode = codecs.escape_decode
1209	n/a	self.assertRaises(ValueError, decode, br"\x")
1210	n/a	self.assertRaises(ValueError, decode, br"[\x]")
1211	n/a	self.assertEqual(decode(br"[\x]\x", "ignore"), (b"[]", 6))
1212	n/a	self.assertEqual(decode(br"[\x]\x", "replace"), (b"[?]?", 6))
1213	n/a	self.assertRaises(ValueError, decode, br"\x0")
1214	n/a	self.assertRaises(ValueError, decode, br"[\x0]")
1215	n/a	self.assertEqual(decode(br"[\x0]\x0", "ignore"), (b"[]", 8))
1216	n/a	self.assertEqual(decode(br"[\x0]\x0", "replace"), (b"[?]?", 8))
1217	n/a
1218	n/a
1219	n/a	class RecodingTest(unittest.TestCase):
1220	n/a	def test_recoding(self):
1221	n/a	f = io.BytesIO()
1222	n/a	f2 = codecs.EncodedFile(f, "unicode_internal", "utf-8")
1223	n/a	f2.write("a")
1224	n/a	f2.close()
1225	n/a	# Python used to crash on this at exit because of a refcount
1226	n/a	# bug in _codecsmodule.c
1227	n/a
1228	n/a	self.assertTrue(f.closed)
1229	n/a
1230	n/a	# From RFC 3492
1231	n/a	punycode_testcases = [
1232	n/a	# A Arabic (Egyptian):
1233	n/a	("\u0644\u064A\u0647\u0645\u0627\u0628\u062A\u0643\u0644"
1234	n/a	"\u0645\u0648\u0634\u0639\u0631\u0628\u064A\u061F",
1235	n/a	b"egbpdaj6bu4bxfgehfvwxn"),
1236	n/a	# B Chinese (simplified):
1237	n/a	("\u4ED6\u4EEC\u4E3A\u4EC0\u4E48\u4E0D\u8BF4\u4E2D\u6587",
1238	n/a	b"ihqwcrb4cv8a8dqg056pqjye"),
1239	n/a	# C Chinese (traditional):
1240	n/a	("\u4ED6\u5011\u7232\u4EC0\u9EBD\u4E0D\u8AAA\u4E2D\u6587",
1241	n/a	b"ihqwctvzc91f659drss3x8bo0yb"),
1242	n/a	# D Czech: Pro<ccaron>prost<ecaron>nemluv<iacute><ccaron>esky
1243	n/a	("\u0050\u0072\u006F\u010D\u0070\u0072\u006F\u0073\u0074"
1244	n/a	"\u011B\u006E\u0065\u006D\u006C\u0075\u0076\u00ED\u010D"
1245	n/a	"\u0065\u0073\u006B\u0079",
1246	n/a	b"Proprostnemluvesky-uyb24dma41a"),
1247	n/a	# E Hebrew:
1248	n/a	("\u05DC\u05DE\u05D4\u05D4\u05DD\u05E4\u05E9\u05D5\u05D8"
1249	n/a	"\u05DC\u05D0\u05DE\u05D3\u05D1\u05E8\u05D9\u05DD\u05E2"
1250	n/a	"\u05D1\u05E8\u05D9\u05EA",
1251	n/a	b"4dbcagdahymbxekheh6e0a7fei0b"),
1252	n/a	# F Hindi (Devanagari):
1253	n/a	("\u092F\u0939\u0932\u094B\u0917\u0939\u093F\u0928\u094D"
1254	n/a	"\u0926\u0940\u0915\u094D\u092F\u094B\u0902\u0928\u0939"
1255	n/a	"\u0940\u0902\u092C\u094B\u0932\u0938\u0915\u0924\u0947"
1256	n/a	"\u0939\u0948\u0902",
1257	n/a	b"i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd"),
1258	n/a
1259	n/a	#(G) Japanese (kanji and hiragana):
1260	n/a	("\u306A\u305C\u307F\u3093\u306A\u65E5\u672C\u8A9E\u3092"
1261	n/a	"\u8A71\u3057\u3066\u304F\u308C\u306A\u3044\u306E\u304B",
1262	n/a	b"n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa"),
1263	n/a
1264	n/a	# (H) Korean (Hangul syllables):
1265	n/a	("\uC138\uACC4\uC758\uBAA8\uB4E0\uC0AC\uB78C\uB4E4\uC774"
1266	n/a	"\uD55C\uAD6D\uC5B4\uB97C\uC774\uD574\uD55C\uB2E4\uBA74"
1267	n/a	"\uC5BC\uB9C8\uB098\uC88B\uC744\uAE4C",
1268	n/a	b"989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5j"
1269	n/a	b"psd879ccm6fea98c"),
1270	n/a
1271	n/a	# (I) Russian (Cyrillic):
1272	n/a	("\u043F\u043E\u0447\u0435\u043C\u0443\u0436\u0435\u043E"
1273	n/a	"\u043D\u0438\u043D\u0435\u0433\u043E\u0432\u043E\u0440"
1274	n/a	"\u044F\u0442\u043F\u043E\u0440\u0443\u0441\u0441\u043A"
1275	n/a	"\u0438",
1276	n/a	b"b1abfaaepdrnnbgefbaDotcwatmq2g4l"),
1277	n/a
1278	n/a	# (J) Spanish: Porqu<eacute>nopuedensimplementehablarenEspa<ntilde>ol
1279	n/a	("\u0050\u006F\u0072\u0071\u0075\u00E9\u006E\u006F\u0070"
1280	n/a	"\u0075\u0065\u0064\u0065\u006E\u0073\u0069\u006D\u0070"
1281	n/a	"\u006C\u0065\u006D\u0065\u006E\u0074\u0065\u0068\u0061"
1282	n/a	"\u0062\u006C\u0061\u0072\u0065\u006E\u0045\u0073\u0070"
1283	n/a	"\u0061\u00F1\u006F\u006C",
1284	n/a	b"PorqunopuedensimplementehablarenEspaol-fmd56a"),
1285	n/a
1286	n/a	# (K) Vietnamese:
1287	n/a	# T<adotbelow>isaoh<odotbelow>kh<ocirc>ngth<ecirchookabove>ch\
1288	n/a	# <ihookabove>n<oacute>iti<ecircacute>ngVi<ecircdotbelow>t
1289	n/a	("\u0054\u1EA1\u0069\u0073\u0061\u006F\u0068\u1ECD\u006B"
1290	n/a	"\u0068\u00F4\u006E\u0067\u0074\u0068\u1EC3\u0063\u0068"
1291	n/a	"\u1EC9\u006E\u00F3\u0069\u0074\u0069\u1EBF\u006E\u0067"
1292	n/a	"\u0056\u0069\u1EC7\u0074",
1293	n/a	b"TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g"),
1294	n/a
1295	n/a	#(L) 3<nen>B<gumi><kinpachi><sensei>
1296	n/a	("\u0033\u5E74\u0042\u7D44\u91D1\u516B\u5148\u751F",
1297	n/a	b"3B-ww4c5e180e575a65lsy2b"),
1298	n/a
1299	n/a	# (M) <amuro><namie>-with-SUPER-MONKEYS
1300	n/a	("\u5B89\u5BA4\u5948\u7F8E\u6075\u002D\u0077\u0069\u0074"
1301	n/a	"\u0068\u002D\u0053\u0055\u0050\u0045\u0052\u002D\u004D"
1302	n/a	"\u004F\u004E\u004B\u0045\u0059\u0053",
1303	n/a	b"-with-SUPER-MONKEYS-pc58ag80a8qai00g7n9n"),
1304	n/a
1305	n/a	# (N) Hello-Another-Way-<sorezore><no><basho>
1306	n/a	("\u0048\u0065\u006C\u006C\u006F\u002D\u0041\u006E\u006F"
1307	n/a	"\u0074\u0068\u0065\u0072\u002D\u0057\u0061\u0079\u002D"
1308	n/a	"\u305D\u308C\u305E\u308C\u306E\u5834\u6240",
1309	n/a	b"Hello-Another-Way--fc4qua05auwb3674vfr0b"),
1310	n/a
1311	n/a	# (O) <hitotsu><yane><no><shita>2
1312	n/a	("\u3072\u3068\u3064\u5C4B\u6839\u306E\u4E0B\u0032",
1313	n/a	b"2-u9tlzr9756bt3uc0v"),
1314	n/a
1315	n/a	# (P) Maji<de>Koi<suru>5<byou><mae>
1316	n/a	("\u004D\u0061\u006A\u0069\u3067\u004B\u006F\u0069\u3059"
1317	n/a	"\u308B\u0035\u79D2\u524D",
1318	n/a	b"MajiKoi5-783gue6qz075azm5e"),
1319	n/a
1320	n/a	# (Q) <pafii>de<runba>
1321	n/a	("\u30D1\u30D5\u30A3\u30FC\u0064\u0065\u30EB\u30F3\u30D0",
1322	n/a	b"de-jg4avhby1noc0d"),
1323	n/a
1324	n/a	# (R) <sono><supiido><de>
1325	n/a	("\u305D\u306E\u30B9\u30D4\u30FC\u30C9\u3067",
1326	n/a	b"d9juau41awczczp"),
1327	n/a
1328	n/a	# (S) -> $1.00 <-
1329	n/a	("\u002D\u003E\u0020\u0024\u0031\u002E\u0030\u0030\u0020"
1330	n/a	"\u003C\u002D",
1331	n/a	b"-> $1.00 <--")
1332	n/a	]
1333	n/a
1334	n/a	for i in punycode_testcases:
1335	n/a	if len(i)!=2:
1336	n/a	print(repr(i))
1337	n/a
1338	n/a
1339	n/a	class PunycodeTest(unittest.TestCase):
1340	n/a	def test_encode(self):
1341	n/a	for uni, puny in punycode_testcases:
1342	n/a	# Need to convert both strings to lower case, since
1343	n/a	# some of the extended encodings use upper case, but our
1344	n/a	# code produces only lower case. Converting just puny to
1345	n/a	# lower is also insufficient, since some of the input characters
1346	n/a	# are upper case.
1347	n/a	self.assertEqual(
1348	n/a	str(uni.encode("punycode"), "ascii").lower(),
1349	n/a	str(puny, "ascii").lower()
1350	n/a	)
1351	n/a
1352	n/a	def test_decode(self):
1353	n/a	for uni, puny in punycode_testcases:
1354	n/a	self.assertEqual(uni, puny.decode("punycode"))
1355	n/a	puny = puny.decode("ascii").encode("ascii")
1356	n/a	self.assertEqual(uni, puny.decode("punycode"))
1357	n/a
1358	n/a
1359	n/a	class UnicodeInternalTest(unittest.TestCase):
1360	n/a	@unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t')
1361	n/a	def test_bug1251300(self):
1362	n/a	# Decoding with unicode_internal used to not correctly handle "code
1363	n/a	# points" above 0x10ffff on UCS-4 builds.
1364	n/a	ok = [
1365	n/a	(b"\x00\x10\xff\xff", "\U0010ffff"),
1366	n/a	(b"\x00\x00\x01\x01", "\U00000101"),
1367	n/a	(b"", ""),
1368	n/a	]
1369	n/a	not_ok = [
1370	n/a	b"\x7f\xff\xff\xff",
1371	n/a	b"\x80\x00\x00\x00",
1372	n/a	b"\x81\x00\x00\x00",
1373	n/a	b"\x00",
1374	n/a	b"\x00\x00\x00\x00\x00",
1375	n/a	]
1376	n/a	for internal, uni in ok:
1377	n/a	if sys.byteorder == "little":
1378	n/a	internal = bytes(reversed(internal))
1379	n/a	with support.check_warnings():
1380	n/a	self.assertEqual(uni, internal.decode("unicode_internal"))
1381	n/a	for internal in not_ok:
1382	n/a	if sys.byteorder == "little":
1383	n/a	internal = bytes(reversed(internal))
1384	n/a	with support.check_warnings(('unicode_internal codec has been '
1385	n/a	'deprecated', DeprecationWarning)):
1386	n/a	self.assertRaises(UnicodeDecodeError, internal.decode,
1387	n/a	"unicode_internal")
1388	n/a	if sys.byteorder == "little":
1389	n/a	invalid = b"\x00\x00\x11\x00"
1390	n/a	invalid_backslashreplace = r"\x00\x00\x11\x00"
1391	n/a	else:
1392	n/a	invalid = b"\x00\x11\x00\x00"
1393	n/a	invalid_backslashreplace = r"\x00\x11\x00\x00"
1394	n/a	with support.check_warnings():
1395	n/a	self.assertRaises(UnicodeDecodeError,
1396	n/a	invalid.decode, "unicode_internal")
1397	n/a	with support.check_warnings():
1398	n/a	self.assertEqual(invalid.decode("unicode_internal", "replace"),
1399	n/a	'\ufffd')
1400	n/a	with support.check_warnings():
1401	n/a	self.assertEqual(invalid.decode("unicode_internal", "backslashreplace"),
1402	n/a	invalid_backslashreplace)
1403	n/a
1404	n/a	@unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t')
1405	n/a	def test_decode_error_attributes(self):
1406	n/a	try:
1407	n/a	with support.check_warnings(('unicode_internal codec has been '
1408	n/a	'deprecated', DeprecationWarning)):
1409	n/a	b"\x00\x00\x00\x00\x00\x11\x11\x00".decode("unicode_internal")
1410	n/a	except UnicodeDecodeError as ex:
1411	n/a	self.assertEqual("unicode_internal", ex.encoding)
1412	n/a	self.assertEqual(b"\x00\x00\x00\x00\x00\x11\x11\x00", ex.object)
1413	n/a	self.assertEqual(4, ex.start)
1414	n/a	self.assertEqual(8, ex.end)
1415	n/a	else:
1416	n/a	self.fail()
1417	n/a
1418	n/a	@unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t')
1419	n/a	def test_decode_callback(self):
1420	n/a	codecs.register_error("UnicodeInternalTest", codecs.ignore_errors)
1421	n/a	decoder = codecs.getdecoder("unicode_internal")
1422	n/a	with support.check_warnings(('unicode_internal codec has been '
1423	n/a	'deprecated', DeprecationWarning)):
1424	n/a	ab = "ab".encode("unicode_internal").decode()
1425	n/a	ignored = decoder(bytes("%s\x22\x22\x22\x22%s" % (ab[:4], ab[4:]),
1426	n/a	"ascii"),
1427	n/a	"UnicodeInternalTest")
1428	n/a	self.assertEqual(("ab", 12), ignored)
1429	n/a
1430	n/a	def test_encode_length(self):
1431	n/a	with support.check_warnings(('unicode_internal codec has been '
1432	n/a	'deprecated', DeprecationWarning)):
1433	n/a	# Issue 3739
1434	n/a	encoder = codecs.getencoder("unicode_internal")
1435	n/a	self.assertEqual(encoder("a")[1], 1)
1436	n/a	self.assertEqual(encoder("\xe9\u0142")[1], 2)
1437	n/a
1438	n/a	self.assertEqual(codecs.escape_encode(br'\x00')[1], 4)
1439	n/a
1440	n/a	# From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html
1441	n/a	nameprep_tests = [
1442	n/a	# 3.1 Map to nothing.
1443	n/a	(b'foo\xc2\xad\xcd\x8f\xe1\xa0\x86\xe1\xa0\x8bbar'
1444	n/a	b'\xe2\x80\x8b\xe2\x81\xa0baz\xef\xb8\x80\xef\xb8\x88\xef'
1445	n/a	b'\xb8\x8f\xef\xbb\xbf',
1446	n/a	b'foobarbaz'),
1447	n/a	# 3.2 Case folding ASCII U+0043 U+0041 U+0046 U+0045.
1448	n/a	(b'CAFE',
1449	n/a	b'cafe'),
1450	n/a	# 3.3 Case folding 8bit U+00DF (german sharp s).
1451	n/a	# The original test case is bogus; it says \xc3\xdf
1452	n/a	(b'\xc3\x9f',
1453	n/a	b'ss'),
1454	n/a	# 3.4 Case folding U+0130 (turkish capital I with dot).
1455	n/a	(b'\xc4\xb0',
1456	n/a	b'i\xcc\x87'),
1457	n/a	# 3.5 Case folding multibyte U+0143 U+037A.
1458	n/a	(b'\xc5\x83\xcd\xba',
1459	n/a	b'\xc5\x84 \xce\xb9'),
1460	n/a	# 3.6 Case folding U+2121 U+33C6 U+1D7BB.
1461	n/a	# XXX: skip this as it fails in UCS-2 mode
1462	n/a	#('\xe2\x84\xa1\xe3\x8f\x86\xf0\x9d\x9e\xbb',
1463	n/a	# 'telc\xe2\x88\x95kg\xcf\x83'),
1464	n/a	(None, None),
1465	n/a	# 3.7 Normalization of U+006a U+030c U+00A0 U+00AA.
1466	n/a	(b'j\xcc\x8c\xc2\xa0\xc2\xaa',
1467	n/a	b'\xc7\xb0 a'),
1468	n/a	# 3.8 Case folding U+1FB7 and normalization.
1469	n/a	(b'\xe1\xbe\xb7',
1470	n/a	b'\xe1\xbe\xb6\xce\xb9'),
1471	n/a	# 3.9 Self-reverting case folding U+01F0 and normalization.
1472	n/a	# The original test case is bogus, it says `\xc7\xf0'
1473	n/a	(b'\xc7\xb0',
1474	n/a	b'\xc7\xb0'),
1475	n/a	# 3.10 Self-reverting case folding U+0390 and normalization.
1476	n/a	(b'\xce\x90',
1477	n/a	b'\xce\x90'),
1478	n/a	# 3.11 Self-reverting case folding U+03B0 and normalization.
1479	n/a	(b'\xce\xb0',
1480	n/a	b'\xce\xb0'),
1481	n/a	# 3.12 Self-reverting case folding U+1E96 and normalization.
1482	n/a	(b'\xe1\xba\x96',
1483	n/a	b'\xe1\xba\x96'),
1484	n/a	# 3.13 Self-reverting case folding U+1F56 and normalization.
1485	n/a	(b'\xe1\xbd\x96',
1486	n/a	b'\xe1\xbd\x96'),
1487	n/a	# 3.14 ASCII space character U+0020.
1488	n/a	(b' ',
1489	n/a	b' '),
1490	n/a	# 3.15 Non-ASCII 8bit space character U+00A0.
1491	n/a	(b'\xc2\xa0',
1492	n/a	b' '),
1493	n/a	# 3.16 Non-ASCII multibyte space character U+1680.
1494	n/a	(b'\xe1\x9a\x80',
1495	n/a	None),
1496	n/a	# 3.17 Non-ASCII multibyte space character U+2000.
1497	n/a	(b'\xe2\x80\x80',
1498	n/a	b' '),
1499	n/a	# 3.18 Zero Width Space U+200b.
1500	n/a	(b'\xe2\x80\x8b',
1501	n/a	b''),
1502	n/a	# 3.19 Non-ASCII multibyte space character U+3000.
1503	n/a	(b'\xe3\x80\x80',
1504	n/a	b' '),
1505	n/a	# 3.20 ASCII control characters U+0010 U+007F.
1506	n/a	(b'\x10\x7f',
1507	n/a	b'\x10\x7f'),
1508	n/a	# 3.21 Non-ASCII 8bit control character U+0085.
1509	n/a	(b'\xc2\x85',
1510	n/a	None),
1511	n/a	# 3.22 Non-ASCII multibyte control character U+180E.
1512	n/a	(b'\xe1\xa0\x8e',
1513	n/a	None),
1514	n/a	# 3.23 Zero Width No-Break Space U+FEFF.
1515	n/a	(b'\xef\xbb\xbf',
1516	n/a	b''),
1517	n/a	# 3.24 Non-ASCII control character U+1D175.
1518	n/a	(b'\xf0\x9d\x85\xb5',
1519	n/a	None),
1520	n/a	# 3.25 Plane 0 private use character U+F123.
1521	n/a	(b'\xef\x84\xa3',
1522	n/a	None),
1523	n/a	# 3.26 Plane 15 private use character U+F1234.
1524	n/a	(b'\xf3\xb1\x88\xb4',
1525	n/a	None),
1526	n/a	# 3.27 Plane 16 private use character U+10F234.
1527	n/a	(b'\xf4\x8f\x88\xb4',
1528	n/a	None),
1529	n/a	# 3.28 Non-character code point U+8FFFE.
1530	n/a	(b'\xf2\x8f\xbf\xbe',
1531	n/a	None),
1532	n/a	# 3.29 Non-character code point U+10FFFF.
1533	n/a	(b'\xf4\x8f\xbf\xbf',
1534	n/a	None),
1535	n/a	# 3.30 Surrogate code U+DF42.
1536	n/a	(b'\xed\xbd\x82',
1537	n/a	None),
1538	n/a	# 3.31 Non-plain text character U+FFFD.
1539	n/a	(b'\xef\xbf\xbd',
1540	n/a	None),
1541	n/a	# 3.32 Ideographic description character U+2FF5.
1542	n/a	(b'\xe2\xbf\xb5',
1543	n/a	None),
1544	n/a	# 3.33 Display property character U+0341.
1545	n/a	(b'\xcd\x81',
1546	n/a	b'\xcc\x81'),
1547	n/a	# 3.34 Left-to-right mark U+200E.
1548	n/a	(b'\xe2\x80\x8e',
1549	n/a	None),
1550	n/a	# 3.35 Deprecated U+202A.
1551	n/a	(b'\xe2\x80\xaa',
1552	n/a	None),
1553	n/a	# 3.36 Language tagging character U+E0001.
1554	n/a	(b'\xf3\xa0\x80\x81',
1555	n/a	None),
1556	n/a	# 3.37 Language tagging character U+E0042.
1557	n/a	(b'\xf3\xa0\x81\x82',
1558	n/a	None),
1559	n/a	# 3.38 Bidi: RandALCat character U+05BE and LCat characters.
1560	n/a	(b'foo\xd6\xbebar',
1561	n/a	None),
1562	n/a	# 3.39 Bidi: RandALCat character U+FD50 and LCat characters.
1563	n/a	(b'foo\xef\xb5\x90bar',
1564	n/a	None),
1565	n/a	# 3.40 Bidi: RandALCat character U+FB38 and LCat characters.
1566	n/a	(b'foo\xef\xb9\xb6bar',
1567	n/a	b'foo \xd9\x8ebar'),
1568	n/a	# 3.41 Bidi: RandALCat without trailing RandALCat U+0627 U+0031.
1569	n/a	(b'\xd8\xa71',
1570	n/a	None),
1571	n/a	# 3.42 Bidi: RandALCat character U+0627 U+0031 U+0628.
1572	n/a	(b'\xd8\xa71\xd8\xa8',
1573	n/a	b'\xd8\xa71\xd8\xa8'),
1574	n/a	# 3.43 Unassigned code point U+E0002.
1575	n/a	# Skip this test as we allow unassigned
1576	n/a	#(b'\xf3\xa0\x80\x82',
1577	n/a	# None),
1578	n/a	(None, None),
1579	n/a	# 3.44 Larger test (shrinking).
1580	n/a	# Original test case reads \xc3\xdf
1581	n/a	(b'X\xc2\xad\xc3\x9f\xc4\xb0\xe2\x84\xa1j\xcc\x8c\xc2\xa0\xc2'
1582	n/a	b'\xaa\xce\xb0\xe2\x80\x80',
1583	n/a	b'xssi\xcc\x87tel\xc7\xb0 a\xce\xb0 '),
1584	n/a	# 3.45 Larger test (expanding).
1585	n/a	# Original test case reads \xc3\x9f
1586	n/a	(b'X\xc3\x9f\xe3\x8c\x96\xc4\xb0\xe2\x84\xa1\xe2\x92\x9f\xe3\x8c'
1587	n/a	b'\x80',
1588	n/a	b'xss\xe3\x82\xad\xe3\x83\xad\xe3\x83\xa1\xe3\x83\xbc\xe3'
1589	n/a	b'\x83\x88\xe3\x83\xabi\xcc\x87tel\x28d\x29\xe3\x82'
1590	n/a	b'\xa2\xe3\x83\x91\xe3\x83\xbc\xe3\x83\x88')
1591	n/a	]
1592	n/a
1593	n/a
1594	n/a	class NameprepTest(unittest.TestCase):
1595	n/a	def test_nameprep(self):
1596	n/a	from encodings.idna import nameprep
1597	n/a	for pos, (orig, prepped) in enumerate(nameprep_tests):
1598	n/a	if orig is None:
1599	n/a	# Skipped
1600	n/a	continue
1601	n/a	# The Unicode strings are given in UTF-8
1602	n/a	orig = str(orig, "utf-8", "surrogatepass")
1603	n/a	if prepped is None:
1604	n/a	# Input contains prohibited characters
1605	n/a	self.assertRaises(UnicodeError, nameprep, orig)
1606	n/a	else:
1607	n/a	prepped = str(prepped, "utf-8", "surrogatepass")
1608	n/a	try:
1609	n/a	self.assertEqual(nameprep(orig), prepped)
1610	n/a	except Exception as e:
1611	n/a	raise support.TestFailed("Test 3.%d: %s" % (pos+1, str(e)))
1612	n/a
1613	n/a
1614	n/a	class IDNACodecTest(unittest.TestCase):
1615	n/a	def test_builtin_decode(self):
1616	n/a	self.assertEqual(str(b"python.org", "idna"), "python.org")
1617	n/a	self.assertEqual(str(b"python.org.", "idna"), "python.org.")
1618	n/a	self.assertEqual(str(b"xn--pythn-mua.org", "idna"), "pyth\xf6n.org")
1619	n/a	self.assertEqual(str(b"xn--pythn-mua.org.", "idna"), "pyth\xf6n.org.")
1620	n/a
1621	n/a	def test_builtin_encode(self):
1622	n/a	self.assertEqual("python.org".encode("idna"), b"python.org")
1623	n/a	self.assertEqual("python.org.".encode("idna"), b"python.org.")
1624	n/a	self.assertEqual("pyth\xf6n.org".encode("idna"), b"xn--pythn-mua.org")
1625	n/a	self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.")
1626	n/a
1627	n/a	def test_stream(self):
1628	n/a	r = codecs.getreader("idna")(io.BytesIO(b"abc"))
1629	n/a	r.read(3)
1630	n/a	self.assertEqual(r.read(), "")
1631	n/a
1632	n/a	def test_incremental_decode(self):
1633	n/a	self.assertEqual(
1634	n/a	"".join(codecs.iterdecode((bytes([c]) for c in b"python.org"), "idna")),
1635	n/a	"python.org"
1636	n/a	)
1637	n/a	self.assertEqual(
1638	n/a	"".join(codecs.iterdecode((bytes([c]) for c in b"python.org."), "idna")),
1639	n/a	"python.org."
1640	n/a	)
1641	n/a	self.assertEqual(
1642	n/a	"".join(codecs.iterdecode((bytes([c]) for c in b"xn--pythn-mua.org."), "idna")),
1643	n/a	"pyth\xf6n.org."
1644	n/a	)
1645	n/a	self.assertEqual(
1646	n/a	"".join(codecs.iterdecode((bytes([c]) for c in b"xn--pythn-mua.org."), "idna")),
1647	n/a	"pyth\xf6n.org."
1648	n/a	)
1649	n/a
1650	n/a	decoder = codecs.getincrementaldecoder("idna")()
1651	n/a	self.assertEqual(decoder.decode(b"xn--xam", ), "")
1652	n/a	self.assertEqual(decoder.decode(b"ple-9ta.o", ), "\xe4xample.")
1653	n/a	self.assertEqual(decoder.decode(b"rg"), "")
1654	n/a	self.assertEqual(decoder.decode(b"", True), "org")
1655	n/a
1656	n/a	decoder.reset()
1657	n/a	self.assertEqual(decoder.decode(b"xn--xam", ), "")
1658	n/a	self.assertEqual(decoder.decode(b"ple-9ta.o", ), "\xe4xample.")
1659	n/a	self.assertEqual(decoder.decode(b"rg."), "org.")
1660	n/a	self.assertEqual(decoder.decode(b"", True), "")
1661	n/a
1662	n/a	def test_incremental_encode(self):
1663	n/a	self.assertEqual(
1664	n/a	b"".join(codecs.iterencode("python.org", "idna")),
1665	n/a	b"python.org"
1666	n/a	)
1667	n/a	self.assertEqual(
1668	n/a	b"".join(codecs.iterencode("python.org.", "idna")),
1669	n/a	b"python.org."
1670	n/a	)
1671	n/a	self.assertEqual(
1672	n/a	b"".join(codecs.iterencode("pyth\xf6n.org.", "idna")),
1673	n/a	b"xn--pythn-mua.org."
1674	n/a	)
1675	n/a	self.assertEqual(
1676	n/a	b"".join(codecs.iterencode("pyth\xf6n.org.", "idna")),
1677	n/a	b"xn--pythn-mua.org."
1678	n/a	)
1679	n/a
1680	n/a	encoder = codecs.getincrementalencoder("idna")()
1681	n/a	self.assertEqual(encoder.encode("\xe4x"), b"")
1682	n/a	self.assertEqual(encoder.encode("ample.org"), b"xn--xample-9ta.")
1683	n/a	self.assertEqual(encoder.encode("", True), b"org")
1684	n/a
1685	n/a	encoder.reset()
1686	n/a	self.assertEqual(encoder.encode("\xe4x"), b"")
1687	n/a	self.assertEqual(encoder.encode("ample.org."), b"xn--xample-9ta.org.")
1688	n/a	self.assertEqual(encoder.encode("", True), b"")
1689	n/a
1690	n/a	def test_errors(self):
1691	n/a	"""Only supports "strict" error handler"""
1692	n/a	"python.org".encode("idna", "strict")
1693	n/a	b"python.org".decode("idna", "strict")
1694	n/a	for errors in ("ignore", "replace", "backslashreplace",
1695	n/a	"surrogateescape"):
1696	n/a	self.assertRaises(Exception, "python.org".encode, "idna", errors)
1697	n/a	self.assertRaises(Exception,
1698	n/a	b"python.org".decode, "idna", errors)
1699	n/a
1700	n/a
1701	n/a	class CodecsModuleTest(unittest.TestCase):
1702	n/a
1703	n/a	def test_decode(self):
1704	n/a	self.assertEqual(codecs.decode(b'\xe4\xf6\xfc', 'latin-1'),
1705	n/a	'\xe4\xf6\xfc')
1706	n/a	self.assertRaises(TypeError, codecs.decode)
1707	n/a	self.assertEqual(codecs.decode(b'abc'), 'abc')
1708	n/a	self.assertRaises(UnicodeDecodeError, codecs.decode, b'\xff', 'ascii')
1709	n/a
1710	n/a	# test keywords
1711	n/a	self.assertEqual(codecs.decode(obj=b'\xe4\xf6\xfc', encoding='latin-1'),
1712	n/a	'\xe4\xf6\xfc')
1713	n/a	self.assertEqual(codecs.decode(b'[\xff]', 'ascii', errors='ignore'),
1714	n/a	'[]')
1715	n/a
1716	n/a	def test_encode(self):
1717	n/a	self.assertEqual(codecs.encode('\xe4\xf6\xfc', 'latin-1'),
1718	n/a	b'\xe4\xf6\xfc')
1719	n/a	self.assertRaises(TypeError, codecs.encode)
1720	n/a	self.assertRaises(LookupError, codecs.encode, "foo", "__spam__")
1721	n/a	self.assertEqual(codecs.encode('abc'), b'abc')
1722	n/a	self.assertRaises(UnicodeEncodeError, codecs.encode, '\xffff', 'ascii')
1723	n/a
1724	n/a	# test keywords
1725	n/a	self.assertEqual(codecs.encode(obj='\xe4\xf6\xfc', encoding='latin-1'),
1726	n/a	b'\xe4\xf6\xfc')
1727	n/a	self.assertEqual(codecs.encode('[\xff]', 'ascii', errors='ignore'),
1728	n/a	b'[]')
1729	n/a
1730	n/a	def test_register(self):
1731	n/a	self.assertRaises(TypeError, codecs.register)
1732	n/a	self.assertRaises(TypeError, codecs.register, 42)
1733	n/a
1734	n/a	def test_lookup(self):
1735	n/a	self.assertRaises(TypeError, codecs.lookup)
1736	n/a	self.assertRaises(LookupError, codecs.lookup, "__spam__")
1737	n/a	self.assertRaises(LookupError, codecs.lookup, " ")
1738	n/a
1739	n/a	def test_getencoder(self):
1740	n/a	self.assertRaises(TypeError, codecs.getencoder)
1741	n/a	self.assertRaises(LookupError, codecs.getencoder, "__spam__")
1742	n/a
1743	n/a	def test_getdecoder(self):
1744	n/a	self.assertRaises(TypeError, codecs.getdecoder)
1745	n/a	self.assertRaises(LookupError, codecs.getdecoder, "__spam__")
1746	n/a
1747	n/a	def test_getreader(self):
1748	n/a	self.assertRaises(TypeError, codecs.getreader)
1749	n/a	self.assertRaises(LookupError, codecs.getreader, "__spam__")
1750	n/a
1751	n/a	def test_getwriter(self):
1752	n/a	self.assertRaises(TypeError, codecs.getwriter)
1753	n/a	self.assertRaises(LookupError, codecs.getwriter, "__spam__")
1754	n/a
1755	n/a	def test_lookup_issue1813(self):
1756	n/a	# Issue #1813: under Turkish locales, lookup of some codecs failed
1757	n/a	# because 'I' is lowercased as "Ä±" (dotless i)
1758	n/a	oldlocale = locale.setlocale(locale.LC_CTYPE)
1759	n/a	self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale)
1760	n/a	try:
1761	n/a	locale.setlocale(locale.LC_CTYPE, 'tr_TR')
1762	n/a	except locale.Error:
1763	n/a	# Unsupported locale on this system
1764	n/a	self.skipTest('test needs Turkish locale')
1765	n/a	c = codecs.lookup('ASCII')
1766	n/a	self.assertEqual(c.name, 'ascii')
1767	n/a
1768	n/a	def test_all(self):
1769	n/a	api = (
1770	n/a	"encode", "decode",
1771	n/a	"register", "CodecInfo", "Codec", "IncrementalEncoder",
1772	n/a	"IncrementalDecoder", "StreamReader", "StreamWriter", "lookup",
1773	n/a	"getencoder", "getdecoder", "getincrementalencoder",
1774	n/a	"getincrementaldecoder", "getreader", "getwriter",
1775	n/a	"register_error", "lookup_error",
1776	n/a	"strict_errors", "replace_errors", "ignore_errors",
1777	n/a	"xmlcharrefreplace_errors", "backslashreplace_errors",
1778	n/a	"namereplace_errors",
1779	n/a	"open", "EncodedFile",
1780	n/a	"iterencode", "iterdecode",
1781	n/a	"BOM", "BOM_BE", "BOM_LE",
1782	n/a	"BOM_UTF8", "BOM_UTF16", "BOM_UTF16_BE", "BOM_UTF16_LE",
1783	n/a	"BOM_UTF32", "BOM_UTF32_BE", "BOM_UTF32_LE",
1784	n/a	"BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE", # Undocumented
1785	n/a	"StreamReaderWriter", "StreamRecoder",
1786	n/a	)
1787	n/a	self.assertCountEqual(api, codecs.__all__)
1788	n/a	for api in codecs.__all__:
1789	n/a	getattr(codecs, api)
1790	n/a
1791	n/a	def test_open(self):
1792	n/a	self.addCleanup(support.unlink, support.TESTFN)
1793	n/a	for mode in ('w', 'r', 'r+', 'w+', 'a', 'a+'):
1794	n/a	with self.subTest(mode), \
1795	n/a	codecs.open(support.TESTFN, mode, 'ascii') as file:
1796	n/a	self.assertIsInstance(file, codecs.StreamReaderWriter)
1797	n/a
1798	n/a	def test_undefined(self):
1799	n/a	self.assertRaises(UnicodeError, codecs.encode, 'abc', 'undefined')
1800	n/a	self.assertRaises(UnicodeError, codecs.decode, b'abc', 'undefined')
1801	n/a	self.assertRaises(UnicodeError, codecs.encode, '', 'undefined')
1802	n/a	self.assertRaises(UnicodeError, codecs.decode, b'', 'undefined')
1803	n/a	for errors in ('strict', 'ignore', 'replace', 'backslashreplace'):
1804	n/a	self.assertRaises(UnicodeError,
1805	n/a	codecs.encode, 'abc', 'undefined', errors)
1806	n/a	self.assertRaises(UnicodeError,
1807	n/a	codecs.decode, b'abc', 'undefined', errors)
1808	n/a
1809	n/a
1810	n/a	class StreamReaderTest(unittest.TestCase):
1811	n/a
1812	n/a	def setUp(self):
1813	n/a	self.reader = codecs.getreader('utf-8')
1814	n/a	self.stream = io.BytesIO(b'\xed\x95\x9c\n\xea\xb8\x80')
1815	n/a
1816	n/a	def test_readlines(self):
1817	n/a	f = self.reader(self.stream)
1818	n/a	self.assertEqual(f.readlines(), ['\ud55c\n', '\uae00'])
1819	n/a
1820	n/a
1821	n/a	class EncodedFileTest(unittest.TestCase):
1822	n/a
1823	n/a	def test_basic(self):
1824	n/a	f = io.BytesIO(b'\xed\x95\x9c\n\xea\xb8\x80')
1825	n/a	ef = codecs.EncodedFile(f, 'utf-16-le', 'utf-8')
1826	n/a	self.assertEqual(ef.read(), b'\\\xd5\n\x00\x00\xae')
1827	n/a
1828	n/a	f = io.BytesIO()
1829	n/a	ef = codecs.EncodedFile(f, 'utf-8', 'latin-1')
1830	n/a	ef.write(b'\xc3\xbc')
1831	n/a	self.assertEqual(f.getvalue(), b'\xfc')
1832	n/a
1833	n/a	all_unicode_encodings = [
1834	n/a	"ascii",
1835	n/a	"big5",
1836	n/a	"big5hkscs",
1837	n/a	"charmap",
1838	n/a	"cp037",
1839	n/a	"cp1006",
1840	n/a	"cp1026",
1841	n/a	"cp1125",
1842	n/a	"cp1140",
1843	n/a	"cp1250",
1844	n/a	"cp1251",
1845	n/a	"cp1252",
1846	n/a	"cp1253",
1847	n/a	"cp1254",
1848	n/a	"cp1255",
1849	n/a	"cp1256",
1850	n/a	"cp1257",
1851	n/a	"cp1258",
1852	n/a	"cp424",
1853	n/a	"cp437",
1854	n/a	"cp500",
1855	n/a	"cp720",
1856	n/a	"cp737",
1857	n/a	"cp775",
1858	n/a	"cp850",
1859	n/a	"cp852",
1860	n/a	"cp855",
1861	n/a	"cp856",
1862	n/a	"cp857",
1863	n/a	"cp858",
1864	n/a	"cp860",
1865	n/a	"cp861",
1866	n/a	"cp862",
1867	n/a	"cp863",
1868	n/a	"cp864",
1869	n/a	"cp865",
1870	n/a	"cp866",
1871	n/a	"cp869",
1872	n/a	"cp874",
1873	n/a	"cp875",
1874	n/a	"cp932",
1875	n/a	"cp949",
1876	n/a	"cp950",
1877	n/a	"euc_jis_2004",
1878	n/a	"euc_jisx0213",
1879	n/a	"euc_jp",
1880	n/a	"euc_kr",
1881	n/a	"gb18030",
1882	n/a	"gb2312",
1883	n/a	"gbk",
1884	n/a	"hp_roman8",
1885	n/a	"hz",
1886	n/a	"idna",
1887	n/a	"iso2022_jp",
1888	n/a	"iso2022_jp_1",
1889	n/a	"iso2022_jp_2",
1890	n/a	"iso2022_jp_2004",
1891	n/a	"iso2022_jp_3",
1892	n/a	"iso2022_jp_ext",
1893	n/a	"iso2022_kr",
1894	n/a	"iso8859_1",
1895	n/a	"iso8859_10",
1896	n/a	"iso8859_11",
1897	n/a	"iso8859_13",
1898	n/a	"iso8859_14",
1899	n/a	"iso8859_15",
1900	n/a	"iso8859_16",
1901	n/a	"iso8859_2",
1902	n/a	"iso8859_3",
1903	n/a	"iso8859_4",
1904	n/a	"iso8859_5",
1905	n/a	"iso8859_6",
1906	n/a	"iso8859_7",
1907	n/a	"iso8859_8",
1908	n/a	"iso8859_9",
1909	n/a	"johab",
1910	n/a	"koi8_r",
1911	n/a	"koi8_t",
1912	n/a	"koi8_u",
1913	n/a	"kz1048",
1914	n/a	"latin_1",
1915	n/a	"mac_cyrillic",
1916	n/a	"mac_greek",
1917	n/a	"mac_iceland",
1918	n/a	"mac_latin2",
1919	n/a	"mac_roman",
1920	n/a	"mac_turkish",
1921	n/a	"palmos",
1922	n/a	"ptcp154",
1923	n/a	"punycode",
1924	n/a	"raw_unicode_escape",
1925	n/a	"shift_jis",
1926	n/a	"shift_jis_2004",
1927	n/a	"shift_jisx0213",
1928	n/a	"tis_620",
1929	n/a	"unicode_escape",
1930	n/a	"unicode_internal",
1931	n/a	"utf_16",
1932	n/a	"utf_16_be",
1933	n/a	"utf_16_le",
1934	n/a	"utf_7",
1935	n/a	"utf_8",
1936	n/a	]
1937	n/a
1938	n/a	if hasattr(codecs, "mbcs_encode"):
1939	n/a	all_unicode_encodings.append("mbcs")
1940	n/a	if hasattr(codecs, "oem_encode"):
1941	n/a	all_unicode_encodings.append("oem")
1942	n/a
1943	n/a	# The following encoding is not tested, because it's not supposed
1944	n/a	# to work:
1945	n/a	# "undefined"
1946	n/a
1947	n/a	# The following encodings don't work in stateful mode
1948	n/a	broken_unicode_with_stateful = [
1949	n/a	"punycode",
1950	n/a	"unicode_internal"
1951	n/a	]
1952	n/a
1953	n/a
1954	n/a	class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling):
1955	n/a	def test_basics(self):
1956	n/a	s = "abc123" # all codecs should be able to encode these
1957	n/a	for encoding in all_unicode_encodings:
1958	n/a	name = codecs.lookup(encoding).name
1959	n/a	if encoding.endswith("_codec"):
1960	n/a	name += "_codec"
1961	n/a	elif encoding == "latin_1":
1962	n/a	name = "latin_1"
1963	n/a	self.assertEqual(encoding.replace("_", "-"), name.replace("_", "-"))
1964	n/a
1965	n/a	with support.check_warnings():
1966	n/a	# unicode-internal has been deprecated
1967	n/a	(b, size) = codecs.getencoder(encoding)(s)
1968	n/a	self.assertEqual(size, len(s), "encoding=%r" % encoding)
1969	n/a	(chars, size) = codecs.getdecoder(encoding)(b)
1970	n/a	self.assertEqual(chars, s, "encoding=%r" % encoding)
1971	n/a
1972	n/a	if encoding not in broken_unicode_with_stateful:
1973	n/a	# check stream reader/writer
1974	n/a	q = Queue(b"")
1975	n/a	writer = codecs.getwriter(encoding)(q)
1976	n/a	encodedresult = b""
1977	n/a	for c in s:
1978	n/a	writer.write(c)
1979	n/a	chunk = q.read()
1980	n/a	self.assertTrue(type(chunk) is bytes, type(chunk))
1981	n/a	encodedresult += chunk
1982	n/a	q = Queue(b"")
1983	n/a	reader = codecs.getreader(encoding)(q)
1984	n/a	decodedresult = ""
1985	n/a	for c in encodedresult:
1986	n/a	q.write(bytes([c]))
1987	n/a	decodedresult += reader.read()
1988	n/a	self.assertEqual(decodedresult, s, "encoding=%r" % encoding)
1989	n/a
1990	n/a	if encoding not in broken_unicode_with_stateful:
1991	n/a	# check incremental decoder/encoder and iterencode()/iterdecode()
1992	n/a	try:
1993	n/a	encoder = codecs.getincrementalencoder(encoding)()
1994	n/a	except LookupError: # no IncrementalEncoder
1995	n/a	pass
1996	n/a	else:
1997	n/a	# check incremental decoder/encoder
1998	n/a	encodedresult = b""
1999	n/a	for c in s:
2000	n/a	encodedresult += encoder.encode(c)
2001	n/a	encodedresult += encoder.encode("", True)
2002	n/a	decoder = codecs.getincrementaldecoder(encoding)()
2003	n/a	decodedresult = ""
2004	n/a	for c in encodedresult:
2005	n/a	decodedresult += decoder.decode(bytes([c]))
2006	n/a	decodedresult += decoder.decode(b"", True)
2007	n/a	self.assertEqual(decodedresult, s,
2008	n/a	"encoding=%r" % encoding)
2009	n/a
2010	n/a	# check iterencode()/iterdecode()
2011	n/a	result = "".join(codecs.iterdecode(
2012	n/a	codecs.iterencode(s, encoding), encoding))
2013	n/a	self.assertEqual(result, s, "encoding=%r" % encoding)
2014	n/a
2015	n/a	# check iterencode()/iterdecode() with empty string
2016	n/a	result = "".join(codecs.iterdecode(
2017	n/a	codecs.iterencode("", encoding), encoding))
2018	n/a	self.assertEqual(result, "")
2019	n/a
2020	n/a	if encoding not in ("idna", "mbcs"):
2021	n/a	# check incremental decoder/encoder with errors argument
2022	n/a	try:
2023	n/a	encoder = codecs.getincrementalencoder(encoding)("ignore")
2024	n/a	except LookupError: # no IncrementalEncoder
2025	n/a	pass
2026	n/a	else:
2027	n/a	encodedresult = b"".join(encoder.encode(c) for c in s)
2028	n/a	decoder = codecs.getincrementaldecoder(encoding)("ignore")
2029	n/a	decodedresult = "".join(decoder.decode(bytes([c]))
2030	n/a	for c in encodedresult)
2031	n/a	self.assertEqual(decodedresult, s,
2032	n/a	"encoding=%r" % encoding)
2033	n/a
2034	n/a	@support.cpython_only
2035	n/a	def test_basics_capi(self):
2036	n/a	from _testcapi import codec_incrementalencoder, codec_incrementaldecoder
2037	n/a	s = "abc123" # all codecs should be able to encode these
2038	n/a	for encoding in all_unicode_encodings:
2039	n/a	if encoding not in broken_unicode_with_stateful:
2040	n/a	# check incremental decoder/encoder (fetched via the C API)
2041	n/a	try:
2042	n/a	cencoder = codec_incrementalencoder(encoding)
2043	n/a	except LookupError: # no IncrementalEncoder
2044	n/a	pass
2045	n/a	else:
2046	n/a	# check C API
2047	n/a	encodedresult = b""
2048	n/a	for c in s:
2049	n/a	encodedresult += cencoder.encode(c)
2050	n/a	encodedresult += cencoder.encode("", True)
2051	n/a	cdecoder = codec_incrementaldecoder(encoding)
2052	n/a	decodedresult = ""
2053	n/a	for c in encodedresult:
2054	n/a	decodedresult += cdecoder.decode(bytes([c]))
2055	n/a	decodedresult += cdecoder.decode(b"", True)
2056	n/a	self.assertEqual(decodedresult, s,
2057	n/a	"encoding=%r" % encoding)
2058	n/a
2059	n/a	if encoding not in ("idna", "mbcs"):
2060	n/a	# check incremental decoder/encoder with errors argument
2061	n/a	try:
2062	n/a	cencoder = codec_incrementalencoder(encoding, "ignore")
2063	n/a	except LookupError: # no IncrementalEncoder
2064	n/a	pass
2065	n/a	else:
2066	n/a	encodedresult = b"".join(cencoder.encode(c) for c in s)
2067	n/a	cdecoder = codec_incrementaldecoder(encoding, "ignore")
2068	n/a	decodedresult = "".join(cdecoder.decode(bytes([c]))
2069	n/a	for c in encodedresult)
2070	n/a	self.assertEqual(decodedresult, s,
2071	n/a	"encoding=%r" % encoding)
2072	n/a
2073	n/a	def test_seek(self):
2074	n/a	# all codecs should be able to encode these
2075	n/a	s = "%s\n%s\n" % (100"abc123", 100"def456")
2076	n/a	for encoding in all_unicode_encodings:
2077	n/a	if encoding == "idna": # FIXME: See SF bug #1163178
2078	n/a	continue
2079	n/a	if encoding in broken_unicode_with_stateful:
2080	n/a	continue
2081	n/a	reader = codecs.getreader(encoding)(io.BytesIO(s.encode(encoding)))
2082	n/a	for t in range(5):
2083	n/a	# Test that calling seek resets the internal codec state and buffers
2084	n/a	reader.seek(0, 0)
2085	n/a	data = reader.read()
2086	n/a	self.assertEqual(s, data)
2087	n/a
2088	n/a	def test_bad_decode_args(self):
2089	n/a	for encoding in all_unicode_encodings:
2090	n/a	decoder = codecs.getdecoder(encoding)
2091	n/a	self.assertRaises(TypeError, decoder)
2092	n/a	if encoding not in ("idna", "punycode"):
2093	n/a	self.assertRaises(TypeError, decoder, 42)
2094	n/a
2095	n/a	def test_bad_encode_args(self):
2096	n/a	for encoding in all_unicode_encodings:
2097	n/a	encoder = codecs.getencoder(encoding)
2098	n/a	with support.check_warnings():
2099	n/a	# unicode-internal has been deprecated
2100	n/a	self.assertRaises(TypeError, encoder)
2101	n/a
2102	n/a	def test_encoding_map_type_initialized(self):
2103	n/a	from encodings import cp1140
2104	n/a	# This used to crash, we are only verifying there's no crash.
2105	n/a	table_type = type(cp1140.encoding_table)
2106	n/a	self.assertEqual(table_type, table_type)
2107	n/a
2108	n/a	def test_decoder_state(self):
2109	n/a	# Check that getstate() and setstate() handle the state properly
2110	n/a	u = "abc123"
2111	n/a	for encoding in all_unicode_encodings:
2112	n/a	if encoding not in broken_unicode_with_stateful:
2113	n/a	self.check_state_handling_decode(encoding, u, u.encode(encoding))
2114	n/a	self.check_state_handling_encode(encoding, u, u.encode(encoding))
2115	n/a
2116	n/a
2117	n/a	class CharmapTest(unittest.TestCase):
2118	n/a	def test_decode_with_string_map(self):
2119	n/a	self.assertEqual(
2120	n/a	codecs.charmap_decode(b"\x00\x01\x02", "strict", "abc"),
2121	n/a	("abc", 3)
2122	n/a	)
2123	n/a
2124	n/a	self.assertEqual(
2125	n/a	codecs.charmap_decode(b"\x00\x01\x02", "strict", "\U0010FFFFbc"),
2126	n/a	("\U0010FFFFbc", 3)
2127	n/a	)
2128	n/a
2129	n/a	self.assertRaises(UnicodeDecodeError,
2130	n/a	codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab"
2131	n/a	)
2132	n/a
2133	n/a	self.assertRaises(UnicodeDecodeError,
2134	n/a	codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab\ufffe"
2135	n/a	)
2136	n/a
2137	n/a	self.assertEqual(
2138	n/a	codecs.charmap_decode(b"\x00\x01\x02", "replace", "ab"),
2139	n/a	("ab\ufffd", 3)
2140	n/a	)
2141	n/a
2142	n/a	self.assertEqual(
2143	n/a	codecs.charmap_decode(b"\x00\x01\x02", "replace", "ab\ufffe"),
2144	n/a	("ab\ufffd", 3)
2145	n/a	)
2146	n/a
2147	n/a	self.assertEqual(
2148	n/a	codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace", "ab"),
2149	n/a	("ab\\x02", 3)
2150	n/a	)
2151	n/a
2152	n/a	self.assertEqual(
2153	n/a	codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace", "ab\ufffe"),
2154	n/a	("ab\\x02", 3)
2155	n/a	)
2156	n/a
2157	n/a	self.assertEqual(
2158	n/a	codecs.charmap_decode(b"\x00\x01\x02", "ignore", "ab"),
2159	n/a	("ab", 3)
2160	n/a	)
2161	n/a
2162	n/a	self.assertEqual(
2163	n/a	codecs.charmap_decode(b"\x00\x01\x02", "ignore", "ab\ufffe"),
2164	n/a	("ab", 3)
2165	n/a	)
2166	n/a
2167	n/a	allbytes = bytes(range(256))
2168	n/a	self.assertEqual(
2169	n/a	codecs.charmap_decode(allbytes, "ignore", ""),
2170	n/a	("", len(allbytes))
2171	n/a	)
2172	n/a
2173	n/a	def test_decode_with_int2str_map(self):
2174	n/a	self.assertEqual(
2175	n/a	codecs.charmap_decode(b"\x00\x01\x02", "strict",
2176	n/a	{0: 'a', 1: 'b', 2: 'c'}),
2177	n/a	("abc", 3)
2178	n/a	)
2179	n/a
2180	n/a	self.assertEqual(
2181	n/a	codecs.charmap_decode(b"\x00\x01\x02", "strict",
2182	n/a	{0: 'Aa', 1: 'Bb', 2: 'Cc'}),
2183	n/a	("AaBbCc", 3)
2184	n/a	)
2185	n/a
2186	n/a	self.assertEqual(
2187	n/a	codecs.charmap_decode(b"\x00\x01\x02", "strict",
2188	n/a	{0: '\U0010FFFF', 1: 'b', 2: 'c'}),
2189	n/a	("\U0010FFFFbc", 3)
2190	n/a	)
2191	n/a
2192	n/a	self.assertEqual(
2193	n/a	codecs.charmap_decode(b"\x00\x01\x02", "strict",
2194	n/a	{0: 'a', 1: 'b', 2: ''}),
2195	n/a	("ab", 3)
2196	n/a	)
2197	n/a
2198	n/a	self.assertRaises(UnicodeDecodeError,
2199	n/a	codecs.charmap_decode, b"\x00\x01\x02", "strict",
2200	n/a	{0: 'a', 1: 'b'}
2201	n/a	)
2202	n/a
2203	n/a	self.assertRaises(UnicodeDecodeError,
2204	n/a	codecs.charmap_decode, b"\x00\x01\x02", "strict",
2205	n/a	{0: 'a', 1: 'b', 2: None}
2206	n/a	)
2207	n/a
2208	n/a	# Issue #14850
2209	n/a	self.assertRaises(UnicodeDecodeError,
2210	n/a	codecs.charmap_decode, b"\x00\x01\x02", "strict",
2211	n/a	{0: 'a', 1: 'b', 2: '\ufffe'}
2212	n/a	)
2213	n/a
2214	n/a	self.assertEqual(
2215	n/a	codecs.charmap_decode(b"\x00\x01\x02", "replace",
2216	n/a	{0: 'a', 1: 'b'}),
2217	n/a	("ab\ufffd", 3)
2218	n/a	)
2219	n/a
2220	n/a	self.assertEqual(
2221	n/a	codecs.charmap_decode(b"\x00\x01\x02", "replace",
2222	n/a	{0: 'a', 1: 'b', 2: None}),
2223	n/a	("ab\ufffd", 3)
2224	n/a	)
2225	n/a
2226	n/a	# Issue #14850
2227	n/a	self.assertEqual(
2228	n/a	codecs.charmap_decode(b"\x00\x01\x02", "replace",
2229	n/a	{0: 'a', 1: 'b', 2: '\ufffe'}),
2230	n/a	("ab\ufffd", 3)
2231	n/a	)
2232	n/a
2233	n/a	self.assertEqual(
2234	n/a	codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
2235	n/a	{0: 'a', 1: 'b'}),
2236	n/a	("ab\\x02", 3)
2237	n/a	)
2238	n/a
2239	n/a	self.assertEqual(
2240	n/a	codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
2241	n/a	{0: 'a', 1: 'b', 2: None}),
2242	n/a	("ab\\x02", 3)
2243	n/a	)
2244	n/a
2245	n/a	# Issue #14850
2246	n/a	self.assertEqual(
2247	n/a	codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
2248	n/a	{0: 'a', 1: 'b', 2: '\ufffe'}),
2249	n/a	("ab\\x02", 3)
2250	n/a	)
2251	n/a
2252	n/a	self.assertEqual(
2253	n/a	codecs.charmap_decode(b"\x00\x01\x02", "ignore",
2254	n/a	{0: 'a', 1: 'b'}),
2255	n/a	("ab", 3)
2256	n/a	)
2257	n/a
2258	n/a	self.assertEqual(
2259	n/a	codecs.charmap_decode(b"\x00\x01\x02", "ignore",
2260	n/a	{0: 'a', 1: 'b', 2: None}),
2261	n/a	("ab", 3)
2262	n/a	)
2263	n/a
2264	n/a	# Issue #14850
2265	n/a	self.assertEqual(
2266	n/a	codecs.charmap_decode(b"\x00\x01\x02", "ignore",
2267	n/a	{0: 'a', 1: 'b', 2: '\ufffe'}),
2268	n/a	("ab", 3)
2269	n/a	)
2270	n/a
2271	n/a	allbytes = bytes(range(256))
2272	n/a	self.assertEqual(
2273	n/a	codecs.charmap_decode(allbytes, "ignore", {}),
2274	n/a	("", len(allbytes))
2275	n/a	)
2276	n/a
2277	n/a	def test_decode_with_int2int_map(self):
2278	n/a	a = ord('a')
2279	n/a	b = ord('b')
2280	n/a	c = ord('c')
2281	n/a
2282	n/a	self.assertEqual(
2283	n/a	codecs.charmap_decode(b"\x00\x01\x02", "strict",
2284	n/a	{0: a, 1: b, 2: c}),
2285	n/a	("abc", 3)
2286	n/a	)
2287	n/a
2288	n/a	# Issue #15379
2289	n/a	self.assertEqual(
2290	n/a	codecs.charmap_decode(b"\x00\x01\x02", "strict",
2291	n/a	{0: 0x10FFFF, 1: b, 2: c}),
2292	n/a	("\U0010FFFFbc", 3)
2293	n/a	)
2294	n/a
2295	n/a	self.assertEqual(
2296	n/a	codecs.charmap_decode(b"\x00\x01\x02", "strict",
2297	n/a	{0: sys.maxunicode, 1: b, 2: c}),
2298	n/a	(chr(sys.maxunicode) + "bc", 3)
2299	n/a	)
2300	n/a
2301	n/a	self.assertRaises(TypeError,
2302	n/a	codecs.charmap_decode, b"\x00\x01\x02", "strict",
2303	n/a	{0: sys.maxunicode + 1, 1: b, 2: c}
2304	n/a	)
2305	n/a
2306	n/a	self.assertRaises(UnicodeDecodeError,
2307	n/a	codecs.charmap_decode, b"\x00\x01\x02", "strict",
2308	n/a	{0: a, 1: b},
2309	n/a	)
2310	n/a
2311	n/a	self.assertRaises(UnicodeDecodeError,
2312	n/a	codecs.charmap_decode, b"\x00\x01\x02", "strict",
2313	n/a	{0: a, 1: b, 2: 0xFFFE},
2314	n/a	)
2315	n/a
2316	n/a	self.assertEqual(
2317	n/a	codecs.charmap_decode(b"\x00\x01\x02", "replace",
2318	n/a	{0: a, 1: b}),
2319	n/a	("ab\ufffd", 3)
2320	n/a	)
2321	n/a
2322	n/a	self.assertEqual(
2323	n/a	codecs.charmap_decode(b"\x00\x01\x02", "replace",
2324	n/a	{0: a, 1: b, 2: 0xFFFE}),
2325	n/a	("ab\ufffd", 3)
2326	n/a	)
2327	n/a
2328	n/a	self.assertEqual(
2329	n/a	codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
2330	n/a	{0: a, 1: b}),
2331	n/a	("ab\\x02", 3)
2332	n/a	)
2333	n/a
2334	n/a	self.assertEqual(
2335	n/a	codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
2336	n/a	{0: a, 1: b, 2: 0xFFFE}),
2337	n/a	("ab\\x02", 3)
2338	n/a	)
2339	n/a
2340	n/a	self.assertEqual(
2341	n/a	codecs.charmap_decode(b"\x00\x01\x02", "ignore",
2342	n/a	{0: a, 1: b}),
2343	n/a	("ab", 3)
2344	n/a	)
2345	n/a
2346	n/a	self.assertEqual(
2347	n/a	codecs.charmap_decode(b"\x00\x01\x02", "ignore",
2348	n/a	{0: a, 1: b, 2: 0xFFFE}),
2349	n/a	("ab", 3)
2350	n/a	)
2351	n/a
2352	n/a
2353	n/a	class WithStmtTest(unittest.TestCase):
2354	n/a	def test_encodedfile(self):
2355	n/a	f = io.BytesIO(b"\xc3\xbc")
2356	n/a	with codecs.EncodedFile(f, "latin-1", "utf-8") as ef:
2357	n/a	self.assertEqual(ef.read(), b"\xfc")
2358	n/a	self.assertTrue(f.closed)
2359	n/a
2360	n/a	def test_streamreaderwriter(self):
2361	n/a	f = io.BytesIO(b"\xc3\xbc")
2362	n/a	info = codecs.lookup("utf-8")
2363	n/a	with codecs.StreamReaderWriter(f, info.streamreader,
2364	n/a	info.streamwriter, 'strict') as srw:
2365	n/a	self.assertEqual(srw.read(), "\xfc")
2366	n/a
2367	n/a
2368	n/a	class TypesTest(unittest.TestCase):
2369	n/a	def test_decode_unicode(self):
2370	n/a	# Most decoders don't accept unicode input
2371	n/a	decoders = [
2372	n/a	codecs.utf_7_decode,
2373	n/a	codecs.utf_8_decode,
2374	n/a	codecs.utf_16_le_decode,
2375	n/a	codecs.utf_16_be_decode,
2376	n/a	codecs.utf_16_ex_decode,
2377	n/a	codecs.utf_32_decode,
2378	n/a	codecs.utf_32_le_decode,
2379	n/a	codecs.utf_32_be_decode,
2380	n/a	codecs.utf_32_ex_decode,
2381	n/a	codecs.latin_1_decode,
2382	n/a	codecs.ascii_decode,
2383	n/a	codecs.charmap_decode,
2384	n/a	]
2385	n/a	if hasattr(codecs, "mbcs_decode"):
2386	n/a	decoders.append(codecs.mbcs_decode)
2387	n/a	for decoder in decoders:
2388	n/a	self.assertRaises(TypeError, decoder, "xxx")
2389	n/a
2390	n/a	def test_unicode_escape(self):
2391	n/a	# Escape-decoding a unicode string is supported and gives the same
2392	n/a	# result as decoding the equivalent ASCII bytes string.
2393	n/a	self.assertEqual(codecs.unicode_escape_decode(r"\u1234"), ("\u1234", 6))
2394	n/a	self.assertEqual(codecs.unicode_escape_decode(br"\u1234"), ("\u1234", 6))
2395	n/a	self.assertEqual(codecs.raw_unicode_escape_decode(r"\u1234"), ("\u1234", 6))
2396	n/a	self.assertEqual(codecs.raw_unicode_escape_decode(br"\u1234"), ("\u1234", 6))
2397	n/a
2398	n/a	self.assertRaises(UnicodeDecodeError, codecs.unicode_escape_decode, br"\U00110000")
2399	n/a	self.assertEqual(codecs.unicode_escape_decode(r"\U00110000", "replace"), ("\ufffd", 10))
2400	n/a	self.assertEqual(codecs.unicode_escape_decode(r"\U00110000", "backslashreplace"),
2401	n/a	(r"\x5c\x55\x30\x30\x31\x31\x30\x30\x30\x30", 10))
2402	n/a
2403	n/a	self.assertRaises(UnicodeDecodeError, codecs.raw_unicode_escape_decode, br"\U00110000")
2404	n/a	self.assertEqual(codecs.raw_unicode_escape_decode(r"\U00110000", "replace"), ("\ufffd", 10))
2405	n/a	self.assertEqual(codecs.raw_unicode_escape_decode(r"\U00110000", "backslashreplace"),
2406	n/a	(r"\x5c\x55\x30\x30\x31\x31\x30\x30\x30\x30", 10))
2407	n/a
2408	n/a
2409	n/a	class UnicodeEscapeTest(unittest.TestCase):
2410	n/a	def test_empty(self):
2411	n/a	self.assertEqual(codecs.unicode_escape_encode(""), (b"", 0))
2412	n/a	self.assertEqual(codecs.unicode_escape_decode(b""), ("", 0))
2413	n/a
2414	n/a	def test_raw_encode(self):
2415	n/a	encode = codecs.unicode_escape_encode
2416	n/a	for b in range(32, 127):
2417	n/a	if b != b'\\'[0]:
2418	n/a	self.assertEqual(encode(chr(b)), (bytes([b]), 1))
2419	n/a
2420	n/a	def test_raw_decode(self):
2421	n/a	decode = codecs.unicode_escape_decode
2422	n/a	for b in range(256):
2423	n/a	if b != b'\\'[0]:
2424	n/a	self.assertEqual(decode(bytes([b]) + b'0'), (chr(b) + '0', 2))
2425	n/a
2426	n/a	def test_escape_encode(self):
2427	n/a	encode = codecs.unicode_escape_encode
2428	n/a	check = coding_checker(self, encode)
2429	n/a	check('\t', br'\t')
2430	n/a	check('\n', br'\n')
2431	n/a	check('\r', br'\r')
2432	n/a	check('\\', br'\\')
2433	n/a	for b in range(32):
2434	n/a	if chr(b) not in '\t\n\r':
2435	n/a	check(chr(b), ('\\x%02x' % b).encode())
2436	n/a	for b in range(127, 256):
2437	n/a	check(chr(b), ('\\x%02x' % b).encode())
2438	n/a	check('\u20ac', br'\u20ac')
2439	n/a	check('\U0001d120', br'\U0001d120')
2440	n/a
2441	n/a	def test_escape_decode(self):
2442	n/a	decode = codecs.unicode_escape_decode
2443	n/a	check = coding_checker(self, decode)
2444	n/a	check(b"[\\\n]", "[]")
2445	n/a	check(br'[\"]', '["]')
2446	n/a	check(br"[\']", "[']")
2447	n/a	check(br"[\\]", r"[\]")
2448	n/a	check(br"[\a]", "[\x07]")
2449	n/a	check(br"[\b]", "[\x08]")
2450	n/a	check(br"[\t]", "[\x09]")
2451	n/a	check(br"[\n]", "[\x0a]")
2452	n/a	check(br"[\v]", "[\x0b]")
2453	n/a	check(br"[\f]", "[\x0c]")
2454	n/a	check(br"[\r]", "[\x0d]")
2455	n/a	check(br"[\7]", "[\x07]")
2456	n/a	check(br"[\78]", "[\x078]")
2457	n/a	check(br"[\41]", "[!]")
2458	n/a	check(br"[\418]", "[!8]")
2459	n/a	check(br"[\101]", "[A]")
2460	n/a	check(br"[\1010]", "[A0]")
2461	n/a	check(br"[\x41]", "[A]")
2462	n/a	check(br"[\x410]", "[A0]")
2463	n/a	check(br"\u20ac", "\u20ac")
2464	n/a	check(br"\U0001d120", "\U0001d120")
2465	n/a	for i in range(97, 123):
2466	n/a	b = bytes([i])
2467	n/a	if b not in b'abfnrtuvx':
2468	n/a	with self.assertWarns(DeprecationWarning):
2469	n/a	check(b"\\" + b, "\\" + chr(i))
2470	n/a	if b.upper() not in b'UN':
2471	n/a	with self.assertWarns(DeprecationWarning):
2472	n/a	check(b"\\" + b.upper(), "\\" + chr(i-32))
2473	n/a	with self.assertWarns(DeprecationWarning):
2474	n/a	check(br"\8", "\\8")
2475	n/a	with self.assertWarns(DeprecationWarning):
2476	n/a	check(br"\9", "\\9")
2477	n/a
2478	n/a	def test_decode_errors(self):
2479	n/a	decode = codecs.unicode_escape_decode
2480	n/a	for c, d in (b'x', 2), (b'u', 4), (b'U', 4):
2481	n/a	for i in range(d):
2482	n/a	self.assertRaises(UnicodeDecodeError, decode,
2483	n/a	b"\\" + c + b"0"*i)
2484	n/a	self.assertRaises(UnicodeDecodeError, decode,
2485	n/a	b"[\\" + c + b"0"*i + b"]")
2486	n/a	data = b"[\\" + c + b"0"i + b"]\\" + c + b"0"i
2487	n/a	self.assertEqual(decode(data, "ignore"), ("[]", len(data)))
2488	n/a	self.assertEqual(decode(data, "replace"),
2489	n/a	("[\ufffd]\ufffd", len(data)))
2490	n/a	self.assertRaises(UnicodeDecodeError, decode, br"\U00110000")
2491	n/a	self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10))
2492	n/a	self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10))
2493	n/a
2494	n/a
2495	n/a	class RawUnicodeEscapeTest(unittest.TestCase):
2496	n/a	def test_empty(self):
2497	n/a	self.assertEqual(codecs.raw_unicode_escape_encode(""), (b"", 0))
2498	n/a	self.assertEqual(codecs.raw_unicode_escape_decode(b""), ("", 0))
2499	n/a
2500	n/a	def test_raw_encode(self):
2501	n/a	encode = codecs.raw_unicode_escape_encode
2502	n/a	for b in range(256):
2503	n/a	self.assertEqual(encode(chr(b)), (bytes([b]), 1))
2504	n/a
2505	n/a	def test_raw_decode(self):
2506	n/a	decode = codecs.raw_unicode_escape_decode
2507	n/a	for b in range(256):
2508	n/a	self.assertEqual(decode(bytes([b]) + b'0'), (chr(b) + '0', 2))
2509	n/a
2510	n/a	def test_escape_encode(self):
2511	n/a	encode = codecs.raw_unicode_escape_encode
2512	n/a	check = coding_checker(self, encode)
2513	n/a	for b in range(256):
2514	n/a	if b not in b'uU':
2515	n/a	check('\\' + chr(b), b'\\' + bytes([b]))
2516	n/a	check('\u20ac', br'\u20ac')
2517	n/a	check('\U0001d120', br'\U0001d120')
2518	n/a
2519	n/a	def test_escape_decode(self):
2520	n/a	decode = codecs.raw_unicode_escape_decode
2521	n/a	check = coding_checker(self, decode)
2522	n/a	for b in range(256):
2523	n/a	if b not in b'uU':
2524	n/a	check(b'\\' + bytes([b]), '\\' + chr(b))
2525	n/a	check(br"\u20ac", "\u20ac")
2526	n/a	check(br"\U0001d120", "\U0001d120")
2527	n/a
2528	n/a	def test_decode_errors(self):
2529	n/a	decode = codecs.raw_unicode_escape_decode
2530	n/a	for c, d in (b'u', 4), (b'U', 4):
2531	n/a	for i in range(d):
2532	n/a	self.assertRaises(UnicodeDecodeError, decode,
2533	n/a	b"\\" + c + b"0"*i)
2534	n/a	self.assertRaises(UnicodeDecodeError, decode,
2535	n/a	b"[\\" + c + b"0"*i + b"]")
2536	n/a	data = b"[\\" + c + b"0"i + b"]\\" + c + b"0"i
2537	n/a	self.assertEqual(decode(data, "ignore"), ("[]", len(data)))
2538	n/a	self.assertEqual(decode(data, "replace"),
2539	n/a	("[\ufffd]\ufffd", len(data)))
2540	n/a	self.assertRaises(UnicodeDecodeError, decode, br"\U00110000")
2541	n/a	self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10))
2542	n/a	self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10))
2543	n/a
2544	n/a
2545	n/a	class EscapeEncodeTest(unittest.TestCase):
2546	n/a
2547	n/a	def test_escape_encode(self):
2548	n/a	tests = [
2549	n/a	(b'', (b'', 0)),
2550	n/a	(b'foobar', (b'foobar', 6)),
2551	n/a	(b'spam\0eggs', (b'spam\\x00eggs', 9)),
2552	n/a	(b'a\'b', (b"a\\'b", 3)),
2553	n/a	(b'b\\c', (b'b\\\\c', 3)),
2554	n/a	(b'c\nd', (b'c\\nd', 3)),
2555	n/a	(b'd\re', (b'd\\re', 3)),
2556	n/a	(b'f\x7fg', (b'f\\x7fg', 3)),
2557	n/a	]
2558	n/a	for data, output in tests:
2559	n/a	with self.subTest(data=data):
2560	n/a	self.assertEqual(codecs.escape_encode(data), output)
2561	n/a	self.assertRaises(TypeError, codecs.escape_encode, 'spam')
2562	n/a	self.assertRaises(TypeError, codecs.escape_encode, bytearray(b'spam'))
2563	n/a
2564	n/a
2565	n/a	class SurrogateEscapeTest(unittest.TestCase):
2566	n/a
2567	n/a	def test_utf8(self):
2568	n/a	# Bad byte
2569	n/a	self.assertEqual(b"foo\x80bar".decode("utf-8", "surrogateescape"),
2570	n/a	"foo\udc80bar")
2571	n/a	self.assertEqual("foo\udc80bar".encode("utf-8", "surrogateescape"),
2572	n/a	b"foo\x80bar")
2573	n/a	# bad-utf-8 encoded surrogate
2574	n/a	self.assertEqual(b"\xed\xb0\x80".decode("utf-8", "surrogateescape"),
2575	n/a	"\udced\udcb0\udc80")
2576	n/a	self.assertEqual("\udced\udcb0\udc80".encode("utf-8", "surrogateescape"),
2577	n/a	b"\xed\xb0\x80")
2578	n/a
2579	n/a	def test_ascii(self):
2580	n/a	# bad byte
2581	n/a	self.assertEqual(b"foo\x80bar".decode("ascii", "surrogateescape"),
2582	n/a	"foo\udc80bar")
2583	n/a	self.assertEqual("foo\udc80bar".encode("ascii", "surrogateescape"),
2584	n/a	b"foo\x80bar")
2585	n/a
2586	n/a	def test_charmap(self):
2587	n/a	# bad byte: \xa5 is unmapped in iso-8859-3
2588	n/a	self.assertEqual(b"foo\xa5bar".decode("iso-8859-3", "surrogateescape"),
2589	n/a	"foo\udca5bar")
2590	n/a	self.assertEqual("foo\udca5bar".encode("iso-8859-3", "surrogateescape"),
2591	n/a	b"foo\xa5bar")
2592	n/a
2593	n/a	def test_latin1(self):
2594	n/a	# Issue6373
2595	n/a	self.assertEqual("\udce4\udceb\udcef\udcf6\udcfc".encode("latin-1", "surrogateescape"),
2596	n/a	b"\xe4\xeb\xef\xf6\xfc")
2597	n/a
2598	n/a
2599	n/a	class BomTest(unittest.TestCase):
2600	n/a	def test_seek0(self):
2601	n/a	data = "1234567890"
2602	n/a	tests = ("utf-16",
2603	n/a	"utf-16-le",
2604	n/a	"utf-16-be",
2605	n/a	"utf-32",
2606	n/a	"utf-32-le",
2607	n/a	"utf-32-be")
2608	n/a	self.addCleanup(support.unlink, support.TESTFN)
2609	n/a	for encoding in tests:
2610	n/a	# Check if the BOM is written only once
2611	n/a	with codecs.open(support.TESTFN, 'w+', encoding=encoding) as f:
2612	n/a	f.write(data)
2613	n/a	f.write(data)
2614	n/a	f.seek(0)
2615	n/a	self.assertEqual(f.read(), data * 2)
2616	n/a	f.seek(0)
2617	n/a	self.assertEqual(f.read(), data * 2)
2618	n/a
2619	n/a	# Check that the BOM is written after a seek(0)
2620	n/a	with codecs.open(support.TESTFN, 'w+', encoding=encoding) as f:
2621	n/a	f.write(data[0])
2622	n/a	self.assertNotEqual(f.tell(), 0)
2623	n/a	f.seek(0)
2624	n/a	f.write(data)
2625	n/a	f.seek(0)
2626	n/a	self.assertEqual(f.read(), data)
2627	n/a
2628	n/a	# (StreamWriter) Check that the BOM is written after a seek(0)
2629	n/a	with codecs.open(support.TESTFN, 'w+', encoding=encoding) as f:
2630	n/a	f.writer.write(data[0])
2631	n/a	self.assertNotEqual(f.writer.tell(), 0)
2632	n/a	f.writer.seek(0)
2633	n/a	f.writer.write(data)
2634	n/a	f.seek(0)
2635	n/a	self.assertEqual(f.read(), data)
2636	n/a
2637	n/a	# Check that the BOM is not written after a seek() at a position
2638	n/a	# different than the start
2639	n/a	with codecs.open(support.TESTFN, 'w+', encoding=encoding) as f:
2640	n/a	f.write(data)
2641	n/a	f.seek(f.tell())
2642	n/a	f.write(data)
2643	n/a	f.seek(0)
2644	n/a	self.assertEqual(f.read(), data * 2)
2645	n/a
2646	n/a	# (StreamWriter) Check that the BOM is not written after a seek()
2647	n/a	# at a position different than the start
2648	n/a	with codecs.open(support.TESTFN, 'w+', encoding=encoding) as f:
2649	n/a	f.writer.write(data)
2650	n/a	f.writer.seek(f.writer.tell())
2651	n/a	f.writer.write(data)
2652	n/a	f.seek(0)
2653	n/a	self.assertEqual(f.read(), data * 2)
2654	n/a
2655	n/a
2656	n/a	bytes_transform_encodings = [
2657	n/a	"base64_codec",
2658	n/a	"uu_codec",
2659	n/a	"quopri_codec",
2660	n/a	"hex_codec",
2661	n/a	]
2662	n/a
2663	n/a	transform_aliases = {
2664	n/a	"base64_codec": ["base64", "base_64"],
2665	n/a	"uu_codec": ["uu"],
2666	n/a	"quopri_codec": ["quopri", "quoted_printable", "quotedprintable"],
2667	n/a	"hex_codec": ["hex"],
2668	n/a	"rot_13": ["rot13"],
2669	n/a	}
2670	n/a
2671	n/a	try:
2672	n/a	import zlib
2673	n/a	except ImportError:
2674	n/a	zlib = None
2675	n/a	else:
2676	n/a	bytes_transform_encodings.append("zlib_codec")
2677	n/a	transform_aliases["zlib_codec"] = ["zip", "zlib"]
2678	n/a	try:
2679	n/a	import bz2
2680	n/a	except ImportError:
2681	n/a	pass
2682	n/a	else:
2683	n/a	bytes_transform_encodings.append("bz2_codec")
2684	n/a	transform_aliases["bz2_codec"] = ["bz2"]
2685	n/a
2686	n/a
2687	n/a	class TransformCodecTest(unittest.TestCase):
2688	n/a
2689	n/a	def test_basics(self):
2690	n/a	binput = bytes(range(256))
2691	n/a	for encoding in bytes_transform_encodings:
2692	n/a	with self.subTest(encoding=encoding):
2693	n/a	# generic codecs interface
2694	n/a	(o, size) = codecs.getencoder(encoding)(binput)
2695	n/a	self.assertEqual(size, len(binput))
2696	n/a	(i, size) = codecs.getdecoder(encoding)(o)
2697	n/a	self.assertEqual(size, len(o))
2698	n/a	self.assertEqual(i, binput)
2699	n/a
2700	n/a	def test_read(self):
2701	n/a	for encoding in bytes_transform_encodings:
2702	n/a	with self.subTest(encoding=encoding):
2703	n/a	sin = codecs.encode(b"\x80", encoding)
2704	n/a	reader = codecs.getreader(encoding)(io.BytesIO(sin))
2705	n/a	sout = reader.read()
2706	n/a	self.assertEqual(sout, b"\x80")
2707	n/a
2708	n/a	def test_readline(self):
2709	n/a	for encoding in bytes_transform_encodings:
2710	n/a	with self.subTest(encoding=encoding):
2711	n/a	sin = codecs.encode(b"\x80", encoding)
2712	n/a	reader = codecs.getreader(encoding)(io.BytesIO(sin))
2713	n/a	sout = reader.readline()
2714	n/a	self.assertEqual(sout, b"\x80")
2715	n/a
2716	n/a	def test_buffer_api_usage(self):
2717	n/a	# We check all the transform codecs accept memoryview input
2718	n/a	# for encoding and decoding
2719	n/a	# and also that they roundtrip correctly
2720	n/a	original = b"12345\x80"
2721	n/a	for encoding in bytes_transform_encodings:
2722	n/a	with self.subTest(encoding=encoding):
2723	n/a	data = original
2724	n/a	view = memoryview(data)
2725	n/a	data = codecs.encode(data, encoding)
2726	n/a	view_encoded = codecs.encode(view, encoding)
2727	n/a	self.assertEqual(view_encoded, data)
2728	n/a	view = memoryview(data)
2729	n/a	data = codecs.decode(data, encoding)
2730	n/a	self.assertEqual(data, original)
2731	n/a	view_decoded = codecs.decode(view, encoding)
2732	n/a	self.assertEqual(view_decoded, data)
2733	n/a
2734	n/a	def test_text_to_binary_blacklists_binary_transforms(self):
2735	n/a	# Check binary -> binary codecs give a good error for str input
2736	n/a	bad_input = "bad input type"
2737	n/a	for encoding in bytes_transform_encodings:
2738	n/a	with self.subTest(encoding=encoding):
2739	n/a	fmt = (r"{!r} is not a text encoding; "
2740	n/a	r"use codecs.encode to handle arbitrary codecs")
2741	n/a	msg = fmt.format(encoding)
2742	n/a	with self.assertRaisesRegex(LookupError, msg) as failure:
2743	n/a	bad_input.encode(encoding)
2744	n/a	self.assertIsNone(failure.exception.__cause__)
2745	n/a
2746	n/a	def test_text_to_binary_blacklists_text_transforms(self):
2747	n/a	# Check str.encode gives a good error message for str -> str codecs
2748	n/a	msg = (r"^'rot_13' is not a text encoding; "
2749	n/a	r"use codecs.encode to handle arbitrary codecs")
2750	n/a	with self.assertRaisesRegex(LookupError, msg):
2751	n/a	"just an example message".encode("rot_13")
2752	n/a
2753	n/a	def test_binary_to_text_blacklists_binary_transforms(self):
2754	n/a	# Check bytes.decode and bytearray.decode give a good error
2755	n/a	# message for binary -> binary codecs
2756	n/a	data = b"encode first to ensure we meet any format restrictions"
2757	n/a	for encoding in bytes_transform_encodings:
2758	n/a	with self.subTest(encoding=encoding):
2759	n/a	encoded_data = codecs.encode(data, encoding)
2760	n/a	fmt = (r"{!r} is not a text encoding; "
2761	n/a	r"use codecs.decode to handle arbitrary codecs")
2762	n/a	msg = fmt.format(encoding)
2763	n/a	with self.assertRaisesRegex(LookupError, msg):
2764	n/a	encoded_data.decode(encoding)
2765	n/a	with self.assertRaisesRegex(LookupError, msg):
2766	n/a	bytearray(encoded_data).decode(encoding)
2767	n/a
2768	n/a	def test_binary_to_text_blacklists_text_transforms(self):
2769	n/a	# Check str -> str codec gives a good error for binary input
2770	n/a	for bad_input in (b"immutable", bytearray(b"mutable")):
2771	n/a	with self.subTest(bad_input=bad_input):
2772	n/a	msg = (r"^'rot_13' is not a text encoding; "
2773	n/a	r"use codecs.decode to handle arbitrary codecs")
2774	n/a	with self.assertRaisesRegex(LookupError, msg) as failure:
2775	n/a	bad_input.decode("rot_13")
2776	n/a	self.assertIsNone(failure.exception.__cause__)
2777	n/a
2778	n/a	@unittest.skipUnless(zlib, "Requires zlib support")
2779	n/a	def test_custom_zlib_error_is_wrapped(self):
2780	n/a	# Check zlib codec gives a good error for malformed input
2781	n/a	msg = "^decoding with 'zlib_codec' codec failed"
2782	n/a	with self.assertRaisesRegex(Exception, msg) as failure:
2783	n/a	codecs.decode(b"hello", "zlib_codec")
2784	n/a	self.assertIsInstance(failure.exception.__cause__,
2785	n/a	type(failure.exception))
2786	n/a
2787	n/a	def test_custom_hex_error_is_wrapped(self):
2788	n/a	# Check hex codec gives a good error for malformed input
2789	n/a	msg = "^decoding with 'hex_codec' codec failed"
2790	n/a	with self.assertRaisesRegex(Exception, msg) as failure:
2791	n/a	codecs.decode(b"hello", "hex_codec")
2792	n/a	self.assertIsInstance(failure.exception.__cause__,
2793	n/a	type(failure.exception))
2794	n/a
2795	n/a	# Unfortunately, the bz2 module throws OSError, which the codec
2796	n/a	# machinery currently can't wrap :(
2797	n/a
2798	n/a	# Ensure codec aliases from http://bugs.python.org/issue7475 work
2799	n/a	def test_aliases(self):
2800	n/a	for codec_name, aliases in transform_aliases.items():
2801	n/a	expected_name = codecs.lookup(codec_name).name
2802	n/a	for alias in aliases:
2803	n/a	with self.subTest(alias=alias):
2804	n/a	info = codecs.lookup(alias)
2805	n/a	self.assertEqual(info.name, expected_name)
2806	n/a
2807	n/a	def test_quopri_stateless(self):
2808	n/a	# Should encode with quotetabs=True
2809	n/a	encoded = codecs.encode(b"space tab\teol \n", "quopri-codec")
2810	n/a	self.assertEqual(encoded, b"space=20tab=09eol=20\n")
2811	n/a	# But should still support unescaped tabs and spaces
2812	n/a	unescaped = b"space tab eol\n"
2813	n/a	self.assertEqual(codecs.decode(unescaped, "quopri-codec"), unescaped)
2814	n/a
2815	n/a	def test_uu_invalid(self):
2816	n/a	# Missing "begin" line
2817	n/a	self.assertRaises(ValueError, codecs.decode, b"", "uu-codec")
2818	n/a
2819	n/a
2820	n/a	# The codec system tries to wrap exceptions in order to ensure the error
2821	n/a	# mentions the operation being performed and the codec involved. We
2822	n/a	# currently only want this to happen for relatively stateless
2823	n/a	# exceptions, where the only significant information they contain is their
2824	n/a	# type and a single str argument.
2825	n/a
2826	n/a	# Use a local codec registry to avoid appearing to leak objects when
2827	n/a	# registering multiple search functions
2828	n/a	_TEST_CODECS = {}
2829	n/a
2830	n/a	def _get_test_codec(codec_name):
2831	n/a	return _TEST_CODECS.get(codec_name)
2832	n/a	codecs.register(_get_test_codec) # Returns None, not usable as a decorator
2833	n/a
2834	n/a	try:
2835	n/a	# Issue #22166: Also need to clear the internal cache in CPython
2836	n/a	from _codecs import _forget_codec
2837	n/a	except ImportError:
2838	n/a	def _forget_codec(codec_name):
2839	n/a	pass
2840	n/a
2841	n/a
2842	n/a	class ExceptionChainingTest(unittest.TestCase):
2843	n/a
2844	n/a	def setUp(self):
2845	n/a	# There's no way to unregister a codec search function, so we just
2846	n/a	# ensure we render this one fairly harmless after the test
2847	n/a	# case finishes by using the test case repr as the codec name
2848	n/a	# The codecs module normalizes codec names, although this doesn't
2849	n/a	# appear to be formally documented...
2850	n/a	# We also make sure we use a truly unique id for the custom codec
2851	n/a	# to avoid issues with the codec cache when running these tests
2852	n/a	# multiple times (e.g. when hunting for refleaks)
2853	n/a	unique_id = repr(self) + str(id(self))
2854	n/a	self.codec_name = encodings.normalize_encoding(unique_id).lower()
2855	n/a
2856	n/a	# We store the object to raise on the instance because of a bad
2857	n/a	# interaction between the codec caching (which means we can't
2858	n/a	# recreate the codec entry) and regrtest refleak hunting (which
2859	n/a	# runs the same test instance multiple times). This means we
2860	n/a	# need to ensure the codecs call back in to the instance to find
2861	n/a	# out which exception to raise rather than binding them in a
2862	n/a	# closure to an object that may change on the next run
2863	n/a	self.obj_to_raise = RuntimeError
2864	n/a
2865	n/a	def tearDown(self):
2866	n/a	_TEST_CODECS.pop(self.codec_name, None)
2867	n/a	# Issue #22166: Also pop from caches to avoid appearance of ref leaks
2868	n/a	encodings._cache.pop(self.codec_name, None)
2869	n/a	try:
2870	n/a	_forget_codec(self.codec_name)
2871	n/a	except KeyError:
2872	n/a	pass
2873	n/a
2874	n/a	def set_codec(self, encode, decode):
2875	n/a	codec_info = codecs.CodecInfo(encode, decode,
2876	n/a	name=self.codec_name)
2877	n/a	_TEST_CODECS[self.codec_name] = codec_info
2878	n/a
2879	n/a	@contextlib.contextmanager
2880	n/a	def assertWrapped(self, operation, exc_type, msg):
2881	n/a	full_msg = r"{} with {!r} codec failed ${}: {}$".format(
2882	n/a	operation, self.codec_name, exc_type.__name__, msg)
2883	n/a	with self.assertRaisesRegex(exc_type, full_msg) as caught:
2884	n/a	yield caught
2885	n/a	self.assertIsInstance(caught.exception.__cause__, exc_type)
2886	n/a	self.assertIsNotNone(caught.exception.__cause__.__traceback__)
2887	n/a
2888	n/a	def raise_obj(self, args, *kwds):
2889	n/a	# Helper to dynamically change the object raised by a test codec
2890	n/a	raise self.obj_to_raise
2891	n/a
2892	n/a	def check_wrapped(self, obj_to_raise, msg, exc_type=RuntimeError):
2893	n/a	self.obj_to_raise = obj_to_raise
2894	n/a	self.set_codec(self.raise_obj, self.raise_obj)
2895	n/a	with self.assertWrapped("encoding", exc_type, msg):
2896	n/a	"str_input".encode(self.codec_name)
2897	n/a	with self.assertWrapped("encoding", exc_type, msg):
2898	n/a	codecs.encode("str_input", self.codec_name)
2899	n/a	with self.assertWrapped("decoding", exc_type, msg):
2900	n/a	b"bytes input".decode(self.codec_name)
2901	n/a	with self.assertWrapped("decoding", exc_type, msg):
2902	n/a	codecs.decode(b"bytes input", self.codec_name)
2903	n/a
2904	n/a	def test_raise_by_type(self):
2905	n/a	self.check_wrapped(RuntimeError, "")
2906	n/a
2907	n/a	def test_raise_by_value(self):
2908	n/a	msg = "This should be wrapped"
2909	n/a	self.check_wrapped(RuntimeError(msg), msg)
2910	n/a
2911	n/a	def test_raise_grandchild_subclass_exact_size(self):
2912	n/a	msg = "This should be wrapped"
2913	n/a	class MyRuntimeError(RuntimeError):
2914	n/a	__slots__ = ()
2915	n/a	self.check_wrapped(MyRuntimeError(msg), msg, MyRuntimeError)
2916	n/a
2917	n/a	def test_raise_subclass_with_weakref_support(self):
2918	n/a	msg = "This should be wrapped"
2919	n/a	class MyRuntimeError(RuntimeError):
2920	n/a	pass
2921	n/a	self.check_wrapped(MyRuntimeError(msg), msg, MyRuntimeError)
2922	n/a
2923	n/a	def check_not_wrapped(self, obj_to_raise, msg):
2924	n/a	def raise_obj(args, *kwds):
2925	n/a	raise obj_to_raise
2926	n/a	self.set_codec(raise_obj, raise_obj)
2927	n/a	with self.assertRaisesRegex(RuntimeError, msg):
2928	n/a	"str input".encode(self.codec_name)
2929	n/a	with self.assertRaisesRegex(RuntimeError, msg):
2930	n/a	codecs.encode("str input", self.codec_name)
2931	n/a	with self.assertRaisesRegex(RuntimeError, msg):
2932	n/a	b"bytes input".decode(self.codec_name)
2933	n/a	with self.assertRaisesRegex(RuntimeError, msg):
2934	n/a	codecs.decode(b"bytes input", self.codec_name)
2935	n/a
2936	n/a	def test_init_override_is_not_wrapped(self):
2937	n/a	class CustomInit(RuntimeError):
2938	n/a	def __init__(self):
2939	n/a	pass
2940	n/a	self.check_not_wrapped(CustomInit, "")
2941	n/a
2942	n/a	def test_new_override_is_not_wrapped(self):
2943	n/a	class CustomNew(RuntimeError):
2944	n/a	def __new__(cls):
2945	n/a	return super().__new__(cls)
2946	n/a	self.check_not_wrapped(CustomNew, "")
2947	n/a
2948	n/a	def test_instance_attribute_is_not_wrapped(self):
2949	n/a	msg = "This should NOT be wrapped"
2950	n/a	exc = RuntimeError(msg)
2951	n/a	exc.attr = 1
2952	n/a	self.check_not_wrapped(exc, "^{}$".format(msg))
2953	n/a
2954	n/a	def test_non_str_arg_is_not_wrapped(self):
2955	n/a	self.check_not_wrapped(RuntimeError(1), "1")
2956	n/a
2957	n/a	def test_multiple_args_is_not_wrapped(self):
2958	n/a	msg_re = r"^$'a', 'b', 'c'$$"
2959	n/a	self.check_not_wrapped(RuntimeError('a', 'b', 'c'), msg_re)
2960	n/a
2961	n/a	# http://bugs.python.org/issue19609
2962	n/a	def test_codec_lookup_failure_not_wrapped(self):
2963	n/a	msg = "^unknown encoding: {}$".format(self.codec_name)
2964	n/a	# The initial codec lookup should not be wrapped
2965	n/a	with self.assertRaisesRegex(LookupError, msg):
2966	n/a	"str input".encode(self.codec_name)
2967	n/a	with self.assertRaisesRegex(LookupError, msg):
2968	n/a	codecs.encode("str input", self.codec_name)
2969	n/a	with self.assertRaisesRegex(LookupError, msg):
2970	n/a	b"bytes input".decode(self.codec_name)
2971	n/a	with self.assertRaisesRegex(LookupError, msg):
2972	n/a	codecs.decode(b"bytes input", self.codec_name)
2973	n/a
2974	n/a	def test_unflagged_non_text_codec_handling(self):
2975	n/a	# The stdlib non-text codecs are now marked so they're
2976	n/a	# pre-emptively skipped by the text model related methods
2977	n/a	# However, third party codecs won't be flagged, so we still make
2978	n/a	# sure the case where an inappropriate output type is produced is
2979	n/a	# handled appropriately
2980	n/a	def encode_to_str(args, *kwds):
2981	n/a	return "not bytes!", 0
2982	n/a	def decode_to_bytes(args, *kwds):
2983	n/a	return b"not str!", 0
2984	n/a	self.set_codec(encode_to_str, decode_to_bytes)
2985	n/a	# No input or output type checks on the codecs module functions
2986	n/a	encoded = codecs.encode(None, self.codec_name)
2987	n/a	self.assertEqual(encoded, "not bytes!")
2988	n/a	decoded = codecs.decode(None, self.codec_name)
2989	n/a	self.assertEqual(decoded, b"not str!")
2990	n/a	# Text model methods should complain
2991	n/a	fmt = (r"^{!r} encoder returned 'str' instead of 'bytes'; "
2992	n/a	r"use codecs.encode to encode to arbitrary types$")
2993	n/a	msg = fmt.format(self.codec_name)
2994	n/a	with self.assertRaisesRegex(TypeError, msg):
2995	n/a	"str_input".encode(self.codec_name)
2996	n/a	fmt = (r"^{!r} decoder returned 'bytes' instead of 'str'; "
2997	n/a	r"use codecs.decode to decode to arbitrary types$")
2998	n/a	msg = fmt.format(self.codec_name)
2999	n/a	with self.assertRaisesRegex(TypeError, msg):
3000	n/a	b"bytes input".decode(self.codec_name)
3001	n/a
3002	n/a
3003	n/a
3004	n/a	@unittest.skipUnless(sys.platform == 'win32',
3005	n/a	'code pages are specific to Windows')
3006	n/a	class CodePageTest(unittest.TestCase):
3007	n/a	# CP_UTF8 is already tested by CP65001Test
3008	n/a	CP_UTF8 = 65001
3009	n/a
3010	n/a	def test_invalid_code_page(self):
3011	n/a	self.assertRaises(ValueError, codecs.code_page_encode, -1, 'a')
3012	n/a	self.assertRaises(ValueError, codecs.code_page_decode, -1, b'a')
3013	n/a	self.assertRaises(OSError, codecs.code_page_encode, 123, 'a')
3014	n/a	self.assertRaises(OSError, codecs.code_page_decode, 123, b'a')
3015	n/a
3016	n/a	def test_code_page_name(self):
3017	n/a	self.assertRaisesRegex(UnicodeEncodeError, 'cp932',
3018	n/a	codecs.code_page_encode, 932, '\xff')
3019	n/a	self.assertRaisesRegex(UnicodeDecodeError, 'cp932',
3020	n/a	codecs.code_page_decode, 932, b'\x81\x00', 'strict', True)
3021	n/a	self.assertRaisesRegex(UnicodeDecodeError, 'CP_UTF8',
3022	n/a	codecs.code_page_decode, self.CP_UTF8, b'\xff', 'strict', True)
3023	n/a
3024	n/a	def check_decode(self, cp, tests):
3025	n/a	for raw, errors, expected in tests:
3026	n/a	if expected is not None:
3027	n/a	try:
3028	n/a	decoded = codecs.code_page_decode(cp, raw, errors, True)
3029	n/a	except UnicodeDecodeError as err:
3030	n/a	self.fail('Unable to decode %a from "cp%s" with '
3031	n/a	'errors=%r: %s' % (raw, cp, errors, err))
3032	n/a	self.assertEqual(decoded[0], expected,
3033	n/a	'%a.decode("cp%s", %r)=%a != %a'
3034	n/a	% (raw, cp, errors, decoded[0], expected))
3035	n/a	# assert 0 <= decoded[1] <= len(raw)
3036	n/a	self.assertGreaterEqual(decoded[1], 0)
3037	n/a	self.assertLessEqual(decoded[1], len(raw))
3038	n/a	else:
3039	n/a	self.assertRaises(UnicodeDecodeError,
3040	n/a	codecs.code_page_decode, cp, raw, errors, True)
3041	n/a
3042	n/a	def check_encode(self, cp, tests):
3043	n/a	for text, errors, expected in tests:
3044	n/a	if expected is not None:
3045	n/a	try:
3046	n/a	encoded = codecs.code_page_encode(cp, text, errors)
3047	n/a	except UnicodeEncodeError as err:
3048	n/a	self.fail('Unable to encode %a to "cp%s" with '
3049	n/a	'errors=%r: %s' % (text, cp, errors, err))
3050	n/a	self.assertEqual(encoded[0], expected,
3051	n/a	'%a.encode("cp%s", %r)=%a != %a'
3052	n/a	% (text, cp, errors, encoded[0], expected))
3053	n/a	self.assertEqual(encoded[1], len(text))
3054	n/a	else:
3055	n/a	self.assertRaises(UnicodeEncodeError,
3056	n/a	codecs.code_page_encode, cp, text, errors)
3057	n/a
3058	n/a	def test_cp932(self):
3059	n/a	self.check_encode(932, (
3060	n/a	('abc', 'strict', b'abc'),
3061	n/a	('\uff44\u9a3e', 'strict', b'\x82\x84\xe9\x80'),
3062	n/a	# test error handlers
3063	n/a	('\xff', 'strict', None),
3064	n/a	('[\xff]', 'ignore', b'[]'),
3065	n/a	('[\xff]', 'replace', b'[y]'),
3066	n/a	('[\u20ac]', 'replace', b'[?]'),
3067	n/a	('[\xff]', 'backslashreplace', b'[\\xff]'),
3068	n/a	('[\xff]', 'namereplace',
3069	n/a	b'[\\N{LATIN SMALL LETTER Y WITH DIAERESIS}]'),
3070	n/a	('[\xff]', 'xmlcharrefreplace', b'[ÿ]'),
3071	n/a	('\udcff', 'strict', None),
3072	n/a	('[\udcff]', 'surrogateescape', b'[\xff]'),
3073	n/a	('[\udcff]', 'surrogatepass', None),
3074	n/a	))
3075	n/a	self.check_decode(932, (
3076	n/a	(b'abc', 'strict', 'abc'),
3077	n/a	(b'\x82\x84\xe9\x80', 'strict', '\uff44\u9a3e'),
3078	n/a	# invalid bytes
3079	n/a	(b'[\xff]', 'strict', None),
3080	n/a	(b'[\xff]', 'ignore', '[]'),
3081	n/a	(b'[\xff]', 'replace', '[\ufffd]'),
3082	n/a	(b'[\xff]', 'backslashreplace', '[\\xff]'),
3083	n/a	(b'[\xff]', 'surrogateescape', '[\udcff]'),
3084	n/a	(b'[\xff]', 'surrogatepass', None),
3085	n/a	(b'\x81\x00abc', 'strict', None),
3086	n/a	(b'\x81\x00abc', 'ignore', '\x00abc'),
3087	n/a	(b'\x81\x00abc', 'replace', '\ufffd\x00abc'),
3088	n/a	(b'\x81\x00abc', 'backslashreplace', '\\x81\x00abc'),
3089	n/a	))
3090	n/a
3091	n/a	def test_cp1252(self):
3092	n/a	self.check_encode(1252, (
3093	n/a	('abc', 'strict', b'abc'),
3094	n/a	('\xe9\u20ac', 'strict', b'\xe9\x80'),
3095	n/a	('\xff', 'strict', b'\xff'),
3096	n/a	# test error handlers
3097	n/a	('\u0141', 'strict', None),
3098	n/a	('\u0141', 'ignore', b''),
3099	n/a	('\u0141', 'replace', b'L'),
3100	n/a	('\udc98', 'surrogateescape', b'\x98'),
3101	n/a	('\udc98', 'surrogatepass', None),
3102	n/a	))
3103	n/a	self.check_decode(1252, (
3104	n/a	(b'abc', 'strict', 'abc'),
3105	n/a	(b'\xe9\x80', 'strict', '\xe9\u20ac'),
3106	n/a	(b'\xff', 'strict', '\xff'),
3107	n/a	))
3108	n/a
3109	n/a	def test_cp_utf7(self):
3110	n/a	cp = 65000
3111	n/a	self.check_encode(cp, (
3112	n/a	('abc', 'strict', b'abc'),
3113	n/a	('\xe9\u20ac', 'strict', b'+AOkgrA-'),
3114	n/a	('\U0010ffff', 'strict', b'+2//f/w-'),
3115	n/a	('\udc80', 'strict', b'+3IA-'),
3116	n/a	('\ufffd', 'strict', b'+//0-'),
3117	n/a	))
3118	n/a	self.check_decode(cp, (
3119	n/a	(b'abc', 'strict', 'abc'),
3120	n/a	(b'+AOkgrA-', 'strict', '\xe9\u20ac'),
3121	n/a	(b'+2//f/w-', 'strict', '\U0010ffff'),
3122	n/a	(b'+3IA-', 'strict', '\udc80'),
3123	n/a	(b'+//0-', 'strict', '\ufffd'),
3124	n/a	# invalid bytes
3125	n/a	(b'[+/]', 'strict', '[]'),
3126	n/a	(b'[\xff]', 'strict', '[\xff]'),
3127	n/a	))
3128	n/a
3129	n/a	def test_multibyte_encoding(self):
3130	n/a	self.check_decode(932, (
3131	n/a	(b'\x84\xe9\x80', 'ignore', '\u9a3e'),
3132	n/a	(b'\x84\xe9\x80', 'replace', '\ufffd\u9a3e'),
3133	n/a	))
3134	n/a	self.check_decode(self.CP_UTF8, (
3135	n/a	(b'\xff\xf4\x8f\xbf\xbf', 'ignore', '\U0010ffff'),
3136	n/a	(b'\xff\xf4\x8f\xbf\xbf', 'replace', '\ufffd\U0010ffff'),
3137	n/a	))
3138	n/a	self.check_encode(self.CP_UTF8, (
3139	n/a	('[\U0010ffff\uDC80]', 'ignore', b'[\xf4\x8f\xbf\xbf]'),
3140	n/a	('[\U0010ffff\uDC80]', 'replace', b'[\xf4\x8f\xbf\xbf?]'),
3141	n/a	))
3142	n/a
3143	n/a	def test_incremental(self):
3144	n/a	decoded = codecs.code_page_decode(932, b'\x82', 'strict', False)
3145	n/a	self.assertEqual(decoded, ('', 0))
3146	n/a
3147	n/a	decoded = codecs.code_page_decode(932,
3148	n/a	b'\xe9\x80\xe9', 'strict',
3149	n/a	False)
3150	n/a	self.assertEqual(decoded, ('\u9a3e', 2))
3151	n/a
3152	n/a	decoded = codecs.code_page_decode(932,
3153	n/a	b'\xe9\x80\xe9\x80', 'strict',
3154	n/a	False)
3155	n/a	self.assertEqual(decoded, ('\u9a3e\u9a3e', 4))
3156	n/a
3157	n/a	decoded = codecs.code_page_decode(932,
3158	n/a	b'abc', 'strict',
3159	n/a	False)
3160	n/a	self.assertEqual(decoded, ('abc', 3))
3161	n/a
3162	n/a	def test_mbcs_alias(self):
3163	n/a	# Check that looking up our 'default' codepage will return
3164	n/a	# mbcs when we don't have a more specific one available
3165	n/a	import _bootlocale
3166	n/a	def _get_fake_codepage(*a):
3167	n/a	return 'cp123'
3168	n/a	old_getpreferredencoding = _bootlocale.getpreferredencoding
3169	n/a	_bootlocale.getpreferredencoding = _get_fake_codepage
3170	n/a	try:
3171	n/a	codec = codecs.lookup('cp123')
3172	n/a	self.assertEqual(codec.name, 'mbcs')
3173	n/a	finally:
3174	n/a	_bootlocale.getpreferredencoding = old_getpreferredencoding
3175	n/a
3176	n/a
3177	n/a	class ASCIITest(unittest.TestCase):
3178	n/a	def test_encode(self):
3179	n/a	self.assertEqual('abc123'.encode('ascii'), b'abc123')
3180	n/a
3181	n/a	def test_encode_error(self):
3182	n/a	for data, error_handler, expected in (
3183	n/a	('[\x80\xff\u20ac]', 'ignore', b'[]'),
3184	n/a	('[\x80\xff\u20ac]', 'replace', b'[???]'),
3185	n/a	('[\x80\xff\u20ac]', 'xmlcharrefreplace', b'[ÿ€]'),
3186	n/a	('[\x80\xff\u20ac\U000abcde]', 'backslashreplace',
3187	n/a	b'[\\x80\\xff\\u20ac\\U000abcde]'),
3188	n/a	('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'),
3189	n/a	):
3190	n/a	with self.subTest(data=data, error_handler=error_handler,
3191	n/a	expected=expected):
3192	n/a	self.assertEqual(data.encode('ascii', error_handler),
3193	n/a	expected)
3194	n/a
3195	n/a	def test_encode_surrogateescape_error(self):
3196	n/a	with self.assertRaises(UnicodeEncodeError):
3197	n/a	# the first character can be decoded, but not the second
3198	n/a	'\udc80\xff'.encode('ascii', 'surrogateescape')
3199	n/a
3200	n/a	def test_decode(self):
3201	n/a	self.assertEqual(b'abc'.decode('ascii'), 'abc')
3202	n/a
3203	n/a	def test_decode_error(self):
3204	n/a	for data, error_handler, expected in (
3205	n/a	(b'[\x80\xff]', 'ignore', '[]'),
3206	n/a	(b'[\x80\xff]', 'replace', '[\ufffd\ufffd]'),
3207	n/a	(b'[\x80\xff]', 'surrogateescape', '[\udc80\udcff]'),
3208	n/a	(b'[\x80\xff]', 'backslashreplace', '[\\x80\\xff]'),
3209	n/a	):
3210	n/a	with self.subTest(data=data, error_handler=error_handler,
3211	n/a	expected=expected):
3212	n/a	self.assertEqual(data.decode('ascii', error_handler),
3213	n/a	expected)
3214	n/a
3215	n/a
3216	n/a	class Latin1Test(unittest.TestCase):
3217	n/a	def test_encode(self):
3218	n/a	for data, expected in (
3219	n/a	('abc', b'abc'),
3220	n/a	('\x80\xe9\xff', b'\x80\xe9\xff'),
3221	n/a	):
3222	n/a	with self.subTest(data=data, expected=expected):
3223	n/a	self.assertEqual(data.encode('latin1'), expected)
3224	n/a
3225	n/a	def test_encode_errors(self):
3226	n/a	for data, error_handler, expected in (
3227	n/a	('[\u20ac\udc80]', 'ignore', b'[]'),
3228	n/a	('[\u20ac\udc80]', 'replace', b'[??]'),
3229	n/a	('[\u20ac\U000abcde]', 'backslashreplace',
3230	n/a	b'[\\u20ac\\U000abcde]'),
3231	n/a	('[\u20ac\udc80]', 'xmlcharrefreplace', b'[€&#56448;]'),
3232	n/a	('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'),
3233	n/a	):
3234	n/a	with self.subTest(data=data, error_handler=error_handler,
3235	n/a	expected=expected):
3236	n/a	self.assertEqual(data.encode('latin1', error_handler),
3237	n/a	expected)
3238	n/a
3239	n/a	def test_encode_surrogateescape_error(self):
3240	n/a	with self.assertRaises(UnicodeEncodeError):
3241	n/a	# the first character can be decoded, but not the second
3242	n/a	'\udc80\u20ac'.encode('latin1', 'surrogateescape')
3243	n/a
3244	n/a	def test_decode(self):
3245	n/a	for data, expected in (
3246	n/a	(b'abc', 'abc'),
3247	n/a	(b'[\x80\xff]', '[\x80\xff]'),
3248	n/a	):
3249	n/a	with self.subTest(data=data, expected=expected):
3250	n/a	self.assertEqual(data.decode('latin1'), expected)
3251	n/a
3252	n/a
3253	n/a	if __name__ == "__main__":
3254	n/a	unittest.main()