Python code coverage for Lib/test/test_unicodedata.py

#	count	content
1	n/a	""" Test script for the unicodedata module.
2	n/a
3	n/a	Written by Marc-Andre Lemburg (mal@lemburg.com).
4	n/a
5	n/a	(c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
6	n/a
7	n/a	"""
8	n/a
9	n/a	import sys
10	n/a	import unittest
11	n/a	import hashlib
12	n/a	from test.support import script_helper
13	n/a
14	n/a	encoding = 'utf-8'
15	n/a	errors = 'surrogatepass'
16	n/a
17	n/a
18	n/a	### Run tests
19	n/a
20	n/a	class UnicodeMethodsTest(unittest.TestCase):
21	n/a
22	n/a	# update this, if the database changes
23	n/a	expectedchecksum = 'c1fa98674a683aa8a8d8dee0c84494f8d36346e6'
24	n/a
25	n/a	def test_method_checksum(self):
26	n/a	h = hashlib.sha1()
27	n/a	for i in range(0x10000):
28	n/a	char = chr(i)
29	n/a	data = [
30	n/a	# Predicates (single char)
31	n/a	"01"[char.isalnum()],
32	n/a	"01"[char.isalpha()],
33	n/a	"01"[char.isdecimal()],
34	n/a	"01"[char.isdigit()],
35	n/a	"01"[char.islower()],
36	n/a	"01"[char.isnumeric()],
37	n/a	"01"[char.isspace()],
38	n/a	"01"[char.istitle()],
39	n/a	"01"[char.isupper()],
40	n/a
41	n/a	# Predicates (multiple chars)
42	n/a	"01"[(char + 'abc').isalnum()],
43	n/a	"01"[(char + 'abc').isalpha()],
44	n/a	"01"[(char + '123').isdecimal()],
45	n/a	"01"[(char + '123').isdigit()],
46	n/a	"01"[(char + 'abc').islower()],
47	n/a	"01"[(char + '123').isnumeric()],
48	n/a	"01"[(char + ' \t').isspace()],
49	n/a	"01"[(char + 'abc').istitle()],
50	n/a	"01"[(char + 'ABC').isupper()],
51	n/a
52	n/a	# Mappings (single char)
53	n/a	char.lower(),
54	n/a	char.upper(),
55	n/a	char.title(),
56	n/a
57	n/a	# Mappings (multiple chars)
58	n/a	(char + 'abc').lower(),
59	n/a	(char + 'ABC').upper(),
60	n/a	(char + 'abc').title(),
61	n/a	(char + 'ABC').title(),
62	n/a
63	n/a	]
64	n/a	h.update(''.join(data).encode(encoding, errors))
65	n/a	result = h.hexdigest()
66	n/a	self.assertEqual(result, self.expectedchecksum)
67	n/a
68	n/a	class UnicodeDatabaseTest(unittest.TestCase):
69	n/a
70	n/a	def setUp(self):
71	n/a	# In case unicodedata is not available, this will raise an ImportError,
72	n/a	# but the other test cases will still be run
73	n/a	import unicodedata
74	n/a	self.db = unicodedata
75	n/a
76	n/a	def tearDown(self):
77	n/a	del self.db
78	n/a
79	n/a	class UnicodeFunctionsTest(UnicodeDatabaseTest):
80	n/a
81	n/a	# Update this if the database changes. Make sure to do a full rebuild
82	n/a	# (e.g. 'make distclean && make') to get the correct checksum.
83	n/a	expectedchecksum = 'f891b1e6430c712531b9bc935a38e22d78ba1bf3'
84	n/a	def test_function_checksum(self):
85	n/a	data = []
86	n/a	h = hashlib.sha1()
87	n/a
88	n/a	for i in range(0x10000):
89	n/a	char = chr(i)
90	n/a	data = [
91	n/a	# Properties
92	n/a	format(self.db.digit(char, -1), '.12g'),
93	n/a	format(self.db.numeric(char, -1), '.12g'),
94	n/a	format(self.db.decimal(char, -1), '.12g'),
95	n/a	self.db.category(char),
96	n/a	self.db.bidirectional(char),
97	n/a	self.db.decomposition(char),
98	n/a	str(self.db.mirrored(char)),
99	n/a	str(self.db.combining(char)),
100	n/a	]
101	n/a	h.update(''.join(data).encode("ascii"))
102	n/a	result = h.hexdigest()
103	n/a	self.assertEqual(result, self.expectedchecksum)
104	n/a
105	n/a	def test_digit(self):
106	n/a	self.assertEqual(self.db.digit('A', None), None)
107	n/a	self.assertEqual(self.db.digit('9'), 9)
108	n/a	self.assertEqual(self.db.digit('\u215b', None), None)
109	n/a	self.assertEqual(self.db.digit('\u2468'), 9)
110	n/a	self.assertEqual(self.db.digit('\U00020000', None), None)
111	n/a	self.assertEqual(self.db.digit('\U0001D7FD'), 7)
112	n/a
113	n/a	self.assertRaises(TypeError, self.db.digit)
114	n/a	self.assertRaises(TypeError, self.db.digit, 'xx')
115	n/a	self.assertRaises(ValueError, self.db.digit, 'x')
116	n/a
117	n/a	def test_numeric(self):
118	n/a	self.assertEqual(self.db.numeric('A',None), None)
119	n/a	self.assertEqual(self.db.numeric('9'), 9)
120	n/a	self.assertEqual(self.db.numeric('\u215b'), 0.125)
121	n/a	self.assertEqual(self.db.numeric('\u2468'), 9.0)
122	n/a	self.assertEqual(self.db.numeric('\ua627'), 7.0)
123	n/a	self.assertEqual(self.db.numeric('\U00020000', None), None)
124	n/a	self.assertEqual(self.db.numeric('\U0001012A'), 9000)
125	n/a
126	n/a	self.assertRaises(TypeError, self.db.numeric)
127	n/a	self.assertRaises(TypeError, self.db.numeric, 'xx')
128	n/a	self.assertRaises(ValueError, self.db.numeric, 'x')
129	n/a
130	n/a	def test_decimal(self):
131	n/a	self.assertEqual(self.db.decimal('A',None), None)
132	n/a	self.assertEqual(self.db.decimal('9'), 9)
133	n/a	self.assertEqual(self.db.decimal('\u215b', None), None)
134	n/a	self.assertEqual(self.db.decimal('\u2468', None), None)
135	n/a	self.assertEqual(self.db.decimal('\U00020000', None), None)
136	n/a	self.assertEqual(self.db.decimal('\U0001D7FD'), 7)
137	n/a
138	n/a	self.assertRaises(TypeError, self.db.decimal)
139	n/a	self.assertRaises(TypeError, self.db.decimal, 'xx')
140	n/a	self.assertRaises(ValueError, self.db.decimal, 'x')
141	n/a
142	n/a	def test_category(self):
143	n/a	self.assertEqual(self.db.category('\uFFFE'), 'Cn')
144	n/a	self.assertEqual(self.db.category('a'), 'Ll')
145	n/a	self.assertEqual(self.db.category('A'), 'Lu')
146	n/a	self.assertEqual(self.db.category('\U00020000'), 'Lo')
147	n/a	self.assertEqual(self.db.category('\U0001012A'), 'No')
148	n/a
149	n/a	self.assertRaises(TypeError, self.db.category)
150	n/a	self.assertRaises(TypeError, self.db.category, 'xx')
151	n/a
152	n/a	def test_bidirectional(self):
153	n/a	self.assertEqual(self.db.bidirectional('\uFFFE'), '')
154	n/a	self.assertEqual(self.db.bidirectional(' '), 'WS')
155	n/a	self.assertEqual(self.db.bidirectional('A'), 'L')
156	n/a	self.assertEqual(self.db.bidirectional('\U00020000'), 'L')
157	n/a
158	n/a	self.assertRaises(TypeError, self.db.bidirectional)
159	n/a	self.assertRaises(TypeError, self.db.bidirectional, 'xx')
160	n/a
161	n/a	def test_decomposition(self):
162	n/a	self.assertEqual(self.db.decomposition('\uFFFE'),'')
163	n/a	self.assertEqual(self.db.decomposition('\u00bc'), '<fraction> 0031 2044 0034')
164	n/a
165	n/a	self.assertRaises(TypeError, self.db.decomposition)
166	n/a	self.assertRaises(TypeError, self.db.decomposition, 'xx')
167	n/a
168	n/a	def test_mirrored(self):
169	n/a	self.assertEqual(self.db.mirrored('\uFFFE'), 0)
170	n/a	self.assertEqual(self.db.mirrored('a'), 0)
171	n/a	self.assertEqual(self.db.mirrored('\u2201'), 1)
172	n/a	self.assertEqual(self.db.mirrored('\U00020000'), 0)
173	n/a
174	n/a	self.assertRaises(TypeError, self.db.mirrored)
175	n/a	self.assertRaises(TypeError, self.db.mirrored, 'xx')
176	n/a
177	n/a	def test_combining(self):
178	n/a	self.assertEqual(self.db.combining('\uFFFE'), 0)
179	n/a	self.assertEqual(self.db.combining('a'), 0)
180	n/a	self.assertEqual(self.db.combining('\u20e1'), 230)
181	n/a	self.assertEqual(self.db.combining('\U00020000'), 0)
182	n/a
183	n/a	self.assertRaises(TypeError, self.db.combining)
184	n/a	self.assertRaises(TypeError, self.db.combining, 'xx')
185	n/a
186	n/a	def test_normalize(self):
187	n/a	self.assertRaises(TypeError, self.db.normalize)
188	n/a	self.assertRaises(ValueError, self.db.normalize, 'unknown', 'xx')
189	n/a	self.assertEqual(self.db.normalize('NFKC', ''), '')
190	n/a	# The rest can be found in test_normalization.py
191	n/a	# which requires an external file.
192	n/a
193	n/a	def test_pr29(self):
194	n/a	# http://www.unicode.org/review/pr-29.html
195	n/a	# See issues #1054943 and #10254.
196	n/a	composed = ("\u0b47\u0300\u0b3e", "\u1100\u0300\u1161",
197	n/a	'Li\u030dt-s\u1e73\u0301',
198	n/a	'\u092e\u093e\u0930\u094d\u0915 \u091c\u093c'
199	n/a	+ '\u0941\u0915\u0947\u0930\u092c\u0930\u094d\u0917',
200	n/a	'\u0915\u093f\u0930\u094d\u0917\u093f\u091c\u093c'
201	n/a	+ '\u0938\u094d\u0924\u093e\u0928')
202	n/a	for text in composed:
203	n/a	self.assertEqual(self.db.normalize('NFC', text), text)
204	n/a
205	n/a	def test_issue10254(self):
206	n/a	# Crash reported in #10254
207	n/a	a = 'C\u0338' * 20 + 'C\u0327'
208	n/a	b = 'C\u0338' * 20 + '\xC7'
209	n/a	self.assertEqual(self.db.normalize('NFC', a), b)
210	n/a
211	n/a	def test_east_asian_width(self):
212	n/a	eaw = self.db.east_asian_width
213	n/a	self.assertRaises(TypeError, eaw, b'a')
214	n/a	self.assertRaises(TypeError, eaw, bytearray())
215	n/a	self.assertRaises(TypeError, eaw, '')
216	n/a	self.assertRaises(TypeError, eaw, 'ra')
217	n/a	self.assertEqual(eaw('\x1e'), 'N')
218	n/a	self.assertEqual(eaw('\x20'), 'Na')
219	n/a	self.assertEqual(eaw('\uC894'), 'W')
220	n/a	self.assertEqual(eaw('\uFF66'), 'H')
221	n/a	self.assertEqual(eaw('\uFF1F'), 'F')
222	n/a	self.assertEqual(eaw('\u2010'), 'A')
223	n/a	self.assertEqual(eaw('\U00020000'), 'W')
224	n/a
225	n/a	def test_east_asian_width_9_0_changes(self):
226	n/a	self.assertEqual(self.db.ucd_3_2_0.east_asian_width('\u231a'), 'N')
227	n/a	self.assertEqual(self.db.east_asian_width('\u231a'), 'W')
228	n/a
229	n/a	class UnicodeMiscTest(UnicodeDatabaseTest):
230	n/a
231	n/a	def test_failed_import_during_compiling(self):
232	n/a	# Issue 4367
233	n/a	# Decoding \N escapes requires the unicodedata module. If it can't be
234	n/a	# imported, we shouldn't segfault.
235	n/a
236	n/a	# This program should raise a SyntaxError in the eval.
237	n/a	code = "import sys;" \
238	n/a	"sys.modules['unicodedata'] = None;" \
239	n/a	"""eval("'\\\\N{SOFT HYPHEN}'")"""
240	n/a	# We use a separate process because the unicodedata module may already
241	n/a	# have been loaded in this process.
242	n/a	result = script_helper.assert_python_failure("-c", code)
243	n/a	error = "SyntaxError: (unicode error) \\N escapes not supported " \
244	n/a	"(can't load unicodedata module)"
245	n/a	self.assertIn(error, result.err.decode("ascii"))
246	n/a
247	n/a	def test_decimal_numeric_consistent(self):
248	n/a	# Test that decimal and numeric are consistent,
249	n/a	# i.e. if a character has a decimal value,
250	n/a	# its numeric value should be the same.
251	n/a	count = 0
252	n/a	for i in range(0x10000):
253	n/a	c = chr(i)
254	n/a	dec = self.db.decimal(c, -1)
255	n/a	if dec != -1:
256	n/a	self.assertEqual(dec, self.db.numeric(c))
257	n/a	count += 1
258	n/a	self.assertTrue(count >= 10) # should have tested at least the ASCII digits
259	n/a
260	n/a	def test_digit_numeric_consistent(self):
261	n/a	# Test that digit and numeric are consistent,
262	n/a	# i.e. if a character has a digit value,
263	n/a	# its numeric value should be the same.
264	n/a	count = 0
265	n/a	for i in range(0x10000):
266	n/a	c = chr(i)
267	n/a	dec = self.db.digit(c, -1)
268	n/a	if dec != -1:
269	n/a	self.assertEqual(dec, self.db.numeric(c))
270	n/a	count += 1
271	n/a	self.assertTrue(count >= 10) # should have tested at least the ASCII digits
272	n/a
273	n/a	def test_bug_1704793(self):
274	n/a	self.assertEqual(self.db.lookup("GOTHIC LETTER FAIHU"), '\U00010346')
275	n/a
276	n/a	def test_ucd_510(self):
277	n/a	import unicodedata
278	n/a	# In UCD 5.1.0, a mirrored property changed wrt. UCD 3.2.0
279	n/a	self.assertTrue(unicodedata.mirrored("\u0f3a"))
280	n/a	self.assertTrue(not unicodedata.ucd_3_2_0.mirrored("\u0f3a"))
281	n/a	# Also, we now have two ways of representing
282	n/a	# the upper-case mapping: as delta, or as absolute value
283	n/a	self.assertTrue("a".upper()=='A')
284	n/a	self.assertTrue("\u1d79".upper()=='\ua77d')
285	n/a	self.assertTrue(".".upper()=='.')
286	n/a
287	n/a	def test_bug_5828(self):
288	n/a	self.assertEqual("\u1d79".lower(), "\u1d79")
289	n/a	# Only U+0000 should have U+0000 as its upper/lower/titlecase variant
290	n/a	self.assertEqual(
291	n/a	[
292	n/a	c for c in range(sys.maxunicode+1)
293	n/a	if "\x00" in chr(c).lower()+chr(c).upper()+chr(c).title()
294	n/a	],
295	n/a	[0]
296	n/a	)
297	n/a
298	n/a	def test_bug_4971(self):
299	n/a	# LETTER DZ WITH CARON: DZ, Dz, dz
300	n/a	self.assertEqual("\u01c4".title(), "\u01c5")
301	n/a	self.assertEqual("\u01c5".title(), "\u01c5")
302	n/a	self.assertEqual("\u01c6".title(), "\u01c5")
303	n/a
304	n/a	def test_linebreak_7643(self):
305	n/a	for i in range(0x10000):
306	n/a	lines = (chr(i) + 'A').splitlines()
307	n/a	if i in (0x0a, 0x0b, 0x0c, 0x0d, 0x85,
308	n/a	0x1c, 0x1d, 0x1e, 0x2028, 0x2029):
309	n/a	self.assertEqual(len(lines), 2,
310	n/a	r"\u%.4x should be a linebreak" % i)
311	n/a	else:
312	n/a	self.assertEqual(len(lines), 1,
313	n/a	r"\u%.4x should not be a linebreak" % i)
314	n/a
315	n/a	if __name__ == "__main__":
316	n/a	unittest.main()