ยปCore Development>Code coverage>Lib/test/test_unicodedata.py

Python code coverage for Lib/test/test_unicodedata.py

#countcontent
1n/a""" Test script for the unicodedata module.
2n/a
3n/a Written by Marc-Andre Lemburg (mal@lemburg.com).
4n/a
5n/a (c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
6n/a
7n/a"""
8n/a
9n/aimport sys
10n/aimport unittest
11n/aimport hashlib
12n/afrom test.support import script_helper
13n/a
14n/aencoding = 'utf-8'
15n/aerrors = 'surrogatepass'
16n/a
17n/a
18n/a### Run tests
19n/a
20n/aclass UnicodeMethodsTest(unittest.TestCase):
21n/a
22n/a # update this, if the database changes
23n/a expectedchecksum = 'c1fa98674a683aa8a8d8dee0c84494f8d36346e6'
24n/a
25n/a def test_method_checksum(self):
26n/a h = hashlib.sha1()
27n/a for i in range(0x10000):
28n/a char = chr(i)
29n/a data = [
30n/a # Predicates (single char)
31n/a "01"[char.isalnum()],
32n/a "01"[char.isalpha()],
33n/a "01"[char.isdecimal()],
34n/a "01"[char.isdigit()],
35n/a "01"[char.islower()],
36n/a "01"[char.isnumeric()],
37n/a "01"[char.isspace()],
38n/a "01"[char.istitle()],
39n/a "01"[char.isupper()],
40n/a
41n/a # Predicates (multiple chars)
42n/a "01"[(char + 'abc').isalnum()],
43n/a "01"[(char + 'abc').isalpha()],
44n/a "01"[(char + '123').isdecimal()],
45n/a "01"[(char + '123').isdigit()],
46n/a "01"[(char + 'abc').islower()],
47n/a "01"[(char + '123').isnumeric()],
48n/a "01"[(char + ' \t').isspace()],
49n/a "01"[(char + 'abc').istitle()],
50n/a "01"[(char + 'ABC').isupper()],
51n/a
52n/a # Mappings (single char)
53n/a char.lower(),
54n/a char.upper(),
55n/a char.title(),
56n/a
57n/a # Mappings (multiple chars)
58n/a (char + 'abc').lower(),
59n/a (char + 'ABC').upper(),
60n/a (char + 'abc').title(),
61n/a (char + 'ABC').title(),
62n/a
63n/a ]
64n/a h.update(''.join(data).encode(encoding, errors))
65n/a result = h.hexdigest()
66n/a self.assertEqual(result, self.expectedchecksum)
67n/a
68n/aclass UnicodeDatabaseTest(unittest.TestCase):
69n/a
70n/a def setUp(self):
71n/a # In case unicodedata is not available, this will raise an ImportError,
72n/a # but the other test cases will still be run
73n/a import unicodedata
74n/a self.db = unicodedata
75n/a
76n/a def tearDown(self):
77n/a del self.db
78n/a
79n/aclass UnicodeFunctionsTest(UnicodeDatabaseTest):
80n/a
81n/a # Update this if the database changes. Make sure to do a full rebuild
82n/a # (e.g. 'make distclean && make') to get the correct checksum.
83n/a expectedchecksum = 'f891b1e6430c712531b9bc935a38e22d78ba1bf3'
84n/a def test_function_checksum(self):
85n/a data = []
86n/a h = hashlib.sha1()
87n/a
88n/a for i in range(0x10000):
89n/a char = chr(i)
90n/a data = [
91n/a # Properties
92n/a format(self.db.digit(char, -1), '.12g'),
93n/a format(self.db.numeric(char, -1), '.12g'),
94n/a format(self.db.decimal(char, -1), '.12g'),
95n/a self.db.category(char),
96n/a self.db.bidirectional(char),
97n/a self.db.decomposition(char),
98n/a str(self.db.mirrored(char)),
99n/a str(self.db.combining(char)),
100n/a ]
101n/a h.update(''.join(data).encode("ascii"))
102n/a result = h.hexdigest()
103n/a self.assertEqual(result, self.expectedchecksum)
104n/a
105n/a def test_digit(self):
106n/a self.assertEqual(self.db.digit('A', None), None)
107n/a self.assertEqual(self.db.digit('9'), 9)
108n/a self.assertEqual(self.db.digit('\u215b', None), None)
109n/a self.assertEqual(self.db.digit('\u2468'), 9)
110n/a self.assertEqual(self.db.digit('\U00020000', None), None)
111n/a self.assertEqual(self.db.digit('\U0001D7FD'), 7)
112n/a
113n/a self.assertRaises(TypeError, self.db.digit)
114n/a self.assertRaises(TypeError, self.db.digit, 'xx')
115n/a self.assertRaises(ValueError, self.db.digit, 'x')
116n/a
117n/a def test_numeric(self):
118n/a self.assertEqual(self.db.numeric('A',None), None)
119n/a self.assertEqual(self.db.numeric('9'), 9)
120n/a self.assertEqual(self.db.numeric('\u215b'), 0.125)
121n/a self.assertEqual(self.db.numeric('\u2468'), 9.0)
122n/a self.assertEqual(self.db.numeric('\ua627'), 7.0)
123n/a self.assertEqual(self.db.numeric('\U00020000', None), None)
124n/a self.assertEqual(self.db.numeric('\U0001012A'), 9000)
125n/a
126n/a self.assertRaises(TypeError, self.db.numeric)
127n/a self.assertRaises(TypeError, self.db.numeric, 'xx')
128n/a self.assertRaises(ValueError, self.db.numeric, 'x')
129n/a
130n/a def test_decimal(self):
131n/a self.assertEqual(self.db.decimal('A',None), None)
132n/a self.assertEqual(self.db.decimal('9'), 9)
133n/a self.assertEqual(self.db.decimal('\u215b', None), None)
134n/a self.assertEqual(self.db.decimal('\u2468', None), None)
135n/a self.assertEqual(self.db.decimal('\U00020000', None), None)
136n/a self.assertEqual(self.db.decimal('\U0001D7FD'), 7)
137n/a
138n/a self.assertRaises(TypeError, self.db.decimal)
139n/a self.assertRaises(TypeError, self.db.decimal, 'xx')
140n/a self.assertRaises(ValueError, self.db.decimal, 'x')
141n/a
142n/a def test_category(self):
143n/a self.assertEqual(self.db.category('\uFFFE'), 'Cn')
144n/a self.assertEqual(self.db.category('a'), 'Ll')
145n/a self.assertEqual(self.db.category('A'), 'Lu')
146n/a self.assertEqual(self.db.category('\U00020000'), 'Lo')
147n/a self.assertEqual(self.db.category('\U0001012A'), 'No')
148n/a
149n/a self.assertRaises(TypeError, self.db.category)
150n/a self.assertRaises(TypeError, self.db.category, 'xx')
151n/a
152n/a def test_bidirectional(self):
153n/a self.assertEqual(self.db.bidirectional('\uFFFE'), '')
154n/a self.assertEqual(self.db.bidirectional(' '), 'WS')
155n/a self.assertEqual(self.db.bidirectional('A'), 'L')
156n/a self.assertEqual(self.db.bidirectional('\U00020000'), 'L')
157n/a
158n/a self.assertRaises(TypeError, self.db.bidirectional)
159n/a self.assertRaises(TypeError, self.db.bidirectional, 'xx')
160n/a
161n/a def test_decomposition(self):
162n/a self.assertEqual(self.db.decomposition('\uFFFE'),'')
163n/a self.assertEqual(self.db.decomposition('\u00bc'), '<fraction> 0031 2044 0034')
164n/a
165n/a self.assertRaises(TypeError, self.db.decomposition)
166n/a self.assertRaises(TypeError, self.db.decomposition, 'xx')
167n/a
168n/a def test_mirrored(self):
169n/a self.assertEqual(self.db.mirrored('\uFFFE'), 0)
170n/a self.assertEqual(self.db.mirrored('a'), 0)
171n/a self.assertEqual(self.db.mirrored('\u2201'), 1)
172n/a self.assertEqual(self.db.mirrored('\U00020000'), 0)
173n/a
174n/a self.assertRaises(TypeError, self.db.mirrored)
175n/a self.assertRaises(TypeError, self.db.mirrored, 'xx')
176n/a
177n/a def test_combining(self):
178n/a self.assertEqual(self.db.combining('\uFFFE'), 0)
179n/a self.assertEqual(self.db.combining('a'), 0)
180n/a self.assertEqual(self.db.combining('\u20e1'), 230)
181n/a self.assertEqual(self.db.combining('\U00020000'), 0)
182n/a
183n/a self.assertRaises(TypeError, self.db.combining)
184n/a self.assertRaises(TypeError, self.db.combining, 'xx')
185n/a
186n/a def test_normalize(self):
187n/a self.assertRaises(TypeError, self.db.normalize)
188n/a self.assertRaises(ValueError, self.db.normalize, 'unknown', 'xx')
189n/a self.assertEqual(self.db.normalize('NFKC', ''), '')
190n/a # The rest can be found in test_normalization.py
191n/a # which requires an external file.
192n/a
193n/a def test_pr29(self):
194n/a # http://www.unicode.org/review/pr-29.html
195n/a # See issues #1054943 and #10254.
196n/a composed = ("\u0b47\u0300\u0b3e", "\u1100\u0300\u1161",
197n/a 'Li\u030dt-s\u1e73\u0301',
198n/a '\u092e\u093e\u0930\u094d\u0915 \u091c\u093c'
199n/a + '\u0941\u0915\u0947\u0930\u092c\u0930\u094d\u0917',
200n/a '\u0915\u093f\u0930\u094d\u0917\u093f\u091c\u093c'
201n/a + '\u0938\u094d\u0924\u093e\u0928')
202n/a for text in composed:
203n/a self.assertEqual(self.db.normalize('NFC', text), text)
204n/a
205n/a def test_issue10254(self):
206n/a # Crash reported in #10254
207n/a a = 'C\u0338' * 20 + 'C\u0327'
208n/a b = 'C\u0338' * 20 + '\xC7'
209n/a self.assertEqual(self.db.normalize('NFC', a), b)
210n/a
211n/a def test_east_asian_width(self):
212n/a eaw = self.db.east_asian_width
213n/a self.assertRaises(TypeError, eaw, b'a')
214n/a self.assertRaises(TypeError, eaw, bytearray())
215n/a self.assertRaises(TypeError, eaw, '')
216n/a self.assertRaises(TypeError, eaw, 'ra')
217n/a self.assertEqual(eaw('\x1e'), 'N')
218n/a self.assertEqual(eaw('\x20'), 'Na')
219n/a self.assertEqual(eaw('\uC894'), 'W')
220n/a self.assertEqual(eaw('\uFF66'), 'H')
221n/a self.assertEqual(eaw('\uFF1F'), 'F')
222n/a self.assertEqual(eaw('\u2010'), 'A')
223n/a self.assertEqual(eaw('\U00020000'), 'W')
224n/a
225n/a def test_east_asian_width_9_0_changes(self):
226n/a self.assertEqual(self.db.ucd_3_2_0.east_asian_width('\u231a'), 'N')
227n/a self.assertEqual(self.db.east_asian_width('\u231a'), 'W')
228n/a
229n/aclass UnicodeMiscTest(UnicodeDatabaseTest):
230n/a
231n/a def test_failed_import_during_compiling(self):
232n/a # Issue 4367
233n/a # Decoding \N escapes requires the unicodedata module. If it can't be
234n/a # imported, we shouldn't segfault.
235n/a
236n/a # This program should raise a SyntaxError in the eval.
237n/a code = "import sys;" \
238n/a "sys.modules['unicodedata'] = None;" \
239n/a """eval("'\\\\N{SOFT HYPHEN}'")"""
240n/a # We use a separate process because the unicodedata module may already
241n/a # have been loaded in this process.
242n/a result = script_helper.assert_python_failure("-c", code)
243n/a error = "SyntaxError: (unicode error) \\N escapes not supported " \
244n/a "(can't load unicodedata module)"
245n/a self.assertIn(error, result.err.decode("ascii"))
246n/a
247n/a def test_decimal_numeric_consistent(self):
248n/a # Test that decimal and numeric are consistent,
249n/a # i.e. if a character has a decimal value,
250n/a # its numeric value should be the same.
251n/a count = 0
252n/a for i in range(0x10000):
253n/a c = chr(i)
254n/a dec = self.db.decimal(c, -1)
255n/a if dec != -1:
256n/a self.assertEqual(dec, self.db.numeric(c))
257n/a count += 1
258n/a self.assertTrue(count >= 10) # should have tested at least the ASCII digits
259n/a
260n/a def test_digit_numeric_consistent(self):
261n/a # Test that digit and numeric are consistent,
262n/a # i.e. if a character has a digit value,
263n/a # its numeric value should be the same.
264n/a count = 0
265n/a for i in range(0x10000):
266n/a c = chr(i)
267n/a dec = self.db.digit(c, -1)
268n/a if dec != -1:
269n/a self.assertEqual(dec, self.db.numeric(c))
270n/a count += 1
271n/a self.assertTrue(count >= 10) # should have tested at least the ASCII digits
272n/a
273n/a def test_bug_1704793(self):
274n/a self.assertEqual(self.db.lookup("GOTHIC LETTER FAIHU"), '\U00010346')
275n/a
276n/a def test_ucd_510(self):
277n/a import unicodedata
278n/a # In UCD 5.1.0, a mirrored property changed wrt. UCD 3.2.0
279n/a self.assertTrue(unicodedata.mirrored("\u0f3a"))
280n/a self.assertTrue(not unicodedata.ucd_3_2_0.mirrored("\u0f3a"))
281n/a # Also, we now have two ways of representing
282n/a # the upper-case mapping: as delta, or as absolute value
283n/a self.assertTrue("a".upper()=='A')
284n/a self.assertTrue("\u1d79".upper()=='\ua77d')
285n/a self.assertTrue(".".upper()=='.')
286n/a
287n/a def test_bug_5828(self):
288n/a self.assertEqual("\u1d79".lower(), "\u1d79")
289n/a # Only U+0000 should have U+0000 as its upper/lower/titlecase variant
290n/a self.assertEqual(
291n/a [
292n/a c for c in range(sys.maxunicode+1)
293n/a if "\x00" in chr(c).lower()+chr(c).upper()+chr(c).title()
294n/a ],
295n/a [0]
296n/a )
297n/a
298n/a def test_bug_4971(self):
299n/a # LETTER DZ WITH CARON: DZ, Dz, dz
300n/a self.assertEqual("\u01c4".title(), "\u01c5")
301n/a self.assertEqual("\u01c5".title(), "\u01c5")
302n/a self.assertEqual("\u01c6".title(), "\u01c5")
303n/a
304n/a def test_linebreak_7643(self):
305n/a for i in range(0x10000):
306n/a lines = (chr(i) + 'A').splitlines()
307n/a if i in (0x0a, 0x0b, 0x0c, 0x0d, 0x85,
308n/a 0x1c, 0x1d, 0x1e, 0x2028, 0x2029):
309n/a self.assertEqual(len(lines), 2,
310n/a r"\u%.4x should be a linebreak" % i)
311n/a else:
312n/a self.assertEqual(len(lines), 1,
313n/a r"\u%.4x should not be a linebreak" % i)
314n/a
315n/aif __name__ == "__main__":
316n/a unittest.main()