ยปCore Development>Code coverage>Lib/test/test_ucn.py

Python code coverage for Lib/test/test_ucn.py

#countcontent
1n/a""" Test script for the Unicode implementation.
2n/a
3n/aWritten by Bill Tutt.
4n/aModified for Python 2.0 by Fredrik Lundh (fredrik@pythonware.com)
5n/a
6n/a(c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
7n/a
8n/a"""#"
9n/a
10n/aimport unittest
11n/aimport unicodedata
12n/a
13n/afrom test import support
14n/afrom http.client import HTTPException
15n/afrom test.test_normalization import check_version
16n/a
17n/atry:
18n/a from _testcapi import INT_MAX, PY_SSIZE_T_MAX, UINT_MAX
19n/aexcept ImportError:
20n/a INT_MAX = PY_SSIZE_T_MAX = UINT_MAX = 2**64 - 1
21n/a
22n/aclass UnicodeNamesTest(unittest.TestCase):
23n/a
24n/a def checkletter(self, name, code):
25n/a # Helper that put all \N escapes inside eval'd raw strings,
26n/a # to make sure this script runs even if the compiler
27n/a # chokes on \N escapes
28n/a res = eval(r'"\N{%s}"' % name)
29n/a self.assertEqual(res, code)
30n/a return res
31n/a
32n/a def test_general(self):
33n/a # General and case insensitivity test:
34n/a chars = [
35n/a "LATIN CAPITAL LETTER T",
36n/a "LATIN SMALL LETTER H",
37n/a "LATIN SMALL LETTER E",
38n/a "SPACE",
39n/a "LATIN SMALL LETTER R",
40n/a "LATIN CAPITAL LETTER E",
41n/a "LATIN SMALL LETTER D",
42n/a "SPACE",
43n/a "LATIN SMALL LETTER f",
44n/a "LATIN CAPITAL LeTtEr o",
45n/a "LATIN SMaLl LETTER x",
46n/a "SPACE",
47n/a "LATIN SMALL LETTER A",
48n/a "LATIN SMALL LETTER T",
49n/a "LATIN SMALL LETTER E",
50n/a "SPACE",
51n/a "LATIN SMALL LETTER T",
52n/a "LATIN SMALL LETTER H",
53n/a "LATIN SMALL LETTER E",
54n/a "SpAcE",
55n/a "LATIN SMALL LETTER S",
56n/a "LATIN SMALL LETTER H",
57n/a "LATIN small LETTER e",
58n/a "LATIN small LETTER e",
59n/a "LATIN SMALL LETTER P",
60n/a "FULL STOP"
61n/a ]
62n/a string = "The rEd fOx ate the sheep."
63n/a
64n/a self.assertEqual(
65n/a "".join([self.checkletter(*args) for args in zip(chars, string)]),
66n/a string
67n/a )
68n/a
69n/a def test_ascii_letters(self):
70n/a for char in "".join(map(chr, range(ord("a"), ord("z")))):
71n/a name = "LATIN SMALL LETTER %s" % char.upper()
72n/a code = unicodedata.lookup(name)
73n/a self.assertEqual(unicodedata.name(code), name)
74n/a
75n/a def test_hangul_syllables(self):
76n/a self.checkletter("HANGUL SYLLABLE GA", "\uac00")
77n/a self.checkletter("HANGUL SYLLABLE GGWEOSS", "\uafe8")
78n/a self.checkletter("HANGUL SYLLABLE DOLS", "\ub3d0")
79n/a self.checkletter("HANGUL SYLLABLE RYAN", "\ub7b8")
80n/a self.checkletter("HANGUL SYLLABLE MWIK", "\ubba0")
81n/a self.checkletter("HANGUL SYLLABLE BBWAEM", "\ubf88")
82n/a self.checkletter("HANGUL SYLLABLE SSEOL", "\uc370")
83n/a self.checkletter("HANGUL SYLLABLE YI", "\uc758")
84n/a self.checkletter("HANGUL SYLLABLE JJYOSS", "\ucb40")
85n/a self.checkletter("HANGUL SYLLABLE KYEOLS", "\ucf28")
86n/a self.checkletter("HANGUL SYLLABLE PAN", "\ud310")
87n/a self.checkletter("HANGUL SYLLABLE HWEOK", "\ud6f8")
88n/a self.checkletter("HANGUL SYLLABLE HIH", "\ud7a3")
89n/a
90n/a self.assertRaises(ValueError, unicodedata.name, "\ud7a4")
91n/a
92n/a def test_cjk_unified_ideographs(self):
93n/a self.checkletter("CJK UNIFIED IDEOGRAPH-3400", "\u3400")
94n/a self.checkletter("CJK UNIFIED IDEOGRAPH-4DB5", "\u4db5")
95n/a self.checkletter("CJK UNIFIED IDEOGRAPH-4E00", "\u4e00")
96n/a self.checkletter("CJK UNIFIED IDEOGRAPH-9FCB", "\u9fCB")
97n/a self.checkletter("CJK UNIFIED IDEOGRAPH-20000", "\U00020000")
98n/a self.checkletter("CJK UNIFIED IDEOGRAPH-2A6D6", "\U0002a6d6")
99n/a self.checkletter("CJK UNIFIED IDEOGRAPH-2A700", "\U0002A700")
100n/a self.checkletter("CJK UNIFIED IDEOGRAPH-2B734", "\U0002B734")
101n/a self.checkletter("CJK UNIFIED IDEOGRAPH-2B740", "\U0002B740")
102n/a self.checkletter("CJK UNIFIED IDEOGRAPH-2B81D", "\U0002B81D")
103n/a
104n/a def test_bmp_characters(self):
105n/a for code in range(0x10000):
106n/a char = chr(code)
107n/a name = unicodedata.name(char, None)
108n/a if name is not None:
109n/a self.assertEqual(unicodedata.lookup(name), char)
110n/a
111n/a def test_misc_symbols(self):
112n/a self.checkletter("PILCROW SIGN", "\u00b6")
113n/a self.checkletter("REPLACEMENT CHARACTER", "\uFFFD")
114n/a self.checkletter("HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK", "\uFF9F")
115n/a self.checkletter("FULLWIDTH LATIN SMALL LETTER A", "\uFF41")
116n/a
117n/a def test_aliases(self):
118n/a # Check that the aliases defined in the NameAliases.txt file work.
119n/a # This should be updated when new aliases are added or the file
120n/a # should be downloaded and parsed instead. See #12753.
121n/a aliases = [
122n/a ('LATIN CAPITAL LETTER GHA', 0x01A2),
123n/a ('LATIN SMALL LETTER GHA', 0x01A3),
124n/a ('KANNADA LETTER LLLA', 0x0CDE),
125n/a ('LAO LETTER FO FON', 0x0E9D),
126n/a ('LAO LETTER FO FAY', 0x0E9F),
127n/a ('LAO LETTER RO', 0x0EA3),
128n/a ('LAO LETTER LO', 0x0EA5),
129n/a ('TIBETAN MARK BKA- SHOG GI MGO RGYAN', 0x0FD0),
130n/a ('YI SYLLABLE ITERATION MARK', 0xA015),
131n/a ('PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRACKET', 0xFE18),
132n/a ('BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS', 0x1D0C5)
133n/a ]
134n/a for alias, codepoint in aliases:
135n/a self.checkletter(alias, chr(codepoint))
136n/a name = unicodedata.name(chr(codepoint))
137n/a self.assertNotEqual(name, alias)
138n/a self.assertEqual(unicodedata.lookup(alias),
139n/a unicodedata.lookup(name))
140n/a with self.assertRaises(KeyError):
141n/a unicodedata.ucd_3_2_0.lookup(alias)
142n/a
143n/a def test_aliases_names_in_pua_range(self):
144n/a # We are storing aliases in the PUA 15, but their names shouldn't leak
145n/a for cp in range(0xf0000, 0xf0100):
146n/a with self.assertRaises(ValueError) as cm:
147n/a unicodedata.name(chr(cp))
148n/a self.assertEqual(str(cm.exception), 'no such name')
149n/a
150n/a def test_named_sequences_names_in_pua_range(self):
151n/a # We are storing named seq in the PUA 15, but their names shouldn't leak
152n/a for cp in range(0xf0100, 0xf0fff):
153n/a with self.assertRaises(ValueError) as cm:
154n/a unicodedata.name(chr(cp))
155n/a self.assertEqual(str(cm.exception), 'no such name')
156n/a
157n/a def test_named_sequences_sample(self):
158n/a # Check a few named sequences. See #12753.
159n/a sequences = [
160n/a ('LATIN SMALL LETTER R WITH TILDE', '\u0072\u0303'),
161n/a ('TAMIL SYLLABLE SAI', '\u0BB8\u0BC8'),
162n/a ('TAMIL SYLLABLE MOO', '\u0BAE\u0BCB'),
163n/a ('TAMIL SYLLABLE NNOO', '\u0BA3\u0BCB'),
164n/a ('TAMIL CONSONANT KSS', '\u0B95\u0BCD\u0BB7\u0BCD'),
165n/a ]
166n/a for seqname, codepoints in sequences:
167n/a self.assertEqual(unicodedata.lookup(seqname), codepoints)
168n/a with self.assertRaises(SyntaxError):
169n/a self.checkletter(seqname, None)
170n/a with self.assertRaises(KeyError):
171n/a unicodedata.ucd_3_2_0.lookup(seqname)
172n/a
173n/a def test_named_sequences_full(self):
174n/a # Check all the named sequences
175n/a url = ("http://www.pythontest.net/unicode/%s/NamedSequences.txt" %
176n/a unicodedata.unidata_version)
177n/a try:
178n/a testdata = support.open_urlresource(url, encoding="utf-8",
179n/a check=check_version)
180n/a except (OSError, HTTPException):
181n/a self.skipTest("Could not retrieve " + url)
182n/a self.addCleanup(testdata.close)
183n/a for line in testdata:
184n/a line = line.strip()
185n/a if not line or line.startswith('#'):
186n/a continue
187n/a seqname, codepoints = line.split(';')
188n/a codepoints = ''.join(chr(int(cp, 16)) for cp in codepoints.split())
189n/a self.assertEqual(unicodedata.lookup(seqname), codepoints)
190n/a with self.assertRaises(SyntaxError):
191n/a self.checkletter(seqname, None)
192n/a with self.assertRaises(KeyError):
193n/a unicodedata.ucd_3_2_0.lookup(seqname)
194n/a
195n/a def test_errors(self):
196n/a self.assertRaises(TypeError, unicodedata.name)
197n/a self.assertRaises(TypeError, unicodedata.name, 'xx')
198n/a self.assertRaises(TypeError, unicodedata.lookup)
199n/a self.assertRaises(KeyError, unicodedata.lookup, 'unknown')
200n/a
201n/a def test_strict_error_handling(self):
202n/a # bogus character name
203n/a self.assertRaises(
204n/a UnicodeError,
205n/a str, b"\\N{blah}", 'unicode-escape', 'strict'
206n/a )
207n/a # long bogus character name
208n/a self.assertRaises(
209n/a UnicodeError,
210n/a str, bytes("\\N{%s}" % ("x" * 100000), "ascii"), 'unicode-escape', 'strict'
211n/a )
212n/a # missing closing brace
213n/a self.assertRaises(
214n/a UnicodeError,
215n/a str, b"\\N{SPACE", 'unicode-escape', 'strict'
216n/a )
217n/a # missing opening brace
218n/a self.assertRaises(
219n/a UnicodeError,
220n/a str, b"\\NSPACE", 'unicode-escape', 'strict'
221n/a )
222n/a
223n/a @support.cpython_only
224n/a @unittest.skipUnless(INT_MAX < PY_SSIZE_T_MAX, "needs UINT_MAX < SIZE_MAX")
225n/a @support.bigmemtest(size=UINT_MAX + 1, memuse=2 + 1, dry_run=False)
226n/a def test_issue16335(self, size):
227n/a # very very long bogus character name
228n/a x = b'\\N{SPACE' + b'x' * (UINT_MAX + 1) + b'}'
229n/a self.assertEqual(len(x), len(b'\\N{SPACE}') + (UINT_MAX + 1))
230n/a self.assertRaisesRegex(UnicodeError,
231n/a 'unknown Unicode character name',
232n/a x.decode, 'unicode-escape'
233n/a )
234n/a
235n/a
236n/aif __name__ == "__main__":
237n/a unittest.main()