ยปCore Development>Code coverage>Lib/email/charset.py

Python code coverage for Lib/email/charset.py

#countcontent
1n/a# Copyright (C) 2001-2007 Python Software Foundation
2n/a# Author: Ben Gertzfield, Barry Warsaw
3n/a# Contact: email-sig@python.org
4n/a
5n/a__all__ = [
6n/a 'Charset',
7n/a 'add_alias',
8n/a 'add_charset',
9n/a 'add_codec',
10n/a ]
11n/a
12n/afrom functools import partial
13n/a
14n/aimport email.base64mime
15n/aimport email.quoprimime
16n/a
17n/afrom email import errors
18n/afrom email.encoders import encode_7or8bit
19n/a
20n/a
21n/a
22n/a# Flags for types of header encodings
23n/aQP = 1 # Quoted-Printable
24n/aBASE64 = 2 # Base64
25n/aSHORTEST = 3 # the shorter of QP and base64, but only for headers
26n/a
27n/a# In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7
28n/aRFC2047_CHROME_LEN = 7
29n/a
30n/aDEFAULT_CHARSET = 'us-ascii'
31n/aUNKNOWN8BIT = 'unknown-8bit'
32n/aEMPTYSTRING = ''
33n/a
34n/a
35n/a
36n/a# Defaults
37n/aCHARSETS = {
38n/a # input header enc body enc output conv
39n/a 'iso-8859-1': (QP, QP, None),
40n/a 'iso-8859-2': (QP, QP, None),
41n/a 'iso-8859-3': (QP, QP, None),
42n/a 'iso-8859-4': (QP, QP, None),
43n/a # iso-8859-5 is Cyrillic, and not especially used
44n/a # iso-8859-6 is Arabic, also not particularly used
45n/a # iso-8859-7 is Greek, QP will not make it readable
46n/a # iso-8859-8 is Hebrew, QP will not make it readable
47n/a 'iso-8859-9': (QP, QP, None),
48n/a 'iso-8859-10': (QP, QP, None),
49n/a # iso-8859-11 is Thai, QP will not make it readable
50n/a 'iso-8859-13': (QP, QP, None),
51n/a 'iso-8859-14': (QP, QP, None),
52n/a 'iso-8859-15': (QP, QP, None),
53n/a 'iso-8859-16': (QP, QP, None),
54n/a 'windows-1252':(QP, QP, None),
55n/a 'viscii': (QP, QP, None),
56n/a 'us-ascii': (None, None, None),
57n/a 'big5': (BASE64, BASE64, None),
58n/a 'gb2312': (BASE64, BASE64, None),
59n/a 'euc-jp': (BASE64, None, 'iso-2022-jp'),
60n/a 'shift_jis': (BASE64, None, 'iso-2022-jp'),
61n/a 'iso-2022-jp': (BASE64, None, None),
62n/a 'koi8-r': (BASE64, BASE64, None),
63n/a 'utf-8': (SHORTEST, BASE64, 'utf-8'),
64n/a }
65n/a
66n/a# Aliases for other commonly-used names for character sets. Map
67n/a# them to the real ones used in email.
68n/aALIASES = {
69n/a 'latin_1': 'iso-8859-1',
70n/a 'latin-1': 'iso-8859-1',
71n/a 'latin_2': 'iso-8859-2',
72n/a 'latin-2': 'iso-8859-2',
73n/a 'latin_3': 'iso-8859-3',
74n/a 'latin-3': 'iso-8859-3',
75n/a 'latin_4': 'iso-8859-4',
76n/a 'latin-4': 'iso-8859-4',
77n/a 'latin_5': 'iso-8859-9',
78n/a 'latin-5': 'iso-8859-9',
79n/a 'latin_6': 'iso-8859-10',
80n/a 'latin-6': 'iso-8859-10',
81n/a 'latin_7': 'iso-8859-13',
82n/a 'latin-7': 'iso-8859-13',
83n/a 'latin_8': 'iso-8859-14',
84n/a 'latin-8': 'iso-8859-14',
85n/a 'latin_9': 'iso-8859-15',
86n/a 'latin-9': 'iso-8859-15',
87n/a 'latin_10':'iso-8859-16',
88n/a 'latin-10':'iso-8859-16',
89n/a 'cp949': 'ks_c_5601-1987',
90n/a 'euc_jp': 'euc-jp',
91n/a 'euc_kr': 'euc-kr',
92n/a 'ascii': 'us-ascii',
93n/a }
94n/a
95n/a
96n/a# Map charsets to their Unicode codec strings.
97n/aCODEC_MAP = {
98n/a 'gb2312': 'eucgb2312_cn',
99n/a 'big5': 'big5_tw',
100n/a # Hack: We don't want *any* conversion for stuff marked us-ascii, as all
101n/a # sorts of garbage might be sent to us in the guise of 7-bit us-ascii.
102n/a # Let that stuff pass through without conversion to/from Unicode.
103n/a 'us-ascii': None,
104n/a }
105n/a
106n/a
107n/a
108n/a# Convenience functions for extending the above mappings
109n/adef add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
110n/a """Add character set properties to the global registry.
111n/a
112n/a charset is the input character set, and must be the canonical name of a
113n/a character set.
114n/a
115n/a Optional header_enc and body_enc is either Charset.QP for
116n/a quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for
117n/a the shortest of qp or base64 encoding, or None for no encoding. SHORTEST
118n/a is only valid for header_enc. It describes how message headers and
119n/a message bodies in the input charset are to be encoded. Default is no
120n/a encoding.
121n/a
122n/a Optional output_charset is the character set that the output should be
123n/a in. Conversions will proceed from input charset, to Unicode, to the
124n/a output charset when the method Charset.convert() is called. The default
125n/a is to output in the same character set as the input.
126n/a
127n/a Both input_charset and output_charset must have Unicode codec entries in
128n/a the module's charset-to-codec mapping; use add_codec(charset, codecname)
129n/a to add codecs the module does not know about. See the codecs module's
130n/a documentation for more information.
131n/a """
132n/a if body_enc == SHORTEST:
133n/a raise ValueError('SHORTEST not allowed for body_enc')
134n/a CHARSETS[charset] = (header_enc, body_enc, output_charset)
135n/a
136n/a
137n/adef add_alias(alias, canonical):
138n/a """Add a character set alias.
139n/a
140n/a alias is the alias name, e.g. latin-1
141n/a canonical is the character set's canonical name, e.g. iso-8859-1
142n/a """
143n/a ALIASES[alias] = canonical
144n/a
145n/a
146n/adef add_codec(charset, codecname):
147n/a """Add a codec that map characters in the given charset to/from Unicode.
148n/a
149n/a charset is the canonical name of a character set. codecname is the name
150n/a of a Python codec, as appropriate for the second argument to the unicode()
151n/a built-in, or to the encode() method of a Unicode string.
152n/a """
153n/a CODEC_MAP[charset] = codecname
154n/a
155n/a
156n/a
157n/a# Convenience function for encoding strings, taking into account
158n/a# that they might be unknown-8bit (ie: have surrogate-escaped bytes)
159n/adef _encode(string, codec):
160n/a if codec == UNKNOWN8BIT:
161n/a return string.encode('ascii', 'surrogateescape')
162n/a else:
163n/a return string.encode(codec)
164n/a
165n/a
166n/a
167n/aclass Charset:
168n/a """Map character sets to their email properties.
169n/a
170n/a This class provides information about the requirements imposed on email
171n/a for a specific character set. It also provides convenience routines for
172n/a converting between character sets, given the availability of the
173n/a applicable codecs. Given a character set, it will do its best to provide
174n/a information on how to use that character set in an email in an
175n/a RFC-compliant way.
176n/a
177n/a Certain character sets must be encoded with quoted-printable or base64
178n/a when used in email headers or bodies. Certain character sets must be
179n/a converted outright, and are not allowed in email. Instances of this
180n/a module expose the following information about a character set:
181n/a
182n/a input_charset: The initial character set specified. Common aliases
183n/a are converted to their `official' email names (e.g. latin_1
184n/a is converted to iso-8859-1). Defaults to 7-bit us-ascii.
185n/a
186n/a header_encoding: If the character set must be encoded before it can be
187n/a used in an email header, this attribute will be set to
188n/a Charset.QP (for quoted-printable), Charset.BASE64 (for
189n/a base64 encoding), or Charset.SHORTEST for the shortest of
190n/a QP or BASE64 encoding. Otherwise, it will be None.
191n/a
192n/a body_encoding: Same as header_encoding, but describes the encoding for the
193n/a mail message's body, which indeed may be different than the
194n/a header encoding. Charset.SHORTEST is not allowed for
195n/a body_encoding.
196n/a
197n/a output_charset: Some character sets must be converted before they can be
198n/a used in email headers or bodies. If the input_charset is
199n/a one of them, this attribute will contain the name of the
200n/a charset output will be converted to. Otherwise, it will
201n/a be None.
202n/a
203n/a input_codec: The name of the Python codec used to convert the
204n/a input_charset to Unicode. If no conversion codec is
205n/a necessary, this attribute will be None.
206n/a
207n/a output_codec: The name of the Python codec used to convert Unicode
208n/a to the output_charset. If no conversion codec is necessary,
209n/a this attribute will have the same value as the input_codec.
210n/a """
211n/a def __init__(self, input_charset=DEFAULT_CHARSET):
212n/a # RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to
213n/a # unicode because its .lower() is locale insensitive. If the argument
214n/a # is already a unicode, we leave it at that, but ensure that the
215n/a # charset is ASCII, as the standard (RFC XXX) requires.
216n/a try:
217n/a if isinstance(input_charset, str):
218n/a input_charset.encode('ascii')
219n/a else:
220n/a input_charset = str(input_charset, 'ascii')
221n/a except UnicodeError:
222n/a raise errors.CharsetError(input_charset)
223n/a input_charset = input_charset.lower()
224n/a # Set the input charset after filtering through the aliases
225n/a self.input_charset = ALIASES.get(input_charset, input_charset)
226n/a # We can try to guess which encoding and conversion to use by the
227n/a # charset_map dictionary. Try that first, but let the user override
228n/a # it.
229n/a henc, benc, conv = CHARSETS.get(self.input_charset,
230n/a (SHORTEST, BASE64, None))
231n/a if not conv:
232n/a conv = self.input_charset
233n/a # Set the attributes, allowing the arguments to override the default.
234n/a self.header_encoding = henc
235n/a self.body_encoding = benc
236n/a self.output_charset = ALIASES.get(conv, conv)
237n/a # Now set the codecs. If one isn't defined for input_charset,
238n/a # guess and try a Unicode codec with the same name as input_codec.
239n/a self.input_codec = CODEC_MAP.get(self.input_charset,
240n/a self.input_charset)
241n/a self.output_codec = CODEC_MAP.get(self.output_charset,
242n/a self.output_charset)
243n/a
244n/a def __str__(self):
245n/a return self.input_charset.lower()
246n/a
247n/a __repr__ = __str__
248n/a
249n/a def __eq__(self, other):
250n/a return str(self) == str(other).lower()
251n/a
252n/a def get_body_encoding(self):
253n/a """Return the content-transfer-encoding used for body encoding.
254n/a
255n/a This is either the string `quoted-printable' or `base64' depending on
256n/a the encoding used, or it is a function in which case you should call
257n/a the function with a single argument, the Message object being
258n/a encoded. The function should then set the Content-Transfer-Encoding
259n/a header itself to whatever is appropriate.
260n/a
261n/a Returns "quoted-printable" if self.body_encoding is QP.
262n/a Returns "base64" if self.body_encoding is BASE64.
263n/a Returns conversion function otherwise.
264n/a """
265n/a assert self.body_encoding != SHORTEST
266n/a if self.body_encoding == QP:
267n/a return 'quoted-printable'
268n/a elif self.body_encoding == BASE64:
269n/a return 'base64'
270n/a else:
271n/a return encode_7or8bit
272n/a
273n/a def get_output_charset(self):
274n/a """Return the output character set.
275n/a
276n/a This is self.output_charset if that is not None, otherwise it is
277n/a self.input_charset.
278n/a """
279n/a return self.output_charset or self.input_charset
280n/a
281n/a def header_encode(self, string):
282n/a """Header-encode a string by converting it first to bytes.
283n/a
284n/a The type of encoding (base64 or quoted-printable) will be based on
285n/a this charset's `header_encoding`.
286n/a
287n/a :param string: A unicode string for the header. It must be possible
288n/a to encode this string to bytes using the character set's
289n/a output codec.
290n/a :return: The encoded string, with RFC 2047 chrome.
291n/a """
292n/a codec = self.output_codec or 'us-ascii'
293n/a header_bytes = _encode(string, codec)
294n/a # 7bit/8bit encodings return the string unchanged (modulo conversions)
295n/a encoder_module = self._get_encoder(header_bytes)
296n/a if encoder_module is None:
297n/a return string
298n/a return encoder_module.header_encode(header_bytes, codec)
299n/a
300n/a def header_encode_lines(self, string, maxlengths):
301n/a """Header-encode a string by converting it first to bytes.
302n/a
303n/a This is similar to `header_encode()` except that the string is fit
304n/a into maximum line lengths as given by the argument.
305n/a
306n/a :param string: A unicode string for the header. It must be possible
307n/a to encode this string to bytes using the character set's
308n/a output codec.
309n/a :param maxlengths: Maximum line length iterator. Each element
310n/a returned from this iterator will provide the next maximum line
311n/a length. This parameter is used as an argument to built-in next()
312n/a and should never be exhausted. The maximum line lengths should
313n/a not count the RFC 2047 chrome. These line lengths are only a
314n/a hint; the splitter does the best it can.
315n/a :return: Lines of encoded strings, each with RFC 2047 chrome.
316n/a """
317n/a # See which encoding we should use.
318n/a codec = self.output_codec or 'us-ascii'
319n/a header_bytes = _encode(string, codec)
320n/a encoder_module = self._get_encoder(header_bytes)
321n/a encoder = partial(encoder_module.header_encode, charset=codec)
322n/a # Calculate the number of characters that the RFC 2047 chrome will
323n/a # contribute to each line.
324n/a charset = self.get_output_charset()
325n/a extra = len(charset) + RFC2047_CHROME_LEN
326n/a # Now comes the hard part. We must encode bytes but we can't split on
327n/a # bytes because some character sets are variable length and each
328n/a # encoded word must stand on its own. So the problem is you have to
329n/a # encode to bytes to figure out this word's length, but you must split
330n/a # on characters. This causes two problems: first, we don't know how
331n/a # many octets a specific substring of unicode characters will get
332n/a # encoded to, and second, we don't know how many ASCII characters
333n/a # those octets will get encoded to. Unless we try it. Which seems
334n/a # inefficient. In the interest of being correct rather than fast (and
335n/a # in the hope that there will be few encoded headers in any such
336n/a # message), brute force it. :(
337n/a lines = []
338n/a current_line = []
339n/a maxlen = next(maxlengths) - extra
340n/a for character in string:
341n/a current_line.append(character)
342n/a this_line = EMPTYSTRING.join(current_line)
343n/a length = encoder_module.header_length(_encode(this_line, charset))
344n/a if length > maxlen:
345n/a # This last character doesn't fit so pop it off.
346n/a current_line.pop()
347n/a # Does nothing fit on the first line?
348n/a if not lines and not current_line:
349n/a lines.append(None)
350n/a else:
351n/a separator = (' ' if lines else '')
352n/a joined_line = EMPTYSTRING.join(current_line)
353n/a header_bytes = _encode(joined_line, codec)
354n/a lines.append(encoder(header_bytes))
355n/a current_line = [character]
356n/a maxlen = next(maxlengths) - extra
357n/a joined_line = EMPTYSTRING.join(current_line)
358n/a header_bytes = _encode(joined_line, codec)
359n/a lines.append(encoder(header_bytes))
360n/a return lines
361n/a
362n/a def _get_encoder(self, header_bytes):
363n/a if self.header_encoding == BASE64:
364n/a return email.base64mime
365n/a elif self.header_encoding == QP:
366n/a return email.quoprimime
367n/a elif self.header_encoding == SHORTEST:
368n/a len64 = email.base64mime.header_length(header_bytes)
369n/a lenqp = email.quoprimime.header_length(header_bytes)
370n/a if len64 < lenqp:
371n/a return email.base64mime
372n/a else:
373n/a return email.quoprimime
374n/a else:
375n/a return None
376n/a
377n/a def body_encode(self, string):
378n/a """Body-encode a string by converting it first to bytes.
379n/a
380n/a The type of encoding (base64 or quoted-printable) will be based on
381n/a self.body_encoding. If body_encoding is None, we assume the
382n/a output charset is a 7bit encoding, so re-encoding the decoded
383n/a string using the ascii codec produces the correct string version
384n/a of the content.
385n/a """
386n/a if not string:
387n/a return string
388n/a if self.body_encoding is BASE64:
389n/a if isinstance(string, str):
390n/a string = string.encode(self.output_charset)
391n/a return email.base64mime.body_encode(string)
392n/a elif self.body_encoding is QP:
393n/a # quopromime.body_encode takes a string, but operates on it as if
394n/a # it were a list of byte codes. For a (minimal) history on why
395n/a # this is so, see changeset 0cf700464177. To correctly encode a
396n/a # character set, then, we must turn it into pseudo bytes via the
397n/a # latin1 charset, which will encode any byte as a single code point
398n/a # between 0 and 255, which is what body_encode is expecting.
399n/a if isinstance(string, str):
400n/a string = string.encode(self.output_charset)
401n/a string = string.decode('latin1')
402n/a return email.quoprimime.body_encode(string)
403n/a else:
404n/a if isinstance(string, str):
405n/a string = string.encode(self.output_charset).decode('ascii')
406n/a return string