ยปCore Development>Code coverage>Lib/email/feedparser.py

Python code coverage for Lib/email/feedparser.py

#countcontent
1n/a# Copyright (C) 2004-2006 Python Software Foundation
2n/a# Authors: Baxter, Wouters and Warsaw
3n/a# Contact: email-sig@python.org
4n/a
5n/a"""FeedParser - An email feed parser.
6n/a
7n/aThe feed parser implements an interface for incrementally parsing an email
8n/amessage, line by line. This has advantages for certain applications, such as
9n/athose reading email messages off a socket.
10n/a
11n/aFeedParser.feed() is the primary interface for pushing new data into the
12n/aparser. It returns when there's nothing more it can do with the available
13n/adata. When you have no more data to push into the parser, call .close().
14n/aThis completes the parsing and returns the root message object.
15n/a
16n/aThe other advantage of this parser is that it will never raise a parsing
17n/aexception. Instead, when it finds something unexpected, it adds a 'defect' to
18n/athe current message. Defects are just instances that live on the message
19n/aobject's .defects attribute.
20n/a"""
21n/a
22n/a__all__ = ['FeedParser', 'BytesFeedParser']
23n/a
24n/aimport re
25n/a
26n/afrom email import errors
27n/afrom email._policybase import compat32
28n/afrom collections import deque
29n/afrom io import StringIO
30n/a
31n/aNLCRE = re.compile(r'\r\n|\r|\n')
32n/aNLCRE_bol = re.compile(r'(\r\n|\r|\n)')
33n/aNLCRE_eol = re.compile(r'(\r\n|\r|\n)\Z')
34n/aNLCRE_crack = re.compile(r'(\r\n|\r|\n)')
35n/a# RFC 2822 $3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character
36n/a# except controls, SP, and ":".
37n/aheaderRE = re.compile(r'^(From |[\041-\071\073-\176]*:|[\t ])')
38n/aEMPTYSTRING = ''
39n/aNL = '\n'
40n/a
41n/aNeedMoreData = object()
42n/a
43n/a
44n/a
45n/aclass BufferedSubFile(object):
46n/a """A file-ish object that can have new data loaded into it.
47n/a
48n/a You can also push and pop line-matching predicates onto a stack. When the
49n/a current predicate matches the current line, a false EOF response
50n/a (i.e. empty string) is returned instead. This lets the parser adhere to a
51n/a simple abstraction -- it parses until EOF closes the current message.
52n/a """
53n/a def __init__(self):
54n/a # Text stream of the last partial line pushed into this object.
55n/a # See issue 22233 for why this is a text stream and not a list.
56n/a self._partial = StringIO(newline='')
57n/a # A deque of full, pushed lines
58n/a self._lines = deque()
59n/a # The stack of false-EOF checking predicates.
60n/a self._eofstack = []
61n/a # A flag indicating whether the file has been closed or not.
62n/a self._closed = False
63n/a
64n/a def push_eof_matcher(self, pred):
65n/a self._eofstack.append(pred)
66n/a
67n/a def pop_eof_matcher(self):
68n/a return self._eofstack.pop()
69n/a
70n/a def close(self):
71n/a # Don't forget any trailing partial line.
72n/a self._partial.seek(0)
73n/a self.pushlines(self._partial.readlines())
74n/a self._partial.seek(0)
75n/a self._partial.truncate()
76n/a self._closed = True
77n/a
78n/a def readline(self):
79n/a if not self._lines:
80n/a if self._closed:
81n/a return ''
82n/a return NeedMoreData
83n/a # Pop the line off the stack and see if it matches the current
84n/a # false-EOF predicate.
85n/a line = self._lines.popleft()
86n/a # RFC 2046, section 5.1.2 requires us to recognize outer level
87n/a # boundaries at any level of inner nesting. Do this, but be sure it's
88n/a # in the order of most to least nested.
89n/a for ateof in reversed(self._eofstack):
90n/a if ateof(line):
91n/a # We're at the false EOF. But push the last line back first.
92n/a self._lines.appendleft(line)
93n/a return ''
94n/a return line
95n/a
96n/a def unreadline(self, line):
97n/a # Let the consumer push a line back into the buffer.
98n/a assert line is not NeedMoreData
99n/a self._lines.appendleft(line)
100n/a
101n/a def push(self, data):
102n/a """Push some new data into this object."""
103n/a self._partial.write(data)
104n/a if '\n' not in data and '\r' not in data:
105n/a # No new complete lines, wait for more.
106n/a return
107n/a
108n/a # Crack into lines, preserving the linesep characters.
109n/a self._partial.seek(0)
110n/a parts = self._partial.readlines()
111n/a self._partial.seek(0)
112n/a self._partial.truncate()
113n/a
114n/a # If the last element of the list does not end in a newline, then treat
115n/a # it as a partial line. We only check for '\n' here because a line
116n/a # ending with '\r' might be a line that was split in the middle of a
117n/a # '\r\n' sequence (see bugs 1555570 and 1721862).
118n/a if not parts[-1].endswith('\n'):
119n/a self._partial.write(parts.pop())
120n/a self.pushlines(parts)
121n/a
122n/a def pushlines(self, lines):
123n/a self._lines.extend(lines)
124n/a
125n/a def __iter__(self):
126n/a return self
127n/a
128n/a def __next__(self):
129n/a line = self.readline()
130n/a if line == '':
131n/a raise StopIteration
132n/a return line
133n/a
134n/a
135n/a
136n/aclass FeedParser:
137n/a """A feed-style parser of email."""
138n/a
139n/a def __init__(self, _factory=None, *, policy=compat32):
140n/a """_factory is called with no arguments to create a new message obj
141n/a
142n/a The policy keyword specifies a policy object that controls a number of
143n/a aspects of the parser's operation. The default policy maintains
144n/a backward compatibility.
145n/a
146n/a """
147n/a self.policy = policy
148n/a self._old_style_factory = False
149n/a if _factory is None:
150n/a if policy.message_factory is None:
151n/a from email.message import Message
152n/a self._factory = Message
153n/a else:
154n/a self._factory = policy.message_factory
155n/a else:
156n/a self._factory = _factory
157n/a try:
158n/a _factory(policy=self.policy)
159n/a except TypeError:
160n/a # Assume this is an old-style factory
161n/a self._old_style_factory = True
162n/a self._input = BufferedSubFile()
163n/a self._msgstack = []
164n/a self._parse = self._parsegen().__next__
165n/a self._cur = None
166n/a self._last = None
167n/a self._headersonly = False
168n/a
169n/a # Non-public interface for supporting Parser's headersonly flag
170n/a def _set_headersonly(self):
171n/a self._headersonly = True
172n/a
173n/a def feed(self, data):
174n/a """Push more data into the parser."""
175n/a self._input.push(data)
176n/a self._call_parse()
177n/a
178n/a def _call_parse(self):
179n/a try:
180n/a self._parse()
181n/a except StopIteration:
182n/a pass
183n/a
184n/a def close(self):
185n/a """Parse all remaining data and return the root message object."""
186n/a self._input.close()
187n/a self._call_parse()
188n/a root = self._pop_message()
189n/a assert not self._msgstack
190n/a # Look for final set of defects
191n/a if root.get_content_maintype() == 'multipart' \
192n/a and not root.is_multipart():
193n/a defect = errors.MultipartInvariantViolationDefect()
194n/a self.policy.handle_defect(root, defect)
195n/a return root
196n/a
197n/a def _new_message(self):
198n/a if self._old_style_factory:
199n/a msg = self._factory()
200n/a else:
201n/a msg = self._factory(policy=self.policy)
202n/a if self._cur and self._cur.get_content_type() == 'multipart/digest':
203n/a msg.set_default_type('message/rfc822')
204n/a if self._msgstack:
205n/a self._msgstack[-1].attach(msg)
206n/a self._msgstack.append(msg)
207n/a self._cur = msg
208n/a self._last = msg
209n/a
210n/a def _pop_message(self):
211n/a retval = self._msgstack.pop()
212n/a if self._msgstack:
213n/a self._cur = self._msgstack[-1]
214n/a else:
215n/a self._cur = None
216n/a return retval
217n/a
218n/a def _parsegen(self):
219n/a # Create a new message and start by parsing headers.
220n/a self._new_message()
221n/a headers = []
222n/a # Collect the headers, searching for a line that doesn't match the RFC
223n/a # 2822 header or continuation pattern (including an empty line).
224n/a for line in self._input:
225n/a if line is NeedMoreData:
226n/a yield NeedMoreData
227n/a continue
228n/a if not headerRE.match(line):
229n/a # If we saw the RFC defined header/body separator
230n/a # (i.e. newline), just throw it away. Otherwise the line is
231n/a # part of the body so push it back.
232n/a if not NLCRE.match(line):
233n/a defect = errors.MissingHeaderBodySeparatorDefect()
234n/a self.policy.handle_defect(self._cur, defect)
235n/a self._input.unreadline(line)
236n/a break
237n/a headers.append(line)
238n/a # Done with the headers, so parse them and figure out what we're
239n/a # supposed to see in the body of the message.
240n/a self._parse_headers(headers)
241n/a # Headers-only parsing is a backwards compatibility hack, which was
242n/a # necessary in the older parser, which could raise errors. All
243n/a # remaining lines in the input are thrown into the message body.
244n/a if self._headersonly:
245n/a lines = []
246n/a while True:
247n/a line = self._input.readline()
248n/a if line is NeedMoreData:
249n/a yield NeedMoreData
250n/a continue
251n/a if line == '':
252n/a break
253n/a lines.append(line)
254n/a self._cur.set_payload(EMPTYSTRING.join(lines))
255n/a return
256n/a if self._cur.get_content_type() == 'message/delivery-status':
257n/a # message/delivery-status contains blocks of headers separated by
258n/a # a blank line. We'll represent each header block as a separate
259n/a # nested message object, but the processing is a bit different
260n/a # than standard message/* types because there is no body for the
261n/a # nested messages. A blank line separates the subparts.
262n/a while True:
263n/a self._input.push_eof_matcher(NLCRE.match)
264n/a for retval in self._parsegen():
265n/a if retval is NeedMoreData:
266n/a yield NeedMoreData
267n/a continue
268n/a break
269n/a msg = self._pop_message()
270n/a # We need to pop the EOF matcher in order to tell if we're at
271n/a # the end of the current file, not the end of the last block
272n/a # of message headers.
273n/a self._input.pop_eof_matcher()
274n/a # The input stream must be sitting at the newline or at the
275n/a # EOF. We want to see if we're at the end of this subpart, so
276n/a # first consume the blank line, then test the next line to see
277n/a # if we're at this subpart's EOF.
278n/a while True:
279n/a line = self._input.readline()
280n/a if line is NeedMoreData:
281n/a yield NeedMoreData
282n/a continue
283n/a break
284n/a while True:
285n/a line = self._input.readline()
286n/a if line is NeedMoreData:
287n/a yield NeedMoreData
288n/a continue
289n/a break
290n/a if line == '':
291n/a break
292n/a # Not at EOF so this is a line we're going to need.
293n/a self._input.unreadline(line)
294n/a return
295n/a if self._cur.get_content_maintype() == 'message':
296n/a # The message claims to be a message/* type, then what follows is
297n/a # another RFC 2822 message.
298n/a for retval in self._parsegen():
299n/a if retval is NeedMoreData:
300n/a yield NeedMoreData
301n/a continue
302n/a break
303n/a self._pop_message()
304n/a return
305n/a if self._cur.get_content_maintype() == 'multipart':
306n/a boundary = self._cur.get_boundary()
307n/a if boundary is None:
308n/a # The message /claims/ to be a multipart but it has not
309n/a # defined a boundary. That's a problem which we'll handle by
310n/a # reading everything until the EOF and marking the message as
311n/a # defective.
312n/a defect = errors.NoBoundaryInMultipartDefect()
313n/a self.policy.handle_defect(self._cur, defect)
314n/a lines = []
315n/a for line in self._input:
316n/a if line is NeedMoreData:
317n/a yield NeedMoreData
318n/a continue
319n/a lines.append(line)
320n/a self._cur.set_payload(EMPTYSTRING.join(lines))
321n/a return
322n/a # Make sure a valid content type was specified per RFC 2045:6.4.
323n/a if (self._cur.get('content-transfer-encoding', '8bit').lower()
324n/a not in ('7bit', '8bit', 'binary')):
325n/a defect = errors.InvalidMultipartContentTransferEncodingDefect()
326n/a self.policy.handle_defect(self._cur, defect)
327n/a # Create a line match predicate which matches the inter-part
328n/a # boundary as well as the end-of-multipart boundary. Don't push
329n/a # this onto the input stream until we've scanned past the
330n/a # preamble.
331n/a separator = '--' + boundary
332n/a boundaryre = re.compile(
333n/a '(?P<sep>' + re.escape(separator) +
334n/a r')(?P<end>--)?(?P<ws>[ \t]*)(?P<linesep>\r\n|\r|\n)?$')
335n/a capturing_preamble = True
336n/a preamble = []
337n/a linesep = False
338n/a close_boundary_seen = False
339n/a while True:
340n/a line = self._input.readline()
341n/a if line is NeedMoreData:
342n/a yield NeedMoreData
343n/a continue
344n/a if line == '':
345n/a break
346n/a mo = boundaryre.match(line)
347n/a if mo:
348n/a # If we're looking at the end boundary, we're done with
349n/a # this multipart. If there was a newline at the end of
350n/a # the closing boundary, then we need to initialize the
351n/a # epilogue with the empty string (see below).
352n/a if mo.group('end'):
353n/a close_boundary_seen = True
354n/a linesep = mo.group('linesep')
355n/a break
356n/a # We saw an inter-part boundary. Were we in the preamble?
357n/a if capturing_preamble:
358n/a if preamble:
359n/a # According to RFC 2046, the last newline belongs
360n/a # to the boundary.
361n/a lastline = preamble[-1]
362n/a eolmo = NLCRE_eol.search(lastline)
363n/a if eolmo:
364n/a preamble[-1] = lastline[:-len(eolmo.group(0))]
365n/a self._cur.preamble = EMPTYSTRING.join(preamble)
366n/a capturing_preamble = False
367n/a self._input.unreadline(line)
368n/a continue
369n/a # We saw a boundary separating two parts. Consume any
370n/a # multiple boundary lines that may be following. Our
371n/a # interpretation of RFC 2046 BNF grammar does not produce
372n/a # body parts within such double boundaries.
373n/a while True:
374n/a line = self._input.readline()
375n/a if line is NeedMoreData:
376n/a yield NeedMoreData
377n/a continue
378n/a mo = boundaryre.match(line)
379n/a if not mo:
380n/a self._input.unreadline(line)
381n/a break
382n/a # Recurse to parse this subpart; the input stream points
383n/a # at the subpart's first line.
384n/a self._input.push_eof_matcher(boundaryre.match)
385n/a for retval in self._parsegen():
386n/a if retval is NeedMoreData:
387n/a yield NeedMoreData
388n/a continue
389n/a break
390n/a # Because of RFC 2046, the newline preceding the boundary
391n/a # separator actually belongs to the boundary, not the
392n/a # previous subpart's payload (or epilogue if the previous
393n/a # part is a multipart).
394n/a if self._last.get_content_maintype() == 'multipart':
395n/a epilogue = self._last.epilogue
396n/a if epilogue == '':
397n/a self._last.epilogue = None
398n/a elif epilogue is not None:
399n/a mo = NLCRE_eol.search(epilogue)
400n/a if mo:
401n/a end = len(mo.group(0))
402n/a self._last.epilogue = epilogue[:-end]
403n/a else:
404n/a payload = self._last._payload
405n/a if isinstance(payload, str):
406n/a mo = NLCRE_eol.search(payload)
407n/a if mo:
408n/a payload = payload[:-len(mo.group(0))]
409n/a self._last._payload = payload
410n/a self._input.pop_eof_matcher()
411n/a self._pop_message()
412n/a # Set the multipart up for newline cleansing, which will
413n/a # happen if we're in a nested multipart.
414n/a self._last = self._cur
415n/a else:
416n/a # I think we must be in the preamble
417n/a assert capturing_preamble
418n/a preamble.append(line)
419n/a # We've seen either the EOF or the end boundary. If we're still
420n/a # capturing the preamble, we never saw the start boundary. Note
421n/a # that as a defect and store the captured text as the payload.
422n/a if capturing_preamble:
423n/a defect = errors.StartBoundaryNotFoundDefect()
424n/a self.policy.handle_defect(self._cur, defect)
425n/a self._cur.set_payload(EMPTYSTRING.join(preamble))
426n/a epilogue = []
427n/a for line in self._input:
428n/a if line is NeedMoreData:
429n/a yield NeedMoreData
430n/a continue
431n/a self._cur.epilogue = EMPTYSTRING.join(epilogue)
432n/a return
433n/a # If we're not processing the preamble, then we might have seen
434n/a # EOF without seeing that end boundary...that is also a defect.
435n/a if not close_boundary_seen:
436n/a defect = errors.CloseBoundaryNotFoundDefect()
437n/a self.policy.handle_defect(self._cur, defect)
438n/a return
439n/a # Everything from here to the EOF is epilogue. If the end boundary
440n/a # ended in a newline, we'll need to make sure the epilogue isn't
441n/a # None
442n/a if linesep:
443n/a epilogue = ['']
444n/a else:
445n/a epilogue = []
446n/a for line in self._input:
447n/a if line is NeedMoreData:
448n/a yield NeedMoreData
449n/a continue
450n/a epilogue.append(line)
451n/a # Any CRLF at the front of the epilogue is not technically part of
452n/a # the epilogue. Also, watch out for an empty string epilogue,
453n/a # which means a single newline.
454n/a if epilogue:
455n/a firstline = epilogue[0]
456n/a bolmo = NLCRE_bol.match(firstline)
457n/a if bolmo:
458n/a epilogue[0] = firstline[len(bolmo.group(0)):]
459n/a self._cur.epilogue = EMPTYSTRING.join(epilogue)
460n/a return
461n/a # Otherwise, it's some non-multipart type, so the entire rest of the
462n/a # file contents becomes the payload.
463n/a lines = []
464n/a for line in self._input:
465n/a if line is NeedMoreData:
466n/a yield NeedMoreData
467n/a continue
468n/a lines.append(line)
469n/a self._cur.set_payload(EMPTYSTRING.join(lines))
470n/a
471n/a def _parse_headers(self, lines):
472n/a # Passed a list of lines that make up the headers for the current msg
473n/a lastheader = ''
474n/a lastvalue = []
475n/a for lineno, line in enumerate(lines):
476n/a # Check for continuation
477n/a if line[0] in ' \t':
478n/a if not lastheader:
479n/a # The first line of the headers was a continuation. This
480n/a # is illegal, so let's note the defect, store the illegal
481n/a # line, and ignore it for purposes of headers.
482n/a defect = errors.FirstHeaderLineIsContinuationDefect(line)
483n/a self.policy.handle_defect(self._cur, defect)
484n/a continue
485n/a lastvalue.append(line)
486n/a continue
487n/a if lastheader:
488n/a self._cur.set_raw(*self.policy.header_source_parse(lastvalue))
489n/a lastheader, lastvalue = '', []
490n/a # Check for envelope header, i.e. unix-from
491n/a if line.startswith('From '):
492n/a if lineno == 0:
493n/a # Strip off the trailing newline
494n/a mo = NLCRE_eol.search(line)
495n/a if mo:
496n/a line = line[:-len(mo.group(0))]
497n/a self._cur.set_unixfrom(line)
498n/a continue
499n/a elif lineno == len(lines) - 1:
500n/a # Something looking like a unix-from at the end - it's
501n/a # probably the first line of the body, so push back the
502n/a # line and stop.
503n/a self._input.unreadline(line)
504n/a return
505n/a else:
506n/a # Weirdly placed unix-from line. Note this as a defect
507n/a # and ignore it.
508n/a defect = errors.MisplacedEnvelopeHeaderDefect(line)
509n/a self._cur.defects.append(defect)
510n/a continue
511n/a # Split the line on the colon separating field name from value.
512n/a # There will always be a colon, because if there wasn't the part of
513n/a # the parser that calls us would have started parsing the body.
514n/a i = line.find(':')
515n/a
516n/a # If the colon is on the start of the line the header is clearly
517n/a # malformed, but we might be able to salvage the rest of the
518n/a # message. Track the error but keep going.
519n/a if i == 0:
520n/a defect = errors.InvalidHeaderDefect("Missing header name.")
521n/a self._cur.defects.append(defect)
522n/a continue
523n/a
524n/a assert i>0, "_parse_headers fed line with no : and no leading WS"
525n/a lastheader = line[:i]
526n/a lastvalue = [line]
527n/a # Done with all the lines, so handle the last header.
528n/a if lastheader:
529n/a self._cur.set_raw(*self.policy.header_source_parse(lastvalue))
530n/a
531n/a
532n/aclass BytesFeedParser(FeedParser):
533n/a """Like FeedParser, but feed accepts bytes."""
534n/a
535n/a def feed(self, data):
536n/a super().feed(data.decode('ascii', 'surrogateescape'))