1 # Copyright (C) 2004-2006 Python Software Foundation
2 # Authors: Baxter, Wouters and Warsaw
3 # Contact: email-sig@python.org
5 """FeedParser - An email feed parser.
7 The feed parser implements an interface for incrementally parsing an email
8 message, line by line. This has advantages for certain applications, such as
9 those reading email messages off a socket.
11 FeedParser.feed() is the primary interface for pushing new data into the
12 parser. It returns when there's nothing more it can do with the available
13 data. When you have no more data to push into the parser, call .close().
14 This completes the parsing and returns the root message object.
16 The other advantage of this parser is that it will never throw a parsing
17 exception. Instead, when it finds something unexpected, it adds a 'defect' to
18 the current message. Defects are just instances that live on the message
19 object's .defects attribute.
23 from email
import Errors
24 from email
import Message
26 NLCRE
= re
.compile('\r\n|\r|\n')
27 NLCRE_bol
= re
.compile('(\r\n|\r|\n)')
28 NLCRE_eol
= re
.compile('(\r\n|\r|\n)$')
29 NLCRE_crack
= re
.compile('(\r\n|\r|\n)')
30 # RFC 2822 $3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character
31 # except controls, SP, and ":".
32 headerRE
= re
.compile(r
'^(From |[\041-\071\073-\176]{1,}:|[\t ])')
36 NeedMoreData
= object()
40 class BufferedSubFile(object):
41 """A file-ish object that can have new data loaded into it.
43 You can also push and pop line-matching predicates onto a stack. When the
44 current predicate matches the current line, a false EOF response
45 (i.e. empty string) is returned instead. This lets the parser adhere to a
46 simple abstraction -- it parses until EOF closes the current message.
49 # The last partial line pushed into this object.
51 # The list of full, pushed lines, in reverse order
53 # The stack of false-EOF checking predicates.
55 # A flag indicating whether the file has been closed or not.
58 def push_eof_matcher(self
, pred
):
59 self
._eofstack
.append(pred
)
61 def pop_eof_matcher(self
):
62 return self
._eofstack
.pop()
65 # Don't forget any trailing partial line.
66 self
._lines
.append(self
._partial
)
75 # Pop the line off the stack and see if it matches the current
76 # false-EOF predicate.
77 line
= self
._lines
.pop()
78 # RFC 2046, section 5.1.2 requires us to recognize outer level
79 # boundaries at any level of inner nesting. Do this, but be sure it's
80 # in the order of most to least nested.
81 for ateof
in self
._eofstack
[::-1]:
83 # We're at the false EOF. But push the last line back first.
84 self
._lines
.append(line
)
88 def unreadline(self
, line
):
89 # Let the consumer push a line back into the buffer.
90 assert line
is not NeedMoreData
91 self
._lines
.append(line
)
94 """Push some new data into this object."""
95 # Handle any previous leftovers
96 data
, self
._partial
= self
._partial
+ data
, ''
97 # Crack into lines, but preserve the newlines on the end of each
98 parts
= NLCRE_crack
.split(data
)
99 # The *ahem* interesting behaviour of re.split when supplied grouping
100 # parentheses is that the last element of the resulting list is the
101 # data after the final RE. In the case of a NL/CR terminated string,
102 # this is the empty string.
103 self
._partial
= parts
.pop()
104 # parts is a list of strings, alternating between the line contents
105 # and the eol character(s). Gather up a list of lines after
106 # re-attaching the newlines.
108 for i
in range(len(parts
) // 2):
109 lines
.append(parts
[i
*2] + parts
[i
*2+1])
110 self
.pushlines(lines
)
112 def pushlines(self
, lines
):
113 # Reverse and insert at the front of the lines.
114 self
._lines
[:0] = lines
[::-1]
123 line
= self
.readline()
131 """A feed-style parser of email."""
133 def __init__(self
, _factory
=Message
.Message
):
134 """_factory is called with no arguments to create a new message obj"""
135 self
._factory
= _factory
136 self
._input
= BufferedSubFile()
138 self
._parse
= self
._parsegen
().next
141 self
._headersonly
= False
143 # Non-public interface for supporting Parser's headersonly flag
144 def _set_headersonly(self
):
145 self
._headersonly
= True
147 def feed(self
, data
):
148 """Push more data into the parser."""
149 self
._input
.push(data
)
152 def _call_parse(self
):
155 except StopIteration:
159 """Parse all remaining data and return the root message object."""
162 root
= self
._pop
_message
()
163 assert not self
._msgstack
164 # Look for final set of defects
165 if root
.get_content_maintype() == 'multipart' \
166 and not root
.is_multipart():
167 root
.defects
.append(Errors
.MultipartInvariantViolationDefect())
170 def _new_message(self
):
171 msg
= self
._factory
()
172 if self
._cur
and self
._cur
.get_content_type() == 'multipart/digest':
173 msg
.set_default_type('message/rfc822')
175 self
._msgstack
[-1].attach(msg
)
176 self
._msgstack
.append(msg
)
180 def _pop_message(self
):
181 retval
= self
._msgstack
.pop()
183 self
._cur
= self
._msgstack
[-1]
189 # Create a new message and start by parsing headers.
192 # Collect the headers, searching for a line that doesn't match the RFC
193 # 2822 header or continuation pattern (including an empty line).
194 for line
in self
._input
:
195 if line
is NeedMoreData
:
198 if not headerRE
.match(line
):
199 # If we saw the RFC defined header/body separator
200 # (i.e. newline), just throw it away. Otherwise the line is
201 # part of the body so push it back.
202 if not NLCRE
.match(line
):
203 self
._input
.unreadline(line
)
206 # Done with the headers, so parse them and figure out what we're
207 # supposed to see in the body of the message.
208 self
._parse
_headers
(headers
)
209 # Headers-only parsing is a backwards compatibility hack, which was
210 # necessary in the older parser, which could throw errors. All
211 # remaining lines in the input are thrown into the message body.
212 if self
._headersonly
:
215 line
= self
._input
.readline()
216 if line
is NeedMoreData
:
222 self
._cur
.set_payload(EMPTYSTRING
.join(lines
))
224 if self
._cur
.get_content_type() == 'message/delivery-status':
225 # message/delivery-status contains blocks of headers separated by
226 # a blank line. We'll represent each header block as a separate
227 # nested message object, but the processing is a bit different
228 # than standard message/* types because there is no body for the
229 # nested messages. A blank line separates the subparts.
231 self
._input
.push_eof_matcher(NLCRE
.match
)
232 for retval
in self
._parsegen
():
233 if retval
is NeedMoreData
:
237 msg
= self
._pop
_message
()
238 # We need to pop the EOF matcher in order to tell if we're at
239 # the end of the current file, not the end of the last block
240 # of message headers.
241 self
._input
.pop_eof_matcher()
242 # The input stream must be sitting at the newline or at the
243 # EOF. We want to see if we're at the end of this subpart, so
244 # first consume the blank line, then test the next line to see
245 # if we're at this subpart's EOF.
247 line
= self
._input
.readline()
248 if line
is NeedMoreData
:
253 line
= self
._input
.readline()
254 if line
is NeedMoreData
:
260 # Not at EOF so this is a line we're going to need.
261 self
._input
.unreadline(line
)
263 if self
._cur
.get_content_maintype() == 'message':
264 # The message claims to be a message/* type, then what follows is
265 # another RFC 2822 message.
266 for retval
in self
._parsegen
():
267 if retval
is NeedMoreData
:
273 if self
._cur
.get_content_maintype() == 'multipart':
274 boundary
= self
._cur
.get_boundary()
276 # The message /claims/ to be a multipart but it has not
277 # defined a boundary. That's a problem which we'll handle by
278 # reading everything until the EOF and marking the message as
280 self
._cur
.defects
.append(Errors
.NoBoundaryInMultipartDefect())
282 for line
in self
._input
:
283 if line
is NeedMoreData
:
287 self
._cur
.set_payload(EMPTYSTRING
.join(lines
))
289 # Create a line match predicate which matches the inter-part
290 # boundary as well as the end-of-multipart boundary. Don't push
291 # this onto the input stream until we've scanned past the
293 separator
= '--' + boundary
294 boundaryre
= re
.compile(
295 '(?P<sep>' + re
.escape(separator
) +
296 r
')(?P<end>--)?(?P<ws>[ \t]*)(?P<linesep>\r\n|\r|\n)?$')
297 capturing_preamble
= True
301 line
= self
._input
.readline()
302 if line
is NeedMoreData
:
307 mo
= boundaryre
.match(line
)
309 # If we're looking at the end boundary, we're done with
310 # this multipart. If there was a newline at the end of
311 # the closing boundary, then we need to initialize the
312 # epilogue with the empty string (see below).
314 linesep
= mo
.group('linesep')
316 # We saw an inter-part boundary. Were we in the preamble?
317 if capturing_preamble
:
319 # According to RFC 2046, the last newline belongs
321 lastline
= preamble
[-1]
322 eolmo
= NLCRE_eol
.search(lastline
)
324 preamble
[-1] = lastline
[:-len(eolmo
.group(0))]
325 self
._cur
.preamble
= EMPTYSTRING
.join(preamble
)
326 capturing_preamble
= False
327 self
._input
.unreadline(line
)
329 # We saw a boundary separating two parts. Consume any
330 # multiple boundary lines that may be following. Our
331 # interpretation of RFC 2046 BNF grammar does not produce
332 # body parts within such double boundaries.
334 line
= self
._input
.readline()
335 if line
is NeedMoreData
:
338 mo
= boundaryre
.match(line
)
340 self
._input
.unreadline(line
)
342 # Recurse to parse this subpart; the input stream points
343 # at the subpart's first line.
344 self
._input
.push_eof_matcher(boundaryre
.match
)
345 for retval
in self
._parsegen
():
346 if retval
is NeedMoreData
:
350 # Because of RFC 2046, the newline preceding the boundary
351 # separator actually belongs to the boundary, not the
352 # previous subpart's payload (or epilogue if the previous
353 # part is a multipart).
354 if self
._last
.get_content_maintype() == 'multipart':
355 epilogue
= self
._last
.epilogue
357 self
._last
.epilogue
= None
358 elif epilogue
is not None:
359 mo
= NLCRE_eol
.search(epilogue
)
361 end
= len(mo
.group(0))
362 self
._last
.epilogue
= epilogue
[:-end
]
364 payload
= self
._last
.get_payload()
365 if isinstance(payload
, basestring
):
366 mo
= NLCRE_eol
.search(payload
)
368 payload
= payload
[:-len(mo
.group(0))]
369 self
._last
.set_payload(payload
)
370 self
._input
.pop_eof_matcher()
372 # Set the multipart up for newline cleansing, which will
373 # happen if we're in a nested multipart.
374 self
._last
= self
._cur
376 # I think we must be in the preamble
377 assert capturing_preamble
378 preamble
.append(line
)
379 # We've seen either the EOF or the end boundary. If we're still
380 # capturing the preamble, we never saw the start boundary. Note
381 # that as a defect and store the captured text as the payload.
382 # Everything from here to the EOF is epilogue.
383 if capturing_preamble
:
384 self
._cur
.defects
.append(Errors
.StartBoundaryNotFoundDefect())
385 self
._cur
.set_payload(EMPTYSTRING
.join(preamble
))
387 for line
in self
._input
:
388 if line
is NeedMoreData
:
391 self
._cur
.epilogue
= EMPTYSTRING
.join(epilogue
)
393 # If the end boundary ended in a newline, we'll need to make sure
394 # the epilogue isn't None
399 for line
in self
._input
:
400 if line
is NeedMoreData
:
403 epilogue
.append(line
)
404 # Any CRLF at the front of the epilogue is not technically part of
405 # the epilogue. Also, watch out for an empty string epilogue,
406 # which means a single newline.
408 firstline
= epilogue
[0]
409 bolmo
= NLCRE_bol
.match(firstline
)
411 epilogue
[0] = firstline
[len(bolmo
.group(0)):]
412 self
._cur
.epilogue
= EMPTYSTRING
.join(epilogue
)
414 # Otherwise, it's some non-multipart type, so the entire rest of the
415 # file contents becomes the payload.
417 for line
in self
._input
:
418 if line
is NeedMoreData
:
422 self
._cur
.set_payload(EMPTYSTRING
.join(lines
))
424 def _parse_headers(self
, lines
):
425 # Passed a list of lines that make up the headers for the current msg
428 for lineno
, line
in enumerate(lines
):
429 # Check for continuation
432 # The first line of the headers was a continuation. This
433 # is illegal, so let's note the defect, store the illegal
434 # line, and ignore it for purposes of headers.
435 defect
= Errors
.FirstHeaderLineIsContinuationDefect(line
)
436 self
._cur
.defects
.append(defect
)
438 lastvalue
.append(line
)
441 # XXX reconsider the joining of folded lines
442 lhdr
= EMPTYSTRING
.join(lastvalue
)[:-1].rstrip('\r\n')
443 self
._cur
[lastheader
] = lhdr
444 lastheader
, lastvalue
= '', []
445 # Check for envelope header, i.e. unix-from
446 if line
.startswith('From '):
448 # Strip off the trailing newline
449 mo
= NLCRE_eol
.search(line
)
451 line
= line
[:-len(mo
.group(0))]
452 self
._cur
.set_unixfrom(line
)
454 elif lineno
== len(lines
) - 1:
455 # Something looking like a unix-from at the end - it's
456 # probably the first line of the body, so push back the
458 self
._input
.unreadline(line
)
461 # Weirdly placed unix-from line. Note this as a defect
463 defect
= Errors
.MisplacedEnvelopeHeaderDefect(line
)
464 self
._cur
.defects
.append(defect
)
466 # Split the line on the colon separating field name from value.
469 defect
= Errors
.MalformedHeaderDefect(line
)
470 self
._cur
.defects
.append(defect
)
472 lastheader
= line
[:i
]
473 lastvalue
= [line
[i
+1:].lstrip()]
474 # Done with all the lines, so handle the last header.
476 # XXX reconsider the joining of folded lines
477 self
._cur
[lastheader
] = EMPTYSTRING
.join(lastvalue
).rstrip('\r\n')