src/mailman/runners/nntp.py

   1 # Copyright (C) 2000-2016 by the Free Software Foundation, Inc.
   2 #
   3 # This file is part of GNU Mailman.
   4 #
   5 # GNU Mailman is free software: you can redistribute it and/or modify it under
   6 # the terms of the GNU General Public License as published by the Free
   7 # Software Foundation, either version 3 of the License, or (at your option)
   8 # any later version.
   9 #
  10 # GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
  11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13 # more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along with
  16 # GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 """NNTP runner."""
  19
  20 __all__ = [
  21     'NNTPRunner',
  22     ]
  23
  24
  25 import re
  26 import email
  27 import socket
  28 import logging
  29 import nntplib
  30
  31 from io import StringIO
  32 from mailman.config import config
  33 from mailman.core.runner import Runner
  34 from mailman.interfaces.nntp import NewsgroupModeration
  35
  36
  37 COMMA = ','
  38 COMMASPACE = ', '
  39 log = logging.getLogger('mailman.error')
  40
  41
  42 # Matches our Mailman crafted Message-IDs.  See Utils.unique_message_id()
  43 # XXX The move to email.utils.make_msgid() breaks this.
  44 mcre = re.compile(r"""
  45     <mailman.                                     # match the prefix
  46     \d+.                                          # serial number
  47     \d+.                                          # time in seconds since epoch
  48     \d+.                                          # pid
  49     (?P<listname>[^@]+)                           # list's internal_name()
  50     @                                             # localpart@dom.ain
  51     (?P<hostname>[^>]+)                           # list's mail_host
  52     >                                             # trailer
  53     """, re.VERBOSE)
  54
  55
  56 \f
  57 class NNTPRunner(Runner):
  58     def _dispose(self, mlist, msg, msgdata):
  59         # Get NNTP server connection information.
  60         host = config.nntp.host.strip()
  61         port = config.nntp.port.strip()
  62         if len(port) == 0:
  63             port = 119
  64         else:
  65             try:
  66                 port = int(port)
  67             except (TypeError, ValueError):
  68                 log.exception('Bad [nntp]port value: {0}'.format(port))
  69                 port = 119
  70         # Make sure we have the most up-to-date state
  71         if not msgdata.get('prepped'):
  72             prepare_message(mlist, msg, msgdata)
  73         # Flatten the message object, sticking it in a StringIO object
  74         fp = StringIO(msg.as_string())
  75         conn = None
  76         try:
  77             conn = nntplib.NNTP(host, port,
  78                                 readermode=True,
  79                                 user=config.nntp.user,
  80                                 password=config.nntp.password)
  81             conn.post(fp)
  82         except nntplib.NNTPTemporaryError:
  83             log.exception('{0} NNTP error for {1}'.format(
  84                 msg.get('message-id', 'n/a'), mlist.fqdn_listname))
  85         except socket.error:
  86             log.exception('{0} NNTP socket error for {1}'.format(
  87                 msg.get('message-id', 'n/a'), mlist.fqdn_listname))
  88         except Exception:
  89             # Some other exception occurred, which we definitely did not
  90             # expect, so set this message up for requeuing.
  91             log.exception('{0} NNTP unexpected exception for {1}'.format(
  92                 msg.get('message-id', 'n/a'), mlist.fqdn_listname))
  93             return True
  94         finally:
  95             if conn:
  96                 conn.quit()
  97         return False
  98
  99
 100 \f
 101 def prepare_message(mlist, msg, msgdata):
 102     # If the newsgroup is moderated, we need to add this header for the Usenet
 103     # software to accept the posting, and not forward it on to the n.g.'s
 104     # moderation address.  The posting would not have gotten here if it hadn't
 105     # already been approved.  1 == open list, mod n.g., 2 == moderated
 106     if mlist.newsgroup_moderation in (NewsgroupModeration.open_moderated,
 107                                       NewsgroupModeration.moderated):
 108         del msg['approved']
 109         msg['Approved'] = mlist.posting_address
 110     # Should we restore the original, non-prefixed subject for gatewayed
 111     # messages? TK: We use stripped_subject (prefix stripped) which was crafted
 112     # in the subject-prefix handler to ensure prefix was stripped from the
 113     # subject came from mailing list user.
 114     stripped_subject = msgdata.get('stripped_subject',
 115                                    msgdata.get('original_subject'))
 116     if not mlist.nntp_prefix_subject_too and stripped_subject is not None:
 117         del msg['subject']
 118         msg['subject'] = stripped_subject
 119     # Add the appropriate Newsgroups header.  Multiple Newsgroups headers are
 120     # generally not allowed so we're not testing for them.
 121     header = msg.get('newsgroups')
 122     if header is None:
 123         msg['Newsgroups'] = mlist.linked_newsgroup
 124     else:
 125         # See if the Newsgroups: header already contains our linked_newsgroup.
 126         # If so, don't add it again.  If not, append our linked_newsgroup to
 127         # the end of the header list
 128         newsgroups = [value.strip() for value in header.split(COMMA)]
 129         if mlist.linked_newsgroup not in newsgroups:
 130             newsgroups.append(mlist.linked_newsgroup)
 131             # Subtitute our new header for the old one.
 132             del msg['newsgroups']
 133             msg['Newsgroups'] = COMMASPACE.join(newsgroups)
 134     # Note: We need to be sure two messages aren't ever sent to the same list
 135     # in the same process, since message ids need to be unique.  Further, if
 136     # messages are crossposted to two gated mailing lists, they must each have
 137     # unique message ids or the nntpd will only accept one of them.  The
 138     # solution here is to substitute any existing message-id that isn't ours
 139     # with one of ours, so we need to parse it to be sure we're not looping.
 140     #
 141     # Our Message-ID format is <mailman.secs.pid.listname@hostname>
 142     #
 143     # XXX 2012-03-31 BAW: What we really want to do is try posting the message
 144     # to the nntpd first, and only if that fails substitute a unique
 145     # Message-ID.  The following should get moved out of prepare_message() and
 146     # into _dispose() above.
 147     msgid = msg['message-id']
 148     hackmsgid = True
 149     if msgid:
 150         mo = mcre.search(msgid)
 151         if mo:
 152             lname, hname = mo.group('listname', 'hostname')
 153             if lname == mlist.internal_name() and hname == mlist.mail_host:
 154                 hackmsgid = False
 155     if hackmsgid:
 156         del msg['message-id']
 157         msg['Message-ID'] = email.utils.make_msgid()
 158     # Lines: is useful.
 159     if msg['Lines'] is None:
 160         # BAW: is there a better way?
 161         count = len(list(email.iterators.body_line_iterator(msg)))
 162         msg['Lines'] = str(count)
 163     # Massage the message headers by remove some and rewriting others.  This
 164     # won't completely sanitize the message, but it will eliminate the bulk of
 165     # the rejections based on message headers.  The NNTP server may still
 166     # reject the message because of other problems.
 167     for header in config.nntp.remove_headers.split():
 168         del msg[header]
 169     dup_headers = config.nntp.rewrite_duplicate_headers.split()
 170     if len(dup_headers) % 2 != 0:
 171         # There are an odd number of headers; ignore the last one.
 172         bad_header = dup_headers.pop()
 173         log.error('Ignoring odd [nntp]rewrite_duplicate_headers: {0}'.format(
 174             bad_header))
 175     dup_headers.reverse()
 176     while dup_headers:
 177         source = dup_headers.pop()
 178         target = dup_headers.pop()
 179         values = msg.get_all(source, [])
 180         if len(values) < 2:
 181             # We only care about duplicates.
 182             continue
 183         # Delete all the original headers.
 184         del msg[source]
 185         # Put the first value back on the original header.
 186         msg[source] = values[0]
 187         # And put all the subsequent values on the destination header.
 188         for value in values[1:]:
 189             msg[target] = value
 190     # Mark this message as prepared in case it has to be requeued.
 191     msgdata['prepped'] = True