mailman/queue/news.py

   1 # Copyright (C) 2000-2008 by the Free Software Foundation, Inc.
   2 #
   3 # This file is part of GNU Mailman.
   4 #
   5 # GNU Mailman is free software: you can redistribute it and/or modify it under
   6 # the terms of the GNU General Public License as published by the Free
   7 # Software Foundation, either version 3 of the License, or (at your option)
   8 # any later version.
   9 #
  10 # GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
  11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13 # more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along with
  16 # GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 """NNTP queue runner."""
  19
  20 import re
  21 import email
  22 import socket
  23 import logging
  24 import nntplib
  25
  26 from cStringIO import StringIO
  27 from email.utils import getaddresses, make_msgid
  28
  29 COMMASPACE = ', '
  30
  31 from mailman import Utils
  32 from mailman.config import config
  33 from mailman.interfaces import NewsModeration
  34 from mailman.queue import Runner
  35
  36 log = logging.getLogger('mailman.error')
  37
  38 # Matches our Mailman crafted Message-IDs.  See Utils.unique_message_id()
  39 # XXX The move to email.utils.make_msgid() breaks this.
  40 mcre = re.compile(r"""
  41     <mailman.                                     # match the prefix
  42     \d+.                                          # serial number
  43     \d+.                                          # time in seconds since epoch
  44     \d+.                                          # pid
  45     (?P<listname>[^@]+)                           # list's internal_name()
  46     @                                             # localpart@dom.ain
  47     (?P<hostname>[^>]+)                           # list's host_name
  48     >                                             # trailer
  49     """, re.VERBOSE)
  50
  51
  52 \f
  53 class NewsRunner(Runner):
  54     QDIR = config.NEWSQUEUE_DIR
  55
  56     def _dispose(self, mlist, msg, msgdata):
  57         # Make sure we have the most up-to-date state
  58         mlist.Load()
  59         if not msgdata.get('prepped'):
  60             prepare_message(mlist, msg, msgdata)
  61         try:
  62             # Flatten the message object, sticking it in a StringIO object
  63             fp = StringIO(msg.as_string())
  64             conn = None
  65             try:
  66                 try:
  67                     nntp_host, nntp_port = Utils.nntpsplit(mlist.nntp_host)
  68                     conn = nntplib.NNTP(nntp_host, nntp_port,
  69                                         readermode=True,
  70                                         user=config.NNTP_USERNAME,
  71                                         password=config.NNTP_PASSWORD)
  72                     conn.post(fp)
  73                 except nntplib.error_temp, e:
  74                     log.error('(NNTPDirect) NNTP error for list "%s": %s',
  75                               mlist.internal_name(), e)
  76                 except socket.error, e:
  77                     log.error('(NNTPDirect) socket error for list "%s": %s',
  78                               mlist.internal_name(), e)
  79             finally:
  80                 if conn:
  81                     conn.quit()
  82         except Exception, e:
  83             # Some other exception occurred, which we definitely did not
  84             # expect, so set this message up for requeuing.
  85             self._log(e)
  86             return True
  87         return False
  88
  89
  90 \f
  91 def prepare_message(mlist, msg, msgdata):
  92     # If the newsgroup is moderated, we need to add this header for the Usenet
  93     # software to accept the posting, and not forward it on to the n.g.'s
  94     # moderation address.  The posting would not have gotten here if it hadn't
  95     # already been approved.  1 == open list, mod n.g., 2 == moderated
  96     if mlist.news_moderation in (NewsModeration.open_moderated,
  97                                  NewsModeration.moderated):
  98         del msg['approved']
  99         msg['Approved'] = mlist.posting_address
 100     # Should we restore the original, non-prefixed subject for gatewayed
 101     # messages? TK: We use stripped_subject (prefix stripped) which was
 102     # crafted in CookHeaders.py to ensure prefix was stripped from the subject
 103     # came from mailing list user.
 104     stripped_subject = msgdata.get('stripped_subject') \
 105                        or msgdata.get('origsubj')
 106     if not mlist.news_prefix_subject_too and stripped_subject is not None:
 107         del msg['subject']
 108         msg['subject'] = stripped_subject
 109     # Add the appropriate Newsgroups: header
 110     ngheader = msg['newsgroups']
 111     if ngheader is not None:
 112         # See if the Newsgroups: header already contains our linked_newsgroup.
 113         # If so, don't add it again.  If not, append our linked_newsgroup to
 114         # the end of the header list
 115         ngroups = [s.strip() for s in ngheader.split(',')]
 116         if mlist.linked_newsgroup not in ngroups:
 117             ngroups.append(mlist.linked_newsgroup)
 118             # Subtitute our new header for the old one.
 119             del msg['newsgroups']
 120             msg['Newsgroups'] = COMMASPACE.join(ngroups)
 121     else:
 122         # Newsgroups: isn't in the message
 123         msg['Newsgroups'] = mlist.linked_newsgroup
 124     # Note: We need to be sure two messages aren't ever sent to the same list
 125     # in the same process, since message ids need to be unique.  Further, if
 126     # messages are crossposted to two Usenet-gated mailing lists, they each
 127     # need to have unique message ids or the nntpd will only accept one of
 128     # them.  The solution here is to substitute any existing message-id that
 129     # isn't ours with one of ours, so we need to parse it to be sure we're not
 130     # looping.
 131     #
 132     # Our Message-ID format is <mailman.secs.pid.listname@hostname>
 133     msgid = msg['message-id']
 134     hackmsgid = True
 135     if msgid:
 136         mo = mcre.search(msgid)
 137         if mo:
 138             lname, hname = mo.group('listname', 'hostname')
 139             if lname == mlist.internal_name() and hname == mlist.host_name:
 140                 hackmsgid = False
 141     if hackmsgid:
 142         del msg['message-id']
 143         msg['Message-ID'] = email.utils.make_msgid()
 144     # Lines: is useful
 145     if msg['Lines'] is None:
 146         # BAW: is there a better way?
 147         count = len(list(email.Iterators.body_line_iterator(msg)))
 148         msg['Lines'] = str(count)
 149     # Massage the message headers by remove some and rewriting others.  This
 150     # woon't completely sanitize the message, but it will eliminate the bulk
 151     # of the rejections based on message headers.  The NNTP server may still
 152     # reject the message because of other problems.
 153     for header in config.NNTP_REMOVE_HEADERS:
 154         del msg[header]
 155     for header, rewrite in config.NNTP_REWRITE_DUPLICATE_HEADERS:
 156         values = msg.get_all(header, [])
 157         if len(values) < 2:
 158             # We only care about duplicates
 159             continue
 160         del msg[header]
 161         # But keep the first one...
 162         msg[header] = values[0]
 163         for v in values[1:]:
 164             msg[rewrite] = v
 165     # Mark this message as prepared in case it has to be requeued
 166     msgdata['prepped'] = True