src/mailman/runners/nntp.py

   1 # Copyright (C) 2000-2023 by the Free Software Foundation, Inc.
   2 #
   3 # This file is part of GNU Mailman.
   4 #
   5 # GNU Mailman is free software: you can redistribute it and/or modify it under
   6 # the terms of the GNU General Public License as published by the Free
   7 # Software Foundation, either version 3 of the License, or (at your option)
   8 # any later version.
   9 #
  10 # GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
  11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13 # more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along with
  16 # GNU Mailman.  If not, see <https://www.gnu.org/licenses/>.
  17
  18 """NNTP runner."""
  19
  20 import os
  21 import re
  22 import sys
  23 import email
  24 import socket
  25 import logging
  26 import nntplib
  27 import subprocess
  28
  29 from datetime import datetime
  30 from io import BytesIO
  31 from lazr.config import as_timedelta
  32 from mailman.config import config
  33 from mailman.core.runner import Runner
  34 from mailman.interfaces.nntp import NewsgroupModeration
  35 from public import public
  36
  37
  38 COMMA = ','
  39 COMMASPACE = ', '
  40 log = logging.getLogger('mailman.error')
  41
  42
  43 # Matches our crafted Message-ID.
  44 mcre = re.compile(r"""
  45     <                                    # match the prefix
  46     \d+\.                                # time in centi-seconds since epoch
  47     \d+\.                                # pid
  48     \d+\.                                # random number
  49     (?P<listname>[^@]+)                  # list's list_name
  50     @                                    # localpart@dom.ain
  51     (?P<hostname>[^>]+)                  # list's mail_host
  52     >                                    # trailer
  53     """, re.VERBOSE)
  54
  55
  56 @public
  57 class NNTPRunner(Runner):
  58     def __init__(self, name, slice=None):
  59         super().__init__(name, slice)
  60         self.lastrun = datetime.min
  61         self.delay = as_timedelta(config.nntp.gatenews_every)
  62         self.slice = slice
  63         python = sys.executable
  64         mailman = os.path.join(config.BIN_DIR, 'mailman')
  65         conf = config.filename
  66         self.cmd = [python, mailman, '-C', conf, 'gatenews']
  67         log.debug(self.cmd)
  68
  69     def _dispose(self, mlist, msg, msgdata):
  70         # Get NNTP server connection information.
  71         host = config.nntp.host.strip()
  72         port = config.nntp.port.strip()
  73         if len(port) == 0:
  74             port = 119
  75         else:
  76             try:
  77                 port = int(port)
  78             except (TypeError, ValueError):
  79                 log.exception('Bad [nntp]port value: {}'.format(port))
  80                 port = 119
  81         # Make sure we have the most up-to-date state
  82         if not msgdata.get('prepped'):
  83             prepare_message(mlist, msg, msgdata)
  84         # Flatten the message object, sticking it in a BytesIO object
  85         fp = BytesIO()
  86         email.generator.BytesGenerator(fp, maxheaderlen=0).flatten(msg)
  87         fp.seek(0)
  88         conn = None
  89         try:
  90             conn = nntplib.NNTP(host, port,
  91                                 readermode=True,
  92                                 user=config.nntp.user,
  93                                 password=config.nntp.password)
  94             conn.post(fp)
  95         except nntplib.NNTPTemporaryError:
  96             # This could be a duplicate Message-ID for a message cross-posted
  97             # to another group.  See if we already munged the Message-ID.
  98             mo = mcre.search(msg.get('message-id', 'n/a'))
  99             if (mo and mo.group('listname') == mlist.list_name and
 100                     mo.group('hostname') == mlist.mail_host):
 101                 # This is our munged Message-ID.  This must be a failure of the
 102                 # requeued message or a Message-ID we added in prepare_message.
 103                 # Get the original Message-ID or the one we added and log it.
 104                 log_message_id = msgdata.get('original_message-id',
 105                                              msg.get('message-id', 'n/a'))
 106                 log.exception('{} NNTP error for {}'.format(
 107                     log_message_id, mlist.fqdn_listname))
 108             else:
 109                 # This might be a duplicate Message-ID.  Munge it and requeue
 110                 # the message, but save the original Message-ID for logging.
 111                 msgdata['original_message-id'] = msg.get('message-id', 'n/a')
 112                 del msg['message-id']
 113                 msg['Message-ID'] = email.utils.make_msgid(mlist.list_name,
 114                                                            mlist.mail_host)
 115                 return True
 116         except socket.error:
 117             log.exception('{} NNTP socket error for {}'.format(
 118                 msg.get('message-id', 'n/a'), mlist.fqdn_listname))
 119         except Exception:
 120             # Some other exception occurred, which we definitely did not
 121             # expect, so set this message up for requeuing.
 122             log.exception('{} NNTP unexpected exception for {}'.format(
 123                 msg.get('message-id', 'n/a'), mlist.fqdn_listname))
 124             return True
 125         finally:
 126             if conn:
 127                 conn.quit()
 128         return False
 129
 130     def _do_periodic(self):
 131         """Invoked periodically by the run() method in the super class."""
 132         if self.lastrun + self.delay > datetime.now():
 133             return                                    # pragma: nocover
 134         if not (self.slice in (None, 0)):
 135             # If queue is sliced, only run for slice = 0.
 136             return                                    # pragma: nocover
 137         self.lastrun = datetime.now()
 138         log.debug('Running nntp runner periodic task gatenews')
 139         os.environ['_MAILMAN_GATENEWS_NNTP'] = 'yes'
 140         result = subprocess.run(self.cmd, stderr=subprocess.PIPE)
 141         if result.returncode != 0:
 142             log.error(f"""\
 143 gatenews failed. status: {result.returncode}
 144 message: {result.stderr}""")
 145
 146
 147 def prepare_message(mlist, msg, msgdata):
 148     # If the newsgroup is moderated, we need to add this header for the Usenet
 149     # software to accept the posting, and not forward it on to the n.g.'s
 150     # moderation address.  The posting would not have gotten here if it hadn't
 151     # already been approved.  1 == open list, mod n.g., 2 == moderated
 152     if mlist.newsgroup_moderation in (NewsgroupModeration.open_moderated,
 153                                       NewsgroupModeration.moderated):
 154         del msg['approved']
 155         msg['Approved'] = mlist.posting_address
 156     # Should we restore the original, non-prefixed subject for gatewayed
 157     # messages? TK: We use stripped_subject (prefix stripped) which was crafted
 158     # in the subject-prefix handler to ensure prefix was stripped from the
 159     # subject came from mailing list user.
 160     stripped_subject = msgdata.get('stripped_subject',
 161                                    msgdata.get('original_subject'))
 162     if not mlist.nntp_prefix_subject_too and stripped_subject is not None:
 163         del msg['subject']
 164         msg['Subject'] = stripped_subject
 165     # Add the appropriate Newsgroups header.  Multiple Newsgroups headers are
 166     # generally not allowed so we're not testing for them.
 167     header = msg.get('newsgroups')
 168     if header is None:
 169         msg['Newsgroups'] = mlist.linked_newsgroup
 170     else:
 171         # See if the Newsgroups: header already contains our linked_newsgroup.
 172         # If so, don't add it again.  If not, append our linked_newsgroup to
 173         # the end of the header list
 174         newsgroups = [value.strip() for value in header.split(COMMA)]
 175         if mlist.linked_newsgroup not in newsgroups:
 176             newsgroups.append(mlist.linked_newsgroup)
 177             # Subtitute our new header for the old one.
 178             del msg['newsgroups']
 179             msg['Newsgroups'] = COMMASPACE.join(newsgroups)
 180     # Ensure we have an unfolded Message-ID.
 181     if not msg.get('message-id'):
 182         msg['Message-ID'] = email.utils.make_msgid(mlist.list_name,
 183                                                    mlist.mail_host)
 184     mid = re.sub(r'[\s]', '', msg.get('message-id'))
 185     msg.replace_header('message-id', mid)
 186     # Lines: is useful.
 187     if msg['Lines'] is None:
 188         # BAW: is there a better way?
 189         count = len(list(email.iterators.body_line_iterator(msg)))
 190         msg['Lines'] = str(count)
 191     # Massage the message headers by remove some and rewriting others.  This
 192     # won't completely sanitize the message, but it will eliminate the bulk of
 193     # the rejections based on message headers.  The NNTP server may still
 194     # reject the message because of other problems.
 195     for header in config.nntp.remove_headers.split():
 196         del msg[header]
 197     dup_headers = config.nntp.rewrite_duplicate_headers.split()
 198     if len(dup_headers) % 2 != 0:
 199         # There are an odd number of headers; ignore the last one.
 200         bad_header = dup_headers.pop()
 201         log.error('Ignoring odd [nntp]rewrite_duplicate_headers: {}'.format(
 202             bad_header))
 203     dup_headers.reverse()
 204     while dup_headers:
 205         source = dup_headers.pop()
 206         target = dup_headers.pop()
 207         values = msg.get_all(source, [])
 208         if len(values) < 2:
 209             # We only care about duplicates.
 210             continue
 211         # Delete all the original headers.
 212         del msg[source]
 213         # Put the first value back on the original header.
 214         msg[source] = values[0]
 215         # And put all the subsequent values on the destination header.
 216         for value in values[1:]:
 217             msg[target] = value
 218     # Mark this message as prepared in case it has to be requeued.
 219     msgdata['prepped'] = True