1 # Copyright (C) 1998-2017 by the Free Software Foundation, Inc.
3 # This file is part of GNU Mailman.
5 # GNU Mailman is free software: you can redistribute it and/or modify it under
6 # the terms of the GNU General Public License as published by the Free
7 # Software Foundation, either version 3 of the License, or (at your option)
10 # GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 # You should have received a copy of the GNU General Public License along with
16 # GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
27 from email
.Parser
import Parser
28 from flufl
.lock
import Lock
, TimeOutError
29 from lazr
.config
import as_host_port
31 from mailman
import MailList
32 from mailman
import Message
33 from mailman
import loginit
34 from mailman
.configuration
import config
35 from mailman
.core
.i18n
import _
36 from mailman
.core
.switchboard
import Switchboard
37 from mailman
.version
import MAILMAN_VERSION
39 # Work around known problems with some RedHat cron daemons
41 signal
.signal(signal
.SIGCHLD
, signal
.SIG_DFL
)
47 class _ContinueLoop(Exception):
53 parser
= optparse
.OptionParser(version
=MAILMAN_VERSION
,
57 Poll the NNTP servers for messages to be gatewayed to mailing lists."""))
58 parser
.add_option('-C', '--config',
59 help=_('Alternative configuration file to use'))
60 opts
, args
= parser
.parse_args()
63 print >> sys
.stderr
, _('Unexpected arguments')
65 return opts
, args
, parser
71 def open_newsgroup(mlist
):
72 # Split host:port if given.
73 nntp_host
, nntp_port
= as_host_port(mlist
.nntp_host
, default_port
=119)
74 # Open up a "mode reader" connection to nntp server. This will be shared
75 # for all the gated lists having the same nntp_host.
76 conn
= _hostcache
.get(mlist
.nntp_host
)
79 conn
= nntplib
.NNTP(nntp_host
, nntp_port
,
81 user
=config
.NNTP_USERNAME
,
82 password
=config
.NNTP_PASSWORD
)
83 except (socket
.error
, nntplib
.NNTPError
, IOError) as e
:
84 log
.error('error opening connection to nntp_host: %s\n%s',
87 _hostcache
[mlist
.nntp_host
] = conn
88 # Get the GROUP information for the list, but we're only really interested
89 # in the first article number and the last article number
90 r
, c
, f
, l
, n
= conn
.group(mlist
.linked_newsgroup
)
91 return conn
, int(f
), int(l
)
95 for conn
in set(_hostcache
.values()):
101 # This function requires the list to be locked.
102 def poll_newsgroup(mlist
, conn
, first
, last
, glock
):
103 listname
= mlist
.internal_name()
104 # NEWNEWS is not portable and has synchronization issues.
105 for num
in range(first
, last
):
108 headers
= conn
.head(repr(num
))[3]
111 for header
in headers
:
113 value
= header
[:i
].lower()
114 if i
> 0 and value
== 'to':
116 # FIXME 2010-02-16 barry use List-Post header.
117 if value
<> 'x-beenthere':
119 if header
[i
:] == ': %s' % mlist
.posting_address
:
123 body
= conn
.body(repr(num
))[3]
124 # Usenet originated messages will not have a Unix envelope
125 # (i.e. "From " header). This breaks Pipermail archiving, so
126 # we will synthesize one. Be sure to use the format searched
127 # for by mailbox.UnixMailbox._isrealfromline(). BAW: We use
128 # the -bounces address here in case any downstream clients use
129 # the envelope sender for bounces; I'm not sure about this,
130 # but it's the closest to the old semantics.
131 lines
= ['From %s %s' % (mlist
.GetBouncesEmail(),
132 time
.ctime(time
.time()))]
133 lines
.extend(headers
)
137 p
= Parser(Message
.Message
)
139 msg
= p
.parsestr(NL
.join(lines
))
140 except email
.Errors
.MessageError
as e
:
141 log
.error('email package exception for %s:%d\n%s',
142 mlist
.linked_newsgroup
, num
, e
)
145 del msg
['X-Originally-To']
146 msg
['X-Originally-To'] = msg
['To']
148 msg
['To'] = mlist
.posting_address
149 # Post the message to the locked list
150 inq
= Switchboard(config
.INQUEUE_DIR
)
152 listid
=mlist
.list_id
,
154 log
.info('posted to list %s: %7d', listname
, num
)
155 except nntplib
.NNTPError
as e
:
156 log
.exception('NNTP error for list %s: %7d', listname
, num
)
157 except _ContinueLoop
:
159 # Even if we don't post the message because it was seen on the
160 # list already, update the watermark
161 mlist
.usenet_watermark
= num
165 def process_lists(glock
):
166 for listname
in config
.list_manager
.names
:
168 # Open the list unlocked just to check to see if it is gating news to
169 # mail. If not, we're done with the list. Otherwise, lock the list
170 # and gate the group.
171 mlist
= MailList
.MailList(listname
, lock
=False)
172 if not mlist
.gateway_to_mail
:
174 # Get the list's watermark, i.e. the last article number that we gated
175 # from news to mail. None means that this list has never polled its
176 # newsgroup and that we should do a catch up.
177 watermark
= getattr(mlist
, 'usenet_watermark', None)
178 # Open the newsgroup, but let most exceptions percolate up.
180 conn
, first
, last
= open_newsgroup(mlist
)
181 except (socket
.error
, nntplib
.NNTPError
):
183 log
.info('%s: [%d..%d]', listname
, first
, last
)
186 if watermark
is None:
187 mlist
.Lock(timeout
=config
.LIST_LOCK_TIMEOUT
)
188 # This is the first time we've tried to gate this
189 # newsgroup. We essentially do a mass catch-up, otherwise
190 # we'd flood the mailing list.
191 mlist
.usenet_watermark
= last
192 log
.info('%s caught up to article %d', listname
, last
)
194 # The list has been polled previously, so now we simply
195 # grab all the messages on the newsgroup that have not
196 # been seen by the mailing list. The first such article
197 # is the maximum of the lowest article available in the
198 # newsgroup and the watermark. It's possible that some
199 # articles have been expired since the last time gate_news
200 # has run. Not much we can do about that.
201 start
= max(watermark
+ 1, first
)
203 log
.info('nothing new for list %s', listname
)
205 mlist
.Lock(timeout
=config
.LIST_LOCK_TIMEOUT
)
206 log
.info('gating %s articles [%d..%d]',
207 listname
, start
, last
)
208 # Use last+1 because poll_newsgroup() employes a for
209 # loop over range, and this will not include the last
210 # element in the list.
211 poll_newsgroup(mlist
, conn
, start
, last
+ 1, glock
)
213 log
.error('Could not acquire list lock: %s', listname
)
218 log
.info('%s watermark: %d', listname
, mlist
.usenet_watermark
)
223 opts
, args
, parser
= parseargs()
224 config
.load(opts
.config
)
226 GATENEWS_LOCK_FILE
= os
.path
.join(config
.LOCK_DIR
, 'gate_news.lock')
227 LOCK_LIFETIME
= config
.hours(2)
229 loginit
.initialize(propagate
=True)
230 log
= logging
.getLogger('mailman.fromusenet')
233 with
Lock(GATENEWS_LOCK_FILE
,
234 # It's okay to hijack this
235 lifetime
=LOCK_LIFETIME
) as lock
:
239 log
.error('Could not acquire gate_news lock')
243 if __name__
== '__main__':