Lib/nntplib.py

   1 """An NNTP client class based on RFC 977: Network News Transfer Protocol.
   2
   3 Example:
   4
   5 >>> from nntplib import NNTP
   6 >>> s = NNTP('news')
   7 >>> resp, count, first, last, name = s.group('comp.lang.python')
   8 >>> print('Group', name, 'has', count, 'articles, range', first, 'to', last)
   9 Group comp.lang.python has 51 articles, range 5770 to 5821
  10 >>> resp, subs = s.xhdr('subject', '{0}-{1}'.format(first, last))
  11 >>> resp = s.quit()
  12 >>>
  13
  14 Here 'resp' is the server response line.
  15 Error responses are turned into exceptions.
  16
  17 To post an article from a file:
  18 >>> f = open(filename, 'rb') # file containing article, including header
  19 >>> resp = s.post(f)
  20 >>>
  21
  22 For descriptions of all methods, read the comments in the code below.
  23 Note that all arguments and return values representing article numbers
  24 are strings, not numbers, since they are rarely used for calculations.
  25 """
  26
  27 # RFC 977 by Brian Kantor and Phil Lapsley.
  28 # xover, xgtitle, xpath, date methods by Kevan Heydon
  29
  30
  31 # Imports
  32 import re
  33 import socket
  34
  35 __all__ = ["NNTP","NNTPReplyError","NNTPTemporaryError",
  36            "NNTPPermanentError","NNTPProtocolError","NNTPDataError",
  37            "error_reply","error_temp","error_perm","error_proto",
  38            "error_data",]
  39
  40 # Exceptions raised when an error or invalid response is received
  41 class NNTPError(Exception):
  42     """Base class for all nntplib exceptions"""
  43     def __init__(self, *args):
  44         Exception.__init__(self, *args)
  45         try:
  46             self.response = args[0]
  47         except IndexError:
  48             self.response = 'No response given'
  49
  50 class NNTPReplyError(NNTPError):
  51     """Unexpected [123]xx reply"""
  52     pass
  53
  54 class NNTPTemporaryError(NNTPError):
  55     """4xx errors"""
  56     pass
  57
  58 class NNTPPermanentError(NNTPError):
  59     """5xx errors"""
  60     pass
  61
  62 class NNTPProtocolError(NNTPError):
  63     """Response does not begin with [1-5]"""
  64     pass
  65
  66 class NNTPDataError(NNTPError):
  67     """Error in response data"""
  68     pass
  69
  70 # for backwards compatibility
  71 error_reply = NNTPReplyError
  72 error_temp = NNTPTemporaryError
  73 error_perm = NNTPPermanentError
  74 error_proto = NNTPProtocolError
  75 error_data = NNTPDataError
  76
  77
  78
  79 # Standard port used by NNTP servers
  80 NNTP_PORT = 119
  81
  82
  83 # Response numbers that are followed by additional text (e.g. article)
  84 LONGRESP = [b'100', b'215', b'220', b'221', b'222', b'224', b'230', b'231', b'282']
  85
  86
  87 # Line terminators (we always output CRLF, but accept any of CRLF, CR, LF)
  88 CRLF = b'\r\n'
  89
  90
  91
  92 # The class itself
  93 class NNTP:
  94     def __init__(self, host, port=NNTP_PORT, user=None, password=None,
  95                  readermode=None, usenetrc=True):
  96         """Initialize an instance.  Arguments:
  97         - host: hostname to connect to
  98         - port: port to connect to (default the standard NNTP port)
  99         - user: username to authenticate with
 100         - password: password to use with username
 101         - readermode: if true, send 'mode reader' command after
 102                       connecting.
 103
 104         readermode is sometimes necessary if you are connecting to an
 105         NNTP server on the local machine and intend to call
 106         reader-specific comamnds, such as `group'.  If you get
 107         unexpected NNTPPermanentErrors, you might need to set
 108         readermode.
 109         """
 110         self.host = host
 111         self.port = port
 112         self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 113         self.sock.connect((self.host, self.port))
 114         self.file = self.sock.makefile('rb')
 115         self.debugging = 0
 116         self.welcome = self.getresp()
 117
 118         # 'mode reader' is sometimes necessary to enable 'reader' mode.
 119         # However, the order in which 'mode reader' and 'authinfo' need to
 120         # arrive differs between some NNTP servers. Try to send
 121         # 'mode reader', and if it fails with an authorization failed
 122         # error, try again after sending authinfo.
 123         readermode_afterauth = 0
 124         if readermode:
 125             try:
 126                 self.welcome = self.shortcmd('mode reader')
 127             except NNTPPermanentError:
 128                 # error 500, probably 'not implemented'
 129                 pass
 130             except NNTPTemporaryError as e:
 131                 if user and e.response.startswith(b'480'):
 132                     # Need authorization before 'mode reader'
 133                     readermode_afterauth = 1
 134                 else:
 135                     raise
 136         # If no login/password was specified, try to get them from ~/.netrc
 137         # Presume that if .netc has an entry, NNRP authentication is required.
 138         try:
 139             if usenetrc and not user:
 140                 import netrc
 141                 credentials = netrc.netrc()
 142                 auth = credentials.authenticators(host)
 143                 if auth:
 144                     user = auth[0]
 145                     password = auth[2]
 146         except IOError:
 147             pass
 148         # Perform NNRP authentication if needed.
 149         if user:
 150             resp = self.shortcmd('authinfo user '+user)
 151             if resp.startswith(b'381'):
 152                 if not password:
 153                     raise NNTPReplyError(resp)
 154                 else:
 155                     resp = self.shortcmd(
 156                             'authinfo pass '+password)
 157                     if not resp.startswith(b'281'):
 158                         raise NNTPPermanentError(resp)
 159             if readermode_afterauth:
 160                 try:
 161                     self.welcome = self.shortcmd('mode reader')
 162                 except NNTPPermanentError:
 163                     # error 500, probably 'not implemented'
 164                     pass
 165
 166
 167     # Get the welcome message from the server
 168     # (this is read and squirreled away by __init__()).
 169     # If the response code is 200, posting is allowed;
 170     # if it 201, posting is not allowed
 171
 172     def getwelcome(self):
 173         """Get the welcome message from the server
 174         (this is read and squirreled away by __init__()).
 175         If the response code is 200, posting is allowed;
 176         if it 201, posting is not allowed."""
 177
 178         if self.debugging: print('*welcome*', repr(self.welcome))
 179         return self.welcome
 180
 181     def set_debuglevel(self, level):
 182         """Set the debugging level.  Argument 'level' means:
 183         0: no debugging output (default)
 184         1: print commands and responses but not body text etc.
 185         2: also print raw lines read and sent before stripping CR/LF"""
 186
 187         self.debugging = level
 188     debug = set_debuglevel
 189
 190     def putline(self, line):
 191         """Internal: send one line to the server, appending CRLF."""
 192         line = line + CRLF
 193         if self.debugging > 1: print('*put*', repr(line))
 194         self.sock.sendall(line)
 195
 196     def putcmd(self, line):
 197         """Internal: send one command to the server (through putline())."""
 198         if self.debugging: print('*cmd*', repr(line))
 199         line = bytes(line, "ASCII")
 200         self.putline(line)
 201
 202     def getline(self):
 203         """Internal: return one line from the server, stripping CRLF.
 204         Raise EOFError if the connection is closed."""
 205         line = self.file.readline()
 206         if self.debugging > 1:
 207             print('*get*', repr(line))
 208         if not line: raise EOFError
 209         if line[-2:] == CRLF:
 210             line = line[:-2]
 211         elif line[-1:] in CRLF:
 212             line = line[:-1]
 213         return line
 214
 215     def getresp(self):
 216         """Internal: get a response from the server.
 217         Raise various errors if the response indicates an error."""
 218         resp = self.getline()
 219         if self.debugging: print('*resp*', repr(resp))
 220         c = resp[:1]
 221         if c == b'4':
 222             raise NNTPTemporaryError(resp)
 223         if c == b'5':
 224             raise NNTPPermanentError(resp)
 225         if c not in b'123':
 226             raise NNTPProtocolError(resp)
 227         return resp
 228
 229     def getlongresp(self, file=None):
 230         """Internal: get a response plus following text from the server.
 231         Raise various errors if the response indicates an error."""
 232
 233         openedFile = None
 234         try:
 235             # If a string was passed then open a file with that name
 236             if isinstance(file, str):
 237                 openedFile = file = open(file, "w")
 238
 239             resp = self.getresp()
 240             if resp[:3] not in LONGRESP:
 241                 raise NNTPReplyError(resp)
 242             list = []
 243             while 1:
 244                 line = self.getline()
 245                 if line == b'.':
 246                     break
 247                 if line.startswith(b'..'):
 248                     line = line[1:]
 249                 if file:
 250                     file.write(line + b'\n')
 251                 else:
 252                     list.append(line)
 253         finally:
 254             # If this method created the file, then it must close it
 255             if openedFile:
 256                 openedFile.close()
 257
 258         return resp, list
 259
 260     def shortcmd(self, line):
 261         """Internal: send a command and get the response."""
 262         self.putcmd(line)
 263         return self.getresp()
 264
 265     def longcmd(self, line, file=None):
 266         """Internal: send a command and get the response plus following text."""
 267         self.putcmd(line)
 268         return self.getlongresp(file)
 269
 270     def newgroups(self, date, time, file=None):
 271         """Process a NEWGROUPS command.  Arguments:
 272         - date: string 'yymmdd' indicating the date
 273         - time: string 'hhmmss' indicating the time
 274         Return:
 275         - resp: server response if successful
 276         - list: list of newsgroup names"""
 277
 278         return self.longcmd('NEWGROUPS ' + date + ' ' + time, file)
 279
 280     def newnews(self, group, date, time, file=None):
 281         """Process a NEWNEWS command.  Arguments:
 282         - group: group name or '*'
 283         - date: string 'yymmdd' indicating the date
 284         - time: string 'hhmmss' indicating the time
 285         Return:
 286         - resp: server response if successful
 287         - list: list of message ids"""
 288
 289         cmd = 'NEWNEWS ' + group + ' ' + date + ' ' + time
 290         return self.longcmd(cmd, file)
 291
 292     def list(self, file=None):
 293         """Process a LIST command.  Return:
 294         - resp: server response if successful
 295         - list: list of (group, last, first, flag) (strings)"""
 296
 297         resp, list = self.longcmd('LIST', file)
 298         for i in range(len(list)):
 299             # Parse lines into "group last first flag"
 300             list[i] = tuple(list[i].split())
 301         return resp, list
 302
 303     def description(self, group):
 304
 305         """Get a description for a single group.  If more than one
 306         group matches ('group' is a pattern), return the first.  If no
 307         group matches, return an empty string.
 308
 309         This elides the response code from the server, since it can
 310         only be '215' or '285' (for xgtitle) anyway.  If the response
 311         code is needed, use the 'descriptions' method.
 312
 313         NOTE: This neither checks for a wildcard in 'group' nor does
 314         it check whether the group actually exists."""
 315
 316         resp, lines = self.descriptions(group)
 317         if len(lines) == 0:
 318             return b''
 319         else:
 320             return lines[0][1]
 321
 322     def descriptions(self, group_pattern):
 323         """Get descriptions for a range of groups."""
 324         line_pat = re.compile(b'^(?P<group>[^ \t]+)[ \t]+(.*)$')
 325         # Try the more std (acc. to RFC2980) LIST NEWSGROUPS first
 326         resp, raw_lines = self.longcmd('LIST NEWSGROUPS ' + group_pattern)
 327         if not resp.startswith(b'215'):
 328             # Now the deprecated XGTITLE.  This either raises an error
 329             # or succeeds with the same output structure as LIST
 330             # NEWSGROUPS.
 331             resp, raw_lines = self.longcmd('XGTITLE ' + group_pattern)
 332         lines = []
 333         for raw_line in raw_lines:
 334             match = line_pat.search(raw_line.strip())
 335             if match:
 336                 lines.append(match.group(1, 2))
 337         return resp, lines
 338
 339     def group(self, name):
 340         """Process a GROUP command.  Argument:
 341         - group: the group name
 342         Returns:
 343         - resp: server response if successful
 344         - count: number of articles (string)
 345         - first: first article number (string)
 346         - last: last article number (string)
 347         - name: the group name"""
 348
 349         resp = self.shortcmd('GROUP ' + name)
 350         if not resp.startswith(b'211'):
 351             raise NNTPReplyError(resp)
 352         words = resp.split()
 353         count = first = last = 0
 354         n = len(words)
 355         if n > 1:
 356             count = words[1]
 357             if n > 2:
 358                 first = words[2]
 359                 if n > 3:
 360                     last = words[3]
 361                     if n > 4:
 362                         name = words[4].lower()
 363         return resp, count, first, last, name
 364
 365     def help(self, file=None):
 366         """Process a HELP command.  Returns:
 367         - resp: server response if successful
 368         - list: list of strings"""
 369
 370         return self.longcmd('HELP',file)
 371
 372     def statparse(self, resp):
 373         """Internal: parse the response of a STAT, NEXT or LAST command."""
 374         if not resp.startswith(b'22'):
 375             raise NNTPReplyError(resp)
 376         words = resp.split()
 377         nr = 0
 378         id = b''
 379         n = len(words)
 380         if n > 1:
 381             nr = words[1]
 382             if n > 2:
 383                 id = words[2]
 384         return resp, nr, id
 385
 386     def statcmd(self, line):
 387         """Internal: process a STAT, NEXT or LAST command."""
 388         resp = self.shortcmd(line)
 389         return self.statparse(resp)
 390
 391     def stat(self, id):
 392         """Process a STAT command.  Argument:
 393         - id: article number or message id
 394         Returns:
 395         - resp: server response if successful
 396         - nr:   the article number
 397         - id:   the message id"""
 398
 399         return self.statcmd('STAT {0}'.format(id))
 400
 401     def next(self):
 402         """Process a NEXT command.  No arguments.  Return as for STAT."""
 403         return self.statcmd('NEXT')
 404
 405     def last(self):
 406         """Process a LAST command.  No arguments.  Return as for STAT."""
 407         return self.statcmd('LAST')
 408
 409     def artcmd(self, line, file=None):
 410         """Internal: process a HEAD, BODY or ARTICLE command."""
 411         resp, list = self.longcmd(line, file)
 412         resp, nr, id = self.statparse(resp)
 413         return resp, nr, id, list
 414
 415     def head(self, id):
 416         """Process a HEAD command.  Argument:
 417         - id: article number or message id
 418         Returns:
 419         - resp: server response if successful
 420         - nr: article number
 421         - id: message id
 422         - list: the lines of the article's header"""
 423
 424         return self.artcmd('HEAD {0}'.format(id))
 425
 426     def body(self, id, file=None):
 427         """Process a BODY command.  Argument:
 428         - id: article number or message id
 429         - file: Filename string or file object to store the article in
 430         Returns:
 431         - resp: server response if successful
 432         - nr: article number
 433         - id: message id
 434         - list: the lines of the article's body or an empty list
 435                 if file was used"""
 436
 437         return self.artcmd('BODY {0}'.format(id), file)
 438
 439     def article(self, id):
 440         """Process an ARTICLE command.  Argument:
 441         - id: article number or message id
 442         Returns:
 443         - resp: server response if successful
 444         - nr: article number
 445         - id: message id
 446         - list: the lines of the article"""
 447
 448         return self.artcmd('ARTICLE {0}'.format(id))
 449
 450     def slave(self):
 451         """Process a SLAVE command.  Returns:
 452         - resp: server response if successful"""
 453
 454         return self.shortcmd('SLAVE')
 455
 456     def xhdr(self, hdr, str, file=None):
 457         """Process an XHDR command (optional server extension).  Arguments:
 458         - hdr: the header type (e.g. 'subject')
 459         - str: an article nr, a message id, or a range nr1-nr2
 460         Returns:
 461         - resp: server response if successful
 462         - list: list of (nr, value) strings"""
 463
 464         pat = re.compile(b'^([0-9]+) ?(.*)\n?')
 465         resp, lines = self.longcmd('XHDR {0} {1}'.format(hdr, str), file)
 466         for i in range(len(lines)):
 467             line = lines[i]
 468             m = pat.match(line)
 469             if m:
 470                 lines[i] = m.group(1, 2)
 471         return resp, lines
 472
 473     def xover(self, start, end, file=None):
 474         """Process an XOVER command (optional server extension) Arguments:
 475         - start: start of range
 476         - end: end of range
 477         Returns:
 478         - resp: server response if successful
 479         - list: list of (art-nr, subject, poster, date,
 480                          id, references, size, lines)"""
 481
 482         resp, lines = self.longcmd('XOVER {0}-{1}'.format(start, end), file)
 483         xover_lines = []
 484         for line in lines:
 485             elem = line.split(b'\t')
 486             try:
 487                 xover_lines.append((elem[0],
 488                                     elem[1],
 489                                     elem[2],
 490                                     elem[3],
 491                                     elem[4],
 492                                     elem[5].split(),
 493                                     elem[6],
 494                                     elem[7]))
 495             except IndexError:
 496                 raise NNTPDataError(line)
 497         return resp,xover_lines
 498
 499     def xgtitle(self, group, file=None):
 500         """Process an XGTITLE command (optional server extension) Arguments:
 501         - group: group name wildcard (i.e. news.*)
 502         Returns:
 503         - resp: server response if successful
 504         - list: list of (name,title) strings"""
 505
 506         line_pat = re.compile(b'^([^ \t]+)[ \t]+(.*)$')
 507         resp, raw_lines = self.longcmd('XGTITLE ' + group, file)
 508         lines = []
 509         for raw_line in raw_lines:
 510             match = line_pat.search(raw_line.strip())
 511             if match:
 512                 lines.append(match.group(1, 2))
 513         return resp, lines
 514
 515     def xpath(self,id):
 516         """Process an XPATH command (optional server extension) Arguments:
 517         - id: Message id of article
 518         Returns:
 519         resp: server response if successful
 520         path: directory path to article"""
 521
 522         resp = self.shortcmd('XPATH {0}'.format(id))
 523         if not resp.startswith(b'223'):
 524             raise NNTPReplyError(resp)
 525         try:
 526             [resp_num, path] = resp.split()
 527         except ValueError:
 528             raise NNTPReplyError(resp)
 529         else:
 530             return resp, path
 531
 532     def date (self):
 533         """Process the DATE command. Arguments:
 534         None
 535         Returns:
 536         resp: server response if successful
 537         date: Date suitable for newnews/newgroups commands etc.
 538         time: Time suitable for newnews/newgroups commands etc."""
 539
 540         resp = self.shortcmd("DATE")
 541         if not resp.startswith(b'111'):
 542             raise NNTPReplyError(resp)
 543         elem = resp.split()
 544         if len(elem) != 2:
 545             raise NNTPDataError(resp)
 546         date = elem[1][2:8]
 547         time = elem[1][-6:]
 548         if len(date) != 6 or len(time) != 6:
 549             raise NNTPDataError(resp)
 550         return resp, date, time
 551
 552     def _post(self, command, f):
 553         resp = self.shortcmd(command)
 554         # Raises error_??? if posting is not allowed
 555         if not resp.startswith(b'3'):
 556             raise NNTPReplyError(resp)
 557         while 1:
 558             line = f.readline()
 559             if not line:
 560                 break
 561             if line.endswith(b'\n'):
 562                 line = line[:-1]
 563             if line.startswith(b'.'):
 564                 line = b'.' + line
 565             self.putline(line)
 566         self.putline(b'.')
 567         return self.getresp()
 568
 569     def post(self, f):
 570         """Process a POST command.  Arguments:
 571         - f: file containing the article
 572         Returns:
 573         - resp: server response if successful"""
 574         return self._post('POST', f)
 575
 576     def ihave(self, id, f):
 577         """Process an IHAVE command.  Arguments:
 578         - id: message-id of the article
 579         - f:  file containing the article
 580         Returns:
 581         - resp: server response if successful
 582         Note that if the server refuses the article an exception is raised."""
 583         return self._post('IHAVE {0}'.format(id), f)
 584
 585     def quit(self):
 586         """Process a QUIT command and close the socket.  Returns:
 587         - resp: server response if successful"""
 588
 589         resp = self.shortcmd('QUIT')
 590         self.file.close()
 591         self.sock.close()
 592         del self.file, self.sock
 593         return resp
 594
 595
 596 # Test retrieval when run as a script.
 597 # Assumption: if there's a local news server, it's called 'news'.
 598 # Assumption: if user queries a remote news server, it's named
 599 # in the environment variable NNTPSERVER (used by slrn and kin)
 600 # and we want readermode off.
 601 if __name__ == '__main__':
 602     import os
 603     newshost = 'news' and os.environ["NNTPSERVER"]
 604     if newshost.find('.') == -1:
 605         mode = 'readermode'
 606     else:
 607         mode = None
 608     s = NNTP(newshost, readermode=mode)
 609     resp, count, first, last, name = s.group('comp.lang.python')
 610     print(resp)
 611     print('Group', name, 'has', count, 'articles, range', first, 'to', last)
 612     resp, subs = s.xhdr('subject', '{0}-{1}'.format(first, last))
 613     print(resp)
 614     for item in subs:
 615         print("%7s %s" % item)
 616     resp = s.quit()
 617     print(resp)