Lib/nntplib.py

   1 """An NNTP client class based on RFC 977: Network News Transfer Protocol.
   2
   3 Example:
   4
   5 >>> from nntplib import NNTP
   6 >>> s = NNTP('news')
   7 >>> resp, count, first, last, name = s.group('comp.lang.python')
   8 >>> print 'Group', name, 'has', count, 'articles, range', first, 'to', last
   9 Group comp.lang.python has 51 articles, range 5770 to 5821
  10 >>> resp, subs = s.xhdr('subject', first + '-' + last)
  11 >>> resp = s.quit()
  12 >>>
  13
  14 Here 'resp' is the server response line.
  15 Error responses are turned into exceptions.
  16
  17 To post an article from a file:
  18 >>> f = open(filename, 'r') # file containing article, including header
  19 >>> resp = s.post(f)
  20 >>>
  21
  22 For descriptions of all methods, read the comments in the code below.
  23 Note that all arguments and return values representing article numbers
  24 are strings, not numbers, since they are rarely used for calculations.
  25 """
  26
  27 # RFC 977 by Brian Kantor and Phil Lapsley.
  28 # xover, xgtitle, xpath, date methods by Kevan Heydon
  29
  30
  31 # Imports
  32 import re
  33 import socket
  34
  35 __all__ = ["NNTP","NNTPReplyError","NNTPTemporaryError",
  36            "NNTPPermanentError","NNTPProtocolError","NNTPDataError",
  37            "error_reply","error_temp","error_perm","error_proto",
  38            "error_data",]
  39
  40 # Exceptions raised when an error or invalid response is received
  41 class NNTPError(Exception):
  42     """Base class for all nntplib exceptions"""
  43     def __init__(self, *args):
  44         Exception.__init__(self, *args)
  45         try:
  46             self.response = args[0]
  47         except IndexError:
  48             self.response = 'No response given'
  49
  50 class NNTPReplyError(NNTPError):
  51     """Unexpected [123]xx reply"""
  52     pass
  53
  54 class NNTPTemporaryError(NNTPError):
  55     """4xx errors"""
  56     pass
  57
  58 class NNTPPermanentError(NNTPError):
  59     """5xx errors"""
  60     pass
  61
  62 class NNTPProtocolError(NNTPError):
  63     """Response does not begin with [1-5]"""
  64     pass
  65
  66 class NNTPDataError(NNTPError):
  67     """Error in response data"""
  68     pass
  69
  70 # for backwards compatibility
  71 error_reply = NNTPReplyError
  72 error_temp = NNTPTemporaryError
  73 error_perm = NNTPPermanentError
  74 error_proto = NNTPProtocolError
  75 error_data = NNTPDataError
  76
  77
  78
  79 # Standard port used by NNTP servers
  80 NNTP_PORT = 119
  81
  82
  83 # Response numbers that are followed by additional text (e.g. article)
  84 LONGRESP = ['100', '215', '220', '221', '222', '224', '230', '231', '282']
  85
  86
  87 # Line terminators (we always output CRLF, but accept any of CRLF, CR, LF)
  88 CRLF = '\r\n'
  89
  90
  91
  92 # The class itself
  93 class NNTP:
  94     def __init__(self, host, port=NNTP_PORT, user=None, password=None,
  95                  readermode=None, usenetrc=True):
  96         """Initialize an instance.  Arguments:
  97         - host: hostname to connect to
  98         - port: port to connect to (default the standard NNTP port)
  99         - user: username to authenticate with
 100         - password: password to use with username
 101         - readermode: if true, send 'mode reader' command after
 102                       connecting.
 103
 104         readermode is sometimes necessary if you are connecting to an
 105         NNTP server on the local machine and intend to call
 106         reader-specific comamnds, such as `group'.  If you get
 107         unexpected NNTPPermanentErrors, you might need to set
 108         readermode.
 109         """
 110         self.host = host
 111         self.port = port
 112         self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 113         self.sock.connect((self.host, self.port))
 114         self.file = self.sock.makefile('rb')
 115         self.debugging = 0
 116         self.welcome = self.getresp()
 117
 118         # 'mode reader' is sometimes necessary to enable 'reader' mode.
 119         # However, the order in which 'mode reader' and 'authinfo' need to
 120         # arrive differs between some NNTP servers. Try to send
 121         # 'mode reader', and if it fails with an authorization failed
 122         # error, try again after sending authinfo.
 123         readermode_afterauth = 0
 124         if readermode:
 125             try:
 126                 self.welcome = self.shortcmd('mode reader')
 127             except NNTPPermanentError:
 128                 # error 500, probably 'not implemented'
 129                 pass
 130             except NNTPTemporaryError, e:
 131                 if user and e.response[:3] == '480':
 132                     # Need authorization before 'mode reader'
 133                     readermode_afterauth = 1
 134                 else:
 135                     raise
 136         # If no login/password was specified, try to get them from ~/.netrc
 137         # Presume that if .netc has an entry, NNRP authentication is required.
 138         try:
 139             if usenetrc and not user:
 140                 import netrc
 141                 credentials = netrc.netrc()
 142                 auth = credentials.authenticators(host)
 143                 if auth:
 144                     user = auth[0]
 145                     password = auth[2]
 146         except IOError:
 147             pass
 148         # Perform NNRP authentication if needed.
 149         if user:
 150             resp = self.shortcmd('authinfo user '+user)
 151             if resp[:3] == '381':
 152                 if not password:
 153                     raise NNTPReplyError(resp)
 154                 else:
 155                     resp = self.shortcmd(
 156                             'authinfo pass '+password)
 157                     if resp[:3] != '281':
 158                         raise NNTPPermanentError(resp)
 159             if readermode_afterauth:
 160                 try:
 161                     self.welcome = self.shortcmd('mode reader')
 162                 except NNTPPermanentError:
 163                     # error 500, probably 'not implemented'
 164                     pass
 165
 166
 167     # Get the welcome message from the server
 168     # (this is read and squirreled away by __init__()).
 169     # If the response code is 200, posting is allowed;
 170     # if it 201, posting is not allowed
 171
 172     def getwelcome(self):
 173         """Get the welcome message from the server
 174         (this is read and squirreled away by __init__()).
 175         If the response code is 200, posting is allowed;
 176         if it 201, posting is not allowed."""
 177
 178         if self.debugging: print '*welcome*', repr(self.welcome)
 179         return self.welcome
 180
 181     def set_debuglevel(self, level):
 182         """Set the debugging level.  Argument 'level' means:
 183         0: no debugging output (default)
 184         1: print commands and responses but not body text etc.
 185         2: also print raw lines read and sent before stripping CR/LF"""
 186
 187         self.debugging = level
 188     debug = set_debuglevel
 189
 190     def putline(self, line):
 191         """Internal: send one line to the server, appending CRLF."""
 192         line = line + CRLF
 193         if self.debugging > 1: print '*put*', repr(line)
 194         self.sock.sendall(line)
 195
 196     def putcmd(self, line):
 197         """Internal: send one command to the server (through putline())."""
 198         if self.debugging: print '*cmd*', repr(line)
 199         self.putline(line)
 200
 201     def getline(self):
 202         """Internal: return one line from the server, stripping CRLF.
 203         Raise EOFError if the connection is closed."""
 204         line = self.file.readline()
 205         if self.debugging > 1:
 206             print '*get*', repr(line)
 207         if not line: raise EOFError
 208         if line[-2:] == CRLF: line = line[:-2]
 209         elif line[-1:] in CRLF: line = line[:-1]
 210         return line
 211
 212     def getresp(self):
 213         """Internal: get a response from the server.
 214         Raise various errors if the response indicates an error."""
 215         resp = self.getline()
 216         if self.debugging: print '*resp*', repr(resp)
 217         c = resp[:1]
 218         if c == '4':
 219             raise NNTPTemporaryError(resp)
 220         if c == '5':
 221             raise NNTPPermanentError(resp)
 222         if c not in '123':
 223             raise NNTPProtocolError(resp)
 224         return resp
 225
 226     def getlongresp(self, file=None):
 227         """Internal: get a response plus following text from the server.
 228         Raise various errors if the response indicates an error."""
 229
 230         openedFile = None
 231         try:
 232             # If a string was passed then open a file with that name
 233             if isinstance(file, str):
 234                 openedFile = file = open(file, "w")
 235
 236             resp = self.getresp()
 237             if resp[:3] not in LONGRESP:
 238                 raise NNTPReplyError(resp)
 239             list = []
 240             while 1:
 241                 line = self.getline()
 242                 if line == '.':
 243                     break
 244                 if line[:2] == '..':
 245                     line = line[1:]
 246                 if file:
 247                     file.write(line + "\n")
 248                 else:
 249                     list.append(line)
 250         finally:
 251             # If this method created the file, then it must close it
 252             if openedFile:
 253                 openedFile.close()
 254
 255         return resp, list
 256
 257     def shortcmd(self, line):
 258         """Internal: send a command and get the response."""
 259         self.putcmd(line)
 260         return self.getresp()
 261
 262     def longcmd(self, line, file=None):
 263         """Internal: send a command and get the response plus following text."""
 264         self.putcmd(line)
 265         return self.getlongresp(file)
 266
 267     def newgroups(self, date, time, file=None):
 268         """Process a NEWGROUPS command.  Arguments:
 269         - date: string 'yymmdd' indicating the date
 270         - time: string 'hhmmss' indicating the time
 271         Return:
 272         - resp: server response if successful
 273         - list: list of newsgroup names"""
 274
 275         return self.longcmd('NEWGROUPS ' + date + ' ' + time, file)
 276
 277     def newnews(self, group, date, time, file=None):
 278         """Process a NEWNEWS command.  Arguments:
 279         - group: group name or '*'
 280         - date: string 'yymmdd' indicating the date
 281         - time: string 'hhmmss' indicating the time
 282         Return:
 283         - resp: server response if successful
 284         - list: list of message ids"""
 285
 286         cmd = 'NEWNEWS ' + group + ' ' + date + ' ' + time
 287         return self.longcmd(cmd, file)
 288
 289     def list(self, file=None):
 290         """Process a LIST command.  Return:
 291         - resp: server response if successful
 292         - list: list of (group, last, first, flag) (strings)"""
 293
 294         resp, list = self.longcmd('LIST', file)
 295         for i in range(len(list)):
 296             # Parse lines into "group last first flag"
 297             list[i] = tuple(list[i].split())
 298         return resp, list
 299
 300     def description(self, group):
 301
 302         """Get a description for a single group.  If more than one
 303         group matches ('group' is a pattern), return the first.  If no
 304         group matches, return an empty string.
 305
 306         This elides the response code from the server, since it can
 307         only be '215' or '285' (for xgtitle) anyway.  If the response
 308         code is needed, use the 'descriptions' method.
 309
 310         NOTE: This neither checks for a wildcard in 'group' nor does
 311         it check whether the group actually exists."""
 312
 313         resp, lines = self.descriptions(group)
 314         if len(lines) == 0:
 315             return ""
 316         else:
 317             return lines[0][1]
 318
 319     def descriptions(self, group_pattern):
 320         """Get descriptions for a range of groups."""
 321         line_pat = re.compile("^(?P<group>[^ \t]+)[ \t]+(.*)$")
 322         # Try the more std (acc. to RFC2980) LIST NEWSGROUPS first
 323         resp, raw_lines = self.longcmd('LIST NEWSGROUPS ' + group_pattern)
 324         if resp[:3] != "215":
 325             # Now the deprecated XGTITLE.  This either raises an error
 326             # or succeeds with the same output structure as LIST
 327             # NEWSGROUPS.
 328             resp, raw_lines = self.longcmd('XGTITLE ' + group_pattern)
 329         lines = []
 330         for raw_line in raw_lines:
 331             match = line_pat.search(raw_line.strip())
 332             if match:
 333                 lines.append(match.group(1, 2))
 334         return resp, lines
 335
 336     def group(self, name):
 337         """Process a GROUP command.  Argument:
 338         - group: the group name
 339         Returns:
 340         - resp: server response if successful
 341         - count: number of articles (string)
 342         - first: first article number (string)
 343         - last: last article number (string)
 344         - name: the group name"""
 345
 346         resp = self.shortcmd('GROUP ' + name)
 347         if resp[:3] != '211':
 348             raise NNTPReplyError(resp)
 349         words = resp.split()
 350         count = first = last = 0
 351         n = len(words)
 352         if n > 1:
 353             count = words[1]
 354             if n > 2:
 355                 first = words[2]
 356                 if n > 3:
 357                     last = words[3]
 358                     if n > 4:
 359                         name = words[4].lower()
 360         return resp, count, first, last, name
 361
 362     def help(self, file=None):
 363         """Process a HELP command.  Returns:
 364         - resp: server response if successful
 365         - list: list of strings"""
 366
 367         return self.longcmd('HELP',file)
 368
 369     def statparse(self, resp):
 370         """Internal: parse the response of a STAT, NEXT or LAST command."""
 371         if resp[:2] != '22':
 372             raise NNTPReplyError(resp)
 373         words = resp.split()
 374         nr = 0
 375         id = ''
 376         n = len(words)
 377         if n > 1:
 378             nr = words[1]
 379             if n > 2:
 380                 id = words[2]
 381         return resp, nr, id
 382
 383     def statcmd(self, line):
 384         """Internal: process a STAT, NEXT or LAST command."""
 385         resp = self.shortcmd(line)
 386         return self.statparse(resp)
 387
 388     def stat(self, id):
 389         """Process a STAT command.  Argument:
 390         - id: article number or message id
 391         Returns:
 392         - resp: server response if successful
 393         - nr:   the article number
 394         - id:   the message id"""
 395
 396         return self.statcmd('STAT ' + id)
 397
 398     def next(self):
 399         """Process a NEXT command.  No arguments.  Return as for STAT."""
 400         return self.statcmd('NEXT')
 401
 402     def last(self):
 403         """Process a LAST command.  No arguments.  Return as for STAT."""
 404         return self.statcmd('LAST')
 405
 406     def artcmd(self, line, file=None):
 407         """Internal: process a HEAD, BODY or ARTICLE command."""
 408         resp, list = self.longcmd(line, file)
 409         resp, nr, id = self.statparse(resp)
 410         return resp, nr, id, list
 411
 412     def head(self, id):
 413         """Process a HEAD command.  Argument:
 414         - id: article number or message id
 415         Returns:
 416         - resp: server response if successful
 417         - nr: article number
 418         - id: message id
 419         - list: the lines of the article's header"""
 420
 421         return self.artcmd('HEAD ' + id)
 422
 423     def body(self, id, file=None):
 424         """Process a BODY command.  Argument:
 425         - id: article number or message id
 426         - file: Filename string or file object to store the article in
 427         Returns:
 428         - resp: server response if successful
 429         - nr: article number
 430         - id: message id
 431         - list: the lines of the article's body or an empty list
 432                 if file was used"""
 433
 434         return self.artcmd('BODY ' + id, file)
 435
 436     def article(self, id):
 437         """Process an ARTICLE command.  Argument:
 438         - id: article number or message id
 439         Returns:
 440         - resp: server response if successful
 441         - nr: article number
 442         - id: message id
 443         - list: the lines of the article"""
 444
 445         return self.artcmd('ARTICLE ' + id)
 446
 447     def slave(self):
 448         """Process a SLAVE command.  Returns:
 449         - resp: server response if successful"""
 450
 451         return self.shortcmd('SLAVE')
 452
 453     def xhdr(self, hdr, str, file=None):
 454         """Process an XHDR command (optional server extension).  Arguments:
 455         - hdr: the header type (e.g. 'subject')
 456         - str: an article nr, a message id, or a range nr1-nr2
 457         Returns:
 458         - resp: server response if successful
 459         - list: list of (nr, value) strings"""
 460
 461         pat = re.compile('^([0-9]+) ?(.*)\n?')
 462         resp, lines = self.longcmd('XHDR ' + hdr + ' ' + str, file)
 463         for i in range(len(lines)):
 464             line = lines[i]
 465             m = pat.match(line)
 466             if m:
 467                 lines[i] = m.group(1, 2)
 468         return resp, lines
 469
 470     def xover(self, start, end, file=None):
 471         """Process an XOVER command (optional server extension) Arguments:
 472         - start: start of range
 473         - end: end of range
 474         Returns:
 475         - resp: server response if successful
 476         - list: list of (art-nr, subject, poster, date,
 477                          id, references, size, lines)"""
 478
 479         resp, lines = self.longcmd('XOVER ' + start + '-' + end, file)
 480         xover_lines = []
 481         for line in lines:
 482             elem = line.split("\t")
 483             try:
 484                 xover_lines.append((elem[0],
 485                                     elem[1],
 486                                     elem[2],
 487                                     elem[3],
 488                                     elem[4],
 489                                     elem[5].split(),
 490                                     elem[6],
 491                                     elem[7]))
 492             except IndexError:
 493                 raise NNTPDataError(line)
 494         return resp,xover_lines
 495
 496     def xgtitle(self, group, file=None):
 497         """Process an XGTITLE command (optional server extension) Arguments:
 498         - group: group name wildcard (i.e. news.*)
 499         Returns:
 500         - resp: server response if successful
 501         - list: list of (name,title) strings"""
 502
 503         line_pat = re.compile("^([^ \t]+)[ \t]+(.*)$")
 504         resp, raw_lines = self.longcmd('XGTITLE ' + group, file)
 505         lines = []
 506         for raw_line in raw_lines:
 507             match = line_pat.search(raw_line.strip())
 508             if match:
 509                 lines.append(match.group(1, 2))
 510         return resp, lines
 511
 512     def xpath(self,id):
 513         """Process an XPATH command (optional server extension) Arguments:
 514         - id: Message id of article
 515         Returns:
 516         resp: server response if successful
 517         path: directory path to article"""
 518
 519         resp = self.shortcmd("XPATH " + id)
 520         if resp[:3] != '223':
 521             raise NNTPReplyError(resp)
 522         try:
 523             [resp_num, path] = resp.split()
 524         except ValueError:
 525             raise NNTPReplyError(resp)
 526         else:
 527             return resp, path
 528
 529     def date (self):
 530         """Process the DATE command. Arguments:
 531         None
 532         Returns:
 533         resp: server response if successful
 534         date: Date suitable for newnews/newgroups commands etc.
 535         time: Time suitable for newnews/newgroups commands etc."""
 536
 537         resp = self.shortcmd("DATE")
 538         if resp[:3] != '111':
 539             raise NNTPReplyError(resp)
 540         elem = resp.split()
 541         if len(elem) != 2:
 542             raise NNTPDataError(resp)
 543         date = elem[1][2:8]
 544         time = elem[1][-6:]
 545         if len(date) != 6 or len(time) != 6:
 546             raise NNTPDataError(resp)
 547         return resp, date, time
 548
 549
 550     def post(self, f):
 551         """Process a POST command.  Arguments:
 552         - f: file containing the article
 553         Returns:
 554         - resp: server response if successful"""
 555
 556         resp = self.shortcmd('POST')
 557         # Raises error_??? if posting is not allowed
 558         if resp[0] != '3':
 559             raise NNTPReplyError(resp)
 560         while 1:
 561             line = f.readline()
 562             if not line:
 563                 break
 564             if line[-1] == '\n':
 565                 line = line[:-1]
 566             if line[:1] == '.':
 567                 line = '.' + line
 568             self.putline(line)
 569         self.putline('.')
 570         return self.getresp()
 571
 572     def ihave(self, id, f):
 573         """Process an IHAVE command.  Arguments:
 574         - id: message-id of the article
 575         - f:  file containing the article
 576         Returns:
 577         - resp: server response if successful
 578         Note that if the server refuses the article an exception is raised."""
 579
 580         resp = self.shortcmd('IHAVE ' + id)
 581         # Raises error_??? if the server already has it
 582         if resp[0] != '3':
 583             raise NNTPReplyError(resp)
 584         while 1:
 585             line = f.readline()
 586             if not line:
 587                 break
 588             if line[-1] == '\n':
 589                 line = line[:-1]
 590             if line[:1] == '.':
 591                 line = '.' + line
 592             self.putline(line)
 593         self.putline('.')
 594         return self.getresp()
 595
 596     def quit(self):
 597         """Process a QUIT command and close the socket.  Returns:
 598         - resp: server response if successful"""
 599
 600         resp = self.shortcmd('QUIT')
 601         self.file.close()
 602         self.sock.close()
 603         del self.file, self.sock
 604         return resp
 605
 606
 607 # Test retrieval when run as a script.
 608 # Assumption: if there's a local news server, it's called 'news'.
 609 # Assumption: if user queries a remote news server, it's named
 610 # in the environment variable NNTPSERVER (used by slrn and kin)
 611 # and we want readermode off.
 612 if __name__ == '__main__':
 613     import os
 614     newshost = 'news' and os.environ["NNTPSERVER"]
 615     if newshost.find('.') == -1:
 616         mode = 'readermode'
 617     else:
 618         mode = None
 619     s = NNTP(newshost, readermode=mode)
 620     resp, count, first, last, name = s.group('comp.lang.python')
 621     print resp
 622     print 'Group', name, 'has', count, 'articles, range', first, 'to', last
 623     resp, subs = s.xhdr('subject', first + '-' + last)
 624     print resp
 625     for item in subs:
 626         print "%7s %s" % item
 627     resp = s.quit()
 628     print resp