Lib/nntplib.py

   1 """An NNTP client class based on RFC 977: Network News Transfer Protocol.
   2
   3 Example:
   4
   5 >>> from nntplib import NNTP
   6 >>> s = NNTP('news')
   7 >>> resp, count, first, last, name = s.group('comp.lang.python')
   8 >>> print 'Group', name, 'has', count, 'articles, range', first, 'to', last
   9 Group comp.lang.python has 51 articles, range 5770 to 5821
  10 >>> resp, subs = s.xhdr('subject', first + '-' + last)
  11 >>> resp = s.quit()
  12 >>>
  13
  14 Here 'resp' is the server response line.
  15 Error responses are turned into exceptions.
  16
  17 To post an article from a file:
  18 >>> f = open(filename, 'r') # file containing article, including header
  19 >>> resp = s.post(f)
  20 >>>
  21
  22 For descriptions of all methods, read the comments in the code below.
  23 Note that all arguments and return values representing article numbers
  24 are strings, not numbers, since they are rarely used for calculations.
  25 """
  26
  27 # RFC 977 by Brian Kantor and Phil Lapsley.
  28 # xover, xgtitle, xpath, date methods by Kevan Heydon
  29
  30
  31 # Imports
  32 import re
  33 import socket
  34
  35 __all__ = ["NNTP","NNTPReplyError","NNTPTemporaryError",
  36            "NNTPPermanentError","NNTPProtocolError","NNTPDataError",
  37            "error_reply","error_temp","error_perm","error_proto",
  38            "error_data",]
  39
  40 # Exceptions raised when an error or invalid response is received
  41 class NNTPError(Exception):
  42     """Base class for all nntplib exceptions"""
  43     def __init__(self, *args):
  44         Exception.__init__(self, *args)
  45         try:
  46             self.response = args[0]
  47         except IndexError:
  48             self.response = 'No response given'
  49
  50 class NNTPReplyError(NNTPError):
  51     """Unexpected [123]xx reply"""
  52     pass
  53
  54 class NNTPTemporaryError(NNTPError):
  55     """4xx errors"""
  56     pass
  57
  58 class NNTPPermanentError(NNTPError):
  59     """5xx errors"""
  60     pass
  61
  62 class NNTPProtocolError(NNTPError):
  63     """Response does not begin with [1-5]"""
  64     pass
  65
  66 class NNTPDataError(NNTPError):
  67     """Error in response data"""
  68     pass
  69
  70 # for backwards compatibility
  71 error_reply = NNTPReplyError
  72 error_temp = NNTPTemporaryError
  73 error_perm = NNTPPermanentError
  74 error_proto = NNTPProtocolError
  75 error_data = NNTPDataError
  76
  77
  78
  79 # Standard port used by NNTP servers
  80 NNTP_PORT = 119
  81
  82
  83 # Response numbers that are followed by additional text (e.g. article)
  84 LONGRESP = ['100', '215', '220', '221', '222', '224', '230', '231', '282']
  85
  86
  87 # Line terminators (we always output CRLF, but accept any of CRLF, CR, LF)
  88 CRLF = '\r\n'
  89
  90
  91
  92 # The class itself
  93 class NNTP:
  94     def __init__(self, host, port=NNTP_PORT, user=None, password=None,
  95                  readermode=None, usenetrc=True):
  96         """Initialize an instance.  Arguments:
  97         - host: hostname to connect to
  98         - port: port to connect to (default the standard NNTP port)
  99         - user: username to authenticate with
 100         - password: password to use with username
 101         - readermode: if true, send 'mode reader' command after
 102                       connecting.
 103
 104         readermode is sometimes necessary if you are connecting to an
 105         NNTP server on the local machine and intend to call
 106         reader-specific comamnds, such as `group'.  If you get
 107         unexpected NNTPPermanentErrors, you might need to set
 108         readermode.
 109         """
 110         self.host = host
 111         self.port = port
 112         self.sock = socket.create_connection((host, port))
 113         self.file = self.sock.makefile('rb')
 114         self.debugging = 0
 115         self.welcome = self.getresp()
 116
 117         # 'mode reader' is sometimes necessary to enable 'reader' mode.
 118         # However, the order in which 'mode reader' and 'authinfo' need to
 119         # arrive differs between some NNTP servers. Try to send
 120         # 'mode reader', and if it fails with an authorization failed
 121         # error, try again after sending authinfo.
 122         readermode_afterauth = 0
 123         if readermode:
 124             try:
 125                 self.welcome = self.shortcmd('mode reader')
 126             except NNTPPermanentError:
 127                 # error 500, probably 'not implemented'
 128                 pass
 129             except NNTPTemporaryError, e:
 130                 if user and e.response[:3] == '480':
 131                     # Need authorization before 'mode reader'
 132                     readermode_afterauth = 1
 133                 else:
 134                     raise
 135         # If no login/password was specified, try to get them from ~/.netrc
 136         # Presume that if .netc has an entry, NNRP authentication is required.
 137         try:
 138             if usenetrc and not user:
 139                 import netrc
 140                 credentials = netrc.netrc()
 141                 auth = credentials.authenticators(host)
 142                 if auth:
 143                     user = auth[0]
 144                     password = auth[2]
 145         except IOError:
 146             pass
 147         # Perform NNRP authentication if needed.
 148         if user:
 149             resp = self.shortcmd('authinfo user '+user)
 150             if resp[:3] == '381':
 151                 if not password:
 152                     raise NNTPReplyError(resp)
 153                 else:
 154                     resp = self.shortcmd(
 155                             'authinfo pass '+password)
 156                     if resp[:3] != '281':
 157                         raise NNTPPermanentError(resp)
 158             if readermode_afterauth:
 159                 try:
 160                     self.welcome = self.shortcmd('mode reader')
 161                 except NNTPPermanentError:
 162                     # error 500, probably 'not implemented'
 163                     pass
 164
 165
 166     # Get the welcome message from the server
 167     # (this is read and squirreled away by __init__()).
 168     # If the response code is 200, posting is allowed;
 169     # if it 201, posting is not allowed
 170
 171     def getwelcome(self):
 172         """Get the welcome message from the server
 173         (this is read and squirreled away by __init__()).
 174         If the response code is 200, posting is allowed;
 175         if it 201, posting is not allowed."""
 176
 177         if self.debugging: print '*welcome*', repr(self.welcome)
 178         return self.welcome
 179
 180     def set_debuglevel(self, level):
 181         """Set the debugging level.  Argument 'level' means:
 182         0: no debugging output (default)
 183         1: print commands and responses but not body text etc.
 184         2: also print raw lines read and sent before stripping CR/LF"""
 185
 186         self.debugging = level
 187     debug = set_debuglevel
 188
 189     def putline(self, line):
 190         """Internal: send one line to the server, appending CRLF."""
 191         line = line + CRLF
 192         if self.debugging > 1: print '*put*', repr(line)
 193         self.sock.sendall(line)
 194
 195     def putcmd(self, line):
 196         """Internal: send one command to the server (through putline())."""
 197         if self.debugging: print '*cmd*', repr(line)
 198         self.putline(line)
 199
 200     def getline(self):
 201         """Internal: return one line from the server, stripping CRLF.
 202         Raise EOFError if the connection is closed."""
 203         line = self.file.readline()
 204         if self.debugging > 1:
 205             print '*get*', repr(line)
 206         if not line: raise EOFError
 207         if line[-2:] == CRLF: line = line[:-2]
 208         elif line[-1:] in CRLF: line = line[:-1]
 209         return line
 210
 211     def getresp(self):
 212         """Internal: get a response from the server.
 213         Raise various errors if the response indicates an error."""
 214         resp = self.getline()
 215         if self.debugging: print '*resp*', repr(resp)
 216         c = resp[:1]
 217         if c == '4':
 218             raise NNTPTemporaryError(resp)
 219         if c == '5':
 220             raise NNTPPermanentError(resp)
 221         if c not in '123':
 222             raise NNTPProtocolError(resp)
 223         return resp
 224
 225     def getlongresp(self, file=None):
 226         """Internal: get a response plus following text from the server.
 227         Raise various errors if the response indicates an error."""
 228
 229         openedFile = None
 230         try:
 231             # If a string was passed then open a file with that name
 232             if isinstance(file, str):
 233                 openedFile = file = open(file, "w")
 234
 235             resp = self.getresp()
 236             if resp[:3] not in LONGRESP:
 237                 raise NNTPReplyError(resp)
 238             list = []
 239             while 1:
 240                 line = self.getline()
 241                 if line == '.':
 242                     break
 243                 if line[:2] == '..':
 244                     line = line[1:]
 245                 if file:
 246                     file.write(line + "\n")
 247                 else:
 248                     list.append(line)
 249         finally:
 250             # If this method created the file, then it must close it
 251             if openedFile:
 252                 openedFile.close()
 253
 254         return resp, list
 255
 256     def shortcmd(self, line):
 257         """Internal: send a command and get the response."""
 258         self.putcmd(line)
 259         return self.getresp()
 260
 261     def longcmd(self, line, file=None):
 262         """Internal: send a command and get the response plus following text."""
 263         self.putcmd(line)
 264         return self.getlongresp(file)
 265
 266     def newgroups(self, date, time, file=None):
 267         """Process a NEWGROUPS command.  Arguments:
 268         - date: string 'yymmdd' indicating the date
 269         - time: string 'hhmmss' indicating the time
 270         Return:
 271         - resp: server response if successful
 272         - list: list of newsgroup names"""
 273
 274         return self.longcmd('NEWGROUPS ' + date + ' ' + time, file)
 275
 276     def newnews(self, group, date, time, file=None):
 277         """Process a NEWNEWS command.  Arguments:
 278         - group: group name or '*'
 279         - date: string 'yymmdd' indicating the date
 280         - time: string 'hhmmss' indicating the time
 281         Return:
 282         - resp: server response if successful
 283         - list: list of message ids"""
 284
 285         cmd = 'NEWNEWS ' + group + ' ' + date + ' ' + time
 286         return self.longcmd(cmd, file)
 287
 288     def list(self, file=None):
 289         """Process a LIST command.  Return:
 290         - resp: server response if successful
 291         - list: list of (group, last, first, flag) (strings)"""
 292
 293         resp, list = self.longcmd('LIST', file)
 294         for i in range(len(list)):
 295             # Parse lines into "group last first flag"
 296             list[i] = tuple(list[i].split())
 297         return resp, list
 298
 299     def description(self, group):
 300
 301         """Get a description for a single group.  If more than one
 302         group matches ('group' is a pattern), return the first.  If no
 303         group matches, return an empty string.
 304
 305         This elides the response code from the server, since it can
 306         only be '215' or '285' (for xgtitle) anyway.  If the response
 307         code is needed, use the 'descriptions' method.
 308
 309         NOTE: This neither checks for a wildcard in 'group' nor does
 310         it check whether the group actually exists."""
 311
 312         resp, lines = self.descriptions(group)
 313         if len(lines) == 0:
 314             return ""
 315         else:
 316             return lines[0][1]
 317
 318     def descriptions(self, group_pattern):
 319         """Get descriptions for a range of groups."""
 320         line_pat = re.compile("^(?P<group>[^ \t]+)[ \t]+(.*)$")
 321         # Try the more std (acc. to RFC2980) LIST NEWSGROUPS first
 322         resp, raw_lines = self.longcmd('LIST NEWSGROUPS ' + group_pattern)
 323         if resp[:3] != "215":
 324             # Now the deprecated XGTITLE.  This either raises an error
 325             # or succeeds with the same output structure as LIST
 326             # NEWSGROUPS.
 327             resp, raw_lines = self.longcmd('XGTITLE ' + group_pattern)
 328         lines = []
 329         for raw_line in raw_lines:
 330             match = line_pat.search(raw_line.strip())
 331             if match:
 332                 lines.append(match.group(1, 2))
 333         return resp, lines
 334
 335     def group(self, name):
 336         """Process a GROUP command.  Argument:
 337         - group: the group name
 338         Returns:
 339         - resp: server response if successful
 340         - count: number of articles (string)
 341         - first: first article number (string)
 342         - last: last article number (string)
 343         - name: the group name"""
 344
 345         resp = self.shortcmd('GROUP ' + name)
 346         if resp[:3] != '211':
 347             raise NNTPReplyError(resp)
 348         words = resp.split()
 349         count = first = last = 0
 350         n = len(words)
 351         if n > 1:
 352             count = words[1]
 353             if n > 2:
 354                 first = words[2]
 355                 if n > 3:
 356                     last = words[3]
 357                     if n > 4:
 358                         name = words[4].lower()
 359         return resp, count, first, last, name
 360
 361     def help(self, file=None):
 362         """Process a HELP command.  Returns:
 363         - resp: server response if successful
 364         - list: list of strings"""
 365
 366         return self.longcmd('HELP',file)
 367
 368     def statparse(self, resp):
 369         """Internal: parse the response of a STAT, NEXT or LAST command."""
 370         if resp[:2] != '22':
 371             raise NNTPReplyError(resp)
 372         words = resp.split()
 373         nr = 0
 374         id = ''
 375         n = len(words)
 376         if n > 1:
 377             nr = words[1]
 378             if n > 2:
 379                 id = words[2]
 380         return resp, nr, id
 381
 382     def statcmd(self, line):
 383         """Internal: process a STAT, NEXT or LAST command."""
 384         resp = self.shortcmd(line)
 385         return self.statparse(resp)
 386
 387     def stat(self, id):
 388         """Process a STAT command.  Argument:
 389         - id: article number or message id
 390         Returns:
 391         - resp: server response if successful
 392         - nr:   the article number
 393         - id:   the message id"""
 394
 395         return self.statcmd('STAT ' + id)
 396
 397     def next(self):
 398         """Process a NEXT command.  No arguments.  Return as for STAT."""
 399         return self.statcmd('NEXT')
 400
 401     def last(self):
 402         """Process a LAST command.  No arguments.  Return as for STAT."""
 403         return self.statcmd('LAST')
 404
 405     def artcmd(self, line, file=None):
 406         """Internal: process a HEAD, BODY or ARTICLE command."""
 407         resp, list = self.longcmd(line, file)
 408         resp, nr, id = self.statparse(resp)
 409         return resp, nr, id, list
 410
 411     def head(self, id):
 412         """Process a HEAD command.  Argument:
 413         - id: article number or message id
 414         Returns:
 415         - resp: server response if successful
 416         - nr: article number
 417         - id: message id
 418         - list: the lines of the article's header"""
 419
 420         return self.artcmd('HEAD ' + id)
 421
 422     def body(self, id, file=None):
 423         """Process a BODY command.  Argument:
 424         - id: article number or message id
 425         - file: Filename string or file object to store the article in
 426         Returns:
 427         - resp: server response if successful
 428         - nr: article number
 429         - id: message id
 430         - list: the lines of the article's body or an empty list
 431                 if file was used"""
 432
 433         return self.artcmd('BODY ' + id, file)
 434
 435     def article(self, id):
 436         """Process an ARTICLE command.  Argument:
 437         - id: article number or message id
 438         Returns:
 439         - resp: server response if successful
 440         - nr: article number
 441         - id: message id
 442         - list: the lines of the article"""
 443
 444         return self.artcmd('ARTICLE ' + id)
 445
 446     def slave(self):
 447         """Process a SLAVE command.  Returns:
 448         - resp: server response if successful"""
 449
 450         return self.shortcmd('SLAVE')
 451
 452     def xhdr(self, hdr, str, file=None):
 453         """Process an XHDR command (optional server extension).  Arguments:
 454         - hdr: the header type (e.g. 'subject')
 455         - str: an article nr, a message id, or a range nr1-nr2
 456         Returns:
 457         - resp: server response if successful
 458         - list: list of (nr, value) strings"""
 459
 460         pat = re.compile('^([0-9]+) ?(.*)\n?')
 461         resp, lines = self.longcmd('XHDR ' + hdr + ' ' + str, file)
 462         for i in range(len(lines)):
 463             line = lines[i]
 464             m = pat.match(line)
 465             if m:
 466                 lines[i] = m.group(1, 2)
 467         return resp, lines
 468
 469     def xover(self, start, end, file=None):
 470         """Process an XOVER command (optional server extension) Arguments:
 471         - start: start of range
 472         - end: end of range
 473         Returns:
 474         - resp: server response if successful
 475         - list: list of (art-nr, subject, poster, date,
 476                          id, references, size, lines)"""
 477
 478         resp, lines = self.longcmd('XOVER ' + start + '-' + end, file)
 479         xover_lines = []
 480         for line in lines:
 481             elem = line.split("\t")
 482             try:
 483                 xover_lines.append((elem[0],
 484                                     elem[1],
 485                                     elem[2],
 486                                     elem[3],
 487                                     elem[4],
 488                                     elem[5].split(),
 489                                     elem[6],
 490                                     elem[7]))
 491             except IndexError:
 492                 raise NNTPDataError(line)
 493         return resp,xover_lines
 494
 495     def xgtitle(self, group, file=None):
 496         """Process an XGTITLE command (optional server extension) Arguments:
 497         - group: group name wildcard (i.e. news.*)
 498         Returns:
 499         - resp: server response if successful
 500         - list: list of (name,title) strings"""
 501
 502         line_pat = re.compile("^([^ \t]+)[ \t]+(.*)$")
 503         resp, raw_lines = self.longcmd('XGTITLE ' + group, file)
 504         lines = []
 505         for raw_line in raw_lines:
 506             match = line_pat.search(raw_line.strip())
 507             if match:
 508                 lines.append(match.group(1, 2))
 509         return resp, lines
 510
 511     def xpath(self,id):
 512         """Process an XPATH command (optional server extension) Arguments:
 513         - id: Message id of article
 514         Returns:
 515         resp: server response if successful
 516         path: directory path to article"""
 517
 518         resp = self.shortcmd("XPATH " + id)
 519         if resp[:3] != '223':
 520             raise NNTPReplyError(resp)
 521         try:
 522             [resp_num, path] = resp.split()
 523         except ValueError:
 524             raise NNTPReplyError(resp)
 525         else:
 526             return resp, path
 527
 528     def date (self):
 529         """Process the DATE command. Arguments:
 530         None
 531         Returns:
 532         resp: server response if successful
 533         date: Date suitable for newnews/newgroups commands etc.
 534         time: Time suitable for newnews/newgroups commands etc."""
 535
 536         resp = self.shortcmd("DATE")
 537         if resp[:3] != '111':
 538             raise NNTPReplyError(resp)
 539         elem = resp.split()
 540         if len(elem) != 2:
 541             raise NNTPDataError(resp)
 542         date = elem[1][2:8]
 543         time = elem[1][-6:]
 544         if len(date) != 6 or len(time) != 6:
 545             raise NNTPDataError(resp)
 546         return resp, date, time
 547
 548
 549     def post(self, f):
 550         """Process a POST command.  Arguments:
 551         - f: file containing the article
 552         Returns:
 553         - resp: server response if successful"""
 554
 555         resp = self.shortcmd('POST')
 556         # Raises error_??? if posting is not allowed
 557         if resp[0] != '3':
 558             raise NNTPReplyError(resp)
 559         while 1:
 560             line = f.readline()
 561             if not line:
 562                 break
 563             if line[-1] == '\n':
 564                 line = line[:-1]
 565             if line[:1] == '.':
 566                 line = '.' + line
 567             self.putline(line)
 568         self.putline('.')
 569         return self.getresp()
 570
 571     def ihave(self, id, f):
 572         """Process an IHAVE command.  Arguments:
 573         - id: message-id of the article
 574         - f:  file containing the article
 575         Returns:
 576         - resp: server response if successful
 577         Note that if the server refuses the article an exception is raised."""
 578
 579         resp = self.shortcmd('IHAVE ' + id)
 580         # Raises error_??? if the server already has it
 581         if resp[0] != '3':
 582             raise NNTPReplyError(resp)
 583         while 1:
 584             line = f.readline()
 585             if not line:
 586                 break
 587             if line[-1] == '\n':
 588                 line = line[:-1]
 589             if line[:1] == '.':
 590                 line = '.' + line
 591             self.putline(line)
 592         self.putline('.')
 593         return self.getresp()
 594
 595     def quit(self):
 596         """Process a QUIT command and close the socket.  Returns:
 597         - resp: server response if successful"""
 598
 599         resp = self.shortcmd('QUIT')
 600         self.file.close()
 601         self.sock.close()
 602         del self.file, self.sock
 603         return resp
 604
 605
 606 # Test retrieval when run as a script.
 607 # Assumption: if there's a local news server, it's called 'news'.
 608 # Assumption: if user queries a remote news server, it's named
 609 # in the environment variable NNTPSERVER (used by slrn and kin)
 610 # and we want readermode off.
 611 if __name__ == '__main__':
 612     import os
 613     newshost = 'news' and os.environ["NNTPSERVER"]
 614     if newshost.find('.') == -1:
 615         mode = 'readermode'
 616     else:
 617         mode = None
 618     s = NNTP(newshost, readermode=mode)
 619     resp, count, first, last, name = s.group('comp.lang.python')
 620     print resp
 621     print 'Group', name, 'has', count, 'articles, range', first, 'to', last
 622     resp, subs = s.xhdr('subject', first + '-' + last)
 623     print resp
 624     for item in subs:
 625         print "%7s %s" % item
 626     resp = s.quit()
 627     print resp