Lib/httplib.py

   1 """HTTP/1.1 client library
   2
   3 <intro stuff goes here>
   4 <other stuff, too>
   5
   6 HTTPConnection goes through a number of "states", which define when a client
   7 may legally make another request or fetch the response for a particular
   8 request. This diagram details these state transitions:
   9
  10     (null)
  11       |
  12       | HTTPConnection()
  13       v
  14     Idle
  15       |
  16       | putrequest()
  17       v
  18     Request-started
  19       |
  20       | ( putheader() )*  endheaders()
  21       v
  22     Request-sent
  23       |
  24       | response = getresponse()
  25       v
  26     Unread-response   [Response-headers-read]
  27       |\____________________
  28       |                     |
  29       | response.read()     | putrequest()
  30       v                     v
  31     Idle                  Req-started-unread-response
  32                      ______/|
  33                    /        |
  34    response.read() |        | ( putheader() )*  endheaders()
  35                    v        v
  36        Request-started    Req-sent-unread-response
  37                             |
  38                             | response.read()
  39                             v
  40                           Request-sent
  41
  42 This diagram presents the following rules:
  43   -- a second request may not be started until {response-headers-read}
  44   -- a response [object] cannot be retrieved until {request-sent}
  45   -- there is no differentiation between an unread response body and a
  46      partially read response body
  47
  48 Note: this enforcement is applied by the HTTPConnection class. The
  49       HTTPResponse class does not enforce this state machine, which
  50       implies sophisticated clients may accelerate the request/response
  51       pipeline. Caution should be taken, though: accelerating the states
  52       beyond the above pattern may imply knowledge of the server's
  53       connection-close behavior for certain requests. For example, it
  54       is impossible to tell whether the server will close the connection
  55       UNTIL the response headers have been read; this means that further
  56       requests cannot be placed into the pipeline until it is known that
  57       the server will NOT be closing the connection.
  58
  59 Logical State                  __state            __response
  60 -------------                  -------            ----------
  61 Idle                           _CS_IDLE           None
  62 Request-started                _CS_REQ_STARTED    None
  63 Request-sent                   _CS_REQ_SENT       None
  64 Unread-response                _CS_IDLE           <response_class>
  65 Req-started-unread-response    _CS_REQ_STARTED    <response_class>
  66 Req-sent-unread-response       _CS_REQ_SENT       <response_class>
  67 """
  68
  69 import errno
  70 import mimetools
  71 import socket
  72 from urlparse import urlsplit
  73
  74 try:
  75     from cStringIO import StringIO
  76 except ImportError:
  77     from StringIO import StringIO
  78
  79 __all__ = ["HTTP", "HTTPResponse", "HTTPConnection", "HTTPSConnection",
  80            "HTTPException", "NotConnected", "UnknownProtocol",
  81            "UnknownTransferEncoding", "UnimplementedFileMode",
  82            "IncompleteRead", "InvalidURL", "ImproperConnectionState",
  83            "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
  84            "BadStatusLine", "error", "responses"]
  85
  86 HTTP_PORT = 80
  87 HTTPS_PORT = 443
  88
  89 _UNKNOWN = 'UNKNOWN'
  90
  91 # connection states
  92 _CS_IDLE = 'Idle'
  93 _CS_REQ_STARTED = 'Request-started'
  94 _CS_REQ_SENT = 'Request-sent'
  95
  96 # status codes
  97 # informational
  98 CONTINUE = 100
  99 SWITCHING_PROTOCOLS = 101
 100 PROCESSING = 102
 101
 102 # successful
 103 OK = 200
 104 CREATED = 201
 105 ACCEPTED = 202
 106 NON_AUTHORITATIVE_INFORMATION = 203
 107 NO_CONTENT = 204
 108 RESET_CONTENT = 205
 109 PARTIAL_CONTENT = 206
 110 MULTI_STATUS = 207
 111 IM_USED = 226
 112
 113 # redirection
 114 MULTIPLE_CHOICES = 300
 115 MOVED_PERMANENTLY = 301
 116 FOUND = 302
 117 SEE_OTHER = 303
 118 NOT_MODIFIED = 304
 119 USE_PROXY = 305
 120 TEMPORARY_REDIRECT = 307
 121
 122 # client error
 123 BAD_REQUEST = 400
 124 UNAUTHORIZED = 401
 125 PAYMENT_REQUIRED = 402
 126 FORBIDDEN = 403
 127 NOT_FOUND = 404
 128 METHOD_NOT_ALLOWED = 405
 129 NOT_ACCEPTABLE = 406
 130 PROXY_AUTHENTICATION_REQUIRED = 407
 131 REQUEST_TIMEOUT = 408
 132 CONFLICT = 409
 133 GONE = 410
 134 LENGTH_REQUIRED = 411
 135 PRECONDITION_FAILED = 412
 136 REQUEST_ENTITY_TOO_LARGE = 413
 137 REQUEST_URI_TOO_LONG = 414
 138 UNSUPPORTED_MEDIA_TYPE = 415
 139 REQUESTED_RANGE_NOT_SATISFIABLE = 416
 140 EXPECTATION_FAILED = 417
 141 UNPROCESSABLE_ENTITY = 422
 142 LOCKED = 423
 143 FAILED_DEPENDENCY = 424
 144 UPGRADE_REQUIRED = 426
 145
 146 # server error
 147 INTERNAL_SERVER_ERROR = 500
 148 NOT_IMPLEMENTED = 501
 149 BAD_GATEWAY = 502
 150 SERVICE_UNAVAILABLE = 503
 151 GATEWAY_TIMEOUT = 504
 152 HTTP_VERSION_NOT_SUPPORTED = 505
 153 INSUFFICIENT_STORAGE = 507
 154 NOT_EXTENDED = 510
 155
 156 # Mapping status codes to official W3C names
 157 responses = {
 158     100: 'Continue',
 159     101: 'Switching Protocols',
 160
 161     200: 'OK',
 162     201: 'Created',
 163     202: 'Accepted',
 164     203: 'Non-Authoritative Information',
 165     204: 'No Content',
 166     205: 'Reset Content',
 167     206: 'Partial Content',
 168
 169     300: 'Multiple Choices',
 170     301: 'Moved Permanently',
 171     302: 'Found',
 172     303: 'See Other',
 173     304: 'Not Modified',
 174     305: 'Use Proxy',
 175     306: '(Unused)',
 176     307: 'Temporary Redirect',
 177
 178     400: 'Bad Request',
 179     401: 'Unauthorized',
 180     402: 'Payment Required',
 181     403: 'Forbidden',
 182     404: 'Not Found',
 183     405: 'Method Not Allowed',
 184     406: 'Not Acceptable',
 185     407: 'Proxy Authentication Required',
 186     408: 'Request Timeout',
 187     409: 'Conflict',
 188     410: 'Gone',
 189     411: 'Length Required',
 190     412: 'Precondition Failed',
 191     413: 'Request Entity Too Large',
 192     414: 'Request-URI Too Long',
 193     415: 'Unsupported Media Type',
 194     416: 'Requested Range Not Satisfiable',
 195     417: 'Expectation Failed',
 196
 197     500: 'Internal Server Error',
 198     501: 'Not Implemented',
 199     502: 'Bad Gateway',
 200     503: 'Service Unavailable',
 201     504: 'Gateway Timeout',
 202     505: 'HTTP Version Not Supported',
 203 }
 204
 205 # maximal amount of data to read at one time in _safe_read
 206 MAXAMOUNT = 1048576
 207
 208 class HTTPMessage(mimetools.Message):
 209
 210     def addheader(self, key, value):
 211         """Add header for field key handling repeats."""
 212         prev = self.dict.get(key)
 213         if prev is None:
 214             self.dict[key] = value
 215         else:
 216             combined = ", ".join((prev, value))
 217             self.dict[key] = combined
 218
 219     def addcontinue(self, key, more):
 220         """Add more field data from a continuation line."""
 221         prev = self.dict[key]
 222         self.dict[key] = prev + "\n " + more
 223
 224     def readheaders(self):
 225         """Read header lines.
 226
 227         Read header lines up to the entirely blank line that terminates them.
 228         The (normally blank) line that ends the headers is skipped, but not
 229         included in the returned list.  If a non-header line ends the headers,
 230         (which is an error), an attempt is made to backspace over it; it is
 231         never included in the returned list.
 232
 233         The variable self.status is set to the empty string if all went well,
 234         otherwise it is an error message.  The variable self.headers is a
 235         completely uninterpreted list of lines contained in the header (so
 236         printing them will reproduce the header exactly as it appears in the
 237         file).
 238
 239         If multiple header fields with the same name occur, they are combined
 240         according to the rules in RFC 2616 sec 4.2:
 241
 242         Appending each subsequent field-value to the first, each separated
 243         by a comma. The order in which header fields with the same field-name
 244         are received is significant to the interpretation of the combined
 245         field value.
 246         """
 247         # XXX The implementation overrides the readheaders() method of
 248         # rfc822.Message.  The base class design isn't amenable to
 249         # customized behavior here so the method here is a copy of the
 250         # base class code with a few small changes.
 251
 252         self.dict = {}
 253         self.unixfrom = ''
 254         self.headers = hlist = []
 255         self.status = ''
 256         headerseen = ""
 257         firstline = 1
 258         startofline = unread = tell = None
 259         if hasattr(self.fp, 'unread'):
 260             unread = self.fp.unread
 261         elif self.seekable:
 262             tell = self.fp.tell
 263         while True:
 264             if tell:
 265                 try:
 266                     startofline = tell()
 267                 except IOError:
 268                     startofline = tell = None
 269                     self.seekable = 0
 270             line = self.fp.readline()
 271             if not line:
 272                 self.status = 'EOF in headers'
 273                 break
 274             # Skip unix From name time lines
 275             if firstline and line.startswith('From '):
 276                 self.unixfrom = self.unixfrom + line
 277                 continue
 278             firstline = 0
 279             if headerseen and line[0] in ' \t':
 280                 # XXX Not sure if continuation lines are handled properly
 281                 # for http and/or for repeating headers
 282                 # It's a continuation line.
 283                 hlist.append(line)
 284                 self.addcontinue(headerseen, line.strip())
 285                 continue
 286             elif self.iscomment(line):
 287                 # It's a comment.  Ignore it.
 288                 continue
 289             elif self.islast(line):
 290                 # Note! No pushback here!  The delimiter line gets eaten.
 291                 break
 292             headerseen = self.isheader(line)
 293             if headerseen:
 294                 # It's a legal header line, save it.
 295                 hlist.append(line)
 296                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
 297                 continue
 298             else:
 299                 # It's not a header line; throw it back and stop here.
 300                 if not self.dict:
 301                     self.status = 'No headers'
 302                 else:
 303                     self.status = 'Non-header line where header expected'
 304                 # Try to undo the read.
 305                 if unread:
 306                     unread(line)
 307                 elif tell:
 308                     self.fp.seek(startofline)
 309                 else:
 310                     self.status = self.status + '; bad seek'
 311                 break
 312
 313 class HTTPResponse:
 314
 315     # strict: If true, raise BadStatusLine if the status line can't be
 316     # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
 317     # false because it prevents clients from talking to HTTP/0.9
 318     # servers.  Note that a response with a sufficiently corrupted
 319     # status line will look like an HTTP/0.9 response.
 320
 321     # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
 322
 323     def __init__(self, sock, debuglevel=0, strict=0, method=None):
 324         self.fp = sock.makefile('rb', 0)
 325         self.debuglevel = debuglevel
 326         self.strict = strict
 327         self._method = method
 328
 329         self.msg = None
 330
 331         # from the Status-Line of the response
 332         self.version = _UNKNOWN # HTTP-Version
 333         self.status = _UNKNOWN  # Status-Code
 334         self.reason = _UNKNOWN  # Reason-Phrase
 335
 336         self.chunked = _UNKNOWN         # is "chunked" being used?
 337         self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
 338         self.length = _UNKNOWN          # number of bytes left in response
 339         self.will_close = _UNKNOWN      # conn will close at end of response
 340
 341     def _read_status(self):
 342         # Initialize with Simple-Response defaults
 343         line = self.fp.readline()
 344         if self.debuglevel > 0:
 345             print "reply:", repr(line)
 346         if not line:
 347             # Presumably, the server closed the connection before
 348             # sending a valid response.
 349             raise BadStatusLine(line)
 350         try:
 351             [version, status, reason] = line.split(None, 2)
 352         except ValueError:
 353             try:
 354                 [version, status] = line.split(None, 1)
 355                 reason = ""
 356             except ValueError:
 357                 # empty version will cause next test to fail and status
 358                 # will be treated as 0.9 response.
 359                 version = ""
 360         if not version.startswith('HTTP/'):
 361             if self.strict:
 362                 self.close()
 363                 raise BadStatusLine(line)
 364             else:
 365                 # assume it's a Simple-Response from an 0.9 server
 366                 self.fp = LineAndFileWrapper(line, self.fp)
 367                 return "HTTP/0.9", 200, ""
 368
 369         # The status code is a three-digit number
 370         try:
 371             status = int(status)
 372             if status < 100 or status > 999:
 373                 raise BadStatusLine(line)
 374         except ValueError:
 375             raise BadStatusLine(line)
 376         return version, status, reason
 377
 378     def begin(self):
 379         if self.msg is not None:
 380             # we've already started reading the response
 381             return
 382
 383         # read until we get a non-100 response
 384         while True:
 385             version, status, reason = self._read_status()
 386             if status != CONTINUE:
 387                 break
 388             # skip the header from the 100 response
 389             while True:
 390                 skip = self.fp.readline().strip()
 391                 if not skip:
 392                     break
 393                 if self.debuglevel > 0:
 394                     print "header:", skip
 395
 396         self.status = status
 397         self.reason = reason.strip()
 398         if version == 'HTTP/1.0':
 399             self.version = 10
 400         elif version.startswith('HTTP/1.'):
 401             self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
 402         elif version == 'HTTP/0.9':
 403             self.version = 9
 404         else:
 405             raise UnknownProtocol(version)
 406
 407         if self.version == 9:
 408             self.length = None
 409             self.chunked = 0
 410             self.will_close = 1
 411             self.msg = HTTPMessage(StringIO())
 412             return
 413
 414         self.msg = HTTPMessage(self.fp, 0)
 415         if self.debuglevel > 0:
 416             for hdr in self.msg.headers:
 417                 print "header:", hdr,
 418
 419         # don't let the msg keep an fp
 420         self.msg.fp = None
 421
 422         # are we using the chunked-style of transfer encoding?
 423         tr_enc = self.msg.getheader('transfer-encoding')
 424         if tr_enc and tr_enc.lower() == "chunked":
 425             self.chunked = 1
 426             self.chunk_left = None
 427         else:
 428             self.chunked = 0
 429
 430         # will the connection close at the end of the response?
 431         self.will_close = self._check_close()
 432
 433         # do we have a Content-Length?
 434         # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
 435         length = self.msg.getheader('content-length')
 436         if length and not self.chunked:
 437             try:
 438                 self.length = int(length)
 439             except ValueError:
 440                 self.length = None
 441         else:
 442             self.length = None
 443
 444         # does the body have a fixed length? (of zero)
 445         if (status == NO_CONTENT or status == NOT_MODIFIED or
 446             100 <= status < 200 or      # 1xx codes
 447             self._method == 'HEAD'):
 448             self.length = 0
 449
 450         # if the connection remains open, and we aren't using chunked, and
 451         # a content-length was not provided, then assume that the connection
 452         # WILL close.
 453         if not self.will_close and \
 454            not self.chunked and \
 455            self.length is None:
 456             self.will_close = 1
 457
 458     def _check_close(self):
 459         conn = self.msg.getheader('connection')
 460         if self.version == 11:
 461             # An HTTP/1.1 proxy is assumed to stay open unless
 462             # explicitly closed.
 463             conn = self.msg.getheader('connection')
 464             if conn and "close" in conn.lower():
 465                 return True
 466             return False
 467
 468         # Some HTTP/1.0 implementations have support for persistent
 469         # connections, using rules different than HTTP/1.1.
 470
 471         # For older HTTP, Keep-Alive indiciates persistent connection.
 472         if self.msg.getheader('keep-alive'):
 473             return False
 474
 475         # At least Akamai returns a "Connection: Keep-Alive" header,
 476         # which was supposed to be sent by the client.
 477         if conn and "keep-alive" in conn.lower():
 478             return False
 479
 480         # Proxy-Connection is a netscape hack.
 481         pconn = self.msg.getheader('proxy-connection')
 482         if pconn and "keep-alive" in pconn.lower():
 483             return False
 484
 485         # otherwise, assume it will close
 486         return True
 487
 488     def close(self):
 489         if self.fp:
 490             self.fp.close()
 491             self.fp = None
 492
 493     def isclosed(self):
 494         # NOTE: it is possible that we will not ever call self.close(). This
 495         #       case occurs when will_close is TRUE, length is None, and we
 496         #       read up to the last byte, but NOT past it.
 497         #
 498         # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
 499         #          called, meaning self.isclosed() is meaningful.
 500         return self.fp is None
 501
 502     # XXX It would be nice to have readline and __iter__ for this, too.
 503
 504     def read(self, amt=None):
 505         if self.fp is None:
 506             return ''
 507
 508         if self.chunked:
 509             return self._read_chunked(amt)
 510
 511         if amt is None:
 512             # unbounded read
 513             if self.length is None:
 514                 s = self.fp.read()
 515             else:
 516                 s = self._safe_read(self.length)
 517                 self.length = 0
 518             self.close()        # we read everything
 519             return s
 520
 521         if self.length is not None:
 522             if amt > self.length:
 523                 # clip the read to the "end of response"
 524                 amt = self.length
 525
 526         # we do not use _safe_read() here because this may be a .will_close
 527         # connection, and the user is reading more bytes than will be provided
 528         # (for example, reading in 1k chunks)
 529         s = self.fp.read(amt)
 530         if self.length is not None:
 531             self.length -= len(s)
 532
 533         return s
 534
 535     def _read_chunked(self, amt):
 536         assert self.chunked != _UNKNOWN
 537         chunk_left = self.chunk_left
 538         value = ''
 539
 540         # XXX This accumulates chunks by repeated string concatenation,
 541         # which is not efficient as the number or size of chunks gets big.
 542         while True:
 543             if chunk_left is None:
 544                 line = self.fp.readline()
 545                 i = line.find(';')
 546                 if i >= 0:
 547                     line = line[:i] # strip chunk-extensions
 548                 chunk_left = int(line, 16)
 549                 if chunk_left == 0:
 550                     break
 551             if amt is None:
 552                 value += self._safe_read(chunk_left)
 553             elif amt < chunk_left:
 554                 value += self._safe_read(amt)
 555                 self.chunk_left = chunk_left - amt
 556                 return value
 557             elif amt == chunk_left:
 558                 value += self._safe_read(amt)
 559                 self._safe_read(2)  # toss the CRLF at the end of the chunk
 560                 self.chunk_left = None
 561                 return value
 562             else:
 563                 value += self._safe_read(chunk_left)
 564                 amt -= chunk_left
 565
 566             # we read the whole chunk, get another
 567             self._safe_read(2)      # toss the CRLF at the end of the chunk
 568             chunk_left = None
 569
 570         # read and discard trailer up to the CRLF terminator
 571         ### note: we shouldn't have any trailers!
 572         while True:
 573             line = self.fp.readline()
 574             if line == '\r\n':
 575                 break
 576
 577         # we read everything; close the "file"
 578         self.close()
 579
 580         return value
 581
 582     def _safe_read(self, amt):
 583         """Read the number of bytes requested, compensating for partial reads.
 584
 585         Normally, we have a blocking socket, but a read() can be interrupted
 586         by a signal (resulting in a partial read).
 587
 588         Note that we cannot distinguish between EOF and an interrupt when zero
 589         bytes have been read. IncompleteRead() will be raised in this
 590         situation.
 591
 592         This function should be used when <amt> bytes "should" be present for
 593         reading. If the bytes are truly not available (due to EOF), then the
 594         IncompleteRead exception can be used to detect the problem.
 595         """
 596         s = []
 597         while amt > 0:
 598             chunk = self.fp.read(min(amt, MAXAMOUNT))
 599             if not chunk:
 600                 raise IncompleteRead(s)
 601             s.append(chunk)
 602             amt -= len(chunk)
 603         return ''.join(s)
 604
 605     def getheader(self, name, default=None):
 606         if self.msg is None:
 607             raise ResponseNotReady()
 608         return self.msg.getheader(name, default)
 609
 610     def getheaders(self):
 611         """Return list of (header, value) tuples."""
 612         if self.msg is None:
 613             raise ResponseNotReady()
 614         return self.msg.items()
 615
 616
 617 class HTTPConnection:
 618
 619     _http_vsn = 11
 620     _http_vsn_str = 'HTTP/1.1'
 621
 622     response_class = HTTPResponse
 623     default_port = HTTP_PORT
 624     auto_open = 1
 625     debuglevel = 0
 626     strict = 0
 627
 628     def __init__(self, host, port=None, strict=None):
 629         self.sock = None
 630         self._buffer = []
 631         self.__response = None
 632         self.__state = _CS_IDLE
 633         self._method = None
 634
 635         self._set_hostport(host, port)
 636         if strict is not None:
 637             self.strict = strict
 638
 639     def _set_hostport(self, host, port):
 640         if port is None:
 641             i = host.rfind(':')
 642             j = host.rfind(']')         # ipv6 addresses have [...]
 643             if i > j:
 644                 try:
 645                     port = int(host[i+1:])
 646                 except ValueError:
 647                     raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
 648                 host = host[:i]
 649             else:
 650                 port = self.default_port
 651             if host and host[0] == '[' and host[-1] == ']':
 652                 host = host[1:-1]
 653         self.host = host
 654         self.port = port
 655
 656     def set_debuglevel(self, level):
 657         self.debuglevel = level
 658
 659     def connect(self):
 660         """Connect to the host and port specified in __init__."""
 661         msg = "getaddrinfo returns an empty list"
 662         for res in socket.getaddrinfo(self.host, self.port, 0,
 663                                       socket.SOCK_STREAM):
 664             af, socktype, proto, canonname, sa = res
 665             try:
 666                 self.sock = socket.socket(af, socktype, proto)
 667                 if self.debuglevel > 0:
 668                     print "connect: (%s, %s)" % (self.host, self.port)
 669                 self.sock.connect(sa)
 670             except socket.error, msg:
 671                 if self.debuglevel > 0:
 672                     print 'connect fail:', (self.host, self.port)
 673                 if self.sock:
 674                     self.sock.close()
 675                 self.sock = None
 676                 continue
 677             break
 678         if not self.sock:
 679             raise socket.error, msg
 680
 681     def close(self):
 682         """Close the connection to the HTTP server."""
 683         if self.sock:
 684             self.sock.close()   # close it manually... there may be other refs
 685             self.sock = None
 686         if self.__response:
 687             self.__response.close()
 688             self.__response = None
 689         self.__state = _CS_IDLE
 690
 691     def send(self, str):
 692         """Send `str' to the server."""
 693         if self.sock is None:
 694             if self.auto_open:
 695                 self.connect()
 696             else:
 697                 raise NotConnected()
 698
 699         # send the data to the server. if we get a broken pipe, then close
 700         # the socket. we want to reconnect when somebody tries to send again.
 701         #
 702         # NOTE: we DO propagate the error, though, because we cannot simply
 703         #       ignore the error... the caller will know if they can retry.
 704         if self.debuglevel > 0:
 705             print "send:", repr(str)
 706         try:
 707             blocksize=8192
 708             if hasattr(str,'read') :
 709                 if self.debuglevel > 0: print "sendIng a read()able"
 710                 data=str.read(blocksize)
 711                 while data:
 712                     self.sock.sendall(data)
 713                     data=str.read(blocksize)
 714             else:
 715                 self.sock.sendall(str)
 716         except socket.error, v:
 717             if v[0] == 32:      # Broken pipe
 718                 self.close()
 719             raise
 720
 721     def _output(self, s):
 722         """Add a line of output to the current request buffer.
 723
 724         Assumes that the line does *not* end with \\r\\n.
 725         """
 726         self._buffer.append(s)
 727
 728     def _send_output(self):
 729         """Send the currently buffered request and clear the buffer.
 730
 731         Appends an extra \\r\\n to the buffer.
 732         """
 733         self._buffer.extend(("", ""))
 734         msg = "\r\n".join(self._buffer)
 735         del self._buffer[:]
 736         self.send(msg)
 737
 738     def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
 739         """Send a request to the server.
 740
 741         `method' specifies an HTTP request method, e.g. 'GET'.
 742         `url' specifies the object being requested, e.g. '/index.html'.
 743         `skip_host' if True does not add automatically a 'Host:' header
 744         `skip_accept_encoding' if True does not add automatically an
 745            'Accept-Encoding:' header
 746         """
 747
 748         # if a prior response has been completed, then forget about it.
 749         if self.__response and self.__response.isclosed():
 750             self.__response = None
 751
 752
 753         # in certain cases, we cannot issue another request on this connection.
 754         # this occurs when:
 755         #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
 756         #   2) a response to a previous request has signalled that it is going
 757         #      to close the connection upon completion.
 758         #   3) the headers for the previous response have not been read, thus
 759         #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
 760         #
 761         # if there is no prior response, then we can request at will.
 762         #
 763         # if point (2) is true, then we will have passed the socket to the
 764         # response (effectively meaning, "there is no prior response"), and
 765         # will open a new one when a new request is made.
 766         #
 767         # Note: if a prior response exists, then we *can* start a new request.
 768         #       We are not allowed to begin fetching the response to this new
 769         #       request, however, until that prior response is complete.
 770         #
 771         if self.__state == _CS_IDLE:
 772             self.__state = _CS_REQ_STARTED
 773         else:
 774             raise CannotSendRequest()
 775
 776         # Save the method we use, we need it later in the response phase
 777         self._method = method
 778         if not url:
 779             url = '/'
 780         str = '%s %s %s' % (method, url, self._http_vsn_str)
 781
 782         self._output(str)
 783
 784         if self._http_vsn == 11:
 785             # Issue some standard headers for better HTTP/1.1 compliance
 786
 787             if not skip_host:
 788                 # this header is issued *only* for HTTP/1.1
 789                 # connections. more specifically, this means it is
 790                 # only issued when the client uses the new
 791                 # HTTPConnection() class. backwards-compat clients
 792                 # will be using HTTP/1.0 and those clients may be
 793                 # issuing this header themselves. we should NOT issue
 794                 # it twice; some web servers (such as Apache) barf
 795                 # when they see two Host: headers
 796
 797                 # If we need a non-standard port,include it in the
 798                 # header.  If the request is going through a proxy,
 799                 # but the host of the actual URL, not the host of the
 800                 # proxy.
 801
 802                 netloc = ''
 803                 if url.startswith('http'):
 804                     nil, netloc, nil, nil, nil = urlsplit(url)
 805
 806                 if netloc:
 807                     try:
 808                         netloc_enc = netloc.encode("ascii")
 809                     except UnicodeEncodeError:
 810                         netloc_enc = netloc.encode("idna")
 811                     self.putheader('Host', netloc_enc)
 812                 else:
 813                     try:
 814                         host_enc = self.host.encode("ascii")
 815                     except UnicodeEncodeError:
 816                         host_enc = self.host.encode("idna")
 817                     if self.port == HTTP_PORT:
 818                         self.putheader('Host', host_enc)
 819                     else:
 820                         self.putheader('Host', "%s:%s" % (host_enc, self.port))
 821
 822             # note: we are assuming that clients will not attempt to set these
 823             #       headers since *this* library must deal with the
 824             #       consequences. this also means that when the supporting
 825             #       libraries are updated to recognize other forms, then this
 826             #       code should be changed (removed or updated).
 827
 828             # we only want a Content-Encoding of "identity" since we don't
 829             # support encodings such as x-gzip or x-deflate.
 830             if not skip_accept_encoding:
 831                 self.putheader('Accept-Encoding', 'identity')
 832
 833             # we can accept "chunked" Transfer-Encodings, but no others
 834             # NOTE: no TE header implies *only* "chunked"
 835             #self.putheader('TE', 'chunked')
 836
 837             # if TE is supplied in the header, then it must appear in a
 838             # Connection header.
 839             #self.putheader('Connection', 'TE')
 840
 841         else:
 842             # For HTTP/1.0, the server will assume "not chunked"
 843             pass
 844
 845     def putheader(self, header, value):
 846         """Send a request header line to the server.
 847
 848         For example: h.putheader('Accept', 'text/html')
 849         """
 850         if self.__state != _CS_REQ_STARTED:
 851             raise CannotSendHeader()
 852
 853         str = '%s: %s' % (header, value)
 854         self._output(str)
 855
 856     def endheaders(self):
 857         """Indicate that the last header line has been sent to the server."""
 858
 859         if self.__state == _CS_REQ_STARTED:
 860             self.__state = _CS_REQ_SENT
 861         else:
 862             raise CannotSendHeader()
 863
 864         self._send_output()
 865
 866     def request(self, method, url, body=None, headers={}):
 867         """Send a complete request to the server."""
 868
 869         try:
 870             self._send_request(method, url, body, headers)
 871         except socket.error, v:
 872             # trap 'Broken pipe' if we're allowed to automatically reconnect
 873             if v[0] != 32 or not self.auto_open:
 874                 raise
 875             # try one more time
 876             self._send_request(method, url, body, headers)
 877
 878     def _send_request(self, method, url, body, headers):
 879         # honour explicitly requested Host: and Accept-Encoding headers
 880         header_names = dict.fromkeys([k.lower() for k in headers])
 881         skips = {}
 882         if 'host' in header_names:
 883             skips['skip_host'] = 1
 884         if 'accept-encoding' in header_names:
 885             skips['skip_accept_encoding'] = 1
 886
 887         self.putrequest(method, url, **skips)
 888
 889         if body and ('content-length' not in header_names):
 890             thelen=None
 891             try:
 892                 thelen=str(len(body))
 893             except TypeError, te:
 894                 # If this is a file-like object, try to
 895                 # fstat its file descriptor
 896                 import os
 897                 try:
 898                     thelen = str(os.fstat(body.fileno()).st_size)
 899                 except (AttributeError, OSError):
 900                     # Don't send a length if this failed
 901                     if self.debuglevel > 0: print "Cannot stat!!"
 902
 903             if thelen is not None:
 904                 self.putheader('Content-Length',thelen)
 905         for hdr, value in headers.iteritems():
 906             self.putheader(hdr, value)
 907         self.endheaders()
 908
 909         if body:
 910             self.send(body)
 911
 912     def getresponse(self):
 913         "Get the response from the server."
 914
 915         # if a prior response has been completed, then forget about it.
 916         if self.__response and self.__response.isclosed():
 917             self.__response = None
 918
 919         #
 920         # if a prior response exists, then it must be completed (otherwise, we
 921         # cannot read this response's header to determine the connection-close
 922         # behavior)
 923         #
 924         # note: if a prior response existed, but was connection-close, then the
 925         # socket and response were made independent of this HTTPConnection
 926         # object since a new request requires that we open a whole new
 927         # connection
 928         #
 929         # this means the prior response had one of two states:
 930         #   1) will_close: this connection was reset and the prior socket and
 931         #                  response operate independently
 932         #   2) persistent: the response was retained and we await its
 933         #                  isclosed() status to become true.
 934         #
 935         if self.__state != _CS_REQ_SENT or self.__response:
 936             raise ResponseNotReady()
 937
 938         if self.debuglevel > 0:
 939             response = self.response_class(self.sock, self.debuglevel,
 940                                            strict=self.strict,
 941                                            method=self._method)
 942         else:
 943             response = self.response_class(self.sock, strict=self.strict,
 944                                            method=self._method)
 945
 946         response.begin()
 947         assert response.will_close != _UNKNOWN
 948         self.__state = _CS_IDLE
 949
 950         if response.will_close:
 951             # Pass the socket to the response
 952             self.sock = None
 953         else:
 954             # remember this, so we can tell when it is complete
 955             self.__response = response
 956
 957         return response
 958
 959 # The next several classes are used to define FakeSocket, a socket-like
 960 # interface to an SSL connection.
 961
 962 # The primary complexity comes from faking a makefile() method.  The
 963 # standard socket makefile() implementation calls dup() on the socket
 964 # file descriptor.  As a consequence, clients can call close() on the
 965 # parent socket and its makefile children in any order.  The underlying
 966 # socket isn't closed until they are all closed.
 967
 968 # The implementation uses reference counting to keep the socket open
 969 # until the last client calls close().  SharedSocket keeps track of
 970 # the reference counting and SharedSocketClient provides an constructor
 971 # and close() method that call incref() and decref() correctly.
 972
 973 class SharedSocket:
 974
 975     def __init__(self, sock):
 976         self.sock = sock
 977         self._refcnt = 0
 978
 979     def incref(self):
 980         self._refcnt += 1
 981
 982     def decref(self):
 983         self._refcnt -= 1
 984         assert self._refcnt >= 0
 985         if self._refcnt == 0:
 986             self.sock.close()
 987
 988     def __del__(self):
 989         self.sock.close()
 990
 991 class SharedSocketClient:
 992
 993     def __init__(self, shared):
 994         self._closed = 0
 995         self._shared = shared
 996         self._shared.incref()
 997         self._sock = shared.sock
 998
 999     def close(self):
1000         if not self._closed:
1001             self._shared.decref()
1002             self._closed = 1
1003             self._shared = None
1004
1005 class SSLFile(SharedSocketClient):
1006     """File-like object wrapping an SSL socket."""
1007
1008     BUFSIZE = 8192
1009
1010     def __init__(self, sock, ssl, bufsize=None):
1011         SharedSocketClient.__init__(self, sock)
1012         self._ssl = ssl
1013         self._buf = ''
1014         self._bufsize = bufsize or self.__class__.BUFSIZE
1015
1016     def _read(self):
1017         buf = ''
1018         # put in a loop so that we retry on transient errors
1019         while True:
1020             try:
1021                 buf = self._ssl.read(self._bufsize)
1022             except socket.sslerror, err:
1023                 if (err[0] == socket.SSL_ERROR_WANT_READ
1024                     or err[0] == socket.SSL_ERROR_WANT_WRITE):
1025                     continue
1026                 if (err[0] == socket.SSL_ERROR_ZERO_RETURN
1027                     or err[0] == socket.SSL_ERROR_EOF):
1028                     break
1029                 raise
1030             except socket.error, err:
1031                 if err[0] == errno.EINTR:
1032                     continue
1033                 if err[0] == errno.EBADF:
1034                     # XXX socket was closed?
1035                     break
1036                 raise
1037             else:
1038                 break
1039         return buf
1040
1041     def read(self, size=None):
1042         L = [self._buf]
1043         avail = len(self._buf)
1044         while size is None or avail < size:
1045             s = self._read()
1046             if s == '':
1047                 break
1048             L.append(s)
1049             avail += len(s)
1050         all = "".join(L)
1051         if size is None:
1052             self._buf = ''
1053             return all
1054         else:
1055             self._buf = all[size:]
1056             return all[:size]
1057
1058     def readline(self):
1059         L = [self._buf]
1060         self._buf = ''
1061         while 1:
1062             i = L[-1].find("\n")
1063             if i >= 0:
1064                 break
1065             s = self._read()
1066             if s == '':
1067                 break
1068             L.append(s)
1069         if i == -1:
1070             # loop exited because there is no more data
1071             return "".join(L)
1072         else:
1073             all = "".join(L)
1074             # XXX could do enough bookkeeping not to do a 2nd search
1075             i = all.find("\n") + 1
1076             line = all[:i]
1077             self._buf = all[i:]
1078             return line
1079
1080     def readlines(self, sizehint=0):
1081         total = 0
1082         list = []
1083         while True:
1084             line = self.readline()
1085             if not line:
1086                 break
1087             list.append(line)
1088             total += len(line)
1089             if sizehint and total >= sizehint:
1090                 break
1091         return list
1092
1093     def fileno(self):
1094         return self._sock.fileno()
1095
1096     def __iter__(self):
1097         return self
1098
1099     def next(self):
1100         line = self.readline()
1101         if not line:
1102             raise StopIteration
1103         return line
1104
1105 class FakeSocket(SharedSocketClient):
1106
1107     class _closedsocket:
1108         def __getattr__(self, name):
1109             raise error(9, 'Bad file descriptor')
1110
1111     def __init__(self, sock, ssl):
1112         sock = SharedSocket(sock)
1113         SharedSocketClient.__init__(self, sock)
1114         self._ssl = ssl
1115
1116     def close(self):
1117         SharedSocketClient.close(self)
1118         self._sock = self.__class__._closedsocket()
1119
1120     def makefile(self, mode, bufsize=None):
1121         if mode != 'r' and mode != 'rb':
1122             raise UnimplementedFileMode()
1123         return SSLFile(self._shared, self._ssl, bufsize)
1124
1125     def send(self, stuff, flags = 0):
1126         return self._ssl.write(stuff)
1127
1128     sendall = send
1129
1130     def recv(self, len = 1024, flags = 0):
1131         return self._ssl.read(len)
1132
1133     def __getattr__(self, attr):
1134         return getattr(self._sock, attr)
1135
1136
1137 class HTTPSConnection(HTTPConnection):
1138     "This class allows communication via SSL."
1139
1140     default_port = HTTPS_PORT
1141
1142     def __init__(self, host, port=None, key_file=None, cert_file=None,
1143                  strict=None):
1144         HTTPConnection.__init__(self, host, port, strict)
1145         self.key_file = key_file
1146         self.cert_file = cert_file
1147
1148     def connect(self):
1149         "Connect to a host on a given (SSL) port."
1150
1151         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1152         sock.connect((self.host, self.port))
1153         ssl = socket.ssl(sock, self.key_file, self.cert_file)
1154         self.sock = FakeSocket(sock, ssl)
1155
1156
1157 class HTTP:
1158     "Compatibility class with httplib.py from 1.5."
1159
1160     _http_vsn = 10
1161     _http_vsn_str = 'HTTP/1.0'
1162
1163     debuglevel = 0
1164
1165     _connection_class = HTTPConnection
1166
1167     def __init__(self, host='', port=None, strict=None):
1168         "Provide a default host, since the superclass requires one."
1169
1170         # some joker passed 0 explicitly, meaning default port
1171         if port == 0:
1172             port = None
1173
1174         # Note that we may pass an empty string as the host; this will throw
1175         # an error when we attempt to connect. Presumably, the client code
1176         # will call connect before then, with a proper host.
1177         self._setup(self._connection_class(host, port, strict))
1178
1179     def _setup(self, conn):
1180         self._conn = conn
1181
1182         # set up delegation to flesh out interface
1183         self.send = conn.send
1184         self.putrequest = conn.putrequest
1185         self.endheaders = conn.endheaders
1186         self.set_debuglevel = conn.set_debuglevel
1187
1188         conn._http_vsn = self._http_vsn
1189         conn._http_vsn_str = self._http_vsn_str
1190
1191         self.file = None
1192
1193     def connect(self, host=None, port=None):
1194         "Accept arguments to set the host/port, since the superclass doesn't."
1195
1196         if host is not None:
1197             self._conn._set_hostport(host, port)
1198         self._conn.connect()
1199
1200     def getfile(self):
1201         "Provide a getfile, since the superclass' does not use this concept."
1202         return self.file
1203
1204     def putheader(self, header, *values):
1205         "The superclass allows only one value argument."
1206         self._conn.putheader(header, '\r\n\t'.join(values))
1207
1208     def getreply(self):
1209         """Compat definition since superclass does not define it.
1210
1211         Returns a tuple consisting of:
1212         - server status code (e.g. '200' if all goes well)
1213         - server "reason" corresponding to status code
1214         - any RFC822 headers in the response from the server
1215         """
1216         try:
1217             response = self._conn.getresponse()
1218         except BadStatusLine, e:
1219             ### hmm. if getresponse() ever closes the socket on a bad request,
1220             ### then we are going to have problems with self.sock
1221
1222             ### should we keep this behavior? do people use it?
1223             # keep the socket open (as a file), and return it
1224             self.file = self._conn.sock.makefile('rb', 0)
1225
1226             # close our socket -- we want to restart after any protocol error
1227             self.close()
1228
1229             self.headers = None
1230             return -1, e.line, None
1231
1232         self.headers = response.msg
1233         self.file = response.fp
1234         return response.status, response.reason, response.msg
1235
1236     def close(self):
1237         self._conn.close()
1238
1239         # note that self.file == response.fp, which gets closed by the
1240         # superclass. just clear the object ref here.
1241         ### hmm. messy. if status==-1, then self.file is owned by us.
1242         ### well... we aren't explicitly closing, but losing this ref will
1243         ### do it
1244         self.file = None
1245
1246 if hasattr(socket, 'ssl'):
1247     class HTTPS(HTTP):
1248         """Compatibility with 1.5 httplib interface
1249
1250         Python 1.5.2 did not have an HTTPS class, but it defined an
1251         interface for sending http requests that is also useful for
1252         https.
1253         """
1254
1255         _connection_class = HTTPSConnection
1256
1257         def __init__(self, host='', port=None, key_file=None, cert_file=None,
1258                      strict=None):
1259             # provide a default host, pass the X509 cert info
1260
1261             # urf. compensate for bad input.
1262             if port == 0:
1263                 port = None
1264             self._setup(self._connection_class(host, port, key_file,
1265                                                cert_file, strict))
1266
1267             # we never actually use these for anything, but we keep them
1268             # here for compatibility with post-1.5.2 CVS.
1269             self.key_file = key_file
1270             self.cert_file = cert_file
1271
1272
1273 class HTTPException(Exception):
1274     # Subclasses that define an __init__ must call Exception.__init__
1275     # or define self.args.  Otherwise, str() will fail.
1276     pass
1277
1278 class NotConnected(HTTPException):
1279     pass
1280
1281 class InvalidURL(HTTPException):
1282     pass
1283
1284 class UnknownProtocol(HTTPException):
1285     def __init__(self, version):
1286         self.args = version,
1287         self.version = version
1288
1289 class UnknownTransferEncoding(HTTPException):
1290     pass
1291
1292 class UnimplementedFileMode(HTTPException):
1293     pass
1294
1295 class IncompleteRead(HTTPException):
1296     def __init__(self, partial):
1297         self.args = partial,
1298         self.partial = partial
1299
1300 class ImproperConnectionState(HTTPException):
1301     pass
1302
1303 class CannotSendRequest(ImproperConnectionState):
1304     pass
1305
1306 class CannotSendHeader(ImproperConnectionState):
1307     pass
1308
1309 class ResponseNotReady(ImproperConnectionState):
1310     pass
1311
1312 class BadStatusLine(HTTPException):
1313     def __init__(self, line):
1314         self.args = line,
1315         self.line = line
1316
1317 # for backwards compatibility
1318 error = HTTPException
1319
1320 class LineAndFileWrapper:
1321     """A limited file-like object for HTTP/0.9 responses."""
1322
1323     # The status-line parsing code calls readline(), which normally
1324     # get the HTTP status line.  For a 0.9 response, however, this is
1325     # actually the first line of the body!  Clients need to get a
1326     # readable file object that contains that line.
1327
1328     def __init__(self, line, file):
1329         self._line = line
1330         self._file = file
1331         self._line_consumed = 0
1332         self._line_offset = 0
1333         self._line_left = len(line)
1334
1335     def __getattr__(self, attr):
1336         return getattr(self._file, attr)
1337
1338     def _done(self):
1339         # called when the last byte is read from the line.  After the
1340         # call, all read methods are delegated to the underlying file
1341         # object.
1342         self._line_consumed = 1
1343         self.read = self._file.read
1344         self.readline = self._file.readline
1345         self.readlines = self._file.readlines
1346
1347     def read(self, amt=None):
1348         if self._line_consumed:
1349             return self._file.read(amt)
1350         assert self._line_left
1351         if amt is None or amt > self._line_left:
1352             s = self._line[self._line_offset:]
1353             self._done()
1354             if amt is None:
1355                 return s + self._file.read()
1356             else:
1357                 return s + self._file.read(amt - len(s))
1358         else:
1359             assert amt <= self._line_left
1360             i = self._line_offset
1361             j = i + amt
1362             s = self._line[i:j]
1363             self._line_offset = j
1364             self._line_left -= amt
1365             if self._line_left == 0:
1366                 self._done()
1367             return s
1368
1369     def readline(self):
1370         if self._line_consumed:
1371             return self._file.readline()
1372         assert self._line_left
1373         s = self._line[self._line_offset:]
1374         self._done()
1375         return s
1376
1377     def readlines(self, size=None):
1378         if self._line_consumed:
1379             return self._file.readlines(size)
1380         assert self._line_left
1381         L = [self._line[self._line_offset:]]
1382         self._done()
1383         if size is None:
1384             return L + self._file.readlines()
1385         else:
1386             return L + self._file.readlines(size)
1387
1388 def test():
1389     """Test this module.
1390
1391     A hodge podge of tests collected here, because they have too many
1392     external dependencies for the regular test suite.
1393     """
1394
1395     import sys
1396     import getopt
1397     opts, args = getopt.getopt(sys.argv[1:], 'd')
1398     dl = 0
1399     for o, a in opts:
1400         if o == '-d': dl = dl + 1
1401     host = 'www.python.org'
1402     selector = '/'
1403     if args[0:]: host = args[0]
1404     if args[1:]: selector = args[1]
1405     h = HTTP()
1406     h.set_debuglevel(dl)
1407     h.connect(host)
1408     h.putrequest('GET', selector)
1409     h.endheaders()
1410     status, reason, headers = h.getreply()
1411     print 'status =', status
1412     print 'reason =', reason
1413     print "read", len(h.getfile().read())
1414     print
1415     if headers:
1416         for header in headers.headers: print header.strip()
1417     print
1418
1419     # minimal test that code to extract host from url works
1420     class HTTP11(HTTP):
1421         _http_vsn = 11
1422         _http_vsn_str = 'HTTP/1.1'
1423
1424     h = HTTP11('www.python.org')
1425     h.putrequest('GET', 'http://www.python.org/~jeremy/')
1426     h.endheaders()
1427     h.getreply()
1428     h.close()
1429
1430     if hasattr(socket, 'ssl'):
1431
1432         for host, selector in (('sourceforge.net', '/projects/python'),
1433                                ):
1434             print "https://%s%s" % (host, selector)
1435             hs = HTTPS()
1436             hs.set_debuglevel(dl)
1437             hs.connect(host)
1438             hs.putrequest('GET', selector)
1439             hs.endheaders()
1440             status, reason, headers = hs.getreply()
1441             print 'status =', status
1442             print 'reason =', reason
1443             print "read", len(hs.getfile().read())
1444             print
1445             if headers:
1446                 for header in headers.headers: print header.strip()
1447             print
1448
1449 if __name__ == '__main__':
1450     test()