Lib/httplib.py

   1 """HTTP/1.1 client library
   2
   3 <intro stuff goes here>
   4 <other stuff, too>
   5
   6 HTTPConnection goes through a number of "states", which define when a client
   7 may legally make another request or fetch the response for a particular
   8 request. This diagram details these state transitions:
   9
  10     (null)
  11       |
  12       | HTTPConnection()
  13       v
  14     Idle
  15       |
  16       | putrequest()
  17       v
  18     Request-started
  19       |
  20       | ( putheader() )*  endheaders()
  21       v
  22     Request-sent
  23       |
  24       | response = getresponse()
  25       v
  26     Unread-response   [Response-headers-read]
  27       |\____________________
  28       |                     |
  29       | response.read()     | putrequest()
  30       v                     v
  31     Idle                  Req-started-unread-response
  32                      ______/|
  33                    /        |
  34    response.read() |        | ( putheader() )*  endheaders()
  35                    v        v
  36        Request-started    Req-sent-unread-response
  37                             |
  38                             | response.read()
  39                             v
  40                           Request-sent
  41
  42 This diagram presents the following rules:
  43   -- a second request may not be started until {response-headers-read}
  44   -- a response [object] cannot be retrieved until {request-sent}
  45   -- there is no differentiation between an unread response body and a
  46      partially read response body
  47
  48 Note: this enforcement is applied by the HTTPConnection class. The
  49       HTTPResponse class does not enforce this state machine, which
  50       implies sophisticated clients may accelerate the request/response
  51       pipeline. Caution should be taken, though: accelerating the states
  52       beyond the above pattern may imply knowledge of the server's
  53       connection-close behavior for certain requests. For example, it
  54       is impossible to tell whether the server will close the connection
  55       UNTIL the response headers have been read; this means that further
  56       requests cannot be placed into the pipeline until it is known that
  57       the server will NOT be closing the connection.
  58
  59 Logical State                  __state            __response
  60 -------------                  -------            ----------
  61 Idle                           _CS_IDLE           None
  62 Request-started                _CS_REQ_STARTED    None
  63 Request-sent                   _CS_REQ_SENT       None
  64 Unread-response                _CS_IDLE           <response_class>
  65 Req-started-unread-response    _CS_REQ_STARTED    <response_class>
  66 Req-sent-unread-response       _CS_REQ_SENT       <response_class>
  67 """
  68
  69 import errno
  70 import mimetools
  71 import socket
  72 from urlparse import urlsplit
  73
  74 try:
  75     from cStringIO import StringIO
  76 except ImportError:
  77     from StringIO import StringIO
  78
  79 __all__ = ["HTTP", "HTTPResponse", "HTTPConnection", "HTTPSConnection",
  80            "HTTPException", "NotConnected", "UnknownProtocol",
  81            "UnknownTransferEncoding", "UnimplementedFileMode",
  82            "IncompleteRead", "InvalidURL", "ImproperConnectionState",
  83            "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
  84            "BadStatusLine", "error", "responses"]
  85
  86 HTTP_PORT = 80
  87 HTTPS_PORT = 443
  88
  89 _UNKNOWN = 'UNKNOWN'
  90
  91 # connection states
  92 _CS_IDLE = 'Idle'
  93 _CS_REQ_STARTED = 'Request-started'
  94 _CS_REQ_SENT = 'Request-sent'
  95
  96 # status codes
  97 # informational
  98 CONTINUE = 100
  99 SWITCHING_PROTOCOLS = 101
 100 PROCESSING = 102
 101
 102 # successful
 103 OK = 200
 104 CREATED = 201
 105 ACCEPTED = 202
 106 NON_AUTHORITATIVE_INFORMATION = 203
 107 NO_CONTENT = 204
 108 RESET_CONTENT = 205
 109 PARTIAL_CONTENT = 206
 110 MULTI_STATUS = 207
 111 IM_USED = 226
 112
 113 # redirection
 114 MULTIPLE_CHOICES = 300
 115 MOVED_PERMANENTLY = 301
 116 FOUND = 302
 117 SEE_OTHER = 303
 118 NOT_MODIFIED = 304
 119 USE_PROXY = 305
 120 TEMPORARY_REDIRECT = 307
 121
 122 # client error
 123 BAD_REQUEST = 400
 124 UNAUTHORIZED = 401
 125 PAYMENT_REQUIRED = 402
 126 FORBIDDEN = 403
 127 NOT_FOUND = 404
 128 METHOD_NOT_ALLOWED = 405
 129 NOT_ACCEPTABLE = 406
 130 PROXY_AUTHENTICATION_REQUIRED = 407
 131 REQUEST_TIMEOUT = 408
 132 CONFLICT = 409
 133 GONE = 410
 134 LENGTH_REQUIRED = 411
 135 PRECONDITION_FAILED = 412
 136 REQUEST_ENTITY_TOO_LARGE = 413
 137 REQUEST_URI_TOO_LONG = 414
 138 UNSUPPORTED_MEDIA_TYPE = 415
 139 REQUESTED_RANGE_NOT_SATISFIABLE = 416
 140 EXPECTATION_FAILED = 417
 141 UNPROCESSABLE_ENTITY = 422
 142 LOCKED = 423
 143 FAILED_DEPENDENCY = 424
 144 UPGRADE_REQUIRED = 426
 145
 146 # server error
 147 INTERNAL_SERVER_ERROR = 500
 148 NOT_IMPLEMENTED = 501
 149 BAD_GATEWAY = 502
 150 SERVICE_UNAVAILABLE = 503
 151 GATEWAY_TIMEOUT = 504
 152 HTTP_VERSION_NOT_SUPPORTED = 505
 153 INSUFFICIENT_STORAGE = 507
 154 NOT_EXTENDED = 510
 155
 156 # Mapping status codes to official W3C names
 157 responses = {
 158     100: 'Continue',
 159     101: 'Switching Protocols',
 160
 161     200: 'OK',
 162     201: 'Created',
 163     202: 'Accepted',
 164     203: 'Non-Authoritative Information',
 165     204: 'No Content',
 166     205: 'Reset Content',
 167     206: 'Partial Content',
 168
 169     300: 'Multiple Choices',
 170     301: 'Moved Permanently',
 171     302: 'Found',
 172     303: 'See Other',
 173     304: 'Not Modified',
 174     305: 'Use Proxy',
 175     306: '(Unused)',
 176     307: 'Temporary Redirect',
 177
 178     400: 'Bad Request',
 179     401: 'Unauthorized',
 180     402: 'Payment Required',
 181     403: 'Forbidden',
 182     404: 'Not Found',
 183     405: 'Method Not Allowed',
 184     406: 'Not Acceptable',
 185     407: 'Proxy Authentication Required',
 186     408: 'Request Timeout',
 187     409: 'Conflict',
 188     410: 'Gone',
 189     411: 'Length Required',
 190     412: 'Precondition Failed',
 191     413: 'Request Entity Too Large',
 192     414: 'Request-URI Too Long',
 193     415: 'Unsupported Media Type',
 194     416: 'Requested Range Not Satisfiable',
 195     417: 'Expectation Failed',
 196
 197     500: 'Internal Server Error',
 198     501: 'Not Implemented',
 199     502: 'Bad Gateway',
 200     503: 'Service Unavailable',
 201     504: 'Gateway Timeout',
 202     505: 'HTTP Version Not Supported',
 203 }
 204
 205 # maximal amount of data to read at one time in _safe_read
 206 MAXAMOUNT = 1048576
 207
 208 class HTTPMessage(mimetools.Message):
 209
 210     def addheader(self, key, value):
 211         """Add header for field key handling repeats."""
 212         prev = self.dict.get(key)
 213         if prev is None:
 214             self.dict[key] = value
 215         else:
 216             combined = ", ".join((prev, value))
 217             self.dict[key] = combined
 218
 219     def addcontinue(self, key, more):
 220         """Add more field data from a continuation line."""
 221         prev = self.dict[key]
 222         self.dict[key] = prev + "\n " + more
 223
 224     def readheaders(self):
 225         """Read header lines.
 226
 227         Read header lines up to the entirely blank line that terminates them.
 228         The (normally blank) line that ends the headers is skipped, but not
 229         included in the returned list.  If a non-header line ends the headers,
 230         (which is an error), an attempt is made to backspace over it; it is
 231         never included in the returned list.
 232
 233         The variable self.status is set to the empty string if all went well,
 234         otherwise it is an error message.  The variable self.headers is a
 235         completely uninterpreted list of lines contained in the header (so
 236         printing them will reproduce the header exactly as it appears in the
 237         file).
 238
 239         If multiple header fields with the same name occur, they are combined
 240         according to the rules in RFC 2616 sec 4.2:
 241
 242         Appending each subsequent field-value to the first, each separated
 243         by a comma. The order in which header fields with the same field-name
 244         are received is significant to the interpretation of the combined
 245         field value.
 246         """
 247         # XXX The implementation overrides the readheaders() method of
 248         # rfc822.Message.  The base class design isn't amenable to
 249         # customized behavior here so the method here is a copy of the
 250         # base class code with a few small changes.
 251
 252         self.dict = {}
 253         self.unixfrom = ''
 254         self.headers = hlist = []
 255         self.status = ''
 256         headerseen = ""
 257         firstline = 1
 258         startofline = unread = tell = None
 259         if hasattr(self.fp, 'unread'):
 260             unread = self.fp.unread
 261         elif self.seekable:
 262             tell = self.fp.tell
 263         while True:
 264             if tell:
 265                 try:
 266                     startofline = tell()
 267                 except IOError:
 268                     startofline = tell = None
 269                     self.seekable = 0
 270             line = self.fp.readline()
 271             if not line:
 272                 self.status = 'EOF in headers'
 273                 break
 274             # Skip unix From name time lines
 275             if firstline and line.startswith('From '):
 276                 self.unixfrom = self.unixfrom + line
 277                 continue
 278             firstline = 0
 279             if headerseen and line[0] in ' \t':
 280                 # XXX Not sure if continuation lines are handled properly
 281                 # for http and/or for repeating headers
 282                 # It's a continuation line.
 283                 hlist.append(line)
 284                 self.addcontinue(headerseen, line.strip())
 285                 continue
 286             elif self.iscomment(line):
 287                 # It's a comment.  Ignore it.
 288                 continue
 289             elif self.islast(line):
 290                 # Note! No pushback here!  The delimiter line gets eaten.
 291                 break
 292             headerseen = self.isheader(line)
 293             if headerseen:
 294                 # It's a legal header line, save it.
 295                 hlist.append(line)
 296                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
 297                 continue
 298             else:
 299                 # It's not a header line; throw it back and stop here.
 300                 if not self.dict:
 301                     self.status = 'No headers'
 302                 else:
 303                     self.status = 'Non-header line where header expected'
 304                 # Try to undo the read.
 305                 if unread:
 306                     unread(line)
 307                 elif tell:
 308                     self.fp.seek(startofline)
 309                 else:
 310                     self.status = self.status + '; bad seek'
 311                 break
 312
 313 class HTTPResponse:
 314
 315     # strict: If true, raise BadStatusLine if the status line can't be
 316     # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
 317     # false because it prevents clients from talking to HTTP/0.9
 318     # servers.  Note that a response with a sufficiently corrupted
 319     # status line will look like an HTTP/0.9 response.
 320
 321     # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
 322
 323     def __init__(self, sock, debuglevel=0, strict=0, method=None):
 324         self.fp = sock.makefile('rb', 0)
 325         self.debuglevel = debuglevel
 326         self.strict = strict
 327         self._method = method
 328
 329         self.msg = None
 330
 331         # from the Status-Line of the response
 332         self.version = _UNKNOWN # HTTP-Version
 333         self.status = _UNKNOWN  # Status-Code
 334         self.reason = _UNKNOWN  # Reason-Phrase
 335
 336         self.chunked = _UNKNOWN         # is "chunked" being used?
 337         self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
 338         self.length = _UNKNOWN          # number of bytes left in response
 339         self.will_close = _UNKNOWN      # conn will close at end of response
 340
 341     def _read_status(self):
 342         # Initialize with Simple-Response defaults
 343         line = self.fp.readline()
 344         if self.debuglevel > 0:
 345             print "reply:", repr(line)
 346         if not line:
 347             # Presumably, the server closed the connection before
 348             # sending a valid response.
 349             raise BadStatusLine(line)
 350         try:
 351             [version, status, reason] = line.split(None, 2)
 352         except ValueError:
 353             try:
 354                 [version, status] = line.split(None, 1)
 355                 reason = ""
 356             except ValueError:
 357                 # empty version will cause next test to fail and status
 358                 # will be treated as 0.9 response.
 359                 version = ""
 360         if not version.startswith('HTTP/'):
 361             if self.strict:
 362                 self.close()
 363                 raise BadStatusLine(line)
 364             else:
 365                 # assume it's a Simple-Response from an 0.9 server
 366                 self.fp = LineAndFileWrapper(line, self.fp)
 367                 return "HTTP/0.9", 200, ""
 368
 369         # The status code is a three-digit number
 370         try:
 371             status = int(status)
 372             if status < 100 or status > 999:
 373                 raise BadStatusLine(line)
 374         except ValueError:
 375             raise BadStatusLine(line)
 376         return version, status, reason
 377
 378     def begin(self):
 379         if self.msg is not None:
 380             # we've already started reading the response
 381             return
 382
 383         # read until we get a non-100 response
 384         while True:
 385             version, status, reason = self._read_status()
 386             if status != CONTINUE:
 387                 break
 388             # skip the header from the 100 response
 389             while True:
 390                 skip = self.fp.readline().strip()
 391                 if not skip:
 392                     break
 393                 if self.debuglevel > 0:
 394                     print "header:", skip
 395
 396         self.status = status
 397         self.reason = reason.strip()
 398         if version == 'HTTP/1.0':
 399             self.version = 10
 400         elif version.startswith('HTTP/1.'):
 401             self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
 402         elif version == 'HTTP/0.9':
 403             self.version = 9
 404         else:
 405             raise UnknownProtocol(version)
 406
 407         if self.version == 9:
 408             self.length = None
 409             self.chunked = 0
 410             self.will_close = 1
 411             self.msg = HTTPMessage(StringIO())
 412             return
 413
 414         self.msg = HTTPMessage(self.fp, 0)
 415         if self.debuglevel > 0:
 416             for hdr in self.msg.headers:
 417                 print "header:", hdr,
 418
 419         # don't let the msg keep an fp
 420         self.msg.fp = None
 421
 422         # are we using the chunked-style of transfer encoding?
 423         tr_enc = self.msg.getheader('transfer-encoding')
 424         if tr_enc and tr_enc.lower() == "chunked":
 425             self.chunked = 1
 426             self.chunk_left = None
 427         else:
 428             self.chunked = 0
 429
 430         # will the connection close at the end of the response?
 431         self.will_close = self._check_close()
 432
 433         # do we have a Content-Length?
 434         # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
 435         length = self.msg.getheader('content-length')
 436         if length and not self.chunked:
 437             try:
 438                 self.length = int(length)
 439             except ValueError:
 440                 self.length = None
 441         else:
 442             self.length = None
 443
 444         # does the body have a fixed length? (of zero)
 445         if (status == NO_CONTENT or status == NOT_MODIFIED or
 446             100 <= status < 200 or      # 1xx codes
 447             self._method == 'HEAD'):
 448             self.length = 0
 449
 450         # if the connection remains open, and we aren't using chunked, and
 451         # a content-length was not provided, then assume that the connection
 452         # WILL close.
 453         if not self.will_close and \
 454            not self.chunked and \
 455            self.length is None:
 456             self.will_close = 1
 457
 458     def _check_close(self):
 459         conn = self.msg.getheader('connection')
 460         if self.version == 11:
 461             # An HTTP/1.1 proxy is assumed to stay open unless
 462             # explicitly closed.
 463             conn = self.msg.getheader('connection')
 464             if conn and "close" in conn.lower():
 465                 return True
 466             return False
 467
 468         # Some HTTP/1.0 implementations have support for persistent
 469         # connections, using rules different than HTTP/1.1.
 470
 471         # For older HTTP, Keep-Alive indiciates persistent connection.
 472         if self.msg.getheader('keep-alive'):
 473             return False
 474
 475         # At least Akamai returns a "Connection: Keep-Alive" header,
 476         # which was supposed to be sent by the client.
 477         if conn and "keep-alive" in conn.lower():
 478             return False
 479
 480         # Proxy-Connection is a netscape hack.
 481         pconn = self.msg.getheader('proxy-connection')
 482         if pconn and "keep-alive" in pconn.lower():
 483             return False
 484
 485         # otherwise, assume it will close
 486         return True
 487
 488     def close(self):
 489         if self.fp:
 490             self.fp.close()
 491             self.fp = None
 492
 493     def isclosed(self):
 494         # NOTE: it is possible that we will not ever call self.close(). This
 495         #       case occurs when will_close is TRUE, length is None, and we
 496         #       read up to the last byte, but NOT past it.
 497         #
 498         # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
 499         #          called, meaning self.isclosed() is meaningful.
 500         return self.fp is None
 501
 502     # XXX It would be nice to have readline and __iter__ for this, too.
 503
 504     def read(self, amt=None):
 505         if self.fp is None:
 506             return ''
 507
 508         if self.chunked:
 509             return self._read_chunked(amt)
 510
 511         if amt is None:
 512             # unbounded read
 513             if self.length is None:
 514                 s = self.fp.read()
 515             else:
 516                 s = self._safe_read(self.length)
 517                 self.length = 0
 518             self.close()        # we read everything
 519             return s
 520
 521         if self.length is not None:
 522             if amt > self.length:
 523                 # clip the read to the "end of response"
 524                 amt = self.length
 525
 526         # we do not use _safe_read() here because this may be a .will_close
 527         # connection, and the user is reading more bytes than will be provided
 528         # (for example, reading in 1k chunks)
 529         s = self.fp.read(amt)
 530         if self.length is not None:
 531             self.length -= len(s)
 532
 533         return s
 534
 535     def _read_chunked(self, amt):
 536         assert self.chunked != _UNKNOWN
 537         chunk_left = self.chunk_left
 538         value = ''
 539
 540         # XXX This accumulates chunks by repeated string concatenation,
 541         # which is not efficient as the number or size of chunks gets big.
 542         while True:
 543             if chunk_left is None:
 544                 line = self.fp.readline()
 545                 i = line.find(';')
 546                 if i >= 0:
 547                     line = line[:i] # strip chunk-extensions
 548                 chunk_left = int(line, 16)
 549                 if chunk_left == 0:
 550                     break
 551             if amt is None:
 552                 value += self._safe_read(chunk_left)
 553             elif amt < chunk_left:
 554                 value += self._safe_read(amt)
 555                 self.chunk_left = chunk_left - amt
 556                 return value
 557             elif amt == chunk_left:
 558                 value += self._safe_read(amt)
 559                 self._safe_read(2)  # toss the CRLF at the end of the chunk
 560                 self.chunk_left = None
 561                 return value
 562             else:
 563                 value += self._safe_read(chunk_left)
 564                 amt -= chunk_left
 565
 566             # we read the whole chunk, get another
 567             self._safe_read(2)      # toss the CRLF at the end of the chunk
 568             chunk_left = None
 569
 570         # read and discard trailer up to the CRLF terminator
 571         ### note: we shouldn't have any trailers!
 572         while True:
 573             line = self.fp.readline()
 574             if line == '\r\n':
 575                 break
 576
 577         # we read everything; close the "file"
 578         self.close()
 579
 580         return value
 581
 582     def _safe_read(self, amt):
 583         """Read the number of bytes requested, compensating for partial reads.
 584
 585         Normally, we have a blocking socket, but a read() can be interrupted
 586         by a signal (resulting in a partial read).
 587
 588         Note that we cannot distinguish between EOF and an interrupt when zero
 589         bytes have been read. IncompleteRead() will be raised in this
 590         situation.
 591
 592         This function should be used when <amt> bytes "should" be present for
 593         reading. If the bytes are truly not available (due to EOF), then the
 594         IncompleteRead exception can be used to detect the problem.
 595         """
 596         s = []
 597         while amt > 0:
 598             chunk = self.fp.read(min(amt, MAXAMOUNT))
 599             if not chunk:
 600                 raise IncompleteRead(s)
 601             s.append(chunk)
 602             amt -= len(chunk)
 603         return ''.join(s)
 604
 605     def getheader(self, name, default=None):
 606         if self.msg is None:
 607             raise ResponseNotReady()
 608         return self.msg.getheader(name, default)
 609
 610     def getheaders(self):
 611         """Return list of (header, value) tuples."""
 612         if self.msg is None:
 613             raise ResponseNotReady()
 614         return self.msg.items()
 615
 616
 617 class HTTPConnection:
 618
 619     _http_vsn = 11
 620     _http_vsn_str = 'HTTP/1.1'
 621
 622     response_class = HTTPResponse
 623     default_port = HTTP_PORT
 624     auto_open = 1
 625     debuglevel = 0
 626     strict = 0
 627
 628     def __init__(self, host, port=None, strict=None, timeout=None):
 629         self.timeout = timeout
 630         self.sock = None
 631         self._buffer = []
 632         self.__response = None
 633         self.__state = _CS_IDLE
 634         self._method = None
 635
 636         self._set_hostport(host, port)
 637         if strict is not None:
 638             self.strict = strict
 639
 640     def _set_hostport(self, host, port):
 641         if port is None:
 642             i = host.rfind(':')
 643             j = host.rfind(']')         # ipv6 addresses have [...]
 644             if i > j:
 645                 try:
 646                     port = int(host[i+1:])
 647                 except ValueError:
 648                     raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
 649                 host = host[:i]
 650             else:
 651                 port = self.default_port
 652             if host and host[0] == '[' and host[-1] == ']':
 653                 host = host[1:-1]
 654         self.host = host
 655         self.port = port
 656
 657     def set_debuglevel(self, level):
 658         self.debuglevel = level
 659
 660     def connect(self):
 661         """Connect to the host and port specified in __init__."""
 662         self.sock = socket.create_connection((self.host,self.port),
 663                                              self.timeout)
 664
 665     def close(self):
 666         """Close the connection to the HTTP server."""
 667         if self.sock:
 668             self.sock.close()   # close it manually... there may be other refs
 669             self.sock = None
 670         if self.__response:
 671             self.__response.close()
 672             self.__response = None
 673         self.__state = _CS_IDLE
 674
 675     def send(self, str):
 676         """Send `str' to the server."""
 677         if self.sock is None:
 678             if self.auto_open:
 679                 self.connect()
 680             else:
 681                 raise NotConnected()
 682
 683         # send the data to the server. if we get a broken pipe, then close
 684         # the socket. we want to reconnect when somebody tries to send again.
 685         #
 686         # NOTE: we DO propagate the error, though, because we cannot simply
 687         #       ignore the error... the caller will know if they can retry.
 688         if self.debuglevel > 0:
 689             print "send:", repr(str)
 690         try:
 691             blocksize=8192
 692             if hasattr(str,'read') :
 693                 if self.debuglevel > 0: print "sendIng a read()able"
 694                 data=str.read(blocksize)
 695                 while data:
 696                     self.sock.sendall(data)
 697                     data=str.read(blocksize)
 698             else:
 699                 self.sock.sendall(str)
 700         except socket.error, v:
 701             if v[0] == 32:      # Broken pipe
 702                 self.close()
 703             raise
 704
 705     def _output(self, s):
 706         """Add a line of output to the current request buffer.
 707
 708         Assumes that the line does *not* end with \\r\\n.
 709         """
 710         self._buffer.append(s)
 711
 712     def _send_output(self):
 713         """Send the currently buffered request and clear the buffer.
 714
 715         Appends an extra \\r\\n to the buffer.
 716         """
 717         self._buffer.extend(("", ""))
 718         msg = "\r\n".join(self._buffer)
 719         del self._buffer[:]
 720         self.send(msg)
 721
 722     def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
 723         """Send a request to the server.
 724
 725         `method' specifies an HTTP request method, e.g. 'GET'.
 726         `url' specifies the object being requested, e.g. '/index.html'.
 727         `skip_host' if True does not add automatically a 'Host:' header
 728         `skip_accept_encoding' if True does not add automatically an
 729            'Accept-Encoding:' header
 730         """
 731
 732         # if a prior response has been completed, then forget about it.
 733         if self.__response and self.__response.isclosed():
 734             self.__response = None
 735
 736
 737         # in certain cases, we cannot issue another request on this connection.
 738         # this occurs when:
 739         #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
 740         #   2) a response to a previous request has signalled that it is going
 741         #      to close the connection upon completion.
 742         #   3) the headers for the previous response have not been read, thus
 743         #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
 744         #
 745         # if there is no prior response, then we can request at will.
 746         #
 747         # if point (2) is true, then we will have passed the socket to the
 748         # response (effectively meaning, "there is no prior response"), and
 749         # will open a new one when a new request is made.
 750         #
 751         # Note: if a prior response exists, then we *can* start a new request.
 752         #       We are not allowed to begin fetching the response to this new
 753         #       request, however, until that prior response is complete.
 754         #
 755         if self.__state == _CS_IDLE:
 756             self.__state = _CS_REQ_STARTED
 757         else:
 758             raise CannotSendRequest()
 759
 760         # Save the method we use, we need it later in the response phase
 761         self._method = method
 762         if not url:
 763             url = '/'
 764         str = '%s %s %s' % (method, url, self._http_vsn_str)
 765
 766         self._output(str)
 767
 768         if self._http_vsn == 11:
 769             # Issue some standard headers for better HTTP/1.1 compliance
 770
 771             if not skip_host:
 772                 # this header is issued *only* for HTTP/1.1
 773                 # connections. more specifically, this means it is
 774                 # only issued when the client uses the new
 775                 # HTTPConnection() class. backwards-compat clients
 776                 # will be using HTTP/1.0 and those clients may be
 777                 # issuing this header themselves. we should NOT issue
 778                 # it twice; some web servers (such as Apache) barf
 779                 # when they see two Host: headers
 780
 781                 # If we need a non-standard port,include it in the
 782                 # header.  If the request is going through a proxy,
 783                 # but the host of the actual URL, not the host of the
 784                 # proxy.
 785
 786                 netloc = ''
 787                 if url.startswith('http'):
 788                     nil, netloc, nil, nil, nil = urlsplit(url)
 789
 790                 if netloc:
 791                     try:
 792                         netloc_enc = netloc.encode("ascii")
 793                     except UnicodeEncodeError:
 794                         netloc_enc = netloc.encode("idna")
 795                     self.putheader('Host', netloc_enc)
 796                 else:
 797                     try:
 798                         host_enc = self.host.encode("ascii")
 799                     except UnicodeEncodeError:
 800                         host_enc = self.host.encode("idna")
 801                     if self.port == HTTP_PORT:
 802                         self.putheader('Host', host_enc)
 803                     else:
 804                         self.putheader('Host', "%s:%s" % (host_enc, self.port))
 805
 806             # note: we are assuming that clients will not attempt to set these
 807             #       headers since *this* library must deal with the
 808             #       consequences. this also means that when the supporting
 809             #       libraries are updated to recognize other forms, then this
 810             #       code should be changed (removed or updated).
 811
 812             # we only want a Content-Encoding of "identity" since we don't
 813             # support encodings such as x-gzip or x-deflate.
 814             if not skip_accept_encoding:
 815                 self.putheader('Accept-Encoding', 'identity')
 816
 817             # we can accept "chunked" Transfer-Encodings, but no others
 818             # NOTE: no TE header implies *only* "chunked"
 819             #self.putheader('TE', 'chunked')
 820
 821             # if TE is supplied in the header, then it must appear in a
 822             # Connection header.
 823             #self.putheader('Connection', 'TE')
 824
 825         else:
 826             # For HTTP/1.0, the server will assume "not chunked"
 827             pass
 828
 829     def putheader(self, header, value):
 830         """Send a request header line to the server.
 831
 832         For example: h.putheader('Accept', 'text/html')
 833         """
 834         if self.__state != _CS_REQ_STARTED:
 835             raise CannotSendHeader()
 836
 837         str = '%s: %s' % (header, value)
 838         self._output(str)
 839
 840     def endheaders(self):
 841         """Indicate that the last header line has been sent to the server."""
 842
 843         if self.__state == _CS_REQ_STARTED:
 844             self.__state = _CS_REQ_SENT
 845         else:
 846             raise CannotSendHeader()
 847
 848         self._send_output()
 849
 850     def request(self, method, url, body=None, headers={}):
 851         """Send a complete request to the server."""
 852
 853         try:
 854             self._send_request(method, url, body, headers)
 855         except socket.error, v:
 856             # trap 'Broken pipe' if we're allowed to automatically reconnect
 857             if v[0] != 32 or not self.auto_open:
 858                 raise
 859             # try one more time
 860             self._send_request(method, url, body, headers)
 861
 862     def _send_request(self, method, url, body, headers):
 863         # honour explicitly requested Host: and Accept-Encoding headers
 864         header_names = dict.fromkeys([k.lower() for k in headers])
 865         skips = {}
 866         if 'host' in header_names:
 867             skips['skip_host'] = 1
 868         if 'accept-encoding' in header_names:
 869             skips['skip_accept_encoding'] = 1
 870
 871         self.putrequest(method, url, **skips)
 872
 873         if body and ('content-length' not in header_names):
 874             thelen=None
 875             try:
 876                 thelen=str(len(body))
 877             except TypeError, te:
 878                 # If this is a file-like object, try to
 879                 # fstat its file descriptor
 880                 import os
 881                 try:
 882                     thelen = str(os.fstat(body.fileno()).st_size)
 883                 except (AttributeError, OSError):
 884                     # Don't send a length if this failed
 885                     if self.debuglevel > 0: print "Cannot stat!!"
 886
 887             if thelen is not None:
 888                 self.putheader('Content-Length',thelen)
 889         for hdr, value in headers.iteritems():
 890             self.putheader(hdr, value)
 891         self.endheaders()
 892
 893         if body:
 894             self.send(body)
 895
 896     def getresponse(self):
 897         "Get the response from the server."
 898
 899         # if a prior response has been completed, then forget about it.
 900         if self.__response and self.__response.isclosed():
 901             self.__response = None
 902
 903         #
 904         # if a prior response exists, then it must be completed (otherwise, we
 905         # cannot read this response's header to determine the connection-close
 906         # behavior)
 907         #
 908         # note: if a prior response existed, but was connection-close, then the
 909         # socket and response were made independent of this HTTPConnection
 910         # object since a new request requires that we open a whole new
 911         # connection
 912         #
 913         # this means the prior response had one of two states:
 914         #   1) will_close: this connection was reset and the prior socket and
 915         #                  response operate independently
 916         #   2) persistent: the response was retained and we await its
 917         #                  isclosed() status to become true.
 918         #
 919         if self.__state != _CS_REQ_SENT or self.__response:
 920             raise ResponseNotReady()
 921
 922         if self.debuglevel > 0:
 923             response = self.response_class(self.sock, self.debuglevel,
 924                                            strict=self.strict,
 925                                            method=self._method)
 926         else:
 927             response = self.response_class(self.sock, strict=self.strict,
 928                                            method=self._method)
 929
 930         response.begin()
 931         assert response.will_close != _UNKNOWN
 932         self.__state = _CS_IDLE
 933
 934         if response.will_close:
 935             # this effectively passes the connection to the response
 936             self.close()
 937         else:
 938             # remember this, so we can tell when it is complete
 939             self.__response = response
 940
 941         return response
 942
 943 # The next several classes are used to define FakeSocket, a socket-like
 944 # interface to an SSL connection.
 945
 946 # The primary complexity comes from faking a makefile() method.  The
 947 # standard socket makefile() implementation calls dup() on the socket
 948 # file descriptor.  As a consequence, clients can call close() on the
 949 # parent socket and its makefile children in any order.  The underlying
 950 # socket isn't closed until they are all closed.
 951
 952 # The implementation uses reference counting to keep the socket open
 953 # until the last client calls close().  SharedSocket keeps track of
 954 # the reference counting and SharedSocketClient provides an constructor
 955 # and close() method that call incref() and decref() correctly.
 956
 957 class SharedSocket:
 958
 959     def __init__(self, sock):
 960         self.sock = sock
 961         self._refcnt = 0
 962
 963     def incref(self):
 964         self._refcnt += 1
 965
 966     def decref(self):
 967         self._refcnt -= 1
 968         assert self._refcnt >= 0
 969         if self._refcnt == 0:
 970             self.sock.close()
 971
 972     def __del__(self):
 973         self.sock.close()
 974
 975 class SharedSocketClient:
 976
 977     def __init__(self, shared):
 978         self._closed = 0
 979         self._shared = shared
 980         self._shared.incref()
 981         self._sock = shared.sock
 982
 983     def close(self):
 984         if not self._closed:
 985             self._shared.decref()
 986             self._closed = 1
 987             self._shared = None
 988
 989 class SSLFile(SharedSocketClient):
 990     """File-like object wrapping an SSL socket."""
 991
 992     BUFSIZE = 8192
 993
 994     def __init__(self, sock, ssl, bufsize=None):
 995         SharedSocketClient.__init__(self, sock)
 996         self._ssl = ssl
 997         self._buf = ''
 998         self._bufsize = bufsize or self.__class__.BUFSIZE
 999
1000     def _read(self):
1001         buf = ''
1002         # put in a loop so that we retry on transient errors
1003         while True:
1004             try:
1005                 buf = self._ssl.read(self._bufsize)
1006             except socket.sslerror, err:
1007                 if (err[0] == socket.SSL_ERROR_WANT_READ
1008                     or err[0] == socket.SSL_ERROR_WANT_WRITE):
1009                     continue
1010                 if (err[0] == socket.SSL_ERROR_ZERO_RETURN
1011                     or err[0] == socket.SSL_ERROR_EOF):
1012                     break
1013                 raise
1014             except socket.error, err:
1015                 if err[0] == errno.EINTR:
1016                     continue
1017                 if err[0] == errno.EBADF:
1018                     # XXX socket was closed?
1019                     break
1020                 raise
1021             else:
1022                 break
1023         return buf
1024
1025     def read(self, size=None):
1026         L = [self._buf]
1027         avail = len(self._buf)
1028         while size is None or avail < size:
1029             s = self._read()
1030             if s == '':
1031                 break
1032             L.append(s)
1033             avail += len(s)
1034         all = "".join(L)
1035         if size is None:
1036             self._buf = ''
1037             return all
1038         else:
1039             self._buf = all[size:]
1040             return all[:size]
1041
1042     def readline(self):
1043         L = [self._buf]
1044         self._buf = ''
1045         while 1:
1046             i = L[-1].find("\n")
1047             if i >= 0:
1048                 break
1049             s = self._read()
1050             if s == '':
1051                 break
1052             L.append(s)
1053         if i == -1:
1054             # loop exited because there is no more data
1055             return "".join(L)
1056         else:
1057             all = "".join(L)
1058             # XXX could do enough bookkeeping not to do a 2nd search
1059             i = all.find("\n") + 1
1060             line = all[:i]
1061             self._buf = all[i:]
1062             return line
1063
1064     def readlines(self, sizehint=0):
1065         total = 0
1066         list = []
1067         while True:
1068             line = self.readline()
1069             if not line:
1070                 break
1071             list.append(line)
1072             total += len(line)
1073             if sizehint and total >= sizehint:
1074                 break
1075         return list
1076
1077     def fileno(self):
1078         return self._sock.fileno()
1079
1080     def __iter__(self):
1081         return self
1082
1083     def next(self):
1084         line = self.readline()
1085         if not line:
1086             raise StopIteration
1087         return line
1088
1089 class FakeSocket(SharedSocketClient):
1090
1091     class _closedsocket:
1092         def __getattr__(self, name):
1093             raise error(9, 'Bad file descriptor')
1094
1095     def __init__(self, sock, ssl):
1096         sock = SharedSocket(sock)
1097         SharedSocketClient.__init__(self, sock)
1098         self._ssl = ssl
1099
1100     def close(self):
1101         SharedSocketClient.close(self)
1102         self._sock = self.__class__._closedsocket()
1103
1104     def makefile(self, mode, bufsize=None):
1105         if mode != 'r' and mode != 'rb':
1106             raise UnimplementedFileMode()
1107         return SSLFile(self._shared, self._ssl, bufsize)
1108
1109     def send(self, stuff, flags = 0):
1110         return self._ssl.write(stuff)
1111
1112     sendall = send
1113
1114     def recv(self, len = 1024, flags = 0):
1115         return self._ssl.read(len)
1116
1117     def __getattr__(self, attr):
1118         return getattr(self._sock, attr)
1119
1120
1121 class HTTPSConnection(HTTPConnection):
1122     "This class allows communication via SSL."
1123
1124     default_port = HTTPS_PORT
1125
1126     def __init__(self, host, port=None, key_file=None, cert_file=None,
1127                  strict=None):
1128         HTTPConnection.__init__(self, host, port, strict)
1129         self.key_file = key_file
1130         self.cert_file = cert_file
1131
1132     def connect(self):
1133         "Connect to a host on a given (SSL) port."
1134
1135         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1136         sock.connect((self.host, self.port))
1137         ssl = socket.ssl(sock, self.key_file, self.cert_file)
1138         self.sock = FakeSocket(sock, ssl)
1139
1140
1141 class HTTP:
1142     "Compatibility class with httplib.py from 1.5."
1143
1144     _http_vsn = 10
1145     _http_vsn_str = 'HTTP/1.0'
1146
1147     debuglevel = 0
1148
1149     _connection_class = HTTPConnection
1150
1151     def __init__(self, host='', port=None, strict=None):
1152         "Provide a default host, since the superclass requires one."
1153
1154         # some joker passed 0 explicitly, meaning default port
1155         if port == 0:
1156             port = None
1157
1158         # Note that we may pass an empty string as the host; this will throw
1159         # an error when we attempt to connect. Presumably, the client code
1160         # will call connect before then, with a proper host.
1161         self._setup(self._connection_class(host, port, strict))
1162
1163     def _setup(self, conn):
1164         self._conn = conn
1165
1166         # set up delegation to flesh out interface
1167         self.send = conn.send
1168         self.putrequest = conn.putrequest
1169         self.endheaders = conn.endheaders
1170         self.set_debuglevel = conn.set_debuglevel
1171
1172         conn._http_vsn = self._http_vsn
1173         conn._http_vsn_str = self._http_vsn_str
1174
1175         self.file = None
1176
1177     def connect(self, host=None, port=None):
1178         "Accept arguments to set the host/port, since the superclass doesn't."
1179
1180         if host is not None:
1181             self._conn._set_hostport(host, port)
1182         self._conn.connect()
1183
1184     def getfile(self):
1185         "Provide a getfile, since the superclass' does not use this concept."
1186         return self.file
1187
1188     def putheader(self, header, *values):
1189         "The superclass allows only one value argument."
1190         self._conn.putheader(header, '\r\n\t'.join(values))
1191
1192     def getreply(self):
1193         """Compat definition since superclass does not define it.
1194
1195         Returns a tuple consisting of:
1196         - server status code (e.g. '200' if all goes well)
1197         - server "reason" corresponding to status code
1198         - any RFC822 headers in the response from the server
1199         """
1200         try:
1201             response = self._conn.getresponse()
1202         except BadStatusLine, e:
1203             ### hmm. if getresponse() ever closes the socket on a bad request,
1204             ### then we are going to have problems with self.sock
1205
1206             ### should we keep this behavior? do people use it?
1207             # keep the socket open (as a file), and return it
1208             self.file = self._conn.sock.makefile('rb', 0)
1209
1210             # close our socket -- we want to restart after any protocol error
1211             self.close()
1212
1213             self.headers = None
1214             return -1, e.line, None
1215
1216         self.headers = response.msg
1217         self.file = response.fp
1218         return response.status, response.reason, response.msg
1219
1220     def close(self):
1221         self._conn.close()
1222
1223         # note that self.file == response.fp, which gets closed by the
1224         # superclass. just clear the object ref here.
1225         ### hmm. messy. if status==-1, then self.file is owned by us.
1226         ### well... we aren't explicitly closing, but losing this ref will
1227         ### do it
1228         self.file = None
1229
1230 if hasattr(socket, 'ssl'):
1231     class HTTPS(HTTP):
1232         """Compatibility with 1.5 httplib interface
1233
1234         Python 1.5.2 did not have an HTTPS class, but it defined an
1235         interface for sending http requests that is also useful for
1236         https.
1237         """
1238
1239         _connection_class = HTTPSConnection
1240
1241         def __init__(self, host='', port=None, key_file=None, cert_file=None,
1242                      strict=None):
1243             # provide a default host, pass the X509 cert info
1244
1245             # urf. compensate for bad input.
1246             if port == 0:
1247                 port = None
1248             self._setup(self._connection_class(host, port, key_file,
1249                                                cert_file, strict))
1250
1251             # we never actually use these for anything, but we keep them
1252             # here for compatibility with post-1.5.2 CVS.
1253             self.key_file = key_file
1254             self.cert_file = cert_file
1255
1256
1257 class HTTPException(Exception):
1258     # Subclasses that define an __init__ must call Exception.__init__
1259     # or define self.args.  Otherwise, str() will fail.
1260     pass
1261
1262 class NotConnected(HTTPException):
1263     pass
1264
1265 class InvalidURL(HTTPException):
1266     pass
1267
1268 class UnknownProtocol(HTTPException):
1269     def __init__(self, version):
1270         self.args = version,
1271         self.version = version
1272
1273 class UnknownTransferEncoding(HTTPException):
1274     pass
1275
1276 class UnimplementedFileMode(HTTPException):
1277     pass
1278
1279 class IncompleteRead(HTTPException):
1280     def __init__(self, partial):
1281         self.args = partial,
1282         self.partial = partial
1283
1284 class ImproperConnectionState(HTTPException):
1285     pass
1286
1287 class CannotSendRequest(ImproperConnectionState):
1288     pass
1289
1290 class CannotSendHeader(ImproperConnectionState):
1291     pass
1292
1293 class ResponseNotReady(ImproperConnectionState):
1294     pass
1295
1296 class BadStatusLine(HTTPException):
1297     def __init__(self, line):
1298         self.args = line,
1299         self.line = line
1300
1301 # for backwards compatibility
1302 error = HTTPException
1303
1304 class LineAndFileWrapper:
1305     """A limited file-like object for HTTP/0.9 responses."""
1306
1307     # The status-line parsing code calls readline(), which normally
1308     # get the HTTP status line.  For a 0.9 response, however, this is
1309     # actually the first line of the body!  Clients need to get a
1310     # readable file object that contains that line.
1311
1312     def __init__(self, line, file):
1313         self._line = line
1314         self._file = file
1315         self._line_consumed = 0
1316         self._line_offset = 0
1317         self._line_left = len(line)
1318
1319     def __getattr__(self, attr):
1320         return getattr(self._file, attr)
1321
1322     def _done(self):
1323         # called when the last byte is read from the line.  After the
1324         # call, all read methods are delegated to the underlying file
1325         # object.
1326         self._line_consumed = 1
1327         self.read = self._file.read
1328         self.readline = self._file.readline
1329         self.readlines = self._file.readlines
1330
1331     def read(self, amt=None):
1332         if self._line_consumed:
1333             return self._file.read(amt)
1334         assert self._line_left
1335         if amt is None or amt > self._line_left:
1336             s = self._line[self._line_offset:]
1337             self._done()
1338             if amt is None:
1339                 return s + self._file.read()
1340             else:
1341                 return s + self._file.read(amt - len(s))
1342         else:
1343             assert amt <= self._line_left
1344             i = self._line_offset
1345             j = i + amt
1346             s = self._line[i:j]
1347             self._line_offset = j
1348             self._line_left -= amt
1349             if self._line_left == 0:
1350                 self._done()
1351             return s
1352
1353     def readline(self):
1354         if self._line_consumed:
1355             return self._file.readline()
1356         assert self._line_left
1357         s = self._line[self._line_offset:]
1358         self._done()
1359         return s
1360
1361     def readlines(self, size=None):
1362         if self._line_consumed:
1363             return self._file.readlines(size)
1364         assert self._line_left
1365         L = [self._line[self._line_offset:]]
1366         self._done()
1367         if size is None:
1368             return L + self._file.readlines()
1369         else:
1370             return L + self._file.readlines(size)
1371
1372 def test():
1373     """Test this module.
1374
1375     A hodge podge of tests collected here, because they have too many
1376     external dependencies for the regular test suite.
1377     """
1378
1379     import sys
1380     import getopt
1381     opts, args = getopt.getopt(sys.argv[1:], 'd')
1382     dl = 0
1383     for o, a in opts:
1384         if o == '-d': dl = dl + 1
1385     host = 'www.python.org'
1386     selector = '/'
1387     if args[0:]: host = args[0]
1388     if args[1:]: selector = args[1]
1389     h = HTTP()
1390     h.set_debuglevel(dl)
1391     h.connect(host)
1392     h.putrequest('GET', selector)
1393     h.endheaders()
1394     status, reason, headers = h.getreply()
1395     print 'status =', status
1396     print 'reason =', reason
1397     print "read", len(h.getfile().read())
1398     print
1399     if headers:
1400         for header in headers.headers: print header.strip()
1401     print
1402
1403     # minimal test that code to extract host from url works
1404     class HTTP11(HTTP):
1405         _http_vsn = 11
1406         _http_vsn_str = 'HTTP/1.1'
1407
1408     h = HTTP11('www.python.org')
1409     h.putrequest('GET', 'http://www.python.org/~jeremy/')
1410     h.endheaders()
1411     h.getreply()
1412     h.close()
1413
1414     if hasattr(socket, 'ssl'):
1415
1416         for host, selector in (('sourceforge.net', '/projects/python'),
1417                                ):
1418             print "https://%s%s" % (host, selector)
1419             hs = HTTPS()
1420             hs.set_debuglevel(dl)
1421             hs.connect(host)
1422             hs.putrequest('GET', selector)
1423             hs.endheaders()
1424             status, reason, headers = hs.getreply()
1425             print 'status =', status
1426             print 'reason =', reason
1427             print "read", len(hs.getfile().read())
1428             print
1429             if headers:
1430                 for header in headers.headers: print header.strip()
1431             print
1432
1433 if __name__ == '__main__':
1434     test()