Lib/httplib.py

   1 """HTTP/1.1 client library
   2
   3 <intro stuff goes here>
   4 <other stuff, too>
   5
   6 HTTPConnection go through a number of "states", which defines when a client
   7 may legally make another request or fetch the response for a particular
   8 request. This diagram details these state transitions:
   9
  10     (null)
  11       |
  12       | HTTPConnection()
  13       v
  14     Idle
  15       |
  16       | putrequest()
  17       v
  18     Request-started
  19       |
  20       | ( putheader() )*  endheaders()
  21       v
  22     Request-sent
  23       |
  24       | response = getresponse()
  25       v
  26     Unread-response   [Response-headers-read]
  27       |\____________________
  28       |                     |
  29       | response.read()     | putrequest()
  30       v                     v
  31     Idle                  Req-started-unread-response
  32                      ______/|
  33                    /        |
  34    response.read() |        | ( putheader() )*  endheaders()
  35                    v        v
  36        Request-started    Req-sent-unread-response
  37                             |
  38                             | response.read()
  39                             v
  40                           Request-sent
  41
  42 This diagram presents the following rules:
  43   -- a second request may not be started until {response-headers-read}
  44   -- a response [object] cannot be retrieved until {request-sent}
  45   -- there is no differentiation between an unread response body and a
  46      partially read response body
  47
  48 Note: this enforcement is applied by the HTTPConnection class. The
  49       HTTPResponse class does not enforce this state machine, which
  50       implies sophisticated clients may accelerate the request/response
  51       pipeline. Caution should be taken, though: accelerating the states
  52       beyond the above pattern may imply knowledge of the server's
  53       connection-close behavior for certain requests. For example, it
  54       is impossible to tell whether the server will close the connection
  55       UNTIL the response headers have been read; this means that further
  56       requests cannot be placed into the pipeline until it is known that
  57       the server will NOT be closing the connection.
  58
  59 Logical State                  __state            __response
  60 -------------                  -------            ----------
  61 Idle                           _CS_IDLE           None
  62 Request-started                _CS_REQ_STARTED    None
  63 Request-sent                   _CS_REQ_SENT       None
  64 Unread-response                _CS_IDLE           <response_class>
  65 Req-started-unread-response    _CS_REQ_STARTED    <response_class>
  66 Req-sent-unread-response       _CS_REQ_SENT       <response_class>
  67 """
  68
  69 import errno
  70 import mimetools
  71 import socket
  72 from urlparse import urlsplit
  73
  74 try:
  75     from cStringIO import StringIO
  76 except ImportError:
  77     from StringIO import StringIO
  78
  79 __all__ = ["HTTP", "HTTPResponse", "HTTPConnection", "HTTPSConnection",
  80            "HTTPException", "NotConnected", "UnknownProtocol",
  81            "UnknownTransferEncoding", "UnimplementedFileMode",
  82            "IncompleteRead", "InvalidURL", "ImproperConnectionState",
  83            "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
  84            "BadStatusLine", "error", "responses"]
  85
  86 HTTP_PORT = 80
  87 HTTPS_PORT = 443
  88
  89 _UNKNOWN = 'UNKNOWN'
  90
  91 # connection states
  92 _CS_IDLE = 'Idle'
  93 _CS_REQ_STARTED = 'Request-started'
  94 _CS_REQ_SENT = 'Request-sent'
  95
  96 # status codes
  97 # informational
  98 CONTINUE = 100
  99 SWITCHING_PROTOCOLS = 101
 100 PROCESSING = 102
 101
 102 # successful
 103 OK = 200
 104 CREATED = 201
 105 ACCEPTED = 202
 106 NON_AUTHORITATIVE_INFORMATION = 203
 107 NO_CONTENT = 204
 108 RESET_CONTENT = 205
 109 PARTIAL_CONTENT = 206
 110 MULTI_STATUS = 207
 111 IM_USED = 226
 112
 113 # redirection
 114 MULTIPLE_CHOICES = 300
 115 MOVED_PERMANENTLY = 301
 116 FOUND = 302
 117 SEE_OTHER = 303
 118 NOT_MODIFIED = 304
 119 USE_PROXY = 305
 120 TEMPORARY_REDIRECT = 307
 121
 122 # client error
 123 BAD_REQUEST = 400
 124 UNAUTHORIZED = 401
 125 PAYMENT_REQUIRED = 402
 126 FORBIDDEN = 403
 127 NOT_FOUND = 404
 128 METHOD_NOT_ALLOWED = 405
 129 NOT_ACCEPTABLE = 406
 130 PROXY_AUTHENTICATION_REQUIRED = 407
 131 REQUEST_TIMEOUT = 408
 132 CONFLICT = 409
 133 GONE = 410
 134 LENGTH_REQUIRED = 411
 135 PRECONDITION_FAILED = 412
 136 REQUEST_ENTITY_TOO_LARGE = 413
 137 REQUEST_URI_TOO_LONG = 414
 138 UNSUPPORTED_MEDIA_TYPE = 415
 139 REQUESTED_RANGE_NOT_SATISFIABLE = 416
 140 EXPECTATION_FAILED = 417
 141 UNPROCESSABLE_ENTITY = 422
 142 LOCKED = 423
 143 FAILED_DEPENDENCY = 424
 144 UPGRADE_REQUIRED = 426
 145
 146 # server error
 147 INTERNAL_SERVER_ERROR = 500
 148 NOT_IMPLEMENTED = 501
 149 BAD_GATEWAY = 502
 150 SERVICE_UNAVAILABLE = 503
 151 GATEWAY_TIMEOUT = 504
 152 HTTP_VERSION_NOT_SUPPORTED = 505
 153 INSUFFICIENT_STORAGE = 507
 154 NOT_EXTENDED = 510
 155
 156 # Mapping status codes to official W3C names
 157 responses = {
 158     100: 'Continue',
 159     101: 'Switching Protocols',
 160
 161     200: 'OK',
 162     201: 'Created',
 163     202: 'Accepted',
 164     203: 'Non-Authoritative Information',
 165     204: 'No Content',
 166     205: 'Reset Content',
 167     206: 'Partial Content',
 168
 169     300: 'Multiple Choices',
 170     301: 'Moved Permanently',
 171     302: 'Found',
 172     303: 'See Other',
 173     304: 'Not Modified',
 174     305: 'Use Proxy',
 175     306: '(Unused)',
 176     307: 'Temporary Redirect',
 177
 178     400: 'Bad Request',
 179     401: 'Unauthorized',
 180     402: 'Payment Required',
 181     403: 'Forbidden',
 182     404: 'Not Found',
 183     405: 'Method Not Allowed',
 184     406: 'Not Acceptable',
 185     407: 'Proxy Authentication Required',
 186     408: 'Request Timeout',
 187     409: 'Conflict',
 188     410: 'Gone',
 189     411: 'Length Required',
 190     412: 'Precondition Failed',
 191     413: 'Request Entity Too Large',
 192     414: 'Request-URI Too Long',
 193     415: 'Unsupported Media Type',
 194     416: 'Requested Range Not Satisfiable',
 195     417: 'Expectation Failed',
 196
 197     500: 'Internal Server Error',
 198     501: 'Not Implemented',
 199     502: 'Bad Gateway',
 200     503: 'Service Unavailable',
 201     504: 'Gateway Timeout',
 202     505: 'HTTP Version Not Supported',
 203 }
 204
 205 # maximal amount of data to read at one time in _safe_read
 206 MAXAMOUNT = 1048576
 207
 208 class HTTPMessage(mimetools.Message):
 209
 210     def addheader(self, key, value):
 211         """Add header for field key handling repeats."""
 212         prev = self.dict.get(key)
 213         if prev is None:
 214             self.dict[key] = value
 215         else:
 216             combined = ", ".join((prev, value))
 217             self.dict[key] = combined
 218
 219     def addcontinue(self, key, more):
 220         """Add more field data from a continuation line."""
 221         prev = self.dict[key]
 222         self.dict[key] = prev + "\n " + more
 223
 224     def readheaders(self):
 225         """Read header lines.
 226
 227         Read header lines up to the entirely blank line that terminates them.
 228         The (normally blank) line that ends the headers is skipped, but not
 229         included in the returned list.  If a non-header line ends the headers,
 230         (which is an error), an attempt is made to backspace over it; it is
 231         never included in the returned list.
 232
 233         The variable self.status is set to the empty string if all went well,
 234         otherwise it is an error message.  The variable self.headers is a
 235         completely uninterpreted list of lines contained in the header (so
 236         printing them will reproduce the header exactly as it appears in the
 237         file).
 238
 239         If multiple header fields with the same name occur, they are combined
 240         according to the rules in RFC 2616 sec 4.2:
 241
 242         Appending each subsequent field-value to the first, each separated
 243         by a comma. The order in which header fields with the same field-name
 244         are received is significant to the interpretation of the combined
 245         field value.
 246         """
 247         # XXX The implementation overrides the readheaders() method of
 248         # rfc822.Message.  The base class design isn't amenable to
 249         # customized behavior here so the method here is a copy of the
 250         # base class code with a few small changes.
 251
 252         self.dict = {}
 253         self.unixfrom = ''
 254         self.headers = hlist = []
 255         self.status = ''
 256         headerseen = ""
 257         firstline = 1
 258         startofline = unread = tell = None
 259         if hasattr(self.fp, 'unread'):
 260             unread = self.fp.unread
 261         elif self.seekable:
 262             tell = self.fp.tell
 263         while True:
 264             if tell:
 265                 try:
 266                     startofline = tell()
 267                 except IOError:
 268                     startofline = tell = None
 269                     self.seekable = 0
 270             line = self.fp.readline()
 271             if not line:
 272                 self.status = 'EOF in headers'
 273                 break
 274             # Skip unix From name time lines
 275             if firstline and line.startswith('From '):
 276                 self.unixfrom = self.unixfrom + line
 277                 continue
 278             firstline = 0
 279             if headerseen and line[0] in ' \t':
 280                 # XXX Not sure if continuation lines are handled properly
 281                 # for http and/or for repeating headers
 282                 # It's a continuation line.
 283                 hlist.append(line)
 284                 self.addcontinue(headerseen, line.strip())
 285                 continue
 286             elif self.iscomment(line):
 287                 # It's a comment.  Ignore it.
 288                 continue
 289             elif self.islast(line):
 290                 # Note! No pushback here!  The delimiter line gets eaten.
 291                 break
 292             headerseen = self.isheader(line)
 293             if headerseen:
 294                 # It's a legal header line, save it.
 295                 hlist.append(line)
 296                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
 297                 continue
 298             else:
 299                 # It's not a header line; throw it back and stop here.
 300                 if not self.dict:
 301                     self.status = 'No headers'
 302                 else:
 303                     self.status = 'Non-header line where header expected'
 304                 # Try to undo the read.
 305                 if unread:
 306                     unread(line)
 307                 elif tell:
 308                     self.fp.seek(startofline)
 309                 else:
 310                     self.status = self.status + '; bad seek'
 311                 break
 312
 313 class HTTPResponse:
 314
 315     # strict: If true, raise BadStatusLine if the status line can't be
 316     # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
 317     # false because it prevents clients from talking to HTTP/0.9
 318     # servers.  Note that a response with a sufficiently corrupted
 319     # status line will look like an HTTP/0.9 response.
 320
 321     # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
 322
 323     def __init__(self, sock, debuglevel=0, strict=0, method=None):
 324         self.fp = sock.makefile('rb', 0)
 325         self.debuglevel = debuglevel
 326         self.strict = strict
 327         self._method = method
 328
 329         self.msg = None
 330
 331         # from the Status-Line of the response
 332         self.version = _UNKNOWN # HTTP-Version
 333         self.status = _UNKNOWN  # Status-Code
 334         self.reason = _UNKNOWN  # Reason-Phrase
 335
 336         self.chunked = _UNKNOWN         # is "chunked" being used?
 337         self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
 338         self.length = _UNKNOWN          # number of bytes left in response
 339         self.will_close = _UNKNOWN      # conn will close at end of response
 340
 341     def _read_status(self):
 342         # Initialize with Simple-Response defaults
 343         line = self.fp.readline()
 344         if self.debuglevel > 0:
 345             print "reply:", repr(line)
 346         if not line:
 347             # Presumably, the server closed the connection before
 348             # sending a valid response.
 349             raise BadStatusLine(line)
 350         try:
 351             [version, status, reason] = line.split(None, 2)
 352         except ValueError:
 353             try:
 354                 [version, status] = line.split(None, 1)
 355                 reason = ""
 356             except ValueError:
 357                 # empty version will cause next test to fail and status
 358                 # will be treated as 0.9 response.
 359                 version = ""
 360         if not version.startswith('HTTP/'):
 361             if self.strict:
 362                 self.close()
 363                 raise BadStatusLine(line)
 364             else:
 365                 # assume it's a Simple-Response from an 0.9 server
 366                 self.fp = LineAndFileWrapper(line, self.fp)
 367                 return "HTTP/0.9", 200, ""
 368
 369         # The status code is a three-digit number
 370         try:
 371             status = int(status)
 372             if status < 100 or status > 999:
 373                 raise BadStatusLine(line)
 374         except ValueError:
 375             raise BadStatusLine(line)
 376         return version, status, reason
 377
 378     def begin(self):
 379         if self.msg is not None:
 380             # we've already started reading the response
 381             return
 382
 383         # read until we get a non-100 response
 384         while True:
 385             version, status, reason = self._read_status()
 386             if status != CONTINUE:
 387                 break
 388             # skip the header from the 100 response
 389             while True:
 390                 skip = self.fp.readline().strip()
 391                 if not skip:
 392                     break
 393                 if self.debuglevel > 0:
 394                     print "header:", skip
 395
 396         self.status = status
 397         self.reason = reason.strip()
 398         if version == 'HTTP/1.0':
 399             self.version = 10
 400         elif version.startswith('HTTP/1.'):
 401             self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
 402         elif version == 'HTTP/0.9':
 403             self.version = 9
 404         else:
 405             raise UnknownProtocol(version)
 406
 407         if self.version == 9:
 408             self.length = None
 409             self.chunked = 0
 410             self.will_close = 1
 411             self.msg = HTTPMessage(StringIO())
 412             return
 413
 414         self.msg = HTTPMessage(self.fp, 0)
 415         if self.debuglevel > 0:
 416             for hdr in self.msg.headers:
 417                 print "header:", hdr,
 418
 419         # don't let the msg keep an fp
 420         self.msg.fp = None
 421
 422         # are we using the chunked-style of transfer encoding?
 423         tr_enc = self.msg.getheader('transfer-encoding')
 424         if tr_enc and tr_enc.lower() == "chunked":
 425             self.chunked = 1
 426             self.chunk_left = None
 427         else:
 428             self.chunked = 0
 429
 430         # will the connection close at the end of the response?
 431         self.will_close = self._check_close()
 432
 433         # do we have a Content-Length?
 434         # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
 435         length = self.msg.getheader('content-length')
 436         if length and not self.chunked:
 437             try:
 438                 self.length = int(length)
 439             except ValueError:
 440                 self.length = None
 441         else:
 442             self.length = None
 443
 444         # does the body have a fixed length? (of zero)
 445         if (status == NO_CONTENT or status == NOT_MODIFIED or
 446             100 <= status < 200 or      # 1xx codes
 447             self._method == 'HEAD'):
 448             self.length = 0
 449
 450         # if the connection remains open, and we aren't using chunked, and
 451         # a content-length was not provided, then assume that the connection
 452         # WILL close.
 453         if not self.will_close and \
 454            not self.chunked and \
 455            self.length is None:
 456             self.will_close = 1
 457
 458     def _check_close(self):
 459         conn = self.msg.getheader('connection')
 460         if self.version == 11:
 461             # An HTTP/1.1 proxy is assumed to stay open unless
 462             # explicitly closed.
 463             conn = self.msg.getheader('connection')
 464             if conn and "close" in conn.lower():
 465                 return True
 466             return False
 467
 468         # Some HTTP/1.0 implementations have support for persistent
 469         # connections, using rules different than HTTP/1.1.
 470
 471         # For older HTTP, Keep-Alive indiciates persistent connection.
 472         if self.msg.getheader('keep-alive'):
 473             return False
 474
 475         # At least Akamai returns a "Connection: Keep-Alive" header,
 476         # which was supposed to be sent by the client.
 477         if conn and "keep-alive" in conn.lower():
 478             return False
 479
 480         # Proxy-Connection is a netscape hack.
 481         pconn = self.msg.getheader('proxy-connection')
 482         if pconn and "keep-alive" in pconn.lower():
 483             return False
 484
 485         # otherwise, assume it will close
 486         return True
 487
 488     def close(self):
 489         if self.fp:
 490             self.fp.close()
 491             self.fp = None
 492
 493     def isclosed(self):
 494         # NOTE: it is possible that we will not ever call self.close(). This
 495         #       case occurs when will_close is TRUE, length is None, and we
 496         #       read up to the last byte, but NOT past it.
 497         #
 498         # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
 499         #          called, meaning self.isclosed() is meaningful.
 500         return self.fp is None
 501
 502     # XXX It would be nice to have readline and __iter__ for this, too.
 503
 504     def read(self, amt=None):
 505         if self.fp is None:
 506             return ''
 507
 508         if self.chunked:
 509             return self._read_chunked(amt)
 510
 511         if amt is None:
 512             # unbounded read
 513             if self.length is None:
 514                 s = self.fp.read()
 515             else:
 516                 s = self._safe_read(self.length)
 517                 self.length = 0
 518             self.close()        # we read everything
 519             return s
 520
 521         if self.length is not None:
 522             if amt > self.length:
 523                 # clip the read to the "end of response"
 524                 amt = self.length
 525
 526         # we do not use _safe_read() here because this may be a .will_close
 527         # connection, and the user is reading more bytes than will be provided
 528         # (for example, reading in 1k chunks)
 529         s = self.fp.read(amt)
 530         if self.length is not None:
 531             self.length -= len(s)
 532
 533         return s
 534
 535     def _read_chunked(self, amt):
 536         assert self.chunked != _UNKNOWN
 537         chunk_left = self.chunk_left
 538         value = ''
 539
 540         # XXX This accumulates chunks by repeated string concatenation,
 541         # which is not efficient as the number or size of chunks gets big.
 542         while True:
 543             if chunk_left is None:
 544                 line = self.fp.readline()
 545                 i = line.find(';')
 546                 if i >= 0:
 547                     line = line[:i] # strip chunk-extensions
 548                 chunk_left = int(line, 16)
 549                 if chunk_left == 0:
 550                     break
 551             if amt is None:
 552                 value += self._safe_read(chunk_left)
 553             elif amt < chunk_left:
 554                 value += self._safe_read(amt)
 555                 self.chunk_left = chunk_left - amt
 556                 return value
 557             elif amt == chunk_left:
 558                 value += self._safe_read(amt)
 559                 self._safe_read(2)  # toss the CRLF at the end of the chunk
 560                 self.chunk_left = None
 561                 return value
 562             else:
 563                 value += self._safe_read(chunk_left)
 564                 amt -= chunk_left
 565
 566             # we read the whole chunk, get another
 567             self._safe_read(2)      # toss the CRLF at the end of the chunk
 568             chunk_left = None
 569
 570         # read and discard trailer up to the CRLF terminator
 571         ### note: we shouldn't have any trailers!
 572         while True:
 573             line = self.fp.readline()
 574             if line == '\r\n':
 575                 break
 576
 577         # we read everything; close the "file"
 578         self.close()
 579
 580         return value
 581
 582     def _safe_read(self, amt):
 583         """Read the number of bytes requested, compensating for partial reads.
 584
 585         Normally, we have a blocking socket, but a read() can be interrupted
 586         by a signal (resulting in a partial read).
 587
 588         Note that we cannot distinguish between EOF and an interrupt when zero
 589         bytes have been read. IncompleteRead() will be raised in this
 590         situation.
 591
 592         This function should be used when <amt> bytes "should" be present for
 593         reading. If the bytes are truly not available (due to EOF), then the
 594         IncompleteRead exception can be used to detect the problem.
 595         """
 596         s = []
 597         while amt > 0:
 598             chunk = self.fp.read(min(amt, MAXAMOUNT))
 599             if not chunk:
 600                 raise IncompleteRead(s)
 601             s.append(chunk)
 602             amt -= len(chunk)
 603         return ''.join(s)
 604
 605     def getheader(self, name, default=None):
 606         if self.msg is None:
 607             raise ResponseNotReady()
 608         return self.msg.getheader(name, default)
 609
 610     def getheaders(self):
 611         """Return list of (header, value) tuples."""
 612         if self.msg is None:
 613             raise ResponseNotReady()
 614         return self.msg.items()
 615
 616
 617 class HTTPConnection:
 618
 619     _http_vsn = 11
 620     _http_vsn_str = 'HTTP/1.1'
 621
 622     response_class = HTTPResponse
 623     default_port = HTTP_PORT
 624     auto_open = 1
 625     debuglevel = 0
 626     strict = 0
 627
 628     def __init__(self, host, port=None, strict=None):
 629         self.sock = None
 630         self._buffer = []
 631         self.__response = None
 632         self.__state = _CS_IDLE
 633         self._method = None
 634
 635         self._set_hostport(host, port)
 636         if strict is not None:
 637             self.strict = strict
 638
 639     def _set_hostport(self, host, port):
 640         if port is None:
 641             i = host.rfind(':')
 642             j = host.rfind(']')         # ipv6 addresses have [...]
 643             if i > j:
 644                 try:
 645                     port = int(host[i+1:])
 646                 except ValueError:
 647                     raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
 648                 host = host[:i]
 649             else:
 650                 port = self.default_port
 651             if host and host[0] == '[' and host[-1] == ']':
 652                 host = host[1:-1]
 653         self.host = host
 654         self.port = port
 655
 656     def set_debuglevel(self, level):
 657         self.debuglevel = level
 658
 659     def connect(self):
 660         """Connect to the host and port specified in __init__."""
 661         msg = "getaddrinfo returns an empty list"
 662         for res in socket.getaddrinfo(self.host, self.port, 0,
 663                                       socket.SOCK_STREAM):
 664             af, socktype, proto, canonname, sa = res
 665             try:
 666                 self.sock = socket.socket(af, socktype, proto)
 667                 if self.debuglevel > 0:
 668                     print "connect: (%s, %s)" % (self.host, self.port)
 669                 self.sock.connect(sa)
 670             except socket.error, msg:
 671                 if self.debuglevel > 0:
 672                     print 'connect fail:', (self.host, self.port)
 673                 if self.sock:
 674                     self.sock.close()
 675                 self.sock = None
 676                 continue
 677             break
 678         if not self.sock:
 679             raise socket.error, msg
 680
 681     def close(self):
 682         """Close the connection to the HTTP server."""
 683         if self.sock:
 684             self.sock.close()   # close it manually... there may be other refs
 685             self.sock = None
 686         if self.__response:
 687             self.__response.close()
 688             self.__response = None
 689         self.__state = _CS_IDLE
 690
 691     def send(self, str):
 692         """Send `str' to the server."""
 693         if self.sock is None:
 694             if self.auto_open:
 695                 self.connect()
 696             else:
 697                 raise NotConnected()
 698
 699         # send the data to the server. if we get a broken pipe, then close
 700         # the socket. we want to reconnect when somebody tries to send again.
 701         #
 702         # NOTE: we DO propagate the error, though, because we cannot simply
 703         #       ignore the error... the caller will know if they can retry.
 704         if self.debuglevel > 0:
 705             print "send:", repr(str)
 706         try:
 707             self.sock.sendall(str)
 708         except socket.error, v:
 709             if v[0] == 32:      # Broken pipe
 710                 self.close()
 711             raise
 712
 713     def _output(self, s):
 714         """Add a line of output to the current request buffer.
 715
 716         Assumes that the line does *not* end with \\r\\n.
 717         """
 718         self._buffer.append(s)
 719
 720     def _send_output(self):
 721         """Send the currently buffered request and clear the buffer.
 722
 723         Appends an extra \\r\\n to the buffer.
 724         """
 725         self._buffer.extend(("", ""))
 726         msg = "\r\n".join(self._buffer)
 727         del self._buffer[:]
 728         self.send(msg)
 729
 730     def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
 731         """Send a request to the server.
 732
 733         `method' specifies an HTTP request method, e.g. 'GET'.
 734         `url' specifies the object being requested, e.g. '/index.html'.
 735         `skip_host' if True does not add automatically a 'Host:' header
 736         `skip_accept_encoding' if True does not add automatically an
 737            'Accept-Encoding:' header
 738         """
 739
 740         # if a prior response has been completed, then forget about it.
 741         if self.__response and self.__response.isclosed():
 742             self.__response = None
 743
 744
 745         # in certain cases, we cannot issue another request on this connection.
 746         # this occurs when:
 747         #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
 748         #   2) a response to a previous request has signalled that it is going
 749         #      to close the connection upon completion.
 750         #   3) the headers for the previous response have not been read, thus
 751         #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
 752         #
 753         # if there is no prior response, then we can request at will.
 754         #
 755         # if point (2) is true, then we will have passed the socket to the
 756         # response (effectively meaning, "there is no prior response"), and
 757         # will open a new one when a new request is made.
 758         #
 759         # Note: if a prior response exists, then we *can* start a new request.
 760         #       We are not allowed to begin fetching the response to this new
 761         #       request, however, until that prior response is complete.
 762         #
 763         if self.__state == _CS_IDLE:
 764             self.__state = _CS_REQ_STARTED
 765         else:
 766             raise CannotSendRequest()
 767
 768         # Save the method we use, we need it later in the response phase
 769         self._method = method
 770         if not url:
 771             url = '/'
 772         str = '%s %s %s' % (method, url, self._http_vsn_str)
 773
 774         self._output(str)
 775
 776         if self._http_vsn == 11:
 777             # Issue some standard headers for better HTTP/1.1 compliance
 778
 779             if not skip_host:
 780                 # this header is issued *only* for HTTP/1.1
 781                 # connections. more specifically, this means it is
 782                 # only issued when the client uses the new
 783                 # HTTPConnection() class. backwards-compat clients
 784                 # will be using HTTP/1.0 and those clients may be
 785                 # issuing this header themselves. we should NOT issue
 786                 # it twice; some web servers (such as Apache) barf
 787                 # when they see two Host: headers
 788
 789                 # If we need a non-standard port,include it in the
 790                 # header.  If the request is going through a proxy,
 791                 # but the host of the actual URL, not the host of the
 792                 # proxy.
 793
 794                 netloc = ''
 795                 if url.startswith('http'):
 796                     nil, netloc, nil, nil, nil = urlsplit(url)
 797
 798                 if netloc:
 799                     self.putheader('Host', netloc.encode("idna"))
 800                 elif self.port == HTTP_PORT:
 801                     self.putheader('Host', self.host.encode("idna"))
 802                 else:
 803                     self.putheader('Host', "%s:%s" % (self.host.encode("idna"), self.port))
 804
 805             # note: we are assuming that clients will not attempt to set these
 806             #       headers since *this* library must deal with the
 807             #       consequences. this also means that when the supporting
 808             #       libraries are updated to recognize other forms, then this
 809             #       code should be changed (removed or updated).
 810
 811             # we only want a Content-Encoding of "identity" since we don't
 812             # support encodings such as x-gzip or x-deflate.
 813             if not skip_accept_encoding:
 814                 self.putheader('Accept-Encoding', 'identity')
 815
 816             # we can accept "chunked" Transfer-Encodings, but no others
 817             # NOTE: no TE header implies *only* "chunked"
 818             #self.putheader('TE', 'chunked')
 819
 820             # if TE is supplied in the header, then it must appear in a
 821             # Connection header.
 822             #self.putheader('Connection', 'TE')
 823
 824         else:
 825             # For HTTP/1.0, the server will assume "not chunked"
 826             pass
 827
 828     def putheader(self, header, value):
 829         """Send a request header line to the server.
 830
 831         For example: h.putheader('Accept', 'text/html')
 832         """
 833         if self.__state != _CS_REQ_STARTED:
 834             raise CannotSendHeader()
 835
 836         str = '%s: %s' % (header, value)
 837         self._output(str)
 838
 839     def endheaders(self):
 840         """Indicate that the last header line has been sent to the server."""
 841
 842         if self.__state == _CS_REQ_STARTED:
 843             self.__state = _CS_REQ_SENT
 844         else:
 845             raise CannotSendHeader()
 846
 847         self._send_output()
 848
 849     def request(self, method, url, body=None, headers={}):
 850         """Send a complete request to the server."""
 851
 852         try:
 853             self._send_request(method, url, body, headers)
 854         except socket.error, v:
 855             # trap 'Broken pipe' if we're allowed to automatically reconnect
 856             if v[0] != 32 or not self.auto_open:
 857                 raise
 858             # try one more time
 859             self._send_request(method, url, body, headers)
 860
 861     def _send_request(self, method, url, body, headers):
 862         # honour explicitly requested Host: and Accept-Encoding headers
 863         header_names = dict.fromkeys([k.lower() for k in headers])
 864         skips = {}
 865         if 'host' in header_names:
 866             skips['skip_host'] = 1
 867         if 'accept-encoding' in header_names:
 868             skips['skip_accept_encoding'] = 1
 869
 870         self.putrequest(method, url, **skips)
 871
 872         if body and ('content-length' not in header_names):
 873             self.putheader('Content-Length', str(len(body)))
 874         for hdr, value in headers.iteritems():
 875             self.putheader(hdr, value)
 876         self.endheaders()
 877
 878         if body:
 879             self.send(body)
 880
 881     def getresponse(self):
 882         "Get the response from the server."
 883
 884         # if a prior response has been completed, then forget about it.
 885         if self.__response and self.__response.isclosed():
 886             self.__response = None
 887
 888         #
 889         # if a prior response exists, then it must be completed (otherwise, we
 890         # cannot read this response's header to determine the connection-close
 891         # behavior)
 892         #
 893         # note: if a prior response existed, but was connection-close, then the
 894         # socket and response were made independent of this HTTPConnection
 895         # object since a new request requires that we open a whole new
 896         # connection
 897         #
 898         # this means the prior response had one of two states:
 899         #   1) will_close: this connection was reset and the prior socket and
 900         #                  response operate independently
 901         #   2) persistent: the response was retained and we await its
 902         #                  isclosed() status to become true.
 903         #
 904         if self.__state != _CS_REQ_SENT or self.__response:
 905             raise ResponseNotReady()
 906
 907         if self.debuglevel > 0:
 908             response = self.response_class(self.sock, self.debuglevel,
 909                                            strict=self.strict,
 910                                            method=self._method)
 911         else:
 912             response = self.response_class(self.sock, strict=self.strict,
 913                                            method=self._method)
 914
 915         response.begin()
 916         assert response.will_close != _UNKNOWN
 917         self.__state = _CS_IDLE
 918
 919         if response.will_close:
 920             # this effectively passes the connection to the response
 921             self.close()
 922         else:
 923             # remember this, so we can tell when it is complete
 924             self.__response = response
 925
 926         return response
 927
 928 # The next several classes are used to define FakeSocket,a socket-like
 929 # interface to an SSL connection.
 930
 931 # The primary complexity comes from faking a makefile() method.  The
 932 # standard socket makefile() implementation calls dup() on the socket
 933 # file descriptor.  As a consequence, clients can call close() on the
 934 # parent socket and its makefile children in any order.  The underlying
 935 # socket isn't closed until they are all closed.
 936
 937 # The implementation uses reference counting to keep the socket open
 938 # until the last client calls close().  SharedSocket keeps track of
 939 # the reference counting and SharedSocketClient provides an constructor
 940 # and close() method that call incref() and decref() correctly.
 941
 942 class SharedSocket:
 943
 944     def __init__(self, sock):
 945         self.sock = sock
 946         self._refcnt = 0
 947
 948     def incref(self):
 949         self._refcnt += 1
 950
 951     def decref(self):
 952         self._refcnt -= 1
 953         assert self._refcnt >= 0
 954         if self._refcnt == 0:
 955             self.sock.close()
 956
 957     def __del__(self):
 958         self.sock.close()
 959
 960 class SharedSocketClient:
 961
 962     def __init__(self, shared):
 963         self._closed = 0
 964         self._shared = shared
 965         self._shared.incref()
 966         self._sock = shared.sock
 967
 968     def close(self):
 969         if not self._closed:
 970             self._shared.decref()
 971             self._closed = 1
 972             self._shared = None
 973
 974 class SSLFile(SharedSocketClient):
 975     """File-like object wrapping an SSL socket."""
 976
 977     BUFSIZE = 8192
 978
 979     def __init__(self, sock, ssl, bufsize=None):
 980         SharedSocketClient.__init__(self, sock)
 981         self._ssl = ssl
 982         self._buf = ''
 983         self._bufsize = bufsize or self.__class__.BUFSIZE
 984
 985     def _read(self):
 986         buf = ''
 987         # put in a loop so that we retry on transient errors
 988         while True:
 989             try:
 990                 buf = self._ssl.read(self._bufsize)
 991             except socket.sslerror, err:
 992                 if (err[0] == socket.SSL_ERROR_WANT_READ
 993                     or err[0] == socket.SSL_ERROR_WANT_WRITE):
 994                     continue
 995                 if (err[0] == socket.SSL_ERROR_ZERO_RETURN
 996                     or err[0] == socket.SSL_ERROR_EOF):
 997                     break
 998                 raise
 999             except socket.error, err:
1000                 if err[0] == errno.EINTR:
1001                     continue
1002                 if err[0] == errno.EBADF:
1003                     # XXX socket was closed?
1004                     break
1005                 raise
1006             else:
1007                 break
1008         return buf
1009
1010     def read(self, size=None):
1011         L = [self._buf]
1012         avail = len(self._buf)
1013         while size is None or avail < size:
1014             s = self._read()
1015             if s == '':
1016                 break
1017             L.append(s)
1018             avail += len(s)
1019         all = "".join(L)
1020         if size is None:
1021             self._buf = ''
1022             return all
1023         else:
1024             self._buf = all[size:]
1025             return all[:size]
1026
1027     def readline(self):
1028         L = [self._buf]
1029         self._buf = ''
1030         while 1:
1031             i = L[-1].find("\n")
1032             if i >= 0:
1033                 break
1034             s = self._read()
1035             if s == '':
1036                 break
1037             L.append(s)
1038         if i == -1:
1039             # loop exited because there is no more data
1040             return "".join(L)
1041         else:
1042             all = "".join(L)
1043             # XXX could do enough bookkeeping not to do a 2nd search
1044             i = all.find("\n") + 1
1045             line = all[:i]
1046             self._buf = all[i:]
1047             return line
1048
1049     def readlines(self, sizehint=0):
1050         total = 0
1051         list = []
1052         while True:
1053             line = self.readline()
1054             if not line:
1055                 break
1056             list.append(line)
1057             total += len(line)
1058             if sizehint and total >= sizehint:
1059                 break
1060         return list
1061
1062     def fileno(self):
1063         return self._sock.fileno()
1064
1065     def __iter__(self):
1066         return self
1067
1068     def next(self):
1069         line = self.readline()
1070         if not line:
1071             raise StopIteration
1072         return line
1073
1074 class FakeSocket(SharedSocketClient):
1075
1076     class _closedsocket:
1077         def __getattr__(self, name):
1078             raise error(9, 'Bad file descriptor')
1079
1080     def __init__(self, sock, ssl):
1081         sock = SharedSocket(sock)
1082         SharedSocketClient.__init__(self, sock)
1083         self._ssl = ssl
1084
1085     def close(self):
1086         SharedSocketClient.close(self)
1087         self._sock = self.__class__._closedsocket()
1088
1089     def makefile(self, mode, bufsize=None):
1090         if mode != 'r' and mode != 'rb':
1091             raise UnimplementedFileMode()
1092         return SSLFile(self._shared, self._ssl, bufsize)
1093
1094     def send(self, stuff, flags = 0):
1095         return self._ssl.write(stuff)
1096
1097     sendall = send
1098
1099     def recv(self, len = 1024, flags = 0):
1100         return self._ssl.read(len)
1101
1102     def __getattr__(self, attr):
1103         return getattr(self._sock, attr)
1104
1105
1106 class HTTPSConnection(HTTPConnection):
1107     "This class allows communication via SSL."
1108
1109     default_port = HTTPS_PORT
1110
1111     def __init__(self, host, port=None, key_file=None, cert_file=None,
1112                  strict=None):
1113         HTTPConnection.__init__(self, host, port, strict)
1114         self.key_file = key_file
1115         self.cert_file = cert_file
1116
1117     def connect(self):
1118         "Connect to a host on a given (SSL) port."
1119
1120         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1121         sock.connect((self.host, self.port))
1122         ssl = socket.ssl(sock, self.key_file, self.cert_file)
1123         self.sock = FakeSocket(sock, ssl)
1124
1125
1126 class HTTP:
1127     "Compatibility class with httplib.py from 1.5."
1128
1129     _http_vsn = 10
1130     _http_vsn_str = 'HTTP/1.0'
1131
1132     debuglevel = 0
1133
1134     _connection_class = HTTPConnection
1135
1136     def __init__(self, host='', port=None, strict=None):
1137         "Provide a default host, since the superclass requires one."
1138
1139         # some joker passed 0 explicitly, meaning default port
1140         if port == 0:
1141             port = None
1142
1143         # Note that we may pass an empty string as the host; this will throw
1144         # an error when we attempt to connect. Presumably, the client code
1145         # will call connect before then, with a proper host.
1146         self._setup(self._connection_class(host, port, strict))
1147
1148     def _setup(self, conn):
1149         self._conn = conn
1150
1151         # set up delegation to flesh out interface
1152         self.send = conn.send
1153         self.putrequest = conn.putrequest
1154         self.endheaders = conn.endheaders
1155         self.set_debuglevel = conn.set_debuglevel
1156
1157         conn._http_vsn = self._http_vsn
1158         conn._http_vsn_str = self._http_vsn_str
1159
1160         self.file = None
1161
1162     def connect(self, host=None, port=None):
1163         "Accept arguments to set the host/port, since the superclass doesn't."
1164
1165         if host is not None:
1166             self._conn._set_hostport(host, port)
1167         self._conn.connect()
1168
1169     def getfile(self):
1170         "Provide a getfile, since the superclass' does not use this concept."
1171         return self.file
1172
1173     def putheader(self, header, *values):
1174         "The superclass allows only one value argument."
1175         self._conn.putheader(header, '\r\n\t'.join(values))
1176
1177     def getreply(self):
1178         """Compat definition since superclass does not define it.
1179
1180         Returns a tuple consisting of:
1181         - server status code (e.g. '200' if all goes well)
1182         - server "reason" corresponding to status code
1183         - any RFC822 headers in the response from the server
1184         """
1185         try:
1186             response = self._conn.getresponse()
1187         except BadStatusLine, e:
1188             ### hmm. if getresponse() ever closes the socket on a bad request,
1189             ### then we are going to have problems with self.sock
1190
1191             ### should we keep this behavior? do people use it?
1192             # keep the socket open (as a file), and return it
1193             self.file = self._conn.sock.makefile('rb', 0)
1194
1195             # close our socket -- we want to restart after any protocol error
1196             self.close()
1197
1198             self.headers = None
1199             return -1, e.line, None
1200
1201         self.headers = response.msg
1202         self.file = response.fp
1203         return response.status, response.reason, response.msg
1204
1205     def close(self):
1206         self._conn.close()
1207
1208         # note that self.file == response.fp, which gets closed by the
1209         # superclass. just clear the object ref here.
1210         ### hmm. messy. if status==-1, then self.file is owned by us.
1211         ### well... we aren't explicitly closing, but losing this ref will
1212         ### do it
1213         self.file = None
1214
1215 if hasattr(socket, 'ssl'):
1216     class HTTPS(HTTP):
1217         """Compatibility with 1.5 httplib interface
1218
1219         Python 1.5.2 did not have an HTTPS class, but it defined an
1220         interface for sending http requests that is also useful for
1221         https.
1222         """
1223
1224         _connection_class = HTTPSConnection
1225
1226         def __init__(self, host='', port=None, key_file=None, cert_file=None,
1227                      strict=None):
1228             # provide a default host, pass the X509 cert info
1229
1230             # urf. compensate for bad input.
1231             if port == 0:
1232                 port = None
1233             self._setup(self._connection_class(host, port, key_file,
1234                                                cert_file, strict))
1235
1236             # we never actually use these for anything, but we keep them
1237             # here for compatibility with post-1.5.2 CVS.
1238             self.key_file = key_file
1239             self.cert_file = cert_file
1240
1241
1242 class HTTPException(Exception):
1243     # Subclasses that define an __init__ must call Exception.__init__
1244     # or define self.args.  Otherwise, str() will fail.
1245     pass
1246
1247 class NotConnected(HTTPException):
1248     pass
1249
1250 class InvalidURL(HTTPException):
1251     pass
1252
1253 class UnknownProtocol(HTTPException):
1254     def __init__(self, version):
1255         self.args = version,
1256         self.version = version
1257
1258 class UnknownTransferEncoding(HTTPException):
1259     pass
1260
1261 class UnimplementedFileMode(HTTPException):
1262     pass
1263
1264 class IncompleteRead(HTTPException):
1265     def __init__(self, partial):
1266         self.args = partial,
1267         self.partial = partial
1268
1269 class ImproperConnectionState(HTTPException):
1270     pass
1271
1272 class CannotSendRequest(ImproperConnectionState):
1273     pass
1274
1275 class CannotSendHeader(ImproperConnectionState):
1276     pass
1277
1278 class ResponseNotReady(ImproperConnectionState):
1279     pass
1280
1281 class BadStatusLine(HTTPException):
1282     def __init__(self, line):
1283         self.args = line,
1284         self.line = line
1285
1286 # for backwards compatibility
1287 error = HTTPException
1288
1289 class LineAndFileWrapper:
1290     """A limited file-like object for HTTP/0.9 responses."""
1291
1292     # The status-line parsing code calls readline(), which normally
1293     # get the HTTP status line.  For a 0.9 response, however, this is
1294     # actually the first line of the body!  Clients need to get a
1295     # readable file object that contains that line.
1296
1297     def __init__(self, line, file):
1298         self._line = line
1299         self._file = file
1300         self._line_consumed = 0
1301         self._line_offset = 0
1302         self._line_left = len(line)
1303
1304     def __getattr__(self, attr):
1305         return getattr(self._file, attr)
1306
1307     def _done(self):
1308         # called when the last byte is read from the line.  After the
1309         # call, all read methods are delegated to the underlying file
1310         # object.
1311         self._line_consumed = 1
1312         self.read = self._file.read
1313         self.readline = self._file.readline
1314         self.readlines = self._file.readlines
1315
1316     def read(self, amt=None):
1317         if self._line_consumed:
1318             return self._file.read(amt)
1319         assert self._line_left
1320         if amt is None or amt > self._line_left:
1321             s = self._line[self._line_offset:]
1322             self._done()
1323             if amt is None:
1324                 return s + self._file.read()
1325             else:
1326                 return s + self._file.read(amt - len(s))
1327         else:
1328             assert amt <= self._line_left
1329             i = self._line_offset
1330             j = i + amt
1331             s = self._line[i:j]
1332             self._line_offset = j
1333             self._line_left -= amt
1334             if self._line_left == 0:
1335                 self._done()
1336             return s
1337
1338     def readline(self):
1339         if self._line_consumed:
1340             return self._file.readline()
1341         assert self._line_left
1342         s = self._line[self._line_offset:]
1343         self._done()
1344         return s
1345
1346     def readlines(self, size=None):
1347         if self._line_consumed:
1348             return self._file.readlines(size)
1349         assert self._line_left
1350         L = [self._line[self._line_offset:]]
1351         self._done()
1352         if size is None:
1353             return L + self._file.readlines()
1354         else:
1355             return L + self._file.readlines(size)
1356
1357 def test():
1358     """Test this module.
1359
1360     A hodge podge of tests collected here, because they have too many
1361     external dependencies for the regular test suite.
1362     """
1363
1364     import sys
1365     import getopt
1366     opts, args = getopt.getopt(sys.argv[1:], 'd')
1367     dl = 0
1368     for o, a in opts:
1369         if o == '-d': dl = dl + 1
1370     host = 'www.python.org'
1371     selector = '/'
1372     if args[0:]: host = args[0]
1373     if args[1:]: selector = args[1]
1374     h = HTTP()
1375     h.set_debuglevel(dl)
1376     h.connect(host)
1377     h.putrequest('GET', selector)
1378     h.endheaders()
1379     status, reason, headers = h.getreply()
1380     print 'status =', status
1381     print 'reason =', reason
1382     print "read", len(h.getfile().read())
1383     print
1384     if headers:
1385         for header in headers.headers: print header.strip()
1386     print
1387
1388     # minimal test that code to extract host from url works
1389     class HTTP11(HTTP):
1390         _http_vsn = 11
1391         _http_vsn_str = 'HTTP/1.1'
1392
1393     h = HTTP11('www.python.org')
1394     h.putrequest('GET', 'http://www.python.org/~jeremy/')
1395     h.endheaders()
1396     h.getreply()
1397     h.close()
1398
1399     if hasattr(socket, 'ssl'):
1400
1401         for host, selector in (('sourceforge.net', '/projects/python'),
1402                                ):
1403             print "https://%s%s" % (host, selector)
1404             hs = HTTPS()
1405             hs.set_debuglevel(dl)
1406             hs.connect(host)
1407             hs.putrequest('GET', selector)
1408             hs.endheaders()
1409             status, reason, headers = hs.getreply()
1410             print 'status =', status
1411             print 'reason =', reason
1412             print "read", len(hs.getfile().read())
1413             print
1414             if headers:
1415                 for header in headers.headers: print header.strip()
1416             print
1417
1418 if __name__ == '__main__':
1419     test()