Lib/httplib.py

   1 """HTTP/1.1 client library
   2
   3 <intro stuff goes here>
   4 <other stuff, too>
   5
   6 HTTPConnection go through a number of "states", which defines when a client
   7 may legally make another request or fetch the response for a particular
   8 request. This diagram details these state transitions:
   9
  10     (null)
  11       |
  12       | HTTPConnection()
  13       v
  14     Idle
  15       |
  16       | putrequest()
  17       v
  18     Request-started
  19       |
  20       | ( putheader() )*  endheaders()
  21       v
  22     Request-sent
  23       |
  24       | response = getresponse()
  25       v
  26     Unread-response   [Response-headers-read]
  27       |\____________________
  28       |                     |
  29       | response.read()     | putrequest()
  30       v                     v
  31     Idle                  Req-started-unread-response
  32                      ______/|
  33                    /        |
  34    response.read() |        | ( putheader() )*  endheaders()
  35                    v        v
  36        Request-started    Req-sent-unread-response
  37                             |
  38                             | response.read()
  39                             v
  40                           Request-sent
  41
  42 This diagram presents the following rules:
  43   -- a second request may not be started until {response-headers-read}
  44   -- a response [object] cannot be retrieved until {request-sent}
  45   -- there is no differentiation between an unread response body and a
  46      partially read response body
  47
  48 Note: this enforcement is applied by the HTTPConnection class. The
  49       HTTPResponse class does not enforce this state machine, which
  50       implies sophisticated clients may accelerate the request/response
  51       pipeline. Caution should be taken, though: accelerating the states
  52       beyond the above pattern may imply knowledge of the server's
  53       connection-close behavior for certain requests. For example, it
  54       is impossible to tell whether the server will close the connection
  55       UNTIL the response headers have been read; this means that further
  56       requests cannot be placed into the pipeline until it is known that
  57       the server will NOT be closing the connection.
  58
  59 Logical State                  __state            __response
  60 -------------                  -------            ----------
  61 Idle                           _CS_IDLE           None
  62 Request-started                _CS_REQ_STARTED    None
  63 Request-sent                   _CS_REQ_SENT       None
  64 Unread-response                _CS_IDLE           <response_class>
  65 Req-started-unread-response    _CS_REQ_STARTED    <response_class>
  66 Req-sent-unread-response       _CS_REQ_SENT       <response_class>
  67 """
  68
  69 import errno
  70 import mimetools
  71 import socket
  72 from urlparse import urlsplit
  73
  74 try:
  75     from cStringIO import StringIO
  76 except ImportError:
  77     from StringIO import StringIO
  78
  79 __all__ = ["HTTP", "HTTPResponse", "HTTPConnection", "HTTPSConnection",
  80            "HTTPException", "NotConnected", "UnknownProtocol",
  81            "UnknownTransferEncoding", "UnimplementedFileMode",
  82            "IncompleteRead", "InvalidURL", "ImproperConnectionState",
  83            "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
  84            "BadStatusLine", "error", "responses"]
  85
  86 HTTP_PORT = 80
  87 HTTPS_PORT = 443
  88
  89 _UNKNOWN = 'UNKNOWN'
  90
  91 # connection states
  92 _CS_IDLE = 'Idle'
  93 _CS_REQ_STARTED = 'Request-started'
  94 _CS_REQ_SENT = 'Request-sent'
  95
  96 # status codes
  97 # informational
  98 CONTINUE = 100
  99 SWITCHING_PROTOCOLS = 101
 100 PROCESSING = 102
 101
 102 # successful
 103 OK = 200
 104 CREATED = 201
 105 ACCEPTED = 202
 106 NON_AUTHORITATIVE_INFORMATION = 203
 107 NO_CONTENT = 204
 108 RESET_CONTENT = 205
 109 PARTIAL_CONTENT = 206
 110 MULTI_STATUS = 207
 111 IM_USED = 226
 112
 113 # redirection
 114 MULTIPLE_CHOICES = 300
 115 MOVED_PERMANENTLY = 301
 116 FOUND = 302
 117 SEE_OTHER = 303
 118 NOT_MODIFIED = 304
 119 USE_PROXY = 305
 120 TEMPORARY_REDIRECT = 307
 121
 122 # client error
 123 BAD_REQUEST = 400
 124 UNAUTHORIZED = 401
 125 PAYMENT_REQUIRED = 402
 126 FORBIDDEN = 403
 127 NOT_FOUND = 404
 128 METHOD_NOT_ALLOWED = 405
 129 NOT_ACCEPTABLE = 406
 130 PROXY_AUTHENTICATION_REQUIRED = 407
 131 REQUEST_TIMEOUT = 408
 132 CONFLICT = 409
 133 GONE = 410
 134 LENGTH_REQUIRED = 411
 135 PRECONDITION_FAILED = 412
 136 REQUEST_ENTITY_TOO_LARGE = 413
 137 REQUEST_URI_TOO_LONG = 414
 138 UNSUPPORTED_MEDIA_TYPE = 415
 139 REQUESTED_RANGE_NOT_SATISFIABLE = 416
 140 EXPECTATION_FAILED = 417
 141 UNPROCESSABLE_ENTITY = 422
 142 LOCKED = 423
 143 FAILED_DEPENDENCY = 424
 144 UPGRADE_REQUIRED = 426
 145
 146 # server error
 147 INTERNAL_SERVER_ERROR = 500
 148 NOT_IMPLEMENTED = 501
 149 BAD_GATEWAY = 502
 150 SERVICE_UNAVAILABLE = 503
 151 GATEWAY_TIMEOUT = 504
 152 HTTP_VERSION_NOT_SUPPORTED = 505
 153 INSUFFICIENT_STORAGE = 507
 154 NOT_EXTENDED = 510
 155
 156 # Mapping status codes to official W3C names
 157 responses = {
 158     100: 'Continue',
 159     101: 'Switching Protocols',
 160
 161     200: 'OK',
 162     201: 'Created',
 163     202: 'Accepted',
 164     203: 'Non-Authoritative Information',
 165     204: 'No Content',
 166     205: 'Reset Content',
 167     206: 'Partial Content',
 168
 169     300: 'Multiple Choices',
 170     301: 'Moved Permanently',
 171     302: 'Found',
 172     303: 'See Other',
 173     304: 'Not Modified',
 174     305: 'Use Proxy',
 175     306: '(Unused)',
 176     307: 'Temporary Redirect',
 177
 178     400: 'Bad Request',
 179     401: 'Unauthorized',
 180     402: 'Payment Required',
 181     403: 'Forbidden',
 182     404: 'Not Found',
 183     405: 'Method Not Allowed',
 184     406: 'Not Acceptable',
 185     407: 'Proxy Authentication Required',
 186     408: 'Request Timeout',
 187     409: 'Conflict',
 188     410: 'Gone',
 189     411: 'Length Required',
 190     412: 'Precondition Failed',
 191     413: 'Request Entity Too Large',
 192     414: 'Request-URI Too Long',
 193     415: 'Unsupported Media Type',
 194     416: 'Requested Range Not Satisfiable',
 195     417: 'Expectation Failed',
 196
 197     500: 'Internal Server Error',
 198     501: 'Not Implemented',
 199     502: 'Bad Gateway',
 200     503: 'Service Unavailable',
 201     504: 'Gateway Timeout',
 202     505: 'HTTP Version Not Supported',
 203 }
 204
 205 # maximal amount of data to read at one time in _safe_read
 206 MAXAMOUNT = 1048576
 207
 208 class HTTPMessage(mimetools.Message):
 209
 210     def addheader(self, key, value):
 211         """Add header for field key handling repeats."""
 212         prev = self.dict.get(key)
 213         if prev is None:
 214             self.dict[key] = value
 215         else:
 216             combined = ", ".join((prev, value))
 217             self.dict[key] = combined
 218
 219     def addcontinue(self, key, more):
 220         """Add more field data from a continuation line."""
 221         prev = self.dict[key]
 222         self.dict[key] = prev + "\n " + more
 223
 224     def readheaders(self):
 225         """Read header lines.
 226
 227         Read header lines up to the entirely blank line that terminates them.
 228         The (normally blank) line that ends the headers is skipped, but not
 229         included in the returned list.  If a non-header line ends the headers,
 230         (which is an error), an attempt is made to backspace over it; it is
 231         never included in the returned list.
 232
 233         The variable self.status is set to the empty string if all went well,
 234         otherwise it is an error message.  The variable self.headers is a
 235         completely uninterpreted list of lines contained in the header (so
 236         printing them will reproduce the header exactly as it appears in the
 237         file).
 238
 239         If multiple header fields with the same name occur, they are combined
 240         according to the rules in RFC 2616 sec 4.2:
 241
 242         Appending each subsequent field-value to the first, each separated
 243         by a comma. The order in which header fields with the same field-name
 244         are received is significant to the interpretation of the combined
 245         field value.
 246         """
 247         # XXX The implementation overrides the readheaders() method of
 248         # rfc822.Message.  The base class design isn't amenable to
 249         # customized behavior here so the method here is a copy of the
 250         # base class code with a few small changes.
 251
 252         self.dict = {}
 253         self.unixfrom = ''
 254         self.headers = hlist = []
 255         self.status = ''
 256         headerseen = ""
 257         firstline = 1
 258         startofline = unread = tell = None
 259         if hasattr(self.fp, 'unread'):
 260             unread = self.fp.unread
 261         elif self.seekable:
 262             tell = self.fp.tell
 263         while True:
 264             if tell:
 265                 try:
 266                     startofline = tell()
 267                 except IOError:
 268                     startofline = tell = None
 269                     self.seekable = 0
 270             line = self.fp.readline()
 271             if not line:
 272                 self.status = 'EOF in headers'
 273                 break
 274             # Skip unix From name time lines
 275             if firstline and line.startswith('From '):
 276                 self.unixfrom = self.unixfrom + line
 277                 continue
 278             firstline = 0
 279             if headerseen and line[0] in ' \t':
 280                 # XXX Not sure if continuation lines are handled properly
 281                 # for http and/or for repeating headers
 282                 # It's a continuation line.
 283                 hlist.append(line)
 284                 self.addcontinue(headerseen, line.strip())
 285                 continue
 286             elif self.iscomment(line):
 287                 # It's a comment.  Ignore it.
 288                 continue
 289             elif self.islast(line):
 290                 # Note! No pushback here!  The delimiter line gets eaten.
 291                 break
 292             headerseen = self.isheader(line)
 293             if headerseen:
 294                 # It's a legal header line, save it.
 295                 hlist.append(line)
 296                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
 297                 continue
 298             else:
 299                 # It's not a header line; throw it back and stop here.
 300                 if not self.dict:
 301                     self.status = 'No headers'
 302                 else:
 303                     self.status = 'Non-header line where header expected'
 304                 # Try to undo the read.
 305                 if unread:
 306                     unread(line)
 307                 elif tell:
 308                     self.fp.seek(startofline)
 309                 else:
 310                     self.status = self.status + '; bad seek'
 311                 break
 312
 313 class HTTPResponse:
 314
 315     # strict: If true, raise BadStatusLine if the status line can't be
 316     # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
 317     # false because it prevents clients from talking to HTTP/0.9
 318     # servers.  Note that a response with a sufficiently corrupted
 319     # status line will look like an HTTP/0.9 response.
 320
 321     # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
 322
 323     def __init__(self, sock, debuglevel=0, strict=0, method=None):
 324         self.fp = sock.makefile('rb', 0)
 325         self.debuglevel = debuglevel
 326         self.strict = strict
 327         self._method = method
 328
 329         self.msg = None
 330
 331         # from the Status-Line of the response
 332         self.version = _UNKNOWN # HTTP-Version
 333         self.status = _UNKNOWN  # Status-Code
 334         self.reason = _UNKNOWN  # Reason-Phrase
 335
 336         self.chunked = _UNKNOWN         # is "chunked" being used?
 337         self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
 338         self.length = _UNKNOWN          # number of bytes left in response
 339         self.will_close = _UNKNOWN      # conn will close at end of response
 340
 341     def _read_status(self):
 342         # Initialize with Simple-Response defaults
 343         line = self.fp.readline()
 344         if self.debuglevel > 0:
 345             print "reply:", repr(line)
 346         if not line:
 347             # Presumably, the server closed the connection before
 348             # sending a valid response.
 349             raise BadStatusLine(line)
 350         try:
 351             [version, status, reason] = line.split(None, 2)
 352         except ValueError:
 353             try:
 354                 [version, status] = line.split(None, 1)
 355                 reason = ""
 356             except ValueError:
 357                 # empty version will cause next test to fail and status
 358                 # will be treated as 0.9 response.
 359                 version = ""
 360         if not version.startswith('HTTP/'):
 361             if self.strict:
 362                 self.close()
 363                 raise BadStatusLine(line)
 364             else:
 365                 # assume it's a Simple-Response from an 0.9 server
 366                 self.fp = LineAndFileWrapper(line, self.fp)
 367                 return "HTTP/0.9", 200, ""
 368
 369         # The status code is a three-digit number
 370         try:
 371             status = int(status)
 372             if status < 100 or status > 999:
 373                 raise BadStatusLine(line)
 374         except ValueError:
 375             raise BadStatusLine(line)
 376         return version, status, reason
 377
 378     def begin(self):
 379         if self.msg is not None:
 380             # we've already started reading the response
 381             return
 382
 383         # read until we get a non-100 response
 384         while True:
 385             version, status, reason = self._read_status()
 386             if status != CONTINUE:
 387                 break
 388             # skip the header from the 100 response
 389             while True:
 390                 skip = self.fp.readline().strip()
 391                 if not skip:
 392                     break
 393                 if self.debuglevel > 0:
 394                     print "header:", skip
 395
 396         self.status = status
 397         self.reason = reason.strip()
 398         if version == 'HTTP/1.0':
 399             self.version = 10
 400         elif version.startswith('HTTP/1.'):
 401             self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
 402         elif version == 'HTTP/0.9':
 403             self.version = 9
 404         else:
 405             raise UnknownProtocol(version)
 406
 407         if self.version == 9:
 408             self.length = None
 409             self.chunked = 0
 410             self.will_close = 1
 411             self.msg = HTTPMessage(StringIO())
 412             return
 413
 414         self.msg = HTTPMessage(self.fp, 0)
 415         if self.debuglevel > 0:
 416             for hdr in self.msg.headers:
 417                 print "header:", hdr,
 418
 419         # don't let the msg keep an fp
 420         self.msg.fp = None
 421
 422         # are we using the chunked-style of transfer encoding?
 423         tr_enc = self.msg.getheader('transfer-encoding')
 424         if tr_enc and tr_enc.lower() == "chunked":
 425             self.chunked = 1
 426             self.chunk_left = None
 427         else:
 428             self.chunked = 0
 429
 430         # will the connection close at the end of the response?
 431         self.will_close = self._check_close()
 432
 433         # do we have a Content-Length?
 434         # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
 435         length = self.msg.getheader('content-length')
 436         if length and not self.chunked:
 437             try:
 438                 self.length = int(length)
 439             except ValueError:
 440                 self.length = None
 441         else:
 442             self.length = None
 443
 444         # does the body have a fixed length? (of zero)
 445         if (status == NO_CONTENT or status == NOT_MODIFIED or
 446             100 <= status < 200 or      # 1xx codes
 447             self._method == 'HEAD'):
 448             self.length = 0
 449
 450         # if the connection remains open, and we aren't using chunked, and
 451         # a content-length was not provided, then assume that the connection
 452         # WILL close.
 453         if not self.will_close and \
 454            not self.chunked and \
 455            self.length is None:
 456             self.will_close = 1
 457
 458     def _check_close(self):
 459         conn = self.msg.getheader('connection')
 460         if self.version == 11:
 461             # An HTTP/1.1 proxy is assumed to stay open unless
 462             # explicitly closed.
 463             conn = self.msg.getheader('connection')
 464             if conn and "close" in conn.lower():
 465                 return True
 466             return False
 467
 468         # Some HTTP/1.0 implementations have support for persistent
 469         # connections, using rules different than HTTP/1.1.
 470
 471         # For older HTTP, Keep-Alive indiciates persistent connection.
 472         if self.msg.getheader('keep-alive'):
 473             return False
 474
 475         # At least Akamai returns a "Connection: Keep-Alive" header,
 476         # which was supposed to be sent by the client.
 477         if conn and "keep-alive" in conn.lower():
 478             return False
 479
 480         # Proxy-Connection is a netscape hack.
 481         pconn = self.msg.getheader('proxy-connection')
 482         if pconn and "keep-alive" in pconn.lower():
 483             return False
 484
 485         # otherwise, assume it will close
 486         return True
 487
 488     def close(self):
 489         if self.fp:
 490             self.fp.close()
 491             self.fp = None
 492
 493     def isclosed(self):
 494         # NOTE: it is possible that we will not ever call self.close(). This
 495         #       case occurs when will_close is TRUE, length is None, and we
 496         #       read up to the last byte, but NOT past it.
 497         #
 498         # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
 499         #          called, meaning self.isclosed() is meaningful.
 500         return self.fp is None
 501
 502     # XXX It would be nice to have readline and __iter__ for this, too.
 503
 504     def read(self, amt=None):
 505         if self.fp is None:
 506             return ''
 507
 508         if self.chunked:
 509             return self._read_chunked(amt)
 510
 511         if amt is None:
 512             # unbounded read
 513             if self.length is None:
 514                 s = self.fp.read()
 515             else:
 516                 s = self._safe_read(self.length)
 517                 self.length = 0
 518             self.close()        # we read everything
 519             return s
 520
 521         if self.length is not None:
 522             if amt > self.length:
 523                 # clip the read to the "end of response"
 524                 amt = self.length
 525
 526         # we do not use _safe_read() here because this may be a .will_close
 527         # connection, and the user is reading more bytes than will be provided
 528         # (for example, reading in 1k chunks)
 529         s = self.fp.read(amt)
 530         if self.length is not None:
 531             self.length -= len(s)
 532
 533         return s
 534
 535     def _read_chunked(self, amt):
 536         assert self.chunked != _UNKNOWN
 537         chunk_left = self.chunk_left
 538         value = ''
 539
 540         # XXX This accumulates chunks by repeated string concatenation,
 541         # which is not efficient as the number or size of chunks gets big.
 542         while True:
 543             if chunk_left is None:
 544                 line = self.fp.readline()
 545                 i = line.find(';')
 546                 if i >= 0:
 547                     line = line[:i] # strip chunk-extensions
 548                 chunk_left = int(line, 16)
 549                 if chunk_left == 0:
 550                     break
 551             if amt is None:
 552                 value += self._safe_read(chunk_left)
 553             elif amt < chunk_left:
 554                 value += self._safe_read(amt)
 555                 self.chunk_left = chunk_left - amt
 556                 return value
 557             elif amt == chunk_left:
 558                 value += self._safe_read(amt)
 559                 self._safe_read(2)  # toss the CRLF at the end of the chunk
 560                 self.chunk_left = None
 561                 return value
 562             else:
 563                 value += self._safe_read(chunk_left)
 564                 amt -= chunk_left
 565
 566             # we read the whole chunk, get another
 567             self._safe_read(2)      # toss the CRLF at the end of the chunk
 568             chunk_left = None
 569
 570         # read and discard trailer up to the CRLF terminator
 571         ### note: we shouldn't have any trailers!
 572         while True:
 573             line = self.fp.readline()
 574             if line == '\r\n':
 575                 break
 576
 577         # we read everything; close the "file"
 578         self.close()
 579
 580         return value
 581
 582     def _safe_read(self, amt):
 583         """Read the number of bytes requested, compensating for partial reads.
 584
 585         Normally, we have a blocking socket, but a read() can be interrupted
 586         by a signal (resulting in a partial read).
 587
 588         Note that we cannot distinguish between EOF and an interrupt when zero
 589         bytes have been read. IncompleteRead() will be raised in this
 590         situation.
 591
 592         This function should be used when <amt> bytes "should" be present for
 593         reading. If the bytes are truly not available (due to EOF), then the
 594         IncompleteRead exception can be used to detect the problem.
 595         """
 596         s = []
 597         while amt > 0:
 598             chunk = self.fp.read(min(amt, MAXAMOUNT))
 599             if not chunk:
 600                 raise IncompleteRead(s)
 601             s.append(chunk)
 602             amt -= len(chunk)
 603         return ''.join(s)
 604
 605     def getheader(self, name, default=None):
 606         if self.msg is None:
 607             raise ResponseNotReady()
 608         return self.msg.getheader(name, default)
 609
 610     def getheaders(self):
 611         """Return list of (header, value) tuples."""
 612         if self.msg is None:
 613             raise ResponseNotReady()
 614         return self.msg.items()
 615
 616
 617 class HTTPConnection:
 618
 619     _http_vsn = 11
 620     _http_vsn_str = 'HTTP/1.1'
 621
 622     response_class = HTTPResponse
 623     default_port = HTTP_PORT
 624     auto_open = 1
 625     debuglevel = 0
 626     strict = 0
 627
 628     def __init__(self, host, port=None, strict=None):
 629         self.sock = None
 630         self._buffer = []
 631         self.__response = None
 632         self.__state = _CS_IDLE
 633         self._method = None
 634
 635         self._set_hostport(host, port)
 636         if strict is not None:
 637             self.strict = strict
 638
 639     def _set_hostport(self, host, port):
 640         if port is None:
 641             i = host.rfind(':')
 642             j = host.rfind(']')         # ipv6 addresses have [...]
 643             if i > j:
 644                 try:
 645                     port = int(host[i+1:])
 646                 except ValueError:
 647                     raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
 648                 host = host[:i]
 649             else:
 650                 port = self.default_port
 651             if host and host[0] == '[' and host[-1] == ']':
 652                 host = host[1:-1]
 653         self.host = host
 654         self.port = port
 655
 656     def set_debuglevel(self, level):
 657         self.debuglevel = level
 658
 659     def connect(self):
 660         """Connect to the host and port specified in __init__."""
 661         msg = "getaddrinfo returns an empty list"
 662         for res in socket.getaddrinfo(self.host, self.port, 0,
 663                                       socket.SOCK_STREAM):
 664             af, socktype, proto, canonname, sa = res
 665             try:
 666                 self.sock = socket.socket(af, socktype, proto)
 667                 if self.debuglevel > 0:
 668                     print "connect: (%s, %s)" % (self.host, self.port)
 669                 self.sock.connect(sa)
 670             except socket.error, msg:
 671                 if self.debuglevel > 0:
 672                     print 'connect fail:', (self.host, self.port)
 673                 if self.sock:
 674                     self.sock.close()
 675                 self.sock = None
 676                 continue
 677             break
 678         if not self.sock:
 679             raise socket.error, msg
 680
 681     def close(self):
 682         """Close the connection to the HTTP server."""
 683         if self.sock:
 684             self.sock.close()   # close it manually... there may be other refs
 685             self.sock = None
 686         if self.__response:
 687             self.__response.close()
 688             self.__response = None
 689         self.__state = _CS_IDLE
 690
 691     def send(self, str):
 692         """Send `str' to the server."""
 693         if self.sock is None:
 694             if self.auto_open:
 695                 self.connect()
 696             else:
 697                 raise NotConnected()
 698
 699         # send the data to the server. if we get a broken pipe, then close
 700         # the socket. we want to reconnect when somebody tries to send again.
 701         #
 702         # NOTE: we DO propagate the error, though, because we cannot simply
 703         #       ignore the error... the caller will know if they can retry.
 704         if self.debuglevel > 0:
 705             print "send:", repr(str)
 706         try:
 707             self.sock.sendall(str)
 708         except socket.error, v:
 709             if v[0] == 32:      # Broken pipe
 710                 self.close()
 711             raise
 712
 713     def _output(self, s):
 714         """Add a line of output to the current request buffer.
 715
 716         Assumes that the line does *not* end with \\r\\n.
 717         """
 718         self._buffer.append(s)
 719
 720     def _send_output(self):
 721         """Send the currently buffered request and clear the buffer.
 722
 723         Appends an extra \\r\\n to the buffer.
 724         """
 725         self._buffer.extend(("", ""))
 726         msg = "\r\n".join(self._buffer)
 727         del self._buffer[:]
 728         self.send(msg)
 729
 730     def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
 731         """Send a request to the server.
 732
 733         `method' specifies an HTTP request method, e.g. 'GET'.
 734         `url' specifies the object being requested, e.g. '/index.html'.
 735         `skip_host' if True does not add automatically a 'Host:' header
 736         `skip_accept_encoding' if True does not add automatically an
 737            'Accept-Encoding:' header
 738         """
 739
 740         # if a prior response has been completed, then forget about it.
 741         if self.__response and self.__response.isclosed():
 742             self.__response = None
 743
 744
 745         # in certain cases, we cannot issue another request on this connection.
 746         # this occurs when:
 747         #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
 748         #   2) a response to a previous request has signalled that it is going
 749         #      to close the connection upon completion.
 750         #   3) the headers for the previous response have not been read, thus
 751         #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
 752         #
 753         # if there is no prior response, then we can request at will.
 754         #
 755         # if point (2) is true, then we will have passed the socket to the
 756         # response (effectively meaning, "there is no prior response"), and
 757         # will open a new one when a new request is made.
 758         #
 759         # Note: if a prior response exists, then we *can* start a new request.
 760         #       We are not allowed to begin fetching the response to this new
 761         #       request, however, until that prior response is complete.
 762         #
 763         if self.__state == _CS_IDLE:
 764             self.__state = _CS_REQ_STARTED
 765         else:
 766             raise CannotSendRequest()
 767
 768         # Save the method we use, we need it later in the response phase
 769         self._method = method
 770         if not url:
 771             url = '/'
 772         str = '%s %s %s' % (method, url, self._http_vsn_str)
 773
 774         self._output(str)
 775
 776         if self._http_vsn == 11:
 777             # Issue some standard headers for better HTTP/1.1 compliance
 778
 779             if not skip_host:
 780                 # this header is issued *only* for HTTP/1.1
 781                 # connections. more specifically, this means it is
 782                 # only issued when the client uses the new
 783                 # HTTPConnection() class. backwards-compat clients
 784                 # will be using HTTP/1.0 and those clients may be
 785                 # issuing this header themselves. we should NOT issue
 786                 # it twice; some web servers (such as Apache) barf
 787                 # when they see two Host: headers
 788
 789                 # If we need a non-standard port,include it in the
 790                 # header.  If the request is going through a proxy,
 791                 # but the host of the actual URL, not the host of the
 792                 # proxy.
 793
 794                 netloc = ''
 795                 if url.startswith('http'):
 796                     nil, netloc, nil, nil, nil = urlsplit(url)
 797
 798                 if netloc:
 799                     try:
 800                         netloc_enc = netloc.encode("ascii")
 801                     except UnicodeEncodeError:
 802                         netloc_enc = netloc.encode("idna")
 803                     self.putheader('Host', netloc_enc)
 804                 else:
 805                     try:
 806                         host_enc = self.host.encode("ascii")
 807                     except UnicodeEncodeError:
 808                         host_enc = self.host.encode("idna")
 809                     if self.port == HTTP_PORT:
 810                         self.putheader('Host', host_enc)
 811                     else:
 812                         self.putheader('Host', "%s:%s" % (host_enc, self.port))
 813
 814             # note: we are assuming that clients will not attempt to set these
 815             #       headers since *this* library must deal with the
 816             #       consequences. this also means that when the supporting
 817             #       libraries are updated to recognize other forms, then this
 818             #       code should be changed (removed or updated).
 819
 820             # we only want a Content-Encoding of "identity" since we don't
 821             # support encodings such as x-gzip or x-deflate.
 822             if not skip_accept_encoding:
 823                 self.putheader('Accept-Encoding', 'identity')
 824
 825             # we can accept "chunked" Transfer-Encodings, but no others
 826             # NOTE: no TE header implies *only* "chunked"
 827             #self.putheader('TE', 'chunked')
 828
 829             # if TE is supplied in the header, then it must appear in a
 830             # Connection header.
 831             #self.putheader('Connection', 'TE')
 832
 833         else:
 834             # For HTTP/1.0, the server will assume "not chunked"
 835             pass
 836
 837     def putheader(self, header, value):
 838         """Send a request header line to the server.
 839
 840         For example: h.putheader('Accept', 'text/html')
 841         """
 842         if self.__state != _CS_REQ_STARTED:
 843             raise CannotSendHeader()
 844
 845         str = '%s: %s' % (header, value)
 846         self._output(str)
 847
 848     def endheaders(self):
 849         """Indicate that the last header line has been sent to the server."""
 850
 851         if self.__state == _CS_REQ_STARTED:
 852             self.__state = _CS_REQ_SENT
 853         else:
 854             raise CannotSendHeader()
 855
 856         self._send_output()
 857
 858     def request(self, method, url, body=None, headers={}):
 859         """Send a complete request to the server."""
 860
 861         try:
 862             self._send_request(method, url, body, headers)
 863         except socket.error, v:
 864             # trap 'Broken pipe' if we're allowed to automatically reconnect
 865             if v[0] != 32 or not self.auto_open:
 866                 raise
 867             # try one more time
 868             self._send_request(method, url, body, headers)
 869
 870     def _send_request(self, method, url, body, headers):
 871         # honour explicitly requested Host: and Accept-Encoding headers
 872         header_names = dict.fromkeys([k.lower() for k in headers])
 873         skips = {}
 874         if 'host' in header_names:
 875             skips['skip_host'] = 1
 876         if 'accept-encoding' in header_names:
 877             skips['skip_accept_encoding'] = 1
 878
 879         self.putrequest(method, url, **skips)
 880
 881         if body and ('content-length' not in header_names):
 882             self.putheader('Content-Length', str(len(body)))
 883         for hdr, value in headers.iteritems():
 884             self.putheader(hdr, value)
 885         self.endheaders()
 886
 887         if body:
 888             self.send(body)
 889
 890     def getresponse(self):
 891         "Get the response from the server."
 892
 893         # if a prior response has been completed, then forget about it.
 894         if self.__response and self.__response.isclosed():
 895             self.__response = None
 896
 897         #
 898         # if a prior response exists, then it must be completed (otherwise, we
 899         # cannot read this response's header to determine the connection-close
 900         # behavior)
 901         #
 902         # note: if a prior response existed, but was connection-close, then the
 903         # socket and response were made independent of this HTTPConnection
 904         # object since a new request requires that we open a whole new
 905         # connection
 906         #
 907         # this means the prior response had one of two states:
 908         #   1) will_close: this connection was reset and the prior socket and
 909         #                  response operate independently
 910         #   2) persistent: the response was retained and we await its
 911         #                  isclosed() status to become true.
 912         #
 913         if self.__state != _CS_REQ_SENT or self.__response:
 914             raise ResponseNotReady()
 915
 916         if self.debuglevel > 0:
 917             response = self.response_class(self.sock, self.debuglevel,
 918                                            strict=self.strict,
 919                                            method=self._method)
 920         else:
 921             response = self.response_class(self.sock, strict=self.strict,
 922                                            method=self._method)
 923
 924         response.begin()
 925         assert response.will_close != _UNKNOWN
 926         self.__state = _CS_IDLE
 927
 928         if response.will_close:
 929             # this effectively passes the connection to the response
 930             self.close()
 931         else:
 932             # remember this, so we can tell when it is complete
 933             self.__response = response
 934
 935         return response
 936
 937 # The next several classes are used to define FakeSocket,a socket-like
 938 # interface to an SSL connection.
 939
 940 # The primary complexity comes from faking a makefile() method.  The
 941 # standard socket makefile() implementation calls dup() on the socket
 942 # file descriptor.  As a consequence, clients can call close() on the
 943 # parent socket and its makefile children in any order.  The underlying
 944 # socket isn't closed until they are all closed.
 945
 946 # The implementation uses reference counting to keep the socket open
 947 # until the last client calls close().  SharedSocket keeps track of
 948 # the reference counting and SharedSocketClient provides an constructor
 949 # and close() method that call incref() and decref() correctly.
 950
 951 class SharedSocket:
 952
 953     def __init__(self, sock):
 954         self.sock = sock
 955         self._refcnt = 0
 956
 957     def incref(self):
 958         self._refcnt += 1
 959
 960     def decref(self):
 961         self._refcnt -= 1
 962         assert self._refcnt >= 0
 963         if self._refcnt == 0:
 964             self.sock.close()
 965
 966     def __del__(self):
 967         self.sock.close()
 968
 969 class SharedSocketClient:
 970
 971     def __init__(self, shared):
 972         self._closed = 0
 973         self._shared = shared
 974         self._shared.incref()
 975         self._sock = shared.sock
 976
 977     def close(self):
 978         if not self._closed:
 979             self._shared.decref()
 980             self._closed = 1
 981             self._shared = None
 982
 983 class SSLFile(SharedSocketClient):
 984     """File-like object wrapping an SSL socket."""
 985
 986     BUFSIZE = 8192
 987
 988     def __init__(self, sock, ssl, bufsize=None):
 989         SharedSocketClient.__init__(self, sock)
 990         self._ssl = ssl
 991         self._buf = ''
 992         self._bufsize = bufsize or self.__class__.BUFSIZE
 993
 994     def _read(self):
 995         buf = ''
 996         # put in a loop so that we retry on transient errors
 997         while True:
 998             try:
 999                 buf = self._ssl.read(self._bufsize)
1000             except socket.sslerror, err:
1001                 if (err[0] == socket.SSL_ERROR_WANT_READ
1002                     or err[0] == socket.SSL_ERROR_WANT_WRITE):
1003                     continue
1004                 if (err[0] == socket.SSL_ERROR_ZERO_RETURN
1005                     or err[0] == socket.SSL_ERROR_EOF):
1006                     break
1007                 raise
1008             except socket.error, err:
1009                 if err[0] == errno.EINTR:
1010                     continue
1011                 if err[0] == errno.EBADF:
1012                     # XXX socket was closed?
1013                     break
1014                 raise
1015             else:
1016                 break
1017         return buf
1018
1019     def read(self, size=None):
1020         L = [self._buf]
1021         avail = len(self._buf)
1022         while size is None or avail < size:
1023             s = self._read()
1024             if s == '':
1025                 break
1026             L.append(s)
1027             avail += len(s)
1028         all = "".join(L)
1029         if size is None:
1030             self._buf = ''
1031             return all
1032         else:
1033             self._buf = all[size:]
1034             return all[:size]
1035
1036     def readline(self):
1037         L = [self._buf]
1038         self._buf = ''
1039         while 1:
1040             i = L[-1].find("\n")
1041             if i >= 0:
1042                 break
1043             s = self._read()
1044             if s == '':
1045                 break
1046             L.append(s)
1047         if i == -1:
1048             # loop exited because there is no more data
1049             return "".join(L)
1050         else:
1051             all = "".join(L)
1052             # XXX could do enough bookkeeping not to do a 2nd search
1053             i = all.find("\n") + 1
1054             line = all[:i]
1055             self._buf = all[i:]
1056             return line
1057
1058     def readlines(self, sizehint=0):
1059         total = 0
1060         list = []
1061         while True:
1062             line = self.readline()
1063             if not line:
1064                 break
1065             list.append(line)
1066             total += len(line)
1067             if sizehint and total >= sizehint:
1068                 break
1069         return list
1070
1071     def fileno(self):
1072         return self._sock.fileno()
1073
1074     def __iter__(self):
1075         return self
1076
1077     def next(self):
1078         line = self.readline()
1079         if not line:
1080             raise StopIteration
1081         return line
1082
1083 class FakeSocket(SharedSocketClient):
1084
1085     class _closedsocket:
1086         def __getattr__(self, name):
1087             raise error(9, 'Bad file descriptor')
1088
1089     def __init__(self, sock, ssl):
1090         sock = SharedSocket(sock)
1091         SharedSocketClient.__init__(self, sock)
1092         self._ssl = ssl
1093
1094     def close(self):
1095         SharedSocketClient.close(self)
1096         self._sock = self.__class__._closedsocket()
1097
1098     def makefile(self, mode, bufsize=None):
1099         if mode != 'r' and mode != 'rb':
1100             raise UnimplementedFileMode()
1101         return SSLFile(self._shared, self._ssl, bufsize)
1102
1103     def send(self, stuff, flags = 0):
1104         return self._ssl.write(stuff)
1105
1106     sendall = send
1107
1108     def recv(self, len = 1024, flags = 0):
1109         return self._ssl.read(len)
1110
1111     def __getattr__(self, attr):
1112         return getattr(self._sock, attr)
1113
1114
1115 class HTTPSConnection(HTTPConnection):
1116     "This class allows communication via SSL."
1117
1118     default_port = HTTPS_PORT
1119
1120     def __init__(self, host, port=None, key_file=None, cert_file=None,
1121                  strict=None):
1122         HTTPConnection.__init__(self, host, port, strict)
1123         self.key_file = key_file
1124         self.cert_file = cert_file
1125
1126     def connect(self):
1127         "Connect to a host on a given (SSL) port."
1128
1129         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1130         sock.connect((self.host, self.port))
1131         ssl = socket.ssl(sock, self.key_file, self.cert_file)
1132         self.sock = FakeSocket(sock, ssl)
1133
1134
1135 class HTTP:
1136     "Compatibility class with httplib.py from 1.5."
1137
1138     _http_vsn = 10
1139     _http_vsn_str = 'HTTP/1.0'
1140
1141     debuglevel = 0
1142
1143     _connection_class = HTTPConnection
1144
1145     def __init__(self, host='', port=None, strict=None):
1146         "Provide a default host, since the superclass requires one."
1147
1148         # some joker passed 0 explicitly, meaning default port
1149         if port == 0:
1150             port = None
1151
1152         # Note that we may pass an empty string as the host; this will throw
1153         # an error when we attempt to connect. Presumably, the client code
1154         # will call connect before then, with a proper host.
1155         self._setup(self._connection_class(host, port, strict))
1156
1157     def _setup(self, conn):
1158         self._conn = conn
1159
1160         # set up delegation to flesh out interface
1161         self.send = conn.send
1162         self.putrequest = conn.putrequest
1163         self.endheaders = conn.endheaders
1164         self.set_debuglevel = conn.set_debuglevel
1165
1166         conn._http_vsn = self._http_vsn
1167         conn._http_vsn_str = self._http_vsn_str
1168
1169         self.file = None
1170
1171     def connect(self, host=None, port=None):
1172         "Accept arguments to set the host/port, since the superclass doesn't."
1173
1174         if host is not None:
1175             self._conn._set_hostport(host, port)
1176         self._conn.connect()
1177
1178     def getfile(self):
1179         "Provide a getfile, since the superclass' does not use this concept."
1180         return self.file
1181
1182     def putheader(self, header, *values):
1183         "The superclass allows only one value argument."
1184         self._conn.putheader(header, '\r\n\t'.join(values))
1185
1186     def getreply(self):
1187         """Compat definition since superclass does not define it.
1188
1189         Returns a tuple consisting of:
1190         - server status code (e.g. '200' if all goes well)
1191         - server "reason" corresponding to status code
1192         - any RFC822 headers in the response from the server
1193         """
1194         try:
1195             response = self._conn.getresponse()
1196         except BadStatusLine, e:
1197             ### hmm. if getresponse() ever closes the socket on a bad request,
1198             ### then we are going to have problems with self.sock
1199
1200             ### should we keep this behavior? do people use it?
1201             # keep the socket open (as a file), and return it
1202             self.file = self._conn.sock.makefile('rb', 0)
1203
1204             # close our socket -- we want to restart after any protocol error
1205             self.close()
1206
1207             self.headers = None
1208             return -1, e.line, None
1209
1210         self.headers = response.msg
1211         self.file = response.fp
1212         return response.status, response.reason, response.msg
1213
1214     def close(self):
1215         self._conn.close()
1216
1217         # note that self.file == response.fp, which gets closed by the
1218         # superclass. just clear the object ref here.
1219         ### hmm. messy. if status==-1, then self.file is owned by us.
1220         ### well... we aren't explicitly closing, but losing this ref will
1221         ### do it
1222         self.file = None
1223
1224 if hasattr(socket, 'ssl'):
1225     class HTTPS(HTTP):
1226         """Compatibility with 1.5 httplib interface
1227
1228         Python 1.5.2 did not have an HTTPS class, but it defined an
1229         interface for sending http requests that is also useful for
1230         https.
1231         """
1232
1233         _connection_class = HTTPSConnection
1234
1235         def __init__(self, host='', port=None, key_file=None, cert_file=None,
1236                      strict=None):
1237             # provide a default host, pass the X509 cert info
1238
1239             # urf. compensate for bad input.
1240             if port == 0:
1241                 port = None
1242             self._setup(self._connection_class(host, port, key_file,
1243                                                cert_file, strict))
1244
1245             # we never actually use these for anything, but we keep them
1246             # here for compatibility with post-1.5.2 CVS.
1247             self.key_file = key_file
1248             self.cert_file = cert_file
1249
1250
1251 class HTTPException(Exception):
1252     # Subclasses that define an __init__ must call Exception.__init__
1253     # or define self.args.  Otherwise, str() will fail.
1254     pass
1255
1256 class NotConnected(HTTPException):
1257     pass
1258
1259 class InvalidURL(HTTPException):
1260     pass
1261
1262 class UnknownProtocol(HTTPException):
1263     def __init__(self, version):
1264         self.args = version,
1265         self.version = version
1266
1267 class UnknownTransferEncoding(HTTPException):
1268     pass
1269
1270 class UnimplementedFileMode(HTTPException):
1271     pass
1272
1273 class IncompleteRead(HTTPException):
1274     def __init__(self, partial):
1275         self.args = partial,
1276         self.partial = partial
1277
1278 class ImproperConnectionState(HTTPException):
1279     pass
1280
1281 class CannotSendRequest(ImproperConnectionState):
1282     pass
1283
1284 class CannotSendHeader(ImproperConnectionState):
1285     pass
1286
1287 class ResponseNotReady(ImproperConnectionState):
1288     pass
1289
1290 class BadStatusLine(HTTPException):
1291     def __init__(self, line):
1292         self.args = line,
1293         self.line = line
1294
1295 # for backwards compatibility
1296 error = HTTPException
1297
1298 class LineAndFileWrapper:
1299     """A limited file-like object for HTTP/0.9 responses."""
1300
1301     # The status-line parsing code calls readline(), which normally
1302     # get the HTTP status line.  For a 0.9 response, however, this is
1303     # actually the first line of the body!  Clients need to get a
1304     # readable file object that contains that line.
1305
1306     def __init__(self, line, file):
1307         self._line = line
1308         self._file = file
1309         self._line_consumed = 0
1310         self._line_offset = 0
1311         self._line_left = len(line)
1312
1313     def __getattr__(self, attr):
1314         return getattr(self._file, attr)
1315
1316     def _done(self):
1317         # called when the last byte is read from the line.  After the
1318         # call, all read methods are delegated to the underlying file
1319         # object.
1320         self._line_consumed = 1
1321         self.read = self._file.read
1322         self.readline = self._file.readline
1323         self.readlines = self._file.readlines
1324
1325     def read(self, amt=None):
1326         if self._line_consumed:
1327             return self._file.read(amt)
1328         assert self._line_left
1329         if amt is None or amt > self._line_left:
1330             s = self._line[self._line_offset:]
1331             self._done()
1332             if amt is None:
1333                 return s + self._file.read()
1334             else:
1335                 return s + self._file.read(amt - len(s))
1336         else:
1337             assert amt <= self._line_left
1338             i = self._line_offset
1339             j = i + amt
1340             s = self._line[i:j]
1341             self._line_offset = j
1342             self._line_left -= amt
1343             if self._line_left == 0:
1344                 self._done()
1345             return s
1346
1347     def readline(self):
1348         if self._line_consumed:
1349             return self._file.readline()
1350         assert self._line_left
1351         s = self._line[self._line_offset:]
1352         self._done()
1353         return s
1354
1355     def readlines(self, size=None):
1356         if self._line_consumed:
1357             return self._file.readlines(size)
1358         assert self._line_left
1359         L = [self._line[self._line_offset:]]
1360         self._done()
1361         if size is None:
1362             return L + self._file.readlines()
1363         else:
1364             return L + self._file.readlines(size)
1365
1366 def test():
1367     """Test this module.
1368
1369     A hodge podge of tests collected here, because they have too many
1370     external dependencies for the regular test suite.
1371     """
1372
1373     import sys
1374     import getopt
1375     opts, args = getopt.getopt(sys.argv[1:], 'd')
1376     dl = 0
1377     for o, a in opts:
1378         if o == '-d': dl = dl + 1
1379     host = 'www.python.org'
1380     selector = '/'
1381     if args[0:]: host = args[0]
1382     if args[1:]: selector = args[1]
1383     h = HTTP()
1384     h.set_debuglevel(dl)
1385     h.connect(host)
1386     h.putrequest('GET', selector)
1387     h.endheaders()
1388     status, reason, headers = h.getreply()
1389     print 'status =', status
1390     print 'reason =', reason
1391     print "read", len(h.getfile().read())
1392     print
1393     if headers:
1394         for header in headers.headers: print header.strip()
1395     print
1396
1397     # minimal test that code to extract host from url works
1398     class HTTP11(HTTP):
1399         _http_vsn = 11
1400         _http_vsn_str = 'HTTP/1.1'
1401
1402     h = HTTP11('www.python.org')
1403     h.putrequest('GET', 'http://www.python.org/~jeremy/')
1404     h.endheaders()
1405     h.getreply()
1406     h.close()
1407
1408     if hasattr(socket, 'ssl'):
1409
1410         for host, selector in (('sourceforge.net', '/projects/python'),
1411                                ):
1412             print "https://%s%s" % (host, selector)
1413             hs = HTTPS()
1414             hs.set_debuglevel(dl)
1415             hs.connect(host)
1416             hs.putrequest('GET', selector)
1417             hs.endheaders()
1418             status, reason, headers = hs.getreply()
1419             print 'status =', status
1420             print 'reason =', reason
1421             print "read", len(hs.getfile().read())
1422             print
1423             if headers:
1424                 for header in headers.headers: print header.strip()
1425             print
1426
1427 if __name__ == '__main__':
1428     test()