Lib/httplib.py

   1 """HTTP/1.1 client library
   2
   3 <intro stuff goes here>
   4 <other stuff, too>
   5
   6 HTTPConnection goes through a number of "states", which define when a client
   7 may legally make another request or fetch the response for a particular
   8 request. This diagram details these state transitions:
   9
  10     (null)
  11       |
  12       | HTTPConnection()
  13       v
  14     Idle
  15       |
  16       | putrequest()
  17       v
  18     Request-started
  19       |
  20       | ( putheader() )*  endheaders()
  21       v
  22     Request-sent
  23       |
  24       | response = getresponse()
  25       v
  26     Unread-response   [Response-headers-read]
  27       |\____________________
  28       |                     |
  29       | response.read()     | putrequest()
  30       v                     v
  31     Idle                  Req-started-unread-response
  32                      ______/|
  33                    /        |
  34    response.read() |        | ( putheader() )*  endheaders()
  35                    v        v
  36        Request-started    Req-sent-unread-response
  37                             |
  38                             | response.read()
  39                             v
  40                           Request-sent
  41
  42 This diagram presents the following rules:
  43   -- a second request may not be started until {response-headers-read}
  44   -- a response [object] cannot be retrieved until {request-sent}
  45   -- there is no differentiation between an unread response body and a
  46      partially read response body
  47
  48 Note: this enforcement is applied by the HTTPConnection class. The
  49       HTTPResponse class does not enforce this state machine, which
  50       implies sophisticated clients may accelerate the request/response
  51       pipeline. Caution should be taken, though: accelerating the states
  52       beyond the above pattern may imply knowledge of the server's
  53       connection-close behavior for certain requests. For example, it
  54       is impossible to tell whether the server will close the connection
  55       UNTIL the response headers have been read; this means that further
  56       requests cannot be placed into the pipeline until it is known that
  57       the server will NOT be closing the connection.
  58
  59 Logical State                  __state            __response
  60 -------------                  -------            ----------
  61 Idle                           _CS_IDLE           None
  62 Request-started                _CS_REQ_STARTED    None
  63 Request-sent                   _CS_REQ_SENT       None
  64 Unread-response                _CS_IDLE           <response_class>
  65 Req-started-unread-response    _CS_REQ_STARTED    <response_class>
  66 Req-sent-unread-response       _CS_REQ_SENT       <response_class>
  67 """
  68
  69 import mimetools
  70 import socket
  71 from urlparse import urlsplit
  72 import warnings
  73
  74 try:
  75     from cStringIO import StringIO
  76 except ImportError:
  77     from StringIO import StringIO
  78
  79 __all__ = ["HTTP", "HTTPResponse", "HTTPConnection",
  80            "HTTPException", "NotConnected", "UnknownProtocol",
  81            "UnknownTransferEncoding", "UnimplementedFileMode",
  82            "IncompleteRead", "InvalidURL", "ImproperConnectionState",
  83            "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
  84            "BadStatusLine", "error", "responses"]
  85
  86 HTTP_PORT = 80
  87 HTTPS_PORT = 443
  88
  89 _UNKNOWN = 'UNKNOWN'
  90
  91 # connection states
  92 _CS_IDLE = 'Idle'
  93 _CS_REQ_STARTED = 'Request-started'
  94 _CS_REQ_SENT = 'Request-sent'
  95
  96 # status codes
  97 # informational
  98 CONTINUE = 100
  99 SWITCHING_PROTOCOLS = 101
 100 PROCESSING = 102
 101
 102 # successful
 103 OK = 200
 104 CREATED = 201
 105 ACCEPTED = 202
 106 NON_AUTHORITATIVE_INFORMATION = 203
 107 NO_CONTENT = 204
 108 RESET_CONTENT = 205
 109 PARTIAL_CONTENT = 206
 110 MULTI_STATUS = 207
 111 IM_USED = 226
 112
 113 # redirection
 114 MULTIPLE_CHOICES = 300
 115 MOVED_PERMANENTLY = 301
 116 FOUND = 302
 117 SEE_OTHER = 303
 118 NOT_MODIFIED = 304
 119 USE_PROXY = 305
 120 TEMPORARY_REDIRECT = 307
 121
 122 # client error
 123 BAD_REQUEST = 400
 124 UNAUTHORIZED = 401
 125 PAYMENT_REQUIRED = 402
 126 FORBIDDEN = 403
 127 NOT_FOUND = 404
 128 METHOD_NOT_ALLOWED = 405
 129 NOT_ACCEPTABLE = 406
 130 PROXY_AUTHENTICATION_REQUIRED = 407
 131 REQUEST_TIMEOUT = 408
 132 CONFLICT = 409
 133 GONE = 410
 134 LENGTH_REQUIRED = 411
 135 PRECONDITION_FAILED = 412
 136 REQUEST_ENTITY_TOO_LARGE = 413
 137 REQUEST_URI_TOO_LONG = 414
 138 UNSUPPORTED_MEDIA_TYPE = 415
 139 REQUESTED_RANGE_NOT_SATISFIABLE = 416
 140 EXPECTATION_FAILED = 417
 141 UNPROCESSABLE_ENTITY = 422
 142 LOCKED = 423
 143 FAILED_DEPENDENCY = 424
 144 UPGRADE_REQUIRED = 426
 145
 146 # server error
 147 INTERNAL_SERVER_ERROR = 500
 148 NOT_IMPLEMENTED = 501
 149 BAD_GATEWAY = 502
 150 SERVICE_UNAVAILABLE = 503
 151 GATEWAY_TIMEOUT = 504
 152 HTTP_VERSION_NOT_SUPPORTED = 505
 153 INSUFFICIENT_STORAGE = 507
 154 NOT_EXTENDED = 510
 155
 156 # Mapping status codes to official W3C names
 157 responses = {
 158     100: 'Continue',
 159     101: 'Switching Protocols',
 160
 161     200: 'OK',
 162     201: 'Created',
 163     202: 'Accepted',
 164     203: 'Non-Authoritative Information',
 165     204: 'No Content',
 166     205: 'Reset Content',
 167     206: 'Partial Content',
 168
 169     300: 'Multiple Choices',
 170     301: 'Moved Permanently',
 171     302: 'Found',
 172     303: 'See Other',
 173     304: 'Not Modified',
 174     305: 'Use Proxy',
 175     306: '(Unused)',
 176     307: 'Temporary Redirect',
 177
 178     400: 'Bad Request',
 179     401: 'Unauthorized',
 180     402: 'Payment Required',
 181     403: 'Forbidden',
 182     404: 'Not Found',
 183     405: 'Method Not Allowed',
 184     406: 'Not Acceptable',
 185     407: 'Proxy Authentication Required',
 186     408: 'Request Timeout',
 187     409: 'Conflict',
 188     410: 'Gone',
 189     411: 'Length Required',
 190     412: 'Precondition Failed',
 191     413: 'Request Entity Too Large',
 192     414: 'Request-URI Too Long',
 193     415: 'Unsupported Media Type',
 194     416: 'Requested Range Not Satisfiable',
 195     417: 'Expectation Failed',
 196
 197     500: 'Internal Server Error',
 198     501: 'Not Implemented',
 199     502: 'Bad Gateway',
 200     503: 'Service Unavailable',
 201     504: 'Gateway Timeout',
 202     505: 'HTTP Version Not Supported',
 203 }
 204
 205 # maximal amount of data to read at one time in _safe_read
 206 MAXAMOUNT = 1048576
 207
 208 class HTTPMessage(mimetools.Message):
 209
 210     def addheader(self, key, value):
 211         """Add header for field key handling repeats."""
 212         prev = self.dict.get(key)
 213         if prev is None:
 214             self.dict[key] = value
 215         else:
 216             combined = ", ".join((prev, value))
 217             self.dict[key] = combined
 218
 219     def addcontinue(self, key, more):
 220         """Add more field data from a continuation line."""
 221         prev = self.dict[key]
 222         self.dict[key] = prev + "\n " + more
 223
 224     def readheaders(self):
 225         """Read header lines.
 226
 227         Read header lines up to the entirely blank line that terminates them.
 228         The (normally blank) line that ends the headers is skipped, but not
 229         included in the returned list.  If a non-header line ends the headers,
 230         (which is an error), an attempt is made to backspace over it; it is
 231         never included in the returned list.
 232
 233         The variable self.status is set to the empty string if all went well,
 234         otherwise it is an error message.  The variable self.headers is a
 235         completely uninterpreted list of lines contained in the header (so
 236         printing them will reproduce the header exactly as it appears in the
 237         file).
 238
 239         If multiple header fields with the same name occur, they are combined
 240         according to the rules in RFC 2616 sec 4.2:
 241
 242         Appending each subsequent field-value to the first, each separated
 243         by a comma. The order in which header fields with the same field-name
 244         are received is significant to the interpretation of the combined
 245         field value.
 246         """
 247         # XXX The implementation overrides the readheaders() method of
 248         # rfc822.Message.  The base class design isn't amenable to
 249         # customized behavior here so the method here is a copy of the
 250         # base class code with a few small changes.
 251
 252         self.dict = {}
 253         self.unixfrom = ''
 254         self.headers = hlist = []
 255         self.status = ''
 256         headerseen = ""
 257         firstline = 1
 258         startofline = unread = tell = None
 259         if hasattr(self.fp, 'unread'):
 260             unread = self.fp.unread
 261         elif self.seekable:
 262             tell = self.fp.tell
 263         while True:
 264             if tell:
 265                 try:
 266                     startofline = tell()
 267                 except IOError:
 268                     startofline = tell = None
 269                     self.seekable = 0
 270             line = self.fp.readline()
 271             if not line:
 272                 self.status = 'EOF in headers'
 273                 break
 274             # Skip unix From name time lines
 275             if firstline and line.startswith('From '):
 276                 self.unixfrom = self.unixfrom + line
 277                 continue
 278             firstline = 0
 279             if headerseen and line[0] in ' \t':
 280                 # XXX Not sure if continuation lines are handled properly
 281                 # for http and/or for repeating headers
 282                 # It's a continuation line.
 283                 hlist.append(line)
 284                 self.addcontinue(headerseen, line.strip())
 285                 continue
 286             elif self.iscomment(line):
 287                 # It's a comment.  Ignore it.
 288                 continue
 289             elif self.islast(line):
 290                 # Note! No pushback here!  The delimiter line gets eaten.
 291                 break
 292             headerseen = self.isheader(line)
 293             if headerseen:
 294                 # It's a legal header line, save it.
 295                 hlist.append(line)
 296                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
 297                 continue
 298             else:
 299                 # It's not a header line; throw it back and stop here.
 300                 if not self.dict:
 301                     self.status = 'No headers'
 302                 else:
 303                     self.status = 'Non-header line where header expected'
 304                 # Try to undo the read.
 305                 if unread:
 306                     unread(line)
 307                 elif tell:
 308                     self.fp.seek(startofline)
 309                 else:
 310                     self.status = self.status + '; bad seek'
 311                 break
 312
 313 class HTTPResponse:
 314
 315     # strict: If true, raise BadStatusLine if the status line can't be
 316     # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
 317     # false because it prevents clients from talking to HTTP/0.9
 318     # servers.  Note that a response with a sufficiently corrupted
 319     # status line will look like an HTTP/0.9 response.
 320
 321     # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
 322
 323     def __init__(self, sock, debuglevel=0, strict=0, method=None):
 324         self.fp = sock.makefile('rb', 0)
 325         self.debuglevel = debuglevel
 326         self.strict = strict
 327         self._method = method
 328
 329         self.msg = None
 330
 331         # from the Status-Line of the response
 332         self.version = _UNKNOWN # HTTP-Version
 333         self.status = _UNKNOWN  # Status-Code
 334         self.reason = _UNKNOWN  # Reason-Phrase
 335
 336         self.chunked = _UNKNOWN         # is "chunked" being used?
 337         self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
 338         self.length = _UNKNOWN          # number of bytes left in response
 339         self.will_close = _UNKNOWN      # conn will close at end of response
 340
 341     def _read_status(self):
 342         # Initialize with Simple-Response defaults
 343         line = self.fp.readline()
 344         if self.debuglevel > 0:
 345             print "reply:", repr(line)
 346         if not line:
 347             # Presumably, the server closed the connection before
 348             # sending a valid response.
 349             raise BadStatusLine(line)
 350         try:
 351             [version, status, reason] = line.split(None, 2)
 352         except ValueError:
 353             try:
 354                 [version, status] = line.split(None, 1)
 355                 reason = ""
 356             except ValueError:
 357                 # empty version will cause next test to fail and status
 358                 # will be treated as 0.9 response.
 359                 version = ""
 360         if not version.startswith('HTTP/'):
 361             if self.strict:
 362                 self.close()
 363                 raise BadStatusLine(line)
 364             else:
 365                 # assume it's a Simple-Response from an 0.9 server
 366                 self.fp = LineAndFileWrapper(line, self.fp)
 367                 return "HTTP/0.9", 200, ""
 368
 369         # The status code is a three-digit number
 370         try:
 371             status = int(status)
 372             if status < 100 or status > 999:
 373                 raise BadStatusLine(line)
 374         except ValueError:
 375             raise BadStatusLine(line)
 376         return version, status, reason
 377
 378     def begin(self):
 379         if self.msg is not None:
 380             # we've already started reading the response
 381             return
 382
 383         # read until we get a non-100 response
 384         while True:
 385             version, status, reason = self._read_status()
 386             if status != CONTINUE:
 387                 break
 388             # skip the header from the 100 response
 389             while True:
 390                 skip = self.fp.readline().strip()
 391                 if not skip:
 392                     break
 393                 if self.debuglevel > 0:
 394                     print "header:", skip
 395
 396         self.status = status
 397         self.reason = reason.strip()
 398         if version == 'HTTP/1.0':
 399             self.version = 10
 400         elif version.startswith('HTTP/1.'):
 401             self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
 402         elif version == 'HTTP/0.9':
 403             self.version = 9
 404         else:
 405             raise UnknownProtocol(version)
 406
 407         if self.version == 9:
 408             self.length = None
 409             self.chunked = 0
 410             self.will_close = 1
 411             self.msg = HTTPMessage(StringIO())
 412             return
 413
 414         self.msg = HTTPMessage(self.fp, 0)
 415         if self.debuglevel > 0:
 416             for hdr in self.msg.headers:
 417                 print "header:", hdr,
 418
 419         # don't let the msg keep an fp
 420         self.msg.fp = None
 421
 422         # are we using the chunked-style of transfer encoding?
 423         tr_enc = self.msg.getheader('transfer-encoding')
 424         if tr_enc and tr_enc.lower() == "chunked":
 425             self.chunked = 1
 426             self.chunk_left = None
 427         else:
 428             self.chunked = 0
 429
 430         # will the connection close at the end of the response?
 431         self.will_close = self._check_close()
 432
 433         # do we have a Content-Length?
 434         # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
 435         length = self.msg.getheader('content-length')
 436         if length and not self.chunked:
 437             try:
 438                 self.length = int(length)
 439             except ValueError:
 440                 self.length = None
 441             else:
 442                 if self.length < 0:  # ignore nonsensical negative lengths
 443                     self.length = None
 444         else:
 445             self.length = None
 446
 447         # does the body have a fixed length? (of zero)
 448         if (status == NO_CONTENT or status == NOT_MODIFIED or
 449             100 <= status < 200 or      # 1xx codes
 450             self._method == 'HEAD'):
 451             self.length = 0
 452
 453         # if the connection remains open, and we aren't using chunked, and
 454         # a content-length was not provided, then assume that the connection
 455         # WILL close.
 456         if not self.will_close and \
 457            not self.chunked and \
 458            self.length is None:
 459             self.will_close = 1
 460
 461     def _check_close(self):
 462         conn = self.msg.getheader('connection')
 463         if self.version == 11:
 464             # An HTTP/1.1 proxy is assumed to stay open unless
 465             # explicitly closed.
 466             conn = self.msg.getheader('connection')
 467             if conn and "close" in conn.lower():
 468                 return True
 469             return False
 470
 471         # Some HTTP/1.0 implementations have support for persistent
 472         # connections, using rules different than HTTP/1.1.
 473
 474         # For older HTTP, Keep-Alive indicates persistent connection.
 475         if self.msg.getheader('keep-alive'):
 476             return False
 477
 478         # At least Akamai returns a "Connection: Keep-Alive" header,
 479         # which was supposed to be sent by the client.
 480         if conn and "keep-alive" in conn.lower():
 481             return False
 482
 483         # Proxy-Connection is a netscape hack.
 484         pconn = self.msg.getheader('proxy-connection')
 485         if pconn and "keep-alive" in pconn.lower():
 486             return False
 487
 488         # otherwise, assume it will close
 489         return True
 490
 491     def close(self):
 492         if self.fp:
 493             self.fp.close()
 494             self.fp = None
 495
 496     def isclosed(self):
 497         # NOTE: it is possible that we will not ever call self.close(). This
 498         #       case occurs when will_close is TRUE, length is None, and we
 499         #       read up to the last byte, but NOT past it.
 500         #
 501         # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
 502         #          called, meaning self.isclosed() is meaningful.
 503         return self.fp is None
 504
 505     # XXX It would be nice to have readline and __iter__ for this, too.
 506
 507     def read(self, amt=None):
 508         if self.fp is None:
 509             return ''
 510
 511         if self.chunked:
 512             return self._read_chunked(amt)
 513
 514         if amt is None:
 515             # unbounded read
 516             if self.length is None:
 517                 s = self.fp.read()
 518             else:
 519                 s = self._safe_read(self.length)
 520                 self.length = 0
 521             self.close()        # we read everything
 522             return s
 523
 524         if self.length is not None:
 525             if amt > self.length:
 526                 # clip the read to the "end of response"
 527                 amt = self.length
 528
 529         # we do not use _safe_read() here because this may be a .will_close
 530         # connection, and the user is reading more bytes than will be provided
 531         # (for example, reading in 1k chunks)
 532         s = self.fp.read(amt)
 533         if self.length is not None:
 534             self.length -= len(s)
 535             if not self.length:
 536                 self.close()
 537         return s
 538
 539     def _read_chunked(self, amt):
 540         assert self.chunked != _UNKNOWN
 541         chunk_left = self.chunk_left
 542         value = ''
 543
 544         # XXX This accumulates chunks by repeated string concatenation,
 545         # which is not efficient as the number or size of chunks gets big.
 546         while True:
 547             if chunk_left is None:
 548                 line = self.fp.readline()
 549                 i = line.find(';')
 550                 if i >= 0:
 551                     line = line[:i] # strip chunk-extensions
 552                 try:
 553                     chunk_left = int(line, 16)
 554                 except ValueError:
 555                     # close the connection as protocol synchronisation is
 556                     # probably lost
 557                     self.close()
 558                     raise IncompleteRead(value)
 559                 if chunk_left == 0:
 560                     break
 561             if amt is None:
 562                 value += self._safe_read(chunk_left)
 563             elif amt < chunk_left:
 564                 value += self._safe_read(amt)
 565                 self.chunk_left = chunk_left - amt
 566                 return value
 567             elif amt == chunk_left:
 568                 value += self._safe_read(amt)
 569                 self._safe_read(2)  # toss the CRLF at the end of the chunk
 570                 self.chunk_left = None
 571                 return value
 572             else:
 573                 value += self._safe_read(chunk_left)
 574                 amt -= chunk_left
 575
 576             # we read the whole chunk, get another
 577             self._safe_read(2)      # toss the CRLF at the end of the chunk
 578             chunk_left = None
 579
 580         # read and discard trailer up to the CRLF terminator
 581         ### note: we shouldn't have any trailers!
 582         while True:
 583             line = self.fp.readline()
 584             if not line:
 585                 # a vanishingly small number of sites EOF without
 586                 # sending the trailer
 587                 break
 588             if line == '\r\n':
 589                 break
 590
 591         # we read everything; close the "file"
 592         self.close()
 593
 594         return value
 595
 596     def _safe_read(self, amt):
 597         """Read the number of bytes requested, compensating for partial reads.
 598
 599         Normally, we have a blocking socket, but a read() can be interrupted
 600         by a signal (resulting in a partial read).
 601
 602         Note that we cannot distinguish between EOF and an interrupt when zero
 603         bytes have been read. IncompleteRead() will be raised in this
 604         situation.
 605
 606         This function should be used when <amt> bytes "should" be present for
 607         reading. If the bytes are truly not available (due to EOF), then the
 608         IncompleteRead exception can be used to detect the problem.
 609         """
 610         s = []
 611         while amt > 0:
 612             chunk = self.fp.read(min(amt, MAXAMOUNT))
 613             if not chunk:
 614                 raise IncompleteRead(s)
 615             s.append(chunk)
 616             amt -= len(chunk)
 617         return ''.join(s)
 618
 619     def getheader(self, name, default=None):
 620         if self.msg is None:
 621             raise ResponseNotReady()
 622         return self.msg.getheader(name, default)
 623
 624     def getheaders(self):
 625         """Return list of (header, value) tuples."""
 626         if self.msg is None:
 627             raise ResponseNotReady()
 628         return self.msg.items()
 629
 630
 631 class HTTPConnection:
 632
 633     _http_vsn = 11
 634     _http_vsn_str = 'HTTP/1.1'
 635
 636     response_class = HTTPResponse
 637     default_port = HTTP_PORT
 638     auto_open = 1
 639     debuglevel = 0
 640     strict = 0
 641
 642     def __init__(self, host, port=None, strict=None, timeout=None):
 643         self.timeout = timeout
 644         self.sock = None
 645         self._buffer = []
 646         self.__response = None
 647         self.__state = _CS_IDLE
 648         self._method = None
 649
 650         self._set_hostport(host, port)
 651         if strict is not None:
 652             self.strict = strict
 653
 654     def _set_hostport(self, host, port):
 655         if port is None:
 656             i = host.rfind(':')
 657             j = host.rfind(']')         # ipv6 addresses have [...]
 658             if i > j:
 659                 try:
 660                     port = int(host[i+1:])
 661                 except ValueError:
 662                     raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
 663                 host = host[:i]
 664             else:
 665                 port = self.default_port
 666             if host and host[0] == '[' and host[-1] == ']':
 667                 host = host[1:-1]
 668         self.host = host
 669         self.port = port
 670
 671     def set_debuglevel(self, level):
 672         self.debuglevel = level
 673
 674     def connect(self):
 675         """Connect to the host and port specified in __init__."""
 676         self.sock = socket.create_connection((self.host,self.port),
 677                                              self.timeout)
 678
 679     def close(self):
 680         """Close the connection to the HTTP server."""
 681         if self.sock:
 682             self.sock.close()   # close it manually... there may be other refs
 683             self.sock = None
 684         if self.__response:
 685             self.__response.close()
 686             self.__response = None
 687         self.__state = _CS_IDLE
 688
 689     def send(self, str):
 690         """Send `str' to the server."""
 691         if self.sock is None:
 692             if self.auto_open:
 693                 self.connect()
 694             else:
 695                 raise NotConnected()
 696
 697         # send the data to the server. if we get a broken pipe, then close
 698         # the socket. we want to reconnect when somebody tries to send again.
 699         #
 700         # NOTE: we DO propagate the error, though, because we cannot simply
 701         #       ignore the error... the caller will know if they can retry.
 702         if self.debuglevel > 0:
 703             print "send:", repr(str)
 704         try:
 705             blocksize=8192
 706             if hasattr(str,'read') :
 707                 if self.debuglevel > 0: print "sendIng a read()able"
 708                 data=str.read(blocksize)
 709                 while data:
 710                     self.sock.sendall(data)
 711                     data=str.read(blocksize)
 712             else:
 713                 self.sock.sendall(str)
 714         except socket.error, v:
 715             if v[0] == 32:      # Broken pipe
 716                 self.close()
 717             raise
 718
 719     def _output(self, s):
 720         """Add a line of output to the current request buffer.
 721
 722         Assumes that the line does *not* end with \\r\\n.
 723         """
 724         self._buffer.append(s)
 725
 726     def _send_output(self):
 727         """Send the currently buffered request and clear the buffer.
 728
 729         Appends an extra \\r\\n to the buffer.
 730         """
 731         self._buffer.extend(("", ""))
 732         msg = "\r\n".join(self._buffer)
 733         del self._buffer[:]
 734         self.send(msg)
 735
 736     def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
 737         """Send a request to the server.
 738
 739         `method' specifies an HTTP request method, e.g. 'GET'.
 740         `url' specifies the object being requested, e.g. '/index.html'.
 741         `skip_host' if True does not add automatically a 'Host:' header
 742         `skip_accept_encoding' if True does not add automatically an
 743            'Accept-Encoding:' header
 744         """
 745
 746         # if a prior response has been completed, then forget about it.
 747         if self.__response and self.__response.isclosed():
 748             self.__response = None
 749
 750
 751         # in certain cases, we cannot issue another request on this connection.
 752         # this occurs when:
 753         #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
 754         #   2) a response to a previous request has signalled that it is going
 755         #      to close the connection upon completion.
 756         #   3) the headers for the previous response have not been read, thus
 757         #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
 758         #
 759         # if there is no prior response, then we can request at will.
 760         #
 761         # if point (2) is true, then we will have passed the socket to the
 762         # response (effectively meaning, "there is no prior response"), and
 763         # will open a new one when a new request is made.
 764         #
 765         # Note: if a prior response exists, then we *can* start a new request.
 766         #       We are not allowed to begin fetching the response to this new
 767         #       request, however, until that prior response is complete.
 768         #
 769         if self.__state == _CS_IDLE:
 770             self.__state = _CS_REQ_STARTED
 771         else:
 772             raise CannotSendRequest()
 773
 774         # Save the method we use, we need it later in the response phase
 775         self._method = method
 776         if not url:
 777             url = '/'
 778         str = '%s %s %s' % (method, url, self._http_vsn_str)
 779
 780         self._output(str)
 781
 782         if self._http_vsn == 11:
 783             # Issue some standard headers for better HTTP/1.1 compliance
 784
 785             if not skip_host:
 786                 # this header is issued *only* for HTTP/1.1
 787                 # connections. more specifically, this means it is
 788                 # only issued when the client uses the new
 789                 # HTTPConnection() class. backwards-compat clients
 790                 # will be using HTTP/1.0 and those clients may be
 791                 # issuing this header themselves. we should NOT issue
 792                 # it twice; some web servers (such as Apache) barf
 793                 # when they see two Host: headers
 794
 795                 # If we need a non-standard port,include it in the
 796                 # header.  If the request is going through a proxy,
 797                 # but the host of the actual URL, not the host of the
 798                 # proxy.
 799
 800                 netloc = ''
 801                 if url.startswith('http'):
 802                     nil, netloc, nil, nil, nil = urlsplit(url)
 803
 804                 if netloc:
 805                     try:
 806                         netloc_enc = netloc.encode("ascii")
 807                     except UnicodeEncodeError:
 808                         netloc_enc = netloc.encode("idna")
 809                     self.putheader('Host', netloc_enc)
 810                 else:
 811                     try:
 812                         host_enc = self.host.encode("ascii")
 813                     except UnicodeEncodeError:
 814                         host_enc = self.host.encode("idna")
 815                     if self.port == HTTP_PORT:
 816                         self.putheader('Host', host_enc)
 817                     else:
 818                         self.putheader('Host', "%s:%s" % (host_enc, self.port))
 819
 820             # note: we are assuming that clients will not attempt to set these
 821             #       headers since *this* library must deal with the
 822             #       consequences. this also means that when the supporting
 823             #       libraries are updated to recognize other forms, then this
 824             #       code should be changed (removed or updated).
 825
 826             # we only want a Content-Encoding of "identity" since we don't
 827             # support encodings such as x-gzip or x-deflate.
 828             if not skip_accept_encoding:
 829                 self.putheader('Accept-Encoding', 'identity')
 830
 831             # we can accept "chunked" Transfer-Encodings, but no others
 832             # NOTE: no TE header implies *only* "chunked"
 833             #self.putheader('TE', 'chunked')
 834
 835             # if TE is supplied in the header, then it must appear in a
 836             # Connection header.
 837             #self.putheader('Connection', 'TE')
 838
 839         else:
 840             # For HTTP/1.0, the server will assume "not chunked"
 841             pass
 842
 843     def putheader(self, header, value):
 844         """Send a request header line to the server.
 845
 846         For example: h.putheader('Accept', 'text/html')
 847         """
 848         if self.__state != _CS_REQ_STARTED:
 849             raise CannotSendHeader()
 850
 851         str = '%s: %s' % (header, value)
 852         self._output(str)
 853
 854     def endheaders(self):
 855         """Indicate that the last header line has been sent to the server."""
 856
 857         if self.__state == _CS_REQ_STARTED:
 858             self.__state = _CS_REQ_SENT
 859         else:
 860             raise CannotSendHeader()
 861
 862         self._send_output()
 863
 864     def request(self, method, url, body=None, headers={}):
 865         """Send a complete request to the server."""
 866
 867         try:
 868             self._send_request(method, url, body, headers)
 869         except socket.error, v:
 870             # trap 'Broken pipe' if we're allowed to automatically reconnect
 871             if v[0] != 32 or not self.auto_open:
 872                 raise
 873             # try one more time
 874             self._send_request(method, url, body, headers)
 875
 876     def _send_request(self, method, url, body, headers):
 877         # honour explicitly requested Host: and Accept-Encoding headers
 878         header_names = dict.fromkeys([k.lower() for k in headers])
 879         skips = {}
 880         if 'host' in header_names:
 881             skips['skip_host'] = 1
 882         if 'accept-encoding' in header_names:
 883             skips['skip_accept_encoding'] = 1
 884
 885         self.putrequest(method, url, **skips)
 886
 887         if body and ('content-length' not in header_names):
 888             thelen=None
 889             try:
 890                 thelen=str(len(body))
 891             except TypeError, te:
 892                 # If this is a file-like object, try to
 893                 # fstat its file descriptor
 894                 import os
 895                 try:
 896                     thelen = str(os.fstat(body.fileno()).st_size)
 897                 except (AttributeError, OSError):
 898                     # Don't send a length if this failed
 899                     if self.debuglevel > 0: print "Cannot stat!!"
 900
 901             if thelen is not None:
 902                 self.putheader('Content-Length',thelen)
 903         for hdr, value in headers.iteritems():
 904             self.putheader(hdr, value)
 905         self.endheaders()
 906
 907         if body:
 908             self.send(body)
 909
 910     def getresponse(self):
 911         "Get the response from the server."
 912
 913         # if a prior response has been completed, then forget about it.
 914         if self.__response and self.__response.isclosed():
 915             self.__response = None
 916
 917         #
 918         # if a prior response exists, then it must be completed (otherwise, we
 919         # cannot read this response's header to determine the connection-close
 920         # behavior)
 921         #
 922         # note: if a prior response existed, but was connection-close, then the
 923         # socket and response were made independent of this HTTPConnection
 924         # object since a new request requires that we open a whole new
 925         # connection
 926         #
 927         # this means the prior response had one of two states:
 928         #   1) will_close: this connection was reset and the prior socket and
 929         #                  response operate independently
 930         #   2) persistent: the response was retained and we await its
 931         #                  isclosed() status to become true.
 932         #
 933         if self.__state != _CS_REQ_SENT or self.__response:
 934             raise ResponseNotReady()
 935
 936         if self.debuglevel > 0:
 937             response = self.response_class(self.sock, self.debuglevel,
 938                                            strict=self.strict,
 939                                            method=self._method)
 940         else:
 941             response = self.response_class(self.sock, strict=self.strict,
 942                                            method=self._method)
 943
 944         response.begin()
 945         assert response.will_close != _UNKNOWN
 946         self.__state = _CS_IDLE
 947
 948         if response.will_close:
 949             # this effectively passes the connection to the response
 950             self.close()
 951         else:
 952             # remember this, so we can tell when it is complete
 953             self.__response = response
 954
 955         return response
 956
 957
 958 class HTTP:
 959     "Compatibility class with httplib.py from 1.5."
 960
 961     _http_vsn = 10
 962     _http_vsn_str = 'HTTP/1.0'
 963
 964     debuglevel = 0
 965
 966     _connection_class = HTTPConnection
 967
 968     def __init__(self, host='', port=None, strict=None):
 969         "Provide a default host, since the superclass requires one."
 970
 971         # some joker passed 0 explicitly, meaning default port
 972         if port == 0:
 973             port = None
 974
 975         # Note that we may pass an empty string as the host; this will throw
 976         # an error when we attempt to connect. Presumably, the client code
 977         # will call connect before then, with a proper host.
 978         self._setup(self._connection_class(host, port, strict))
 979
 980     def _setup(self, conn):
 981         self._conn = conn
 982
 983         # set up delegation to flesh out interface
 984         self.send = conn.send
 985         self.putrequest = conn.putrequest
 986         self.endheaders = conn.endheaders
 987         self.set_debuglevel = conn.set_debuglevel
 988
 989         conn._http_vsn = self._http_vsn
 990         conn._http_vsn_str = self._http_vsn_str
 991
 992         self.file = None
 993
 994     def connect(self, host=None, port=None):
 995         "Accept arguments to set the host/port, since the superclass doesn't."
 996
 997         if host is not None:
 998             self._conn._set_hostport(host, port)
 999         self._conn.connect()
1000
1001     def getfile(self):
1002         "Provide a getfile, since the superclass' does not use this concept."
1003         return self.file
1004
1005     def putheader(self, header, *values):
1006         "The superclass allows only one value argument."
1007         self._conn.putheader(header, '\r\n\t'.join(values))
1008
1009     def getreply(self):
1010         """Compat definition since superclass does not define it.
1011
1012         Returns a tuple consisting of:
1013         - server status code (e.g. '200' if all goes well)
1014         - server "reason" corresponding to status code
1015         - any RFC822 headers in the response from the server
1016         """
1017         try:
1018             response = self._conn.getresponse()
1019         except BadStatusLine, e:
1020             ### hmm. if getresponse() ever closes the socket on a bad request,
1021             ### then we are going to have problems with self.sock
1022
1023             ### should we keep this behavior? do people use it?
1024             # keep the socket open (as a file), and return it
1025             self.file = self._conn.sock.makefile('rb', 0)
1026
1027             # close our socket -- we want to restart after any protocol error
1028             self.close()
1029
1030             self.headers = None
1031             return -1, e.line, None
1032
1033         self.headers = response.msg
1034         self.file = response.fp
1035         return response.status, response.reason, response.msg
1036
1037     def close(self):
1038         self._conn.close()
1039
1040         # note that self.file == response.fp, which gets closed by the
1041         # superclass. just clear the object ref here.
1042         ### hmm. messy. if status==-1, then self.file is owned by us.
1043         ### well... we aren't explicitly closing, but losing this ref will
1044         ### do it
1045         self.file = None
1046
1047 try:
1048     import ssl
1049 except ImportError:
1050     pass
1051 else:
1052     class HTTPSConnection(HTTPConnection):
1053         "This class allows communication via SSL."
1054
1055         default_port = HTTPS_PORT
1056
1057         def __init__(self, host, port=None, key_file=None, cert_file=None,
1058                      strict=None, timeout=None):
1059             HTTPConnection.__init__(self, host, port, strict, timeout)
1060             self.key_file = key_file
1061             self.cert_file = cert_file
1062
1063         def connect(self):
1064             "Connect to a host on a given (SSL) port."
1065
1066             sock = socket.create_connection((self.host, self.port), self.timeout)
1067             self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
1068
1069     __all__.append("HTTPSConnection")
1070
1071     class HTTPS(HTTP):
1072         """Compatibility with 1.5 httplib interface
1073
1074         Python 1.5.2 did not have an HTTPS class, but it defined an
1075         interface for sending http requests that is also useful for
1076         https.
1077         """
1078
1079         _connection_class = HTTPSConnection
1080
1081         def __init__(self, host='', port=None, key_file=None, cert_file=None,
1082                      strict=None):
1083             # provide a default host, pass the X509 cert info
1084
1085             # urf. compensate for bad input.
1086             if port == 0:
1087                 port = None
1088             self._setup(self._connection_class(host, port, key_file,
1089                                                cert_file, strict))
1090
1091             # we never actually use these for anything, but we keep them
1092             # here for compatibility with post-1.5.2 CVS.
1093             self.key_file = key_file
1094             self.cert_file = cert_file
1095
1096
1097     def FakeSocket (sock, sslobj):
1098         warnings.warn("FakeSocket is deprecated, and won't be in 3.x.  " +
1099                       "Use the result of ssl.wrap_socket() directly instead.",
1100                       DeprecationWarning, stacklevel=2)
1101         return sslobj
1102
1103
1104 class HTTPException(Exception):
1105     # Subclasses that define an __init__ must call Exception.__init__
1106     # or define self.args.  Otherwise, str() will fail.
1107     pass
1108
1109 class NotConnected(HTTPException):
1110     pass
1111
1112 class InvalidURL(HTTPException):
1113     pass
1114
1115 class UnknownProtocol(HTTPException):
1116     def __init__(self, version):
1117         self.args = version,
1118         self.version = version
1119
1120 class UnknownTransferEncoding(HTTPException):
1121     pass
1122
1123 class UnimplementedFileMode(HTTPException):
1124     pass
1125
1126 class IncompleteRead(HTTPException):
1127     def __init__(self, partial):
1128         self.args = partial,
1129         self.partial = partial
1130
1131 class ImproperConnectionState(HTTPException):
1132     pass
1133
1134 class CannotSendRequest(ImproperConnectionState):
1135     pass
1136
1137 class CannotSendHeader(ImproperConnectionState):
1138     pass
1139
1140 class ResponseNotReady(ImproperConnectionState):
1141     pass
1142
1143 class BadStatusLine(HTTPException):
1144     def __init__(self, line):
1145         self.args = line,
1146         self.line = line
1147
1148 # for backwards compatibility
1149 error = HTTPException
1150
1151 class LineAndFileWrapper:
1152     """A limited file-like object for HTTP/0.9 responses."""
1153
1154     # The status-line parsing code calls readline(), which normally
1155     # get the HTTP status line.  For a 0.9 response, however, this is
1156     # actually the first line of the body!  Clients need to get a
1157     # readable file object that contains that line.
1158
1159     def __init__(self, line, file):
1160         self._line = line
1161         self._file = file
1162         self._line_consumed = 0
1163         self._line_offset = 0
1164         self._line_left = len(line)
1165
1166     def __getattr__(self, attr):
1167         return getattr(self._file, attr)
1168
1169     def _done(self):
1170         # called when the last byte is read from the line.  After the
1171         # call, all read methods are delegated to the underlying file
1172         # object.
1173         self._line_consumed = 1
1174         self.read = self._file.read
1175         self.readline = self._file.readline
1176         self.readlines = self._file.readlines
1177
1178     def read(self, amt=None):
1179         if self._line_consumed:
1180             return self._file.read(amt)
1181         assert self._line_left
1182         if amt is None or amt > self._line_left:
1183             s = self._line[self._line_offset:]
1184             self._done()
1185             if amt is None:
1186                 return s + self._file.read()
1187             else:
1188                 return s + self._file.read(amt - len(s))
1189         else:
1190             assert amt <= self._line_left
1191             i = self._line_offset
1192             j = i + amt
1193             s = self._line[i:j]
1194             self._line_offset = j
1195             self._line_left -= amt
1196             if self._line_left == 0:
1197                 self._done()
1198             return s
1199
1200     def readline(self):
1201         if self._line_consumed:
1202             return self._file.readline()
1203         assert self._line_left
1204         s = self._line[self._line_offset:]
1205         self._done()
1206         return s
1207
1208     def readlines(self, size=None):
1209         if self._line_consumed:
1210             return self._file.readlines(size)
1211         assert self._line_left
1212         L = [self._line[self._line_offset:]]
1213         self._done()
1214         if size is None:
1215             return L + self._file.readlines()
1216         else:
1217             return L + self._file.readlines(size)
1218
1219 def test():
1220     """Test this module.
1221
1222     A hodge podge of tests collected here, because they have too many
1223     external dependencies for the regular test suite.
1224     """
1225
1226     import sys
1227     import getopt
1228     opts, args = getopt.getopt(sys.argv[1:], 'd')
1229     dl = 0
1230     for o, a in opts:
1231         if o == '-d': dl = dl + 1
1232     host = 'www.python.org'
1233     selector = '/'
1234     if args[0:]: host = args[0]
1235     if args[1:]: selector = args[1]
1236     h = HTTP()
1237     h.set_debuglevel(dl)
1238     h.connect(host)
1239     h.putrequest('GET', selector)
1240     h.endheaders()
1241     status, reason, headers = h.getreply()
1242     print 'status =', status
1243     print 'reason =', reason
1244     print "read", len(h.getfile().read())
1245     print
1246     if headers:
1247         for header in headers.headers: print header.strip()
1248     print
1249
1250     # minimal test that code to extract host from url works
1251     class HTTP11(HTTP):
1252         _http_vsn = 11
1253         _http_vsn_str = 'HTTP/1.1'
1254
1255     h = HTTP11('www.python.org')
1256     h.putrequest('GET', 'http://www.python.org/~jeremy/')
1257     h.endheaders()
1258     h.getreply()
1259     h.close()
1260
1261     try:
1262         import ssl
1263     except ImportError:
1264         pass
1265     else:
1266
1267         for host, selector in (('sourceforge.net', '/projects/python'),
1268                                ):
1269             print "https://%s%s" % (host, selector)
1270             hs = HTTPS()
1271             hs.set_debuglevel(dl)
1272             hs.connect(host)
1273             hs.putrequest('GET', selector)
1274             hs.endheaders()
1275             status, reason, headers = hs.getreply()
1276             print 'status =', status
1277             print 'reason =', reason
1278             print "read", len(hs.getfile().read())
1279             print
1280             if headers:
1281                 for header in headers.headers: print header.strip()
1282             print
1283
1284 if __name__ == '__main__':
1285     test()