testing/web-platform/tests/tools/third_party/hyper/hyper/http11/response.py

   1 # -*- coding: utf-8 -*-
   2 """
   3 hyper/http11/response
   4 ~~~~~~~~~~~~~~~~~~~~~
   5
   6 Contains the HTTP/1.1 equivalent of the HTTPResponse object defined in
   7 httplib/http.client.
   8 """
   9 import logging
  10 import weakref
  11 import zlib
  12
  13 from ..common.decoder import DeflateDecoder
  14 from ..common.exceptions import ChunkedDecodeError, InvalidResponseError
  15 from ..common.exceptions import ConnectionResetError
  16
  17 log = logging.getLogger(__name__)
  18
  19
  20 class HTTP11Response(object):
  21     """
  22     An ``HTTP11Response`` wraps the HTTP/1.1 response from the server. It
  23     provides access to the response headers and the entity body. The response
  24     is an iterable object and can be used in a with statement.
  25     """
  26     def __init__(self, code, reason, headers, sock, connection=None):
  27         #: The reason phrase returned by the server.
  28         self.reason = reason
  29
  30         #: The status code returned by the server.
  31         self.status = code
  32
  33         #: The response headers. These are determined upon creation, assigned
  34         #: once, and never assigned again.
  35         self.headers = headers
  36
  37         #: The response trailers. These are always intially ``None``.
  38         self.trailers = None
  39
  40         # The socket this response is being sent over.
  41         self._sock = sock
  42
  43         # Whether we expect the connection to be closed. If we do, we don't
  44         # bother checking for content-length, we just keep reading until
  45         # we no longer can.
  46         self._expect_close = False
  47         if b'close' in self.headers.get(b'connection', []):
  48             self._expect_close = True
  49
  50         # The expected length of the body.
  51         try:
  52             self._length = int(self.headers[b'content-length'][0])
  53         except KeyError:
  54             self._length = None
  55
  56         # Whether we expect a chunked response.
  57         self._chunked = (
  58             b'chunked' in self.headers.get(b'transfer-encoding', [])
  59         )
  60
  61         # One of the following must be true: we must expect that the connection
  62         # will be closed following the body, or that a content-length was sent,
  63         # or that we're getting a chunked response.
  64         # FIXME: Remove naked assert, replace with something better.
  65         assert self._expect_close or self._length is not None or self._chunked
  66
  67         # This object is used for decompressing gzipped request bodies. Right
  68         # now we only support gzip because that's all the RFC mandates of us.
  69         # Later we'll add support for more encodings.
  70         # This 16 + MAX_WBITS nonsense is to force gzip. See this
  71         # Stack Overflow answer for more:
  72         # http://stackoverflow.com/a/2695466/1401686
  73         if b'gzip' in self.headers.get(b'content-encoding', []):
  74             self._decompressobj = zlib.decompressobj(16 + zlib.MAX_WBITS)
  75         elif b'deflate' in self.headers.get(b'content-encoding', []):
  76             self._decompressobj = DeflateDecoder()
  77         else:
  78             self._decompressobj = None
  79
  80         # This is a reference that allows for the Response class to tell the
  81         # parent connection object to throw away its socket object. This is to
  82         # be used when the connection is genuinely closed, so that the user
  83         # can keep using the Connection object.
  84         # Strictly, we take a weakreference to this so that we don't set up a
  85         # reference cycle.
  86         if connection is not None:
  87             self._parent = weakref.ref(connection)
  88         else:
  89             self._parent = None
  90
  91         self._buffered_data = b''
  92         self._chunker = None
  93
  94     def read(self, amt=None, decode_content=True):
  95         """
  96         Reads the response body, or up to the next ``amt`` bytes.
  97
  98         :param amt: (optional) The amount of data to read. If not provided, all
  99             the data will be read from the response.
 100         :param decode_content: (optional) If ``True``, will transparently
 101             decode the response data.
 102         :returns: The read data. Note that if ``decode_content`` is set to
 103             ``True``, the actual amount of data returned may be different to
 104             the amount requested.
 105         """
 106         # Return early if we've lost our connection.
 107         if self._sock is None:
 108             return b''
 109
 110         if self._chunked:
 111             return self._normal_read_chunked(amt, decode_content)
 112
 113         # If we're asked to do a read without a length, we need to read
 114         # everything. That means either the entire content length, or until the
 115         # socket is closed, depending.
 116         if amt is None:
 117             if self._length is not None:
 118                 amt = self._length
 119             elif self._expect_close:
 120                 return self._read_expect_closed(decode_content)
 121             else:  # pragma: no cover
 122                 raise InvalidResponseError(
 123                     "Response must either have length or Connection: close"
 124                 )
 125
 126         # Otherwise, we've been asked to do a bounded read. We should read no
 127         # more than the remaining length, obviously.
 128         # FIXME: Handle cases without _length
 129         if self._length is not None:
 130             amt = min(amt, self._length)
 131
 132         # Now, issue reads until we read that length. This is to account for
 133         # the fact that it's possible that we'll be asked to read more than
 134         # 65kB in one shot.
 135         to_read = amt
 136         chunks = []
 137
 138         # Ideally I'd like this to read 'while to_read', but I want to be
 139         # defensive against the admittedly unlikely case that the socket
 140         # returns *more* data than I want.
 141         while to_read > 0:
 142             chunk = self._sock.recv(amt).tobytes()
 143
 144             # If we got an empty read, but were expecting more, the remote end
 145             # has hung up. Raise an exception if we were expecting more data,
 146             # but if we were expecting the remote end to close then it's ok.
 147             if not chunk:
 148                 if self._length is not None or not self._expect_close:
 149                     self.close(socket_close=True)
 150                     raise ConnectionResetError("Remote end hung up!")
 151
 152                 break
 153
 154             to_read -= len(chunk)
 155             chunks.append(chunk)
 156
 157         data = b''.join(chunks)
 158         if self._length is not None:
 159             self._length -= len(data)
 160
 161         # If we're at the end of the request, we have some cleaning up to do.
 162         # Close the stream, and if necessary flush the buffer. Checking that
 163         # we're at the end is actually obscenely complex: either we've read the
 164         # full content-length or, if we were expecting a closed connection,
 165         # we've had a read shorter than the requested amount. We also have to
 166         # do this before we try to decompress the body.
 167         end_of_request = (self._length == 0 or
 168                           (self._expect_close and len(data) < amt))
 169
 170         # We may need to decode the body.
 171         if decode_content and self._decompressobj and data:
 172             data = self._decompressobj.decompress(data)
 173
 174         if decode_content and self._decompressobj and end_of_request:
 175             data += self._decompressobj.flush()
 176
 177         # We're at the end. Close the connection. Explicit check for zero here
 178         # because self._length might be None.
 179         if end_of_request:
 180             self.close(socket_close=self._expect_close)
 181
 182         return data
 183
 184     def read_chunked(self, decode_content=True):
 185         """
 186         Reads chunked transfer encoded bodies. This method returns a generator:
 187         each iteration of which yields one chunk *unless* the chunks are
 188         compressed, in which case it yields whatever the decompressor provides
 189         for each chunk.
 190
 191         .. warning:: This may yield the empty string, without that being the
 192                      end of the body!
 193         """
 194         if not self._chunked:
 195             raise ChunkedDecodeError(
 196                 "Attempted chunked read of non-chunked body."
 197             )
 198
 199         # Return early if possible.
 200         if self._sock is None:
 201             return
 202
 203         while True:
 204             # Read to the newline to get the chunk length. This is a
 205             # hexadecimal integer.
 206             chunk_length = int(self._sock.readline().tobytes().strip(), 16)
 207             data = b''
 208
 209             # If the chunk length is zero, consume the newline and then we're
 210             # done. If we were decompressing data, return the remaining data.
 211             if not chunk_length:
 212                 self._sock.readline()
 213
 214                 if decode_content and self._decompressobj:
 215                     yield self._decompressobj.flush()
 216
 217                 self.close(socket_close=self._expect_close)
 218                 break
 219
 220             # Then read that many bytes.
 221             while chunk_length > 0:
 222                 chunk = self._sock.recv(chunk_length).tobytes()
 223                 data += chunk
 224                 chunk_length -= len(chunk)
 225
 226             assert chunk_length == 0
 227
 228             # Now, consume the newline.
 229             self._sock.readline()
 230
 231             # We may need to decode the body.
 232             if decode_content and self._decompressobj and data:
 233                 data = self._decompressobj.decompress(data)
 234
 235             yield data
 236
 237         return
 238
 239     def close(self, socket_close=False):
 240         """
 241         Close the response. This causes the Response to lose access to the
 242         backing socket. In some cases, it can also cause the backing connection
 243         to be torn down.
 244
 245         :param socket_close: Whether to close the backing socket.
 246         :returns: Nothing.
 247         """
 248         if socket_close and self._parent is not None:
 249             # The double call is necessary because we need to dereference the
 250             # weakref. If the weakref is no longer valid, that's fine, there's
 251             # no connection object to tell.
 252             parent = self._parent()
 253             if parent is not None:
 254                 parent.close()
 255
 256         self._sock = None
 257
 258     def _read_expect_closed(self, decode_content):
 259         """
 260         Implements the logic for an unbounded read on a socket that we expect
 261         to be closed by the remote end.
 262         """
 263         # In this case, just read until we cannot read anymore. Then, close the
 264         # socket, becuase we know we have to.
 265         chunks = []
 266         while True:
 267             try:
 268                 chunk = self._sock.recv(65535).tobytes()
 269                 if not chunk:
 270                     break
 271             except ConnectionResetError:
 272                 break
 273             else:
 274                 chunks.append(chunk)
 275
 276         self.close(socket_close=True)
 277
 278         # We may need to decompress the data.
 279         data = b''.join(chunks)
 280         if decode_content and self._decompressobj:
 281             data = self._decompressobj.decompress(data)
 282             data += self._decompressobj.flush()
 283
 284         return data
 285
 286     def _normal_read_chunked(self, amt, decode_content):
 287         """
 288         Implements the logic for calling ``read()`` on a chunked response.
 289         """
 290         # If we're doing a full read, read it as chunked and then just join
 291         # the chunks together!
 292         if amt is None:
 293             return self._buffered_data + b''.join(self.read_chunked())
 294
 295         if self._chunker is None:
 296             self._chunker = self.read_chunked()
 297
 298         # Otherwise, we have a certain amount of data we want to read.
 299         current_amount = len(self._buffered_data)
 300
 301         extra_data = [self._buffered_data]
 302         while current_amount < amt:
 303             try:
 304                 chunk = next(self._chunker)
 305             except StopIteration:
 306                 self.close(socket_close=self._expect_close)
 307                 break
 308
 309             current_amount += len(chunk)
 310             extra_data.append(chunk)
 311
 312         data = b''.join(extra_data)
 313         self._buffered_data = data[amt:]
 314         return data[:amt]
 315
 316     # The following methods implement the context manager protocol.
 317     def __enter__(self):
 318         return self
 319
 320     def __exit__(self, *args):
 321         self.close()
 322         return False  # Never swallow exceptions.