third_party/python/requests/requests/adapters.py

   1 """
   2 requests.adapters
   3 ~~~~~~~~~~~~~~~~~
   4
   5 This module contains the transport adapters that Requests uses to define
   6 and maintain connections.
   7 """
   8
   9 import os.path
  10 import socket  # noqa: F401
  11
  12 from urllib3.exceptions import ClosedPoolError, ConnectTimeoutError
  13 from urllib3.exceptions import HTTPError as _HTTPError
  14 from urllib3.exceptions import InvalidHeader as _InvalidHeader
  15 from urllib3.exceptions import (
  16     LocationValueError,
  17     MaxRetryError,
  18     NewConnectionError,
  19     ProtocolError,
  20 )
  21 from urllib3.exceptions import ProxyError as _ProxyError
  22 from urllib3.exceptions import ReadTimeoutError, ResponseError
  23 from urllib3.exceptions import SSLError as _SSLError
  24 from urllib3.poolmanager import PoolManager, proxy_from_url
  25 from urllib3.util import Timeout as TimeoutSauce
  26 from urllib3.util import parse_url
  27 from urllib3.util.retry import Retry
  28
  29 from .auth import _basic_auth_str
  30 from .compat import basestring, urlparse
  31 from .cookies import extract_cookies_to_jar
  32 from .exceptions import (
  33     ConnectionError,
  34     ConnectTimeout,
  35     InvalidHeader,
  36     InvalidProxyURL,
  37     InvalidSchema,
  38     InvalidURL,
  39     ProxyError,
  40     ReadTimeout,
  41     RetryError,
  42     SSLError,
  43 )
  44 from .models import Response
  45 from .structures import CaseInsensitiveDict
  46 from .utils import (
  47     DEFAULT_CA_BUNDLE_PATH,
  48     extract_zipped_paths,
  49     get_auth_from_url,
  50     get_encoding_from_headers,
  51     prepend_scheme_if_needed,
  52     select_proxy,
  53     urldefragauth,
  54 )
  55
  56 try:
  57     from urllib3.contrib.socks import SOCKSProxyManager
  58 except ImportError:
  59
  60     def SOCKSProxyManager(*args, **kwargs):
  61         raise InvalidSchema("Missing dependencies for SOCKS support.")
  62
  63
  64 DEFAULT_POOLBLOCK = False
  65 DEFAULT_POOLSIZE = 10
  66 DEFAULT_RETRIES = 0
  67 DEFAULT_POOL_TIMEOUT = None
  68
  69
  70 class BaseAdapter:
  71     """The Base Transport Adapter"""
  72
  73     def __init__(self):
  74         super().__init__()
  75
  76     def send(
  77         self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
  78     ):
  79         """Sends PreparedRequest object. Returns Response object.
  80
  81         :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
  82         :param stream: (optional) Whether to stream the request content.
  83         :param timeout: (optional) How long to wait for the server to send
  84             data before giving up, as a float, or a :ref:`(connect timeout,
  85             read timeout) <timeouts>` tuple.
  86         :type timeout: float or tuple
  87         :param verify: (optional) Either a boolean, in which case it controls whether we verify
  88             the server's TLS certificate, or a string, in which case it must be a path
  89             to a CA bundle to use
  90         :param cert: (optional) Any user-provided SSL certificate to be trusted.
  91         :param proxies: (optional) The proxies dictionary to apply to the request.
  92         """
  93         raise NotImplementedError
  94
  95     def close(self):
  96         """Cleans up adapter specific items."""
  97         raise NotImplementedError
  98
  99
 100 class HTTPAdapter(BaseAdapter):
 101     """The built-in HTTP Adapter for urllib3.
 102
 103     Provides a general-case interface for Requests sessions to contact HTTP and
 104     HTTPS urls by implementing the Transport Adapter interface. This class will
 105     usually be created by the :class:`Session <Session>` class under the
 106     covers.
 107
 108     :param pool_connections: The number of urllib3 connection pools to cache.
 109     :param pool_maxsize: The maximum number of connections to save in the pool.
 110     :param max_retries: The maximum number of retries each connection
 111         should attempt. Note, this applies only to failed DNS lookups, socket
 112         connections and connection timeouts, never to requests where data has
 113         made it to the server. By default, Requests does not retry failed
 114         connections. If you need granular control over the conditions under
 115         which we retry a request, import urllib3's ``Retry`` class and pass
 116         that instead.
 117     :param pool_block: Whether the connection pool should block for connections.
 118
 119     Usage::
 120
 121       >>> import requests
 122       >>> s = requests.Session()
 123       >>> a = requests.adapters.HTTPAdapter(max_retries=3)
 124       >>> s.mount('http://', a)
 125     """
 126
 127     __attrs__ = [
 128         "max_retries",
 129         "config",
 130         "_pool_connections",
 131         "_pool_maxsize",
 132         "_pool_block",
 133     ]
 134
 135     def __init__(
 136         self,
 137         pool_connections=DEFAULT_POOLSIZE,
 138         pool_maxsize=DEFAULT_POOLSIZE,
 139         max_retries=DEFAULT_RETRIES,
 140         pool_block=DEFAULT_POOLBLOCK,
 141     ):
 142         if max_retries == DEFAULT_RETRIES:
 143             self.max_retries = Retry(0, read=False)
 144         else:
 145             self.max_retries = Retry.from_int(max_retries)
 146         self.config = {}
 147         self.proxy_manager = {}
 148
 149         super().__init__()
 150
 151         self._pool_connections = pool_connections
 152         self._pool_maxsize = pool_maxsize
 153         self._pool_block = pool_block
 154
 155         self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)
 156
 157     def __getstate__(self):
 158         return {attr: getattr(self, attr, None) for attr in self.__attrs__}
 159
 160     def __setstate__(self, state):
 161         # Can't handle by adding 'proxy_manager' to self.__attrs__ because
 162         # self.poolmanager uses a lambda function, which isn't pickleable.
 163         self.proxy_manager = {}
 164         self.config = {}
 165
 166         for attr, value in state.items():
 167             setattr(self, attr, value)
 168
 169         self.init_poolmanager(
 170             self._pool_connections, self._pool_maxsize, block=self._pool_block
 171         )
 172
 173     def init_poolmanager(
 174         self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs
 175     ):
 176         """Initializes a urllib3 PoolManager.
 177
 178         This method should not be called from user code, and is only
 179         exposed for use when subclassing the
 180         :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
 181
 182         :param connections: The number of urllib3 connection pools to cache.
 183         :param maxsize: The maximum number of connections to save in the pool.
 184         :param block: Block when no free connections are available.
 185         :param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager.
 186         """
 187         # save these values for pickling
 188         self._pool_connections = connections
 189         self._pool_maxsize = maxsize
 190         self._pool_block = block
 191
 192         self.poolmanager = PoolManager(
 193             num_pools=connections,
 194             maxsize=maxsize,
 195             block=block,
 196             **pool_kwargs,
 197         )
 198
 199     def proxy_manager_for(self, proxy, **proxy_kwargs):
 200         """Return urllib3 ProxyManager for the given proxy.
 201
 202         This method should not be called from user code, and is only
 203         exposed for use when subclassing the
 204         :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
 205
 206         :param proxy: The proxy to return a urllib3 ProxyManager for.
 207         :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager.
 208         :returns: ProxyManager
 209         :rtype: urllib3.ProxyManager
 210         """
 211         if proxy in self.proxy_manager:
 212             manager = self.proxy_manager[proxy]
 213         elif proxy.lower().startswith("socks"):
 214             username, password = get_auth_from_url(proxy)
 215             manager = self.proxy_manager[proxy] = SOCKSProxyManager(
 216                 proxy,
 217                 username=username,
 218                 password=password,
 219                 num_pools=self._pool_connections,
 220                 maxsize=self._pool_maxsize,
 221                 block=self._pool_block,
 222                 **proxy_kwargs,
 223             )
 224         else:
 225             proxy_headers = self.proxy_headers(proxy)
 226             manager = self.proxy_manager[proxy] = proxy_from_url(
 227                 proxy,
 228                 proxy_headers=proxy_headers,
 229                 num_pools=self._pool_connections,
 230                 maxsize=self._pool_maxsize,
 231                 block=self._pool_block,
 232                 **proxy_kwargs,
 233             )
 234
 235         return manager
 236
 237     def cert_verify(self, conn, url, verify, cert):
 238         """Verify a SSL certificate. This method should not be called from user
 239         code, and is only exposed for use when subclassing the
 240         :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
 241
 242         :param conn: The urllib3 connection object associated with the cert.
 243         :param url: The requested URL.
 244         :param verify: Either a boolean, in which case it controls whether we verify
 245             the server's TLS certificate, or a string, in which case it must be a path
 246             to a CA bundle to use
 247         :param cert: The SSL certificate to verify.
 248         """
 249         if url.lower().startswith("https") and verify:
 250
 251             cert_loc = None
 252
 253             # Allow self-specified cert location.
 254             if verify is not True:
 255                 cert_loc = verify
 256
 257             if not cert_loc:
 258                 cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH)
 259
 260             if not cert_loc or not os.path.exists(cert_loc):
 261                 raise OSError(
 262                     f"Could not find a suitable TLS CA certificate bundle, "
 263                     f"invalid path: {cert_loc}"
 264                 )
 265
 266             conn.cert_reqs = "CERT_REQUIRED"
 267
 268             if not os.path.isdir(cert_loc):
 269                 conn.ca_certs = cert_loc
 270             else:
 271                 conn.ca_cert_dir = cert_loc
 272         else:
 273             conn.cert_reqs = "CERT_NONE"
 274             conn.ca_certs = None
 275             conn.ca_cert_dir = None
 276
 277         if cert:
 278             if not isinstance(cert, basestring):
 279                 conn.cert_file = cert[0]
 280                 conn.key_file = cert[1]
 281             else:
 282                 conn.cert_file = cert
 283                 conn.key_file = None
 284             if conn.cert_file and not os.path.exists(conn.cert_file):
 285                 raise OSError(
 286                     f"Could not find the TLS certificate file, "
 287                     f"invalid path: {conn.cert_file}"
 288                 )
 289             if conn.key_file and not os.path.exists(conn.key_file):
 290                 raise OSError(
 291                     f"Could not find the TLS key file, invalid path: {conn.key_file}"
 292                 )
 293
 294     def build_response(self, req, resp):
 295         """Builds a :class:`Response <requests.Response>` object from a urllib3
 296         response. This should not be called from user code, and is only exposed
 297         for use when subclassing the
 298         :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
 299
 300         :param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
 301         :param resp: The urllib3 response object.
 302         :rtype: requests.Response
 303         """
 304         response = Response()
 305
 306         # Fallback to None if there's no status_code, for whatever reason.
 307         response.status_code = getattr(resp, "status", None)
 308
 309         # Make headers case-insensitive.
 310         response.headers = CaseInsensitiveDict(getattr(resp, "headers", {}))
 311
 312         # Set encoding.
 313         response.encoding = get_encoding_from_headers(response.headers)
 314         response.raw = resp
 315         response.reason = response.raw.reason
 316
 317         if isinstance(req.url, bytes):
 318             response.url = req.url.decode("utf-8")
 319         else:
 320             response.url = req.url
 321
 322         # Add new cookies from the server.
 323         extract_cookies_to_jar(response.cookies, req, resp)
 324
 325         # Give the Response some context.
 326         response.request = req
 327         response.connection = self
 328
 329         return response
 330
 331     def get_connection(self, url, proxies=None):
 332         """Returns a urllib3 connection for the given URL. This should not be
 333         called from user code, and is only exposed for use when subclassing the
 334         :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
 335
 336         :param url: The URL to connect to.
 337         :param proxies: (optional) A Requests-style dictionary of proxies used on this request.
 338         :rtype: urllib3.ConnectionPool
 339         """
 340         proxy = select_proxy(url, proxies)
 341
 342         if proxy:
 343             proxy = prepend_scheme_if_needed(proxy, "http")
 344             proxy_url = parse_url(proxy)
 345             if not proxy_url.host:
 346                 raise InvalidProxyURL(
 347                     "Please check proxy URL. It is malformed "
 348                     "and could be missing the host."
 349                 )
 350             proxy_manager = self.proxy_manager_for(proxy)
 351             conn = proxy_manager.connection_from_url(url)
 352         else:
 353             # Only scheme should be lower case
 354             parsed = urlparse(url)
 355             url = parsed.geturl()
 356             conn = self.poolmanager.connection_from_url(url)
 357
 358         return conn
 359
 360     def close(self):
 361         """Disposes of any internal state.
 362
 363         Currently, this closes the PoolManager and any active ProxyManager,
 364         which closes any pooled connections.
 365         """
 366         self.poolmanager.clear()
 367         for proxy in self.proxy_manager.values():
 368             proxy.clear()
 369
 370     def request_url(self, request, proxies):
 371         """Obtain the url to use when making the final request.
 372
 373         If the message is being sent through a HTTP proxy, the full URL has to
 374         be used. Otherwise, we should only use the path portion of the URL.
 375
 376         This should not be called from user code, and is only exposed for use
 377         when subclassing the
 378         :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
 379
 380         :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
 381         :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs.
 382         :rtype: str
 383         """
 384         proxy = select_proxy(request.url, proxies)
 385         scheme = urlparse(request.url).scheme
 386
 387         is_proxied_http_request = proxy and scheme != "https"
 388         using_socks_proxy = False
 389         if proxy:
 390             proxy_scheme = urlparse(proxy).scheme.lower()
 391             using_socks_proxy = proxy_scheme.startswith("socks")
 392
 393         url = request.path_url
 394         if is_proxied_http_request and not using_socks_proxy:
 395             url = urldefragauth(request.url)
 396
 397         return url
 398
 399     def add_headers(self, request, **kwargs):
 400         """Add any headers needed by the connection. As of v2.0 this does
 401         nothing by default, but is left for overriding by users that subclass
 402         the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
 403
 404         This should not be called from user code, and is only exposed for use
 405         when subclassing the
 406         :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
 407
 408         :param request: The :class:`PreparedRequest <PreparedRequest>` to add headers to.
 409         :param kwargs: The keyword arguments from the call to send().
 410         """
 411         pass
 412
 413     def proxy_headers(self, proxy):
 414         """Returns a dictionary of the headers to add to any request sent
 415         through a proxy. This works with urllib3 magic to ensure that they are
 416         correctly sent to the proxy, rather than in a tunnelled request if
 417         CONNECT is being used.
 418
 419         This should not be called from user code, and is only exposed for use
 420         when subclassing the
 421         :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
 422
 423         :param proxy: The url of the proxy being used for this request.
 424         :rtype: dict
 425         """
 426         headers = {}
 427         username, password = get_auth_from_url(proxy)
 428
 429         if username:
 430             headers["Proxy-Authorization"] = _basic_auth_str(username, password)
 431
 432         return headers
 433
 434     def send(
 435         self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
 436     ):
 437         """Sends PreparedRequest object. Returns Response object.
 438
 439         :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
 440         :param stream: (optional) Whether to stream the request content.
 441         :param timeout: (optional) How long to wait for the server to send
 442             data before giving up, as a float, or a :ref:`(connect timeout,
 443             read timeout) <timeouts>` tuple.
 444         :type timeout: float or tuple or urllib3 Timeout object
 445         :param verify: (optional) Either a boolean, in which case it controls whether
 446             we verify the server's TLS certificate, or a string, in which case it
 447             must be a path to a CA bundle to use
 448         :param cert: (optional) Any user-provided SSL certificate to be trusted.
 449         :param proxies: (optional) The proxies dictionary to apply to the request.
 450         :rtype: requests.Response
 451         """
 452
 453         try:
 454             conn = self.get_connection(request.url, proxies)
 455         except LocationValueError as e:
 456             raise InvalidURL(e, request=request)
 457
 458         self.cert_verify(conn, request.url, verify, cert)
 459         url = self.request_url(request, proxies)
 460         self.add_headers(
 461             request,
 462             stream=stream,
 463             timeout=timeout,
 464             verify=verify,
 465             cert=cert,
 466             proxies=proxies,
 467         )
 468
 469         chunked = not (request.body is None or "Content-Length" in request.headers)
 470
 471         if isinstance(timeout, tuple):
 472             try:
 473                 connect, read = timeout
 474                 timeout = TimeoutSauce(connect=connect, read=read)
 475             except ValueError:
 476                 raise ValueError(
 477                     f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
 478                     f"or a single float to set both timeouts to the same value."
 479                 )
 480         elif isinstance(timeout, TimeoutSauce):
 481             pass
 482         else:
 483             timeout = TimeoutSauce(connect=timeout, read=timeout)
 484
 485         try:
 486             resp = conn.urlopen(
 487                 method=request.method,
 488                 url=url,
 489                 body=request.body,
 490                 headers=request.headers,
 491                 redirect=False,
 492                 assert_same_host=False,
 493                 preload_content=False,
 494                 decode_content=False,
 495                 retries=self.max_retries,
 496                 timeout=timeout,
 497                 chunked=chunked,
 498             )
 499
 500         except (ProtocolError, OSError) as err:
 501             raise ConnectionError(err, request=request)
 502
 503         except MaxRetryError as e:
 504             if isinstance(e.reason, ConnectTimeoutError):
 505                 # TODO: Remove this in 3.0.0: see #2811
 506                 if not isinstance(e.reason, NewConnectionError):
 507                     raise ConnectTimeout(e, request=request)
 508
 509             if isinstance(e.reason, ResponseError):
 510                 raise RetryError(e, request=request)
 511
 512             if isinstance(e.reason, _ProxyError):
 513                 raise ProxyError(e, request=request)
 514
 515             if isinstance(e.reason, _SSLError):
 516                 # This branch is for urllib3 v1.22 and later.
 517                 raise SSLError(e, request=request)
 518
 519             raise ConnectionError(e, request=request)
 520
 521         except ClosedPoolError as e:
 522             raise ConnectionError(e, request=request)
 523
 524         except _ProxyError as e:
 525             raise ProxyError(e)
 526
 527         except (_SSLError, _HTTPError) as e:
 528             if isinstance(e, _SSLError):
 529                 # This branch is for urllib3 versions earlier than v1.22
 530                 raise SSLError(e, request=request)
 531             elif isinstance(e, ReadTimeoutError):
 532                 raise ReadTimeout(e, request=request)
 533             elif isinstance(e, _InvalidHeader):
 534                 raise InvalidHeader(e, request=request)
 535             else:
 536                 raise
 537
 538         return self.build_response(request, resp)