1 // Copyright 2011 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // HTTP client implementation. See RFC 2616.
7 // This is the low-level Transport implementation of RoundTripper.
8 // The high-level interface is in client.go.
31 "golang_org/x/net/lex/httplex"
32 "golang_org/x/net/proxy"
35 // DefaultTransport is the default implementation of Transport and is
36 // used by DefaultClient. It establishes network connections as needed
37 // and caches them for reuse by subsequent calls. It uses HTTP proxies
38 // as directed by the $HTTP_PROXY and $NO_PROXY (or $http_proxy and
39 // $no_proxy) environment variables.
40 var DefaultTransport RoundTripper
= &Transport
{
41 Proxy
: ProxyFromEnvironment
,
42 DialContext
: (&net
.Dialer
{
43 Timeout
: 30 * time
.Second
,
44 KeepAlive
: 30 * time
.Second
,
48 IdleConnTimeout
: 90 * time
.Second
,
49 TLSHandshakeTimeout
: 10 * time
.Second
,
50 ExpectContinueTimeout
: 1 * time
.Second
,
53 // DefaultMaxIdleConnsPerHost is the default value of Transport's
54 // MaxIdleConnsPerHost.
55 const DefaultMaxIdleConnsPerHost
= 2
57 // Transport is an implementation of RoundTripper that supports HTTP,
58 // HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT).
60 // By default, Transport caches connections for future re-use.
61 // This may leave many open connections when accessing many hosts.
62 // This behavior can be managed using Transport's CloseIdleConnections method
63 // and the MaxIdleConnsPerHost and DisableKeepAlives fields.
65 // Transports should be reused instead of created as needed.
66 // Transports are safe for concurrent use by multiple goroutines.
68 // A Transport is a low-level primitive for making HTTP and HTTPS requests.
69 // For high-level functionality, such as cookies and redirects, see Client.
71 // Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2
72 // for HTTPS URLs, depending on whether the server supports HTTP/2,
73 // and how the Transport is configured. The DefaultTransport supports HTTP/2.
74 // To explicitly enable HTTP/2 on a transport, use golang.org/x/net/http2
75 // and call ConfigureTransport. See the package docs for more about HTTP/2.
76 type Transport
struct {
78 wantIdle
bool // user has requested to close all idle conns
79 idleConn
map[connectMethodKey
][]*persistConn
// most recently used at end
80 idleConnCh
map[connectMethodKey
]chan *persistConn
84 reqCanceler
map[*Request
]func(error
)
86 altMu sync
.Mutex
// guards changing altProto only
87 altProto atomic
.Value
// of nil or map[string]RoundTripper, key is URI scheme
89 // Proxy specifies a function to return a proxy for a given
90 // Request. If the function returns a non-nil error, the
91 // request is aborted with the provided error.
93 // The proxy type is determined by the URL scheme. "http"
94 // and "socks5" are supported. If the scheme is empty,
97 // If Proxy is nil or returns a nil *URL, no proxy is used.
98 Proxy
func(*Request
) (*url
.URL
, error
)
100 // DialContext specifies the dial function for creating unencrypted TCP connections.
101 // If DialContext is nil (and the deprecated Dial below is also nil),
102 // then the transport dials using package net.
103 DialContext
func(ctx context
.Context
, network
, addr
string) (net
.Conn
, error
)
105 // Dial specifies the dial function for creating unencrypted TCP connections.
107 // Deprecated: Use DialContext instead, which allows the transport
108 // to cancel dials as soon as they are no longer needed.
109 // If both are set, DialContext takes priority.
110 Dial
func(network
, addr
string) (net
.Conn
, error
)
112 // DialTLS specifies an optional dial function for creating
113 // TLS connections for non-proxied HTTPS requests.
115 // If DialTLS is nil, Dial and TLSClientConfig are used.
117 // If DialTLS is set, the Dial hook is not used for HTTPS
118 // requests and the TLSClientConfig and TLSHandshakeTimeout
119 // are ignored. The returned net.Conn is assumed to already be
120 // past the TLS handshake.
121 DialTLS
func(network
, addr
string) (net
.Conn
, error
)
123 // TLSClientConfig specifies the TLS configuration to use with
125 // If nil, the default configuration is used.
126 // If non-nil, HTTP/2 support may not be enabled by default.
127 TLSClientConfig
*tls
.Config
129 // TLSHandshakeTimeout specifies the maximum amount of time waiting to
130 // wait for a TLS handshake. Zero means no timeout.
131 TLSHandshakeTimeout time
.Duration
133 // DisableKeepAlives, if true, prevents re-use of TCP connections
134 // between different HTTP requests.
135 DisableKeepAlives
bool
137 // DisableCompression, if true, prevents the Transport from
138 // requesting compression with an "Accept-Encoding: gzip"
139 // request header when the Request contains no existing
140 // Accept-Encoding value. If the Transport requests gzip on
141 // its own and gets a gzipped response, it's transparently
142 // decoded in the Response.Body. However, if the user
143 // explicitly requested gzip it is not automatically
145 DisableCompression
bool
147 // MaxIdleConns controls the maximum number of idle (keep-alive)
148 // connections across all hosts. Zero means no limit.
151 // MaxIdleConnsPerHost, if non-zero, controls the maximum idle
152 // (keep-alive) connections to keep per-host. If zero,
153 // DefaultMaxIdleConnsPerHost is used.
154 MaxIdleConnsPerHost
int
156 // IdleConnTimeout is the maximum amount of time an idle
157 // (keep-alive) connection will remain idle before closing
159 // Zero means no limit.
160 IdleConnTimeout time
.Duration
162 // ResponseHeaderTimeout, if non-zero, specifies the amount of
163 // time to wait for a server's response headers after fully
164 // writing the request (including its body, if any). This
165 // time does not include the time to read the response body.
166 ResponseHeaderTimeout time
.Duration
168 // ExpectContinueTimeout, if non-zero, specifies the amount of
169 // time to wait for a server's first response headers after fully
170 // writing the request headers if the request has an
171 // "Expect: 100-continue" header. Zero means no timeout and
172 // causes the body to be sent immediately, without
173 // waiting for the server to approve.
174 // This time does not include the time to send the request header.
175 ExpectContinueTimeout time
.Duration
177 // TLSNextProto specifies how the Transport switches to an
178 // alternate protocol (such as HTTP/2) after a TLS NPN/ALPN
179 // protocol negotiation. If Transport dials an TLS connection
180 // with a non-empty protocol name and TLSNextProto contains a
181 // map entry for that key (such as "h2"), then the func is
182 // called with the request's authority (such as "example.com"
183 // or "example.com:1234") and the TLS connection. The function
184 // must return a RoundTripper that then handles the request.
185 // If TLSNextProto is not nil, HTTP/2 support is not enabled
187 TLSNextProto
map[string]func(authority
string, c
*tls
.Conn
) RoundTripper
189 // ProxyConnectHeader optionally specifies headers to send to
190 // proxies during CONNECT requests.
191 ProxyConnectHeader Header
193 // MaxResponseHeaderBytes specifies a limit on how many
194 // response bytes are allowed in the server's response
197 // Zero means to use a default limit.
198 MaxResponseHeaderBytes
int64
200 // nextProtoOnce guards initialization of TLSNextProto and
201 // h2transport (via onceSetNextProtoDefaults)
202 nextProtoOnce sync
.Once
203 h2transport
*http2Transport
// non-nil if http2 wired up
205 // TODO: tunable on max per-host TCP dials in flight (Issue 13957)
208 // onceSetNextProtoDefaults initializes TLSNextProto.
209 // It must be called via t.nextProtoOnce.Do.
210 func (t
*Transport
) onceSetNextProtoDefaults() {
211 if strings
.Contains(os
.Getenv("GODEBUG"), "http2client=0") {
214 if t
.TLSNextProto
!= nil {
215 // This is the documented way to disable http2 on a
219 if t
.TLSClientConfig
!= nil || t
.Dial
!= nil || t
.DialTLS
!= nil {
220 // Be conservative and don't automatically enable
221 // http2 if they've specified a custom TLS config or
222 // custom dialers. Let them opt-in themselves via
223 // http2.ConfigureTransport so we don't surprise them
224 // by modifying their tls.Config. Issue 14275.
227 t2
, err
:= http2configureTransport(t
)
229 log
.Printf("Error enabling Transport HTTP/2 support: %v", err
)
234 // Auto-configure the http2.Transport's MaxHeaderListSize from
235 // the http.Transport's MaxResponseHeaderBytes. They don't
236 // exactly mean the same thing, but they're close.
238 // TODO: also add this to x/net/http2.Configure Transport, behind
239 // a +build go1.7 build tag:
240 if limit1
:= t
.MaxResponseHeaderBytes
; limit1
!= 0 && t2
.MaxHeaderListSize
== 0 {
241 const h2max
= 1<<32 - 1
243 t2
.MaxHeaderListSize
= h2max
245 t2
.MaxHeaderListSize
= uint32(limit1
)
250 // ProxyFromEnvironment returns the URL of the proxy to use for a
251 // given request, as indicated by the environment variables
252 // HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions
253 // thereof). HTTPS_PROXY takes precedence over HTTP_PROXY for https
256 // The environment values may be either a complete URL or a
257 // "host[:port]", in which case the "http" scheme is assumed.
258 // An error is returned if the value is a different form.
260 // A nil URL and nil error are returned if no proxy is defined in the
261 // environment, or a proxy should not be used for the given request,
262 // as defined by NO_PROXY.
264 // As a special case, if req.URL.Host is "localhost" (with or without
265 // a port number), then a nil URL and nil error will be returned.
266 func ProxyFromEnvironment(req
*Request
) (*url
.URL
, error
) {
268 if req
.URL
.Scheme
== "https" {
269 proxy
= httpsProxyEnv
.Get()
272 proxy
= httpProxyEnv
.Get()
273 if proxy
!= "" && os
.Getenv("REQUEST_METHOD") != "" {
274 return nil, errors
.New("net/http: refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy")
280 if !useProxy(canonicalAddr(req
.URL
)) {
283 proxyURL
, err
:= url
.Parse(proxy
)
285 (proxyURL
.Scheme
!= "http" &&
286 proxyURL
.Scheme
!= "https" &&
287 proxyURL
.Scheme
!= "socks5") {
288 // proxy was bogus. Try prepending "http://" to it and
289 // see if that parses correctly. If not, we fall
290 // through and complain about the original one.
291 if proxyURL
, err
:= url
.Parse("http://" + proxy
); err
== nil {
297 return nil, fmt
.Errorf("invalid proxy address %q: %v", proxy
, err
)
302 // ProxyURL returns a proxy function (for use in a Transport)
303 // that always returns the same URL.
304 func ProxyURL(fixedURL
*url
.URL
) func(*Request
) (*url
.URL
, error
) {
305 return func(*Request
) (*url
.URL
, error
) {
310 // transportRequest is a wrapper around a *Request that adds
311 // optional extra headers to write and stores any error to return
313 type transportRequest
struct {
314 *Request
// original request, not to be mutated
315 extra Header
// extra headers to write, or nil
316 trace
*httptrace
.ClientTrace
// optional
318 mu sync
.Mutex
// guards err
319 err error
// first setError value for mapRoundTripError to consider
322 func (tr
*transportRequest
) extraHeaders() Header
{
324 tr
.extra
= make(Header
)
329 func (tr
*transportRequest
) setError(err error
) {
337 // RoundTrip implements the RoundTripper interface.
339 // For higher-level HTTP client support (such as handling of cookies
340 // and redirects), see Get, Post, and the Client type.
341 func (t
*Transport
) RoundTrip(req
*Request
) (*Response
, error
) {
342 t
.nextProtoOnce
.Do(t
.onceSetNextProtoDefaults
)
344 trace
:= httptrace
.ContextClientTrace(ctx
)
348 return nil, errors
.New("http: nil Request.URL")
350 if req
.Header
== nil {
352 return nil, errors
.New("http: nil Request.Header")
354 scheme
:= req
.URL
.Scheme
355 isHTTP
:= scheme
== "http" || scheme
== "https"
357 for k
, vv
:= range req
.Header
{
358 if !httplex
.ValidHeaderFieldName(k
) {
359 return nil, fmt
.Errorf("net/http: invalid header field name %q", k
)
361 for _
, v
:= range vv
{
362 if !httplex
.ValidHeaderFieldValue(v
) {
363 return nil, fmt
.Errorf("net/http: invalid header field value %q for key %v", v
, k
)
369 altProto
, _
:= t
.altProto
.Load().(map[string]RoundTripper
)
370 if altRT
:= altProto
[scheme
]; altRT
!= nil {
371 if resp
, err
:= altRT
.RoundTrip(req
); err
!= ErrSkipAltProtocol
{
377 return nil, &badStringError
{"unsupported protocol scheme", scheme
}
379 if req
.Method
!= "" && !validMethod(req
.Method
) {
380 return nil, fmt
.Errorf("net/http: invalid method %q", req
.Method
)
382 if req
.URL
.Host
== "" {
384 return nil, errors
.New("http: no Host in request URL")
388 // treq gets modified by roundTrip, so we need to recreate for each retry.
389 treq
:= &transportRequest
{Request
: req
, trace
: trace
}
390 cm
, err
:= t
.connectMethodForRequest(treq
)
396 // Get the cached or newly-created connection to either the
397 // host (for http or https), the http proxy, or the http proxy
398 // pre-CONNECTed to https server. In any case, we'll be ready
399 // to send it requests.
400 pconn
, err
:= t
.getConn(treq
, cm
)
402 t
.setReqCanceler(req
, nil)
408 if pconn
.alt
!= nil {
410 t
.setReqCanceler(req
, nil) // not cancelable with CancelRequest
411 resp
, err
= pconn
.alt
.RoundTrip(req
)
413 resp
, err
= pconn
.roundTrip(treq
)
418 if !pconn
.shouldRetryRequest(req
, err
) {
419 // Issue 16465: return underlying net.Conn.Read error from peek,
420 // as we've historically done.
421 if e
, ok
:= err
.(transportReadFromServerError
); ok
{
426 testHookRoundTripRetried()
428 // Rewind the body if we're able to. (HTTP/2 does this itself so we only
429 // need to do it for HTTP/1.1 connections.)
430 if req
.GetBody
!= nil && pconn
.alt
== nil {
433 newReq
.Body
, err
= req
.GetBody()
442 // shouldRetryRequest reports whether we should retry sending a failed
443 // HTTP request on a new connection. The non-nil input error is the
444 // error from roundTrip.
445 func (pc
*persistConn
) shouldRetryRequest(req
*Request
, err error
) bool {
446 if err
== http2ErrNoCachedConn
{
447 // Issue 16582: if the user started a bunch of
448 // requests at once, they can all pick the same conn
449 // and violate the server's max concurrent streams.
450 // Instead, match the HTTP/1 behavior for now and dial
451 // again to get a new TCP connection, rather than failing
455 if err
== errMissingHost
{
460 // This was a fresh connection. There's no reason the server
461 // should've hung up on us.
463 // Also, if we retried now, we could loop forever
464 // creating new connections and retrying if the server
465 // is just hanging up on us because it doesn't like
466 // our request (as opposed to sending an error).
469 if _
, ok
:= err
.(nothingWrittenError
); ok
{
470 // We never wrote anything, so it's safe to retry, if there's no body or we
471 // can "rewind" the body with GetBody.
472 return req
.outgoingLength() == 0 || req
.GetBody
!= nil
474 if !req
.isReplayable() {
475 // Don't retry non-idempotent requests.
478 if _
, ok
:= err
.(transportReadFromServerError
); ok
{
479 // We got some non-EOF net.Conn.Read failure reading
480 // the 1st response byte from the server.
483 if err
== errServerClosedIdle
{
484 // The server replied with io.EOF while we were trying to
485 // read the response. Probably an unfortunately keep-alive
486 // timeout, just as the client was writing a request.
489 return false // conservatively
492 // ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol.
493 var ErrSkipAltProtocol
= errors
.New("net/http: skip alternate protocol")
495 // RegisterProtocol registers a new protocol with scheme.
496 // The Transport will pass requests using the given scheme to rt.
497 // It is rt's responsibility to simulate HTTP request semantics.
499 // RegisterProtocol can be used by other packages to provide
500 // implementations of protocol schemes like "ftp" or "file".
502 // If rt.RoundTrip returns ErrSkipAltProtocol, the Transport will
503 // handle the RoundTrip itself for that one request, as if the
504 // protocol were not registered.
505 func (t
*Transport
) RegisterProtocol(scheme
string, rt RoundTripper
) {
507 defer t
.altMu
.Unlock()
508 oldMap
, _
:= t
.altProto
.Load().(map[string]RoundTripper
)
509 if _
, exists
:= oldMap
[scheme
]; exists
{
510 panic("protocol " + scheme
+ " already registered")
512 newMap
:= make(map[string]RoundTripper
)
513 for k
, v
:= range oldMap
{
517 t
.altProto
.Store(newMap
)
520 // CloseIdleConnections closes any connections which were previously
521 // connected from previous requests but are now sitting idle in
522 // a "keep-alive" state. It does not interrupt any connections currently
524 func (t
*Transport
) CloseIdleConnections() {
525 t
.nextProtoOnce
.Do(t
.onceSetNextProtoDefaults
)
531 t
.idleLRU
= connLRU
{}
533 for _
, conns
:= range m
{
534 for _
, pconn
:= range conns
{
535 pconn
.close(errCloseIdleConns
)
538 if t2
:= t
.h2transport
; t2
!= nil {
539 t2
.CloseIdleConnections()
543 // CancelRequest cancels an in-flight request by closing its connection.
544 // CancelRequest should only be called after RoundTrip has returned.
546 // Deprecated: Use Request.WithContext to create a request with a
547 // cancelable context instead. CancelRequest cannot cancel HTTP/2
549 func (t
*Transport
) CancelRequest(req
*Request
) {
550 t
.cancelRequest(req
, errRequestCanceled
)
553 // Cancel an in-flight request, recording the error value.
554 func (t
*Transport
) cancelRequest(req
*Request
, err error
) {
556 cancel
:= t
.reqCanceler
[req
]
557 delete(t
.reqCanceler
, req
)
565 // Private implementation past this point.
569 httpProxyEnv
= &envOnce
{
570 names
: []string{"HTTP_PROXY", "http_proxy"},
572 httpsProxyEnv
= &envOnce
{
573 names
: []string{"HTTPS_PROXY", "https_proxy"},
575 noProxyEnv
= &envOnce
{
576 names
: []string{"NO_PROXY", "no_proxy"},
580 // envOnce looks up an environment variable (optionally by multiple
581 // names) once. It mitigates expensive lookups on some platforms
583 type envOnce
struct {
589 func (e
*envOnce
) Get() string {
594 func (e
*envOnce
) init() {
595 for _
, n
:= range e
.names
{
603 // reset is used by tests
604 func (e
*envOnce
) reset() {
609 func (t
*Transport
) connectMethodForRequest(treq
*transportRequest
) (cm connectMethod
, err error
) {
610 if port
:= treq
.URL
.Port(); !validPort(port
) {
611 return cm
, fmt
.Errorf("invalid URL port %q", port
)
613 cm
.targetScheme
= treq
.URL
.Scheme
614 cm
.targetAddr
= canonicalAddr(treq
.URL
)
616 cm
.proxyURL
, err
= t
.Proxy(treq
.Request
)
617 if err
== nil && cm
.proxyURL
!= nil {
618 if port
:= cm
.proxyURL
.Port(); !validPort(port
) {
619 return cm
, fmt
.Errorf("invalid proxy URL port %q", port
)
626 // proxyAuth returns the Proxy-Authorization header to set
627 // on requests, if applicable.
628 func (cm
*connectMethod
) proxyAuth() string {
629 if cm
.proxyURL
== nil {
632 if u
:= cm
.proxyURL
.User
; u
!= nil {
633 username
:= u
.Username()
634 password
, _
:= u
.Password()
635 return "Basic " + basicAuth(username
, password
)
640 // error values for debugging and testing, not seen by users.
642 errKeepAlivesDisabled
= errors
.New("http: putIdleConn: keep alives disabled")
643 errConnBroken
= errors
.New("http: putIdleConn: connection is in bad state")
644 errWantIdle
= errors
.New("http: putIdleConn: CloseIdleConnections was called")
645 errTooManyIdle
= errors
.New("http: putIdleConn: too many idle connections")
646 errTooManyIdleHost
= errors
.New("http: putIdleConn: too many idle connections for host")
647 errCloseIdleConns
= errors
.New("http: CloseIdleConnections called")
648 errReadLoopExiting
= errors
.New("http: persistConn.readLoop exiting")
649 errServerClosedIdle
= errors
.New("http: server closed idle connection")
650 errIdleConnTimeout
= errors
.New("http: idle connection timeout")
651 errNotCachingH2Conn
= errors
.New("http: not caching alternate protocol's connections")
654 // transportReadFromServerError is used by Transport.readLoop when the
655 // 1 byte peek read fails and we're actually anticipating a response.
656 // Usually this is just due to the inherent keep-alive shut down race,
657 // where the server closed the connection at the same time the client
658 // wrote. The underlying err field is usually io.EOF or some
659 // ECONNRESET sort of thing which varies by platform. But it might be
660 // the user's custom net.Conn.Read error too, so we carry it along for
661 // them to return from Transport.RoundTrip.
662 type transportReadFromServerError
struct {
666 func (e transportReadFromServerError
) Error() string {
667 return fmt
.Sprintf("net/http: Transport failed to read from server: %v", e
.err
)
670 func (t
*Transport
) putOrCloseIdleConn(pconn
*persistConn
) {
671 if err
:= t
.tryPutIdleConn(pconn
); err
!= nil {
676 func (t
*Transport
) maxIdleConnsPerHost() int {
677 if v
:= t
.MaxIdleConnsPerHost
; v
!= 0 {
680 return DefaultMaxIdleConnsPerHost
683 // tryPutIdleConn adds pconn to the list of idle persistent connections awaiting
685 // If pconn is no longer needed or not in a good state, tryPutIdleConn returns
686 // an error explaining why it wasn't registered.
687 // tryPutIdleConn does not close pconn. Use putOrCloseIdleConn instead for that.
688 func (t
*Transport
) tryPutIdleConn(pconn
*persistConn
) error
{
689 if t
.DisableKeepAlives || t
.MaxIdleConnsPerHost
< 0 {
690 return errKeepAlivesDisabled
692 if pconn
.isBroken() {
695 if pconn
.alt
!= nil {
696 return errNotCachingH2Conn
699 key
:= pconn
.cacheKey
702 defer t
.idleMu
.Unlock()
704 waitingDialer
:= t
.idleConnCh
[key
]
706 case waitingDialer
<- pconn
:
707 // We're done with this pconn and somebody else is
708 // currently waiting for a conn of this type (they're
709 // actively dialing, but this conn is ready
710 // first). Chrome calls this socket late binding. See
711 // https://insouciant.org/tech/connection-management-in-chromium/
714 if waitingDialer
!= nil {
715 // They had populated this, but their dial won
716 // first, so we can clean up this map entry.
717 delete(t
.idleConnCh
, key
)
723 if t
.idleConn
== nil {
724 t
.idleConn
= make(map[connectMethodKey
][]*persistConn
)
726 idles
:= t
.idleConn
[key
]
727 if len(idles
) >= t
.maxIdleConnsPerHost() {
728 return errTooManyIdleHost
730 for _
, exist
:= range idles
{
732 log
.Fatalf("dup idle pconn %p in freelist", pconn
)
735 t
.idleConn
[key
] = append(idles
, pconn
)
737 if t
.MaxIdleConns
!= 0 && t
.idleLRU
.len() > t
.MaxIdleConns
{
738 oldest
:= t
.idleLRU
.removeOldest()
739 oldest
.close(errTooManyIdle
)
740 t
.removeIdleConnLocked(oldest
)
742 if t
.IdleConnTimeout
> 0 {
743 if pconn
.idleTimer
!= nil {
744 pconn
.idleTimer
.Reset(t
.IdleConnTimeout
)
746 pconn
.idleTimer
= time
.AfterFunc(t
.IdleConnTimeout
, pconn
.closeConnIfStillIdle
)
749 pconn
.idleAt
= time
.Now()
753 // getIdleConnCh returns a channel to receive and return idle
754 // persistent connection for the given connectMethod.
755 // It may return nil, if persistent connections are not being used.
756 func (t
*Transport
) getIdleConnCh(cm connectMethod
) chan *persistConn
{
757 if t
.DisableKeepAlives
{
762 defer t
.idleMu
.Unlock()
764 if t
.idleConnCh
== nil {
765 t
.idleConnCh
= make(map[connectMethodKey
]chan *persistConn
)
767 ch
, ok
:= t
.idleConnCh
[key
]
769 ch
= make(chan *persistConn
)
770 t
.idleConnCh
[key
] = ch
775 func (t
*Transport
) getIdleConn(cm connectMethod
) (pconn
*persistConn
, idleSince time
.Time
) {
778 defer t
.idleMu
.Unlock()
780 pconns
, ok
:= t
.idleConn
[key
]
782 return nil, time
.Time
{}
784 if len(pconns
) == 1 {
786 delete(t
.idleConn
, key
)
788 // 2 or more cached connections; use the most
789 // recently used one at the end.
790 pconn
= pconns
[len(pconns
)-1]
791 t
.idleConn
[key
] = pconns
[:len(pconns
)-1]
793 t
.idleLRU
.remove(pconn
)
794 if pconn
.isBroken() {
795 // There is a tiny window where this is
796 // possible, between the connecting dying and
797 // the persistConn readLoop calling
798 // Transport.removeIdleConn. Just skip it and
802 if pconn
.idleTimer
!= nil && !pconn
.idleTimer
.Stop() {
803 // We picked this conn at the ~same time it
804 // was expiring and it's trying to close
805 // itself in another goroutine. Don't use it.
808 return pconn
, pconn
.idleAt
812 // removeIdleConn marks pconn as dead.
813 func (t
*Transport
) removeIdleConn(pconn
*persistConn
) {
815 defer t
.idleMu
.Unlock()
816 t
.removeIdleConnLocked(pconn
)
819 // t.idleMu must be held.
820 func (t
*Transport
) removeIdleConnLocked(pconn
*persistConn
) {
821 if pconn
.idleTimer
!= nil {
822 pconn
.idleTimer
.Stop()
824 t
.idleLRU
.remove(pconn
)
825 key
:= pconn
.cacheKey
826 pconns
:= t
.idleConn
[key
]
831 if pconns
[0] == pconn
{
832 delete(t
.idleConn
, key
)
835 for i
, v
:= range pconns
{
839 // Slide down, keeping most recently-used
841 copy(pconns
[i
:], pconns
[i
+1:])
842 t
.idleConn
[key
] = pconns
[:len(pconns
)-1]
848 func (t
*Transport
) setReqCanceler(r
*Request
, fn
func(error
)) {
850 defer t
.reqMu
.Unlock()
851 if t
.reqCanceler
== nil {
852 t
.reqCanceler
= make(map[*Request
]func(error
))
855 t
.reqCanceler
[r
] = fn
857 delete(t
.reqCanceler
, r
)
861 // replaceReqCanceler replaces an existing cancel function. If there is no cancel function
862 // for the request, we don't set the function and return false.
863 // Since CancelRequest will clear the canceler, we can use the return value to detect if
864 // the request was canceled since the last setReqCancel call.
865 func (t
*Transport
) replaceReqCanceler(r
*Request
, fn
func(error
)) bool {
867 defer t
.reqMu
.Unlock()
868 _
, ok
:= t
.reqCanceler
[r
]
873 t
.reqCanceler
[r
] = fn
875 delete(t
.reqCanceler
, r
)
880 var zeroDialer net
.Dialer
882 func (t
*Transport
) dial(ctx context
.Context
, network
, addr
string) (net
.Conn
, error
) {
883 if t
.DialContext
!= nil {
884 return t
.DialContext(ctx
, network
, addr
)
887 c
, err
:= t
.Dial(network
, addr
)
888 if c
== nil && err
== nil {
889 err
= errors
.New("net/http: Transport.Dial hook returned (nil, nil)")
893 return zeroDialer
.DialContext(ctx
, network
, addr
)
896 // getConn dials and creates a new persistConn to the target as
897 // specified in the connectMethod. This includes doing a proxy CONNECT
898 // and/or setting up TLS. If this doesn't return an error, the persistConn
899 // is ready to write requests to.
900 func (t
*Transport
) getConn(treq
*transportRequest
, cm connectMethod
) (*persistConn
, error
) {
904 if trace
!= nil && trace
.GetConn
!= nil {
905 trace
.GetConn(cm
.addr())
907 if pc
, idleSince
:= t
.getIdleConn(cm
); pc
!= nil {
908 if trace
!= nil && trace
.GotConn
!= nil {
909 trace
.GotConn(pc
.gotIdleConnTrace(idleSince
))
911 // set request canceler to some non-nil function so we
912 // can detect whether it was cleared between now and when
913 // we enter roundTrip
914 t
.setReqCanceler(req
, func(error
) {})
918 type dialRes
struct {
922 dialc
:= make(chan dialRes
)
924 // Copy these hooks so we don't race on the postPendingDial in
925 // the goroutine we launch. Issue 11136.
926 testHookPrePendingDial
:= testHookPrePendingDial
927 testHookPostPendingDial
:= testHookPostPendingDial
929 handlePendingDial
:= func() {
930 testHookPrePendingDial()
932 if v
:= <-dialc
; v
.err
== nil {
933 t
.putOrCloseIdleConn(v
.pc
)
935 testHookPostPendingDial()
939 cancelc
:= make(chan error
, 1)
940 t
.setReqCanceler(req
, func(err error
) { cancelc
<- err
})
943 pc
, err
:= t
.dialConn(ctx
, cm
)
944 dialc
<- dialRes
{pc
, err
}
947 idleConnCh
:= t
.getIdleConnCh(cm
)
950 // Our dial finished.
952 if trace
!= nil && trace
.GotConn
!= nil && v
.pc
.alt
== nil {
953 trace
.GotConn(httptrace
.GotConnInfo
{Conn
: v
.pc
.conn
})
957 // Our dial failed. See why to return a nicer error
961 // It was an error due to cancelation, so prioritize that
962 // error value. (Issue 16049)
963 return nil, errRequestCanceledConn
964 case <-req
.Context().Done():
965 return nil, req
.Context().Err()
966 case err
:= <-cancelc
:
967 if err
== errRequestCanceled
{
968 err
= errRequestCanceledConn
972 // It wasn't an error due to cancelation, so
973 // return the original error message:
976 case pc
:= <-idleConnCh
:
977 // Another request finished first and its net.Conn
978 // became available before our dial. Or somebody
979 // else's dial that they didn't use.
980 // But our dial is still going, so give it away
983 if trace
!= nil && trace
.GotConn
!= nil {
984 trace
.GotConn(httptrace
.GotConnInfo
{Conn
: pc
.conn
, Reused
: pc
.isReused()})
989 return nil, errRequestCanceledConn
990 case <-req
.Context().Done():
992 return nil, req
.Context().Err()
993 case err
:= <-cancelc
:
995 if err
== errRequestCanceled
{
996 err
= errRequestCanceledConn
1002 type oneConnDialer
<-chan net
.Conn
1004 func newOneConnDialer(c net
.Conn
) proxy
.Dialer
{
1005 ch
:= make(chan net
.Conn
, 1)
1007 return oneConnDialer(ch
)
1010 func (d oneConnDialer
) Dial(network
, addr
string) (net
.Conn
, error
) {
1019 func (t
*Transport
) dialConn(ctx context
.Context
, cm connectMethod
) (*persistConn
, error
) {
1020 pconn
:= &persistConn
{
1023 reqch
: make(chan requestAndChan
, 1),
1024 writech
: make(chan writeRequest
, 1),
1025 closech
: make(chan struct{}),
1026 writeErrCh
: make(chan error
, 1),
1027 writeLoopDone
: make(chan struct{}),
1029 trace
:= httptrace
.ContextClientTrace(ctx
)
1030 tlsDial
:= t
.DialTLS
!= nil && cm
.targetScheme
== "https" && cm
.proxyURL
== nil
1033 pconn
.conn
, err
= t
.DialTLS("tcp", cm
.addr())
1037 if pconn
.conn
== nil {
1038 return nil, errors
.New("net/http: Transport.DialTLS returned (nil, nil)")
1040 if tc
, ok
:= pconn
.conn
.(*tls
.Conn
); ok
{
1041 // Handshake here, in case DialTLS didn't. TLSNextProto below
1042 // depends on it for knowing the connection state.
1043 if trace
!= nil && trace
.TLSHandshakeStart
!= nil {
1044 trace
.TLSHandshakeStart()
1046 if err
:= tc
.Handshake(); err
!= nil {
1047 go pconn
.conn
.Close()
1048 if trace
!= nil && trace
.TLSHandshakeDone
!= nil {
1049 trace
.TLSHandshakeDone(tls
.ConnectionState
{}, err
)
1053 cs
:= tc
.ConnectionState()
1054 if trace
!= nil && trace
.TLSHandshakeDone
!= nil {
1055 trace
.TLSHandshakeDone(cs
, nil)
1057 pconn
.tlsState
= &cs
1060 conn
, err
:= t
.dial(ctx
, "tcp", cm
.addr())
1062 if cm
.proxyURL
!= nil {
1063 // Return a typed error, per Issue 16997:
1064 err
= &net
.OpError
{Op
: "proxyconnect", Net
: "tcp", Err
: err
}
1073 case cm
.proxyURL
== nil:
1074 // Do nothing. Not using a proxy.
1075 case cm
.proxyURL
.Scheme
== "socks5":
1077 var auth
*proxy
.Auth
1078 if u
:= cm
.proxyURL
.User
; u
!= nil {
1079 auth
= &proxy
.Auth
{}
1080 auth
.User
= u
.Username()
1081 auth
.Password
, _
= u
.Password()
1083 p
, err
:= proxy
.SOCKS5("", cm
.addr(), auth
, newOneConnDialer(conn
))
1088 if _
, err
:= p
.Dial("tcp", cm
.targetAddr
); err
!= nil {
1092 case cm
.targetScheme
== "http":
1093 pconn
.isProxy
= true
1094 if pa
:= cm
.proxyAuth(); pa
!= "" {
1095 pconn
.mutateHeaderFunc
= func(h Header
) {
1096 h
.Set("Proxy-Authorization", pa
)
1099 case cm
.targetScheme
== "https":
1101 hdr
:= t
.ProxyConnectHeader
1105 connectReq
:= &Request
{
1107 URL
: &url
.URL
{Opaque
: cm
.targetAddr
},
1108 Host
: cm
.targetAddr
,
1111 if pa
:= cm
.proxyAuth(); pa
!= "" {
1112 connectReq
.Header
.Set("Proxy-Authorization", pa
)
1114 connectReq
.Write(conn
)
1117 // Okay to use and discard buffered reader here, because
1118 // TLS server will not speak until spoken to.
1119 br
:= bufio
.NewReader(conn
)
1120 resp
, err
:= ReadResponse(br
, connectReq
)
1125 if resp
.StatusCode
!= 200 {
1126 f
:= strings
.SplitN(resp
.Status
, " ", 2)
1128 return nil, errors
.New(f
[1])
1132 if cm
.targetScheme
== "https" && !tlsDial
{
1133 // Initiate TLS and check remote host name against certificate.
1134 cfg
:= cloneTLSConfig(t
.TLSClientConfig
)
1135 if cfg
.ServerName
== "" {
1136 cfg
.ServerName
= cm
.tlsHost()
1138 plainConn
:= pconn
.conn
1139 tlsConn
:= tls
.Client(plainConn
, cfg
)
1140 errc
:= make(chan error
, 2)
1141 var timer
*time
.Timer
// for canceling TLS handshake
1142 if d
:= t
.TLSHandshakeTimeout
; d
!= 0 {
1143 timer
= time
.AfterFunc(d
, func() {
1144 errc
<- tlsHandshakeTimeoutError
{}
1148 if trace
!= nil && trace
.TLSHandshakeStart
!= nil {
1149 trace
.TLSHandshakeStart()
1151 err
:= tlsConn
.Handshake()
1157 if err
:= <-errc
; err
!= nil {
1159 if trace
!= nil && trace
.TLSHandshakeDone
!= nil {
1160 trace
.TLSHandshakeDone(tls
.ConnectionState
{}, err
)
1164 if !cfg
.InsecureSkipVerify
{
1165 if err
:= tlsConn
.VerifyHostname(cfg
.ServerName
); err
!= nil {
1170 cs
:= tlsConn
.ConnectionState()
1171 if trace
!= nil && trace
.TLSHandshakeDone
!= nil {
1172 trace
.TLSHandshakeDone(cs
, nil)
1174 pconn
.tlsState
= &cs
1175 pconn
.conn
= tlsConn
1178 if s
:= pconn
.tlsState
; s
!= nil && s
.NegotiatedProtocolIsMutual
&& s
.NegotiatedProtocol
!= "" {
1179 if next
, ok
:= t
.TLSNextProto
[s
.NegotiatedProtocol
]; ok
{
1180 return &persistConn
{alt
: next(cm
.targetAddr
, pconn
.conn
.(*tls
.Conn
))}, nil
1184 pconn
.br
= bufio
.NewReader(pconn
)
1185 pconn
.bw
= bufio
.NewWriter(persistConnWriter
{pconn
})
1187 go pconn
.writeLoop()
1191 // persistConnWriter is the io.Writer written to by pc.bw.
1192 // It accumulates the number of bytes written to the underlying conn,
1193 // so the retry logic can determine whether any bytes made it across
1195 // This is exactly 1 pointer field wide so it can go into an interface
1196 // without allocation.
1197 type persistConnWriter
struct {
1201 func (w persistConnWriter
) Write(p
[]byte) (n
int, err error
) {
1202 n
, err
= w
.pc
.conn
.Write(p
)
1203 w
.pc
.nwrite
+= int64(n
)
1207 // useProxy reports whether requests to addr should use a proxy,
1208 // according to the NO_PROXY or no_proxy environment variable.
1209 // addr is always a canonicalAddr with a host and port.
1210 func useProxy(addr
string) bool {
1214 host
, _
, err
:= net
.SplitHostPort(addr
)
1218 if host
== "localhost" {
1221 if ip
:= net
.ParseIP(host
); ip
!= nil {
1222 if ip
.IsLoopback() {
1227 no_proxy
:= noProxyEnv
.Get()
1228 if no_proxy
== "*" {
1232 addr
= strings
.ToLower(strings
.TrimSpace(addr
))
1234 addr
= addr
[:strings
.LastIndex(addr
, ":")]
1237 for _
, p
:= range strings
.Split(no_proxy
, ",") {
1238 p
= strings
.ToLower(strings
.TrimSpace(p
))
1243 p
= p
[:strings
.LastIndex(p
, ":")]
1249 // There is no host part, likely the entry is malformed; ignore.
1252 if p
[0] == '.' && (strings
.HasSuffix(addr
, p
) || addr
== p
[1:]) {
1253 // no_proxy ".foo.com" matches "bar.foo.com" or "foo.com"
1256 if p
[0] != '.' && strings
.HasSuffix(addr
, p
) && addr
[len(addr
)-len(p
)-1] == '.' {
1257 // no_proxy "foo.com" matches "bar.foo.com"
1264 // connectMethod is the map key (in its String form) for keeping persistent
1265 // TCP connections alive for subsequent HTTP requests.
1267 // A connect method may be of the following types:
1269 // Cache key form Description
1270 // ----------------- -------------------------
1271 // |http|foo.com http directly to server, no proxy
1272 // |https|foo.com https directly to server, no proxy
1273 // http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com
1274 // http://proxy.com|http http to proxy, http to anywhere after that
1275 // socks5://proxy.com|http|foo.com socks5 to proxy, then http to foo.com
1276 // socks5://proxy.com|https|foo.com socks5 to proxy, then https to foo.com
1278 // Note: no support to https to the proxy yet.
1280 type connectMethod
struct {
1281 proxyURL
*url
.URL
// nil for no proxy, else full proxy URL
1282 targetScheme
string // "http" or "https"
1283 targetAddr
string // Not used if http proxy + http targetScheme (4th example in table)
1286 func (cm
*connectMethod
) key() connectMethodKey
{
1288 targetAddr
:= cm
.targetAddr
1289 if cm
.proxyURL
!= nil {
1290 proxyStr
= cm
.proxyURL
.String()
1291 if strings
.HasPrefix(cm
.proxyURL
.Scheme
, "http") && cm
.targetScheme
== "http" {
1295 return connectMethodKey
{
1297 scheme
: cm
.targetScheme
,
1302 // addr returns the first hop "host:port" to which we need to TCP connect.
1303 func (cm
*connectMethod
) addr() string {
1304 if cm
.proxyURL
!= nil {
1305 return canonicalAddr(cm
.proxyURL
)
1307 return cm
.targetAddr
1310 // tlsHost returns the host name to match against the peer's
1312 func (cm
*connectMethod
) tlsHost() string {
1315 h
= h
[:strings
.LastIndex(h
, ":")]
1320 // connectMethodKey is the map key version of connectMethod, with a
1321 // stringified proxy URL (or the empty string) instead of a pointer to
1323 type connectMethodKey
struct {
1324 proxy
, scheme
, addr
string
1327 func (k connectMethodKey
) String() string {
1328 // Only used by tests.
1329 return fmt
.Sprintf("%s|%s|%s", k
.proxy
, k
.scheme
, k
.addr
)
1332 // persistConn wraps a connection, usually a persistent one
1333 // (but may be used for non-keep-alive requests as well)
1334 type persistConn
struct {
1335 // alt optionally specifies the TLS NextProto RoundTripper.
1336 // This is used for HTTP/2 today and future protocols later.
1337 // If it's non-nil, the rest of the fields are unused.
1341 cacheKey connectMethodKey
1343 tlsState
*tls
.ConnectionState
1344 br
*bufio
.Reader
// from conn
1345 bw
*bufio
.Writer
// to conn
1346 nwrite
int64 // bytes written
1347 reqch
chan requestAndChan
// written by roundTrip; read by readLoop
1348 writech
chan writeRequest
// written by roundTrip; read by writeLoop
1349 closech
chan struct{} // closed when conn closed
1351 sawEOF
bool // whether we've seen EOF from conn; owned by readLoop
1352 readLimit
int64 // bytes allowed to be read; owned by readLoop
1353 // writeErrCh passes the request write error (usually nil)
1354 // from the writeLoop goroutine to the readLoop which passes
1355 // it off to the res.Body reader, which then uses it to decide
1356 // whether or not a connection can be reused. Issue 7569.
1357 writeErrCh
chan error
1359 writeLoopDone
chan struct{} // closed when write loop ends
1361 // Both guarded by Transport.idleMu:
1362 idleAt time
.Time
// time it last become idle
1363 idleTimer
*time
.Timer
// holding an AfterFunc to close it
1365 mu sync
.Mutex
// guards following fields
1366 numExpectedResponses
int
1367 closed error
// set non-nil when conn is closed, before closech is closed
1368 canceledErr error
// set non-nil if conn is canceled
1369 broken
bool // an error has happened on this connection; marked broken so it's not reused.
1370 reused
bool // whether conn has had successful request/response and is being reused.
1371 // mutateHeaderFunc is an optional func to modify extra
1372 // headers on each outbound request before it's written. (the
1373 // original Request given to RoundTrip is not modified)
1374 mutateHeaderFunc
func(Header
)
1377 func (pc
*persistConn
) maxHeaderResponseSize() int64 {
1378 if v
:= pc
.t
.MaxResponseHeaderBytes
; v
!= 0 {
1381 return 10 << 20 // conservative default; same as http2
1384 func (pc
*persistConn
) Read(p
[]byte) (n
int, err error
) {
1385 if pc
.readLimit
<= 0 {
1386 return 0, fmt
.Errorf("read limit of %d bytes exhausted", pc
.maxHeaderResponseSize())
1388 if int64(len(p
)) > pc
.readLimit
{
1389 p
= p
[:pc
.readLimit
]
1391 n
, err
= pc
.conn
.Read(p
)
1395 pc
.readLimit
-= int64(n
)
1399 // isBroken reports whether this connection is in a known broken state.
1400 func (pc
*persistConn
) isBroken() bool {
1402 b
:= pc
.closed != nil
1407 // canceled returns non-nil if the connection was closed due to
1408 // CancelRequest or due to context cancelation.
1409 func (pc
*persistConn
) canceled() error
{
1411 defer pc
.mu
.Unlock()
1412 return pc
.canceledErr
1415 // isReused reports whether this connection is in a known broken state.
1416 func (pc
*persistConn
) isReused() bool {
1423 func (pc
*persistConn
) gotIdleConnTrace(idleAt time
.Time
) (t httptrace
.GotConnInfo
) {
1425 defer pc
.mu
.Unlock()
1426 t
.Reused
= pc
.reused
1429 if !idleAt
.IsZero() {
1430 t
.IdleTime
= time
.Since(idleAt
)
1435 func (pc
*persistConn
) cancelRequest(err error
) {
1437 defer pc
.mu
.Unlock()
1438 pc
.canceledErr
= err
1439 pc
.closeLocked(errRequestCanceled
)
1442 // closeConnIfStillIdle closes the connection if it's still sitting idle.
1443 // This is what's called by the persistConn's idleTimer, and is run in its
1445 func (pc
*persistConn
) closeConnIfStillIdle() {
1448 defer t
.idleMu
.Unlock()
1449 if _
, ok
:= t
.idleLRU
.m
[pc
]; !ok
{
1453 t
.removeIdleConnLocked(pc
)
1454 pc
.close(errIdleConnTimeout
)
1457 // mapRoundTripError returns the appropriate error value for
1458 // persistConn.roundTrip.
1460 // The provided err is the first error that (*persistConn).roundTrip
1461 // happened to receive from its select statement.
1463 // The startBytesWritten value should be the value of pc.nwrite before the roundTrip
1464 // started writing the request.
1465 func (pc
*persistConn
) mapRoundTripError(req
*transportRequest
, startBytesWritten
int64, err error
) error
{
1470 // If the request was canceled, that's better than network
1471 // failures that were likely the result of tearing down the
1473 if cerr
:= pc
.canceled(); cerr
!= nil {
1477 // See if an error was set explicitly.
1485 if err
== errServerClosedIdle
{
1490 if _
, ok
:= err
.(transportReadFromServerError
); ok
{
1496 if pc
.nwrite
== startBytesWritten
{
1497 return nothingWrittenError
{err
}
1499 return fmt
.Errorf("net/http: HTTP/1.x transport connection broken: %v", err
)
1504 func (pc
*persistConn
) readLoop() {
1505 closeErr
:= errReadLoopExiting
// default value, if not changed below
1508 pc
.t
.removeIdleConn(pc
)
1511 tryPutIdleConn
:= func(trace
*httptrace
.ClientTrace
) bool {
1512 if err
:= pc
.t
.tryPutIdleConn(pc
); err
!= nil {
1514 if trace
!= nil && trace
.PutIdleConn
!= nil && err
!= errKeepAlivesDisabled
{
1515 trace
.PutIdleConn(err
)
1519 if trace
!= nil && trace
.PutIdleConn
!= nil {
1520 trace
.PutIdleConn(nil)
1525 // eofc is used to block caller goroutines reading from Response.Body
1526 // at EOF until this goroutines has (potentially) added the connection
1527 // back to the idle pool.
1528 eofc
:= make(chan struct{})
1529 defer close(eofc
) // unblock reader on errors
1531 // Read this once, before loop starts. (to avoid races in tests)
1533 testHookReadLoopBeforeNextRead
:= testHookReadLoopBeforeNextRead
1538 pc
.readLimit
= pc
.maxHeaderResponseSize()
1539 _
, err
:= pc
.br
.Peek(1)
1542 if pc
.numExpectedResponses
== 0 {
1543 pc
.readLoopPeekFailLocked(err
)
1550 trace
:= httptrace
.ContextClientTrace(rc
.req
.Context())
1554 resp
, err
= pc
.readResponse(rc
, trace
)
1556 err
= transportReadFromServerError
{err
}
1561 if pc
.readLimit
<= 0 {
1562 err
= fmt
.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc
.maxHeaderResponseSize())
1566 case rc
.ch
<- responseAndError
{err
: err
}:
1567 case <-rc
.callerGone
:
1572 pc
.readLimit
= maxInt64
// effictively no limit for response bodies
1575 pc
.numExpectedResponses
--
1578 hasBody
:= rc
.req
.Method
!= "HEAD" && resp
.ContentLength
!= 0
1580 if resp
.Close || rc
.req
.Close || resp
.StatusCode
<= 199 {
1581 // Don't do keep-alive on error if either party requested a close
1582 // or we get an unexpected informational (1xx) response.
1583 // StatusCode 100 is already handled above.
1588 pc
.t
.setReqCanceler(rc
.req
, nil)
1590 // Put the idle conn back into the pool before we send the response
1591 // so if they process it quickly and make another request, they'll
1592 // get this same conn. But we use the unbuffered channel 'rc'
1593 // to guarantee that persistConn.roundTrip got out of its select
1594 // potentially waiting for this persistConn to close.
1598 pc
.wroteRequest() &&
1599 tryPutIdleConn(trace
)
1602 case rc
.ch
<- responseAndError
{res
: resp
}:
1603 case <-rc
.callerGone
:
1607 // Now that they've read from the unbuffered channel, they're safely
1608 // out of the select that also waits on this goroutine to die, so
1609 // we're allowed to exit now if needed (if alive is false)
1610 testHookReadLoopBeforeNextRead()
1614 waitForBodyRead
:= make(chan bool, 2)
1615 body
:= &bodyEOFSignal
{
1617 earlyCloseFn
: func() error
{
1618 waitForBodyRead
<- false
1622 fn
: func(err error
) error
{
1623 isEOF
:= err
== io
.EOF
1624 waitForBodyRead
<- isEOF
1626 <-eofc
// see comment above eofc declaration
1627 } else if err
!= nil {
1628 if cerr
:= pc
.canceled(); cerr
!= nil {
1637 if rc
.addedGzip
&& strings
.EqualFold(resp
.Header
.Get("Content-Encoding"), "gzip") {
1638 resp
.Body
= &gzipReader
{body
: body
}
1639 resp
.Header
.Del("Content-Encoding")
1640 resp
.Header
.Del("Content-Length")
1641 resp
.ContentLength
= -1
1642 resp
.Uncompressed
= true
1646 case rc
.ch
<- responseAndError
{res
: resp
}:
1647 case <-rc
.callerGone
:
1651 // Before looping back to the top of this function and peeking on
1652 // the bufio.Reader, wait for the caller goroutine to finish
1653 // reading the response body. (or for cancelation or death)
1655 case bodyEOF
:= <-waitForBodyRead
:
1656 pc
.t
.setReqCanceler(rc
.req
, nil) // before pc might return to idle pool
1660 pc
.wroteRequest() &&
1661 tryPutIdleConn(trace
)
1665 case <-rc
.req
.Cancel
:
1667 pc
.t
.CancelRequest(rc
.req
)
1668 case <-rc
.req
.Context().Done():
1670 pc
.t
.cancelRequest(rc
.req
, rc
.req
.Context().Err())
1675 testHookReadLoopBeforeNextRead()
1679 func (pc
*persistConn
) readLoopPeekFailLocked(peekErr error
) {
1680 if pc
.closed != nil {
1683 if n
:= pc
.br
.Buffered(); n
> 0 {
1684 buf
, _
:= pc
.br
.Peek(n
)
1685 log
.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf
, peekErr
)
1687 if peekErr
== io
.EOF
{
1689 pc
.closeLocked(errServerClosedIdle
)
1691 pc
.closeLocked(fmt
.Errorf("readLoopPeekFailLocked: %v", peekErr
))
1695 // readResponse reads an HTTP response (or two, in the case of "Expect:
1696 // 100-continue") from the server. It returns the final non-100 one.
1697 // trace is optional.
1698 func (pc
*persistConn
) readResponse(rc requestAndChan
, trace
*httptrace
.ClientTrace
) (resp
*Response
, err error
) {
1699 if trace
!= nil && trace
.GotFirstResponseByte
!= nil {
1700 if peek
, err
:= pc
.br
.Peek(1); err
== nil && len(peek
) == 1 {
1701 trace
.GotFirstResponseByte()
1704 resp
, err
= ReadResponse(pc
.br
, rc
.req
)
1708 if rc
.continueCh
!= nil {
1709 if resp
.StatusCode
== 100 {
1710 if trace
!= nil && trace
.Got100Continue
!= nil {
1711 trace
.Got100Continue()
1713 rc
.continueCh
<- struct{}{}
1715 close(rc
.continueCh
)
1718 if resp
.StatusCode
== 100 {
1719 pc
.readLimit
= pc
.maxHeaderResponseSize() // reset the limit
1720 resp
, err
= ReadResponse(pc
.br
, rc
.req
)
1725 resp
.TLS
= pc
.tlsState
1729 // waitForContinue returns the function to block until
1730 // any response, timeout or connection close. After any of them,
1731 // the function returns a bool which indicates if the body should be sent.
1732 func (pc
*persistConn
) waitForContinue(continueCh
<-chan struct{}) func() bool {
1733 if continueCh
== nil {
1736 return func() bool {
1737 timer
:= time
.NewTimer(pc
.t
.ExpectContinueTimeout
)
1741 case _
, ok
:= <-continueCh
:
1751 // nothingWrittenError wraps a write errors which ended up writing zero bytes.
1752 type nothingWrittenError
struct {
1756 func (pc
*persistConn
) writeLoop() {
1757 defer close(pc
.writeLoopDone
)
1760 case wr
:= <-pc
.writech
:
1761 startBytesWritten
:= pc
.nwrite
1762 err
:= wr
.req
.Request
.write(pc
.bw
, pc
.isProxy
, wr
.req
.extra
, pc
.waitForContinue(wr
.continueCh
))
1763 if bre
, ok
:= err
.(requestBodyReadError
); ok
{
1765 // Errors reading from the user's
1766 // Request.Body are high priority.
1767 // Set it here before sending on the
1768 // channels below or calling
1769 // pc.close() which tears town
1770 // connections and causes other
1772 wr
.req
.setError(err
)
1778 wr
.req
.Request
.closeBody()
1779 if pc
.nwrite
== startBytesWritten
{
1780 err
= nothingWrittenError
{err
}
1783 pc
.writeErrCh
<- err
// to the body reader, which might recycle us
1784 wr
.ch
<- err
// to the roundTrip function
1795 // wroteRequest is a check before recycling a connection that the previous write
1796 // (from writeLoop above) happened and was successful.
1797 func (pc
*persistConn
) wroteRequest() bool {
1799 case err
:= <-pc
.writeErrCh
:
1800 // Common case: the write happened well before the response, so
1801 // avoid creating a timer.
1804 // Rare case: the request was written in writeLoop above but
1805 // before it could send to pc.writeErrCh, the reader read it
1806 // all, processed it, and called us here. In this case, give the
1807 // write goroutine a bit of time to finish its send.
1809 // Less rare case: We also get here in the legitimate case of
1810 // Issue 7569, where the writer is still writing (or stalled),
1811 // but the server has already replied. In this case, we don't
1812 // want to wait too long, and we want to return false so this
1813 // connection isn't re-used.
1815 case err
:= <-pc
.writeErrCh
:
1817 case <-time
.After(50 * time
.Millisecond
):
1823 // responseAndError is how the goroutine reading from an HTTP/1 server
1824 // communicates with the goroutine doing the RoundTrip.
1825 type responseAndError
struct {
1826 res
*Response
// else use this response (see res method)
1830 type requestAndChan
struct {
1832 ch
chan responseAndError
// unbuffered; always send in select on callerGone
1834 // whether the Transport (as opposed to the user client code)
1835 // added the Accept-Encoding gzip header. If the Transport
1836 // set it, only then do we transparently decode the gzip.
1839 // Optional blocking chan for Expect: 100-continue (for send).
1840 // If the request has an "Expect: 100-continue" header and
1841 // the server responds 100 Continue, readLoop send a value
1842 // to writeLoop via this chan.
1843 continueCh
chan<- struct{}
1845 callerGone
<-chan struct{} // closed when roundTrip caller has returned
1848 // A writeRequest is sent by the readLoop's goroutine to the
1849 // writeLoop's goroutine to write a request while the read loop
1850 // concurrently waits on both the write response and the server's
1852 type writeRequest
struct {
1853 req
*transportRequest
1856 // Optional blocking chan for Expect: 100-continue (for receive).
1857 // If not nil, writeLoop blocks sending request body until
1858 // it receives from this chan.
1859 continueCh
<-chan struct{}
1862 type httpError
struct {
1867 func (e
*httpError
) Error() string { return e
.err
}
1868 func (e
*httpError
) Timeout() bool { return e
.timeout
}
1869 func (e
*httpError
) Temporary() bool { return true }
1871 var errTimeout error
= &httpError
{err
: "net/http: timeout awaiting response headers", timeout
: true}
1872 var errRequestCanceled
= errors
.New("net/http: request canceled")
1873 var errRequestCanceledConn
= errors
.New("net/http: request canceled while waiting for connection") // TODO: unify?
1877 // testHooks. Always non-nil.
1879 testHookEnterRoundTrip
= nop
1880 testHookWaitResLoop
= nop
1881 testHookRoundTripRetried
= nop
1882 testHookPrePendingDial
= nop
1883 testHookPostPendingDial
= nop
1885 testHookMu sync
.Locker
= fakeLocker
{} // guards following
1886 testHookReadLoopBeforeNextRead
= nop
1889 func (pc
*persistConn
) roundTrip(req
*transportRequest
) (resp
*Response
, err error
) {
1890 testHookEnterRoundTrip()
1891 if !pc
.t
.replaceReqCanceler(req
.Request
, pc
.cancelRequest
) {
1892 pc
.t
.putOrCloseIdleConn(pc
)
1893 return nil, errRequestCanceled
1896 pc
.numExpectedResponses
++
1897 headerFn
:= pc
.mutateHeaderFunc
1900 if headerFn
!= nil {
1901 headerFn(req
.extraHeaders())
1904 // Ask for a compressed version if the caller didn't set their
1905 // own value for Accept-Encoding. We only attempt to
1906 // uncompress the gzip stream if we were the layer that
1908 requestedGzip
:= false
1909 if !pc
.t
.DisableCompression
&&
1910 req
.Header
.Get("Accept-Encoding") == "" &&
1911 req
.Header
.Get("Range") == "" &&
1912 req
.Method
!= "HEAD" {
1913 // Request gzip only, not deflate. Deflate is ambiguous and
1914 // not as universally supported anyway.
1915 // See: http://www.gzip.org/zlib/zlib_faq.html#faq38
1917 // Note that we don't request this for HEAD requests,
1918 // due to a bug in nginx:
1919 // http://trac.nginx.org/nginx/ticket/358
1920 // https://golang.org/issue/5522
1922 // We don't request gzip if the request is for a range, since
1923 // auto-decoding a portion of a gzipped document will just fail
1924 // anyway. See https://golang.org/issue/8923
1925 requestedGzip
= true
1926 req
.extraHeaders().Set("Accept-Encoding", "gzip")
1929 var continueCh
chan struct{}
1930 if req
.ProtoAtLeast(1, 1) && req
.Body
!= nil && req
.expectsContinue() {
1931 continueCh
= make(chan struct{}, 1)
1934 if pc
.t
.DisableKeepAlives
{
1935 req
.extraHeaders().Set("Connection", "close")
1938 gone
:= make(chan struct{})
1943 pc
.t
.setReqCanceler(req
.Request
, nil)
1947 const debugRoundTrip
= false
1949 // Write the request concurrently with waiting for a response,
1950 // in case the server decides to reply before reading our full
1952 startBytesWritten
:= pc
.nwrite
1953 writeErrCh
:= make(chan error
, 1)
1954 pc
.writech
<- writeRequest
{req
, writeErrCh
, continueCh
}
1956 resc
:= make(chan responseAndError
)
1957 pc
.reqch
<- requestAndChan
{
1960 addedGzip
: requestedGzip
,
1961 continueCh
: continueCh
,
1965 var respHeaderTimer
<-chan time
.Time
1966 cancelChan
:= req
.Request
.Cancel
1967 ctxDoneChan
:= req
.Context().Done()
1969 testHookWaitResLoop()
1971 case err
:= <-writeErrCh
:
1973 req
.logf("writeErrCh resv: %T/%#v", err
, err
)
1976 pc
.close(fmt
.Errorf("write error: %v", err
))
1977 return nil, pc
.mapRoundTripError(req
, startBytesWritten
, err
)
1979 if d
:= pc
.t
.ResponseHeaderTimeout
; d
> 0 {
1981 req
.logf("starting timer for %v", d
)
1983 timer
:= time
.NewTimer(d
)
1984 defer timer
.Stop() // prevent leaks
1985 respHeaderTimer
= timer
.C
1989 req
.logf("closech recv: %T %#v", pc
.closed, pc
.closed)
1991 return nil, pc
.mapRoundTripError(req
, startBytesWritten
, pc
.closed)
1992 case <-respHeaderTimer
:
1994 req
.logf("timeout waiting for response headers.")
1996 pc
.close(errTimeout
)
1997 return nil, errTimeout
1999 if (re
.res
== nil) == (re
.err
== nil) {
2000 panic(fmt
.Sprintf("internal error: exactly one of res or err should be set; nil=%v", re
.res
== nil))
2003 req
.logf("resc recv: %p, %T/%#v", re
.res
, re
.err
, re
.err
)
2006 return nil, pc
.mapRoundTripError(req
, startBytesWritten
, re
.err
)
2010 pc
.t
.CancelRequest(req
.Request
)
2013 pc
.t
.cancelRequest(req
.Request
, req
.Context().Err())
2020 // tLogKey is a context WithValue key for test debugging contexts containing
2021 // a t.Logf func. See export_test.go's Request.WithT method.
2022 type tLogKey
struct{}
2024 func (r
*transportRequest
) logf(format
string, args
...interface{}) {
2025 if logf
, ok
:= r
.Request
.Context().Value(tLogKey
{}).(func(string, ...interface{})); ok
{
2026 logf(time
.Now().Format(time
.RFC3339Nano
)+": "+format
, args
...)
2030 // markReused marks this connection as having been successfully used for a
2031 // request and response.
2032 func (pc
*persistConn
) markReused() {
2038 // close closes the underlying TCP connection and closes
2039 // the pc.closech channel.
2041 // The provided err is only for testing and debugging; in normal
2042 // circumstances it should never be seen by users.
2043 func (pc
*persistConn
) close(err error
) {
2045 defer pc
.mu
.Unlock()
2049 func (pc
*persistConn
) closeLocked(err error
) {
2054 if pc
.closed == nil {
2057 // Do nothing; can only get here via getConn's
2058 // handlePendingDial's putOrCloseIdleConn when
2059 // it turns out the abandoned connection in
2060 // flight ended up negotiating an alternate
2061 // protocol. We don't use the connection
2062 // freelist for http2. That's done by the
2063 // alternate protocol's RoundTripper.
2069 pc
.mutateHeaderFunc
= nil
2072 var portMap
= map[string]string{
2078 // canonicalAddr returns url.Host but always with a ":port" suffix
2079 func canonicalAddr(url
*url
.URL
) string {
2080 addr
:= url
.Hostname()
2081 if v
, err
:= idnaASCII(addr
); err
== nil {
2086 port
= portMap
[url
.Scheme
]
2088 return net
.JoinHostPort(addr
, port
)
2091 // bodyEOFSignal is used by the HTTP/1 transport when reading response
2092 // bodies to make sure we see the end of a response body before
2093 // proceeding and reading on the connection again.
2095 // It wraps a ReadCloser but runs fn (if non-nil) at most
2096 // once, right before its final (error-producing) Read or Close call
2097 // returns. fn should return the new error to return from Read or Close.
2099 // If earlyCloseFn is non-nil and Close is called before io.EOF is
2100 // seen, earlyCloseFn is called instead of fn, and its return value is
2101 // the return value from Close.
2102 type bodyEOFSignal
struct {
2104 mu sync
.Mutex
// guards following 4 fields
2105 closed bool // whether Close has been called
2106 rerr error
// sticky Read error
2107 fn
func(error
) error
// err will be nil on Read io.EOF
2108 earlyCloseFn
func() error
// optional alt Close func used if io.EOF not seen
2111 var errReadOnClosedResBody
= errors
.New("http: read on closed response body")
2113 func (es
*bodyEOFSignal
) Read(p
[]byte) (n
int, err error
) {
2115 closed, rerr
:= es
.closed, es
.rerr
2118 return 0, errReadOnClosedResBody
2124 n
, err
= es
.body
.Read(p
)
2127 defer es
.mu
.Unlock()
2131 err
= es
.condfn(err
)
2136 func (es
*bodyEOFSignal
) Close() error
{
2138 defer es
.mu
.Unlock()
2143 if es
.earlyCloseFn
!= nil && es
.rerr
!= io
.EOF
{
2144 return es
.earlyCloseFn()
2146 err
:= es
.body
.Close()
2147 return es
.condfn(err
)
2150 // caller must hold es.mu.
2151 func (es
*bodyEOFSignal
) condfn(err error
) error
{
2160 // gzipReader wraps a response body so it can lazily
2161 // call gzip.NewReader on the first call to Read
2162 type gzipReader
struct {
2163 body
*bodyEOFSignal
// underlying HTTP/1 response body framing
2164 zr
*gzip
.Reader
// lazily-initialized gzip reader
2165 zerr error
// any error from gzip.NewReader; sticky
2168 func (gz
*gzipReader
) Read(p
[]byte) (n
int, err error
) {
2171 gz
.zr
, gz
.zerr
= gzip
.NewReader(gz
.body
)
2180 err
= errReadOnClosedResBody
2187 return gz
.zr
.Read(p
)
2190 func (gz
*gzipReader
) Close() error
{
2191 return gz
.body
.Close()
2194 type readerAndCloser
struct {
2199 type tlsHandshakeTimeoutError
struct{}
2201 func (tlsHandshakeTimeoutError
) Timeout() bool { return true }
2202 func (tlsHandshakeTimeoutError
) Temporary() bool { return true }
2203 func (tlsHandshakeTimeoutError
) Error() string { return "net/http: TLS handshake timeout" }
2205 // fakeLocker is a sync.Locker which does nothing. It's used to guard
2206 // test-only fields when not under test, to avoid runtime atomic
2208 type fakeLocker
struct{}
2210 func (fakeLocker
) Lock() {}
2211 func (fakeLocker
) Unlock() {}
2213 // clneTLSConfig returns a shallow clone of cfg, or a new zero tls.Config if
2214 // cfg is nil. This is safe to call even if cfg is in active use by a TLS
2215 // client or server.
2216 func cloneTLSConfig(cfg
*tls
.Config
) *tls
.Config
{
2218 return &tls
.Config
{}
2223 type connLRU
struct {
2224 ll
*list
.List
// list.Element.Value type is of *persistConn
2225 m
map[*persistConn
]*list
.Element
2228 // add adds pc to the head of the linked list.
2229 func (cl
*connLRU
) add(pc
*persistConn
) {
2232 cl
.m
= make(map[*persistConn
]*list
.Element
)
2234 ele
:= cl
.ll
.PushFront(pc
)
2235 if _
, ok
:= cl
.m
[pc
]; ok
{
2236 panic("persistConn was already in LRU")
2241 func (cl
*connLRU
) removeOldest() *persistConn
{
2243 pc
:= ele
.Value
.(*persistConn
)
2249 // remove removes pc from cl.
2250 func (cl
*connLRU
) remove(pc
*persistConn
) {
2251 if ele
, ok
:= cl
.m
[pc
]; ok
{
2257 // len returns the number of items in the cache.
2258 func (cl
*connLRU
) len() int {
2262 // validPort reports whether p (without the colon) is a valid port in
2263 // a URL, per RFC 3986 Section 3.2.3, which says the port may be
2264 // empty, or only contain digits.
2265 func validPort(p
string) bool {
2266 for _
, r
:= range []byte(p
) {
2267 if r
< '0' || r
> '9' {