net/socket/client_socket_pool_base.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/socket/client_socket_pool_base.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/compiler_specific.h"
  10 #include "base/format_macros.h"
  11 #include "base/location.h"
  12 #include "base/logging.h"
  13 #include "base/single_thread_task_runner.h"
  14 #include "base/stl_util.h"
  15 #include "base/strings/string_util.h"
  16 #include "base/thread_task_runner_handle.h"
  17 #include "base/time/time.h"
  18 #include "base/values.h"
  19 #include "net/base/net_errors.h"
  20 #include "net/log/net_log.h"
  21
  22 using base::TimeDelta;
  23
  24 namespace net {
  25
  26 namespace {
  27
  28 // Indicate whether we should enable idle socket cleanup timer. When timer is
  29 // disabled, sockets are closed next time a socket request is made.
  30 bool g_cleanup_timer_enabled = true;
  31
  32 // The timeout value, in seconds, used to clean up idle sockets that can't be
  33 // reused.
  34 //
  35 // Note: It's important to close idle sockets that have received data as soon
  36 // as possible because the received data may cause BSOD on Windows XP under
  37 // some conditions.  See http://crbug.com/4606.
  38 const int kCleanupInterval = 10;  // DO NOT INCREASE THIS TIMEOUT.
  39
  40 // Indicate whether or not we should establish a new transport layer connection
  41 // after a certain timeout has passed without receiving an ACK.
  42 bool g_connect_backup_jobs_enabled = true;
  43
  44 }  // namespace
  45
  46 ConnectJob::ConnectJob(const std::string& group_name,
  47                        base::TimeDelta timeout_duration,
  48                        RequestPriority priority,
  49                        Delegate* delegate,
  50                        const BoundNetLog& net_log)
  51     : group_name_(group_name),
  52       timeout_duration_(timeout_duration),
  53       priority_(priority),
  54       delegate_(delegate),
  55       net_log_(net_log),
  56       idle_(true) {
  57   DCHECK(!group_name.empty());
  58   DCHECK(delegate);
  59   net_log.BeginEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB,
  60                      NetLog::StringCallback("group_name", &group_name_));
  61 }
  62
  63 ConnectJob::~ConnectJob() {
  64   net_log().EndEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB);
  65 }
  66
  67 scoped_ptr<StreamSocket> ConnectJob::PassSocket() {
  68   return socket_.Pass();
  69 }
  70
  71 int ConnectJob::Connect() {
  72   if (timeout_duration_ != base::TimeDelta())
  73     timer_.Start(FROM_HERE, timeout_duration_, this, &ConnectJob::OnTimeout);
  74
  75   idle_ = false;
  76
  77   LogConnectStart();
  78
  79   int rv = ConnectInternal();
  80
  81   if (rv != ERR_IO_PENDING) {
  82     LogConnectCompletion(rv);
  83     delegate_ = NULL;
  84   }
  85
  86   return rv;
  87 }
  88
  89 void ConnectJob::SetSocket(scoped_ptr<StreamSocket> socket) {
  90   if (socket) {
  91     net_log().AddEvent(NetLog::TYPE_CONNECT_JOB_SET_SOCKET,
  92                        socket->NetLog().source().ToEventParametersCallback());
  93   }
  94   socket_ = socket.Pass();
  95 }
  96
  97 void ConnectJob::NotifyDelegateOfCompletion(int rv) {
  98   // The delegate will own |this|.
  99   Delegate* delegate = delegate_;
 100   delegate_ = NULL;
 101
 102   LogConnectCompletion(rv);
 103   delegate->OnConnectJobComplete(rv, this);
 104 }
 105
 106 void ConnectJob::ResetTimer(base::TimeDelta remaining_time) {
 107   timer_.Stop();
 108   timer_.Start(FROM_HERE, remaining_time, this, &ConnectJob::OnTimeout);
 109 }
 110
 111 void ConnectJob::LogConnectStart() {
 112   connect_timing_.connect_start = base::TimeTicks::Now();
 113   net_log().BeginEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_CONNECT);
 114 }
 115
 116 void ConnectJob::LogConnectCompletion(int net_error) {
 117   connect_timing_.connect_end = base::TimeTicks::Now();
 118   net_log().EndEventWithNetErrorCode(
 119       NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_CONNECT, net_error);
 120 }
 121
 122 void ConnectJob::OnTimeout() {
 123   // Make sure the socket is NULL before calling into |delegate|.
 124   SetSocket(scoped_ptr<StreamSocket>());
 125
 126   net_log_.AddEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_TIMED_OUT);
 127
 128   NotifyDelegateOfCompletion(ERR_TIMED_OUT);
 129 }
 130
 131 namespace internal {
 132
 133 ClientSocketPoolBaseHelper::Request::Request(
 134     ClientSocketHandle* handle,
 135     const CompletionCallback& callback,
 136     RequestPriority priority,
 137     bool ignore_limits,
 138     Flags flags,
 139     const BoundNetLog& net_log)
 140     : handle_(handle),
 141       callback_(callback),
 142       priority_(priority),
 143       ignore_limits_(ignore_limits),
 144       flags_(flags),
 145       net_log_(net_log) {
 146   if (ignore_limits_)
 147     DCHECK_EQ(priority_, MAXIMUM_PRIORITY);
 148 }
 149
 150 ClientSocketPoolBaseHelper::Request::~Request() {
 151   liveness_ = DEAD;
 152 }
 153
 154 void ClientSocketPoolBaseHelper::Request::CrashIfInvalid() const {
 155   CHECK_EQ(liveness_, ALIVE);
 156 }
 157
 158 ClientSocketPoolBaseHelper::ClientSocketPoolBaseHelper(
 159     HigherLayeredPool* pool,
 160     int max_sockets,
 161     int max_sockets_per_group,
 162     base::TimeDelta unused_idle_socket_timeout,
 163     base::TimeDelta used_idle_socket_timeout,
 164     ConnectJobFactory* connect_job_factory)
 165     : idle_socket_count_(0),
 166       connecting_socket_count_(0),
 167       handed_out_socket_count_(0),
 168       max_sockets_(max_sockets),
 169       max_sockets_per_group_(max_sockets_per_group),
 170       use_cleanup_timer_(g_cleanup_timer_enabled),
 171       unused_idle_socket_timeout_(unused_idle_socket_timeout),
 172       used_idle_socket_timeout_(used_idle_socket_timeout),
 173       connect_job_factory_(connect_job_factory),
 174       connect_backup_jobs_enabled_(false),
 175       pool_generation_number_(0),
 176       pool_(pool),
 177       weak_factory_(this) {
 178   DCHECK_LE(0, max_sockets_per_group);
 179   DCHECK_LE(max_sockets_per_group, max_sockets);
 180
 181   NetworkChangeNotifier::AddIPAddressObserver(this);
 182 }
 183
 184 ClientSocketPoolBaseHelper::~ClientSocketPoolBaseHelper() {
 185   // Clean up any idle sockets and pending connect jobs.  Assert that we have no
 186   // remaining active sockets or pending requests.  They should have all been
 187   // cleaned up prior to |this| being destroyed.
 188   FlushWithError(ERR_ABORTED);
 189   DCHECK(group_map_.empty());
 190   DCHECK(pending_callback_map_.empty());
 191   DCHECK_EQ(0, connecting_socket_count_);
 192   CHECK(higher_pools_.empty());
 193
 194   NetworkChangeNotifier::RemoveIPAddressObserver(this);
 195
 196   // Remove from lower layer pools.
 197   for (std::set<LowerLayeredPool*>::iterator it = lower_pools_.begin();
 198        it != lower_pools_.end();
 199        ++it) {
 200     (*it)->RemoveHigherLayeredPool(pool_);
 201   }
 202 }
 203
 204 ClientSocketPoolBaseHelper::CallbackResultPair::CallbackResultPair()
 205     : result(OK) {
 206 }
 207
 208 ClientSocketPoolBaseHelper::CallbackResultPair::CallbackResultPair(
 209     const CompletionCallback& callback_in, int result_in)
 210     : callback(callback_in),
 211       result(result_in) {
 212 }
 213
 214 ClientSocketPoolBaseHelper::CallbackResultPair::~CallbackResultPair() {}
 215
 216 bool ClientSocketPoolBaseHelper::IsStalled() const {
 217   // If a lower layer pool is stalled, consider |this| stalled as well.
 218   for (std::set<LowerLayeredPool*>::const_iterator it = lower_pools_.begin();
 219        it != lower_pools_.end();
 220        ++it) {
 221     if ((*it)->IsStalled())
 222       return true;
 223   }
 224
 225   // If fewer than |max_sockets_| are in use, then clearly |this| is not
 226   // stalled.
 227   if ((handed_out_socket_count_ + connecting_socket_count_) < max_sockets_)
 228     return false;
 229   // So in order to be stalled, |this| must be using at least |max_sockets_| AND
 230   // |this| must have a request that is actually stalled on the global socket
 231   // limit.  To find such a request, look for a group that has more requests
 232   // than jobs AND where the number of sockets is less than
 233   // |max_sockets_per_group_|.  (If the number of sockets is equal to
 234   // |max_sockets_per_group_|, then the request is stalled on the group limit,
 235   // which does not count.)
 236   for (GroupMap::const_iterator it = group_map_.begin();
 237        it != group_map_.end(); ++it) {
 238     if (it->second->CanUseAdditionalSocketSlot(max_sockets_per_group_))
 239       return true;
 240   }
 241   return false;
 242 }
 243
 244 void ClientSocketPoolBaseHelper::AddLowerLayeredPool(
 245     LowerLayeredPool* lower_pool) {
 246   DCHECK(pool_);
 247   CHECK(!ContainsKey(lower_pools_, lower_pool));
 248   lower_pools_.insert(lower_pool);
 249   lower_pool->AddHigherLayeredPool(pool_);
 250 }
 251
 252 void ClientSocketPoolBaseHelper::AddHigherLayeredPool(
 253     HigherLayeredPool* higher_pool) {
 254   CHECK(higher_pool);
 255   CHECK(!ContainsKey(higher_pools_, higher_pool));
 256   higher_pools_.insert(higher_pool);
 257 }
 258
 259 void ClientSocketPoolBaseHelper::RemoveHigherLayeredPool(
 260     HigherLayeredPool* higher_pool) {
 261   CHECK(higher_pool);
 262   CHECK(ContainsKey(higher_pools_, higher_pool));
 263   higher_pools_.erase(higher_pool);
 264 }
 265
 266 int ClientSocketPoolBaseHelper::RequestSocket(
 267     const std::string& group_name,
 268     scoped_ptr<const Request> request) {
 269   CHECK(!request->callback().is_null());
 270   CHECK(request->handle());
 271
 272   // Cleanup any timed-out idle sockets if no timer is used.
 273   if (!use_cleanup_timer_)
 274     CleanupIdleSockets(false);
 275
 276   request->net_log().BeginEvent(NetLog::TYPE_SOCKET_POOL);
 277   Group* group = GetOrCreateGroup(group_name);
 278
 279   int rv = RequestSocketInternal(group_name, *request);
 280   if (rv != ERR_IO_PENDING) {
 281     request->net_log().EndEventWithNetErrorCode(NetLog::TYPE_SOCKET_POOL, rv);
 282     CHECK(!request->handle()->is_initialized());
 283     request.reset();
 284   } else {
 285     group->InsertPendingRequest(request.Pass());
 286     // Have to do this asynchronously, as closing sockets in higher level pools
 287     // call back in to |this|, which will cause all sorts of fun and exciting
 288     // re-entrancy issues if the socket pool is doing something else at the
 289     // time.
 290     if (group->CanUseAdditionalSocketSlot(max_sockets_per_group_)) {
 291       base::ThreadTaskRunnerHandle::Get()->PostTask(
 292           FROM_HERE,
 293           base::Bind(
 294               &ClientSocketPoolBaseHelper::TryToCloseSocketsInLayeredPools,
 295               weak_factory_.GetWeakPtr()));
 296     }
 297   }
 298   return rv;
 299 }
 300
 301 void ClientSocketPoolBaseHelper::RequestSockets(
 302     const std::string& group_name,
 303     const Request& request,
 304     int num_sockets) {
 305   DCHECK(request.callback().is_null());
 306   DCHECK(!request.handle());
 307
 308   // Cleanup any timed out idle sockets if no timer is used.
 309   if (!use_cleanup_timer_)
 310     CleanupIdleSockets(false);
 311
 312   if (num_sockets > max_sockets_per_group_) {
 313     num_sockets = max_sockets_per_group_;
 314   }
 315
 316   request.net_log().BeginEvent(
 317       NetLog::TYPE_SOCKET_POOL_CONNECTING_N_SOCKETS,
 318       NetLog::IntegerCallback("num_sockets", num_sockets));
 319
 320   Group* group = GetOrCreateGroup(group_name);
 321
 322   // RequestSocketsInternal() may delete the group.
 323   bool deleted_group = false;
 324
 325   int rv = OK;
 326   for (int num_iterations_left = num_sockets;
 327        group->NumActiveSocketSlots() < num_sockets &&
 328        num_iterations_left > 0 ; num_iterations_left--) {
 329     rv = RequestSocketInternal(group_name, request);
 330     if (rv < 0 && rv != ERR_IO_PENDING) {
 331       // We're encountering a synchronous error.  Give up.
 332       if (!ContainsKey(group_map_, group_name))
 333         deleted_group = true;
 334       break;
 335     }
 336     if (!ContainsKey(group_map_, group_name)) {
 337       // Unexpected.  The group should only be getting deleted on synchronous
 338       // error.
 339       NOTREACHED();
 340       deleted_group = true;
 341       break;
 342     }
 343   }
 344
 345   if (!deleted_group && group->IsEmpty())
 346     RemoveGroup(group_name);
 347
 348   if (rv == ERR_IO_PENDING)
 349     rv = OK;
 350   request.net_log().EndEventWithNetErrorCode(
 351       NetLog::TYPE_SOCKET_POOL_CONNECTING_N_SOCKETS, rv);
 352 }
 353
 354 int ClientSocketPoolBaseHelper::RequestSocketInternal(
 355     const std::string& group_name,
 356     const Request& request) {
 357   ClientSocketHandle* const handle = request.handle();
 358   const bool preconnecting = !handle;
 359   Group* group = GetOrCreateGroup(group_name);
 360
 361   if (!(request.flags() & NO_IDLE_SOCKETS)) {
 362     // Try to reuse a socket.
 363     if (AssignIdleSocketToRequest(request, group))
 364       return OK;
 365   }
 366
 367   // If there are more ConnectJobs than pending requests, don't need to do
 368   // anything.  Can just wait for the extra job to connect, and then assign it
 369   // to the request.
 370   if (!preconnecting && group->TryToUseUnassignedConnectJob())
 371     return ERR_IO_PENDING;
 372
 373   // Can we make another active socket now?
 374   if (!group->HasAvailableSocketSlot(max_sockets_per_group_) &&
 375       !request.ignore_limits()) {
 376     // TODO(willchan): Consider whether or not we need to close a socket in a
 377     // higher layered group. I don't think this makes sense since we would just
 378     // reuse that socket then if we needed one and wouldn't make it down to this
 379     // layer.
 380     request.net_log().AddEvent(
 381         NetLog::TYPE_SOCKET_POOL_STALLED_MAX_SOCKETS_PER_GROUP);
 382     return ERR_IO_PENDING;
 383   }
 384
 385   if (ReachedMaxSocketsLimit() && !request.ignore_limits()) {
 386     // NOTE(mmenke):  Wonder if we really need different code for each case
 387     // here.  Only reason for them now seems to be preconnects.
 388     if (idle_socket_count() > 0) {
 389       // There's an idle socket in this pool. Either that's because there's
 390       // still one in this group, but we got here due to preconnecting bypassing
 391       // idle sockets, or because there's an idle socket in another group.
 392       bool closed = CloseOneIdleSocketExceptInGroup(group);
 393       if (preconnecting && !closed)
 394         return ERR_PRECONNECT_MAX_SOCKET_LIMIT;
 395     } else {
 396       // We could check if we really have a stalled group here, but it requires
 397       // a scan of all groups, so just flip a flag here, and do the check later.
 398       request.net_log().AddEvent(NetLog::TYPE_SOCKET_POOL_STALLED_MAX_SOCKETS);
 399       return ERR_IO_PENDING;
 400     }
 401   }
 402
 403   // We couldn't find a socket to reuse, and there's space to allocate one,
 404   // so allocate and connect a new one.
 405   scoped_ptr<ConnectJob> connect_job(
 406       connect_job_factory_->NewConnectJob(group_name, request, this));
 407
 408   int rv = connect_job->Connect();
 409   if (rv == OK) {
 410     LogBoundConnectJobToRequest(connect_job->net_log().source(), request);
 411     if (!preconnecting) {
 412       HandOutSocket(connect_job->PassSocket(), ClientSocketHandle::UNUSED,
 413                     connect_job->connect_timing(), handle, base::TimeDelta(),
 414                     group, request.net_log());
 415     } else {
 416       AddIdleSocket(connect_job->PassSocket(), group);
 417     }
 418   } else if (rv == ERR_IO_PENDING) {
 419     // If we don't have any sockets in this group, set a timer for potentially
 420     // creating a new one.  If the SYN is lost, this backup socket may complete
 421     // before the slow socket, improving end user latency.
 422     if (connect_backup_jobs_enabled_ && group->IsEmpty()) {
 423       group->StartBackupJobTimer(group_name, this);
 424     }
 425
 426     connecting_socket_count_++;
 427
 428     group->AddJob(connect_job.Pass(), preconnecting);
 429   } else {
 430     LogBoundConnectJobToRequest(connect_job->net_log().source(), request);
 431     scoped_ptr<StreamSocket> error_socket;
 432     if (!preconnecting) {
 433       DCHECK(handle);
 434       connect_job->GetAdditionalErrorState(handle);
 435       error_socket = connect_job->PassSocket();
 436     }
 437     if (error_socket) {
 438       HandOutSocket(error_socket.Pass(), ClientSocketHandle::UNUSED,
 439                     connect_job->connect_timing(), handle, base::TimeDelta(),
 440                     group, request.net_log());
 441     } else if (group->IsEmpty()) {
 442       RemoveGroup(group_name);
 443     }
 444   }
 445
 446   return rv;
 447 }
 448
 449 bool ClientSocketPoolBaseHelper::AssignIdleSocketToRequest(
 450     const Request& request, Group* group) {
 451   std::list<IdleSocket>* idle_sockets = group->mutable_idle_sockets();
 452   std::list<IdleSocket>::iterator idle_socket_it = idle_sockets->end();
 453
 454   // Iterate through the idle sockets forwards (oldest to newest)
 455   //   * Delete any disconnected ones.
 456   //   * If we find a used idle socket, assign to |idle_socket|.  At the end,
 457   //   the |idle_socket_it| will be set to the newest used idle socket.
 458   for (std::list<IdleSocket>::iterator it = idle_sockets->begin();
 459        it != idle_sockets->end();) {
 460     if (!it->IsUsable()) {
 461       DecrementIdleCount();
 462       delete it->socket;
 463       it = idle_sockets->erase(it);
 464       continue;
 465     }
 466
 467     if (it->socket->WasEverUsed()) {
 468       // We found one we can reuse!
 469       idle_socket_it = it;
 470     }
 471
 472     ++it;
 473   }
 474
 475   // If we haven't found an idle socket, that means there are no used idle
 476   // sockets.  Pick the oldest (first) idle socket (FIFO).
 477
 478   if (idle_socket_it == idle_sockets->end() && !idle_sockets->empty())
 479     idle_socket_it = idle_sockets->begin();
 480
 481   if (idle_socket_it != idle_sockets->end()) {
 482     DecrementIdleCount();
 483     base::TimeDelta idle_time =
 484         base::TimeTicks::Now() - idle_socket_it->start_time;
 485     IdleSocket idle_socket = *idle_socket_it;
 486     idle_sockets->erase(idle_socket_it);
 487     // TODO(davidben): If |idle_time| is under some low watermark, consider
 488     // treating as UNUSED rather than UNUSED_IDLE. This will avoid
 489     // HttpNetworkTransaction retrying on some errors.
 490     ClientSocketHandle::SocketReuseType reuse_type =
 491         idle_socket.socket->WasEverUsed() ?
 492             ClientSocketHandle::REUSED_IDLE :
 493             ClientSocketHandle::UNUSED_IDLE;
 494
 495     // If this socket took multiple attempts to obtain, don't report those
 496     // every time it's reused, just to the first user.
 497     if (idle_socket.socket->WasEverUsed())
 498       idle_socket.socket->ClearConnectionAttempts();
 499
 500     HandOutSocket(
 501         scoped_ptr<StreamSocket>(idle_socket.socket),
 502         reuse_type,
 503         LoadTimingInfo::ConnectTiming(),
 504         request.handle(),
 505         idle_time,
 506         group,
 507         request.net_log());
 508     return true;
 509   }
 510
 511   return false;
 512 }
 513
 514 // static
 515 void ClientSocketPoolBaseHelper::LogBoundConnectJobToRequest(
 516     const NetLog::Source& connect_job_source, const Request& request) {
 517   request.net_log().AddEvent(NetLog::TYPE_SOCKET_POOL_BOUND_TO_CONNECT_JOB,
 518                              connect_job_source.ToEventParametersCallback());
 519 }
 520
 521 void ClientSocketPoolBaseHelper::CancelRequest(
 522     const std::string& group_name, ClientSocketHandle* handle) {
 523   PendingCallbackMap::iterator callback_it = pending_callback_map_.find(handle);
 524   if (callback_it != pending_callback_map_.end()) {
 525     int result = callback_it->second.result;
 526     pending_callback_map_.erase(callback_it);
 527     scoped_ptr<StreamSocket> socket = handle->PassSocket();
 528     if (socket) {
 529       if (result != OK)
 530         socket->Disconnect();
 531       ReleaseSocket(handle->group_name(), socket.Pass(), handle->id());
 532     }
 533     return;
 534   }
 535
 536   CHECK(ContainsKey(group_map_, group_name));
 537
 538   Group* group = GetOrCreateGroup(group_name);
 539
 540   // Search pending_requests for matching handle.
 541   scoped_ptr<const Request> request =
 542       group->FindAndRemovePendingRequest(handle);
 543   if (request) {
 544     request->net_log().AddEvent(NetLog::TYPE_CANCELLED);
 545     request->net_log().EndEvent(NetLog::TYPE_SOCKET_POOL);
 546
 547     // We let the job run, unless we're at the socket limit and there is
 548     // not another request waiting on the job.
 549     if (group->jobs().size() > group->pending_request_count() &&
 550         ReachedMaxSocketsLimit()) {
 551       RemoveConnectJob(*group->jobs().begin(), group);
 552       CheckForStalledSocketGroups();
 553     }
 554   }
 555 }
 556
 557 bool ClientSocketPoolBaseHelper::HasGroup(const std::string& group_name) const {
 558   return ContainsKey(group_map_, group_name);
 559 }
 560
 561 void ClientSocketPoolBaseHelper::CloseIdleSockets() {
 562   CleanupIdleSockets(true);
 563   DCHECK_EQ(0, idle_socket_count_);
 564 }
 565
 566 int ClientSocketPoolBaseHelper::IdleSocketCountInGroup(
 567     const std::string& group_name) const {
 568   GroupMap::const_iterator i = group_map_.find(group_name);
 569   CHECK(i != group_map_.end());
 570
 571   return i->second->idle_sockets().size();
 572 }
 573
 574 LoadState ClientSocketPoolBaseHelper::GetLoadState(
 575     const std::string& group_name,
 576     const ClientSocketHandle* handle) const {
 577   if (ContainsKey(pending_callback_map_, handle))
 578     return LOAD_STATE_CONNECTING;
 579
 580   GroupMap::const_iterator group_it = group_map_.find(group_name);
 581   if (group_it == group_map_.end()) {
 582     // TODO(mmenke):  This is actually reached in the wild, for unknown reasons.
 583     // Would be great to understand why, and if it's a bug, fix it.  If not,
 584     // should have a test for that case.
 585     NOTREACHED();
 586     return LOAD_STATE_IDLE;
 587   }
 588
 589   const Group& group = *group_it->second;
 590   if (group.HasConnectJobForHandle(handle)) {
 591     // Just return the state of the oldest ConnectJob.
 592     return (*group.jobs().begin())->GetLoadState();
 593   }
 594
 595   if (group.CanUseAdditionalSocketSlot(max_sockets_per_group_))
 596     return LOAD_STATE_WAITING_FOR_STALLED_SOCKET_POOL;
 597   return LOAD_STATE_WAITING_FOR_AVAILABLE_SOCKET;
 598 }
 599
 600 scoped_ptr<base::DictionaryValue> ClientSocketPoolBaseHelper::GetInfoAsValue(
 601     const std::string& name, const std::string& type) const {
 602   scoped_ptr<base::DictionaryValue> dict(new base::DictionaryValue());
 603   dict->SetString("name", name);
 604   dict->SetString("type", type);
 605   dict->SetInteger("handed_out_socket_count", handed_out_socket_count_);
 606   dict->SetInteger("connecting_socket_count", connecting_socket_count_);
 607   dict->SetInteger("idle_socket_count", idle_socket_count_);
 608   dict->SetInteger("max_socket_count", max_sockets_);
 609   dict->SetInteger("max_sockets_per_group", max_sockets_per_group_);
 610   dict->SetInteger("pool_generation_number", pool_generation_number_);
 611
 612   if (group_map_.empty())
 613     return dict.Pass();
 614
 615   base::DictionaryValue* all_groups_dict = new base::DictionaryValue();
 616   for (GroupMap::const_iterator it = group_map_.begin();
 617        it != group_map_.end(); it++) {
 618     const Group* group = it->second;
 619     base::DictionaryValue* group_dict = new base::DictionaryValue();
 620
 621     group_dict->SetInteger("pending_request_count",
 622                            group->pending_request_count());
 623     if (group->has_pending_requests()) {
 624       group_dict->SetString(
 625           "top_pending_priority",
 626           RequestPriorityToString(group->TopPendingPriority()));
 627     }
 628
 629     group_dict->SetInteger("active_socket_count", group->active_socket_count());
 630
 631     base::ListValue* idle_socket_list = new base::ListValue();
 632     std::list<IdleSocket>::const_iterator idle_socket;
 633     for (idle_socket = group->idle_sockets().begin();
 634          idle_socket != group->idle_sockets().end();
 635          idle_socket++) {
 636       int source_id = idle_socket->socket->NetLog().source().id;
 637       idle_socket_list->Append(new base::FundamentalValue(source_id));
 638     }
 639     group_dict->Set("idle_sockets", idle_socket_list);
 640
 641     base::ListValue* connect_jobs_list = new base::ListValue();
 642     std::list<ConnectJob*>::const_iterator job = group->jobs().begin();
 643     for (job = group->jobs().begin(); job != group->jobs().end(); job++) {
 644       int source_id = (*job)->net_log().source().id;
 645       connect_jobs_list->Append(new base::FundamentalValue(source_id));
 646     }
 647     group_dict->Set("connect_jobs", connect_jobs_list);
 648
 649     group_dict->SetBoolean("is_stalled", group->CanUseAdditionalSocketSlot(
 650                                              max_sockets_per_group_));
 651     group_dict->SetBoolean("backup_job_timer_is_running",
 652                            group->BackupJobTimerIsRunning());
 653
 654     all_groups_dict->SetWithoutPathExpansion(it->first, group_dict);
 655   }
 656   dict->Set("groups", all_groups_dict);
 657   return dict.Pass();
 658 }
 659
 660 bool ClientSocketPoolBaseHelper::IdleSocket::IsUsable() const {
 661   if (socket->WasEverUsed())
 662     return socket->IsConnectedAndIdle();
 663   return socket->IsConnected();
 664 }
 665
 666 bool ClientSocketPoolBaseHelper::IdleSocket::ShouldCleanup(
 667     base::TimeTicks now,
 668     base::TimeDelta timeout) const {
 669   bool timed_out = (now - start_time) >= timeout;
 670   if (timed_out)
 671     return true;
 672   return !IsUsable();
 673 }
 674
 675 void ClientSocketPoolBaseHelper::CleanupIdleSockets(bool force) {
 676   if (idle_socket_count_ == 0)
 677     return;
 678
 679   // Current time value. Retrieving it once at the function start rather than
 680   // inside the inner loop, since it shouldn't change by any meaningful amount.
 681   base::TimeTicks now = base::TimeTicks::Now();
 682
 683   GroupMap::iterator i = group_map_.begin();
 684   while (i != group_map_.end()) {
 685     Group* group = i->second;
 686
 687     std::list<IdleSocket>::iterator j = group->mutable_idle_sockets()->begin();
 688     while (j != group->idle_sockets().end()) {
 689       base::TimeDelta timeout =
 690           j->socket->WasEverUsed() ?
 691           used_idle_socket_timeout_ : unused_idle_socket_timeout_;
 692       if (force || j->ShouldCleanup(now, timeout)) {
 693         delete j->socket;
 694         j = group->mutable_idle_sockets()->erase(j);
 695         DecrementIdleCount();
 696       } else {
 697         ++j;
 698       }
 699     }
 700
 701     // Delete group if no longer needed.
 702     if (group->IsEmpty()) {
 703       RemoveGroup(i++);
 704     } else {
 705       ++i;
 706     }
 707   }
 708 }
 709
 710 ClientSocketPoolBaseHelper::Group* ClientSocketPoolBaseHelper::GetOrCreateGroup(
 711     const std::string& group_name) {
 712   GroupMap::iterator it = group_map_.find(group_name);
 713   if (it != group_map_.end())
 714     return it->second;
 715   Group* group = new Group;
 716   group_map_[group_name] = group;
 717   return group;
 718 }
 719
 720 void ClientSocketPoolBaseHelper::RemoveGroup(const std::string& group_name) {
 721   GroupMap::iterator it = group_map_.find(group_name);
 722   CHECK(it != group_map_.end());
 723
 724   RemoveGroup(it);
 725 }
 726
 727 void ClientSocketPoolBaseHelper::RemoveGroup(GroupMap::iterator it) {
 728   delete it->second;
 729   group_map_.erase(it);
 730 }
 731
 732 // static
 733 bool ClientSocketPoolBaseHelper::connect_backup_jobs_enabled() {
 734   return g_connect_backup_jobs_enabled;
 735 }
 736
 737 // static
 738 bool ClientSocketPoolBaseHelper::set_connect_backup_jobs_enabled(bool enabled) {
 739   bool old_value = g_connect_backup_jobs_enabled;
 740   g_connect_backup_jobs_enabled = enabled;
 741   return old_value;
 742 }
 743
 744 void ClientSocketPoolBaseHelper::EnableConnectBackupJobs() {
 745   connect_backup_jobs_enabled_ = g_connect_backup_jobs_enabled;
 746 }
 747
 748 void ClientSocketPoolBaseHelper::IncrementIdleCount() {
 749   if (++idle_socket_count_ == 1 && use_cleanup_timer_)
 750     StartIdleSocketTimer();
 751 }
 752
 753 void ClientSocketPoolBaseHelper::DecrementIdleCount() {
 754   if (--idle_socket_count_ == 0)
 755     timer_.Stop();
 756 }
 757
 758 // static
 759 bool ClientSocketPoolBaseHelper::cleanup_timer_enabled() {
 760   return g_cleanup_timer_enabled;
 761 }
 762
 763 // static
 764 bool ClientSocketPoolBaseHelper::set_cleanup_timer_enabled(bool enabled) {
 765   bool old_value = g_cleanup_timer_enabled;
 766   g_cleanup_timer_enabled = enabled;
 767   return old_value;
 768 }
 769
 770 void ClientSocketPoolBaseHelper::StartIdleSocketTimer() {
 771   timer_.Start(FROM_HERE, TimeDelta::FromSeconds(kCleanupInterval), this,
 772                &ClientSocketPoolBaseHelper::OnCleanupTimerFired);
 773 }
 774
 775 void ClientSocketPoolBaseHelper::ReleaseSocket(const std::string& group_name,
 776                                                scoped_ptr<StreamSocket> socket,
 777                                                int id) {
 778   GroupMap::iterator i = group_map_.find(group_name);
 779   CHECK(i != group_map_.end());
 780
 781   Group* group = i->second;
 782
 783   CHECK_GT(handed_out_socket_count_, 0);
 784   handed_out_socket_count_--;
 785
 786   CHECK_GT(group->active_socket_count(), 0);
 787   group->DecrementActiveSocketCount();
 788
 789   const bool can_reuse = socket->IsConnectedAndIdle() &&
 790       id == pool_generation_number_;
 791   if (can_reuse) {
 792     // Add it to the idle list.
 793     AddIdleSocket(socket.Pass(), group);
 794     OnAvailableSocketSlot(group_name, group);
 795   } else {
 796     socket.reset();
 797   }
 798
 799   CheckForStalledSocketGroups();
 800 }
 801
 802 void ClientSocketPoolBaseHelper::CheckForStalledSocketGroups() {
 803   // If we have idle sockets, see if we can give one to the top-stalled group.
 804   std::string top_group_name;
 805   Group* top_group = NULL;
 806   if (!FindTopStalledGroup(&top_group, &top_group_name)) {
 807     // There may still be a stalled group in a lower level pool.
 808     for (std::set<LowerLayeredPool*>::iterator it = lower_pools_.begin();
 809          it != lower_pools_.end();
 810          ++it) {
 811        if ((*it)->IsStalled()) {
 812          CloseOneIdleSocket();
 813          break;
 814        }
 815     }
 816     return;
 817   }
 818
 819   if (ReachedMaxSocketsLimit()) {
 820     if (idle_socket_count() > 0) {
 821       CloseOneIdleSocket();
 822     } else {
 823       // We can't activate more sockets since we're already at our global
 824       // limit.
 825       return;
 826     }
 827   }
 828
 829   // Note:  we don't loop on waking stalled groups.  If the stalled group is at
 830   //        its limit, may be left with other stalled groups that could be
 831   //        woken.  This isn't optimal, but there is no starvation, so to avoid
 832   //        the looping we leave it at this.
 833   OnAvailableSocketSlot(top_group_name, top_group);
 834 }
 835
 836 // Search for the highest priority pending request, amongst the groups that
 837 // are not at the |max_sockets_per_group_| limit. Note: for requests with
 838 // the same priority, the winner is based on group hash ordering (and not
 839 // insertion order).
 840 bool ClientSocketPoolBaseHelper::FindTopStalledGroup(
 841     Group** group,
 842     std::string* group_name) const {
 843   CHECK((group && group_name) || (!group && !group_name));
 844   Group* top_group = NULL;
 845   const std::string* top_group_name = NULL;
 846   bool has_stalled_group = false;
 847   for (GroupMap::const_iterator i = group_map_.begin();
 848        i != group_map_.end(); ++i) {
 849     Group* curr_group = i->second;
 850     if (!curr_group->has_pending_requests())
 851       continue;
 852     if (curr_group->CanUseAdditionalSocketSlot(max_sockets_per_group_)) {
 853       if (!group)
 854         return true;
 855       has_stalled_group = true;
 856       bool has_higher_priority = !top_group ||
 857           curr_group->TopPendingPriority() > top_group->TopPendingPriority();
 858       if (has_higher_priority) {
 859         top_group = curr_group;
 860         top_group_name = &i->first;
 861       }
 862     }
 863   }
 864
 865   if (top_group) {
 866     CHECK(group);
 867     *group = top_group;
 868     *group_name = *top_group_name;
 869   } else {
 870     CHECK(!has_stalled_group);
 871   }
 872   return has_stalled_group;
 873 }
 874
 875 void ClientSocketPoolBaseHelper::OnConnectJobComplete(
 876     int result, ConnectJob* job) {
 877   DCHECK_NE(ERR_IO_PENDING, result);
 878   const std::string group_name = job->group_name();
 879   GroupMap::iterator group_it = group_map_.find(group_name);
 880   CHECK(group_it != group_map_.end());
 881   Group* group = group_it->second;
 882
 883   scoped_ptr<StreamSocket> socket = job->PassSocket();
 884
 885   // Copies of these are needed because |job| may be deleted before they are
 886   // accessed.
 887   BoundNetLog job_log = job->net_log();
 888   LoadTimingInfo::ConnectTiming connect_timing = job->connect_timing();
 889
 890   // RemoveConnectJob(job, _) must be called by all branches below;
 891   // otherwise, |job| will be leaked.
 892
 893   if (result == OK) {
 894     DCHECK(socket.get());
 895     RemoveConnectJob(job, group);
 896     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 897     if (request) {
 898       LogBoundConnectJobToRequest(job_log.source(), *request);
 899       HandOutSocket(
 900           socket.Pass(), ClientSocketHandle::UNUSED, connect_timing,
 901           request->handle(), base::TimeDelta(), group, request->net_log());
 902       request->net_log().EndEvent(NetLog::TYPE_SOCKET_POOL);
 903       InvokeUserCallbackLater(request->handle(), request->callback(), result);
 904     } else {
 905       AddIdleSocket(socket.Pass(), group);
 906       OnAvailableSocketSlot(group_name, group);
 907       CheckForStalledSocketGroups();
 908     }
 909   } else {
 910     // If we got a socket, it must contain error information so pass that
 911     // up so that the caller can retrieve it.
 912     bool handed_out_socket = false;
 913     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 914     if (request) {
 915       LogBoundConnectJobToRequest(job_log.source(), *request);
 916       job->GetAdditionalErrorState(request->handle());
 917       RemoveConnectJob(job, group);
 918       if (socket.get()) {
 919         handed_out_socket = true;
 920         HandOutSocket(socket.Pass(), ClientSocketHandle::UNUSED,
 921                       connect_timing, request->handle(), base::TimeDelta(),
 922                       group, request->net_log());
 923       }
 924       request->net_log().EndEventWithNetErrorCode(
 925           NetLog::TYPE_SOCKET_POOL, result);
 926       InvokeUserCallbackLater(request->handle(), request->callback(), result);
 927     } else {
 928       RemoveConnectJob(job, group);
 929     }
 930     if (!handed_out_socket) {
 931       OnAvailableSocketSlot(group_name, group);
 932       CheckForStalledSocketGroups();
 933     }
 934   }
 935 }
 936
 937 void ClientSocketPoolBaseHelper::OnIPAddressChanged() {
 938   FlushWithError(ERR_NETWORK_CHANGED);
 939 }
 940
 941 void ClientSocketPoolBaseHelper::FlushWithError(int error) {
 942   pool_generation_number_++;
 943   CancelAllConnectJobs();
 944   CloseIdleSockets();
 945   CancelAllRequestsWithError(error);
 946 }
 947
 948 void ClientSocketPoolBaseHelper::RemoveConnectJob(ConnectJob* job,
 949                                                   Group* group) {
 950   CHECK_GT(connecting_socket_count_, 0);
 951   connecting_socket_count_--;
 952
 953   DCHECK(group);
 954   group->RemoveJob(job);
 955 }
 956
 957 void ClientSocketPoolBaseHelper::OnAvailableSocketSlot(
 958     const std::string& group_name, Group* group) {
 959   DCHECK(ContainsKey(group_map_, group_name));
 960   if (group->IsEmpty()) {
 961     RemoveGroup(group_name);
 962   } else if (group->has_pending_requests()) {
 963     ProcessPendingRequest(group_name, group);
 964   }
 965 }
 966
 967 void ClientSocketPoolBaseHelper::ProcessPendingRequest(
 968     const std::string& group_name, Group* group) {
 969   const Request* next_request = group->GetNextPendingRequest();
 970   DCHECK(next_request);
 971
 972   // If the group has no idle sockets, and can't make use of an additional slot,
 973   // either because it's at the limit or because it's at the socket per group
 974   // limit, then there's nothing to do.
 975   if (group->idle_sockets().empty() &&
 976       !group->CanUseAdditionalSocketSlot(max_sockets_per_group_)) {
 977     return;
 978   }
 979
 980   int rv = RequestSocketInternal(group_name, *next_request);
 981   if (rv != ERR_IO_PENDING) {
 982     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 983     DCHECK(request);
 984     if (group->IsEmpty())
 985       RemoveGroup(group_name);
 986
 987     request->net_log().EndEventWithNetErrorCode(NetLog::TYPE_SOCKET_POOL, rv);
 988     InvokeUserCallbackLater(request->handle(), request->callback(), rv);
 989   }
 990 }
 991
 992 void ClientSocketPoolBaseHelper::HandOutSocket(
 993     scoped_ptr<StreamSocket> socket,
 994     ClientSocketHandle::SocketReuseType reuse_type,
 995     const LoadTimingInfo::ConnectTiming& connect_timing,
 996     ClientSocketHandle* handle,
 997     base::TimeDelta idle_time,
 998     Group* group,
 999     const BoundNetLog& net_log) {
1000   DCHECK(socket);
1001   handle->SetSocket(socket.Pass());
1002   handle->set_reuse_type(reuse_type);
1003   handle->set_idle_time(idle_time);
1004   handle->set_pool_id(pool_generation_number_);
1005   handle->set_connect_timing(connect_timing);
1006
1007   if (handle->is_reused()) {
1008     net_log.AddEvent(
1009         NetLog::TYPE_SOCKET_POOL_REUSED_AN_EXISTING_SOCKET,
1010         NetLog::IntegerCallback(
1011             "idle_ms", static_cast<int>(idle_time.InMilliseconds())));
1012   }
1013
1014   net_log.AddEvent(
1015       NetLog::TYPE_SOCKET_POOL_BOUND_TO_SOCKET,
1016       handle->socket()->NetLog().source().ToEventParametersCallback());
1017
1018   handed_out_socket_count_++;
1019   group->IncrementActiveSocketCount();
1020 }
1021
1022 void ClientSocketPoolBaseHelper::AddIdleSocket(
1023     scoped_ptr<StreamSocket> socket,
1024     Group* group) {
1025   DCHECK(socket);
1026   IdleSocket idle_socket;
1027   idle_socket.socket = socket.release();
1028   idle_socket.start_time = base::TimeTicks::Now();
1029
1030   group->mutable_idle_sockets()->push_back(idle_socket);
1031   IncrementIdleCount();
1032 }
1033
1034 void ClientSocketPoolBaseHelper::CancelAllConnectJobs() {
1035   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end();) {
1036     Group* group = i->second;
1037     connecting_socket_count_ -= group->jobs().size();
1038     group->RemoveAllJobs();
1039
1040     // Delete group if no longer needed.
1041     if (group->IsEmpty()) {
1042       // RemoveGroup() will call .erase() which will invalidate the iterator,
1043       // but i will already have been incremented to a valid iterator before
1044       // RemoveGroup() is called.
1045       RemoveGroup(i++);
1046     } else {
1047       ++i;
1048     }
1049   }
1050   DCHECK_EQ(0, connecting_socket_count_);
1051 }
1052
1053 void ClientSocketPoolBaseHelper::CancelAllRequestsWithError(int error) {
1054   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end();) {
1055     Group* group = i->second;
1056
1057     while (true) {
1058       scoped_ptr<const Request> request = group->PopNextPendingRequest();
1059       if (!request)
1060         break;
1061       InvokeUserCallbackLater(request->handle(), request->callback(), error);
1062     }
1063
1064     // Delete group if no longer needed.
1065     if (group->IsEmpty()) {
1066       // RemoveGroup() will call .erase() which will invalidate the iterator,
1067       // but i will already have been incremented to a valid iterator before
1068       // RemoveGroup() is called.
1069       RemoveGroup(i++);
1070     } else {
1071       ++i;
1072     }
1073   }
1074 }
1075
1076 bool ClientSocketPoolBaseHelper::ReachedMaxSocketsLimit() const {
1077   // Each connecting socket will eventually connect and be handed out.
1078   int total = handed_out_socket_count_ + connecting_socket_count_ +
1079       idle_socket_count();
1080   // There can be more sockets than the limit since some requests can ignore
1081   // the limit
1082   if (total < max_sockets_)
1083     return false;
1084   return true;
1085 }
1086
1087 bool ClientSocketPoolBaseHelper::CloseOneIdleSocket() {
1088   if (idle_socket_count() == 0)
1089     return false;
1090   return CloseOneIdleSocketExceptInGroup(NULL);
1091 }
1092
1093 bool ClientSocketPoolBaseHelper::CloseOneIdleSocketExceptInGroup(
1094     const Group* exception_group) {
1095   CHECK_GT(idle_socket_count(), 0);
1096
1097   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end(); ++i) {
1098     Group* group = i->second;
1099     if (exception_group == group)
1100       continue;
1101     std::list<IdleSocket>* idle_sockets = group->mutable_idle_sockets();
1102
1103     if (!idle_sockets->empty()) {
1104       delete idle_sockets->front().socket;
1105       idle_sockets->pop_front();
1106       DecrementIdleCount();
1107       if (group->IsEmpty())
1108         RemoveGroup(i);
1109
1110       return true;
1111     }
1112   }
1113
1114   return false;
1115 }
1116
1117 bool ClientSocketPoolBaseHelper::CloseOneIdleConnectionInHigherLayeredPool() {
1118   // This pool doesn't have any idle sockets. It's possible that a pool at a
1119   // higher layer is holding one of this sockets active, but it's actually idle.
1120   // Query the higher layers.
1121   for (std::set<HigherLayeredPool*>::const_iterator it = higher_pools_.begin();
1122        it != higher_pools_.end(); ++it) {
1123     if ((*it)->CloseOneIdleConnection())
1124       return true;
1125   }
1126   return false;
1127 }
1128
1129 void ClientSocketPoolBaseHelper::InvokeUserCallbackLater(
1130     ClientSocketHandle* handle, const CompletionCallback& callback, int rv) {
1131   CHECK(!ContainsKey(pending_callback_map_, handle));
1132   pending_callback_map_[handle] = CallbackResultPair(callback, rv);
1133   base::ThreadTaskRunnerHandle::Get()->PostTask(
1134       FROM_HERE, base::Bind(&ClientSocketPoolBaseHelper::InvokeUserCallback,
1135                             weak_factory_.GetWeakPtr(), handle));
1136 }
1137
1138 void ClientSocketPoolBaseHelper::InvokeUserCallback(
1139     ClientSocketHandle* handle) {
1140   PendingCallbackMap::iterator it = pending_callback_map_.find(handle);
1141
1142   // Exit if the request has already been cancelled.
1143   if (it == pending_callback_map_.end())
1144     return;
1145
1146   CHECK(!handle->is_initialized());
1147   CompletionCallback callback = it->second.callback;
1148   int result = it->second.result;
1149   pending_callback_map_.erase(it);
1150   callback.Run(result);
1151 }
1152
1153 void ClientSocketPoolBaseHelper::TryToCloseSocketsInLayeredPools() {
1154   while (IsStalled()) {
1155     // Closing a socket will result in calling back into |this| to use the freed
1156     // socket slot, so nothing else is needed.
1157     if (!CloseOneIdleConnectionInHigherLayeredPool())
1158       return;
1159   }
1160 }
1161
1162 ClientSocketPoolBaseHelper::Group::Group()
1163     : unassigned_job_count_(0),
1164       pending_requests_(NUM_PRIORITIES),
1165       active_socket_count_(0) {}
1166
1167 ClientSocketPoolBaseHelper::Group::~Group() {
1168   DCHECK_EQ(0u, unassigned_job_count_);
1169 }
1170
1171 void ClientSocketPoolBaseHelper::Group::StartBackupJobTimer(
1172     const std::string& group_name,
1173     ClientSocketPoolBaseHelper* pool) {
1174   // Only allow one timer to run at a time.
1175   if (BackupJobTimerIsRunning())
1176     return;
1177
1178   // Unretained here is okay because |backup_job_timer_| is
1179   // automatically cancelled when it's destroyed.
1180   backup_job_timer_.Start(
1181       FROM_HERE, pool->ConnectRetryInterval(),
1182       base::Bind(&Group::OnBackupJobTimerFired, base::Unretained(this),
1183                  group_name, pool));
1184 }
1185
1186 bool ClientSocketPoolBaseHelper::Group::BackupJobTimerIsRunning() const {
1187   return backup_job_timer_.IsRunning();
1188 }
1189
1190 bool ClientSocketPoolBaseHelper::Group::TryToUseUnassignedConnectJob() {
1191   SanityCheck();
1192
1193   if (unassigned_job_count_ == 0)
1194     return false;
1195   --unassigned_job_count_;
1196   return true;
1197 }
1198
1199 void ClientSocketPoolBaseHelper::Group::AddJob(scoped_ptr<ConnectJob> job,
1200                                                bool is_preconnect) {
1201   SanityCheck();
1202
1203   if (is_preconnect)
1204     ++unassigned_job_count_;
1205   jobs_.push_back(job.release());
1206 }
1207
1208 void ClientSocketPoolBaseHelper::Group::RemoveJob(ConnectJob* job) {
1209   scoped_ptr<ConnectJob> owned_job(job);
1210   SanityCheck();
1211
1212   // Check that |job| is in the list.
1213   DCHECK_EQ(*std::find(jobs_.begin(), jobs_.end(), job), job);
1214   jobs_.remove(job);
1215   size_t job_count = jobs_.size();
1216   if (job_count < unassigned_job_count_)
1217     unassigned_job_count_ = job_count;
1218
1219   // If we've got no more jobs for this group, then we no longer need a
1220   // backup job either.
1221   if (jobs_.empty())
1222     backup_job_timer_.Stop();
1223 }
1224
1225 void ClientSocketPoolBaseHelper::Group::OnBackupJobTimerFired(
1226     std::string group_name,
1227     ClientSocketPoolBaseHelper* pool) {
1228   // If there are no more jobs pending, there is no work to do.
1229   // If we've done our cleanups correctly, this should not happen.
1230   if (jobs_.empty()) {
1231     NOTREACHED();
1232     return;
1233   }
1234
1235   // If our old job is waiting on DNS, or if we can't create any sockets
1236   // right now due to limits, just reset the timer.
1237   if (pool->ReachedMaxSocketsLimit() ||
1238       !HasAvailableSocketSlot(pool->max_sockets_per_group_) ||
1239       (*jobs_.begin())->GetLoadState() == LOAD_STATE_RESOLVING_HOST) {
1240     StartBackupJobTimer(group_name, pool);
1241     return;
1242   }
1243
1244   if (pending_requests_.empty())
1245     return;
1246
1247   scoped_ptr<ConnectJob> backup_job =
1248       pool->connect_job_factory_->NewConnectJob(
1249           group_name, *pending_requests_.FirstMax().value(), pool);
1250   backup_job->net_log().AddEvent(NetLog::TYPE_BACKUP_CONNECT_JOB_CREATED);
1251   int rv = backup_job->Connect();
1252   pool->connecting_socket_count_++;
1253   ConnectJob* raw_backup_job = backup_job.get();
1254   AddJob(backup_job.Pass(), false);
1255   if (rv != ERR_IO_PENDING)
1256     pool->OnConnectJobComplete(rv, raw_backup_job);
1257 }
1258
1259 void ClientSocketPoolBaseHelper::Group::SanityCheck() {
1260   DCHECK_LE(unassigned_job_count_, jobs_.size());
1261 }
1262
1263 void ClientSocketPoolBaseHelper::Group::RemoveAllJobs() {
1264   SanityCheck();
1265
1266   // Delete active jobs.
1267   STLDeleteElements(&jobs_);
1268   unassigned_job_count_ = 0;
1269
1270   // Stop backup job timer.
1271   backup_job_timer_.Stop();
1272 }
1273
1274 const ClientSocketPoolBaseHelper::Request*
1275 ClientSocketPoolBaseHelper::Group::GetNextPendingRequest() const {
1276   return
1277       pending_requests_.empty() ? NULL : pending_requests_.FirstMax().value();
1278 }
1279
1280 bool ClientSocketPoolBaseHelper::Group::HasConnectJobForHandle(
1281     const ClientSocketHandle* handle) const {
1282   // Search the first |jobs_.size()| pending requests for |handle|.
1283   // If it's farther back in the deque than that, it doesn't have a
1284   // corresponding ConnectJob.
1285   size_t i = 0;
1286   for (RequestQueue::Pointer pointer = pending_requests_.FirstMax();
1287        !pointer.is_null() && i < jobs_.size();
1288        pointer = pending_requests_.GetNextTowardsLastMin(pointer), ++i) {
1289     if (pointer.value()->handle() == handle)
1290       return true;
1291   }
1292   return false;
1293 }
1294
1295 void ClientSocketPoolBaseHelper::Group::InsertPendingRequest(
1296     scoped_ptr<const Request> request) {
1297   // This value must be cached before we release |request|.
1298   RequestPriority priority = request->priority();
1299   if (request->ignore_limits()) {
1300     // Put requests with ignore_limits == true (which should have
1301     // priority == MAXIMUM_PRIORITY) ahead of other requests with
1302     // MAXIMUM_PRIORITY.
1303     DCHECK_EQ(priority, MAXIMUM_PRIORITY);
1304     pending_requests_.InsertAtFront(request.release(), priority);
1305   } else {
1306     pending_requests_.Insert(request.release(), priority);
1307   }
1308 }
1309
1310 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1311 ClientSocketPoolBaseHelper::Group::PopNextPendingRequest() {
1312   if (pending_requests_.empty())
1313     return scoped_ptr<const ClientSocketPoolBaseHelper::Request>();
1314   return RemovePendingRequest(pending_requests_.FirstMax());
1315 }
1316
1317 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1318 ClientSocketPoolBaseHelper::Group::FindAndRemovePendingRequest(
1319     ClientSocketHandle* handle) {
1320   for (RequestQueue::Pointer pointer = pending_requests_.FirstMax();
1321        !pointer.is_null();
1322        pointer = pending_requests_.GetNextTowardsLastMin(pointer)) {
1323     if (pointer.value()->handle() == handle) {
1324       scoped_ptr<const Request> request = RemovePendingRequest(pointer);
1325       return request.Pass();
1326     }
1327   }
1328   return scoped_ptr<const ClientSocketPoolBaseHelper::Request>();
1329 }
1330
1331 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1332 ClientSocketPoolBaseHelper::Group::RemovePendingRequest(
1333     const RequestQueue::Pointer& pointer) {
1334   // TODO(eroman): Temporary for debugging http://crbug.com/467797.
1335   CHECK(!pointer.is_null());
1336   scoped_ptr<const Request> request(pointer.value());
1337   pending_requests_.Erase(pointer);
1338   // If there are no more requests, kill the backup timer.
1339   if (pending_requests_.empty())
1340     backup_job_timer_.Stop();
1341   request->CrashIfInvalid();
1342   return request.Pass();
1343 }
1344
1345 }  // namespace internal
1346
1347 }  // namespace net