net/url_request/url_fetcher_core.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_fetcher_core.h"
   6
   7 #include <stdint.h>
   8
   9 #include "base/bind.h"
  10 #include "base/logging.h"
  11 #include "base/metrics/histogram_macros.h"
  12 #include "base/profiler/scoped_tracker.h"
  13 #include "base/sequenced_task_runner.h"
  14 #include "base/single_thread_task_runner.h"
  15 #include "base/stl_util.h"
  16 #include "base/thread_task_runner_handle.h"
  17 #include "base/tracked_objects.h"
  18 #include "net/base/elements_upload_data_stream.h"
  19 #include "net/base/io_buffer.h"
  20 #include "net/base/load_flags.h"
  21 #include "net/base/net_errors.h"
  22 #include "net/base/request_priority.h"
  23 #include "net/base/upload_bytes_element_reader.h"
  24 #include "net/base/upload_data_stream.h"
  25 #include "net/base/upload_file_element_reader.h"
  26 #include "net/http/http_response_headers.h"
  27 #include "net/url_request/redirect_info.h"
  28 #include "net/url_request/url_fetcher_delegate.h"
  29 #include "net/url_request/url_fetcher_response_writer.h"
  30 #include "net/url_request/url_request_context.h"
  31 #include "net/url_request/url_request_context_getter.h"
  32 #include "net/url_request/url_request_throttler_manager.h"
  33
  34 namespace {
  35
  36 const int kBufferSize = 4096;
  37 const int kUploadProgressTimerInterval = 100;
  38 bool g_ignore_certificate_requests = false;
  39
  40 void EmptyCompletionCallback(int result) {}
  41
  42 }  // namespace
  43
  44 namespace net {
  45
  46 // URLFetcherCore::Registry ---------------------------------------------------
  47
  48 URLFetcherCore::Registry::Registry() {}
  49 URLFetcherCore::Registry::~Registry() {}
  50
  51 void URLFetcherCore::Registry::AddURLFetcherCore(URLFetcherCore* core) {
  52   DCHECK(!ContainsKey(fetchers_, core));
  53   fetchers_.insert(core);
  54 }
  55
  56 void URLFetcherCore::Registry::RemoveURLFetcherCore(URLFetcherCore* core) {
  57   DCHECK(ContainsKey(fetchers_, core));
  58   fetchers_.erase(core);
  59 }
  60
  61 void URLFetcherCore::Registry::CancelAll() {
  62   while (!fetchers_.empty())
  63     (*fetchers_.begin())->CancelURLRequest(ERR_ABORTED);
  64 }
  65
  66 // URLFetcherCore -------------------------------------------------------------
  67
  68 // static
  69 base::LazyInstance<URLFetcherCore::Registry>
  70     URLFetcherCore::g_registry = LAZY_INSTANCE_INITIALIZER;
  71
  72 URLFetcherCore::URLFetcherCore(URLFetcher* fetcher,
  73                                const GURL& original_url,
  74                                URLFetcher::RequestType request_type,
  75                                URLFetcherDelegate* d)
  76     : fetcher_(fetcher),
  77       original_url_(original_url),
  78       request_type_(request_type),
  79       delegate_(d),
  80       delegate_task_runner_(base::ThreadTaskRunnerHandle::Get()),
  81       load_flags_(LOAD_NORMAL),
  82       response_code_(URLFetcher::RESPONSE_CODE_INVALID),
  83       buffer_(new IOBuffer(kBufferSize)),
  84       url_request_data_key_(NULL),
  85       was_fetched_via_proxy_(false),
  86       was_cached_(false),
  87       received_response_content_length_(0),
  88       total_received_bytes_(0),
  89       upload_content_set_(false),
  90       upload_range_offset_(0),
  91       upload_range_length_(0),
  92       referrer_policy_(
  93           URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE),
  94       is_chunked_upload_(false),
  95       was_cancelled_(false),
  96       stop_on_redirect_(false),
  97       stopped_on_redirect_(false),
  98       automatically_retry_on_5xx_(true),
  99       num_retries_on_5xx_(0),
 100       max_retries_on_5xx_(0),
 101       num_retries_on_network_changes_(0),
 102       max_retries_on_network_changes_(0),
 103       current_upload_bytes_(-1),
 104       current_response_bytes_(0),
 105       total_response_bytes_(-1) {
 106   CHECK(original_url_.is_valid());
 107 }
 108
 109 void URLFetcherCore::Start() {
 110   DCHECK(delegate_task_runner_.get());
 111   DCHECK(request_context_getter_.get()) << "We need an URLRequestContext!";
 112   if (network_task_runner_.get()) {
 113     DCHECK_EQ(network_task_runner_,
 114               request_context_getter_->GetNetworkTaskRunner());
 115   } else {
 116     network_task_runner_ = request_context_getter_->GetNetworkTaskRunner();
 117   }
 118   DCHECK(network_task_runner_.get()) << "We need an IO task runner";
 119
 120   network_task_runner_->PostTask(
 121       FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
 122 }
 123
 124 void URLFetcherCore::Stop() {
 125   if (delegate_task_runner_.get())  // May be NULL in tests.
 126     DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 127
 128   delegate_ = NULL;
 129   fetcher_ = NULL;
 130   if (!network_task_runner_.get())
 131     return;
 132   if (network_task_runner_->RunsTasksOnCurrentThread()) {
 133     CancelURLRequest(ERR_ABORTED);
 134   } else {
 135     network_task_runner_->PostTask(
 136         FROM_HERE,
 137         base::Bind(&URLFetcherCore::CancelURLRequest, this, ERR_ABORTED));
 138   }
 139 }
 140
 141 void URLFetcherCore::SetUploadData(const std::string& upload_content_type,
 142                                    const std::string& upload_content) {
 143   AssertHasNoUploadData();
 144   DCHECK(!is_chunked_upload_);
 145   DCHECK(upload_content_type_.empty());
 146
 147   // Empty |upload_content_type| is allowed iff the |upload_content| is empty.
 148   DCHECK(upload_content.empty() || !upload_content_type.empty());
 149
 150   upload_content_type_ = upload_content_type;
 151   upload_content_ = upload_content;
 152   upload_content_set_ = true;
 153 }
 154
 155 void URLFetcherCore::SetUploadFilePath(
 156     const std::string& upload_content_type,
 157     const base::FilePath& file_path,
 158     uint64 range_offset,
 159     uint64 range_length,
 160     scoped_refptr<base::TaskRunner> file_task_runner) {
 161   AssertHasNoUploadData();
 162   DCHECK(!is_chunked_upload_);
 163   DCHECK_EQ(upload_range_offset_, 0ULL);
 164   DCHECK_EQ(upload_range_length_, 0ULL);
 165   DCHECK(upload_content_type_.empty());
 166   DCHECK(!upload_content_type.empty());
 167
 168   upload_content_type_ = upload_content_type;
 169   upload_file_path_ = file_path;
 170   upload_range_offset_ = range_offset;
 171   upload_range_length_ = range_length;
 172   upload_file_task_runner_ = file_task_runner;
 173   upload_content_set_ = true;
 174 }
 175
 176 void URLFetcherCore::SetUploadStreamFactory(
 177     const std::string& upload_content_type,
 178     const URLFetcher::CreateUploadStreamCallback& factory) {
 179   AssertHasNoUploadData();
 180   DCHECK(!is_chunked_upload_);
 181   DCHECK(upload_content_type_.empty());
 182
 183   upload_content_type_ = upload_content_type;
 184   upload_stream_factory_ = factory;
 185   upload_content_set_ = true;
 186 }
 187
 188 void URLFetcherCore::SetChunkedUpload(const std::string& content_type) {
 189   if (!is_chunked_upload_) {
 190     AssertHasNoUploadData();
 191     DCHECK(upload_content_type_.empty());
 192   }
 193
 194   // Empty |content_type| is not allowed here, because it is impossible
 195   // to ensure non-empty upload content as it is not yet supplied.
 196   DCHECK(!content_type.empty());
 197
 198   upload_content_type_ = content_type;
 199   upload_content_.clear();
 200   is_chunked_upload_ = true;
 201 }
 202
 203 void URLFetcherCore::AppendChunkToUpload(const std::string& content,
 204                                          bool is_last_chunk) {
 205   DCHECK(delegate_task_runner_.get());
 206   DCHECK(network_task_runner_.get());
 207   network_task_runner_->PostTask(
 208       FROM_HERE,
 209       base::Bind(&URLFetcherCore::CompleteAddingUploadDataChunk, this, content,
 210                  is_last_chunk));
 211 }
 212
 213 void URLFetcherCore::SetLoadFlags(int load_flags) {
 214   load_flags_ = load_flags;
 215 }
 216
 217 int URLFetcherCore::GetLoadFlags() const {
 218   return load_flags_;
 219 }
 220
 221 void URLFetcherCore::SetReferrer(const std::string& referrer) {
 222   referrer_ = referrer;
 223 }
 224
 225 void URLFetcherCore::SetReferrerPolicy(
 226     URLRequest::ReferrerPolicy referrer_policy) {
 227   referrer_policy_ = referrer_policy;
 228 }
 229
 230 void URLFetcherCore::SetExtraRequestHeaders(
 231     const std::string& extra_request_headers) {
 232   extra_request_headers_.Clear();
 233   extra_request_headers_.AddHeadersFromString(extra_request_headers);
 234 }
 235
 236 void URLFetcherCore::AddExtraRequestHeader(const std::string& header_line) {
 237   extra_request_headers_.AddHeaderFromString(header_line);
 238 }
 239
 240 void URLFetcherCore::SetRequestContext(
 241     URLRequestContextGetter* request_context_getter) {
 242   DCHECK(!request_context_getter_.get());
 243   DCHECK(request_context_getter);
 244   request_context_getter_ = request_context_getter;
 245 }
 246
 247 void URLFetcherCore::SetFirstPartyForCookies(
 248     const GURL& first_party_for_cookies) {
 249   DCHECK(first_party_for_cookies_.is_empty());
 250   first_party_for_cookies_ = first_party_for_cookies;
 251 }
 252
 253 void URLFetcherCore::SetURLRequestUserData(
 254     const void* key,
 255     const URLFetcher::CreateDataCallback& create_data_callback) {
 256   DCHECK(key);
 257   DCHECK(!create_data_callback.is_null());
 258   url_request_data_key_ = key;
 259   url_request_create_data_callback_ = create_data_callback;
 260 }
 261
 262 void URLFetcherCore::SetStopOnRedirect(bool stop_on_redirect) {
 263   stop_on_redirect_ = stop_on_redirect;
 264 }
 265
 266 void URLFetcherCore::SetAutomaticallyRetryOn5xx(bool retry) {
 267   automatically_retry_on_5xx_ = retry;
 268 }
 269
 270 void URLFetcherCore::SetMaxRetriesOn5xx(int max_retries) {
 271   max_retries_on_5xx_ = max_retries;
 272 }
 273
 274 int URLFetcherCore::GetMaxRetriesOn5xx() const {
 275   return max_retries_on_5xx_;
 276 }
 277
 278 base::TimeDelta URLFetcherCore::GetBackoffDelay() const {
 279   return backoff_delay_;
 280 }
 281
 282 void URLFetcherCore::SetAutomaticallyRetryOnNetworkChanges(int max_retries) {
 283   max_retries_on_network_changes_ = max_retries;
 284 }
 285
 286 void URLFetcherCore::SaveResponseToFileAtPath(
 287     const base::FilePath& file_path,
 288     scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
 289   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 290   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
 291       new URLFetcherFileWriter(file_task_runner, file_path)));
 292 }
 293
 294 void URLFetcherCore::SaveResponseToTemporaryFile(
 295     scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
 296   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 297   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
 298       new URLFetcherFileWriter(file_task_runner, base::FilePath())));
 299 }
 300
 301 void URLFetcherCore::SaveResponseWithWriter(
 302     scoped_ptr<URLFetcherResponseWriter> response_writer) {
 303   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 304   response_writer_ = response_writer.Pass();
 305 }
 306
 307 HttpResponseHeaders* URLFetcherCore::GetResponseHeaders() const {
 308   return response_headers_.get();
 309 }
 310
 311 // TODO(panayiotis): socket_address_ is written in the IO thread,
 312 // if this is accessed in the UI thread, this could result in a race.
 313 // Same for response_headers_ above and was_fetched_via_proxy_ below.
 314 HostPortPair URLFetcherCore::GetSocketAddress() const {
 315   return socket_address_;
 316 }
 317
 318 bool URLFetcherCore::WasFetchedViaProxy() const {
 319   return was_fetched_via_proxy_;
 320 }
 321
 322 bool URLFetcherCore::WasCached() const {
 323   return was_cached_;
 324 }
 325
 326 int64_t URLFetcherCore::GetReceivedResponseContentLength() const {
 327   return received_response_content_length_;
 328 }
 329
 330 int64_t URLFetcherCore::GetTotalReceivedBytes() const {
 331   return total_received_bytes_;
 332 }
 333
 334 const GURL& URLFetcherCore::GetOriginalURL() const {
 335   return original_url_;
 336 }
 337
 338 const GURL& URLFetcherCore::GetURL() const {
 339   return url_;
 340 }
 341
 342 const URLRequestStatus& URLFetcherCore::GetStatus() const {
 343   return status_;
 344 }
 345
 346 int URLFetcherCore::GetResponseCode() const {
 347   return response_code_;
 348 }
 349
 350 const ResponseCookies& URLFetcherCore::GetCookies() const {
 351   return cookies_;
 352 }
 353
 354 void URLFetcherCore::ReceivedContentWasMalformed() {
 355   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 356   if (network_task_runner_.get()) {
 357     network_task_runner_->PostTask(
 358         FROM_HERE, base::Bind(&URLFetcherCore::NotifyMalformedContent, this));
 359   }
 360 }
 361
 362 bool URLFetcherCore::GetResponseAsString(
 363     std::string* out_response_string) const {
 364   URLFetcherStringWriter* string_writer =
 365       response_writer_ ? response_writer_->AsStringWriter() : NULL;
 366   if (!string_writer)
 367     return false;
 368
 369   *out_response_string = string_writer->data();
 370   UMA_HISTOGRAM_MEMORY_KB("UrlFetcher.StringResponseSize",
 371                           (string_writer->data().length() / 1024));
 372   return true;
 373 }
 374
 375 bool URLFetcherCore::GetResponseAsFilePath(bool take_ownership,
 376                                            base::FilePath* out_response_path) {
 377   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 378
 379   URLFetcherFileWriter* file_writer =
 380       response_writer_ ? response_writer_->AsFileWriter() : NULL;
 381   if (!file_writer)
 382     return false;
 383
 384   *out_response_path = file_writer->file_path();
 385
 386   if (take_ownership) {
 387     // Intentionally calling a file_writer_ method directly without posting
 388     // the task to network_task_runner_.
 389     //
 390     // This is for correctly handling the case when file_writer_->DisownFile()
 391     // is soon followed by URLFetcherCore::Stop(). We have to make sure that
 392     // DisownFile takes effect before Stop deletes file_writer_.
 393     //
 394     // This direct call should be thread-safe, since DisownFile itself does no
 395     // file operation. It just flips the state to be referred in destruction.
 396     file_writer->DisownFile();
 397   }
 398   return true;
 399 }
 400
 401 void URLFetcherCore::OnReceivedRedirect(URLRequest* request,
 402                                         const RedirectInfo& redirect_info,
 403                                         bool* defer_redirect) {
 404   DCHECK_EQ(request, request_.get());
 405   DCHECK(network_task_runner_->BelongsToCurrentThread());
 406   if (stop_on_redirect_) {
 407     stopped_on_redirect_ = true;
 408     url_ = redirect_info.new_url;
 409     response_code_ = request_->GetResponseCode();
 410     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
 411     was_cached_ = request_->was_cached();
 412     total_received_bytes_ += request_->GetTotalReceivedBytes();
 413     request->Cancel();
 414     OnReadCompleted(request, 0);
 415   }
 416 }
 417
 418 void URLFetcherCore::OnResponseStarted(URLRequest* request) {
 419   DCHECK_EQ(request, request_.get());
 420   DCHECK(network_task_runner_->BelongsToCurrentThread());
 421   if (request_->status().is_success()) {
 422     response_code_ = request_->GetResponseCode();
 423     response_headers_ = request_->response_headers();
 424     socket_address_ = request_->GetSocketAddress();
 425     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
 426     was_cached_ = request_->was_cached();
 427     total_response_bytes_ = request_->GetExpectedContentSize();
 428   }
 429
 430   ReadResponse();
 431 }
 432
 433 void URLFetcherCore::OnCertificateRequested(
 434     URLRequest* request,
 435     SSLCertRequestInfo* cert_request_info) {
 436   DCHECK_EQ(request, request_.get());
 437   DCHECK(network_task_runner_->BelongsToCurrentThread());
 438
 439   if (g_ignore_certificate_requests) {
 440     request->ContinueWithCertificate(NULL);
 441   } else {
 442     request->Cancel();
 443   }
 444 }
 445
 446 void URLFetcherCore::OnReadCompleted(URLRequest* request,
 447                                      int bytes_read) {
 448   DCHECK(request == request_);
 449   DCHECK(network_task_runner_->BelongsToCurrentThread());
 450
 451   if (!stopped_on_redirect_)
 452     url_ = request->url();
 453   URLRequestThrottlerManager* throttler_manager =
 454       request->context()->throttler_manager();
 455   if (throttler_manager)
 456     url_throttler_entry_ = throttler_manager->RegisterRequestUrl(url_);
 457
 458   do {
 459     if (!request_->status().is_success() || bytes_read <= 0)
 460       break;
 461
 462     current_response_bytes_ += bytes_read;
 463     InformDelegateDownloadProgress();
 464
 465     const int result =
 466         WriteBuffer(new DrainableIOBuffer(buffer_.get(), bytes_read));
 467     if (result < 0) {
 468       // Write failed or waiting for write completion.
 469       return;
 470     }
 471   } while (request_->Read(buffer_.get(), kBufferSize, &bytes_read));
 472
 473   const URLRequestStatus status = request_->status();
 474   if (status.is_success())
 475     request_->GetResponseCookies(&cookies_);
 476
 477   // See comments re: HEAD requests in ReadResponse().
 478   if (!status.is_io_pending() || request_type_ == URLFetcher::HEAD) {
 479     status_ = status;
 480     received_response_content_length_ =
 481         request_->received_response_content_length();
 482     total_received_bytes_ += request_->GetTotalReceivedBytes();
 483     ReleaseRequest();
 484
 485     // No more data to write.
 486     const int result = response_writer_->Finish(
 487         base::Bind(&URLFetcherCore::DidFinishWriting, this));
 488     if (result != ERR_IO_PENDING)
 489       DidFinishWriting(result);
 490   }
 491 }
 492
 493 void URLFetcherCore::OnContextShuttingDown() {
 494   DCHECK(request_);
 495   CancelRequestAndInformDelegate(ERR_CONTEXT_SHUT_DOWN);
 496 }
 497
 498 void URLFetcherCore::CancelAll() {
 499   g_registry.Get().CancelAll();
 500 }
 501
 502 int URLFetcherCore::GetNumFetcherCores() {
 503   return g_registry.Get().size();
 504 }
 505
 506 void URLFetcherCore::SetIgnoreCertificateRequests(bool ignored) {
 507   g_ignore_certificate_requests = ignored;
 508 }
 509
 510 URLFetcherCore::~URLFetcherCore() {
 511   // |request_| should be NULL. If not, it's unsafe to delete it here since we
 512   // may not be on the IO thread.
 513   DCHECK(!request_.get());
 514 }
 515
 516 void URLFetcherCore::StartOnIOThread() {
 517   DCHECK(network_task_runner_->BelongsToCurrentThread());
 518
 519   if (!response_writer_)
 520     response_writer_.reset(new URLFetcherStringWriter);
 521
 522   const int result = response_writer_->Initialize(
 523       base::Bind(&URLFetcherCore::DidInitializeWriter, this));
 524   if (result != ERR_IO_PENDING)
 525     DidInitializeWriter(result);
 526 }
 527
 528 void URLFetcherCore::StartURLRequest() {
 529   DCHECK(network_task_runner_->BelongsToCurrentThread());
 530
 531   if (was_cancelled_) {
 532     // Since StartURLRequest() is posted as a *delayed* task, it may
 533     // run after the URLFetcher was already stopped.
 534     return;
 535   }
 536
 537   if (!request_context_getter_->GetURLRequestContext()) {
 538     CancelRequestAndInformDelegate(ERR_CONTEXT_SHUT_DOWN);
 539     return;
 540   }
 541
 542   DCHECK(request_context_getter_.get());
 543   DCHECK(!request_.get());
 544
 545   g_registry.Get().AddURLFetcherCore(this);
 546   current_response_bytes_ = 0;
 547   request_context_getter_->AddObserver(this);
 548   request_ = request_context_getter_->GetURLRequestContext()->CreateRequest(
 549       original_url_, DEFAULT_PRIORITY, this);
 550   request_->set_stack_trace(stack_trace_);
 551   int flags = request_->load_flags() | load_flags_;
 552
 553   if (is_chunked_upload_)
 554     request_->EnableChunkedUpload();
 555   request_->SetLoadFlags(flags);
 556   request_->SetReferrer(referrer_);
 557   request_->set_referrer_policy(referrer_policy_);
 558   request_->set_first_party_for_cookies(first_party_for_cookies_.is_empty() ?
 559       original_url_ : first_party_for_cookies_);
 560   if (url_request_data_key_ && !url_request_create_data_callback_.is_null()) {
 561     request_->SetUserData(url_request_data_key_,
 562                           url_request_create_data_callback_.Run());
 563   }
 564
 565   switch (request_type_) {
 566     case URLFetcher::GET:
 567       break;
 568
 569     case URLFetcher::POST:
 570     case URLFetcher::PUT:
 571     case URLFetcher::PATCH: {
 572       // Upload content must be set.
 573       DCHECK(is_chunked_upload_ || upload_content_set_);
 574
 575       request_->set_method(
 576           request_type_ == URLFetcher::POST ? "POST" :
 577           request_type_ == URLFetcher::PUT ? "PUT" : "PATCH");
 578       if (!upload_content_type_.empty()) {
 579         extra_request_headers_.SetHeader(HttpRequestHeaders::kContentType,
 580                                          upload_content_type_);
 581       }
 582       if (!upload_content_.empty()) {
 583         scoped_ptr<UploadElementReader> reader(new UploadBytesElementReader(
 584             upload_content_.data(), upload_content_.size()));
 585         request_->set_upload(
 586             ElementsUploadDataStream::CreateWithReader(reader.Pass(), 0));
 587       } else if (!upload_file_path_.empty()) {
 588         scoped_ptr<UploadElementReader> reader(
 589             new UploadFileElementReader(upload_file_task_runner_.get(),
 590                                         upload_file_path_,
 591                                         upload_range_offset_,
 592                                         upload_range_length_,
 593                                         base::Time()));
 594         request_->set_upload(
 595             ElementsUploadDataStream::CreateWithReader(reader.Pass(), 0));
 596       } else if (!upload_stream_factory_.is_null()) {
 597         scoped_ptr<UploadDataStream> stream = upload_stream_factory_.Run();
 598         DCHECK(stream);
 599         request_->set_upload(stream.Pass());
 600       }
 601
 602       current_upload_bytes_ = -1;
 603       // TODO(kinaba): http://crbug.com/118103. Implement upload callback in the
 604       //  layer and avoid using timer here.
 605       upload_progress_checker_timer_.reset(
 606           new base::RepeatingTimer<URLFetcherCore>());
 607       upload_progress_checker_timer_->Start(
 608           FROM_HERE,
 609           base::TimeDelta::FromMilliseconds(kUploadProgressTimerInterval),
 610           this,
 611           &URLFetcherCore::InformDelegateUploadProgress);
 612       break;
 613     }
 614
 615     case URLFetcher::HEAD:
 616       request_->set_method("HEAD");
 617       break;
 618
 619     case URLFetcher::DELETE_REQUEST:
 620       request_->set_method("DELETE");
 621       break;
 622
 623     default:
 624       NOTREACHED();
 625   }
 626
 627   if (!extra_request_headers_.IsEmpty())
 628     request_->SetExtraRequestHeaders(extra_request_headers_);
 629
 630   request_->Start();
 631 }
 632
 633 void URLFetcherCore::DidInitializeWriter(int result) {
 634   if (result != OK) {
 635     CancelRequestAndInformDelegate(result);
 636     return;
 637   }
 638   StartURLRequestWhenAppropriate();
 639 }
 640
 641 void URLFetcherCore::StartURLRequestWhenAppropriate() {
 642   DCHECK(network_task_runner_->BelongsToCurrentThread());
 643
 644   if (was_cancelled_)
 645     return;
 646
 647   DCHECK(request_context_getter_.get());
 648
 649   // Check if the request should be delayed, and if so, post a task to start it
 650   // after the delay has expired.  Otherwise, start it now.
 651
 652   URLRequestContext* context = request_context_getter_->GetURLRequestContext();
 653   // If the context has been shut down, or there's no ThrottlerManager, just
 654   // start the request.  In the former case, StartURLRequest() will just inform
 655   // the URLFetcher::Delegate the request has been canceled.
 656   if (context && context->throttler_manager()) {
 657     if (!original_url_throttler_entry_.get()) {
 658       original_url_throttler_entry_ =
 659           context->throttler_manager()->RegisterRequestUrl(original_url_);
 660     }
 661
 662     if (original_url_throttler_entry_.get()) {
 663       int64 delay =
 664           original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
 665               GetBackoffReleaseTime());
 666       if (delay != 0) {
 667         base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 668             FROM_HERE, base::Bind(&URLFetcherCore::StartURLRequest, this),
 669             base::TimeDelta::FromMilliseconds(delay));
 670         return;
 671       }
 672     }
 673   }
 674
 675   StartURLRequest();
 676 }
 677
 678 void URLFetcherCore::CancelURLRequest(int error) {
 679   DCHECK(network_task_runner_->BelongsToCurrentThread());
 680
 681   if (request_.get()) {
 682     request_->CancelWithError(error);
 683     ReleaseRequest();
 684   }
 685
 686   // Set the error manually.
 687   // Normally, calling URLRequest::CancelWithError() results in calling
 688   // OnReadCompleted() with bytes_read = -1 via an asynchronous task posted by
 689   // URLRequestJob::NotifyDone(). But, because the request was released
 690   // immediately after being canceled, the request could not call
 691   // OnReadCompleted() which overwrites |status_| with the error status.
 692   status_ = URLRequestStatus(URLRequestStatus::CANCELED, error);
 693
 694   // Release the reference to the request context. There could be multiple
 695   // references to URLFetcher::Core at this point so it may take a while to
 696   // delete the object, but we cannot delay the destruction of the request
 697   // context.
 698   request_context_getter_ = NULL;
 699   first_party_for_cookies_ = GURL();
 700   url_request_data_key_ = NULL;
 701   url_request_create_data_callback_.Reset();
 702   was_cancelled_ = true;
 703 }
 704
 705 void URLFetcherCore::OnCompletedURLRequest(
 706     base::TimeDelta backoff_delay) {
 707   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 708
 709   // Save the status and backoff_delay so that delegates can read it.
 710   if (delegate_) {
 711     backoff_delay_ = backoff_delay;
 712     InformDelegateFetchIsComplete();
 713   }
 714 }
 715
 716 void URLFetcherCore::InformDelegateFetchIsComplete() {
 717   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 718   if (delegate_)
 719     delegate_->OnURLFetchComplete(fetcher_);
 720 }
 721
 722 void URLFetcherCore::NotifyMalformedContent() {
 723   DCHECK(network_task_runner_->BelongsToCurrentThread());
 724   if (url_throttler_entry_.get()) {
 725     int status_code = response_code_;
 726     if (status_code == URLFetcher::RESPONSE_CODE_INVALID) {
 727       // The status code will generally be known by the time clients
 728       // call the |ReceivedContentWasMalformed()| function (which ends up
 729       // calling the current function) but if it's not, we need to assume
 730       // the response was successful so that the total failure count
 731       // used to calculate exponential back-off goes up.
 732       status_code = 200;
 733     }
 734     url_throttler_entry_->ReceivedContentWasMalformed(status_code);
 735   }
 736 }
 737
 738 void URLFetcherCore::DidFinishWriting(int result) {
 739   if (result != OK) {
 740     CancelRequestAndInformDelegate(result);
 741     return;
 742   }
 743   // If the file was successfully closed, then the URL request is complete.
 744   RetryOrCompleteUrlFetch();
 745 }
 746
 747 void URLFetcherCore::RetryOrCompleteUrlFetch() {
 748   DCHECK(network_task_runner_->BelongsToCurrentThread());
 749   base::TimeDelta backoff_delay;
 750
 751   // Checks the response from server.
 752   if (response_code_ >= 500 ||
 753       status_.error() == ERR_TEMPORARILY_THROTTLED) {
 754     // When encountering a server error, we will send the request again
 755     // after backoff time.
 756     ++num_retries_on_5xx_;
 757
 758     // Note that backoff_delay may be 0 because (a) the
 759     // URLRequestThrottlerManager and related code does not
 760     // necessarily back off on the first error, (b) it only backs off
 761     // on some of the 5xx status codes, (c) not all URLRequestContexts
 762     // have a throttler manager.
 763     base::TimeTicks backoff_release_time = GetBackoffReleaseTime();
 764     backoff_delay = backoff_release_time - base::TimeTicks::Now();
 765     if (backoff_delay < base::TimeDelta())
 766       backoff_delay = base::TimeDelta();
 767
 768     if (automatically_retry_on_5xx_ &&
 769         num_retries_on_5xx_ <= max_retries_on_5xx_) {
 770       StartOnIOThread();
 771       return;
 772     }
 773   } else {
 774     backoff_delay = base::TimeDelta();
 775   }
 776
 777   // Retry if the request failed due to network changes.
 778   if (status_.error() == ERR_NETWORK_CHANGED &&
 779       num_retries_on_network_changes_ < max_retries_on_network_changes_) {
 780     ++num_retries_on_network_changes_;
 781
 782     // Retry soon, after flushing all the current tasks which may include
 783     // further network change observers.
 784     network_task_runner_->PostTask(
 785         FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
 786     return;
 787   }
 788
 789   request_context_getter_ = NULL;
 790   first_party_for_cookies_ = GURL();
 791   url_request_data_key_ = NULL;
 792   url_request_create_data_callback_.Reset();
 793   bool posted = delegate_task_runner_->PostTask(
 794       FROM_HERE,
 795       base::Bind(&URLFetcherCore::OnCompletedURLRequest, this, backoff_delay));
 796
 797   // If the delegate message loop does not exist any more, then the delegate
 798   // should be gone too.
 799   DCHECK(posted || !delegate_);
 800 }
 801
 802 void URLFetcherCore::CancelRequestAndInformDelegate(int result) {
 803   CancelURLRequest(result);
 804   delegate_task_runner_->PostTask(
 805       FROM_HERE,
 806       base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
 807 }
 808
 809 void URLFetcherCore::ReleaseRequest() {
 810   request_context_getter_->RemoveObserver(this);
 811   upload_progress_checker_timer_.reset();
 812   request_.reset();
 813   g_registry.Get().RemoveURLFetcherCore(this);
 814 }
 815
 816 base::TimeTicks URLFetcherCore::GetBackoffReleaseTime() {
 817   DCHECK(network_task_runner_->BelongsToCurrentThread());
 818
 819   if (!original_url_throttler_entry_.get())
 820     return base::TimeTicks();
 821
 822   base::TimeTicks original_url_backoff =
 823       original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
 824   base::TimeTicks destination_url_backoff;
 825   if (url_throttler_entry_.get() &&
 826       original_url_throttler_entry_.get() != url_throttler_entry_.get()) {
 827     destination_url_backoff =
 828         url_throttler_entry_->GetExponentialBackoffReleaseTime();
 829   }
 830
 831   return original_url_backoff > destination_url_backoff ?
 832       original_url_backoff : destination_url_backoff;
 833 }
 834
 835 void URLFetcherCore::CompleteAddingUploadDataChunk(
 836     const std::string& content, bool is_last_chunk) {
 837   if (was_cancelled_) {
 838     // Since CompleteAddingUploadDataChunk() is posted as a *delayed* task, it
 839     // may run after the URLFetcher was already stopped.
 840     return;
 841   }
 842   DCHECK(is_chunked_upload_);
 843   DCHECK(request_.get());
 844   DCHECK(!content.empty());
 845   request_->AppendChunkToUpload(content.data(),
 846                                 static_cast<int>(content.length()),
 847                                 is_last_chunk);
 848 }
 849
 850 int URLFetcherCore::WriteBuffer(scoped_refptr<DrainableIOBuffer> data) {
 851   while (data->BytesRemaining() > 0) {
 852     const int result = response_writer_->Write(
 853         data.get(),
 854         data->BytesRemaining(),
 855         base::Bind(&URLFetcherCore::DidWriteBuffer, this, data));
 856     if (result < 0) {
 857       if (result != ERR_IO_PENDING)
 858         DidWriteBuffer(data, result);
 859       return result;
 860     }
 861     data->DidConsume(result);
 862   }
 863   return OK;
 864 }
 865
 866 void URLFetcherCore::DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data,
 867                                     int result) {
 868   if (result < 0) {  // Handle errors.
 869     response_writer_->Finish(base::Bind(&EmptyCompletionCallback));
 870     CancelRequestAndInformDelegate(result);
 871     return;
 872   }
 873
 874   // Continue writing.
 875   data->DidConsume(result);
 876   if (WriteBuffer(data) < 0)
 877     return;
 878
 879   // Finished writing buffer_. Read some more, unless the request has been
 880   // cancelled and deleted.
 881   DCHECK_EQ(0, data->BytesRemaining());
 882   if (request_.get())
 883     ReadResponse();
 884 }
 885
 886 void URLFetcherCore::ReadResponse() {
 887   // Some servers may treat HEAD requests as GET requests. To free up the
 888   // network connection as soon as possible, signal that the request has
 889   // completed immediately, without trying to read any data back (all we care
 890   // about is the response code and headers, which we already have).
 891   int bytes_read = 0;
 892   if (request_->status().is_success() &&
 893       (request_type_ != URLFetcher::HEAD)) {
 894     if (!request_->Read(buffer_.get(), kBufferSize, &bytes_read))
 895       bytes_read = -1;  // Match OnReadCompleted() interface contract.
 896   }
 897   OnReadCompleted(request_.get(), bytes_read);
 898 }
 899
 900 void URLFetcherCore::InformDelegateUploadProgress() {
 901   DCHECK(network_task_runner_->BelongsToCurrentThread());
 902   if (request_.get()) {
 903     int64 current = request_->GetUploadProgress().position();
 904     if (current_upload_bytes_ != current) {
 905       current_upload_bytes_ = current;
 906       int64 total = -1;
 907       if (!is_chunked_upload_) {
 908         total = static_cast<int64>(request_->GetUploadProgress().size());
 909         // Total may be zero if the UploadDataStream::Init has not been called
 910         // yet. Don't send the upload progress until the size is initialized.
 911         if (!total)
 912           return;
 913       }
 914       delegate_task_runner_->PostTask(
 915           FROM_HERE,
 916           base::Bind(
 917               &URLFetcherCore::InformDelegateUploadProgressInDelegateThread,
 918               this, current, total));
 919     }
 920   }
 921 }
 922
 923 void URLFetcherCore::InformDelegateUploadProgressInDelegateThread(
 924     int64 current, int64 total) {
 925   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 926   if (delegate_)
 927     delegate_->OnURLFetchUploadProgress(fetcher_, current, total);
 928 }
 929
 930 void URLFetcherCore::InformDelegateDownloadProgress() {
 931   DCHECK(network_task_runner_->BelongsToCurrentThread());
 932
 933   // TODO(pkasting): Remove ScopedTracker below once crbug.com/455952 is fixed.
 934   tracked_objects::ScopedTracker tracking_profile2(
 935       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 936           "455952 delegate_task_runner_->PostTask()"));
 937
 938   delegate_task_runner_->PostTask(
 939       FROM_HERE,
 940       base::Bind(
 941           &URLFetcherCore::InformDelegateDownloadProgressInDelegateThread,
 942           this, current_response_bytes_, total_response_bytes_));
 943 }
 944
 945 void URLFetcherCore::InformDelegateDownloadProgressInDelegateThread(
 946     int64 current, int64 total) {
 947   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 948   if (delegate_)
 949     delegate_->OnURLFetchDownloadProgress(fetcher_, current, total);
 950 }
 951
 952 void URLFetcherCore::AssertHasNoUploadData() const {
 953   DCHECK(!upload_content_set_);
 954   DCHECK(upload_content_.empty());
 955   DCHECK(upload_file_path_.empty());
 956   DCHECK(upload_stream_factory_.is_null());
 957 }
 958
 959 }  // namespace net