net/url_request/url_request_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request_job.h"
   6
   7 #include "base/bind.h"
   8 #include "base/compiler_specific.h"
   9 #include "base/location.h"
  10 #include "base/metrics/histogram_macros.h"
  11 #include "base/power_monitor/power_monitor.h"
  12 #include "base/profiler/scoped_tracker.h"
  13 #include "base/single_thread_task_runner.h"
  14 #include "base/strings/string_number_conversions.h"
  15 #include "base/strings/string_util.h"
  16 #include "base/thread_task_runner_handle.h"
  17 #include "base/values.h"
  18 #include "net/base/auth.h"
  19 #include "net/base/host_port_pair.h"
  20 #include "net/base/io_buffer.h"
  21 #include "net/base/load_flags.h"
  22 #include "net/base/load_states.h"
  23 #include "net/base/net_errors.h"
  24 #include "net/base/network_delegate.h"
  25 #include "net/base/network_quality_estimator.h"
  26 #include "net/filter/filter.h"
  27 #include "net/http/http_response_headers.h"
  28 #include "net/url_request/url_request_context.h"
  29
  30 namespace net {
  31
  32 namespace {
  33
  34 // Callback for TYPE_URL_REQUEST_FILTERS_SET net-internals event.
  35 scoped_ptr<base::Value> FiltersSetCallback(
  36     Filter* filter,
  37     NetLogCaptureMode /* capture_mode */) {
  38   scoped_ptr<base::DictionaryValue> event_params(new base::DictionaryValue());
  39   event_params->SetString("filters", filter->OrderedFilterList());
  40   return event_params.Pass();
  41 }
  42
  43 std::string ComputeMethodForRedirect(const std::string& method,
  44                                      int http_status_code) {
  45   // For 303 redirects, all request methods except HEAD are converted to GET,
  46   // as per the latest httpbis draft.  The draft also allows POST requests to
  47   // be converted to GETs when following 301/302 redirects, for historical
  48   // reasons. Most major browsers do this and so shall we.  Both RFC 2616 and
  49   // the httpbis draft say to prompt the user to confirm the generation of new
  50   // requests, other than GET and HEAD requests, but IE omits these prompts and
  51   // so shall we.
  52   // See:
  53   // https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-17#section-7.3
  54   if ((http_status_code == 303 && method != "HEAD") ||
  55       ((http_status_code == 301 || http_status_code == 302) &&
  56        method == "POST")) {
  57     return "GET";
  58   }
  59   return method;
  60 }
  61
  62 }  // namespace
  63
  64 URLRequestJob::URLRequestJob(URLRequest* request,
  65                              NetworkDelegate* network_delegate)
  66     : request_(request),
  67       done_(false),
  68       prefilter_bytes_read_(0),
  69       postfilter_bytes_read_(0),
  70       filter_needs_more_output_space_(false),
  71       filtered_read_buffer_len_(0),
  72       has_handled_response_(false),
  73       expected_content_size_(-1),
  74       network_delegate_(network_delegate),
  75       last_notified_total_received_bytes_(0),
  76       weak_factory_(this) {
  77   base::PowerMonitor* power_monitor = base::PowerMonitor::Get();
  78   if (power_monitor)
  79     power_monitor->AddObserver(this);
  80 }
  81
  82 void URLRequestJob::SetUpload(UploadDataStream* upload) {
  83 }
  84
  85 void URLRequestJob::SetExtraRequestHeaders(const HttpRequestHeaders& headers) {
  86 }
  87
  88 void URLRequestJob::SetPriority(RequestPriority priority) {
  89 }
  90
  91 void URLRequestJob::Kill() {
  92   weak_factory_.InvalidateWeakPtrs();
  93   // Make sure the request is notified that we are done.  We assume that the
  94   // request took care of setting its error status before calling Kill.
  95   if (request_)
  96     NotifyCanceled();
  97 }
  98
  99 void URLRequestJob::DetachRequest() {
 100   request_ = NULL;
 101 }
 102
 103 // This function calls ReadData to get stream data. If a filter exists, passes
 104 // the data to the attached filter. Then returns the output from filter back to
 105 // the caller.
 106 bool URLRequestJob::Read(IOBuffer* buf, int buf_size, int *bytes_read) {
 107   bool rv = false;
 108
 109   DCHECK_LT(buf_size, 1000000);  // Sanity check.
 110   DCHECK(buf);
 111   DCHECK(bytes_read);
 112   DCHECK(filtered_read_buffer_.get() == NULL);
 113   DCHECK_EQ(0, filtered_read_buffer_len_);
 114
 115   *bytes_read = 0;
 116
 117   // Skip Filter if not present.
 118   if (!filter_.get()) {
 119     rv = ReadRawDataHelper(buf, buf_size, bytes_read);
 120   } else {
 121     // Save the caller's buffers while we do IO
 122     // in the filter's buffers.
 123     filtered_read_buffer_ = buf;
 124     filtered_read_buffer_len_ = buf_size;
 125
 126     if (ReadFilteredData(bytes_read)) {
 127       rv = true;   // We have data to return.
 128
 129       // It is fine to call DoneReading even if ReadFilteredData receives 0
 130       // bytes from the net, but we avoid making that call if we know for
 131       // sure that's the case (ReadRawDataHelper path).
 132       if (*bytes_read == 0)
 133         DoneReading();
 134     } else {
 135       rv = false;  // Error, or a new IO is pending.
 136     }
 137   }
 138
 139   if (rv && *bytes_read == 0)
 140     NotifyDone(URLRequestStatus());
 141   return rv;
 142 }
 143
 144 void URLRequestJob::StopCaching() {
 145   // Nothing to do here.
 146 }
 147
 148 bool URLRequestJob::GetFullRequestHeaders(HttpRequestHeaders* headers) const {
 149   // Most job types don't send request headers.
 150   return false;
 151 }
 152
 153 int64 URLRequestJob::GetTotalReceivedBytes() const {
 154   return 0;
 155 }
 156
 157 LoadState URLRequestJob::GetLoadState() const {
 158   return LOAD_STATE_IDLE;
 159 }
 160
 161 UploadProgress URLRequestJob::GetUploadProgress() const {
 162   return UploadProgress();
 163 }
 164
 165 bool URLRequestJob::GetCharset(std::string* charset) {
 166   return false;
 167 }
 168
 169 void URLRequestJob::GetResponseInfo(HttpResponseInfo* info) {
 170 }
 171
 172 void URLRequestJob::GetLoadTimingInfo(LoadTimingInfo* load_timing_info) const {
 173   // Only certain request types return more than just request start times.
 174 }
 175
 176 bool URLRequestJob::GetResponseCookies(std::vector<std::string>* cookies) {
 177   return false;
 178 }
 179
 180 Filter* URLRequestJob::SetupFilter() const {
 181   return NULL;
 182 }
 183
 184 bool URLRequestJob::IsRedirectResponse(GURL* location,
 185                                        int* http_status_code) {
 186   // For non-HTTP jobs, headers will be null.
 187   HttpResponseHeaders* headers = request_->response_headers();
 188   if (!headers)
 189     return false;
 190
 191   std::string value;
 192   if (!headers->IsRedirect(&value))
 193     return false;
 194
 195   *location = request_->url().Resolve(value);
 196   *http_status_code = headers->response_code();
 197   return true;
 198 }
 199
 200 bool URLRequestJob::CopyFragmentOnRedirect(const GURL& location) const {
 201   return true;
 202 }
 203
 204 bool URLRequestJob::IsSafeRedirect(const GURL& location) {
 205   return true;
 206 }
 207
 208 bool URLRequestJob::NeedsAuth() {
 209   return false;
 210 }
 211
 212 void URLRequestJob::GetAuthChallengeInfo(
 213     scoped_refptr<AuthChallengeInfo>* auth_info) {
 214   // This will only be called if NeedsAuth() returns true, in which
 215   // case the derived class should implement this!
 216   NOTREACHED();
 217 }
 218
 219 void URLRequestJob::SetAuth(const AuthCredentials& credentials) {
 220   // This will only be called if NeedsAuth() returns true, in which
 221   // case the derived class should implement this!
 222   NOTREACHED();
 223 }
 224
 225 void URLRequestJob::CancelAuth() {
 226   // This will only be called if NeedsAuth() returns true, in which
 227   // case the derived class should implement this!
 228   NOTREACHED();
 229 }
 230
 231 void URLRequestJob::ContinueWithCertificate(
 232     X509Certificate* client_cert) {
 233   // The derived class should implement this!
 234   NOTREACHED();
 235 }
 236
 237 void URLRequestJob::ContinueDespiteLastError() {
 238   // Implementations should know how to recover from errors they generate.
 239   // If this code was reached, we are trying to recover from an error that
 240   // we don't know how to recover from.
 241   NOTREACHED();
 242 }
 243
 244 void URLRequestJob::FollowDeferredRedirect() {
 245   DCHECK_NE(-1, deferred_redirect_info_.status_code);
 246
 247   // NOTE: deferred_redirect_info_ may be invalid, and attempting to follow it
 248   // will fail inside FollowRedirect.  The DCHECK above asserts that we called
 249   // OnReceivedRedirect.
 250
 251   // It is also possible that FollowRedirect will drop the last reference to
 252   // this job, so we need to reset our members before calling it.
 253
 254   RedirectInfo redirect_info = deferred_redirect_info_;
 255   deferred_redirect_info_ = RedirectInfo();
 256   FollowRedirect(redirect_info);
 257 }
 258
 259 void URLRequestJob::ResumeNetworkStart() {
 260   // This should only be called for HTTP Jobs, and implemented in the derived
 261   // class.
 262   NOTREACHED();
 263 }
 264
 265 bool URLRequestJob::GetMimeType(std::string* mime_type) const {
 266   return false;
 267 }
 268
 269 int URLRequestJob::GetResponseCode() const {
 270   return -1;
 271 }
 272
 273 HostPortPair URLRequestJob::GetSocketAddress() const {
 274   return HostPortPair();
 275 }
 276
 277 void URLRequestJob::OnSuspend() {
 278   Kill();
 279 }
 280
 281 void URLRequestJob::NotifyURLRequestDestroyed() {
 282 }
 283
 284 void URLRequestJob::GetConnectionAttempts(ConnectionAttempts* out) const {
 285   out->clear();
 286 }
 287
 288 // static
 289 GURL URLRequestJob::ComputeReferrerForRedirect(
 290     URLRequest::ReferrerPolicy policy,
 291     const std::string& referrer,
 292     const GURL& redirect_destination) {
 293   GURL original_referrer(referrer);
 294   bool secure_referrer_but_insecure_destination =
 295       original_referrer.SchemeIsCryptographic() &&
 296       !redirect_destination.SchemeIsCryptographic();
 297   bool same_origin =
 298       original_referrer.GetOrigin() == redirect_destination.GetOrigin();
 299   switch (policy) {
 300     case URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE:
 301       return secure_referrer_but_insecure_destination ? GURL()
 302                                                       : original_referrer;
 303
 304     case URLRequest::REDUCE_REFERRER_GRANULARITY_ON_TRANSITION_CROSS_ORIGIN:
 305       if (same_origin) {
 306         return original_referrer;
 307       } else if (secure_referrer_but_insecure_destination) {
 308         return GURL();
 309       } else {
 310         return original_referrer.GetOrigin();
 311       }
 312
 313     case URLRequest::ORIGIN_ONLY_ON_TRANSITION_CROSS_ORIGIN:
 314       return same_origin ? original_referrer : original_referrer.GetOrigin();
 315
 316     case URLRequest::NEVER_CLEAR_REFERRER:
 317       return original_referrer;
 318   }
 319
 320   NOTREACHED();
 321   return GURL();
 322 }
 323
 324 URLRequestJob::~URLRequestJob() {
 325   base::PowerMonitor* power_monitor = base::PowerMonitor::Get();
 326   if (power_monitor)
 327     power_monitor->RemoveObserver(this);
 328 }
 329
 330 void URLRequestJob::NotifyCertificateRequested(
 331     SSLCertRequestInfo* cert_request_info) {
 332   if (!request_)
 333     return;  // The request was destroyed, so there is no more work to do.
 334
 335   request_->NotifyCertificateRequested(cert_request_info);
 336 }
 337
 338 void URLRequestJob::NotifySSLCertificateError(const SSLInfo& ssl_info,
 339                                               bool fatal) {
 340   if (!request_)
 341     return;  // The request was destroyed, so there is no more work to do.
 342
 343   request_->NotifySSLCertificateError(ssl_info, fatal);
 344 }
 345
 346 bool URLRequestJob::CanGetCookies(const CookieList& cookie_list) const {
 347   if (!request_)
 348     return false;  // The request was destroyed, so there is no more work to do.
 349
 350   return request_->CanGetCookies(cookie_list);
 351 }
 352
 353 bool URLRequestJob::CanSetCookie(const std::string& cookie_line,
 354                                  CookieOptions* options) const {
 355   if (!request_)
 356     return false;  // The request was destroyed, so there is no more work to do.
 357
 358   return request_->CanSetCookie(cookie_line, options);
 359 }
 360
 361 bool URLRequestJob::CanEnablePrivacyMode() const {
 362   if (!request_)
 363     return false;  // The request was destroyed, so there is no more work to do.
 364
 365   return request_->CanEnablePrivacyMode();
 366 }
 367
 368 void URLRequestJob::NotifyBeforeNetworkStart(bool* defer) {
 369   if (!request_)
 370     return;
 371
 372   request_->NotifyBeforeNetworkStart(defer);
 373 }
 374
 375 void URLRequestJob::NotifyHeadersComplete() {
 376   if (!request_ || !request_->has_delegate())
 377     return;  // The request was destroyed, so there is no more work to do.
 378
 379   if (has_handled_response_)
 380     return;
 381
 382   // This should not be called on error, and the job type should have cleared
 383   // IO_PENDING state before calling this method.
 384   DCHECK(request_->status().is_success());
 385
 386   // Initialize to the current time, and let the subclass optionally override
 387   // the time stamps if it has that information.  The default request_time is
 388   // set by URLRequest before it calls our Start method.
 389   request_->response_info_.response_time = base::Time::Now();
 390   GetResponseInfo(&request_->response_info_);
 391
 392   // When notifying the delegate, the delegate can release the request
 393   // (and thus release 'this').  After calling to the delgate, we must
 394   // check the request pointer to see if it still exists, and return
 395   // immediately if it has been destroyed.  self_preservation ensures our
 396   // survival until we can get out of this method.
 397   scoped_refptr<URLRequestJob> self_preservation(this);
 398
 399   MaybeNotifyNetworkBytes();
 400
 401   if (request_)
 402     request_->OnHeadersComplete();
 403
 404   GURL new_location;
 405   int http_status_code;
 406   if (IsRedirectResponse(&new_location, &http_status_code)) {
 407     // Redirect response bodies are not read. Notify the transaction
 408     // so it does not treat being stopped as an error.
 409     DoneReadingRedirectResponse();
 410
 411     RedirectInfo redirect_info =
 412         ComputeRedirectInfo(new_location, http_status_code);
 413     bool defer_redirect = false;
 414     request_->NotifyReceivedRedirect(redirect_info, &defer_redirect);
 415
 416     // Ensure that the request wasn't detached or destroyed in
 417     // NotifyReceivedRedirect
 418     if (!request_ || !request_->has_delegate())
 419       return;
 420
 421     // If we were not cancelled, then maybe follow the redirect.
 422     if (request_->status().is_success()) {
 423       if (defer_redirect) {
 424         deferred_redirect_info_ = redirect_info;
 425       } else {
 426         FollowRedirect(redirect_info);
 427       }
 428       return;
 429     }
 430   } else if (NeedsAuth()) {
 431     scoped_refptr<AuthChallengeInfo> auth_info;
 432     GetAuthChallengeInfo(&auth_info);
 433
 434     // Need to check for a NULL auth_info because the server may have failed
 435     // to send a challenge with the 401 response.
 436     if (auth_info.get()) {
 437       request_->NotifyAuthRequired(auth_info.get());
 438       // Wait for SetAuth or CancelAuth to be called.
 439       return;
 440     }
 441   }
 442
 443   has_handled_response_ = true;
 444   if (request_->status().is_success())
 445     filter_.reset(SetupFilter());
 446
 447   if (!filter_.get()) {
 448     std::string content_length;
 449     request_->GetResponseHeaderByName("content-length", &content_length);
 450     if (!content_length.empty())
 451       base::StringToInt64(content_length, &expected_content_size_);
 452   } else {
 453     request_->net_log().AddEvent(
 454         NetLog::TYPE_URL_REQUEST_FILTERS_SET,
 455         base::Bind(&FiltersSetCallback, base::Unretained(filter_.get())));
 456   }
 457
 458   request_->NotifyResponseStarted();
 459 }
 460
 461 void URLRequestJob::NotifyReadComplete(int bytes_read) {
 462   // TODO(cbentzel): Remove ScopedTracker below once crbug.com/475755 is fixed.
 463   tracked_objects::ScopedTracker tracking_profile(
 464       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 465           "475755 URLRequestJob::NotifyReadComplete"));
 466
 467   if (!request_ || !request_->has_delegate())
 468     return;  // The request was destroyed, so there is no more work to do.
 469
 470   // TODO(darin): Bug 1004233. Re-enable this test once all of the chrome
 471   // unit_tests have been fixed to not trip this.
 472 #if 0
 473   DCHECK(!request_->status().is_io_pending());
 474 #endif
 475   // The headers should be complete before reads complete
 476   DCHECK(has_handled_response_);
 477
 478   OnRawReadComplete(bytes_read);
 479
 480   // Don't notify if we had an error.
 481   if (!request_->status().is_success())
 482     return;
 483
 484   // When notifying the delegate, the delegate can release the request
 485   // (and thus release 'this').  After calling to the delegate, we must
 486   // check the request pointer to see if it still exists, and return
 487   // immediately if it has been destroyed.  self_preservation ensures our
 488   // survival until we can get out of this method.
 489   scoped_refptr<URLRequestJob> self_preservation(this);
 490
 491   if (filter_.get()) {
 492     // Tell the filter that it has more data
 493     FilteredDataRead(bytes_read);
 494
 495     // Filter the data.
 496     int filter_bytes_read = 0;
 497     if (ReadFilteredData(&filter_bytes_read)) {
 498       if (!filter_bytes_read)
 499         DoneReading();
 500       request_->NotifyReadCompleted(filter_bytes_read);
 501     }
 502   } else {
 503     request_->NotifyReadCompleted(bytes_read);
 504   }
 505   DVLOG(1) << __FUNCTION__ << "() "
 506            << "\"" << (request_ ? request_->url().spec() : "???") << "\""
 507            << " pre bytes read = " << bytes_read
 508            << " pre total = " << prefilter_bytes_read_
 509            << " post total = " << postfilter_bytes_read_;
 510 }
 511
 512 void URLRequestJob::NotifyStartError(const URLRequestStatus &status) {
 513   DCHECK(!has_handled_response_);
 514   has_handled_response_ = true;
 515   if (request_) {
 516     // There may be relevant information in the response info even in the
 517     // error case.
 518     GetResponseInfo(&request_->response_info_);
 519
 520     request_->set_status(status);
 521     request_->NotifyResponseStarted();
 522     // We may have been deleted.
 523   }
 524 }
 525
 526 void URLRequestJob::NotifyDone(const URLRequestStatus &status) {
 527   DCHECK(!done_) << "Job sending done notification twice";
 528   if (done_)
 529     return;
 530   done_ = true;
 531
 532   // Unless there was an error, we should have at least tried to handle
 533   // the response before getting here.
 534   DCHECK(has_handled_response_ || !status.is_success());
 535
 536   // As with NotifyReadComplete, we need to take care to notice if we were
 537   // destroyed during a delegate callback.
 538   if (request_) {
 539     request_->set_is_pending(false);
 540     // With async IO, it's quite possible to have a few outstanding
 541     // requests.  We could receive a request to Cancel, followed shortly
 542     // by a successful IO.  For tracking the status(), once there is
 543     // an error, we do not change the status back to success.  To
 544     // enforce this, only set the status if the job is so far
 545     // successful.
 546     if (request_->status().is_success()) {
 547       if (status.status() == URLRequestStatus::FAILED) {
 548         request_->net_log().AddEventWithNetErrorCode(NetLog::TYPE_FAILED,
 549                                                      status.error());
 550       }
 551       request_->set_status(status);
 552     }
 553
 554     // If the request succeeded (And wasn't cancelled) and the response code was
 555     // 4xx or 5xx, record whether or not the main frame was blank.  This is
 556     // intended to be a short-lived histogram, used to figure out how important
 557     // fixing http://crbug.com/331745 is.
 558     if (request_->status().is_success()) {
 559       int response_code = GetResponseCode();
 560       if (400 <= response_code && response_code <= 599) {
 561         bool page_has_content = (postfilter_bytes_read_ != 0);
 562         if (request_->load_flags() & net::LOAD_MAIN_FRAME) {
 563           UMA_HISTOGRAM_BOOLEAN("Net.ErrorResponseHasContentMainFrame",
 564                                 page_has_content);
 565         } else {
 566           UMA_HISTOGRAM_BOOLEAN("Net.ErrorResponseHasContentNonMainFrame",
 567                                 page_has_content);
 568         }
 569       }
 570     }
 571   }
 572
 573   MaybeNotifyNetworkBytes();
 574
 575   // Complete this notification later.  This prevents us from re-entering the
 576   // delegate if we're done because of a synchronous call.
 577   base::ThreadTaskRunnerHandle::Get()->PostTask(
 578       FROM_HERE, base::Bind(&URLRequestJob::CompleteNotifyDone,
 579                             weak_factory_.GetWeakPtr()));
 580 }
 581
 582 void URLRequestJob::CompleteNotifyDone() {
 583   // Check if we should notify the delegate that we're done because of an error.
 584   if (request_ &&
 585       !request_->status().is_success() &&
 586       request_->has_delegate()) {
 587     // We report the error differently depending on whether we've called
 588     // OnResponseStarted yet.
 589     if (has_handled_response_) {
 590       // We signal the error by calling OnReadComplete with a bytes_read of -1.
 591       request_->NotifyReadCompleted(-1);
 592     } else {
 593       has_handled_response_ = true;
 594       request_->NotifyResponseStarted();
 595     }
 596   }
 597 }
 598
 599 void URLRequestJob::NotifyCanceled() {
 600   if (!done_) {
 601     NotifyDone(URLRequestStatus(URLRequestStatus::CANCELED, ERR_ABORTED));
 602   }
 603 }
 604
 605 void URLRequestJob::NotifyRestartRequired() {
 606   DCHECK(!has_handled_response_);
 607   if (GetStatus().status() != URLRequestStatus::CANCELED)
 608     request_->Restart();
 609 }
 610
 611 void URLRequestJob::OnCallToDelegate() {
 612   request_->OnCallToDelegate();
 613 }
 614
 615 void URLRequestJob::OnCallToDelegateComplete() {
 616   request_->OnCallToDelegateComplete();
 617 }
 618
 619 bool URLRequestJob::ReadRawData(IOBuffer* buf, int buf_size,
 620                                 int *bytes_read) {
 621   DCHECK(bytes_read);
 622   *bytes_read = 0;
 623   return true;
 624 }
 625
 626 void URLRequestJob::DoneReading() {
 627   // Do nothing.
 628 }
 629
 630 void URLRequestJob::DoneReadingRedirectResponse() {
 631 }
 632
 633 void URLRequestJob::FilteredDataRead(int bytes_read) {
 634   DCHECK(filter_);
 635   filter_->FlushStreamBuffer(bytes_read);
 636 }
 637
 638 bool URLRequestJob::ReadFilteredData(int* bytes_read) {
 639   DCHECK(filter_);
 640   DCHECK(filtered_read_buffer_.get());
 641   DCHECK_GT(filtered_read_buffer_len_, 0);
 642   DCHECK_LT(filtered_read_buffer_len_, 1000000);  // Sanity check.
 643   DCHECK(!raw_read_buffer_.get());
 644
 645   *bytes_read = 0;
 646   bool rv = false;
 647
 648   for (;;) {
 649     if (is_done())
 650       return true;
 651
 652     if (!filter_needs_more_output_space_ && !filter_->stream_data_len()) {
 653       // We don't have any raw data to work with, so read from the transaction.
 654       int filtered_data_read;
 655       if (ReadRawDataForFilter(&filtered_data_read)) {
 656         if (filtered_data_read > 0) {
 657           // Give data to filter.
 658           filter_->FlushStreamBuffer(filtered_data_read);
 659         } else {
 660           return true;  // EOF.
 661         }
 662       } else {
 663         return false;  // IO Pending (or error).
 664       }
 665     }
 666
 667     if ((filter_->stream_data_len() || filter_needs_more_output_space_) &&
 668         !is_done()) {
 669       // Get filtered data.
 670       int filtered_data_len = filtered_read_buffer_len_;
 671       int output_buffer_size = filtered_data_len;
 672       Filter::FilterStatus status =
 673           filter_->ReadData(filtered_read_buffer_->data(), &filtered_data_len);
 674
 675       if (filter_needs_more_output_space_ && !filtered_data_len) {
 676         // filter_needs_more_output_space_ was mistaken... there are no more
 677         // bytes and we should have at least tried to fill up the filter's input
 678         // buffer. Correct the state, and try again.
 679         filter_needs_more_output_space_ = false;
 680         continue;
 681       }
 682       filter_needs_more_output_space_ =
 683           (filtered_data_len == output_buffer_size);
 684
 685       switch (status) {
 686         case Filter::FILTER_DONE: {
 687           filter_needs_more_output_space_ = false;
 688           *bytes_read = filtered_data_len;
 689           postfilter_bytes_read_ += filtered_data_len;
 690           rv = true;
 691           break;
 692         }
 693         case Filter::FILTER_NEED_MORE_DATA: {
 694           // We have finished filtering all data currently in the buffer.
 695           // There might be some space left in the output buffer. One can
 696           // consider reading more data from the stream to feed the filter
 697           // and filling up the output buffer. This leads to more complicated
 698           // buffer management and data notification mechanisms.
 699           // We can revisit this issue if there is a real perf need.
 700           if (filtered_data_len > 0) {
 701             *bytes_read = filtered_data_len;
 702             postfilter_bytes_read_ += filtered_data_len;
 703             rv = true;
 704           } else {
 705             // Read again since we haven't received enough data yet (e.g., we
 706             // may not have a complete gzip header yet).
 707             continue;
 708           }
 709           break;
 710         }
 711         case Filter::FILTER_OK: {
 712           *bytes_read = filtered_data_len;
 713           postfilter_bytes_read_ += filtered_data_len;
 714           rv = true;
 715           break;
 716         }
 717         case Filter::FILTER_ERROR: {
 718           DVLOG(1) << __FUNCTION__ << "() "
 719                    << "\"" << (request_ ? request_->url().spec() : "???")
 720                    << "\"" << " Filter Error";
 721           filter_needs_more_output_space_ = false;
 722           NotifyDone(URLRequestStatus(URLRequestStatus::FAILED,
 723                      ERR_CONTENT_DECODING_FAILED));
 724           rv = false;
 725           break;
 726         }
 727         default: {
 728           NOTREACHED();
 729           filter_needs_more_output_space_ = false;
 730           rv = false;
 731           break;
 732         }
 733       }
 734
 735       // If logging all bytes is enabled, log the filtered bytes read.
 736       if (rv && request() && filtered_data_len > 0 &&
 737           request()->net_log().IsCapturing()) {
 738         request()->net_log().AddByteTransferEvent(
 739             NetLog::TYPE_URL_REQUEST_JOB_FILTERED_BYTES_READ, filtered_data_len,
 740             filtered_read_buffer_->data());
 741       }
 742     } else {
 743       // we are done, or there is no data left.
 744       rv = true;
 745     }
 746     break;
 747   }
 748
 749   if (rv) {
 750     // When we successfully finished a read, we no longer need to save the
 751     // caller's buffers. Release our reference.
 752     filtered_read_buffer_ = NULL;
 753     filtered_read_buffer_len_ = 0;
 754   }
 755   return rv;
 756 }
 757
 758 void URLRequestJob::DestroyFilters() {
 759   filter_.reset();
 760 }
 761
 762 const URLRequestStatus URLRequestJob::GetStatus() {
 763   if (request_)
 764     return request_->status();
 765   // If the request is gone, we must be cancelled.
 766   return URLRequestStatus(URLRequestStatus::CANCELED,
 767                           ERR_ABORTED);
 768 }
 769
 770 void URLRequestJob::SetStatus(const URLRequestStatus &status) {
 771   if (request_) {
 772     // An error status should never be replaced by a non-error status by a
 773     // URLRequestJob.  URLRequest has some retry paths, but it resets the status
 774     // itself, if needed.
 775     DCHECK(request_->status().is_io_pending() ||
 776            request_->status().is_success() ||
 777            (!status.is_success() && !status.is_io_pending()));
 778     request_->set_status(status);
 779   }
 780 }
 781
 782 void URLRequestJob::SetProxyServer(const HostPortPair& proxy_server) {
 783   request_->proxy_server_ = proxy_server;
 784 }
 785
 786 bool URLRequestJob::ReadRawDataForFilter(int* bytes_read) {
 787   bool rv = false;
 788
 789   DCHECK(bytes_read);
 790   DCHECK(filter_.get());
 791
 792   *bytes_read = 0;
 793
 794   // Get more pre-filtered data if needed.
 795   // TODO(mbelshe): is it possible that the filter needs *MORE* data
 796   //    when there is some data already in the buffer?
 797   if (!filter_->stream_data_len() && !is_done()) {
 798     IOBuffer* stream_buffer = filter_->stream_buffer();
 799     int stream_buffer_size = filter_->stream_buffer_size();
 800     rv = ReadRawDataHelper(stream_buffer, stream_buffer_size, bytes_read);
 801   }
 802   return rv;
 803 }
 804
 805 bool URLRequestJob::ReadRawDataHelper(IOBuffer* buf, int buf_size,
 806                                       int* bytes_read) {
 807   DCHECK(!request_->status().is_io_pending());
 808   DCHECK(raw_read_buffer_.get() == NULL);
 809
 810   // Keep a pointer to the read buffer, so we have access to it in the
 811   // OnRawReadComplete() callback in the event that the read completes
 812   // asynchronously.
 813   raw_read_buffer_ = buf;
 814   bool rv = ReadRawData(buf, buf_size, bytes_read);
 815
 816   if (!request_->status().is_io_pending()) {
 817     // If the read completes synchronously, either success or failure,
 818     // invoke the OnRawReadComplete callback so we can account for the
 819     // completed read.
 820     OnRawReadComplete(*bytes_read);
 821   }
 822   return rv;
 823 }
 824
 825 void URLRequestJob::FollowRedirect(const RedirectInfo& redirect_info) {
 826   int rv = request_->Redirect(redirect_info);
 827   if (rv != OK)
 828     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, rv));
 829 }
 830
 831 void URLRequestJob::OnRawReadComplete(int bytes_read) {
 832   DCHECK(raw_read_buffer_.get());
 833   // If |filter_| is non-NULL, bytes will be logged after it is applied instead.
 834   if (!filter_.get() && request() && bytes_read > 0 &&
 835       request()->net_log().IsCapturing()) {
 836     request()->net_log().AddByteTransferEvent(
 837         NetLog::TYPE_URL_REQUEST_JOB_BYTES_READ,
 838         bytes_read, raw_read_buffer_->data());
 839   }
 840
 841   if (bytes_read > 0) {
 842     RecordBytesRead(bytes_read);
 843   }
 844   raw_read_buffer_ = NULL;
 845 }
 846
 847 void URLRequestJob::RecordBytesRead(int bytes_read) {
 848   DCHECK_GT(bytes_read, 0);
 849   prefilter_bytes_read_ += bytes_read;
 850
 851   // On first read, notify NetworkQualityEstimator that response headers have
 852   // been received.
 853   // TODO(tbansal): Move this to url_request_http_job.cc. This may catch
 854   // Service Worker jobs twice.
 855   // If prefilter_bytes_read_ is equal to bytes_read, it indicates this is the
 856   // first raw read of the response body. This is used as the signal that
 857   // response headers have been received.
 858   if (request_ && request_->context()->network_quality_estimator() &&
 859       prefilter_bytes_read_ == bytes_read) {
 860     request_->context()->network_quality_estimator()->NotifyHeadersReceived(
 861         *request_);
 862   }
 863
 864   if (!filter_.get())
 865     postfilter_bytes_read_ += bytes_read;
 866   DVLOG(2) << __FUNCTION__ << "() "
 867            << "\"" << (request_ ? request_->url().spec() : "???") << "\""
 868            << " pre bytes read = " << bytes_read
 869            << " pre total = " << prefilter_bytes_read_
 870            << " post total = " << postfilter_bytes_read_;
 871   UpdatePacketReadTimes();  // Facilitate stats recording if it is active.
 872
 873   // Notify observers if any additional network usage has occurred. Note that
 874   // the number of received bytes over the network sent by this notification
 875   // could be vastly different from |bytes_read|, such as when a large chunk of
 876   // network bytes is received before multiple smaller raw reads are performed
 877   // on it.
 878   MaybeNotifyNetworkBytes();
 879 }
 880
 881 bool URLRequestJob::FilterHasData() {
 882     return filter_.get() && filter_->stream_data_len();
 883 }
 884
 885 void URLRequestJob::UpdatePacketReadTimes() {
 886 }
 887
 888 RedirectInfo URLRequestJob::ComputeRedirectInfo(const GURL& location,
 889                                                 int http_status_code) {
 890   const GURL& url = request_->url();
 891
 892   RedirectInfo redirect_info;
 893
 894   redirect_info.status_code = http_status_code;
 895
 896   // The request method may change, depending on the status code.
 897   redirect_info.new_method =
 898       ComputeMethodForRedirect(request_->method(), http_status_code);
 899
 900   // Move the reference fragment of the old location to the new one if the
 901   // new one has none. This duplicates mozilla's behavior.
 902   if (url.is_valid() && url.has_ref() && !location.has_ref() &&
 903       CopyFragmentOnRedirect(location)) {
 904     GURL::Replacements replacements;
 905     // Reference the |ref| directly out of the original URL to avoid a
 906     // malloc.
 907     replacements.SetRef(url.spec().data(),
 908                         url.parsed_for_possibly_invalid_spec().ref);
 909     redirect_info.new_url = location.ReplaceComponents(replacements);
 910   } else {
 911     redirect_info.new_url = location;
 912   }
 913
 914   // Update the first-party URL if appropriate.
 915   if (request_->first_party_url_policy() ==
 916           URLRequest::UPDATE_FIRST_PARTY_URL_ON_REDIRECT) {
 917     redirect_info.new_first_party_for_cookies = redirect_info.new_url;
 918   } else {
 919     redirect_info.new_first_party_for_cookies =
 920         request_->first_party_for_cookies();
 921   }
 922
 923   // Alter the referrer if redirecting cross-origin (especially HTTP->HTTPS).
 924   redirect_info.new_referrer =
 925       ComputeReferrerForRedirect(request_->referrer_policy(),
 926                                  request_->referrer(),
 927                                  redirect_info.new_url).spec();
 928
 929   return redirect_info;
 930 }
 931
 932 void URLRequestJob::MaybeNotifyNetworkBytes() {
 933   if (!request_ || !network_delegate_)
 934     return;
 935
 936   int64_t total_received_bytes = GetTotalReceivedBytes();
 937   DCHECK_GE(total_received_bytes, last_notified_total_received_bytes_);
 938   if (total_received_bytes > last_notified_total_received_bytes_) {
 939     network_delegate_->NotifyNetworkBytesReceived(
 940         *request_, total_received_bytes - last_notified_total_received_bytes_);
 941   }
 942   last_notified_total_received_bytes_ = total_received_bytes;
 943 }
 944
 945 }  // namespace net