mcs/class/System/System.Net/HttpWebRequest.cs

   1 //
   2 // System.Net.HttpWebRequest
   3 //
   4 // Authors:
   5 //      Lawrence Pit (loz@cable.a2000.nl)
   6 //      Gonzalo Paniagua Javier (gonzalo@ximian.com)
   7 //      Martin Baulig <mabaul@microsoft.com>
   8 //
   9 // (c) 2002 Lawrence Pit
  10 // (c) 2003 Ximian, Inc. (http://www.ximian.com)
  11 // (c) 2004 Novell, Inc. (http://www.novell.com)
  12 // Copyright (c) 2017 Xamarin Inc. (http://www.xamarin.com)
  13 //
  14
  15 //
  16 // Permission is hereby granted, free of charge, to any person obtaining
  17 // a copy of this software and associated documentation files (the
  18 // "Software"), to deal in the Software without restriction, including
  19 // without limitation the rights to use, copy, modify, merge, publish,
  20 // distribute, sublicense, and/or sell copies of the Software, and to
  21 // permit persons to whom the Software is furnished to do so, subject to
  22 // the following conditions:
  23 //
  24 // The above copyright notice and this permission notice shall be
  25 // included in all copies or substantial portions of the Software.
  26 //
  27 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  31 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  32 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  33 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  34 //
  35 #if SECURITY_DEP
  36 #if MONO_SECURITY_ALIAS
  37 extern alias MonoSecurity;
  38 using MonoSecurity::Mono.Security.Interface;
  39 #else
  40 using Mono.Security.Interface;
  41 #endif
  42 #endif
  43
  44 using System;
  45 using System.Collections;
  46 using System.Configuration;
  47 using System.Globalization;
  48 using System.IO;
  49 using System.Net;
  50 using System.Net.Cache;
  51 using System.Net.Sockets;
  52 using System.Net.Security;
  53 using System.Runtime.Remoting.Messaging;
  54 using System.Runtime.Serialization;
  55 using System.Security.Cryptography.X509Certificates;
  56 using System.Text;
  57 using System.Threading;
  58 using System.Threading.Tasks;
  59 using Mono.Net.Security;
  60
  61 namespace System.Net
  62 {
  63         [Serializable]
  64         public class HttpWebRequest : WebRequest, ISerializable
  65         {
  66                 Uri requestUri;
  67                 Uri actualUri;
  68                 bool hostChanged;
  69                 bool allowAutoRedirect = true;
  70                 bool allowBuffering = true;
  71                 bool allowReadStreamBuffering;
  72                 X509CertificateCollection certificates;
  73                 string connectionGroup;
  74                 bool haveContentLength;
  75                 long contentLength = -1;
  76                 HttpContinueDelegate continueDelegate;
  77                 CookieContainer cookieContainer;
  78                 ICredentials credentials;
  79                 bool haveResponse;
  80                 bool requestSent;
  81                 WebHeaderCollection webHeaders;
  82                 bool keepAlive = true;
  83                 int maxAutoRedirect = 50;
  84                 string mediaType = String.Empty;
  85                 string method = "GET";
  86                 string initialMethod = "GET";
  87                 bool pipelined = true;
  88                 bool preAuthenticate;
  89                 bool usedPreAuth;
  90                 Version version = HttpVersion.Version11;
  91                 bool force_version;
  92                 Version actualVersion;
  93                 IWebProxy proxy;
  94                 bool sendChunked;
  95                 ServicePoint servicePoint;
  96                 int timeout = 100000;
  97                 int continueTimeout = 350;
  98
  99                 WebRequestStream writeStream;
 100                 HttpWebResponse webResponse;
 101                 WebCompletionSource responseTask;
 102                 WebOperation currentOperation;
 103                 int aborted;
 104                 bool gotRequestStream;
 105                 int redirects;
 106                 bool expectContinue;
 107                 bool getResponseCalled;
 108                 object locker = new object ();
 109                 bool finished_reading;
 110                 DecompressionMethods auto_decomp;
 111                 int maxResponseHeadersLength;
 112                 static int defaultMaxResponseHeadersLength;
 113                 static int defaultMaximumErrorResponseLength;
 114                 static RequestCachePolicy defaultCachePolicy;
 115                 int readWriteTimeout = 300000; // ms
 116 #if SECURITY_DEP
 117                 MonoTlsProvider tlsProvider;
 118                 MonoTlsSettings tlsSettings;
 119 #endif
 120                 ServerCertValidationCallback certValidationCallback;
 121
 122                 // stores the user provided Host header as Uri. If the user specified a default port explicitly we'll lose
 123                 // that information when converting the host string to a Uri. _HostHasPort will store that information.
 124                 bool hostHasPort;
 125                 Uri hostUri;
 126
 127                 enum NtlmAuthState
 128                 {
 129                         None,
 130                         Challenge,
 131                         Response
 132                 }
 133                 AuthorizationState auth_state, proxy_auth_state;
 134
 135                 [NonSerialized]
 136                 internal Func<Stream, Task> ResendContentFactory;
 137
 138                 // Constructors
 139                 static HttpWebRequest ()
 140                 {
 141                         defaultMaxResponseHeadersLength = 64;
 142                         defaultMaximumErrorResponseLength = 64;
 143                         defaultCachePolicy = new RequestCachePolicy (RequestCacheLevel.BypassCache);
 144 #if !MOBILE
 145 #pragma warning disable 618
 146                         NetConfig config = ConfigurationSettings.GetConfig ("system.net/settings") as NetConfig;
 147 #pragma warning restore 618
 148                         if (config != null)
 149                                 defaultMaxResponseHeadersLength = config.MaxResponseHeadersLength;
 150 #endif
 151                 }
 152
 153 #if MOBILE
 154                 public
 155 #else
 156                 internal
 157 #endif
 158                 HttpWebRequest (Uri uri)
 159                 {
 160                         this.requestUri = uri;
 161                         this.actualUri = uri;
 162                         this.proxy = InternalDefaultWebProxy;
 163                         this.webHeaders = new WebHeaderCollection (WebHeaderCollectionType.HttpWebRequest);
 164                         ThrowOnError = true;
 165                         ResetAuthorization ();
 166                 }
 167
 168 #if SECURITY_DEP
 169                 internal HttpWebRequest (Uri uri, MonoTlsProvider tlsProvider, MonoTlsSettings settings = null)
 170                         : this (uri)
 171                 {
 172                         this.tlsProvider = tlsProvider;
 173                         this.tlsSettings = settings;
 174                 }
 175 #endif
 176
 177                 [Obsolete ("Serialization is obsoleted for this type.  http://go.microsoft.com/fwlink/?linkid=14202")]
 178                 protected HttpWebRequest (SerializationInfo serializationInfo, StreamingContext streamingContext)
 179                 {
 180                         // In CoreFX, attempting to serialize this class fails due to
 181                         // non-serializable fields, so this constructor never gets called.
 182                         // They're throwing PlatformNotSupportedException() in here.
 183                         throw new SerializationException ();
 184                 }
 185
 186 #if MONO_WEB_DEBUG
 187                 static int nextId;
 188                 internal readonly int ID = ++nextId;
 189 #else
 190                 internal readonly int ID;
 191 #endif
 192
 193                 void ResetAuthorization ()
 194                 {
 195                         auth_state = new AuthorizationState (this, false);
 196                         proxy_auth_state = new AuthorizationState (this, true);
 197                 }
 198
 199                 // Properties
 200
 201                 void SetSpecialHeaders (string HeaderName, string value)
 202                 {
 203                         value = WebHeaderCollection.CheckBadChars (value, true);
 204                         webHeaders.RemoveInternal (HeaderName);
 205                         if (value.Length != 0) {
 206                                 webHeaders.AddInternal (HeaderName, value);
 207                         }
 208                 }
 209
 210                 public string Accept {
 211                         get { return webHeaders["Accept"]; }
 212                         set {
 213                                 CheckRequestStarted ();
 214                                 SetSpecialHeaders ("Accept", value);
 215                         }
 216                 }
 217
 218                 public Uri Address {
 219                         get { return actualUri; }
 220                         internal set { actualUri = value; } // Used by Ftp+proxy
 221                 }
 222
 223                 public virtual bool AllowAutoRedirect {
 224                         get { return allowAutoRedirect; }
 225                         set { this.allowAutoRedirect = value; }
 226                 }
 227
 228                 public virtual bool AllowWriteStreamBuffering {
 229                         get { return allowBuffering; }
 230                         set { allowBuffering = value; }
 231                 }
 232
 233                 public virtual bool AllowReadStreamBuffering {
 234                         get { return allowReadStreamBuffering; }
 235                         set { allowReadStreamBuffering = value; }
 236                 }
 237
 238                 static Exception GetMustImplement ()
 239                 {
 240                         return new NotImplementedException ();
 241                 }
 242
 243                 public DecompressionMethods AutomaticDecompression {
 244                         get {
 245                                 return auto_decomp;
 246                         }
 247                         set {
 248                                 CheckRequestStarted ();
 249                                 auto_decomp = value;
 250                         }
 251                 }
 252
 253                 internal bool InternalAllowBuffering {
 254                         get {
 255                                 return allowBuffering && MethodWithBuffer;
 256                         }
 257                 }
 258
 259                 bool MethodWithBuffer {
 260                         get {
 261                                 return method != "HEAD" && method != "GET" &&
 262                                 method != "MKCOL" && method != "CONNECT" &&
 263                                 method != "TRACE";
 264                         }
 265                 }
 266
 267 #if SECURITY_DEP
 268                 internal MonoTlsProvider TlsProvider {
 269                         get { return tlsProvider; }
 270                 }
 271
 272                 internal MonoTlsSettings TlsSettings {
 273                         get { return tlsSettings; }
 274                 }
 275 #endif
 276
 277                 public X509CertificateCollection ClientCertificates {
 278                         get {
 279                                 if (certificates == null)
 280                                         certificates = new X509CertificateCollection ();
 281                                 return certificates;
 282                         }
 283                         set {
 284                                 if (value == null)
 285                                         throw new ArgumentNullException ("value");
 286                                 certificates = value;
 287                         }
 288                 }
 289
 290                 public string Connection {
 291                         get { return webHeaders["Connection"]; }
 292                         set {
 293                                 CheckRequestStarted ();
 294
 295                                 if (string.IsNullOrWhiteSpace (value)) {
 296                                         webHeaders.RemoveInternal ("Connection");
 297                                         return;
 298                                 }
 299
 300                                 string val = value.ToLowerInvariant ();
 301                                 if (val.Contains ("keep-alive") || val.Contains ("close"))
 302                                         throw new ArgumentException (SR.net_connarg, nameof (value));
 303
 304                                 string checkedValue = HttpValidationHelpers.CheckBadHeaderValueChars (value);
 305                                 webHeaders.CheckUpdate ("Connection", checkedValue);
 306                         }
 307                 }
 308
 309                 public override string ConnectionGroupName {
 310                         get { return connectionGroup; }
 311                         set { connectionGroup = value; }
 312                 }
 313
 314                 public override long ContentLength {
 315                         get { return contentLength; }
 316                         set {
 317                                 CheckRequestStarted ();
 318                                 if (value < 0)
 319                                         throw new ArgumentOutOfRangeException ("value", "Content-Length must be >= 0");
 320
 321                                 contentLength = value;
 322                                 haveContentLength = true;
 323                         }
 324                 }
 325
 326                 internal long InternalContentLength {
 327                         set { contentLength = value; }
 328                 }
 329
 330                 internal bool ThrowOnError { get; set; }
 331
 332                 public override string ContentType {
 333                         get { return webHeaders["Content-Type"]; }
 334                         set {
 335                                 SetSpecialHeaders ("Content-Type", value);
 336                         }
 337                 }
 338
 339                 public HttpContinueDelegate ContinueDelegate {
 340                         get { return continueDelegate; }
 341                         set { continueDelegate = value; }
 342                 }
 343
 344                 virtual
 345                 public CookieContainer CookieContainer {
 346                         get { return cookieContainer; }
 347                         set { cookieContainer = value; }
 348                 }
 349
 350                 public override ICredentials Credentials {
 351                         get { return credentials; }
 352                         set { credentials = value; }
 353                 }
 354                 public DateTime Date {
 355                         get {
 356                                 string date = webHeaders["Date"];
 357                                 if (date == null)
 358                                         return DateTime.MinValue;
 359                                 return DateTime.ParseExact (date, "r", CultureInfo.InvariantCulture).ToLocalTime ();
 360                         }
 361                         set {
 362                                 SetDateHeaderHelper ("Date", value);
 363                         }
 364                 }
 365
 366                 void SetDateHeaderHelper (string headerName, DateTime dateTime)
 367                 {
 368                         if (dateTime == DateTime.MinValue)
 369                                 SetSpecialHeaders (headerName, null); // remove header
 370                         else
 371                                 SetSpecialHeaders (headerName, HttpProtocolUtils.date2string (dateTime));
 372                 }
 373
 374 #if !MOBILE
 375                 [MonoTODO]
 376                 public static new RequestCachePolicy DefaultCachePolicy {
 377                         get { return defaultCachePolicy; }
 378                         set { defaultCachePolicy = value; }
 379                 }
 380 #endif
 381
 382                 [MonoTODO]
 383                 public static int DefaultMaximumErrorResponseLength {
 384                         get { return defaultMaximumErrorResponseLength; }
 385                         set { defaultMaximumErrorResponseLength = value; }
 386                 }
 387
 388                 public string Expect {
 389                         get { return webHeaders["Expect"]; }
 390                         set {
 391                                 CheckRequestStarted ();
 392                                 string val = value;
 393                                 if (val != null)
 394                                         val = val.Trim ().ToLower ();
 395
 396                                 if (val == null || val.Length == 0) {
 397                                         webHeaders.RemoveInternal ("Expect");
 398                                         return;
 399                                 }
 400
 401                                 if (val == "100-continue")
 402                                         throw new ArgumentException ("100-Continue cannot be set with this property.",
 403                                                                      "value");
 404
 405                                 webHeaders.CheckUpdate ("Expect", value);
 406                         }
 407                 }
 408
 409                 virtual
 410                 public bool HaveResponse {
 411                         get { return haveResponse; }
 412                 }
 413
 414                 public override WebHeaderCollection Headers {
 415                         get { return webHeaders; }
 416                         set {
 417                                 CheckRequestStarted ();
 418
 419                                 WebHeaderCollection webHeaders = value;
 420                                 WebHeaderCollection newWebHeaders = new WebHeaderCollection (WebHeaderCollectionType.HttpWebRequest);
 421
 422                                 // Copy And Validate -
 423                                 // Handle the case where their object tries to change
 424                                 //  name, value pairs after they call set, so therefore,
 425                                 //  we need to clone their headers.
 426                                 //
 427
 428                                 foreach (String headerName in webHeaders.AllKeys) {
 429                                         newWebHeaders.Add (headerName, webHeaders[headerName]);
 430                                 }
 431
 432                                 this.webHeaders = newWebHeaders;
 433                         }
 434                 }
 435
 436                 public string Host {
 437
 438                         get {
 439                                 Uri uri = hostUri ?? Address;
 440                                 return (hostUri == null || !hostHasPort) && Address.IsDefaultPort ?
 441                                     uri.Host : uri.Host + ":" + uri.Port;
 442                         }
 443                         set {
 444                                 CheckRequestStarted ();
 445
 446                                 if (value == null)
 447                                         throw new ArgumentNullException (nameof (value));
 448
 449                                 Uri uri;
 450                                 if ((value.IndexOf ('/') != -1) || (!TryGetHostUri (value, out uri)))
 451                                         throw new ArgumentException (SR.net_invalid_host, nameof (value));
 452
 453                                 hostUri = uri;
 454
 455                                 // Determine if the user provided string contains a port
 456                                 if (!hostUri.IsDefaultPort) {
 457                                         hostHasPort = true;
 458                                 } else if (value.IndexOf (':') == -1) {
 459                                         hostHasPort = false;
 460                                 } else {
 461                                         int endOfIPv6Address = value.IndexOf (']');
 462                                         hostHasPort = endOfIPv6Address == -1 || value.LastIndexOf (':') > endOfIPv6Address;
 463                                 }
 464                         }
 465                 }
 466
 467                 bool TryGetHostUri (string hostName, out Uri hostUri)
 468                 {
 469                         string s = Address.Scheme + "://" + hostName + Address.PathAndQuery;
 470                         return Uri.TryCreate (s, UriKind.Absolute, out hostUri);
 471                 }
 472
 473                 public DateTime IfModifiedSince {
 474                         get {
 475                                 string str = webHeaders["If-Modified-Since"];
 476                                 if (str == null)
 477                                         return DateTime.Now;
 478                                 try {
 479                                         return MonoHttpDate.Parse (str);
 480                                 } catch (Exception) {
 481                                         return DateTime.Now;
 482                                 }
 483                         }
 484                         set {
 485                                 CheckRequestStarted ();
 486                                 // rfc-1123 pattern
 487                                 webHeaders.SetInternal ("If-Modified-Since",
 488                                         value.ToUniversalTime ().ToString ("r", null));
 489                                 // TODO: check last param when using different locale
 490                         }
 491                 }
 492
 493                 public bool KeepAlive {
 494                         get {
 495                                 return keepAlive;
 496                         }
 497                         set {
 498                                 keepAlive = value;
 499                         }
 500                 }
 501
 502                 public int MaximumAutomaticRedirections {
 503                         get { return maxAutoRedirect; }
 504                         set {
 505                                 if (value <= 0)
 506                                         throw new ArgumentException ("Must be > 0", "value");
 507
 508                                 maxAutoRedirect = value;
 509                         }
 510                 }
 511
 512                 [MonoTODO ("Use this")]
 513                 public int MaximumResponseHeadersLength {
 514                         get { return maxResponseHeadersLength; }
 515                         set {
 516                                 CheckRequestStarted ();
 517                                 if (value < 0 && value != System.Threading.Timeout.Infinite)
 518                                         throw new ArgumentOutOfRangeException (nameof (value), SR.net_toosmall);
 519
 520                                 maxResponseHeadersLength = value;
 521                         }
 522                 }
 523
 524                 [MonoTODO ("Use this")]
 525                 public static int DefaultMaximumResponseHeadersLength {
 526                         get { return defaultMaxResponseHeadersLength; }
 527                         set { defaultMaxResponseHeadersLength = value; }
 528                 }
 529
 530                 public int ReadWriteTimeout {
 531                         get { return readWriteTimeout; }
 532                         set {
 533                                 CheckRequestStarted ();
 534
 535                                 if (value <= 0 && value != System.Threading.Timeout.Infinite)
 536                                         throw new ArgumentOutOfRangeException (nameof (value), SR.net_io_timeout_use_gt_zero);
 537
 538                                 readWriteTimeout = value;
 539                         }
 540                 }
 541
 542                 [MonoTODO]
 543                 public int ContinueTimeout {
 544                         get {
 545                                 return continueTimeout;
 546                         }
 547                         set {
 548                                 CheckRequestStarted ();
 549                                 if ((value < 0) && (value != System.Threading.Timeout.Infinite))
 550                                         throw new ArgumentOutOfRangeException (nameof (value), SR.net_io_timeout_use_ge_zero);
 551                                 continueTimeout = value;
 552                         }
 553                 }
 554
 555                 public string MediaType {
 556                         get { return mediaType; }
 557                         set {
 558                                 mediaType = value;
 559                         }
 560                 }
 561
 562                 public override string Method {
 563                         get { return this.method; }
 564                         set {
 565                                 if (string.IsNullOrEmpty (value))
 566                                         throw new ArgumentException (SR.net_badmethod, nameof (value));
 567                                 if (HttpValidationHelpers.IsInvalidMethodOrHeaderString (value))
 568                                         throw new ArgumentException (SR.net_badmethod, nameof (value));
 569
 570                                 method = value.ToUpperInvariant ();
 571                                 if (method != "HEAD" && method != "GET" && method != "POST" && method != "PUT" &&
 572                                         method != "DELETE" && method != "CONNECT" && method != "TRACE" &&
 573                                         method != "MKCOL") {
 574                                         method = value;
 575                                 }
 576                         }
 577                 }
 578
 579                 public bool Pipelined {
 580                         get { return pipelined; }
 581                         set { pipelined = value; }
 582                 }
 583
 584                 public override bool PreAuthenticate {
 585                         get { return preAuthenticate; }
 586                         set { preAuthenticate = value; }
 587                 }
 588
 589                 public Version ProtocolVersion {
 590                         get { return version; }
 591                         set {
 592                                 if (value != HttpVersion.Version10 && value != HttpVersion.Version11)
 593                                         throw new ArgumentException (SR.net_wrongversion, nameof (value));
 594
 595                                 force_version = true;
 596                                 version = value;
 597                         }
 598                 }
 599
 600                 public override IWebProxy Proxy {
 601                         get { return proxy; }
 602                         set {
 603                                 CheckRequestStarted ();
 604                                 proxy = value;
 605                                 servicePoint = null; // we may need a new one
 606                                 GetServicePoint ();
 607                         }
 608                 }
 609
 610                 public string Referer {
 611                         get { return webHeaders["Referer"]; }
 612                         set {
 613                                 CheckRequestStarted ();
 614                                 if (value == null || value.Trim ().Length == 0) {
 615                                         webHeaders.RemoveInternal ("Referer");
 616                                         return;
 617                                 }
 618                                 webHeaders.SetInternal ("Referer", value);
 619                         }
 620                 }
 621
 622                 public override Uri RequestUri {
 623                         get { return requestUri; }
 624                 }
 625
 626                 public bool SendChunked {
 627                         get { return sendChunked; }
 628                         set {
 629                                 CheckRequestStarted ();
 630                                 sendChunked = value;
 631                         }
 632                 }
 633
 634                 public ServicePoint ServicePoint {
 635                         get { return GetServicePoint (); }
 636                 }
 637
 638                 internal ServicePoint ServicePointNoLock {
 639                         get { return servicePoint; }
 640                 }
 641                 public virtual bool SupportsCookieContainer {
 642                         get {
 643                                 // The managed implementation supports the cookie container
 644                                 // it is only Silverlight that returns false here
 645                                 return true;
 646                         }
 647                 }
 648                 public override int Timeout {
 649                         get { return timeout; }
 650                         set {
 651                                 if (value < -1)
 652                                         throw new ArgumentOutOfRangeException ("value");
 653
 654                                 timeout = value;
 655                         }
 656                 }
 657
 658                 public string TransferEncoding {
 659                         get { return webHeaders["Transfer-Encoding"]; }
 660                         set {
 661                                 CheckRequestStarted ();
 662
 663                                 if (string.IsNullOrWhiteSpace (value)) {
 664                                         webHeaders.RemoveInternal ("Transfer-Encoding");
 665                                         return;
 666                                 }
 667
 668                                 string val = value.ToLower ();
 669                                 //
 670                                 // prevent them from adding chunked, or from adding an Encoding without
 671                                 // turning on chunked, the reason is due to the HTTP Spec which prevents
 672                                 // additional encoding types from being used without chunked
 673                                 //
 674                                 if (val.Contains ("chunked"))
 675                                         throw new ArgumentException (SR.net_nochunked, nameof (value));
 676                                 else if (!SendChunked)
 677                                         throw new InvalidOperationException (SR.net_needchunked);
 678
 679                                 string checkedValue = HttpValidationHelpers.CheckBadHeaderValueChars (value);
 680                                 webHeaders.CheckUpdate ("Transfer-Encoding", checkedValue);
 681                         }
 682                 }
 683
 684                 public override bool UseDefaultCredentials {
 685                         get { return CredentialCache.DefaultCredentials == Credentials; }
 686                         set { Credentials = value ? CredentialCache.DefaultCredentials : null; }
 687                 }
 688
 689                 public string UserAgent {
 690                         get { return webHeaders["User-Agent"]; }
 691                         set { webHeaders.SetInternal ("User-Agent", value); }
 692                 }
 693
 694                 bool unsafe_auth_blah;
 695                 public bool UnsafeAuthenticatedConnectionSharing {
 696                         get { return unsafe_auth_blah; }
 697                         set { unsafe_auth_blah = value; }
 698                 }
 699
 700                 internal bool GotRequestStream {
 701                         get { return gotRequestStream; }
 702                 }
 703
 704                 internal bool ExpectContinue {
 705                         get { return expectContinue; }
 706                         set { expectContinue = value; }
 707                 }
 708
 709                 internal Uri AuthUri {
 710                         get { return actualUri; }
 711                 }
 712
 713                 internal bool ProxyQuery {
 714                         get { return servicePoint.UsesProxy && !servicePoint.UseConnect; }
 715                 }
 716
 717                 internal ServerCertValidationCallback ServerCertValidationCallback {
 718                         get { return certValidationCallback; }
 719                 }
 720
 721                 public RemoteCertificateValidationCallback ServerCertificateValidationCallback {
 722                         get {
 723                                 if (certValidationCallback == null)
 724                                         return null;
 725                                 return certValidationCallback.ValidationCallback;
 726                         }
 727                         set {
 728                                 if (value == null)
 729                                         certValidationCallback = null;
 730                                 else
 731                                         certValidationCallback = new ServerCertValidationCallback (value);
 732                         }
 733                 }
 734
 735                 // Methods
 736
 737                 internal ServicePoint GetServicePoint ()
 738                 {
 739                         lock (locker) {
 740                                 if (hostChanged || servicePoint == null) {
 741                                         servicePoint = ServicePointManager.FindServicePoint (actualUri, proxy);
 742                                         hostChanged = false;
 743                                 }
 744                         }
 745
 746                         return servicePoint;
 747                 }
 748
 749                 public void AddRange (int range)
 750                 {
 751                         AddRange ("bytes", (long)range);
 752                 }
 753
 754                 public void AddRange (int from, int to)
 755                 {
 756                         AddRange ("bytes", (long)from, (long)to);
 757                 }
 758
 759                 public void AddRange (string rangeSpecifier, int range)
 760                 {
 761                         AddRange (rangeSpecifier, (long)range);
 762                 }
 763
 764                 public void AddRange (string rangeSpecifier, int from, int to)
 765                 {
 766                         AddRange (rangeSpecifier, (long)from, (long)to);
 767                 }
 768                 public
 769                 void AddRange (long range)
 770                 {
 771                         AddRange ("bytes", (long)range);
 772                 }
 773
 774                 public
 775                 void AddRange (long from, long to)
 776                 {
 777                         AddRange ("bytes", from, to);
 778                 }
 779
 780                 public
 781                 void AddRange (string rangeSpecifier, long range)
 782                 {
 783                         if (rangeSpecifier == null)
 784                                 throw new ArgumentNullException ("rangeSpecifier");
 785                         if (!WebHeaderCollection.IsValidToken (rangeSpecifier))
 786                                 throw new ArgumentException ("Invalid range specifier", "rangeSpecifier");
 787
 788                         string r = webHeaders["Range"];
 789                         if (r == null)
 790                                 r = rangeSpecifier + "=";
 791                         else {
 792                                 string old_specifier = r.Substring (0, r.IndexOf ('='));
 793                                 if (String.Compare (old_specifier, rangeSpecifier, StringComparison.OrdinalIgnoreCase) != 0)
 794                                         throw new InvalidOperationException ("A different range specifier is already in use");
 795                                 r += ",";
 796                         }
 797
 798                         string n = range.ToString (CultureInfo.InvariantCulture);
 799                         if (range < 0)
 800                                 r = r + "0" + n;
 801                         else
 802                                 r = r + n + "-";
 803                         webHeaders.ChangeInternal ("Range", r);
 804                 }
 805
 806                 public
 807                 void AddRange (string rangeSpecifier, long from, long to)
 808                 {
 809                         if (rangeSpecifier == null)
 810                                 throw new ArgumentNullException ("rangeSpecifier");
 811                         if (!WebHeaderCollection.IsValidToken (rangeSpecifier))
 812                                 throw new ArgumentException ("Invalid range specifier", "rangeSpecifier");
 813                         if (from > to || from < 0)
 814                                 throw new ArgumentOutOfRangeException ("from");
 815                         if (to < 0)
 816                                 throw new ArgumentOutOfRangeException ("to");
 817
 818                         string r = webHeaders["Range"];
 819                         if (r == null)
 820                                 r = rangeSpecifier + "=";
 821                         else
 822                                 r += ",";
 823
 824                         r = String.Format ("{0}{1}-{2}", r, from, to);
 825                         webHeaders.ChangeInternal ("Range", r);
 826                 }
 827
 828                 WebOperation SendRequest (bool redirecting, BufferOffsetSize writeBuffer, CancellationToken cancellationToken)
 829                 {
 830                         lock (locker) {
 831                                 WebConnection.Debug ($"HWR SEND REQUEST: Req={ID} requestSent={requestSent} redirecting={redirecting}");
 832
 833                                 WebOperation operation;
 834                                 if (!redirecting) {
 835                                         if (requestSent) {
 836                                                 operation = currentOperation;
 837                                                 if (operation == null)
 838                                                         throw new InvalidOperationException ("Should never happen!");
 839                                                 return operation;
 840                                         }
 841                                 }
 842
 843                                 operation = new WebOperation (this, writeBuffer, false, cancellationToken);
 844                                 if (Interlocked.CompareExchange (ref currentOperation, operation, null) != null)
 845                                         throw new InvalidOperationException ("Invalid nested call.");
 846
 847                                 requestSent = true;
 848                                 if (!redirecting)
 849                                         redirects = 0;
 850                                 servicePoint = GetServicePoint ();
 851                                 servicePoint.SendRequest (operation, connectionGroup);
 852                                 return operation;
 853                         }
 854                 }
 855
 856                 Task<Stream> MyGetRequestStreamAsync (CancellationToken cancellationToken)
 857                 {
 858                         if (Aborted)
 859                                 throw CreateRequestAbortedException ();
 860
 861                         bool send = !(method == "GET" || method == "CONNECT" || method == "HEAD" ||
 862                                         method == "TRACE");
 863                         if (method == null || !send)
 864                                 throw new ProtocolViolationException ("Cannot send data when method is: " + method);
 865
 866                         if (contentLength == -1 && !sendChunked && !allowBuffering && KeepAlive)
 867                                 throw new ProtocolViolationException ("Content-Length not set");
 868
 869                         string transferEncoding = TransferEncoding;
 870                         if (!sendChunked && transferEncoding != null && transferEncoding.Trim () != "")
 871                                 throw new ProtocolViolationException ("SendChunked should be true.");
 872
 873                         WebOperation operation;
 874                         lock (locker) {
 875                                 if (getResponseCalled)
 876                                         throw new InvalidOperationException ("The operation cannot be performed once the request has been submitted.");
 877
 878                                 operation = currentOperation;
 879                                 if (operation == null) {
 880                                         initialMethod = method;
 881
 882                                         gotRequestStream = true;
 883                                         operation = SendRequest (false, null, cancellationToken);
 884                                 }
 885                         }
 886
 887                         return operation.GetRequestStream ();
 888                 }
 889
 890                 public override IAsyncResult BeginGetRequestStream (AsyncCallback callback, object state)
 891                 {
 892                         return TaskToApm.Begin (RunWithTimeout (MyGetRequestStreamAsync), callback, state);
 893                 }
 894
 895                 public override Stream EndGetRequestStream (IAsyncResult asyncResult)
 896                 {
 897                         if (asyncResult == null)
 898                                 throw new ArgumentNullException ("asyncResult");
 899
 900                         try {
 901                                 return TaskToApm.End<Stream> (asyncResult);
 902                         } catch (Exception e) {
 903                                 throw FlattenException (e);
 904                         }
 905                 }
 906
 907                 public override Stream GetRequestStream ()
 908                 {
 909                         try {
 910                                 return GetRequestStreamAsync ().Result;
 911                         } catch (Exception e) {
 912                                 throw FlattenException (e);
 913                         }
 914                 }
 915
 916                 [MonoTODO]
 917                 public Stream GetRequestStream (out TransportContext context)
 918                 {
 919                         throw new NotImplementedException ();
 920                 }
 921
 922                 public override Task<Stream> GetRequestStreamAsync ()
 923                 {
 924                         return RunWithTimeout (MyGetRequestStreamAsync);
 925                 }
 926
 927                 internal static Task<T> RunWithTimeout<T> (
 928                         Func<CancellationToken, Task<T>> func, int timeout, Action abort)
 929                 {
 930                         // Call `func` here to propagate any potential exception that it
 931                         // might throw to our caller rather than returning a faulted task.
 932                         var cts = new CancellationTokenSource ();
 933                         var workerTask = func (cts.Token);
 934                         return RunWithTimeoutWorker (workerTask, timeout, abort, cts);
 935                 }
 936
 937                 static async Task<T> RunWithTimeoutWorker<T> (
 938                         Task<T> workerTask, int timeout, Action abort,
 939                         CancellationTokenSource cts)
 940                 {
 941                         try {
 942                                 if (await ServicePointScheduler.WaitAsync (workerTask, timeout).ConfigureAwait (false))
 943                                         return workerTask.Result;
 944                                 try {
 945                                         cts.Cancel ();
 946                                         abort ();
 947                                 } catch {
 948                                         // Ignore; we report the timeout.
 949                                 }
 950                                 throw new WebException (SR.net_timeout, WebExceptionStatus.Timeout);
 951                         } catch (Exception ex) {
 952                                 throw FlattenException (ex);
 953                         } finally {
 954                                 cts.Dispose ();
 955                         }
 956                 }
 957
 958                 Task<T> RunWithTimeout<T> (Func<CancellationToken, Task<T>> func)
 959                 {
 960                         return RunWithTimeout (func, timeout, Abort);
 961                 }
 962
 963                 async Task<HttpWebResponse> MyGetResponseAsync (CancellationToken cancellationToken)
 964                 {
 965                         if (Aborted)
 966                                 throw CreateRequestAbortedException ();
 967
 968                         if (method == null)
 969                                 throw new ProtocolViolationException ("Method is null.");
 970
 971                         string transferEncoding = TransferEncoding;
 972                         if (!sendChunked && transferEncoding != null && transferEncoding.Trim () != "")
 973                                 throw new ProtocolViolationException ("SendChunked should be true.");
 974
 975                         var completion = new WebCompletionSource ();
 976                         WebOperation operation;
 977                         lock (locker) {
 978                                 getResponseCalled = true;
 979                                 var oldCompletion = Interlocked.CompareExchange (ref responseTask, completion, null);
 980                                 WebConnection.Debug ($"HWR GET RESPONSE: Req={ID} {oldCompletion != null}");
 981                                 if (oldCompletion != null) {
 982                                         oldCompletion.ThrowOnError ();
 983                                         if (haveResponse && oldCompletion.Task.IsCompleted)
 984                                                 return webResponse;
 985                                         throw new InvalidOperationException ("Cannot re-call start of asynchronous " +
 986                                                                 "method while a previous call is still in progress.");
 987                                 }
 988
 989                                 operation = currentOperation;
 990                                 if (currentOperation != null)
 991                                         writeStream = currentOperation.WriteStream;
 992
 993                                 initialMethod = method;
 994
 995                                 operation = SendRequest (false, null, cancellationToken);
 996                         }
 997
 998                         while (true) {
 999                                 WebException throwMe = null;
1000                                 HttpWebResponse response = null;
1001                                 WebResponseStream stream = null;
1002                                 bool redirect = false;
1003                                 bool mustReadAll = false;
1004                                 WebOperation ntlm = null;
1005                                 BufferOffsetSize writeBuffer = null;
1006
1007                                 try {
1008                                         cancellationToken.ThrowIfCancellationRequested ();
1009
1010                                         WebConnection.Debug ($"HWR GET RESPONSE LOOP: Req={ID} {auth_state.NtlmAuthState}");
1011
1012                                         writeStream = await operation.GetRequestStreamInternal ();
1013                                         await writeStream.WriteRequestAsync (cancellationToken).ConfigureAwait (false);
1014
1015                                         stream = await operation.GetResponseStream ();
1016
1017                                         WebConnection.Debug ($"HWR RESPONSE LOOP #0: Req={ID} - {stream?.Headers != null}");
1018
1019                                         (response, redirect, mustReadAll, writeBuffer, ntlm) = await GetResponseFromData (
1020                                                 stream, cancellationToken).ConfigureAwait (false);
1021                                 } catch (Exception e) {
1022                                         throwMe = GetWebException (e);
1023                                 }
1024
1025                                 WebConnection.Debug ($"HWR GET RESPONSE LOOP #1: Req={ID} - redirect={redirect} mustReadAll={mustReadAll} writeBuffer={writeBuffer != null} ntlm={ntlm != null} - {throwMe != null}");
1026
1027                                 lock (locker) {
1028                                         if (throwMe != null) {
1029                                                 WebConnection.Debug ($"HWR GET RESPONSE LOOP #1 EX: Req={ID} {throwMe.Status} {throwMe.InnerException?.GetType ()}");
1030                                                 haveResponse = true;
1031                                                 completion.TrySetException (throwMe);
1032                                                 throw throwMe;
1033                                         }
1034
1035                                         if (!redirect) {
1036                                                 haveResponse = true;
1037                                                 webResponse = response;
1038                                                 completion.TrySetCompleted ();
1039                                                 return response;
1040                                         }
1041
1042                                         finished_reading = false;
1043                                         haveResponse = false;
1044                                         webResponse = null;
1045                                         currentOperation = ntlm;
1046                                         WebConnection.Debug ($"HWR GET RESPONSE LOOP #2: Req={ID} {mustReadAll} {ntlm}");
1047                                 }
1048
1049                                 try {
1050                                         if (mustReadAll)
1051                                                 await stream.ReadAllAsync (redirect || ntlm != null, cancellationToken).ConfigureAwait (false);
1052                                         operation.Finish (true);
1053                                         response.Close ();
1054                                 } catch (Exception e) {
1055                                         throwMe = GetWebException (e);
1056                                 }
1057
1058                                 lock (locker) {
1059                                         WebConnection.Debug ($"HWR GET RESPONSE LOOP #3: Req={ID} {writeBuffer != null} {ntlm != null}");
1060                                         if (throwMe != null) {
1061                                                 WebConnection.Debug ($"HWR GET RESPONSE LOOP #3 EX: Req={ID} {throwMe.Status} {throwMe.InnerException?.GetType ()}");
1062                                                 haveResponse = true;
1063                                                 stream?.Close ();
1064                                                 completion.TrySetException (throwMe);
1065                                                 throw throwMe;
1066                                         }
1067
1068                                         if (ntlm == null) {
1069                                                 operation = SendRequest (true, writeBuffer, cancellationToken);
1070                                         } else {
1071                                                 operation = ntlm;
1072                                         }
1073                                 }
1074                         }
1075                 }
1076
1077                 async Task<(HttpWebResponse response, bool redirect, bool mustReadAll, BufferOffsetSize writeBuffer, WebOperation ntlm)>
1078                         GetResponseFromData (WebResponseStream stream, CancellationToken cancellationToken)
1079                 {
1080                         /*
1081                          * WebConnection has either called SetResponseData() or SetResponseError().
1082                         */
1083
1084                         var response = new HttpWebResponse (actualUri, method, stream, cookieContainer);
1085
1086                         WebException throwMe = null;
1087                         bool redirect = false;
1088                         bool mustReadAll = false;
1089                         WebOperation ntlm = null;
1090                         Task<BufferOffsetSize> rewriteHandler = null;
1091                         BufferOffsetSize writeBuffer = null;
1092
1093                         lock (locker) {
1094                                 (redirect, mustReadAll, rewriteHandler, throwMe) = CheckFinalStatus (response);
1095                         }
1096
1097                         if (throwMe != null) {
1098                                 if (mustReadAll)
1099                                         await stream.ReadAllAsync (false, cancellationToken).ConfigureAwait (false);
1100                                 throw throwMe;
1101                         }
1102
1103                         if (rewriteHandler != null) {
1104                                 writeBuffer = await rewriteHandler.ConfigureAwait (false);
1105                         }
1106
1107                         lock (locker) {
1108                                 bool isProxy = ProxyQuery && proxy != null && !proxy.IsBypassed (actualUri);
1109
1110                                 if (!redirect) {
1111                                         if ((isProxy ? proxy_auth_state : auth_state).IsNtlmAuthenticated && (int)response.StatusCode < 400) {
1112                                                 stream.Connection.NtlmAuthenticated = true;
1113                                         }
1114
1115                                         // clear internal buffer so that it does not
1116                                         // hold possible big buffer (bug #397627)
1117                                         if (writeStream != null)
1118                                                 writeStream.KillBuffer ();
1119
1120                                         return (response, false, false, writeBuffer, null);
1121                                 }
1122
1123                                 if (sendChunked) {
1124                                         sendChunked = false;
1125                                         webHeaders.RemoveInternal ("Transfer-Encoding");
1126                                 }
1127
1128                                 bool isChallenge;
1129                                 (ntlm, isChallenge) = HandleNtlmAuth (stream, response, writeBuffer, cancellationToken);
1130                                 WebConnection.Debug ($"HWR REDIRECT: {ntlm} {isChallenge} {mustReadAll}");
1131                         }
1132
1133                         return (response, true, mustReadAll, writeBuffer, ntlm);
1134                 }
1135
1136                 internal static Exception FlattenException (Exception e)
1137                 {
1138                         if (e is AggregateException ae) {
1139                                 ae = ae.Flatten ();
1140                                 if (ae.InnerExceptions.Count == 1)
1141                                         return ae.InnerException;
1142                         }
1143
1144                         return e;
1145                 }
1146
1147                 WebException GetWebException (Exception e)
1148                 {
1149                         e = FlattenException (e);
1150                         if (e is WebException wexc) {
1151                                 if (!Aborted || wexc.Status == WebExceptionStatus.RequestCanceled || wexc.Status == WebExceptionStatus.Timeout)
1152                                         return wexc;
1153                         }
1154                         if (Aborted || e is OperationCanceledException || e is ObjectDisposedException)
1155                                 return CreateRequestAbortedException ();
1156                         return new WebException (e.Message, e, WebExceptionStatus.UnknownError, null);
1157                 }
1158
1159                 internal static WebException CreateRequestAbortedException ()
1160                 {
1161                         return new WebException (SR.Format (SR.net_reqaborted, WebExceptionStatus.RequestCanceled), WebExceptionStatus.RequestCanceled);
1162                 }
1163
1164                 public override IAsyncResult BeginGetResponse (AsyncCallback callback, object state)
1165                 {
1166                         if (Aborted)
1167                                 throw CreateRequestAbortedException ();
1168
1169                         return TaskToApm.Begin (RunWithTimeout (MyGetResponseAsync), callback, state);
1170                 }
1171
1172                 public override WebResponse EndGetResponse (IAsyncResult asyncResult)
1173                 {
1174                         if (asyncResult == null)
1175                                 throw new ArgumentNullException (nameof (asyncResult));
1176
1177                         try {
1178                                 return TaskToApm.End<HttpWebResponse> (asyncResult);
1179                         } catch (Exception e) {
1180                                 throw FlattenException (e);
1181                         }
1182                 }
1183
1184                 public Stream EndGetRequestStream (IAsyncResult asyncResult, out TransportContext context)
1185                 {
1186                         if (asyncResult == null)
1187                                 throw new ArgumentNullException (nameof (asyncResult));
1188
1189                         context = null;
1190                         return EndGetRequestStream (asyncResult);
1191                 }
1192
1193                 public override WebResponse GetResponse ()
1194                 {
1195                         try {
1196                                 return GetResponseAsync ().Result;
1197                         } catch (Exception e) {
1198                                 throw FlattenException (e);
1199                         }
1200                 }
1201
1202                 internal bool FinishedReading {
1203                         get { return finished_reading; }
1204                         set { finished_reading = value; }
1205                 }
1206
1207                 internal bool Aborted {
1208                         get { return Interlocked.CompareExchange (ref aborted, 0, 0) == 1; }
1209                 }
1210
1211                 public override void Abort ()
1212                 {
1213                         if (Interlocked.CompareExchange (ref aborted, 1, 0) == 1)
1214                                 return;
1215
1216                         WebConnection.Debug ($"HWR ABORT: Req={ID}");
1217
1218                         haveResponse = true;
1219                         var operation = currentOperation;
1220                         if (operation != null)
1221                                 operation.Abort ();
1222
1223                         responseTask?.TrySetCanceled ();
1224
1225                         if (webResponse != null) {
1226                                 try {
1227                                         webResponse.Close ();
1228                                         webResponse = null;
1229                                 } catch { }
1230                         }
1231                 }
1232
1233                 void ISerializable.GetObjectData (SerializationInfo serializationInfo,
1234                                                   StreamingContext streamingContext)
1235                 {
1236                         throw new SerializationException ();
1237                 }
1238
1239                 protected override void GetObjectData (SerializationInfo serializationInfo,
1240                         StreamingContext streamingContext)
1241                 {
1242                         throw new SerializationException ();
1243                 }
1244
1245                 void CheckRequestStarted ()
1246                 {
1247                         if (requestSent)
1248                                 throw new InvalidOperationException ("request started");
1249                 }
1250
1251                 internal void DoContinueDelegate (int statusCode, WebHeaderCollection headers)
1252                 {
1253                         if (continueDelegate != null)
1254                                 continueDelegate (statusCode, headers);
1255                 }
1256
1257                 void RewriteRedirectToGet ()
1258                 {
1259                         method = "GET";
1260                         webHeaders.RemoveInternal ("Transfer-Encoding");
1261                         sendChunked = false;
1262                 }
1263
1264                 bool Redirect (HttpStatusCode code, WebResponse response)
1265                 {
1266                         redirects++;
1267                         Exception e = null;
1268                         string uriString = null;
1269                         switch (code) {
1270                         case HttpStatusCode.Ambiguous: // 300
1271                                 e = new WebException ("Ambiguous redirect.");
1272                                 break;
1273                         case HttpStatusCode.MovedPermanently: // 301
1274                         case HttpStatusCode.Redirect: // 302
1275                                 if (method == "POST")
1276                                         RewriteRedirectToGet ();
1277                                 break;
1278                         case HttpStatusCode.TemporaryRedirect: // 307
1279                                 break;
1280                         case HttpStatusCode.SeeOther: //303
1281                                 RewriteRedirectToGet ();
1282                                 break;
1283                         case HttpStatusCode.NotModified: // 304
1284                                 return false;
1285                         case HttpStatusCode.UseProxy: // 305
1286                                 e = new NotImplementedException ("Proxy support not available.");
1287                                 break;
1288                         case HttpStatusCode.Unused: // 306
1289                         default:
1290                                 e = new ProtocolViolationException ("Invalid status code: " + (int)code);
1291                                 break;
1292                         }
1293
1294                         if (method != "GET" && !InternalAllowBuffering && ResendContentFactory == null &&
1295                             (writeStream.WriteBufferLength > 0 || contentLength > 0))
1296                                 e = new WebException ("The request requires buffering data to succeed.", null, WebExceptionStatus.ProtocolError, response);
1297
1298                         if (e != null)
1299                                 throw e;
1300
1301                         if (AllowWriteStreamBuffering || method == "GET")
1302                                 contentLength = -1;
1303
1304                         uriString = response.Headers["Location"];
1305
1306                         if (uriString == null)
1307                                 throw new WebException ($"No Location header found for {(int)code}", null,
1308                                                         WebExceptionStatus.ProtocolError, response);
1309
1310                         Uri prev = actualUri;
1311                         try {
1312                                 actualUri = new Uri (actualUri, uriString);
1313                         } catch (Exception) {
1314                                 throw new WebException ($"Invalid URL ({uriString}) for {(int)code}",
1315                                                         null, WebExceptionStatus.ProtocolError, response);
1316                         }
1317
1318                         hostChanged = (actualUri.Scheme != prev.Scheme || Host != prev.Authority);
1319                         return true;
1320                 }
1321
1322                 string GetHeaders ()
1323                 {
1324                         bool continue100 = false;
1325                         if (sendChunked) {
1326                                 continue100 = true;
1327                                 webHeaders.ChangeInternal ("Transfer-Encoding", "chunked");
1328                                 webHeaders.RemoveInternal ("Content-Length");
1329                         } else if (contentLength != -1) {
1330                                 if (auth_state.NtlmAuthState == NtlmAuthState.Challenge || proxy_auth_state.NtlmAuthState == NtlmAuthState.Challenge) {
1331                                         // We don't send any body with the NTLM Challenge request.
1332                                         if (haveContentLength || gotRequestStream || contentLength > 0)
1333                                                 webHeaders.SetInternal ("Content-Length", "0");
1334                                         else
1335                                                 webHeaders.RemoveInternal ("Content-Length");
1336                                 } else {
1337                                         if (contentLength > 0)
1338                                                 continue100 = true;
1339
1340                                         if (haveContentLength || gotRequestStream || contentLength > 0)
1341                                                 webHeaders.SetInternal ("Content-Length", contentLength.ToString ());
1342                                 }
1343                                 webHeaders.RemoveInternal ("Transfer-Encoding");
1344                         } else {
1345                                 webHeaders.RemoveInternal ("Content-Length");
1346                         }
1347
1348                         if (actualVersion == HttpVersion.Version11 && continue100 &&
1349                             servicePoint.SendContinue) { // RFC2616 8.2.3
1350                                 webHeaders.ChangeInternal ("Expect", "100-continue");
1351                                 expectContinue = true;
1352                         } else {
1353                                 webHeaders.RemoveInternal ("Expect");
1354                                 expectContinue = false;
1355                         }
1356
1357                         bool proxy_query = ProxyQuery;
1358                         string connectionHeader = (proxy_query) ? "Proxy-Connection" : "Connection";
1359                         webHeaders.RemoveInternal ((!proxy_query) ? "Proxy-Connection" : "Connection");
1360                         Version proto_version = servicePoint.ProtocolVersion;
1361                         bool spoint10 = (proto_version == null || proto_version == HttpVersion.Version10);
1362
1363                         if (keepAlive && (version == HttpVersion.Version10 || spoint10)) {
1364                                 if (webHeaders[connectionHeader] == null
1365                                     || webHeaders[connectionHeader].IndexOf ("keep-alive", StringComparison.OrdinalIgnoreCase) == -1)
1366                                         webHeaders.ChangeInternal (connectionHeader, "keep-alive");
1367                         } else if (!keepAlive && version == HttpVersion.Version11) {
1368                                 webHeaders.ChangeInternal (connectionHeader, "close");
1369                         }
1370
1371                         string host;
1372                         if (hostUri != null) {
1373                                 if (hostHasPort)
1374                                         host = hostUri.GetComponents (UriComponents.HostAndPort, UriFormat.Unescaped);
1375                                 else
1376                                         host = hostUri.GetComponents (UriComponents.Host, UriFormat.Unescaped);
1377                         } else if (Address.IsDefaultPort) {
1378                                 host = Address.GetComponents (UriComponents.Host, UriFormat.Unescaped);
1379                         } else {
1380                                 host = Address.GetComponents (UriComponents.HostAndPort, UriFormat.Unescaped);
1381                         }
1382                         webHeaders.SetInternal ("Host", host);
1383
1384                         if (cookieContainer != null) {
1385                                 string cookieHeader = cookieContainer.GetCookieHeader (actualUri);
1386                                 if (cookieHeader != "")
1387                                         webHeaders.ChangeInternal ("Cookie", cookieHeader);
1388                                 else
1389                                         webHeaders.RemoveInternal ("Cookie");
1390                         }
1391
1392                         string accept_encoding = null;
1393                         if ((auto_decomp & DecompressionMethods.GZip) != 0)
1394                                 accept_encoding = "gzip";
1395                         if ((auto_decomp & DecompressionMethods.Deflate) != 0)
1396                                 accept_encoding = accept_encoding != null ? "gzip, deflate" : "deflate";
1397                         if (accept_encoding != null)
1398                                 webHeaders.ChangeInternal ("Accept-Encoding", accept_encoding);
1399
1400                         if (!usedPreAuth && preAuthenticate)
1401                                 DoPreAuthenticate ();
1402
1403                         return webHeaders.ToString ();
1404                 }
1405
1406                 void DoPreAuthenticate ()
1407                 {
1408                         bool isProxy = (proxy != null && !proxy.IsBypassed (actualUri));
1409                         ICredentials creds = (!isProxy || credentials != null) ? credentials : proxy.Credentials;
1410                         Authorization auth = AuthenticationManager.PreAuthenticate (this, creds);
1411                         if (auth == null)
1412                                 return;
1413
1414                         webHeaders.RemoveInternal ("Proxy-Authorization");
1415                         webHeaders.RemoveInternal ("Authorization");
1416                         string authHeader = (isProxy && credentials == null) ? "Proxy-Authorization" : "Authorization";
1417                         webHeaders[authHeader] = auth.Message;
1418                         usedPreAuth = true;
1419                 }
1420
1421                 internal byte[] GetRequestHeaders ()
1422                 {
1423                         StringBuilder req = new StringBuilder ();
1424                         string query;
1425                         if (!ProxyQuery) {
1426                                 query = actualUri.PathAndQuery;
1427                         } else {
1428                                 query = String.Format ("{0}://{1}{2}", actualUri.Scheme,
1429                                                                         Host,
1430                                                                         actualUri.PathAndQuery);
1431                         }
1432
1433                         if (!force_version && servicePoint.ProtocolVersion != null && servicePoint.ProtocolVersion < version) {
1434                                 actualVersion = servicePoint.ProtocolVersion;
1435                         } else {
1436                                 actualVersion = version;
1437                         }
1438
1439                         req.AppendFormat ("{0} {1} HTTP/{2}.{3}\r\n", method, query,
1440                                                                 actualVersion.Major, actualVersion.Minor);
1441                         req.Append (GetHeaders ());
1442                         string reqstr = req.ToString ();
1443                         return Encoding.UTF8.GetBytes (reqstr);
1444                 }
1445
1446                 (WebOperation, bool) HandleNtlmAuth (WebResponseStream stream, HttpWebResponse response,
1447                                                      BufferOffsetSize writeBuffer, CancellationToken cancellationToken)
1448                 {
1449                         bool isProxy = response.StatusCode == HttpStatusCode.ProxyAuthenticationRequired;
1450                         if ((isProxy ? proxy_auth_state : auth_state).NtlmAuthState == NtlmAuthState.None)
1451                                 return (null, false);
1452
1453                         var isChallenge = auth_state.NtlmAuthState == NtlmAuthState.Challenge || proxy_auth_state.NtlmAuthState == NtlmAuthState.Challenge;
1454
1455                         var operation = new WebOperation (this, writeBuffer, isChallenge, cancellationToken);
1456                         stream.Operation.SetPriorityRequest (operation);
1457                         var creds = (!isProxy || proxy == null) ? credentials : proxy.Credentials;
1458                         if (creds != null) {
1459                                 stream.Connection.NtlmCredential = creds.GetCredential (requestUri, "NTLM");
1460                                 stream.Connection.UnsafeAuthenticatedConnectionSharing = unsafe_auth_blah;
1461                         }
1462                         return (operation, isChallenge);
1463                 }
1464
1465                 struct AuthorizationState
1466                 {
1467                         readonly HttpWebRequest request;
1468                         readonly bool isProxy;
1469                         bool isCompleted;
1470                         NtlmAuthState ntlm_auth_state;
1471
1472                         public bool IsCompleted {
1473                                 get { return isCompleted; }
1474                         }
1475
1476                         public NtlmAuthState NtlmAuthState {
1477                                 get { return ntlm_auth_state; }
1478                         }
1479
1480                         public bool IsNtlmAuthenticated {
1481                                 get { return isCompleted && ntlm_auth_state != NtlmAuthState.None; }
1482                         }
1483
1484                         public AuthorizationState (HttpWebRequest request, bool isProxy)
1485                         {
1486                                 this.request = request;
1487                                 this.isProxy = isProxy;
1488                                 isCompleted = false;
1489                                 ntlm_auth_state = NtlmAuthState.None;
1490                         }
1491
1492                         public bool CheckAuthorization (WebResponse response, HttpStatusCode code)
1493                         {
1494                                 isCompleted = false;
1495                                 if (code == HttpStatusCode.Unauthorized && request.credentials == null)
1496                                         return false;
1497
1498                                 // FIXME: This should never happen!
1499                                 if (isProxy != (code == HttpStatusCode.ProxyAuthenticationRequired))
1500                                         return false;
1501
1502                                 if (isProxy && (request.proxy == null || request.proxy.Credentials == null))
1503                                         return false;
1504
1505                                 string[] authHeaders = response.Headers.GetValues (isProxy ? "Proxy-Authenticate" : "WWW-Authenticate");
1506                                 if (authHeaders == null || authHeaders.Length == 0)
1507                                         return false;
1508
1509                                 ICredentials creds = (!isProxy) ? request.credentials : request.proxy.Credentials;
1510                                 Authorization auth = null;
1511                                 foreach (string authHeader in authHeaders) {
1512                                         auth = AuthenticationManager.Authenticate (authHeader, request, creds);
1513                                         if (auth != null)
1514                                                 break;
1515                                 }
1516                                 if (auth == null)
1517                                         return false;
1518                                 request.webHeaders[isProxy ? "Proxy-Authorization" : "Authorization"] = auth.Message;
1519                                 isCompleted = auth.Complete;
1520                                 bool is_ntlm = (auth.ModuleAuthenticationType == "NTLM");
1521                                 if (is_ntlm)
1522                                         ntlm_auth_state = (NtlmAuthState)((int)ntlm_auth_state + 1);
1523                                 return true;
1524                         }
1525
1526                         public void Reset ()
1527                         {
1528                                 isCompleted = false;
1529                                 ntlm_auth_state = NtlmAuthState.None;
1530                                 request.webHeaders.RemoveInternal (isProxy ? "Proxy-Authorization" : "Authorization");
1531                         }
1532
1533                         public override string ToString ()
1534                         {
1535                                 return string.Format ("{0}AuthState [{1}:{2}]", isProxy ? "Proxy" : "", isCompleted, ntlm_auth_state);
1536                         }
1537                 }
1538
1539                 bool CheckAuthorization (WebResponse response, HttpStatusCode code)
1540                 {
1541                         bool isProxy = code == HttpStatusCode.ProxyAuthenticationRequired;
1542                         return isProxy ? proxy_auth_state.CheckAuthorization (response, code) : auth_state.CheckAuthorization (response, code);
1543                 }
1544
1545                 (Task<BufferOffsetSize> task, WebException throwMe) GetRewriteHandler (HttpWebResponse response, bool redirect)
1546                 {
1547                         if (redirect) {
1548                                 if (!MethodWithBuffer)
1549                                         return (null, null);
1550
1551                                 if (writeStream.WriteBufferLength == 0 || contentLength == 0)
1552                                         return (null, null);
1553                         }
1554
1555                         // Keep the written body, so it can be rewritten in the retry
1556                         if (AllowWriteStreamBuffering)
1557                                 return (Task.FromResult (writeStream.GetWriteBuffer ()), null);
1558
1559                         if (ResendContentFactory == null)
1560                                 return (null, new WebException (
1561                                         "The request requires buffering data to succeed.", null, WebExceptionStatus.ProtocolError, response));
1562
1563                         Func<Task<BufferOffsetSize>> handleResendContentFactory = async () => {
1564                                 using (var ms = new MemoryStream ()) {
1565                                         await ResendContentFactory (ms).ConfigureAwait (false);
1566                                         var buffer = ms.ToArray ();
1567                                         return new BufferOffsetSize (buffer, 0, buffer.Length, false);
1568                                 }
1569                         };
1570
1571                         //
1572                         // Buffering is not allowed but we have alternative way to get same content (we
1573                         // need to resent it due to NTLM Authentication).
1574                         //
1575                         return (handleResendContentFactory (), null);
1576                 }
1577
1578                 // Returns true if redirected
1579                 (bool redirect, bool mustReadAll, Task<BufferOffsetSize> writeBuffer, WebException throwMe) CheckFinalStatus (HttpWebResponse response)
1580                 {
1581                         WebException throwMe = null;
1582
1583                         bool mustReadAll = false;
1584                         HttpStatusCode code = 0;
1585                         Task<BufferOffsetSize> rewriteHandler = null;
1586
1587                         code = response.StatusCode;
1588                         if ((!auth_state.IsCompleted && code == HttpStatusCode.Unauthorized && credentials != null) ||
1589                                 (ProxyQuery && !proxy_auth_state.IsCompleted && code == HttpStatusCode.ProxyAuthenticationRequired)) {
1590                                 if (!usedPreAuth && CheckAuthorization (response, code)) {
1591                                         mustReadAll = true;
1592
1593                                         // HEAD, GET, MKCOL, CONNECT, TRACE
1594                                         if (!MethodWithBuffer)
1595                                                 return (true, mustReadAll, null, null);
1596
1597                                         (rewriteHandler, throwMe) = GetRewriteHandler (response, false);
1598                                         if (throwMe == null)
1599                                                 return (true, mustReadAll, rewriteHandler, null);
1600
1601                                         if (!ThrowOnError)
1602                                                 return (false, mustReadAll, null, null);
1603
1604                                         writeStream.InternalClose ();
1605                                         writeStream = null;
1606                                         response.Close ();
1607
1608                                         return (false, mustReadAll, null, throwMe);
1609                                 }
1610                         }
1611
1612                         if ((int)code >= 400) {
1613                                 string err = String.Format ("The remote server returned an error: ({0}) {1}.",
1614                                                             (int)code, response.StatusDescription);
1615                                 throwMe = new WebException (err, null, WebExceptionStatus.ProtocolError, response);
1616                                 mustReadAll = true;
1617                         } else if ((int)code == 304 && allowAutoRedirect) {
1618                                 string err = String.Format ("The remote server returned an error: ({0}) {1}.",
1619                                                             (int)code, response.StatusDescription);
1620                                 throwMe = new WebException (err, null, WebExceptionStatus.ProtocolError, response);
1621                         } else if ((int)code >= 300 && allowAutoRedirect && redirects >= maxAutoRedirect) {
1622                                 throwMe = new WebException ("Max. redirections exceeded.", null,
1623                                                             WebExceptionStatus.ProtocolError, response);
1624                                 mustReadAll = true;
1625                         }
1626
1627                         if (throwMe == null) {
1628                                 int c = (int)code;
1629                                 bool b = false;
1630                                 if (allowAutoRedirect && c >= 300) {
1631                                         b = Redirect (code, response);
1632                                         (rewriteHandler, throwMe) = GetRewriteHandler (response, true);
1633                                         if (b && !unsafe_auth_blah) {
1634                                                 auth_state.Reset ();
1635                                                 proxy_auth_state.Reset ();
1636                                         }
1637                                 }
1638
1639                                 if (c >= 300 && c != 304)
1640                                         mustReadAll = true;
1641
1642                                 if (throwMe == null)
1643                                         return (b, mustReadAll, rewriteHandler, null);
1644                         }
1645
1646                         if (!ThrowOnError)
1647                                 return (false, mustReadAll, null, null);
1648
1649                         if (writeStream != null) {
1650                                 writeStream.InternalClose ();
1651                                 writeStream = null;
1652                         }
1653
1654                         return (false, mustReadAll, null, throwMe);
1655                 }
1656
1657                 internal bool ReuseConnection {
1658                         get;
1659                         set;
1660                 }
1661
1662 #region referencesource
1663         internal static StringBuilder GenerateConnectionGroup(string connectionGroupName, bool unsafeConnectionGroup, bool isInternalGroup)
1664         {
1665             StringBuilder connectionLine = new StringBuilder(connectionGroupName);
1666
1667             connectionLine.Append(unsafeConnectionGroup ? "U>" : "S>");
1668
1669             if (isInternalGroup)
1670             {
1671                 connectionLine.Append("I>");
1672             }
1673
1674             return connectionLine;
1675         }
1676 #endregion
1677         }
1678 }
1679