ArchiSteamFarm/Web/WebBrowser.cs

   1 // ----------------------------------------------------------------------------------------------
   2 //     _                _      _  ____   _                           _____
   3 //    / \    _ __  ___ | |__  (_)/ ___| | |_  ___   __ _  _ __ ___  |  ___|__ _  _ __  _ __ ___
   4 //   / _ \  | '__|/ __|| '_ \ | |\___ \ | __|/ _ \ / _` || '_ ` _ \ | |_  / _` || '__|| '_ ` _ \
   5 //  / ___ \ | |  | (__ | | | || | ___) || |_|  __/| (_| || | | | | ||  _|| (_| || |   | | | | | |
   6 // /_/   \_\|_|   \___||_| |_||_||____/  \__|\___| \__,_||_| |_| |_||_|   \__,_||_|   |_| |_| |_|
   7 // ----------------------------------------------------------------------------------------------
   8 // |
   9 // Copyright 2015-2024 Łukasz "JustArchi" Domeradzki
  10 // Contact: JustArchi@JustArchi.net
  11 // |
  12 // Licensed under the Apache License, Version 2.0 (the "License");
  13 // you may not use this file except in compliance with the License.
  14 // You may obtain a copy of the License at
  15 // |
  16 // http://www.apache.org/licenses/LICENSE-2.0
  17 // |
  18 // Unless required by applicable law or agreed to in writing, software
  19 // distributed under the License is distributed on an "AS IS" BASIS,
  20 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  21 // See the License for the specific language governing permissions and
  22 // limitations under the License.
  23
  24 using System;
  25 using System.Buffers;
  26 using System.Collections.Generic;
  27 using System.ComponentModel;
  28 using System.Diagnostics.CodeAnalysis;
  29 using System.Globalization;
  30 using System.IO;
  31 using System.Linq;
  32 using System.Net;
  33 using System.Net.Http;
  34 using System.Net.Http.Headers;
  35 using System.Net.Http.Json;
  36 using System.Threading;
  37 using System.Threading.Tasks;
  38 using ArchiSteamFarm.Core;
  39 using ArchiSteamFarm.Helpers.Json;
  40 using ArchiSteamFarm.Localization;
  41 using ArchiSteamFarm.NLog;
  42 using ArchiSteamFarm.Storage;
  43 using ArchiSteamFarm.Web.Responses;
  44 using JetBrains.Annotations;
  45
  46 namespace ArchiSteamFarm.Web;
  47
  48 public sealed class WebBrowser : IDisposable {
  49         [PublicAPI]
  50         public const byte MaxTries = 5; // Defines maximum number of recommended tries for a single request
  51
  52         internal const byte MaxConnections = 5; // Defines maximum number of connections per ServicePoint. Be careful, as it also defines maximum number of sockets in CLOSE_WAIT state
  53
  54         private const ushort ExtendedTimeout = 600; // Defines timeout for WebBrowsers dealing with huge data (ASF update)
  55         private const byte MaxIdleTime = 15; // Defines in seconds, how long socket is allowed to stay in CLOSE_WAIT state after there are no connections to it
  56
  57         [PublicAPI]
  58         public CookieContainer CookieContainer { get; } = new();
  59
  60         [PublicAPI]
  61         public TimeSpan Timeout => HttpClient.Timeout;
  62
  63         private readonly ArchiLogger ArchiLogger;
  64         private readonly HttpClient HttpClient;
  65         private readonly HttpClientHandler HttpClientHandler;
  66
  67         internal WebBrowser(ArchiLogger archiLogger, IWebProxy? webProxy = null, bool extendedTimeout = false) {
  68                 ArgumentNullException.ThrowIfNull(archiLogger);
  69
  70                 ArchiLogger = archiLogger;
  71
  72                 HttpClientHandler = new HttpClientHandler {
  73                         AllowAutoRedirect = false, // This must be false if we want to handle custom redirection schemes such as "steammobile://"
  74                         AutomaticDecompression = DecompressionMethods.All,
  75                         CookieContainer = CookieContainer,
  76                         MaxConnectionsPerServer = MaxConnections
  77                 };
  78
  79                 if (webProxy != null) {
  80                         HttpClientHandler.Proxy = webProxy;
  81                         HttpClientHandler.UseProxy = true;
  82
  83                         if (webProxy.Credentials != null) {
  84                                 // We can be pretty sure that user knows what he's doing and that proxy indeed requires authentication, save roundtrip
  85                                 HttpClientHandler.PreAuthenticate = true;
  86                         }
  87                 }
  88
  89                 HttpClient = GenerateDisposableHttpClient(extendedTimeout);
  90         }
  91
  92         public void Dispose() {
  93                 HttpClient.Dispose();
  94                 HttpClientHandler.Dispose();
  95         }
  96
  97         [PublicAPI]
  98         public HttpClient GenerateDisposableHttpClient(bool extendedTimeout = false) {
  99                 byte connectionTimeout = ASF.GlobalConfig?.ConnectionTimeout ?? GlobalConfig.DefaultConnectionTimeout;
 100
 101                 HttpClient result = new(HttpClientHandler, false) {
 102                         DefaultRequestVersion = HttpVersion.Version30,
 103                         Timeout = TimeSpan.FromSeconds(extendedTimeout ? ExtendedTimeout : connectionTimeout)
 104                 };
 105
 106                 // Most web services expect that UserAgent is set, so we declare it globally
 107                 // If you by any chance came here with a very "clever" idea of hiding your ass by changing default ASF user-agent then here is a very good advice from me: don't, for your own safety - you've been warned
 108                 result.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue(SharedInfo.PublicIdentifier, SharedInfo.Version.ToString()));
 109                 result.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue($"({SharedInfo.BuildInfo.Variant}; {OS.Version.Replace("(", "", StringComparison.Ordinal).Replace(")", "", StringComparison.Ordinal)}; +{SharedInfo.ProjectURL})"));
 110
 111                 // Inform websites that we visit about our preference in language, if possible
 112                 result.DefaultRequestHeaders.AcceptLanguage.Add(new StringWithQualityHeaderValue("en-US", 0.9));
 113                 result.DefaultRequestHeaders.AcceptLanguage.Add(new StringWithQualityHeaderValue("en", 0.8));
 114
 115                 return result;
 116         }
 117
 118         [PublicAPI]
 119         public async Task<BinaryResponse?> UrlGetToBinary(Uri request, IReadOnlyCollection<KeyValuePair<string, string>>? headers = null, Uri? referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries, int rateLimitingDelay = 0, IProgress<byte>? progressReporter = null, CancellationToken cancellationToken = default) {
 120                 ArgumentNullException.ThrowIfNull(request);
 121                 ArgumentOutOfRangeException.ThrowIfZero(maxTries);
 122                 ArgumentOutOfRangeException.ThrowIfNegative(rateLimitingDelay);
 123
 124                 for (byte i = 0; i < maxTries; i++) {
 125                         if ((i > 0) && (rateLimitingDelay > 0)) {
 126                                 await Task.Delay(rateLimitingDelay, cancellationToken).ConfigureAwait(false);
 127                         }
 128
 129                         StreamResponse? response = await UrlGetToStream(request, headers, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1, rateLimitingDelay, cancellationToken).ConfigureAwait(false);
 130
 131                         if (response == null) {
 132                                 // Request timed out, try again
 133                                 continue;
 134                         }
 135
 136                         await using (response.ConfigureAwait(false)) {
 137                                 if (response.StatusCode.IsRedirectionCode()) {
 138                                         if (!requestOptions.HasFlag(ERequestOptions.ReturnRedirections)) {
 139                                                 break;
 140                                         }
 141                                 }
 142
 143                                 if (response.StatusCode.IsClientErrorCode()) {
 144                                         if (!requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) {
 145                                                 break;
 146                                         }
 147                                 }
 148
 149                                 if (response.StatusCode.IsServerErrorCode()) {
 150                                         if (!requestOptions.HasFlag(ERequestOptions.ReturnServerErrors)) {
 151                                                 continue;
 152                                         }
 153                                 }
 154
 155                                 if (response.Content == null) {
 156                                         throw new InvalidOperationException(nameof(response.Content));
 157                                 }
 158
 159                                 if (response.Length > Array.MaxLength) {
 160                                         throw new InvalidOperationException(nameof(response.Length));
 161                                 }
 162
 163                                 progressReporter?.Report(0);
 164
 165                                 MemoryStream ms = new((int) response.Length);
 166
 167                                 await using (ms.ConfigureAwait(false)) {
 168                                         byte batch = 0;
 169                                         long readThisBatch = 0;
 170                                         long batchIncreaseSize = response.Length / 100;
 171
 172                                         ArrayPool<byte> bytePool = ArrayPool<byte>.Shared;
 173
 174                                         // This is HttpClient's buffer, using more doesn't make sense
 175                                         byte[] buffer = bytePool.Rent(8192);
 176
 177                                         try {
 178                                                 while (response.Content.CanRead) {
 179                                                         int read = await response.Content.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken).ConfigureAwait(false);
 180
 181                                                         if (read <= 0) {
 182                                                                 break;
 183                                                         }
 184
 185                                                         // Report progress in-between downloading only if file is big enough to justify it
 186                                                         // Current logic below will report progress if file is bigger than ~800 KB
 187                                                         if (batchIncreaseSize >= buffer.Length) {
 188                                                                 readThisBatch += read;
 189
 190                                                                 for (; (readThisBatch >= batchIncreaseSize) && (batch < 99); readThisBatch -= batchIncreaseSize) {
 191                                                                         // We need a copy of variable being passed when in for loops, as loop will proceed before our event is launched
 192                                                                         byte progress = ++batch;
 193
 194                                                                         progressReporter?.Report(progress);
 195                                                                 }
 196                                                         }
 197
 198                                                         await ms.WriteAsync(buffer.AsMemory(0, read), cancellationToken).ConfigureAwait(false);
 199                                                 }
 200                                         } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) {
 201                                                 throw;
 202                                         } catch (Exception e) {
 203                                                 ArchiLogger.LogGenericWarningException(e);
 204                                                 ArchiLogger.LogGenericDebug(string.Format(CultureInfo.CurrentCulture, Strings.ErrorFailingRequest, request));
 205
 206                                                 return null;
 207                                         } finally {
 208                                                 bytePool.Return(buffer);
 209                                         }
 210
 211                                         progressReporter?.Report(100);
 212
 213                                         return new BinaryResponse(response, ms.ToArray());
 214                                 }
 215                         }
 216                 }
 217
 218                 ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.ErrorRequestFailedTooManyTimes, maxTries));
 219                 ArchiLogger.LogGenericDebug(string.Format(CultureInfo.CurrentCulture, Strings.ErrorFailingRequest, request));
 220
 221                 return null;
 222         }
 223
 224         [PublicAPI]
 225         public async Task<HtmlDocumentResponse?> UrlGetToHtmlDocument(Uri request, IReadOnlyCollection<KeyValuePair<string, string>>? headers = null, Uri? referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries, int rateLimitingDelay = 0, CancellationToken cancellationToken = default) {
 226                 ArgumentNullException.ThrowIfNull(request);
 227                 ArgumentOutOfRangeException.ThrowIfZero(maxTries);
 228                 ArgumentOutOfRangeException.ThrowIfNegative(rateLimitingDelay);
 229
 230                 for (byte i = 0; i < maxTries; i++) {
 231                         if ((i > 0) && (rateLimitingDelay > 0)) {
 232                                 await Task.Delay(rateLimitingDelay, cancellationToken).ConfigureAwait(false);
 233                         }
 234
 235                         StreamResponse? response = await UrlGetToStream(request, headers, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1, rateLimitingDelay, cancellationToken).ConfigureAwait(false);
 236
 237                         if (response == null) {
 238                                 // Request timed out, try again
 239                                 continue;
 240                         }
 241
 242                         await using (response.ConfigureAwait(false)) {
 243                                 if (response.StatusCode.IsRedirectionCode()) {
 244                                         if (!requestOptions.HasFlag(ERequestOptions.ReturnRedirections)) {
 245                                                 break;
 246                                         }
 247                                 }
 248
 249                                 if (response.StatusCode.IsClientErrorCode()) {
 250                                         if (!requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) {
 251                                                 break;
 252                                         }
 253                                 }
 254
 255                                 if (response.StatusCode.IsServerErrorCode()) {
 256                                         if (!requestOptions.HasFlag(ERequestOptions.ReturnServerErrors)) {
 257                                                 continue;
 258                                         }
 259                                 }
 260
 261                                 if (response.Content == null) {
 262                                         throw new InvalidOperationException(nameof(response.Content));
 263                                 }
 264
 265                                 try {
 266                                         return await HtmlDocumentResponse.Create(response, cancellationToken).ConfigureAwait(false);
 267                                 } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) {
 268                                         throw;
 269                                 } catch (Exception e) {
 270                                         if ((requestOptions.HasFlag(ERequestOptions.AllowInvalidBodyOnSuccess) && response.StatusCode.IsSuccessCode()) || (requestOptions.HasFlag(ERequestOptions.AllowInvalidBodyOnErrors) && !response.StatusCode.IsSuccessCode())) {
 271                                                 return new HtmlDocumentResponse(response);
 272                                         }
 273
 274                                         ArchiLogger.LogGenericWarningException(e);
 275                                         ArchiLogger.LogGenericDebug(string.Format(CultureInfo.CurrentCulture, Strings.ErrorFailingRequest, request));
 276                                 }
 277                         }
 278                 }
 279
 280                 ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.ErrorRequestFailedTooManyTimes, maxTries));
 281                 ArchiLogger.LogGenericDebug(string.Format(CultureInfo.CurrentCulture, Strings.ErrorFailingRequest, request));
 282
 283                 return null;
 284         }
 285
 286         [PublicAPI]
 287         public async Task<ObjectResponse<T>?> UrlGetToJsonObject<T>(Uri request, IReadOnlyCollection<KeyValuePair<string, string>>? headers = null, Uri? referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries, int rateLimitingDelay = 0, CancellationToken cancellationToken = default) {
 288                 ArgumentNullException.ThrowIfNull(request);
 289                 ArgumentOutOfRangeException.ThrowIfZero(maxTries);
 290                 ArgumentOutOfRangeException.ThrowIfNegative(rateLimitingDelay);
 291
 292                 for (byte i = 0; i < maxTries; i++) {
 293                         if ((i > 0) && (rateLimitingDelay > 0)) {
 294                                 await Task.Delay(rateLimitingDelay, cancellationToken).ConfigureAwait(false);
 295                         }
 296
 297                         StreamResponse? response = await UrlGetToStream(request, headers, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1, rateLimitingDelay, cancellationToken).ConfigureAwait(false);
 298
 299                         if (response == null) {
 300                                 // Request timed out, try again
 301                                 continue;
 302                         }
 303
 304                         await using (response.ConfigureAwait(false)) {
 305                                 if (response.StatusCode.IsRedirectionCode()) {
 306                                         if (!requestOptions.HasFlag(ERequestOptions.ReturnRedirections)) {
 307                                                 break;
 308                                         }
 309                                 }
 310
 311                                 if (response.StatusCode.IsClientErrorCode()) {
 312                                         if (!requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) {
 313                                                 break;
 314                                         }
 315                                 }
 316
 317                                 if (response.StatusCode.IsServerErrorCode()) {
 318                                         if (!requestOptions.HasFlag(ERequestOptions.ReturnServerErrors)) {
 319                                                 continue;
 320                                         }
 321                                 }
 322
 323                                 if (response.Content == null) {
 324                                         throw new InvalidOperationException(nameof(response.Content));
 325                                 }
 326
 327                                 T? obj;
 328
 329                                 try {
 330                                         obj = await response.Content.ToJsonObject<T>(cancellationToken).ConfigureAwait(false);
 331                                 } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) {
 332                                         throw;
 333                                 } catch (Exception e) {
 334                                         if ((requestOptions.HasFlag(ERequestOptions.AllowInvalidBodyOnSuccess) && response.StatusCode.IsSuccessCode()) || (requestOptions.HasFlag(ERequestOptions.AllowInvalidBodyOnErrors) && !response.StatusCode.IsSuccessCode())) {
 335                                                 return new ObjectResponse<T>(response);
 336                                         }
 337
 338                                         ArchiLogger.LogGenericWarningException(e);
 339                                         ArchiLogger.LogGenericDebug(string.Format(CultureInfo.CurrentCulture, Strings.ErrorFailingRequest, request));
 340
 341                                         continue;
 342                                 }
 343
 344                                 if (obj is null) {
 345                                         if ((requestOptions.HasFlag(ERequestOptions.AllowInvalidBodyOnSuccess) && response.StatusCode.IsSuccessCode()) || (requestOptions.HasFlag(ERequestOptions.AllowInvalidBodyOnErrors) && !response.StatusCode.IsSuccessCode())) {
 346                                                 return new ObjectResponse<T>(response);
 347                                         }
 348
 349                                         ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.ErrorIsEmpty, nameof(obj)));
 350
 351                                         continue;
 352                                 }
 353
 354                                 return new ObjectResponse<T>(response, obj);
 355                         }
 356                 }
 357
 358                 ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.ErrorRequestFailedTooManyTimes, maxTries));
 359                 ArchiLogger.LogGenericDebug(string.Format(CultureInfo.CurrentCulture, Strings.ErrorFailingRequest, request));
 360
 361                 return null;
 362         }
 363
 364         [PublicAPI]
 365         public async Task<StreamResponse?> UrlGetToStream(Uri request, IReadOnlyCollection<KeyValuePair<string, string>>? headers = null, Uri? referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries, int rateLimitingDelay = 0, CancellationToken cancellationToken = default) {
 366                 ArgumentNullException.ThrowIfNull(request);
 367                 ArgumentOutOfRangeException.ThrowIfZero(maxTries);
 368                 ArgumentOutOfRangeException.ThrowIfNegative(rateLimitingDelay);
 369
 370                 for (byte i = 0; i < maxTries; i++) {
 371                         if ((i > 0) && (rateLimitingDelay > 0)) {
 372                                 await Task.Delay(rateLimitingDelay, cancellationToken).ConfigureAwait(false);
 373                         }
 374
 375                         HttpResponseMessage? response = await InternalGet(request, headers, referer, requestOptions, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
 376
 377                         if (response == null) {
 378                                 // Request timed out, try again
 379                                 continue;
 380                         }
 381
 382                         if (response.StatusCode.IsRedirectionCode()) {
 383                                 if (!requestOptions.HasFlag(ERequestOptions.ReturnRedirections)) {
 384                                         break;
 385                                 }
 386                         }
 387
 388                         if (response.StatusCode.IsClientErrorCode()) {
 389                                 if (!requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) {
 390                                         break;
 391                                 }
 392                         }
 393
 394                         if (response.StatusCode.IsServerErrorCode()) {
 395                                 if (!requestOptions.HasFlag(ERequestOptions.ReturnServerErrors)) {
 396                                         continue;
 397                                 }
 398                         }
 399
 400                         return new StreamResponse(response, await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false));
 401                 }
 402
 403                 ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.ErrorRequestFailedTooManyTimes, maxTries));
 404                 ArchiLogger.LogGenericDebug(string.Format(CultureInfo.CurrentCulture, Strings.ErrorFailingRequest, request));
 405
 406                 return null;
 407         }
 408
 409         [PublicAPI]
 410         public async Task<BasicResponse?> UrlHead(Uri request, IReadOnlyCollection<KeyValuePair<string, string>>? headers = null, Uri? referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries, int rateLimitingDelay = 0, CancellationToken cancellationToken = default) {
 411                 ArgumentNullException.ThrowIfNull(request);
 412                 ArgumentOutOfRangeException.ThrowIfZero(maxTries);
 413                 ArgumentOutOfRangeException.ThrowIfNegative(rateLimitingDelay);
 414
 415                 for (byte i = 0; i < maxTries; i++) {
 416                         if ((i > 0) && (rateLimitingDelay > 0)) {
 417                                 await Task.Delay(rateLimitingDelay, cancellationToken).ConfigureAwait(false);
 418                         }
 419
 420                         using HttpResponseMessage? response = await InternalHead(request, headers, referer, requestOptions, cancellationToken: cancellationToken).ConfigureAwait(false);
 421
 422                         if (response == null) {
 423                                 continue;
 424                         }
 425
 426                         if (response.StatusCode.IsRedirectionCode()) {
 427                                 if (!requestOptions.HasFlag(ERequestOptions.ReturnRedirections)) {
 428                                         break;
 429                                 }
 430                         }
 431
 432                         if (response.StatusCode.IsClientErrorCode()) {
 433                                 if (!requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) {
 434                                         break;
 435                                 }
 436                         }
 437
 438                         if (response.StatusCode.IsServerErrorCode()) {
 439                                 if (!requestOptions.HasFlag(ERequestOptions.ReturnServerErrors)) {
 440                                         continue;
 441                                 }
 442                         }
 443
 444                         return new BasicResponse(response);
 445                 }
 446
 447                 ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.ErrorRequestFailedTooManyTimes, maxTries));
 448                 ArchiLogger.LogGenericDebug(string.Format(CultureInfo.CurrentCulture, Strings.ErrorFailingRequest, request));
 449
 450                 return null;
 451         }
 452
 453         [PublicAPI]
 454         public async Task<BasicResponse?> UrlPost<T>(Uri request, IReadOnlyCollection<KeyValuePair<string, string>>? headers = null, T? data = null, Uri? referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries, int rateLimitingDelay = 0, CancellationToken cancellationToken = default) where T : class {
 455                 ArgumentNullException.ThrowIfNull(request);
 456                 ArgumentOutOfRangeException.ThrowIfZero(maxTries);
 457                 ArgumentOutOfRangeException.ThrowIfNegative(rateLimitingDelay);
 458
 459                 for (byte i = 0; i < maxTries; i++) {
 460                         if ((i > 0) && (rateLimitingDelay > 0)) {
 461                                 await Task.Delay(rateLimitingDelay, cancellationToken).ConfigureAwait(false);
 462                         }
 463
 464                         using HttpResponseMessage? response = await InternalPost(request, headers, data, referer, requestOptions, cancellationToken: cancellationToken).ConfigureAwait(false);
 465
 466                         if (response == null) {
 467                                 continue;
 468                         }
 469
 470                         if (response.StatusCode.IsRedirectionCode()) {
 471                                 if (!requestOptions.HasFlag(ERequestOptions.ReturnRedirections)) {
 472                                         break;
 473                                 }
 474                         }
 475
 476                         if (response.StatusCode.IsClientErrorCode()) {
 477                                 if (!requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) {
 478                                         break;
 479                                 }
 480                         }
 481
 482                         if (response.StatusCode.IsServerErrorCode()) {
 483                                 if (!requestOptions.HasFlag(ERequestOptions.ReturnServerErrors)) {
 484                                         continue;
 485                                 }
 486                         }
 487
 488                         return new BasicResponse(response);
 489                 }
 490
 491                 ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.ErrorRequestFailedTooManyTimes, maxTries));
 492                 ArchiLogger.LogGenericDebug(string.Format(CultureInfo.CurrentCulture, Strings.ErrorFailingRequest, request));
 493
 494                 return null;
 495         }
 496
 497         [PublicAPI]
 498         public async Task<HtmlDocumentResponse?> UrlPostToHtmlDocument<T>(Uri request, IReadOnlyCollection<KeyValuePair<string, string>>? headers = null, T? data = null, Uri? referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries, int rateLimitingDelay = 0, CancellationToken cancellationToken = default) where T : class {
 499                 ArgumentNullException.ThrowIfNull(request);
 500                 ArgumentOutOfRangeException.ThrowIfZero(maxTries);
 501                 ArgumentOutOfRangeException.ThrowIfNegative(rateLimitingDelay);
 502
 503                 for (byte i = 0; i < maxTries; i++) {
 504                         if ((i > 0) && (rateLimitingDelay > 0)) {
 505                                 await Task.Delay(rateLimitingDelay, cancellationToken).ConfigureAwait(false);
 506                         }
 507
 508                         StreamResponse? response = await UrlPostToStream(request, headers, data, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1, rateLimitingDelay, cancellationToken).ConfigureAwait(false);
 509
 510                         if (response == null) {
 511                                 // Request timed out, try again
 512                                 continue;
 513                         }
 514
 515                         await using (response.ConfigureAwait(false)) {
 516                                 if (response.StatusCode.IsRedirectionCode()) {
 517                                         if (!requestOptions.HasFlag(ERequestOptions.ReturnRedirections)) {
 518                                                 break;
 519                                         }
 520                                 }
 521
 522                                 if (response.StatusCode.IsClientErrorCode()) {
 523                                         if (!requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) {
 524                                                 break;
 525                                         }
 526                                 }
 527
 528                                 if (response.StatusCode.IsServerErrorCode()) {
 529                                         if (!requestOptions.HasFlag(ERequestOptions.ReturnServerErrors)) {
 530                                                 continue;
 531                                         }
 532                                 }
 533
 534                                 if (response.Content == null) {
 535                                         throw new InvalidOperationException(nameof(response.Content));
 536                                 }
 537
 538                                 try {
 539                                         return await HtmlDocumentResponse.Create(response, cancellationToken).ConfigureAwait(false);
 540                                 } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) {
 541                                         throw;
 542                                 } catch (Exception e) {
 543                                         if ((requestOptions.HasFlag(ERequestOptions.AllowInvalidBodyOnSuccess) && response.StatusCode.IsSuccessCode()) || (requestOptions.HasFlag(ERequestOptions.AllowInvalidBodyOnErrors) && !response.StatusCode.IsSuccessCode())) {
 544                                                 return new HtmlDocumentResponse(response);
 545                                         }
 546
 547                                         ArchiLogger.LogGenericWarningException(e);
 548                                         ArchiLogger.LogGenericDebug(string.Format(CultureInfo.CurrentCulture, Strings.ErrorFailingRequest, request));
 549                                 }
 550                         }
 551                 }
 552
 553                 ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.ErrorRequestFailedTooManyTimes, maxTries));
 554                 ArchiLogger.LogGenericDebug(string.Format(CultureInfo.CurrentCulture, Strings.ErrorFailingRequest, request));
 555
 556                 return null;
 557         }
 558
 559         [PublicAPI]
 560         public async Task<ObjectResponse<TResult>?> UrlPostToJsonObject<TResult, TData>(Uri request, IReadOnlyCollection<KeyValuePair<string, string>>? headers = null, TData? data = null, Uri? referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries, int rateLimitingDelay = 0, CancellationToken cancellationToken = default) where TData : class {
 561                 ArgumentNullException.ThrowIfNull(request);
 562                 ArgumentOutOfRangeException.ThrowIfZero(maxTries);
 563                 ArgumentOutOfRangeException.ThrowIfNegative(rateLimitingDelay);
 564
 565                 for (byte i = 0; i < maxTries; i++) {
 566                         if ((i > 0) && (rateLimitingDelay > 0)) {
 567                                 await Task.Delay(rateLimitingDelay, cancellationToken).ConfigureAwait(false);
 568                         }
 569
 570                         StreamResponse? response = await UrlPostToStream(request, headers, data, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1, rateLimitingDelay, cancellationToken).ConfigureAwait(false);
 571
 572                         if (response == null) {
 573                                 // Request timed out, try again
 574                                 continue;
 575                         }
 576
 577                         await using (response.ConfigureAwait(false)) {
 578                                 if (response.StatusCode.IsRedirectionCode()) {
 579                                         if (!requestOptions.HasFlag(ERequestOptions.ReturnRedirections)) {
 580                                                 break;
 581                                         }
 582                                 }
 583
 584                                 if (response.StatusCode.IsClientErrorCode()) {
 585                                         if (!requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) {
 586                                                 break;
 587                                         }
 588                                 }
 589
 590                                 if (response.StatusCode.IsServerErrorCode()) {
 591                                         if (!requestOptions.HasFlag(ERequestOptions.ReturnServerErrors)) {
 592                                                 continue;
 593                                         }
 594                                 }
 595
 596                                 if (response.Content == null) {
 597                                         throw new InvalidOperationException(nameof(response.Content));
 598                                 }
 599
 600                                 TResult? obj;
 601
 602                                 try {
 603                                         obj = await response.Content.ToJsonObject<TResult>(cancellationToken).ConfigureAwait(false);
 604                                 } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) {
 605                                         throw;
 606                                 } catch (Exception e) {
 607                                         if ((requestOptions.HasFlag(ERequestOptions.AllowInvalidBodyOnSuccess) && response.StatusCode.IsSuccessCode()) || (requestOptions.HasFlag(ERequestOptions.AllowInvalidBodyOnErrors) && !response.StatusCode.IsSuccessCode())) {
 608                                                 return new ObjectResponse<TResult>(response);
 609                                         }
 610
 611                                         ArchiLogger.LogGenericWarningException(e);
 612                                         ArchiLogger.LogGenericDebug(string.Format(CultureInfo.CurrentCulture, Strings.ErrorFailingRequest, request));
 613
 614                                         continue;
 615                                 }
 616
 617                                 if (obj is null) {
 618                                         if ((requestOptions.HasFlag(ERequestOptions.AllowInvalidBodyOnSuccess) && response.StatusCode.IsSuccessCode()) || (requestOptions.HasFlag(ERequestOptions.AllowInvalidBodyOnErrors) && !response.StatusCode.IsSuccessCode())) {
 619                                                 return new ObjectResponse<TResult>(response);
 620                                         }
 621
 622                                         ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.ErrorIsEmpty, nameof(obj)));
 623
 624                                         continue;
 625                                 }
 626
 627                                 return new ObjectResponse<TResult>(response, obj);
 628                         }
 629                 }
 630
 631                 ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.ErrorRequestFailedTooManyTimes, maxTries));
 632                 ArchiLogger.LogGenericDebug(string.Format(CultureInfo.CurrentCulture, Strings.ErrorFailingRequest, request));
 633
 634                 return null;
 635         }
 636
 637         [PublicAPI]
 638         public async Task<StreamResponse?> UrlPostToStream<T>(Uri request, IReadOnlyCollection<KeyValuePair<string, string>>? headers = null, T? data = null, Uri? referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries, int rateLimitingDelay = 0, CancellationToken cancellationToken = default) where T : class {
 639                 ArgumentNullException.ThrowIfNull(request);
 640                 ArgumentOutOfRangeException.ThrowIfZero(maxTries);
 641                 ArgumentOutOfRangeException.ThrowIfNegative(rateLimitingDelay);
 642
 643                 for (byte i = 0; i < maxTries; i++) {
 644                         if ((i > 0) && (rateLimitingDelay > 0)) {
 645                                 await Task.Delay(rateLimitingDelay, cancellationToken).ConfigureAwait(false);
 646                         }
 647
 648                         HttpResponseMessage? response = await InternalPost(request, headers, data, referer, requestOptions, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
 649
 650                         if (response == null) {
 651                                 // Request timed out, try again
 652                                 continue;
 653                         }
 654
 655                         if (response.StatusCode.IsRedirectionCode()) {
 656                                 if (!requestOptions.HasFlag(ERequestOptions.ReturnRedirections)) {
 657                                         break;
 658                                 }
 659                         }
 660
 661                         if (response.StatusCode.IsClientErrorCode()) {
 662                                 if (!requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) {
 663                                         break;
 664                                 }
 665                         }
 666
 667                         if (response.StatusCode.IsServerErrorCode()) {
 668                                 if (!requestOptions.HasFlag(ERequestOptions.ReturnServerErrors)) {
 669                                         continue;
 670                                 }
 671                         }
 672
 673                         return new StreamResponse(response, await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false));
 674                 }
 675
 676                 ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.ErrorRequestFailedTooManyTimes, maxTries));
 677                 ArchiLogger.LogGenericDebug(string.Format(CultureInfo.CurrentCulture, Strings.ErrorFailingRequest, request));
 678
 679                 return null;
 680         }
 681
 682         internal static void Init() {
 683                 // Set max connection limit from default of 2 to desired value
 684                 ServicePointManager.DefaultConnectionLimit = MaxConnections;
 685
 686                 // Set max idle time from default of 100 seconds (100 * 1000) to desired value
 687                 ServicePointManager.MaxServicePointIdleTime = MaxIdleTime * 1000;
 688
 689                 // Don't use Expect100Continue, we're sure about our POSTs, save some TCP packets
 690                 ServicePointManager.Expect100Continue = false;
 691
 692                 // Reuse ports if possible
 693                 ServicePointManager.ReusePort = true;
 694         }
 695
 696         private async Task<HttpResponseMessage?> InternalGet(Uri request, IReadOnlyCollection<KeyValuePair<string, string>>? headers = null, Uri? referer = null, ERequestOptions requestOptions = ERequestOptions.None, HttpCompletionOption httpCompletionOption = HttpCompletionOption.ResponseContentRead, CancellationToken cancellationToken = default) {
 697                 ArgumentNullException.ThrowIfNull(request);
 698
 699                 if (!Enum.IsDefined(httpCompletionOption)) {
 700                         throw new InvalidEnumArgumentException(nameof(httpCompletionOption), (int) httpCompletionOption, typeof(HttpCompletionOption));
 701                 }
 702
 703                 return await InternalRequest<object>(request, HttpMethod.Get, headers, null, referer, requestOptions, httpCompletionOption, cancellationToken: cancellationToken).ConfigureAwait(false);
 704         }
 705
 706         private async Task<HttpResponseMessage?> InternalHead(Uri request, IReadOnlyCollection<KeyValuePair<string, string>>? headers = null, Uri? referer = null, ERequestOptions requestOptions = ERequestOptions.None, HttpCompletionOption httpCompletionOption = HttpCompletionOption.ResponseContentRead, CancellationToken cancellationToken = default) {
 707                 ArgumentNullException.ThrowIfNull(request);
 708
 709                 if (!Enum.IsDefined(httpCompletionOption)) {
 710                         throw new InvalidEnumArgumentException(nameof(httpCompletionOption), (int) httpCompletionOption, typeof(HttpCompletionOption));
 711                 }
 712
 713                 return await InternalRequest<object>(request, HttpMethod.Head, headers, null, referer, requestOptions, httpCompletionOption, cancellationToken: cancellationToken).ConfigureAwait(false);
 714         }
 715
 716         private async Task<HttpResponseMessage?> InternalPost<T>(Uri request, IReadOnlyCollection<KeyValuePair<string, string>>? headers = null, T? data = null, Uri? referer = null, ERequestOptions requestOptions = ERequestOptions.None, HttpCompletionOption httpCompletionOption = HttpCompletionOption.ResponseContentRead, CancellationToken cancellationToken = default) where T : class {
 717                 ArgumentNullException.ThrowIfNull(request);
 718
 719                 if (!Enum.IsDefined(httpCompletionOption)) {
 720                         throw new InvalidEnumArgumentException(nameof(httpCompletionOption), (int) httpCompletionOption, typeof(HttpCompletionOption));
 721                 }
 722
 723                 return await InternalRequest(request, HttpMethod.Post, headers, data, referer, requestOptions, httpCompletionOption, cancellationToken: cancellationToken).ConfigureAwait(false);
 724         }
 725
 726         [UnconditionalSuppressMessage("AssemblyLoadTrimming", "IL2026:RequiresUnreferencedCode", Justification = "We don't care about trimmed assemblies, as we need it to work only with the known (used) ones")]
 727         private async Task<HttpResponseMessage?> InternalRequest<T>(Uri request, HttpMethod httpMethod, IReadOnlyCollection<KeyValuePair<string, string>>? headers = null, T? data = null, Uri? referer = null, ERequestOptions requestOptions = ERequestOptions.None, HttpCompletionOption httpCompletionOption = HttpCompletionOption.ResponseContentRead, byte maxRedirections = MaxTries, CancellationToken cancellationToken = default) where T : class {
 728                 ArgumentNullException.ThrowIfNull(request);
 729                 ArgumentNullException.ThrowIfNull(httpMethod);
 730
 731                 if (!Enum.IsDefined(httpCompletionOption)) {
 732                         throw new InvalidEnumArgumentException(nameof(httpCompletionOption), (int) httpCompletionOption, typeof(HttpCompletionOption));
 733                 }
 734
 735                 HttpResponseMessage response;
 736
 737                 while (true) {
 738                         using (HttpRequestMessage requestMessage = new(httpMethod, request)) {
 739                                 requestMessage.Version = HttpClient.DefaultRequestVersion;
 740
 741                                 if (headers != null) {
 742                                         foreach ((string header, string value) in headers) {
 743                                                 requestMessage.Headers.Add(header, value);
 744                                         }
 745                                 }
 746
 747                                 if (data != null) {
 748                                         switch (data) {
 749                                                 case HttpContent content:
 750                                                         requestMessage.Content = content;
 751
 752                                                         break;
 753                                                 case IReadOnlyCollection<KeyValuePair<string, string>> nameValueCollection:
 754                                                         try {
 755                                                                 requestMessage.Content = new FormUrlEncodedContent(nameValueCollection);
 756                                                         } catch (UriFormatException) {
 757                                                                 requestMessage.Content = new StringContent(string.Join('&', nameValueCollection.Select(static kv => $"{Uri.EscapeDataString(kv.Key)}={Uri.EscapeDataString(kv.Value)}")), null, "application/x-www-form-urlencoded");
 758                                                         }
 759
 760                                                         break;
 761                                                 case string text:
 762                                                         requestMessage.Content = new StringContent(text);
 763
 764                                                         break;
 765                                                 default:
 766                                                         requestMessage.Content = JsonContent.Create(data, options: JsonUtilities.DefaultJsonSerialierOptions);
 767
 768                                                         break;
 769                                         }
 770
 771                                         // Compress the request if caller specified it, so they know that the server supports it, and the content is not compressed yet
 772                                         if (requestOptions.HasFlag(ERequestOptions.CompressRequest) && (requestMessage.Content.Headers.ContentEncoding.Count == 0)) {
 773                                                 HttpContent originalContent = requestMessage.Content;
 774
 775                                                 requestMessage.Content = await WebBrowserUtilities.CreateCompressedHttpContent(originalContent).ConfigureAwait(false);
 776
 777                                                 if (data is not HttpContent) {
 778                                                         // We don't need to keep old HttpContent around anymore, help GC
 779                                                         originalContent.Dispose();
 780                                                 }
 781                                         }
 782                                 }
 783
 784                                 if (referer != null) {
 785                                         requestMessage.Headers.Referrer = referer;
 786                                 }
 787
 788                                 if (Debugging.IsUserDebugging) {
 789                                         ArchiLogger.LogGenericDebug($"{httpMethod} {request}");
 790                                 }
 791
 792                                 try {
 793                                         response = await HttpClient.SendAsync(requestMessage, httpCompletionOption, cancellationToken).ConfigureAwait(false);
 794                                 } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) {
 795                                         throw;
 796                                 } catch (Exception e) {
 797                                         ArchiLogger.LogGenericDebuggingException(e);
 798
 799                                         return null;
 800                                 } finally {
 801                                         if (data is HttpContent) {
 802                                                 // We reset the request content to null, as our http content will get disposed otherwise, and we still need it for subsequent calls, such as redirections or retries
 803                                                 requestMessage.Content = null;
 804                                         }
 805                                 }
 806                         }
 807
 808                         if (Debugging.IsUserDebugging) {
 809                                 ArchiLogger.LogGenericDebug($"{response.StatusCode} <- {httpMethod} {request}");
 810                         }
 811
 812                         if (response.IsSuccessStatusCode) {
 813                                 return response;
 814                         }
 815
 816                         // WARNING: We still have not disposed response by now, make sure to dispose it ASAP if we're not returning it!
 817                         if (response.StatusCode.IsRedirectionCode() && (maxRedirections > 0)) {
 818                                 if (requestOptions.HasFlag(ERequestOptions.ReturnRedirections)) {
 819                                         // User wants to handle it manually, that's alright
 820                                         return response;
 821                                 }
 822
 823                                 Uri? redirectUri = response.Headers.Location;
 824
 825                                 if (redirectUri == null) {
 826                                         ArchiLogger.LogNullError(redirectUri);
 827
 828                                         return null;
 829                                 }
 830
 831                                 if (redirectUri.IsAbsoluteUri) {
 832                                         switch (redirectUri.Scheme) {
 833                                                 case "http" or "https":
 834                                                         break;
 835                                                 case "steammobile":
 836                                                         // Those redirections are invalid, but we're aware of that and we have extra logic for them
 837                                                         return response;
 838                                                 default:
 839                                                         // We have no clue about those, but maybe HttpClient can handle them for us
 840                                                         ArchiLogger.LogGenericError(string.Format(CultureInfo.CurrentCulture, Strings.WarningUnknownValuePleaseReport, nameof(redirectUri.Scheme), redirectUri.Scheme));
 841
 842                                                         break;
 843                                         }
 844                                 } else {
 845                                         redirectUri = new Uri(request, redirectUri);
 846                                 }
 847
 848                                 switch (response.StatusCode) {
 849                                         case HttpStatusCode.MovedPermanently: // Per https://tools.ietf.org/html/rfc7231#section-6.4.2, a 301 redirect may be performed using a GET request
 850                                         case HttpStatusCode.Redirect: // Per https://tools.ietf.org/html/rfc7231#section-6.4.3, a 302 redirect may be performed using a GET request
 851                                         case HttpStatusCode.SeeOther: // Per https://tools.ietf.org/html/rfc7231#section-6.4.4, a 303 redirect should be performed using a GET request
 852                                                 if (httpMethod != HttpMethod.Head) {
 853                                                         httpMethod = HttpMethod.Get;
 854                                                 }
 855
 856                                                 // Data doesn't make any sense for a fetch request, clear it in case it's being used
 857                                                 data = null;
 858
 859                                                 break;
 860                                 }
 861
 862                                 response.Dispose();
 863
 864                                 // Per https://tools.ietf.org/html/rfc7231#section-7.1.2, a redirect location without a fragment should inherit the fragment from the original URI
 865                                 if (!string.IsNullOrEmpty(request.Fragment) && string.IsNullOrEmpty(redirectUri.Fragment)) {
 866                                         redirectUri = new UriBuilder(redirectUri) { Fragment = request.Fragment }.Uri;
 867                                 }
 868
 869                                 request = redirectUri;
 870                                 maxRedirections--;
 871
 872                                 continue;
 873                         }
 874
 875                         break;
 876                 }
 877
 878                 if (!Debugging.IsUserDebugging) {
 879                         ArchiLogger.LogGenericDebug($"{response.StatusCode} <- {httpMethod} {request}");
 880                 }
 881
 882                 if (response.StatusCode.IsClientErrorCode()) {
 883                         if (Debugging.IsUserDebugging) {
 884                                 ArchiLogger.LogGenericDebug(string.Format(CultureInfo.CurrentCulture, Strings.Content, await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false)));
 885                         }
 886
 887                         // Do not retry on client errors
 888                         return response;
 889                 }
 890
 891                 if (requestOptions.HasFlag(ERequestOptions.ReturnServerErrors) && response.StatusCode.IsServerErrorCode()) {
 892                         if (Debugging.IsUserDebugging) {
 893                                 ArchiLogger.LogGenericDebug(string.Format(CultureInfo.CurrentCulture, Strings.Content, await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false)));
 894                         }
 895
 896                         // Do not retry on server errors in this case
 897                         return response;
 898                 }
 899
 900                 using (response) {
 901                         if (Debugging.IsUserDebugging) {
 902                                 ArchiLogger.LogGenericDebug(string.Format(CultureInfo.CurrentCulture, Strings.Content, await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false)));
 903                         }
 904
 905                         return null;
 906                 }
 907         }
 908
 909         [Flags]
 910         public enum ERequestOptions : byte {
 911                 None = 0,
 912                 ReturnClientErrors = 1,
 913                 ReturnServerErrors = 2,
 914                 ReturnRedirections = 4,
 915                 AllowInvalidBodyOnSuccess = 8,
 916                 AllowInvalidBodyOnErrors = 16,
 917                 CompressRequest = 32
 918         }
 919 }