docs/HtmlAgilityPack/HtmlWeb.cs

   1 // HtmlAgilityPack V1.0 - Simon Mourier <simon underscore mourier at hotmail dot com>
   2 using System;
   3 using System.IO;
   4 using System.Net;
   5 using System.Text;
   6 using System.Xml;
   7 using System.Xml.Serialization;
   8 using System.Xml.Xsl;
   9 using Microsoft.Win32;
  10
  11 namespace HtmlAgilityPack
  12 {
  13     /// <summary>
  14     /// A utility class to get HTML document from HTTP.
  15     /// </summary>
  16     public class HtmlWeb
  17     {
  18         #region Delegates
  19
  20         /// <summary>
  21         /// Represents the method that will handle the PostResponse event.
  22         /// </summary>
  23         public delegate void PostResponseHandler(HttpWebRequest request, HttpWebResponse response);
  24
  25         /// <summary>
  26         /// Represents the method that will handle the PreHandleDocument event.
  27         /// </summary>
  28         public delegate void PreHandleDocumentHandler(HtmlDocument document);
  29
  30         /// <summary>
  31         /// Represents the method that will handle the PreRequest event.
  32         /// </summary>
  33         public delegate bool PreRequestHandler(HttpWebRequest request);
  34
  35         #endregion
  36
  37         #region Fields
  38
  39         private bool _autoDetectEncoding = true;
  40         private bool _cacheOnly;
  41
  42         private string _cachePath;
  43         private bool _fromCache;
  44         private int _requestDuration;
  45         private Uri _responseUri;
  46         private HttpStatusCode _statusCode = HttpStatusCode.OK;
  47         private int _streamBufferSize = 1024;
  48         private bool _useCookies;
  49         private bool _usingCache;
  50
  51         /// <summary>
  52         /// Occurs after an HTTP request has been executed.
  53         /// </summary>
  54         public PostResponseHandler PostResponse;
  55
  56         /// <summary>
  57         /// Occurs before an HTML document is handled.
  58         /// </summary>
  59         public PreHandleDocumentHandler PreHandleDocument;
  60
  61         /// <summary>
  62         /// Occurs before an HTTP request is executed.
  63         /// </summary>
  64         public PreRequestHandler PreRequest;
  65
  66         #endregion
  67
  68         #region Properties
  69
  70         /// <summary>
  71         /// Gets or Sets a value indicating if document encoding must be automatically detected.
  72         /// </summary>
  73         public bool AutoDetectEncoding
  74         {
  75             get { return _autoDetectEncoding; }
  76             set { _autoDetectEncoding = value; }
  77         }
  78
  79         /// <summary>
  80         /// Gets or Sets a value indicating whether to get document only from the cache.
  81         /// If this is set to true and document is not found in the cache, nothing will be loaded.
  82         /// </summary>
  83         public bool CacheOnly
  84         {
  85             get { return _cacheOnly; }
  86             set
  87             {
  88                 if ((value) && !UsingCache)
  89                 {
  90                     throw new HtmlWebException("Cache is not enabled. Set UsingCache to true first.");
  91                 }
  92                 _cacheOnly = value;
  93             }
  94         }
  95
  96         /// <summary>
  97         /// Gets or Sets the cache path. If null, no caching mechanism will be used.
  98         /// </summary>
  99         public string CachePath
 100         {
 101             get { return _cachePath; }
 102             set { _cachePath = value; }
 103         }
 104
 105         /// <summary>
 106         /// Gets a value indicating if the last document was retrieved from the cache.
 107         /// </summary>
 108         public bool FromCache
 109         {
 110             get { return _fromCache; }
 111         }
 112
 113         /// <summary>
 114         /// Gets the last request duration in milliseconds.
 115         /// </summary>
 116         public int RequestDuration
 117         {
 118             get { return _requestDuration; }
 119         }
 120
 121         /// <summary>
 122         /// Gets the URI of the Internet resource that actually responded to the request.
 123         /// </summary>
 124         public Uri ResponseUri
 125         {
 126             get { return _responseUri; }
 127         }
 128
 129         /// <summary>
 130         /// Gets the last request status.
 131         /// </summary>
 132         public HttpStatusCode StatusCode
 133         {
 134             get { return _statusCode; }
 135         }
 136
 137         /// <summary>
 138         /// Gets or Sets the size of the buffer used for memory operations.
 139         /// </summary>
 140         public int StreamBufferSize
 141         {
 142             get { return _streamBufferSize; }
 143             set
 144             {
 145                 if (_streamBufferSize <= 0)
 146                 {
 147                     throw new ArgumentException("Size must be greater than zero.");
 148                 }
 149                 _streamBufferSize = value;
 150             }
 151         }
 152
 153         /// <summary>
 154         /// Gets or Sets a value indicating if cookies will be stored.
 155         /// </summary>
 156         public bool UseCookies
 157         {
 158             get { return _useCookies; }
 159             set { _useCookies = value; }
 160         }
 161
 162         /// <summary>
 163         /// Gets or Sets a value indicating whether the caching mechanisms should be used or not.
 164         /// </summary>
 165         public bool UsingCache
 166         {
 167             get
 168             {
 169                 if (_cachePath == null)
 170                 {
 171                     return false;
 172                 }
 173                 return _usingCache;
 174             }
 175             set
 176             {
 177                 if ((value) && (_cachePath == null))
 178                 {
 179                     throw new HtmlWebException("You need to define a CachePath first.");
 180                 }
 181                 _usingCache = value;
 182             }
 183         }
 184
 185         #endregion
 186
 187         #region Public Methods
 188
 189         /// <summary>
 190         /// Gets the MIME content type for a given path extension.
 191         /// </summary>
 192         /// <param name="extension">The input path extension.</param>
 193         /// <param name="def">The default content type to return if any error occurs.</param>
 194         /// <returns>The path extension's MIME content type.</returns>
 195         public static string GetContentTypeForExtension(string extension, string def)
 196         {
 197             if (string.IsNullOrEmpty(extension))
 198             {
 199                 return def;
 200             }
 201             string contentType = "";
 202             try
 203             {
 204                 RegistryKey reg = Registry.ClassesRoot;
 205                 reg = reg.OpenSubKey(extension, false);
 206                 if (reg != null) contentType = (string)reg.GetValue("", def);
 207             }
 208             catch (Exception)
 209             {
 210                 contentType = def;
 211             }
 212             return contentType;
 213         }
 214
 215         /// <summary>
 216         /// Gets the path extension for a given MIME content type.
 217         /// </summary>
 218         /// <param name="contentType">The input MIME content type.</param>
 219         /// <param name="def">The default path extension to return if any error occurs.</param>
 220         /// <returns>The MIME content type's path extension.</returns>
 221         public static string GetExtensionForContentType(string contentType, string def)
 222         {
 223             if (string.IsNullOrEmpty(contentType))
 224             {
 225                 return def;
 226             }
 227             string ext = "";
 228             try
 229             {
 230                 RegistryKey reg = Registry.ClassesRoot;
 231                 reg = reg.OpenSubKey(@"MIME\Database\Content Type\" + contentType, false);
 232                 if (reg != null) ext = (string)reg.GetValue("Extension", def);
 233             }
 234             catch (Exception)
 235             {
 236                 ext = def;
 237             }
 238             return ext;
 239         }
 240
 241         /// <summary>
 242         /// Creates an instance of the given type from the specified Internet resource.
 243         /// </summary>
 244         /// <param name="url">The requested URL, such as "http://Myserver/Mypath/Myfile.asp".</param>
 245         /// <param name="type">The requested type.</param>
 246         /// <returns>An newly created instance.</returns>
 247         public object CreateInstance(string url, Type type)
 248         {
 249             return CreateInstance(url, null, null, type);
 250         }
 251
 252         /// <summary>
 253         /// Creates an instance of the given type from the specified Internet resource.
 254         /// </summary>
 255         /// <param name="htmlUrl">The requested URL, such as "http://Myserver/Mypath/Myfile.asp".</param>
 256         /// <param name="xsltUrl">The URL that specifies the XSLT stylesheet to load.</param>
 257         /// <param name="xsltArgs">An <see cref="XsltArgumentList"/> containing the namespace-qualified arguments used as input to the transform.</param>
 258         /// <param name="type">The requested type.</param>
 259         /// <returns>An newly created instance.</returns>
 260         public object CreateInstance(string htmlUrl, string xsltUrl, XsltArgumentList xsltArgs, Type type)
 261         {
 262             return CreateInstance(htmlUrl, xsltUrl, xsltArgs, type, null);
 263         }
 264
 265         /// <summary>
 266         /// Creates an instance of the given type from the specified Internet resource.
 267         /// </summary>
 268         /// <param name="htmlUrl">The requested URL, such as "http://Myserver/Mypath/Myfile.asp".</param>
 269         /// <param name="xsltUrl">The URL that specifies the XSLT stylesheet to load.</param>
 270         /// <param name="xsltArgs">An <see cref="XsltArgumentList"/> containing the namespace-qualified arguments used as input to the transform.</param>
 271         /// <param name="type">The requested type.</param>
 272         /// <param name="xmlPath">A file path where the temporary XML before transformation will be saved. Mostly used for debugging purposes.</param>
 273         /// <returns>An newly created instance.</returns>
 274         public object CreateInstance(string htmlUrl, string xsltUrl, XsltArgumentList xsltArgs, Type type,
 275                                      string xmlPath)
 276         {
 277             StringWriter sw = new StringWriter();
 278             XmlTextWriter writer = new XmlTextWriter(sw);
 279             if (xsltUrl == null)
 280             {
 281                 LoadHtmlAsXml(htmlUrl, writer);
 282             }
 283             else
 284             {
 285                 if (xmlPath == null)
 286                 {
 287                     LoadHtmlAsXml(htmlUrl, xsltUrl, xsltArgs, writer);
 288                 }
 289                 else
 290                 {
 291                     LoadHtmlAsXml(htmlUrl, xsltUrl, xsltArgs, writer, xmlPath);
 292                 }
 293             }
 294             writer.Flush();
 295             StringReader sr = new StringReader(sw.ToString());
 296             XmlTextReader reader = new XmlTextReader(sr);
 297             XmlSerializer serializer = new XmlSerializer(type);
 298             object o;
 299             try
 300             {
 301                 o = serializer.Deserialize(reader);
 302             }
 303             catch (InvalidOperationException ex)
 304             {
 305                 throw new Exception(ex + ", --- xml:" + sw);
 306             }
 307             return o;
 308         }
 309
 310         /// <summary>
 311         /// Gets an HTML document from an Internet resource and saves it to the specified file.
 312         /// </summary>
 313         /// <param name="url">The requested URL, such as "http://Myserver/Mypath/Myfile.asp".</param>
 314         /// <param name="path">The location of the file where you want to save the document.</param>
 315         public void Get(string url, string path)
 316         {
 317             Get(url, path, "GET");
 318         }
 319
 320         /// <summary>
 321         /// Gets an HTML document from an Internet resource and saves it to the specified file. - Proxy aware
 322         /// </summary>
 323         /// <param name="url">The requested URL, such as "http://Myserver/Mypath/Myfile.asp".</param>
 324         /// <param name="path">The location of the file where you want to save the document.</param>
 325         /// <param name="proxy"></param>
 326         /// <param name="credentials"></param>
 327         public void Get(string url, string path, WebProxy proxy, NetworkCredential credentials)
 328         {
 329             Get(url, path, proxy, credentials, "GET");
 330         }
 331
 332         /// <summary>
 333         /// Gets an HTML document from an Internet resource and saves it to the specified file.
 334         /// </summary>
 335         /// <param name="url">The requested URL, such as "http://Myserver/Mypath/Myfile.asp".</param>
 336         /// <param name="path">The location of the file where you want to save the document.</param>
 337         /// <param name="method">The HTTP method used to open the connection, such as GET, POST, PUT, or PROPFIND.</param>
 338         public void Get(string url, string path, string method)
 339         {
 340             Uri uri = new Uri(url);
 341             if ((uri.Scheme == Uri.UriSchemeHttps) ||
 342                 (uri.Scheme == Uri.UriSchemeHttp))
 343             {
 344                 Get(uri, method, path, null, null, null);
 345             }
 346             else
 347             {
 348                 throw new HtmlWebException("Unsupported uri scheme: '" + uri.Scheme + "'.");
 349             }
 350         }
 351
 352         /// <summary>
 353         /// Gets an HTML document from an Internet resource and saves it to the specified file.  Understands Proxies
 354         /// </summary>
 355         /// <param name="url">The requested URL, such as "http://Myserver/Mypath/Myfile.asp".</param>
 356         /// <param name="path">The location of the file where you want to save the document.</param>
 357         /// <param name="credentials"></param>
 358         /// <param name="method">The HTTP method used to open the connection, such as GET, POST, PUT, or PROPFIND.</param>
 359         /// <param name="proxy"></param>
 360         public void Get(string url, string path, WebProxy proxy, NetworkCredential credentials, string method)
 361         {
 362             Uri uri = new Uri(url);
 363             if ((uri.Scheme == Uri.UriSchemeHttps) ||
 364                 (uri.Scheme == Uri.UriSchemeHttp))
 365             {
 366                 Get(uri, method, path, null, proxy, credentials);
 367             }
 368             else
 369             {
 370                 throw new HtmlWebException("Unsupported uri scheme: '" + uri.Scheme + "'.");
 371             }
 372         }
 373
 374         /// <summary>
 375         /// Gets the cache file path for a specified url.
 376         /// </summary>
 377         /// <param name="uri">The url fo which to retrieve the cache path. May not be null.</param>
 378         /// <returns>The cache file path.</returns>
 379         public string GetCachePath(Uri uri)
 380         {
 381             if (uri == null)
 382             {
 383                 throw new ArgumentNullException("uri");
 384             }
 385             if (!UsingCache)
 386             {
 387                 throw new HtmlWebException("Cache is not enabled. Set UsingCache to true first.");
 388             }
 389             string cachePath;
 390             if (uri.AbsolutePath == "/")
 391             {
 392                 cachePath = Path.Combine(_cachePath, ".htm");
 393             }
 394             else
 395             {
 396                 cachePath = Path.Combine(_cachePath, (uri.Host + uri.AbsolutePath).Replace('/', '\\'));
 397             }
 398             return cachePath;
 399         }
 400
 401         /// <summary>
 402         /// Gets an HTML document from an Internet resource.
 403         /// </summary>
 404         /// <param name="url">The requested URL, such as "http://Myserver/Mypath/Myfile.asp".</param>
 405         /// <returns>A new HTML document.</returns>
 406         public HtmlDocument Load(string url)
 407         {
 408             return Load(url, "GET");
 409         }
 410
 411         /// <summary>
 412         /// Gets an HTML document from an Internet resource.
 413         /// </summary>
 414         /// <param name="url">The requested URL, such as "http://Myserver/Mypath/Myfile.asp".</param>
 415         /// <param name="proxyHost">Host to use for Proxy</param>
 416         /// <param name="proxyPort">Port the Proxy is on</param>
 417         /// <param name="userId">User Id for Authentication</param>
 418         /// <param name="password">Password for Authentication</param>
 419         /// <returns>A new HTML document.</returns>
 420         public HtmlDocument Load(string url, string proxyHost, int proxyPort, string userId, string password)
 421         {
 422             //Create my proxy
 423             WebProxy myProxy = new WebProxy(proxyHost, proxyPort);
 424             myProxy.BypassProxyOnLocal = true;
 425
 426             //Create my credentials
 427             NetworkCredential myCreds = null;
 428             if ((userId != null) && (password != null))
 429             {
 430                 myCreds = new NetworkCredential(userId, password);
 431                 CredentialCache credCache = new CredentialCache();
 432                 //Add the creds
 433                 credCache.Add(myProxy.Address, "Basic", myCreds);
 434                 credCache.Add(myProxy.Address, "Digest", myCreds);
 435             }
 436
 437             return Load(url, "GET", myProxy, myCreds);
 438         }
 439
 440         /// <summary>
 441         /// Loads an HTML document from an Internet resource.
 442         /// </summary>
 443         /// <param name="url">The requested URL, such as "http://Myserver/Mypath/Myfile.asp".</param>
 444         /// <param name="method">The HTTP method used to open the connection, such as GET, POST, PUT, or PROPFIND.</param>
 445         /// <returns>A new HTML document.</returns>
 446         public HtmlDocument Load(string url, string method)
 447         {
 448             Uri uri = new Uri(url);
 449             HtmlDocument doc;
 450             if ((uri.Scheme == Uri.UriSchemeHttps) ||
 451                 (uri.Scheme == Uri.UriSchemeHttp))
 452             {
 453                 doc = LoadUrl(uri, method, null, null);
 454             }
 455             else
 456             {
 457                 if (uri.Scheme == Uri.UriSchemeFile)
 458                 {
 459                     doc = new HtmlDocument();
 460                     doc.OptionAutoCloseOnEnd = false;
 461                     doc.OptionAutoCloseOnEnd = true;
 462                     doc.DetectEncodingAndLoad(url, _autoDetectEncoding);
 463                 }
 464                 else
 465                 {
 466                     throw new HtmlWebException("Unsupported uri scheme: '" + uri.Scheme + "'.");
 467                 }
 468             }
 469             if (PreHandleDocument != null)
 470             {
 471                 PreHandleDocument(doc);
 472             }
 473             return doc;
 474         }
 475
 476         /// <summary>
 477         /// Loads an HTML document from an Internet resource.
 478         /// </summary>
 479         /// <param name="url">The requested URL, such as "http://Myserver/Mypath/Myfile.asp".</param>
 480         /// <param name="method">The HTTP method used to open the connection, such as GET, POST, PUT, or PROPFIND.</param>
 481         /// <param name="proxy">Proxy to use with this request</param>
 482         /// <param name="credentials">Credentials to use when authenticating</param>
 483         /// <returns>A new HTML document.</returns>
 484         public HtmlDocument Load(string url, string method, WebProxy proxy, NetworkCredential credentials)
 485         {
 486             Uri uri = new Uri(url);
 487             HtmlDocument doc;
 488             if ((uri.Scheme == Uri.UriSchemeHttps) ||
 489                 (uri.Scheme == Uri.UriSchemeHttp))
 490             {
 491                 doc = LoadUrl(uri, method, proxy, credentials);
 492             }
 493             else
 494             {
 495                 if (uri.Scheme == Uri.UriSchemeFile)
 496                 {
 497                     doc = new HtmlDocument();
 498                     doc.OptionAutoCloseOnEnd = false;
 499                     doc.OptionAutoCloseOnEnd = true;
 500                     doc.DetectEncodingAndLoad(url, _autoDetectEncoding);
 501                 }
 502                 else
 503                 {
 504                     throw new HtmlWebException("Unsupported uri scheme: '" + uri.Scheme + "'.");
 505                 }
 506             }
 507             if (PreHandleDocument != null)
 508             {
 509                 PreHandleDocument(doc);
 510             }
 511             return doc;
 512         }
 513
 514         /// <summary>
 515         /// Loads an HTML document from an Internet resource and saves it to the specified XmlTextWriter.
 516         /// </summary>
 517         /// <param name="htmlUrl">The requested URL, such as "http://Myserver/Mypath/Myfile.asp".</param>
 518         /// <param name="writer">The XmlTextWriter to which you want to save.</param>
 519         public void LoadHtmlAsXml(string htmlUrl, XmlTextWriter writer)
 520         {
 521             HtmlDocument doc = Load(htmlUrl);
 522             doc.Save(writer);
 523         }
 524
 525         /// <summary>
 526         /// Loads an HTML document from an Internet resource and saves it to the specified XmlTextWriter, after an XSLT transformation.
 527         /// </summary>
 528         /// <param name="htmlUrl">The requested URL, such as "http://Myserver/Mypath/Myfile.asp".</param>
 529         /// <param name="xsltUrl">The URL that specifies the XSLT stylesheet to load.</param>
 530         /// <param name="xsltArgs">An XsltArgumentList containing the namespace-qualified arguments used as input to the transform.</param>
 531         /// <param name="writer">The XmlTextWriter to which you want to save.</param>
 532         public void LoadHtmlAsXml(string htmlUrl, string xsltUrl, XsltArgumentList xsltArgs, XmlTextWriter writer)
 533         {
 534             LoadHtmlAsXml(htmlUrl, xsltUrl, xsltArgs, writer, null);
 535         }
 536
 537         /// <summary>
 538         /// Loads an HTML document from an Internet resource and saves it to the specified XmlTextWriter, after an XSLT transformation.
 539         /// </summary>
 540         /// <param name="htmlUrl">The requested URL, such as "http://Myserver/Mypath/Myfile.asp". May not be null.</param>
 541         /// <param name="xsltUrl">The URL that specifies the XSLT stylesheet to load.</param>
 542         /// <param name="xsltArgs">An XsltArgumentList containing the namespace-qualified arguments used as input to the transform.</param>
 543         /// <param name="writer">The XmlTextWriter to which you want to save.</param>
 544         /// <param name="xmlPath">A file path where the temporary XML before transformation will be saved. Mostly used for debugging purposes.</param>
 545         public void LoadHtmlAsXml(string htmlUrl, string xsltUrl, XsltArgumentList xsltArgs, XmlTextWriter writer,
 546                                   string xmlPath)
 547         {
 548             if (htmlUrl == null)
 549             {
 550                 throw new ArgumentNullException("htmlUrl");
 551             }
 552
 553             HtmlDocument doc = Load(htmlUrl);
 554
 555             if (xmlPath != null)
 556             {
 557                 XmlTextWriter w = new XmlTextWriter(xmlPath, doc.Encoding);
 558                 doc.Save(w);
 559                 w.Close();
 560             }
 561             if (xsltArgs == null)
 562             {
 563                 xsltArgs = new XsltArgumentList();
 564             }
 565
 566             // add some useful variables to the xslt doc
 567             xsltArgs.AddParam("url", "", htmlUrl);
 568             xsltArgs.AddParam("requestDuration", "", RequestDuration);
 569             xsltArgs.AddParam("fromCache", "", FromCache);
 570
 571             XslCompiledTransform xslt = new XslCompiledTransform();
 572             xslt.Load(xsltUrl);
 573             xslt.Transform(doc, xsltArgs, writer);
 574         }
 575
 576         #endregion
 577
 578         #region Private Methods
 579
 580         private static void FilePreparePath(string target)
 581         {
 582             if (File.Exists(target))
 583             {
 584                 FileAttributes atts = File.GetAttributes(target);
 585                 File.SetAttributes(target, atts & ~FileAttributes.ReadOnly);
 586             }
 587             else
 588             {
 589                 string dir = Path.GetDirectoryName(target);
 590                 if (!Directory.Exists(dir))
 591                 {
 592                     Directory.CreateDirectory(dir);
 593                 }
 594             }
 595         }
 596
 597         private static DateTime RemoveMilliseconds(DateTime t)
 598         {
 599             return new DateTime(t.Year, t.Month, t.Day, t.Hour, t.Minute, t.Second, 0);
 600         }
 601
 602         // ReSharper disable UnusedMethodReturnValue.Local
 603         private static long SaveStream(Stream stream, string path, DateTime touchDate, int streamBufferSize)
 604         // ReSharper restore UnusedMethodReturnValue.Local
 605         {
 606             FilePreparePath(path);
 607             FileStream fs = new FileStream(path, FileMode.Create, FileAccess.Write);
 608             BinaryReader br = null;
 609             BinaryWriter bw = null;
 610             long len = 0;
 611             try
 612             {
 613                 br = new BinaryReader(stream);
 614                 bw = new BinaryWriter(fs);
 615
 616                 byte[] buffer;
 617                 do
 618                 {
 619                     buffer = br.ReadBytes(streamBufferSize);
 620                     len += buffer.Length;
 621                     if (buffer.Length > 0)
 622                     {
 623                         bw.Write(buffer);
 624                     }
 625                 } while (buffer.Length > 0);
 626             }
 627             finally
 628             {
 629                 if (br != null)
 630                 {
 631                     br.Close();
 632                 }
 633                 if (bw != null)
 634                 {
 635                     bw.Flush();
 636                     bw.Close();
 637                 }
 638                 if (fs != null)
 639                 {
 640                     fs.Close();
 641                 }
 642             }
 643             File.SetLastWriteTime(path, touchDate);
 644             return len;
 645         }
 646
 647         private HttpStatusCode Get(Uri uri, string method, string path, HtmlDocument doc, IWebProxy proxy,
 648                                    ICredentials creds)
 649         {
 650             string cachePath = null;
 651             HttpWebRequest req;
 652             bool oldFile = false;
 653
 654             req = WebRequest.Create(uri) as HttpWebRequest;
 655             req.Method = method;
 656
 657             if (proxy != null)
 658             {
 659                 if (creds != null)
 660                 {
 661                     proxy.Credentials = creds;
 662                     req.Credentials = creds;
 663                 }
 664                 else
 665                 {
 666                     proxy.Credentials = CredentialCache.DefaultCredentials;
 667                     req.Credentials = CredentialCache.DefaultCredentials;
 668                 }
 669                 req.Proxy = proxy;
 670             }
 671
 672             _fromCache = false;
 673             _requestDuration = 0;
 674             int tc = Environment.TickCount;
 675             if (UsingCache)
 676             {
 677                 cachePath = GetCachePath(req.RequestUri);
 678                 if (File.Exists(cachePath))
 679                 {
 680                     req.IfModifiedSince = File.GetLastAccessTime(cachePath);
 681                     oldFile = true;
 682                 }
 683             }
 684
 685             if (_cacheOnly)
 686             {
 687                 if (!File.Exists(cachePath))
 688                 {
 689                     throw new HtmlWebException("File was not found at cache path: '" + cachePath + "'");
 690                 }
 691
 692                 if (path != null)
 693                 {
 694                     IOLibrary.CopyAlways(cachePath, path);
 695                     // touch the file
 696                     File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
 697                 }
 698                 _fromCache = true;
 699                 return HttpStatusCode.NotModified;
 700             }
 701
 702             if (_useCookies)
 703             {
 704                 req.CookieContainer = new CookieContainer();
 705             }
 706
 707             if (PreRequest != null)
 708             {
 709                 // allow our user to change the request at will
 710                 if (!PreRequest(req))
 711                 {
 712                     return HttpStatusCode.ResetContent;
 713                 }
 714
 715                 // dump cookie
 716                 //                              if (_useCookies)
 717                 //                              {
 718                 //                                      foreach(Cookie cookie in req.CookieContainer.GetCookies(req.RequestUri))
 719                 //                                      {
 720                 //                                              HtmlLibrary.Trace("Cookie " + cookie.Name + "=" + cookie.Value + " path=" + cookie.Path + " domain=" + cookie.Domain);
 721                 //                                      }
 722                 //                              }
 723             }
 724
 725             HttpWebResponse resp;
 726
 727             try
 728             {
 729                 resp = req.GetResponse() as HttpWebResponse;
 730             }
 731             catch (WebException we)
 732             {
 733                 _requestDuration = Environment.TickCount - tc;
 734                 resp = (HttpWebResponse)we.Response;
 735                 if (resp == null)
 736                 {
 737                     if (oldFile)
 738                     {
 739                         if (path != null)
 740                         {
 741                             IOLibrary.CopyAlways(cachePath, path);
 742                             // touch the file
 743                             File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
 744                         }
 745                         return HttpStatusCode.NotModified;
 746                     }
 747                     throw;
 748                 }
 749             }
 750             catch (Exception)
 751             {
 752                 _requestDuration = Environment.TickCount - tc;
 753                 throw;
 754             }
 755
 756             // allow our user to get some info from the response
 757             if (PostResponse != null)
 758             {
 759                 PostResponse(req, resp);
 760             }
 761
 762             _requestDuration = Environment.TickCount - tc;
 763             _responseUri = resp.ResponseUri;
 764
 765             bool html = IsHtmlContent(resp.ContentType);
 766             Encoding respenc;
 767
 768             if ((resp.ContentEncoding != null) && (resp.ContentEncoding.Length > 0))
 769             {
 770                 respenc = Encoding.GetEncoding(resp.ContentEncoding);
 771             }
 772             else
 773             {
 774                 respenc = null;
 775             }
 776
 777             if (resp.StatusCode == HttpStatusCode.NotModified)
 778             {
 779                 if (UsingCache)
 780                 {
 781                     _fromCache = true;
 782                     if (path != null)
 783                     {
 784                         IOLibrary.CopyAlways(cachePath, path);
 785                         // touch the file
 786                         File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
 787                     }
 788                     return resp.StatusCode;
 789                 }
 790                 else
 791                 {
 792                     // this should *never* happen...
 793                     throw new HtmlWebException("Server has send a NotModifed code, without cache enabled.");
 794                 }
 795             }
 796             Stream s = resp.GetResponseStream();
 797             if (s != null)
 798             {
 799                 if (UsingCache)
 800                 {
 801                     // NOTE: LastModified does not contain milliseconds, so we remove them to the file
 802                     SaveStream(s, cachePath, RemoveMilliseconds(resp.LastModified), _streamBufferSize);
 803
 804                     // save headers
 805                     SaveCacheHeaders(req.RequestUri, resp);
 806
 807                     if (path != null)
 808                     {
 809                         // copy and touch the file
 810                         IOLibrary.CopyAlways(cachePath, path);
 811                         File.SetLastWriteTime(path, File.GetLastWriteTime(cachePath));
 812                     }
 813                 }
 814                 else
 815                 {
 816                     // try to work in-memory
 817                     if ((doc != null) && (html))
 818                     {
 819                         if (respenc != null)
 820                         {
 821                             doc.Load(s, respenc);
 822                         }
 823                         else
 824                         {
 825                             doc.Load(s, true);
 826                         }
 827                     }
 828                 }
 829                 resp.Close();
 830             }
 831             return resp.StatusCode;
 832         }
 833
 834         private string GetCacheHeader(Uri requestUri, string name, string def)
 835         {
 836             // note: some headers are collection (ex: www-authenticate)
 837             // we don't handle that here
 838             XmlDocument doc = new XmlDocument();
 839             doc.Load(GetCacheHeadersPath(requestUri));
 840             XmlNode node =
 841                 doc.SelectSingleNode("//h[translate(@n, 'abcdefghijklmnopqrstuvwxyz','ABCDEFGHIJKLMNOPQRSTUVWXYZ')='" +
 842                                      name.ToUpper() + "']");
 843             if (node == null)
 844             {
 845                 return def;
 846             }
 847             // attribute should exist
 848             return node.Attributes[name].Value;
 849         }
 850
 851         private string GetCacheHeadersPath(Uri uri)
 852         {
 853             //return Path.Combine(GetCachePath(uri), ".h.xml");
 854             return GetCachePath(uri) + ".h.xml";
 855         }
 856
 857         private bool IsCacheHtmlContent(string path)
 858         {
 859             string ct = GetContentTypeForExtension(Path.GetExtension(path), null);
 860             return IsHtmlContent(ct);
 861         }
 862
 863         private bool IsHtmlContent(string contentType)
 864         {
 865             return contentType.ToLower().StartsWith("text/html");
 866         }
 867
 868         private HtmlDocument LoadUrl(Uri uri, string method, WebProxy proxy, NetworkCredential creds)
 869         {
 870             HtmlDocument doc = new HtmlDocument();
 871             doc.OptionAutoCloseOnEnd = false;
 872             doc.OptionFixNestedTags = true;
 873             _statusCode = Get(uri, method, null, doc, proxy, creds);
 874             if (_statusCode == HttpStatusCode.NotModified)
 875             {
 876                 // read cached encoding
 877                 doc.DetectEncodingAndLoad(GetCachePath(uri));
 878             }
 879             return doc;
 880         }
 881
 882         private void SaveCacheHeaders(Uri requestUri, HttpWebResponse resp)
 883         {
 884             // we cache the original headers aside the cached document.
 885             string file = GetCacheHeadersPath(requestUri);
 886             XmlDocument doc = new XmlDocument();
 887             doc.LoadXml("<c></c>");
 888             XmlNode cache = doc.FirstChild;
 889             foreach (string header in resp.Headers)
 890             {
 891                 XmlNode entry = doc.CreateElement("h");
 892                 XmlAttribute att = doc.CreateAttribute("n");
 893                 att.Value = header;
 894                 entry.Attributes.Append(att);
 895
 896                 att = doc.CreateAttribute("v");
 897                 att.Value = resp.Headers[header];
 898                 entry.Attributes.Append(att);
 899
 900                 cache.AppendChild(entry);
 901             }
 902             doc.Save(file);
 903         }
 904
 905         #endregion
 906     }
 907 }