dom/webbrowserpersist/nsWebBrowserPersist.cpp

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this
   4  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #include "mozilla/ArrayUtils.h"
   7 #include "mozilla/TextUtils.h"
   8
   9 #include "nspr.h"
  10
  11 #include "nsIFileStreams.h"  // New Necko file streams
  12 #include <algorithm>
  13
  14 #include "nsNetCID.h"
  15 #include "nsNetUtil.h"
  16 #include "nsIClassOfService.h"
  17 #include "nsIInterfaceRequestorUtils.h"
  18 #include "nsILoadContext.h"
  19 #include "nsIPrivateBrowsingChannel.h"
  20 #include "nsComponentManagerUtils.h"
  21 #include "nsIStorageStream.h"
  22 #include "nsISeekableStream.h"
  23 #include "nsIHttpChannel.h"
  24 #include "nsIEncodedChannel.h"
  25 #include "nsIUploadChannel.h"
  26 #include "nsICacheInfoChannel.h"
  27 #include "nsIFileChannel.h"
  28 #include "nsEscape.h"
  29 #include "nsUnicharUtils.h"
  30 #include "nsIStringEnumerator.h"
  31 #include "nsContentCID.h"
  32 #include "nsStreamUtils.h"
  33
  34 #include "nsCExternalHandlerService.h"
  35
  36 #include "nsIURL.h"
  37 #include "nsIFileURL.h"
  38 #include "nsIWebProgressListener.h"
  39 #include "nsIAuthPrompt.h"
  40 #include "nsIPrompt.h"
  41 #include "nsIFormControl.h"
  42 #include "nsIThreadRetargetableRequest.h"
  43 #include "nsContentUtils.h"
  44
  45 #include "nsIStringBundle.h"
  46 #include "nsIProtocolHandler.h"
  47
  48 #include "nsWebBrowserPersist.h"
  49 #include "WebBrowserPersistLocalDocument.h"
  50
  51 #include "nsIContent.h"
  52 #include "nsIMIMEInfo.h"
  53 #include "mozilla/dom/HTMLInputElement.h"
  54 #include "mozilla/dom/HTMLSharedElement.h"
  55 #include "mozilla/net/CookieJarSettings.h"
  56 #include "mozilla/Mutex.h"
  57 #include "mozilla/Printf.h"
  58 #include "ReferrerInfo.h"
  59 #include "nsIURIMutator.h"
  60 #include "mozilla/WebBrowserPersistDocumentParent.h"
  61 #include "mozilla/dom/CanonicalBrowsingContext.h"
  62 #include "mozilla/dom/WindowGlobalParent.h"
  63 #include "mozilla/dom/ContentParent.h"
  64 #include "mozilla/dom/PContentParent.h"
  65 #include "mozilla/dom/BrowserParent.h"
  66 #include "nsIDocumentEncoder.h"
  67
  68 using namespace mozilla;
  69 using namespace mozilla::dom;
  70
  71 // Buffer file writes in 32kb chunks
  72 #define BUFFERED_OUTPUT_SIZE (1024 * 32)
  73
  74 struct nsWebBrowserPersist::WalkData {
  75   nsCOMPtr<nsIWebBrowserPersistDocument> mDocument;
  76   nsCOMPtr<nsIURI> mFile;
  77   nsCOMPtr<nsIURI> mDataPath;
  78 };
  79
  80 // Information about a DOM document
  81 struct nsWebBrowserPersist::DocData {
  82   nsCOMPtr<nsIURI> mBaseURI;
  83   nsCOMPtr<nsIWebBrowserPersistDocument> mDocument;
  84   nsCOMPtr<nsIURI> mFile;
  85   nsCString mCharset;
  86 };
  87
  88 // Information about a URI
  89 struct nsWebBrowserPersist::URIData {
  90   bool mNeedsPersisting;
  91   bool mSaved;
  92   bool mIsSubFrame;
  93   bool mDataPathIsRelative;
  94   bool mNeedsFixup;
  95   nsString mFilename;
  96   nsString mSubFrameExt;
  97   nsCOMPtr<nsIURI> mFile;
  98   nsCOMPtr<nsIURI> mDataPath;
  99   nsCOMPtr<nsIURI> mRelativeDocumentURI;
 100   nsCOMPtr<nsIPrincipal> mTriggeringPrincipal;
 101   nsCOMPtr<nsICookieJarSettings> mCookieJarSettings;
 102   nsContentPolicyType mContentPolicyType;
 103   nsCString mRelativePathToData;
 104   nsCString mCharset;
 105
 106   nsresult GetLocalURI(nsIURI* targetBaseURI, nsCString& aSpecOut);
 107 };
 108
 109 // Information about the output stream
 110 // Note that this data structure (and the map that nsWebBrowserPersist keeps,
 111 // where these are values) is used from two threads: the main thread,
 112 // and the background task thread.
 113 // The background thread only writes to mStream (from OnDataAvailable), and
 114 // this access is guarded using mStreamMutex. It reads the mFile member, which
 115 // is only written to on the main thread when the object is constructed and
 116 // from OnStartRequest (if mCalcFileExt), both guaranteed to happen before
 117 // OnDataAvailable is fired.
 118 // The main thread gets OnStartRequest, OnStopRequest, and progress sink events,
 119 // and accesses the other members.
 120 struct nsWebBrowserPersist::OutputData {
 121   nsCOMPtr<nsIURI> mFile;
 122   nsCOMPtr<nsIURI> mOriginalLocation;
 123   nsCOMPtr<nsIOutputStream> mStream;
 124   Mutex mStreamMutex;
 125   int64_t mSelfProgress;
 126   int64_t mSelfProgressMax;
 127   bool mCalcFileExt;
 128
 129   OutputData(nsIURI* aFile, nsIURI* aOriginalLocation, bool aCalcFileExt)
 130       : mFile(aFile),
 131         mOriginalLocation(aOriginalLocation),
 132         mStreamMutex("nsWebBrowserPersist::OutputData::mStreamMutex"),
 133         mSelfProgress(0),
 134         mSelfProgressMax(10000),
 135         mCalcFileExt(aCalcFileExt) {}
 136   ~OutputData() {
 137     // Gaining this lock in the destructor is pretty icky. It should be OK
 138     // because the only other place we lock the mutex is in OnDataAvailable,
 139     // which will never itself cause the OutputData instance to be
 140     // destroyed.
 141     MutexAutoLock lock(mStreamMutex);
 142     if (mStream) {
 143       mStream->Close();
 144     }
 145   }
 146 };
 147
 148 struct nsWebBrowserPersist::UploadData {
 149   nsCOMPtr<nsIURI> mFile;
 150   int64_t mSelfProgress;
 151   int64_t mSelfProgressMax;
 152
 153   explicit UploadData(nsIURI* aFile)
 154       : mFile(aFile), mSelfProgress(0), mSelfProgressMax(10000) {}
 155 };
 156
 157 struct nsWebBrowserPersist::CleanupData {
 158   nsCOMPtr<nsIFile> mFile;
 159   // Snapshot of what the file actually is at the time of creation so that if
 160   // it transmutes into something else later on it can be ignored. For example,
 161   // catch files that turn into dirs or vice versa.
 162   bool mIsDirectory;
 163 };
 164
 165 class nsWebBrowserPersist::OnWalk final
 166     : public nsIWebBrowserPersistResourceVisitor {
 167  public:
 168   OnWalk(nsWebBrowserPersist* aParent, nsIURI* aFile, nsIFile* aDataPath)
 169       : mParent(aParent),
 170         mFile(aFile),
 171         mDataPath(aDataPath),
 172         mPendingDocuments(1),
 173         mStatus(NS_OK) {}
 174
 175   NS_DECL_NSIWEBBROWSERPERSISTRESOURCEVISITOR
 176   NS_DECL_ISUPPORTS
 177  private:
 178   RefPtr<nsWebBrowserPersist> mParent;
 179   nsCOMPtr<nsIURI> mFile;
 180   nsCOMPtr<nsIFile> mDataPath;
 181
 182   uint32_t mPendingDocuments;
 183   nsresult mStatus;
 184
 185   virtual ~OnWalk() = default;
 186 };
 187
 188 NS_IMPL_ISUPPORTS(nsWebBrowserPersist::OnWalk,
 189                   nsIWebBrowserPersistResourceVisitor)
 190
 191 class nsWebBrowserPersist::OnRemoteWalk final
 192     : public nsIWebBrowserPersistDocumentReceiver {
 193  public:
 194   OnRemoteWalk(nsIWebBrowserPersistResourceVisitor* aVisitor,
 195                nsIWebBrowserPersistDocument* aDocument)
 196       : mVisitor(aVisitor), mDocument(aDocument) {}
 197
 198   NS_DECL_NSIWEBBROWSERPERSISTDOCUMENTRECEIVER
 199   NS_DECL_ISUPPORTS
 200  private:
 201   nsCOMPtr<nsIWebBrowserPersistResourceVisitor> mVisitor;
 202   nsCOMPtr<nsIWebBrowserPersistDocument> mDocument;
 203
 204   virtual ~OnRemoteWalk() = default;
 205 };
 206
 207 NS_IMPL_ISUPPORTS(nsWebBrowserPersist::OnRemoteWalk,
 208                   nsIWebBrowserPersistDocumentReceiver)
 209
 210 class nsWebBrowserPersist::OnWrite final
 211     : public nsIWebBrowserPersistWriteCompletion {
 212  public:
 213   OnWrite(nsWebBrowserPersist* aParent, nsIURI* aFile, nsIFile* aLocalFile)
 214       : mParent(aParent), mFile(aFile), mLocalFile(aLocalFile) {}
 215
 216   NS_DECL_NSIWEBBROWSERPERSISTWRITECOMPLETION
 217   NS_DECL_ISUPPORTS
 218  private:
 219   RefPtr<nsWebBrowserPersist> mParent;
 220   nsCOMPtr<nsIURI> mFile;
 221   nsCOMPtr<nsIFile> mLocalFile;
 222
 223   virtual ~OnWrite() = default;
 224 };
 225
 226 NS_IMPL_ISUPPORTS(nsWebBrowserPersist::OnWrite,
 227                   nsIWebBrowserPersistWriteCompletion)
 228
 229 class nsWebBrowserPersist::FlatURIMap final
 230     : public nsIWebBrowserPersistURIMap {
 231  public:
 232   explicit FlatURIMap(const nsACString& aTargetBase)
 233       : mTargetBase(aTargetBase) {}
 234
 235   void Add(const nsACString& aMapFrom, const nsACString& aMapTo) {
 236     mMapFrom.AppendElement(aMapFrom);
 237     mMapTo.AppendElement(aMapTo);
 238   }
 239
 240   NS_DECL_NSIWEBBROWSERPERSISTURIMAP
 241   NS_DECL_ISUPPORTS
 242
 243  private:
 244   nsTArray<nsCString> mMapFrom;
 245   nsTArray<nsCString> mMapTo;
 246   nsCString mTargetBase;
 247
 248   virtual ~FlatURIMap() = default;
 249 };
 250
 251 NS_IMPL_ISUPPORTS(nsWebBrowserPersist::FlatURIMap, nsIWebBrowserPersistURIMap)
 252
 253 NS_IMETHODIMP
 254 nsWebBrowserPersist::FlatURIMap::GetNumMappedURIs(uint32_t* aNum) {
 255   MOZ_ASSERT(mMapFrom.Length() == mMapTo.Length());
 256   *aNum = mMapTo.Length();
 257   return NS_OK;
 258 }
 259
 260 NS_IMETHODIMP
 261 nsWebBrowserPersist::FlatURIMap::GetTargetBaseURI(nsACString& aTargetBase) {
 262   aTargetBase = mTargetBase;
 263   return NS_OK;
 264 }
 265
 266 NS_IMETHODIMP
 267 nsWebBrowserPersist::FlatURIMap::GetURIMapping(uint32_t aIndex,
 268                                                nsACString& aMapFrom,
 269                                                nsACString& aMapTo) {
 270   MOZ_ASSERT(mMapFrom.Length() == mMapTo.Length());
 271   if (aIndex >= mMapTo.Length()) {
 272     return NS_ERROR_INVALID_ARG;
 273   }
 274   aMapFrom = mMapFrom[aIndex];
 275   aMapTo = mMapTo[aIndex];
 276   return NS_OK;
 277 }
 278
 279 // Maximum file length constant. The max file name length is
 280 // volume / server dependent but it is difficult to obtain
 281 // that information. Instead this constant is a reasonable value that
 282 // modern systems should able to cope with.
 283 const uint32_t kDefaultMaxFilenameLength = 64;
 284
 285 // Default flags for persistence
 286 const uint32_t kDefaultPersistFlags =
 287     nsIWebBrowserPersist::PERSIST_FLAGS_NO_CONVERSION |
 288     nsIWebBrowserPersist::PERSIST_FLAGS_REPLACE_EXISTING_FILES;
 289
 290 // String bundle where error messages come from
 291 const char* kWebBrowserPersistStringBundle =
 292     "chrome://global/locale/nsWebBrowserPersist.properties";
 293
 294 nsWebBrowserPersist::nsWebBrowserPersist()
 295     : mCurrentDataPathIsRelative(false),
 296       mCurrentThingsToPersist(0),
 297       mOutputMapMutex("nsWebBrowserPersist::mOutputMapMutex"),
 298       mFirstAndOnlyUse(true),
 299       mSavingDocument(false),
 300       mCancel(false),
 301       mEndCalled(false),
 302       mCompleted(false),
 303       mStartSaving(false),
 304       mReplaceExisting(true),
 305       mSerializingOutput(false),
 306       mIsPrivate(false),
 307       mPersistFlags(kDefaultPersistFlags),
 308       mPersistResult(NS_OK),
 309       mTotalCurrentProgress(0),
 310       mTotalMaxProgress(0),
 311       mWrapColumn(72),
 312       mEncodingFlags(0) {}
 313
 314 nsWebBrowserPersist::~nsWebBrowserPersist() { Cleanup(); }
 315
 316 //*****************************************************************************
 317 // nsWebBrowserPersist::nsISupports
 318 //*****************************************************************************
 319
 320 NS_IMPL_ADDREF(nsWebBrowserPersist)
 321 NS_IMPL_RELEASE(nsWebBrowserPersist)
 322
 323 NS_INTERFACE_MAP_BEGIN(nsWebBrowserPersist)
 324   NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsISupports, nsIWebBrowserPersist)
 325   NS_INTERFACE_MAP_ENTRY(nsIWebBrowserPersist)
 326   NS_INTERFACE_MAP_ENTRY(nsICancelable)
 327   NS_INTERFACE_MAP_ENTRY(nsIInterfaceRequestor)
 328   NS_INTERFACE_MAP_ENTRY(nsISupportsWeakReference)
 329   NS_INTERFACE_MAP_ENTRY(nsIStreamListener)
 330   NS_INTERFACE_MAP_ENTRY(nsIThreadRetargetableStreamListener)
 331   NS_INTERFACE_MAP_ENTRY(nsIRequestObserver)
 332   NS_INTERFACE_MAP_ENTRY(nsIProgressEventSink)
 333 NS_INTERFACE_MAP_END
 334
 335 //*****************************************************************************
 336 // nsWebBrowserPersist::nsIInterfaceRequestor
 337 //*****************************************************************************
 338
 339 NS_IMETHODIMP nsWebBrowserPersist::GetInterface(const nsIID& aIID,
 340                                                 void** aIFace) {
 341   NS_ENSURE_ARG_POINTER(aIFace);
 342
 343   *aIFace = nullptr;
 344
 345   nsresult rv = QueryInterface(aIID, aIFace);
 346   if (NS_SUCCEEDED(rv)) {
 347     return rv;
 348   }
 349
 350   if (mProgressListener && (aIID.Equals(NS_GET_IID(nsIAuthPrompt)) ||
 351                             aIID.Equals(NS_GET_IID(nsIPrompt)))) {
 352     mProgressListener->QueryInterface(aIID, aIFace);
 353     if (*aIFace) return NS_OK;
 354   }
 355
 356   nsCOMPtr<nsIInterfaceRequestor> req = do_QueryInterface(mProgressListener);
 357   if (req) {
 358     return req->GetInterface(aIID, aIFace);
 359   }
 360
 361   return NS_ERROR_NO_INTERFACE;
 362 }
 363
 364 //*****************************************************************************
 365 // nsWebBrowserPersist::nsIWebBrowserPersist
 366 //*****************************************************************************
 367
 368 NS_IMETHODIMP nsWebBrowserPersist::GetPersistFlags(uint32_t* aPersistFlags) {
 369   NS_ENSURE_ARG_POINTER(aPersistFlags);
 370   *aPersistFlags = mPersistFlags;
 371   return NS_OK;
 372 }
 373 NS_IMETHODIMP nsWebBrowserPersist::SetPersistFlags(uint32_t aPersistFlags) {
 374   mPersistFlags = aPersistFlags;
 375   mReplaceExisting = (mPersistFlags & PERSIST_FLAGS_REPLACE_EXISTING_FILES);
 376   mSerializingOutput = (mPersistFlags & PERSIST_FLAGS_SERIALIZE_OUTPUT);
 377   return NS_OK;
 378 }
 379
 380 NS_IMETHODIMP nsWebBrowserPersist::GetCurrentState(uint32_t* aCurrentState) {
 381   NS_ENSURE_ARG_POINTER(aCurrentState);
 382   if (mCompleted) {
 383     *aCurrentState = PERSIST_STATE_FINISHED;
 384   } else if (mFirstAndOnlyUse) {
 385     *aCurrentState = PERSIST_STATE_SAVING;
 386   } else {
 387     *aCurrentState = PERSIST_STATE_READY;
 388   }
 389   return NS_OK;
 390 }
 391
 392 NS_IMETHODIMP nsWebBrowserPersist::GetResult(nsresult* aResult) {
 393   NS_ENSURE_ARG_POINTER(aResult);
 394   *aResult = mPersistResult;
 395   return NS_OK;
 396 }
 397
 398 NS_IMETHODIMP nsWebBrowserPersist::GetProgressListener(
 399     nsIWebProgressListener** aProgressListener) {
 400   NS_ENSURE_ARG_POINTER(aProgressListener);
 401   *aProgressListener = mProgressListener;
 402   NS_IF_ADDREF(*aProgressListener);
 403   return NS_OK;
 404 }
 405
 406 NS_IMETHODIMP nsWebBrowserPersist::SetProgressListener(
 407     nsIWebProgressListener* aProgressListener) {
 408   mProgressListener = aProgressListener;
 409   mProgressListener2 = do_QueryInterface(aProgressListener);
 410   mEventSink = do_GetInterface(aProgressListener);
 411   return NS_OK;
 412 }
 413
 414 NS_IMETHODIMP nsWebBrowserPersist::SaveURI(
 415     nsIURI* aURI, nsIPrincipal* aPrincipal, uint32_t aCacheKey,
 416     nsIReferrerInfo* aReferrerInfo, nsICookieJarSettings* aCookieJarSettings,
 417     nsIInputStream* aPostData, const char* aExtraHeaders, nsISupports* aFile,
 418     nsContentPolicyType aContentPolicyType, nsILoadContext* aPrivacyContext) {
 419   bool isPrivate = aPrivacyContext && aPrivacyContext->UsePrivateBrowsing();
 420   return SavePrivacyAwareURI(aURI, aPrincipal, aCacheKey, aReferrerInfo,
 421                              aCookieJarSettings, aPostData, aExtraHeaders,
 422                              aFile, aContentPolicyType, isPrivate);
 423 }
 424
 425 NS_IMETHODIMP nsWebBrowserPersist::SavePrivacyAwareURI(
 426     nsIURI* aURI, nsIPrincipal* aPrincipal, uint32_t aCacheKey,
 427     nsIReferrerInfo* aReferrerInfo, nsICookieJarSettings* aCookieJarSettings,
 428     nsIInputStream* aPostData, const char* aExtraHeaders, nsISupports* aFile,
 429     nsContentPolicyType aContentPolicy, bool aIsPrivate) {
 430   NS_ENSURE_TRUE(mFirstAndOnlyUse, NS_ERROR_FAILURE);
 431   mFirstAndOnlyUse = false;  // Stop people from reusing this object!
 432
 433   nsCOMPtr<nsIURI> fileAsURI;
 434   nsresult rv;
 435   rv = GetValidURIFromObject(aFile, getter_AddRefs(fileAsURI));
 436   NS_ENSURE_SUCCESS(rv, NS_ERROR_INVALID_ARG);
 437
 438   // SaveURI doesn't like broken uris.
 439   mPersistFlags |= PERSIST_FLAGS_FAIL_ON_BROKEN_LINKS;
 440   rv = SaveURIInternal(aURI, aPrincipal, aContentPolicy, aCacheKey,
 441                        aReferrerInfo, aCookieJarSettings, aPostData,
 442                        aExtraHeaders, fileAsURI, false, aIsPrivate);
 443   return NS_FAILED(rv) ? rv : NS_OK;
 444 }
 445
 446 NS_IMETHODIMP nsWebBrowserPersist::SaveChannel(nsIChannel* aChannel,
 447                                                nsISupports* aFile) {
 448   NS_ENSURE_TRUE(mFirstAndOnlyUse, NS_ERROR_FAILURE);
 449   mFirstAndOnlyUse = false;  // Stop people from reusing this object!
 450
 451   nsCOMPtr<nsIURI> fileAsURI;
 452   nsresult rv;
 453   rv = GetValidURIFromObject(aFile, getter_AddRefs(fileAsURI));
 454   NS_ENSURE_SUCCESS(rv, NS_ERROR_INVALID_ARG);
 455
 456   rv = aChannel->GetURI(getter_AddRefs(mURI));
 457   NS_ENSURE_SUCCESS(rv, rv);
 458
 459   // SaveURI doesn't like broken uris.
 460   mPersistFlags |= PERSIST_FLAGS_FAIL_ON_BROKEN_LINKS;
 461   rv = SaveChannelInternal(aChannel, fileAsURI, false);
 462   return NS_FAILED(rv) ? rv : NS_OK;
 463 }
 464
 465 NS_IMETHODIMP nsWebBrowserPersist::SaveDocument(nsISupports* aDocument,
 466                                                 nsISupports* aFile,
 467                                                 nsISupports* aDataPath,
 468                                                 const char* aOutputContentType,
 469                                                 uint32_t aEncodingFlags,
 470                                                 uint32_t aWrapColumn) {
 471   NS_ENSURE_TRUE(mFirstAndOnlyUse, NS_ERROR_FAILURE);
 472   mFirstAndOnlyUse = false;  // Stop people from reusing this object!
 473
 474   // We need a STATE_IS_NETWORK start/stop pair to bracket the
 475   // notification callbacks.  For a whole document we generate those
 476   // here and in EndDownload(), but for the single-request methods
 477   // that's done in On{Start,Stop}Request instead.
 478   mSavingDocument = true;
 479
 480   NS_ENSURE_ARG_POINTER(aDocument);
 481   NS_ENSURE_ARG_POINTER(aFile);
 482
 483   nsCOMPtr<nsIURI> fileAsURI;
 484   nsCOMPtr<nsIURI> datapathAsURI;
 485   nsresult rv;
 486
 487   rv = GetValidURIFromObject(aFile, getter_AddRefs(fileAsURI));
 488   NS_ENSURE_SUCCESS(rv, NS_ERROR_INVALID_ARG);
 489   if (aDataPath) {
 490     rv = GetValidURIFromObject(aDataPath, getter_AddRefs(datapathAsURI));
 491     NS_ENSURE_SUCCESS(rv, NS_ERROR_INVALID_ARG);
 492   }
 493
 494   mWrapColumn = aWrapColumn;
 495   mEncodingFlags = aEncodingFlags;
 496
 497   if (aOutputContentType) {
 498     mContentType.AssignASCII(aOutputContentType);
 499   }
 500
 501   // State start notification
 502   if (mProgressListener) {
 503     mProgressListener->OnStateChange(
 504         nullptr, nullptr,
 505         nsIWebProgressListener::STATE_START |
 506             nsIWebProgressListener::STATE_IS_NETWORK,
 507         NS_OK);
 508   }
 509
 510   nsCOMPtr<nsIWebBrowserPersistDocument> doc = do_QueryInterface(aDocument);
 511   if (!doc) {
 512     nsCOMPtr<Document> localDoc = do_QueryInterface(aDocument);
 513     if (localDoc) {
 514       doc = new mozilla::WebBrowserPersistLocalDocument(localDoc);
 515     } else {
 516       rv = NS_ERROR_NO_INTERFACE;
 517     }
 518   }
 519
 520   bool closed = false;
 521   if (doc && NS_SUCCEEDED(doc->GetIsClosed(&closed)) && !closed) {
 522     rv = SaveDocumentInternal(doc, fileAsURI, datapathAsURI);
 523   }
 524
 525   if (NS_FAILED(rv) || closed) {
 526     SendErrorStatusChange(true, rv, nullptr, mURI);
 527     EndDownload(rv);
 528   }
 529   return rv;
 530 }
 531
 532 NS_IMETHODIMP nsWebBrowserPersist::Cancel(nsresult aReason) {
 533   // No point cancelling if we're already complete.
 534   if (mEndCalled) {
 535     return NS_OK;
 536   }
 537   mCancel = true;
 538   EndDownload(aReason);
 539   return NS_OK;
 540 }
 541
 542 NS_IMETHODIMP nsWebBrowserPersist::CancelSave() {
 543   return Cancel(NS_BINDING_ABORTED);
 544 }
 545
 546 nsresult nsWebBrowserPersist::StartUpload(nsIStorageStream* storStream,
 547                                           nsIURI* aDestinationURI,
 548                                           const nsACString& aContentType) {
 549   // setup the upload channel if the destination is not local
 550   nsCOMPtr<nsIInputStream> inputstream;
 551   nsresult rv = storStream->NewInputStream(0, getter_AddRefs(inputstream));
 552   NS_ENSURE_TRUE(inputstream, NS_ERROR_FAILURE);
 553   NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
 554   return StartUpload(inputstream, aDestinationURI, aContentType);
 555 }
 556
 557 nsresult nsWebBrowserPersist::StartUpload(nsIInputStream* aInputStream,
 558                                           nsIURI* aDestinationURI,
 559                                           const nsACString& aContentType) {
 560   nsCOMPtr<nsIChannel> destChannel;
 561   CreateChannelFromURI(aDestinationURI, getter_AddRefs(destChannel));
 562   nsCOMPtr<nsIUploadChannel> uploadChannel(do_QueryInterface(destChannel));
 563   NS_ENSURE_TRUE(uploadChannel, NS_ERROR_FAILURE);
 564
 565   // Set the upload stream
 566   // NOTE: ALL data must be available in "inputstream"
 567   nsresult rv = uploadChannel->SetUploadStream(aInputStream, aContentType, -1);
 568   NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
 569   rv = destChannel->AsyncOpen(this);
 570   NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
 571
 572   // add this to the upload list
 573   nsCOMPtr<nsISupports> keyPtr = do_QueryInterface(destChannel);
 574   mUploadList.InsertOrUpdate(keyPtr, MakeUnique<UploadData>(aDestinationURI));
 575
 576   return NS_OK;
 577 }
 578
 579 void nsWebBrowserPersist::SerializeNextFile() {
 580   nsresult rv = NS_OK;
 581   MOZ_ASSERT(mWalkStack.Length() == 0);
 582
 583   // First, handle gathered URIs.
 584   // This is potentially O(n^2), when taking into account the
 585   // number of times this method is called.  If it becomes a
 586   // bottleneck, the count of not-yet-persisted URIs could be
 587   // maintained separately, and we can skip iterating mURIMap if there are none.
 588
 589   // Persist each file in the uri map. The document(s)
 590   // will be saved after the last one of these is saved.
 591   for (const auto& entry : mURIMap) {
 592     URIData* data = entry.GetWeak();
 593
 594     if (!data->mNeedsPersisting || data->mSaved) {
 595       continue;
 596     }
 597
 598     // Create a URI from the key.
 599     nsCOMPtr<nsIURI> uri;
 600     rv = NS_NewURI(getter_AddRefs(uri), entry.GetKey(), data->mCharset.get());
 601     if (NS_WARN_IF(NS_FAILED(rv))) {
 602       break;
 603     }
 604
 605     // Make a URI to save the data to.
 606     nsCOMPtr<nsIURI> fileAsURI = data->mDataPath;
 607     rv = AppendPathToURI(fileAsURI, data->mFilename, fileAsURI);
 608     if (NS_WARN_IF(NS_FAILED(rv))) {
 609       break;
 610     }
 611
 612     rv = SaveURIInternal(uri, data->mTriggeringPrincipal,
 613                          data->mContentPolicyType, 0, nullptr,
 614                          data->mCookieJarSettings, nullptr, nullptr, fileAsURI,
 615                          true, mIsPrivate);
 616     // If SaveURIInternal fails, then it will have called EndDownload,
 617     // which means that |data| is no longer valid memory. We MUST bail.
 618     if (NS_WARN_IF(NS_FAILED(rv))) {
 619       break;
 620     }
 621
 622     if (rv == NS_OK) {
 623       // URIData.mFile will be updated to point to the correct
 624       // URI object when it is fixed up with the right file extension
 625       // in OnStartRequest
 626       data->mFile = fileAsURI;
 627       data->mSaved = true;
 628     } else {
 629       data->mNeedsFixup = false;
 630     }
 631
 632     if (mSerializingOutput) {
 633       break;
 634     }
 635   }
 636
 637   // If there are downloads happening, wait until they're done; the
 638   // OnStopRequest handler will call this method again.
 639   if (mOutputMap.Count() > 0) {
 640     return;
 641   }
 642
 643   // If serializing, also wait until last upload is done.
 644   if (mSerializingOutput && mUploadList.Count() > 0) {
 645     return;
 646   }
 647
 648   // If there are also no more documents, then we're done.
 649   if (mDocList.Length() == 0) {
 650     // ...or not quite done, if there are still uploads.
 651     if (mUploadList.Count() > 0) {
 652       return;
 653     }
 654     // Finish and clean things up.  Defer this because the caller
 655     // may have been expecting to use the listeners that that
 656     // method will clear.
 657     NS_DispatchToCurrentThread(
 658         NewRunnableMethod("nsWebBrowserPersist::FinishDownload", this,
 659                           &nsWebBrowserPersist::FinishDownload));
 660     return;
 661   }
 662
 663   // There are no URIs to save, so just save the next document.
 664   mStartSaving = true;
 665   mozilla::UniquePtr<DocData> docData(mDocList.ElementAt(0));
 666   mDocList.RemoveElementAt(0);  // O(n^2) but probably doesn't matter.
 667   MOZ_ASSERT(docData);
 668   if (!docData) {
 669     EndDownload(NS_ERROR_FAILURE);
 670     return;
 671   }
 672
 673   mCurrentBaseURI = docData->mBaseURI;
 674   mCurrentCharset = docData->mCharset;
 675   mTargetBaseURI = docData->mFile;
 676
 677   // Save the document, fixing it up with the new URIs as we do
 678
 679   nsAutoCString targetBaseSpec;
 680   if (mTargetBaseURI) {
 681     rv = mTargetBaseURI->GetSpec(targetBaseSpec);
 682     if (NS_FAILED(rv)) {
 683       SendErrorStatusChange(true, rv, nullptr, nullptr);
 684       EndDownload(rv);
 685       return;
 686     }
 687   }
 688
 689   // mFlatURIMap must be rebuilt each time through SerializeNextFile, as
 690   // mTargetBaseURI is used to create the relative URLs and will be different
 691   // with each serialized document.
 692   RefPtr<FlatURIMap> flatMap = new FlatURIMap(targetBaseSpec);
 693   for (const auto& uriEntry : mURIMap) {
 694     nsAutoCString mapTo;
 695     nsresult rv = uriEntry.GetWeak()->GetLocalURI(mTargetBaseURI, mapTo);
 696     if (NS_SUCCEEDED(rv) || !mapTo.IsVoid()) {
 697       flatMap->Add(uriEntry.GetKey(), mapTo);
 698     }
 699   }
 700   mFlatURIMap = std::move(flatMap);
 701
 702   nsCOMPtr<nsIFile> localFile;
 703   GetLocalFileFromURI(docData->mFile, getter_AddRefs(localFile));
 704   if (localFile) {
 705     // if we're not replacing an existing file but the file
 706     // exists, something is wrong
 707     bool fileExists = false;
 708     rv = localFile->Exists(&fileExists);
 709     if (NS_SUCCEEDED(rv) && !mReplaceExisting && fileExists) {
 710       rv = NS_ERROR_FILE_ALREADY_EXISTS;
 711     }
 712     if (NS_FAILED(rv)) {
 713       SendErrorStatusChange(false, rv, nullptr, docData->mFile);
 714       EndDownload(rv);
 715       return;
 716     }
 717   }
 718   nsCOMPtr<nsIOutputStream> outputStream;
 719   rv = MakeOutputStream(docData->mFile, getter_AddRefs(outputStream));
 720   if (NS_SUCCEEDED(rv) && !outputStream) {
 721     rv = NS_ERROR_FAILURE;
 722   }
 723   if (NS_FAILED(rv)) {
 724     SendErrorStatusChange(false, rv, nullptr, docData->mFile);
 725     EndDownload(rv);
 726     return;
 727   }
 728
 729   RefPtr<OnWrite> finish = new OnWrite(this, docData->mFile, localFile);
 730   rv = docData->mDocument->WriteContent(outputStream, mFlatURIMap,
 731                                         NS_ConvertUTF16toUTF8(mContentType),
 732                                         mEncodingFlags, mWrapColumn, finish);
 733   if (NS_FAILED(rv)) {
 734     SendErrorStatusChange(false, rv, nullptr, docData->mFile);
 735     EndDownload(rv);
 736   }
 737 }
 738
 739 NS_IMETHODIMP
 740 nsWebBrowserPersist::OnWrite::OnFinish(nsIWebBrowserPersistDocument* aDoc,
 741                                        nsIOutputStream* aStream,
 742                                        const nsACString& aContentType,
 743                                        nsresult aStatus) {
 744   nsresult rv = aStatus;
 745
 746   if (NS_FAILED(rv)) {
 747     mParent->SendErrorStatusChange(false, rv, nullptr, mFile);
 748     mParent->EndDownload(rv);
 749     return NS_OK;
 750   }
 751   if (!mLocalFile) {
 752     nsCOMPtr<nsIStorageStream> storStream(do_QueryInterface(aStream));
 753     if (storStream) {
 754       aStream->Close();
 755       rv = mParent->StartUpload(storStream, mFile, aContentType);
 756       if (NS_FAILED(rv)) {
 757         mParent->SendErrorStatusChange(false, rv, nullptr, mFile);
 758         mParent->EndDownload(rv);
 759       }
 760       // Either we failed and we're done, or we're uploading and
 761       // the OnStopRequest callback is responsible for the next
 762       // SerializeNextFile().
 763       return NS_OK;
 764     }
 765   }
 766   NS_DispatchToCurrentThread(
 767       NewRunnableMethod("nsWebBrowserPersist::SerializeNextFile", mParent,
 768                         &nsWebBrowserPersist::SerializeNextFile));
 769   return NS_OK;
 770 }
 771
 772 //*****************************************************************************
 773 // nsWebBrowserPersist::nsIRequestObserver
 774 //*****************************************************************************
 775
 776 NS_IMETHODIMP nsWebBrowserPersist::OnStartRequest(nsIRequest* request) {
 777   if (mProgressListener) {
 778     uint32_t stateFlags = nsIWebProgressListener::STATE_START |
 779                           nsIWebProgressListener::STATE_IS_REQUEST;
 780     if (!mSavingDocument) {
 781       stateFlags |= nsIWebProgressListener::STATE_IS_NETWORK;
 782     }
 783     mProgressListener->OnStateChange(nullptr, request, stateFlags, NS_OK);
 784   }
 785
 786   nsCOMPtr<nsIChannel> channel = do_QueryInterface(request);
 787   NS_ENSURE_TRUE(channel, NS_ERROR_FAILURE);
 788
 789   nsCOMPtr<nsISupports> keyPtr = do_QueryInterface(request);
 790   OutputData* data = mOutputMap.Get(keyPtr);
 791
 792   // NOTE: This code uses the channel as a hash key so it will not
 793   //       recognize redirected channels because the key is not the same.
 794   //       When that happens we remove and add the data entry to use the
 795   //       new channel as the hash key.
 796   if (!data) {
 797     UploadData* upData = mUploadList.Get(keyPtr);
 798     if (!upData) {
 799       // Redirect? Try and fixup the output table
 800       nsresult rv = FixRedirectedChannelEntry(channel);
 801       NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
 802
 803       // Should be able to find the data after fixup unless redirects
 804       // are disabled.
 805       data = mOutputMap.Get(keyPtr);
 806       if (!data) {
 807         return NS_ERROR_FAILURE;
 808       }
 809     }
 810   }
 811
 812   if (data && data->mFile) {
 813     nsCOMPtr<nsIThreadRetargetableRequest> r = do_QueryInterface(request);
 814     // Determine if we're uploading. Only use OMT onDataAvailable if not.
 815     nsCOMPtr<nsIFile> localFile;
 816     GetLocalFileFromURI(data->mFile, getter_AddRefs(localFile));
 817     if (r && localFile) {
 818       if (!mBackgroundQueue) {
 819         NS_CreateBackgroundTaskQueue("WebBrowserPersist",
 820                                      getter_AddRefs(mBackgroundQueue));
 821       }
 822       if (mBackgroundQueue) {
 823         r->RetargetDeliveryTo(mBackgroundQueue);
 824       }
 825     }
 826
 827     // If PERSIST_FLAGS_AUTODETECT_APPLY_CONVERSION is set in mPersistFlags,
 828     // try to determine whether this channel needs to apply Content-Encoding
 829     // conversions.
 830     NS_ASSERTION(
 831         !((mPersistFlags & PERSIST_FLAGS_AUTODETECT_APPLY_CONVERSION) &&
 832           (mPersistFlags & PERSIST_FLAGS_NO_CONVERSION)),
 833         "Conflict in persist flags: both AUTODETECT and NO_CONVERSION set");
 834     if (mPersistFlags & PERSIST_FLAGS_AUTODETECT_APPLY_CONVERSION)
 835       SetApplyConversionIfNeeded(channel);
 836
 837     if (data->mCalcFileExt &&
 838         !(mPersistFlags & PERSIST_FLAGS_DONT_CHANGE_FILENAMES)) {
 839       nsCOMPtr<nsIURI> uriWithExt;
 840       // this is the first point at which the server can tell us the mimetype
 841       nsresult rv = CalculateAndAppendFileExt(
 842           data->mFile, channel, data->mOriginalLocation, uriWithExt);
 843       if (NS_SUCCEEDED(rv)) {
 844         data->mFile = uriWithExt;
 845       }
 846
 847       // now make filename conformant and unique
 848       nsCOMPtr<nsIURI> uniqueFilenameURI;
 849       rv = CalculateUniqueFilename(data->mFile, uniqueFilenameURI);
 850       if (NS_SUCCEEDED(rv)) {
 851         data->mFile = uniqueFilenameURI;
 852       }
 853
 854       // The URIData entry is pointing to the old unfixed URI, so we need
 855       // to update it.
 856       nsCOMPtr<nsIURI> chanURI;
 857       rv = channel->GetOriginalURI(getter_AddRefs(chanURI));
 858       if (NS_SUCCEEDED(rv)) {
 859         nsAutoCString spec;
 860         chanURI->GetSpec(spec);
 861         URIData* uridata;
 862         if (mURIMap.Get(spec, &uridata)) {
 863           uridata->mFile = data->mFile;
 864         }
 865       }
 866     }
 867
 868     // compare uris and bail before we add to output map if they are equal
 869     bool isEqual = false;
 870     if (NS_SUCCEEDED(data->mFile->Equals(data->mOriginalLocation, &isEqual)) &&
 871         isEqual) {
 872       {
 873         MutexAutoLock lock(mOutputMapMutex);
 874         // remove from output map
 875         mOutputMap.Remove(keyPtr);
 876       }
 877
 878       // cancel; we don't need to know any more
 879       // stop request will get called
 880       request->Cancel(NS_BINDING_ABORTED);
 881     }
 882   }
 883
 884   return NS_OK;
 885 }
 886
 887 NS_IMETHODIMP nsWebBrowserPersist::OnStopRequest(nsIRequest* request,
 888                                                  nsresult status) {
 889   nsCOMPtr<nsISupports> keyPtr = do_QueryInterface(request);
 890   OutputData* data = mOutputMap.Get(keyPtr);
 891   if (data) {
 892     if (NS_SUCCEEDED(mPersistResult) && NS_FAILED(status)) {
 893       SendErrorStatusChange(true, status, request, data->mFile);
 894     }
 895
 896     // If there is a stream ref and we weren't canceled,
 897     // close it away from the main thread.
 898     // We don't do this when there's an error/cancelation,
 899     // because our consumer may try to delete the file, which will error
 900     // if we're still holding on to it, so we have to close it pronto.
 901     {
 902       MutexAutoLock lock(data->mStreamMutex);
 903       if (data->mStream && NS_SUCCEEDED(status) && !mCancel) {
 904         if (!mBackgroundQueue) {
 905           nsresult rv = NS_CreateBackgroundTaskQueue(
 906               "WebBrowserPersist", getter_AddRefs(mBackgroundQueue));
 907           if (NS_FAILED(rv)) {
 908             return rv;
 909           }
 910         }
 911         // Now steal the stream ref and close it away from the main thread,
 912         // keeping the promise around so we don't finish before all files
 913         // are flushed and closed.
 914         mFileClosePromises.AppendElement(InvokeAsync(
 915             mBackgroundQueue, __func__, [stream = std::move(data->mStream)]() {
 916               nsresult rv = stream->Close();
 917               // We don't care if closing failed; we don't care in the
 918               // destructor either...
 919               return ClosePromise::CreateAndResolve(rv, __func__);
 920             }));
 921       }
 922     }
 923     MutexAutoLock lock(mOutputMapMutex);
 924     mOutputMap.Remove(keyPtr);
 925   } else {
 926     // if we didn't find the data in mOutputMap, try mUploadList
 927     UploadData* upData = mUploadList.Get(keyPtr);
 928     if (upData) {
 929       mUploadList.Remove(keyPtr);
 930     }
 931   }
 932
 933   // Do more work.
 934   SerializeNextFile();
 935
 936   if (mProgressListener) {
 937     uint32_t stateFlags = nsIWebProgressListener::STATE_STOP |
 938                           nsIWebProgressListener::STATE_IS_REQUEST;
 939     if (!mSavingDocument) {
 940       stateFlags |= nsIWebProgressListener::STATE_IS_NETWORK;
 941     }
 942     mProgressListener->OnStateChange(nullptr, request, stateFlags, status);
 943   }
 944
 945   return NS_OK;
 946 }
 947
 948 //*****************************************************************************
 949 // nsWebBrowserPersist::nsIStreamListener
 950 //*****************************************************************************
 951
 952 // Note: this is supposed to (but not guaranteed to) fire on a background
 953 // thread when used to save to local disk (channels not using local files will
 954 // use the main thread).
 955 // (Read) Access to mOutputMap is guarded via mOutputMapMutex.
 956 // Access to individual OutputData::mStream is guarded via its mStreamMutex.
 957 // mCancel is atomic, as is mPersistFlags (accessed via MakeOutputStream).
 958 // If you end up touching this method and needing other member access, bear
 959 // this in mind.
 960 NS_IMETHODIMP
 961 nsWebBrowserPersist::OnDataAvailable(nsIRequest* request,
 962                                      nsIInputStream* aIStream, uint64_t aOffset,
 963                                      uint32_t aLength) {
 964   // MOZ_ASSERT(!NS_IsMainThread()); // no guarantees, but it's likely.
 965
 966   bool cancel = mCancel;
 967   if (!cancel) {
 968     nsresult rv = NS_OK;
 969     uint32_t bytesRemaining = aLength;
 970
 971     nsCOMPtr<nsIChannel> channel = do_QueryInterface(request);
 972     NS_ENSURE_TRUE(channel, NS_ERROR_FAILURE);
 973
 974     MutexAutoLock lock(mOutputMapMutex);
 975     nsCOMPtr<nsISupports> keyPtr = do_QueryInterface(request);
 976     OutputData* data = mOutputMap.Get(keyPtr);
 977     if (!data) {
 978       // might be uploadData; consume necko's buffer and bail...
 979       uint32_t n;
 980       return aIStream->ReadSegments(NS_DiscardSegment, nullptr, aLength, &n);
 981     }
 982
 983     bool readError = true;
 984
 985     MutexAutoLock streamLock(data->mStreamMutex);
 986     // Make the output stream
 987     if (!data->mStream) {
 988       rv = MakeOutputStream(data->mFile, getter_AddRefs(data->mStream));
 989       if (NS_FAILED(rv)) {
 990         readError = false;
 991         cancel = true;
 992       }
 993     }
 994
 995     // Read data from the input and write to the output
 996     char buffer[8192];
 997     uint32_t bytesRead;
 998     while (!cancel && bytesRemaining) {
 999       readError = true;
1000       rv = aIStream->Read(buffer,
1001                           std::min(uint32_t(sizeof(buffer)), bytesRemaining),
1002                           &bytesRead);
1003       if (NS_SUCCEEDED(rv)) {
1004         readError = false;
1005         // Write out the data until something goes wrong, or, it is
1006         // all written.  We loop because for some errors (e.g., disk
1007         // full), we get NS_OK with some bytes written, then an error.
1008         // So, we want to write again in that case to get the actual
1009         // error code.
1010         const char* bufPtr = buffer;  // Where to write from.
1011         while (NS_SUCCEEDED(rv) && bytesRead) {
1012           uint32_t bytesWritten = 0;
1013           rv = data->mStream->Write(bufPtr, bytesRead, &bytesWritten);
1014           if (NS_SUCCEEDED(rv)) {
1015             bytesRead -= bytesWritten;
1016             bufPtr += bytesWritten;
1017             bytesRemaining -= bytesWritten;
1018             // Force an error if (for some reason) we get NS_OK but
1019             // no bytes written.
1020             if (!bytesWritten) {
1021               rv = NS_ERROR_FAILURE;
1022               cancel = true;
1023             }
1024           } else {
1025             // Disaster - can't write out the bytes - disk full / permission?
1026             cancel = true;
1027           }
1028         }
1029       } else {
1030         // Disaster - can't read the bytes - broken link / file error?
1031         cancel = true;
1032       }
1033     }
1034
1035     int64_t channelContentLength = -1;
1036     if (!cancel &&
1037         NS_SUCCEEDED(channel->GetContentLength(&channelContentLength))) {
1038       // if we get -1 at this point, we didn't get content-length header
1039       // assume that we got all of the data and push what we have;
1040       // that's the best we can do now
1041       if ((-1 == channelContentLength) ||
1042           ((channelContentLength - (aOffset + aLength)) == 0)) {
1043         NS_WARNING_ASSERTION(
1044             channelContentLength != -1,
1045             "nsWebBrowserPersist::OnDataAvailable() no content length "
1046             "header, pushing what we have");
1047         // we're done with this pass; see if we need to do upload
1048         nsAutoCString contentType;
1049         channel->GetContentType(contentType);
1050         // if we don't have the right type of output stream then it's a local
1051         // file
1052         nsCOMPtr<nsIStorageStream> storStream(do_QueryInterface(data->mStream));
1053         if (storStream) {
1054           data->mStream->Close();
1055           data->mStream =
1056               nullptr;  // null out stream so we don't close it later
1057           MOZ_ASSERT(NS_IsMainThread(),
1058                      "Uploads should be on the main thread.");
1059           rv = StartUpload(storStream, data->mFile, contentType);
1060           if (NS_FAILED(rv)) {
1061             readError = false;
1062             cancel = true;
1063           }
1064         }
1065       }
1066     }
1067
1068     // Notify listener if an error occurred.
1069     if (cancel) {
1070       RefPtr<nsIRequest> req = readError ? request : nullptr;
1071       nsCOMPtr<nsIURI> file = data->mFile;
1072       RefPtr<Runnable> errorOnMainThread = NS_NewRunnableFunction(
1073           "nsWebBrowserPersist::SendErrorStatusChange",
1074           [self = RefPtr{this}, req, file, readError, rv]() {
1075             self->SendErrorStatusChange(readError, rv, req, file);
1076           });
1077       NS_DispatchToMainThread(errorOnMainThread);
1078
1079       // And end the download on the main thread.
1080       nsCOMPtr<nsIRunnable> endOnMainThread = NewRunnableMethod<nsresult>(
1081           "nsWebBrowserPersist::EndDownload", this,
1082           &nsWebBrowserPersist::EndDownload, NS_BINDING_ABORTED);
1083       NS_DispatchToMainThread(endOnMainThread);
1084     }
1085   }
1086
1087   return cancel ? NS_BINDING_ABORTED : NS_OK;
1088 }
1089
1090 //*****************************************************************************
1091 // nsWebBrowserPersist::nsIThreadRetargetableStreamListener
1092 //*****************************************************************************
1093
1094 NS_IMETHODIMP nsWebBrowserPersist::CheckListenerChain() { return NS_OK; }
1095
1096 //*****************************************************************************
1097 // nsWebBrowserPersist::nsIProgressEventSink
1098 //*****************************************************************************
1099
1100 NS_IMETHODIMP nsWebBrowserPersist::OnProgress(nsIRequest* request,
1101                                               int64_t aProgress,
1102                                               int64_t aProgressMax) {
1103   if (!mProgressListener) {
1104     return NS_OK;
1105   }
1106
1107   // Store the progress of this request
1108   nsCOMPtr<nsISupports> keyPtr = do_QueryInterface(request);
1109   OutputData* data = mOutputMap.Get(keyPtr);
1110   if (data) {
1111     data->mSelfProgress = aProgress;
1112     data->mSelfProgressMax = aProgressMax;
1113   } else {
1114     UploadData* upData = mUploadList.Get(keyPtr);
1115     if (upData) {
1116       upData->mSelfProgress = aProgress;
1117       upData->mSelfProgressMax = aProgressMax;
1118     }
1119   }
1120
1121   // Notify listener of total progress
1122   CalcTotalProgress();
1123   if (mProgressListener2) {
1124     mProgressListener2->OnProgressChange64(nullptr, request, aProgress,
1125                                            aProgressMax, mTotalCurrentProgress,
1126                                            mTotalMaxProgress);
1127   } else {
1128     // have to truncate 64-bit to 32bit
1129     mProgressListener->OnProgressChange(
1130         nullptr, request, uint64_t(aProgress), uint64_t(aProgressMax),
1131         mTotalCurrentProgress, mTotalMaxProgress);
1132   }
1133
1134   // If our progress listener implements nsIProgressEventSink,
1135   // forward the notification
1136   if (mEventSink) {
1137     mEventSink->OnProgress(request, aProgress, aProgressMax);
1138   }
1139
1140   return NS_OK;
1141 }
1142
1143 NS_IMETHODIMP nsWebBrowserPersist::OnStatus(nsIRequest* request,
1144                                             nsresult status,
1145                                             const char16_t* statusArg) {
1146   if (mProgressListener) {
1147     // We need to filter out non-error error codes.
1148     // Is the only NS_SUCCEEDED value NS_OK?
1149     switch (status) {
1150       case NS_NET_STATUS_RESOLVING_HOST:
1151       case NS_NET_STATUS_RESOLVED_HOST:
1152       case NS_NET_STATUS_CONNECTING_TO:
1153       case NS_NET_STATUS_CONNECTED_TO:
1154       case NS_NET_STATUS_TLS_HANDSHAKE_STARTING:
1155       case NS_NET_STATUS_TLS_HANDSHAKE_ENDED:
1156       case NS_NET_STATUS_SENDING_TO:
1157       case NS_NET_STATUS_RECEIVING_FROM:
1158       case NS_NET_STATUS_WAITING_FOR:
1159       case NS_NET_STATUS_READING:
1160       case NS_NET_STATUS_WRITING:
1161         break;
1162
1163       default:
1164         // Pass other notifications (for legitimate errors) along.
1165         mProgressListener->OnStatusChange(nullptr, request, status, statusArg);
1166         break;
1167     }
1168   }
1169
1170   // If our progress listener implements nsIProgressEventSink,
1171   // forward the notification
1172   if (mEventSink) {
1173     mEventSink->OnStatus(request, status, statusArg);
1174   }
1175
1176   return NS_OK;
1177 }
1178
1179 //*****************************************************************************
1180 // nsWebBrowserPersist private methods
1181 //*****************************************************************************
1182
1183 // Convert error info into proper message text and send OnStatusChange
1184 // notification to the web progress listener.
1185 nsresult nsWebBrowserPersist::SendErrorStatusChange(bool aIsReadError,
1186                                                     nsresult aResult,
1187                                                     nsIRequest* aRequest,
1188                                                     nsIURI* aURI) {
1189   NS_ENSURE_ARG_POINTER(aURI);
1190
1191   if (!mProgressListener) {
1192     // Do nothing
1193     return NS_OK;
1194   }
1195
1196   // Get the file path or spec from the supplied URI
1197   nsCOMPtr<nsIFile> file;
1198   GetLocalFileFromURI(aURI, getter_AddRefs(file));
1199   AutoTArray<nsString, 1> strings;
1200   nsresult rv;
1201   if (file) {
1202     file->GetPath(*strings.AppendElement());
1203   } else {
1204     nsAutoCString fileurl;
1205     rv = aURI->GetSpec(fileurl);
1206     NS_ENSURE_SUCCESS(rv, rv);
1207     CopyUTF8toUTF16(fileurl, *strings.AppendElement());
1208   }
1209
1210   const char* msgId;
1211   switch (aResult) {
1212     case NS_ERROR_FILE_NAME_TOO_LONG:
1213       // File name too long.
1214       msgId = "fileNameTooLongError";
1215       break;
1216     case NS_ERROR_FILE_ALREADY_EXISTS:
1217       // File exists with same name as directory.
1218       msgId = "fileAlreadyExistsError";
1219       break;
1220     case NS_ERROR_FILE_NO_DEVICE_SPACE:
1221       // Out of space on target volume.
1222       msgId = "diskFull";
1223       break;
1224
1225     case NS_ERROR_FILE_READ_ONLY:
1226       // Attempt to write to read/only file.
1227       msgId = "readOnly";
1228       break;
1229
1230     case NS_ERROR_FILE_ACCESS_DENIED:
1231       // Attempt to write without sufficient permissions.
1232       msgId = "accessError";
1233       break;
1234
1235     default:
1236       // Generic read/write error message.
1237       if (aIsReadError)
1238         msgId = "readError";
1239       else
1240         msgId = "writeError";
1241       break;
1242   }
1243   // Get properties file bundle and extract status string.
1244   nsCOMPtr<nsIStringBundleService> s =
1245       do_GetService(NS_STRINGBUNDLE_CONTRACTID, &rv);
1246   NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && s, NS_ERROR_FAILURE);
1247
1248   nsCOMPtr<nsIStringBundle> bundle;
1249   rv = s->CreateBundle(kWebBrowserPersistStringBundle, getter_AddRefs(bundle));
1250   NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && bundle, NS_ERROR_FAILURE);
1251
1252   nsAutoString msgText;
1253   rv = bundle->FormatStringFromName(msgId, strings, msgText);
1254   NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
1255
1256   mProgressListener->OnStatusChange(nullptr, aRequest, aResult, msgText.get());
1257
1258   return NS_OK;
1259 }
1260
1261 nsresult nsWebBrowserPersist::GetValidURIFromObject(nsISupports* aObject,
1262                                                     nsIURI** aURI) const {
1263   NS_ENSURE_ARG_POINTER(aObject);
1264   NS_ENSURE_ARG_POINTER(aURI);
1265
1266   nsCOMPtr<nsIFile> objAsFile = do_QueryInterface(aObject);
1267   if (objAsFile) {
1268     return NS_NewFileURI(aURI, objAsFile);
1269   }
1270   nsCOMPtr<nsIURI> objAsURI = do_QueryInterface(aObject);
1271   if (objAsURI) {
1272     *aURI = objAsURI;
1273     NS_ADDREF(*aURI);
1274     return NS_OK;
1275   }
1276
1277   return NS_ERROR_FAILURE;
1278 }
1279
1280 /* static */
1281 nsresult nsWebBrowserPersist::GetLocalFileFromURI(nsIURI* aURI,
1282                                                   nsIFile** aLocalFile) {
1283   nsresult rv;
1284
1285   nsCOMPtr<nsIFileURL> fileURL = do_QueryInterface(aURI, &rv);
1286   if (NS_FAILED(rv)) return rv;
1287
1288   nsCOMPtr<nsIFile> file;
1289   rv = fileURL->GetFile(getter_AddRefs(file));
1290   if (NS_FAILED(rv)) {
1291     return rv;
1292   }
1293
1294   file.forget(aLocalFile);
1295   return NS_OK;
1296 }
1297
1298 /* static */
1299 nsresult nsWebBrowserPersist::AppendPathToURI(nsIURI* aURI,
1300                                               const nsAString& aPath,
1301                                               nsCOMPtr<nsIURI>& aOutURI) {
1302   NS_ENSURE_ARG_POINTER(aURI);
1303
1304   nsAutoCString newPath;
1305   nsresult rv = aURI->GetPathQueryRef(newPath);
1306   NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
1307
1308   // Append a forward slash if necessary
1309   int32_t len = newPath.Length();
1310   if (len > 0 && newPath.CharAt(len - 1) != '/') {
1311     newPath.Append('/');
1312   }
1313
1314   // Store the path back on the URI
1315   AppendUTF16toUTF8(aPath, newPath);
1316
1317   return NS_MutateURI(aURI).SetPathQueryRef(newPath).Finalize(aOutURI);
1318 }
1319
1320 nsresult nsWebBrowserPersist::SaveURIInternal(
1321     nsIURI* aURI, nsIPrincipal* aTriggeringPrincipal,
1322     nsContentPolicyType aContentPolicyType, uint32_t aCacheKey,
1323     nsIReferrerInfo* aReferrerInfo, nsICookieJarSettings* aCookieJarSettings,
1324     nsIInputStream* aPostData, const char* aExtraHeaders, nsIURI* aFile,
1325     bool aCalcFileExt, bool aIsPrivate) {
1326   NS_ENSURE_ARG_POINTER(aURI);
1327   NS_ENSURE_ARG_POINTER(aFile);
1328   NS_ENSURE_ARG_POINTER(aTriggeringPrincipal);
1329
1330   nsresult rv = NS_OK;
1331
1332   mURI = aURI;
1333
1334   nsLoadFlags loadFlags = nsIRequest::LOAD_NORMAL;
1335   if (mPersistFlags & PERSIST_FLAGS_BYPASS_CACHE) {
1336     loadFlags |= nsIRequest::LOAD_BYPASS_CACHE;
1337   } else if (mPersistFlags & PERSIST_FLAGS_FROM_CACHE) {
1338     loadFlags |= nsIRequest::LOAD_FROM_CACHE;
1339   }
1340
1341   // If there is no cookieJarSetting given, we need to create a new
1342   // cookieJarSettings for this download in order to send cookies based on the
1343   // current state of the prefs/permissions.
1344   nsCOMPtr<nsICookieJarSettings> cookieJarSettings = aCookieJarSettings;
1345   if (!cookieJarSettings) {
1346     cookieJarSettings =
1347         aIsPrivate
1348             ? net::CookieJarSettings::Create(net::CookieJarSettings::ePrivate)
1349             : net::CookieJarSettings::Create(net::CookieJarSettings::eRegular);
1350   }
1351
1352   // Open a channel to the URI
1353   nsCOMPtr<nsIChannel> inputChannel;
1354   rv = NS_NewChannel(getter_AddRefs(inputChannel), aURI, aTriggeringPrincipal,
1355                      nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_SEC_CONTEXT_IS_NULL,
1356                      aContentPolicyType, cookieJarSettings,
1357                      nullptr,  // aPerformanceStorage
1358                      nullptr,  // aLoadGroup
1359                      static_cast<nsIInterfaceRequestor*>(this), loadFlags);
1360
1361   nsCOMPtr<nsIPrivateBrowsingChannel> pbChannel =
1362       do_QueryInterface(inputChannel);
1363   if (pbChannel) {
1364     pbChannel->SetPrivate(aIsPrivate);
1365   }
1366
1367   if (NS_FAILED(rv) || inputChannel == nullptr) {
1368     EndDownload(NS_ERROR_FAILURE);
1369     return NS_ERROR_FAILURE;
1370   }
1371
1372   // Disable content conversion
1373   if (mPersistFlags & PERSIST_FLAGS_NO_CONVERSION) {
1374     nsCOMPtr<nsIEncodedChannel> encodedChannel(do_QueryInterface(inputChannel));
1375     if (encodedChannel) {
1376       encodedChannel->SetApplyConversion(false);
1377     }
1378   }
1379
1380   // Set the referrer, post data and headers if any
1381   nsCOMPtr<nsIHttpChannel> httpChannel(do_QueryInterface(inputChannel));
1382   if (httpChannel) {
1383     if (aReferrerInfo) {
1384       DebugOnly<nsresult> success = httpChannel->SetReferrerInfo(aReferrerInfo);
1385       MOZ_ASSERT(NS_SUCCEEDED(success));
1386     }
1387
1388     // Post data
1389     if (aPostData) {
1390       nsCOMPtr<nsISeekableStream> stream(do_QueryInterface(aPostData));
1391       if (stream) {
1392         // Rewind the postdata stream
1393         stream->Seek(nsISeekableStream::NS_SEEK_SET, 0);
1394         nsCOMPtr<nsIUploadChannel> uploadChannel(
1395             do_QueryInterface(httpChannel));
1396         NS_ASSERTION(uploadChannel, "http must support nsIUploadChannel");
1397         // Attach the postdata to the http channel
1398         uploadChannel->SetUploadStream(aPostData, ""_ns, -1);
1399       }
1400     }
1401
1402     // Cache key
1403     nsCOMPtr<nsICacheInfoChannel> cacheChannel(do_QueryInterface(httpChannel));
1404     if (cacheChannel && aCacheKey != 0) {
1405       cacheChannel->SetCacheKey(aCacheKey);
1406     }
1407
1408     // Headers
1409     if (aExtraHeaders) {
1410       nsAutoCString oneHeader;
1411       nsAutoCString headerName;
1412       nsAutoCString headerValue;
1413       int32_t crlf = 0;
1414       int32_t colon = 0;
1415       const char* kWhitespace = "\b\t\r\n ";
1416       nsAutoCString extraHeaders(aExtraHeaders);
1417       while (true) {
1418         crlf = extraHeaders.Find("\r\n", true);
1419         if (crlf == -1) break;
1420         extraHeaders.Mid(oneHeader, 0, crlf);
1421         extraHeaders.Cut(0, crlf + 2);
1422         colon = oneHeader.Find(":");
1423         if (colon == -1) break;  // Should have a colon
1424         oneHeader.Left(headerName, colon);
1425         colon++;
1426         oneHeader.Mid(headerValue, colon, oneHeader.Length() - colon);
1427         headerName.Trim(kWhitespace);
1428         headerValue.Trim(kWhitespace);
1429         // Add the header (merging if required)
1430         rv = httpChannel->SetRequestHeader(headerName, headerValue, true);
1431         if (NS_FAILED(rv)) {
1432           EndDownload(NS_ERROR_FAILURE);
1433           return NS_ERROR_FAILURE;
1434         }
1435       }
1436     }
1437   }
1438   return SaveChannelInternal(inputChannel, aFile, aCalcFileExt);
1439 }
1440
1441 nsresult nsWebBrowserPersist::SaveChannelInternal(nsIChannel* aChannel,
1442                                                   nsIURI* aFile,
1443                                                   bool aCalcFileExt) {
1444   NS_ENSURE_ARG_POINTER(aChannel);
1445   NS_ENSURE_ARG_POINTER(aFile);
1446
1447   // The default behaviour of SaveChannelInternal is to download the source
1448   // into a storage stream and upload that to the target. MakeOutputStream
1449   // special-cases a file target and creates a file output stream directly.
1450   // We want to special-case a file source and create a file input stream,
1451   // but we don't need to do this in the case of a file target.
1452   nsCOMPtr<nsIFileChannel> fc(do_QueryInterface(aChannel));
1453   nsCOMPtr<nsIFileURL> fu(do_QueryInterface(aFile));
1454
1455   if (fc && !fu) {
1456     nsCOMPtr<nsIInputStream> fileInputStream, bufferedInputStream;
1457     nsresult rv =
1458         NS_MaybeOpenChannelUsingOpen(aChannel, getter_AddRefs(fileInputStream));
1459     NS_ENSURE_SUCCESS(rv, rv);
1460     rv = NS_NewBufferedInputStream(getter_AddRefs(bufferedInputStream),
1461                                    fileInputStream.forget(),
1462                                    BUFFERED_OUTPUT_SIZE);
1463     NS_ENSURE_SUCCESS(rv, rv);
1464     nsAutoCString contentType;
1465     aChannel->GetContentType(contentType);
1466     return StartUpload(bufferedInputStream, aFile, contentType);
1467   }
1468
1469   // Mark save channel as throttleable.
1470   nsCOMPtr<nsIClassOfService> cos(do_QueryInterface(aChannel));
1471   if (cos) {
1472     cos->AddClassFlags(nsIClassOfService::Throttleable);
1473   }
1474
1475   // Read from the input channel
1476   nsresult rv = NS_MaybeOpenChannelUsingAsyncOpen(aChannel, this);
1477   if (rv == NS_ERROR_NO_CONTENT) {
1478     // Assume this is a protocol such as mailto: which does not feed out
1479     // data and just ignore it.
1480     return NS_SUCCESS_DONT_FIXUP;
1481   }
1482
1483   if (NS_FAILED(rv)) {
1484     // Opening failed, but do we care?
1485     if (mPersistFlags & PERSIST_FLAGS_FAIL_ON_BROKEN_LINKS) {
1486       SendErrorStatusChange(true, rv, aChannel, aFile);
1487       EndDownload(NS_ERROR_FAILURE);
1488       return NS_ERROR_FAILURE;
1489     }
1490     return NS_SUCCESS_DONT_FIXUP;
1491   }
1492
1493   MutexAutoLock lock(mOutputMapMutex);
1494   // Add the output transport to the output map with the channel as the key
1495   nsCOMPtr<nsISupports> keyPtr = do_QueryInterface(aChannel);
1496   mOutputMap.InsertOrUpdate(keyPtr,
1497                             MakeUnique<OutputData>(aFile, mURI, aCalcFileExt));
1498
1499   return NS_OK;
1500 }
1501
1502 nsresult nsWebBrowserPersist::GetExtensionForContentType(
1503     const char16_t* aContentType, char16_t** aExt) {
1504   NS_ENSURE_ARG_POINTER(aContentType);
1505   NS_ENSURE_ARG_POINTER(aExt);
1506
1507   *aExt = nullptr;
1508
1509   nsresult rv;
1510   if (!mMIMEService) {
1511     mMIMEService = do_GetService(NS_MIMESERVICE_CONTRACTID, &rv);
1512     NS_ENSURE_TRUE(mMIMEService, NS_ERROR_FAILURE);
1513   }
1514
1515   nsAutoCString contentType;
1516   LossyCopyUTF16toASCII(MakeStringSpan(aContentType), contentType);
1517   nsAutoCString ext;
1518   rv = mMIMEService->GetPrimaryExtension(contentType, ""_ns, ext);
1519   if (NS_SUCCEEDED(rv)) {
1520     *aExt = UTF8ToNewUnicode(ext);
1521     NS_ENSURE_TRUE(*aExt, NS_ERROR_OUT_OF_MEMORY);
1522     return NS_OK;
1523   }
1524
1525   return NS_ERROR_FAILURE;
1526 }
1527
1528 nsresult nsWebBrowserPersist::SaveDocumentDeferred(
1529     mozilla::UniquePtr<WalkData>&& aData) {
1530   nsresult rv =
1531       SaveDocumentInternal(aData->mDocument, aData->mFile, aData->mDataPath);
1532   if (NS_FAILED(rv)) {
1533     SendErrorStatusChange(true, rv, nullptr, mURI);
1534     EndDownload(rv);
1535   }
1536   return rv;
1537 }
1538
1539 nsresult nsWebBrowserPersist::SaveDocumentInternal(
1540     nsIWebBrowserPersistDocument* aDocument, nsIURI* aFile, nsIURI* aDataPath) {
1541   mURI = nullptr;
1542   NS_ENSURE_ARG_POINTER(aDocument);
1543   NS_ENSURE_ARG_POINTER(aFile);
1544
1545   nsresult rv = aDocument->SetPersistFlags(mPersistFlags);
1546   NS_ENSURE_SUCCESS(rv, rv);
1547
1548   rv = aDocument->GetIsPrivate(&mIsPrivate);
1549   NS_ENSURE_SUCCESS(rv, rv);
1550
1551   // See if we can get the local file representation of this URI
1552   nsCOMPtr<nsIFile> localFile;
1553   rv = GetLocalFileFromURI(aFile, getter_AddRefs(localFile));
1554
1555   nsCOMPtr<nsIFile> localDataPath;
1556   if (NS_SUCCEEDED(rv) && aDataPath) {
1557     // See if we can get the local file representation of this URI
1558     rv = GetLocalFileFromURI(aDataPath, getter_AddRefs(localDataPath));
1559     NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
1560   }
1561
1562   // Persist the main document
1563   rv = aDocument->GetCharacterSet(mCurrentCharset);
1564   NS_ENSURE_SUCCESS(rv, rv);
1565   nsAutoCString uriSpec;
1566   rv = aDocument->GetDocumentURI(uriSpec);
1567   NS_ENSURE_SUCCESS(rv, rv);
1568   rv = NS_NewURI(getter_AddRefs(mURI), uriSpec, mCurrentCharset.get());
1569   NS_ENSURE_SUCCESS(rv, rv);
1570   rv = aDocument->GetBaseURI(uriSpec);
1571   NS_ENSURE_SUCCESS(rv, rv);
1572   rv = NS_NewURI(getter_AddRefs(mCurrentBaseURI), uriSpec,
1573                  mCurrentCharset.get());
1574   NS_ENSURE_SUCCESS(rv, rv);
1575
1576   // Does the caller want to fixup the referenced URIs and save those too?
1577   if (aDataPath) {
1578     // Basic steps are these.
1579     //
1580     // 1. Iterate through the document (and subdocuments) building a list
1581     //    of unique URIs.
1582     // 2. For each URI create an OutputData entry and open a channel to save
1583     //    it. As each URI is saved, discover the mime type and fix up the
1584     //    local filename with the correct extension.
1585     // 3. Store the document in a list and wait for URI persistence to finish
1586     // 4. After URI persistence completes save the list of documents,
1587     //    fixing it up as it goes out to file.
1588
1589     mCurrentDataPathIsRelative = false;
1590     mCurrentDataPath = aDataPath;
1591     mCurrentRelativePathToData = "";
1592     mCurrentThingsToPersist = 0;
1593     mTargetBaseURI = aFile;
1594
1595     // Determine if the specified data path is relative to the
1596     // specified file, (e.g. c:\docs\htmldata is relative to
1597     // c:\docs\myfile.htm, but not to d:\foo\data.
1598
1599     // Starting with the data dir work back through its parents
1600     // checking if one of them matches the base directory.
1601
1602     if (localDataPath && localFile) {
1603       nsCOMPtr<nsIFile> baseDir;
1604       localFile->GetParent(getter_AddRefs(baseDir));
1605
1606       nsAutoCString relativePathToData;
1607       nsCOMPtr<nsIFile> dataDirParent;
1608       dataDirParent = localDataPath;
1609       while (dataDirParent) {
1610         bool sameDir = false;
1611         dataDirParent->Equals(baseDir, &sameDir);
1612         if (sameDir) {
1613           mCurrentRelativePathToData = relativePathToData;
1614           mCurrentDataPathIsRelative = true;
1615           break;
1616         }
1617
1618         nsAutoString dirName;
1619         dataDirParent->GetLeafName(dirName);
1620
1621         nsAutoCString newRelativePathToData;
1622         newRelativePathToData =
1623             NS_ConvertUTF16toUTF8(dirName) + "/"_ns + relativePathToData;
1624         relativePathToData = newRelativePathToData;
1625
1626         nsCOMPtr<nsIFile> newDataDirParent;
1627         rv = dataDirParent->GetParent(getter_AddRefs(newDataDirParent));
1628         dataDirParent = newDataDirParent;
1629       }
1630     } else {
1631       // generate a relative path if possible
1632       nsCOMPtr<nsIURL> pathToBaseURL(do_QueryInterface(aFile));
1633       if (pathToBaseURL) {
1634         nsAutoCString relativePath;  // nsACString
1635         if (NS_SUCCEEDED(
1636                 pathToBaseURL->GetRelativeSpec(aDataPath, relativePath))) {
1637           mCurrentDataPathIsRelative = true;
1638           mCurrentRelativePathToData = relativePath;
1639         }
1640       }
1641     }
1642
1643     // Store the document in a list so when URI persistence is done and the
1644     // filenames of saved URIs are known, the documents can be fixed up and
1645     // saved
1646
1647     auto* docData = new DocData;
1648     docData->mBaseURI = mCurrentBaseURI;
1649     docData->mCharset = mCurrentCharset;
1650     docData->mDocument = aDocument;
1651     docData->mFile = aFile;
1652     mDocList.AppendElement(docData);
1653
1654     // Walk the DOM gathering a list of externally referenced URIs in the uri
1655     // map
1656     nsCOMPtr<nsIWebBrowserPersistResourceVisitor> visit =
1657         new OnWalk(this, aFile, localDataPath);
1658     return aDocument->ReadResources(visit);
1659   } else {
1660     auto* docData = new DocData;
1661     docData->mBaseURI = mCurrentBaseURI;
1662     docData->mCharset = mCurrentCharset;
1663     docData->mDocument = aDocument;
1664     docData->mFile = aFile;
1665     mDocList.AppendElement(docData);
1666
1667     // Not walking DOMs, so go directly to serialization.
1668     SerializeNextFile();
1669     return NS_OK;
1670   }
1671 }
1672
1673 NS_IMETHODIMP
1674 nsWebBrowserPersist::OnWalk::VisitResource(
1675     nsIWebBrowserPersistDocument* aDoc, const nsACString& aURI,
1676     nsContentPolicyType aContentPolicyType) {
1677   return mParent->StoreURI(aURI, aDoc, aContentPolicyType);
1678 }
1679
1680 NS_IMETHODIMP
1681 nsWebBrowserPersist::OnWalk::VisitDocument(
1682     nsIWebBrowserPersistDocument* aDoc, nsIWebBrowserPersistDocument* aSubDoc) {
1683   URIData* data = nullptr;
1684   nsAutoCString uriSpec;
1685   nsresult rv = aSubDoc->GetDocumentURI(uriSpec);
1686   NS_ENSURE_SUCCESS(rv, rv);
1687   rv = mParent->StoreURI(uriSpec, aDoc, nsIContentPolicy::TYPE_SUBDOCUMENT,
1688                          false, &data);
1689   NS_ENSURE_SUCCESS(rv, rv);
1690   if (!data) {
1691     // If the URI scheme isn't persistable, then don't persist.
1692     return NS_OK;
1693   }
1694   data->mIsSubFrame = true;
1695   return mParent->SaveSubframeContent(aSubDoc, aDoc, uriSpec, data);
1696 }
1697
1698 NS_IMETHODIMP
1699 nsWebBrowserPersist::OnWalk::VisitBrowsingContext(
1700     nsIWebBrowserPersistDocument* aDoc, BrowsingContext* aContext) {
1701   RefPtr<dom::CanonicalBrowsingContext> context = aContext->Canonical();
1702
1703   if (NS_WARN_IF(!context->GetCurrentWindowGlobal())) {
1704     EndVisit(nullptr, NS_ERROR_FAILURE);
1705     return NS_ERROR_FAILURE;
1706   }
1707
1708   UniquePtr<WebBrowserPersistDocumentParent> actor(
1709       new WebBrowserPersistDocumentParent());
1710
1711   nsCOMPtr<nsIWebBrowserPersistDocumentReceiver> receiver =
1712       new OnRemoteWalk(this, aDoc);
1713   actor->SetOnReady(receiver);
1714
1715   RefPtr<dom::BrowserParent> browserParent =
1716       context->GetCurrentWindowGlobal()->GetBrowserParent();
1717
1718   bool ok =
1719       context->GetContentParent()->SendPWebBrowserPersistDocumentConstructor(
1720           actor.release(), browserParent, context);
1721
1722   if (NS_WARN_IF(!ok)) {
1723     // (The actor will be destroyed on constructor failure.)
1724     EndVisit(nullptr, NS_ERROR_FAILURE);
1725     return NS_ERROR_FAILURE;
1726   }
1727
1728   ++mPendingDocuments;
1729
1730   return NS_OK;
1731 }
1732
1733 NS_IMETHODIMP
1734 nsWebBrowserPersist::OnWalk::EndVisit(nsIWebBrowserPersistDocument* aDoc,
1735                                       nsresult aStatus) {
1736   if (NS_FAILED(mStatus)) {
1737     return mStatus;
1738   }
1739
1740   if (NS_FAILED(aStatus)) {
1741     mStatus = aStatus;
1742     mParent->SendErrorStatusChange(true, aStatus, nullptr, mFile);
1743     mParent->EndDownload(aStatus);
1744     return aStatus;
1745   }
1746
1747   if (--mPendingDocuments) {
1748     // We're not done yet, wait for more.
1749     return NS_OK;
1750   }
1751
1752   mParent->FinishSaveDocumentInternal(mFile, mDataPath);
1753   return NS_OK;
1754 }
1755
1756 NS_IMETHODIMP
1757 nsWebBrowserPersist::OnRemoteWalk::OnDocumentReady(
1758     nsIWebBrowserPersistDocument* aSubDocument) {
1759   mVisitor->VisitDocument(mDocument, aSubDocument);
1760   mVisitor->EndVisit(mDocument, NS_OK);
1761   return NS_OK;
1762 }
1763
1764 NS_IMETHODIMP
1765 nsWebBrowserPersist::OnRemoteWalk::OnError(nsresult aFailure) {
1766   mVisitor->EndVisit(nullptr, aFailure);
1767   return NS_OK;
1768 }
1769
1770 void nsWebBrowserPersist::FinishSaveDocumentInternal(nsIURI* aFile,
1771                                                      nsIFile* aDataPath) {
1772   // If there are things to persist, create a directory to hold them
1773   if (mCurrentThingsToPersist > 0) {
1774     if (aDataPath) {
1775       bool exists = false;
1776       bool haveDir = false;
1777
1778       aDataPath->Exists(&exists);
1779       if (exists) {
1780         aDataPath->IsDirectory(&haveDir);
1781       }
1782       if (!haveDir) {
1783         nsresult rv = aDataPath->Create(nsIFile::DIRECTORY_TYPE, 0755);
1784         if (NS_SUCCEEDED(rv)) {
1785           haveDir = true;
1786         } else {
1787           SendErrorStatusChange(false, rv, nullptr, aFile);
1788         }
1789       }
1790       if (!haveDir) {
1791         EndDownload(NS_ERROR_FAILURE);
1792         return;
1793       }
1794       if (mPersistFlags & PERSIST_FLAGS_CLEANUP_ON_FAILURE) {
1795         // Add to list of things to delete later if all goes wrong
1796         auto* cleanupData = new CleanupData;
1797         cleanupData->mFile = aDataPath;
1798         cleanupData->mIsDirectory = true;
1799         mCleanupList.AppendElement(cleanupData);
1800       }
1801     }
1802   }
1803
1804   if (mWalkStack.Length() > 0) {
1805     mozilla::UniquePtr<WalkData> toWalk = mWalkStack.PopLastElement();
1806     // Bounce this off the event loop to avoid stack overflow.
1807     using WalkStorage = StoreCopyPassByRRef<decltype(toWalk)>;
1808     auto saveMethod = &nsWebBrowserPersist::SaveDocumentDeferred;
1809     nsCOMPtr<nsIRunnable> saveLater = NewRunnableMethod<WalkStorage>(
1810         "nsWebBrowserPersist::FinishSaveDocumentInternal", this, saveMethod,
1811         std::move(toWalk));
1812     NS_DispatchToCurrentThread(saveLater);
1813   } else {
1814     // Done walking DOMs; on to the serialization phase.
1815     SerializeNextFile();
1816   }
1817 }
1818
1819 void nsWebBrowserPersist::Cleanup() {
1820   mURIMap.Clear();
1821   nsClassHashtable<nsISupportsHashKey, OutputData> outputMapCopy;
1822   {
1823     MutexAutoLock lock(mOutputMapMutex);
1824     mOutputMap.SwapElements(outputMapCopy);
1825   }
1826   for (const auto& key : outputMapCopy.Keys()) {
1827     nsCOMPtr<nsIChannel> channel = do_QueryInterface(key);
1828     if (channel) {
1829       channel->Cancel(NS_BINDING_ABORTED);
1830     }
1831   }
1832   outputMapCopy.Clear();
1833
1834   for (const auto& key : mUploadList.Keys()) {
1835     nsCOMPtr<nsIChannel> channel = do_QueryInterface(key);
1836     if (channel) {
1837       channel->Cancel(NS_BINDING_ABORTED);
1838     }
1839   }
1840   mUploadList.Clear();
1841
1842   uint32_t i;
1843   for (i = 0; i < mDocList.Length(); i++) {
1844     DocData* docData = mDocList.ElementAt(i);
1845     delete docData;
1846   }
1847   mDocList.Clear();
1848
1849   for (i = 0; i < mCleanupList.Length(); i++) {
1850     CleanupData* cleanupData = mCleanupList.ElementAt(i);
1851     delete cleanupData;
1852   }
1853   mCleanupList.Clear();
1854
1855   mFilenameList.Clear();
1856 }
1857
1858 void nsWebBrowserPersist::CleanupLocalFiles() {
1859   // Two passes, the first pass cleans up files, the second pass tests
1860   // for and then deletes empty directories. Directories that are not
1861   // empty after the first pass must contain files from something else
1862   // and are not deleted.
1863   int pass;
1864   for (pass = 0; pass < 2; pass++) {
1865     uint32_t i;
1866     for (i = 0; i < mCleanupList.Length(); i++) {
1867       CleanupData* cleanupData = mCleanupList.ElementAt(i);
1868       nsCOMPtr<nsIFile> file = cleanupData->mFile;
1869
1870       // Test if the dir / file exists (something in an earlier loop
1871       // may have already removed it)
1872       bool exists = false;
1873       file->Exists(&exists);
1874       if (!exists) continue;
1875
1876       // Test if the file has changed in between creation and deletion
1877       // in some way that means it should be ignored
1878       bool isDirectory = false;
1879       file->IsDirectory(&isDirectory);
1880       if (isDirectory != cleanupData->mIsDirectory)
1881         continue;  // A file has become a dir or vice versa !
1882
1883       if (pass == 0 && !isDirectory) {
1884         file->Remove(false);
1885       } else if (pass == 1 && isDirectory)  // Directory
1886       {
1887         // Directories are more complicated. Enumerate through
1888         // children looking for files. Any files created by the
1889         // persist object would have been deleted by the first
1890         // pass so if there are any there at this stage, the dir
1891         // cannot be deleted because it has someone else's files
1892         // in it. Empty child dirs are deleted but they must be
1893         // recursed through to ensure they are actually empty.
1894
1895         bool isEmptyDirectory = true;
1896         nsCOMArray<nsIDirectoryEnumerator> dirStack;
1897         int32_t stackSize = 0;
1898
1899         // Push the top level enum onto the stack
1900         nsCOMPtr<nsIDirectoryEnumerator> pos;
1901         if (NS_SUCCEEDED(file->GetDirectoryEntries(getter_AddRefs(pos))))
1902           dirStack.AppendObject(pos);
1903
1904         while (isEmptyDirectory && (stackSize = dirStack.Count())) {
1905           // Pop the last element
1906           nsCOMPtr<nsIDirectoryEnumerator> curPos;
1907           curPos = dirStack[stackSize - 1];
1908           dirStack.RemoveObjectAt(stackSize - 1);
1909
1910           nsCOMPtr<nsIFile> child;
1911           if (NS_FAILED(curPos->GetNextFile(getter_AddRefs(child))) || !child) {
1912             continue;
1913           }
1914
1915           bool childIsSymlink = false;
1916           child->IsSymlink(&childIsSymlink);
1917           bool childIsDir = false;
1918           child->IsDirectory(&childIsDir);
1919           if (!childIsDir || childIsSymlink) {
1920             // Some kind of file or symlink which means dir
1921             // is not empty so just drop out.
1922             isEmptyDirectory = false;
1923             break;
1924           }
1925           // Push parent enumerator followed by child enumerator
1926           nsCOMPtr<nsIDirectoryEnumerator> childPos;
1927           child->GetDirectoryEntries(getter_AddRefs(childPos));
1928           dirStack.AppendObject(curPos);
1929           if (childPos) dirStack.AppendObject(childPos);
1930         }
1931         dirStack.Clear();
1932
1933         // If after all that walking the dir is deemed empty, delete it
1934         if (isEmptyDirectory) {
1935           file->Remove(true);
1936         }
1937       }
1938     }
1939   }
1940 }
1941
1942 nsresult nsWebBrowserPersist::CalculateUniqueFilename(
1943     nsIURI* aURI, nsCOMPtr<nsIURI>& aOutURI) {
1944   nsCOMPtr<nsIURL> url(do_QueryInterface(aURI));
1945   NS_ENSURE_TRUE(url, NS_ERROR_FAILURE);
1946
1947   bool nameHasChanged = false;
1948   nsresult rv;
1949
1950   // Get the old filename
1951   nsAutoCString filename;
1952   rv = url->GetFileName(filename);
1953   NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
1954   nsAutoCString directory;
1955   rv = url->GetDirectory(directory);
1956   NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
1957
1958   // Split the filename into a base and an extension.
1959   // e.g. "foo.html" becomes "foo" & ".html"
1960   //
1961   // The nsIURL methods GetFileBaseName & GetFileExtension don't
1962   // preserve the dot whereas this code does to save some effort
1963   // later when everything is put back together.
1964   int32_t lastDot = filename.RFind(".");
1965   nsAutoCString base;
1966   nsAutoCString ext;
1967   if (lastDot >= 0) {
1968     filename.Mid(base, 0, lastDot);
1969     filename.Mid(ext, lastDot, filename.Length() - lastDot);  // includes dot
1970   } else {
1971     // filename contains no dot
1972     base = filename;
1973   }
1974
1975   // Test if the filename is longer than allowed by the OS
1976   int32_t needToChop = filename.Length() - kDefaultMaxFilenameLength;
1977   if (needToChop > 0) {
1978     // Truncate the base first and then the ext if necessary
1979     if (base.Length() > (uint32_t)needToChop) {
1980       base.Truncate(base.Length() - needToChop);
1981     } else {
1982       needToChop -= base.Length() - 1;
1983       base.Truncate(1);
1984       if (ext.Length() > (uint32_t)needToChop) {
1985         ext.Truncate(ext.Length() - needToChop);
1986       } else {
1987         ext.Truncate(0);
1988       }
1989       // If kDefaultMaxFilenameLength were 1 we'd be in trouble here,
1990       // but that won't happen because it will be set to a sensible
1991       // value.
1992     }
1993
1994     filename.Assign(base);
1995     filename.Append(ext);
1996     nameHasChanged = true;
1997   }
1998
1999   // Ensure the filename is unique
2000   // Create a filename if it's empty, or if the filename / datapath is
2001   // already taken by another URI and create an alternate name.
2002
2003   if (base.IsEmpty() || !mFilenameList.IsEmpty()) {
2004     nsAutoCString tmpPath;
2005     nsAutoCString tmpBase;
2006     uint32_t duplicateCounter = 1;
2007     while (true) {
2008       // Make a file name,
2009       // Foo become foo_001, foo_002, etc.
2010       // Empty files become _001, _002 etc.
2011
2012       if (base.IsEmpty() || duplicateCounter > 1) {
2013         SmprintfPointer tmp = mozilla::Smprintf("_%03d", duplicateCounter);
2014         NS_ENSURE_TRUE(tmp, NS_ERROR_OUT_OF_MEMORY);
2015         if (filename.Length() < kDefaultMaxFilenameLength - 4) {
2016           tmpBase = base;
2017         } else {
2018           base.Mid(tmpBase, 0, base.Length() - 4);
2019         }
2020         tmpBase.Append(tmp.get());
2021       } else {
2022         tmpBase = base;
2023       }
2024
2025       tmpPath.Assign(directory);
2026       tmpPath.Append(tmpBase);
2027       tmpPath.Append(ext);
2028
2029       // Test if the name is a duplicate
2030       if (!mFilenameList.Contains(tmpPath)) {
2031         if (!base.Equals(tmpBase)) {
2032           filename.Assign(tmpBase);
2033           filename.Append(ext);
2034           nameHasChanged = true;
2035         }
2036         break;
2037       }
2038       duplicateCounter++;
2039     }
2040   }
2041
2042   // Add name to list of those already used
2043   nsAutoCString newFilepath(directory);
2044   newFilepath.Append(filename);
2045   mFilenameList.AppendElement(newFilepath);
2046
2047   // Update the uri accordingly if the filename actually changed
2048   if (nameHasChanged) {
2049     // Final sanity test
2050     if (filename.Length() > kDefaultMaxFilenameLength) {
2051       NS_WARNING(
2052           "Filename wasn't truncated less than the max file length - how can "
2053           "that be?");
2054       return NS_ERROR_FAILURE;
2055     }
2056
2057     nsCOMPtr<nsIFile> localFile;
2058     GetLocalFileFromURI(aURI, getter_AddRefs(localFile));
2059
2060     if (localFile) {
2061       nsAutoString filenameAsUnichar;
2062       CopyASCIItoUTF16(filename, filenameAsUnichar);
2063       localFile->SetLeafName(filenameAsUnichar);
2064
2065       // Resync the URI with the file after the extension has been appended
2066       return NS_MutateURI(aURI)
2067           .Apply(&nsIFileURLMutator::SetFile, localFile)
2068           .Finalize(aOutURI);
2069     }
2070     return NS_MutateURI(url)
2071         .Apply(&nsIURLMutator::SetFileName, filename, nullptr)
2072         .Finalize(aOutURI);
2073   }
2074
2075   // TODO (:valentin) This method should always clone aURI
2076   aOutURI = aURI;
2077   return NS_OK;
2078 }
2079
2080 nsresult nsWebBrowserPersist::MakeFilenameFromURI(nsIURI* aURI,
2081                                                   nsString& aFilename) {
2082   // Try to get filename from the URI.
2083   nsAutoString fileName;
2084
2085   // Get a suggested file name from the URL but strip it of characters
2086   // likely to cause the name to be illegal.
2087
2088   nsCOMPtr<nsIURL> url(do_QueryInterface(aURI));
2089   if (url) {
2090     nsAutoCString nameFromURL;
2091     url->GetFileName(nameFromURL);
2092     if (mPersistFlags & PERSIST_FLAGS_DONT_CHANGE_FILENAMES) {
2093       CopyASCIItoUTF16(NS_UnescapeURL(nameFromURL), fileName);
2094       aFilename = fileName;
2095       return NS_OK;
2096     }
2097     if (!nameFromURL.IsEmpty()) {
2098       // Unescape the file name (GetFileName escapes it)
2099       NS_UnescapeURL(nameFromURL);
2100       uint32_t nameLength = 0;
2101       const char* p = nameFromURL.get();
2102       for (; *p && *p != ';' && *p != '?' && *p != '#' && *p != '.'; p++) {
2103         if (IsAsciiAlpha(*p) || IsAsciiDigit(*p) || *p == '.' || *p == '-' ||
2104             *p == '_' || (*p == ' ')) {
2105           fileName.Append(char16_t(*p));
2106           if (++nameLength == kDefaultMaxFilenameLength) {
2107             // Note:
2108             // There is no point going any further since it will be
2109             // truncated in CalculateUniqueFilename anyway.
2110             // More importantly, certain implementations of
2111             // nsIFile (e.g. the Mac impl) might truncate
2112             // names in undesirable ways, such as truncating from
2113             // the middle, inserting ellipsis and so on.
2114             break;
2115           }
2116         }
2117       }
2118     }
2119   }
2120
2121   // Empty filenames can confuse the local file object later
2122   // when it attempts to set the leaf name in CalculateUniqueFilename
2123   // for duplicates and ends up replacing the parent dir. To avoid
2124   // the problem, all filenames are made at least one character long.
2125   if (fileName.IsEmpty()) {
2126     fileName.Append(char16_t('a'));  // 'a' is for arbitrary
2127   }
2128
2129   aFilename = fileName;
2130   return NS_OK;
2131 }
2132
2133 nsresult nsWebBrowserPersist::CalculateAndAppendFileExt(
2134     nsIURI* aURI, nsIChannel* aChannel, nsIURI* aOriginalURIWithExtension,
2135     nsCOMPtr<nsIURI>& aOutURI) {
2136   nsresult rv = NS_OK;
2137
2138   if (!mMIMEService) {
2139     mMIMEService = do_GetService(NS_MIMESERVICE_CONTRACTID, &rv);
2140     NS_ENSURE_TRUE(mMIMEService, NS_ERROR_FAILURE);
2141   }
2142
2143   nsAutoCString contentType;
2144
2145   // Get the content type from the channel
2146   aChannel->GetContentType(contentType);
2147
2148   // Get the content type from the MIME service
2149   if (contentType.IsEmpty()) {
2150     nsCOMPtr<nsIURI> uri;
2151     aChannel->GetOriginalURI(getter_AddRefs(uri));
2152     mMIMEService->GetTypeFromURI(uri, contentType);
2153   }
2154
2155   // Append the extension onto the file
2156   if (!contentType.IsEmpty()) {
2157     nsCOMPtr<nsIMIMEInfo> mimeInfo;
2158     mMIMEService->GetFromTypeAndExtension(contentType, ""_ns,
2159                                           getter_AddRefs(mimeInfo));
2160
2161     nsCOMPtr<nsIFile> localFile;
2162     GetLocalFileFromURI(aURI, getter_AddRefs(localFile));
2163
2164     if (mimeInfo) {
2165       nsCOMPtr<nsIURL> url(do_QueryInterface(aURI));
2166       NS_ENSURE_TRUE(url, NS_ERROR_FAILURE);
2167
2168       nsAutoCString newFileName;
2169       url->GetFileName(newFileName);
2170
2171       // Test if the current extension is current for the mime type
2172       bool hasExtension = false;
2173       int32_t ext = newFileName.RFind(".");
2174       if (ext != -1) {
2175         mimeInfo->ExtensionExists(Substring(newFileName, ext + 1),
2176                                   &hasExtension);
2177       }
2178
2179       // Append the mime file extension
2180       nsAutoCString fileExt;
2181       if (!hasExtension) {
2182         // Test if previous extension is acceptable
2183         nsCOMPtr<nsIURL> oldurl(do_QueryInterface(aOriginalURIWithExtension));
2184         NS_ENSURE_TRUE(oldurl, NS_ERROR_FAILURE);
2185         oldurl->GetFileExtension(fileExt);
2186         bool useOldExt = false;
2187         if (!fileExt.IsEmpty()) {
2188           mimeInfo->ExtensionExists(fileExt, &useOldExt);
2189         }
2190
2191         // If the url doesn't have an extension, or we don't know the extension,
2192         // try to use the primary extension for the type. If we don't know the
2193         // primary extension for the type, just continue with the url extension.
2194         if (!useOldExt) {
2195           nsAutoCString primaryExt;
2196           mimeInfo->GetPrimaryExtension(primaryExt);
2197           if (!primaryExt.IsEmpty()) {
2198             fileExt = primaryExt;
2199           }
2200         }
2201
2202         if (!fileExt.IsEmpty()) {
2203           uint32_t newLength = newFileName.Length() + fileExt.Length() + 1;
2204           if (newLength > kDefaultMaxFilenameLength) {
2205             if (fileExt.Length() > kDefaultMaxFilenameLength / 2)
2206               fileExt.Truncate(kDefaultMaxFilenameLength / 2);
2207
2208             uint32_t diff = kDefaultMaxFilenameLength - 1 - fileExt.Length();
2209             if (newFileName.Length() > diff) newFileName.Truncate(diff);
2210           }
2211           newFileName.Append('.');
2212           newFileName.Append(fileExt);
2213         }
2214
2215         if (localFile) {
2216           localFile->SetLeafName(NS_ConvertUTF8toUTF16(newFileName));
2217
2218           // Resync the URI with the file after the extension has been appended
2219           return NS_MutateURI(url)
2220               .Apply(&nsIFileURLMutator::SetFile, localFile)
2221               .Finalize(aOutURI);
2222         }
2223         return NS_MutateURI(url)
2224             .Apply(&nsIURLMutator::SetFileName, newFileName, nullptr)
2225             .Finalize(aOutURI);
2226       }
2227     }
2228   }
2229
2230   // TODO (:valentin) This method should always clone aURI
2231   aOutURI = aURI;
2232   return NS_OK;
2233 }
2234
2235 // Note: the MakeOutputStream helpers can be called from a background thread.
2236 nsresult nsWebBrowserPersist::MakeOutputStream(
2237     nsIURI* aURI, nsIOutputStream** aOutputStream) {
2238   nsresult rv;
2239
2240   nsCOMPtr<nsIFile> localFile;
2241   GetLocalFileFromURI(aURI, getter_AddRefs(localFile));
2242   if (localFile)
2243     rv = MakeOutputStreamFromFile(localFile, aOutputStream);
2244   else
2245     rv = MakeOutputStreamFromURI(aURI, aOutputStream);
2246
2247   return rv;
2248 }
2249
2250 nsresult nsWebBrowserPersist::MakeOutputStreamFromFile(
2251     nsIFile* aFile, nsIOutputStream** aOutputStream) {
2252   nsresult rv = NS_OK;
2253
2254   nsCOMPtr<nsIFileOutputStream> fileOutputStream =
2255       do_CreateInstance(NS_LOCALFILEOUTPUTSTREAM_CONTRACTID, &rv);
2256   NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
2257
2258   // XXX brade:  get the right flags here!
2259   int32_t ioFlags = -1;
2260   if (mPersistFlags & nsIWebBrowserPersist::PERSIST_FLAGS_APPEND_TO_FILE)
2261     ioFlags = PR_APPEND | PR_CREATE_FILE | PR_WRONLY;
2262   rv = fileOutputStream->Init(aFile, ioFlags, -1, 0);
2263   NS_ENSURE_SUCCESS(rv, rv);
2264
2265   rv = NS_NewBufferedOutputStream(aOutputStream, fileOutputStream.forget(),
2266                                   BUFFERED_OUTPUT_SIZE);
2267   NS_ENSURE_SUCCESS(rv, rv);
2268
2269   if (mPersistFlags & PERSIST_FLAGS_CLEANUP_ON_FAILURE) {
2270     // Add to cleanup list in event of failure
2271     auto* cleanupData = new CleanupData;
2272     cleanupData->mFile = aFile;
2273     cleanupData->mIsDirectory = false;
2274     if (NS_IsMainThread()) {
2275       mCleanupList.AppendElement(cleanupData);
2276     } else {
2277       // If we're on a background thread, add the cleanup back on the main
2278       // thread.
2279       RefPtr<Runnable> addCleanup = NS_NewRunnableFunction(
2280           "nsWebBrowserPersist::AddCleanupToList",
2281           [self = RefPtr{this}, cleanup = std::move(cleanupData)]() {
2282             self->mCleanupList.AppendElement(cleanup);
2283           });
2284       NS_DispatchToMainThread(addCleanup);
2285     }
2286   }
2287
2288   return NS_OK;
2289 }
2290
2291 nsresult nsWebBrowserPersist::MakeOutputStreamFromURI(
2292     nsIURI* aURI, nsIOutputStream** aOutputStream) {
2293   uint32_t segsize = 8192;
2294   uint32_t maxsize = uint32_t(-1);
2295   nsCOMPtr<nsIStorageStream> storStream;
2296   nsresult rv =
2297       NS_NewStorageStream(segsize, maxsize, getter_AddRefs(storStream));
2298   NS_ENSURE_SUCCESS(rv, rv);
2299
2300   NS_ENSURE_SUCCESS(CallQueryInterface(storStream, aOutputStream),
2301                     NS_ERROR_FAILURE);
2302   return NS_OK;
2303 }
2304
2305 void nsWebBrowserPersist::FinishDownload() {
2306   // We call FinishDownload when we run out of things to download for this
2307   // persist operation, by dispatching this method to the main thread. By now,
2308   // it's possible that we have been canceled or encountered an error earlier
2309   // in the download, or something else called EndDownload. In that case, don't
2310   // re-run EndDownload.
2311   if (mEndCalled) {
2312     return;
2313   }
2314   EndDownload(NS_OK);
2315 }
2316
2317 void nsWebBrowserPersist::EndDownload(nsresult aResult) {
2318   MOZ_ASSERT(NS_IsMainThread(), "Should end download on the main thread.");
2319
2320   // Really this should just never happen, but if it does, at least avoid
2321   // no-op notifications or pretending we succeeded if we already failed.
2322   if (mEndCalled && (NS_SUCCEEDED(aResult) || mPersistResult == aResult)) {
2323     return;
2324   }
2325
2326   // Store the error code in the result if it is an error
2327   if (NS_SUCCEEDED(mPersistResult) && NS_FAILED(aResult)) {
2328     mPersistResult = aResult;
2329   }
2330
2331   if (mEndCalled) {
2332     MOZ_ASSERT(!mEndCalled, "Should only end the download once.");
2333     return;
2334   }
2335   mEndCalled = true;
2336
2337   ClosePromise::All(GetCurrentSerialEventTarget(), mFileClosePromises)
2338       ->Then(GetCurrentSerialEventTarget(), __func__,
2339              [self = RefPtr{this}, aResult]() {
2340                self->EndDownloadInternal(aResult);
2341              });
2342 }
2343
2344 void nsWebBrowserPersist::EndDownloadInternal(nsresult aResult) {
2345   // mCompleted needs to be set before issuing the stop notification.
2346   // (Bug 1224437)
2347   mCompleted = true;
2348   // State stop notification
2349   if (mProgressListener) {
2350     mProgressListener->OnStateChange(
2351         nullptr, nullptr,
2352         nsIWebProgressListener::STATE_STOP |
2353             nsIWebProgressListener::STATE_IS_NETWORK,
2354         mPersistResult);
2355   }
2356
2357   // Do file cleanup if required
2358   if (NS_FAILED(aResult) &&
2359       (mPersistFlags & PERSIST_FLAGS_CLEANUP_ON_FAILURE)) {
2360     CleanupLocalFiles();
2361   }
2362
2363   // Cleanup the channels
2364   Cleanup();
2365
2366   mProgressListener = nullptr;
2367   mProgressListener2 = nullptr;
2368   mEventSink = nullptr;
2369 }
2370
2371 nsresult nsWebBrowserPersist::FixRedirectedChannelEntry(
2372     nsIChannel* aNewChannel) {
2373   NS_ENSURE_ARG_POINTER(aNewChannel);
2374
2375   // Iterate through existing open channels looking for one with a URI
2376   // matching the one specified.
2377   nsCOMPtr<nsIURI> originalURI;
2378   aNewChannel->GetOriginalURI(getter_AddRefs(originalURI));
2379   nsISupports* matchingKey = nullptr;
2380   for (nsISupports* key : mOutputMap.Keys()) {
2381     nsCOMPtr<nsIChannel> thisChannel = do_QueryInterface(key);
2382     nsCOMPtr<nsIURI> thisURI;
2383
2384     thisChannel->GetOriginalURI(getter_AddRefs(thisURI));
2385
2386     // Compare this channel's URI to the one passed in.
2387     bool matchingURI = false;
2388     thisURI->Equals(originalURI, &matchingURI);
2389     if (matchingURI) {
2390       matchingKey = key;
2391       break;
2392     }
2393   }
2394
2395   if (matchingKey) {
2396     // We only get called from OnStartRequest, so this is always on the
2397     // main thread. Make sure we don't pull the rug from under anything else.
2398     MutexAutoLock lock(mOutputMapMutex);
2399     // If a match was found, remove the data entry with the old channel
2400     // key and re-add it with the new channel key.
2401     mozilla::UniquePtr<OutputData> outputData;
2402     mOutputMap.Remove(matchingKey, &outputData);
2403     NS_ENSURE_TRUE(outputData, NS_ERROR_FAILURE);
2404
2405     // Store data again with new channel unless told to ignore redirects.
2406     if (!(mPersistFlags & PERSIST_FLAGS_IGNORE_REDIRECTED_DATA)) {
2407       nsCOMPtr<nsISupports> keyPtr = do_QueryInterface(aNewChannel);
2408       mOutputMap.InsertOrUpdate(keyPtr, std::move(outputData));
2409     }
2410   }
2411
2412   return NS_OK;
2413 }
2414
2415 void nsWebBrowserPersist::CalcTotalProgress() {
2416   mTotalCurrentProgress = 0;
2417   mTotalMaxProgress = 0;
2418
2419   if (mOutputMap.Count() > 0) {
2420     // Total up the progress of each output stream
2421     for (const auto& data : mOutputMap.Values()) {
2422       // Only count toward total progress if destination file is local.
2423       nsCOMPtr<nsIFileURL> fileURL = do_QueryInterface(data->mFile);
2424       if (fileURL) {
2425         mTotalCurrentProgress += data->mSelfProgress;
2426         mTotalMaxProgress += data->mSelfProgressMax;
2427       }
2428     }
2429   }
2430
2431   if (mUploadList.Count() > 0) {
2432     // Total up the progress of each upload
2433     for (const auto& data : mUploadList.Values()) {
2434       if (data) {
2435         mTotalCurrentProgress += data->mSelfProgress;
2436         mTotalMaxProgress += data->mSelfProgressMax;
2437       }
2438     }
2439   }
2440
2441   // XXX this code seems pretty bogus and pointless
2442   if (mTotalCurrentProgress == 0 && mTotalMaxProgress == 0) {
2443     // No output streams so we must be complete
2444     mTotalCurrentProgress = 10000;
2445     mTotalMaxProgress = 10000;
2446   }
2447 }
2448
2449 nsresult nsWebBrowserPersist::StoreURI(const nsACString& aURI,
2450                                        nsIWebBrowserPersistDocument* aDoc,
2451                                        nsContentPolicyType aContentPolicyType,
2452                                        bool aNeedsPersisting, URIData** aData) {
2453   nsCOMPtr<nsIURI> uri;
2454   nsresult rv = NS_NewURI(getter_AddRefs(uri), aURI, mCurrentCharset.get(),
2455                           mCurrentBaseURI);
2456   NS_ENSURE_SUCCESS(rv, rv);
2457
2458   return StoreURI(uri, aDoc, aContentPolicyType, aNeedsPersisting, aData);
2459 }
2460
2461 nsresult nsWebBrowserPersist::StoreURI(nsIURI* aURI,
2462                                        nsIWebBrowserPersistDocument* aDoc,
2463                                        nsContentPolicyType aContentPolicyType,
2464                                        bool aNeedsPersisting, URIData** aData) {
2465   NS_ENSURE_ARG_POINTER(aURI);
2466   if (aData) {
2467     *aData = nullptr;
2468   }
2469
2470   // Test if this URI should be persisted. By default
2471   // we should assume the URI  is persistable.
2472   bool doNotPersistURI;
2473   nsresult rv = NS_URIChainHasFlags(
2474       aURI, nsIProtocolHandler::URI_NON_PERSISTABLE, &doNotPersistURI);
2475   if (NS_FAILED(rv)) {
2476     doNotPersistURI = false;
2477   }
2478
2479   if (doNotPersistURI) {
2480     return NS_OK;
2481   }
2482
2483   URIData* data = nullptr;
2484   MakeAndStoreLocalFilenameInURIMap(aURI, aDoc, aContentPolicyType,
2485                                     aNeedsPersisting, &data);
2486   if (aData) {
2487     *aData = data;
2488   }
2489
2490   return NS_OK;
2491 }
2492
2493 nsresult nsWebBrowserPersist::URIData::GetLocalURI(nsIURI* targetBaseURI,
2494                                                    nsCString& aSpecOut) {
2495   aSpecOut.SetIsVoid(true);
2496   if (!mNeedsFixup) {
2497     return NS_OK;
2498   }
2499   nsresult rv;
2500   nsCOMPtr<nsIURI> fileAsURI;
2501   if (mFile) {
2502     fileAsURI = mFile;
2503   } else {
2504     fileAsURI = mDataPath;
2505     rv = AppendPathToURI(fileAsURI, mFilename, fileAsURI);
2506     NS_ENSURE_SUCCESS(rv, rv);
2507   }
2508
2509   // remove username/password if present
2510   Unused << NS_MutateURI(fileAsURI).SetUserPass(""_ns).Finalize(fileAsURI);
2511
2512   // reset node attribute
2513   // Use relative or absolute links
2514   if (mDataPathIsRelative) {
2515     bool isEqual = false;
2516     if (NS_SUCCEEDED(mRelativeDocumentURI->Equals(targetBaseURI, &isEqual)) &&
2517         isEqual) {
2518       nsCOMPtr<nsIURL> url(do_QueryInterface(fileAsURI));
2519       if (!url) {
2520         return NS_ERROR_FAILURE;
2521       }
2522
2523       nsAutoCString filename;
2524       url->GetFileName(filename);
2525
2526       nsAutoCString rawPathURL(mRelativePathToData);
2527       rawPathURL.Append(filename);
2528
2529       rv = NS_EscapeURL(rawPathURL, esc_FilePath, aSpecOut, fallible);
2530       NS_ENSURE_SUCCESS(rv, rv);
2531     } else {
2532       nsAutoCString rawPathURL;
2533
2534       nsCOMPtr<nsIFile> dataFile;
2535       rv = GetLocalFileFromURI(mFile, getter_AddRefs(dataFile));
2536       NS_ENSURE_SUCCESS(rv, rv);
2537
2538       nsCOMPtr<nsIFile> docFile;
2539       rv = GetLocalFileFromURI(targetBaseURI, getter_AddRefs(docFile));
2540       NS_ENSURE_SUCCESS(rv, rv);
2541
2542       nsCOMPtr<nsIFile> parentDir;
2543       rv = docFile->GetParent(getter_AddRefs(parentDir));
2544       NS_ENSURE_SUCCESS(rv, rv);
2545
2546       rv = dataFile->GetRelativePath(parentDir, rawPathURL);
2547       NS_ENSURE_SUCCESS(rv, rv);
2548
2549       rv = NS_EscapeURL(rawPathURL, esc_FilePath, aSpecOut, fallible);
2550       NS_ENSURE_SUCCESS(rv, rv);
2551     }
2552   } else {
2553     fileAsURI->GetSpec(aSpecOut);
2554   }
2555   if (mIsSubFrame) {
2556     AppendUTF16toUTF8(mSubFrameExt, aSpecOut);
2557   }
2558
2559   return NS_OK;
2560 }
2561
2562 bool nsWebBrowserPersist::DocumentEncoderExists(const char* aContentType) {
2563   return do_getDocumentTypeSupportedForEncoding(aContentType);
2564 }
2565
2566 nsresult nsWebBrowserPersist::SaveSubframeContent(
2567     nsIWebBrowserPersistDocument* aFrameContent,
2568     nsIWebBrowserPersistDocument* aParentDocument, const nsCString& aURISpec,
2569     URIData* aData) {
2570   NS_ENSURE_ARG_POINTER(aData);
2571
2572   // Extract the content type for the frame's contents.
2573   nsAutoCString contentType;
2574   nsresult rv = aFrameContent->GetContentType(contentType);
2575   NS_ENSURE_SUCCESS(rv, rv);
2576
2577   nsString ext;
2578   GetExtensionForContentType(NS_ConvertASCIItoUTF16(contentType).get(),
2579                              getter_Copies(ext));
2580
2581   // We must always have an extension so we will try to re-assign
2582   // the original extension if GetExtensionForContentType fails.
2583   if (ext.IsEmpty()) {
2584     nsCOMPtr<nsIURI> docURI;
2585     rv = NS_NewURI(getter_AddRefs(docURI), aURISpec, mCurrentCharset.get());
2586     NS_ENSURE_SUCCESS(rv, rv);
2587
2588     nsCOMPtr<nsIURL> url(do_QueryInterface(docURI, &rv));
2589     nsAutoCString extension;
2590     if (NS_SUCCEEDED(rv)) {
2591       url->GetFileExtension(extension);
2592     } else {
2593       extension.AssignLiteral("htm");
2594     }
2595     aData->mSubFrameExt.Assign(char16_t('.'));
2596     AppendUTF8toUTF16(extension, aData->mSubFrameExt);
2597   } else {
2598     aData->mSubFrameExt.Assign(char16_t('.'));
2599     aData->mSubFrameExt.Append(ext);
2600   }
2601
2602   nsString filenameWithExt = aData->mFilename;
2603   filenameWithExt.Append(aData->mSubFrameExt);
2604
2605   // Work out the path for the subframe
2606   nsCOMPtr<nsIURI> frameURI = mCurrentDataPath;
2607   rv = AppendPathToURI(frameURI, filenameWithExt, frameURI);
2608   NS_ENSURE_SUCCESS(rv, rv);
2609
2610   // Work out the path for the subframe data
2611   nsCOMPtr<nsIURI> frameDataURI = mCurrentDataPath;
2612   nsAutoString newFrameDataPath(aData->mFilename);
2613
2614   // Append _data
2615   newFrameDataPath.AppendLiteral("_data");
2616   rv = AppendPathToURI(frameDataURI, newFrameDataPath, frameDataURI);
2617   NS_ENSURE_SUCCESS(rv, rv);
2618
2619   // Make frame document & data path conformant and unique
2620   nsCOMPtr<nsIURI> out;
2621   rv = CalculateUniqueFilename(frameURI, out);
2622   NS_ENSURE_SUCCESS(rv, rv);
2623   frameURI = out;
2624
2625   rv = CalculateUniqueFilename(frameDataURI, out);
2626   NS_ENSURE_SUCCESS(rv, rv);
2627   frameDataURI = out;
2628
2629   mCurrentThingsToPersist++;
2630
2631   // We shouldn't use SaveDocumentInternal for the contents
2632   // of frames that are not documents, e.g. images.
2633   if (DocumentEncoderExists(contentType.get())) {
2634     auto toWalk = mozilla::MakeUnique<WalkData>();
2635     toWalk->mDocument = aFrameContent;
2636     toWalk->mFile = frameURI;
2637     toWalk->mDataPath = frameDataURI;
2638     mWalkStack.AppendElement(std::move(toWalk));
2639   } else {
2640     nsContentPolicyType policyType = nsIContentPolicy::TYPE_OTHER;
2641     if (StringBeginsWith(contentType, "image/"_ns)) {
2642       policyType = nsIContentPolicy::TYPE_IMAGE;
2643     } else if (StringBeginsWith(contentType, "audio/"_ns) ||
2644                StringBeginsWith(contentType, "video/"_ns)) {
2645       policyType = nsIContentPolicy::TYPE_MEDIA;
2646     }
2647     rv = StoreURI(aURISpec, aParentDocument, policyType);
2648   }
2649   NS_ENSURE_SUCCESS(rv, rv);
2650
2651   // Store the updated uri to the frame
2652   aData->mFile = frameURI;
2653   aData->mSubFrameExt.Truncate();  // we already put this in frameURI
2654
2655   return NS_OK;
2656 }
2657
2658 nsresult nsWebBrowserPersist::CreateChannelFromURI(nsIURI* aURI,
2659                                                    nsIChannel** aChannel) {
2660   nsresult rv = NS_OK;
2661   *aChannel = nullptr;
2662
2663   rv = NS_NewChannel(aChannel, aURI, nsContentUtils::GetSystemPrincipal(),
2664                      nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_SEC_CONTEXT_IS_NULL,
2665                      nsIContentPolicy::TYPE_OTHER);
2666   NS_ENSURE_SUCCESS(rv, rv);
2667   NS_ENSURE_ARG_POINTER(*aChannel);
2668
2669   rv = (*aChannel)->SetNotificationCallbacks(
2670       static_cast<nsIInterfaceRequestor*>(this));
2671   NS_ENSURE_SUCCESS(rv, rv);
2672   return NS_OK;
2673 }
2674
2675 // we store the current location as the key (absolutized version of domnode's
2676 // attribute's value)
2677 nsresult nsWebBrowserPersist::MakeAndStoreLocalFilenameInURIMap(
2678     nsIURI* aURI, nsIWebBrowserPersistDocument* aDoc,
2679     nsContentPolicyType aContentPolicyType, bool aNeedsPersisting,
2680     URIData** aData) {
2681   NS_ENSURE_ARG_POINTER(aURI);
2682
2683   nsAutoCString spec;
2684   nsresult rv = aURI->GetSpec(spec);
2685   NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
2686
2687   // Create a sensibly named filename for the URI and store in the URI map
2688   URIData* data;
2689   if (mURIMap.Get(spec, &data)) {
2690     if (aNeedsPersisting) {
2691       data->mNeedsPersisting = true;
2692     }
2693     if (aData) {
2694       *aData = data;
2695     }
2696     return NS_OK;
2697   }
2698
2699   // Create a unique file name for the uri
2700   nsString filename;
2701   rv = MakeFilenameFromURI(aURI, filename);
2702   NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
2703
2704   // Store the file name
2705   data = new URIData;
2706
2707   data->mContentPolicyType = aContentPolicyType;
2708   data->mNeedsPersisting = aNeedsPersisting;
2709   data->mNeedsFixup = true;
2710   data->mFilename = filename;
2711   data->mSaved = false;
2712   data->mIsSubFrame = false;
2713   data->mDataPath = mCurrentDataPath;
2714   data->mDataPathIsRelative = mCurrentDataPathIsRelative;
2715   data->mRelativePathToData = mCurrentRelativePathToData;
2716   data->mRelativeDocumentURI = mTargetBaseURI;
2717   data->mCharset = mCurrentCharset;
2718
2719   aDoc->GetPrincipal(getter_AddRefs(data->mTriggeringPrincipal));
2720   aDoc->GetCookieJarSettings(getter_AddRefs(data->mCookieJarSettings));
2721
2722   if (aNeedsPersisting) mCurrentThingsToPersist++;
2723
2724   mURIMap.InsertOrUpdate(spec, UniquePtr<URIData>(data));
2725   if (aData) {
2726     *aData = data;
2727   }
2728
2729   return NS_OK;
2730 }
2731
2732 // Decide if we need to apply conversion to the passed channel.
2733 void nsWebBrowserPersist::SetApplyConversionIfNeeded(nsIChannel* aChannel) {
2734   nsresult rv = NS_OK;
2735   nsCOMPtr<nsIEncodedChannel> encChannel = do_QueryInterface(aChannel, &rv);
2736   if (NS_FAILED(rv)) return;
2737
2738   // Set the default conversion preference:
2739   encChannel->SetApplyConversion(false);
2740
2741   nsCOMPtr<nsIURI> thisURI;
2742   aChannel->GetURI(getter_AddRefs(thisURI));
2743   nsCOMPtr<nsIURL> sourceURL(do_QueryInterface(thisURI));
2744   if (!sourceURL) return;
2745   nsAutoCString extension;
2746   sourceURL->GetFileExtension(extension);
2747
2748   nsCOMPtr<nsIUTF8StringEnumerator> encEnum;
2749   encChannel->GetContentEncodings(getter_AddRefs(encEnum));
2750   if (!encEnum) return;
2751   nsCOMPtr<nsIExternalHelperAppService> helperAppService =
2752       do_GetService(NS_EXTERNALHELPERAPPSERVICE_CONTRACTID, &rv);
2753   if (NS_FAILED(rv)) return;
2754   bool hasMore;
2755   rv = encEnum->HasMore(&hasMore);
2756   if (NS_SUCCEEDED(rv) && hasMore) {
2757     nsAutoCString encType;
2758     rv = encEnum->GetNext(encType);
2759     if (NS_SUCCEEDED(rv)) {
2760       bool applyConversion = false;
2761       rv = helperAppService->ApplyDecodingForExtension(extension, encType,
2762                                                        &applyConversion);
2763       if (NS_SUCCEEDED(rv)) encChannel->SetApplyConversion(applyConversion);
2764     }
2765   }
2766 }