parser/html/nsHtml5TreeOpExecutor.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set sw=2 ts=2 et tw=79: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "mozilla/DebugOnly.h"
   8 #include "mozilla/Likely.h"
   9 #include "mozilla/dom/nsCSPService.h"
  10
  11 #include "nsError.h"
  12 #include "nsHtml5TreeOpExecutor.h"
  13 #include "nsScriptLoader.h"
  14 #include "nsIContentViewer.h"
  15 #include "nsIContentSecurityPolicy.h"
  16 #include "nsIDocShellTreeItem.h"
  17 #include "nsIDocShell.h"
  18 #include "nsIDOMDocument.h"
  19 #include "nsIScriptGlobalObject.h"
  20 #include "nsIScriptSecurityManager.h"
  21 #include "nsIWebShellServices.h"
  22 #include "nsContentUtils.h"
  23 #include "mozAutoDocUpdate.h"
  24 #include "nsNetUtil.h"
  25 #include "nsHtml5Parser.h"
  26 #include "nsHtml5Tokenizer.h"
  27 #include "nsHtml5TreeBuilder.h"
  28 #include "nsHtml5StreamParser.h"
  29 #include "mozilla/css/Loader.h"
  30 #include "GeckoProfiler.h"
  31 #include "nsIScriptError.h"
  32 #include "nsIScriptContext.h"
  33 #include "mozilla/Preferences.h"
  34 #include "nsIHTMLDocument.h"
  35 #include "nsIViewSourceChannel.h"
  36
  37 using namespace mozilla;
  38
  39 NS_INTERFACE_TABLE_HEAD_CYCLE_COLLECTION_INHERITED(nsHtml5TreeOpExecutor)
  40   NS_INTERFACE_TABLE_INHERITED(nsHtml5TreeOpExecutor,
  41                                nsIContentSink)
  42 NS_INTERFACE_TABLE_TAIL_INHERITING(nsHtml5DocumentBuilder)
  43
  44 NS_IMPL_ADDREF_INHERITED(nsHtml5TreeOpExecutor, nsContentSink)
  45
  46 NS_IMPL_RELEASE_INHERITED(nsHtml5TreeOpExecutor, nsContentSink)
  47
  48 class nsHtml5ExecutorReflusher : public nsRunnable
  49 {
  50   private:
  51     RefPtr<nsHtml5TreeOpExecutor> mExecutor;
  52   public:
  53     explicit nsHtml5ExecutorReflusher(nsHtml5TreeOpExecutor* aExecutor)
  54       : mExecutor(aExecutor)
  55     {}
  56     NS_IMETHODIMP Run()
  57     {
  58       mExecutor->RunFlushLoop();
  59       return NS_OK;
  60     }
  61 };
  62
  63 static mozilla::LinkedList<nsHtml5TreeOpExecutor>* gBackgroundFlushList = nullptr;
  64 static nsITimer* gFlushTimer = nullptr;
  65
  66 nsHtml5TreeOpExecutor::nsHtml5TreeOpExecutor()
  67   : nsHtml5DocumentBuilder(false)
  68   , mPreloadedURLs(23)  // Mean # of preloadable resources per page on dmoz
  69   , mSpeculationReferrerPolicy(mozilla::net::RP_Default)
  70 {
  71   // zeroing operator new for everything else
  72 }
  73
  74 nsHtml5TreeOpExecutor::~nsHtml5TreeOpExecutor()
  75 {
  76   if (gBackgroundFlushList && isInList()) {
  77     mOpQueue.Clear();
  78     removeFrom(*gBackgroundFlushList);
  79     if (gBackgroundFlushList->isEmpty()) {
  80       delete gBackgroundFlushList;
  81       gBackgroundFlushList = nullptr;
  82       if (gFlushTimer) {
  83         gFlushTimer->Cancel();
  84         NS_RELEASE(gFlushTimer);
  85       }
  86     }
  87   }
  88   NS_ASSERTION(mOpQueue.IsEmpty(), "Somehow there's stuff in the op queue.");
  89 }
  90
  91 // nsIContentSink
  92 NS_IMETHODIMP
  93 nsHtml5TreeOpExecutor::WillParse()
  94 {
  95   NS_NOTREACHED("No one should call this");
  96   return NS_ERROR_NOT_IMPLEMENTED;
  97 }
  98
  99 NS_IMETHODIMP
 100 nsHtml5TreeOpExecutor::WillBuildModel(nsDTDMode aDTDMode)
 101 {
 102   mDocument->AddObserver(this);
 103   WillBuildModelImpl();
 104   GetDocument()->BeginLoad();
 105   if (mDocShell && !GetDocument()->GetWindow() &&
 106       !IsExternalViewSource()) {
 107     // Not loading as data but script global object not ready
 108     return MarkAsBroken(NS_ERROR_DOM_INVALID_STATE_ERR);
 109   }
 110   return NS_OK;
 111 }
 112
 113
 114 // This is called when the tree construction has ended
 115 NS_IMETHODIMP
 116 nsHtml5TreeOpExecutor::DidBuildModel(bool aTerminated)
 117 {
 118   if (!aTerminated) {
 119     // This is needed to avoid unblocking loads too many times on one hand
 120     // and on the other hand to avoid destroying the frame constructor from
 121     // within an update batch. See bug 537683.
 122     EndDocUpdate();
 123
 124     // If the above caused a call to nsIParser::Terminate(), let that call
 125     // win.
 126     if (!mParser) {
 127       return NS_OK;
 128     }
 129   }
 130
 131   if (mRunsToCompletion) {
 132     return NS_OK;
 133   }
 134
 135   GetParser()->DropStreamParser();
 136
 137   // This comes from nsXMLContentSink and nsHTMLContentSink
 138   // If this parser has been marked as broken, treat the end of parse as
 139   // forced termination.
 140   DidBuildModelImpl(aTerminated || NS_FAILED(IsBroken()));
 141
 142   if (!mLayoutStarted) {
 143     // We never saw the body, and layout never got started. Force
 144     // layout *now*, to get an initial reflow.
 145
 146     // NOTE: only force the layout if we are NOT destroying the
 147     // docshell. If we are destroying it, then starting layout will
 148     // likely cause us to crash, or at best waste a lot of time as we
 149     // are just going to tear it down anyway.
 150     bool destroying = true;
 151     if (mDocShell) {
 152       mDocShell->IsBeingDestroyed(&destroying);
 153     }
 154
 155     if (!destroying) {
 156       nsContentSink::StartLayout(false);
 157     }
 158   }
 159
 160   ScrollToRef();
 161   mDocument->RemoveObserver(this);
 162   if (!mParser) {
 163     // DidBuildModelImpl may cause mParser to be nulled out
 164     // Return early to avoid unblocking the onload event too many times.
 165     return NS_OK;
 166   }
 167
 168   // We may not have called BeginLoad() if loading is terminated before
 169   // OnStartRequest call.
 170   if (mStarted) {
 171     mDocument->EndLoad();
 172   }
 173   DropParserAndPerfHint();
 174 #ifdef GATHER_DOCWRITE_STATISTICS
 175   printf("UNSAFE SCRIPTS: %d\n", sUnsafeDocWrites);
 176   printf("TOKENIZER-SAFE SCRIPTS: %d\n", sTokenSafeDocWrites);
 177   printf("TREEBUILDER-SAFE SCRIPTS: %d\n", sTreeSafeDocWrites);
 178 #endif
 179 #ifdef DEBUG_NS_HTML5_TREE_OP_EXECUTOR_FLUSH
 180   printf("MAX NOTIFICATION BATCH LEN: %d\n", sAppendBatchMaxSize);
 181   if (sAppendBatchExaminations != 0) {
 182     printf("AVERAGE SLOTS EXAMINED: %d\n", sAppendBatchSlotsExamined / sAppendBatchExaminations);
 183   }
 184 #endif
 185   return NS_OK;
 186 }
 187
 188 NS_IMETHODIMP
 189 nsHtml5TreeOpExecutor::WillInterrupt()
 190 {
 191   NS_NOTREACHED("Don't call. For interface compat only.");
 192   return NS_ERROR_NOT_IMPLEMENTED;
 193 }
 194
 195 NS_IMETHODIMP
 196 nsHtml5TreeOpExecutor::WillResume()
 197 {
 198   NS_NOTREACHED("Don't call. For interface compat only.");
 199   return NS_ERROR_NOT_IMPLEMENTED;
 200 }
 201
 202 NS_IMETHODIMP
 203 nsHtml5TreeOpExecutor::SetParser(nsParserBase* aParser)
 204 {
 205   mParser = aParser;
 206   return NS_OK;
 207 }
 208
 209 void
 210 nsHtml5TreeOpExecutor::FlushPendingNotifications(mozFlushType aType)
 211 {
 212   if (aType >= Flush_InterruptibleLayout) {
 213     // Bug 577508 / 253951
 214     nsContentSink::StartLayout(true);
 215   }
 216 }
 217
 218 nsISupports*
 219 nsHtml5TreeOpExecutor::GetTarget()
 220 {
 221   return mDocument;
 222 }
 223
 224 nsresult
 225 nsHtml5TreeOpExecutor::MarkAsBroken(nsresult aReason)
 226 {
 227   NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
 228   mBroken = aReason;
 229   if (mStreamParser) {
 230     mStreamParser->Terminate();
 231   }
 232   // We are under memory pressure, but let's hope the following allocation
 233   // works out so that we get to terminate and clean up the parser from
 234   // a safer point.
 235   if (mParser) { // can mParser ever be null here?
 236     nsCOMPtr<nsIRunnable> terminator =
 237       NS_NewRunnableMethod(GetParser(), &nsHtml5Parser::Terminate);
 238     if (NS_FAILED(NS_DispatchToMainThread(terminator))) {
 239       NS_WARNING("failed to dispatch executor flush event");
 240     }
 241   }
 242   return aReason;
 243 }
 244
 245 void
 246 FlushTimerCallback(nsITimer* aTimer, void* aClosure)
 247 {
 248   RefPtr<nsHtml5TreeOpExecutor> ex = gBackgroundFlushList->popFirst();
 249   if (ex) {
 250     ex->RunFlushLoop();
 251   }
 252   if (gBackgroundFlushList && gBackgroundFlushList->isEmpty()) {
 253     delete gBackgroundFlushList;
 254     gBackgroundFlushList = nullptr;
 255     gFlushTimer->Cancel();
 256     NS_RELEASE(gFlushTimer);
 257   }
 258 }
 259
 260 void
 261 nsHtml5TreeOpExecutor::ContinueInterruptedParsingAsync()
 262 {
 263   if (!mDocument || !mDocument->IsInBackgroundWindow()) {
 264     nsCOMPtr<nsIRunnable> flusher = new nsHtml5ExecutorReflusher(this);
 265     if (NS_FAILED(NS_DispatchToMainThread(flusher))) {
 266       NS_WARNING("failed to dispatch executor flush event");
 267     }
 268   } else {
 269     if (!gBackgroundFlushList) {
 270       gBackgroundFlushList = new mozilla::LinkedList<nsHtml5TreeOpExecutor>();
 271     }
 272     if (!isInList()) {
 273       gBackgroundFlushList->insertBack(this);
 274     }
 275     if (!gFlushTimer) {
 276       nsCOMPtr<nsITimer> t = do_CreateInstance("@mozilla.org/timer;1");
 277       t.swap(gFlushTimer);
 278       // The timer value 50 should not hopefully slow down background pages too
 279       // much, yet lets event loop to process enough between ticks.
 280       // See bug 734015.
 281       gFlushTimer->InitWithNamedFuncCallback(FlushTimerCallback, nullptr,
 282                                              50, nsITimer::TYPE_REPEATING_SLACK,
 283                                              "FlushTimerCallback");
 284     }
 285   }
 286 }
 287
 288 void
 289 nsHtml5TreeOpExecutor::FlushSpeculativeLoads()
 290 {
 291   nsTArray<nsHtml5SpeculativeLoad> speculativeLoadQueue;
 292   mStage.MoveSpeculativeLoadsTo(speculativeLoadQueue);
 293   const nsHtml5SpeculativeLoad* start = speculativeLoadQueue.Elements();
 294   const nsHtml5SpeculativeLoad* end = start + speculativeLoadQueue.Length();
 295   for (nsHtml5SpeculativeLoad* iter = const_cast<nsHtml5SpeculativeLoad*>(start);
 296        iter < end;
 297        ++iter) {
 298     if (MOZ_UNLIKELY(!mParser)) {
 299       // An extension terminated the parser from a HTTP observer.
 300       return;
 301     }
 302     iter->Perform(this);
 303   }
 304 }
 305
 306 class nsHtml5FlushLoopGuard
 307 {
 308   private:
 309     RefPtr<nsHtml5TreeOpExecutor> mExecutor;
 310     #ifdef DEBUG_NS_HTML5_TREE_OP_EXECUTOR_FLUSH
 311     uint32_t mStartTime;
 312     #endif
 313   public:
 314     explicit nsHtml5FlushLoopGuard(nsHtml5TreeOpExecutor* aExecutor)
 315       : mExecutor(aExecutor)
 316     #ifdef DEBUG_NS_HTML5_TREE_OP_EXECUTOR_FLUSH
 317       , mStartTime(PR_IntervalToMilliseconds(PR_IntervalNow()))
 318     #endif
 319     {
 320       mExecutor->mRunFlushLoopOnStack = true;
 321     }
 322     ~nsHtml5FlushLoopGuard()
 323     {
 324       #ifdef DEBUG_NS_HTML5_TREE_OP_EXECUTOR_FLUSH
 325         uint32_t timeOffTheEventLoop =
 326           PR_IntervalToMilliseconds(PR_IntervalNow()) - mStartTime;
 327         if (timeOffTheEventLoop >
 328             nsHtml5TreeOpExecutor::sLongestTimeOffTheEventLoop) {
 329           nsHtml5TreeOpExecutor::sLongestTimeOffTheEventLoop =
 330             timeOffTheEventLoop;
 331         }
 332         printf("Longest time off the event loop: %d\n",
 333           nsHtml5TreeOpExecutor::sLongestTimeOffTheEventLoop);
 334       #endif
 335
 336       mExecutor->mRunFlushLoopOnStack = false;
 337     }
 338 };
 339
 340 /**
 341  * The purpose of the loop here is to avoid returning to the main event loop
 342  */
 343 void
 344 nsHtml5TreeOpExecutor::RunFlushLoop()
 345 {
 346   PROFILER_LABEL("nsHtml5TreeOpExecutor", "RunFlushLoop",
 347     js::ProfileEntry::Category::OTHER);
 348
 349   if (mRunFlushLoopOnStack) {
 350     // There's already a RunFlushLoop() on the call stack.
 351     return;
 352   }
 353
 354   nsHtml5FlushLoopGuard guard(this); // this is also the self-kungfu!
 355
 356   nsCOMPtr<nsISupports> parserKungFuDeathGrip(mParser);
 357
 358   // Remember the entry time
 359   (void) nsContentSink::WillParseImpl();
 360
 361   for (;;) {
 362     if (!mParser) {
 363       // Parse has terminated.
 364       mOpQueue.Clear(); // clear in order to be able to assert in destructor
 365       return;
 366     }
 367
 368     if (NS_FAILED(IsBroken())) {
 369       return;
 370     }
 371
 372     if (!mParser->IsParserEnabled()) {
 373       // The parser is blocked.
 374       return;
 375     }
 376
 377     if (mFlushState != eNotFlushing) {
 378       // XXX Can this happen? In case it can, let's avoid crashing.
 379       return;
 380     }
 381
 382     // If there are scripts executing, then the content sink is jumping the gun
 383     // (probably due to a synchronous XMLHttpRequest) and will re-enable us
 384     // later, see bug 460706.
 385     if (IsScriptExecuting()) {
 386       return;
 387     }
 388
 389     if (mReadingFromStage) {
 390       nsTArray<nsHtml5SpeculativeLoad> speculativeLoadQueue;
 391       mStage.MoveOpsAndSpeculativeLoadsTo(mOpQueue, speculativeLoadQueue);
 392       // Make sure speculative loads never start after the corresponding
 393       // normal loads for the same URLs.
 394       const nsHtml5SpeculativeLoad* start = speculativeLoadQueue.Elements();
 395       const nsHtml5SpeculativeLoad* end = start + speculativeLoadQueue.Length();
 396       for (nsHtml5SpeculativeLoad* iter = (nsHtml5SpeculativeLoad*)start;
 397            iter < end;
 398            ++iter) {
 399         iter->Perform(this);
 400         if (MOZ_UNLIKELY(!mParser)) {
 401           // An extension terminated the parser from a HTTP observer.
 402           mOpQueue.Clear(); // clear in order to be able to assert in destructor
 403           return;
 404         }
 405       }
 406     } else {
 407       FlushSpeculativeLoads(); // Make sure speculative loads never start after
 408                                // the corresponding normal loads for the same
 409                                // URLs.
 410       if (MOZ_UNLIKELY(!mParser)) {
 411         // An extension terminated the parser from a HTTP observer.
 412         mOpQueue.Clear(); // clear in order to be able to assert in destructor
 413         return;
 414       }
 415       // Not sure if this grip is still needed, but previously, the code
 416       // gripped before calling ParseUntilBlocked();
 417       RefPtr<nsHtml5StreamParser> streamKungFuDeathGrip =
 418         GetParser()->GetStreamParser();
 419       // Now parse content left in the document.write() buffer queue if any.
 420       // This may generate tree ops on its own or dequeue a speculation.
 421       nsresult rv = GetParser()->ParseUntilBlocked();
 422       if (NS_FAILED(rv)) {
 423         MarkAsBroken(rv);
 424         return;
 425       }
 426     }
 427
 428     if (mOpQueue.IsEmpty()) {
 429       // Avoid bothering the rest of the engine with a doc update if there's
 430       // nothing to do.
 431       return;
 432     }
 433
 434     mFlushState = eInFlush;
 435
 436     nsIContent* scriptElement = nullptr;
 437
 438     BeginDocUpdate();
 439
 440     uint32_t numberOfOpsToFlush = mOpQueue.Length();
 441
 442     const nsHtml5TreeOperation* first = mOpQueue.Elements();
 443     const nsHtml5TreeOperation* last = first + numberOfOpsToFlush - 1;
 444     for (nsHtml5TreeOperation* iter = const_cast<nsHtml5TreeOperation*>(first);;) {
 445       if (MOZ_UNLIKELY(!mParser)) {
 446         // The previous tree op caused a call to nsIParser::Terminate().
 447         break;
 448       }
 449       NS_ASSERTION(mFlushState == eInDocUpdate,
 450         "Tried to perform tree op outside update batch.");
 451       nsresult rv = iter->Perform(this, &scriptElement);
 452       if (NS_FAILED(rv)) {
 453         MarkAsBroken(rv);
 454         break;
 455       }
 456
 457       // Be sure not to check the deadline if the last op was just performed.
 458       if (MOZ_UNLIKELY(iter == last)) {
 459         break;
 460       } else if (MOZ_UNLIKELY(nsContentSink::DidProcessATokenImpl() ==
 461                  NS_ERROR_HTMLPARSER_INTERRUPTED)) {
 462         mOpQueue.RemoveElementsAt(0, (iter - first) + 1);
 463
 464         EndDocUpdate();
 465
 466         mFlushState = eNotFlushing;
 467
 468         #ifdef DEBUG_NS_HTML5_TREE_OP_EXECUTOR_FLUSH
 469           printf("REFLUSH SCHEDULED (executing ops): %d\n",
 470             ++sTimesFlushLoopInterrupted);
 471         #endif
 472         nsHtml5TreeOpExecutor::ContinueInterruptedParsingAsync();
 473         return;
 474       }
 475       ++iter;
 476     }
 477
 478     mOpQueue.Clear();
 479
 480     EndDocUpdate();
 481
 482     mFlushState = eNotFlushing;
 483
 484     if (MOZ_UNLIKELY(!mParser)) {
 485       // The parse ended already.
 486       return;
 487     }
 488
 489     if (scriptElement) {
 490       // must be tail call when mFlushState is eNotFlushing
 491       RunScript(scriptElement);
 492
 493       // Always check the clock in nsContentSink right after a script
 494       StopDeflecting();
 495       if (nsContentSink::DidProcessATokenImpl() ==
 496           NS_ERROR_HTMLPARSER_INTERRUPTED) {
 497         #ifdef DEBUG_NS_HTML5_TREE_OP_EXECUTOR_FLUSH
 498           printf("REFLUSH SCHEDULED (after script): %d\n",
 499             ++sTimesFlushLoopInterrupted);
 500         #endif
 501         nsHtml5TreeOpExecutor::ContinueInterruptedParsingAsync();
 502         return;
 503       }
 504     }
 505   }
 506 }
 507
 508 nsresult
 509 nsHtml5TreeOpExecutor::FlushDocumentWrite()
 510 {
 511   nsresult rv = IsBroken();
 512   NS_ENSURE_SUCCESS(rv, rv);
 513
 514   FlushSpeculativeLoads(); // Make sure speculative loads never start after the
 515                 // corresponding normal loads for the same URLs.
 516
 517   if (MOZ_UNLIKELY(!mParser)) {
 518     // The parse has ended.
 519     mOpQueue.Clear(); // clear in order to be able to assert in destructor
 520     return rv;
 521   }
 522
 523   if (mFlushState != eNotFlushing) {
 524     // XXX Can this happen? In case it can, let's avoid crashing.
 525     return rv;
 526   }
 527
 528   mFlushState = eInFlush;
 529
 530   // avoid crashing near EOF
 531   RefPtr<nsHtml5TreeOpExecutor> kungFuDeathGrip(this);
 532   RefPtr<nsParserBase> parserKungFuDeathGrip(mParser);
 533
 534   NS_ASSERTION(!mReadingFromStage,
 535     "Got doc write flush when reading from stage");
 536
 537 #ifdef DEBUG
 538   mStage.AssertEmpty();
 539 #endif
 540
 541   nsIContent* scriptElement = nullptr;
 542
 543   BeginDocUpdate();
 544
 545   uint32_t numberOfOpsToFlush = mOpQueue.Length();
 546
 547   const nsHtml5TreeOperation* start = mOpQueue.Elements();
 548   const nsHtml5TreeOperation* end = start + numberOfOpsToFlush;
 549   for (nsHtml5TreeOperation* iter = const_cast<nsHtml5TreeOperation*>(start);
 550        iter < end;
 551        ++iter) {
 552     if (MOZ_UNLIKELY(!mParser)) {
 553       // The previous tree op caused a call to nsIParser::Terminate().
 554       break;
 555     }
 556     NS_ASSERTION(mFlushState == eInDocUpdate,
 557       "Tried to perform tree op outside update batch.");
 558     rv = iter->Perform(this, &scriptElement);
 559     if (NS_FAILED(rv)) {
 560       MarkAsBroken(rv);
 561       break;
 562     }
 563   }
 564
 565   mOpQueue.Clear();
 566
 567   EndDocUpdate();
 568
 569   mFlushState = eNotFlushing;
 570
 571   if (MOZ_UNLIKELY(!mParser)) {
 572     // Ending the doc update caused a call to nsIParser::Terminate().
 573     return rv;
 574   }
 575
 576   if (scriptElement) {
 577     // must be tail call when mFlushState is eNotFlushing
 578     RunScript(scriptElement);
 579   }
 580   return rv;
 581 }
 582
 583 // copied from HTML content sink
 584 bool
 585 nsHtml5TreeOpExecutor::IsScriptEnabled()
 586 {
 587   if (!mDocument || !mDocShell)
 588     return true;
 589   nsCOMPtr<nsIScriptGlobalObject> globalObject = do_QueryInterface(mDocument->GetInnerWindow());
 590   // Getting context is tricky if the document hasn't had its
 591   // GlobalObject set yet
 592   if (!globalObject) {
 593     globalObject = mDocShell->GetScriptGlobalObject();
 594     NS_ENSURE_TRUE(globalObject, true);
 595   }
 596   NS_ENSURE_TRUE(globalObject && globalObject->GetGlobalJSObject(), true);
 597   return nsContentUtils::GetSecurityManager()->
 598            ScriptAllowed(globalObject->GetGlobalJSObject());
 599 }
 600
 601 void
 602 nsHtml5TreeOpExecutor::StartLayout() {
 603   if (mLayoutStarted || !mDocument) {
 604     return;
 605   }
 606
 607   EndDocUpdate();
 608
 609   if (MOZ_UNLIKELY(!mParser)) {
 610     // got terminate
 611     return;
 612   }
 613
 614   nsContentSink::StartLayout(false);
 615
 616   BeginDocUpdate();
 617 }
 618
 619 /**
 620  * The reason why this code is here and not in the tree builder even in the
 621  * main-thread case is to allow the control to return from the tokenizer
 622  * before scripts run. This way, the tokenizer is not invoked re-entrantly
 623  * although the parser is.
 624  *
 625  * The reason why this is called as a tail call when mFlushState is set to
 626  * eNotFlushing is to allow re-entry to Flush() but only after the current
 627  * Flush() has cleared the op queue and is otherwise done cleaning up after
 628  * itself.
 629  */
 630 void
 631 nsHtml5TreeOpExecutor::RunScript(nsIContent* aScriptElement)
 632 {
 633   if (mRunsToCompletion) {
 634     // We are in createContextualFragment() or in the upcoming document.parse().
 635     // Do nothing. Let's not even mark scripts malformed here, because that
 636     // could cause serialization weirdness later.
 637     return;
 638   }
 639
 640   NS_ASSERTION(aScriptElement, "No script to run");
 641   nsCOMPtr<nsIScriptElement> sele = do_QueryInterface(aScriptElement);
 642
 643   if (!mParser) {
 644     NS_ASSERTION(sele->IsMalformed(), "Script wasn't marked as malformed.");
 645     // We got here not because of an end tag but because the tree builder
 646     // popped an incomplete script element on EOF. Returning here to avoid
 647     // calling back into mParser anymore.
 648     return;
 649   }
 650
 651   if (sele->GetScriptDeferred() || sele->GetScriptAsync()) {
 652     DebugOnly<bool> block = sele->AttemptToExecute();
 653     NS_ASSERTION(!block, "Defer or async script tried to block.");
 654     return;
 655   }
 656
 657   NS_ASSERTION(mFlushState == eNotFlushing, "Tried to run script when flushing.");
 658
 659   mReadingFromStage = false;
 660
 661   sele->SetCreatorParser(GetParser());
 662
 663   // Copied from nsXMLContentSink
 664   // Now tell the script that it's ready to go. This may execute the script
 665   // or return true, or neither if the script doesn't need executing.
 666   bool block = sele->AttemptToExecute();
 667
 668   // If the act of insertion evaluated the script, we're fine.
 669   // Else, block the parser till the script has loaded.
 670   if (block) {
 671     if (mParser) {
 672       GetParser()->BlockParser();
 673     }
 674   } else {
 675     // mParser may have been nulled out by now, but the flusher deals
 676
 677     // If this event isn't needed, it doesn't do anything. It is sometimes
 678     // necessary for the parse to continue after complex situations.
 679     nsHtml5TreeOpExecutor::ContinueInterruptedParsingAsync();
 680   }
 681 }
 682
 683 void
 684 nsHtml5TreeOpExecutor::Start()
 685 {
 686   NS_PRECONDITION(!mStarted, "Tried to start when already started.");
 687   mStarted = true;
 688 }
 689
 690 void
 691 nsHtml5TreeOpExecutor::NeedsCharsetSwitchTo(const char* aEncoding,
 692                                             int32_t aSource,
 693                                             uint32_t aLineNumber)
 694 {
 695   EndDocUpdate();
 696
 697   if (MOZ_UNLIKELY(!mParser)) {
 698     // got terminate
 699     return;
 700   }
 701
 702   nsCOMPtr<nsIWebShellServices> wss = do_QueryInterface(mDocShell);
 703   if (!wss) {
 704     return;
 705   }
 706
 707   // ask the webshellservice to load the URL
 708   if (NS_SUCCEEDED(wss->StopDocumentLoad())) {
 709     wss->ReloadDocument(aEncoding, aSource);
 710   }
 711   // if the charset switch was accepted, wss has called Terminate() on the
 712   // parser by now
 713
 714   if (!mParser) {
 715     // success
 716     if (aSource == kCharsetFromMetaTag) {
 717       MaybeComplainAboutCharset("EncLateMetaReload", false, aLineNumber);
 718     }
 719     return;
 720   }
 721
 722   if (aSource == kCharsetFromMetaTag) {
 723     MaybeComplainAboutCharset("EncLateMetaTooLate", true, aLineNumber);
 724   }
 725
 726   GetParser()->ContinueAfterFailedCharsetSwitch();
 727
 728   BeginDocUpdate();
 729 }
 730
 731 void
 732 nsHtml5TreeOpExecutor::MaybeComplainAboutCharset(const char* aMsgId,
 733                                                  bool aError,
 734                                                  uint32_t aLineNumber)
 735 {
 736   if (mAlreadyComplainedAboutCharset) {
 737     return;
 738   }
 739   // The EncNoDeclaration case for advertising iframes is so common that it
 740   // would result is way too many errors. The iframe case doesn't matter
 741   // when the ad is an image or a Flash animation anyway. When the ad is
 742   // textual, a misrendered ad probably isn't a huge loss for users.
 743   // Let's suppress the message in this case.
 744   // This means that errors about other different-origin iframes in mashups
 745   // are lost as well, but generally, the site author isn't in control of
 746   // the embedded different-origin pages anyway and can't fix problems even
 747   // if alerted about them.
 748   if (!strcmp(aMsgId, "EncNoDeclaration") && mDocShell) {
 749     nsCOMPtr<nsIDocShellTreeItem> parent;
 750     mDocShell->GetSameTypeParent(getter_AddRefs(parent));
 751     if (parent) {
 752       return;
 753     }
 754   }
 755   mAlreadyComplainedAboutCharset = true;
 756   nsContentUtils::ReportToConsole(aError ? nsIScriptError::errorFlag
 757                                          : nsIScriptError::warningFlag,
 758                                   NS_LITERAL_CSTRING("HTML parser"),
 759                                   mDocument,
 760                                   nsContentUtils::eHTMLPARSER_PROPERTIES,
 761                                   aMsgId,
 762                                   nullptr,
 763                                   0,
 764                                   nullptr,
 765                                   EmptyString(),
 766                                   aLineNumber);
 767 }
 768
 769 void
 770 nsHtml5TreeOpExecutor::ComplainAboutBogusProtocolCharset(nsIDocument* aDoc)
 771 {
 772   NS_ASSERTION(!mAlreadyComplainedAboutCharset,
 773                "How come we already managed to complain?");
 774   mAlreadyComplainedAboutCharset = true;
 775   nsContentUtils::ReportToConsole(nsIScriptError::errorFlag,
 776                                   NS_LITERAL_CSTRING("HTML parser"),
 777                                   aDoc,
 778                                   nsContentUtils::eHTMLPARSER_PROPERTIES,
 779                                   "EncProtocolUnsupported");
 780 }
 781
 782 nsHtml5Parser*
 783 nsHtml5TreeOpExecutor::GetParser()
 784 {
 785   MOZ_ASSERT(!mRunsToCompletion);
 786   return static_cast<nsHtml5Parser*>(mParser.get());
 787 }
 788
 789 void
 790 nsHtml5TreeOpExecutor::MoveOpsFrom(nsTArray<nsHtml5TreeOperation>& aOpQueue)
 791 {
 792   NS_PRECONDITION(mFlushState == eNotFlushing, "mOpQueue modified during tree op execution.");
 793   mOpQueue.AppendElements(Move(aOpQueue));
 794 }
 795
 796 void
 797 nsHtml5TreeOpExecutor::InitializeDocWriteParserState(nsAHtml5TreeBuilderState* aState, int32_t aLine)
 798 {
 799   GetParser()->InitializeDocWriteParserState(aState, aLine);
 800 }
 801
 802 nsIURI*
 803 nsHtml5TreeOpExecutor::GetViewSourceBaseURI()
 804 {
 805   if (!mViewSourceBaseURI) {
 806
 807     // We query the channel for the baseURI because in certain situations it
 808     // cannot otherwise be determined. If this process fails, fall back to the
 809     // standard method.
 810     nsCOMPtr<nsIViewSourceChannel> vsc =
 811       do_QueryInterface(mDocument->GetChannel());
 812     if (vsc) {
 813       nsresult rv =  vsc->GetBaseURI(getter_AddRefs(mViewSourceBaseURI));
 814       if (NS_SUCCEEDED(rv) && mViewSourceBaseURI) {
 815         return mViewSourceBaseURI;
 816       }
 817     }
 818
 819     nsCOMPtr<nsIURI> orig = mDocument->GetOriginalURI();
 820     bool isViewSource;
 821     orig->SchemeIs("view-source", &isViewSource);
 822     if (isViewSource) {
 823       nsCOMPtr<nsINestedURI> nested = do_QueryInterface(orig);
 824       NS_ASSERTION(nested, "URI with scheme view-source didn't QI to nested!");
 825       nested->GetInnerURI(getter_AddRefs(mViewSourceBaseURI));
 826     } else {
 827       // Fail gracefully if the base URL isn't a view-source: URL.
 828       // Not sure if this can ever happen.
 829       mViewSourceBaseURI = orig;
 830     }
 831   }
 832   return mViewSourceBaseURI;
 833 }
 834
 835 //static
 836 void
 837 nsHtml5TreeOpExecutor::InitializeStatics()
 838 {
 839   mozilla::Preferences::AddBoolVarCache(&sExternalViewSource,
 840                                         "view_source.editor.external");
 841 }
 842
 843 bool
 844 nsHtml5TreeOpExecutor::IsExternalViewSource()
 845 {
 846   if (!sExternalViewSource) {
 847     return false;
 848   }
 849   bool isViewSource = false;
 850   if (mDocumentURI) {
 851     mDocumentURI->SchemeIs("view-source", &isViewSource);
 852   }
 853   return isViewSource;
 854 }
 855
 856 // Speculative loading
 857
 858 nsIURI*
 859 nsHtml5TreeOpExecutor::BaseURIForPreload()
 860 {
 861   // The URL of the document without <base>
 862   nsIURI* documentURI = mDocument->GetDocumentURI();
 863   // The URL of the document with non-speculative <base>
 864   nsIURI* documentBaseURI = mDocument->GetDocBaseURI();
 865
 866   // If the two above are different, use documentBaseURI. If they are the same,
 867   // the document object isn't aware of a <base>, so attempt to use the
 868   // mSpeculationBaseURI or, failing, that, documentURI.
 869   return (documentURI == documentBaseURI) ?
 870           (mSpeculationBaseURI ?
 871            mSpeculationBaseURI.get() : documentURI)
 872          : documentBaseURI;
 873 }
 874
 875 already_AddRefed<nsIURI>
 876 nsHtml5TreeOpExecutor::ConvertIfNotPreloadedYet(const nsAString& aURL)
 877 {
 878   if (aURL.IsEmpty()) {
 879     return nullptr;
 880   }
 881
 882   nsIURI* base = BaseURIForPreload();
 883   const nsCString& charset = mDocument->GetDocumentCharacterSet();
 884   nsCOMPtr<nsIURI> uri;
 885   nsresult rv = NS_NewURI(getter_AddRefs(uri), aURL, charset.get(), base);
 886   if (NS_FAILED(rv)) {
 887     NS_WARNING("Failed to create a URI");
 888     return nullptr;
 889   }
 890
 891   if (ShouldPreloadURI(uri)) {
 892     return uri.forget();
 893   }
 894
 895   return nullptr;
 896 }
 897
 898 bool
 899 nsHtml5TreeOpExecutor::ShouldPreloadURI(nsIURI *aURI)
 900 {
 901   nsAutoCString spec;
 902   aURI->GetSpec(spec);
 903   if (mPreloadedURLs.Contains(spec)) {
 904     return false;
 905   }
 906   mPreloadedURLs.PutEntry(spec);
 907   return true;
 908 }
 909
 910 void
 911 nsHtml5TreeOpExecutor::PreloadScript(const nsAString& aURL,
 912                                      const nsAString& aCharset,
 913                                      const nsAString& aType,
 914                                      const nsAString& aCrossOrigin,
 915                                      const nsAString& aIntegrity,
 916                                      bool aScriptFromHead)
 917 {
 918   nsCOMPtr<nsIURI> uri = ConvertIfNotPreloadedYet(aURL);
 919   if (!uri) {
 920     return;
 921   }
 922   mDocument->ScriptLoader()->PreloadURI(uri, aCharset, aType, aCrossOrigin,
 923                                         aIntegrity, aScriptFromHead,
 924                                         mSpeculationReferrerPolicy);
 925 }
 926
 927 void
 928 nsHtml5TreeOpExecutor::PreloadStyle(const nsAString& aURL,
 929                                     const nsAString& aCharset,
 930                                     const nsAString& aCrossOrigin,
 931                                     const nsAString& aIntegrity)
 932 {
 933   nsCOMPtr<nsIURI> uri = ConvertIfNotPreloadedYet(aURL);
 934   if (!uri) {
 935     return;
 936   }
 937   mDocument->PreloadStyle(uri, aCharset, aCrossOrigin,
 938                           mSpeculationReferrerPolicy, aIntegrity);
 939 }
 940
 941 void
 942 nsHtml5TreeOpExecutor::PreloadImage(const nsAString& aURL,
 943                                     const nsAString& aCrossOrigin,
 944                                     const nsAString& aSrcset,
 945                                     const nsAString& aSizes,
 946                                     const nsAString& aImageReferrerPolicy)
 947 {
 948   nsCOMPtr<nsIURI> baseURI = BaseURIForPreload();
 949   nsCOMPtr<nsIURI> uri = mDocument->ResolvePreloadImage(baseURI, aURL, aSrcset,
 950                                                         aSizes);
 951   if (uri && ShouldPreloadURI(uri)) {
 952     // use document wide referrer policy
 953     mozilla::net::ReferrerPolicy referrerPolicy = mSpeculationReferrerPolicy;
 954     // if enabled in preferences, use the referrer attribute from the image, if provided
 955     bool referrerAttributeEnabled = Preferences::GetBool("network.http.enablePerElementReferrer", false);
 956     if (referrerAttributeEnabled) {
 957       mozilla::net::ReferrerPolicy imageReferrerPolicy = mozilla::net::ReferrerPolicyFromString(aImageReferrerPolicy);
 958       if (imageReferrerPolicy != mozilla::net::RP_Unset) {
 959         referrerPolicy = imageReferrerPolicy;
 960       }
 961     }
 962
 963     mDocument->MaybePreLoadImage(uri, aCrossOrigin, referrerPolicy);
 964   }
 965 }
 966
 967 // These calls inform the document of picture state and seen sources, such that
 968 // it can use them to inform ResolvePreLoadImage as necessary
 969 void
 970 nsHtml5TreeOpExecutor::PreloadPictureSource(const nsAString& aSrcset,
 971                                             const nsAString& aSizes,
 972                                             const nsAString& aType,
 973                                             const nsAString& aMedia)
 974 {
 975   mDocument->PreloadPictureImageSource(aSrcset, aSizes, aType, aMedia);
 976 }
 977
 978 void
 979 nsHtml5TreeOpExecutor::PreloadOpenPicture()
 980 {
 981   mDocument->PreloadPictureOpened();
 982 }
 983
 984 void
 985 nsHtml5TreeOpExecutor::PreloadEndPicture()
 986 {
 987   mDocument->PreloadPictureClosed();
 988 }
 989
 990 void
 991 nsHtml5TreeOpExecutor::AddBase(const nsAString& aURL)
 992 {
 993   const nsCString& charset = mDocument->GetDocumentCharacterSet();
 994   nsresult rv = NS_NewURI(getter_AddRefs(mViewSourceBaseURI), aURL,
 995                                      charset.get(), GetViewSourceBaseURI());
 996   if (NS_FAILED(rv)) {
 997     mViewSourceBaseURI = nullptr;
 998   }
 999 }
1000 void
1001 nsHtml5TreeOpExecutor::SetSpeculationBase(const nsAString& aURL)
1002 {
1003   if (mSpeculationBaseURI) {
1004     // the first one wins
1005     return;
1006   }
1007   const nsCString& charset = mDocument->GetDocumentCharacterSet();
1008   DebugOnly<nsresult> rv = NS_NewURI(getter_AddRefs(mSpeculationBaseURI), aURL,
1009                                      charset.get(), mDocument->GetDocumentURI());
1010   NS_WARN_IF_FALSE(NS_SUCCEEDED(rv), "Failed to create a URI");
1011 }
1012
1013 void
1014 nsHtml5TreeOpExecutor::SetSpeculationReferrerPolicy(const nsAString& aReferrerPolicy)
1015 {
1016   ReferrerPolicy policy = mozilla::net::ReferrerPolicyFromString(aReferrerPolicy);
1017   return SetSpeculationReferrerPolicy(policy);
1018 }
1019
1020 void
1021 nsHtml5TreeOpExecutor::AddSpeculationCSP(const nsAString& aCSP)
1022 {
1023   if (!CSPService::sCSPEnabled) {
1024     return;
1025   }
1026
1027   NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
1028
1029   nsIPrincipal* principal = mDocument->NodePrincipal();
1030   nsCOMPtr<nsIContentSecurityPolicy> preloadCsp;
1031   nsCOMPtr<nsIDOMDocument> domDoc = do_QueryInterface(mDocument);
1032   nsresult rv = principal->EnsurePreloadCSP(domDoc, getter_AddRefs(preloadCsp));
1033   NS_ENSURE_SUCCESS_VOID(rv);
1034
1035   // please note that meta CSPs and CSPs delivered through a header need
1036   // to be joined together.
1037   rv = preloadCsp->AppendPolicy(aCSP,
1038                                 false, // csp via meta tag can not be report only
1039                                 true); // delivered through the meta tag
1040   NS_ENSURE_SUCCESS_VOID(rv);
1041
1042   // Record "speculated" referrer policy for preloads
1043   bool hasReferrerPolicy = false;
1044   uint32_t referrerPolicy = mozilla::net::RP_Default;
1045   rv = preloadCsp->GetReferrerPolicy(&referrerPolicy, &hasReferrerPolicy);
1046   NS_ENSURE_SUCCESS_VOID(rv);
1047   if (hasReferrerPolicy) {
1048     SetSpeculationReferrerPolicy(static_cast<ReferrerPolicy>(referrerPolicy));
1049   }
1050
1051   mDocument->ApplySettingsFromCSP(true);
1052 }
1053
1054 void
1055 nsHtml5TreeOpExecutor::SetSpeculationReferrerPolicy(ReferrerPolicy aReferrerPolicy)
1056 {
1057   // Record "speculated" referrer policy locally and thread through the
1058   // speculation phase.  The actual referrer policy will be set by
1059   // HTMLMetaElement::BindToTree().
1060   mSpeculationReferrerPolicy = aReferrerPolicy;
1061 }
1062
1063 #ifdef DEBUG_NS_HTML5_TREE_OP_EXECUTOR_FLUSH
1064 uint32_t nsHtml5TreeOpExecutor::sAppendBatchMaxSize = 0;
1065 uint32_t nsHtml5TreeOpExecutor::sAppendBatchSlotsExamined = 0;
1066 uint32_t nsHtml5TreeOpExecutor::sAppendBatchExaminations = 0;
1067 uint32_t nsHtml5TreeOpExecutor::sLongestTimeOffTheEventLoop = 0;
1068 uint32_t nsHtml5TreeOpExecutor::sTimesFlushLoopInterrupted = 0;
1069 #endif
1070 bool nsHtml5TreeOpExecutor::sExternalViewSource = false;