1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set sw=2 ts=2 et tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "nsHtml5StreamParser.h"
13 #include <type_traits>
15 #include "ErrorList.h"
16 #include "GeckoProfiler.h"
18 #include "mozilla/ArrayIterator.h"
19 #include "mozilla/Buffer.h"
20 #include "mozilla/CheckedInt.h"
21 #include "mozilla/DebugOnly.h"
22 #include "mozilla/Encoding.h"
23 #include "mozilla/EncodingDetector.h"
24 #include "mozilla/Likely.h"
25 #include "mozilla/Maybe.h"
26 #include "mozilla/SchedulerGroup.h"
27 #include "mozilla/ScopeExit.h"
28 #include "mozilla/Services.h"
29 #include "mozilla/StaticPrefs_html5.h"
30 #include "mozilla/StaticPrefs_intl.h"
31 #include "mozilla/TaskCategory.h"
32 #include "mozilla/TextUtils.h"
34 #include "mozilla/UniquePtrExtensions.h"
35 #include "mozilla/Unused.h"
36 #include "mozilla/dom/BindingDeclarations.h"
37 #include "mozilla/dom/BrowsingContext.h"
38 #include "mozilla/dom/DebuggerUtilsBinding.h"
39 #include "mozilla/dom/DocGroup.h"
40 #include "mozilla/dom/Document.h"
41 #include "mozilla/mozalloc.h"
42 #include "mozilla/Vector.h"
43 #include "nsContentSink.h"
44 #include "nsContentUtils.h"
45 #include "nsCycleCollectionTraversalCallback.h"
46 #include "nsHtml5AtomTable.h"
47 #include "nsHtml5ByteReadable.h"
48 #include "nsHtml5Highlighter.h"
49 #include "nsHtml5Module.h"
50 #include "nsHtml5OwningUTF16Buffer.h"
51 #include "nsHtml5Parser.h"
52 #include "nsHtml5Speculation.h"
53 #include "nsHtml5StreamParserPtr.h"
54 #include "nsHtml5Tokenizer.h"
55 #include "nsHtml5TreeBuilder.h"
56 #include "nsHtml5TreeOpExecutor.h"
57 #include "nsHtml5TreeOpStage.h"
58 #include "nsIChannel.h"
59 #include "nsIContentSink.h"
62 #include "nsIDocShell.h"
63 #include "nsIEventTarget.h"
64 #include "nsIHttpChannel.h"
65 #include "nsIInputStream.h"
66 #include "nsINestedURI.h"
67 #include "nsIObserverService.h"
68 #include "nsIRequest.h"
69 #include "nsIRunnable.h"
70 #include "nsIScriptError.h"
71 #include "nsIThread.h"
72 #include "nsIThreadRetargetableRequest.h"
73 #include "nsIThreadRetargetableStreamListener.h"
76 #include "nsJSEnvironment.h"
77 #include "nsLiteralString.h"
78 #include "nsNetUtil.h"
80 #include "nsTPromiseFlatString.h"
81 #include "nsThreadUtils.h"
82 #include "nsXULAppAPI.h"
85 // Defined in intl/encoding_glue/src/lib.rs
86 const mozilla::Encoding
* xmldecl_parse(const uint8_t* buf
, size_t buf_len
);
89 using namespace mozilla
;
90 using namespace mozilla::dom
;
93 * Note that nsHtml5StreamParser implements cycle collecting AddRef and
94 * Release. Therefore, nsHtml5StreamParser must never be refcounted from
97 * To work around this limitation, runnables posted by the main thread to the
98 * parser thread hold their reference to the stream parser in an
99 * nsHtml5StreamParserPtr. Upon creation, nsHtml5StreamParserPtr addrefs the
101 * just like a regular nsRefPtr. This is OK, since the creation of the
102 * runnable and the nsHtml5StreamParserPtr happens on the main thread.
104 * When the runnable is done on the parser thread, the destructor of
105 * nsHtml5StreamParserPtr runs there. It doesn't call Release on the held object
106 * directly. Instead, it posts another runnable back to the main thread where
107 * that runnable calls Release on the wrapped object.
109 * When posting runnables in the other direction, the runnables have to be
110 * created on the main thread when nsHtml5StreamParser is instantiated and
111 * held for the lifetime of the nsHtml5StreamParser. This works, because the
112 * same runnabled can be dispatched multiple times and currently runnables
113 * posted from the parser thread to main thread don't need to wrap any
114 * runnable-specific data. (In the other direction, the runnables most notably
115 * wrap the byte data of the stream.)
117 NS_IMPL_CYCLE_COLLECTING_ADDREF(nsHtml5StreamParser
)
118 NS_IMPL_CYCLE_COLLECTING_RELEASE(nsHtml5StreamParser
)
120 NS_INTERFACE_TABLE_HEAD(nsHtml5StreamParser
)
121 NS_INTERFACE_TABLE(nsHtml5StreamParser
, nsISupports
)
122 NS_INTERFACE_TABLE_TO_MAP_SEGUE_CYCLE_COLLECTION(nsHtml5StreamParser
)
125 NS_IMPL_CYCLE_COLLECTION_CLASS(nsHtml5StreamParser
)
127 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(nsHtml5StreamParser
)
129 NS_IMPL_CYCLE_COLLECTION_UNLINK(mRequest
)
130 NS_IMPL_CYCLE_COLLECTION_UNLINK(mOwner
)
131 tmp
->mExecutorFlusher
= nullptr;
132 tmp
->mLoadFlusher
= nullptr;
133 tmp
->mExecutor
= nullptr;
134 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
136 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(nsHtml5StreamParser
)
137 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mRequest
)
138 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOwner
)
139 // hack: count the strongly owned edge wrapped in the runnable
140 if (tmp
->mExecutorFlusher
) {
141 NS_CYCLE_COLLECTION_NOTE_EDGE_NAME(cb
, "mExecutorFlusher->mExecutor");
142 cb
.NoteXPCOMChild(static_cast<nsIContentSink
*>(tmp
->mExecutor
));
144 // hack: count the strongly owned edge wrapped in the runnable
145 if (tmp
->mLoadFlusher
) {
146 NS_CYCLE_COLLECTION_NOTE_EDGE_NAME(cb
, "mLoadFlusher->mExecutor");
147 cb
.NoteXPCOMChild(static_cast<nsIContentSink
*>(tmp
->mExecutor
));
149 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
151 class nsHtml5ExecutorFlusher
: public Runnable
{
153 RefPtr
<nsHtml5TreeOpExecutor
> mExecutor
;
156 explicit nsHtml5ExecutorFlusher(nsHtml5TreeOpExecutor
* aExecutor
)
157 : Runnable("nsHtml5ExecutorFlusher"), mExecutor(aExecutor
) {}
158 NS_IMETHOD
Run() override
{
159 if (!mExecutor
->isInList()) {
160 Document
* doc
= mExecutor
->GetDocument();
161 if (XRE_IsContentProcess() &&
163 HighPriorityEventPendingForTopLevelDocumentBeforeContentfulPaint(
165 // Possible early paint pending, reuse the runnable and try to
166 // call RunFlushLoop later.
167 nsCOMPtr
<nsIRunnable
> flusher
= this;
169 doc
->Dispatch(TaskCategory::Network
, flusher
.forget()))) {
170 PROFILER_MARKER_UNTYPED("HighPrio blocking parser flushing(1)", DOM
);
174 mExecutor
->RunFlushLoop();
180 class nsHtml5LoadFlusher
: public Runnable
{
182 RefPtr
<nsHtml5TreeOpExecutor
> mExecutor
;
185 explicit nsHtml5LoadFlusher(nsHtml5TreeOpExecutor
* aExecutor
)
186 : Runnable("nsHtml5LoadFlusher"), mExecutor(aExecutor
) {}
187 NS_IMETHOD
Run() override
{
188 mExecutor
->FlushSpeculativeLoads();
193 nsHtml5StreamParser::nsHtml5StreamParser(nsHtml5TreeOpExecutor
* aExecutor
,
194 nsHtml5Parser
* aOwner
,
196 : mBomState(eBomState::BOM_SNIFFING_NOT_STARTED
),
197 mCharsetSource(kCharsetUninitialized
),
198 mEncodingSwitchSource(kCharsetUninitialized
),
199 mEncoding(X_USER_DEFINED_ENCODING
), // Obviously bogus value to notice if
201 mNeedsEncodingSwitchTo(nullptr),
202 mSeenEligibleMetaCharset(false),
205 mStartedFeedingDetector(false),
206 mStartedFeedingDevTools(false),
208 mReparseForbidden(false),
209 mForceAutoDetection(false),
210 mChannelHadCharset(false),
211 mLookingForMetaCharset(false),
212 mStartsWithLtQuestion(false),
213 mLookingForXmlDeclarationForXmlViewSource(false),
214 mTemplatePushedOrHeadPopped(false),
217 mLastBuffer(nullptr), // Will be filled when starting
218 mExecutor(aExecutor
),
219 mTreeBuilder(new nsHtml5TreeBuilder(
220 (aMode
== VIEW_SOURCE_HTML
|| aMode
== VIEW_SOURCE_XML
)
222 : mExecutor
->GetStage(),
223 mExecutor
->GetStage(), aMode
== NORMAL
)),
225 new nsHtml5Tokenizer(mTreeBuilder
.get(), aMode
== VIEW_SOURCE_XML
)),
226 mTokenizerMutex("nsHtml5StreamParser mTokenizerMutex"),
229 mStreamState(eHtml5StreamState::STREAM_NOT_STARTED
),
232 mSpeculationMutex("nsHtml5StreamParser mSpeculationMutex"),
233 mSpeculationFailureCount(0),
234 mNumBytesBuffered(0),
237 mEventTarget(nsHtml5Module::GetStreamParserEventTarget()),
238 mExecutorFlusher(new nsHtml5ExecutorFlusher(aExecutor
)),
239 mLoadFlusher(new nsHtml5LoadFlusher(aExecutor
)),
240 mInitialEncodingWasFromParentFrame(false),
241 mHasHadErrors(false),
242 mDetectorHasSeenNonAscii(false),
243 mDecodingLocalFileWithoutTokenizing(false),
244 mBufferingBytes(false),
245 mFlushTimer(NS_NewTimer(mEventTarget
)),
246 mFlushTimerMutex("nsHtml5StreamParser mFlushTimerMutex"),
247 mFlushTimerArmed(false),
248 mFlushTimerEverFired(false),
250 NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
252 mAtomTable
.SetPermittedLookupEventTarget(mEventTarget
);
254 mTokenizer
->setInterner(&mAtomTable
);
255 mTokenizer
->setEncodingDeclarationHandler(this);
257 if (aMode
== VIEW_SOURCE_HTML
|| aMode
== VIEW_SOURCE_XML
) {
258 nsHtml5Highlighter
* highlighter
=
259 new nsHtml5Highlighter(mExecutor
->GetStage());
260 mTokenizer
->EnableViewSource(highlighter
); // takes ownership
261 mTreeBuilder
->EnableViewSource(highlighter
); // doesn't own
264 // There's a zeroing operator new for everything else
267 nsHtml5StreamParser::~nsHtml5StreamParser() {
268 NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
272 mozilla::MutexAutoLock
flushTimerLock(mFlushTimerMutex
);
273 MOZ_ASSERT(!mFlushTimer
, "Flush timer was not dropped before dtor!");
276 mUnicodeDecoder
= nullptr;
277 mFirstBuffer
= nullptr;
279 mTreeBuilder
= nullptr;
280 mTokenizer
= nullptr;
285 nsresult
nsHtml5StreamParser::GetChannel(nsIChannel
** aChannel
) {
286 NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
287 return mRequest
? CallQueryInterface(mRequest
, aChannel
)
288 : NS_ERROR_NOT_AVAILABLE
;
291 std::tuple
<NotNull
<const Encoding
*>, nsCharsetSource
>
292 nsHtml5StreamParser::GuessEncoding(bool aInitial
) {
294 mCharsetSource
!= kCharsetFromFinalUserForcedAutoDetection
&&
296 kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII
&&
298 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Generic
&&
300 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8GenericInitialWasASCII
&&
302 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Content
&&
304 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8ContentInitialWasASCII
&&
306 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD
&&
308 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLDInitialWasASCII
&&
309 mCharsetSource
!= kCharsetFromFinalAutoDetectionFile
);
310 auto ifHadBeenForced
= mDetector
->Guess(EmptyCString(), true);
314 : mDetector
->Guess(mTLD
, mDecodingLocalFileWithoutTokenizing
);
315 nsCharsetSource source
=
317 ? (mForceAutoDetection
318 ? kCharsetFromInitialUserForcedAutoDetection
319 : (mDecodingLocalFileWithoutTokenizing
320 ? kCharsetFromFinalAutoDetectionFile
321 : kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Generic
))
322 : (mForceAutoDetection
323 ? kCharsetFromFinalUserForcedAutoDetection
324 : (mDecodingLocalFileWithoutTokenizing
325 ? kCharsetFromFinalAutoDetectionFile
326 : kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Generic
));
327 if (source
== kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Generic
) {
328 if (encoding
== ISO_2022_JP_ENCODING
) {
329 if (EncodingDetector::TldMayAffectGuess(mTLD
)) {
330 source
= kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Content
;
332 } else if (!mDetectorHasSeenNonAscii
) {
333 source
= kCharsetFromInitialAutoDetectionASCII
; // deliberately Initial
334 } else if (ifHadBeenForced
== UTF_8_ENCODING
) {
335 MOZ_ASSERT(mCharsetSource
== kCharsetFromInitialAutoDetectionASCII
||
337 kCharsetFromInitialAutoDetectionWouldHaveBeenUTF8
||
338 mEncoding
== ISO_2022_JP_ENCODING
);
339 source
= kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII
;
340 } else if (encoding
!= ifHadBeenForced
) {
341 if (mCharsetSource
== kCharsetFromInitialAutoDetectionASCII
) {
343 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLDInitialWasASCII
;
346 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD
;
348 } else if (EncodingDetector::TldMayAffectGuess(mTLD
)) {
349 if (mCharsetSource
== kCharsetFromInitialAutoDetectionASCII
) {
351 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8ContentInitialWasASCII
;
353 source
= kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Content
;
355 } else if (mCharsetSource
== kCharsetFromInitialAutoDetectionASCII
) {
357 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8GenericInitialWasASCII
;
360 kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Generic
) {
361 if (encoding
== ISO_2022_JP_ENCODING
) {
362 if (EncodingDetector::TldMayAffectGuess(mTLD
)) {
363 source
= kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Content
;
365 } else if (!mDetectorHasSeenNonAscii
) {
366 source
= kCharsetFromInitialAutoDetectionASCII
;
367 } else if (ifHadBeenForced
== UTF_8_ENCODING
) {
368 source
= kCharsetFromInitialAutoDetectionWouldHaveBeenUTF8
;
369 } else if (encoding
!= ifHadBeenForced
) {
371 kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD
;
372 } else if (EncodingDetector::TldMayAffectGuess(mTLD
)) {
373 source
= kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Content
;
376 return {encoding
, source
};
379 void nsHtml5StreamParser::FeedDetector(Span
<const uint8_t> aBuffer
) {
381 mStartedFeedingDetector
= true;
383 MOZ_ASSERT(!mChardetEof
);
384 mDetectorHasSeenNonAscii
= mDetector
->Feed(aBuffer
, false);
387 void nsHtml5StreamParser::DetectorEof() {
389 mStartedFeedingDetector
= true;
395 mDetectorHasSeenNonAscii
= mDetector
->Feed(Span
<const uint8_t>(), true);
398 void nsHtml5StreamParser::SetViewSourceTitle(nsIURI
* aURL
) {
399 MOZ_ASSERT(NS_IsMainThread());
401 BrowsingContext
* browsingContext
=
402 mExecutor
->GetDocument()->GetBrowsingContext();
403 if (browsingContext
&& browsingContext
->WatchedByDevTools()) {
404 mURIToSendToDevtools
= aURL
;
407 nsresult rv
= nsID::GenerateUUIDInPlace(uuid
);
408 if (!NS_FAILED(rv
)) {
409 char buffer
[NSID_LENGTH
];
410 uuid
.ToProvidedString(buffer
);
411 mUUIDForDevtools
= NS_ConvertASCIItoUTF16(buffer
);
416 nsCOMPtr
<nsIURI
> temp
;
417 if (aURL
->SchemeIs("view-source")) {
418 nsCOMPtr
<nsINestedURI
> nested
= do_QueryInterface(aURL
);
419 nested
->GetInnerURI(getter_AddRefs(temp
));
423 if (temp
->SchemeIs("data")) {
424 // Avoid showing potentially huge data: URLs. The three last bytes are
425 // UTF-8 for an ellipsis.
426 mViewSourceTitle
.AssignLiteral("data:\xE2\x80\xA6");
428 nsresult rv
= temp
->GetSpec(mViewSourceTitle
);
430 mViewSourceTitle
.AssignLiteral("\xE2\x80\xA6");
437 nsHtml5StreamParser::SetupDecodingAndWriteSniffingBufferAndCurrentSegment(
438 Span
<const uint8_t> aPrefix
, Span
<const uint8_t> aFromSegment
) {
439 NS_ASSERTION(IsParserThread(), "Wrong thread!");
440 mUnicodeDecoder
= mEncoding
->NewDecoderWithBOMRemoval();
441 nsresult rv
= WriteStreamBytes(aPrefix
);
442 NS_ENSURE_SUCCESS(rv
, rv
);
443 return WriteStreamBytes(aFromSegment
);
446 void nsHtml5StreamParser::SetupDecodingFromBom(
447 NotNull
<const Encoding
*> aEncoding
) {
448 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
449 mEncoding
= aEncoding
;
450 mDecodingLocalFileWithoutTokenizing
= false;
451 mLookingForMetaCharset
= false;
452 mBufferingBytes
= false;
453 mUnicodeDecoder
= mEncoding
->NewDecoderWithoutBOMHandling();
454 mCharsetSource
= kCharsetFromByteOrderMark
;
455 mForceAutoDetection
= false;
456 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
457 mBomState
= BOM_SNIFFING_OVER
;
458 if (mMode
== VIEW_SOURCE_HTML
) {
459 mTokenizer
->StartViewSourceCharacters();
463 void nsHtml5StreamParser::SetupDecodingFromUtf16BogoXml(
464 NotNull
<const Encoding
*> aEncoding
) {
465 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
466 mEncoding
= aEncoding
;
467 mDecodingLocalFileWithoutTokenizing
= false;
468 mLookingForMetaCharset
= false;
469 mBufferingBytes
= false;
470 mUnicodeDecoder
= mEncoding
->NewDecoderWithoutBOMHandling();
471 mCharsetSource
= kCharsetFromXmlDeclarationUtf16
;
472 mForceAutoDetection
= false;
473 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
474 mBomState
= BOM_SNIFFING_OVER
;
475 if (mMode
== VIEW_SOURCE_HTML
) {
476 mTokenizer
->StartViewSourceCharacters();
478 auto dst
= mLastBuffer
->TailAsSpan(READ_BUFFER_SIZE
);
482 mLastBuffer
->AdvanceEnd(3);
483 MOZ_ASSERT(!mStartedFeedingDevTools
);
484 OnNewContent(dst
.To(3));
487 size_t nsHtml5StreamParser::LengthOfLtContainingPrefixInSecondBuffer() {
488 MOZ_ASSERT(mBufferedBytes
.Length() <= 2);
489 if (mBufferedBytes
.Length() < 2) {
492 Buffer
<uint8_t>& second
= mBufferedBytes
[1];
493 const uint8_t* elements
= second
.Elements();
494 const uint8_t* lt
= (const uint8_t*)memchr(elements
, '>', second
.Length());
496 return (lt
- elements
) + 1;
501 nsresult
nsHtml5StreamParser::SniffStreamBytes(Span
<const uint8_t> aFromSegment
,
503 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
504 MOZ_ASSERT_IF(aEof
, aFromSegment
.IsEmpty());
506 if (mCharsetSource
>=
507 kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII
&&
508 mCharsetSource
<= kCharsetFromFinalUserForcedAutoDetection
) {
509 if (mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
) {
510 mTreeBuilder
->MaybeComplainAboutCharset("EncDetectorReloadPlain", true,
513 mTreeBuilder
->MaybeComplainAboutCharset("EncDetectorReload", true, 0);
517 // mEncoding and mCharsetSource potentially have come from channel or higher
518 // by now. If we find a BOM, SetupDecodingFromBom() will overwrite them.
519 // If we don't find a BOM, the previously set values of mEncoding and
520 // mCharsetSource are not modified by the BOM sniffing here.
521 static uint8_t utf8
[] = {0xEF, 0xBB};
522 static uint8_t utf16le
[] = {0xFF};
523 static uint8_t utf16be
[] = {0xFE};
524 static uint8_t utf16leXml
[] = {'<', 0x00, '?', 0x00, 'x'};
525 static uint8_t utf16beXml
[] = {0x00, '<', 0x00, '?', 0x00};
526 // Buffer for replaying past bytes based on state machine state. If
527 // writing this from scratch, probably wouldn't do it this way, but
528 // let's keep the changes to a minimum.
529 const uint8_t* prefix
= utf8
;
530 size_t prefixLength
= 0;
531 if (aEof
&& mBomState
== BOM_SNIFFING_NOT_STARTED
) {
532 // Avoid handling aEof in the BOM_SNIFFING_NOT_STARTED state below.
533 mBomState
= BOM_SNIFFING_OVER
;
536 (i
< aFromSegment
.Length() && mBomState
!= BOM_SNIFFING_OVER
) || aEof
;
539 case BOM_SNIFFING_NOT_STARTED
:
540 MOZ_ASSERT(i
== 0, "Bad BOM sniffing state.");
541 MOZ_ASSERT(!aEof
, "Should have checked for aEof above!");
542 switch (aFromSegment
[0]) {
544 mBomState
= SEEN_UTF_8_FIRST_BYTE
;
547 mBomState
= SEEN_UTF_16_LE_FIRST_BYTE
;
550 mBomState
= SEEN_UTF_16_BE_FIRST_BYTE
;
553 if (mCharsetSource
< kCharsetFromXmlDeclarationUtf16
&&
554 mCharsetSource
!= kCharsetFromChannel
) {
555 mBomState
= SEEN_UTF_16_BE_XML_FIRST
;
557 mBomState
= BOM_SNIFFING_OVER
;
561 if (mCharsetSource
< kCharsetFromXmlDeclarationUtf16
&&
562 mCharsetSource
!= kCharsetFromChannel
) {
563 mBomState
= SEEN_UTF_16_LE_XML_FIRST
;
565 mBomState
= BOM_SNIFFING_OVER
;
569 mBomState
= BOM_SNIFFING_OVER
;
573 case SEEN_UTF_16_LE_FIRST_BYTE
:
574 if (!aEof
&& aFromSegment
[i
] == 0xFE) {
575 SetupDecodingFromBom(UTF_16LE_ENCODING
);
576 return WriteStreamBytes(aFromSegment
.From(i
+ 1));
579 prefixLength
= 1 - i
;
580 mBomState
= BOM_SNIFFING_OVER
;
582 case SEEN_UTF_16_BE_FIRST_BYTE
:
583 if (!aEof
&& aFromSegment
[i
] == 0xFF) {
584 SetupDecodingFromBom(UTF_16BE_ENCODING
);
585 return WriteStreamBytes(aFromSegment
.From(i
+ 1));
588 prefixLength
= 1 - i
;
589 mBomState
= BOM_SNIFFING_OVER
;
591 case SEEN_UTF_8_FIRST_BYTE
:
592 if (!aEof
&& aFromSegment
[i
] == 0xBB) {
593 mBomState
= SEEN_UTF_8_SECOND_BYTE
;
595 prefixLength
= 1 - i
;
596 mBomState
= BOM_SNIFFING_OVER
;
599 case SEEN_UTF_8_SECOND_BYTE
:
600 if (!aEof
&& aFromSegment
[i
] == 0xBF) {
601 SetupDecodingFromBom(UTF_8_ENCODING
);
602 return WriteStreamBytes(aFromSegment
.From(i
+ 1));
604 prefixLength
= 2 - i
;
605 mBomState
= BOM_SNIFFING_OVER
;
607 case SEEN_UTF_16_BE_XML_FIRST
:
608 if (!aEof
&& aFromSegment
[i
] == '<') {
609 mBomState
= SEEN_UTF_16_BE_XML_SECOND
;
612 prefixLength
= 1 - i
;
613 mBomState
= BOM_SNIFFING_OVER
;
616 case SEEN_UTF_16_BE_XML_SECOND
:
617 if (!aEof
&& aFromSegment
[i
] == 0x00) {
618 mBomState
= SEEN_UTF_16_BE_XML_THIRD
;
621 prefixLength
= 2 - i
;
622 mBomState
= BOM_SNIFFING_OVER
;
625 case SEEN_UTF_16_BE_XML_THIRD
:
626 if (!aEof
&& aFromSegment
[i
] == '?') {
627 mBomState
= SEEN_UTF_16_BE_XML_FOURTH
;
630 prefixLength
= 3 - i
;
631 mBomState
= BOM_SNIFFING_OVER
;
634 case SEEN_UTF_16_BE_XML_FOURTH
:
635 if (!aEof
&& aFromSegment
[i
] == 0x00) {
636 mBomState
= SEEN_UTF_16_BE_XML_FIFTH
;
639 prefixLength
= 4 - i
;
640 mBomState
= BOM_SNIFFING_OVER
;
643 case SEEN_UTF_16_BE_XML_FIFTH
:
644 if (!aEof
&& aFromSegment
[i
] == 'x') {
645 SetupDecodingFromUtf16BogoXml(UTF_16BE_ENCODING
);
646 return WriteStreamBytes(aFromSegment
.From(i
+ 1));
649 prefixLength
= 5 - i
;
650 mBomState
= BOM_SNIFFING_OVER
;
652 case SEEN_UTF_16_LE_XML_FIRST
:
653 if (!aEof
&& aFromSegment
[i
] == 0x00) {
654 mBomState
= SEEN_UTF_16_LE_XML_SECOND
;
656 if (!aEof
&& aFromSegment
[i
] == '?' &&
657 !(mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
)) {
658 mStartsWithLtQuestion
= true;
661 prefixLength
= 1 - i
;
662 mBomState
= BOM_SNIFFING_OVER
;
665 case SEEN_UTF_16_LE_XML_SECOND
:
666 if (!aEof
&& aFromSegment
[i
] == '?') {
667 mBomState
= SEEN_UTF_16_LE_XML_THIRD
;
670 prefixLength
= 2 - i
;
671 mBomState
= BOM_SNIFFING_OVER
;
674 case SEEN_UTF_16_LE_XML_THIRD
:
675 if (!aEof
&& aFromSegment
[i
] == 0x00) {
676 mBomState
= SEEN_UTF_16_LE_XML_FOURTH
;
679 prefixLength
= 3 - i
;
680 mBomState
= BOM_SNIFFING_OVER
;
683 case SEEN_UTF_16_LE_XML_FOURTH
:
684 if (!aEof
&& aFromSegment
[i
] == 'x') {
685 mBomState
= SEEN_UTF_16_LE_XML_FIFTH
;
688 prefixLength
= 4 - i
;
689 mBomState
= BOM_SNIFFING_OVER
;
692 case SEEN_UTF_16_LE_XML_FIFTH
:
693 if (!aEof
&& aFromSegment
[i
] == 0x00) {
694 SetupDecodingFromUtf16BogoXml(UTF_16LE_ENCODING
);
695 return WriteStreamBytes(aFromSegment
.From(i
+ 1));
698 prefixLength
= 5 - i
;
699 mBomState
= BOM_SNIFFING_OVER
;
702 mBomState
= BOM_SNIFFING_OVER
;
709 // if we get here, there either was no BOM or the BOM sniffing isn't complete
712 MOZ_ASSERT(mCharsetSource
!= kCharsetFromByteOrderMark
,
713 "Should not come here if BOM was found.");
714 MOZ_ASSERT(mCharsetSource
!= kCharsetFromXmlDeclarationUtf16
,
715 "Should not come here if UTF-16 bogo-XML declaration was found.");
716 MOZ_ASSERT(mCharsetSource
!= kCharsetFromOtherComponent
,
717 "kCharsetFromOtherComponent is for XSLT.");
719 if (mBomState
== BOM_SNIFFING_OVER
) {
720 if (mMode
== VIEW_SOURCE_XML
&& mStartsWithLtQuestion
&&
721 mCharsetSource
< kCharsetFromChannel
) {
722 // Sniff for XML declaration only.
723 MOZ_ASSERT(!mLookingForXmlDeclarationForXmlViewSource
);
725 MOZ_ASSERT(!mLookingForMetaCharset
);
726 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
);
727 // Maybe we've already buffered a '>'.
728 MOZ_ASSERT(!mBufferedBytes
.IsEmpty(),
729 "How did at least <? not get buffered?");
730 Buffer
<uint8_t>& first
= mBufferedBytes
[0];
731 const Encoding
* encoding
=
732 xmldecl_parse(first
.Elements(), first
.Length());
734 mEncoding
= WrapNotNull(encoding
);
735 mCharsetSource
= kCharsetFromXmlDeclaration
;
736 } else if (memchr(first
.Elements(), '>', first
.Length())) {
737 // There was a '>', but an encoding still wasn't found.
738 ; // fall through to commit to the UTF-8 default.
739 } else if (size_t lengthOfPrefix
=
740 LengthOfLtContainingPrefixInSecondBuffer()) {
741 // This can only happen if the first buffer was a lone '<', because
742 // we come here upon seeing the second byte '?' if the first two bytes
743 // were "<?". That is, the only way how we aren't dealing with the first
744 // buffer is if the first buffer only contained a single '<' and we are
745 // dealing with the second buffer that starts with '?'.
746 MOZ_ASSERT(first
.Length() == 1);
747 MOZ_ASSERT(mBufferedBytes
[1][0] == '?');
748 // Our scanner for XML declaration-like syntax wants to see a contiguous
749 // buffer, so let's linearize the data. (Ideally, the XML declaration
750 // scanner would be incremental, but this is the rare path anyway.)
751 Vector
<uint8_t> contiguous
;
752 if (!contiguous
.append(first
.Elements(), first
.Length())) {
753 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
754 return NS_ERROR_OUT_OF_MEMORY
;
756 if (!contiguous
.append(mBufferedBytes
[1].Elements(), lengthOfPrefix
)) {
757 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
758 return NS_ERROR_OUT_OF_MEMORY
;
760 encoding
= xmldecl_parse(contiguous
.begin(), contiguous
.length());
762 mEncoding
= WrapNotNull(encoding
);
763 mCharsetSource
= kCharsetFromXmlDeclaration
;
765 // else no XML decl, commit to the UTF-8 default.
767 MOZ_ASSERT(mBufferingBytes
);
768 mLookingForXmlDeclarationForXmlViewSource
= true;
771 } else if (mMode
!= VIEW_SOURCE_XML
&&
772 (mForceAutoDetection
|| mCharsetSource
< kCharsetFromChannel
)) {
773 // In order to use the buffering logic for meta with mForceAutoDetection,
774 // we set mLookingForMetaCharset but still actually potentially ignore the
776 mFirstBufferOfMetaScan
= mFirstBuffer
;
777 MOZ_ASSERT(mLookingForMetaCharset
);
779 if (mMode
== VIEW_SOURCE_HTML
) {
780 auto r
= mTokenizer
->FlushViewSource();
782 return r
.unwrapErr();
785 auto r
= mTreeBuilder
->Flush();
787 return r
.unwrapErr();
789 // Encoding committer flushes the ops on the main thread.
791 mozilla::MutexAutoLock
speculationAutoLock(mSpeculationMutex
);
792 nsHtml5Speculation
* speculation
= new nsHtml5Speculation(
793 mFirstBuffer
, mFirstBuffer
->getStart(), mTokenizer
->getLineNumber(),
794 mTokenizer
->getColumnNumber(), mTreeBuilder
->newSnapshot());
795 MOZ_ASSERT(!mFlushTimerArmed
, "How did we end up arming the timer?");
796 if (mMode
== VIEW_SOURCE_HTML
) {
797 mTokenizer
->SetViewSourceOpSink(speculation
);
798 mTokenizer
->StartViewSourceCharacters();
800 MOZ_ASSERT(mMode
!= VIEW_SOURCE_XML
);
801 mTreeBuilder
->SetOpSink(speculation
);
803 mSpeculations
.AppendElement(speculation
); // adopts the pointer
806 mLookingForMetaCharset
= false;
807 mBufferingBytes
= false;
808 mDecodingLocalFileWithoutTokenizing
= false;
809 if (mMode
== VIEW_SOURCE_HTML
) {
810 mTokenizer
->StartViewSourceCharacters();
813 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
814 return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(
815 Span(prefix
, prefixLength
), aFromSegment
);
821 class AddContentRunnable
: public Runnable
{
823 AddContentRunnable(const nsAString
& aParserID
, nsIURI
* aURI
,
824 Span
<const char16_t
> aData
, bool aComplete
)
825 : Runnable("AddContent") {
828 mData
.mUri
.Construct(NS_ConvertUTF8toUTF16(spec
));
829 mData
.mParserID
.Construct(aParserID
);
830 mData
.mContents
.Construct(aData
.Elements(), aData
.Length());
831 mData
.mComplete
.Construct(aComplete
);
834 NS_IMETHOD
Run() override
{
836 if (!mData
.ToJSON(json
)) {
837 return NS_ERROR_FAILURE
;
840 nsCOMPtr
<nsIObserverService
> obsService
= services::GetObserverService();
842 obsService
->NotifyObservers(nullptr, "devtools-html-content",
843 PromiseFlatString(json
).get());
852 inline void nsHtml5StreamParser::OnNewContent(Span
<const char16_t
> aData
) {
854 mStartedFeedingDevTools
= true;
856 if (mURIToSendToDevtools
) {
857 if (aData
.IsEmpty()) {
858 // Optimize out the runnable.
861 NS_DispatchToMainThread(new AddContentRunnable(mUUIDForDevtools
,
862 mURIToSendToDevtools
, aData
,
863 /* aComplete */ false));
867 inline void nsHtml5StreamParser::OnContentComplete() {
869 mStartedFeedingDevTools
= true;
871 if (mURIToSendToDevtools
) {
872 NS_DispatchToMainThread(new AddContentRunnable(
873 mUUIDForDevtools
, mURIToSendToDevtools
, Span
<const char16_t
>(),
874 /* aComplete */ true));
875 mURIToSendToDevtools
= nullptr;
879 nsresult
nsHtml5StreamParser::WriteStreamBytes(
880 Span
<const uint8_t> aFromSegment
) {
881 NS_ASSERTION(IsParserThread(), "Wrong thread!");
882 mTokenizerMutex
.AssertCurrentThreadOwns();
883 // mLastBuffer should always point to a buffer of the size
886 NS_WARNING("mLastBuffer should not be null!");
887 MarkAsBroken(NS_ERROR_NULL_POINTER
);
888 return NS_ERROR_NULL_POINTER
;
890 size_t totalRead
= 0;
891 auto src
= aFromSegment
;
893 auto dst
= mLastBuffer
->TailAsSpan(READ_BUFFER_SIZE
);
894 auto [result
, read
, written
, hadErrors
] =
895 mUnicodeDecoder
->DecodeToUTF16(src
, dst
, false);
896 if (!(mLookingForMetaCharset
|| mDecodingLocalFileWithoutTokenizing
)) {
897 OnNewContent(dst
.To(written
));
899 if (hadErrors
&& !mHasHadErrors
) {
900 mHasHadErrors
= true;
901 if (mEncoding
== UTF_8_ENCODING
) {
902 mTreeBuilder
->TryToEnableEncodingMenu();
905 src
= src
.From(read
);
907 mLastBuffer
->AdvanceEnd(written
);
908 if (result
== kOutputFull
) {
909 RefPtr
<nsHtml5OwningUTF16Buffer
> newBuf
=
910 nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE
);
912 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
913 return NS_ERROR_OUT_OF_MEMORY
;
915 mLastBuffer
= (mLastBuffer
->next
= std::move(newBuf
));
917 MOZ_ASSERT(totalRead
== aFromSegment
.Length(),
918 "The Unicode decoder consumed the wrong number of bytes.");
920 if (!mLookingForMetaCharset
&& mDecodingLocalFileWithoutTokenizing
&&
921 mNumBytesBuffered
== LOCAL_FILE_UTF_8_BUFFER_SIZE
) {
922 MOZ_ASSERT(!mStartedFeedingDetector
);
923 for (auto&& buffer
: mBufferedBytes
) {
924 FeedDetector(buffer
);
926 // If the file is exactly LOCAL_FILE_UTF_8_BUFFER_SIZE bytes long
927 // we end up not considering the EOF. That's not fatal, since we
928 // don't consider the EOF if the file is
929 // LOCAL_FILE_UTF_8_BUFFER_SIZE + 1 bytes long.
930 auto [encoding
, source
] = GuessEncoding(true);
931 mCharsetSource
= source
;
932 if (encoding
!= mEncoding
) {
933 mEncoding
= encoding
;
934 nsresult rv
= ReDecodeLocalFile();
939 MOZ_ASSERT(mEncoding
== UTF_8_ENCODING
);
940 nsresult rv
= CommitLocalFileToEncoding();
951 [[nodiscard
]] nsresult
nsHtml5StreamParser::ReDecodeLocalFile() {
952 MOZ_ASSERT(mDecodingLocalFileWithoutTokenizing
&& !mLookingForMetaCharset
);
953 MOZ_ASSERT(mFirstBufferOfMetaScan
);
954 MOZ_ASSERT(mCharsetSource
== kCharsetFromFinalAutoDetectionFile
||
955 (mForceAutoDetection
&&
956 mCharsetSource
== kCharsetFromInitialUserForcedAutoDetection
));
958 DiscardMetaSpeculation();
960 MOZ_ASSERT(mEncoding
!= UTF_8_ENCODING
);
962 mDecodingLocalFileWithoutTokenizing
= false;
964 mEncoding
->NewDecoderWithBOMRemovalInto(*mUnicodeDecoder
);
965 mHasHadErrors
= false;
967 // Throw away previous decoded data
968 mLastBuffer
= mFirstBuffer
;
969 mLastBuffer
->next
= nullptr;
970 mLastBuffer
->setStart(0);
971 mLastBuffer
->setEnd(0);
973 mBufferingBytes
= false;
974 mForceAutoDetection
= false; // To stop feeding the detector
975 mFirstBufferOfMetaScan
= nullptr;
977 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, true);
980 for (auto&& buffer
: mBufferedBytes
) {
981 DoDataAvailable(buffer
);
984 if (mMode
== VIEW_SOURCE_HTML
) {
985 auto r
= mTokenizer
->FlushViewSource();
987 return r
.unwrapErr();
990 auto r
= mTreeBuilder
->Flush();
992 return r
.unwrapErr();
997 [[nodiscard
]] nsresult
nsHtml5StreamParser::CommitLocalFileToEncoding() {
998 MOZ_ASSERT(mDecodingLocalFileWithoutTokenizing
&& !mLookingForMetaCharset
);
999 MOZ_ASSERT(mFirstBufferOfMetaScan
);
1000 mDecodingLocalFileWithoutTokenizing
= false;
1001 MOZ_ASSERT(mCharsetSource
== kCharsetFromFinalAutoDetectionFile
||
1002 (mForceAutoDetection
&&
1003 mCharsetSource
== kCharsetFromInitialUserForcedAutoDetection
));
1004 MOZ_ASSERT(mEncoding
== UTF_8_ENCODING
);
1006 MOZ_ASSERT(!mStartedFeedingDevTools
);
1007 if (mURIToSendToDevtools
) {
1008 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBufferOfMetaScan
;
1010 Span
<const char16_t
> data(buffer
->getBuffer() + buffer
->getStart(),
1011 buffer
->getLength());
1013 buffer
= buffer
->next
;
1017 mFirstBufferOfMetaScan
= nullptr;
1019 mBufferingBytes
= false;
1020 mForceAutoDetection
= false; // To stop feeding the detector
1021 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, true);
1022 if (mMode
== VIEW_SOURCE_HTML
) {
1023 auto r
= mTokenizer
->FlushViewSource();
1025 return r
.unwrapErr();
1028 auto r
= mTreeBuilder
->Flush();
1030 return r
.unwrapErr();
1035 class MaybeRunCollector
: public Runnable
{
1037 explicit MaybeRunCollector(nsIDocShell
* aDocShell
)
1038 : Runnable("MaybeRunCollector"), mDocShell(aDocShell
) {}
1040 NS_IMETHOD
Run() override
{
1041 nsJSContext::MaybeRunNextCollectorSlice(mDocShell
,
1042 JS::GCReason::HTML_PARSER
);
1046 nsCOMPtr
<nsIDocShell
> mDocShell
;
1049 nsresult
nsHtml5StreamParser::OnStartRequest(nsIRequest
* aRequest
) {
1050 MOZ_RELEASE_ASSERT(STREAM_NOT_STARTED
== mStreamState
,
1051 "Got OnStartRequest when the stream had already started.");
1053 !mExecutor
->HasStarted(),
1054 "Got OnStartRequest at the wrong stage in the executor life cycle.");
1055 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
1057 // To avoid the cost of instantiating the detector when it's not needed,
1058 // let's instantiate only if we make it out of this method with the
1059 // intent to use it.
1060 auto detectorCreator
= MakeScopeExit([&] {
1061 if ((mForceAutoDetection
|| mCharsetSource
< kCharsetFromParentFrame
) ||
1062 !(mMode
== LOAD_AS_DATA
|| mMode
== VIEW_SOURCE_XML
)) {
1063 mDetector
= mozilla::EncodingDetector::Create();
1067 mRequest
= aRequest
;
1069 mStreamState
= STREAM_BEING_READ
;
1071 // For View Source, the parser should run with scripts "enabled" if a normal
1072 // load would have scripts enabled.
1073 bool scriptingEnabled
=
1074 mMode
== LOAD_AS_DATA
? false : mExecutor
->IsScriptEnabled();
1075 mOwner
->StartTokenizer(scriptingEnabled
);
1077 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
);
1078 bool isSrcdoc
= false;
1079 nsCOMPtr
<nsIChannel
> channel
;
1080 nsresult rv
= GetChannel(getter_AddRefs(channel
));
1081 if (NS_SUCCEEDED(rv
)) {
1082 isSrcdoc
= NS_IsSrcdocChannel(channel
);
1083 if (!isSrcdoc
&& mCharsetSource
<= kCharsetFromFallback
) {
1084 nsCOMPtr
<nsIURI
> originalURI
;
1085 rv
= channel
->GetOriginalURI(getter_AddRefs(originalURI
));
1086 if (NS_SUCCEEDED(rv
)) {
1087 if (originalURI
->SchemeIs("resource")) {
1088 mCharsetSource
= kCharsetFromBuiltIn
;
1089 mEncoding
= UTF_8_ENCODING
;
1090 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
1092 nsCOMPtr
<nsIURI
> currentURI
;
1093 rv
= channel
->GetURI(getter_AddRefs(currentURI
));
1094 if (NS_SUCCEEDED(rv
)) {
1095 nsCOMPtr
<nsIURI
> innermost
= NS_GetInnermostURI(currentURI
);
1096 if (innermost
->SchemeIs("file")) {
1097 MOZ_ASSERT(mEncoding
== UTF_8_ENCODING
);
1098 if (!(mMode
== LOAD_AS_DATA
|| mMode
== VIEW_SOURCE_XML
)) {
1099 mDecodingLocalFileWithoutTokenizing
= true;
1103 innermost
->GetAsciiHost(host
);
1104 if (!host
.IsEmpty()) {
1105 // First let's see if the host is DNS-absolute and ends with a
1106 // dot and get rid of that one.
1107 if (host
.Last() == '.') {
1108 host
.SetLength(host
.Length() - 1);
1110 int32_t index
= host
.RFindChar('.');
1111 if (index
!= kNotFound
) {
1112 // We tolerate an IPv4 component as generic "TLD", so don't
1115 Substring(host
, index
+ 1, host
.Length() - (index
+ 1)),
1125 mTreeBuilder
->setIsSrcdocDocument(isSrcdoc
);
1126 mTreeBuilder
->setScriptingEnabled(scriptingEnabled
);
1127 mTreeBuilder
->SetPreventScriptExecution(
1128 !((mMode
== NORMAL
) && scriptingEnabled
));
1129 mTokenizer
->start();
1131 mExecutor
->StartReadingFromStage();
1133 if (mMode
== PLAIN_TEXT
) {
1134 mTreeBuilder
->StartPlainText();
1135 mTokenizer
->StartPlainText();
1137 mTemplatePushedOrHeadPopped
); // Needed to force 1024-byte sniffing
1138 // Flush the ops to put them where ContinueAfterScriptsOrEncodingCommitment
1140 auto r
= mTreeBuilder
->Flush();
1142 return mExecutor
->MarkAsBroken(r
.unwrapErr());
1144 } else if (mMode
== VIEW_SOURCE_PLAIN
) {
1145 nsAutoString viewSourceTitle
;
1146 CopyUTF8toUTF16(mViewSourceTitle
, viewSourceTitle
);
1147 mTreeBuilder
->EnsureBufferSpace(viewSourceTitle
.Length());
1148 mTreeBuilder
->StartPlainTextViewSource(viewSourceTitle
);
1149 mTokenizer
->StartPlainText();
1151 mTemplatePushedOrHeadPopped
); // Needed to force 1024-byte sniffing
1152 // Flush the ops to put them where ContinueAfterScriptsOrEncodingCommitment
1154 auto r
= mTreeBuilder
->Flush();
1156 return mExecutor
->MarkAsBroken(r
.unwrapErr());
1158 } else if (mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
) {
1159 // Generate and flush the View Source document up to and including the
1160 // pre element start.
1161 mTokenizer
->StartViewSource(NS_ConvertUTF8toUTF16(mViewSourceTitle
));
1162 if (mMode
== VIEW_SOURCE_XML
) {
1163 mTokenizer
->StartViewSourceCharacters();
1165 // Flush the ops to put them where ContinueAfterScriptsOrEncodingCommitment
1167 auto r
= mTokenizer
->FlushViewSource();
1169 return mExecutor
->MarkAsBroken(r
.unwrapErr());
1174 * If you move the following line, be very careful not to cause
1175 * WillBuildModel to be called before the document has had its
1176 * script global object set.
1178 rv
= mExecutor
->WillBuildModel();
1179 NS_ENSURE_SUCCESS(rv
, rv
);
1181 RefPtr
<nsHtml5OwningUTF16Buffer
> newBuf
=
1182 nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE
);
1184 // marks this stream parser as terminated,
1185 // which prevents entry to code paths that
1186 // would use mFirstBuffer or mLastBuffer.
1187 return mExecutor
->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1189 MOZ_ASSERT(!mFirstBuffer
, "How come we have the first buffer set?");
1190 MOZ_ASSERT(!mLastBuffer
, "How come we have the last buffer set?");
1191 mFirstBuffer
= mLastBuffer
= newBuf
;
1195 mNetworkEventTarget
=
1196 mExecutor
->GetDocument()->EventTargetFor(TaskCategory::Network
);
1198 nsCOMPtr
<nsIHttpChannel
> httpChannel(do_QueryInterface(mRequest
, &rv
));
1199 if (NS_SUCCEEDED(rv
)) {
1200 // Non-HTTP channels are bogus enough that we let them work with unlabeled
1201 // runnables for now. Asserting for HTTP channels only.
1202 MOZ_ASSERT(mNetworkEventTarget
|| mMode
== LOAD_AS_DATA
,
1203 "How come the network event target is still null?");
1205 nsAutoCString method
;
1206 Unused
<< httpChannel
->GetRequestMethod(method
);
1207 // XXX does Necko have a way to renavigate POST, etc. without hitting
1209 if (!method
.EqualsLiteral("GET")) {
1210 // This is the old Gecko behavior but the HTML5 spec disagrees.
1211 // Don't reparse on POST.
1212 mReparseForbidden
= true;
1216 // Attempt to retarget delivery of data (via OnDataAvailable) to the parser
1217 // thread, rather than through the main thread.
1218 nsCOMPtr
<nsIThreadRetargetableRequest
> threadRetargetableRequest
=
1219 do_QueryInterface(mRequest
, &rv
);
1220 if (threadRetargetableRequest
) {
1221 rv
= threadRetargetableRequest
->RetargetDeliveryTo(mEventTarget
);
1222 if (NS_SUCCEEDED(rv
)) {
1223 // Parser thread should be now ready to get data from necko and parse it
1224 // and main thread might have a chance to process a collector slice.
1225 // We need to do this asynchronously so that necko may continue processing
1227 nsCOMPtr
<nsIRunnable
> runnable
=
1228 new MaybeRunCollector(mExecutor
->GetDocument()->GetDocShell());
1229 mozilla::SchedulerGroup::Dispatch(
1230 mozilla::TaskCategory::GarbageCollection
, runnable
.forget());
1234 if (NS_FAILED(rv
)) {
1235 NS_WARNING("Failed to retarget HTML data delivery to the parser thread.");
1238 if (mCharsetSource
== kCharsetFromParentFrame
) {
1239 // Remember this for error reporting.
1240 mInitialEncodingWasFromParentFrame
= true;
1241 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
);
1244 if (mForceAutoDetection
|| mCharsetSource
< kCharsetFromChannel
) {
1245 mBufferingBytes
= true;
1246 if (mMode
!= VIEW_SOURCE_XML
) {
1247 // We need to set mLookingForMetaCharset to true here in case the first
1248 // buffer to arrive is larger than 1024. We need the code that splits
1249 // the buffers at 1024 bytes to work even in that case.
1250 mLookingForMetaCharset
= true;
1254 if (mCharsetSource
< kCharsetFromUtf8OnlyMime
) {
1255 // we aren't ready to commit to an encoding yet
1256 // leave converter uninstantiated for now
1260 MOZ_ASSERT(!(mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
));
1262 MOZ_ASSERT(mEncoding
== UTF_8_ENCODING
,
1263 "How come UTF-8-only MIME type didn't set encoding to UTF-8?");
1265 // We are loading JSON/WebVTT/etc. into a browsing context.
1266 // There's no need to remove the BOM manually here, because
1267 // the UTF-8 decoder removes it.
1268 mReparseForbidden
= true;
1269 mForceAutoDetection
= false;
1271 // Instantiate the converter here to avoid BOM sniffing.
1272 mDecodingLocalFileWithoutTokenizing
= false;
1273 mUnicodeDecoder
= mEncoding
->NewDecoderWithBOMRemoval();
1277 void nsHtml5StreamParser::DoStopRequest() {
1278 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1279 MOZ_RELEASE_ASSERT(STREAM_BEING_READ
== mStreamState
,
1280 "Stream ended without being open.");
1281 mTokenizerMutex
.AssertCurrentThreadOwns();
1283 auto guard
= MakeScopeExit([&] { OnContentComplete(); });
1285 if (IsTerminated()) {
1289 if (MOZ_UNLIKELY(mLookingForXmlDeclarationForXmlViewSource
)) {
1290 mLookingForXmlDeclarationForXmlViewSource
= false;
1291 mBufferingBytes
= false;
1292 mUnicodeDecoder
= mEncoding
->NewDecoderWithoutBOMHandling();
1293 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
1295 for (auto&& buffer
: mBufferedBytes
) {
1296 nsresult rv
= WriteStreamBytes(buffer
);
1297 if (NS_FAILED(rv
)) {
1302 } else if (!mUnicodeDecoder
) {
1304 if (NS_FAILED(rv
= SniffStreamBytes(Span
<const uint8_t>(), true))) {
1310 MOZ_ASSERT(mUnicodeDecoder
,
1311 "Should have a decoder after finalizing sniffing.");
1313 // mLastBuffer should always point to a buffer of the size
1314 // READ_BUFFER_SIZE.
1316 NS_WARNING("mLastBuffer should not be null!");
1317 MarkAsBroken(NS_ERROR_NULL_POINTER
);
1321 Span
<uint8_t> src
; // empty span
1323 auto dst
= mLastBuffer
->TailAsSpan(READ_BUFFER_SIZE
);
1328 // Do not use structured binding lest deal with [-Werror=unused-variable]
1329 std::tie(result
, read
, written
, hadErrors
) =
1330 mUnicodeDecoder
->DecodeToUTF16(src
, dst
, true);
1331 if (!(mLookingForMetaCharset
|| mDecodingLocalFileWithoutTokenizing
)) {
1332 OnNewContent(dst
.To(written
));
1335 mHasHadErrors
= true;
1337 MOZ_ASSERT(read
== 0, "How come an empty span was read form?");
1338 mLastBuffer
->AdvanceEnd(written
);
1339 if (result
== kOutputFull
) {
1340 RefPtr
<nsHtml5OwningUTF16Buffer
> newBuf
=
1341 nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE
);
1343 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1346 mLastBuffer
= (mLastBuffer
->next
= std::move(newBuf
));
1348 if (!mLookingForMetaCharset
&& mDecodingLocalFileWithoutTokenizing
) {
1349 MOZ_ASSERT(mNumBytesBuffered
< LOCAL_FILE_UTF_8_BUFFER_SIZE
);
1350 MOZ_ASSERT(!mStartedFeedingDetector
);
1351 for (auto&& buffer
: mBufferedBytes
) {
1352 FeedDetector(buffer
);
1354 MOZ_ASSERT(!mChardetEof
);
1356 auto [encoding
, source
] = GuessEncoding(true);
1357 mCharsetSource
= source
;
1358 if (encoding
!= mEncoding
) {
1359 mEncoding
= encoding
;
1360 nsresult rv
= ReDecodeLocalFile();
1361 if (NS_FAILED(rv
)) {
1368 MOZ_ASSERT(mEncoding
== UTF_8_ENCODING
);
1369 nsresult rv
= CommitLocalFileToEncoding();
1370 if (NS_FAILED(rv
)) {
1379 mStreamState
= STREAM_ENDED
;
1381 if (IsTerminatedOrInterrupted()) {
1385 ParseAvailableData();
1388 class nsHtml5RequestStopper
: public Runnable
{
1390 nsHtml5StreamParserPtr mStreamParser
;
1393 explicit nsHtml5RequestStopper(nsHtml5StreamParser
* aStreamParser
)
1394 : Runnable("nsHtml5RequestStopper"), mStreamParser(aStreamParser
) {}
1395 NS_IMETHOD
Run() override
{
1396 mozilla::MutexAutoLock
autoLock(mStreamParser
->mTokenizerMutex
);
1397 mStreamParser
->DoStopRequest();
1398 mStreamParser
->PostLoadFlusher();
1403 nsresult
nsHtml5StreamParser::OnStopRequest(nsIRequest
* aRequest
,
1405 MOZ_ASSERT(mRequest
== aRequest
, "Got Stop on wrong stream.");
1406 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
1407 nsCOMPtr
<nsIRunnable
> stopper
= new nsHtml5RequestStopper(this);
1408 if (NS_FAILED(mEventTarget
->Dispatch(stopper
, nsIThread::DISPATCH_NORMAL
))) {
1409 NS_WARNING("Dispatching StopRequest event failed.");
1414 void nsHtml5StreamParser::DoDataAvailableBuffer(
1415 mozilla::Buffer
<uint8_t>&& aBuffer
) {
1416 if (MOZ_UNLIKELY(!mBufferingBytes
)) {
1417 DoDataAvailable(aBuffer
);
1420 if (MOZ_UNLIKELY(mLookingForXmlDeclarationForXmlViewSource
)) {
1421 const uint8_t* elements
= aBuffer
.Elements();
1422 size_t length
= aBuffer
.Length();
1423 const uint8_t* lt
= (const uint8_t*)memchr(elements
, '>', length
);
1425 mBufferedBytes
.AppendElement(std::move(aBuffer
));
1429 // We found an '>'. Now there either is or isn't an XML decl.
1430 length
= (lt
- elements
) + 1;
1431 Vector
<uint8_t> contiguous
;
1432 for (auto&& buffer
: mBufferedBytes
) {
1433 if (!contiguous
.append(buffer
.Elements(), buffer
.Length())) {
1434 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1438 if (!contiguous
.append(elements
, length
)) {
1439 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1443 const Encoding
* encoding
=
1444 xmldecl_parse(contiguous
.begin(), contiguous
.length());
1446 mEncoding
= WrapNotNull(encoding
);
1447 mCharsetSource
= kCharsetFromXmlDeclaration
;
1450 mLookingForXmlDeclarationForXmlViewSource
= false;
1451 mBufferingBytes
= false;
1452 mUnicodeDecoder
= mEncoding
->NewDecoderWithoutBOMHandling();
1453 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
1455 for (auto&& buffer
: mBufferedBytes
) {
1456 DoDataAvailable(buffer
);
1458 DoDataAvailable(aBuffer
);
1459 mBufferedBytes
.Clear();
1462 CheckedInt
<size_t> bufferedPlusLength(aBuffer
.Length());
1463 bufferedPlusLength
+= mNumBytesBuffered
;
1464 if (!bufferedPlusLength
.isValid()) {
1465 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1468 // Ensure that WriteStreamBytes() sees buffers ending
1469 // exactly at the two special boundaries.
1470 bool metaBoundaryWithinBuffer
=
1471 mLookingForMetaCharset
&&
1472 mNumBytesBuffered
< UNCONDITIONAL_META_SCAN_BOUNDARY
&&
1473 bufferedPlusLength
.value() > UNCONDITIONAL_META_SCAN_BOUNDARY
;
1474 bool localFileLimitWithinBuffer
=
1475 mDecodingLocalFileWithoutTokenizing
&&
1476 mNumBytesBuffered
< LOCAL_FILE_UTF_8_BUFFER_SIZE
&&
1477 bufferedPlusLength
.value() > LOCAL_FILE_UTF_8_BUFFER_SIZE
;
1478 if (!metaBoundaryWithinBuffer
&& !localFileLimitWithinBuffer
) {
1479 // Truncation OK, because we just checked the range.
1480 mNumBytesBuffered
= bufferedPlusLength
.value();
1481 mBufferedBytes
.AppendElement(std::move(aBuffer
));
1482 DoDataAvailable(mBufferedBytes
.LastElement());
1485 !(metaBoundaryWithinBuffer
&& localFileLimitWithinBuffer
),
1486 "How can Necko give us a buffer this large?");
1487 size_t boundary
= metaBoundaryWithinBuffer
1488 ? UNCONDITIONAL_META_SCAN_BOUNDARY
1489 : LOCAL_FILE_UTF_8_BUFFER_SIZE
;
1490 // Truncation OK, because the constant is small enough.
1491 size_t overBoundary
= bufferedPlusLength
.value() - boundary
;
1492 MOZ_RELEASE_ASSERT(overBoundary
< aBuffer
.Length());
1493 size_t untilBoundary
= aBuffer
.Length() - overBoundary
;
1494 auto span
= aBuffer
.AsSpan();
1495 auto head
= span
.To(untilBoundary
);
1496 auto tail
= span
.From(untilBoundary
);
1497 MOZ_RELEASE_ASSERT(mNumBytesBuffered
+ untilBoundary
== boundary
);
1498 // The following copies may end up being useless, but optimizing
1499 // them away would add complexity.
1500 Maybe
<Buffer
<uint8_t>> maybeHead
= Buffer
<uint8_t>::CopyFrom(head
);
1501 if (maybeHead
.isNothing()) {
1502 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1505 mNumBytesBuffered
= boundary
;
1506 mBufferedBytes
.AppendElement(std::move(*maybeHead
));
1507 DoDataAvailable(mBufferedBytes
.LastElement());
1508 // Re-decode may have happened here.
1510 Maybe
<Buffer
<uint8_t>> maybeTail
= Buffer
<uint8_t>::CopyFrom(tail
);
1511 if (maybeTail
.isNothing()) {
1512 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1515 mNumBytesBuffered
+= tail
.Length();
1516 mBufferedBytes
.AppendElement(std::move(*maybeTail
));
1517 DoDataAvailable(mBufferedBytes
.LastElement());
1519 // Do this clean-up here to avoid use-after-free when
1520 // DoDataAvailable is passed a span pointing into an
1521 // element of mBufferedBytes.
1522 if (!mBufferingBytes
) {
1523 mBufferedBytes
.Clear();
1527 void nsHtml5StreamParser::DoDataAvailable(Span
<const uint8_t> aBuffer
) {
1528 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1529 MOZ_RELEASE_ASSERT(STREAM_BEING_READ
== mStreamState
,
1530 "DoDataAvailable called when stream not open.");
1531 mTokenizerMutex
.AssertCurrentThreadOwns();
1533 if (IsTerminated()) {
1539 if ((mForceAutoDetection
|| mCharsetSource
< kCharsetFromParentFrame
) &&
1540 !mBufferingBytes
&& !mReparseForbidden
&&
1541 !(mMode
== LOAD_AS_DATA
|| mMode
== VIEW_SOURCE_XML
)) {
1542 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
,
1543 "How is mBufferingBytes false if "
1544 "mDecodingLocalFileWithoutTokenizing is true?");
1545 FeedDetector(aBuffer
);
1547 rv
= WriteStreamBytes(aBuffer
);
1549 rv
= SniffStreamBytes(aBuffer
, false);
1551 if (NS_FAILED(rv
)) {
1556 if (IsTerminatedOrInterrupted()) {
1560 if (!mLookingForMetaCharset
&& mDecodingLocalFileWithoutTokenizing
) {
1564 ParseAvailableData();
1566 if (mBomState
!= BOM_SNIFFING_OVER
|| mFlushTimerArmed
|| mSpeculating
) {
1571 mozilla::MutexAutoLock
flushTimerLock(mFlushTimerMutex
);
1572 mFlushTimer
->InitWithNamedFuncCallback(
1573 nsHtml5StreamParser::TimerCallback
, static_cast<void*>(this),
1574 mFlushTimerEverFired
? StaticPrefs::html5_flushtimer_initialdelay()
1575 : StaticPrefs::html5_flushtimer_subsequentdelay(),
1576 nsITimer::TYPE_ONE_SHOT
, "nsHtml5StreamParser::DoDataAvailable");
1578 mFlushTimerArmed
= true;
1581 class nsHtml5DataAvailable
: public Runnable
{
1583 nsHtml5StreamParserPtr mStreamParser
;
1584 Buffer
<uint8_t> mData
;
1587 nsHtml5DataAvailable(nsHtml5StreamParser
* aStreamParser
,
1588 Buffer
<uint8_t>&& aData
)
1589 : Runnable("nsHtml5DataAvailable"),
1590 mStreamParser(aStreamParser
),
1591 mData(std::move(aData
)) {}
1592 NS_IMETHOD
Run() override
{
1593 mozilla::MutexAutoLock
autoLock(mStreamParser
->mTokenizerMutex
);
1594 mStreamParser
->DoDataAvailableBuffer(std::move(mData
));
1595 mStreamParser
->PostLoadFlusher();
1600 nsresult
nsHtml5StreamParser::OnDataAvailable(nsIRequest
* aRequest
,
1601 nsIInputStream
* aInStream
,
1602 uint64_t aSourceOffset
,
1606 MOZ_ASSERT(mRequest
== aRequest
, "Got data on wrong stream.");
1608 // Main thread to parser thread dispatch requires copying to buffer first.
1609 if (MOZ_UNLIKELY(NS_IsMainThread())) {
1610 if (NS_FAILED(rv
= mExecutor
->IsBroken())) {
1613 Maybe
<Buffer
<uint8_t>> maybe
= Buffer
<uint8_t>::Alloc(aLength
);
1614 if (maybe
.isNothing()) {
1615 return mExecutor
->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1617 Buffer
<uint8_t> data(std::move(*maybe
));
1618 rv
= aInStream
->Read(reinterpret_cast<char*>(data
.Elements()),
1619 data
.Length(), &totalRead
);
1620 NS_ENSURE_SUCCESS(rv
, rv
);
1621 MOZ_ASSERT(totalRead
== aLength
);
1623 nsCOMPtr
<nsIRunnable
> dataAvailable
=
1624 new nsHtml5DataAvailable(this, std::move(data
));
1625 if (NS_FAILED(mEventTarget
->Dispatch(dataAvailable
,
1626 nsIThread::DISPATCH_NORMAL
))) {
1627 NS_WARNING("Dispatching DataAvailable event failed.");
1632 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1633 mozilla::MutexAutoLock
autoLock(mTokenizerMutex
);
1635 if (NS_FAILED(rv
= mTreeBuilder
->IsBroken())) {
1639 // Since we're getting OnDataAvailable directly on the parser thread,
1640 // there is no nsHtml5DataAvailable that would call PostLoadFlusher.
1641 // Hence, we need to call PostLoadFlusher() before this method returns.
1642 // Braces for RAII clarity relative to the mutex despite not being
1643 // strictly necessary.
1645 auto speculationFlusher
= MakeScopeExit([&] { PostLoadFlusher(); });
1647 if (mBufferingBytes
) {
1648 Maybe
<Buffer
<uint8_t>> maybe
= Buffer
<uint8_t>::Alloc(aLength
);
1649 if (maybe
.isNothing()) {
1650 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1651 return NS_ERROR_OUT_OF_MEMORY
;
1653 Buffer
<uint8_t> data(std::move(*maybe
));
1654 rv
= aInStream
->Read(reinterpret_cast<char*>(data
.Elements()),
1655 data
.Length(), &totalRead
);
1656 NS_ENSURE_SUCCESS(rv
, rv
);
1657 MOZ_ASSERT(totalRead
== aLength
);
1658 DoDataAvailableBuffer(std::move(data
));
1661 // Read directly from response buffer.
1662 rv
= aInStream
->ReadSegments(CopySegmentsToParser
, this, aLength
,
1664 NS_ENSURE_SUCCESS(rv
, rv
);
1665 MOZ_ASSERT(totalRead
== aLength
);
1670 // Called under lock by function ptr
1672 nsresult
nsHtml5StreamParser::CopySegmentsToParser(
1673 nsIInputStream
* aInStream
, void* aClosure
, const char* aFromSegment
,
1674 uint32_t aToOffset
, uint32_t aCount
,
1675 uint32_t* aWriteCount
) MOZ_NO_THREAD_SAFETY_ANALYSIS
{
1676 nsHtml5StreamParser
* parser
= static_cast<nsHtml5StreamParser
*>(aClosure
);
1678 parser
->DoDataAvailable(AsBytes(Span(aFromSegment
, aCount
)));
1679 // Assume DoDataAvailable consumed all available bytes.
1680 *aWriteCount
= aCount
;
1684 const Encoding
* nsHtml5StreamParser::PreferredForInternalEncodingDecl(
1685 const nsAString
& aEncoding
) {
1686 const Encoding
* newEncoding
= Encoding::ForLabel(aEncoding
);
1688 // the encoding name is bogus
1689 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaUnsupported", true,
1690 mTokenizer
->getLineNumber());
1694 if (newEncoding
== UTF_16BE_ENCODING
|| newEncoding
== UTF_16LE_ENCODING
) {
1695 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaUtf16", true,
1696 mTokenizer
->getLineNumber());
1697 newEncoding
= UTF_8_ENCODING
;
1700 if (newEncoding
== X_USER_DEFINED_ENCODING
) {
1701 // WebKit/Blink hack for Indian and Armenian legacy sites
1702 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaUserDefined", true,
1703 mTokenizer
->getLineNumber());
1704 newEncoding
= WINDOWS_1252_ENCODING
;
1707 if (newEncoding
== REPLACEMENT_ENCODING
) {
1708 // No line number, because the replacement encoding doesn't allow
1709 // showing the lines.
1710 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaReplacement", true, 0);
1716 bool nsHtml5StreamParser::internalEncodingDeclaration(nsHtml5String aEncoding
) {
1717 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1718 if ((mCharsetSource
>= kCharsetFromMetaTag
&&
1719 mCharsetSource
!= kCharsetFromFinalAutoDetectionFile
) ||
1720 mSeenEligibleMetaCharset
) {
1724 nsString newEncoding
; // Not Auto, because using it to hold nsStringBuffer*
1725 aEncoding
.ToString(newEncoding
);
1726 auto encoding
= PreferredForInternalEncodingDecl(newEncoding
);
1731 mSeenEligibleMetaCharset
= true;
1733 if (!mLookingForMetaCharset
) {
1734 if (mInitialEncodingWasFromParentFrame
) {
1735 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaTooLateFrame", true,
1736 mTokenizer
->getLineNumber());
1738 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaTooLate", true,
1739 mTokenizer
->getLineNumber());
1743 if (mTemplatePushedOrHeadPopped
) {
1744 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaAfterHeadInKilobyte", false,
1745 mTokenizer
->getLineNumber());
1748 if (mForceAutoDetection
&&
1749 (encoding
->IsAsciiCompatible() || encoding
== ISO_2022_JP_ENCODING
)) {
1753 mNeedsEncodingSwitchTo
= encoding
;
1754 mEncodingSwitchSource
= kCharsetFromMetaTag
;
1758 bool nsHtml5StreamParser::TemplatePushedOrHeadPopped() {
1760 IsParserThread() || mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
,
1762 mTemplatePushedOrHeadPopped
= true;
1763 return mNumBytesBuffered
>= UNCONDITIONAL_META_SCAN_BOUNDARY
;
1766 void nsHtml5StreamParser::RememberGt(int32_t aPos
) {
1767 if (mLookingForMetaCharset
) {
1768 mGtBuffer
= mFirstBuffer
;
1773 void nsHtml5StreamParser::PostLoadFlusher() {
1774 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1775 mTokenizerMutex
.AssertCurrentThreadOwns();
1777 mTreeBuilder
->FlushLoads();
1778 // Dispatch this runnable unconditionally, because the loads
1779 // that need flushing may have been flushed earlier even if the
1780 // flush right above here did nothing. (Is this still true?)
1781 nsCOMPtr
<nsIRunnable
> runnable(mLoadFlusher
);
1783 DispatchToMain(CreateRenderBlockingRunnable(runnable
.forget())))) {
1784 NS_WARNING("failed to dispatch load flush event");
1787 if ((mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
) &&
1788 mTokenizer
->ShouldFlushViewSource()) {
1789 auto r
= mTreeBuilder
->Flush(); // delete useless ops
1790 MOZ_ASSERT(r
.isOk(), "Should have null sink with View Source");
1791 r
= mTokenizer
->FlushViewSource();
1793 MarkAsBroken(r
.unwrapErr());
1797 nsCOMPtr
<nsIRunnable
> runnable(mExecutorFlusher
);
1798 if (NS_FAILED(DispatchToMain(runnable
.forget()))) {
1799 NS_WARNING("failed to dispatch executor flush event");
1805 void nsHtml5StreamParser::FlushTreeOpsAndDisarmTimer() {
1806 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1807 if (mFlushTimerArmed
) {
1808 // avoid calling Cancel if the flush timer isn't armed to avoid acquiring
1811 mozilla::MutexAutoLock
flushTimerLock(mFlushTimerMutex
);
1812 mFlushTimer
->Cancel();
1814 mFlushTimerArmed
= false;
1816 if (mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
) {
1817 auto r
= mTokenizer
->FlushViewSource();
1819 MarkAsBroken(r
.unwrapErr());
1822 auto r
= mTreeBuilder
->Flush();
1824 MarkAsBroken(r
.unwrapErr());
1826 nsCOMPtr
<nsIRunnable
> runnable(mExecutorFlusher
);
1827 if (NS_FAILED(DispatchToMain(runnable
.forget()))) {
1828 NS_WARNING("failed to dispatch executor flush event");
1832 void nsHtml5StreamParser::SwitchDecoderIfAsciiSoFar(
1833 NotNull
<const Encoding
*> aEncoding
) {
1834 if (mEncoding
== aEncoding
) {
1835 MOZ_ASSERT(!mStartedFeedingDevTools
);
1836 // Report all already-decoded buffers to the dev tools if needed.
1837 if (mURIToSendToDevtools
) {
1838 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBufferOfMetaScan
;
1840 auto s
= Span(buffer
->getBuffer(), buffer
->getEnd());
1842 buffer
= buffer
->next
;
1847 if (!mEncoding
->IsAsciiCompatible() || !aEncoding
->IsAsciiCompatible()) {
1850 size_t numAscii
= 0;
1851 MOZ_ASSERT(mFirstBufferOfMetaScan
,
1852 "Why did we come here without starting meta scan?");
1853 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBufferOfMetaScan
;
1854 while (buffer
!= mFirstBuffer
) {
1855 MOZ_ASSERT(buffer
, "mFirstBuffer should have acted as sentinel!");
1856 MOZ_ASSERT(buffer
->getStart() == buffer
->getEnd(),
1857 "Why wasn't an early buffer fully consumed?");
1858 auto s
= Span(buffer
->getBuffer(), buffer
->getStart());
1862 numAscii
+= s
.Length();
1863 buffer
= buffer
->next
;
1865 auto s
= Span(mFirstBuffer
->getBuffer(), mFirstBuffer
->getStart());
1869 numAscii
+= s
.Length();
1871 MOZ_ASSERT(!mStartedFeedingDevTools
);
1872 // Report the ASCII prefix to dev tools if needed
1873 if (mURIToSendToDevtools
) {
1874 buffer
= mFirstBufferOfMetaScan
;
1875 while (buffer
!= mFirstBuffer
) {
1876 MOZ_ASSERT(buffer
, "mFirstBuffer should have acted as sentinel!");
1877 MOZ_ASSERT(buffer
->getStart() == buffer
->getEnd(),
1878 "Why wasn't an early buffer fully consumed?");
1879 auto s
= Span(buffer
->getBuffer(), buffer
->getStart());
1881 buffer
= buffer
->next
;
1883 auto s
= Span(mFirstBuffer
->getBuffer(), mFirstBuffer
->getStart());
1887 // Success! Now let's get rid of the already-decoded but not tokenized data:
1888 mFirstBuffer
->setEnd(mFirstBuffer
->getStart());
1889 mLastBuffer
= mFirstBuffer
;
1890 mFirstBuffer
->next
= nullptr;
1892 // Note: We could have scanned further for ASCII, which could avoid some
1893 // buffer deallocation and reallocation. However, chances are that if we got
1894 // until meta without non-ASCII before, there's going to be a title with
1895 // non-ASCII soon after anyway, so let's avoid the complexity of finding out.
1897 MOZ_ASSERT(mUnicodeDecoder
, "How come we scanned meta without a decoder?");
1898 mEncoding
= aEncoding
;
1899 mEncoding
->NewDecoderWithoutBOMHandlingInto(*mUnicodeDecoder
);
1900 mHasHadErrors
= false;
1902 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
,
1903 "Must have set mDecodingLocalFileWithoutTokenizing to false to "
1904 "report data to dev tools below");
1905 MOZ_ASSERT(!mLookingForMetaCharset
,
1906 "Must have set mLookingForMetaCharset to false to report data to "
1909 // Now skip over as many bytes and redecode the tail of the
1912 for (auto&& buffer
: mBufferedBytes
) {
1913 size_t nextSkipped
= skipped
+ buffer
.Length();
1914 if (nextSkipped
<= numAscii
) {
1915 skipped
= nextSkipped
;
1918 if (skipped
>= numAscii
) {
1919 WriteStreamBytes(buffer
);
1920 skipped
= nextSkipped
;
1923 size_t tailLength
= nextSkipped
- numAscii
;
1924 WriteStreamBytes(Span
<uint8_t>(buffer
).From(buffer
.Length() - tailLength
));
1925 skipped
= nextSkipped
;
1929 size_t nsHtml5StreamParser::CountGts() {
1934 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBufferOfMetaScan
;
1936 MOZ_ASSERT(buffer
, "How did we walk past mGtBuffer?");
1937 char16_t
* buf
= buffer
->getBuffer();
1938 if (buffer
== mGtBuffer
) {
1939 for (int32_t i
= 0; i
<= mGtPos
; ++i
) {
1940 if (buf
[i
] == u
'>') {
1946 for (int32_t i
= 0; i
< buffer
->getEnd(); ++i
) {
1947 if (buf
[i
] == u
'>') {
1951 buffer
= buffer
->next
;
1956 void nsHtml5StreamParser::DiscardMetaSpeculation() {
1957 mozilla::MutexAutoLock
speculationAutoLock(mSpeculationMutex
);
1958 // Rewind the stream
1959 MOZ_ASSERT(!mAtEOF
, "How did we end up setting this?");
1960 mTokenizer
->resetToDataState();
1961 mTokenizer
->setLineNumber(1);
1964 if (mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
) {
1965 // resetToDataState() above logically rewinds to the state before
1966 // the plain text start, so we need to start plain text again to
1967 // put the tokenizer into the plain text state.
1968 mTokenizer
->StartPlainText();
1971 mFirstBuffer
= mLastBuffer
;
1972 mFirstBuffer
->setStart(0);
1973 mFirstBuffer
->setEnd(0);
1974 mFirstBuffer
->next
= nullptr;
1976 mTreeBuilder
->flushCharacters(); // empty the pending buffer
1977 mTreeBuilder
->ClearOps(); // now get rid of the failed ops
1979 if (mMode
== VIEW_SOURCE_HTML
) {
1980 mTokenizer
->RewindViewSource();
1984 // We know that this resets the tree builder back to the start state.
1985 // This must happen _after_ the flushCharacters() call above!
1986 const auto& speculation
= mSpeculations
.ElementAt(0);
1987 mTreeBuilder
->loadState(speculation
->GetSnapshot());
1990 // Experimentation suggests that we don't need to do anything special
1991 // for ignoring the leading LF in View Source here.
1993 mSpeculations
.Clear(); // potentially a huge number of destructors
1994 // run here synchronously...
1996 // Now set up a new speculation for the main thread to find.
1997 // Note that we stay in the speculating state, because the main thread
1998 // knows how to come out of that state and this thread does not.
2000 nsHtml5Speculation
* speculation
= new nsHtml5Speculation(
2001 mFirstBuffer
, mFirstBuffer
->getStart(), mTokenizer
->getLineNumber(),
2002 mTokenizer
->getColumnNumber(), mTreeBuilder
->newSnapshot());
2003 MOZ_ASSERT(!mFlushTimerArmed
, "How did we end up arming the timer?");
2004 if (mMode
== VIEW_SOURCE_HTML
) {
2005 mTokenizer
->SetViewSourceOpSink(speculation
);
2006 mTokenizer
->StartViewSourceCharacters();
2008 MOZ_ASSERT(mMode
!= VIEW_SOURCE_XML
);
2009 mTreeBuilder
->SetOpSink(speculation
);
2011 mSpeculations
.AppendElement(speculation
); // adopts the pointer
2012 MOZ_ASSERT(mSpeculating
, "How did we end speculating?");
2016 * The general idea is to match WebKit and Blink exactly for meta
2019 * 1. WebKit and Blink look for meta as if scripting was disabled
2020 * for `noscript` purposes. This implementation matches the
2021 * `noscript` treatment of the observable DOM building (in order
2022 * to be able to use the same tree builder run).
2023 * 2. WebKit and Blink look for meta as if the foreign content
2024 * feedback from the tree builder to the tokenizer didn't exist.
2025 * This implementation considers the foreign content rules in
2026 * order to be able to use the same tree builder run for meta
2027 * and the observable DOM building. Note that since <svg> and
2028 * <math> imply the end of head, this only matters for meta after
2029 * head but starting within the 1024-byte zone.
2031 * Template is treated specially, because that WebKit/Blink behavior
2032 * is easy to emulate unlike the above two exceptions. In general,
2033 * the meta scan token handler in WebKit and Blink behaves as if there
2034 * was a scripting-disabled tree builder predating the introduction
2035 * of foreign content and template.
2037 * Meta is honored if it _starts_ within the first 1024 kilobytes or,
2038 * if by the 1024-byte boundary head hasn't ended and a template
2039 * element hasn't started, a meta occurs before the first of the head
2040 * ending or a template element starting.
2042 * If a meta isn't honored according to the above definition, and
2043 * we aren't dealing with plain text, the buffered bytes, which by
2044 * now have to contain `>` character unless we encountered EOF, are
2045 * scanned for syntax resembling an XML declaration.
2047 * If neither a meta nor syntax resembling an XML declaration has
2048 * been honored and we aren't inheriting the encoding from a
2049 * same-origin parent or parsing for XHR, chardetng is used.
2050 * chardetng runs first for the part of the document that was searched
2051 * for meta and then at EOF. The part searched for meta is defined as
2052 * follows in order to avoid network buffer boundary-dependent
2055 * 1. At least the first 1024 bytes. (This is what happens for plain
2057 * 2. If the 1024-byte boundary is within a tag, comment, doctype,
2058 * or CDATA section, at least up to the end of that token or CDATA
2059 * section. (Exception: If the 1024-byte boundary is in an RCDATA
2060 * end tag that hasn't yet been decided to be an end tag, the
2061 * token is not considered.)
2062 * 3. If at the 1024-byte boundary, head hasn't ended and there hasn't
2063 * been a template tag, up to the end of the first template tag
2064 * or token ending the head, whichever comes first.
2065 * 4. Except if head is ended by a text token, only to the end of the
2066 * most recent tag, comment, or doctype token. (Because text is
2067 * coalesced, so it would be harder to correlate the text to the
2070 * An encoding-related reload is still possible if chardetng's guess
2071 * at EOF differs from its initial guess.
2073 bool nsHtml5StreamParser::ProcessLookingForMetaCharset(bool aEof
) {
2074 MOZ_ASSERT(mBomState
== BOM_SNIFFING_OVER
);
2075 MOZ_ASSERT(mMode
!= VIEW_SOURCE_XML
);
2076 bool rewound
= false;
2077 MOZ_ASSERT(mForceAutoDetection
||
2078 mCharsetSource
< kCharsetFromInitialAutoDetectionASCII
||
2079 mCharsetSource
== kCharsetFromParentFrame
,
2080 "Why are we looking for meta charset if we've seen it?");
2081 // NOTE! We may come here multiple times with
2082 // mNumBytesBuffered == UNCONDITIONAL_META_SCAN_BOUNDARY
2083 // if the tokenizer suspends multiple times after decoding has reached
2084 // mNumBytesBuffered == UNCONDITIONAL_META_SCAN_BOUNDARY. That's why
2085 // we need to also check whether the we are at the end of the last
2087 // Note that DoDataAvailableBuffer() ensures that the code here has
2088 // the opportunity to run at the exact UNCONDITIONAL_META_SCAN_BOUNDARY
2089 // even if there isn't a network buffer boundary there.
2090 bool atKilobyte
= false;
2091 if ((mNumBytesBuffered
== UNCONDITIONAL_META_SCAN_BOUNDARY
&&
2092 mFirstBuffer
== mLastBuffer
&& !mFirstBuffer
->hasMore())) {
2094 mTokenizer
->AtKilobyteBoundary();
2096 if (!mNeedsEncodingSwitchTo
&&
2097 (aEof
|| (mTemplatePushedOrHeadPopped
&&
2098 !mTokenizer
->IsInTokenStartedAtKilobyteBoundary() &&
2100 mNumBytesBuffered
> UNCONDITIONAL_META_SCAN_BOUNDARY
)))) {
2101 // meta charset was not found
2102 mLookingForMetaCharset
= false;
2103 if (mStartsWithLtQuestion
&& mCharsetSource
< kCharsetFromXmlDeclaration
) {
2104 // Look for bogo XML declaration.
2105 // Search the first buffer in the hope that '>' is within it.
2106 MOZ_ASSERT(!mBufferedBytes
.IsEmpty(),
2107 "How did at least <? not get buffered?");
2108 Buffer
<uint8_t>& first
= mBufferedBytes
[0];
2109 const Encoding
* encoding
=
2110 xmldecl_parse(first
.Elements(), first
.Length());
2112 // Our bogo XML declaration scanner wants to see a contiguous buffer, so
2113 // let's linearize the data. (Ideally, the XML declaration scanner would
2114 // be incremental, but this is the rare path anyway.)
2115 Vector
<uint8_t> contiguous
;
2116 if (!contiguous
.append(first
.Elements(), first
.Length())) {
2117 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2120 for (size_t i
= 1; i
< mBufferedBytes
.Length(); ++i
) {
2121 Buffer
<uint8_t>& buffer
= mBufferedBytes
[i
];
2122 const uint8_t* elements
= buffer
.Elements();
2123 size_t length
= buffer
.Length();
2124 const uint8_t* lt
= (const uint8_t*)memchr(elements
, '>', length
);
2127 length
= (lt
- elements
) + 1;
2130 if (!contiguous
.append(elements
, length
)) {
2131 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2135 // Avoid linearizing all buffered bytes unnecessarily.
2139 encoding
= xmldecl_parse(contiguous
.begin(), contiguous
.length());
2142 if (!(mForceAutoDetection
&& (encoding
->IsAsciiCompatible() ||
2143 encoding
== ISO_2022_JP_ENCODING
))) {
2144 mForceAutoDetection
= false;
2145 mNeedsEncodingSwitchTo
= encoding
;
2146 mEncodingSwitchSource
= kCharsetFromXmlDeclaration
;
2150 // Check again in case we found an encoding in the bogo XML declaration.
2151 if (!mNeedsEncodingSwitchTo
&&
2152 (mForceAutoDetection
||
2153 mCharsetSource
< kCharsetFromInitialAutoDetectionASCII
) &&
2154 !(mMode
== LOAD_AS_DATA
|| mMode
== VIEW_SOURCE_XML
) &&
2155 !(mDecodingLocalFileWithoutTokenizing
&& !aEof
&&
2156 mNumBytesBuffered
<= LOCAL_FILE_UTF_8_BUFFER_SIZE
)) {
2157 MOZ_ASSERT(!mStartedFeedingDetector
);
2158 if (mNumBytesBuffered
== UNCONDITIONAL_META_SCAN_BOUNDARY
|| aEof
) {
2159 // We know that all the buffered bytes have been tokenized, so feed
2160 // them all to chardetng.
2161 for (auto&& buffer
: mBufferedBytes
) {
2162 FeedDetector(buffer
);
2165 MOZ_ASSERT(!mChardetEof
);
2168 auto [encoding
, source
] = GuessEncoding(true);
2169 mNeedsEncodingSwitchTo
= encoding
;
2170 mEncodingSwitchSource
= source
;
2171 } else if (mNumBytesBuffered
> UNCONDITIONAL_META_SCAN_BOUNDARY
) {
2172 size_t gtsLeftToFind
= CountGts();
2173 size_t bytesSeen
= 0;
2174 // We sync the bytes to the UTF-16 code units seen to avoid depending
2175 // on network buffer boundaries. We do the syncing by counting '>'
2176 // bytes / code units. However, we always scan at least 1024 bytes.
2177 // The 1024-byte boundary is guaranteed to be between buffers.
2178 // The guarantee is implemented in DoDataAvailableBuffer().
2179 for (auto&& buffer
: mBufferedBytes
) {
2180 if (!mNeedsEncodingSwitchTo
) {
2181 if (gtsLeftToFind
) {
2182 auto span
= buffer
.AsSpan();
2184 for (size_t i
= 0; i
< span
.Length(); ++i
) {
2185 if (span
[i
] == uint8_t('>')) {
2187 if (!gtsLeftToFind
) {
2188 if (bytesSeen
< UNCONDITIONAL_META_SCAN_BOUNDARY
) {
2192 FeedDetector(span
.To(i
));
2193 auto [encoding
, source
] = GuessEncoding(true);
2194 mNeedsEncodingSwitchTo
= encoding
;
2195 mEncodingSwitchSource
= source
;
2196 FeedDetector(span
.From(i
));
2197 bytesSeen
+= buffer
.Length();
2198 // No need to update bytesSeen anymore, but let's do it for
2200 // We should do `continue outer;` but C++ can't.
2207 FeedDetector(buffer
);
2208 bytesSeen
+= buffer
.Length();
2212 if (bytesSeen
== UNCONDITIONAL_META_SCAN_BOUNDARY
) {
2213 auto [encoding
, source
] = GuessEncoding(true);
2214 mNeedsEncodingSwitchTo
= encoding
;
2215 mEncodingSwitchSource
= source
;
2218 FeedDetector(buffer
);
2219 bytesSeen
+= buffer
.Length();
2222 MOZ_ASSERT(mNeedsEncodingSwitchTo
,
2223 "How come we didn't call GuessEncoding()?");
2226 if (mNeedsEncodingSwitchTo
) {
2227 mDecodingLocalFileWithoutTokenizing
= false;
2228 mLookingForMetaCharset
= false;
2230 auto needsEncodingSwitchTo
= WrapNotNull(mNeedsEncodingSwitchTo
);
2231 mNeedsEncodingSwitchTo
= nullptr;
2233 SwitchDecoderIfAsciiSoFar(needsEncodingSwitchTo
);
2234 // The above line may have changed mEncoding so that mEncoding equals
2235 // needsEncodingSwitchTo.
2237 mCharsetSource
= mEncodingSwitchSource
;
2239 if (mMode
== VIEW_SOURCE_HTML
) {
2240 auto r
= mTokenizer
->FlushViewSource();
2242 MarkAsBroken(r
.unwrapErr());
2246 auto r
= mTreeBuilder
->Flush();
2248 MarkAsBroken(r
.unwrapErr());
2252 if (mEncoding
!= needsEncodingSwitchTo
) {
2253 // Speculation failed
2256 if (mEncoding
== ISO_2022_JP_ENCODING
||
2257 needsEncodingSwitchTo
== ISO_2022_JP_ENCODING
) {
2258 // Chances are no Web author will fix anything due to this message, so
2259 // this is here to help understanding issues when debugging sites made
2261 mTreeBuilder
->MaybeComplainAboutCharset("EncSpeculationFail2022", false,
2262 mTokenizer
->getLineNumber());
2264 if (mCharsetSource
== kCharsetFromMetaTag
) {
2265 mTreeBuilder
->MaybeComplainAboutCharset(
2266 "EncSpeculationFailMeta", false, mTokenizer
->getLineNumber());
2267 } else if (mCharsetSource
== kCharsetFromXmlDeclaration
) {
2268 // This intentionally refers to the line number of how far ahead
2269 // the document was parsed even though the bogo XML decl is always
2271 mTreeBuilder
->MaybeComplainAboutCharset(
2272 "EncSpeculationFailXml", false, mTokenizer
->getLineNumber());
2276 DiscardMetaSpeculation();
2277 // Redecode the stream.
2278 mEncoding
= needsEncodingSwitchTo
;
2279 mUnicodeDecoder
= mEncoding
->NewDecoderWithBOMRemoval();
2280 mHasHadErrors
= false;
2282 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
,
2283 "Must have set mDecodingLocalFileWithoutTokenizing to false "
2284 "to report data to dev tools below");
2285 MOZ_ASSERT(!mLookingForMetaCharset
,
2286 "Must have set mLookingForMetaCharset to false to report data "
2287 "to dev tools below");
2288 for (auto&& buffer
: mBufferedBytes
) {
2289 nsresult rv
= WriteStreamBytes(buffer
);
2290 if (NS_FAILED(rv
)) {
2296 } else if (!mLookingForMetaCharset
&& !mDecodingLocalFileWithoutTokenizing
) {
2297 MOZ_ASSERT(!mStartedFeedingDevTools
);
2298 // Report all already-decoded buffers to the dev tools if needed.
2299 if (mURIToSendToDevtools
) {
2300 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBufferOfMetaScan
;
2302 auto s
= Span(buffer
->getBuffer(), buffer
->getEnd());
2304 buffer
= buffer
->next
;
2308 if (!mLookingForMetaCharset
) {
2309 mGtBuffer
= nullptr;
2312 if (!mDecodingLocalFileWithoutTokenizing
) {
2313 mFirstBufferOfMetaScan
= nullptr;
2314 mBufferingBytes
= false;
2315 mBufferedBytes
.Clear();
2316 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, true);
2317 if (mMode
== VIEW_SOURCE_HTML
) {
2318 auto r
= mTokenizer
->FlushViewSource();
2320 MarkAsBroken(r
.unwrapErr());
2324 auto r
= mTreeBuilder
->Flush();
2326 MarkAsBroken(r
.unwrapErr());
2334 void nsHtml5StreamParser::ParseAvailableData() {
2335 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
2336 mTokenizerMutex
.AssertCurrentThreadOwns();
2337 MOZ_ASSERT(!(mDecodingLocalFileWithoutTokenizing
&& !mLookingForMetaCharset
));
2339 if (IsTerminatedOrInterrupted()) {
2343 if (mSpeculating
&& !IsSpeculationEnabled()) {
2347 bool requestedReload
= false;
2349 if (!mFirstBuffer
->hasMore()) {
2350 if (mFirstBuffer
== mLastBuffer
) {
2351 switch (mStreamState
) {
2352 case STREAM_BEING_READ
:
2353 // never release the last buffer.
2354 if (!mSpeculating
) {
2355 // reuse buffer space if not speculating
2356 mFirstBuffer
->setStart(0);
2357 mFirstBuffer
->setEnd(0);
2359 return; // no more data for now but expecting more
2364 if (mLookingForMetaCharset
) {
2365 // When called with aEof=true, ProcessLookingForMetaCharset()
2366 // is guaranteed to set mLookingForMetaCharset to false so
2367 // that we can't come here twice.
2368 if (ProcessLookingForMetaCharset(true)) {
2369 if (IsTerminatedOrInterrupted()) {
2374 } else if ((mForceAutoDetection
||
2375 mCharsetSource
< kCharsetFromParentFrame
) &&
2376 !(mMode
== LOAD_AS_DATA
|| mMode
== VIEW_SOURCE_XML
) &&
2377 !mReparseForbidden
) {
2378 // An earlier DetectorEof() call is possible in which case
2379 // the one here is a no-op.
2381 auto [encoding
, source
] = GuessEncoding(false);
2382 if (encoding
!= mEncoding
) {
2383 // Request a reload from the docshell.
2386 kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII
&&
2388 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLDInitialWasASCII
) ||
2389 source
== kCharsetFromFinalUserForcedAutoDetection
);
2390 mTreeBuilder
->NeedsCharsetSwitchTo(encoding
, source
, 0);
2391 requestedReload
= true;
2392 } else if (mCharsetSource
==
2393 kCharsetFromInitialAutoDetectionASCII
&&
2394 mDetectorHasSeenNonAscii
) {
2395 mCharsetSource
= source
;
2396 mTreeBuilder
->UpdateCharsetSource(mCharsetSource
);
2401 if (!mForceAutoDetection
&& !requestedReload
) {
2402 if (mCharsetSource
== kCharsetFromParentFrame
) {
2403 mTreeBuilder
->MaybeComplainAboutCharset("EncNoDeclarationFrame",
2405 } else if (mCharsetSource
== kCharsetFromXmlDeclaration
) {
2406 // We know the bogo XML decl is always on the first line.
2407 mTreeBuilder
->MaybeComplainAboutCharset("EncXmlDecl", false, 1);
2410 kCharsetFromInitialAutoDetectionWouldHaveBeenUTF8
&&
2412 kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD
) {
2413 if (mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
) {
2414 mTreeBuilder
->MaybeComplainAboutCharset("EncNoDeclPlain",
2417 mTreeBuilder
->MaybeComplainAboutCharset("EncNoDecl", true, 0);
2421 if (mHasHadErrors
&& mEncoding
!= REPLACEMENT_ENCODING
) {
2422 if (mEncoding
== UTF_8_ENCODING
) {
2423 mTreeBuilder
->TryToEnableEncodingMenu();
2425 if (mCharsetSource
== kCharsetFromParentFrame
) {
2426 if (mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
) {
2427 mTreeBuilder
->MaybeComplainAboutCharset(
2428 "EncErrorFramePlain", true, 0);
2430 mTreeBuilder
->MaybeComplainAboutCharset("EncErrorFrame",
2434 mCharsetSource
>= kCharsetFromXmlDeclaration
&&
2436 kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII
&&
2438 kCharsetFromFinalUserForcedAutoDetection
)) {
2439 mTreeBuilder
->MaybeComplainAboutCharset("EncError", true, 0);
2443 if (NS_SUCCEEDED(mTreeBuilder
->IsBroken())) {
2446 if (NS_FAILED((rv
= mTreeBuilder
->IsBroken()))) {
2449 mTreeBuilder
->StreamEnded();
2450 if (mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
) {
2451 if (!mTokenizer
->EndViewSource()) {
2452 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2457 FlushTreeOpsAndDisarmTimer();
2458 return; // no more data and not expecting more
2460 MOZ_ASSERT_UNREACHABLE("It should be impossible to reach this.");
2464 mFirstBuffer
= mFirstBuffer
->next
;
2468 // now we have a non-empty buffer
2469 mFirstBuffer
->adjust(mLastWasCR
);
2471 if (mFirstBuffer
->hasMore()) {
2472 if (!mTokenizer
->EnsureBufferSpace(mFirstBuffer
->getLength())) {
2473 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2476 mLastWasCR
= mTokenizer
->tokenizeBuffer(mFirstBuffer
);
2478 if (NS_FAILED((rv
= mTreeBuilder
->IsBroken()))) {
2482 if (mTreeBuilder
->HasScript()) {
2483 // HasScript() cannot return true if the tree builder is preventing
2484 // script execution.
2485 MOZ_ASSERT(mMode
== NORMAL
);
2486 mozilla::MutexAutoLock
speculationAutoLock(mSpeculationMutex
);
2487 nsHtml5Speculation
* speculation
= new nsHtml5Speculation(
2488 mFirstBuffer
, mFirstBuffer
->getStart(), mTokenizer
->getLineNumber(),
2489 mTokenizer
->getColumnNumber(), mTreeBuilder
->newSnapshot());
2490 mTreeBuilder
->AddSnapshotToScript(speculation
->GetSnapshot(),
2491 speculation
->GetStartLineNumber());
2492 if (mLookingForMetaCharset
) {
2493 if (mMode
== VIEW_SOURCE_HTML
) {
2494 auto r
= mTokenizer
->FlushViewSource();
2496 MarkAsBroken(r
.unwrapErr());
2500 auto r
= mTreeBuilder
->Flush();
2502 MarkAsBroken(r
.unwrapErr());
2506 FlushTreeOpsAndDisarmTimer();
2508 mTreeBuilder
->SetOpSink(speculation
);
2509 mSpeculations
.AppendElement(speculation
); // adopts the pointer
2510 mSpeculating
= true;
2512 if (IsTerminatedOrInterrupted()) {
2516 if (mLookingForMetaCharset
) {
2517 Unused
<< ProcessLookingForMetaCharset(false);
2522 class nsHtml5StreamParserContinuation
: public Runnable
{
2524 nsHtml5StreamParserPtr mStreamParser
;
2527 explicit nsHtml5StreamParserContinuation(nsHtml5StreamParser
* aStreamParser
)
2528 : Runnable("nsHtml5StreamParserContinuation"),
2529 mStreamParser(aStreamParser
) {}
2530 NS_IMETHOD
Run() override
{
2531 mozilla::MutexAutoLock
autoLock(mStreamParser
->mTokenizerMutex
);
2532 mStreamParser
->Uninterrupt();
2533 mStreamParser
->ParseAvailableData();
2538 void nsHtml5StreamParser::ContinueAfterScriptsOrEncodingCommitment(
2539 nsHtml5Tokenizer
* aTokenizer
, nsHtml5TreeBuilder
* aTreeBuilder
,
2541 // nullptr for aTokenizer means encoding commitment as opposed to the "after
2544 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
2545 MOZ_ASSERT(mMode
!= VIEW_SOURCE_XML
,
2546 "ContinueAfterScriptsOrEncodingCommitment called in XML view "
2548 MOZ_ASSERT(!(aTokenizer
&& mMode
== VIEW_SOURCE_HTML
),
2549 "ContinueAfterScriptsOrEncodingCommitment called with non-null "
2550 "tokenizer in HTML view "
2552 if (NS_FAILED(mExecutor
->IsBroken())) {
2555 MOZ_ASSERT(!(aTokenizer
&& mMode
!= NORMAL
),
2556 "We should only be executing scripts in the normal mode.");
2557 if (!aTokenizer
&& (mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
||
2558 mMode
== VIEW_SOURCE_HTML
)) {
2559 // Take the ops that were generated from OnStartRequest for the synthetic
2560 // head section of the document for plain text and HTML View Source.
2561 // XML View Source never needs this kind of encoding commitment.
2562 // We need to take the ops here so that they end up in the queue before
2563 // the ops that we take from a speculation later in this method.
2564 if (!mExecutor
->TakeOpsFromStage()) {
2565 mExecutor
->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2570 mExecutor
->AssertStageEmpty();
2573 bool speculationFailed
= false;
2575 mozilla::MutexAutoLock
speculationAutoLock(mSpeculationMutex
);
2576 if (mSpeculations
.IsEmpty()) {
2577 MOZ_ASSERT_UNREACHABLE(
2578 "ContinueAfterScriptsOrEncodingCommitment called without "
2583 const auto& speculation
= mSpeculations
.ElementAt(0);
2585 (aLastWasCR
|| !aTokenizer
->isInDataState() ||
2586 !aTreeBuilder
->snapshotMatches(speculation
->GetSnapshot()))) {
2587 speculationFailed
= true;
2588 // We've got a failed speculation :-(
2589 MaybeDisableFutureSpeculation();
2590 Interrupt(); // Make the parser thread release the tokenizer mutex sooner
2591 // Note that the interrupted state continues across possible intervening
2592 // Necko events until the nsHtml5StreamParserContinuation posted at the
2593 // end of this method runs. Therefore, this thread is guaranteed to
2594 // acquire mTokenizerMutex soon even if an intervening Necko event grabbed
2595 // it between now and the acquisition below.
2597 // now fall out of the speculationAutoLock into the tokenizerAutoLock
2600 // We've got a successful speculation!
2601 if (mSpeculations
.Length() > 1) {
2602 // the first speculation isn't the current speculation, so there's
2603 // no need to bother the parser thread.
2604 if (!speculation
->FlushToSink(mExecutor
)) {
2605 mExecutor
->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2608 MOZ_ASSERT(!mExecutor
->IsScriptExecuting(),
2609 "ParseUntilBlocked() was supposed to ensure we don't come "
2610 "here when scripts are executing.");
2611 MOZ_ASSERT(!aTokenizer
|| mExecutor
->IsInFlushLoop(),
2612 "How are we here if "
2613 "RunFlushLoop() didn't call ParseUntilBlocked() or we're "
2614 "not committing to an encoding?");
2615 mSpeculations
.RemoveElementAt(0);
2619 Interrupt(); // Make the parser thread release the tokenizer mutex sooner
2620 // Note that the interrupted state continues across possible intervening
2621 // Necko events until the nsHtml5StreamParserContinuation posted at the
2622 // end of this method runs. Therefore, this thread is guaranteed to
2623 // acquire mTokenizerMutex soon even if an intervening Necko event grabbed
2624 // it between now and the acquisition below.
2627 // the first speculation is the current speculation. Need to
2628 // release the the speculation mutex and acquire the tokenizer
2629 // mutex. (Just acquiring the other mutex here would deadlock)
2633 mozilla::MutexAutoLock
tokenizerAutoLock(mTokenizerMutex
);
2636 mAtomTable
.SetPermittedLookupEventTarget(
2637 GetMainThreadSerialEventTarget());
2640 // In principle, the speculation mutex should be acquired here,
2641 // but there's no point, because the parser thread only acquires it
2642 // when it has also acquired the tokenizer mutex and we are already
2643 // holding the tokenizer mutex.
2644 if (speculationFailed
) {
2645 MOZ_ASSERT(mMode
== NORMAL
);
2646 // Rewind the stream
2648 const auto& speculation
= mSpeculations
.ElementAt(0);
2649 mFirstBuffer
= speculation
->GetBuffer();
2650 mFirstBuffer
->setStart(speculation
->GetStart());
2651 mTokenizer
->setLineNumber(speculation
->GetStartLineNumber());
2652 mTokenizer
->setColumnNumberAndResetNextLine(
2653 speculation
->GetStartColumnNumber());
2655 nsContentUtils::ReportToConsole(
2656 nsIScriptError::warningFlag
, "DOM Events"_ns
,
2657 mExecutor
->GetDocument(), nsContentUtils::eDOM_PROPERTIES
,
2658 "SpeculationFailed2", nsTArray
<nsString
>(), nullptr, u
""_ns
,
2659 speculation
->GetStartLineNumber(),
2660 speculation
->GetStartColumnNumber());
2662 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBuffer
->next
;
2664 buffer
->setStart(0);
2665 buffer
= buffer
->next
;
2668 mSpeculations
.Clear(); // potentially a huge number of destructors
2669 // run here synchronously on the main thread...
2671 mTreeBuilder
->flushCharacters(); // empty the pending buffer
2672 mTreeBuilder
->ClearOps(); // now get rid of the failed ops
2674 mTreeBuilder
->SetOpSink(mExecutor
->GetStage());
2675 mExecutor
->StartReadingFromStage();
2676 mSpeculating
= false;
2679 mLastWasCR
= aLastWasCR
;
2680 mTokenizer
->loadState(aTokenizer
);
2681 mTreeBuilder
->loadState(aTreeBuilder
);
2683 // We've got a successful speculation and at least a moment ago it was
2684 // the current speculation
2685 if (!mSpeculations
.ElementAt(0)->FlushToSink(mExecutor
)) {
2686 mExecutor
->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2689 MOZ_ASSERT(!mExecutor
->IsScriptExecuting(),
2690 "ParseUntilBlocked() was supposed to ensure we don't come "
2691 "here when scripts are executing.");
2692 MOZ_ASSERT(!aTokenizer
|| mExecutor
->IsInFlushLoop(),
2693 "How are we here if "
2694 "RunFlushLoop() didn't call ParseUntilBlocked() or we're not "
2695 "committing to an encoding?");
2696 mSpeculations
.RemoveElementAt(0);
2697 if (mSpeculations
.IsEmpty()) {
2698 if (mMode
== VIEW_SOURCE_HTML
) {
2699 // If we looked for meta charset in the HTML View Source case.
2700 mTokenizer
->SetViewSourceOpSink(mExecutor
->GetStage());
2702 // yes, it was still the only speculation. Now stop speculating
2703 // However, before telling the executor to read from stage, flush
2704 // any pending ops straight to the executor, because otherwise
2705 // they remain unflushed until we get more data from the network.
2706 mTreeBuilder
->SetOpSink(mExecutor
);
2707 auto r
= mTreeBuilder
->Flush(true);
2709 mExecutor
->MarkAsBroken(r
.unwrapErr());
2712 mTreeBuilder
->SetOpSink(mExecutor
->GetStage());
2714 mExecutor
->StartReadingFromStage();
2715 mSpeculating
= false;
2718 nsCOMPtr
<nsIRunnable
> event
= new nsHtml5StreamParserContinuation(this);
2719 if (NS_FAILED(mEventTarget
->Dispatch(event
, nsIThread::DISPATCH_NORMAL
))) {
2720 NS_WARNING("Failed to dispatch nsHtml5StreamParserContinuation");
2722 // A stream event might run before this event runs, but that's harmless.
2724 mAtomTable
.SetPermittedLookupEventTarget(mEventTarget
);
2729 void nsHtml5StreamParser::ContinueAfterFailedCharsetSwitch() {
2730 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
2731 nsCOMPtr
<nsIRunnable
> event
= new nsHtml5StreamParserContinuation(this);
2732 if (NS_FAILED(mEventTarget
->Dispatch(event
, nsIThread::DISPATCH_NORMAL
))) {
2733 NS_WARNING("Failed to dispatch nsHtml5StreamParserContinuation");
2737 class nsHtml5TimerKungFu
: public Runnable
{
2739 nsHtml5StreamParserPtr mStreamParser
;
2742 explicit nsHtml5TimerKungFu(nsHtml5StreamParser
* aStreamParser
)
2743 : Runnable("nsHtml5TimerKungFu"), mStreamParser(aStreamParser
) {}
2744 NS_IMETHOD
Run() override
{
2745 mozilla::MutexAutoLock
flushTimerLock(mStreamParser
->mFlushTimerMutex
);
2746 if (mStreamParser
->mFlushTimer
) {
2747 mStreamParser
->mFlushTimer
->Cancel();
2748 mStreamParser
->mFlushTimer
= nullptr;
2754 void nsHtml5StreamParser::DropTimer() {
2755 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
2757 * Simply nulling out the timer wouldn't work, because if the timer is
2758 * armed, it needs to be canceled first. Simply canceling it first wouldn't
2759 * work, because nsTimerImpl::Cancel is not safe for calling from outside
2760 * the thread where nsTimerImpl::Fire would run. It's not safe to
2761 * dispatch a runnable to cancel the timer from the destructor of this
2762 * class, because the timer has a weak (void*) pointer back to this instance
2763 * of the stream parser and having the timer fire before the runnable
2764 * cancels it would make the timer access a deleted object.
2766 * This DropTimer method addresses these issues. This method must be called
2767 * on the main thread before the destructor of this class is reached.
2768 * The nsHtml5TimerKungFu object has an nsHtml5StreamParserPtr that addrefs
2770 * stream parser object to keep it alive until the runnable is done.
2771 * The runnable cancels the timer on the parser thread, drops the timer
2772 * and lets nsHtml5StreamParserPtr send a runnable back to the main thread to
2773 * release the stream parser.
2775 mozilla::MutexAutoLock
flushTimerLock(mFlushTimerMutex
);
2777 nsCOMPtr
<nsIRunnable
> event
= new nsHtml5TimerKungFu(this);
2778 if (NS_FAILED(mEventTarget
->Dispatch(event
, nsIThread::DISPATCH_NORMAL
))) {
2779 NS_WARNING("Failed to dispatch TimerKungFu event");
2784 // Using a static, because the method name Notify is taken by the chardet
2786 void nsHtml5StreamParser::TimerCallback(nsITimer
* aTimer
, void* aClosure
) {
2787 (static_cast<nsHtml5StreamParser
*>(aClosure
))->TimerFlush();
2790 void nsHtml5StreamParser::TimerFlush() {
2791 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
2792 mozilla::MutexAutoLock
autoLock(mTokenizerMutex
);
2794 MOZ_ASSERT(!mSpeculating
, "Flush timer fired while speculating.");
2796 // The timer fired if we got here. No need to cancel it. Mark it as
2797 // not armed, though.
2798 mFlushTimerArmed
= false;
2800 mFlushTimerEverFired
= true;
2802 if (IsTerminatedOrInterrupted()) {
2806 if (mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
) {
2807 auto r
= mTreeBuilder
->Flush(); // delete useless ops
2809 MarkAsBroken(r
.unwrapErr());
2812 r
= mTokenizer
->FlushViewSource();
2814 MarkAsBroken(r
.unwrapErr());
2818 nsCOMPtr
<nsIRunnable
> runnable(mExecutorFlusher
);
2819 if (NS_FAILED(DispatchToMain(runnable
.forget()))) {
2820 NS_WARNING("failed to dispatch executor flush event");
2824 // we aren't speculating and we don't know when new data is
2825 // going to arrive. Send data to the main thread.
2826 auto r
= mTreeBuilder
->Flush(true);
2828 MarkAsBroken(r
.unwrapErr());
2832 nsCOMPtr
<nsIRunnable
> runnable(mExecutorFlusher
);
2833 if (NS_FAILED(DispatchToMain(runnable
.forget()))) {
2834 NS_WARNING("failed to dispatch executor flush event");
2840 void nsHtml5StreamParser::MarkAsBroken(nsresult aRv
) {
2841 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
2842 mTokenizerMutex
.AssertCurrentThreadOwns();
2845 mTreeBuilder
->MarkAsBroken(aRv
);
2846 auto r
= mTreeBuilder
->Flush(false);
2848 MOZ_ASSERT(r
.unwrap(), "Should have had the markAsBroken op!");
2850 MOZ_CRASH("OOM prevents propagation of OOM state");
2852 nsCOMPtr
<nsIRunnable
> runnable(mExecutorFlusher
);
2853 if (NS_FAILED(DispatchToMain(runnable
.forget()))) {
2854 NS_WARNING("failed to dispatch executor flush event");
2858 nsresult
nsHtml5StreamParser::DispatchToMain(
2859 already_AddRefed
<nsIRunnable
>&& aRunnable
) {
2860 if (mNetworkEventTarget
) {
2861 return mNetworkEventTarget
->Dispatch(std::move(aRunnable
));
2863 return SchedulerGroup::Dispatch(TaskCategory::Network
, std::move(aRunnable
));