1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set sw=2 ts=2 et tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "nsHtml5StreamParser.h"
12 #include "ErrorList.h"
13 #include "GeckoProfiler.h"
15 #include "mozilla/Buffer.h"
16 #include "mozilla/CheckedInt.h"
17 #include "mozilla/Encoding.h"
18 #include "mozilla/EncodingDetector.h"
19 #include "mozilla/Likely.h"
20 #include "mozilla/Maybe.h"
21 #include "mozilla/SchedulerGroup.h"
22 #include "mozilla/ScopeExit.h"
23 #include "mozilla/Services.h"
24 #include "mozilla/StaticPrefs_html5.h"
25 #include "mozilla/StaticPrefs_network.h"
26 #include "mozilla/TextUtils.h"
27 #include "mozilla/glean/GleanMetrics.h"
29 #include "mozilla/Unused.h"
30 #include "mozilla/dom/BindingDeclarations.h"
31 #include "mozilla/dom/BrowsingContext.h"
32 #include "mozilla/dom/DebuggerUtilsBinding.h"
33 #include "mozilla/dom/Document.h"
34 #include "mozilla/Vector.h"
35 #include "nsContentSink.h"
36 #include "nsContentUtils.h"
37 #include "nsCycleCollectionTraversalCallback.h"
38 #include "nsHtml5AtomTable.h"
39 #include "nsHtml5Highlighter.h"
40 #include "nsHtml5Module.h"
41 #include "nsHtml5OwningUTF16Buffer.h"
42 #include "nsHtml5Parser.h"
43 #include "nsHtml5Speculation.h"
44 #include "nsHtml5StreamParserPtr.h"
45 #include "nsHtml5Tokenizer.h"
46 #include "nsHtml5TreeBuilder.h"
47 #include "nsHtml5TreeOpExecutor.h"
48 #include "nsIChannel.h"
49 #include "nsIContentSink.h"
52 #include "nsIDocShell.h"
53 #include "nsIHttpChannel.h"
54 #include "nsIInputStream.h"
55 #include "nsINestedURI.h"
56 #include "nsIObserverService.h"
57 #include "nsIRequest.h"
58 #include "nsIRunnable.h"
59 #include "nsIScriptError.h"
60 #include "nsIThread.h"
61 #include "nsIThreadRetargetableRequest.h"
64 #include "nsJSEnvironment.h"
65 #include "nsLiteralString.h"
66 #include "nsNetUtil.h"
68 #include "nsTPromiseFlatString.h"
69 #include "nsThreadUtils.h"
70 #include "nsXULAppAPI.h"
73 // Defined in intl/encoding_glue/src/lib.rs
74 const mozilla::Encoding
* xmldecl_parse(const uint8_t* buf
, size_t buf_len
);
77 using namespace mozilla
;
78 using namespace mozilla::dom
;
81 * Note that nsHtml5StreamParser implements cycle collecting AddRef and
82 * Release. Therefore, nsHtml5StreamParser must never be refcounted from
85 * To work around this limitation, runnables posted by the main thread to the
86 * parser thread hold their reference to the stream parser in an
87 * nsHtml5StreamParserPtr. Upon creation, nsHtml5StreamParserPtr addrefs the
89 * just like a regular nsRefPtr. This is OK, since the creation of the
90 * runnable and the nsHtml5StreamParserPtr happens on the main thread.
92 * When the runnable is done on the parser thread, the destructor of
93 * nsHtml5StreamParserPtr runs there. It doesn't call Release on the held object
94 * directly. Instead, it posts another runnable back to the main thread where
95 * that runnable calls Release on the wrapped object.
97 * When posting runnables in the other direction, the runnables have to be
98 * created on the main thread when nsHtml5StreamParser is instantiated and
99 * held for the lifetime of the nsHtml5StreamParser. This works, because the
100 * same runnabled can be dispatched multiple times and currently runnables
101 * posted from the parser thread to main thread don't need to wrap any
102 * runnable-specific data. (In the other direction, the runnables most notably
103 * wrap the byte data of the stream.)
105 NS_IMPL_CYCLE_COLLECTING_ADDREF(nsHtml5StreamParser
)
106 NS_IMPL_CYCLE_COLLECTING_RELEASE(nsHtml5StreamParser
)
108 NS_INTERFACE_TABLE_HEAD(nsHtml5StreamParser
)
109 NS_INTERFACE_TABLE(nsHtml5StreamParser
, nsISupports
)
110 NS_INTERFACE_TABLE_TO_MAP_SEGUE_CYCLE_COLLECTION(nsHtml5StreamParser
)
113 NS_IMPL_CYCLE_COLLECTION_CLASS(nsHtml5StreamParser
)
115 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(nsHtml5StreamParser
)
117 NS_IMPL_CYCLE_COLLECTION_UNLINK(mRequest
)
118 NS_IMPL_CYCLE_COLLECTION_UNLINK(mOwner
)
119 tmp
->mExecutorFlusher
= nullptr;
120 tmp
->mLoadFlusher
= nullptr;
121 tmp
->mExecutor
= nullptr;
122 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
124 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(nsHtml5StreamParser
)
125 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mRequest
)
126 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOwner
)
127 // hack: count the strongly owned edge wrapped in the runnable
128 if (tmp
->mExecutorFlusher
) {
129 NS_CYCLE_COLLECTION_NOTE_EDGE_NAME(cb
, "mExecutorFlusher->mExecutor");
130 cb
.NoteXPCOMChild(static_cast<nsIContentSink
*>(tmp
->mExecutor
));
132 // hack: count the strongly owned edge wrapped in the runnable
133 if (tmp
->mLoadFlusher
) {
134 NS_CYCLE_COLLECTION_NOTE_EDGE_NAME(cb
, "mLoadFlusher->mExecutor");
135 cb
.NoteXPCOMChild(static_cast<nsIContentSink
*>(tmp
->mExecutor
));
137 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
139 class nsHtml5ExecutorFlusher
: public Runnable
{
141 RefPtr
<nsHtml5TreeOpExecutor
> mExecutor
;
144 explicit nsHtml5ExecutorFlusher(nsHtml5TreeOpExecutor
* aExecutor
)
145 : Runnable("nsHtml5ExecutorFlusher"), mExecutor(aExecutor
) {}
146 NS_IMETHOD
Run() override
{
147 if (!mExecutor
->isInList()) {
148 Document
* doc
= mExecutor
->GetDocument();
149 if (XRE_IsContentProcess() &&
151 HighPriorityEventPendingForTopLevelDocumentBeforeContentfulPaint(
153 // Possible early paint pending, reuse the runnable and try to
154 // call RunFlushLoop later.
155 nsCOMPtr
<nsIRunnable
> flusher
= this;
156 if (NS_SUCCEEDED(doc
->Dispatch(flusher
.forget()))) {
157 PROFILER_MARKER_UNTYPED("HighPrio blocking parser flushing(1)", DOM
);
161 mExecutor
->RunFlushLoop();
167 class nsHtml5LoadFlusher
: public Runnable
{
169 RefPtr
<nsHtml5TreeOpExecutor
> mExecutor
;
172 explicit nsHtml5LoadFlusher(nsHtml5TreeOpExecutor
* aExecutor
)
173 : Runnable("nsHtml5LoadFlusher"), mExecutor(aExecutor
) {}
174 NS_IMETHOD
Run() override
{
175 mExecutor
->FlushSpeculativeLoads();
180 nsHtml5StreamParser::nsHtml5StreamParser(nsHtml5TreeOpExecutor
* aExecutor
,
181 nsHtml5Parser
* aOwner
,
183 : mBomState(eBomState::BOM_SNIFFING_NOT_STARTED
),
184 mCharsetSource(kCharsetUninitialized
),
185 mEncodingSwitchSource(kCharsetUninitialized
),
186 mEncoding(X_USER_DEFINED_ENCODING
), // Obviously bogus value to notice if
188 mNeedsEncodingSwitchTo(nullptr),
189 mSeenEligibleMetaCharset(false),
192 mStartedFeedingDetector(false),
193 mStartedFeedingDevTools(false),
195 mReparseForbidden(false),
196 mForceAutoDetection(false),
197 mChannelHadCharset(false),
198 mLookingForMetaCharset(false),
199 mStartsWithLtQuestion(false),
200 mLookingForXmlDeclarationForXmlViewSource(false),
201 mTemplatePushedOrHeadPopped(false),
204 mLastBuffer(nullptr), // Will be filled when starting
205 mExecutor(aExecutor
),
206 mTreeBuilder(new nsHtml5TreeBuilder(
207 (aMode
== VIEW_SOURCE_HTML
|| aMode
== VIEW_SOURCE_XML
)
209 : mExecutor
->GetStage(),
210 mExecutor
->GetStage(), aMode
== NORMAL
)),
212 new nsHtml5Tokenizer(mTreeBuilder
.get(), aMode
== VIEW_SOURCE_XML
)),
213 mTokenizerMutex("nsHtml5StreamParser mTokenizerMutex"),
216 mStreamState(eHtml5StreamState::STREAM_NOT_STARTED
),
219 mSpeculationMutex("nsHtml5StreamParser mSpeculationMutex"),
220 mSpeculationFailureCount(0),
221 mNumBytesBuffered(0),
224 mEventTarget(nsHtml5Module::GetStreamParserEventTarget()),
225 mExecutorFlusher(new nsHtml5ExecutorFlusher(aExecutor
)),
226 mLoadFlusher(new nsHtml5LoadFlusher(aExecutor
)),
227 mInitialEncodingWasFromParentFrame(false),
228 mHasHadErrors(false),
229 mDetectorHasSeenNonAscii(false),
230 mDecodingLocalFileWithoutTokenizing(false),
231 mBufferingBytes(false),
232 mFlushTimer(NS_NewTimer(mEventTarget
)),
233 mFlushTimerMutex("nsHtml5StreamParser mFlushTimerMutex"),
234 mFlushTimerArmed(false),
235 mFlushTimerEverFired(false),
237 NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
239 mAtomTable
.SetPermittedLookupEventTarget(mEventTarget
);
241 mTokenizer
->setInterner(&mAtomTable
);
242 mTokenizer
->setEncodingDeclarationHandler(this);
244 if (aMode
== VIEW_SOURCE_HTML
|| aMode
== VIEW_SOURCE_XML
) {
245 nsHtml5Highlighter
* highlighter
=
246 new nsHtml5Highlighter(mExecutor
->GetStage());
247 mTokenizer
->EnableViewSource(highlighter
); // takes ownership
248 mTreeBuilder
->EnableViewSource(highlighter
); // doesn't own
251 // There's a zeroing operator new for everything else
254 nsHtml5StreamParser::~nsHtml5StreamParser() {
255 NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
259 mozilla::MutexAutoLock
flushTimerLock(mFlushTimerMutex
);
260 MOZ_ASSERT(!mFlushTimer
, "Flush timer was not dropped before dtor!");
263 mUnicodeDecoder
= nullptr;
264 mFirstBuffer
= nullptr;
266 mTreeBuilder
= nullptr;
267 mTokenizer
= nullptr;
272 nsresult
nsHtml5StreamParser::GetChannel(nsIChannel
** aChannel
) {
273 NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
274 return mRequest
? CallQueryInterface(mRequest
, aChannel
)
275 : NS_ERROR_NOT_AVAILABLE
;
278 std::tuple
<NotNull
<const Encoding
*>, nsCharsetSource
>
279 nsHtml5StreamParser::GuessEncoding(bool aInitial
) {
281 mCharsetSource
!= kCharsetFromFinalUserForcedAutoDetection
&&
283 kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII
&&
285 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Generic
&&
287 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8GenericInitialWasASCII
&&
289 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Content
&&
291 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8ContentInitialWasASCII
&&
293 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD
&&
295 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLDInitialWasASCII
&&
296 mCharsetSource
!= kCharsetFromFinalAutoDetectionFile
);
297 auto ifHadBeenForced
= mDetector
->Guess(EmptyCString(), true);
301 : mDetector
->Guess(mTLD
, mDecodingLocalFileWithoutTokenizing
);
302 nsCharsetSource source
=
304 ? (mForceAutoDetection
305 ? kCharsetFromInitialUserForcedAutoDetection
306 : (mDecodingLocalFileWithoutTokenizing
307 ? kCharsetFromFinalAutoDetectionFile
308 : kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Generic
))
309 : (mForceAutoDetection
310 ? kCharsetFromFinalUserForcedAutoDetection
311 : (mDecodingLocalFileWithoutTokenizing
312 ? kCharsetFromFinalAutoDetectionFile
313 : kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Generic
));
314 if (source
== kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Generic
) {
315 if (encoding
== ISO_2022_JP_ENCODING
) {
316 if (EncodingDetector::TldMayAffectGuess(mTLD
)) {
317 source
= kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Content
;
319 } else if (!mDetectorHasSeenNonAscii
) {
320 source
= kCharsetFromInitialAutoDetectionASCII
; // deliberately Initial
321 } else if (ifHadBeenForced
== UTF_8_ENCODING
) {
322 MOZ_ASSERT(mCharsetSource
== kCharsetFromInitialAutoDetectionASCII
||
324 kCharsetFromInitialAutoDetectionWouldHaveBeenUTF8
||
325 mEncoding
== ISO_2022_JP_ENCODING
);
326 source
= kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII
;
327 } else if (encoding
!= ifHadBeenForced
) {
328 if (mCharsetSource
== kCharsetFromInitialAutoDetectionASCII
) {
330 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLDInitialWasASCII
;
333 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD
;
335 } else if (EncodingDetector::TldMayAffectGuess(mTLD
)) {
336 if (mCharsetSource
== kCharsetFromInitialAutoDetectionASCII
) {
338 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8ContentInitialWasASCII
;
340 source
= kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Content
;
342 } else if (mCharsetSource
== kCharsetFromInitialAutoDetectionASCII
) {
344 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8GenericInitialWasASCII
;
347 kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Generic
) {
348 if (encoding
== ISO_2022_JP_ENCODING
) {
349 if (EncodingDetector::TldMayAffectGuess(mTLD
)) {
350 source
= kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Content
;
352 } else if (!mDetectorHasSeenNonAscii
) {
353 source
= kCharsetFromInitialAutoDetectionASCII
;
354 } else if (ifHadBeenForced
== UTF_8_ENCODING
) {
355 source
= kCharsetFromInitialAutoDetectionWouldHaveBeenUTF8
;
356 } else if (encoding
!= ifHadBeenForced
) {
358 kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD
;
359 } else if (EncodingDetector::TldMayAffectGuess(mTLD
)) {
360 source
= kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Content
;
363 return {encoding
, source
};
366 void nsHtml5StreamParser::FeedDetector(Span
<const uint8_t> aBuffer
) {
368 mStartedFeedingDetector
= true;
370 MOZ_ASSERT(!mChardetEof
);
371 mDetectorHasSeenNonAscii
= mDetector
->Feed(aBuffer
, false);
374 void nsHtml5StreamParser::DetectorEof() {
376 mStartedFeedingDetector
= true;
382 mDetectorHasSeenNonAscii
= mDetector
->Feed(Span
<const uint8_t>(), true);
385 void nsHtml5StreamParser::SetViewSourceTitle(nsIURI
* aURL
) {
386 MOZ_ASSERT(NS_IsMainThread());
388 BrowsingContext
* browsingContext
=
389 mExecutor
->GetDocument()->GetBrowsingContext();
390 if (browsingContext
&& browsingContext
->WatchedByDevTools()) {
391 mURIToSendToDevtools
= aURL
;
394 nsresult rv
= nsID::GenerateUUIDInPlace(uuid
);
395 if (!NS_FAILED(rv
)) {
396 char buffer
[NSID_LENGTH
];
397 uuid
.ToProvidedString(buffer
);
398 mUUIDForDevtools
= NS_ConvertASCIItoUTF16(buffer
);
403 nsCOMPtr
<nsIURI
> temp
;
404 if (aURL
->SchemeIs("view-source")) {
405 nsCOMPtr
<nsINestedURI
> nested
= do_QueryInterface(aURL
);
406 nested
->GetInnerURI(getter_AddRefs(temp
));
410 if (temp
->SchemeIs("data")) {
411 // Avoid showing potentially huge data: URLs. The three last bytes are
412 // UTF-8 for an ellipsis.
413 mViewSourceTitle
.AssignLiteral("data:\xE2\x80\xA6");
415 nsresult rv
= temp
->GetSpec(mViewSourceTitle
);
417 mViewSourceTitle
.AssignLiteral("\xE2\x80\xA6");
424 nsHtml5StreamParser::SetupDecodingAndWriteSniffingBufferAndCurrentSegment(
425 Span
<const uint8_t> aPrefix
, Span
<const uint8_t> aFromSegment
) {
426 NS_ASSERTION(IsParserThread(), "Wrong thread!");
427 mUnicodeDecoder
= mEncoding
->NewDecoderWithBOMRemoval();
428 nsresult rv
= WriteStreamBytes(aPrefix
);
429 NS_ENSURE_SUCCESS(rv
, rv
);
430 return WriteStreamBytes(aFromSegment
);
433 void nsHtml5StreamParser::SetupDecodingFromBom(
434 NotNull
<const Encoding
*> aEncoding
) {
435 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
436 mEncoding
= aEncoding
;
437 mDecodingLocalFileWithoutTokenizing
= false;
438 mLookingForMetaCharset
= false;
439 mBufferingBytes
= false;
440 mUnicodeDecoder
= mEncoding
->NewDecoderWithoutBOMHandling();
441 mCharsetSource
= kCharsetFromByteOrderMark
;
442 mForceAutoDetection
= false;
443 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
444 mBomState
= BOM_SNIFFING_OVER
;
445 if (mMode
== VIEW_SOURCE_HTML
) {
446 mTokenizer
->StartViewSourceCharacters();
450 void nsHtml5StreamParser::SetupDecodingFromUtf16BogoXml(
451 NotNull
<const Encoding
*> aEncoding
) {
452 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
453 mEncoding
= aEncoding
;
454 mDecodingLocalFileWithoutTokenizing
= false;
455 mLookingForMetaCharset
= false;
456 mBufferingBytes
= false;
457 mUnicodeDecoder
= mEncoding
->NewDecoderWithoutBOMHandling();
458 mCharsetSource
= kCharsetFromXmlDeclarationUtf16
;
459 mForceAutoDetection
= false;
460 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
461 mBomState
= BOM_SNIFFING_OVER
;
462 if (mMode
== VIEW_SOURCE_HTML
) {
463 mTokenizer
->StartViewSourceCharacters();
465 auto dst
= mLastBuffer
->TailAsSpan(READ_BUFFER_SIZE
);
469 mLastBuffer
->AdvanceEnd(3);
470 MOZ_ASSERT(!mStartedFeedingDevTools
);
471 OnNewContent(dst
.To(3));
474 size_t nsHtml5StreamParser::LengthOfLtContainingPrefixInSecondBuffer() {
475 MOZ_ASSERT(mBufferedBytes
.Length() <= 2);
476 if (mBufferedBytes
.Length() < 2) {
479 Buffer
<uint8_t>& second
= mBufferedBytes
[1];
480 const uint8_t* elements
= second
.Elements();
481 const uint8_t* lt
= (const uint8_t*)memchr(elements
, '>', second
.Length());
483 return (lt
- elements
) + 1;
488 nsresult
nsHtml5StreamParser::SniffStreamBytes(Span
<const uint8_t> aFromSegment
,
490 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
491 MOZ_ASSERT_IF(aEof
, aFromSegment
.IsEmpty());
493 if (mCharsetSource
>=
494 kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII
&&
495 mCharsetSource
<= kCharsetFromFinalUserForcedAutoDetection
) {
496 if (mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
) {
497 mTreeBuilder
->MaybeComplainAboutCharset("EncDetectorReloadPlain", true,
500 mTreeBuilder
->MaybeComplainAboutCharset("EncDetectorReload", true, 0);
504 // mEncoding and mCharsetSource potentially have come from channel or higher
505 // by now. If we find a BOM, SetupDecodingFromBom() will overwrite them.
506 // If we don't find a BOM, the previously set values of mEncoding and
507 // mCharsetSource are not modified by the BOM sniffing here.
508 static uint8_t utf8
[] = {0xEF, 0xBB};
509 static uint8_t utf16le
[] = {0xFF};
510 static uint8_t utf16be
[] = {0xFE};
511 static uint8_t utf16leXml
[] = {'<', 0x00, '?', 0x00, 'x'};
512 static uint8_t utf16beXml
[] = {0x00, '<', 0x00, '?', 0x00};
513 // Buffer for replaying past bytes based on state machine state. If
514 // writing this from scratch, probably wouldn't do it this way, but
515 // let's keep the changes to a minimum.
516 const uint8_t* prefix
= utf8
;
517 size_t prefixLength
= 0;
518 if (aEof
&& mBomState
== BOM_SNIFFING_NOT_STARTED
) {
519 // Avoid handling aEof in the BOM_SNIFFING_NOT_STARTED state below.
520 mBomState
= BOM_SNIFFING_OVER
;
523 (i
< aFromSegment
.Length() && mBomState
!= BOM_SNIFFING_OVER
) || aEof
;
526 case BOM_SNIFFING_NOT_STARTED
:
527 MOZ_ASSERT(i
== 0, "Bad BOM sniffing state.");
528 MOZ_ASSERT(!aEof
, "Should have checked for aEof above!");
529 switch (aFromSegment
[0]) {
531 mBomState
= SEEN_UTF_8_FIRST_BYTE
;
534 mBomState
= SEEN_UTF_16_LE_FIRST_BYTE
;
537 mBomState
= SEEN_UTF_16_BE_FIRST_BYTE
;
540 if (mCharsetSource
< kCharsetFromXmlDeclarationUtf16
&&
541 mCharsetSource
!= kCharsetFromChannel
) {
542 mBomState
= SEEN_UTF_16_BE_XML_FIRST
;
544 mBomState
= BOM_SNIFFING_OVER
;
548 if (mCharsetSource
< kCharsetFromXmlDeclarationUtf16
&&
549 mCharsetSource
!= kCharsetFromChannel
) {
550 mBomState
= SEEN_UTF_16_LE_XML_FIRST
;
552 mBomState
= BOM_SNIFFING_OVER
;
556 mBomState
= BOM_SNIFFING_OVER
;
560 case SEEN_UTF_16_LE_FIRST_BYTE
:
561 if (!aEof
&& aFromSegment
[i
] == 0xFE) {
562 SetupDecodingFromBom(UTF_16LE_ENCODING
);
563 return WriteStreamBytes(aFromSegment
.From(i
+ 1));
566 prefixLength
= 1 - i
;
567 mBomState
= BOM_SNIFFING_OVER
;
569 case SEEN_UTF_16_BE_FIRST_BYTE
:
570 if (!aEof
&& aFromSegment
[i
] == 0xFF) {
571 SetupDecodingFromBom(UTF_16BE_ENCODING
);
572 return WriteStreamBytes(aFromSegment
.From(i
+ 1));
575 prefixLength
= 1 - i
;
576 mBomState
= BOM_SNIFFING_OVER
;
578 case SEEN_UTF_8_FIRST_BYTE
:
579 if (!aEof
&& aFromSegment
[i
] == 0xBB) {
580 mBomState
= SEEN_UTF_8_SECOND_BYTE
;
582 prefixLength
= 1 - i
;
583 mBomState
= BOM_SNIFFING_OVER
;
586 case SEEN_UTF_8_SECOND_BYTE
:
587 if (!aEof
&& aFromSegment
[i
] == 0xBF) {
588 SetupDecodingFromBom(UTF_8_ENCODING
);
589 return WriteStreamBytes(aFromSegment
.From(i
+ 1));
591 prefixLength
= 2 - i
;
592 mBomState
= BOM_SNIFFING_OVER
;
594 case SEEN_UTF_16_BE_XML_FIRST
:
595 if (!aEof
&& aFromSegment
[i
] == '<') {
596 mBomState
= SEEN_UTF_16_BE_XML_SECOND
;
599 prefixLength
= 1 - i
;
600 mBomState
= BOM_SNIFFING_OVER
;
603 case SEEN_UTF_16_BE_XML_SECOND
:
604 if (!aEof
&& aFromSegment
[i
] == 0x00) {
605 mBomState
= SEEN_UTF_16_BE_XML_THIRD
;
608 prefixLength
= 2 - i
;
609 mBomState
= BOM_SNIFFING_OVER
;
612 case SEEN_UTF_16_BE_XML_THIRD
:
613 if (!aEof
&& aFromSegment
[i
] == '?') {
614 mBomState
= SEEN_UTF_16_BE_XML_FOURTH
;
617 prefixLength
= 3 - i
;
618 mBomState
= BOM_SNIFFING_OVER
;
621 case SEEN_UTF_16_BE_XML_FOURTH
:
622 if (!aEof
&& aFromSegment
[i
] == 0x00) {
623 mBomState
= SEEN_UTF_16_BE_XML_FIFTH
;
626 prefixLength
= 4 - i
;
627 mBomState
= BOM_SNIFFING_OVER
;
630 case SEEN_UTF_16_BE_XML_FIFTH
:
631 if (!aEof
&& aFromSegment
[i
] == 'x') {
632 SetupDecodingFromUtf16BogoXml(UTF_16BE_ENCODING
);
633 return WriteStreamBytes(aFromSegment
.From(i
+ 1));
636 prefixLength
= 5 - i
;
637 mBomState
= BOM_SNIFFING_OVER
;
639 case SEEN_UTF_16_LE_XML_FIRST
:
640 if (!aEof
&& aFromSegment
[i
] == 0x00) {
641 mBomState
= SEEN_UTF_16_LE_XML_SECOND
;
643 if (!aEof
&& aFromSegment
[i
] == '?' &&
644 !(mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
)) {
645 mStartsWithLtQuestion
= true;
648 prefixLength
= 1 - i
;
649 mBomState
= BOM_SNIFFING_OVER
;
652 case SEEN_UTF_16_LE_XML_SECOND
:
653 if (!aEof
&& aFromSegment
[i
] == '?') {
654 mBomState
= SEEN_UTF_16_LE_XML_THIRD
;
657 prefixLength
= 2 - i
;
658 mBomState
= BOM_SNIFFING_OVER
;
661 case SEEN_UTF_16_LE_XML_THIRD
:
662 if (!aEof
&& aFromSegment
[i
] == 0x00) {
663 mBomState
= SEEN_UTF_16_LE_XML_FOURTH
;
666 prefixLength
= 3 - i
;
667 mBomState
= BOM_SNIFFING_OVER
;
670 case SEEN_UTF_16_LE_XML_FOURTH
:
671 if (!aEof
&& aFromSegment
[i
] == 'x') {
672 mBomState
= SEEN_UTF_16_LE_XML_FIFTH
;
675 prefixLength
= 4 - i
;
676 mBomState
= BOM_SNIFFING_OVER
;
679 case SEEN_UTF_16_LE_XML_FIFTH
:
680 if (!aEof
&& aFromSegment
[i
] == 0x00) {
681 SetupDecodingFromUtf16BogoXml(UTF_16LE_ENCODING
);
682 return WriteStreamBytes(aFromSegment
.From(i
+ 1));
685 prefixLength
= 5 - i
;
686 mBomState
= BOM_SNIFFING_OVER
;
689 mBomState
= BOM_SNIFFING_OVER
;
696 // if we get here, there either was no BOM or the BOM sniffing isn't complete
699 MOZ_ASSERT(mCharsetSource
!= kCharsetFromByteOrderMark
,
700 "Should not come here if BOM was found.");
701 MOZ_ASSERT(mCharsetSource
!= kCharsetFromXmlDeclarationUtf16
,
702 "Should not come here if UTF-16 bogo-XML declaration was found.");
703 MOZ_ASSERT(mCharsetSource
!= kCharsetFromOtherComponent
,
704 "kCharsetFromOtherComponent is for XSLT.");
706 if (mBomState
== BOM_SNIFFING_OVER
) {
707 if (mMode
== VIEW_SOURCE_XML
&& mStartsWithLtQuestion
&&
708 mCharsetSource
< kCharsetFromChannel
) {
709 // Sniff for XML declaration only.
710 MOZ_ASSERT(!mLookingForXmlDeclarationForXmlViewSource
);
712 MOZ_ASSERT(!mLookingForMetaCharset
);
713 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
);
714 // Maybe we've already buffered a '>'.
715 MOZ_ASSERT(!mBufferedBytes
.IsEmpty(),
716 "How did at least <? not get buffered?");
717 Buffer
<uint8_t>& first
= mBufferedBytes
[0];
718 const Encoding
* encoding
=
719 xmldecl_parse(first
.Elements(), first
.Length());
721 mEncoding
= WrapNotNull(encoding
);
722 mCharsetSource
= kCharsetFromXmlDeclaration
;
723 } else if (memchr(first
.Elements(), '>', first
.Length())) {
724 // There was a '>', but an encoding still wasn't found.
725 ; // fall through to commit to the UTF-8 default.
726 } else if (size_t lengthOfPrefix
=
727 LengthOfLtContainingPrefixInSecondBuffer()) {
728 // This can only happen if the first buffer was a lone '<', because
729 // we come here upon seeing the second byte '?' if the first two bytes
730 // were "<?". That is, the only way how we aren't dealing with the first
731 // buffer is if the first buffer only contained a single '<' and we are
732 // dealing with the second buffer that starts with '?'.
733 MOZ_ASSERT(first
.Length() == 1);
734 MOZ_ASSERT(mBufferedBytes
[1][0] == '?');
735 // Our scanner for XML declaration-like syntax wants to see a contiguous
736 // buffer, so let's linearize the data. (Ideally, the XML declaration
737 // scanner would be incremental, but this is the rare path anyway.)
738 Vector
<uint8_t> contiguous
;
739 if (!contiguous
.append(first
.Elements(), first
.Length())) {
740 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
741 return NS_ERROR_OUT_OF_MEMORY
;
743 if (!contiguous
.append(mBufferedBytes
[1].Elements(), lengthOfPrefix
)) {
744 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
745 return NS_ERROR_OUT_OF_MEMORY
;
747 encoding
= xmldecl_parse(contiguous
.begin(), contiguous
.length());
749 mEncoding
= WrapNotNull(encoding
);
750 mCharsetSource
= kCharsetFromXmlDeclaration
;
752 // else no XML decl, commit to the UTF-8 default.
754 MOZ_ASSERT(mBufferingBytes
);
755 mLookingForXmlDeclarationForXmlViewSource
= true;
758 } else if (mMode
!= VIEW_SOURCE_XML
&&
759 (mForceAutoDetection
|| mCharsetSource
< kCharsetFromChannel
)) {
760 // In order to use the buffering logic for meta with mForceAutoDetection,
761 // we set mLookingForMetaCharset but still actually potentially ignore the
763 mFirstBufferOfMetaScan
= mFirstBuffer
;
764 MOZ_ASSERT(mLookingForMetaCharset
);
766 if (mMode
== VIEW_SOURCE_HTML
) {
767 auto r
= mTokenizer
->FlushViewSource();
769 return r
.unwrapErr();
772 auto r
= mTreeBuilder
->Flush();
774 return r
.unwrapErr();
776 // Encoding committer flushes the ops on the main thread.
778 mozilla::MutexAutoLock
speculationAutoLock(mSpeculationMutex
);
779 nsHtml5Speculation
* speculation
= new nsHtml5Speculation(
780 mFirstBuffer
, mFirstBuffer
->getStart(), mTokenizer
->getLineNumber(),
781 mTokenizer
->getColumnNumber(), mTreeBuilder
->newSnapshot());
782 MOZ_ASSERT(!mFlushTimerArmed
, "How did we end up arming the timer?");
783 if (mMode
== VIEW_SOURCE_HTML
) {
784 mTokenizer
->SetViewSourceOpSink(speculation
);
785 mTokenizer
->StartViewSourceCharacters();
787 MOZ_ASSERT(mMode
!= VIEW_SOURCE_XML
);
788 mTreeBuilder
->SetOpSink(speculation
);
790 mSpeculations
.AppendElement(speculation
); // adopts the pointer
793 mLookingForMetaCharset
= false;
794 mBufferingBytes
= false;
795 mDecodingLocalFileWithoutTokenizing
= false;
796 if (mMode
== VIEW_SOURCE_HTML
) {
797 mTokenizer
->StartViewSourceCharacters();
800 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
801 return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(
802 Span(prefix
, prefixLength
), aFromSegment
);
808 class AddContentRunnable
: public Runnable
{
810 AddContentRunnable(const nsAString
& aParserID
, nsIURI
* aURI
,
811 Span
<const char16_t
> aData
, bool aComplete
)
812 : Runnable("AddContent") {
815 mData
.mUri
.Construct(NS_ConvertUTF8toUTF16(spec
));
816 mData
.mParserID
.Construct(aParserID
);
817 mData
.mContents
.Construct(aData
.Elements(), aData
.Length());
818 mData
.mComplete
.Construct(aComplete
);
821 NS_IMETHOD
Run() override
{
823 if (!mData
.ToJSON(json
)) {
824 return NS_ERROR_FAILURE
;
827 nsCOMPtr
<nsIObserverService
> obsService
= services::GetObserverService();
829 obsService
->NotifyObservers(nullptr, "devtools-html-content",
830 PromiseFlatString(json
).get());
839 inline void nsHtml5StreamParser::OnNewContent(Span
<const char16_t
> aData
) {
841 mStartedFeedingDevTools
= true;
843 if (mURIToSendToDevtools
) {
844 if (aData
.IsEmpty()) {
845 // Optimize out the runnable.
848 NS_DispatchToMainThread(new AddContentRunnable(mUUIDForDevtools
,
849 mURIToSendToDevtools
, aData
,
850 /* aComplete */ false));
854 inline void nsHtml5StreamParser::OnContentComplete() {
856 mStartedFeedingDevTools
= true;
858 if (mURIToSendToDevtools
) {
859 NS_DispatchToMainThread(new AddContentRunnable(
860 mUUIDForDevtools
, mURIToSendToDevtools
, Span
<const char16_t
>(),
861 /* aComplete */ true));
862 mURIToSendToDevtools
= nullptr;
866 nsresult
nsHtml5StreamParser::WriteStreamBytes(
867 Span
<const uint8_t> aFromSegment
) {
868 NS_ASSERTION(IsParserThread(), "Wrong thread!");
869 mTokenizerMutex
.AssertCurrentThreadOwns();
870 // mLastBuffer should always point to a buffer of the size
873 NS_WARNING("mLastBuffer should not be null!");
874 MarkAsBroken(NS_ERROR_NULL_POINTER
);
875 return NS_ERROR_NULL_POINTER
;
877 size_t totalRead
= 0;
878 auto src
= aFromSegment
;
880 auto dst
= mLastBuffer
->TailAsSpan(READ_BUFFER_SIZE
);
881 auto [result
, read
, written
, hadErrors
] =
882 mUnicodeDecoder
->DecodeToUTF16(src
, dst
, false);
883 if (!(mLookingForMetaCharset
|| mDecodingLocalFileWithoutTokenizing
)) {
884 OnNewContent(dst
.To(written
));
886 if (hadErrors
&& !mHasHadErrors
) {
887 mHasHadErrors
= true;
888 if (mEncoding
== UTF_8_ENCODING
) {
889 mTreeBuilder
->TryToEnableEncodingMenu();
892 src
= src
.From(read
);
894 mLastBuffer
->AdvanceEnd(written
);
895 if (result
== kOutputFull
) {
896 RefPtr
<nsHtml5OwningUTF16Buffer
> newBuf
=
897 nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE
);
899 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
900 return NS_ERROR_OUT_OF_MEMORY
;
902 mLastBuffer
= (mLastBuffer
->next
= std::move(newBuf
));
904 MOZ_ASSERT(totalRead
== aFromSegment
.Length(),
905 "The Unicode decoder consumed the wrong number of bytes.");
907 if (!mLookingForMetaCharset
&& mDecodingLocalFileWithoutTokenizing
&&
908 mNumBytesBuffered
== LOCAL_FILE_UTF_8_BUFFER_SIZE
) {
909 MOZ_ASSERT(!mStartedFeedingDetector
);
910 for (auto&& buffer
: mBufferedBytes
) {
911 FeedDetector(buffer
);
913 // If the file is exactly LOCAL_FILE_UTF_8_BUFFER_SIZE bytes long
914 // we end up not considering the EOF. That's not fatal, since we
915 // don't consider the EOF if the file is
916 // LOCAL_FILE_UTF_8_BUFFER_SIZE + 1 bytes long.
917 auto [encoding
, source
] = GuessEncoding(true);
918 mCharsetSource
= source
;
919 if (encoding
!= mEncoding
) {
920 mEncoding
= encoding
;
921 nsresult rv
= ReDecodeLocalFile();
926 MOZ_ASSERT(mEncoding
== UTF_8_ENCODING
);
927 nsresult rv
= CommitLocalFileToEncoding();
938 [[nodiscard
]] nsresult
nsHtml5StreamParser::ReDecodeLocalFile() {
939 MOZ_ASSERT(mDecodingLocalFileWithoutTokenizing
&& !mLookingForMetaCharset
);
940 MOZ_ASSERT(mFirstBufferOfMetaScan
);
941 MOZ_ASSERT(mCharsetSource
== kCharsetFromFinalAutoDetectionFile
||
942 (mForceAutoDetection
&&
943 mCharsetSource
== kCharsetFromInitialUserForcedAutoDetection
));
945 DiscardMetaSpeculation();
947 MOZ_ASSERT(mEncoding
!= UTF_8_ENCODING
);
949 mDecodingLocalFileWithoutTokenizing
= false;
951 mEncoding
->NewDecoderWithBOMRemovalInto(*mUnicodeDecoder
);
952 mHasHadErrors
= false;
954 // Throw away previous decoded data
955 mLastBuffer
= mFirstBuffer
;
956 mLastBuffer
->next
= nullptr;
957 mLastBuffer
->setStart(0);
958 mLastBuffer
->setEnd(0);
960 mBufferingBytes
= false;
961 mForceAutoDetection
= false; // To stop feeding the detector
962 mFirstBufferOfMetaScan
= nullptr;
964 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, true);
967 for (auto&& buffer
: mBufferedBytes
) {
968 DoDataAvailable(buffer
);
971 if (mMode
== VIEW_SOURCE_HTML
) {
972 auto r
= mTokenizer
->FlushViewSource();
974 return r
.unwrapErr();
977 auto r
= mTreeBuilder
->Flush();
979 return r
.unwrapErr();
984 [[nodiscard
]] nsresult
nsHtml5StreamParser::CommitLocalFileToEncoding() {
985 MOZ_ASSERT(mDecodingLocalFileWithoutTokenizing
&& !mLookingForMetaCharset
);
986 MOZ_ASSERT(mFirstBufferOfMetaScan
);
987 mDecodingLocalFileWithoutTokenizing
= false;
988 MOZ_ASSERT(mCharsetSource
== kCharsetFromFinalAutoDetectionFile
||
989 (mForceAutoDetection
&&
990 mCharsetSource
== kCharsetFromInitialUserForcedAutoDetection
));
991 MOZ_ASSERT(mEncoding
== UTF_8_ENCODING
);
993 MOZ_ASSERT(!mStartedFeedingDevTools
);
994 if (mURIToSendToDevtools
) {
995 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBufferOfMetaScan
;
997 Span
<const char16_t
> data(buffer
->getBuffer() + buffer
->getStart(),
998 buffer
->getLength());
1000 buffer
= buffer
->next
;
1004 mFirstBufferOfMetaScan
= nullptr;
1006 mBufferingBytes
= false;
1007 mForceAutoDetection
= false; // To stop feeding the detector
1008 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, true);
1009 if (mMode
== VIEW_SOURCE_HTML
) {
1010 auto r
= mTokenizer
->FlushViewSource();
1012 return r
.unwrapErr();
1015 auto r
= mTreeBuilder
->Flush();
1017 return r
.unwrapErr();
1022 class MaybeRunCollector
: public Runnable
{
1024 explicit MaybeRunCollector(nsIDocShell
* aDocShell
)
1025 : Runnable("MaybeRunCollector"), mDocShell(aDocShell
) {}
1027 NS_IMETHOD
Run() override
{
1028 nsJSContext::MaybeRunNextCollectorSlice(mDocShell
,
1029 JS::GCReason::HTML_PARSER
);
1033 nsCOMPtr
<nsIDocShell
> mDocShell
;
1036 nsresult
nsHtml5StreamParser::OnStartRequest(nsIRequest
* aRequest
) {
1037 MOZ_RELEASE_ASSERT(STREAM_NOT_STARTED
== mStreamState
,
1038 "Got OnStartRequest when the stream had already started.");
1040 !mExecutor
->HasStarted(),
1041 "Got OnStartRequest at the wrong stage in the executor life cycle.");
1042 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
1044 // To avoid the cost of instantiating the detector when it's not needed,
1045 // let's instantiate only if we make it out of this method with the
1046 // intent to use it.
1047 auto detectorCreator
= MakeScopeExit([&] {
1048 if ((mForceAutoDetection
|| mCharsetSource
< kCharsetFromParentFrame
) ||
1049 !(mMode
== LOAD_AS_DATA
|| mMode
== VIEW_SOURCE_XML
)) {
1050 mDetector
= mozilla::EncodingDetector::Create();
1054 mRequest
= aRequest
;
1056 mStreamState
= STREAM_BEING_READ
;
1058 // For View Source, the parser should run with scripts "enabled" if a normal
1059 // load would have scripts enabled.
1060 bool scriptingEnabled
=
1061 mMode
== LOAD_AS_DATA
? false : mExecutor
->IsScriptEnabled();
1062 mOwner
->StartTokenizer(scriptingEnabled
);
1064 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
);
1065 bool isSrcdoc
= false;
1066 nsCOMPtr
<nsIChannel
> channel
;
1067 nsresult rv
= GetChannel(getter_AddRefs(channel
));
1068 if (NS_SUCCEEDED(rv
)) {
1069 isSrcdoc
= NS_IsSrcdocChannel(channel
);
1070 if (!isSrcdoc
&& mCharsetSource
<= kCharsetFromFallback
) {
1071 nsCOMPtr
<nsIURI
> originalURI
;
1072 rv
= channel
->GetOriginalURI(getter_AddRefs(originalURI
));
1073 if (NS_SUCCEEDED(rv
)) {
1074 if (originalURI
->SchemeIs("resource")) {
1075 mCharsetSource
= kCharsetFromBuiltIn
;
1076 mEncoding
= UTF_8_ENCODING
;
1077 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
1079 nsCOMPtr
<nsIURI
> currentURI
;
1080 rv
= channel
->GetURI(getter_AddRefs(currentURI
));
1081 if (NS_SUCCEEDED(rv
)) {
1082 nsCOMPtr
<nsIURI
> innermost
= NS_GetInnermostURI(currentURI
);
1083 if (innermost
->SchemeIs("file")) {
1084 MOZ_ASSERT(mEncoding
== UTF_8_ENCODING
);
1085 if (!(mMode
== LOAD_AS_DATA
|| mMode
== VIEW_SOURCE_XML
)) {
1086 mDecodingLocalFileWithoutTokenizing
= true;
1090 innermost
->GetAsciiHost(host
);
1091 if (!host
.IsEmpty()) {
1092 // First let's see if the host is DNS-absolute and ends with a
1093 // dot and get rid of that one.
1094 if (host
.Last() == '.') {
1095 host
.SetLength(host
.Length() - 1);
1097 int32_t index
= host
.RFindChar('.');
1098 if (index
!= kNotFound
) {
1099 // We tolerate an IPv4 component as generic "TLD", so don't
1102 Substring(host
, index
+ 1, host
.Length() - (index
+ 1)),
1112 mTreeBuilder
->setIsSrcdocDocument(isSrcdoc
);
1113 mTreeBuilder
->setScriptingEnabled(scriptingEnabled
);
1114 mTreeBuilder
->SetPreventScriptExecution(
1115 !((mMode
== NORMAL
) && scriptingEnabled
));
1116 mTreeBuilder
->setAllowDeclarativeShadowRoots(
1117 mExecutor
->GetDocument()->AllowsDeclarativeShadowRoots());
1118 mTokenizer
->start();
1120 mExecutor
->StartReadingFromStage();
1122 if (mMode
== PLAIN_TEXT
) {
1123 mTreeBuilder
->StartPlainText();
1124 mTokenizer
->StartPlainText();
1126 mTemplatePushedOrHeadPopped
); // Needed to force 1024-byte sniffing
1127 // Flush the ops to put them where ContinueAfterScriptsOrEncodingCommitment
1129 auto r
= mTreeBuilder
->Flush();
1131 return mExecutor
->MarkAsBroken(r
.unwrapErr());
1133 } else if (mMode
== VIEW_SOURCE_PLAIN
) {
1134 nsAutoString viewSourceTitle
;
1135 CopyUTF8toUTF16(mViewSourceTitle
, viewSourceTitle
);
1136 mTreeBuilder
->EnsureBufferSpace(viewSourceTitle
.Length());
1137 mTreeBuilder
->StartPlainTextViewSource(viewSourceTitle
);
1138 mTokenizer
->StartPlainText();
1140 mTemplatePushedOrHeadPopped
); // Needed to force 1024-byte sniffing
1141 // Flush the ops to put them where ContinueAfterScriptsOrEncodingCommitment
1143 auto r
= mTreeBuilder
->Flush();
1145 return mExecutor
->MarkAsBroken(r
.unwrapErr());
1147 } else if (mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
) {
1148 // Generate and flush the View Source document up to and including the
1149 // pre element start.
1150 mTokenizer
->StartViewSource(NS_ConvertUTF8toUTF16(mViewSourceTitle
));
1151 if (mMode
== VIEW_SOURCE_XML
) {
1152 mTokenizer
->StartViewSourceCharacters();
1154 // Flush the ops to put them where ContinueAfterScriptsOrEncodingCommitment
1156 auto r
= mTokenizer
->FlushViewSource();
1158 return mExecutor
->MarkAsBroken(r
.unwrapErr());
1163 * If you move the following line, be very careful not to cause
1164 * WillBuildModel to be called before the document has had its
1165 * script global object set.
1167 rv
= mExecutor
->WillBuildModel();
1168 NS_ENSURE_SUCCESS(rv
, rv
);
1170 RefPtr
<nsHtml5OwningUTF16Buffer
> newBuf
=
1171 nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE
);
1173 // marks this stream parser as terminated,
1174 // which prevents entry to code paths that
1175 // would use mFirstBuffer or mLastBuffer.
1176 return mExecutor
->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1178 MOZ_ASSERT(!mFirstBuffer
, "How come we have the first buffer set?");
1179 MOZ_ASSERT(!mLastBuffer
, "How come we have the last buffer set?");
1180 mFirstBuffer
= mLastBuffer
= newBuf
;
1184 nsCOMPtr
<nsIHttpChannel
> httpChannel(do_QueryInterface(mRequest
, &rv
));
1185 if (NS_SUCCEEDED(rv
)) {
1186 nsAutoCString method
;
1187 Unused
<< httpChannel
->GetRequestMethod(method
);
1188 // XXX does Necko have a way to renavigate POST, etc. without hitting
1190 if (!method
.EqualsLiteral("GET")) {
1191 // This is the old Gecko behavior but the HTML5 spec disagrees.
1192 // Don't reparse on POST.
1193 mReparseForbidden
= true;
1197 // Attempt to retarget delivery of data (via OnDataAvailable) to the parser
1198 // thread, rather than through the main thread.
1199 nsCOMPtr
<nsIThreadRetargetableRequest
> threadRetargetableRequest
=
1200 do_QueryInterface(mRequest
, &rv
);
1201 if (threadRetargetableRequest
) {
1202 rv
= threadRetargetableRequest
->RetargetDeliveryTo(mEventTarget
);
1203 if (NS_SUCCEEDED(rv
)) {
1204 // Parser thread should be now ready to get data from necko and parse it
1205 // and main thread might have a chance to process a collector slice.
1206 // We need to do this asynchronously so that necko may continue processing
1208 nsCOMPtr
<nsIRunnable
> runnable
=
1209 new MaybeRunCollector(mExecutor
->GetDocument()->GetDocShell());
1210 mozilla::SchedulerGroup::Dispatch(runnable
.forget());
1214 if (NS_FAILED(rv
)) {
1215 NS_WARNING("Failed to retarget HTML data delivery to the parser thread.");
1218 if (mCharsetSource
== kCharsetFromParentFrame
) {
1219 // Remember this for error reporting.
1220 mInitialEncodingWasFromParentFrame
= true;
1221 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
);
1224 if (mForceAutoDetection
|| mCharsetSource
< kCharsetFromChannel
) {
1225 mBufferingBytes
= true;
1226 if (mMode
!= VIEW_SOURCE_XML
) {
1227 // We need to set mLookingForMetaCharset to true here in case the first
1228 // buffer to arrive is larger than 1024. We need the code that splits
1229 // the buffers at 1024 bytes to work even in that case.
1230 mLookingForMetaCharset
= true;
1234 if (mCharsetSource
< kCharsetFromUtf8OnlyMime
) {
1235 // we aren't ready to commit to an encoding yet
1236 // leave converter uninstantiated for now
1240 MOZ_ASSERT(!(mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
));
1242 MOZ_ASSERT(mEncoding
== UTF_8_ENCODING
,
1243 "How come UTF-8-only MIME type didn't set encoding to UTF-8?");
1245 // We are loading JSON/WebVTT/etc. into a browsing context.
1246 // There's no need to remove the BOM manually here, because
1247 // the UTF-8 decoder removes it.
1248 mReparseForbidden
= true;
1249 mForceAutoDetection
= false;
1251 // Instantiate the converter here to avoid BOM sniffing.
1252 mDecodingLocalFileWithoutTokenizing
= false;
1253 mUnicodeDecoder
= mEncoding
->NewDecoderWithBOMRemoval();
1257 void nsHtml5StreamParser::DoStopRequest() {
1258 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1259 MOZ_RELEASE_ASSERT(STREAM_BEING_READ
== mStreamState
,
1260 "Stream ended without being open.");
1261 mTokenizerMutex
.AssertCurrentThreadOwns();
1263 auto guard
= MakeScopeExit([&] { OnContentComplete(); });
1265 if (IsTerminated()) {
1269 if (MOZ_UNLIKELY(mLookingForXmlDeclarationForXmlViewSource
)) {
1270 mLookingForXmlDeclarationForXmlViewSource
= false;
1271 mBufferingBytes
= false;
1272 mUnicodeDecoder
= mEncoding
->NewDecoderWithoutBOMHandling();
1273 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
1275 for (auto&& buffer
: mBufferedBytes
) {
1276 nsresult rv
= WriteStreamBytes(buffer
);
1277 if (NS_FAILED(rv
)) {
1282 } else if (!mUnicodeDecoder
) {
1284 if (NS_FAILED(rv
= SniffStreamBytes(Span
<const uint8_t>(), true))) {
1290 MOZ_ASSERT(mUnicodeDecoder
,
1291 "Should have a decoder after finalizing sniffing.");
1293 // mLastBuffer should always point to a buffer of the size
1294 // READ_BUFFER_SIZE.
1296 NS_WARNING("mLastBuffer should not be null!");
1297 MarkAsBroken(NS_ERROR_NULL_POINTER
);
1301 Span
<uint8_t> src
; // empty span
1303 auto dst
= mLastBuffer
->TailAsSpan(READ_BUFFER_SIZE
);
1308 // Do not use structured binding lest deal with [-Werror=unused-variable]
1309 std::tie(result
, read
, written
, hadErrors
) =
1310 mUnicodeDecoder
->DecodeToUTF16(src
, dst
, true);
1311 if (!(mLookingForMetaCharset
|| mDecodingLocalFileWithoutTokenizing
)) {
1312 OnNewContent(dst
.To(written
));
1315 mHasHadErrors
= true;
1317 MOZ_ASSERT(read
== 0, "How come an empty span was read form?");
1318 mLastBuffer
->AdvanceEnd(written
);
1319 if (result
== kOutputFull
) {
1320 RefPtr
<nsHtml5OwningUTF16Buffer
> newBuf
=
1321 nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE
);
1323 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1326 mLastBuffer
= (mLastBuffer
->next
= std::move(newBuf
));
1328 if (!mLookingForMetaCharset
&& mDecodingLocalFileWithoutTokenizing
) {
1329 MOZ_ASSERT(mNumBytesBuffered
< LOCAL_FILE_UTF_8_BUFFER_SIZE
);
1330 MOZ_ASSERT(!mStartedFeedingDetector
);
1331 for (auto&& buffer
: mBufferedBytes
) {
1332 FeedDetector(buffer
);
1334 MOZ_ASSERT(!mChardetEof
);
1336 auto [encoding
, source
] = GuessEncoding(true);
1337 mCharsetSource
= source
;
1338 if (encoding
!= mEncoding
) {
1339 mEncoding
= encoding
;
1340 nsresult rv
= ReDecodeLocalFile();
1341 if (NS_FAILED(rv
)) {
1348 MOZ_ASSERT(mEncoding
== UTF_8_ENCODING
);
1349 nsresult rv
= CommitLocalFileToEncoding();
1350 if (NS_FAILED(rv
)) {
1359 mStreamState
= STREAM_ENDED
;
1361 if (IsTerminatedOrInterrupted()) {
1365 ParseAvailableData();
1368 class nsHtml5RequestStopper
: public Runnable
{
1370 nsHtml5StreamParserPtr mStreamParser
;
1373 explicit nsHtml5RequestStopper(nsHtml5StreamParser
* aStreamParser
)
1374 : Runnable("nsHtml5RequestStopper"), mStreamParser(aStreamParser
) {}
1375 NS_IMETHOD
Run() override
{
1376 mozilla::MutexAutoLock
autoLock(mStreamParser
->mTokenizerMutex
);
1377 mStreamParser
->DoStopRequest();
1378 mStreamParser
->PostLoadFlusher();
1383 nsresult
nsHtml5StreamParser::OnStopRequest(
1384 nsIRequest
* aRequest
, nsresult status
,
1385 const mozilla::ReentrantMonitorAutoEnter
& aProofOfLock
) {
1386 MOZ_ASSERT_IF(aRequest
, mRequest
== aRequest
);
1387 if (mOnStopCalled
) {
1388 if (mOnDataFinishedTime
) {
1389 mOnStopRequestTime
= TimeStamp::Now();
1391 mOnDataFinishedTime
= TimeStamp::Now();
1394 mOnStopCalled
= true;
1396 if (MOZ_UNLIKELY(NS_IsMainThread())) {
1397 mOnStopRequestTime
= TimeStamp::Now();
1398 nsCOMPtr
<nsIRunnable
> stopper
= new nsHtml5RequestStopper(this);
1400 mEventTarget
->Dispatch(stopper
, nsIThread::DISPATCH_NORMAL
))) {
1401 NS_WARNING("Dispatching StopRequest event failed.");
1404 mOnDataFinishedTime
= TimeStamp::Now();
1406 if (StaticPrefs::network_send_OnDataFinished_html5parser()) {
1407 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1409 mozilla::MutexAutoLock
autoLock(mTokenizerMutex
);
1413 // Let the MainThread event handle this, even though it will just
1414 // send it back to this thread, so we can accurately judge the impact
1415 // of this change. This should eventually be removed
1416 mOnStopCalled
= false;
1417 // don't record any telemetry for this
1422 if (!mOnStopRequestTime
.IsNull() && !mOnDataFinishedTime
.IsNull()) {
1423 TimeDuration delta
= (mOnStopRequestTime
- mOnDataFinishedTime
);
1424 if (delta
.ToMilliseconds() < 0) {
1425 // Because Telemetry can't handle negatives
1428 http_content_html5parser_ondatafinished_to_onstop_delay_negative
1429 .AccumulateRawDuration(delta
);
1431 glean::networking::http_content_html5parser_ondatafinished_to_onstop_delay
1432 .AccumulateRawDuration(delta
);
1438 void nsHtml5StreamParser::DoDataAvailableBuffer(
1439 mozilla::Buffer
<uint8_t>&& aBuffer
) {
1440 if (MOZ_UNLIKELY(!mBufferingBytes
)) {
1441 DoDataAvailable(aBuffer
);
1444 if (MOZ_UNLIKELY(mLookingForXmlDeclarationForXmlViewSource
)) {
1445 const uint8_t* elements
= aBuffer
.Elements();
1446 size_t length
= aBuffer
.Length();
1447 const uint8_t* lt
= (const uint8_t*)memchr(elements
, '>', length
);
1449 mBufferedBytes
.AppendElement(std::move(aBuffer
));
1453 // We found an '>'. Now there either is or isn't an XML decl.
1454 length
= (lt
- elements
) + 1;
1455 Vector
<uint8_t> contiguous
;
1456 for (auto&& buffer
: mBufferedBytes
) {
1457 if (!contiguous
.append(buffer
.Elements(), buffer
.Length())) {
1458 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1462 if (!contiguous
.append(elements
, length
)) {
1463 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1467 const Encoding
* encoding
=
1468 xmldecl_parse(contiguous
.begin(), contiguous
.length());
1470 mEncoding
= WrapNotNull(encoding
);
1471 mCharsetSource
= kCharsetFromXmlDeclaration
;
1474 mLookingForXmlDeclarationForXmlViewSource
= false;
1475 mBufferingBytes
= false;
1476 mUnicodeDecoder
= mEncoding
->NewDecoderWithoutBOMHandling();
1477 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
1479 for (auto&& buffer
: mBufferedBytes
) {
1480 DoDataAvailable(buffer
);
1482 DoDataAvailable(aBuffer
);
1483 mBufferedBytes
.Clear();
1486 CheckedInt
<size_t> bufferedPlusLength(aBuffer
.Length());
1487 bufferedPlusLength
+= mNumBytesBuffered
;
1488 if (!bufferedPlusLength
.isValid()) {
1489 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1492 // Ensure that WriteStreamBytes() sees buffers ending
1493 // exactly at the two special boundaries.
1494 bool metaBoundaryWithinBuffer
=
1495 mLookingForMetaCharset
&&
1496 mNumBytesBuffered
< UNCONDITIONAL_META_SCAN_BOUNDARY
&&
1497 bufferedPlusLength
.value() > UNCONDITIONAL_META_SCAN_BOUNDARY
;
1498 bool localFileLimitWithinBuffer
=
1499 mDecodingLocalFileWithoutTokenizing
&&
1500 mNumBytesBuffered
< LOCAL_FILE_UTF_8_BUFFER_SIZE
&&
1501 bufferedPlusLength
.value() > LOCAL_FILE_UTF_8_BUFFER_SIZE
;
1502 if (!metaBoundaryWithinBuffer
&& !localFileLimitWithinBuffer
) {
1503 // Truncation OK, because we just checked the range.
1504 mNumBytesBuffered
= bufferedPlusLength
.value();
1505 mBufferedBytes
.AppendElement(std::move(aBuffer
));
1506 DoDataAvailable(mBufferedBytes
.LastElement());
1509 !(metaBoundaryWithinBuffer
&& localFileLimitWithinBuffer
),
1510 "How can Necko give us a buffer this large?");
1511 size_t boundary
= metaBoundaryWithinBuffer
1512 ? UNCONDITIONAL_META_SCAN_BOUNDARY
1513 : LOCAL_FILE_UTF_8_BUFFER_SIZE
;
1514 // Truncation OK, because the constant is small enough.
1515 size_t overBoundary
= bufferedPlusLength
.value() - boundary
;
1516 MOZ_RELEASE_ASSERT(overBoundary
< aBuffer
.Length());
1517 size_t untilBoundary
= aBuffer
.Length() - overBoundary
;
1518 auto span
= aBuffer
.AsSpan();
1519 auto head
= span
.To(untilBoundary
);
1520 auto tail
= span
.From(untilBoundary
);
1521 MOZ_RELEASE_ASSERT(mNumBytesBuffered
+ untilBoundary
== boundary
);
1522 // The following copies may end up being useless, but optimizing
1523 // them away would add complexity.
1524 Maybe
<Buffer
<uint8_t>> maybeHead
= Buffer
<uint8_t>::CopyFrom(head
);
1525 if (maybeHead
.isNothing()) {
1526 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1529 mNumBytesBuffered
= boundary
;
1530 mBufferedBytes
.AppendElement(std::move(*maybeHead
));
1531 DoDataAvailable(mBufferedBytes
.LastElement());
1532 // Re-decode may have happened here.
1534 Maybe
<Buffer
<uint8_t>> maybeTail
= Buffer
<uint8_t>::CopyFrom(tail
);
1535 if (maybeTail
.isNothing()) {
1536 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1539 mNumBytesBuffered
+= tail
.Length();
1540 mBufferedBytes
.AppendElement(std::move(*maybeTail
));
1541 DoDataAvailable(mBufferedBytes
.LastElement());
1543 // Do this clean-up here to avoid use-after-free when
1544 // DoDataAvailable is passed a span pointing into an
1545 // element of mBufferedBytes.
1546 if (!mBufferingBytes
) {
1547 mBufferedBytes
.Clear();
1551 void nsHtml5StreamParser::DoDataAvailable(Span
<const uint8_t> aBuffer
) {
1552 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1553 MOZ_RELEASE_ASSERT(STREAM_BEING_READ
== mStreamState
,
1554 "DoDataAvailable called when stream not open.");
1555 mTokenizerMutex
.AssertCurrentThreadOwns();
1557 if (IsTerminated()) {
1563 if ((mForceAutoDetection
|| mCharsetSource
< kCharsetFromParentFrame
) &&
1564 !mBufferingBytes
&& !mReparseForbidden
&&
1565 !(mMode
== LOAD_AS_DATA
|| mMode
== VIEW_SOURCE_XML
)) {
1566 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
,
1567 "How is mBufferingBytes false if "
1568 "mDecodingLocalFileWithoutTokenizing is true?");
1569 FeedDetector(aBuffer
);
1571 rv
= WriteStreamBytes(aBuffer
);
1573 rv
= SniffStreamBytes(aBuffer
, false);
1575 if (NS_FAILED(rv
)) {
1580 if (IsTerminatedOrInterrupted()) {
1584 if (!mLookingForMetaCharset
&& mDecodingLocalFileWithoutTokenizing
) {
1588 ParseAvailableData();
1590 if (mBomState
!= BOM_SNIFFING_OVER
|| mFlushTimerArmed
|| mSpeculating
) {
1595 mozilla::MutexAutoLock
flushTimerLock(mFlushTimerMutex
);
1596 mFlushTimer
->InitWithNamedFuncCallback(
1597 nsHtml5StreamParser::TimerCallback
, static_cast<void*>(this),
1598 mFlushTimerEverFired
? StaticPrefs::html5_flushtimer_initialdelay()
1599 : StaticPrefs::html5_flushtimer_subsequentdelay(),
1600 nsITimer::TYPE_ONE_SHOT
, "nsHtml5StreamParser::DoDataAvailable");
1602 mFlushTimerArmed
= true;
1605 class nsHtml5DataAvailable
: public Runnable
{
1607 nsHtml5StreamParserPtr mStreamParser
;
1608 Buffer
<uint8_t> mData
;
1611 nsHtml5DataAvailable(nsHtml5StreamParser
* aStreamParser
,
1612 Buffer
<uint8_t>&& aData
)
1613 : Runnable("nsHtml5DataAvailable"),
1614 mStreamParser(aStreamParser
),
1615 mData(std::move(aData
)) {}
1616 NS_IMETHOD
Run() override
{
1617 mozilla::MutexAutoLock
autoLock(mStreamParser
->mTokenizerMutex
);
1618 mStreamParser
->DoDataAvailableBuffer(std::move(mData
));
1619 mStreamParser
->PostLoadFlusher();
1624 nsresult
nsHtml5StreamParser::OnDataAvailable(nsIRequest
* aRequest
,
1625 nsIInputStream
* aInStream
,
1626 uint64_t aSourceOffset
,
1630 MOZ_ASSERT(mRequest
== aRequest
, "Got data on wrong stream.");
1632 // Main thread to parser thread dispatch requires copying to buffer first.
1633 if (MOZ_UNLIKELY(NS_IsMainThread())) {
1634 if (NS_FAILED(rv
= mExecutor
->IsBroken())) {
1637 Maybe
<Buffer
<uint8_t>> maybe
= Buffer
<uint8_t>::Alloc(aLength
);
1638 if (maybe
.isNothing()) {
1639 return mExecutor
->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1641 Buffer
<uint8_t> data(std::move(*maybe
));
1642 rv
= aInStream
->Read(reinterpret_cast<char*>(data
.Elements()),
1643 data
.Length(), &totalRead
);
1644 NS_ENSURE_SUCCESS(rv
, rv
);
1645 MOZ_ASSERT(totalRead
== aLength
);
1647 nsCOMPtr
<nsIRunnable
> dataAvailable
=
1648 new nsHtml5DataAvailable(this, std::move(data
));
1649 if (NS_FAILED(mEventTarget
->Dispatch(dataAvailable
,
1650 nsIThread::DISPATCH_NORMAL
))) {
1651 NS_WARNING("Dispatching DataAvailable event failed.");
1656 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1657 mozilla::MutexAutoLock
autoLock(mTokenizerMutex
);
1659 if (NS_FAILED(rv
= mTreeBuilder
->IsBroken())) {
1663 // Since we're getting OnDataAvailable directly on the parser thread,
1664 // there is no nsHtml5DataAvailable that would call PostLoadFlusher.
1665 // Hence, we need to call PostLoadFlusher() before this method returns.
1666 // Braces for RAII clarity relative to the mutex despite not being
1667 // strictly necessary.
1669 auto speculationFlusher
= MakeScopeExit([&] { PostLoadFlusher(); });
1671 if (mBufferingBytes
) {
1672 Maybe
<Buffer
<uint8_t>> maybe
= Buffer
<uint8_t>::Alloc(aLength
);
1673 if (maybe
.isNothing()) {
1674 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1675 return NS_ERROR_OUT_OF_MEMORY
;
1677 Buffer
<uint8_t> data(std::move(*maybe
));
1678 rv
= aInStream
->Read(reinterpret_cast<char*>(data
.Elements()),
1679 data
.Length(), &totalRead
);
1680 NS_ENSURE_SUCCESS(rv
, rv
);
1681 MOZ_ASSERT(totalRead
== aLength
);
1682 DoDataAvailableBuffer(std::move(data
));
1685 // Read directly from response buffer.
1686 rv
= aInStream
->ReadSegments(CopySegmentsToParser
, this, aLength
,
1688 NS_ENSURE_SUCCESS(rv
, rv
);
1689 MOZ_ASSERT(totalRead
== aLength
);
1694 // Called under lock by function ptr
1696 nsresult
nsHtml5StreamParser::CopySegmentsToParser(
1697 nsIInputStream
* aInStream
, void* aClosure
, const char* aFromSegment
,
1698 uint32_t aToOffset
, uint32_t aCount
,
1699 uint32_t* aWriteCount
) MOZ_NO_THREAD_SAFETY_ANALYSIS
{
1700 nsHtml5StreamParser
* parser
= static_cast<nsHtml5StreamParser
*>(aClosure
);
1702 parser
->DoDataAvailable(AsBytes(Span(aFromSegment
, aCount
)));
1703 // Assume DoDataAvailable consumed all available bytes.
1704 *aWriteCount
= aCount
;
1708 const Encoding
* nsHtml5StreamParser::PreferredForInternalEncodingDecl(
1709 const nsAString
& aEncoding
) {
1710 const Encoding
* newEncoding
= Encoding::ForLabel(aEncoding
);
1712 // the encoding name is bogus
1713 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaUnsupported", true,
1714 mTokenizer
->getLineNumber());
1718 if (newEncoding
== UTF_16BE_ENCODING
|| newEncoding
== UTF_16LE_ENCODING
) {
1719 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaUtf16", true,
1720 mTokenizer
->getLineNumber());
1721 newEncoding
= UTF_8_ENCODING
;
1724 if (newEncoding
== X_USER_DEFINED_ENCODING
) {
1725 // WebKit/Blink hack for Indian and Armenian legacy sites
1726 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaUserDefined", true,
1727 mTokenizer
->getLineNumber());
1728 newEncoding
= WINDOWS_1252_ENCODING
;
1731 if (newEncoding
== REPLACEMENT_ENCODING
) {
1732 // No line number, because the replacement encoding doesn't allow
1733 // showing the lines.
1734 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaReplacement", true, 0);
1740 bool nsHtml5StreamParser::internalEncodingDeclaration(nsHtml5String aEncoding
) {
1741 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1742 if ((mCharsetSource
>= kCharsetFromMetaTag
&&
1743 mCharsetSource
!= kCharsetFromFinalAutoDetectionFile
) ||
1744 mSeenEligibleMetaCharset
) {
1748 nsString newEncoding
; // Not Auto, because using it to hold nsStringBuffer*
1749 aEncoding
.ToString(newEncoding
);
1750 auto encoding
= PreferredForInternalEncodingDecl(newEncoding
);
1755 mSeenEligibleMetaCharset
= true;
1757 if (!mLookingForMetaCharset
) {
1758 if (mInitialEncodingWasFromParentFrame
) {
1759 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaTooLateFrame", true,
1760 mTokenizer
->getLineNumber());
1762 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaTooLate", true,
1763 mTokenizer
->getLineNumber());
1767 if (mTemplatePushedOrHeadPopped
) {
1768 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaAfterHeadInKilobyte", false,
1769 mTokenizer
->getLineNumber());
1772 if (mForceAutoDetection
&&
1773 (encoding
->IsAsciiCompatible() || encoding
== ISO_2022_JP_ENCODING
)) {
1777 mNeedsEncodingSwitchTo
= encoding
;
1778 mEncodingSwitchSource
= kCharsetFromMetaTag
;
1782 bool nsHtml5StreamParser::TemplatePushedOrHeadPopped() {
1784 IsParserThread() || mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
,
1786 mTemplatePushedOrHeadPopped
= true;
1787 return mNumBytesBuffered
>= UNCONDITIONAL_META_SCAN_BOUNDARY
;
1790 void nsHtml5StreamParser::RememberGt(int32_t aPos
) {
1791 if (mLookingForMetaCharset
) {
1792 mGtBuffer
= mFirstBuffer
;
1797 void nsHtml5StreamParser::PostLoadFlusher() {
1798 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1799 mTokenizerMutex
.AssertCurrentThreadOwns();
1801 mTreeBuilder
->FlushLoads();
1802 // Dispatch this runnable unconditionally, because the loads
1803 // that need flushing may have been flushed earlier even if the
1804 // flush right above here did nothing. (Is this still true?)
1805 nsCOMPtr
<nsIRunnable
> runnable(mLoadFlusher
);
1807 DispatchToMain(CreateRenderBlockingRunnable(runnable
.forget())))) {
1808 NS_WARNING("failed to dispatch load flush event");
1811 if ((mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
) &&
1812 mTokenizer
->ShouldFlushViewSource()) {
1813 auto r
= mTreeBuilder
->Flush(); // delete useless ops
1814 MOZ_ASSERT(r
.isOk(), "Should have null sink with View Source");
1815 r
= mTokenizer
->FlushViewSource();
1817 MarkAsBroken(r
.unwrapErr());
1821 nsCOMPtr
<nsIRunnable
> runnable(mExecutorFlusher
);
1822 if (NS_FAILED(DispatchToMain(runnable
.forget()))) {
1823 NS_WARNING("failed to dispatch executor flush event");
1829 void nsHtml5StreamParser::FlushTreeOpsAndDisarmTimer() {
1830 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1831 if (mFlushTimerArmed
) {
1832 // avoid calling Cancel if the flush timer isn't armed to avoid acquiring
1835 mozilla::MutexAutoLock
flushTimerLock(mFlushTimerMutex
);
1836 mFlushTimer
->Cancel();
1838 mFlushTimerArmed
= false;
1840 if (mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
) {
1841 auto r
= mTokenizer
->FlushViewSource();
1843 MarkAsBroken(r
.unwrapErr());
1846 auto r
= mTreeBuilder
->Flush();
1848 MarkAsBroken(r
.unwrapErr());
1850 nsCOMPtr
<nsIRunnable
> runnable(mExecutorFlusher
);
1851 if (NS_FAILED(DispatchToMain(runnable
.forget()))) {
1852 NS_WARNING("failed to dispatch executor flush event");
1856 void nsHtml5StreamParser::SwitchDecoderIfAsciiSoFar(
1857 NotNull
<const Encoding
*> aEncoding
) {
1858 if (mEncoding
== aEncoding
) {
1859 MOZ_ASSERT(!mStartedFeedingDevTools
);
1860 // Report all already-decoded buffers to the dev tools if needed.
1861 if (mURIToSendToDevtools
) {
1862 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBufferOfMetaScan
;
1864 auto s
= Span(buffer
->getBuffer(), buffer
->getEnd());
1866 buffer
= buffer
->next
;
1871 if (!mEncoding
->IsAsciiCompatible() || !aEncoding
->IsAsciiCompatible()) {
1874 size_t numAscii
= 0;
1875 MOZ_ASSERT(mFirstBufferOfMetaScan
,
1876 "Why did we come here without starting meta scan?");
1877 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBufferOfMetaScan
;
1878 while (buffer
!= mFirstBuffer
) {
1879 MOZ_ASSERT(buffer
, "mFirstBuffer should have acted as sentinel!");
1880 MOZ_ASSERT(buffer
->getStart() == buffer
->getEnd(),
1881 "Why wasn't an early buffer fully consumed?");
1882 auto s
= Span(buffer
->getBuffer(), buffer
->getStart());
1886 numAscii
+= s
.Length();
1887 buffer
= buffer
->next
;
1889 auto s
= Span(mFirstBuffer
->getBuffer(), mFirstBuffer
->getStart());
1893 numAscii
+= s
.Length();
1895 MOZ_ASSERT(!mStartedFeedingDevTools
);
1896 // Report the ASCII prefix to dev tools if needed
1897 if (mURIToSendToDevtools
) {
1898 buffer
= mFirstBufferOfMetaScan
;
1899 while (buffer
!= mFirstBuffer
) {
1900 MOZ_ASSERT(buffer
, "mFirstBuffer should have acted as sentinel!");
1901 MOZ_ASSERT(buffer
->getStart() == buffer
->getEnd(),
1902 "Why wasn't an early buffer fully consumed?");
1903 auto s
= Span(buffer
->getBuffer(), buffer
->getStart());
1905 buffer
= buffer
->next
;
1907 auto s
= Span(mFirstBuffer
->getBuffer(), mFirstBuffer
->getStart());
1911 // Success! Now let's get rid of the already-decoded but not tokenized data:
1912 mFirstBuffer
->setEnd(mFirstBuffer
->getStart());
1913 mLastBuffer
= mFirstBuffer
;
1914 mFirstBuffer
->next
= nullptr;
1916 // Note: We could have scanned further for ASCII, which could avoid some
1917 // buffer deallocation and reallocation. However, chances are that if we got
1918 // until meta without non-ASCII before, there's going to be a title with
1919 // non-ASCII soon after anyway, so let's avoid the complexity of finding out.
1921 MOZ_ASSERT(mUnicodeDecoder
, "How come we scanned meta without a decoder?");
1922 mEncoding
= aEncoding
;
1923 mEncoding
->NewDecoderWithoutBOMHandlingInto(*mUnicodeDecoder
);
1924 mHasHadErrors
= false;
1926 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
,
1927 "Must have set mDecodingLocalFileWithoutTokenizing to false to "
1928 "report data to dev tools below");
1929 MOZ_ASSERT(!mLookingForMetaCharset
,
1930 "Must have set mLookingForMetaCharset to false to report data to "
1933 // Now skip over as many bytes and redecode the tail of the
1936 for (auto&& buffer
: mBufferedBytes
) {
1937 size_t nextSkipped
= skipped
+ buffer
.Length();
1938 if (nextSkipped
<= numAscii
) {
1939 skipped
= nextSkipped
;
1942 if (skipped
>= numAscii
) {
1943 WriteStreamBytes(buffer
);
1944 skipped
= nextSkipped
;
1947 size_t tailLength
= nextSkipped
- numAscii
;
1948 WriteStreamBytes(Span
<uint8_t>(buffer
).From(buffer
.Length() - tailLength
));
1949 skipped
= nextSkipped
;
1953 size_t nsHtml5StreamParser::CountGts() {
1958 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBufferOfMetaScan
;
1960 MOZ_ASSERT(buffer
, "How did we walk past mGtBuffer?");
1961 char16_t
* buf
= buffer
->getBuffer();
1962 if (buffer
== mGtBuffer
) {
1963 for (int32_t i
= 0; i
<= mGtPos
; ++i
) {
1964 if (buf
[i
] == u
'>') {
1970 for (int32_t i
= 0; i
< buffer
->getEnd(); ++i
) {
1971 if (buf
[i
] == u
'>') {
1975 buffer
= buffer
->next
;
1980 void nsHtml5StreamParser::DiscardMetaSpeculation() {
1981 mozilla::MutexAutoLock
speculationAutoLock(mSpeculationMutex
);
1982 // Rewind the stream
1983 MOZ_ASSERT(!mAtEOF
, "How did we end up setting this?");
1984 mTokenizer
->resetToDataState();
1985 mTokenizer
->setLineNumber(1);
1988 if (mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
) {
1989 // resetToDataState() above logically rewinds to the state before
1990 // the plain text start, so we need to start plain text again to
1991 // put the tokenizer into the plain text state.
1992 mTokenizer
->StartPlainText();
1995 mFirstBuffer
= mLastBuffer
;
1996 mFirstBuffer
->setStart(0);
1997 mFirstBuffer
->setEnd(0);
1998 mFirstBuffer
->next
= nullptr;
2000 mTreeBuilder
->flushCharacters(); // empty the pending buffer
2001 mTreeBuilder
->ClearOps(); // now get rid of the failed ops
2003 if (mMode
== VIEW_SOURCE_HTML
) {
2004 mTokenizer
->RewindViewSource();
2008 // We know that this resets the tree builder back to the start state.
2009 // This must happen _after_ the flushCharacters() call above!
2010 const auto& speculation
= mSpeculations
.ElementAt(0);
2011 mTreeBuilder
->loadState(speculation
->GetSnapshot());
2014 // Experimentation suggests that we don't need to do anything special
2015 // for ignoring the leading LF in View Source here.
2017 mSpeculations
.Clear(); // potentially a huge number of destructors
2018 // run here synchronously...
2020 // Now set up a new speculation for the main thread to find.
2021 // Note that we stay in the speculating state, because the main thread
2022 // knows how to come out of that state and this thread does not.
2024 nsHtml5Speculation
* speculation
= new nsHtml5Speculation(
2025 mFirstBuffer
, mFirstBuffer
->getStart(), mTokenizer
->getLineNumber(),
2026 mTokenizer
->getColumnNumber(), mTreeBuilder
->newSnapshot());
2027 MOZ_ASSERT(!mFlushTimerArmed
, "How did we end up arming the timer?");
2028 if (mMode
== VIEW_SOURCE_HTML
) {
2029 mTokenizer
->SetViewSourceOpSink(speculation
);
2030 mTokenizer
->StartViewSourceCharacters();
2032 MOZ_ASSERT(mMode
!= VIEW_SOURCE_XML
);
2033 mTreeBuilder
->SetOpSink(speculation
);
2035 mSpeculations
.AppendElement(speculation
); // adopts the pointer
2036 MOZ_ASSERT(mSpeculating
, "How did we end speculating?");
2040 * The general idea is to match WebKit and Blink exactly for meta
2043 * 1. WebKit and Blink look for meta as if scripting was disabled
2044 * for `noscript` purposes. This implementation matches the
2045 * `noscript` treatment of the observable DOM building (in order
2046 * to be able to use the same tree builder run).
2047 * 2. WebKit and Blink look for meta as if the foreign content
2048 * feedback from the tree builder to the tokenizer didn't exist.
2049 * This implementation considers the foreign content rules in
2050 * order to be able to use the same tree builder run for meta
2051 * and the observable DOM building. Note that since <svg> and
2052 * <math> imply the end of head, this only matters for meta after
2053 * head but starting within the 1024-byte zone.
2055 * Template is treated specially, because that WebKit/Blink behavior
2056 * is easy to emulate unlike the above two exceptions. In general,
2057 * the meta scan token handler in WebKit and Blink behaves as if there
2058 * was a scripting-disabled tree builder predating the introduction
2059 * of foreign content and template.
2061 * Meta is honored if it _starts_ within the first 1024 kilobytes or,
2062 * if by the 1024-byte boundary head hasn't ended and a template
2063 * element hasn't started, a meta occurs before the first of the head
2064 * ending or a template element starting.
2066 * If a meta isn't honored according to the above definition, and
2067 * we aren't dealing with plain text, the buffered bytes, which by
2068 * now have to contain `>` character unless we encountered EOF, are
2069 * scanned for syntax resembling an XML declaration.
2071 * If neither a meta nor syntax resembling an XML declaration has
2072 * been honored and we aren't inheriting the encoding from a
2073 * same-origin parent or parsing for XHR, chardetng is used.
2074 * chardetng runs first for the part of the document that was searched
2075 * for meta and then at EOF. The part searched for meta is defined as
2076 * follows in order to avoid network buffer boundary-dependent
2079 * 1. At least the first 1024 bytes. (This is what happens for plain
2081 * 2. If the 1024-byte boundary is within a tag, comment, doctype,
2082 * or CDATA section, at least up to the end of that token or CDATA
2083 * section. (Exception: If the 1024-byte boundary is in an RCDATA
2084 * end tag that hasn't yet been decided to be an end tag, the
2085 * token is not considered.)
2086 * 3. If at the 1024-byte boundary, head hasn't ended and there hasn't
2087 * been a template tag, up to the end of the first template tag
2088 * or token ending the head, whichever comes first.
2089 * 4. Except if head is ended by a text token, only to the end of the
2090 * most recent tag, comment, or doctype token. (Because text is
2091 * coalesced, so it would be harder to correlate the text to the
2094 * An encoding-related reload is still possible if chardetng's guess
2095 * at EOF differs from its initial guess.
2097 bool nsHtml5StreamParser::ProcessLookingForMetaCharset(bool aEof
) {
2098 MOZ_ASSERT(mBomState
== BOM_SNIFFING_OVER
);
2099 MOZ_ASSERT(mMode
!= VIEW_SOURCE_XML
);
2100 bool rewound
= false;
2101 MOZ_ASSERT(mForceAutoDetection
||
2102 mCharsetSource
< kCharsetFromInitialAutoDetectionASCII
||
2103 mCharsetSource
== kCharsetFromParentFrame
,
2104 "Why are we looking for meta charset if we've seen it?");
2105 // NOTE! We may come here multiple times with
2106 // mNumBytesBuffered == UNCONDITIONAL_META_SCAN_BOUNDARY
2107 // if the tokenizer suspends multiple times after decoding has reached
2108 // mNumBytesBuffered == UNCONDITIONAL_META_SCAN_BOUNDARY. That's why
2109 // we need to also check whether the we are at the end of the last
2111 // Note that DoDataAvailableBuffer() ensures that the code here has
2112 // the opportunity to run at the exact UNCONDITIONAL_META_SCAN_BOUNDARY
2113 // even if there isn't a network buffer boundary there.
2114 bool atKilobyte
= false;
2115 if ((mNumBytesBuffered
== UNCONDITIONAL_META_SCAN_BOUNDARY
&&
2116 mFirstBuffer
== mLastBuffer
&& !mFirstBuffer
->hasMore())) {
2118 mTokenizer
->AtKilobyteBoundary();
2120 if (!mNeedsEncodingSwitchTo
&&
2121 (aEof
|| (mTemplatePushedOrHeadPopped
&&
2122 !mTokenizer
->IsInTokenStartedAtKilobyteBoundary() &&
2124 mNumBytesBuffered
> UNCONDITIONAL_META_SCAN_BOUNDARY
)))) {
2125 // meta charset was not found
2126 mLookingForMetaCharset
= false;
2127 if (mStartsWithLtQuestion
&& mCharsetSource
< kCharsetFromXmlDeclaration
) {
2128 // Look for bogo XML declaration.
2129 // Search the first buffer in the hope that '>' is within it.
2130 MOZ_ASSERT(!mBufferedBytes
.IsEmpty(),
2131 "How did at least <? not get buffered?");
2132 Buffer
<uint8_t>& first
= mBufferedBytes
[0];
2133 const Encoding
* encoding
=
2134 xmldecl_parse(first
.Elements(), first
.Length());
2136 // Our bogo XML declaration scanner wants to see a contiguous buffer, so
2137 // let's linearize the data. (Ideally, the XML declaration scanner would
2138 // be incremental, but this is the rare path anyway.)
2139 Vector
<uint8_t> contiguous
;
2140 if (!contiguous
.append(first
.Elements(), first
.Length())) {
2141 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2144 for (size_t i
= 1; i
< mBufferedBytes
.Length(); ++i
) {
2145 Buffer
<uint8_t>& buffer
= mBufferedBytes
[i
];
2146 const uint8_t* elements
= buffer
.Elements();
2147 size_t length
= buffer
.Length();
2148 const uint8_t* lt
= (const uint8_t*)memchr(elements
, '>', length
);
2151 length
= (lt
- elements
) + 1;
2154 if (!contiguous
.append(elements
, length
)) {
2155 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2159 // Avoid linearizing all buffered bytes unnecessarily.
2163 encoding
= xmldecl_parse(contiguous
.begin(), contiguous
.length());
2166 if (!(mForceAutoDetection
&& (encoding
->IsAsciiCompatible() ||
2167 encoding
== ISO_2022_JP_ENCODING
))) {
2168 mForceAutoDetection
= false;
2169 mNeedsEncodingSwitchTo
= encoding
;
2170 mEncodingSwitchSource
= kCharsetFromXmlDeclaration
;
2174 // Check again in case we found an encoding in the bogo XML declaration.
2175 if (!mNeedsEncodingSwitchTo
&&
2176 (mForceAutoDetection
||
2177 mCharsetSource
< kCharsetFromInitialAutoDetectionASCII
) &&
2178 !(mMode
== LOAD_AS_DATA
|| mMode
== VIEW_SOURCE_XML
) &&
2179 !(mDecodingLocalFileWithoutTokenizing
&& !aEof
&&
2180 mNumBytesBuffered
<= LOCAL_FILE_UTF_8_BUFFER_SIZE
)) {
2181 MOZ_ASSERT(!mStartedFeedingDetector
);
2182 if (mNumBytesBuffered
== UNCONDITIONAL_META_SCAN_BOUNDARY
|| aEof
) {
2183 // We know that all the buffered bytes have been tokenized, so feed
2184 // them all to chardetng.
2185 for (auto&& buffer
: mBufferedBytes
) {
2186 FeedDetector(buffer
);
2189 MOZ_ASSERT(!mChardetEof
);
2192 auto [encoding
, source
] = GuessEncoding(true);
2193 mNeedsEncodingSwitchTo
= encoding
;
2194 mEncodingSwitchSource
= source
;
2195 } else if (mNumBytesBuffered
> UNCONDITIONAL_META_SCAN_BOUNDARY
) {
2196 size_t gtsLeftToFind
= CountGts();
2197 size_t bytesSeen
= 0;
2198 // We sync the bytes to the UTF-16 code units seen to avoid depending
2199 // on network buffer boundaries. We do the syncing by counting '>'
2200 // bytes / code units. However, we always scan at least 1024 bytes.
2201 // The 1024-byte boundary is guaranteed to be between buffers.
2202 // The guarantee is implemented in DoDataAvailableBuffer().
2203 for (auto&& buffer
: mBufferedBytes
) {
2204 if (!mNeedsEncodingSwitchTo
) {
2205 if (gtsLeftToFind
) {
2206 auto span
= buffer
.AsSpan();
2208 for (size_t i
= 0; i
< span
.Length(); ++i
) {
2209 if (span
[i
] == uint8_t('>')) {
2211 if (!gtsLeftToFind
) {
2212 if (bytesSeen
< UNCONDITIONAL_META_SCAN_BOUNDARY
) {
2216 FeedDetector(span
.To(i
));
2217 auto [encoding
, source
] = GuessEncoding(true);
2218 mNeedsEncodingSwitchTo
= encoding
;
2219 mEncodingSwitchSource
= source
;
2220 FeedDetector(span
.From(i
));
2221 bytesSeen
+= buffer
.Length();
2222 // No need to update bytesSeen anymore, but let's do it for
2224 // We should do `continue outer;` but C++ can't.
2231 FeedDetector(buffer
);
2232 bytesSeen
+= buffer
.Length();
2236 if (bytesSeen
== UNCONDITIONAL_META_SCAN_BOUNDARY
) {
2237 auto [encoding
, source
] = GuessEncoding(true);
2238 mNeedsEncodingSwitchTo
= encoding
;
2239 mEncodingSwitchSource
= source
;
2242 FeedDetector(buffer
);
2243 bytesSeen
+= buffer
.Length();
2246 MOZ_ASSERT(mNeedsEncodingSwitchTo
,
2247 "How come we didn't call GuessEncoding()?");
2250 if (mNeedsEncodingSwitchTo
) {
2251 mDecodingLocalFileWithoutTokenizing
= false;
2252 mLookingForMetaCharset
= false;
2254 auto needsEncodingSwitchTo
= WrapNotNull(mNeedsEncodingSwitchTo
);
2255 mNeedsEncodingSwitchTo
= nullptr;
2257 SwitchDecoderIfAsciiSoFar(needsEncodingSwitchTo
);
2258 // The above line may have changed mEncoding so that mEncoding equals
2259 // needsEncodingSwitchTo.
2261 mCharsetSource
= mEncodingSwitchSource
;
2263 if (mMode
== VIEW_SOURCE_HTML
) {
2264 auto r
= mTokenizer
->FlushViewSource();
2266 MarkAsBroken(r
.unwrapErr());
2270 auto r
= mTreeBuilder
->Flush();
2272 MarkAsBroken(r
.unwrapErr());
2276 if (mEncoding
!= needsEncodingSwitchTo
) {
2277 // Speculation failed
2280 if (mEncoding
== ISO_2022_JP_ENCODING
||
2281 needsEncodingSwitchTo
== ISO_2022_JP_ENCODING
) {
2282 // Chances are no Web author will fix anything due to this message, so
2283 // this is here to help understanding issues when debugging sites made
2285 mTreeBuilder
->MaybeComplainAboutCharset("EncSpeculationFail2022", false,
2286 mTokenizer
->getLineNumber());
2288 if (mCharsetSource
== kCharsetFromMetaTag
) {
2289 mTreeBuilder
->MaybeComplainAboutCharset(
2290 "EncSpeculationFailMeta", false, mTokenizer
->getLineNumber());
2291 } else if (mCharsetSource
== kCharsetFromXmlDeclaration
) {
2292 // This intentionally refers to the line number of how far ahead
2293 // the document was parsed even though the bogo XML decl is always
2295 mTreeBuilder
->MaybeComplainAboutCharset(
2296 "EncSpeculationFailXml", false, mTokenizer
->getLineNumber());
2300 DiscardMetaSpeculation();
2301 // Redecode the stream.
2302 mEncoding
= needsEncodingSwitchTo
;
2303 mUnicodeDecoder
= mEncoding
->NewDecoderWithBOMRemoval();
2304 mHasHadErrors
= false;
2306 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
,
2307 "Must have set mDecodingLocalFileWithoutTokenizing to false "
2308 "to report data to dev tools below");
2309 MOZ_ASSERT(!mLookingForMetaCharset
,
2310 "Must have set mLookingForMetaCharset to false to report data "
2311 "to dev tools below");
2312 for (auto&& buffer
: mBufferedBytes
) {
2313 nsresult rv
= WriteStreamBytes(buffer
);
2314 if (NS_FAILED(rv
)) {
2320 } else if (!mLookingForMetaCharset
&& !mDecodingLocalFileWithoutTokenizing
) {
2321 MOZ_ASSERT(!mStartedFeedingDevTools
);
2322 // Report all already-decoded buffers to the dev tools if needed.
2323 if (mURIToSendToDevtools
) {
2324 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBufferOfMetaScan
;
2326 auto s
= Span(buffer
->getBuffer(), buffer
->getEnd());
2328 buffer
= buffer
->next
;
2332 if (!mLookingForMetaCharset
) {
2333 mGtBuffer
= nullptr;
2336 if (!mDecodingLocalFileWithoutTokenizing
) {
2337 mFirstBufferOfMetaScan
= nullptr;
2338 mBufferingBytes
= false;
2339 mBufferedBytes
.Clear();
2340 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, true);
2341 if (mMode
== VIEW_SOURCE_HTML
) {
2342 auto r
= mTokenizer
->FlushViewSource();
2344 MarkAsBroken(r
.unwrapErr());
2348 auto r
= mTreeBuilder
->Flush();
2350 MarkAsBroken(r
.unwrapErr());
2358 void nsHtml5StreamParser::ParseAvailableData() {
2359 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
2360 mTokenizerMutex
.AssertCurrentThreadOwns();
2361 MOZ_ASSERT(!(mDecodingLocalFileWithoutTokenizing
&& !mLookingForMetaCharset
));
2363 if (IsTerminatedOrInterrupted()) {
2367 if (mSpeculating
&& !IsSpeculationEnabled()) {
2371 bool requestedReload
= false;
2373 if (!mFirstBuffer
->hasMore()) {
2374 if (mFirstBuffer
== mLastBuffer
) {
2375 switch (mStreamState
) {
2376 case STREAM_BEING_READ
:
2377 // never release the last buffer.
2378 if (!mSpeculating
) {
2379 // reuse buffer space if not speculating
2380 mFirstBuffer
->setStart(0);
2381 mFirstBuffer
->setEnd(0);
2383 return; // no more data for now but expecting more
2388 if (mLookingForMetaCharset
) {
2389 // When called with aEof=true, ProcessLookingForMetaCharset()
2390 // is guaranteed to set mLookingForMetaCharset to false so
2391 // that we can't come here twice.
2392 if (ProcessLookingForMetaCharset(true)) {
2393 if (IsTerminatedOrInterrupted()) {
2398 } else if ((mForceAutoDetection
||
2399 mCharsetSource
< kCharsetFromParentFrame
) &&
2400 !(mMode
== LOAD_AS_DATA
|| mMode
== VIEW_SOURCE_XML
) &&
2401 !mReparseForbidden
) {
2402 // An earlier DetectorEof() call is possible in which case
2403 // the one here is a no-op.
2405 auto [encoding
, source
] = GuessEncoding(false);
2406 if (encoding
!= mEncoding
) {
2407 // Request a reload from the docshell.
2410 kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII
&&
2412 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLDInitialWasASCII
) ||
2413 source
== kCharsetFromFinalUserForcedAutoDetection
);
2414 mTreeBuilder
->NeedsCharsetSwitchTo(encoding
, source
, 0);
2415 requestedReload
= true;
2416 } else if (mCharsetSource
==
2417 kCharsetFromInitialAutoDetectionASCII
&&
2418 mDetectorHasSeenNonAscii
) {
2419 mCharsetSource
= source
;
2420 mTreeBuilder
->UpdateCharsetSource(mCharsetSource
);
2425 if (!mForceAutoDetection
&& !requestedReload
) {
2426 if (mCharsetSource
== kCharsetFromParentFrame
) {
2427 mTreeBuilder
->MaybeComplainAboutCharset("EncNoDeclarationFrame",
2429 } else if (mCharsetSource
== kCharsetFromXmlDeclaration
) {
2430 // We know the bogo XML decl is always on the first line.
2431 mTreeBuilder
->MaybeComplainAboutCharset("EncXmlDecl", false, 1);
2434 kCharsetFromInitialAutoDetectionWouldHaveBeenUTF8
&&
2436 kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD
) {
2437 if (mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
) {
2438 mTreeBuilder
->MaybeComplainAboutCharset("EncNoDeclPlain",
2441 mTreeBuilder
->MaybeComplainAboutCharset("EncNoDecl", true, 0);
2445 if (mHasHadErrors
&& mEncoding
!= REPLACEMENT_ENCODING
) {
2446 if (mEncoding
== UTF_8_ENCODING
) {
2447 mTreeBuilder
->TryToEnableEncodingMenu();
2449 if (mCharsetSource
== kCharsetFromParentFrame
) {
2450 if (mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
) {
2451 mTreeBuilder
->MaybeComplainAboutCharset(
2452 "EncErrorFramePlain", true, 0);
2454 mTreeBuilder
->MaybeComplainAboutCharset("EncErrorFrame",
2458 mCharsetSource
>= kCharsetFromXmlDeclaration
&&
2460 kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII
&&
2462 kCharsetFromFinalUserForcedAutoDetection
)) {
2463 mTreeBuilder
->MaybeComplainAboutCharset("EncError", true, 0);
2467 if (NS_SUCCEEDED(mTreeBuilder
->IsBroken())) {
2470 if (NS_FAILED((rv
= mTreeBuilder
->IsBroken()))) {
2473 mTreeBuilder
->StreamEnded();
2474 if (mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
) {
2475 if (!mTokenizer
->EndViewSource()) {
2476 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2481 FlushTreeOpsAndDisarmTimer();
2482 return; // no more data and not expecting more
2484 MOZ_ASSERT_UNREACHABLE("It should be impossible to reach this.");
2488 mFirstBuffer
= mFirstBuffer
->next
;
2492 // now we have a non-empty buffer
2493 mFirstBuffer
->adjust(mLastWasCR
);
2495 if (mFirstBuffer
->hasMore()) {
2496 if (!mTokenizer
->EnsureBufferSpace(mFirstBuffer
->getLength())) {
2497 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2500 mLastWasCR
= mTokenizer
->tokenizeBuffer(mFirstBuffer
);
2502 if (NS_FAILED((rv
= mTreeBuilder
->IsBroken()))) {
2506 if (mTreeBuilder
->HasScriptThatMayDocumentWriteOrBlock()) {
2507 // `HasScriptThatMayDocumentWriteOrBlock()` cannot return true if the
2508 // tree builder is preventing script execution.
2509 MOZ_ASSERT(mMode
== NORMAL
);
2510 mozilla::MutexAutoLock
speculationAutoLock(mSpeculationMutex
);
2511 nsHtml5Speculation
* speculation
= new nsHtml5Speculation(
2512 mFirstBuffer
, mFirstBuffer
->getStart(), mTokenizer
->getLineNumber(),
2513 mTokenizer
->getColumnNumber(), mTreeBuilder
->newSnapshot());
2514 mTreeBuilder
->AddSnapshotToScript(speculation
->GetSnapshot(),
2515 speculation
->GetStartLineNumber());
2516 if (mLookingForMetaCharset
) {
2517 if (mMode
== VIEW_SOURCE_HTML
) {
2518 auto r
= mTokenizer
->FlushViewSource();
2520 MarkAsBroken(r
.unwrapErr());
2524 auto r
= mTreeBuilder
->Flush();
2526 MarkAsBroken(r
.unwrapErr());
2530 FlushTreeOpsAndDisarmTimer();
2532 mTreeBuilder
->SetOpSink(speculation
);
2533 mSpeculations
.AppendElement(speculation
); // adopts the pointer
2534 mSpeculating
= true;
2536 if (IsTerminatedOrInterrupted()) {
2540 if (mLookingForMetaCharset
) {
2541 Unused
<< ProcessLookingForMetaCharset(false);
2546 class nsHtml5StreamParserContinuation
: public Runnable
{
2548 nsHtml5StreamParserPtr mStreamParser
;
2551 explicit nsHtml5StreamParserContinuation(nsHtml5StreamParser
* aStreamParser
)
2552 : Runnable("nsHtml5StreamParserContinuation"),
2553 mStreamParser(aStreamParser
) {}
2554 NS_IMETHOD
Run() override
{
2555 mozilla::MutexAutoLock
autoLock(mStreamParser
->mTokenizerMutex
);
2556 mStreamParser
->Uninterrupt();
2557 mStreamParser
->ParseAvailableData();
2562 void nsHtml5StreamParser::ContinueAfterScriptsOrEncodingCommitment(
2563 nsHtml5Tokenizer
* aTokenizer
, nsHtml5TreeBuilder
* aTreeBuilder
,
2565 // nullptr for aTokenizer means encoding commitment as opposed to the "after
2568 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
2569 MOZ_ASSERT(mMode
!= VIEW_SOURCE_XML
,
2570 "ContinueAfterScriptsOrEncodingCommitment called in XML view "
2572 MOZ_ASSERT(!(aTokenizer
&& mMode
== VIEW_SOURCE_HTML
),
2573 "ContinueAfterScriptsOrEncodingCommitment called with non-null "
2574 "tokenizer in HTML view "
2576 if (NS_FAILED(mExecutor
->IsBroken())) {
2579 MOZ_ASSERT(!(aTokenizer
&& mMode
!= NORMAL
),
2580 "We should only be executing scripts in the normal mode.");
2581 if (!aTokenizer
&& (mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
||
2582 mMode
== VIEW_SOURCE_HTML
)) {
2583 // Take the ops that were generated from OnStartRequest for the synthetic
2584 // head section of the document for plain text and HTML View Source.
2585 // XML View Source never needs this kind of encoding commitment.
2586 // We need to take the ops here so that they end up in the queue before
2587 // the ops that we take from a speculation later in this method.
2588 if (!mExecutor
->TakeOpsFromStage()) {
2589 mExecutor
->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2594 mExecutor
->AssertStageEmpty();
2597 bool speculationFailed
= false;
2599 mozilla::MutexAutoLock
speculationAutoLock(mSpeculationMutex
);
2600 if (mSpeculations
.IsEmpty()) {
2601 MOZ_ASSERT_UNREACHABLE(
2602 "ContinueAfterScriptsOrEncodingCommitment called without "
2607 const auto& speculation
= mSpeculations
.ElementAt(0);
2609 (aLastWasCR
|| !aTokenizer
->isInDataState() ||
2610 !aTreeBuilder
->snapshotMatches(speculation
->GetSnapshot()))) {
2611 speculationFailed
= true;
2612 // We've got a failed speculation :-(
2613 MaybeDisableFutureSpeculation();
2614 Interrupt(); // Make the parser thread release the tokenizer mutex sooner
2615 // Note that the interrupted state continues across possible intervening
2616 // Necko events until the nsHtml5StreamParserContinuation posted at the
2617 // end of this method runs. Therefore, this thread is guaranteed to
2618 // acquire mTokenizerMutex soon even if an intervening Necko event grabbed
2619 // it between now and the acquisition below.
2621 // now fall out of the speculationAutoLock into the tokenizerAutoLock
2624 // We've got a successful speculation!
2625 if (mSpeculations
.Length() > 1) {
2626 // the first speculation isn't the current speculation, so there's
2627 // no need to bother the parser thread.
2628 if (!speculation
->FlushToSink(mExecutor
)) {
2629 mExecutor
->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2632 MOZ_ASSERT(!mExecutor
->IsScriptExecuting(),
2633 "ParseUntilBlocked() was supposed to ensure we don't come "
2634 "here when scripts are executing.");
2635 MOZ_ASSERT(!aTokenizer
|| mExecutor
->IsInFlushLoop(),
2636 "How are we here if "
2637 "RunFlushLoop() didn't call ParseUntilBlocked() or we're "
2638 "not committing to an encoding?");
2639 mSpeculations
.RemoveElementAt(0);
2643 Interrupt(); // Make the parser thread release the tokenizer mutex sooner
2644 // Note that the interrupted state continues across possible intervening
2645 // Necko events until the nsHtml5StreamParserContinuation posted at the
2646 // end of this method runs. Therefore, this thread is guaranteed to
2647 // acquire mTokenizerMutex soon even if an intervening Necko event grabbed
2648 // it between now and the acquisition below.
2651 // the first speculation is the current speculation. Need to
2652 // release the the speculation mutex and acquire the tokenizer
2653 // mutex. (Just acquiring the other mutex here would deadlock)
2657 mozilla::MutexAutoLock
tokenizerAutoLock(mTokenizerMutex
);
2660 mAtomTable
.SetPermittedLookupEventTarget(
2661 GetMainThreadSerialEventTarget());
2664 // In principle, the speculation mutex should be acquired here,
2665 // but there's no point, because the parser thread only acquires it
2666 // when it has also acquired the tokenizer mutex and we are already
2667 // holding the tokenizer mutex.
2668 if (speculationFailed
) {
2669 MOZ_ASSERT(mMode
== NORMAL
);
2670 // Rewind the stream
2672 const auto& speculation
= mSpeculations
.ElementAt(0);
2673 mFirstBuffer
= speculation
->GetBuffer();
2674 mFirstBuffer
->setStart(speculation
->GetStart());
2675 mTokenizer
->setLineNumber(speculation
->GetStartLineNumber());
2676 mTokenizer
->setColumnNumberAndResetNextLine(
2677 speculation
->GetStartColumnNumber());
2679 nsContentUtils::ReportToConsole(
2680 nsIScriptError::warningFlag
, "DOM Events"_ns
,
2681 mExecutor
->GetDocument(), nsContentUtils::eDOM_PROPERTIES
,
2682 "SpeculationFailed2", nsTArray
<nsString
>(), nullptr, u
""_ns
,
2683 speculation
->GetStartLineNumber(),
2684 speculation
->GetStartColumnNumber());
2686 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBuffer
->next
;
2688 buffer
->setStart(0);
2689 buffer
= buffer
->next
;
2692 mSpeculations
.Clear(); // potentially a huge number of destructors
2693 // run here synchronously on the main thread...
2695 mTreeBuilder
->flushCharacters(); // empty the pending buffer
2696 mTreeBuilder
->ClearOps(); // now get rid of the failed ops
2698 mTreeBuilder
->SetOpSink(mExecutor
->GetStage());
2699 mExecutor
->StartReadingFromStage();
2700 mSpeculating
= false;
2703 mLastWasCR
= aLastWasCR
;
2704 mTokenizer
->loadState(aTokenizer
);
2705 mTreeBuilder
->loadState(aTreeBuilder
);
2707 // We've got a successful speculation and at least a moment ago it was
2708 // the current speculation
2709 if (!mSpeculations
.ElementAt(0)->FlushToSink(mExecutor
)) {
2710 mExecutor
->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2713 MOZ_ASSERT(!mExecutor
->IsScriptExecuting(),
2714 "ParseUntilBlocked() was supposed to ensure we don't come "
2715 "here when scripts are executing.");
2716 MOZ_ASSERT(!aTokenizer
|| mExecutor
->IsInFlushLoop(),
2717 "How are we here if "
2718 "RunFlushLoop() didn't call ParseUntilBlocked() or we're not "
2719 "committing to an encoding?");
2720 mSpeculations
.RemoveElementAt(0);
2721 if (mSpeculations
.IsEmpty()) {
2722 if (mMode
== VIEW_SOURCE_HTML
) {
2723 // If we looked for meta charset in the HTML View Source case.
2724 mTokenizer
->SetViewSourceOpSink(mExecutor
->GetStage());
2726 // yes, it was still the only speculation. Now stop speculating
2727 // However, before telling the executor to read from stage, flush
2728 // any pending ops straight to the executor, because otherwise
2729 // they remain unflushed until we get more data from the network.
2730 mTreeBuilder
->SetOpSink(mExecutor
);
2731 auto r
= mTreeBuilder
->Flush(true);
2733 mExecutor
->MarkAsBroken(r
.unwrapErr());
2736 mTreeBuilder
->SetOpSink(mExecutor
->GetStage());
2738 mExecutor
->StartReadingFromStage();
2739 mSpeculating
= false;
2742 nsCOMPtr
<nsIRunnable
> event
= new nsHtml5StreamParserContinuation(this);
2743 if (NS_FAILED(mEventTarget
->Dispatch(event
, nsIThread::DISPATCH_NORMAL
))) {
2744 NS_WARNING("Failed to dispatch nsHtml5StreamParserContinuation");
2746 // A stream event might run before this event runs, but that's harmless.
2748 mAtomTable
.SetPermittedLookupEventTarget(mEventTarget
);
2753 void nsHtml5StreamParser::ContinueAfterFailedCharsetSwitch() {
2754 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
2755 nsCOMPtr
<nsIRunnable
> event
= new nsHtml5StreamParserContinuation(this);
2756 if (NS_FAILED(mEventTarget
->Dispatch(event
, nsIThread::DISPATCH_NORMAL
))) {
2757 NS_WARNING("Failed to dispatch nsHtml5StreamParserContinuation");
2761 class nsHtml5TimerKungFu
: public Runnable
{
2763 nsHtml5StreamParserPtr mStreamParser
;
2766 explicit nsHtml5TimerKungFu(nsHtml5StreamParser
* aStreamParser
)
2767 : Runnable("nsHtml5TimerKungFu"), mStreamParser(aStreamParser
) {}
2768 NS_IMETHOD
Run() override
{
2769 mozilla::MutexAutoLock
flushTimerLock(mStreamParser
->mFlushTimerMutex
);
2770 if (mStreamParser
->mFlushTimer
) {
2771 mStreamParser
->mFlushTimer
->Cancel();
2772 mStreamParser
->mFlushTimer
= nullptr;
2778 void nsHtml5StreamParser::DropTimer() {
2779 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
2781 * Simply nulling out the timer wouldn't work, because if the timer is
2782 * armed, it needs to be canceled first. Simply canceling it first wouldn't
2783 * work, because nsTimerImpl::Cancel is not safe for calling from outside
2784 * the thread where nsTimerImpl::Fire would run. It's not safe to
2785 * dispatch a runnable to cancel the timer from the destructor of this
2786 * class, because the timer has a weak (void*) pointer back to this instance
2787 * of the stream parser and having the timer fire before the runnable
2788 * cancels it would make the timer access a deleted object.
2790 * This DropTimer method addresses these issues. This method must be called
2791 * on the main thread before the destructor of this class is reached.
2792 * The nsHtml5TimerKungFu object has an nsHtml5StreamParserPtr that addrefs
2794 * stream parser object to keep it alive until the runnable is done.
2795 * The runnable cancels the timer on the parser thread, drops the timer
2796 * and lets nsHtml5StreamParserPtr send a runnable back to the main thread to
2797 * release the stream parser.
2799 mozilla::MutexAutoLock
flushTimerLock(mFlushTimerMutex
);
2801 nsCOMPtr
<nsIRunnable
> event
= new nsHtml5TimerKungFu(this);
2802 if (NS_FAILED(mEventTarget
->Dispatch(event
, nsIThread::DISPATCH_NORMAL
))) {
2803 NS_WARNING("Failed to dispatch TimerKungFu event");
2808 // Using a static, because the method name Notify is taken by the chardet
2810 void nsHtml5StreamParser::TimerCallback(nsITimer
* aTimer
, void* aClosure
) {
2811 (static_cast<nsHtml5StreamParser
*>(aClosure
))->TimerFlush();
2814 void nsHtml5StreamParser::TimerFlush() {
2815 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
2816 mozilla::MutexAutoLock
autoLock(mTokenizerMutex
);
2818 MOZ_ASSERT(!mSpeculating
, "Flush timer fired while speculating.");
2820 // The timer fired if we got here. No need to cancel it. Mark it as
2821 // not armed, though.
2822 mFlushTimerArmed
= false;
2824 mFlushTimerEverFired
= true;
2826 if (IsTerminatedOrInterrupted()) {
2830 if (mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
) {
2831 auto r
= mTreeBuilder
->Flush(); // delete useless ops
2833 MarkAsBroken(r
.unwrapErr());
2836 r
= mTokenizer
->FlushViewSource();
2838 MarkAsBroken(r
.unwrapErr());
2842 nsCOMPtr
<nsIRunnable
> runnable(mExecutorFlusher
);
2843 if (NS_FAILED(DispatchToMain(runnable
.forget()))) {
2844 NS_WARNING("failed to dispatch executor flush event");
2848 // we aren't speculating and we don't know when new data is
2849 // going to arrive. Send data to the main thread.
2850 auto r
= mTreeBuilder
->Flush(true);
2852 MarkAsBroken(r
.unwrapErr());
2856 nsCOMPtr
<nsIRunnable
> runnable(mExecutorFlusher
);
2857 if (NS_FAILED(DispatchToMain(runnable
.forget()))) {
2858 NS_WARNING("failed to dispatch executor flush event");
2864 void nsHtml5StreamParser::MarkAsBroken(nsresult aRv
) {
2865 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
2866 mTokenizerMutex
.AssertCurrentThreadOwns();
2869 mTreeBuilder
->MarkAsBroken(aRv
);
2870 auto r
= mTreeBuilder
->Flush(false);
2872 MOZ_ASSERT(r
.unwrap(), "Should have had the markAsBroken op!");
2874 MOZ_CRASH("OOM prevents propagation of OOM state");
2876 nsCOMPtr
<nsIRunnable
> runnable(mExecutorFlusher
);
2877 if (NS_FAILED(DispatchToMain(runnable
.forget()))) {
2878 NS_WARNING("failed to dispatch executor flush event");
2882 nsresult
nsHtml5StreamParser::DispatchToMain(
2883 already_AddRefed
<nsIRunnable
>&& aRunnable
) {
2884 return SchedulerGroup::Dispatch(std::move(aRunnable
));