1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set sw=2 ts=2 et tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "nsHtml5StreamParser.h"
13 #include <type_traits>
15 #include "ErrorList.h"
16 #include "GeckoProfiler.h"
18 #include "mozilla/ArrayIterator.h"
19 #include "mozilla/Buffer.h"
20 #include "mozilla/CheckedInt.h"
21 #include "mozilla/DebugOnly.h"
22 #include "mozilla/Encoding.h"
23 #include "mozilla/EncodingDetector.h"
24 #include "mozilla/Likely.h"
25 #include "mozilla/Maybe.h"
26 #include "mozilla/SchedulerGroup.h"
27 #include "mozilla/ScopeExit.h"
28 #include "mozilla/Services.h"
29 #include "mozilla/StaticPrefs_html5.h"
30 #include "mozilla/StaticPrefs_intl.h"
31 #include "mozilla/StaticPrefs_network.h"
32 #include "mozilla/TextUtils.h"
34 #include "mozilla/UniquePtrExtensions.h"
35 #include "mozilla/Unused.h"
36 #include "mozilla/dom/BindingDeclarations.h"
37 #include "mozilla/dom/BrowsingContext.h"
38 #include "mozilla/dom/DebuggerUtilsBinding.h"
39 #include "mozilla/dom/DocGroup.h"
40 #include "mozilla/dom/Document.h"
41 #include "mozilla/mozalloc.h"
42 #include "mozilla/Vector.h"
43 #include "nsContentSink.h"
44 #include "nsContentUtils.h"
45 #include "nsCycleCollectionTraversalCallback.h"
46 #include "nsHtml5AtomTable.h"
47 #include "nsHtml5ByteReadable.h"
48 #include "nsHtml5Highlighter.h"
49 #include "nsHtml5Module.h"
50 #include "nsHtml5OwningUTF16Buffer.h"
51 #include "nsHtml5Parser.h"
52 #include "nsHtml5Speculation.h"
53 #include "nsHtml5StreamParserPtr.h"
54 #include "nsHtml5Tokenizer.h"
55 #include "nsHtml5TreeBuilder.h"
56 #include "nsHtml5TreeOpExecutor.h"
57 #include "nsHtml5TreeOpStage.h"
58 #include "nsIChannel.h"
59 #include "nsIContentSink.h"
62 #include "nsIDocShell.h"
63 #include "nsIEventTarget.h"
64 #include "nsIHttpChannel.h"
65 #include "nsIInputStream.h"
66 #include "nsINestedURI.h"
67 #include "nsIObserverService.h"
68 #include "nsIRequest.h"
69 #include "nsIRunnable.h"
70 #include "nsIScriptError.h"
71 #include "nsIThread.h"
72 #include "nsIThreadRetargetableRequest.h"
73 #include "nsIThreadRetargetableStreamListener.h"
76 #include "nsJSEnvironment.h"
77 #include "nsLiteralString.h"
78 #include "nsNetUtil.h"
80 #include "nsTPromiseFlatString.h"
81 #include "nsThreadUtils.h"
82 #include "nsXULAppAPI.h"
85 // Defined in intl/encoding_glue/src/lib.rs
86 const mozilla::Encoding
* xmldecl_parse(const uint8_t* buf
, size_t buf_len
);
89 using namespace mozilla
;
90 using namespace mozilla::dom
;
93 * Note that nsHtml5StreamParser implements cycle collecting AddRef and
94 * Release. Therefore, nsHtml5StreamParser must never be refcounted from
97 * To work around this limitation, runnables posted by the main thread to the
98 * parser thread hold their reference to the stream parser in an
99 * nsHtml5StreamParserPtr. Upon creation, nsHtml5StreamParserPtr addrefs the
101 * just like a regular nsRefPtr. This is OK, since the creation of the
102 * runnable and the nsHtml5StreamParserPtr happens on the main thread.
104 * When the runnable is done on the parser thread, the destructor of
105 * nsHtml5StreamParserPtr runs there. It doesn't call Release on the held object
106 * directly. Instead, it posts another runnable back to the main thread where
107 * that runnable calls Release on the wrapped object.
109 * When posting runnables in the other direction, the runnables have to be
110 * created on the main thread when nsHtml5StreamParser is instantiated and
111 * held for the lifetime of the nsHtml5StreamParser. This works, because the
112 * same runnabled can be dispatched multiple times and currently runnables
113 * posted from the parser thread to main thread don't need to wrap any
114 * runnable-specific data. (In the other direction, the runnables most notably
115 * wrap the byte data of the stream.)
117 NS_IMPL_CYCLE_COLLECTING_ADDREF(nsHtml5StreamParser
)
118 NS_IMPL_CYCLE_COLLECTING_RELEASE(nsHtml5StreamParser
)
120 NS_INTERFACE_TABLE_HEAD(nsHtml5StreamParser
)
121 NS_INTERFACE_TABLE(nsHtml5StreamParser
, nsISupports
)
122 NS_INTERFACE_TABLE_TO_MAP_SEGUE_CYCLE_COLLECTION(nsHtml5StreamParser
)
125 NS_IMPL_CYCLE_COLLECTION_CLASS(nsHtml5StreamParser
)
127 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(nsHtml5StreamParser
)
129 NS_IMPL_CYCLE_COLLECTION_UNLINK(mRequest
)
130 NS_IMPL_CYCLE_COLLECTION_UNLINK(mOwner
)
131 tmp
->mExecutorFlusher
= nullptr;
132 tmp
->mLoadFlusher
= nullptr;
133 tmp
->mExecutor
= nullptr;
134 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
136 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(nsHtml5StreamParser
)
137 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mRequest
)
138 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOwner
)
139 // hack: count the strongly owned edge wrapped in the runnable
140 if (tmp
->mExecutorFlusher
) {
141 NS_CYCLE_COLLECTION_NOTE_EDGE_NAME(cb
, "mExecutorFlusher->mExecutor");
142 cb
.NoteXPCOMChild(static_cast<nsIContentSink
*>(tmp
->mExecutor
));
144 // hack: count the strongly owned edge wrapped in the runnable
145 if (tmp
->mLoadFlusher
) {
146 NS_CYCLE_COLLECTION_NOTE_EDGE_NAME(cb
, "mLoadFlusher->mExecutor");
147 cb
.NoteXPCOMChild(static_cast<nsIContentSink
*>(tmp
->mExecutor
));
149 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
151 class nsHtml5ExecutorFlusher
: public Runnable
{
153 RefPtr
<nsHtml5TreeOpExecutor
> mExecutor
;
156 explicit nsHtml5ExecutorFlusher(nsHtml5TreeOpExecutor
* aExecutor
)
157 : Runnable("nsHtml5ExecutorFlusher"), mExecutor(aExecutor
) {}
158 NS_IMETHOD
Run() override
{
159 if (!mExecutor
->isInList()) {
160 Document
* doc
= mExecutor
->GetDocument();
161 if (XRE_IsContentProcess() &&
163 HighPriorityEventPendingForTopLevelDocumentBeforeContentfulPaint(
165 // Possible early paint pending, reuse the runnable and try to
166 // call RunFlushLoop later.
167 nsCOMPtr
<nsIRunnable
> flusher
= this;
168 if (NS_SUCCEEDED(doc
->Dispatch(flusher
.forget()))) {
169 PROFILER_MARKER_UNTYPED("HighPrio blocking parser flushing(1)", DOM
);
173 mExecutor
->RunFlushLoop();
179 class nsHtml5LoadFlusher
: public Runnable
{
181 RefPtr
<nsHtml5TreeOpExecutor
> mExecutor
;
184 explicit nsHtml5LoadFlusher(nsHtml5TreeOpExecutor
* aExecutor
)
185 : Runnable("nsHtml5LoadFlusher"), mExecutor(aExecutor
) {}
186 NS_IMETHOD
Run() override
{
187 mExecutor
->FlushSpeculativeLoads();
192 nsHtml5StreamParser::nsHtml5StreamParser(nsHtml5TreeOpExecutor
* aExecutor
,
193 nsHtml5Parser
* aOwner
,
195 : mBomState(eBomState::BOM_SNIFFING_NOT_STARTED
),
196 mCharsetSource(kCharsetUninitialized
),
197 mEncodingSwitchSource(kCharsetUninitialized
),
198 mEncoding(X_USER_DEFINED_ENCODING
), // Obviously bogus value to notice if
200 mNeedsEncodingSwitchTo(nullptr),
201 mSeenEligibleMetaCharset(false),
204 mStartedFeedingDetector(false),
205 mStartedFeedingDevTools(false),
207 mReparseForbidden(false),
208 mForceAutoDetection(false),
209 mChannelHadCharset(false),
210 mLookingForMetaCharset(false),
211 mStartsWithLtQuestion(false),
212 mLookingForXmlDeclarationForXmlViewSource(false),
213 mTemplatePushedOrHeadPopped(false),
216 mLastBuffer(nullptr), // Will be filled when starting
217 mExecutor(aExecutor
),
218 mTreeBuilder(new nsHtml5TreeBuilder(
219 (aMode
== VIEW_SOURCE_HTML
|| aMode
== VIEW_SOURCE_XML
)
221 : mExecutor
->GetStage(),
222 mExecutor
->GetStage(), aMode
== NORMAL
)),
224 new nsHtml5Tokenizer(mTreeBuilder
.get(), aMode
== VIEW_SOURCE_XML
)),
225 mTokenizerMutex("nsHtml5StreamParser mTokenizerMutex"),
228 mStreamState(eHtml5StreamState::STREAM_NOT_STARTED
),
231 mSpeculationMutex("nsHtml5StreamParser mSpeculationMutex"),
232 mSpeculationFailureCount(0),
233 mNumBytesBuffered(0),
236 mEventTarget(nsHtml5Module::GetStreamParserEventTarget()),
237 mExecutorFlusher(new nsHtml5ExecutorFlusher(aExecutor
)),
238 mLoadFlusher(new nsHtml5LoadFlusher(aExecutor
)),
239 mInitialEncodingWasFromParentFrame(false),
240 mHasHadErrors(false),
241 mDetectorHasSeenNonAscii(false),
242 mDecodingLocalFileWithoutTokenizing(false),
243 mBufferingBytes(false),
244 mFlushTimer(NS_NewTimer(mEventTarget
)),
245 mFlushTimerMutex("nsHtml5StreamParser mFlushTimerMutex"),
246 mFlushTimerArmed(false),
247 mFlushTimerEverFired(false),
249 NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
251 mAtomTable
.SetPermittedLookupEventTarget(mEventTarget
);
253 mTokenizer
->setInterner(&mAtomTable
);
254 mTokenizer
->setEncodingDeclarationHandler(this);
256 if (aMode
== VIEW_SOURCE_HTML
|| aMode
== VIEW_SOURCE_XML
) {
257 nsHtml5Highlighter
* highlighter
=
258 new nsHtml5Highlighter(mExecutor
->GetStage());
259 mTokenizer
->EnableViewSource(highlighter
); // takes ownership
260 mTreeBuilder
->EnableViewSource(highlighter
); // doesn't own
263 // There's a zeroing operator new for everything else
266 nsHtml5StreamParser::~nsHtml5StreamParser() {
267 NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
271 mozilla::MutexAutoLock
flushTimerLock(mFlushTimerMutex
);
272 MOZ_ASSERT(!mFlushTimer
, "Flush timer was not dropped before dtor!");
275 mUnicodeDecoder
= nullptr;
276 mFirstBuffer
= nullptr;
278 mTreeBuilder
= nullptr;
279 mTokenizer
= nullptr;
284 nsresult
nsHtml5StreamParser::GetChannel(nsIChannel
** aChannel
) {
285 NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
286 return mRequest
? CallQueryInterface(mRequest
, aChannel
)
287 : NS_ERROR_NOT_AVAILABLE
;
290 std::tuple
<NotNull
<const Encoding
*>, nsCharsetSource
>
291 nsHtml5StreamParser::GuessEncoding(bool aInitial
) {
293 mCharsetSource
!= kCharsetFromFinalUserForcedAutoDetection
&&
295 kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII
&&
297 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Generic
&&
299 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8GenericInitialWasASCII
&&
301 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Content
&&
303 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8ContentInitialWasASCII
&&
305 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD
&&
307 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLDInitialWasASCII
&&
308 mCharsetSource
!= kCharsetFromFinalAutoDetectionFile
);
309 auto ifHadBeenForced
= mDetector
->Guess(EmptyCString(), true);
313 : mDetector
->Guess(mTLD
, mDecodingLocalFileWithoutTokenizing
);
314 nsCharsetSource source
=
316 ? (mForceAutoDetection
317 ? kCharsetFromInitialUserForcedAutoDetection
318 : (mDecodingLocalFileWithoutTokenizing
319 ? kCharsetFromFinalAutoDetectionFile
320 : kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Generic
))
321 : (mForceAutoDetection
322 ? kCharsetFromFinalUserForcedAutoDetection
323 : (mDecodingLocalFileWithoutTokenizing
324 ? kCharsetFromFinalAutoDetectionFile
325 : kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Generic
));
326 if (source
== kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Generic
) {
327 if (encoding
== ISO_2022_JP_ENCODING
) {
328 if (EncodingDetector::TldMayAffectGuess(mTLD
)) {
329 source
= kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Content
;
331 } else if (!mDetectorHasSeenNonAscii
) {
332 source
= kCharsetFromInitialAutoDetectionASCII
; // deliberately Initial
333 } else if (ifHadBeenForced
== UTF_8_ENCODING
) {
334 MOZ_ASSERT(mCharsetSource
== kCharsetFromInitialAutoDetectionASCII
||
336 kCharsetFromInitialAutoDetectionWouldHaveBeenUTF8
||
337 mEncoding
== ISO_2022_JP_ENCODING
);
338 source
= kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII
;
339 } else if (encoding
!= ifHadBeenForced
) {
340 if (mCharsetSource
== kCharsetFromInitialAutoDetectionASCII
) {
342 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLDInitialWasASCII
;
345 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD
;
347 } else if (EncodingDetector::TldMayAffectGuess(mTLD
)) {
348 if (mCharsetSource
== kCharsetFromInitialAutoDetectionASCII
) {
350 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8ContentInitialWasASCII
;
352 source
= kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Content
;
354 } else if (mCharsetSource
== kCharsetFromInitialAutoDetectionASCII
) {
356 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8GenericInitialWasASCII
;
359 kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Generic
) {
360 if (encoding
== ISO_2022_JP_ENCODING
) {
361 if (EncodingDetector::TldMayAffectGuess(mTLD
)) {
362 source
= kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Content
;
364 } else if (!mDetectorHasSeenNonAscii
) {
365 source
= kCharsetFromInitialAutoDetectionASCII
;
366 } else if (ifHadBeenForced
== UTF_8_ENCODING
) {
367 source
= kCharsetFromInitialAutoDetectionWouldHaveBeenUTF8
;
368 } else if (encoding
!= ifHadBeenForced
) {
370 kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD
;
371 } else if (EncodingDetector::TldMayAffectGuess(mTLD
)) {
372 source
= kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Content
;
375 return {encoding
, source
};
378 void nsHtml5StreamParser::FeedDetector(Span
<const uint8_t> aBuffer
) {
380 mStartedFeedingDetector
= true;
382 MOZ_ASSERT(!mChardetEof
);
383 mDetectorHasSeenNonAscii
= mDetector
->Feed(aBuffer
, false);
386 void nsHtml5StreamParser::DetectorEof() {
388 mStartedFeedingDetector
= true;
394 mDetectorHasSeenNonAscii
= mDetector
->Feed(Span
<const uint8_t>(), true);
397 void nsHtml5StreamParser::SetViewSourceTitle(nsIURI
* aURL
) {
398 MOZ_ASSERT(NS_IsMainThread());
400 BrowsingContext
* browsingContext
=
401 mExecutor
->GetDocument()->GetBrowsingContext();
402 if (browsingContext
&& browsingContext
->WatchedByDevTools()) {
403 mURIToSendToDevtools
= aURL
;
406 nsresult rv
= nsID::GenerateUUIDInPlace(uuid
);
407 if (!NS_FAILED(rv
)) {
408 char buffer
[NSID_LENGTH
];
409 uuid
.ToProvidedString(buffer
);
410 mUUIDForDevtools
= NS_ConvertASCIItoUTF16(buffer
);
415 nsCOMPtr
<nsIURI
> temp
;
416 if (aURL
->SchemeIs("view-source")) {
417 nsCOMPtr
<nsINestedURI
> nested
= do_QueryInterface(aURL
);
418 nested
->GetInnerURI(getter_AddRefs(temp
));
422 if (temp
->SchemeIs("data")) {
423 // Avoid showing potentially huge data: URLs. The three last bytes are
424 // UTF-8 for an ellipsis.
425 mViewSourceTitle
.AssignLiteral("data:\xE2\x80\xA6");
427 nsresult rv
= temp
->GetSpec(mViewSourceTitle
);
429 mViewSourceTitle
.AssignLiteral("\xE2\x80\xA6");
436 nsHtml5StreamParser::SetupDecodingAndWriteSniffingBufferAndCurrentSegment(
437 Span
<const uint8_t> aPrefix
, Span
<const uint8_t> aFromSegment
) {
438 NS_ASSERTION(IsParserThread(), "Wrong thread!");
439 mUnicodeDecoder
= mEncoding
->NewDecoderWithBOMRemoval();
440 nsresult rv
= WriteStreamBytes(aPrefix
);
441 NS_ENSURE_SUCCESS(rv
, rv
);
442 return WriteStreamBytes(aFromSegment
);
445 void nsHtml5StreamParser::SetupDecodingFromBom(
446 NotNull
<const Encoding
*> aEncoding
) {
447 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
448 mEncoding
= aEncoding
;
449 mDecodingLocalFileWithoutTokenizing
= false;
450 mLookingForMetaCharset
= false;
451 mBufferingBytes
= false;
452 mUnicodeDecoder
= mEncoding
->NewDecoderWithoutBOMHandling();
453 mCharsetSource
= kCharsetFromByteOrderMark
;
454 mForceAutoDetection
= false;
455 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
456 mBomState
= BOM_SNIFFING_OVER
;
457 if (mMode
== VIEW_SOURCE_HTML
) {
458 mTokenizer
->StartViewSourceCharacters();
462 void nsHtml5StreamParser::SetupDecodingFromUtf16BogoXml(
463 NotNull
<const Encoding
*> aEncoding
) {
464 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
465 mEncoding
= aEncoding
;
466 mDecodingLocalFileWithoutTokenizing
= false;
467 mLookingForMetaCharset
= false;
468 mBufferingBytes
= false;
469 mUnicodeDecoder
= mEncoding
->NewDecoderWithoutBOMHandling();
470 mCharsetSource
= kCharsetFromXmlDeclarationUtf16
;
471 mForceAutoDetection
= false;
472 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
473 mBomState
= BOM_SNIFFING_OVER
;
474 if (mMode
== VIEW_SOURCE_HTML
) {
475 mTokenizer
->StartViewSourceCharacters();
477 auto dst
= mLastBuffer
->TailAsSpan(READ_BUFFER_SIZE
);
481 mLastBuffer
->AdvanceEnd(3);
482 MOZ_ASSERT(!mStartedFeedingDevTools
);
483 OnNewContent(dst
.To(3));
486 size_t nsHtml5StreamParser::LengthOfLtContainingPrefixInSecondBuffer() {
487 MOZ_ASSERT(mBufferedBytes
.Length() <= 2);
488 if (mBufferedBytes
.Length() < 2) {
491 Buffer
<uint8_t>& second
= mBufferedBytes
[1];
492 const uint8_t* elements
= second
.Elements();
493 const uint8_t* lt
= (const uint8_t*)memchr(elements
, '>', second
.Length());
495 return (lt
- elements
) + 1;
500 nsresult
nsHtml5StreamParser::SniffStreamBytes(Span
<const uint8_t> aFromSegment
,
502 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
503 MOZ_ASSERT_IF(aEof
, aFromSegment
.IsEmpty());
505 if (mCharsetSource
>=
506 kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII
&&
507 mCharsetSource
<= kCharsetFromFinalUserForcedAutoDetection
) {
508 if (mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
) {
509 mTreeBuilder
->MaybeComplainAboutCharset("EncDetectorReloadPlain", true,
512 mTreeBuilder
->MaybeComplainAboutCharset("EncDetectorReload", true, 0);
516 // mEncoding and mCharsetSource potentially have come from channel or higher
517 // by now. If we find a BOM, SetupDecodingFromBom() will overwrite them.
518 // If we don't find a BOM, the previously set values of mEncoding and
519 // mCharsetSource are not modified by the BOM sniffing here.
520 static uint8_t utf8
[] = {0xEF, 0xBB};
521 static uint8_t utf16le
[] = {0xFF};
522 static uint8_t utf16be
[] = {0xFE};
523 static uint8_t utf16leXml
[] = {'<', 0x00, '?', 0x00, 'x'};
524 static uint8_t utf16beXml
[] = {0x00, '<', 0x00, '?', 0x00};
525 // Buffer for replaying past bytes based on state machine state. If
526 // writing this from scratch, probably wouldn't do it this way, but
527 // let's keep the changes to a minimum.
528 const uint8_t* prefix
= utf8
;
529 size_t prefixLength
= 0;
530 if (aEof
&& mBomState
== BOM_SNIFFING_NOT_STARTED
) {
531 // Avoid handling aEof in the BOM_SNIFFING_NOT_STARTED state below.
532 mBomState
= BOM_SNIFFING_OVER
;
535 (i
< aFromSegment
.Length() && mBomState
!= BOM_SNIFFING_OVER
) || aEof
;
538 case BOM_SNIFFING_NOT_STARTED
:
539 MOZ_ASSERT(i
== 0, "Bad BOM sniffing state.");
540 MOZ_ASSERT(!aEof
, "Should have checked for aEof above!");
541 switch (aFromSegment
[0]) {
543 mBomState
= SEEN_UTF_8_FIRST_BYTE
;
546 mBomState
= SEEN_UTF_16_LE_FIRST_BYTE
;
549 mBomState
= SEEN_UTF_16_BE_FIRST_BYTE
;
552 if (mCharsetSource
< kCharsetFromXmlDeclarationUtf16
&&
553 mCharsetSource
!= kCharsetFromChannel
) {
554 mBomState
= SEEN_UTF_16_BE_XML_FIRST
;
556 mBomState
= BOM_SNIFFING_OVER
;
560 if (mCharsetSource
< kCharsetFromXmlDeclarationUtf16
&&
561 mCharsetSource
!= kCharsetFromChannel
) {
562 mBomState
= SEEN_UTF_16_LE_XML_FIRST
;
564 mBomState
= BOM_SNIFFING_OVER
;
568 mBomState
= BOM_SNIFFING_OVER
;
572 case SEEN_UTF_16_LE_FIRST_BYTE
:
573 if (!aEof
&& aFromSegment
[i
] == 0xFE) {
574 SetupDecodingFromBom(UTF_16LE_ENCODING
);
575 return WriteStreamBytes(aFromSegment
.From(i
+ 1));
578 prefixLength
= 1 - i
;
579 mBomState
= BOM_SNIFFING_OVER
;
581 case SEEN_UTF_16_BE_FIRST_BYTE
:
582 if (!aEof
&& aFromSegment
[i
] == 0xFF) {
583 SetupDecodingFromBom(UTF_16BE_ENCODING
);
584 return WriteStreamBytes(aFromSegment
.From(i
+ 1));
587 prefixLength
= 1 - i
;
588 mBomState
= BOM_SNIFFING_OVER
;
590 case SEEN_UTF_8_FIRST_BYTE
:
591 if (!aEof
&& aFromSegment
[i
] == 0xBB) {
592 mBomState
= SEEN_UTF_8_SECOND_BYTE
;
594 prefixLength
= 1 - i
;
595 mBomState
= BOM_SNIFFING_OVER
;
598 case SEEN_UTF_8_SECOND_BYTE
:
599 if (!aEof
&& aFromSegment
[i
] == 0xBF) {
600 SetupDecodingFromBom(UTF_8_ENCODING
);
601 return WriteStreamBytes(aFromSegment
.From(i
+ 1));
603 prefixLength
= 2 - i
;
604 mBomState
= BOM_SNIFFING_OVER
;
606 case SEEN_UTF_16_BE_XML_FIRST
:
607 if (!aEof
&& aFromSegment
[i
] == '<') {
608 mBomState
= SEEN_UTF_16_BE_XML_SECOND
;
611 prefixLength
= 1 - i
;
612 mBomState
= BOM_SNIFFING_OVER
;
615 case SEEN_UTF_16_BE_XML_SECOND
:
616 if (!aEof
&& aFromSegment
[i
] == 0x00) {
617 mBomState
= SEEN_UTF_16_BE_XML_THIRD
;
620 prefixLength
= 2 - i
;
621 mBomState
= BOM_SNIFFING_OVER
;
624 case SEEN_UTF_16_BE_XML_THIRD
:
625 if (!aEof
&& aFromSegment
[i
] == '?') {
626 mBomState
= SEEN_UTF_16_BE_XML_FOURTH
;
629 prefixLength
= 3 - i
;
630 mBomState
= BOM_SNIFFING_OVER
;
633 case SEEN_UTF_16_BE_XML_FOURTH
:
634 if (!aEof
&& aFromSegment
[i
] == 0x00) {
635 mBomState
= SEEN_UTF_16_BE_XML_FIFTH
;
638 prefixLength
= 4 - i
;
639 mBomState
= BOM_SNIFFING_OVER
;
642 case SEEN_UTF_16_BE_XML_FIFTH
:
643 if (!aEof
&& aFromSegment
[i
] == 'x') {
644 SetupDecodingFromUtf16BogoXml(UTF_16BE_ENCODING
);
645 return WriteStreamBytes(aFromSegment
.From(i
+ 1));
648 prefixLength
= 5 - i
;
649 mBomState
= BOM_SNIFFING_OVER
;
651 case SEEN_UTF_16_LE_XML_FIRST
:
652 if (!aEof
&& aFromSegment
[i
] == 0x00) {
653 mBomState
= SEEN_UTF_16_LE_XML_SECOND
;
655 if (!aEof
&& aFromSegment
[i
] == '?' &&
656 !(mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
)) {
657 mStartsWithLtQuestion
= true;
660 prefixLength
= 1 - i
;
661 mBomState
= BOM_SNIFFING_OVER
;
664 case SEEN_UTF_16_LE_XML_SECOND
:
665 if (!aEof
&& aFromSegment
[i
] == '?') {
666 mBomState
= SEEN_UTF_16_LE_XML_THIRD
;
669 prefixLength
= 2 - i
;
670 mBomState
= BOM_SNIFFING_OVER
;
673 case SEEN_UTF_16_LE_XML_THIRD
:
674 if (!aEof
&& aFromSegment
[i
] == 0x00) {
675 mBomState
= SEEN_UTF_16_LE_XML_FOURTH
;
678 prefixLength
= 3 - i
;
679 mBomState
= BOM_SNIFFING_OVER
;
682 case SEEN_UTF_16_LE_XML_FOURTH
:
683 if (!aEof
&& aFromSegment
[i
] == 'x') {
684 mBomState
= SEEN_UTF_16_LE_XML_FIFTH
;
687 prefixLength
= 4 - i
;
688 mBomState
= BOM_SNIFFING_OVER
;
691 case SEEN_UTF_16_LE_XML_FIFTH
:
692 if (!aEof
&& aFromSegment
[i
] == 0x00) {
693 SetupDecodingFromUtf16BogoXml(UTF_16LE_ENCODING
);
694 return WriteStreamBytes(aFromSegment
.From(i
+ 1));
697 prefixLength
= 5 - i
;
698 mBomState
= BOM_SNIFFING_OVER
;
701 mBomState
= BOM_SNIFFING_OVER
;
708 // if we get here, there either was no BOM or the BOM sniffing isn't complete
711 MOZ_ASSERT(mCharsetSource
!= kCharsetFromByteOrderMark
,
712 "Should not come here if BOM was found.");
713 MOZ_ASSERT(mCharsetSource
!= kCharsetFromXmlDeclarationUtf16
,
714 "Should not come here if UTF-16 bogo-XML declaration was found.");
715 MOZ_ASSERT(mCharsetSource
!= kCharsetFromOtherComponent
,
716 "kCharsetFromOtherComponent is for XSLT.");
718 if (mBomState
== BOM_SNIFFING_OVER
) {
719 if (mMode
== VIEW_SOURCE_XML
&& mStartsWithLtQuestion
&&
720 mCharsetSource
< kCharsetFromChannel
) {
721 // Sniff for XML declaration only.
722 MOZ_ASSERT(!mLookingForXmlDeclarationForXmlViewSource
);
724 MOZ_ASSERT(!mLookingForMetaCharset
);
725 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
);
726 // Maybe we've already buffered a '>'.
727 MOZ_ASSERT(!mBufferedBytes
.IsEmpty(),
728 "How did at least <? not get buffered?");
729 Buffer
<uint8_t>& first
= mBufferedBytes
[0];
730 const Encoding
* encoding
=
731 xmldecl_parse(first
.Elements(), first
.Length());
733 mEncoding
= WrapNotNull(encoding
);
734 mCharsetSource
= kCharsetFromXmlDeclaration
;
735 } else if (memchr(first
.Elements(), '>', first
.Length())) {
736 // There was a '>', but an encoding still wasn't found.
737 ; // fall through to commit to the UTF-8 default.
738 } else if (size_t lengthOfPrefix
=
739 LengthOfLtContainingPrefixInSecondBuffer()) {
740 // This can only happen if the first buffer was a lone '<', because
741 // we come here upon seeing the second byte '?' if the first two bytes
742 // were "<?". That is, the only way how we aren't dealing with the first
743 // buffer is if the first buffer only contained a single '<' and we are
744 // dealing with the second buffer that starts with '?'.
745 MOZ_ASSERT(first
.Length() == 1);
746 MOZ_ASSERT(mBufferedBytes
[1][0] == '?');
747 // Our scanner for XML declaration-like syntax wants to see a contiguous
748 // buffer, so let's linearize the data. (Ideally, the XML declaration
749 // scanner would be incremental, but this is the rare path anyway.)
750 Vector
<uint8_t> contiguous
;
751 if (!contiguous
.append(first
.Elements(), first
.Length())) {
752 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
753 return NS_ERROR_OUT_OF_MEMORY
;
755 if (!contiguous
.append(mBufferedBytes
[1].Elements(), lengthOfPrefix
)) {
756 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
757 return NS_ERROR_OUT_OF_MEMORY
;
759 encoding
= xmldecl_parse(contiguous
.begin(), contiguous
.length());
761 mEncoding
= WrapNotNull(encoding
);
762 mCharsetSource
= kCharsetFromXmlDeclaration
;
764 // else no XML decl, commit to the UTF-8 default.
766 MOZ_ASSERT(mBufferingBytes
);
767 mLookingForXmlDeclarationForXmlViewSource
= true;
770 } else if (mMode
!= VIEW_SOURCE_XML
&&
771 (mForceAutoDetection
|| mCharsetSource
< kCharsetFromChannel
)) {
772 // In order to use the buffering logic for meta with mForceAutoDetection,
773 // we set mLookingForMetaCharset but still actually potentially ignore the
775 mFirstBufferOfMetaScan
= mFirstBuffer
;
776 MOZ_ASSERT(mLookingForMetaCharset
);
778 if (mMode
== VIEW_SOURCE_HTML
) {
779 auto r
= mTokenizer
->FlushViewSource();
781 return r
.unwrapErr();
784 auto r
= mTreeBuilder
->Flush();
786 return r
.unwrapErr();
788 // Encoding committer flushes the ops on the main thread.
790 mozilla::MutexAutoLock
speculationAutoLock(mSpeculationMutex
);
791 nsHtml5Speculation
* speculation
= new nsHtml5Speculation(
792 mFirstBuffer
, mFirstBuffer
->getStart(), mTokenizer
->getLineNumber(),
793 mTokenizer
->getColumnNumber(), mTreeBuilder
->newSnapshot());
794 MOZ_ASSERT(!mFlushTimerArmed
, "How did we end up arming the timer?");
795 if (mMode
== VIEW_SOURCE_HTML
) {
796 mTokenizer
->SetViewSourceOpSink(speculation
);
797 mTokenizer
->StartViewSourceCharacters();
799 MOZ_ASSERT(mMode
!= VIEW_SOURCE_XML
);
800 mTreeBuilder
->SetOpSink(speculation
);
802 mSpeculations
.AppendElement(speculation
); // adopts the pointer
805 mLookingForMetaCharset
= false;
806 mBufferingBytes
= false;
807 mDecodingLocalFileWithoutTokenizing
= false;
808 if (mMode
== VIEW_SOURCE_HTML
) {
809 mTokenizer
->StartViewSourceCharacters();
812 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
813 return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(
814 Span(prefix
, prefixLength
), aFromSegment
);
820 class AddContentRunnable
: public Runnable
{
822 AddContentRunnable(const nsAString
& aParserID
, nsIURI
* aURI
,
823 Span
<const char16_t
> aData
, bool aComplete
)
824 : Runnable("AddContent") {
827 mData
.mUri
.Construct(NS_ConvertUTF8toUTF16(spec
));
828 mData
.mParserID
.Construct(aParserID
);
829 mData
.mContents
.Construct(aData
.Elements(), aData
.Length());
830 mData
.mComplete
.Construct(aComplete
);
833 NS_IMETHOD
Run() override
{
835 if (!mData
.ToJSON(json
)) {
836 return NS_ERROR_FAILURE
;
839 nsCOMPtr
<nsIObserverService
> obsService
= services::GetObserverService();
841 obsService
->NotifyObservers(nullptr, "devtools-html-content",
842 PromiseFlatString(json
).get());
851 inline void nsHtml5StreamParser::OnNewContent(Span
<const char16_t
> aData
) {
853 mStartedFeedingDevTools
= true;
855 if (mURIToSendToDevtools
) {
856 if (aData
.IsEmpty()) {
857 // Optimize out the runnable.
860 NS_DispatchToMainThread(new AddContentRunnable(mUUIDForDevtools
,
861 mURIToSendToDevtools
, aData
,
862 /* aComplete */ false));
866 inline void nsHtml5StreamParser::OnContentComplete() {
868 mStartedFeedingDevTools
= true;
870 if (mURIToSendToDevtools
) {
871 NS_DispatchToMainThread(new AddContentRunnable(
872 mUUIDForDevtools
, mURIToSendToDevtools
, Span
<const char16_t
>(),
873 /* aComplete */ true));
874 mURIToSendToDevtools
= nullptr;
878 nsresult
nsHtml5StreamParser::WriteStreamBytes(
879 Span
<const uint8_t> aFromSegment
) {
880 NS_ASSERTION(IsParserThread(), "Wrong thread!");
881 mTokenizerMutex
.AssertCurrentThreadOwns();
882 // mLastBuffer should always point to a buffer of the size
885 NS_WARNING("mLastBuffer should not be null!");
886 MarkAsBroken(NS_ERROR_NULL_POINTER
);
887 return NS_ERROR_NULL_POINTER
;
889 size_t totalRead
= 0;
890 auto src
= aFromSegment
;
892 auto dst
= mLastBuffer
->TailAsSpan(READ_BUFFER_SIZE
);
893 auto [result
, read
, written
, hadErrors
] =
894 mUnicodeDecoder
->DecodeToUTF16(src
, dst
, false);
895 if (!(mLookingForMetaCharset
|| mDecodingLocalFileWithoutTokenizing
)) {
896 OnNewContent(dst
.To(written
));
898 if (hadErrors
&& !mHasHadErrors
) {
899 mHasHadErrors
= true;
900 if (mEncoding
== UTF_8_ENCODING
) {
901 mTreeBuilder
->TryToEnableEncodingMenu();
904 src
= src
.From(read
);
906 mLastBuffer
->AdvanceEnd(written
);
907 if (result
== kOutputFull
) {
908 RefPtr
<nsHtml5OwningUTF16Buffer
> newBuf
=
909 nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE
);
911 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
912 return NS_ERROR_OUT_OF_MEMORY
;
914 mLastBuffer
= (mLastBuffer
->next
= std::move(newBuf
));
916 MOZ_ASSERT(totalRead
== aFromSegment
.Length(),
917 "The Unicode decoder consumed the wrong number of bytes.");
919 if (!mLookingForMetaCharset
&& mDecodingLocalFileWithoutTokenizing
&&
920 mNumBytesBuffered
== LOCAL_FILE_UTF_8_BUFFER_SIZE
) {
921 MOZ_ASSERT(!mStartedFeedingDetector
);
922 for (auto&& buffer
: mBufferedBytes
) {
923 FeedDetector(buffer
);
925 // If the file is exactly LOCAL_FILE_UTF_8_BUFFER_SIZE bytes long
926 // we end up not considering the EOF. That's not fatal, since we
927 // don't consider the EOF if the file is
928 // LOCAL_FILE_UTF_8_BUFFER_SIZE + 1 bytes long.
929 auto [encoding
, source
] = GuessEncoding(true);
930 mCharsetSource
= source
;
931 if (encoding
!= mEncoding
) {
932 mEncoding
= encoding
;
933 nsresult rv
= ReDecodeLocalFile();
938 MOZ_ASSERT(mEncoding
== UTF_8_ENCODING
);
939 nsresult rv
= CommitLocalFileToEncoding();
950 [[nodiscard
]] nsresult
nsHtml5StreamParser::ReDecodeLocalFile() {
951 MOZ_ASSERT(mDecodingLocalFileWithoutTokenizing
&& !mLookingForMetaCharset
);
952 MOZ_ASSERT(mFirstBufferOfMetaScan
);
953 MOZ_ASSERT(mCharsetSource
== kCharsetFromFinalAutoDetectionFile
||
954 (mForceAutoDetection
&&
955 mCharsetSource
== kCharsetFromInitialUserForcedAutoDetection
));
957 DiscardMetaSpeculation();
959 MOZ_ASSERT(mEncoding
!= UTF_8_ENCODING
);
961 mDecodingLocalFileWithoutTokenizing
= false;
963 mEncoding
->NewDecoderWithBOMRemovalInto(*mUnicodeDecoder
);
964 mHasHadErrors
= false;
966 // Throw away previous decoded data
967 mLastBuffer
= mFirstBuffer
;
968 mLastBuffer
->next
= nullptr;
969 mLastBuffer
->setStart(0);
970 mLastBuffer
->setEnd(0);
972 mBufferingBytes
= false;
973 mForceAutoDetection
= false; // To stop feeding the detector
974 mFirstBufferOfMetaScan
= nullptr;
976 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, true);
979 for (auto&& buffer
: mBufferedBytes
) {
980 DoDataAvailable(buffer
);
983 if (mMode
== VIEW_SOURCE_HTML
) {
984 auto r
= mTokenizer
->FlushViewSource();
986 return r
.unwrapErr();
989 auto r
= mTreeBuilder
->Flush();
991 return r
.unwrapErr();
996 [[nodiscard
]] nsresult
nsHtml5StreamParser::CommitLocalFileToEncoding() {
997 MOZ_ASSERT(mDecodingLocalFileWithoutTokenizing
&& !mLookingForMetaCharset
);
998 MOZ_ASSERT(mFirstBufferOfMetaScan
);
999 mDecodingLocalFileWithoutTokenizing
= false;
1000 MOZ_ASSERT(mCharsetSource
== kCharsetFromFinalAutoDetectionFile
||
1001 (mForceAutoDetection
&&
1002 mCharsetSource
== kCharsetFromInitialUserForcedAutoDetection
));
1003 MOZ_ASSERT(mEncoding
== UTF_8_ENCODING
);
1005 MOZ_ASSERT(!mStartedFeedingDevTools
);
1006 if (mURIToSendToDevtools
) {
1007 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBufferOfMetaScan
;
1009 Span
<const char16_t
> data(buffer
->getBuffer() + buffer
->getStart(),
1010 buffer
->getLength());
1012 buffer
= buffer
->next
;
1016 mFirstBufferOfMetaScan
= nullptr;
1018 mBufferingBytes
= false;
1019 mForceAutoDetection
= false; // To stop feeding the detector
1020 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, true);
1021 if (mMode
== VIEW_SOURCE_HTML
) {
1022 auto r
= mTokenizer
->FlushViewSource();
1024 return r
.unwrapErr();
1027 auto r
= mTreeBuilder
->Flush();
1029 return r
.unwrapErr();
1034 class MaybeRunCollector
: public Runnable
{
1036 explicit MaybeRunCollector(nsIDocShell
* aDocShell
)
1037 : Runnable("MaybeRunCollector"), mDocShell(aDocShell
) {}
1039 NS_IMETHOD
Run() override
{
1040 nsJSContext::MaybeRunNextCollectorSlice(mDocShell
,
1041 JS::GCReason::HTML_PARSER
);
1045 nsCOMPtr
<nsIDocShell
> mDocShell
;
1048 nsresult
nsHtml5StreamParser::OnStartRequest(nsIRequest
* aRequest
) {
1049 MOZ_RELEASE_ASSERT(STREAM_NOT_STARTED
== mStreamState
,
1050 "Got OnStartRequest when the stream had already started.");
1052 !mExecutor
->HasStarted(),
1053 "Got OnStartRequest at the wrong stage in the executor life cycle.");
1054 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
1056 // To avoid the cost of instantiating the detector when it's not needed,
1057 // let's instantiate only if we make it out of this method with the
1058 // intent to use it.
1059 auto detectorCreator
= MakeScopeExit([&] {
1060 if ((mForceAutoDetection
|| mCharsetSource
< kCharsetFromParentFrame
) ||
1061 !(mMode
== LOAD_AS_DATA
|| mMode
== VIEW_SOURCE_XML
)) {
1062 mDetector
= mozilla::EncodingDetector::Create();
1066 mRequest
= aRequest
;
1068 mStreamState
= STREAM_BEING_READ
;
1070 // For View Source, the parser should run with scripts "enabled" if a normal
1071 // load would have scripts enabled.
1072 bool scriptingEnabled
=
1073 mMode
== LOAD_AS_DATA
? false : mExecutor
->IsScriptEnabled();
1074 mOwner
->StartTokenizer(scriptingEnabled
);
1076 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
);
1077 bool isSrcdoc
= false;
1078 nsCOMPtr
<nsIChannel
> channel
;
1079 nsresult rv
= GetChannel(getter_AddRefs(channel
));
1080 if (NS_SUCCEEDED(rv
)) {
1081 isSrcdoc
= NS_IsSrcdocChannel(channel
);
1082 if (!isSrcdoc
&& mCharsetSource
<= kCharsetFromFallback
) {
1083 nsCOMPtr
<nsIURI
> originalURI
;
1084 rv
= channel
->GetOriginalURI(getter_AddRefs(originalURI
));
1085 if (NS_SUCCEEDED(rv
)) {
1086 if (originalURI
->SchemeIs("resource")) {
1087 mCharsetSource
= kCharsetFromBuiltIn
;
1088 mEncoding
= UTF_8_ENCODING
;
1089 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
1091 nsCOMPtr
<nsIURI
> currentURI
;
1092 rv
= channel
->GetURI(getter_AddRefs(currentURI
));
1093 if (NS_SUCCEEDED(rv
)) {
1094 nsCOMPtr
<nsIURI
> innermost
= NS_GetInnermostURI(currentURI
);
1095 if (innermost
->SchemeIs("file")) {
1096 MOZ_ASSERT(mEncoding
== UTF_8_ENCODING
);
1097 if (!(mMode
== LOAD_AS_DATA
|| mMode
== VIEW_SOURCE_XML
)) {
1098 mDecodingLocalFileWithoutTokenizing
= true;
1102 innermost
->GetAsciiHost(host
);
1103 if (!host
.IsEmpty()) {
1104 // First let's see if the host is DNS-absolute and ends with a
1105 // dot and get rid of that one.
1106 if (host
.Last() == '.') {
1107 host
.SetLength(host
.Length() - 1);
1109 int32_t index
= host
.RFindChar('.');
1110 if (index
!= kNotFound
) {
1111 // We tolerate an IPv4 component as generic "TLD", so don't
1114 Substring(host
, index
+ 1, host
.Length() - (index
+ 1)),
1124 mTreeBuilder
->setIsSrcdocDocument(isSrcdoc
);
1125 mTreeBuilder
->setScriptingEnabled(scriptingEnabled
);
1126 mTreeBuilder
->SetPreventScriptExecution(
1127 !((mMode
== NORMAL
) && scriptingEnabled
));
1128 mTreeBuilder
->setAllowDeclarativeShadowRoots(
1129 mExecutor
->GetDocument()->AllowsDeclarativeShadowRoots());
1130 mTokenizer
->start();
1132 mExecutor
->StartReadingFromStage();
1134 if (mMode
== PLAIN_TEXT
) {
1135 mTreeBuilder
->StartPlainText();
1136 mTokenizer
->StartPlainText();
1138 mTemplatePushedOrHeadPopped
); // Needed to force 1024-byte sniffing
1139 // Flush the ops to put them where ContinueAfterScriptsOrEncodingCommitment
1141 auto r
= mTreeBuilder
->Flush();
1143 return mExecutor
->MarkAsBroken(r
.unwrapErr());
1145 } else if (mMode
== VIEW_SOURCE_PLAIN
) {
1146 nsAutoString viewSourceTitle
;
1147 CopyUTF8toUTF16(mViewSourceTitle
, viewSourceTitle
);
1148 mTreeBuilder
->EnsureBufferSpace(viewSourceTitle
.Length());
1149 mTreeBuilder
->StartPlainTextViewSource(viewSourceTitle
);
1150 mTokenizer
->StartPlainText();
1152 mTemplatePushedOrHeadPopped
); // Needed to force 1024-byte sniffing
1153 // Flush the ops to put them where ContinueAfterScriptsOrEncodingCommitment
1155 auto r
= mTreeBuilder
->Flush();
1157 return mExecutor
->MarkAsBroken(r
.unwrapErr());
1159 } else if (mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
) {
1160 // Generate and flush the View Source document up to and including the
1161 // pre element start.
1162 mTokenizer
->StartViewSource(NS_ConvertUTF8toUTF16(mViewSourceTitle
));
1163 if (mMode
== VIEW_SOURCE_XML
) {
1164 mTokenizer
->StartViewSourceCharacters();
1166 // Flush the ops to put them where ContinueAfterScriptsOrEncodingCommitment
1168 auto r
= mTokenizer
->FlushViewSource();
1170 return mExecutor
->MarkAsBroken(r
.unwrapErr());
1175 * If you move the following line, be very careful not to cause
1176 * WillBuildModel to be called before the document has had its
1177 * script global object set.
1179 rv
= mExecutor
->WillBuildModel();
1180 NS_ENSURE_SUCCESS(rv
, rv
);
1182 RefPtr
<nsHtml5OwningUTF16Buffer
> newBuf
=
1183 nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE
);
1185 // marks this stream parser as terminated,
1186 // which prevents entry to code paths that
1187 // would use mFirstBuffer or mLastBuffer.
1188 return mExecutor
->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1190 MOZ_ASSERT(!mFirstBuffer
, "How come we have the first buffer set?");
1191 MOZ_ASSERT(!mLastBuffer
, "How come we have the last buffer set?");
1192 mFirstBuffer
= mLastBuffer
= newBuf
;
1196 nsCOMPtr
<nsIHttpChannel
> httpChannel(do_QueryInterface(mRequest
, &rv
));
1197 if (NS_SUCCEEDED(rv
)) {
1198 nsAutoCString method
;
1199 Unused
<< httpChannel
->GetRequestMethod(method
);
1200 // XXX does Necko have a way to renavigate POST, etc. without hitting
1202 if (!method
.EqualsLiteral("GET")) {
1203 // This is the old Gecko behavior but the HTML5 spec disagrees.
1204 // Don't reparse on POST.
1205 mReparseForbidden
= true;
1209 // Attempt to retarget delivery of data (via OnDataAvailable) to the parser
1210 // thread, rather than through the main thread.
1211 nsCOMPtr
<nsIThreadRetargetableRequest
> threadRetargetableRequest
=
1212 do_QueryInterface(mRequest
, &rv
);
1213 if (threadRetargetableRequest
) {
1214 rv
= threadRetargetableRequest
->RetargetDeliveryTo(mEventTarget
);
1215 if (NS_SUCCEEDED(rv
)) {
1216 // Parser thread should be now ready to get data from necko and parse it
1217 // and main thread might have a chance to process a collector slice.
1218 // We need to do this asynchronously so that necko may continue processing
1220 nsCOMPtr
<nsIRunnable
> runnable
=
1221 new MaybeRunCollector(mExecutor
->GetDocument()->GetDocShell());
1222 mozilla::SchedulerGroup::Dispatch(runnable
.forget());
1226 if (NS_FAILED(rv
)) {
1227 NS_WARNING("Failed to retarget HTML data delivery to the parser thread.");
1230 if (mCharsetSource
== kCharsetFromParentFrame
) {
1231 // Remember this for error reporting.
1232 mInitialEncodingWasFromParentFrame
= true;
1233 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
);
1236 if (mForceAutoDetection
|| mCharsetSource
< kCharsetFromChannel
) {
1237 mBufferingBytes
= true;
1238 if (mMode
!= VIEW_SOURCE_XML
) {
1239 // We need to set mLookingForMetaCharset to true here in case the first
1240 // buffer to arrive is larger than 1024. We need the code that splits
1241 // the buffers at 1024 bytes to work even in that case.
1242 mLookingForMetaCharset
= true;
1246 if (mCharsetSource
< kCharsetFromUtf8OnlyMime
) {
1247 // we aren't ready to commit to an encoding yet
1248 // leave converter uninstantiated for now
1252 MOZ_ASSERT(!(mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
));
1254 MOZ_ASSERT(mEncoding
== UTF_8_ENCODING
,
1255 "How come UTF-8-only MIME type didn't set encoding to UTF-8?");
1257 // We are loading JSON/WebVTT/etc. into a browsing context.
1258 // There's no need to remove the BOM manually here, because
1259 // the UTF-8 decoder removes it.
1260 mReparseForbidden
= true;
1261 mForceAutoDetection
= false;
1263 // Instantiate the converter here to avoid BOM sniffing.
1264 mDecodingLocalFileWithoutTokenizing
= false;
1265 mUnicodeDecoder
= mEncoding
->NewDecoderWithBOMRemoval();
1269 void nsHtml5StreamParser::DoStopRequest() {
1270 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1271 MOZ_RELEASE_ASSERT(STREAM_BEING_READ
== mStreamState
,
1272 "Stream ended without being open.");
1273 mTokenizerMutex
.AssertCurrentThreadOwns();
1275 auto guard
= MakeScopeExit([&] { OnContentComplete(); });
1277 if (IsTerminated()) {
1281 if (MOZ_UNLIKELY(mLookingForXmlDeclarationForXmlViewSource
)) {
1282 mLookingForXmlDeclarationForXmlViewSource
= false;
1283 mBufferingBytes
= false;
1284 mUnicodeDecoder
= mEncoding
->NewDecoderWithoutBOMHandling();
1285 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
1287 for (auto&& buffer
: mBufferedBytes
) {
1288 nsresult rv
= WriteStreamBytes(buffer
);
1289 if (NS_FAILED(rv
)) {
1294 } else if (!mUnicodeDecoder
) {
1296 if (NS_FAILED(rv
= SniffStreamBytes(Span
<const uint8_t>(), true))) {
1302 MOZ_ASSERT(mUnicodeDecoder
,
1303 "Should have a decoder after finalizing sniffing.");
1305 // mLastBuffer should always point to a buffer of the size
1306 // READ_BUFFER_SIZE.
1308 NS_WARNING("mLastBuffer should not be null!");
1309 MarkAsBroken(NS_ERROR_NULL_POINTER
);
1313 Span
<uint8_t> src
; // empty span
1315 auto dst
= mLastBuffer
->TailAsSpan(READ_BUFFER_SIZE
);
1320 // Do not use structured binding lest deal with [-Werror=unused-variable]
1321 std::tie(result
, read
, written
, hadErrors
) =
1322 mUnicodeDecoder
->DecodeToUTF16(src
, dst
, true);
1323 if (!(mLookingForMetaCharset
|| mDecodingLocalFileWithoutTokenizing
)) {
1324 OnNewContent(dst
.To(written
));
1327 mHasHadErrors
= true;
1329 MOZ_ASSERT(read
== 0, "How come an empty span was read form?");
1330 mLastBuffer
->AdvanceEnd(written
);
1331 if (result
== kOutputFull
) {
1332 RefPtr
<nsHtml5OwningUTF16Buffer
> newBuf
=
1333 nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE
);
1335 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1338 mLastBuffer
= (mLastBuffer
->next
= std::move(newBuf
));
1340 if (!mLookingForMetaCharset
&& mDecodingLocalFileWithoutTokenizing
) {
1341 MOZ_ASSERT(mNumBytesBuffered
< LOCAL_FILE_UTF_8_BUFFER_SIZE
);
1342 MOZ_ASSERT(!mStartedFeedingDetector
);
1343 for (auto&& buffer
: mBufferedBytes
) {
1344 FeedDetector(buffer
);
1346 MOZ_ASSERT(!mChardetEof
);
1348 auto [encoding
, source
] = GuessEncoding(true);
1349 mCharsetSource
= source
;
1350 if (encoding
!= mEncoding
) {
1351 mEncoding
= encoding
;
1352 nsresult rv
= ReDecodeLocalFile();
1353 if (NS_FAILED(rv
)) {
1360 MOZ_ASSERT(mEncoding
== UTF_8_ENCODING
);
1361 nsresult rv
= CommitLocalFileToEncoding();
1362 if (NS_FAILED(rv
)) {
1371 mStreamState
= STREAM_ENDED
;
1373 if (IsTerminatedOrInterrupted()) {
1377 ParseAvailableData();
1380 class nsHtml5RequestStopper
: public Runnable
{
1382 nsHtml5StreamParserPtr mStreamParser
;
1385 explicit nsHtml5RequestStopper(nsHtml5StreamParser
* aStreamParser
)
1386 : Runnable("nsHtml5RequestStopper"), mStreamParser(aStreamParser
) {}
1387 NS_IMETHOD
Run() override
{
1388 mozilla::MutexAutoLock
autoLock(mStreamParser
->mTokenizerMutex
);
1389 mStreamParser
->DoStopRequest();
1390 mStreamParser
->PostLoadFlusher();
1395 nsresult
nsHtml5StreamParser::OnStopRequest(
1396 nsIRequest
* aRequest
, nsresult status
,
1397 const mozilla::ReentrantMonitorAutoEnter
& aProofOfLock
) {
1398 MOZ_ASSERT_IF(aRequest
, mRequest
== aRequest
);
1399 if (mOnStopCalled
) {
1400 if (mOnDataFinishedTime
) {
1401 mOnStopRequestTime
= TimeStamp::Now();
1403 mOnDataFinishedTime
= TimeStamp::Now();
1406 mOnStopCalled
= true;
1408 if (MOZ_UNLIKELY(NS_IsMainThread())) {
1409 mOnStopRequestTime
= TimeStamp::Now();
1410 nsCOMPtr
<nsIRunnable
> stopper
= new nsHtml5RequestStopper(this);
1412 mEventTarget
->Dispatch(stopper
, nsIThread::DISPATCH_NORMAL
))) {
1413 NS_WARNING("Dispatching StopRequest event failed.");
1416 mOnDataFinishedTime
= TimeStamp::Now();
1418 if (StaticPrefs::network_send_OnDataFinished_html5parser()) {
1419 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1421 mozilla::MutexAutoLock
autoLock(mTokenizerMutex
);
1425 // Let the MainThread event handle this, even though it will just
1426 // send it back to this thread, so we can accurately judge the impact
1427 // of this change. This should eventually be removed
1428 mOnStopCalled
= false;
1429 // don't record any telemetry for this
1434 if (!mOnStopRequestTime
.IsNull() && !mOnDataFinishedTime
.IsNull()) {
1435 TimeDuration delta
= (mOnStopRequestTime
- mOnDataFinishedTime
);
1436 if (delta
.ToMilliseconds() < 0) {
1437 // Because Telemetry can't handle negatives
1440 http_content_html5parser_ondatafinished_to_onstop_delay_negative
1441 .AccumulateRawDuration(delta
);
1443 glean::networking::http_content_html5parser_ondatafinished_to_onstop_delay
1444 .AccumulateRawDuration(delta
);
1450 void nsHtml5StreamParser::DoDataAvailableBuffer(
1451 mozilla::Buffer
<uint8_t>&& aBuffer
) {
1452 if (MOZ_UNLIKELY(!mBufferingBytes
)) {
1453 DoDataAvailable(aBuffer
);
1456 if (MOZ_UNLIKELY(mLookingForXmlDeclarationForXmlViewSource
)) {
1457 const uint8_t* elements
= aBuffer
.Elements();
1458 size_t length
= aBuffer
.Length();
1459 const uint8_t* lt
= (const uint8_t*)memchr(elements
, '>', length
);
1461 mBufferedBytes
.AppendElement(std::move(aBuffer
));
1465 // We found an '>'. Now there either is or isn't an XML decl.
1466 length
= (lt
- elements
) + 1;
1467 Vector
<uint8_t> contiguous
;
1468 for (auto&& buffer
: mBufferedBytes
) {
1469 if (!contiguous
.append(buffer
.Elements(), buffer
.Length())) {
1470 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1474 if (!contiguous
.append(elements
, length
)) {
1475 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1479 const Encoding
* encoding
=
1480 xmldecl_parse(contiguous
.begin(), contiguous
.length());
1482 mEncoding
= WrapNotNull(encoding
);
1483 mCharsetSource
= kCharsetFromXmlDeclaration
;
1486 mLookingForXmlDeclarationForXmlViewSource
= false;
1487 mBufferingBytes
= false;
1488 mUnicodeDecoder
= mEncoding
->NewDecoderWithoutBOMHandling();
1489 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, false);
1491 for (auto&& buffer
: mBufferedBytes
) {
1492 DoDataAvailable(buffer
);
1494 DoDataAvailable(aBuffer
);
1495 mBufferedBytes
.Clear();
1498 CheckedInt
<size_t> bufferedPlusLength(aBuffer
.Length());
1499 bufferedPlusLength
+= mNumBytesBuffered
;
1500 if (!bufferedPlusLength
.isValid()) {
1501 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1504 // Ensure that WriteStreamBytes() sees buffers ending
1505 // exactly at the two special boundaries.
1506 bool metaBoundaryWithinBuffer
=
1507 mLookingForMetaCharset
&&
1508 mNumBytesBuffered
< UNCONDITIONAL_META_SCAN_BOUNDARY
&&
1509 bufferedPlusLength
.value() > UNCONDITIONAL_META_SCAN_BOUNDARY
;
1510 bool localFileLimitWithinBuffer
=
1511 mDecodingLocalFileWithoutTokenizing
&&
1512 mNumBytesBuffered
< LOCAL_FILE_UTF_8_BUFFER_SIZE
&&
1513 bufferedPlusLength
.value() > LOCAL_FILE_UTF_8_BUFFER_SIZE
;
1514 if (!metaBoundaryWithinBuffer
&& !localFileLimitWithinBuffer
) {
1515 // Truncation OK, because we just checked the range.
1516 mNumBytesBuffered
= bufferedPlusLength
.value();
1517 mBufferedBytes
.AppendElement(std::move(aBuffer
));
1518 DoDataAvailable(mBufferedBytes
.LastElement());
1521 !(metaBoundaryWithinBuffer
&& localFileLimitWithinBuffer
),
1522 "How can Necko give us a buffer this large?");
1523 size_t boundary
= metaBoundaryWithinBuffer
1524 ? UNCONDITIONAL_META_SCAN_BOUNDARY
1525 : LOCAL_FILE_UTF_8_BUFFER_SIZE
;
1526 // Truncation OK, because the constant is small enough.
1527 size_t overBoundary
= bufferedPlusLength
.value() - boundary
;
1528 MOZ_RELEASE_ASSERT(overBoundary
< aBuffer
.Length());
1529 size_t untilBoundary
= aBuffer
.Length() - overBoundary
;
1530 auto span
= aBuffer
.AsSpan();
1531 auto head
= span
.To(untilBoundary
);
1532 auto tail
= span
.From(untilBoundary
);
1533 MOZ_RELEASE_ASSERT(mNumBytesBuffered
+ untilBoundary
== boundary
);
1534 // The following copies may end up being useless, but optimizing
1535 // them away would add complexity.
1536 Maybe
<Buffer
<uint8_t>> maybeHead
= Buffer
<uint8_t>::CopyFrom(head
);
1537 if (maybeHead
.isNothing()) {
1538 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1541 mNumBytesBuffered
= boundary
;
1542 mBufferedBytes
.AppendElement(std::move(*maybeHead
));
1543 DoDataAvailable(mBufferedBytes
.LastElement());
1544 // Re-decode may have happened here.
1546 Maybe
<Buffer
<uint8_t>> maybeTail
= Buffer
<uint8_t>::CopyFrom(tail
);
1547 if (maybeTail
.isNothing()) {
1548 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1551 mNumBytesBuffered
+= tail
.Length();
1552 mBufferedBytes
.AppendElement(std::move(*maybeTail
));
1553 DoDataAvailable(mBufferedBytes
.LastElement());
1555 // Do this clean-up here to avoid use-after-free when
1556 // DoDataAvailable is passed a span pointing into an
1557 // element of mBufferedBytes.
1558 if (!mBufferingBytes
) {
1559 mBufferedBytes
.Clear();
1563 void nsHtml5StreamParser::DoDataAvailable(Span
<const uint8_t> aBuffer
) {
1564 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1565 MOZ_RELEASE_ASSERT(STREAM_BEING_READ
== mStreamState
,
1566 "DoDataAvailable called when stream not open.");
1567 mTokenizerMutex
.AssertCurrentThreadOwns();
1569 if (IsTerminated()) {
1575 if ((mForceAutoDetection
|| mCharsetSource
< kCharsetFromParentFrame
) &&
1576 !mBufferingBytes
&& !mReparseForbidden
&&
1577 !(mMode
== LOAD_AS_DATA
|| mMode
== VIEW_SOURCE_XML
)) {
1578 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
,
1579 "How is mBufferingBytes false if "
1580 "mDecodingLocalFileWithoutTokenizing is true?");
1581 FeedDetector(aBuffer
);
1583 rv
= WriteStreamBytes(aBuffer
);
1585 rv
= SniffStreamBytes(aBuffer
, false);
1587 if (NS_FAILED(rv
)) {
1592 if (IsTerminatedOrInterrupted()) {
1596 if (!mLookingForMetaCharset
&& mDecodingLocalFileWithoutTokenizing
) {
1600 ParseAvailableData();
1602 if (mBomState
!= BOM_SNIFFING_OVER
|| mFlushTimerArmed
|| mSpeculating
) {
1607 mozilla::MutexAutoLock
flushTimerLock(mFlushTimerMutex
);
1608 mFlushTimer
->InitWithNamedFuncCallback(
1609 nsHtml5StreamParser::TimerCallback
, static_cast<void*>(this),
1610 mFlushTimerEverFired
? StaticPrefs::html5_flushtimer_initialdelay()
1611 : StaticPrefs::html5_flushtimer_subsequentdelay(),
1612 nsITimer::TYPE_ONE_SHOT
, "nsHtml5StreamParser::DoDataAvailable");
1614 mFlushTimerArmed
= true;
1617 class nsHtml5DataAvailable
: public Runnable
{
1619 nsHtml5StreamParserPtr mStreamParser
;
1620 Buffer
<uint8_t> mData
;
1623 nsHtml5DataAvailable(nsHtml5StreamParser
* aStreamParser
,
1624 Buffer
<uint8_t>&& aData
)
1625 : Runnable("nsHtml5DataAvailable"),
1626 mStreamParser(aStreamParser
),
1627 mData(std::move(aData
)) {}
1628 NS_IMETHOD
Run() override
{
1629 mozilla::MutexAutoLock
autoLock(mStreamParser
->mTokenizerMutex
);
1630 mStreamParser
->DoDataAvailableBuffer(std::move(mData
));
1631 mStreamParser
->PostLoadFlusher();
1636 nsresult
nsHtml5StreamParser::OnDataAvailable(nsIRequest
* aRequest
,
1637 nsIInputStream
* aInStream
,
1638 uint64_t aSourceOffset
,
1642 MOZ_ASSERT(mRequest
== aRequest
, "Got data on wrong stream.");
1644 // Main thread to parser thread dispatch requires copying to buffer first.
1645 if (MOZ_UNLIKELY(NS_IsMainThread())) {
1646 if (NS_FAILED(rv
= mExecutor
->IsBroken())) {
1649 Maybe
<Buffer
<uint8_t>> maybe
= Buffer
<uint8_t>::Alloc(aLength
);
1650 if (maybe
.isNothing()) {
1651 return mExecutor
->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1653 Buffer
<uint8_t> data(std::move(*maybe
));
1654 rv
= aInStream
->Read(reinterpret_cast<char*>(data
.Elements()),
1655 data
.Length(), &totalRead
);
1656 NS_ENSURE_SUCCESS(rv
, rv
);
1657 MOZ_ASSERT(totalRead
== aLength
);
1659 nsCOMPtr
<nsIRunnable
> dataAvailable
=
1660 new nsHtml5DataAvailable(this, std::move(data
));
1661 if (NS_FAILED(mEventTarget
->Dispatch(dataAvailable
,
1662 nsIThread::DISPATCH_NORMAL
))) {
1663 NS_WARNING("Dispatching DataAvailable event failed.");
1668 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1669 mozilla::MutexAutoLock
autoLock(mTokenizerMutex
);
1671 if (NS_FAILED(rv
= mTreeBuilder
->IsBroken())) {
1675 // Since we're getting OnDataAvailable directly on the parser thread,
1676 // there is no nsHtml5DataAvailable that would call PostLoadFlusher.
1677 // Hence, we need to call PostLoadFlusher() before this method returns.
1678 // Braces for RAII clarity relative to the mutex despite not being
1679 // strictly necessary.
1681 auto speculationFlusher
= MakeScopeExit([&] { PostLoadFlusher(); });
1683 if (mBufferingBytes
) {
1684 Maybe
<Buffer
<uint8_t>> maybe
= Buffer
<uint8_t>::Alloc(aLength
);
1685 if (maybe
.isNothing()) {
1686 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
1687 return NS_ERROR_OUT_OF_MEMORY
;
1689 Buffer
<uint8_t> data(std::move(*maybe
));
1690 rv
= aInStream
->Read(reinterpret_cast<char*>(data
.Elements()),
1691 data
.Length(), &totalRead
);
1692 NS_ENSURE_SUCCESS(rv
, rv
);
1693 MOZ_ASSERT(totalRead
== aLength
);
1694 DoDataAvailableBuffer(std::move(data
));
1697 // Read directly from response buffer.
1698 rv
= aInStream
->ReadSegments(CopySegmentsToParser
, this, aLength
,
1700 NS_ENSURE_SUCCESS(rv
, rv
);
1701 MOZ_ASSERT(totalRead
== aLength
);
1706 // Called under lock by function ptr
1708 nsresult
nsHtml5StreamParser::CopySegmentsToParser(
1709 nsIInputStream
* aInStream
, void* aClosure
, const char* aFromSegment
,
1710 uint32_t aToOffset
, uint32_t aCount
,
1711 uint32_t* aWriteCount
) MOZ_NO_THREAD_SAFETY_ANALYSIS
{
1712 nsHtml5StreamParser
* parser
= static_cast<nsHtml5StreamParser
*>(aClosure
);
1714 parser
->DoDataAvailable(AsBytes(Span(aFromSegment
, aCount
)));
1715 // Assume DoDataAvailable consumed all available bytes.
1716 *aWriteCount
= aCount
;
1720 const Encoding
* nsHtml5StreamParser::PreferredForInternalEncodingDecl(
1721 const nsAString
& aEncoding
) {
1722 const Encoding
* newEncoding
= Encoding::ForLabel(aEncoding
);
1724 // the encoding name is bogus
1725 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaUnsupported", true,
1726 mTokenizer
->getLineNumber());
1730 if (newEncoding
== UTF_16BE_ENCODING
|| newEncoding
== UTF_16LE_ENCODING
) {
1731 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaUtf16", true,
1732 mTokenizer
->getLineNumber());
1733 newEncoding
= UTF_8_ENCODING
;
1736 if (newEncoding
== X_USER_DEFINED_ENCODING
) {
1737 // WebKit/Blink hack for Indian and Armenian legacy sites
1738 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaUserDefined", true,
1739 mTokenizer
->getLineNumber());
1740 newEncoding
= WINDOWS_1252_ENCODING
;
1743 if (newEncoding
== REPLACEMENT_ENCODING
) {
1744 // No line number, because the replacement encoding doesn't allow
1745 // showing the lines.
1746 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaReplacement", true, 0);
1752 bool nsHtml5StreamParser::internalEncodingDeclaration(nsHtml5String aEncoding
) {
1753 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1754 if ((mCharsetSource
>= kCharsetFromMetaTag
&&
1755 mCharsetSource
!= kCharsetFromFinalAutoDetectionFile
) ||
1756 mSeenEligibleMetaCharset
) {
1760 nsString newEncoding
; // Not Auto, because using it to hold nsStringBuffer*
1761 aEncoding
.ToString(newEncoding
);
1762 auto encoding
= PreferredForInternalEncodingDecl(newEncoding
);
1767 mSeenEligibleMetaCharset
= true;
1769 if (!mLookingForMetaCharset
) {
1770 if (mInitialEncodingWasFromParentFrame
) {
1771 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaTooLateFrame", true,
1772 mTokenizer
->getLineNumber());
1774 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaTooLate", true,
1775 mTokenizer
->getLineNumber());
1779 if (mTemplatePushedOrHeadPopped
) {
1780 mTreeBuilder
->MaybeComplainAboutCharset("EncMetaAfterHeadInKilobyte", false,
1781 mTokenizer
->getLineNumber());
1784 if (mForceAutoDetection
&&
1785 (encoding
->IsAsciiCompatible() || encoding
== ISO_2022_JP_ENCODING
)) {
1789 mNeedsEncodingSwitchTo
= encoding
;
1790 mEncodingSwitchSource
= kCharsetFromMetaTag
;
1794 bool nsHtml5StreamParser::TemplatePushedOrHeadPopped() {
1796 IsParserThread() || mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
,
1798 mTemplatePushedOrHeadPopped
= true;
1799 return mNumBytesBuffered
>= UNCONDITIONAL_META_SCAN_BOUNDARY
;
1802 void nsHtml5StreamParser::RememberGt(int32_t aPos
) {
1803 if (mLookingForMetaCharset
) {
1804 mGtBuffer
= mFirstBuffer
;
1809 void nsHtml5StreamParser::PostLoadFlusher() {
1810 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1811 mTokenizerMutex
.AssertCurrentThreadOwns();
1813 mTreeBuilder
->FlushLoads();
1814 // Dispatch this runnable unconditionally, because the loads
1815 // that need flushing may have been flushed earlier even if the
1816 // flush right above here did nothing. (Is this still true?)
1817 nsCOMPtr
<nsIRunnable
> runnable(mLoadFlusher
);
1819 DispatchToMain(CreateRenderBlockingRunnable(runnable
.forget())))) {
1820 NS_WARNING("failed to dispatch load flush event");
1823 if ((mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
) &&
1824 mTokenizer
->ShouldFlushViewSource()) {
1825 auto r
= mTreeBuilder
->Flush(); // delete useless ops
1826 MOZ_ASSERT(r
.isOk(), "Should have null sink with View Source");
1827 r
= mTokenizer
->FlushViewSource();
1829 MarkAsBroken(r
.unwrapErr());
1833 nsCOMPtr
<nsIRunnable
> runnable(mExecutorFlusher
);
1834 if (NS_FAILED(DispatchToMain(runnable
.forget()))) {
1835 NS_WARNING("failed to dispatch executor flush event");
1841 void nsHtml5StreamParser::FlushTreeOpsAndDisarmTimer() {
1842 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1843 if (mFlushTimerArmed
) {
1844 // avoid calling Cancel if the flush timer isn't armed to avoid acquiring
1847 mozilla::MutexAutoLock
flushTimerLock(mFlushTimerMutex
);
1848 mFlushTimer
->Cancel();
1850 mFlushTimerArmed
= false;
1852 if (mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
) {
1853 auto r
= mTokenizer
->FlushViewSource();
1855 MarkAsBroken(r
.unwrapErr());
1858 auto r
= mTreeBuilder
->Flush();
1860 MarkAsBroken(r
.unwrapErr());
1862 nsCOMPtr
<nsIRunnable
> runnable(mExecutorFlusher
);
1863 if (NS_FAILED(DispatchToMain(runnable
.forget()))) {
1864 NS_WARNING("failed to dispatch executor flush event");
1868 void nsHtml5StreamParser::SwitchDecoderIfAsciiSoFar(
1869 NotNull
<const Encoding
*> aEncoding
) {
1870 if (mEncoding
== aEncoding
) {
1871 MOZ_ASSERT(!mStartedFeedingDevTools
);
1872 // Report all already-decoded buffers to the dev tools if needed.
1873 if (mURIToSendToDevtools
) {
1874 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBufferOfMetaScan
;
1876 auto s
= Span(buffer
->getBuffer(), buffer
->getEnd());
1878 buffer
= buffer
->next
;
1883 if (!mEncoding
->IsAsciiCompatible() || !aEncoding
->IsAsciiCompatible()) {
1886 size_t numAscii
= 0;
1887 MOZ_ASSERT(mFirstBufferOfMetaScan
,
1888 "Why did we come here without starting meta scan?");
1889 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBufferOfMetaScan
;
1890 while (buffer
!= mFirstBuffer
) {
1891 MOZ_ASSERT(buffer
, "mFirstBuffer should have acted as sentinel!");
1892 MOZ_ASSERT(buffer
->getStart() == buffer
->getEnd(),
1893 "Why wasn't an early buffer fully consumed?");
1894 auto s
= Span(buffer
->getBuffer(), buffer
->getStart());
1898 numAscii
+= s
.Length();
1899 buffer
= buffer
->next
;
1901 auto s
= Span(mFirstBuffer
->getBuffer(), mFirstBuffer
->getStart());
1905 numAscii
+= s
.Length();
1907 MOZ_ASSERT(!mStartedFeedingDevTools
);
1908 // Report the ASCII prefix to dev tools if needed
1909 if (mURIToSendToDevtools
) {
1910 buffer
= mFirstBufferOfMetaScan
;
1911 while (buffer
!= mFirstBuffer
) {
1912 MOZ_ASSERT(buffer
, "mFirstBuffer should have acted as sentinel!");
1913 MOZ_ASSERT(buffer
->getStart() == buffer
->getEnd(),
1914 "Why wasn't an early buffer fully consumed?");
1915 auto s
= Span(buffer
->getBuffer(), buffer
->getStart());
1917 buffer
= buffer
->next
;
1919 auto s
= Span(mFirstBuffer
->getBuffer(), mFirstBuffer
->getStart());
1923 // Success! Now let's get rid of the already-decoded but not tokenized data:
1924 mFirstBuffer
->setEnd(mFirstBuffer
->getStart());
1925 mLastBuffer
= mFirstBuffer
;
1926 mFirstBuffer
->next
= nullptr;
1928 // Note: We could have scanned further for ASCII, which could avoid some
1929 // buffer deallocation and reallocation. However, chances are that if we got
1930 // until meta without non-ASCII before, there's going to be a title with
1931 // non-ASCII soon after anyway, so let's avoid the complexity of finding out.
1933 MOZ_ASSERT(mUnicodeDecoder
, "How come we scanned meta without a decoder?");
1934 mEncoding
= aEncoding
;
1935 mEncoding
->NewDecoderWithoutBOMHandlingInto(*mUnicodeDecoder
);
1936 mHasHadErrors
= false;
1938 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
,
1939 "Must have set mDecodingLocalFileWithoutTokenizing to false to "
1940 "report data to dev tools below");
1941 MOZ_ASSERT(!mLookingForMetaCharset
,
1942 "Must have set mLookingForMetaCharset to false to report data to "
1945 // Now skip over as many bytes and redecode the tail of the
1948 for (auto&& buffer
: mBufferedBytes
) {
1949 size_t nextSkipped
= skipped
+ buffer
.Length();
1950 if (nextSkipped
<= numAscii
) {
1951 skipped
= nextSkipped
;
1954 if (skipped
>= numAscii
) {
1955 WriteStreamBytes(buffer
);
1956 skipped
= nextSkipped
;
1959 size_t tailLength
= nextSkipped
- numAscii
;
1960 WriteStreamBytes(Span
<uint8_t>(buffer
).From(buffer
.Length() - tailLength
));
1961 skipped
= nextSkipped
;
1965 size_t nsHtml5StreamParser::CountGts() {
1970 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBufferOfMetaScan
;
1972 MOZ_ASSERT(buffer
, "How did we walk past mGtBuffer?");
1973 char16_t
* buf
= buffer
->getBuffer();
1974 if (buffer
== mGtBuffer
) {
1975 for (int32_t i
= 0; i
<= mGtPos
; ++i
) {
1976 if (buf
[i
] == u
'>') {
1982 for (int32_t i
= 0; i
< buffer
->getEnd(); ++i
) {
1983 if (buf
[i
] == u
'>') {
1987 buffer
= buffer
->next
;
1992 void nsHtml5StreamParser::DiscardMetaSpeculation() {
1993 mozilla::MutexAutoLock
speculationAutoLock(mSpeculationMutex
);
1994 // Rewind the stream
1995 MOZ_ASSERT(!mAtEOF
, "How did we end up setting this?");
1996 mTokenizer
->resetToDataState();
1997 mTokenizer
->setLineNumber(1);
2000 if (mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
) {
2001 // resetToDataState() above logically rewinds to the state before
2002 // the plain text start, so we need to start plain text again to
2003 // put the tokenizer into the plain text state.
2004 mTokenizer
->StartPlainText();
2007 mFirstBuffer
= mLastBuffer
;
2008 mFirstBuffer
->setStart(0);
2009 mFirstBuffer
->setEnd(0);
2010 mFirstBuffer
->next
= nullptr;
2012 mTreeBuilder
->flushCharacters(); // empty the pending buffer
2013 mTreeBuilder
->ClearOps(); // now get rid of the failed ops
2015 if (mMode
== VIEW_SOURCE_HTML
) {
2016 mTokenizer
->RewindViewSource();
2020 // We know that this resets the tree builder back to the start state.
2021 // This must happen _after_ the flushCharacters() call above!
2022 const auto& speculation
= mSpeculations
.ElementAt(0);
2023 mTreeBuilder
->loadState(speculation
->GetSnapshot());
2026 // Experimentation suggests that we don't need to do anything special
2027 // for ignoring the leading LF in View Source here.
2029 mSpeculations
.Clear(); // potentially a huge number of destructors
2030 // run here synchronously...
2032 // Now set up a new speculation for the main thread to find.
2033 // Note that we stay in the speculating state, because the main thread
2034 // knows how to come out of that state and this thread does not.
2036 nsHtml5Speculation
* speculation
= new nsHtml5Speculation(
2037 mFirstBuffer
, mFirstBuffer
->getStart(), mTokenizer
->getLineNumber(),
2038 mTokenizer
->getColumnNumber(), mTreeBuilder
->newSnapshot());
2039 MOZ_ASSERT(!mFlushTimerArmed
, "How did we end up arming the timer?");
2040 if (mMode
== VIEW_SOURCE_HTML
) {
2041 mTokenizer
->SetViewSourceOpSink(speculation
);
2042 mTokenizer
->StartViewSourceCharacters();
2044 MOZ_ASSERT(mMode
!= VIEW_SOURCE_XML
);
2045 mTreeBuilder
->SetOpSink(speculation
);
2047 mSpeculations
.AppendElement(speculation
); // adopts the pointer
2048 MOZ_ASSERT(mSpeculating
, "How did we end speculating?");
2052 * The general idea is to match WebKit and Blink exactly for meta
2055 * 1. WebKit and Blink look for meta as if scripting was disabled
2056 * for `noscript` purposes. This implementation matches the
2057 * `noscript` treatment of the observable DOM building (in order
2058 * to be able to use the same tree builder run).
2059 * 2. WebKit and Blink look for meta as if the foreign content
2060 * feedback from the tree builder to the tokenizer didn't exist.
2061 * This implementation considers the foreign content rules in
2062 * order to be able to use the same tree builder run for meta
2063 * and the observable DOM building. Note that since <svg> and
2064 * <math> imply the end of head, this only matters for meta after
2065 * head but starting within the 1024-byte zone.
2067 * Template is treated specially, because that WebKit/Blink behavior
2068 * is easy to emulate unlike the above two exceptions. In general,
2069 * the meta scan token handler in WebKit and Blink behaves as if there
2070 * was a scripting-disabled tree builder predating the introduction
2071 * of foreign content and template.
2073 * Meta is honored if it _starts_ within the first 1024 kilobytes or,
2074 * if by the 1024-byte boundary head hasn't ended and a template
2075 * element hasn't started, a meta occurs before the first of the head
2076 * ending or a template element starting.
2078 * If a meta isn't honored according to the above definition, and
2079 * we aren't dealing with plain text, the buffered bytes, which by
2080 * now have to contain `>` character unless we encountered EOF, are
2081 * scanned for syntax resembling an XML declaration.
2083 * If neither a meta nor syntax resembling an XML declaration has
2084 * been honored and we aren't inheriting the encoding from a
2085 * same-origin parent or parsing for XHR, chardetng is used.
2086 * chardetng runs first for the part of the document that was searched
2087 * for meta and then at EOF. The part searched for meta is defined as
2088 * follows in order to avoid network buffer boundary-dependent
2091 * 1. At least the first 1024 bytes. (This is what happens for plain
2093 * 2. If the 1024-byte boundary is within a tag, comment, doctype,
2094 * or CDATA section, at least up to the end of that token or CDATA
2095 * section. (Exception: If the 1024-byte boundary is in an RCDATA
2096 * end tag that hasn't yet been decided to be an end tag, the
2097 * token is not considered.)
2098 * 3. If at the 1024-byte boundary, head hasn't ended and there hasn't
2099 * been a template tag, up to the end of the first template tag
2100 * or token ending the head, whichever comes first.
2101 * 4. Except if head is ended by a text token, only to the end of the
2102 * most recent tag, comment, or doctype token. (Because text is
2103 * coalesced, so it would be harder to correlate the text to the
2106 * An encoding-related reload is still possible if chardetng's guess
2107 * at EOF differs from its initial guess.
2109 bool nsHtml5StreamParser::ProcessLookingForMetaCharset(bool aEof
) {
2110 MOZ_ASSERT(mBomState
== BOM_SNIFFING_OVER
);
2111 MOZ_ASSERT(mMode
!= VIEW_SOURCE_XML
);
2112 bool rewound
= false;
2113 MOZ_ASSERT(mForceAutoDetection
||
2114 mCharsetSource
< kCharsetFromInitialAutoDetectionASCII
||
2115 mCharsetSource
== kCharsetFromParentFrame
,
2116 "Why are we looking for meta charset if we've seen it?");
2117 // NOTE! We may come here multiple times with
2118 // mNumBytesBuffered == UNCONDITIONAL_META_SCAN_BOUNDARY
2119 // if the tokenizer suspends multiple times after decoding has reached
2120 // mNumBytesBuffered == UNCONDITIONAL_META_SCAN_BOUNDARY. That's why
2121 // we need to also check whether the we are at the end of the last
2123 // Note that DoDataAvailableBuffer() ensures that the code here has
2124 // the opportunity to run at the exact UNCONDITIONAL_META_SCAN_BOUNDARY
2125 // even if there isn't a network buffer boundary there.
2126 bool atKilobyte
= false;
2127 if ((mNumBytesBuffered
== UNCONDITIONAL_META_SCAN_BOUNDARY
&&
2128 mFirstBuffer
== mLastBuffer
&& !mFirstBuffer
->hasMore())) {
2130 mTokenizer
->AtKilobyteBoundary();
2132 if (!mNeedsEncodingSwitchTo
&&
2133 (aEof
|| (mTemplatePushedOrHeadPopped
&&
2134 !mTokenizer
->IsInTokenStartedAtKilobyteBoundary() &&
2136 mNumBytesBuffered
> UNCONDITIONAL_META_SCAN_BOUNDARY
)))) {
2137 // meta charset was not found
2138 mLookingForMetaCharset
= false;
2139 if (mStartsWithLtQuestion
&& mCharsetSource
< kCharsetFromXmlDeclaration
) {
2140 // Look for bogo XML declaration.
2141 // Search the first buffer in the hope that '>' is within it.
2142 MOZ_ASSERT(!mBufferedBytes
.IsEmpty(),
2143 "How did at least <? not get buffered?");
2144 Buffer
<uint8_t>& first
= mBufferedBytes
[0];
2145 const Encoding
* encoding
=
2146 xmldecl_parse(first
.Elements(), first
.Length());
2148 // Our bogo XML declaration scanner wants to see a contiguous buffer, so
2149 // let's linearize the data. (Ideally, the XML declaration scanner would
2150 // be incremental, but this is the rare path anyway.)
2151 Vector
<uint8_t> contiguous
;
2152 if (!contiguous
.append(first
.Elements(), first
.Length())) {
2153 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2156 for (size_t i
= 1; i
< mBufferedBytes
.Length(); ++i
) {
2157 Buffer
<uint8_t>& buffer
= mBufferedBytes
[i
];
2158 const uint8_t* elements
= buffer
.Elements();
2159 size_t length
= buffer
.Length();
2160 const uint8_t* lt
= (const uint8_t*)memchr(elements
, '>', length
);
2163 length
= (lt
- elements
) + 1;
2166 if (!contiguous
.append(elements
, length
)) {
2167 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2171 // Avoid linearizing all buffered bytes unnecessarily.
2175 encoding
= xmldecl_parse(contiguous
.begin(), contiguous
.length());
2178 if (!(mForceAutoDetection
&& (encoding
->IsAsciiCompatible() ||
2179 encoding
== ISO_2022_JP_ENCODING
))) {
2180 mForceAutoDetection
= false;
2181 mNeedsEncodingSwitchTo
= encoding
;
2182 mEncodingSwitchSource
= kCharsetFromXmlDeclaration
;
2186 // Check again in case we found an encoding in the bogo XML declaration.
2187 if (!mNeedsEncodingSwitchTo
&&
2188 (mForceAutoDetection
||
2189 mCharsetSource
< kCharsetFromInitialAutoDetectionASCII
) &&
2190 !(mMode
== LOAD_AS_DATA
|| mMode
== VIEW_SOURCE_XML
) &&
2191 !(mDecodingLocalFileWithoutTokenizing
&& !aEof
&&
2192 mNumBytesBuffered
<= LOCAL_FILE_UTF_8_BUFFER_SIZE
)) {
2193 MOZ_ASSERT(!mStartedFeedingDetector
);
2194 if (mNumBytesBuffered
== UNCONDITIONAL_META_SCAN_BOUNDARY
|| aEof
) {
2195 // We know that all the buffered bytes have been tokenized, so feed
2196 // them all to chardetng.
2197 for (auto&& buffer
: mBufferedBytes
) {
2198 FeedDetector(buffer
);
2201 MOZ_ASSERT(!mChardetEof
);
2204 auto [encoding
, source
] = GuessEncoding(true);
2205 mNeedsEncodingSwitchTo
= encoding
;
2206 mEncodingSwitchSource
= source
;
2207 } else if (mNumBytesBuffered
> UNCONDITIONAL_META_SCAN_BOUNDARY
) {
2208 size_t gtsLeftToFind
= CountGts();
2209 size_t bytesSeen
= 0;
2210 // We sync the bytes to the UTF-16 code units seen to avoid depending
2211 // on network buffer boundaries. We do the syncing by counting '>'
2212 // bytes / code units. However, we always scan at least 1024 bytes.
2213 // The 1024-byte boundary is guaranteed to be between buffers.
2214 // The guarantee is implemented in DoDataAvailableBuffer().
2215 for (auto&& buffer
: mBufferedBytes
) {
2216 if (!mNeedsEncodingSwitchTo
) {
2217 if (gtsLeftToFind
) {
2218 auto span
= buffer
.AsSpan();
2220 for (size_t i
= 0; i
< span
.Length(); ++i
) {
2221 if (span
[i
] == uint8_t('>')) {
2223 if (!gtsLeftToFind
) {
2224 if (bytesSeen
< UNCONDITIONAL_META_SCAN_BOUNDARY
) {
2228 FeedDetector(span
.To(i
));
2229 auto [encoding
, source
] = GuessEncoding(true);
2230 mNeedsEncodingSwitchTo
= encoding
;
2231 mEncodingSwitchSource
= source
;
2232 FeedDetector(span
.From(i
));
2233 bytesSeen
+= buffer
.Length();
2234 // No need to update bytesSeen anymore, but let's do it for
2236 // We should do `continue outer;` but C++ can't.
2243 FeedDetector(buffer
);
2244 bytesSeen
+= buffer
.Length();
2248 if (bytesSeen
== UNCONDITIONAL_META_SCAN_BOUNDARY
) {
2249 auto [encoding
, source
] = GuessEncoding(true);
2250 mNeedsEncodingSwitchTo
= encoding
;
2251 mEncodingSwitchSource
= source
;
2254 FeedDetector(buffer
);
2255 bytesSeen
+= buffer
.Length();
2258 MOZ_ASSERT(mNeedsEncodingSwitchTo
,
2259 "How come we didn't call GuessEncoding()?");
2262 if (mNeedsEncodingSwitchTo
) {
2263 mDecodingLocalFileWithoutTokenizing
= false;
2264 mLookingForMetaCharset
= false;
2266 auto needsEncodingSwitchTo
= WrapNotNull(mNeedsEncodingSwitchTo
);
2267 mNeedsEncodingSwitchTo
= nullptr;
2269 SwitchDecoderIfAsciiSoFar(needsEncodingSwitchTo
);
2270 // The above line may have changed mEncoding so that mEncoding equals
2271 // needsEncodingSwitchTo.
2273 mCharsetSource
= mEncodingSwitchSource
;
2275 if (mMode
== VIEW_SOURCE_HTML
) {
2276 auto r
= mTokenizer
->FlushViewSource();
2278 MarkAsBroken(r
.unwrapErr());
2282 auto r
= mTreeBuilder
->Flush();
2284 MarkAsBroken(r
.unwrapErr());
2288 if (mEncoding
!= needsEncodingSwitchTo
) {
2289 // Speculation failed
2292 if (mEncoding
== ISO_2022_JP_ENCODING
||
2293 needsEncodingSwitchTo
== ISO_2022_JP_ENCODING
) {
2294 // Chances are no Web author will fix anything due to this message, so
2295 // this is here to help understanding issues when debugging sites made
2297 mTreeBuilder
->MaybeComplainAboutCharset("EncSpeculationFail2022", false,
2298 mTokenizer
->getLineNumber());
2300 if (mCharsetSource
== kCharsetFromMetaTag
) {
2301 mTreeBuilder
->MaybeComplainAboutCharset(
2302 "EncSpeculationFailMeta", false, mTokenizer
->getLineNumber());
2303 } else if (mCharsetSource
== kCharsetFromXmlDeclaration
) {
2304 // This intentionally refers to the line number of how far ahead
2305 // the document was parsed even though the bogo XML decl is always
2307 mTreeBuilder
->MaybeComplainAboutCharset(
2308 "EncSpeculationFailXml", false, mTokenizer
->getLineNumber());
2312 DiscardMetaSpeculation();
2313 // Redecode the stream.
2314 mEncoding
= needsEncodingSwitchTo
;
2315 mUnicodeDecoder
= mEncoding
->NewDecoderWithBOMRemoval();
2316 mHasHadErrors
= false;
2318 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing
,
2319 "Must have set mDecodingLocalFileWithoutTokenizing to false "
2320 "to report data to dev tools below");
2321 MOZ_ASSERT(!mLookingForMetaCharset
,
2322 "Must have set mLookingForMetaCharset to false to report data "
2323 "to dev tools below");
2324 for (auto&& buffer
: mBufferedBytes
) {
2325 nsresult rv
= WriteStreamBytes(buffer
);
2326 if (NS_FAILED(rv
)) {
2332 } else if (!mLookingForMetaCharset
&& !mDecodingLocalFileWithoutTokenizing
) {
2333 MOZ_ASSERT(!mStartedFeedingDevTools
);
2334 // Report all already-decoded buffers to the dev tools if needed.
2335 if (mURIToSendToDevtools
) {
2336 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBufferOfMetaScan
;
2338 auto s
= Span(buffer
->getBuffer(), buffer
->getEnd());
2340 buffer
= buffer
->next
;
2344 if (!mLookingForMetaCharset
) {
2345 mGtBuffer
= nullptr;
2348 if (!mDecodingLocalFileWithoutTokenizing
) {
2349 mFirstBufferOfMetaScan
= nullptr;
2350 mBufferingBytes
= false;
2351 mBufferedBytes
.Clear();
2352 mTreeBuilder
->SetDocumentCharset(mEncoding
, mCharsetSource
, true);
2353 if (mMode
== VIEW_SOURCE_HTML
) {
2354 auto r
= mTokenizer
->FlushViewSource();
2356 MarkAsBroken(r
.unwrapErr());
2360 auto r
= mTreeBuilder
->Flush();
2362 MarkAsBroken(r
.unwrapErr());
2370 void nsHtml5StreamParser::ParseAvailableData() {
2371 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
2372 mTokenizerMutex
.AssertCurrentThreadOwns();
2373 MOZ_ASSERT(!(mDecodingLocalFileWithoutTokenizing
&& !mLookingForMetaCharset
));
2375 if (IsTerminatedOrInterrupted()) {
2379 if (mSpeculating
&& !IsSpeculationEnabled()) {
2383 bool requestedReload
= false;
2385 if (!mFirstBuffer
->hasMore()) {
2386 if (mFirstBuffer
== mLastBuffer
) {
2387 switch (mStreamState
) {
2388 case STREAM_BEING_READ
:
2389 // never release the last buffer.
2390 if (!mSpeculating
) {
2391 // reuse buffer space if not speculating
2392 mFirstBuffer
->setStart(0);
2393 mFirstBuffer
->setEnd(0);
2395 return; // no more data for now but expecting more
2400 if (mLookingForMetaCharset
) {
2401 // When called with aEof=true, ProcessLookingForMetaCharset()
2402 // is guaranteed to set mLookingForMetaCharset to false so
2403 // that we can't come here twice.
2404 if (ProcessLookingForMetaCharset(true)) {
2405 if (IsTerminatedOrInterrupted()) {
2410 } else if ((mForceAutoDetection
||
2411 mCharsetSource
< kCharsetFromParentFrame
) &&
2412 !(mMode
== LOAD_AS_DATA
|| mMode
== VIEW_SOURCE_XML
) &&
2413 !mReparseForbidden
) {
2414 // An earlier DetectorEof() call is possible in which case
2415 // the one here is a no-op.
2417 auto [encoding
, source
] = GuessEncoding(false);
2418 if (encoding
!= mEncoding
) {
2419 // Request a reload from the docshell.
2422 kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII
&&
2424 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLDInitialWasASCII
) ||
2425 source
== kCharsetFromFinalUserForcedAutoDetection
);
2426 mTreeBuilder
->NeedsCharsetSwitchTo(encoding
, source
, 0);
2427 requestedReload
= true;
2428 } else if (mCharsetSource
==
2429 kCharsetFromInitialAutoDetectionASCII
&&
2430 mDetectorHasSeenNonAscii
) {
2431 mCharsetSource
= source
;
2432 mTreeBuilder
->UpdateCharsetSource(mCharsetSource
);
2437 if (!mForceAutoDetection
&& !requestedReload
) {
2438 if (mCharsetSource
== kCharsetFromParentFrame
) {
2439 mTreeBuilder
->MaybeComplainAboutCharset("EncNoDeclarationFrame",
2441 } else if (mCharsetSource
== kCharsetFromXmlDeclaration
) {
2442 // We know the bogo XML decl is always on the first line.
2443 mTreeBuilder
->MaybeComplainAboutCharset("EncXmlDecl", false, 1);
2446 kCharsetFromInitialAutoDetectionWouldHaveBeenUTF8
&&
2448 kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD
) {
2449 if (mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
) {
2450 mTreeBuilder
->MaybeComplainAboutCharset("EncNoDeclPlain",
2453 mTreeBuilder
->MaybeComplainAboutCharset("EncNoDecl", true, 0);
2457 if (mHasHadErrors
&& mEncoding
!= REPLACEMENT_ENCODING
) {
2458 if (mEncoding
== UTF_8_ENCODING
) {
2459 mTreeBuilder
->TryToEnableEncodingMenu();
2461 if (mCharsetSource
== kCharsetFromParentFrame
) {
2462 if (mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
) {
2463 mTreeBuilder
->MaybeComplainAboutCharset(
2464 "EncErrorFramePlain", true, 0);
2466 mTreeBuilder
->MaybeComplainAboutCharset("EncErrorFrame",
2470 mCharsetSource
>= kCharsetFromXmlDeclaration
&&
2472 kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII
&&
2474 kCharsetFromFinalUserForcedAutoDetection
)) {
2475 mTreeBuilder
->MaybeComplainAboutCharset("EncError", true, 0);
2479 if (NS_SUCCEEDED(mTreeBuilder
->IsBroken())) {
2482 if (NS_FAILED((rv
= mTreeBuilder
->IsBroken()))) {
2485 mTreeBuilder
->StreamEnded();
2486 if (mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
) {
2487 if (!mTokenizer
->EndViewSource()) {
2488 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2493 FlushTreeOpsAndDisarmTimer();
2494 return; // no more data and not expecting more
2496 MOZ_ASSERT_UNREACHABLE("It should be impossible to reach this.");
2500 mFirstBuffer
= mFirstBuffer
->next
;
2504 // now we have a non-empty buffer
2505 mFirstBuffer
->adjust(mLastWasCR
);
2507 if (mFirstBuffer
->hasMore()) {
2508 if (!mTokenizer
->EnsureBufferSpace(mFirstBuffer
->getLength())) {
2509 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2512 mLastWasCR
= mTokenizer
->tokenizeBuffer(mFirstBuffer
);
2514 if (NS_FAILED((rv
= mTreeBuilder
->IsBroken()))) {
2518 if (mTreeBuilder
->HasScriptThatMayDocumentWriteOrBlock()) {
2519 // `HasScriptThatMayDocumentWriteOrBlock()` cannot return true if the
2520 // tree builder is preventing script execution.
2521 MOZ_ASSERT(mMode
== NORMAL
);
2522 mozilla::MutexAutoLock
speculationAutoLock(mSpeculationMutex
);
2523 nsHtml5Speculation
* speculation
= new nsHtml5Speculation(
2524 mFirstBuffer
, mFirstBuffer
->getStart(), mTokenizer
->getLineNumber(),
2525 mTokenizer
->getColumnNumber(), mTreeBuilder
->newSnapshot());
2526 mTreeBuilder
->AddSnapshotToScript(speculation
->GetSnapshot(),
2527 speculation
->GetStartLineNumber());
2528 if (mLookingForMetaCharset
) {
2529 if (mMode
== VIEW_SOURCE_HTML
) {
2530 auto r
= mTokenizer
->FlushViewSource();
2532 MarkAsBroken(r
.unwrapErr());
2536 auto r
= mTreeBuilder
->Flush();
2538 MarkAsBroken(r
.unwrapErr());
2542 FlushTreeOpsAndDisarmTimer();
2544 mTreeBuilder
->SetOpSink(speculation
);
2545 mSpeculations
.AppendElement(speculation
); // adopts the pointer
2546 mSpeculating
= true;
2548 if (IsTerminatedOrInterrupted()) {
2552 if (mLookingForMetaCharset
) {
2553 Unused
<< ProcessLookingForMetaCharset(false);
2558 class nsHtml5StreamParserContinuation
: public Runnable
{
2560 nsHtml5StreamParserPtr mStreamParser
;
2563 explicit nsHtml5StreamParserContinuation(nsHtml5StreamParser
* aStreamParser
)
2564 : Runnable("nsHtml5StreamParserContinuation"),
2565 mStreamParser(aStreamParser
) {}
2566 NS_IMETHOD
Run() override
{
2567 mozilla::MutexAutoLock
autoLock(mStreamParser
->mTokenizerMutex
);
2568 mStreamParser
->Uninterrupt();
2569 mStreamParser
->ParseAvailableData();
2574 void nsHtml5StreamParser::ContinueAfterScriptsOrEncodingCommitment(
2575 nsHtml5Tokenizer
* aTokenizer
, nsHtml5TreeBuilder
* aTreeBuilder
,
2577 // nullptr for aTokenizer means encoding commitment as opposed to the "after
2580 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
2581 MOZ_ASSERT(mMode
!= VIEW_SOURCE_XML
,
2582 "ContinueAfterScriptsOrEncodingCommitment called in XML view "
2584 MOZ_ASSERT(!(aTokenizer
&& mMode
== VIEW_SOURCE_HTML
),
2585 "ContinueAfterScriptsOrEncodingCommitment called with non-null "
2586 "tokenizer in HTML view "
2588 if (NS_FAILED(mExecutor
->IsBroken())) {
2591 MOZ_ASSERT(!(aTokenizer
&& mMode
!= NORMAL
),
2592 "We should only be executing scripts in the normal mode.");
2593 if (!aTokenizer
&& (mMode
== PLAIN_TEXT
|| mMode
== VIEW_SOURCE_PLAIN
||
2594 mMode
== VIEW_SOURCE_HTML
)) {
2595 // Take the ops that were generated from OnStartRequest for the synthetic
2596 // head section of the document for plain text and HTML View Source.
2597 // XML View Source never needs this kind of encoding commitment.
2598 // We need to take the ops here so that they end up in the queue before
2599 // the ops that we take from a speculation later in this method.
2600 if (!mExecutor
->TakeOpsFromStage()) {
2601 mExecutor
->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2606 mExecutor
->AssertStageEmpty();
2609 bool speculationFailed
= false;
2611 mozilla::MutexAutoLock
speculationAutoLock(mSpeculationMutex
);
2612 if (mSpeculations
.IsEmpty()) {
2613 MOZ_ASSERT_UNREACHABLE(
2614 "ContinueAfterScriptsOrEncodingCommitment called without "
2619 const auto& speculation
= mSpeculations
.ElementAt(0);
2621 (aLastWasCR
|| !aTokenizer
->isInDataState() ||
2622 !aTreeBuilder
->snapshotMatches(speculation
->GetSnapshot()))) {
2623 speculationFailed
= true;
2624 // We've got a failed speculation :-(
2625 MaybeDisableFutureSpeculation();
2626 Interrupt(); // Make the parser thread release the tokenizer mutex sooner
2627 // Note that the interrupted state continues across possible intervening
2628 // Necko events until the nsHtml5StreamParserContinuation posted at the
2629 // end of this method runs. Therefore, this thread is guaranteed to
2630 // acquire mTokenizerMutex soon even if an intervening Necko event grabbed
2631 // it between now and the acquisition below.
2633 // now fall out of the speculationAutoLock into the tokenizerAutoLock
2636 // We've got a successful speculation!
2637 if (mSpeculations
.Length() > 1) {
2638 // the first speculation isn't the current speculation, so there's
2639 // no need to bother the parser thread.
2640 if (!speculation
->FlushToSink(mExecutor
)) {
2641 mExecutor
->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2644 MOZ_ASSERT(!mExecutor
->IsScriptExecuting(),
2645 "ParseUntilBlocked() was supposed to ensure we don't come "
2646 "here when scripts are executing.");
2647 MOZ_ASSERT(!aTokenizer
|| mExecutor
->IsInFlushLoop(),
2648 "How are we here if "
2649 "RunFlushLoop() didn't call ParseUntilBlocked() or we're "
2650 "not committing to an encoding?");
2651 mSpeculations
.RemoveElementAt(0);
2655 Interrupt(); // Make the parser thread release the tokenizer mutex sooner
2656 // Note that the interrupted state continues across possible intervening
2657 // Necko events until the nsHtml5StreamParserContinuation posted at the
2658 // end of this method runs. Therefore, this thread is guaranteed to
2659 // acquire mTokenizerMutex soon even if an intervening Necko event grabbed
2660 // it between now and the acquisition below.
2663 // the first speculation is the current speculation. Need to
2664 // release the the speculation mutex and acquire the tokenizer
2665 // mutex. (Just acquiring the other mutex here would deadlock)
2669 mozilla::MutexAutoLock
tokenizerAutoLock(mTokenizerMutex
);
2672 mAtomTable
.SetPermittedLookupEventTarget(
2673 GetMainThreadSerialEventTarget());
2676 // In principle, the speculation mutex should be acquired here,
2677 // but there's no point, because the parser thread only acquires it
2678 // when it has also acquired the tokenizer mutex and we are already
2679 // holding the tokenizer mutex.
2680 if (speculationFailed
) {
2681 MOZ_ASSERT(mMode
== NORMAL
);
2682 // Rewind the stream
2684 const auto& speculation
= mSpeculations
.ElementAt(0);
2685 mFirstBuffer
= speculation
->GetBuffer();
2686 mFirstBuffer
->setStart(speculation
->GetStart());
2687 mTokenizer
->setLineNumber(speculation
->GetStartLineNumber());
2688 mTokenizer
->setColumnNumberAndResetNextLine(
2689 speculation
->GetStartColumnNumber());
2691 nsContentUtils::ReportToConsole(
2692 nsIScriptError::warningFlag
, "DOM Events"_ns
,
2693 mExecutor
->GetDocument(), nsContentUtils::eDOM_PROPERTIES
,
2694 "SpeculationFailed2", nsTArray
<nsString
>(), nullptr, u
""_ns
,
2695 speculation
->GetStartLineNumber(),
2696 speculation
->GetStartColumnNumber());
2698 nsHtml5OwningUTF16Buffer
* buffer
= mFirstBuffer
->next
;
2700 buffer
->setStart(0);
2701 buffer
= buffer
->next
;
2704 mSpeculations
.Clear(); // potentially a huge number of destructors
2705 // run here synchronously on the main thread...
2707 mTreeBuilder
->flushCharacters(); // empty the pending buffer
2708 mTreeBuilder
->ClearOps(); // now get rid of the failed ops
2710 mTreeBuilder
->SetOpSink(mExecutor
->GetStage());
2711 mExecutor
->StartReadingFromStage();
2712 mSpeculating
= false;
2715 mLastWasCR
= aLastWasCR
;
2716 mTokenizer
->loadState(aTokenizer
);
2717 mTreeBuilder
->loadState(aTreeBuilder
);
2719 // We've got a successful speculation and at least a moment ago it was
2720 // the current speculation
2721 if (!mSpeculations
.ElementAt(0)->FlushToSink(mExecutor
)) {
2722 mExecutor
->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY
);
2725 MOZ_ASSERT(!mExecutor
->IsScriptExecuting(),
2726 "ParseUntilBlocked() was supposed to ensure we don't come "
2727 "here when scripts are executing.");
2728 MOZ_ASSERT(!aTokenizer
|| mExecutor
->IsInFlushLoop(),
2729 "How are we here if "
2730 "RunFlushLoop() didn't call ParseUntilBlocked() or we're not "
2731 "committing to an encoding?");
2732 mSpeculations
.RemoveElementAt(0);
2733 if (mSpeculations
.IsEmpty()) {
2734 if (mMode
== VIEW_SOURCE_HTML
) {
2735 // If we looked for meta charset in the HTML View Source case.
2736 mTokenizer
->SetViewSourceOpSink(mExecutor
->GetStage());
2738 // yes, it was still the only speculation. Now stop speculating
2739 // However, before telling the executor to read from stage, flush
2740 // any pending ops straight to the executor, because otherwise
2741 // they remain unflushed until we get more data from the network.
2742 mTreeBuilder
->SetOpSink(mExecutor
);
2743 auto r
= mTreeBuilder
->Flush(true);
2745 mExecutor
->MarkAsBroken(r
.unwrapErr());
2748 mTreeBuilder
->SetOpSink(mExecutor
->GetStage());
2750 mExecutor
->StartReadingFromStage();
2751 mSpeculating
= false;
2754 nsCOMPtr
<nsIRunnable
> event
= new nsHtml5StreamParserContinuation(this);
2755 if (NS_FAILED(mEventTarget
->Dispatch(event
, nsIThread::DISPATCH_NORMAL
))) {
2756 NS_WARNING("Failed to dispatch nsHtml5StreamParserContinuation");
2758 // A stream event might run before this event runs, but that's harmless.
2760 mAtomTable
.SetPermittedLookupEventTarget(mEventTarget
);
2765 void nsHtml5StreamParser::ContinueAfterFailedCharsetSwitch() {
2766 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
2767 nsCOMPtr
<nsIRunnable
> event
= new nsHtml5StreamParserContinuation(this);
2768 if (NS_FAILED(mEventTarget
->Dispatch(event
, nsIThread::DISPATCH_NORMAL
))) {
2769 NS_WARNING("Failed to dispatch nsHtml5StreamParserContinuation");
2773 class nsHtml5TimerKungFu
: public Runnable
{
2775 nsHtml5StreamParserPtr mStreamParser
;
2778 explicit nsHtml5TimerKungFu(nsHtml5StreamParser
* aStreamParser
)
2779 : Runnable("nsHtml5TimerKungFu"), mStreamParser(aStreamParser
) {}
2780 NS_IMETHOD
Run() override
{
2781 mozilla::MutexAutoLock
flushTimerLock(mStreamParser
->mFlushTimerMutex
);
2782 if (mStreamParser
->mFlushTimer
) {
2783 mStreamParser
->mFlushTimer
->Cancel();
2784 mStreamParser
->mFlushTimer
= nullptr;
2790 void nsHtml5StreamParser::DropTimer() {
2791 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
2793 * Simply nulling out the timer wouldn't work, because if the timer is
2794 * armed, it needs to be canceled first. Simply canceling it first wouldn't
2795 * work, because nsTimerImpl::Cancel is not safe for calling from outside
2796 * the thread where nsTimerImpl::Fire would run. It's not safe to
2797 * dispatch a runnable to cancel the timer from the destructor of this
2798 * class, because the timer has a weak (void*) pointer back to this instance
2799 * of the stream parser and having the timer fire before the runnable
2800 * cancels it would make the timer access a deleted object.
2802 * This DropTimer method addresses these issues. This method must be called
2803 * on the main thread before the destructor of this class is reached.
2804 * The nsHtml5TimerKungFu object has an nsHtml5StreamParserPtr that addrefs
2806 * stream parser object to keep it alive until the runnable is done.
2807 * The runnable cancels the timer on the parser thread, drops the timer
2808 * and lets nsHtml5StreamParserPtr send a runnable back to the main thread to
2809 * release the stream parser.
2811 mozilla::MutexAutoLock
flushTimerLock(mFlushTimerMutex
);
2813 nsCOMPtr
<nsIRunnable
> event
= new nsHtml5TimerKungFu(this);
2814 if (NS_FAILED(mEventTarget
->Dispatch(event
, nsIThread::DISPATCH_NORMAL
))) {
2815 NS_WARNING("Failed to dispatch TimerKungFu event");
2820 // Using a static, because the method name Notify is taken by the chardet
2822 void nsHtml5StreamParser::TimerCallback(nsITimer
* aTimer
, void* aClosure
) {
2823 (static_cast<nsHtml5StreamParser
*>(aClosure
))->TimerFlush();
2826 void nsHtml5StreamParser::TimerFlush() {
2827 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
2828 mozilla::MutexAutoLock
autoLock(mTokenizerMutex
);
2830 MOZ_ASSERT(!mSpeculating
, "Flush timer fired while speculating.");
2832 // The timer fired if we got here. No need to cancel it. Mark it as
2833 // not armed, though.
2834 mFlushTimerArmed
= false;
2836 mFlushTimerEverFired
= true;
2838 if (IsTerminatedOrInterrupted()) {
2842 if (mMode
== VIEW_SOURCE_HTML
|| mMode
== VIEW_SOURCE_XML
) {
2843 auto r
= mTreeBuilder
->Flush(); // delete useless ops
2845 MarkAsBroken(r
.unwrapErr());
2848 r
= mTokenizer
->FlushViewSource();
2850 MarkAsBroken(r
.unwrapErr());
2854 nsCOMPtr
<nsIRunnable
> runnable(mExecutorFlusher
);
2855 if (NS_FAILED(DispatchToMain(runnable
.forget()))) {
2856 NS_WARNING("failed to dispatch executor flush event");
2860 // we aren't speculating and we don't know when new data is
2861 // going to arrive. Send data to the main thread.
2862 auto r
= mTreeBuilder
->Flush(true);
2864 MarkAsBroken(r
.unwrapErr());
2868 nsCOMPtr
<nsIRunnable
> runnable(mExecutorFlusher
);
2869 if (NS_FAILED(DispatchToMain(runnable
.forget()))) {
2870 NS_WARNING("failed to dispatch executor flush event");
2876 void nsHtml5StreamParser::MarkAsBroken(nsresult aRv
) {
2877 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
2878 mTokenizerMutex
.AssertCurrentThreadOwns();
2881 mTreeBuilder
->MarkAsBroken(aRv
);
2882 auto r
= mTreeBuilder
->Flush(false);
2884 MOZ_ASSERT(r
.unwrap(), "Should have had the markAsBroken op!");
2886 MOZ_CRASH("OOM prevents propagation of OOM state");
2888 nsCOMPtr
<nsIRunnable
> runnable(mExecutorFlusher
);
2889 if (NS_FAILED(DispatchToMain(runnable
.forget()))) {
2890 NS_WARNING("failed to dispatch executor flush event");
2894 nsresult
nsHtml5StreamParser::DispatchToMain(
2895 already_AddRefed
<nsIRunnable
>&& aRunnable
) {
2896 return SchedulerGroup::Dispatch(std::move(aRunnable
));