1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set sw=2 ts=8 et tw=80 : */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "nsHTTPCompressConv.h"
11 #include "nsIThreadRetargetableStreamListener.h"
12 #include "nsStreamUtils.h"
13 #include "nsStringStream.h"
14 #include "nsComponentManagerUtils.h"
15 #include "nsThreadUtils.h"
16 #include "mozilla/Preferences.h"
17 #include "mozilla/StaticPrefs_network.h"
18 #include "mozilla/Logging.h"
19 #include "nsIForcePendingChannel.h"
20 #include "nsIRequest.h"
21 #include "mozilla/UniquePtrExtensions.h"
27 #include "brotli/decode.h"
32 extern LazyLogModule gHttpLog
;
34 MOZ_LOG(mozilla::net::gHttpLog, mozilla::LogLevel::Debug, args)
39 BrotliDecoderStateInit(&mState
, nullptr, nullptr, nullptr);
41 ~BrotliWrapper() { BrotliDecoderStateCleanup(&mState
); }
43 BrotliDecoderState mState
{};
44 Atomic
<size_t, Relaxed
> mTotalOut
{0};
45 nsresult mStatus
= NS_OK
;
46 Atomic
<bool, Relaxed
> mBrotliStateIsStreamEnd
{false};
48 nsIRequest
* mRequest
{nullptr};
49 nsISupports
* mContext
{nullptr};
50 uint64_t mSourceOffset
{0};
53 // nsISupports implementation
54 NS_IMPL_ISUPPORTS(nsHTTPCompressConv
, nsIStreamConverter
, nsIStreamListener
,
55 nsIRequestObserver
, nsICompressConvStats
,
56 nsIThreadRetargetableStreamListener
)
58 // nsFTPDirListingConv methods
59 nsHTTPCompressConv::nsHTTPCompressConv() {
60 LOG(("nsHttpCompresssConv %p ctor\n", this));
61 if (NS_IsMainThread()) {
63 Preferences::GetBool("network.http.enforce-framing.http", false);
65 mFailUncleanStops
= false;
69 nsHTTPCompressConv::~nsHTTPCompressConv() {
70 LOG(("nsHttpCompresssConv %p dtor\n", this));
79 // For some reason we are not getting Z_STREAM_END. But this was also seen
80 // for mozilla bug 198133. Need to handle this case.
81 if (mStreamInitialized
&& !mStreamEnded
) {
82 inflateEnd(&d_stream
);
87 nsHTTPCompressConv::GetDecodedDataLength(uint64_t* aDecodedDataLength
) {
88 *aDecodedDataLength
= mDecodedDataLength
;
93 nsHTTPCompressConv::AsyncConvertData(const char* aFromType
, const char* aToType
,
94 nsIStreamListener
* aListener
,
96 if (!nsCRT::strncasecmp(aFromType
, HTTP_COMPRESS_TYPE
,
97 sizeof(HTTP_COMPRESS_TYPE
) - 1) ||
98 !nsCRT::strncasecmp(aFromType
, HTTP_X_COMPRESS_TYPE
,
99 sizeof(HTTP_X_COMPRESS_TYPE
) - 1)) {
100 mMode
= HTTP_COMPRESS_COMPRESS
;
101 } else if (!nsCRT::strncasecmp(aFromType
, HTTP_GZIP_TYPE
,
102 sizeof(HTTP_GZIP_TYPE
) - 1) ||
103 !nsCRT::strncasecmp(aFromType
, HTTP_X_GZIP_TYPE
,
104 sizeof(HTTP_X_GZIP_TYPE
) - 1)) {
105 mMode
= HTTP_COMPRESS_GZIP
;
106 } else if (!nsCRT::strncasecmp(aFromType
, HTTP_DEFLATE_TYPE
,
107 sizeof(HTTP_DEFLATE_TYPE
) - 1)) {
108 mMode
= HTTP_COMPRESS_DEFLATE
;
109 } else if (!nsCRT::strncasecmp(aFromType
, HTTP_BROTLI_TYPE
,
110 sizeof(HTTP_BROTLI_TYPE
) - 1)) {
111 mMode
= HTTP_COMPRESS_BROTLI
;
113 LOG(("nsHttpCompresssConv %p AsyncConvertData %s %s mode %d\n", this,
114 aFromType
, aToType
, (CompressMode
)mMode
));
116 MutexAutoLock
lock(mMutex
);
117 // hook ourself up with the receiving listener.
118 mListener
= aListener
;
124 nsHTTPCompressConv::GetConvertedType(const nsACString
& aFromType
,
125 nsIChannel
* aChannel
,
126 nsACString
& aToType
) {
127 return NS_ERROR_NOT_IMPLEMENTED
;
131 nsHTTPCompressConv::OnStartRequest(nsIRequest
* request
) {
132 LOG(("nsHttpCompresssConv %p onstart\n", this));
133 nsCOMPtr
<nsIStreamListener
> listener
;
135 MutexAutoLock
lock(mMutex
);
136 listener
= mListener
;
138 return listener
->OnStartRequest(request
);
142 nsHTTPCompressConv::OnStopRequest(nsIRequest
* request
, nsresult aStatus
) {
143 nsresult status
= aStatus
;
144 LOG(("nsHttpCompresssConv %p onstop %" PRIx32
"\n", this,
145 static_cast<uint32_t>(aStatus
)));
147 // Framing integrity is enforced for content-encoding: gzip, but not for
148 // content-encoding: deflate. Note that gzip vs deflate is NOT determined
149 // by content sniffing but only via header.
150 if (!mStreamEnded
&& NS_SUCCEEDED(status
) &&
151 (mFailUncleanStops
&& (mMode
== HTTP_COMPRESS_GZIP
))) {
152 // This is not a clean end of gzip stream: the transfer is incomplete.
153 status
= NS_ERROR_NET_PARTIAL_TRANSFER
;
154 LOG(("nsHttpCompresssConv %p onstop partial gzip\n", this));
156 if (NS_SUCCEEDED(status
) && mMode
== HTTP_COMPRESS_BROTLI
) {
157 nsCOMPtr
<nsIForcePendingChannel
> fpChannel
= do_QueryInterface(request
);
158 bool isPending
= false;
160 request
->IsPending(&isPending
);
162 if (fpChannel
&& !isPending
) {
163 fpChannel
->ForcePending(true);
165 bool allowTruncatedEmpty
=
166 StaticPrefs::network_compress_allow_truncated_empty_brotli();
167 if (mBrotli
&& ((allowTruncatedEmpty
&& NS_FAILED(mBrotli
->mStatus
)) ||
168 (!allowTruncatedEmpty
&& mBrotli
->mTotalOut
== 0 &&
169 !mBrotli
->mBrotliStateIsStreamEnd
))) {
170 status
= NS_ERROR_INVALID_CONTENT_ENCODING
;
172 LOG(("nsHttpCompresssConv %p onstop brotlihandler rv %" PRIx32
"\n", this,
173 static_cast<uint32_t>(status
)));
174 if (fpChannel
&& !isPending
) {
175 fpChannel
->ForcePending(false);
179 nsCOMPtr
<nsIStreamListener
> listener
;
181 MutexAutoLock
lock(mMutex
);
182 listener
= mListener
;
184 return listener
->OnStopRequest(request
, status
);
188 nsresult
nsHTTPCompressConv::BrotliHandler(nsIInputStream
* stream
,
189 void* closure
, const char* dataIn
,
190 uint32_t, uint32_t aAvail
,
191 uint32_t* countRead
) {
193 nsHTTPCompressConv
* self
= static_cast<nsHTTPCompressConv
*>(closure
);
196 const size_t kOutSize
= 128 * 1024; // just a chunk size, we call in a loop
199 size_t avail
= aAvail
;
200 BrotliDecoderResult res
;
202 if (!self
->mBrotli
) {
207 auto outBuffer
= MakeUniqueFallible
<uint8_t[]>(kOutSize
);
208 if (outBuffer
== nullptr) {
209 self
->mBrotli
->mStatus
= NS_ERROR_OUT_OF_MEMORY
;
210 return self
->mBrotli
->mStatus
;
214 outPtr
= outBuffer
.get();
216 // brotli api is documented in brotli/dec/decode.h and brotli/dec/decode.c
217 LOG(("nsHttpCompresssConv %p brotlihandler decompress %zu\n", self
, avail
));
218 size_t totalOut
= self
->mBrotli
->mTotalOut
;
219 res
= ::BrotliDecoderDecompressStream(
220 &self
->mBrotli
->mState
, &avail
,
221 reinterpret_cast<const unsigned char**>(&dataIn
), &outSize
, &outPtr
,
223 outSize
= kOutSize
- outSize
;
224 self
->mBrotli
->mTotalOut
= totalOut
;
225 self
->mBrotli
->mBrotliStateIsStreamEnd
=
226 BrotliDecoderIsFinished(&self
->mBrotli
->mState
);
227 LOG(("nsHttpCompresssConv %p brotlihandler decompress rv=%" PRIx32
229 self
, static_cast<uint32_t>(res
), outSize
));
231 if (res
== BROTLI_DECODER_RESULT_ERROR
) {
232 LOG(("nsHttpCompressConv %p marking invalid encoding", self
));
233 self
->mBrotli
->mStatus
= NS_ERROR_INVALID_CONTENT_ENCODING
;
234 return self
->mBrotli
->mStatus
;
237 // in 'the current implementation' brotli must consume everything before
238 // asking for more input
239 if (res
== BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT
) {
242 LOG(("nsHttpCompressConv %p did not consume all input", self
));
243 self
->mBrotli
->mStatus
= NS_ERROR_UNEXPECTED
;
244 return self
->mBrotli
->mStatus
;
248 auto callOnDataAvailable
= [&](uint64_t aSourceOffset
, const char* aBuffer
,
250 nsresult rv
= self
->do_OnDataAvailable(self
->mBrotli
->mRequest
,
251 aSourceOffset
, aBuffer
, aCount
);
252 LOG(("nsHttpCompressConv %p BrotliHandler ODA rv=%" PRIx32
, self
,
253 static_cast<uint32_t>(rv
)));
255 self
->mBrotli
->mStatus
= rv
;
262 if (NS_FAILED(callOnDataAvailable(
263 self
->mBrotli
->mSourceOffset
,
264 reinterpret_cast<const char*>(outBuffer
.get()), outSize
))) {
265 return self
->mBrotli
->mStatus
;
267 self
->mBrotli
->mSourceOffset
+= outSize
;
270 // See bug 1759745. If the decoder has more output data, take it.
271 while (::BrotliDecoderHasMoreOutput(&self
->mBrotli
->mState
)) {
273 const uint8_t* buffer
=
274 ::BrotliDecoderTakeOutput(&self
->mBrotli
->mState
, &outSize
);
275 if (NS_FAILED(callOnDataAvailable(self
->mBrotli
->mSourceOffset
,
276 reinterpret_cast<const char*>(buffer
),
278 return self
->mBrotli
->mStatus
;
280 self
->mBrotli
->mSourceOffset
+= outSize
;
283 if (res
== BROTLI_DECODER_RESULT_SUCCESS
||
284 res
== BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT
) {
288 MOZ_ASSERT(res
== BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT
);
289 } while (res
== BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT
);
291 self
->mBrotli
->mStatus
= NS_ERROR_UNEXPECTED
;
292 return self
->mBrotli
->mStatus
;
296 nsHTTPCompressConv::OnDataAvailable(nsIRequest
* request
, nsIInputStream
* iStr
,
297 uint64_t aSourceOffset
, uint32_t aCount
) {
298 nsresult rv
= NS_ERROR_INVALID_CONTENT_ENCODING
;
299 uint32_t streamLen
= aCount
;
300 LOG(("nsHttpCompressConv %p OnDataAvailable aSourceOffset:%" PRIu64
302 this, aSourceOffset
, aCount
));
304 if (streamLen
== 0) {
305 NS_ERROR("count of zero passed to OnDataAvailable");
306 return NS_ERROR_UNEXPECTED
;
310 // Hmm... this may just indicate that the data stream is done and that
311 // what's left is either metadata or padding of some sort.... throwing
312 // it out is probably the safe thing to do.
314 return iStr
->ReadSegments(NS_DiscardSegment
, nullptr, streamLen
, &n
);
318 case HTTP_COMPRESS_GZIP
:
319 streamLen
= check_header(iStr
, streamLen
, &rv
);
325 if (streamLen
== 0) {
331 case HTTP_COMPRESS_DEFLATE
:
333 if (mInpBuffer
!= nullptr && streamLen
> mInpBufferLen
) {
334 unsigned char* originalInpBuffer
= mInpBuffer
;
335 if (!(mInpBuffer
= (unsigned char*)realloc(
336 originalInpBuffer
, mInpBufferLen
= streamLen
))) {
337 free(originalInpBuffer
);
340 if (mOutBufferLen
< streamLen
* 2) {
341 unsigned char* originalOutBuffer
= mOutBuffer
;
342 if (!(mOutBuffer
= (unsigned char*)realloc(
343 mOutBuffer
, mOutBufferLen
= streamLen
* 3))) {
344 free(originalOutBuffer
);
348 if (mInpBuffer
== nullptr || mOutBuffer
== nullptr) {
349 return NS_ERROR_OUT_OF_MEMORY
;
353 if (mInpBuffer
== nullptr) {
354 mInpBuffer
= (unsigned char*)malloc(mInpBufferLen
= streamLen
);
357 if (mOutBuffer
== nullptr) {
358 mOutBuffer
= (unsigned char*)malloc(mOutBufferLen
= streamLen
* 3);
361 if (mInpBuffer
== nullptr || mOutBuffer
== nullptr) {
362 return NS_ERROR_OUT_OF_MEMORY
;
366 iStr
->Read((char*)mInpBuffer
, streamLen
, &unused
);
368 if (mMode
== HTTP_COMPRESS_DEFLATE
) {
369 if (!mStreamInitialized
) {
370 memset(&d_stream
, 0, sizeof(d_stream
));
372 if (inflateInit(&d_stream
) != Z_OK
) {
373 return NS_ERROR_FAILURE
;
376 mStreamInitialized
= true;
378 d_stream
.next_in
= mInpBuffer
;
379 d_stream
.avail_in
= (uInt
)streamLen
;
381 mDummyStreamInitialised
= false;
383 d_stream
.next_out
= mOutBuffer
;
384 d_stream
.avail_out
= (uInt
)mOutBufferLen
;
386 int code
= inflate(&d_stream
, Z_NO_FLUSH
);
387 unsigned bytesWritten
= (uInt
)mOutBufferLen
- d_stream
.avail_out
;
389 if (code
== Z_STREAM_END
) {
391 rv
= do_OnDataAvailable(request
, aSourceOffset
, (char*)mOutBuffer
,
398 inflateEnd(&d_stream
);
404 rv
= do_OnDataAvailable(request
, aSourceOffset
, (char*)mOutBuffer
,
410 } else if (code
== Z_BUF_ERROR
) {
412 rv
= do_OnDataAvailable(request
, aSourceOffset
, (char*)mOutBuffer
,
419 } else if (code
== Z_DATA_ERROR
) {
420 // some servers (notably Apache with mod_deflate) don't generate
421 // zlib headers insert a dummy header and try again
422 static char dummy_head
[2] = {
424 (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF,
426 inflateReset(&d_stream
);
427 d_stream
.next_in
= (Bytef
*)dummy_head
;
428 d_stream
.avail_in
= sizeof(dummy_head
);
430 code
= inflate(&d_stream
, Z_NO_FLUSH
);
432 return NS_ERROR_FAILURE
;
435 // stop an endless loop caused by non-deflate data being labelled as
437 if (mDummyStreamInitialised
) {
439 "endless loop detected"
440 " - invalid deflate");
441 return NS_ERROR_INVALID_CONTENT_ENCODING
;
443 mDummyStreamInitialised
= true;
444 // reset stream pointers to our original data
445 d_stream
.next_in
= mInpBuffer
;
446 d_stream
.avail_in
= (uInt
)streamLen
;
448 return NS_ERROR_INVALID_CONTENT_ENCODING
;
452 if (!mStreamInitialized
) {
453 memset(&d_stream
, 0, sizeof(d_stream
));
455 if (inflateInit2(&d_stream
, -MAX_WBITS
) != Z_OK
) {
456 return NS_ERROR_FAILURE
;
459 mStreamInitialized
= true;
462 d_stream
.next_in
= mInpBuffer
;
463 d_stream
.avail_in
= (uInt
)streamLen
;
466 d_stream
.next_out
= mOutBuffer
;
467 d_stream
.avail_out
= (uInt
)mOutBufferLen
;
469 int code
= inflate(&d_stream
, Z_NO_FLUSH
);
470 unsigned bytesWritten
= (uInt
)mOutBufferLen
- d_stream
.avail_out
;
472 if (code
== Z_STREAM_END
) {
474 rv
= do_OnDataAvailable(request
, aSourceOffset
, (char*)mOutBuffer
,
481 inflateEnd(&d_stream
);
487 rv
= do_OnDataAvailable(request
, aSourceOffset
, (char*)mOutBuffer
,
493 } else if (code
== Z_BUF_ERROR
) {
495 rv
= do_OnDataAvailable(request
, aSourceOffset
, (char*)mOutBuffer
,
503 return NS_ERROR_INVALID_CONTENT_ENCODING
;
509 case HTTP_COMPRESS_BROTLI
: {
511 mBrotli
= MakeUnique
<BrotliWrapper
>();
514 mBrotli
->mRequest
= request
;
515 mBrotli
->mContext
= nullptr;
516 mBrotli
->mSourceOffset
= aSourceOffset
;
519 rv
= iStr
->ReadSegments(BrotliHandler
, this, streamLen
, &countRead
);
520 if (NS_SUCCEEDED(rv
)) {
521 rv
= mBrotli
->mStatus
;
529 nsCOMPtr
<nsIStreamListener
> listener
;
531 MutexAutoLock
lock(mMutex
);
532 listener
= mListener
;
534 rv
= listener
->OnDataAvailable(request
, iStr
, aSourceOffset
, aCount
);
541 } /* OnDataAvailable */
543 // XXX/ruslan: need to implement this too
546 nsHTTPCompressConv::Convert(nsIInputStream
* aFromStream
, const char* aFromType
,
547 const char* aToType
, nsISupports
* aCtxt
,
548 nsIInputStream
** _retval
) {
549 return NS_ERROR_NOT_IMPLEMENTED
;
552 nsresult
nsHTTPCompressConv::do_OnDataAvailable(nsIRequest
* request
,
557 mStream
= do_CreateInstance(NS_STRINGINPUTSTREAM_CONTRACTID
);
558 NS_ENSURE_STATE(mStream
);
561 mStream
->ShareData(buffer
, count
);
563 nsCOMPtr
<nsIStreamListener
> listener
;
565 MutexAutoLock
lock(mMutex
);
566 listener
= mListener
;
568 LOG(("nsHTTPCompressConv::do_OnDataAvailable req:%p offset: offset:%" PRIu64
570 request
, offset
, count
));
571 nsresult rv
= listener
->OnDataAvailable(request
, mStream
, offset
, count
);
573 // Make sure the stream no longer references |buffer| in case our listener
574 // is crazy enough to try to read from |mStream| after ODA.
575 mStream
->ShareData("", 0);
576 mDecodedDataLength
+= count
;
581 #define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */
582 #define HEAD_CRC 0x02 /* bit 1 set: header CRC present */
583 #define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */
584 #define ORIG_NAME 0x08 /* bit 3 set: original file name present */
585 #define COMMENT 0x10 /* bit 4 set: file comment present */
586 #define RESERVED 0xE0 /* bits 5..7: reserved */
588 static unsigned gz_magic
[2] = {0x1f, 0x8b}; /* gzip magic header */
590 uint32_t nsHTTPCompressConv::check_header(nsIInputStream
* iStr
,
591 uint32_t streamLen
, nsresult
* rs
) {
606 if (mCheckHeaderDone
) {
614 iStr
->Read(&c
, 1, &unused
);
617 if (mSkipCount
== 0 && ((unsigned)c
& 0377) != gz_magic
[0]) {
618 *rs
= NS_ERROR_INVALID_CONTENT_ENCODING
;
622 if (mSkipCount
== 1 && ((unsigned)c
& 0377) != gz_magic
[1]) {
623 *rs
= NS_ERROR_INVALID_CONTENT_ENCODING
;
627 if (mSkipCount
== 2 && ((unsigned)c
& 0377) != Z_DEFLATED
) {
628 *rs
= NS_ERROR_INVALID_CONTENT_ENCODING
;
633 if (mSkipCount
== 4) {
634 mFlags
= (unsigned)c
& 0377;
635 if (mFlags
& RESERVED
) {
636 *rs
= NS_ERROR_INVALID_CONTENT_ENCODING
;
645 iStr
->Read(&c
, 1, &unused
);
649 if (mSkipCount
== 6) {
655 if (mFlags
& EXTRA_FIELD
) {
656 iStr
->Read(&c
, 1, &unused
);
658 mLen
= (uInt
)c
& 0377;
666 iStr
->Read(&c
, 1, &unused
);
668 mLen
|= ((uInt
)c
& 0377) << 8;
674 if (mSkipCount
== mLen
) {
677 iStr
->Read(&c
, 1, &unused
);
684 if (mFlags
& ORIG_NAME
) {
685 iStr
->Read(&c
, 1, &unused
);
687 if (c
== 0) hMode
= GZIP_COMMENT
;
689 hMode
= GZIP_COMMENT
;
694 if (mFlags
& COMMENT
) {
695 iStr
->Read(&c
, 1, &unused
);
708 if (mFlags
& HEAD_CRC
) {
709 iStr
->Read(&c
, 1, &unused
);
712 if (mSkipCount
== 2) {
713 mCheckHeaderDone
= true;
717 mCheckHeaderDone
= true;
727 nsHTTPCompressConv::CheckListenerChain() {
728 nsCOMPtr
<nsIThreadRetargetableStreamListener
> listener
;
730 MutexAutoLock
lock(mMutex
);
731 listener
= do_QueryInterface(mListener
);
735 return NS_ERROR_NO_INTERFACE
;
738 return listener
->CheckListenerChain();
742 nsHTTPCompressConv::OnDataFinished(nsresult aStatus
) {
743 nsCOMPtr
<nsIThreadRetargetableStreamListener
> listener
;
746 MutexAutoLock
lock(mMutex
);
747 listener
= do_QueryInterface(mListener
);
751 return listener
->OnDataFinished(aStatus
);
758 } // namespace mozilla
760 nsresult
NS_NewHTTPCompressConv(
761 mozilla::net::nsHTTPCompressConv
** aHTTPCompressConv
) {
762 MOZ_ASSERT(aHTTPCompressConv
!= nullptr, "null ptr");
763 if (!aHTTPCompressConv
) {
764 return NS_ERROR_NULL_POINTER
;
767 RefPtr
<mozilla::net::nsHTTPCompressConv
> outVal
=
768 new mozilla::net::nsHTTPCompressConv();
770 return NS_ERROR_OUT_OF_MEMORY
;
772 outVal
.forget(aHTTPCompressConv
);