1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
8 * The storage stream provides an internal buffer that can be filled by a
9 * client using a single output stream. One or more independent input streams
10 * can be created to read the data out non-destructively. The implementation
11 * uses a segmented buffer internally to avoid realloc'ing of large buffers,
12 * with the attendant performance loss and heap fragmentation.
15 #include "mozilla/Mutex.h"
16 #include "nsAlgorithm.h"
17 #include "nsStorageStream.h"
18 #include "nsSegmentedBuffer.h"
19 #include "nsStreamUtils.h"
21 #include "nsICloneableInputStream.h"
22 #include "nsIInputStream.h"
23 #include "nsIIPCSerializableInputStream.h"
24 #include "nsISeekableStream.h"
25 #include "mozilla/Logging.h"
26 #include "mozilla/Attributes.h"
27 #include "mozilla/Likely.h"
28 #include "mozilla/MathAlgorithms.h"
29 #include "mozilla/ipc/InputStreamUtils.h"
31 using mozilla::MutexAutoLock
;
32 using mozilla::ipc::InputStreamParams
;
33 using mozilla::ipc::StringInputStreamParams
;
36 // Log module for StorageStream logging...
38 // To enable logging (see prlog.h for full details):
40 // set MOZ_LOG=StorageStreamLog:5
41 // set MOZ_LOG_FILE=storage.log
43 // This enables LogLevel::Debug level information and places all output in
44 // the file storage.log.
46 static mozilla::LazyLogModule
sStorageStreamLog("nsStorageStream");
50 #define LOG(args) MOZ_LOG(sStorageStreamLog, mozilla::LogLevel::Debug, args)
52 nsStorageStream::nsStorageStream() {
53 LOG(("Creating nsStorageStream [%p].\n", this));
56 nsStorageStream::~nsStorageStream() { delete mSegmentedBuffer
; }
58 NS_IMPL_ISUPPORTS(nsStorageStream
, nsIStorageStream
, nsIOutputStream
)
61 nsStorageStream::Init(uint32_t aSegmentSize
, uint32_t aMaxSize
) {
62 MutexAutoLock
lock(mMutex
);
63 mSegmentedBuffer
= new nsSegmentedBuffer();
64 mSegmentSize
= aSegmentSize
;
65 mSegmentSizeLog2
= mozilla::FloorLog2(aSegmentSize
);
66 mMaxLogicalLength
= aMaxSize
;
68 // Segment size must be a power of two
69 if (mSegmentSize
!= ((uint32_t)1 << mSegmentSizeLog2
)) {
70 return NS_ERROR_INVALID_ARG
;
73 return mSegmentedBuffer
->Init(aSegmentSize
);
77 nsStorageStream::GetOutputStream(int32_t aStartingOffset
,
78 nsIOutputStream
** aOutputStream
) {
79 if (NS_WARN_IF(!aOutputStream
)) {
80 return NS_ERROR_INVALID_ARG
;
83 MutexAutoLock
lock(mMutex
);
84 if (NS_WARN_IF(!mSegmentedBuffer
)) {
85 return NS_ERROR_NOT_INITIALIZED
;
88 if (mWriteInProgress
) {
89 return NS_ERROR_NOT_AVAILABLE
;
92 if (mActiveSegmentBorrows
> 0) {
93 return NS_ERROR_NOT_AVAILABLE
;
96 nsresult rv
= Seek(aStartingOffset
);
101 // Enlarge the last segment in the buffer so that it is the same size as
102 // all the other segments in the buffer. (It may have been realloc'ed
103 // smaller in the Close() method.)
104 if (mLastSegmentNum
>= 0)
105 if (mSegmentedBuffer
->ReallocLastSegment(mSegmentSize
)) {
106 // Need to re-Seek, since realloc changed segment base pointer
107 rv
= Seek(aStartingOffset
);
114 *aOutputStream
= static_cast<nsIOutputStream
*>(this);
115 mWriteInProgress
= true;
120 nsStorageStream::Close() {
121 MutexAutoLock
lock(mMutex
);
122 if (NS_WARN_IF(!mSegmentedBuffer
)) {
123 return NS_ERROR_NOT_INITIALIZED
;
126 mWriteInProgress
= false;
128 int32_t segmentOffset
= SegOffset(mLogicalLength
);
130 // Shrink the final segment in the segmented buffer to the minimum size
131 // needed to contain the data, so as to conserve memory.
132 if (segmentOffset
&& !mActiveSegmentBorrows
) {
133 mSegmentedBuffer
->ReallocLastSegment(segmentOffset
);
139 LOG(("nsStorageStream [%p] Close mWriteCursor=%p mSegmentEnd=%p\n", this,
140 mWriteCursor
, mSegmentEnd
));
146 nsStorageStream::Flush() { return NS_OK
; }
149 nsStorageStream::StreamStatus() {
150 MutexAutoLock
lock(mMutex
);
151 if (!mSegmentedBuffer
) {
152 return NS_ERROR_NOT_INITIALIZED
;
158 nsStorageStream::Write(const char* aBuffer
, uint32_t aCount
,
159 uint32_t* aNumWritten
) {
160 if (NS_WARN_IF(!aNumWritten
) || NS_WARN_IF(!aBuffer
)) {
161 return NS_ERROR_INVALID_ARG
;
164 MutexAutoLock
lock(mMutex
);
165 if (NS_WARN_IF(!mSegmentedBuffer
)) {
166 return NS_ERROR_NOT_INITIALIZED
;
169 if (NS_WARN_IF(mLogicalLength
>= mMaxLogicalLength
)) {
170 return NS_ERROR_OUT_OF_MEMORY
;
173 LOG(("nsStorageStream [%p] Write mWriteCursor=%p mSegmentEnd=%p aCount=%d\n",
174 this, mWriteCursor
, mSegmentEnd
, aCount
));
176 uint32_t remaining
= aCount
;
177 const char* readCursor
= aBuffer
;
179 remaining
= std::min(remaining
, mMaxLogicalLength
- mLogicalLength
);
181 auto onExit
= mozilla::MakeScopeExit([&] {
182 mMutex
.AssertCurrentThreadOwns();
183 *aNumWritten
= aCount
- remaining
;
184 mLogicalLength
+= *aNumWritten
;
187 ("nsStorageStream [%p] Wrote mWriteCursor=%p mSegmentEnd=%p "
189 this, mWriteCursor
, mSegmentEnd
, *aNumWritten
));
192 // If no segments have been created yet, create one even if we don't have
193 // to write any data; this enables creating an input stream which reads from
194 // the very end of the data for any amount of data in the stream (i.e.
195 // this stream contains N bytes of data and newInputStream(N) is called),
196 // even for N=0 (with the caveat that we require .write("", 0) be called to
197 // initialize internal buffers).
198 bool firstTime
= mSegmentedBuffer
->GetSegmentCount() == 0;
199 while (remaining
|| MOZ_UNLIKELY(firstTime
)) {
201 uint32_t availableInSegment
= mSegmentEnd
- mWriteCursor
;
202 if (!availableInSegment
) {
203 mWriteCursor
= mSegmentedBuffer
->AppendNewSegment();
206 return NS_ERROR_OUT_OF_MEMORY
;
209 mSegmentEnd
= mWriteCursor
+ mSegmentSize
;
210 availableInSegment
= mSegmentEnd
- mWriteCursor
;
212 ("nsStorageStream [%p] Write (new seg) mWriteCursor=%p "
214 this, mWriteCursor
, mSegmentEnd
));
217 uint32_t count
= XPCOM_MIN(availableInSegment
, remaining
);
218 memcpy(mWriteCursor
, readCursor
, count
);
221 mWriteCursor
+= count
;
223 ("nsStorageStream [%p] Writing mWriteCursor=%p mSegmentEnd=%p "
225 this, mWriteCursor
, mSegmentEnd
, count
));
232 nsStorageStream::WriteFrom(nsIInputStream
* aInStr
, uint32_t aCount
,
234 return NS_ERROR_NOT_IMPLEMENTED
;
238 nsStorageStream::WriteSegments(nsReadSegmentFun aReader
, void* aClosure
,
239 uint32_t aCount
, uint32_t* aResult
) {
240 return NS_ERROR_NOT_IMPLEMENTED
;
244 nsStorageStream::IsNonBlocking(bool* aNonBlocking
) {
245 *aNonBlocking
= false;
250 nsStorageStream::GetLength(uint32_t* aLength
) {
251 MutexAutoLock
lock(mMutex
);
252 *aLength
= mLogicalLength
;
257 nsStorageStream::SetLength(uint32_t aLength
) {
258 MutexAutoLock
lock(mMutex
);
259 return SetLengthLocked(aLength
);
262 // Truncate the buffer by deleting the end segments
263 nsresult
nsStorageStream::SetLengthLocked(uint32_t aLength
) {
264 if (NS_WARN_IF(!mSegmentedBuffer
)) {
265 return NS_ERROR_NOT_INITIALIZED
;
268 if (mWriteInProgress
) {
269 return NS_ERROR_NOT_AVAILABLE
;
272 if (mActiveSegmentBorrows
) {
273 return NS_ERROR_NOT_AVAILABLE
;
276 if (aLength
> mLogicalLength
) {
277 return NS_ERROR_INVALID_ARG
;
280 int32_t newLastSegmentNum
= SegNum(aLength
);
281 int32_t segmentOffset
= SegOffset(aLength
);
282 if (segmentOffset
== 0) {
286 while (newLastSegmentNum
< mLastSegmentNum
) {
287 mSegmentedBuffer
->DeleteLastSegment();
291 mLogicalLength
= aLength
;
296 nsStorageStream::GetWriteInProgress(bool* aWriteInProgress
) {
297 MutexAutoLock
lock(mMutex
);
298 *aWriteInProgress
= mWriteInProgress
;
302 nsresult
nsStorageStream::Seek(int32_t aPosition
) {
303 if (NS_WARN_IF(!mSegmentedBuffer
)) {
304 return NS_ERROR_NOT_INITIALIZED
;
307 // An argument of -1 means "seek to end of stream"
308 if (aPosition
== -1) {
309 aPosition
= mLogicalLength
;
312 // Seeking beyond the buffer end is illegal
313 if ((uint32_t)aPosition
> mLogicalLength
) {
314 return NS_ERROR_INVALID_ARG
;
317 // Seeking backwards in the write stream results in truncation
318 SetLengthLocked(aPosition
);
320 // Special handling for seek to start-of-buffer
321 if (aPosition
== 0) {
324 LOG(("nsStorageStream [%p] Seek mWriteCursor=%p mSegmentEnd=%p\n", this,
325 mWriteCursor
, mSegmentEnd
));
329 // Segment may have changed, so reset pointers
330 mWriteCursor
= mSegmentedBuffer
->GetSegment(mLastSegmentNum
);
331 NS_ASSERTION(mWriteCursor
, "null mWriteCursor");
332 mSegmentEnd
= mWriteCursor
+ mSegmentSize
;
334 // Adjust write cursor for current segment offset. This test is necessary
335 // because SegNum may reference the next-to-be-allocated segment, in which
336 // case we need to be pointing at the end of the last segment.
337 int32_t segmentOffset
= SegOffset(aPosition
);
338 if (segmentOffset
== 0 && (SegNum(aPosition
) > (uint32_t)mLastSegmentNum
)) {
339 mWriteCursor
= mSegmentEnd
;
341 mWriteCursor
+= segmentOffset
;
344 LOG(("nsStorageStream [%p] Seek mWriteCursor=%p mSegmentEnd=%p\n", this,
345 mWriteCursor
, mSegmentEnd
));
349 ////////////////////////////////////////////////////////////////////////////////
351 // There can be many nsStorageInputStreams for a single nsStorageStream
352 class nsStorageInputStream final
: public nsIInputStream
,
353 public nsISeekableStream
,
354 public nsIIPCSerializableInputStream
,
355 public nsICloneableInputStream
{
357 nsStorageInputStream(nsStorageStream
* aStorageStream
, uint32_t aSegmentSize
)
358 : mStorageStream(aStorageStream
),
362 mSegmentSize(aSegmentSize
),
366 NS_DECL_THREADSAFE_ISUPPORTS
367 NS_DECL_NSIINPUTSTREAM
368 NS_DECL_NSISEEKABLESTREAM
369 NS_DECL_NSITELLABLESTREAM
370 NS_DECL_NSIIPCSERIALIZABLEINPUTSTREAM
371 NS_DECL_NSICLONEABLEINPUTSTREAM
374 ~nsStorageInputStream() = default;
377 nsresult
Seek(uint32_t aPosition
) MOZ_REQUIRES(mStorageStream
->mMutex
);
379 friend class nsStorageStream
;
382 RefPtr
<nsStorageStream
> mStorageStream
;
383 uint32_t mReadCursor
; // Next memory location to read byte, or 0
384 uint32_t mSegmentEnd
; // One byte past end of current buffer segment
385 uint32_t mSegmentNum
; // Segment number containing read cursor
386 uint32_t mSegmentSize
; // All segments, except the last, are of this size
387 uint32_t mLogicalCursor
; // Logical offset into stream
390 uint32_t SegNum(uint32_t aPosition
) MOZ_REQUIRES(mStorageStream
->mMutex
) {
391 return aPosition
>> mStorageStream
->mSegmentSizeLog2
;
393 uint32_t SegOffset(uint32_t aPosition
) {
394 return aPosition
& (mSegmentSize
- 1);
398 NS_IMPL_ISUPPORTS(nsStorageInputStream
, nsIInputStream
, nsISeekableStream
,
399 nsITellableStream
, nsIIPCSerializableInputStream
,
400 nsICloneableInputStream
)
403 nsStorageStream::NewInputStream(int32_t aStartingOffset
,
404 nsIInputStream
** aInputStream
) {
405 MutexAutoLock
lock(mMutex
);
406 if (NS_WARN_IF(!mSegmentedBuffer
)) {
407 return NS_ERROR_NOT_INITIALIZED
;
410 RefPtr
<nsStorageInputStream
> inputStream
=
411 new nsStorageInputStream(this, mSegmentSize
);
413 inputStream
->mStorageStream
->mMutex
.AssertCurrentThreadOwns();
414 nsresult rv
= inputStream
->Seek(aStartingOffset
);
419 inputStream
.forget(aInputStream
);
424 nsStorageInputStream::Close() {
425 mStatus
= NS_BASE_STREAM_CLOSED
;
430 nsStorageInputStream::Available(uint64_t* aAvailable
) {
431 if (NS_FAILED(mStatus
)) {
435 MutexAutoLock
lock(mStorageStream
->mMutex
);
436 *aAvailable
= mStorageStream
->mLogicalLength
- mLogicalCursor
;
441 nsStorageInputStream::StreamStatus() { return mStatus
; }
444 nsStorageInputStream::Read(char* aBuffer
, uint32_t aCount
, uint32_t* aNumRead
) {
445 return ReadSegments(NS_CopySegmentToBuffer
, aBuffer
, aCount
, aNumRead
);
449 nsStorageInputStream::ReadSegments(nsWriteSegmentFun aWriter
, void* aClosure
,
450 uint32_t aCount
, uint32_t* aNumRead
) {
452 if (mStatus
== NS_BASE_STREAM_CLOSED
) {
455 if (NS_FAILED(mStatus
)) {
459 uint32_t count
, availableInSegment
, remainingCapacity
, bytesConsumed
;
462 remainingCapacity
= aCount
;
463 while (remainingCapacity
) {
464 const char* cur
= nullptr;
466 MutexAutoLock
lock(mStorageStream
->mMutex
);
467 availableInSegment
= mSegmentEnd
- mReadCursor
;
468 if (!availableInSegment
) {
469 uint32_t available
= mStorageStream
->mLogicalLength
- mLogicalCursor
;
474 // We have data in the stream, but if mSegmentEnd is zero, then we
475 // were likely constructed prior to any data being written into
476 // the stream. Therefore, if mSegmentEnd is non-zero, we should
477 // move into the next segment; otherwise, we should stay in this
478 // segment so our input state can be updated and we can properly
479 // perform the initial read.
480 if (mSegmentEnd
> 0) {
484 mSegmentEnd
= XPCOM_MIN(mSegmentSize
, available
);
485 availableInSegment
= mSegmentEnd
;
487 cur
= mStorageStream
->mSegmentedBuffer
->GetSegment(mSegmentNum
);
488 mStorageStream
->mActiveSegmentBorrows
++;
490 auto dropBorrow
= mozilla::MakeScopeExit([&] {
491 MutexAutoLock
lock(mStorageStream
->mMutex
);
492 mStorageStream
->mActiveSegmentBorrows
--;
495 count
= XPCOM_MIN(availableInSegment
, remainingCapacity
);
496 rv
= aWriter(this, aClosure
, cur
+ mReadCursor
, aCount
- remainingCapacity
,
497 count
, &bytesConsumed
);
498 if (NS_FAILED(rv
) || (bytesConsumed
== 0)) {
501 remainingCapacity
-= bytesConsumed
;
502 mReadCursor
+= bytesConsumed
;
503 mLogicalCursor
+= bytesConsumed
;
506 *aNumRead
= aCount
- remainingCapacity
;
508 bool isWriteInProgress
= false;
509 if (NS_FAILED(mStorageStream
->GetWriteInProgress(&isWriteInProgress
))) {
510 isWriteInProgress
= false;
513 if (*aNumRead
== 0 && isWriteInProgress
) {
514 return NS_BASE_STREAM_WOULD_BLOCK
;
521 nsStorageInputStream::IsNonBlocking(bool* aNonBlocking
) {
522 // TODO: This class should implement nsIAsyncInputStream so that callers
523 // have some way of dealing with NS_BASE_STREAM_WOULD_BLOCK errors.
525 *aNonBlocking
= true;
530 nsStorageInputStream::Seek(int32_t aWhence
, int64_t aOffset
) {
531 if (NS_FAILED(mStatus
)) {
535 MutexAutoLock
lock(mStorageStream
->mMutex
);
536 int64_t pos
= aOffset
;
542 pos
+= mLogicalCursor
;
545 pos
+= mStorageStream
->mLogicalLength
;
548 MOZ_ASSERT_UNREACHABLE("unexpected whence value");
549 return NS_ERROR_UNEXPECTED
;
551 if (pos
== int64_t(mLogicalCursor
)) {
559 nsStorageInputStream::Tell(int64_t* aResult
) {
560 if (NS_FAILED(mStatus
)) {
564 *aResult
= mLogicalCursor
;
569 nsStorageInputStream::SetEOF() {
570 MOZ_ASSERT_UNREACHABLE("nsStorageInputStream::SetEOF");
571 return NS_ERROR_NOT_IMPLEMENTED
;
574 nsresult
nsStorageInputStream::Seek(uint32_t aPosition
) {
575 uint32_t length
= mStorageStream
->mLogicalLength
;
576 if (aPosition
> length
) {
577 return NS_ERROR_INVALID_ARG
;
584 mSegmentNum
= SegNum(aPosition
);
585 mReadCursor
= SegOffset(aPosition
);
586 uint32_t available
= length
- aPosition
;
587 mSegmentEnd
= mReadCursor
+ XPCOM_MIN(mSegmentSize
- mReadCursor
, available
);
588 mLogicalCursor
= aPosition
;
592 void nsStorageInputStream::SerializedComplexity(uint32_t aMaxSize
,
595 uint32_t* aTransferables
) {
596 uint64_t remaining
= 0;
597 mozilla::DebugOnly
<nsresult
> rv
= Available(&remaining
);
598 MOZ_ASSERT(NS_SUCCEEDED(rv
));
600 if (remaining
>= aMaxSize
) {
603 *aSizeUsed
= remaining
;
607 void nsStorageInputStream::Serialize(InputStreamParams
& aParams
,
608 uint32_t aMaxSize
, uint32_t* aSizeUsed
) {
609 MOZ_ASSERT(aSizeUsed
);
612 uint64_t remaining
= 0;
613 mozilla::DebugOnly
<nsresult
> rv
= Available(&remaining
);
614 MOZ_ASSERT(NS_SUCCEEDED(rv
));
616 if (remaining
>= aMaxSize
) {
617 mozilla::ipc::InputStreamHelper::SerializeInputStreamAsPipe(this, aParams
);
621 *aSizeUsed
= remaining
;
626 MOZ_ASSERT(NS_SUCCEEDED(rv
));
628 auto handleOrErr
= combined
.BulkWrite(remaining
, 0, false);
629 MOZ_ASSERT(!handleOrErr
.isErr());
631 auto handle
= handleOrErr
.unwrap();
633 uint32_t numRead
= 0;
635 rv
= Read(handle
.Elements(), remaining
, &numRead
);
636 MOZ_ASSERT(NS_SUCCEEDED(rv
));
638 MOZ_ASSERT(numRead
== remaining
);
639 handle
.Finish(numRead
, false);
641 rv
= Seek(NS_SEEK_SET
, offset
);
642 MOZ_ASSERT(NS_SUCCEEDED(rv
));
644 StringInputStreamParams params
;
645 params
.data() = combined
;
649 bool nsStorageInputStream::Deserialize(const InputStreamParams
& aParams
) {
650 MOZ_ASSERT_UNREACHABLE(
651 "We should never attempt to deserialize a storage "
657 nsStorageInputStream::GetCloneable(bool* aCloneableOut
) {
658 *aCloneableOut
= true;
663 nsStorageInputStream::Clone(nsIInputStream
** aCloneOut
) {
664 return mStorageStream
->NewInputStream(mLogicalCursor
, aCloneOut
);
667 nsresult
NS_NewStorageStream(uint32_t aSegmentSize
, uint32_t aMaxSize
,
668 nsIStorageStream
** aResult
) {
669 RefPtr
<nsStorageStream
> storageStream
= new nsStorageStream();
670 nsresult rv
= storageStream
->Init(aSegmentSize
, aMaxSize
);
674 storageStream
.forget(aResult
);
678 // Undefine LOG, so that other .cpp files (or their includes) won't complain
679 // about it already being defined, when we build in unified mode.