Backed out 3 changesets (bug 1877678, bug 1849175) for causing failures on browser_op...
[gecko.git] / xpcom / io / nsStorageStream.cpp
blobcd3dfd6645f78738b4a38085fc3f98ec36406a18
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 /*
8 * The storage stream provides an internal buffer that can be filled by a
9 * client using a single output stream. One or more independent input streams
10 * can be created to read the data out non-destructively. The implementation
11 * uses a segmented buffer internally to avoid realloc'ing of large buffers,
12 * with the attendant performance loss and heap fragmentation.
15 #include "mozilla/Mutex.h"
16 #include "nsAlgorithm.h"
17 #include "nsStorageStream.h"
18 #include "nsSegmentedBuffer.h"
19 #include "nsStreamUtils.h"
20 #include "nsCOMPtr.h"
21 #include "nsICloneableInputStream.h"
22 #include "nsIInputStream.h"
23 #include "nsIIPCSerializableInputStream.h"
24 #include "nsISeekableStream.h"
25 #include "mozilla/Logging.h"
26 #include "mozilla/Attributes.h"
27 #include "mozilla/Likely.h"
28 #include "mozilla/MathAlgorithms.h"
29 #include "mozilla/ipc/InputStreamUtils.h"
31 using mozilla::MutexAutoLock;
32 using mozilla::ipc::InputStreamParams;
33 using mozilla::ipc::StringInputStreamParams;
36 // Log module for StorageStream logging...
38 // To enable logging (see prlog.h for full details):
40 // set MOZ_LOG=StorageStreamLog:5
41 // set MOZ_LOG_FILE=storage.log
43 // This enables LogLevel::Debug level information and places all output in
44 // the file storage.log.
46 static mozilla::LazyLogModule sStorageStreamLog("nsStorageStream");
47 #ifdef LOG
48 # undef LOG
49 #endif
50 #define LOG(args) MOZ_LOG(sStorageStreamLog, mozilla::LogLevel::Debug, args)
52 nsStorageStream::nsStorageStream() {
53 LOG(("Creating nsStorageStream [%p].\n", this));
56 nsStorageStream::~nsStorageStream() { delete mSegmentedBuffer; }
58 NS_IMPL_ISUPPORTS(nsStorageStream, nsIStorageStream, nsIOutputStream)
60 NS_IMETHODIMP
61 nsStorageStream::Init(uint32_t aSegmentSize, uint32_t aMaxSize) {
62 MutexAutoLock lock(mMutex);
63 mSegmentedBuffer = new nsSegmentedBuffer();
64 mSegmentSize = aSegmentSize;
65 mSegmentSizeLog2 = mozilla::FloorLog2(aSegmentSize);
66 mMaxLogicalLength = aMaxSize;
68 // Segment size must be a power of two
69 if (mSegmentSize != ((uint32_t)1 << mSegmentSizeLog2)) {
70 return NS_ERROR_INVALID_ARG;
73 return mSegmentedBuffer->Init(aSegmentSize);
76 NS_IMETHODIMP
77 nsStorageStream::GetOutputStream(int32_t aStartingOffset,
78 nsIOutputStream** aOutputStream) {
79 if (NS_WARN_IF(!aOutputStream)) {
80 return NS_ERROR_INVALID_ARG;
83 MutexAutoLock lock(mMutex);
84 if (NS_WARN_IF(!mSegmentedBuffer)) {
85 return NS_ERROR_NOT_INITIALIZED;
88 if (mWriteInProgress) {
89 return NS_ERROR_NOT_AVAILABLE;
92 if (mActiveSegmentBorrows > 0) {
93 return NS_ERROR_NOT_AVAILABLE;
96 nsresult rv = Seek(aStartingOffset);
97 if (NS_FAILED(rv)) {
98 return rv;
101 // Enlarge the last segment in the buffer so that it is the same size as
102 // all the other segments in the buffer. (It may have been realloc'ed
103 // smaller in the Close() method.)
104 if (mLastSegmentNum >= 0)
105 if (mSegmentedBuffer->ReallocLastSegment(mSegmentSize)) {
106 // Need to re-Seek, since realloc changed segment base pointer
107 rv = Seek(aStartingOffset);
108 if (NS_FAILED(rv)) {
109 return rv;
113 NS_ADDREF(this);
114 *aOutputStream = static_cast<nsIOutputStream*>(this);
115 mWriteInProgress = true;
116 return NS_OK;
119 NS_IMETHODIMP
120 nsStorageStream::Close() {
121 MutexAutoLock lock(mMutex);
122 if (NS_WARN_IF(!mSegmentedBuffer)) {
123 return NS_ERROR_NOT_INITIALIZED;
126 mWriteInProgress = false;
128 int32_t segmentOffset = SegOffset(mLogicalLength);
130 // Shrink the final segment in the segmented buffer to the minimum size
131 // needed to contain the data, so as to conserve memory.
132 if (segmentOffset && !mActiveSegmentBorrows) {
133 mSegmentedBuffer->ReallocLastSegment(segmentOffset);
136 mWriteCursor = 0;
137 mSegmentEnd = 0;
139 LOG(("nsStorageStream [%p] Close mWriteCursor=%p mSegmentEnd=%p\n", this,
140 mWriteCursor, mSegmentEnd));
142 return NS_OK;
145 NS_IMETHODIMP
146 nsStorageStream::Flush() { return NS_OK; }
148 NS_IMETHODIMP
149 nsStorageStream::StreamStatus() {
150 MutexAutoLock lock(mMutex);
151 if (!mSegmentedBuffer) {
152 return NS_ERROR_NOT_INITIALIZED;
154 return NS_OK;
157 NS_IMETHODIMP
158 nsStorageStream::Write(const char* aBuffer, uint32_t aCount,
159 uint32_t* aNumWritten) {
160 if (NS_WARN_IF(!aNumWritten) || NS_WARN_IF(!aBuffer)) {
161 return NS_ERROR_INVALID_ARG;
164 MutexAutoLock lock(mMutex);
165 if (NS_WARN_IF(!mSegmentedBuffer)) {
166 return NS_ERROR_NOT_INITIALIZED;
169 if (NS_WARN_IF(mLogicalLength >= mMaxLogicalLength)) {
170 return NS_ERROR_OUT_OF_MEMORY;
173 LOG(("nsStorageStream [%p] Write mWriteCursor=%p mSegmentEnd=%p aCount=%d\n",
174 this, mWriteCursor, mSegmentEnd, aCount));
176 uint32_t remaining = aCount;
177 const char* readCursor = aBuffer;
179 remaining = std::min(remaining, mMaxLogicalLength - mLogicalLength);
181 auto onExit = mozilla::MakeScopeExit([&] {
182 mMutex.AssertCurrentThreadOwns();
183 *aNumWritten = aCount - remaining;
184 mLogicalLength += *aNumWritten;
186 LOG(
187 ("nsStorageStream [%p] Wrote mWriteCursor=%p mSegmentEnd=%p "
188 "numWritten=%d\n",
189 this, mWriteCursor, mSegmentEnd, *aNumWritten));
192 // If no segments have been created yet, create one even if we don't have
193 // to write any data; this enables creating an input stream which reads from
194 // the very end of the data for any amount of data in the stream (i.e.
195 // this stream contains N bytes of data and newInputStream(N) is called),
196 // even for N=0 (with the caveat that we require .write("", 0) be called to
197 // initialize internal buffers).
198 bool firstTime = mSegmentedBuffer->GetSegmentCount() == 0;
199 while (remaining || MOZ_UNLIKELY(firstTime)) {
200 firstTime = false;
201 uint32_t availableInSegment = mSegmentEnd - mWriteCursor;
202 if (!availableInSegment) {
203 mWriteCursor = mSegmentedBuffer->AppendNewSegment();
204 if (!mWriteCursor) {
205 mSegmentEnd = 0;
206 return NS_ERROR_OUT_OF_MEMORY;
208 mLastSegmentNum++;
209 mSegmentEnd = mWriteCursor + mSegmentSize;
210 availableInSegment = mSegmentEnd - mWriteCursor;
211 LOG(
212 ("nsStorageStream [%p] Write (new seg) mWriteCursor=%p "
213 "mSegmentEnd=%p\n",
214 this, mWriteCursor, mSegmentEnd));
217 uint32_t count = XPCOM_MIN(availableInSegment, remaining);
218 memcpy(mWriteCursor, readCursor, count);
219 remaining -= count;
220 readCursor += count;
221 mWriteCursor += count;
222 LOG(
223 ("nsStorageStream [%p] Writing mWriteCursor=%p mSegmentEnd=%p "
224 "count=%d\n",
225 this, mWriteCursor, mSegmentEnd, count));
228 return NS_OK;
231 NS_IMETHODIMP
232 nsStorageStream::WriteFrom(nsIInputStream* aInStr, uint32_t aCount,
233 uint32_t* aResult) {
234 return NS_ERROR_NOT_IMPLEMENTED;
237 NS_IMETHODIMP
238 nsStorageStream::WriteSegments(nsReadSegmentFun aReader, void* aClosure,
239 uint32_t aCount, uint32_t* aResult) {
240 return NS_ERROR_NOT_IMPLEMENTED;
243 NS_IMETHODIMP
244 nsStorageStream::IsNonBlocking(bool* aNonBlocking) {
245 *aNonBlocking = false;
246 return NS_OK;
249 NS_IMETHODIMP
250 nsStorageStream::GetLength(uint32_t* aLength) {
251 MutexAutoLock lock(mMutex);
252 *aLength = mLogicalLength;
253 return NS_OK;
256 NS_IMETHODIMP
257 nsStorageStream::SetLength(uint32_t aLength) {
258 MutexAutoLock lock(mMutex);
259 return SetLengthLocked(aLength);
262 // Truncate the buffer by deleting the end segments
263 nsresult nsStorageStream::SetLengthLocked(uint32_t aLength) {
264 if (NS_WARN_IF(!mSegmentedBuffer)) {
265 return NS_ERROR_NOT_INITIALIZED;
268 if (mWriteInProgress) {
269 return NS_ERROR_NOT_AVAILABLE;
272 if (mActiveSegmentBorrows) {
273 return NS_ERROR_NOT_AVAILABLE;
276 if (aLength > mLogicalLength) {
277 return NS_ERROR_INVALID_ARG;
280 int32_t newLastSegmentNum = SegNum(aLength);
281 int32_t segmentOffset = SegOffset(aLength);
282 if (segmentOffset == 0) {
283 newLastSegmentNum--;
286 while (newLastSegmentNum < mLastSegmentNum) {
287 mSegmentedBuffer->DeleteLastSegment();
288 mLastSegmentNum--;
291 mLogicalLength = aLength;
292 return NS_OK;
295 NS_IMETHODIMP
296 nsStorageStream::GetWriteInProgress(bool* aWriteInProgress) {
297 MutexAutoLock lock(mMutex);
298 *aWriteInProgress = mWriteInProgress;
299 return NS_OK;
302 nsresult nsStorageStream::Seek(int32_t aPosition) {
303 if (NS_WARN_IF(!mSegmentedBuffer)) {
304 return NS_ERROR_NOT_INITIALIZED;
307 // An argument of -1 means "seek to end of stream"
308 if (aPosition == -1) {
309 aPosition = mLogicalLength;
312 // Seeking beyond the buffer end is illegal
313 if ((uint32_t)aPosition > mLogicalLength) {
314 return NS_ERROR_INVALID_ARG;
317 // Seeking backwards in the write stream results in truncation
318 SetLengthLocked(aPosition);
320 // Special handling for seek to start-of-buffer
321 if (aPosition == 0) {
322 mWriteCursor = 0;
323 mSegmentEnd = 0;
324 LOG(("nsStorageStream [%p] Seek mWriteCursor=%p mSegmentEnd=%p\n", this,
325 mWriteCursor, mSegmentEnd));
326 return NS_OK;
329 // Segment may have changed, so reset pointers
330 mWriteCursor = mSegmentedBuffer->GetSegment(mLastSegmentNum);
331 NS_ASSERTION(mWriteCursor, "null mWriteCursor");
332 mSegmentEnd = mWriteCursor + mSegmentSize;
334 // Adjust write cursor for current segment offset. This test is necessary
335 // because SegNum may reference the next-to-be-allocated segment, in which
336 // case we need to be pointing at the end of the last segment.
337 int32_t segmentOffset = SegOffset(aPosition);
338 if (segmentOffset == 0 && (SegNum(aPosition) > (uint32_t)mLastSegmentNum)) {
339 mWriteCursor = mSegmentEnd;
340 } else {
341 mWriteCursor += segmentOffset;
344 LOG(("nsStorageStream [%p] Seek mWriteCursor=%p mSegmentEnd=%p\n", this,
345 mWriteCursor, mSegmentEnd));
346 return NS_OK;
349 ////////////////////////////////////////////////////////////////////////////////
351 // There can be many nsStorageInputStreams for a single nsStorageStream
352 class nsStorageInputStream final : public nsIInputStream,
353 public nsISeekableStream,
354 public nsIIPCSerializableInputStream,
355 public nsICloneableInputStream {
356 public:
357 nsStorageInputStream(nsStorageStream* aStorageStream, uint32_t aSegmentSize)
358 : mStorageStream(aStorageStream),
359 mReadCursor(0),
360 mSegmentEnd(0),
361 mSegmentNum(0),
362 mSegmentSize(aSegmentSize),
363 mLogicalCursor(0),
364 mStatus(NS_OK) {}
366 NS_DECL_THREADSAFE_ISUPPORTS
367 NS_DECL_NSIINPUTSTREAM
368 NS_DECL_NSISEEKABLESTREAM
369 NS_DECL_NSITELLABLESTREAM
370 NS_DECL_NSIIPCSERIALIZABLEINPUTSTREAM
371 NS_DECL_NSICLONEABLEINPUTSTREAM
373 private:
374 ~nsStorageInputStream() = default;
376 protected:
377 nsresult Seek(uint32_t aPosition) MOZ_REQUIRES(mStorageStream->mMutex);
379 friend class nsStorageStream;
381 private:
382 RefPtr<nsStorageStream> mStorageStream;
383 uint32_t mReadCursor; // Next memory location to read byte, or 0
384 uint32_t mSegmentEnd; // One byte past end of current buffer segment
385 uint32_t mSegmentNum; // Segment number containing read cursor
386 uint32_t mSegmentSize; // All segments, except the last, are of this size
387 uint32_t mLogicalCursor; // Logical offset into stream
388 nsresult mStatus;
390 uint32_t SegNum(uint32_t aPosition) MOZ_REQUIRES(mStorageStream->mMutex) {
391 return aPosition >> mStorageStream->mSegmentSizeLog2;
393 uint32_t SegOffset(uint32_t aPosition) {
394 return aPosition & (mSegmentSize - 1);
398 NS_IMPL_ISUPPORTS(nsStorageInputStream, nsIInputStream, nsISeekableStream,
399 nsITellableStream, nsIIPCSerializableInputStream,
400 nsICloneableInputStream)
402 NS_IMETHODIMP
403 nsStorageStream::NewInputStream(int32_t aStartingOffset,
404 nsIInputStream** aInputStream) {
405 MutexAutoLock lock(mMutex);
406 if (NS_WARN_IF(!mSegmentedBuffer)) {
407 return NS_ERROR_NOT_INITIALIZED;
410 RefPtr<nsStorageInputStream> inputStream =
411 new nsStorageInputStream(this, mSegmentSize);
413 inputStream->mStorageStream->mMutex.AssertCurrentThreadOwns();
414 nsresult rv = inputStream->Seek(aStartingOffset);
415 if (NS_FAILED(rv)) {
416 return rv;
419 inputStream.forget(aInputStream);
420 return NS_OK;
423 NS_IMETHODIMP
424 nsStorageInputStream::Close() {
425 mStatus = NS_BASE_STREAM_CLOSED;
426 return NS_OK;
429 NS_IMETHODIMP
430 nsStorageInputStream::Available(uint64_t* aAvailable) {
431 if (NS_FAILED(mStatus)) {
432 return mStatus;
435 MutexAutoLock lock(mStorageStream->mMutex);
436 *aAvailable = mStorageStream->mLogicalLength - mLogicalCursor;
437 return NS_OK;
440 NS_IMETHODIMP
441 nsStorageInputStream::StreamStatus() { return mStatus; }
443 NS_IMETHODIMP
444 nsStorageInputStream::Read(char* aBuffer, uint32_t aCount, uint32_t* aNumRead) {
445 return ReadSegments(NS_CopySegmentToBuffer, aBuffer, aCount, aNumRead);
448 NS_IMETHODIMP
449 nsStorageInputStream::ReadSegments(nsWriteSegmentFun aWriter, void* aClosure,
450 uint32_t aCount, uint32_t* aNumRead) {
451 *aNumRead = 0;
452 if (mStatus == NS_BASE_STREAM_CLOSED) {
453 return NS_OK;
455 if (NS_FAILED(mStatus)) {
456 return mStatus;
459 uint32_t count, availableInSegment, remainingCapacity, bytesConsumed;
460 nsresult rv;
462 remainingCapacity = aCount;
463 while (remainingCapacity) {
464 const char* cur = nullptr;
466 MutexAutoLock lock(mStorageStream->mMutex);
467 availableInSegment = mSegmentEnd - mReadCursor;
468 if (!availableInSegment) {
469 uint32_t available = mStorageStream->mLogicalLength - mLogicalCursor;
470 if (!available) {
471 break;
474 // We have data in the stream, but if mSegmentEnd is zero, then we
475 // were likely constructed prior to any data being written into
476 // the stream. Therefore, if mSegmentEnd is non-zero, we should
477 // move into the next segment; otherwise, we should stay in this
478 // segment so our input state can be updated and we can properly
479 // perform the initial read.
480 if (mSegmentEnd > 0) {
481 mSegmentNum++;
483 mReadCursor = 0;
484 mSegmentEnd = XPCOM_MIN(mSegmentSize, available);
485 availableInSegment = mSegmentEnd;
487 cur = mStorageStream->mSegmentedBuffer->GetSegment(mSegmentNum);
488 mStorageStream->mActiveSegmentBorrows++;
490 auto dropBorrow = mozilla::MakeScopeExit([&] {
491 MutexAutoLock lock(mStorageStream->mMutex);
492 mStorageStream->mActiveSegmentBorrows--;
495 count = XPCOM_MIN(availableInSegment, remainingCapacity);
496 rv = aWriter(this, aClosure, cur + mReadCursor, aCount - remainingCapacity,
497 count, &bytesConsumed);
498 if (NS_FAILED(rv) || (bytesConsumed == 0)) {
499 break;
501 remainingCapacity -= bytesConsumed;
502 mReadCursor += bytesConsumed;
503 mLogicalCursor += bytesConsumed;
506 *aNumRead = aCount - remainingCapacity;
508 bool isWriteInProgress = false;
509 if (NS_FAILED(mStorageStream->GetWriteInProgress(&isWriteInProgress))) {
510 isWriteInProgress = false;
513 if (*aNumRead == 0 && isWriteInProgress) {
514 return NS_BASE_STREAM_WOULD_BLOCK;
517 return NS_OK;
520 NS_IMETHODIMP
521 nsStorageInputStream::IsNonBlocking(bool* aNonBlocking) {
522 // TODO: This class should implement nsIAsyncInputStream so that callers
523 // have some way of dealing with NS_BASE_STREAM_WOULD_BLOCK errors.
525 *aNonBlocking = true;
526 return NS_OK;
529 NS_IMETHODIMP
530 nsStorageInputStream::Seek(int32_t aWhence, int64_t aOffset) {
531 if (NS_FAILED(mStatus)) {
532 return mStatus;
535 MutexAutoLock lock(mStorageStream->mMutex);
536 int64_t pos = aOffset;
538 switch (aWhence) {
539 case NS_SEEK_SET:
540 break;
541 case NS_SEEK_CUR:
542 pos += mLogicalCursor;
543 break;
544 case NS_SEEK_END:
545 pos += mStorageStream->mLogicalLength;
546 break;
547 default:
548 MOZ_ASSERT_UNREACHABLE("unexpected whence value");
549 return NS_ERROR_UNEXPECTED;
551 if (pos == int64_t(mLogicalCursor)) {
552 return NS_OK;
555 return Seek(pos);
558 NS_IMETHODIMP
559 nsStorageInputStream::Tell(int64_t* aResult) {
560 if (NS_FAILED(mStatus)) {
561 return mStatus;
564 *aResult = mLogicalCursor;
565 return NS_OK;
568 NS_IMETHODIMP
569 nsStorageInputStream::SetEOF() {
570 MOZ_ASSERT_UNREACHABLE("nsStorageInputStream::SetEOF");
571 return NS_ERROR_NOT_IMPLEMENTED;
574 nsresult nsStorageInputStream::Seek(uint32_t aPosition) {
575 uint32_t length = mStorageStream->mLogicalLength;
576 if (aPosition > length) {
577 return NS_ERROR_INVALID_ARG;
580 if (length == 0) {
581 return NS_OK;
584 mSegmentNum = SegNum(aPosition);
585 mReadCursor = SegOffset(aPosition);
586 uint32_t available = length - aPosition;
587 mSegmentEnd = mReadCursor + XPCOM_MIN(mSegmentSize - mReadCursor, available);
588 mLogicalCursor = aPosition;
589 return NS_OK;
592 void nsStorageInputStream::SerializedComplexity(uint32_t aMaxSize,
593 uint32_t* aSizeUsed,
594 uint32_t* aPipes,
595 uint32_t* aTransferables) {
596 uint64_t remaining = 0;
597 mozilla::DebugOnly<nsresult> rv = Available(&remaining);
598 MOZ_ASSERT(NS_SUCCEEDED(rv));
600 if (remaining >= aMaxSize) {
601 *aPipes = 1;
602 } else {
603 *aSizeUsed = remaining;
607 void nsStorageInputStream::Serialize(InputStreamParams& aParams,
608 uint32_t aMaxSize, uint32_t* aSizeUsed) {
609 MOZ_ASSERT(aSizeUsed);
610 *aSizeUsed = 0;
612 uint64_t remaining = 0;
613 mozilla::DebugOnly<nsresult> rv = Available(&remaining);
614 MOZ_ASSERT(NS_SUCCEEDED(rv));
616 if (remaining >= aMaxSize) {
617 mozilla::ipc::InputStreamHelper::SerializeInputStreamAsPipe(this, aParams);
618 return;
621 *aSizeUsed = remaining;
623 nsCString combined;
624 int64_t offset;
625 rv = Tell(&offset);
626 MOZ_ASSERT(NS_SUCCEEDED(rv));
628 auto handleOrErr = combined.BulkWrite(remaining, 0, false);
629 MOZ_ASSERT(!handleOrErr.isErr());
631 auto handle = handleOrErr.unwrap();
633 uint32_t numRead = 0;
635 rv = Read(handle.Elements(), remaining, &numRead);
636 MOZ_ASSERT(NS_SUCCEEDED(rv));
638 MOZ_ASSERT(numRead == remaining);
639 handle.Finish(numRead, false);
641 rv = Seek(NS_SEEK_SET, offset);
642 MOZ_ASSERT(NS_SUCCEEDED(rv));
644 StringInputStreamParams params;
645 params.data() = combined;
646 aParams = params;
649 bool nsStorageInputStream::Deserialize(const InputStreamParams& aParams) {
650 MOZ_ASSERT_UNREACHABLE(
651 "We should never attempt to deserialize a storage "
652 "input stream.");
653 return false;
656 NS_IMETHODIMP
657 nsStorageInputStream::GetCloneable(bool* aCloneableOut) {
658 *aCloneableOut = true;
659 return NS_OK;
662 NS_IMETHODIMP
663 nsStorageInputStream::Clone(nsIInputStream** aCloneOut) {
664 return mStorageStream->NewInputStream(mLogicalCursor, aCloneOut);
667 nsresult NS_NewStorageStream(uint32_t aSegmentSize, uint32_t aMaxSize,
668 nsIStorageStream** aResult) {
669 RefPtr<nsStorageStream> storageStream = new nsStorageStream();
670 nsresult rv = storageStream->Init(aSegmentSize, aMaxSize);
671 if (NS_FAILED(rv)) {
672 return rv;
674 storageStream.forget(aResult);
675 return NS_OK;
678 // Undefine LOG, so that other .cpp files (or their includes) won't complain
679 // about it already being defined, when we build in unified mode.
680 #undef LOG