1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
8 #include "PLDHashTable.h"
9 #include "mozilla/IOInterposer.h"
10 #include "mozilla/MemoryReporting.h"
11 #include "mozilla/scache/StartupCache.h"
13 #include "nsAutoPtr.h"
14 #include "nsClassHashtable.h"
15 #include "nsComponentManagerUtils.h"
16 #include "nsDirectoryServiceUtils.h"
17 #include "nsIClassInfo.h"
19 #include "nsIObserver.h"
20 #include "nsIObserverService.h"
21 #include "nsIOutputStream.h"
22 #include "nsIStorageStream.h"
23 #include "nsIStreamBufferAccess.h"
24 #include "nsIStringStream.h"
25 #include "nsISupports.h"
27 #include "nsIZipWriter.h"
28 #include "nsIZipReader.h"
29 #include "nsZipArchive.h"
30 #include "mozilla/Omnijar.h"
32 #include "mozilla/Telemetry.h"
33 #include "nsThreadUtils.h"
34 #include "nsXULAppAPI.h"
35 #include "nsIProtocolHandler.h"
36 #include "GeckoProfiler.h"
39 # define SC_ENDIAN "big"
41 # define SC_ENDIAN "little"
44 #if PR_BYTES_PER_WORD == 4
45 # define SC_WORDSIZE "4"
47 # define SC_WORDSIZE "8"
53 MOZ_DEFINE_MALLOC_SIZE_OF(StartupCacheMallocSizeOf
)
56 StartupCache::CollectReports(nsIHandleReportCallback
* aHandleReport
,
57 nsISupports
* aData
, bool aAnonymize
) {
59 "explicit/startup-cache/mapping", KIND_NONHEAP
, UNITS_BYTES
,
61 "Memory used to hold the mapping of the startup cache from file. "
62 "This memory is likely to be swapped out shortly after start-up.");
64 MOZ_COLLECT_REPORT("explicit/startup-cache/data", KIND_HEAP
, UNITS_BYTES
,
65 HeapSizeOfIncludingThis(StartupCacheMallocSizeOf
),
66 "Memory used by the startup cache for things other than "
72 #define STARTUP_CACHE_NAME "startupCache." SC_WORDSIZE "." SC_ENDIAN
74 StartupCache
* StartupCache::GetSingleton() {
76 if (!XRE_IsParentProcess()) {
79 #ifdef MOZ_DISABLE_STARTUPCACHE
82 StartupCache::InitSingleton();
86 return StartupCache::gStartupCache
;
89 void StartupCache::DeleteSingleton() { StartupCache::gStartupCache
= nullptr; }
91 nsresult
StartupCache::InitSingleton() {
93 StartupCache::gStartupCache
= new StartupCache();
95 rv
= StartupCache::gStartupCache
->Init();
97 StartupCache::gStartupCache
= nullptr;
102 StaticRefPtr
<StartupCache
> StartupCache::gStartupCache
;
103 bool StartupCache::gShutdownInitiated
;
104 bool StartupCache::gIgnoreDiskCache
;
106 NS_IMPL_ISUPPORTS(StartupCache
, nsIMemoryReporter
)
108 StartupCache::StartupCache()
109 : mArchive(nullptr), mStartupWriteInitiated(false), mWriteThread(nullptr) {}
111 StartupCache::~StartupCache() {
116 // Generally, the in-memory table should be empty here,
117 // but an early shutdown means either mTimer didn't run
118 // or the write thread is still running.
121 // If we shutdown quickly timer wont have fired. Instead of writing
122 // it on the main thread and block the shutdown we simply wont update
123 // the startup cache. Always do this if the file doesn't exist since
124 // we use it part of the package step.
129 UnregisterWeakMemoryReporter(this);
132 nsresult
StartupCache::Init() {
133 // workaround for bug 653936
134 nsCOMPtr
<nsIProtocolHandler
> jarInitializer(
135 do_GetService(NS_NETWORK_PROTOCOL_CONTRACTID_PREFIX
"jar"));
139 // This allows to override the startup cache filename
140 // which is useful from xpcshell, when there is no ProfLDS directory to keep
142 char* env
= PR_GetEnv("MOZ_STARTUP_CACHE");
144 rv
= NS_NewLocalFile(NS_ConvertUTF8toUTF16(env
), false,
145 getter_AddRefs(mFile
));
147 nsCOMPtr
<nsIFile
> file
;
148 rv
= NS_GetSpecialDirectory("ProfLDS", getter_AddRefs(file
));
150 // return silently, this will fail in mochitests's xpcshell process.
154 nsCOMPtr
<nsIFile
> profDir
;
155 NS_GetSpecialDirectory("ProfDS", getter_AddRefs(profDir
));
158 if (NS_SUCCEEDED(profDir
->Equals(file
, &same
)) && !same
) {
159 // We no longer store the startup cache in the main profile
160 // directory, so we should cleanup the old one.
162 profDir
->AppendNative(NS_LITERAL_CSTRING("startupCache")))) {
163 profDir
->Remove(true);
168 rv
= file
->AppendNative(NS_LITERAL_CSTRING("startupCache"));
169 NS_ENSURE_SUCCESS(rv
, rv
);
171 // Try to create the directory if it's not there yet
172 rv
= file
->Create(nsIFile::DIRECTORY_TYPE
, 0777);
173 if (NS_FAILED(rv
) && rv
!= NS_ERROR_FILE_ALREADY_EXISTS
) return rv
;
175 rv
= file
->AppendNative(NS_LITERAL_CSTRING(STARTUP_CACHE_NAME
));
177 NS_ENSURE_SUCCESS(rv
, rv
);
182 NS_ENSURE_TRUE(mFile
, NS_ERROR_UNEXPECTED
);
184 mObserverService
= do_GetService("@mozilla.org/observer-service;1");
186 if (!mObserverService
) {
187 NS_WARNING("Could not get observerService.");
188 return NS_ERROR_UNEXPECTED
;
191 mListener
= new StartupCacheListener();
192 rv
= mObserverService
->AddObserver(mListener
, NS_XPCOM_SHUTDOWN_OBSERVER_ID
,
194 NS_ENSURE_SUCCESS(rv
, rv
);
195 rv
= mObserverService
->AddObserver(mListener
, "startupcache-invalidate",
197 NS_ENSURE_SUCCESS(rv
, rv
);
201 // Sometimes we don't have a cache yet, that's ok.
202 // If it's corrupted, just remove it and start over.
203 if (gIgnoreDiskCache
|| (NS_FAILED(rv
) && rv
!= NS_ERROR_FILE_NOT_FOUND
)) {
204 NS_WARNING("Failed to load startupcache file correctly, removing!");
208 RegisterWeakMemoryReporter(this);
214 * LoadArchive can be called from the main thread or while reloading cache on
217 nsresult
StartupCache::LoadArchive() {
218 if (gIgnoreDiskCache
) return NS_ERROR_FAILURE
;
222 nsresult rv
= mFile
->Exists(&exists
);
223 if (NS_FAILED(rv
) || !exists
) return NS_ERROR_FILE_NOT_FOUND
;
225 mArchive
= new nsZipArchive();
226 rv
= mArchive
->OpenArchive(mFile
);
232 nsresult
GetBufferFromZipArchive(nsZipArchive
* zip
, bool doCRC
, const char* id
,
233 UniquePtr
<char[]>* outbuf
, uint32_t* length
) {
234 if (!zip
) return NS_ERROR_NOT_AVAILABLE
;
236 nsZipItemPtr
<char> zipItem(zip
, id
, doCRC
);
237 if (!zipItem
) return NS_ERROR_NOT_AVAILABLE
;
239 *outbuf
= zipItem
.Forget();
240 *length
= zipItem
.Length();
244 } /* anonymous namespace */
246 // NOTE: this will not find a new entry until it has been written to disk!
247 // Consumer should take ownership of the resulting buffer.
248 nsresult
StartupCache::GetBuffer(const char* id
, UniquePtr
<char[]>* outbuf
,
250 AUTO_PROFILER_LABEL("StartupCache::GetBuffer", OTHER
);
252 NS_ASSERTION(NS_IsMainThread(),
253 "Startup cache only available on main thread");
256 if (!mStartupWriteInitiated
) {
258 nsDependentCString
idStr(id
);
259 mTable
.Get(idStr
, &entry
);
261 *outbuf
= MakeUnique
<char[]>(entry
->size
);
262 memcpy(outbuf
->get(), entry
->data
.get(), entry
->size
);
263 *length
= entry
->size
;
268 nsresult rv
= GetBufferFromZipArchive(mArchive
, true, id
, outbuf
, length
);
269 if (NS_SUCCEEDED(rv
)) return rv
;
271 RefPtr
<nsZipArchive
> omnijar
=
272 mozilla::Omnijar::GetReader(mozilla::Omnijar::APP
);
273 // no need to checksum omnijarred entries
274 rv
= GetBufferFromZipArchive(omnijar
, false, id
, outbuf
, length
);
275 if (NS_SUCCEEDED(rv
)) return rv
;
277 omnijar
= mozilla::Omnijar::GetReader(mozilla::Omnijar::GRE
);
278 // no need to checksum omnijarred entries
279 return GetBufferFromZipArchive(omnijar
, false, id
, outbuf
, length
);
282 // Makes a copy of the buffer, client retains ownership of inbuf.
283 nsresult
StartupCache::PutBuffer(const char* id
, UniquePtr
<char[]>&& inbuf
,
285 NS_ASSERTION(NS_IsMainThread(),
286 "Startup cache only available on main thread");
288 if (StartupCache::gShutdownInitiated
) {
289 return NS_ERROR_NOT_AVAILABLE
;
292 nsDependentCString
idStr(id
);
293 // Cache it for now, we'll write all together later.
294 auto entry
= mTable
.LookupForAdd(idStr
);
297 NS_WARNING("Existing entry in StartupCache.");
298 // Double-caching is undesirable but not an error.
304 nsZipItem
* zipItem
= mArchive
->GetItem(id
);
305 NS_ASSERTION(zipItem
== nullptr, "Existing entry in disk StartupCache.");
310 [&inbuf
, &len
]() { return new CacheEntry(std::move(inbuf
), len
); });
311 mPendingWrites
.AppendElement(idStr
);
312 return ResetStartupWriteTimer();
315 size_t StartupCache::SizeOfMapping() {
316 return mArchive
? mArchive
->SizeOfMapping() : 0;
319 size_t StartupCache::HeapSizeOfIncludingThis(
320 mozilla::MallocSizeOf aMallocSizeOf
) const {
321 // This function could measure more members, but they haven't been found by
322 // DMD to be significant. They can be added later if necessary.
324 size_t n
= aMallocSizeOf(this);
326 n
+= mTable
.ShallowSizeOfExcludingThis(aMallocSizeOf
);
327 for (auto iter
= mTable
.ConstIter(); !iter
.Done(); iter
.Next()) {
328 n
+= iter
.Data()->SizeOfIncludingThis(aMallocSizeOf
);
331 n
+= mPendingWrites
.ShallowSizeOfExcludingThis(aMallocSizeOf
);
336 struct CacheWriteHolder
{
337 nsCOMPtr
<nsIZipWriter
> writer
;
338 nsCOMPtr
<nsIStringInputStream
> stream
;
342 static void CacheCloseHelper(const nsACString
& key
, const CacheEntry
* data
,
343 const CacheWriteHolder
* holder
) {
344 MOZ_ASSERT(data
); // assert key was found in mTable.
347 nsIStringInputStream
* stream
= holder
->stream
;
348 nsIZipWriter
* writer
= holder
->writer
;
350 stream
->ShareData(data
->data
.get(), data
->size
);
354 rv
= writer
->HasEntry(key
, &hasEntry
);
355 NS_ASSERTION(NS_SUCCEEDED(rv
) && hasEntry
== false,
356 "Existing entry in disk StartupCache.");
358 rv
= writer
->AddEntryStream(key
, holder
->time
, true, stream
, false);
361 NS_WARNING("cache entry deleted but not written to disk.");
366 * WriteToDisk writes the cache out to disk. Callers of WriteToDisk need to call
367 * WaitOnWriteThread to make sure there isn't a write happening on another
370 void StartupCache::WriteToDisk() {
372 mStartupWriteInitiated
= true;
374 if (mTable
.Count() == 0) return;
376 nsCOMPtr
<nsIZipWriter
> zipW
= do_CreateInstance("@mozilla.org/zipwriter;1");
379 rv
= zipW
->Open(mFile
, PR_RDWR
| PR_CREATE_FILE
);
381 NS_WARNING("could not open zipfile for write");
385 // If we didn't have an mArchive member, that means that we failed to
386 // open the startup cache for reading. Therefore, we need to record
387 // the time of creation in a zipfile comment; this has been useful for
388 // Telemetry statistics.
389 PRTime now
= PR_Now();
392 comment
.Assign((char*)&now
, sizeof(now
));
393 zipW
->SetComment(comment
);
396 nsCOMPtr
<nsIStringInputStream
> stream
=
397 do_CreateInstance("@mozilla.org/io/string-input-stream;1", &rv
);
399 NS_WARNING("Couldn't create string input stream.");
403 CacheWriteHolder holder
;
404 holder
.stream
= stream
;
405 holder
.writer
= zipW
;
408 for (auto& key
: mPendingWrites
) {
409 CacheCloseHelper(key
, mTable
.Get(key
), &holder
);
411 mPendingWrites
.Clear();
414 // Close the archive so Windows doesn't choke.
418 // We succesfully wrote the archive to disk; mark the disk file as trusted
419 gIgnoreDiskCache
= false;
421 // Our reader's view of the archive is outdated now, reload it.
425 void StartupCache::InvalidateCache() {
427 mPendingWrites
.Clear();
430 nsresult rv
= mFile
->Remove(false);
431 if (NS_FAILED(rv
) && rv
!= NS_ERROR_FILE_TARGET_DOES_NOT_EXIST
&&
432 rv
!= NS_ERROR_FILE_NOT_FOUND
) {
433 gIgnoreDiskCache
= true;
436 gIgnoreDiskCache
= false;
440 void StartupCache::IgnoreDiskCache() {
441 gIgnoreDiskCache
= true;
442 if (gStartupCache
) gStartupCache
->InvalidateCache();
446 * WaitOnWriteThread() is called from a main thread to wait for the worker
447 * thread to finish. However since the same code is used in the worker thread
448 * and main thread, the worker thread can also call WaitOnWriteThread() which is
451 void StartupCache::WaitOnWriteThread() {
452 NS_ASSERTION(NS_IsMainThread(),
453 "Startup cache should only wait for io thread on main thread");
454 if (!mWriteThread
|| mWriteThread
== PR_GetCurrentThread()) return;
456 PR_JoinThread(mWriteThread
);
457 mWriteThread
= nullptr;
460 void StartupCache::ThreadedWrite(void* aClosure
) {
461 AUTO_PROFILER_REGISTER_THREAD("StartupCache");
462 NS_SetCurrentThreadName("StartupCache");
463 mozilla::IOInterposer::RegisterCurrentThread();
465 * It is safe to use the pointer passed in aClosure to reference the
466 * StartupCache object because the thread's lifetime is tightly coupled to
467 * the lifetime of the StartupCache object; this thread is joined in the
468 * StartupCache destructor, guaranteeing that this function runs if and only
469 * if the StartupCache object is valid.
471 StartupCache
* startupCacheObj
= static_cast<StartupCache
*>(aClosure
);
472 startupCacheObj
->WriteToDisk();
473 mozilla::IOInterposer::UnregisterCurrentThread();
477 * The write-thread is spawned on a timeout(which is reset with every write).
478 * This can avoid a slow shutdown. After writing out the cache, the zipreader is
479 * reloaded on the worker thread.
481 void StartupCache::WriteTimeout(nsITimer
* aTimer
, void* aClosure
) {
483 * It is safe to use the pointer passed in aClosure to reference the
484 * StartupCache object because the timer's lifetime is tightly coupled to
485 * the lifetime of the StartupCache object; this timer is canceled in the
486 * StartupCache destructor, guaranteeing that this function runs if and only
487 * if the StartupCache object is valid.
489 StartupCache
* startupCacheObj
= static_cast<StartupCache
*>(aClosure
);
490 startupCacheObj
->mWriteThread
= PR_CreateThread(
491 PR_USER_THREAD
, StartupCache::ThreadedWrite
, startupCacheObj
,
492 PR_PRIORITY_NORMAL
, PR_GLOBAL_THREAD
, PR_JOINABLE_THREAD
, 0);
495 // We don't want to refcount StartupCache, so we'll just
496 // hold a ref to this and pass it to observerService instead.
497 NS_IMPL_ISUPPORTS(StartupCacheListener
, nsIObserver
)
499 nsresult
StartupCacheListener::Observe(nsISupports
* subject
, const char* topic
,
500 const char16_t
* data
) {
501 StartupCache
* sc
= StartupCache::GetSingleton();
502 if (!sc
) return NS_OK
;
504 if (strcmp(topic
, NS_XPCOM_SHUTDOWN_OBSERVER_ID
) == 0) {
505 // Do not leave the thread running past xpcom shutdown
506 sc
->WaitOnWriteThread();
507 StartupCache::gShutdownInitiated
= true;
508 } else if (strcmp(topic
, "startupcache-invalidate") == 0) {
509 sc
->InvalidateCache();
514 nsresult
StartupCache::GetDebugObjectOutputStream(
515 nsIObjectOutputStream
* aStream
, nsIObjectOutputStream
** aOutStream
) {
516 NS_ENSURE_ARG_POINTER(aStream
);
518 auto* stream
= new StartupCacheDebugOutputStream(aStream
, &mWriteObjectMap
);
519 NS_ADDREF(*aOutStream
= stream
);
521 NS_ADDREF(*aOutStream
= aStream
);
527 nsresult
StartupCache::ResetStartupWriteTimer() {
528 mStartupWriteInitiated
= false;
531 mTimer
= NS_NewTimer();
533 rv
= mTimer
->Cancel();
534 NS_ENSURE_SUCCESS(rv
, rv
);
535 // Wait for 10 seconds, then write out the cache.
536 mTimer
->InitWithNamedFuncCallback(StartupCache::WriteTimeout
, this, 60000,
537 nsITimer::TYPE_ONE_SHOT
,
538 "StartupCache::WriteTimeout");
542 bool StartupCache::StartupWriteComplete() {
544 return mStartupWriteInitiated
&& mTable
.Count() == 0;
547 // StartupCacheDebugOutputStream implementation
549 NS_IMPL_ISUPPORTS(StartupCacheDebugOutputStream
, nsIObjectOutputStream
,
550 nsIBinaryOutputStream
, nsIOutputStream
)
552 bool StartupCacheDebugOutputStream::CheckReferences(nsISupports
* aObject
) {
555 nsCOMPtr
<nsIClassInfo
> classInfo
= do_QueryInterface(aObject
);
557 NS_ERROR("aObject must implement nsIClassInfo");
562 rv
= classInfo
->GetFlags(&flags
);
563 NS_ENSURE_SUCCESS(rv
, false);
564 if (flags
& nsIClassInfo::SINGLETON
) return true;
566 nsISupportsHashKey
* key
= mObjectMap
->GetEntry(aObject
);
569 "non-singleton aObject is referenced multiple times in this"
570 "serialization, we don't support that.");
574 mObjectMap
->PutEntry(aObject
);
578 // nsIObjectOutputStream implementation
579 nsresult
StartupCacheDebugOutputStream::WriteObject(nsISupports
* aObject
,
581 nsCOMPtr
<nsISupports
> rootObject(do_QueryInterface(aObject
));
583 NS_ASSERTION(rootObject
.get() == aObject
,
584 "bad call to WriteObject -- call WriteCompoundObject!");
585 bool check
= CheckReferences(aObject
);
586 NS_ENSURE_TRUE(check
, NS_ERROR_FAILURE
);
587 return mBinaryStream
->WriteObject(aObject
, aIsStrongRef
);
590 nsresult
StartupCacheDebugOutputStream::WriteSingleRefObject(
591 nsISupports
* aObject
) {
592 nsCOMPtr
<nsISupports
> rootObject(do_QueryInterface(aObject
));
594 NS_ASSERTION(rootObject
.get() == aObject
,
595 "bad call to WriteSingleRefObject -- call WriteCompoundObject!");
596 bool check
= CheckReferences(aObject
);
597 NS_ENSURE_TRUE(check
, NS_ERROR_FAILURE
);
598 return mBinaryStream
->WriteSingleRefObject(aObject
);
601 nsresult
StartupCacheDebugOutputStream::WriteCompoundObject(
602 nsISupports
* aObject
, const nsIID
& aIID
, bool aIsStrongRef
) {
603 nsCOMPtr
<nsISupports
> rootObject(do_QueryInterface(aObject
));
605 nsCOMPtr
<nsISupports
> roundtrip
;
606 rootObject
->QueryInterface(aIID
, getter_AddRefs(roundtrip
));
607 NS_ASSERTION(roundtrip
.get() == aObject
,
608 "bad aggregation or multiple inheritance detected by call to "
609 "WriteCompoundObject!");
611 bool check
= CheckReferences(aObject
);
612 NS_ENSURE_TRUE(check
, NS_ERROR_FAILURE
);
613 return mBinaryStream
->WriteCompoundObject(aObject
, aIID
, aIsStrongRef
);
616 nsresult
StartupCacheDebugOutputStream::WriteID(nsID
const& aID
) {
617 return mBinaryStream
->WriteID(aID
);
620 char* StartupCacheDebugOutputStream::GetBuffer(uint32_t aLength
,
621 uint32_t aAlignMask
) {
622 return mBinaryStream
->GetBuffer(aLength
, aAlignMask
);
625 void StartupCacheDebugOutputStream::PutBuffer(char* aBuffer
, uint32_t aLength
) {
626 mBinaryStream
->PutBuffer(aBuffer
, aLength
);
630 } // namespace scache
631 } // namespace mozilla