1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
8 #include "PLDHashTable.h"
9 #include "mozilla/IOInterposer.h"
10 #include "mozilla/MemoryReporting.h"
11 #include "mozilla/scache/StartupCache.h"
13 #include "nsAutoPtr.h"
14 #include "nsClassHashtable.h"
15 #include "nsComponentManagerUtils.h"
17 #include "nsDirectoryServiceUtils.h"
18 #include "nsIClassInfo.h"
20 #include "nsIObserver.h"
21 #include "nsIObserverService.h"
22 #include "nsIOutputStream.h"
23 #include "nsIStorageStream.h"
24 #include "nsIStreamBufferAccess.h"
25 #include "nsIStringStream.h"
26 #include "nsISupports.h"
28 #include "nsIZipWriter.h"
29 #include "nsIZipReader.h"
30 #include "nsZipArchive.h"
31 #include "mozilla/Omnijar.h"
33 #include "mozilla/Telemetry.h"
34 #include "nsThreadUtils.h"
35 #include "nsXULAppAPI.h"
36 #include "nsIProtocolHandler.h"
37 #include "GeckoProfiler.h"
40 # define SC_ENDIAN "big"
42 # define SC_ENDIAN "little"
45 #if PR_BYTES_PER_WORD == 4
46 # define SC_WORDSIZE "4"
48 # define SC_WORDSIZE "8"
54 MOZ_DEFINE_MALLOC_SIZE_OF(StartupCacheMallocSizeOf
)
57 StartupCache::CollectReports(nsIHandleReportCallback
* aHandleReport
,
58 nsISupports
* aData
, bool aAnonymize
) {
60 "explicit/startup-cache/mapping", KIND_NONHEAP
, UNITS_BYTES
,
62 "Memory used to hold the mapping of the startup cache from file. "
63 "This memory is likely to be swapped out shortly after start-up.");
65 MOZ_COLLECT_REPORT("explicit/startup-cache/data", KIND_HEAP
, UNITS_BYTES
,
66 HeapSizeOfIncludingThis(StartupCacheMallocSizeOf
),
67 "Memory used by the startup cache for things other than "
73 #define STARTUP_CACHE_NAME "startupCache." SC_WORDSIZE "." SC_ENDIAN
75 StartupCache
* StartupCache::GetSingleton() {
77 if (!XRE_IsParentProcess()) {
80 #ifdef MOZ_DISABLE_STARTUPCACHE
83 StartupCache::InitSingleton();
87 return StartupCache::gStartupCache
;
90 void StartupCache::DeleteSingleton() { StartupCache::gStartupCache
= nullptr; }
92 nsresult
StartupCache::InitSingleton() {
94 StartupCache::gStartupCache
= new StartupCache();
96 rv
= StartupCache::gStartupCache
->Init();
98 StartupCache::gStartupCache
= nullptr;
103 StaticRefPtr
<StartupCache
> StartupCache::gStartupCache
;
104 bool StartupCache::gShutdownInitiated
;
105 bool StartupCache::gIgnoreDiskCache
;
107 NS_IMPL_ISUPPORTS(StartupCache
, nsIMemoryReporter
)
109 StartupCache::StartupCache()
110 : mArchive(nullptr), mStartupWriteInitiated(false), mWriteThread(nullptr) {}
112 StartupCache::~StartupCache() {
117 // Generally, the in-memory table should be empty here,
118 // but an early shutdown means either mTimer didn't run
119 // or the write thread is still running.
122 // If we shutdown quickly timer wont have fired. Instead of writing
123 // it on the main thread and block the shutdown we simply wont update
124 // the startup cache. Always do this if the file doesn't exist since
125 // we use it part of the package step.
130 UnregisterWeakMemoryReporter(this);
133 nsresult
StartupCache::Init() {
134 // workaround for bug 653936
135 nsCOMPtr
<nsIProtocolHandler
> jarInitializer(
136 do_GetService(NS_NETWORK_PROTOCOL_CONTRACTID_PREFIX
"jar"));
140 // This allows to override the startup cache filename
141 // which is useful from xpcshell, when there is no ProfLDS directory to keep
143 char* env
= PR_GetEnv("MOZ_STARTUP_CACHE");
145 rv
= NS_NewLocalFile(NS_ConvertUTF8toUTF16(env
), false,
146 getter_AddRefs(mFile
));
148 nsCOMPtr
<nsIFile
> file
;
149 rv
= NS_GetSpecialDirectory("ProfLDS", getter_AddRefs(file
));
151 // return silently, this will fail in mochitests's xpcshell process.
155 rv
= file
->AppendNative(NS_LITERAL_CSTRING("startupCache"));
156 NS_ENSURE_SUCCESS(rv
, rv
);
158 // Try to create the directory if it's not there yet
159 rv
= file
->Create(nsIFile::DIRECTORY_TYPE
, 0777);
160 if (NS_FAILED(rv
) && rv
!= NS_ERROR_FILE_ALREADY_EXISTS
) return rv
;
162 rv
= file
->AppendNative(NS_LITERAL_CSTRING(STARTUP_CACHE_NAME
));
164 NS_ENSURE_SUCCESS(rv
, rv
);
169 NS_ENSURE_TRUE(mFile
, NS_ERROR_UNEXPECTED
);
171 mObserverService
= do_GetService("@mozilla.org/observer-service;1");
173 if (!mObserverService
) {
174 NS_WARNING("Could not get observerService.");
175 return NS_ERROR_UNEXPECTED
;
178 mListener
= new StartupCacheListener();
179 rv
= mObserverService
->AddObserver(mListener
, NS_XPCOM_SHUTDOWN_OBSERVER_ID
,
181 NS_ENSURE_SUCCESS(rv
, rv
);
182 rv
= mObserverService
->AddObserver(mListener
, "startupcache-invalidate",
184 NS_ENSURE_SUCCESS(rv
, rv
);
188 // Sometimes we don't have a cache yet, that's ok.
189 // If it's corrupted, just remove it and start over.
190 if (gIgnoreDiskCache
|| (NS_FAILED(rv
) && rv
!= NS_ERROR_FILE_NOT_FOUND
)) {
191 NS_WARNING("Failed to load startupcache file correctly, removing!");
195 RegisterWeakMemoryReporter(this);
201 * LoadArchive can be called from the main thread or while reloading cache on
204 nsresult
StartupCache::LoadArchive() {
205 if (gIgnoreDiskCache
) return NS_ERROR_FAILURE
;
209 nsresult rv
= mFile
->Exists(&exists
);
210 if (NS_FAILED(rv
) || !exists
) return NS_ERROR_FILE_NOT_FOUND
;
212 mArchive
= new nsZipArchive();
213 rv
= mArchive
->OpenArchive(mFile
);
219 nsresult
GetBufferFromZipArchive(nsZipArchive
* zip
, bool doCRC
, const char* id
,
220 UniquePtr
<char[]>* outbuf
, uint32_t* length
) {
221 if (!zip
) return NS_ERROR_NOT_AVAILABLE
;
223 nsZipItemPtr
<char> zipItem(zip
, id
, doCRC
);
224 if (!zipItem
) return NS_ERROR_NOT_AVAILABLE
;
226 *outbuf
= zipItem
.Forget();
227 *length
= zipItem
.Length();
231 } /* anonymous namespace */
233 // NOTE: this will not find a new entry until it has been written to disk!
234 // Consumer should take ownership of the resulting buffer.
235 nsresult
StartupCache::GetBuffer(const char* id
, UniquePtr
<char[]>* outbuf
,
237 AUTO_PROFILER_LABEL("StartupCache::GetBuffer", OTHER
);
239 NS_ASSERTION(NS_IsMainThread(),
240 "Startup cache only available on main thread");
243 if (!mStartupWriteInitiated
) {
245 nsDependentCString
idStr(id
);
246 mTable
.Get(idStr
, &entry
);
248 *outbuf
= MakeUnique
<char[]>(entry
->size
);
249 memcpy(outbuf
->get(), entry
->data
.get(), entry
->size
);
250 *length
= entry
->size
;
251 Telemetry::AccumulateCategorical(
252 Telemetry::LABELS_STARTUP_CACHE_REQUESTS::HitMemory
);
257 nsresult rv
= GetBufferFromZipArchive(mArchive
, true, id
, outbuf
, length
);
258 if (NS_SUCCEEDED(rv
)) {
259 Telemetry::AccumulateCategorical(
260 Telemetry::LABELS_STARTUP_CACHE_REQUESTS::HitDisk
);
264 Telemetry::AccumulateCategorical(
265 Telemetry::LABELS_STARTUP_CACHE_REQUESTS::Miss
);
267 RefPtr
<nsZipArchive
> omnijar
=
268 mozilla::Omnijar::GetReader(mozilla::Omnijar::APP
);
269 // no need to checksum omnijarred entries
270 rv
= GetBufferFromZipArchive(omnijar
, false, id
, outbuf
, length
);
271 if (NS_SUCCEEDED(rv
)) return rv
;
273 omnijar
= mozilla::Omnijar::GetReader(mozilla::Omnijar::GRE
);
274 // no need to checksum omnijarred entries
275 return GetBufferFromZipArchive(omnijar
, false, id
, outbuf
, length
);
278 // Makes a copy of the buffer, client retains ownership of inbuf.
279 nsresult
StartupCache::PutBuffer(const char* id
, UniquePtr
<char[]>&& inbuf
,
281 NS_ASSERTION(NS_IsMainThread(),
282 "Startup cache only available on main thread");
284 if (StartupCache::gShutdownInitiated
) {
285 return NS_ERROR_NOT_AVAILABLE
;
288 nsDependentCString
idStr(id
);
289 // Cache it for now, we'll write all together later.
290 auto entry
= mTable
.LookupForAdd(idStr
);
293 NS_WARNING("Existing entry in StartupCache.");
294 // Double-caching is undesirable but not an error.
300 nsZipItem
* zipItem
= mArchive
->GetItem(id
);
301 NS_ASSERTION(zipItem
== nullptr, "Existing entry in disk StartupCache.");
306 [&inbuf
, &len
]() { return new CacheEntry(std::move(inbuf
), len
); });
307 mPendingWrites
.AppendElement(idStr
);
308 return ResetStartupWriteTimer();
311 size_t StartupCache::SizeOfMapping() {
312 return mArchive
? mArchive
->SizeOfMapping() : 0;
315 size_t StartupCache::HeapSizeOfIncludingThis(
316 mozilla::MallocSizeOf aMallocSizeOf
) const {
317 // This function could measure more members, but they haven't been found by
318 // DMD to be significant. They can be added later if necessary.
320 size_t n
= aMallocSizeOf(this);
322 n
+= mTable
.ShallowSizeOfExcludingThis(aMallocSizeOf
);
323 for (auto iter
= mTable
.ConstIter(); !iter
.Done(); iter
.Next()) {
324 n
+= iter
.Data()->SizeOfIncludingThis(aMallocSizeOf
);
327 n
+= mPendingWrites
.ShallowSizeOfExcludingThis(aMallocSizeOf
);
332 struct CacheWriteHolder
{
333 nsCOMPtr
<nsIZipWriter
> writer
;
334 nsCOMPtr
<nsIStringInputStream
> stream
;
338 static void CacheCloseHelper(const nsACString
& key
, const CacheEntry
* data
,
339 const CacheWriteHolder
* holder
) {
340 MOZ_ASSERT(data
); // assert key was found in mTable.
343 nsIStringInputStream
* stream
= holder
->stream
;
344 nsIZipWriter
* writer
= holder
->writer
;
346 stream
->ShareData(data
->data
.get(), data
->size
);
350 rv
= writer
->HasEntry(key
, &hasEntry
);
351 NS_ASSERTION(NS_SUCCEEDED(rv
) && hasEntry
== false,
352 "Existing entry in disk StartupCache.");
354 rv
= writer
->AddEntryStream(key
, holder
->time
, true, stream
, false);
357 NS_WARNING("cache entry deleted but not written to disk.");
362 * WriteToDisk writes the cache out to disk. Callers of WriteToDisk need to call
363 * WaitOnWriteThread to make sure there isn't a write happening on another
366 void StartupCache::WriteToDisk() {
368 mStartupWriteInitiated
= true;
370 if (mTable
.Count() == 0) return;
372 nsCOMPtr
<nsIZipWriter
> zipW
= do_CreateInstance("@mozilla.org/zipwriter;1");
375 rv
= zipW
->Open(mFile
, PR_RDWR
| PR_CREATE_FILE
);
377 NS_WARNING("could not open zipfile for write");
381 // If we didn't have an mArchive member, that means that we failed to
382 // open the startup cache for reading. Therefore, we need to record
383 // the time of creation in a zipfile comment; this has been useful for
384 // Telemetry statistics.
385 PRTime now
= PR_Now();
388 comment
.Assign((char*)&now
, sizeof(now
));
389 zipW
->SetComment(comment
);
392 nsCOMPtr
<nsIStringInputStream
> stream
=
393 do_CreateInstance("@mozilla.org/io/string-input-stream;1", &rv
);
395 NS_WARNING("Couldn't create string input stream.");
399 CacheWriteHolder holder
;
400 holder
.stream
= stream
;
401 holder
.writer
= zipW
;
404 for (auto& key
: mPendingWrites
) {
405 CacheCloseHelper(key
, mTable
.Get(key
), &holder
);
407 mPendingWrites
.Clear();
410 // Close the archive so Windows doesn't choke.
414 // We succesfully wrote the archive to disk; mark the disk file as trusted
415 gIgnoreDiskCache
= false;
417 // Our reader's view of the archive is outdated now, reload it.
421 void StartupCache::InvalidateCache(bool memoryOnly
) {
423 // The memoryOnly option is just for testing purposes. We want to ensure
424 // that we're nuking the in-memory form but that we preserve everything
430 mPendingWrites
.Clear();
433 nsresult rv
= mFile
->Remove(false);
434 if (NS_FAILED(rv
) && rv
!= NS_ERROR_FILE_TARGET_DOES_NOT_EXIST
&&
435 rv
!= NS_ERROR_FILE_NOT_FOUND
) {
436 gIgnoreDiskCache
= true;
439 gIgnoreDiskCache
= false;
443 void StartupCache::IgnoreDiskCache() {
444 gIgnoreDiskCache
= true;
445 if (gStartupCache
) gStartupCache
->InvalidateCache();
449 * WaitOnWriteThread() is called from a main thread to wait for the worker
450 * thread to finish. However since the same code is used in the worker thread
451 * and main thread, the worker thread can also call WaitOnWriteThread() which is
454 void StartupCache::WaitOnWriteThread() {
455 NS_ASSERTION(NS_IsMainThread(),
456 "Startup cache should only wait for io thread on main thread");
457 if (!mWriteThread
|| mWriteThread
== PR_GetCurrentThread()) return;
459 PR_JoinThread(mWriteThread
);
460 mWriteThread
= nullptr;
463 void StartupCache::ThreadedWrite(void* aClosure
) {
464 AUTO_PROFILER_REGISTER_THREAD("StartupCache");
465 NS_SetCurrentThreadName("StartupCache");
466 mozilla::IOInterposer::RegisterCurrentThread();
468 * It is safe to use the pointer passed in aClosure to reference the
469 * StartupCache object because the thread's lifetime is tightly coupled to
470 * the lifetime of the StartupCache object; this thread is joined in the
471 * StartupCache destructor, guaranteeing that this function runs if and only
472 * if the StartupCache object is valid.
474 StartupCache
* startupCacheObj
= static_cast<StartupCache
*>(aClosure
);
475 startupCacheObj
->WriteToDisk();
476 mozilla::IOInterposer::UnregisterCurrentThread();
480 * The write-thread is spawned on a timeout(which is reset with every write).
481 * This can avoid a slow shutdown. After writing out the cache, the zipreader is
482 * reloaded on the worker thread.
484 void StartupCache::WriteTimeout(nsITimer
* aTimer
, void* aClosure
) {
486 * It is safe to use the pointer passed in aClosure to reference the
487 * StartupCache object because the timer's lifetime is tightly coupled to
488 * the lifetime of the StartupCache object; this timer is canceled in the
489 * StartupCache destructor, guaranteeing that this function runs if and only
490 * if the StartupCache object is valid.
492 StartupCache
* startupCacheObj
= static_cast<StartupCache
*>(aClosure
);
493 startupCacheObj
->mWriteThread
= PR_CreateThread(
494 PR_USER_THREAD
, StartupCache::ThreadedWrite
, startupCacheObj
,
495 PR_PRIORITY_NORMAL
, PR_GLOBAL_THREAD
, PR_JOINABLE_THREAD
, 0);
498 // We don't want to refcount StartupCache, so we'll just
499 // hold a ref to this and pass it to observerService instead.
500 NS_IMPL_ISUPPORTS(StartupCacheListener
, nsIObserver
)
502 nsresult
StartupCacheListener::Observe(nsISupports
* subject
, const char* topic
,
503 const char16_t
* data
) {
504 StartupCache
* sc
= StartupCache::GetSingleton();
505 if (!sc
) return NS_OK
;
507 if (strcmp(topic
, NS_XPCOM_SHUTDOWN_OBSERVER_ID
) == 0) {
508 // Do not leave the thread running past xpcom shutdown
509 sc
->WaitOnWriteThread();
510 StartupCache::gShutdownInitiated
= true;
511 } else if (strcmp(topic
, "startupcache-invalidate") == 0) {
512 sc
->InvalidateCache(data
&& nsCRT::strcmp(data
, u
"memoryOnly") == 0);
517 nsresult
StartupCache::GetDebugObjectOutputStream(
518 nsIObjectOutputStream
* aStream
, nsIObjectOutputStream
** aOutStream
) {
519 NS_ENSURE_ARG_POINTER(aStream
);
521 auto* stream
= new StartupCacheDebugOutputStream(aStream
, &mWriteObjectMap
);
522 NS_ADDREF(*aOutStream
= stream
);
524 NS_ADDREF(*aOutStream
= aStream
);
530 nsresult
StartupCache::ResetStartupWriteTimer() {
531 mStartupWriteInitiated
= false;
534 mTimer
= NS_NewTimer();
536 rv
= mTimer
->Cancel();
537 NS_ENSURE_SUCCESS(rv
, rv
);
538 // Wait for 10 seconds, then write out the cache.
539 mTimer
->InitWithNamedFuncCallback(StartupCache::WriteTimeout
, this, 60000,
540 nsITimer::TYPE_ONE_SHOT
,
541 "StartupCache::WriteTimeout");
545 bool StartupCache::StartupWriteComplete() {
547 return mStartupWriteInitiated
&& mTable
.Count() == 0;
550 // StartupCacheDebugOutputStream implementation
552 NS_IMPL_ISUPPORTS(StartupCacheDebugOutputStream
, nsIObjectOutputStream
,
553 nsIBinaryOutputStream
, nsIOutputStream
)
555 bool StartupCacheDebugOutputStream::CheckReferences(nsISupports
* aObject
) {
558 nsCOMPtr
<nsIClassInfo
> classInfo
= do_QueryInterface(aObject
);
560 NS_ERROR("aObject must implement nsIClassInfo");
565 rv
= classInfo
->GetFlags(&flags
);
566 NS_ENSURE_SUCCESS(rv
, false);
567 if (flags
& nsIClassInfo::SINGLETON
) return true;
569 nsISupportsHashKey
* key
= mObjectMap
->GetEntry(aObject
);
572 "non-singleton aObject is referenced multiple times in this"
573 "serialization, we don't support that.");
577 mObjectMap
->PutEntry(aObject
);
581 // nsIObjectOutputStream implementation
582 nsresult
StartupCacheDebugOutputStream::WriteObject(nsISupports
* aObject
,
584 nsCOMPtr
<nsISupports
> rootObject(do_QueryInterface(aObject
));
586 NS_ASSERTION(rootObject
.get() == aObject
,
587 "bad call to WriteObject -- call WriteCompoundObject!");
588 bool check
= CheckReferences(aObject
);
589 NS_ENSURE_TRUE(check
, NS_ERROR_FAILURE
);
590 return mBinaryStream
->WriteObject(aObject
, aIsStrongRef
);
593 nsresult
StartupCacheDebugOutputStream::WriteSingleRefObject(
594 nsISupports
* aObject
) {
595 nsCOMPtr
<nsISupports
> rootObject(do_QueryInterface(aObject
));
597 NS_ASSERTION(rootObject
.get() == aObject
,
598 "bad call to WriteSingleRefObject -- call WriteCompoundObject!");
599 bool check
= CheckReferences(aObject
);
600 NS_ENSURE_TRUE(check
, NS_ERROR_FAILURE
);
601 return mBinaryStream
->WriteSingleRefObject(aObject
);
604 nsresult
StartupCacheDebugOutputStream::WriteCompoundObject(
605 nsISupports
* aObject
, const nsIID
& aIID
, bool aIsStrongRef
) {
606 nsCOMPtr
<nsISupports
> rootObject(do_QueryInterface(aObject
));
608 nsCOMPtr
<nsISupports
> roundtrip
;
609 rootObject
->QueryInterface(aIID
, getter_AddRefs(roundtrip
));
610 NS_ASSERTION(roundtrip
.get() == aObject
,
611 "bad aggregation or multiple inheritance detected by call to "
612 "WriteCompoundObject!");
614 bool check
= CheckReferences(aObject
);
615 NS_ENSURE_TRUE(check
, NS_ERROR_FAILURE
);
616 return mBinaryStream
->WriteCompoundObject(aObject
, aIID
, aIsStrongRef
);
619 nsresult
StartupCacheDebugOutputStream::WriteID(nsID
const& aID
) {
620 return mBinaryStream
->WriteID(aID
);
623 char* StartupCacheDebugOutputStream::GetBuffer(uint32_t aLength
,
624 uint32_t aAlignMask
) {
625 return mBinaryStream
->GetBuffer(aLength
, aAlignMask
);
628 void StartupCacheDebugOutputStream::PutBuffer(char* aBuffer
, uint32_t aLength
) {
629 mBinaryStream
->PutBuffer(aBuffer
, aLength
);
633 } // namespace scache
634 } // namespace mozilla