Bug 1545089 - Make the subgrid markup badge toggle the grid highlighter. r=pbro
[gecko.git] / startupcache / StartupCache.cpp
blobd310a2a2fc54c93a33cc2cf176c9343dd6700d28
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "prio.h"
8 #include "PLDHashTable.h"
9 #include "mozilla/IOInterposer.h"
10 #include "mozilla/MemoryReporting.h"
11 #include "mozilla/scache/StartupCache.h"
13 #include "nsAutoPtr.h"
14 #include "nsClassHashtable.h"
15 #include "nsComponentManagerUtils.h"
16 #include "nsCRT.h"
17 #include "nsDirectoryServiceUtils.h"
18 #include "nsIClassInfo.h"
19 #include "nsIFile.h"
20 #include "nsIObserver.h"
21 #include "nsIObserverService.h"
22 #include "nsIOutputStream.h"
23 #include "nsIStorageStream.h"
24 #include "nsIStreamBufferAccess.h"
25 #include "nsIStringStream.h"
26 #include "nsISupports.h"
27 #include "nsITimer.h"
28 #include "nsIZipWriter.h"
29 #include "nsIZipReader.h"
30 #include "nsZipArchive.h"
31 #include "mozilla/Omnijar.h"
32 #include "prenv.h"
33 #include "mozilla/Telemetry.h"
34 #include "nsThreadUtils.h"
35 #include "nsXULAppAPI.h"
36 #include "nsIProtocolHandler.h"
37 #include "GeckoProfiler.h"
39 #ifdef IS_BIG_ENDIAN
40 # define SC_ENDIAN "big"
41 #else
42 # define SC_ENDIAN "little"
43 #endif
45 #if PR_BYTES_PER_WORD == 4
46 # define SC_WORDSIZE "4"
47 #else
48 # define SC_WORDSIZE "8"
49 #endif
51 namespace mozilla {
52 namespace scache {
54 MOZ_DEFINE_MALLOC_SIZE_OF(StartupCacheMallocSizeOf)
56 NS_IMETHODIMP
57 StartupCache::CollectReports(nsIHandleReportCallback* aHandleReport,
58 nsISupports* aData, bool aAnonymize) {
59 MOZ_COLLECT_REPORT(
60 "explicit/startup-cache/mapping", KIND_NONHEAP, UNITS_BYTES,
61 SizeOfMapping(),
62 "Memory used to hold the mapping of the startup cache from file. "
63 "This memory is likely to be swapped out shortly after start-up.");
65 MOZ_COLLECT_REPORT("explicit/startup-cache/data", KIND_HEAP, UNITS_BYTES,
66 HeapSizeOfIncludingThis(StartupCacheMallocSizeOf),
67 "Memory used by the startup cache for things other than "
68 "the file mapping.");
70 return NS_OK;
73 #define STARTUP_CACHE_NAME "startupCache." SC_WORDSIZE "." SC_ENDIAN
75 StartupCache* StartupCache::GetSingleton() {
76 if (!gStartupCache) {
77 if (!XRE_IsParentProcess()) {
78 return nullptr;
80 #ifdef MOZ_DISABLE_STARTUPCACHE
81 return nullptr;
82 #else
83 StartupCache::InitSingleton();
84 #endif
87 return StartupCache::gStartupCache;
90 void StartupCache::DeleteSingleton() { StartupCache::gStartupCache = nullptr; }
92 nsresult StartupCache::InitSingleton() {
93 nsresult rv;
94 StartupCache::gStartupCache = new StartupCache();
96 rv = StartupCache::gStartupCache->Init();
97 if (NS_FAILED(rv)) {
98 StartupCache::gStartupCache = nullptr;
100 return rv;
103 StaticRefPtr<StartupCache> StartupCache::gStartupCache;
104 bool StartupCache::gShutdownInitiated;
105 bool StartupCache::gIgnoreDiskCache;
107 NS_IMPL_ISUPPORTS(StartupCache, nsIMemoryReporter)
109 StartupCache::StartupCache()
110 : mArchive(nullptr), mStartupWriteInitiated(false), mWriteThread(nullptr) {}
112 StartupCache::~StartupCache() {
113 if (mTimer) {
114 mTimer->Cancel();
117 // Generally, the in-memory table should be empty here,
118 // but an early shutdown means either mTimer didn't run
119 // or the write thread is still running.
120 WaitOnWriteThread();
122 // If we shutdown quickly timer wont have fired. Instead of writing
123 // it on the main thread and block the shutdown we simply wont update
124 // the startup cache. Always do this if the file doesn't exist since
125 // we use it part of the package step.
126 if (!mArchive) {
127 WriteToDisk();
130 UnregisterWeakMemoryReporter(this);
133 nsresult StartupCache::Init() {
134 // workaround for bug 653936
135 nsCOMPtr<nsIProtocolHandler> jarInitializer(
136 do_GetService(NS_NETWORK_PROTOCOL_CONTRACTID_PREFIX "jar"));
138 nsresult rv;
140 // This allows to override the startup cache filename
141 // which is useful from xpcshell, when there is no ProfLDS directory to keep
142 // cache in.
143 char* env = PR_GetEnv("MOZ_STARTUP_CACHE");
144 if (env && *env) {
145 rv = NS_NewLocalFile(NS_ConvertUTF8toUTF16(env), false,
146 getter_AddRefs(mFile));
147 } else {
148 nsCOMPtr<nsIFile> file;
149 rv = NS_GetSpecialDirectory("ProfLDS", getter_AddRefs(file));
150 if (NS_FAILED(rv)) {
151 // return silently, this will fail in mochitests's xpcshell process.
152 return rv;
155 rv = file->AppendNative(NS_LITERAL_CSTRING("startupCache"));
156 NS_ENSURE_SUCCESS(rv, rv);
158 // Try to create the directory if it's not there yet
159 rv = file->Create(nsIFile::DIRECTORY_TYPE, 0777);
160 if (NS_FAILED(rv) && rv != NS_ERROR_FILE_ALREADY_EXISTS) return rv;
162 rv = file->AppendNative(NS_LITERAL_CSTRING(STARTUP_CACHE_NAME));
164 NS_ENSURE_SUCCESS(rv, rv);
166 mFile = file;
169 NS_ENSURE_TRUE(mFile, NS_ERROR_UNEXPECTED);
171 mObserverService = do_GetService("@mozilla.org/observer-service;1");
173 if (!mObserverService) {
174 NS_WARNING("Could not get observerService.");
175 return NS_ERROR_UNEXPECTED;
178 mListener = new StartupCacheListener();
179 rv = mObserverService->AddObserver(mListener, NS_XPCOM_SHUTDOWN_OBSERVER_ID,
180 false);
181 NS_ENSURE_SUCCESS(rv, rv);
182 rv = mObserverService->AddObserver(mListener, "startupcache-invalidate",
183 false);
184 NS_ENSURE_SUCCESS(rv, rv);
186 rv = LoadArchive();
188 // Sometimes we don't have a cache yet, that's ok.
189 // If it's corrupted, just remove it and start over.
190 if (gIgnoreDiskCache || (NS_FAILED(rv) && rv != NS_ERROR_FILE_NOT_FOUND)) {
191 NS_WARNING("Failed to load startupcache file correctly, removing!");
192 InvalidateCache();
195 RegisterWeakMemoryReporter(this);
197 return NS_OK;
201 * LoadArchive can be called from the main thread or while reloading cache on
202 * write thread.
204 nsresult StartupCache::LoadArchive() {
205 if (gIgnoreDiskCache) return NS_ERROR_FAILURE;
207 bool exists;
208 mArchive = nullptr;
209 nsresult rv = mFile->Exists(&exists);
210 if (NS_FAILED(rv) || !exists) return NS_ERROR_FILE_NOT_FOUND;
212 mArchive = new nsZipArchive();
213 rv = mArchive->OpenArchive(mFile);
214 return rv;
217 namespace {
219 nsresult GetBufferFromZipArchive(nsZipArchive* zip, bool doCRC, const char* id,
220 UniquePtr<char[]>* outbuf, uint32_t* length) {
221 if (!zip) return NS_ERROR_NOT_AVAILABLE;
223 nsZipItemPtr<char> zipItem(zip, id, doCRC);
224 if (!zipItem) return NS_ERROR_NOT_AVAILABLE;
226 *outbuf = zipItem.Forget();
227 *length = zipItem.Length();
228 return NS_OK;
231 } /* anonymous namespace */
233 // NOTE: this will not find a new entry until it has been written to disk!
234 // Consumer should take ownership of the resulting buffer.
235 nsresult StartupCache::GetBuffer(const char* id, UniquePtr<char[]>* outbuf,
236 uint32_t* length) {
237 AUTO_PROFILER_LABEL("StartupCache::GetBuffer", OTHER);
239 NS_ASSERTION(NS_IsMainThread(),
240 "Startup cache only available on main thread");
242 WaitOnWriteThread();
243 if (!mStartupWriteInitiated) {
244 CacheEntry* entry;
245 nsDependentCString idStr(id);
246 mTable.Get(idStr, &entry);
247 if (entry) {
248 *outbuf = MakeUnique<char[]>(entry->size);
249 memcpy(outbuf->get(), entry->data.get(), entry->size);
250 *length = entry->size;
251 Telemetry::AccumulateCategorical(
252 Telemetry::LABELS_STARTUP_CACHE_REQUESTS::HitMemory);
253 return NS_OK;
257 nsresult rv = GetBufferFromZipArchive(mArchive, true, id, outbuf, length);
258 if (NS_SUCCEEDED(rv)) {
259 Telemetry::AccumulateCategorical(
260 Telemetry::LABELS_STARTUP_CACHE_REQUESTS::HitDisk);
261 return rv;
264 Telemetry::AccumulateCategorical(
265 Telemetry::LABELS_STARTUP_CACHE_REQUESTS::Miss);
267 RefPtr<nsZipArchive> omnijar =
268 mozilla::Omnijar::GetReader(mozilla::Omnijar::APP);
269 // no need to checksum omnijarred entries
270 rv = GetBufferFromZipArchive(omnijar, false, id, outbuf, length);
271 if (NS_SUCCEEDED(rv)) return rv;
273 omnijar = mozilla::Omnijar::GetReader(mozilla::Omnijar::GRE);
274 // no need to checksum omnijarred entries
275 return GetBufferFromZipArchive(omnijar, false, id, outbuf, length);
278 // Makes a copy of the buffer, client retains ownership of inbuf.
279 nsresult StartupCache::PutBuffer(const char* id, UniquePtr<char[]>&& inbuf,
280 uint32_t len) {
281 NS_ASSERTION(NS_IsMainThread(),
282 "Startup cache only available on main thread");
283 WaitOnWriteThread();
284 if (StartupCache::gShutdownInitiated) {
285 return NS_ERROR_NOT_AVAILABLE;
288 nsDependentCString idStr(id);
289 // Cache it for now, we'll write all together later.
290 auto entry = mTable.LookupForAdd(idStr);
292 if (entry) {
293 NS_WARNING("Existing entry in StartupCache.");
294 // Double-caching is undesirable but not an error.
295 return NS_OK;
298 #ifdef DEBUG
299 if (mArchive) {
300 nsZipItem* zipItem = mArchive->GetItem(id);
301 NS_ASSERTION(zipItem == nullptr, "Existing entry in disk StartupCache.");
303 #endif
305 entry.OrInsert(
306 [&inbuf, &len]() { return new CacheEntry(std::move(inbuf), len); });
307 mPendingWrites.AppendElement(idStr);
308 return ResetStartupWriteTimer();
311 size_t StartupCache::SizeOfMapping() {
312 return mArchive ? mArchive->SizeOfMapping() : 0;
315 size_t StartupCache::HeapSizeOfIncludingThis(
316 mozilla::MallocSizeOf aMallocSizeOf) const {
317 // This function could measure more members, but they haven't been found by
318 // DMD to be significant. They can be added later if necessary.
320 size_t n = aMallocSizeOf(this);
322 n += mTable.ShallowSizeOfExcludingThis(aMallocSizeOf);
323 for (auto iter = mTable.ConstIter(); !iter.Done(); iter.Next()) {
324 n += iter.Data()->SizeOfIncludingThis(aMallocSizeOf);
327 n += mPendingWrites.ShallowSizeOfExcludingThis(aMallocSizeOf);
329 return n;
332 struct CacheWriteHolder {
333 nsCOMPtr<nsIZipWriter> writer;
334 nsCOMPtr<nsIStringInputStream> stream;
335 PRTime time;
338 static void CacheCloseHelper(const nsACString& key, const CacheEntry* data,
339 const CacheWriteHolder* holder) {
340 MOZ_ASSERT(data); // assert key was found in mTable.
342 nsresult rv;
343 nsIStringInputStream* stream = holder->stream;
344 nsIZipWriter* writer = holder->writer;
346 stream->ShareData(data->data.get(), data->size);
348 #ifdef DEBUG
349 bool hasEntry;
350 rv = writer->HasEntry(key, &hasEntry);
351 NS_ASSERTION(NS_SUCCEEDED(rv) && hasEntry == false,
352 "Existing entry in disk StartupCache.");
353 #endif
354 rv = writer->AddEntryStream(key, holder->time, true, stream, false);
356 if (NS_FAILED(rv)) {
357 NS_WARNING("cache entry deleted but not written to disk.");
362 * WriteToDisk writes the cache out to disk. Callers of WriteToDisk need to call
363 * WaitOnWriteThread to make sure there isn't a write happening on another
364 * thread
366 void StartupCache::WriteToDisk() {
367 nsresult rv;
368 mStartupWriteInitiated = true;
370 if (mTable.Count() == 0) return;
372 nsCOMPtr<nsIZipWriter> zipW = do_CreateInstance("@mozilla.org/zipwriter;1");
373 if (!zipW) return;
375 rv = zipW->Open(mFile, PR_RDWR | PR_CREATE_FILE);
376 if (NS_FAILED(rv)) {
377 NS_WARNING("could not open zipfile for write");
378 return;
381 // If we didn't have an mArchive member, that means that we failed to
382 // open the startup cache for reading. Therefore, we need to record
383 // the time of creation in a zipfile comment; this has been useful for
384 // Telemetry statistics.
385 PRTime now = PR_Now();
386 if (!mArchive) {
387 nsCString comment;
388 comment.Assign((char*)&now, sizeof(now));
389 zipW->SetComment(comment);
392 nsCOMPtr<nsIStringInputStream> stream =
393 do_CreateInstance("@mozilla.org/io/string-input-stream;1", &rv);
394 if (NS_FAILED(rv)) {
395 NS_WARNING("Couldn't create string input stream.");
396 return;
399 CacheWriteHolder holder;
400 holder.stream = stream;
401 holder.writer = zipW;
402 holder.time = now;
404 for (auto& key : mPendingWrites) {
405 CacheCloseHelper(key, mTable.Get(key), &holder);
407 mPendingWrites.Clear();
408 mTable.Clear();
410 // Close the archive so Windows doesn't choke.
411 mArchive = nullptr;
412 zipW->Close();
414 // We succesfully wrote the archive to disk; mark the disk file as trusted
415 gIgnoreDiskCache = false;
417 // Our reader's view of the archive is outdated now, reload it.
418 LoadArchive();
421 void StartupCache::InvalidateCache(bool memoryOnly) {
422 if (memoryOnly) {
423 // The memoryOnly option is just for testing purposes. We want to ensure
424 // that we're nuking the in-memory form but that we preserve everything
425 // on disk.
426 WriteToDisk();
427 return;
429 WaitOnWriteThread();
430 mPendingWrites.Clear();
431 mTable.Clear();
432 mArchive = nullptr;
433 nsresult rv = mFile->Remove(false);
434 if (NS_FAILED(rv) && rv != NS_ERROR_FILE_TARGET_DOES_NOT_EXIST &&
435 rv != NS_ERROR_FILE_NOT_FOUND) {
436 gIgnoreDiskCache = true;
437 return;
439 gIgnoreDiskCache = false;
440 LoadArchive();
443 void StartupCache::IgnoreDiskCache() {
444 gIgnoreDiskCache = true;
445 if (gStartupCache) gStartupCache->InvalidateCache();
449 * WaitOnWriteThread() is called from a main thread to wait for the worker
450 * thread to finish. However since the same code is used in the worker thread
451 * and main thread, the worker thread can also call WaitOnWriteThread() which is
452 * a no-op.
454 void StartupCache::WaitOnWriteThread() {
455 NS_ASSERTION(NS_IsMainThread(),
456 "Startup cache should only wait for io thread on main thread");
457 if (!mWriteThread || mWriteThread == PR_GetCurrentThread()) return;
459 PR_JoinThread(mWriteThread);
460 mWriteThread = nullptr;
463 void StartupCache::ThreadedWrite(void* aClosure) {
464 AUTO_PROFILER_REGISTER_THREAD("StartupCache");
465 NS_SetCurrentThreadName("StartupCache");
466 mozilla::IOInterposer::RegisterCurrentThread();
468 * It is safe to use the pointer passed in aClosure to reference the
469 * StartupCache object because the thread's lifetime is tightly coupled to
470 * the lifetime of the StartupCache object; this thread is joined in the
471 * StartupCache destructor, guaranteeing that this function runs if and only
472 * if the StartupCache object is valid.
474 StartupCache* startupCacheObj = static_cast<StartupCache*>(aClosure);
475 startupCacheObj->WriteToDisk();
476 mozilla::IOInterposer::UnregisterCurrentThread();
480 * The write-thread is spawned on a timeout(which is reset with every write).
481 * This can avoid a slow shutdown. After writing out the cache, the zipreader is
482 * reloaded on the worker thread.
484 void StartupCache::WriteTimeout(nsITimer* aTimer, void* aClosure) {
486 * It is safe to use the pointer passed in aClosure to reference the
487 * StartupCache object because the timer's lifetime is tightly coupled to
488 * the lifetime of the StartupCache object; this timer is canceled in the
489 * StartupCache destructor, guaranteeing that this function runs if and only
490 * if the StartupCache object is valid.
492 StartupCache* startupCacheObj = static_cast<StartupCache*>(aClosure);
493 startupCacheObj->mWriteThread = PR_CreateThread(
494 PR_USER_THREAD, StartupCache::ThreadedWrite, startupCacheObj,
495 PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, PR_JOINABLE_THREAD, 0);
498 // We don't want to refcount StartupCache, so we'll just
499 // hold a ref to this and pass it to observerService instead.
500 NS_IMPL_ISUPPORTS(StartupCacheListener, nsIObserver)
502 nsresult StartupCacheListener::Observe(nsISupports* subject, const char* topic,
503 const char16_t* data) {
504 StartupCache* sc = StartupCache::GetSingleton();
505 if (!sc) return NS_OK;
507 if (strcmp(topic, NS_XPCOM_SHUTDOWN_OBSERVER_ID) == 0) {
508 // Do not leave the thread running past xpcom shutdown
509 sc->WaitOnWriteThread();
510 StartupCache::gShutdownInitiated = true;
511 } else if (strcmp(topic, "startupcache-invalidate") == 0) {
512 sc->InvalidateCache(data && nsCRT::strcmp(data, u"memoryOnly") == 0);
514 return NS_OK;
517 nsresult StartupCache::GetDebugObjectOutputStream(
518 nsIObjectOutputStream* aStream, nsIObjectOutputStream** aOutStream) {
519 NS_ENSURE_ARG_POINTER(aStream);
520 #ifdef DEBUG
521 auto* stream = new StartupCacheDebugOutputStream(aStream, &mWriteObjectMap);
522 NS_ADDREF(*aOutStream = stream);
523 #else
524 NS_ADDREF(*aOutStream = aStream);
525 #endif
527 return NS_OK;
530 nsresult StartupCache::ResetStartupWriteTimer() {
531 mStartupWriteInitiated = false;
532 nsresult rv = NS_OK;
533 if (!mTimer)
534 mTimer = NS_NewTimer();
535 else
536 rv = mTimer->Cancel();
537 NS_ENSURE_SUCCESS(rv, rv);
538 // Wait for 10 seconds, then write out the cache.
539 mTimer->InitWithNamedFuncCallback(StartupCache::WriteTimeout, this, 60000,
540 nsITimer::TYPE_ONE_SHOT,
541 "StartupCache::WriteTimeout");
542 return NS_OK;
545 bool StartupCache::StartupWriteComplete() {
546 WaitOnWriteThread();
547 return mStartupWriteInitiated && mTable.Count() == 0;
550 // StartupCacheDebugOutputStream implementation
551 #ifdef DEBUG
552 NS_IMPL_ISUPPORTS(StartupCacheDebugOutputStream, nsIObjectOutputStream,
553 nsIBinaryOutputStream, nsIOutputStream)
555 bool StartupCacheDebugOutputStream::CheckReferences(nsISupports* aObject) {
556 nsresult rv;
558 nsCOMPtr<nsIClassInfo> classInfo = do_QueryInterface(aObject);
559 if (!classInfo) {
560 NS_ERROR("aObject must implement nsIClassInfo");
561 return false;
564 uint32_t flags;
565 rv = classInfo->GetFlags(&flags);
566 NS_ENSURE_SUCCESS(rv, false);
567 if (flags & nsIClassInfo::SINGLETON) return true;
569 nsISupportsHashKey* key = mObjectMap->GetEntry(aObject);
570 if (key) {
571 NS_ERROR(
572 "non-singleton aObject is referenced multiple times in this"
573 "serialization, we don't support that.");
574 return false;
577 mObjectMap->PutEntry(aObject);
578 return true;
581 // nsIObjectOutputStream implementation
582 nsresult StartupCacheDebugOutputStream::WriteObject(nsISupports* aObject,
583 bool aIsStrongRef) {
584 nsCOMPtr<nsISupports> rootObject(do_QueryInterface(aObject));
586 NS_ASSERTION(rootObject.get() == aObject,
587 "bad call to WriteObject -- call WriteCompoundObject!");
588 bool check = CheckReferences(aObject);
589 NS_ENSURE_TRUE(check, NS_ERROR_FAILURE);
590 return mBinaryStream->WriteObject(aObject, aIsStrongRef);
593 nsresult StartupCacheDebugOutputStream::WriteSingleRefObject(
594 nsISupports* aObject) {
595 nsCOMPtr<nsISupports> rootObject(do_QueryInterface(aObject));
597 NS_ASSERTION(rootObject.get() == aObject,
598 "bad call to WriteSingleRefObject -- call WriteCompoundObject!");
599 bool check = CheckReferences(aObject);
600 NS_ENSURE_TRUE(check, NS_ERROR_FAILURE);
601 return mBinaryStream->WriteSingleRefObject(aObject);
604 nsresult StartupCacheDebugOutputStream::WriteCompoundObject(
605 nsISupports* aObject, const nsIID& aIID, bool aIsStrongRef) {
606 nsCOMPtr<nsISupports> rootObject(do_QueryInterface(aObject));
608 nsCOMPtr<nsISupports> roundtrip;
609 rootObject->QueryInterface(aIID, getter_AddRefs(roundtrip));
610 NS_ASSERTION(roundtrip.get() == aObject,
611 "bad aggregation or multiple inheritance detected by call to "
612 "WriteCompoundObject!");
614 bool check = CheckReferences(aObject);
615 NS_ENSURE_TRUE(check, NS_ERROR_FAILURE);
616 return mBinaryStream->WriteCompoundObject(aObject, aIID, aIsStrongRef);
619 nsresult StartupCacheDebugOutputStream::WriteID(nsID const& aID) {
620 return mBinaryStream->WriteID(aID);
623 char* StartupCacheDebugOutputStream::GetBuffer(uint32_t aLength,
624 uint32_t aAlignMask) {
625 return mBinaryStream->GetBuffer(aLength, aAlignMask);
628 void StartupCacheDebugOutputStream::PutBuffer(char* aBuffer, uint32_t aLength) {
629 mBinaryStream->PutBuffer(aBuffer, aLength);
631 #endif // DEBUG
633 } // namespace scache
634 } // namespace mozilla