1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sw=2 et tw=80:
4 * This Source Code Form is subject to the terms of the Mozilla Public
5 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
6 * You can obtain one at http://mozilla.org/MPL/2.0/. */
11 #include "mozilla/EnumeratedArray.h"
12 #include "mozilla/TimeStamp.h"
14 #include "gc/GCProbes.h"
16 #include "gc/MallocedBlockCache.h"
17 #include "gc/Pretenuring.h"
18 #include "js/AllocPolicy.h"
21 #include "js/TypeDecls.h"
22 #include "js/UniquePtr.h"
23 #include "js/Vector.h"
25 #define FOR_EACH_NURSERY_PROFILE_TIME(_) \
26 /* Key Header text */ \
28 _(TraceValues, "mkVals") \
29 _(TraceCells, "mkClls") \
30 _(TraceSlots, "mkSlts") \
31 _(TraceWasmAnyRefs, "mkWars") \
32 _(TraceWholeCells, "mcWCll") \
33 _(TraceGenericEntries, "mkGnrc") \
34 _(CheckHashTables, "ckTbls") \
35 _(MarkRuntime, "mkRntm") \
36 _(MarkDebugger, "mkDbgr") \
37 _(SweepCaches, "swpCch") \
38 _(CollectToObjFP, "colObj") \
39 _(CollectToStrFP, "colStr") \
40 _(ObjectsTenuredCallback, "tenCB") \
42 _(UpdateJitActivations, "updtIn") \
43 _(FreeMallocedBuffers, "frSlts") \
44 _(FreeTrailerBlocks, "frTrBs") \
45 _(ClearStoreBuffer, "clrSB") \
46 _(ClearNursery, "clear") \
47 _(PurgeStringToAtomCache, "pStoA") \
48 _(Pretenure, "pretnr")
56 class AutoLockGCBgAlloc
;
63 class JS_PUBLIC_API Sprinter
;
68 class GCSchedulingTunables
;
74 explicit Nursery(gc::GCRuntime
* gc
);
77 [[nodiscard
]] bool init(AutoLockGCBgAlloc
& lock
);
79 // Number of allocated (ready to use) chunks.
80 unsigned allocatedChunkCount() const { return chunks_
.length(); }
82 // Total number of chunks and the capacity of the nursery. Chunks will be
83 // lazilly allocated and added to the chunks array up to this limit, after
84 // that the nursery must be collected, this limit may be raised during
86 unsigned maxChunkCount() const {
87 MOZ_ASSERT(capacity());
88 return HowMany(capacity(), gc::ChunkSize
);
93 bool isEnabled() const { return capacity() != 0; }
96 void disableStrings();
97 bool canAllocateStrings() const { return canAllocateStrings_
; }
100 void disableBigInts();
101 bool canAllocateBigInts() const { return canAllocateBigInts_
; }
103 // Return true if no allocations have been made since the last collection.
104 bool isEmpty() const;
106 // Check whether an arbitrary pointer is within the nursery. This is
107 // slower than IsInsideNursery(Cell*), but works on all types of pointers.
108 MOZ_ALWAYS_INLINE
bool isInside(gc::Cell
* cellp
) const = delete;
109 MOZ_ALWAYS_INLINE
bool isInside(const void* p
) const {
110 for (auto* chunk
: chunks_
) {
111 if (uintptr_t(p
) - uintptr_t(chunk
) < gc::ChunkSize
) {
118 template <typename T
>
119 inline bool isInside(const SharedMem
<T
>& p
) const;
121 // Allocate and return a pointer to a new GC thing. Returns nullptr if the
123 void* allocateCell(gc::AllocSite
* site
, size_t size
, JS::TraceKind kind
);
125 // Allocate and return a pointer to a new GC thing. Returns nullptr if the
126 // handleAllocationFailure() needs to be called before retrying.
127 void* tryAllocateCell(gc::AllocSite
* site
, size_t size
, JS::TraceKind kind
) {
128 // Ensure there's enough space to replace the contents with a
129 // RelocationOverlay.
130 // MOZ_ASSERT(size >= sizeof(RelocationOverlay));
131 MOZ_ASSERT(size
% gc::CellAlignBytes
== 0);
132 MOZ_ASSERT(size_t(kind
) < gc::NurseryTraceKinds
);
133 MOZ_ASSERT_IF(kind
== JS::TraceKind::String
, canAllocateStrings());
134 MOZ_ASSERT_IF(kind
== JS::TraceKind::BigInt
, canAllocateBigInts());
136 void* ptr
= tryAllocate(sizeof(gc::NurseryCellHeader
) + size
);
137 if (MOZ_UNLIKELY(!ptr
)) {
141 new (ptr
) gc::NurseryCellHeader(site
, kind
);
144 reinterpret_cast<void*>(uintptr_t(ptr
) + sizeof(gc::NurseryCellHeader
));
146 MOZ_MAKE_COMPILER_ASSUME_IS_UNREACHABLE(
147 "Successful allocation cannot result in nullptr");
150 // Update the allocation site. This code is also inlined in
151 // MacroAssembler::updateAllocSite.
152 uint32_t allocCount
= site
->incAllocCount();
153 if (allocCount
== 1) {
154 pretenuringNursery
.insertIntoAllocatedList(site
);
156 MOZ_ASSERT_IF(site
->isNormal(), site
->isInAllocatedList());
158 gc::gcprobes::NurseryAlloc(cell
, kind
);
162 // Attempt to handle the failure of tryAllocate. Returns a GCReason if minor
163 // GC is required, or NO_REASON if the failure was handled and allocation will
165 [[nodiscard
]] JS::GCReason
handleAllocationFailure();
167 static size_t nurseryCellHeaderSize() {
168 return sizeof(gc::NurseryCellHeader
);
171 // Allocate a buffer for a given zone, using the nursery if possible.
172 void* allocateBuffer(JS::Zone
* zone
, size_t nbytes
);
174 // Allocate a buffer for a given object, using the nursery if possible and
175 // obj is in the nursery.
176 void* allocateBuffer(JS::Zone
* zone
, JSObject
* obj
, size_t nbytes
);
178 // Allocate a buffer for a given object, always using the nursery if obj is
179 // in the nursery. The requested size must be less than or equal to
180 // MaxNurseryBufferSize.
181 void* allocateBufferSameLocation(JSObject
* obj
, size_t nbytes
);
183 // Allocate a zero-initialized buffer for a given zone, using the nursery if
184 // possible. If the buffer isn't allocated in the nursery, the given arena is
186 void* allocateZeroedBuffer(JS::Zone
* zone
, size_t nbytes
,
187 arena_id_t arena
= js::MallocArena
);
189 // Allocate a zero-initialized buffer for a given object, using the nursery if
190 // possible and obj is in the nursery. If the buffer isn't allocated in the
191 // nursery, the given arena is used.
192 void* allocateZeroedBuffer(JSObject
* obj
, size_t nbytes
,
193 arena_id_t arena
= js::MallocArena
);
195 // Resize an existing buffer.
196 void* reallocateBuffer(JS::Zone
* zone
, gc::Cell
* cell
, void* oldBuffer
,
197 size_t oldBytes
, size_t newBytes
);
199 // Allocate a digits buffer for a given BigInt, using the nursery if possible
200 // and |bi| is in the nursery.
201 void* allocateBuffer(JS::BigInt
* bi
, size_t nbytes
);
203 // Free an object buffer.
204 void freeBuffer(void* buffer
, size_t nbytes
);
206 // The maximum number of bytes allowed to reside in nursery buffers.
207 static const size_t MaxNurseryBufferSize
= 1024;
209 // Do a minor collection.
210 void collect(JS::GCOptions options
, JS::GCReason reason
);
212 // If the thing at |*ref| in the Nursery has been forwarded, set |*ref| to
213 // the new location and return true. Otherwise return false and leave
215 [[nodiscard
]] MOZ_ALWAYS_INLINE
static bool getForwardedPointer(
218 // Forward a slots/elements pointer stored in an Ion frame.
219 void forwardBufferPointer(uintptr_t* pSlotsElems
);
221 inline void maybeSetForwardingPointer(JSTracer
* trc
, void* oldData
,
222 void* newData
, bool direct
);
223 inline void setForwardingPointerWhileTenuring(void* oldData
, void* newData
,
226 // Register a malloced buffer that is held by a nursery object, which
227 // should be freed at the end of a minor GC. Buffers are unregistered when
228 // their owning objects are tenured.
229 [[nodiscard
]] bool registerMallocedBuffer(void* buffer
, size_t nbytes
);
231 // Mark a malloced buffer as no longer needing to be freed.
232 void removeMallocedBuffer(void* buffer
, size_t nbytes
) {
233 MOZ_ASSERT(mallocedBuffers
.has(buffer
));
234 MOZ_ASSERT(nbytes
> 0);
235 MOZ_ASSERT(mallocedBufferBytes
>= nbytes
);
236 mallocedBuffers
.remove(buffer
);
237 mallocedBufferBytes
-= nbytes
;
240 // Mark a malloced buffer as no longer needing to be freed during minor
241 // GC. There's no need to account for the size here since all remaining
242 // buffers will soon be freed.
243 void removeMallocedBufferDuringMinorGC(void* buffer
) {
244 MOZ_ASSERT(JS::RuntimeHeapIsMinorCollecting());
245 MOZ_ASSERT(mallocedBuffers
.has(buffer
));
246 mallocedBuffers
.remove(buffer
);
249 [[nodiscard
]] bool addedUniqueIdToCell(gc::Cell
* cell
) {
250 MOZ_ASSERT(IsInsideNursery(cell
));
251 MOZ_ASSERT(isEnabled());
252 return cellsWithUid_
.append(cell
);
255 size_t sizeOfMallocedBuffers(mozilla::MallocSizeOf mallocSizeOf
) const;
257 // Wasm "trailer" (C++-heap-allocated) blocks.
259 // All involved blocks are allocated/deallocated via this nursery's
260 // `mallocedBlockCache_`. Hence we must store both the block address and
261 // its freelist ID, wrapped up in a PointerAndUint7.
263 // Trailer blocks registered here are added to `trailersAdded_`. Those that
264 // are later deregistered as a result of `obj_moved` calls that indicate
265 // tenuring, should be added to `trailersRemoved_`.
267 // Unfortunately ::unregisterTrailer cannot be allowed to OOM. To get
268 // around this we rely on the observation that all deregistered blocks
269 // should previously have been registered, so the deregistered set can never
270 // be larger than the registered set. Hence ::registerTrailer effectively
271 // preallocates space in `trailersRemoved_` so as to ensure that, in the
272 // worst case, all registered blocks can be handed to ::unregisterTrailer
273 // without needing to resize `trailersRemoved_` in ::unregisterTrailer.
275 // The downside is that most of the space in `trailersRemoved_` is wasted in
276 // the case where there are few blocks deregistered. This is unfortunate
277 // but it's hard to see how to avoid it.
279 // At the end of a minor collection, all blocks in the set `trailersAdded_ -
280 // trailersRemoved_[0 .. trailersRemovedUsed_ - 1]` are handed back to the
281 // `mallocedBlockCache_`.
282 [[nodiscard
]] inline bool registerTrailer(PointerAndUint7 blockAndListID
,
284 MOZ_ASSERT(trailersAdded_
.length() == trailersRemoved_
.length());
285 MOZ_ASSERT(nBytes
> 0);
286 if (MOZ_UNLIKELY(!trailersAdded_
.append(blockAndListID
))) {
289 if (MOZ_UNLIKELY(!trailersRemoved_
.append(nullptr))) {
290 trailersAdded_
.popBack();
294 // This is a clone of the logic in ::registerMallocedBuffer. It may be
295 // that some other heuristic is better, once we know more about the
296 // typical behaviour of wasm-GC applications.
297 trailerBytes_
+= nBytes
;
298 if (MOZ_UNLIKELY(trailerBytes_
> capacity() * 8)) {
299 requestMinorGC(JS::GCReason::NURSERY_TRAILERS
);
304 void inline unregisterTrailer(void* block
) {
305 MOZ_ASSERT(trailersRemovedUsed_
< trailersRemoved_
.length());
306 trailersRemoved_
[trailersRemovedUsed_
] = block
;
307 trailersRemovedUsed_
++;
310 size_t sizeOfTrailerBlockSets(mozilla::MallocSizeOf mallocSizeOf
) const;
312 // The number of bytes from the start position to the end of the nursery.
313 // pass maxChunkCount(), allocatedChunkCount() or chunkCountLimit()
314 // to calculate the nursery size, current lazy-allocated size or nursery
315 // limit respectively.
316 size_t spaceToEnd(unsigned chunkCount
) const;
318 size_t capacity() const { return capacity_
; }
319 size_t committed() const { return spaceToEnd(allocatedChunkCount()); }
321 // Used and free space both include chunk headers for that part of the
324 // usedSpace() + freeSpace() == capacity()
326 MOZ_ALWAYS_INLINE
size_t usedSpace() const {
327 return capacity() - freeSpace();
329 MOZ_ALWAYS_INLINE
size_t freeSpace() const {
330 MOZ_ASSERT(isEnabled());
331 MOZ_ASSERT(currentEnd_
- position_
<= NurseryChunkUsableSize
);
332 MOZ_ASSERT(currentChunk_
< maxChunkCount());
333 return (currentEnd_
- position_
) +
334 (maxChunkCount() - currentChunk_
- 1) * gc::ChunkSize
;
338 void enterZealMode();
339 void leaveZealMode();
342 // Write profile time JSON on JSONPrinter.
343 void renderProfileJSON(JSONPrinter
& json
) const;
345 // Print header line for profile times.
346 void printProfileHeader();
348 // Print total profile times on shutdown.
349 void printTotalProfileTimes();
351 void* addressOfPosition() const { return (void**)&position_
; }
352 static constexpr int32_t offsetOfCurrentEndFromPosition() {
353 return offsetof(Nursery
, currentEnd_
) - offsetof(Nursery
, position_
);
356 void* addressOfNurseryAllocatedSites() {
357 return pretenuringNursery
.addressOfAllocatedSites();
360 void requestMinorGC(JS::GCReason reason
);
362 bool minorGCRequested() const {
363 return minorGCTriggerReason_
!= JS::GCReason::NO_REASON
;
365 JS::GCReason
minorGCTriggerReason() const { return minorGCTriggerReason_
; }
367 bool shouldCollect() const;
368 bool isNearlyFull() const;
369 bool isUnderused() const;
371 bool enableProfiling() const { return enableProfiling_
; }
373 bool addMapWithNurseryMemory(MapObject
* obj
) {
374 MOZ_ASSERT_IF(!mapsWithNurseryMemory_
.empty(),
375 mapsWithNurseryMemory_
.back() != obj
);
376 return mapsWithNurseryMemory_
.append(obj
);
378 bool addSetWithNurseryMemory(SetObject
* obj
) {
379 MOZ_ASSERT_IF(!setsWithNurseryMemory_
.empty(),
380 setsWithNurseryMemory_
.back() != obj
);
381 return setsWithNurseryMemory_
.append(obj
);
384 // The amount of space in the mapped nursery available to allocations.
385 static const size_t NurseryChunkUsableSize
=
386 gc::ChunkSize
- sizeof(gc::ChunkBase
);
388 void joinDecommitTask();
390 mozilla::TimeStamp
collectionStartTime() {
391 return startTimes_
[ProfileKey::Total
];
394 bool canCreateAllocSite() { return pretenuringNursery
.canCreateAllocSite(); }
395 void noteAllocSiteCreated() { pretenuringNursery
.noteAllocSiteCreated(); }
396 bool reportPretenuring() const { return reportPretenuring_
; }
397 void maybeStopPretenuring(gc::GCRuntime
* gc
) {
398 pretenuringNursery
.maybeStopPretenuring(gc
);
401 void setAllocFlagsForZone(JS::Zone
* zone
);
403 // Round a size in bytes to the nearest valid nursery size.
404 static size_t roundSize(size_t size
);
406 // The malloc'd block cache.
407 gc::MallocedBlockCache
& mallocedBlockCache() { return mallocedBlockCache_
; }
408 size_t sizeOfMallocedBlockCache(mozilla::MallocSizeOf mallocSizeOf
) const {
409 return mallocedBlockCache_
.sizeOfExcludingThis(mallocSizeOf
);
413 // Fields used during allocation fast path are grouped first:
415 // Pointer to the first unallocated byte in the nursery.
418 // Pointer to the last byte of space in the current chunk.
419 uintptr_t currentEnd_
;
421 // Other fields not necessarily used during allocation follow:
423 gc::GCRuntime
* const gc
;
425 // Vector of allocated chunks to allocate from.
426 Vector
<NurseryChunk
*, 0, SystemAllocPolicy
> chunks_
;
428 // The index of the chunk that is currently being allocated from.
429 uint32_t currentChunk_
;
431 // These fields refer to the beginning of the nursery. They're normally 0
432 // and chunk(0).start() respectively. Except when a generational GC zeal
433 // mode is active, then they may be arbitrary (see Nursery::clear()).
434 uint32_t currentStartChunk_
;
435 uintptr_t currentStartPosition_
;
437 // The current nursery capacity measured in bytes. It may grow up to this
438 // value without a collection, allocating chunks on demand. This limit may be
439 // changed by maybeResizeNursery() each collection. It includes chunk headers.
442 gc::PretenuringNursery pretenuringNursery
;
444 mozilla::TimeDuration timeInChunkAlloc_
;
446 // Report minor collections taking at least this long, if enabled.
447 bool enableProfiling_
= false;
448 bool profileWorkers_
= false;
450 mozilla::TimeDuration profileThreshold_
;
452 // Whether we will nursery-allocate strings.
453 bool canAllocateStrings_
;
455 // Whether we will nursery-allocate BigInts.
456 bool canAllocateBigInts_
;
458 // Report how many strings were deduplicated.
459 bool reportDeduplications_
;
461 // Whether to report information on pretenuring, and if so the allocation
462 // threshold at which to report details of each allocation site.
463 bool reportPretenuring_
;
464 size_t reportPretenuringThreshold_
;
466 // Whether and why a collection of this nursery has been requested. When this
467 // happens |prevPosition_| is set to the current position and |position_| set
468 // to the end of the chunk to force the next allocation to fail.
469 JS::GCReason minorGCTriggerReason_
;
470 uintptr_t prevPosition_
;
474 enum class ProfileKey
{
475 #define DEFINE_TIME_KEY(name, text) name,
476 FOR_EACH_NURSERY_PROFILE_TIME(DEFINE_TIME_KEY
)
477 #undef DEFINE_TIME_KEY
482 mozilla::EnumeratedArray
<ProfileKey
, ProfileKey::KeyCount
,
484 using ProfileDurations
=
485 mozilla::EnumeratedArray
<ProfileKey
, ProfileKey::KeyCount
,
486 mozilla::TimeDuration
>;
488 ProfileTimes startTimes_
;
489 ProfileDurations profileDurations_
;
490 ProfileDurations totalDurations_
;
492 // Data about the previous collection.
494 JS::GCReason reason
= JS::GCReason::NO_REASON
;
495 size_t nurseryCapacity
= 0;
496 size_t nurseryCommitted
= 0;
497 size_t nurseryUsedBytes
= 0;
498 size_t nurseryUsedChunkCount
= 0;
499 size_t tenuredBytes
= 0;
500 size_t tenuredCells
= 0;
501 mozilla::TimeStamp endTime
;
503 PreviousGC previousGC
;
505 bool hasRecentGrowthData
;
506 double smoothedTargetSize
;
508 // Calculate the promotion rate of the most recent minor GC.
509 // The valid_for_tenuring parameter is used to return whether this
510 // promotion rate is accurate enough (the nursery was full enough) to be
511 // used for tenuring and other decisions.
513 // Must only be called if the previousGC data is initialised.
514 double calcPromotionRate(bool* validForTenuring
) const;
516 // The set of externally malloced buffers potentially kept live by objects
517 // stored in the nursery. Any external buffers that do not belong to a
518 // tenured thing at the end of a minor GC must be freed.
519 using BufferRelocationOverlay
= void*;
520 using BufferSet
= HashSet
<void*, PointerHasher
<void*>, SystemAllocPolicy
>;
521 BufferSet mallocedBuffers
;
522 size_t mallocedBufferBytes
= 0;
524 // Wasm "trailer" (C++-heap-allocated) blocks. See comments above on
525 // ::registerTrailer and ::unregisterTrailer.
526 Vector
<PointerAndUint7
, 0, SystemAllocPolicy
> trailersAdded_
;
527 Vector
<void*, 0, SystemAllocPolicy
> trailersRemoved_
;
528 size_t trailersRemovedUsed_
= 0;
529 size_t trailerBytes_
= 0;
531 void freeTrailerBlocks();
533 // During a collection most hoisted slot and element buffers indicate their
534 // new location with a forwarding pointer at the base. This does not work
535 // for buffers whose length is less than pointer width, or when different
536 // buffers might overlap each other. For these, an entry in the following
538 using ForwardedBufferMap
=
539 HashMap
<void*, void*, PointerHasher
<void*>, SystemAllocPolicy
>;
540 ForwardedBufferMap forwardedBuffers
;
542 // When we assign a unique id to cell in the nursery, that almost always
543 // means that the cell will be in a hash table, and thus, held live,
544 // automatically moving the uid from the nursery to its new home in
545 // tenured. It is possible, if rare, for an object that acquired a uid to
546 // be dead before the next collection, in which case we need to know to
547 // remove it when we sweep.
549 // Note: we store the pointers as Cell* here, resulting in an ugly cast in
550 // sweep. This is because this structure is used to help implement
551 // stable object hashing and we have to break the cycle somehow.
552 using CellsWithUniqueIdVector
= Vector
<gc::Cell
*, 8, SystemAllocPolicy
>;
553 CellsWithUniqueIdVector cellsWithUid_
;
555 // Lists of map and set objects allocated in the nursery or with iterators
556 // allocated there. Such objects need to be swept after minor GC.
557 Vector
<MapObject
*, 0, SystemAllocPolicy
> mapsWithNurseryMemory_
;
558 Vector
<SetObject
*, 0, SystemAllocPolicy
> setsWithNurseryMemory_
;
560 UniquePtr
<NurseryDecommitTask
> decommitTask
;
562 // A cache of small C++-heap allocated blocks associated with this Nursery.
563 // This provided so as to provide cheap allocation/deallocation of
564 // out-of-line storage areas as used by WasmStructObject and
565 // WasmArrayObject, although the mechanism is general and not specific to
566 // these object types. Regarding lifetimes, because the cache holds only
567 // blocks that are not currently in use, it can be flushed at any point with
568 // no correctness impact, only a performance impact.
569 gc::MallocedBlockCache mallocedBlockCache_
;
571 NurseryChunk
& chunk(unsigned index
) const { return *chunks_
[index
]; }
573 // Set the current chunk. This updates the currentChunk_, position_ and
574 // currentEnd_ values as appropriate. It'll also poison the chunk, either a
575 // portion of the chunk if it is already the current chunk, or the whole chunk
576 // if fullPoison is true or it is not the current chunk.
577 void setCurrentChunk(unsigned chunkno
);
579 bool initFirstChunk(AutoLockGCBgAlloc
& lock
);
581 // extent is advisory, it will be ignored in sub-chunk and generational zeal
582 // modes. It will be clamped to Min(NurseryChunkUsableSize, capacity_).
583 void poisonAndInitCurrentChunk(size_t extent
= gc::ChunkSize
);
585 void setCurrentEnd();
586 void setStartPosition();
588 // Allocate the next chunk, or the first chunk for initialization.
589 // Callers will probably want to call setCurrentChunk(0) next.
590 [[nodiscard
]] bool allocateNextChunk(unsigned chunkno
,
591 AutoLockGCBgAlloc
& lock
);
593 uintptr_t currentEnd() const { return currentEnd_
; }
595 uintptr_t position() const { return position_
; }
597 MOZ_ALWAYS_INLINE
bool isSubChunkMode() const;
599 JSRuntime
* runtime() const;
600 gcstats::Statistics
& stats() const;
602 const js::gc::GCSchedulingTunables
& tunables() const;
604 void getAllocFlagsForZone(JS::Zone
* zone
, bool* allocObjectsOut
,
605 bool* allocStringsOut
, bool* allocBigIntsOut
);
606 void updateAllZoneAllocFlags();
607 void updateAllocFlagsForZone(JS::Zone
* zone
);
608 void discardCodeAndSetJitFlagsForZone(JS::Zone
* zone
);
610 void* allocate(size_t size
);
612 // Common internal allocator function. If this fails, call
613 // handleAllocationFailure to see whether it's possible to retry.
614 void* tryAllocate(size_t size
) {
615 MOZ_ASSERT(isEnabled());
616 MOZ_ASSERT(!JS::RuntimeHeapIsBusy());
617 MOZ_ASSERT_IF(currentChunk_
== currentStartChunk_
,
618 position() >= currentStartPosition_
);
619 MOZ_ASSERT(size
% gc::CellAlignBytes
== 0);
620 MOZ_ASSERT(position() % gc::CellAlignBytes
== 0);
622 if (MOZ_UNLIKELY(currentEnd() < position() + size
)) {
626 void* ptr
= reinterpret_cast<void*>(position());
628 MOZ_MAKE_COMPILER_ASSUME_IS_UNREACHABLE(
629 "Successful allocation cannot result in nullptr");
632 position_
= position() + size
;
634 DebugOnlyPoison(ptr
, JS_ALLOCATED_NURSERY_PATTERN
, size
,
635 MemCheckKind::MakeUndefined
);
640 [[nodiscard
]] bool moveToNextChunk();
642 struct CollectionResult
{
646 CollectionResult
doCollection(gc::AutoGCSession
& session
,
647 JS::GCOptions options
, JS::GCReason reason
);
648 void traceRoots(gc::AutoGCSession
& session
, gc::TenuringTracer
& mover
);
650 size_t doPretenuring(JSRuntime
* rt
, JS::GCReason reason
,
651 bool validPromotionRate
, double promotionRate
);
653 // Handle relocation of slots/elements pointers stored in Ion frames.
654 inline void setForwardingPointer(void* oldData
, void* newData
, bool direct
);
656 inline void setDirectForwardingPointer(void* oldData
, void* newData
);
657 void setIndirectForwardingPointer(void* oldData
, void* newData
);
659 inline void setSlotsForwardingPointer(HeapSlot
* oldSlots
, HeapSlot
* newSlots
,
661 inline void setElementsForwardingPointer(ObjectElements
* oldHeader
,
662 ObjectElements
* newHeader
,
666 bool checkForwardingPointerLocation(void* ptr
, bool expectedInside
);
669 // Updates pointers to nursery objects that have been tenured and discards
670 // pointers to objects that have been freed.
673 // Reset the current chunk and position after a minor collection. Also poison
674 // the nursery on debug & nightly builds.
677 void sweepMapAndSetObjects();
679 // Change the allocable space provided by the nursery.
680 void maybeResizeNursery(JS::GCOptions options
, JS::GCReason reason
);
681 size_t targetSize(JS::GCOptions options
, JS::GCReason reason
);
682 void clearRecentGrowthData();
683 void growAllocableSpace(size_t newCapacity
);
684 void shrinkAllocableSpace(size_t newCapacity
);
685 void minimizeAllocableSpace();
687 // Free the chunks starting at firstFreeChunk until the end of the chunks
688 // vector. Shrinks the vector but does not update maxChunkCount().
689 void freeChunksFrom(unsigned firstFreeChunk
);
691 void sendTelemetry(JS::GCReason reason
, mozilla::TimeDuration totalTime
,
692 bool wasEmpty
, double promotionRate
,
693 size_t sitesPretenured
);
695 void printCollectionProfile(JS::GCReason reason
, double promotionRate
);
696 void printDeduplicationData(js::StringStats
& prev
, js::StringStats
& curr
);
698 // Profile recording and printing.
699 void maybeClearProfileDurations();
700 void startProfile(ProfileKey key
);
701 void endProfile(ProfileKey key
);
702 static bool printProfileDurations(const ProfileDurations
& times
,
705 mozilla::TimeStamp
collectionStartTime() const;
706 mozilla::TimeStamp
lastCollectionEndTime() const;
708 friend class gc::GCRuntime
;
709 friend class gc::TenuringTracer
;
710 friend struct NurseryChunk
;
715 #endif // gc_Nursery_h