js/src/gc/Nursery-inl.h

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=4 sw=2 et tw=80 ft=cpp:
   3  *
   4  * This Source Code Form is subject to the terms of the Mozilla Public
   5  * License, v. 2.0. If a copy of the MPL was not distributed with this
   6  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   7
   8 #ifndef gc_Nursery_inl_h
   9 #define gc_Nursery_inl_h
  10
  11 #include "gc/Nursery.h"
  12
  13 #include "gc/GCRuntime.h"
  14 #include "gc/RelocationOverlay.h"
  15 #include "js/TracingAPI.h"
  16 #include "vm/JSContext.h"
  17 #include "vm/NativeObject.h"
  18
  19 namespace js {
  20 namespace gc {
  21 struct Cell;
  22 }  // namespace gc
  23 }  // namespace js
  24
  25 inline JSRuntime* js::Nursery::runtime() const { return gc->rt; }
  26
  27 template <typename T>
  28 bool js::Nursery::isInside(const SharedMem<T>& p) const {
  29   return isInside(p.unwrap(/*safe - used for value in comparison above*/));
  30 }
  31
  32 inline bool js::Nursery::shouldTenure(gc::Cell* cell) {
  33   MOZ_ASSERT(semispaceEnabled());
  34   MOZ_ASSERT(inCollectedRegion(cell));
  35
  36   size_t offset = fromSpace.offsetFromAddress(uintptr_t(cell));
  37   MOZ_ASSERT(offset >=
  38              fromSpace.offsetFromExclusiveAddress(fromSpace.startPosition_));
  39   return offset <= tenureThreshold_;
  40 }
  41
  42 inline bool js::Nursery::inCollectedRegion(gc::Cell* cell) const {
  43   gc::ChunkBase* chunk = gc::detail::GetCellChunkBase(cell);
  44   return chunk->getKind() == gc::ChunkKind::NurseryFromSpace;
  45 }
  46
  47 inline bool js::Nursery::inCollectedRegion(void* ptr) const {
  48   if (!semispaceEnabled()) {
  49     return toSpace.isInside(ptr);
  50   }
  51
  52   return fromSpace.isInside(ptr);
  53 }
  54
  55 inline size_t js::Nursery::Space::offsetFromExclusiveAddress(
  56     uintptr_t addr) const {
  57   if ((addr & gc::ChunkMask) == 0) {
  58     // |addr| points one past the end of the previous chunk.
  59     return offsetFromAddress(addr - 1) + 1;
  60   }
  61
  62   return offsetFromAddress(addr);
  63 }
  64
  65 inline size_t js::Nursery::Space::offsetFromAddress(uintptr_t addr) const {
  66   gc::ChunkBase* chunk =
  67       gc::detail::GetCellChunkBase(reinterpret_cast<gc::Cell*>(addr));
  68   MOZ_ASSERT(chunk->getKind() == kind);
  69   MOZ_ASSERT(findChunkIndex(addr & ~gc::ChunkMask) == chunk->nurseryChunkIndex);
  70
  71   uint32_t offset = addr & gc::ChunkMask;
  72   MOZ_ASSERT(offset >= sizeof(gc::ChunkBase));
  73   return (chunk->nurseryChunkIndex << gc::ChunkShift) | offset;
  74 }
  75
  76 MOZ_ALWAYS_INLINE /* static */ bool js::Nursery::getForwardedPointer(
  77     js::gc::Cell** ref) {
  78   js::gc::Cell* cell = (*ref);
  79   MOZ_ASSERT(IsInsideNursery(cell));
  80   if (!cell->isForwarded()) {
  81     return false;
  82   }
  83   const gc::RelocationOverlay* overlay = gc::RelocationOverlay::fromCell(cell);
  84   *ref = overlay->forwardingAddress();
  85   return true;
  86 }
  87
  88 inline void js::Nursery::maybeSetForwardingPointer(JSTracer* trc, void* oldData,
  89                                                    void* newData, bool direct) {
  90   if (trc->isTenuringTracer()) {
  91     setForwardingPointerWhileTenuring(oldData, newData, direct);
  92   }
  93 }
  94
  95 inline void js::Nursery::setForwardingPointerWhileTenuring(void* oldData,
  96                                                            void* newData,
  97                                                            bool direct) {
  98   if (isInside(oldData)) {
  99     setForwardingPointer(oldData, newData, direct);
 100   }
 101 }
 102
 103 inline void js::Nursery::setSlotsForwardingPointer(HeapSlot* oldSlots,
 104                                                    HeapSlot* newSlots,
 105                                                    uint32_t nslots) {
 106   // Slot arrays always have enough space for a forwarding pointer, since the
 107   // number of slots is never zero.
 108   MOZ_ASSERT(nslots > 0);
 109   setDirectForwardingPointer(oldSlots, newSlots);
 110 }
 111
 112 inline void js::Nursery::setElementsForwardingPointer(ObjectElements* oldHeader,
 113                                                       ObjectElements* newHeader,
 114                                                       uint32_t capacity) {
 115   // Only use a direct forwarding pointer if there is enough space for one.
 116   setForwardingPointer(oldHeader->elements(), newHeader->elements(),
 117                        capacity > 0);
 118 }
 119
 120 inline void js::Nursery::setForwardingPointer(void* oldData, void* newData,
 121                                               bool direct) {
 122   if (direct) {
 123     setDirectForwardingPointer(oldData, newData);
 124     return;
 125   }
 126
 127   setIndirectForwardingPointer(oldData, newData);
 128 }
 129
 130 inline void js::Nursery::setDirectForwardingPointer(void* oldData,
 131                                                     void* newData) {
 132   MOZ_ASSERT(isInside(oldData));
 133   MOZ_ASSERT_IF(isInside(newData), !inCollectedRegion(newData));
 134
 135   new (oldData) BufferRelocationOverlay{newData};
 136 }
 137
 138 inline void* js::Nursery::tryAllocateCell(gc::AllocSite* site, size_t size,
 139                                           JS::TraceKind kind) {
 140   // Ensure there's enough space to replace the contents with a
 141   // RelocationOverlay.
 142   // MOZ_ASSERT(size >= sizeof(RelocationOverlay));
 143   MOZ_ASSERT(size % gc::CellAlignBytes == 0);
 144   MOZ_ASSERT(size_t(kind) < gc::NurseryTraceKinds);
 145   MOZ_ASSERT_IF(kind == JS::TraceKind::String, canAllocateStrings());
 146   MOZ_ASSERT_IF(kind == JS::TraceKind::BigInt, canAllocateBigInts());
 147
 148   void* ptr = tryAllocate(sizeof(gc::NurseryCellHeader) + size);
 149   if (MOZ_UNLIKELY(!ptr)) {
 150     return nullptr;
 151   }
 152
 153   new (ptr) gc::NurseryCellHeader(site, kind);
 154
 155   void* cell =
 156       reinterpret_cast<void*>(uintptr_t(ptr) + sizeof(gc::NurseryCellHeader));
 157   if (!cell) {
 158     MOZ_MAKE_COMPILER_ASSUME_IS_UNREACHABLE(
 159         "Successful allocation cannot result in nullptr");
 160   }
 161
 162   // Update the allocation site. This code is also inlined in
 163   // MacroAssembler::updateAllocSite.
 164   uint32_t allocCount = site->incAllocCount();
 165   if (allocCount == 1) {
 166     pretenuringNursery.insertIntoAllocatedList(site);
 167   }
 168   MOZ_ASSERT_IF(site->isNormal(), site->isInAllocatedList());
 169
 170   gc::gcprobes::NurseryAlloc(cell, kind);
 171   return cell;
 172 }
 173
 174 inline void* js::Nursery::tryAllocate(size_t size) {
 175   MOZ_ASSERT(isEnabled());
 176   MOZ_ASSERT_IF(JS::RuntimeHeapIsBusy(), JS::RuntimeHeapIsMinorCollecting());
 177   MOZ_ASSERT_IF(currentChunk() == startChunk(), position() >= startPosition());
 178   MOZ_ASSERT(size % gc::CellAlignBytes == 0);
 179   MOZ_ASSERT(position() % gc::CellAlignBytes == 0);
 180
 181   if (MOZ_UNLIKELY(currentEnd() < position() + size)) {
 182     return nullptr;
 183   }
 184
 185   void* ptr = reinterpret_cast<void*>(position());
 186   if (!ptr) {
 187     MOZ_MAKE_COMPILER_ASSUME_IS_UNREACHABLE(
 188         "Successful allocation cannot result in nullptr");
 189   }
 190
 191   toSpace.position_ = position() + size;
 192
 193   DebugOnlyPoison(ptr, JS_ALLOCATED_NURSERY_PATTERN, size,
 194                   MemCheckKind::MakeUndefined);
 195
 196   return ptr;
 197 }
 198
 199 inline bool js::Nursery::registerTrailer(PointerAndUint7 blockAndListID,
 200                                          size_t nBytes) {
 201   MOZ_ASSERT(toSpace.trailersAdded_.length() ==
 202              toSpace.trailersRemoved_.length());
 203   MOZ_ASSERT(nBytes > 0);
 204   if (MOZ_UNLIKELY(!toSpace.trailersAdded_.append(blockAndListID))) {
 205     return false;
 206   }
 207   if (MOZ_UNLIKELY(!toSpace.trailersRemoved_.append(nullptr))) {
 208     toSpace.trailersAdded_.popBack();
 209     return false;
 210   }
 211
 212   // This is a clone of the logic in ::registerMallocedBuffer.  It may be
 213   // that some other heuristic is better, once we know more about the
 214   // typical behaviour of wasm-GC applications.
 215   toSpace.trailerBytes_ += nBytes;
 216   if (MOZ_UNLIKELY(toSpace.trailerBytes_ > capacity() * 8)) {
 217     requestMinorGC(JS::GCReason::NURSERY_TRAILERS);
 218   }
 219   return true;
 220 }
 221
 222 inline void js::Nursery::unregisterTrailer(void* block) {
 223   // Unlike removeMallocedBuffer this is only called during minor GC.
 224   MOZ_ASSERT(fromSpace.trailersRemovedUsed_ <
 225              fromSpace.trailersRemoved_.length());
 226   fromSpace.trailersRemoved_[fromSpace.trailersRemovedUsed_] = block;
 227   fromSpace.trailersRemovedUsed_++;
 228 }
 229
 230 namespace js {
 231
 232 // The allocation methods below will not run the garbage collector. If the
 233 // nursery cannot accomodate the allocation, the malloc heap will be used
 234 // instead.
 235
 236 template <typename T>
 237 static inline T* AllocateCellBuffer(Nursery& nursery, gc::Cell* cell,
 238                                     uint32_t count) {
 239   size_t nbytes = RoundUp(count * sizeof(T), sizeof(Value));
 240   return static_cast<T*>(
 241       nursery.allocateBuffer(cell->zone(), cell, nbytes, js::MallocArena));
 242 }
 243
 244 template <typename T>
 245 static inline T* AllocateCellBuffer(JSContext* cx, gc::Cell* cell,
 246                                     uint32_t count) {
 247   T* buffer = AllocateCellBuffer<T>(cx->nursery(), cell, count);
 248   if (!buffer) {
 249     ReportOutOfMemory(cx);
 250     return nullptr;
 251   }
 252
 253   return buffer;
 254 }
 255
 256 // If this returns null then the old buffer will be left alone.
 257 template <typename T>
 258 static inline T* ReallocateCellBuffer(JSContext* cx, gc::Cell* cell,
 259                                       T* oldBuffer, uint32_t oldCount,
 260                                       uint32_t newCount, arena_id_t arenaId) {
 261   size_t oldBytes = RoundUp(oldCount * sizeof(T), sizeof(Value));
 262   size_t newBytes = RoundUp(newCount * sizeof(T), sizeof(Value));
 263
 264   T* buffer = static_cast<T*>(cx->nursery().reallocateBuffer(
 265       cell->zone(), cell, oldBuffer, oldBytes, newBytes, arenaId));
 266   if (!buffer) {
 267     ReportOutOfMemory(cx);
 268   }
 269
 270   return buffer;
 271 }
 272
 273 }  // namespace js
 274
 275 #endif /* gc_Nursery_inl_h */