js/src/gc/Nursery-inl.h

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=4 sw=2 et tw=80 ft=cpp:
   3  *
   4  * This Source Code Form is subject to the terms of the Mozilla Public
   5  * License, v. 2.0. If a copy of the MPL was not distributed with this
   6  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   7
   8 #ifndef gc_Nursery_inl_h
   9 #define gc_Nursery_inl_h
  10
  11 #include "gc/Nursery.h"
  12
  13 #include "gc/GCRuntime.h"
  14 #include "gc/RelocationOverlay.h"
  15 #include "js/TracingAPI.h"
  16 #include "vm/JSContext.h"
  17 #include "vm/NativeObject.h"
  18
  19 inline JSRuntime* js::Nursery::runtime() const { return gc->rt; }
  20
  21 template <typename T>
  22 bool js::Nursery::isInside(const SharedMem<T>& p) const {
  23   return isInside(p.unwrap(/*safe - used for value in comparison above*/));
  24 }
  25
  26 MOZ_ALWAYS_INLINE /* static */ bool js::Nursery::getForwardedPointer(
  27     js::gc::Cell** ref) {
  28   js::gc::Cell* cell = (*ref);
  29   MOZ_ASSERT(IsInsideNursery(cell));
  30   if (!cell->isForwarded()) {
  31     return false;
  32   }
  33   const gc::RelocationOverlay* overlay = gc::RelocationOverlay::fromCell(cell);
  34   *ref = overlay->forwardingAddress();
  35   return true;
  36 }
  37
  38 inline void js::Nursery::maybeSetForwardingPointer(JSTracer* trc, void* oldData,
  39                                                    void* newData, bool direct) {
  40   if (trc->isTenuringTracer()) {
  41     setForwardingPointerWhileTenuring(oldData, newData, direct);
  42   }
  43 }
  44
  45 inline void js::Nursery::setForwardingPointerWhileTenuring(void* oldData,
  46                                                            void* newData,
  47                                                            bool direct) {
  48   if (isInside(oldData)) {
  49     setForwardingPointer(oldData, newData, direct);
  50   }
  51 }
  52
  53 inline void js::Nursery::setSlotsForwardingPointer(HeapSlot* oldSlots,
  54                                                    HeapSlot* newSlots,
  55                                                    uint32_t nslots) {
  56   // Slot arrays always have enough space for a forwarding pointer, since the
  57   // number of slots is never zero.
  58   MOZ_ASSERT(nslots > 0);
  59   setDirectForwardingPointer(oldSlots, newSlots);
  60 }
  61
  62 inline void js::Nursery::setElementsForwardingPointer(ObjectElements* oldHeader,
  63                                                       ObjectElements* newHeader,
  64                                                       uint32_t capacity) {
  65   // Only use a direct forwarding pointer if there is enough space for one.
  66   setForwardingPointer(oldHeader->elements(), newHeader->elements(),
  67                        capacity > 0);
  68 }
  69
  70 inline void js::Nursery::setForwardingPointer(void* oldData, void* newData,
  71                                               bool direct) {
  72   if (direct) {
  73     setDirectForwardingPointer(oldData, newData);
  74     return;
  75   }
  76
  77   setIndirectForwardingPointer(oldData, newData);
  78 }
  79
  80 inline void js::Nursery::setDirectForwardingPointer(void* oldData,
  81                                                     void* newData) {
  82   MOZ_ASSERT(isInside(oldData));
  83   MOZ_ASSERT(!isInside(newData));
  84
  85   new (oldData) BufferRelocationOverlay{newData};
  86 }
  87
  88 inline void* js::Nursery::tryAllocateCell(gc::AllocSite* site, size_t size,
  89                                           JS::TraceKind kind) {
  90   // Ensure there's enough space to replace the contents with a
  91   // RelocationOverlay.
  92   // MOZ_ASSERT(size >= sizeof(RelocationOverlay));
  93   MOZ_ASSERT(size % gc::CellAlignBytes == 0);
  94   MOZ_ASSERT(size_t(kind) < gc::NurseryTraceKinds);
  95   MOZ_ASSERT_IF(kind == JS::TraceKind::String, canAllocateStrings());
  96   MOZ_ASSERT_IF(kind == JS::TraceKind::BigInt, canAllocateBigInts());
  97
  98   void* ptr = tryAllocate(sizeof(gc::NurseryCellHeader) + size);
  99   if (MOZ_UNLIKELY(!ptr)) {
 100     return nullptr;
 101   }
 102
 103   new (ptr) gc::NurseryCellHeader(site, kind);
 104
 105   void* cell =
 106       reinterpret_cast<void*>(uintptr_t(ptr) + sizeof(gc::NurseryCellHeader));
 107   if (!cell) {
 108     MOZ_MAKE_COMPILER_ASSUME_IS_UNREACHABLE(
 109         "Successful allocation cannot result in nullptr");
 110   }
 111
 112   // Update the allocation site. This code is also inlined in
 113   // MacroAssembler::updateAllocSite.
 114   uint32_t allocCount = site->incAllocCount();
 115   if (allocCount == 1) {
 116     pretenuringNursery.insertIntoAllocatedList(site);
 117   }
 118   MOZ_ASSERT_IF(site->isNormal(), site->isInAllocatedList());
 119
 120   gc::gcprobes::NurseryAlloc(cell, kind);
 121   return cell;
 122 }
 123
 124 inline void* js::Nursery::tryAllocate(size_t size) {
 125   MOZ_ASSERT(isEnabled());
 126   MOZ_ASSERT(!JS::RuntimeHeapIsBusy());
 127   MOZ_ASSERT_IF(currentChunk_ == startChunk_, position() >= startPosition_);
 128   MOZ_ASSERT(size % gc::CellAlignBytes == 0);
 129   MOZ_ASSERT(position() % gc::CellAlignBytes == 0);
 130
 131   if (MOZ_UNLIKELY(currentEnd() < position() + size)) {
 132     return nullptr;
 133   }
 134
 135   void* ptr = reinterpret_cast<void*>(position());
 136   if (!ptr) {
 137     MOZ_MAKE_COMPILER_ASSUME_IS_UNREACHABLE(
 138         "Successful allocation cannot result in nullptr");
 139   }
 140
 141   position_ = position() + size;
 142
 143   DebugOnlyPoison(ptr, JS_ALLOCATED_NURSERY_PATTERN, size,
 144                   MemCheckKind::MakeUndefined);
 145
 146   return ptr;
 147 }
 148
 149 inline bool js::Nursery::registerTrailer(PointerAndUint7 blockAndListID,
 150                                          size_t nBytes) {
 151   MOZ_ASSERT(trailersAdded_.length() == trailersRemoved_.length());
 152   MOZ_ASSERT(nBytes > 0);
 153   if (MOZ_UNLIKELY(!trailersAdded_.append(blockAndListID))) {
 154     return false;
 155   }
 156   if (MOZ_UNLIKELY(!trailersRemoved_.append(nullptr))) {
 157     trailersAdded_.popBack();
 158     return false;
 159   }
 160
 161   // This is a clone of the logic in ::registerMallocedBuffer.  It may be
 162   // that some other heuristic is better, once we know more about the
 163   // typical behaviour of wasm-GC applications.
 164   trailerBytes_ += nBytes;
 165   if (MOZ_UNLIKELY(trailerBytes_ > capacity() * 8)) {
 166     requestMinorGC(JS::GCReason::NURSERY_TRAILERS);
 167   }
 168   return true;
 169 }
 170
 171 inline void js::Nursery::unregisterTrailer(void* block) {
 172   MOZ_ASSERT(trailersRemovedUsed_ < trailersRemoved_.length());
 173   trailersRemoved_[trailersRemovedUsed_] = block;
 174   trailersRemovedUsed_++;
 175 }
 176
 177 namespace js {
 178
 179 // The allocation methods below will not run the garbage collector. If the
 180 // nursery cannot accomodate the allocation, the malloc heap will be used
 181 // instead.
 182
 183 template <typename T>
 184 static inline T* AllocateCellBuffer(JSContext* cx, gc::Cell* cell,
 185                                     uint32_t count) {
 186   size_t nbytes = RoundUp(count * sizeof(T), sizeof(Value));
 187   auto* buffer =
 188       static_cast<T*>(cx->nursery().allocateBuffer(cell->zone(), cell, nbytes));
 189   if (!buffer) {
 190     ReportOutOfMemory(cx);
 191   }
 192
 193   return buffer;
 194 }
 195
 196 // If this returns null then the old buffer will be left alone.
 197 template <typename T>
 198 static inline T* ReallocateCellBuffer(JSContext* cx, gc::Cell* cell,
 199                                       T* oldBuffer, uint32_t oldCount,
 200                                       uint32_t newCount) {
 201   size_t oldBytes = RoundUp(oldCount * sizeof(T), sizeof(Value));
 202   size_t newBytes = RoundUp(newCount * sizeof(T), sizeof(Value));
 203
 204   T* buffer = static_cast<T*>(cx->nursery().reallocateBuffer(
 205       cell->zone(), cell, oldBuffer, oldBytes, newBytes));
 206   if (!buffer) {
 207     ReportOutOfMemory(cx);
 208   }
 209
 210   return buffer;
 211 }
 212
 213 }  // namespace js
 214
 215 #endif /* gc_Nursery_inl_h */