From 788031e3e29a0a8ea82a6ea999671c095b157a21 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Wed, 7 Jul 2021 08:24:36 -0700 Subject: [PATCH] free slabs during shutdown Summary: MemoryManager may grab slabs from SlabManager, which often manages a list of slabs from hugetlb pages. When the server is about to shutdown (but is still finishing ongoing requests), we want to free the hugetlb pages faster. Reviewed By: ottoni Differential Revision: D29352348 fbshipit-source-id: 0322d4692da7207571fa7e302fea77d1ad1f94b6 --- hphp/runtime/base/memory-manager.h | 12 +++++++++++- hphp/runtime/base/sparse-heap.cpp | 14 +++++++++++++- hphp/runtime/server/http-server.cpp | 8 ++++++++ hphp/util/alloc.cpp | 7 +++++++ hphp/util/alloc.h | 1 + hphp/util/slab-manager.h | 30 +++++++++++++++++++++++++++++- 6 files changed, 69 insertions(+), 3 deletions(-) diff --git a/hphp/runtime/base/memory-manager.h b/hphp/runtime/base/memory-manager.h index 8f8a9bb1753..df798001eaa 100644 --- a/hphp/runtime/base/memory-manager.h +++ b/hphp/runtime/base/memory-manager.h @@ -539,6 +539,14 @@ struct SparseHeap { */ MemBlock slab_range() const { return m_slab_range; } + /* + * When the process is going to shutdown soon, try to free the slabs directly + * instead of handling them to slab manager. + */ + static void PrepareToStop(bool val = true) { + s_shutdown.exchange(val, std::memory_order_release); + } + protected: struct SlabInfo { SlabInfo(void* p, uint16_t v) : ptr(p), version(v) {} @@ -550,6 +558,9 @@ struct SparseHeap { MemBlock m_slab_range; int64_t m_hugeBytes{0}; // compare with RequestHugeMaxBytes SlabManager* m_slabManager{nullptr}; + // When the process is shutting down, we try to unmap the slabs directly at + // the end of the request. + static std::atomic_bool s_shutdown; }; using HeapImpl = SparseHeap; @@ -1127,4 +1138,3 @@ void reset_alloc_sampling(); } #include "hphp/runtime/base/memory-manager-inl.h" - diff --git a/hphp/runtime/base/sparse-heap.cpp b/hphp/runtime/base/sparse-heap.cpp index 4e8889e7a16..d184879da4a 100644 --- a/hphp/runtime/base/sparse-heap.cpp +++ b/hphp/runtime/base/sparse-heap.cpp @@ -19,11 +19,14 @@ #include "hphp/util/alloc.h" #include "hphp/util/safe-cast.h" #include "hphp/util/trace.h" +#include namespace HPHP { TRACE_SET_MOD(mm); +std::atomic_bool SparseHeap::s_shutdown = false; + void SparseHeap::threadInit() { #ifdef USE_JEMALLOC m_slabManager = get_local_slab_manager(s_numaNode); @@ -59,10 +62,19 @@ void SparseHeap::reset() { }; TaggedSlabList pooledSlabs; void* pooledSlabTail = nullptr; + auto const UNUSED isShuttingDown = s_shutdown.load(std::memory_order_acquire); for (auto& slab : m_pooled_slabs) { + m_bigs.erase(slab.ptr); +#ifdef __linux__ + if (isShuttingDown) { + // Free the slab by remapping to overwrite it. This may still fail (e.g., + // when the slab comes from weird things such as a 1G page and the kernel + // doesn't handle it properly); so check the result. + if (SlabManager::UnmapSlab(slab.ptr)) continue; + } +#endif if (!pooledSlabTail) pooledSlabTail = slab.ptr; pooledSlabs.push_front(slab.ptr, slab.version); - m_bigs.erase(slab.ptr); } if (pooledSlabTail) { m_slabManager->merge(std::move(pooledSlabs), pooledSlabTail); diff --git a/hphp/runtime/server/http-server.cpp b/hphp/runtime/server/http-server.cpp index 5adf2bd8ae2..53ee00ddeaa 100644 --- a/hphp/runtime/server/http-server.cpp +++ b/hphp/runtime/server/http-server.cpp @@ -229,6 +229,10 @@ void HttpServer::onServerShutdown() { static std::atomic_flag flag = ATOMIC_FLAG_INIT; if (flag.test_and_set()) return; + SparseHeap::PrepareToStop(); +#ifdef USE_JEMALLOC + shutdown_slab_managers(); +#endif InitFiniNode::ServerFini(); Eval::Debugger::Stop(); @@ -602,6 +606,10 @@ void HttpServer::PrepareToStop() { MarkShutdownStat(ShutdownEvent::SHUTDOWN_PREPARE); PrepareToStopTime.store(time(nullptr), std::memory_order_release); EvictFileCache(); + SparseHeap::PrepareToStop(); +#ifdef USE_JEMALLOC + shutdown_slab_managers(); +#endif } void HttpServer::createPid() { diff --git a/hphp/util/alloc.cpp b/hphp/util/alloc.cpp index e4a66339547..11211c351b5 100644 --- a/hphp/util/alloc.cpp +++ b/hphp/util/alloc.cpp @@ -559,6 +559,7 @@ void arenas_thread_exit() { #endif // USE_JEMALLOC_EXTENT_HOOKS std::vector s_slab_managers; + void setup_local_arenas(PageSpec spec, unsigned slabs) { s_slab_managers.reserve(num_numa_nodes()); slabs /= num_numa_nodes(); @@ -646,6 +647,12 @@ SlabManager* get_local_slab_manager(uint32_t node) { return s_slab_managers[node]; } +void shutdown_slab_managers() { + for (auto slab_manager : s_slab_managers) { + if (slab_manager) slab_manager->shutdown(); + } +} + #endif // USE_JEMALLOC ssize_t get_free_slab_bytes() { diff --git a/hphp/util/alloc.h b/hphp/util/alloc.h index b85cd1ea28f..b45630c9f7f 100644 --- a/hphp/util/alloc.h +++ b/hphp/util/alloc.h @@ -119,6 +119,7 @@ struct PageSpec { void setup_local_arenas(PageSpec, unsigned slabs); unsigned get_local_arena(uint32_t node); SlabManager* get_local_slab_manager(uint32_t node); +void shutdown_slab_managers(); void setup_arena0(PageSpec); diff --git a/hphp/util/slab-manager.h b/hphp/util/slab-manager.h index 551f356f165..e105e6d7cb1 100644 --- a/hphp/util/slab-manager.h +++ b/hphp/util/slab-manager.h @@ -22,6 +22,8 @@ #include #include +#include + namespace HPHP { constexpr unsigned kLgSlabSize = 21; @@ -189,7 +191,33 @@ struct SlabManager : TaggedSlabList { } } // otherwise currHead is updated with latest value of m_head. } + + // Try to unmap the slabs when the server is about to shut down. + void shutdown() { + // List of slabs that are still in memory after efforts to unmap them. + TaggedSlabList failed; + void* failedTail = nullptr; + while (auto p = tryAlloc()) { + auto ptr = p.ptr(); + auto tag = p.tag(); + if (!UnmapSlab(ptr)) { + failed.push_front(ptr, tag); + if (!failedTail) failedTail = ptr; + } + } + if (failedTail) merge(std::move(failed), failedTail); + } + + // Return whether unmapping was successful. + static bool UnmapSlab(void* ptr) { + // The Linux man page for munmap mentions "All pages containing a part of + // the indicated range are unmapped", which could mean that the entire 1GB + // page could get unmapped when we only intend to unmap a single slab in it. + // Thus, we try to create a new mapping (not in RSS) to replace this slab. + return mmap(ptr, kSlabSize, PROT_NONE, + MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, + -1 , 0) == ptr; + } }; } - -- 2.11.4.GIT