hphp/util/numa.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16 #include "hphp/util/numa.h"
  17
  18 #ifdef HAVE_NUMA
  19 #include "hphp/util/portability.h"
  20 #include <folly/Bits.h>
  21 extern "C" {
  22 HHVM_ATTRIBUTE_WEAK extern void numa_init();
  23 }
  24 #endif
  25
  26 namespace HPHP {
  27
  28 // Treat the system as having a single NUMA node if HAVE_NUMA not defined.
  29 // Otherwise, initNuma() will calculate the number of allowed NUMA nodes.
  30 uint32_t numa_num_nodes = 1;
  31 uint32_t numa_node_set = 1;
  32
  33 #ifdef HAVE_NUMA
  34
  35 uint32_t numa_node_mask;
  36 bool use_numa = false;
  37 std::vector<bitmask*> node_to_cpu_mask;
  38
  39 void init_numa() {
  40   if (getenv("HHVM_DISABLE_NUMA")) return;
  41
  42   // When linked dynamically numa_init() is called before JEMallocInitializer()
  43   // numa_init is not exported by libnuma.so so it will be NULL
  44   // however when linked statically numa_init() is not guaranteed to be called
  45   // before JEMallocInitializer(), so call it here.
  46   if (&numa_init) {
  47     numa_init();
  48   }
  49   if (numa_available() < 0) return;
  50
  51   // set interleave for early code. we'll then force interleave for a few
  52   // regions, and switch to local for the threads
  53   numa_set_interleave_mask(numa_all_nodes_ptr);
  54
  55   int max_node = numa_max_node();
  56   if (max_node < 1 || max_node >= 32) return;
  57
  58   bool ret = true;
  59   bitmask* run_nodes = numa_get_run_node_mask();
  60   bitmask* mem_nodes = numa_get_mems_allowed();
  61   numa_num_nodes = 0;
  62   numa_node_set = 0;
  63   for (int i = 0; i <= max_node; i++) {
  64     if (!numa_bitmask_isbitset(run_nodes, i) ||
  65         !numa_bitmask_isbitset(mem_nodes, i)) {
  66       // Only deal with the case of a contiguous set of nodes where we can
  67       // run/allocate memory on each node.
  68       ret = false;
  69       break;
  70     }
  71     numa_node_set |= 1u << i;
  72     numa_num_nodes++;
  73   }
  74   numa_bitmask_free(run_nodes);
  75   numa_bitmask_free(mem_nodes);
  76
  77   if (!ret || numa_num_nodes <= 1) return;
  78
  79   numa_node_mask = folly::nextPowTwo(uint32_t(max_node) + 1) - 1;
  80 }
  81
  82 void enable_numa() {
  83   /*
  84    * Check if NUMA shouldn't be used, for reasons including: (1) only one NUMA
  85    * node allowed; (2) NUMA disabled via environmental variable; (3) unsupported
  86    * cases, e.g., allowed nodes non-contiguous, more than 32 nodes; (4) errors
  87    * calling NUMA API in init_numa().
  88    */
  89   if (numa_num_nodes <= 1 || numa_node_mask == 0) return;
  90
  91   /*
  92    * libnuma is only partially aware of taskset. If on entry, you have
  93    * completely disabled a node via taskset, the node will not be available, and
  94    * calling numa_run_on_node will not work for that node. But if only some of
  95    * the cpu's on a node were disabled, then calling numa_run_on_node will
  96    * enable them all. To prevent this, compute the actual masks up front.
  97    */
  98   bitmask* enabled = numa_allocate_cpumask();
  99   if (numa_sched_getaffinity(0, enabled) < 0) {
 100     return;
 101   }
 102
 103   int num_cpus = numa_num_configured_cpus();
 104   int max_node = numa_max_node();
 105   assert(max_node >= 1 && max_node < 32);
 106   for (int i = 0; i <= max_node; i++) {
 107     bitmask* cpus_for_node = numa_allocate_cpumask();
 108     numa_node_to_cpus(i, cpus_for_node);
 109     for (int j = 0; j < num_cpus; j++) {
 110       if (!numa_bitmask_isbitset(enabled, j)) {
 111         numa_bitmask_clearbit(cpus_for_node, j);
 112       }
 113     }
 114     assert(node_to_cpu_mask.size() == i);
 115     node_to_cpu_mask.push_back(cpus_for_node);
 116   }
 117   numa_bitmask_free(enabled);
 118
 119   use_numa = true;
 120 }
 121
 122 uint32_t next_numa_node(std::atomic<uint32_t>& curr_node) {
 123   if (!use_numa) return 0;
 124   uint32_t node;
 125   do {
 126     node = curr_node.fetch_add(1u, std::memory_order_relaxed);
 127     node &= numa_node_mask;
 128   } while (!((numa_node_set >> node) & 1));
 129   return node;
 130 }
 131
 132 void numa_interleave(void* start, size_t size) {
 133   if (!use_numa) return;
 134   numa_interleave_memory(start, size, numa_all_nodes_ptr);
 135 }
 136
 137 void numa_bind_to(void* start, size_t size, uint32_t node) {
 138   if (!use_numa) return;
 139   numa_tonode_memory(start, size, (int)node);
 140 }
 141
 142 #endif
 143 }