arch/x86/mm/srat_64.c

   1 /*
   2  * ACPI 3.0 based NUMA setup
   3  * Copyright 2004 Andi Kleen, SuSE Labs.
   4  *
   5  * Reads the ACPI SRAT table to figure out what memory belongs to which CPUs.
   6  *
   7  * Called from acpi_numa_init while reading the SRAT and SLIT tables.
   8  * Assumes all memory regions belonging to a single proximity domain
   9  * are in one chunk. Holes between them will be included in the node.
  10  */
  11
  12 #include <linux/kernel.h>
  13 #include <linux/acpi.h>
  14 #include <linux/mmzone.h>
  15 #include <linux/bitmap.h>
  16 #include <linux/module.h>
  17 #include <linux/topology.h>
  18 #include <linux/bootmem.h>
  19 #include <linux/memblock.h>
  20 #include <linux/mm.h>
  21 #include <asm/proto.h>
  22 #include <asm/numa.h>
  23 #include <asm/e820.h>
  24 #include <asm/apic.h>
  25 #include <asm/uv/uv.h>
  26
  27 int acpi_numa __initdata;
  28
  29 static struct acpi_table_slit *acpi_slit;
  30
  31 static struct bootnode nodes_add[MAX_NUMNODES];
  32
  33 static __init int setup_node(int pxm)
  34 {
  35         return acpi_map_pxm_to_node(pxm);
  36 }
  37
  38 static __init void bad_srat(void)
  39 {
  40         printk(KERN_ERR "SRAT: SRAT not used.\n");
  41         acpi_numa = -1;
  42         memset(nodes_add, 0, sizeof(nodes_add));
  43 }
  44
  45 static __init inline int srat_disabled(void)
  46 {
  47         return acpi_numa < 0;
  48 }
  49
  50 /* Callback for SLIT parsing */
  51 void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
  52 {
  53         unsigned length;
  54         unsigned long phys;
  55
  56         length = slit->header.length;
  57         phys = memblock_find_in_range(0, max_pfn_mapped<<PAGE_SHIFT, length,
  58                  PAGE_SIZE);
  59
  60         if (phys == MEMBLOCK_ERROR)
  61                 panic(" Can not save slit!\n");
  62
  63         acpi_slit = __va(phys);
  64         memcpy(acpi_slit, slit, length);
  65         memblock_x86_reserve_range(phys, phys + length, "ACPI SLIT");
  66 }
  67
  68 /* Callback for Proximity Domain -> x2APIC mapping */
  69 void __init
  70 acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
  71 {
  72         int pxm, node;
  73         int apic_id;
  74
  75         if (srat_disabled())
  76                 return;
  77         if (pa->header.length < sizeof(struct acpi_srat_x2apic_cpu_affinity)) {
  78                 bad_srat();
  79                 return;
  80         }
  81         if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
  82                 return;
  83         pxm = pa->proximity_domain;
  84         node = setup_node(pxm);
  85         if (node < 0) {
  86                 printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
  87                 bad_srat();
  88                 return;
  89         }
  90
  91         apic_id = pa->apic_id;
  92         if (apic_id >= MAX_LOCAL_APIC) {
  93                 printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u skipped apicid that is too big\n", pxm, apic_id, node);
  94                 return;
  95         }
  96         set_apicid_to_node(apic_id, node);
  97         node_set(node, numa_nodes_parsed);
  98         acpi_numa = 1;
  99         printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u\n",
 100                pxm, apic_id, node);
 101 }
 102
 103 /* Callback for Proximity Domain -> LAPIC mapping */
 104 void __init
 105 acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
 106 {
 107         int pxm, node;
 108         int apic_id;
 109
 110         if (srat_disabled())
 111                 return;
 112         if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) {
 113                 bad_srat();
 114                 return;
 115         }
 116         if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
 117                 return;
 118         pxm = pa->proximity_domain_lo;
 119         node = setup_node(pxm);
 120         if (node < 0) {
 121                 printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
 122                 bad_srat();
 123                 return;
 124         }
 125
 126         if (get_uv_system_type() >= UV_X2APIC)
 127                 apic_id = (pa->apic_id << 8) | pa->local_sapic_eid;
 128         else
 129                 apic_id = pa->apic_id;
 130
 131         if (apic_id >= MAX_LOCAL_APIC) {
 132                 printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u skipped apicid that is too big\n", pxm, apic_id, node);
 133                 return;
 134         }
 135
 136         set_apicid_to_node(apic_id, node);
 137         node_set(node, numa_nodes_parsed);
 138         acpi_numa = 1;
 139         printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u\n",
 140                pxm, apic_id, node);
 141 }
 142
 143 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
 144 static inline int save_add_info(void) {return 1;}
 145 #else
 146 static inline int save_add_info(void) {return 0;}
 147 #endif
 148 /*
 149  * Update nodes_add[]
 150  * This code supports one contiguous hot add area per node
 151  */
 152 static void __init
 153 update_nodes_add(int node, unsigned long start, unsigned long end)
 154 {
 155         unsigned long s_pfn = start >> PAGE_SHIFT;
 156         unsigned long e_pfn = end >> PAGE_SHIFT;
 157         int changed = 0;
 158         struct bootnode *nd = &nodes_add[node];
 159
 160         /* I had some trouble with strange memory hotadd regions breaking
 161            the boot. Be very strict here and reject anything unexpected.
 162            If you want working memory hotadd write correct SRATs.
 163
 164            The node size check is a basic sanity check to guard against
 165            mistakes */
 166         if ((signed long)(end - start) < NODE_MIN_SIZE) {
 167                 printk(KERN_ERR "SRAT: Hotplug area too small\n");
 168                 return;
 169         }
 170
 171         /* This check might be a bit too strict, but I'm keeping it for now. */
 172         if (absent_pages_in_range(s_pfn, e_pfn) != e_pfn - s_pfn) {
 173                 printk(KERN_ERR
 174                         "SRAT: Hotplug area %lu -> %lu has existing memory\n",
 175                         s_pfn, e_pfn);
 176                 return;
 177         }
 178
 179         /* Looks good */
 180
 181         if (nd->start == nd->end) {
 182                 nd->start = start;
 183                 nd->end = end;
 184                 changed = 1;
 185         } else {
 186                 if (nd->start == end) {
 187                         nd->start = start;
 188                         changed = 1;
 189                 }
 190                 if (nd->end == start) {
 191                         nd->end = end;
 192                         changed = 1;
 193                 }
 194                 if (!changed)
 195                         printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n");
 196         }
 197
 198         if (changed) {
 199                 node_set(node, numa_nodes_parsed);
 200                 printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n",
 201                                  nd->start, nd->end);
 202         }
 203 }
 204
 205 /* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
 206 void __init
 207 acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
 208 {
 209         unsigned long start, end;
 210         int node, pxm;
 211
 212         if (srat_disabled())
 213                 return;
 214         if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) {
 215                 bad_srat();
 216                 return;
 217         }
 218         if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
 219                 return;
 220
 221         if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info())
 222                 return;
 223         start = ma->base_address;
 224         end = start + ma->length;
 225         pxm = ma->proximity_domain;
 226         node = setup_node(pxm);
 227         if (node < 0) {
 228                 printk(KERN_ERR "SRAT: Too many proximity domains.\n");
 229                 bad_srat();
 230                 return;
 231         }
 232
 233         if (numa_add_memblk(node, start, end) < 0) {
 234                 bad_srat();
 235                 return;
 236         }
 237
 238         printk(KERN_INFO "SRAT: Node %u PXM %u %lx-%lx\n", node, pxm,
 239                start, end);
 240
 241         if (!(ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE))
 242                 node_set(node, mem_nodes_parsed);
 243         else
 244                 update_nodes_add(node, start, end);
 245 }
 246
 247 void __init acpi_numa_arch_fixup(void) {}
 248
 249 int __init x86_acpi_numa_init(void)
 250 {
 251         int ret;
 252
 253         ret = acpi_numa_init();
 254         if (ret < 0)
 255                 return ret;
 256         return srat_disabled() ? -EINVAL : 0;
 257 }
 258
 259 #ifdef CONFIG_NUMA_EMU
 260 static int fake_node_to_pxm_map[MAX_NUMNODES] __initdata = {
 261         [0 ... MAX_NUMNODES-1] = PXM_INVAL
 262 };
 263 static s16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = {
 264         [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
 265 };
 266
 267 /*
 268  * In NUMA emulation, we need to setup proximity domain (_PXM) to node ID
 269  * mappings that respect the real ACPI topology but reflect our emulated
 270  * environment.  For each emulated node, we find which real node it appears on
 271  * and create PXM to NID mappings for those fake nodes which mirror that
 272  * locality.  SLIT will now represent the correct distances between emulated
 273  * nodes as a result of the real topology.
 274  */
 275 void __init acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes)
 276 {
 277         int i, j;
 278
 279         for (i = 0; i < num_nodes; i++) {
 280                 int nid, pxm;
 281
 282                 nid = find_node_by_addr(fake_nodes[i].start);
 283                 if (nid == NUMA_NO_NODE)
 284                         continue;
 285                 pxm = node_to_pxm(nid);
 286                 if (pxm == PXM_INVAL)
 287                         continue;
 288                 fake_node_to_pxm_map[i] = pxm;
 289                 /*
 290                  * For each apicid_to_node mapping that exists for this real
 291                  * node, it must now point to the fake node ID.
 292                  */
 293                 for (j = 0; j < MAX_LOCAL_APIC; j++)
 294                         if (__apicid_to_node[j] == nid &&
 295                             fake_apicid_to_node[j] == NUMA_NO_NODE)
 296                                 fake_apicid_to_node[j] = i;
 297         }
 298
 299         /*
 300          * If there are apicid-to-node mappings for physical nodes that do not
 301          * have a corresponding emulated node, it should default to a guaranteed
 302          * value.
 303          */
 304         for (i = 0; i < MAX_LOCAL_APIC; i++)
 305                 if (__apicid_to_node[i] != NUMA_NO_NODE &&
 306                     fake_apicid_to_node[i] == NUMA_NO_NODE)
 307                         fake_apicid_to_node[i] = 0;
 308
 309         for (i = 0; i < num_nodes; i++)
 310                 __acpi_map_pxm_to_node(fake_node_to_pxm_map[i], i);
 311         memcpy(__apicid_to_node, fake_apicid_to_node, sizeof(__apicid_to_node));
 312
 313         nodes_clear(mem_nodes_parsed);
 314         for (i = 0; i < num_nodes; i++)
 315                 if (fake_nodes[i].start != fake_nodes[i].end)
 316                         node_set(i, mem_nodes_parsed);
 317 }
 318
 319 static int null_slit_node_compare(int a, int b)
 320 {
 321         return node_to_pxm(a) == node_to_pxm(b);
 322 }
 323 #else
 324 static int null_slit_node_compare(int a, int b)
 325 {
 326         return a == b;
 327 }
 328 #endif /* CONFIG_NUMA_EMU */
 329
 330 int __node_distance(int a, int b)
 331 {
 332         int index;
 333
 334         if (!acpi_slit)
 335                 return null_slit_node_compare(a, b) ? LOCAL_DISTANCE :
 336                                                       REMOTE_DISTANCE;
 337         index = acpi_slit->locality_count * node_to_pxm(a);
 338         return acpi_slit->entry[index + node_to_pxm(b)];
 339 }
 340
 341 EXPORT_SYMBOL(__node_distance);
 342
 343 #if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) || defined(CONFIG_ACPI_HOTPLUG_MEMORY)
 344 int memory_add_physaddr_to_nid(u64 start)
 345 {
 346         int i, ret = 0;
 347
 348         for_each_node(i)
 349                 if (nodes_add[i].start <= start && nodes_add[i].end > start)
 350                         ret = i;
 351
 352         return ret;
 353 }
 354 EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
 355 #endif