arch/x86/mm/srat_64.c

   1 /*
   2  * ACPI 3.0 based NUMA setup
   3  * Copyright 2004 Andi Kleen, SuSE Labs.
   4  *
   5  * Reads the ACPI SRAT table to figure out what memory belongs to which CPUs.
   6  *
   7  * Called from acpi_numa_init while reading the SRAT and SLIT tables.
   8  * Assumes all memory regions belonging to a single proximity domain
   9  * are in one chunk. Holes between them will be included in the node.
  10  */
  11
  12 #include <linux/kernel.h>
  13 #include <linux/acpi.h>
  14 #include <linux/mmzone.h>
  15 #include <linux/bitmap.h>
  16 #include <linux/module.h>
  17 #include <linux/topology.h>
  18 #include <linux/bootmem.h>
  19 #include <linux/memblock.h>
  20 #include <linux/mm.h>
  21 #include <asm/proto.h>
  22 #include <asm/numa.h>
  23 #include <asm/e820.h>
  24 #include <asm/apic.h>
  25 #include <asm/uv/uv.h>
  26
  27 int acpi_numa __initdata;
  28
  29 static struct acpi_table_slit *acpi_slit;
  30
  31 static struct bootnode nodes_add[MAX_NUMNODES];
  32
  33 static __init int setup_node(int pxm)
  34 {
  35         return acpi_map_pxm_to_node(pxm);
  36 }
  37
  38 static __init void bad_srat(void)
  39 {
  40         int i;
  41         printk(KERN_ERR "SRAT: SRAT not used.\n");
  42         acpi_numa = -1;
  43         for (i = 0; i < MAX_NUMNODES; i++) {
  44                 numa_nodes[i].start = numa_nodes[i].end = 0;
  45                 nodes_add[i].start = nodes_add[i].end = 0;
  46         }
  47 }
  48
  49 static __init inline int srat_disabled(void)
  50 {
  51         return acpi_numa < 0;
  52 }
  53
  54 /* Callback for SLIT parsing */
  55 void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
  56 {
  57         unsigned length;
  58         unsigned long phys;
  59
  60         length = slit->header.length;
  61         phys = memblock_find_in_range(0, max_pfn_mapped<<PAGE_SHIFT, length,
  62                  PAGE_SIZE);
  63
  64         if (phys == MEMBLOCK_ERROR)
  65                 panic(" Can not save slit!\n");
  66
  67         acpi_slit = __va(phys);
  68         memcpy(acpi_slit, slit, length);
  69         memblock_x86_reserve_range(phys, phys + length, "ACPI SLIT");
  70 }
  71
  72 /* Callback for Proximity Domain -> x2APIC mapping */
  73 void __init
  74 acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
  75 {
  76         int pxm, node;
  77         int apic_id;
  78
  79         if (srat_disabled())
  80                 return;
  81         if (pa->header.length < sizeof(struct acpi_srat_x2apic_cpu_affinity)) {
  82                 bad_srat();
  83                 return;
  84         }
  85         if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
  86                 return;
  87         pxm = pa->proximity_domain;
  88         node = setup_node(pxm);
  89         if (node < 0) {
  90                 printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
  91                 bad_srat();
  92                 return;
  93         }
  94
  95         apic_id = pa->apic_id;
  96         if (apic_id >= MAX_LOCAL_APIC) {
  97                 printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u skipped apicid that is too big\n", pxm, apic_id, node);
  98                 return;
  99         }
 100         set_apicid_to_node(apic_id, node);
 101         node_set(node, cpu_nodes_parsed);
 102         acpi_numa = 1;
 103         printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u\n",
 104                pxm, apic_id, node);
 105 }
 106
 107 /* Callback for Proximity Domain -> LAPIC mapping */
 108 void __init
 109 acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
 110 {
 111         int pxm, node;
 112         int apic_id;
 113
 114         if (srat_disabled())
 115                 return;
 116         if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) {
 117                 bad_srat();
 118                 return;
 119         }
 120         if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
 121                 return;
 122         pxm = pa->proximity_domain_lo;
 123         node = setup_node(pxm);
 124         if (node < 0) {
 125                 printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
 126                 bad_srat();
 127                 return;
 128         }
 129
 130         if (get_uv_system_type() >= UV_X2APIC)
 131                 apic_id = (pa->apic_id << 8) | pa->local_sapic_eid;
 132         else
 133                 apic_id = pa->apic_id;
 134
 135         if (apic_id >= MAX_LOCAL_APIC) {
 136                 printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u skipped apicid that is too big\n", pxm, apic_id, node);
 137                 return;
 138         }
 139
 140         set_apicid_to_node(apic_id, node);
 141         node_set(node, cpu_nodes_parsed);
 142         acpi_numa = 1;
 143         printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u\n",
 144                pxm, apic_id, node);
 145 }
 146
 147 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
 148 static inline int save_add_info(void) {return 1;}
 149 #else
 150 static inline int save_add_info(void) {return 0;}
 151 #endif
 152 /*
 153  * Update nodes_add[]
 154  * This code supports one contiguous hot add area per node
 155  */
 156 static void __init
 157 update_nodes_add(int node, unsigned long start, unsigned long end)
 158 {
 159         unsigned long s_pfn = start >> PAGE_SHIFT;
 160         unsigned long e_pfn = end >> PAGE_SHIFT;
 161         int changed = 0;
 162         struct bootnode *nd = &nodes_add[node];
 163
 164         /* I had some trouble with strange memory hotadd regions breaking
 165            the boot. Be very strict here and reject anything unexpected.
 166            If you want working memory hotadd write correct SRATs.
 167
 168            The node size check is a basic sanity check to guard against
 169            mistakes */
 170         if ((signed long)(end - start) < NODE_MIN_SIZE) {
 171                 printk(KERN_ERR "SRAT: Hotplug area too small\n");
 172                 return;
 173         }
 174
 175         /* This check might be a bit too strict, but I'm keeping it for now. */
 176         if (absent_pages_in_range(s_pfn, e_pfn) != e_pfn - s_pfn) {
 177                 printk(KERN_ERR
 178                         "SRAT: Hotplug area %lu -> %lu has existing memory\n",
 179                         s_pfn, e_pfn);
 180                 return;
 181         }
 182
 183         /* Looks good */
 184
 185         if (nd->start == nd->end) {
 186                 nd->start = start;
 187                 nd->end = end;
 188                 changed = 1;
 189         } else {
 190                 if (nd->start == end) {
 191                         nd->start = start;
 192                         changed = 1;
 193                 }
 194                 if (nd->end == start) {
 195                         nd->end = end;
 196                         changed = 1;
 197                 }
 198                 if (!changed)
 199                         printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n");
 200         }
 201
 202         if (changed) {
 203                 node_set(node, cpu_nodes_parsed);
 204                 printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n",
 205                                  nd->start, nd->end);
 206         }
 207 }
 208
 209 /* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
 210 void __init
 211 acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
 212 {
 213         struct bootnode *nd;
 214         unsigned long start, end;
 215         int node, pxm;
 216
 217         if (srat_disabled())
 218                 return;
 219         if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) {
 220                 bad_srat();
 221                 return;
 222         }
 223         if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
 224                 return;
 225
 226         if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info())
 227                 return;
 228         start = ma->base_address;
 229         end = start + ma->length;
 230         pxm = ma->proximity_domain;
 231         node = setup_node(pxm);
 232         if (node < 0) {
 233                 printk(KERN_ERR "SRAT: Too many proximity domains.\n");
 234                 bad_srat();
 235                 return;
 236         }
 237
 238         if (numa_add_memblk(node, start, end) < 0) {
 239                 bad_srat();
 240                 return;
 241         }
 242
 243         printk(KERN_INFO "SRAT: Node %u PXM %u %lx-%lx\n", node, pxm,
 244                start, end);
 245
 246         if (!(ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE)) {
 247                 nd = &numa_nodes[node];
 248                 if (!node_test_and_set(node, mem_nodes_parsed)) {
 249                         nd->start = start;
 250                         nd->end = end;
 251                 } else {
 252                         if (start < nd->start)
 253                                 nd->start = start;
 254                         if (nd->end < end)
 255                                 nd->end = end;
 256                 }
 257         } else
 258                 update_nodes_add(node, start, end);
 259 }
 260
 261 void __init acpi_numa_arch_fixup(void) {}
 262
 263 int __init x86_acpi_numa_init(void)
 264 {
 265         int ret;
 266
 267         ret = acpi_numa_init();
 268         if (ret < 0)
 269                 return ret;
 270         return srat_disabled() ? -EINVAL : 0;
 271 }
 272
 273 /* Use the information discovered above to actually set up the nodes. */
 274 int __init acpi_scan_nodes(void)
 275 {
 276         if (acpi_numa <= 0)
 277                 return -1;
 278         return 0;
 279 }
 280
 281 #ifdef CONFIG_NUMA_EMU
 282 static int fake_node_to_pxm_map[MAX_NUMNODES] __initdata = {
 283         [0 ... MAX_NUMNODES-1] = PXM_INVAL
 284 };
 285 static s16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = {
 286         [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
 287 };
 288 static int __init find_node_by_addr(unsigned long addr)
 289 {
 290         int ret = NUMA_NO_NODE;
 291         int i;
 292
 293         for_each_node_mask(i, mem_nodes_parsed) {
 294                 /*
 295                  * Find the real node that this emulated node appears on.  For
 296                  * the sake of simplicity, we only use a real node's starting
 297                  * address to determine which emulated node it appears on.
 298                  */
 299                 if (addr >= numa_nodes[i].start && addr < numa_nodes[i].end) {
 300                         ret = i;
 301                         break;
 302                 }
 303         }
 304         return ret;
 305 }
 306
 307 /*
 308  * In NUMA emulation, we need to setup proximity domain (_PXM) to node ID
 309  * mappings that respect the real ACPI topology but reflect our emulated
 310  * environment.  For each emulated node, we find which real node it appears on
 311  * and create PXM to NID mappings for those fake nodes which mirror that
 312  * locality.  SLIT will now represent the correct distances between emulated
 313  * nodes as a result of the real topology.
 314  */
 315 void __init acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes)
 316 {
 317         int i, j;
 318
 319         for (i = 0; i < num_nodes; i++) {
 320                 int nid, pxm;
 321
 322                 nid = find_node_by_addr(fake_nodes[i].start);
 323                 if (nid == NUMA_NO_NODE)
 324                         continue;
 325                 pxm = node_to_pxm(nid);
 326                 if (pxm == PXM_INVAL)
 327                         continue;
 328                 fake_node_to_pxm_map[i] = pxm;
 329                 /*
 330                  * For each apicid_to_node mapping that exists for this real
 331                  * node, it must now point to the fake node ID.
 332                  */
 333                 for (j = 0; j < MAX_LOCAL_APIC; j++)
 334                         if (__apicid_to_node[j] == nid &&
 335                             fake_apicid_to_node[j] == NUMA_NO_NODE)
 336                                 fake_apicid_to_node[j] = i;
 337         }
 338
 339         /*
 340          * If there are apicid-to-node mappings for physical nodes that do not
 341          * have a corresponding emulated node, it should default to a guaranteed
 342          * value.
 343          */
 344         for (i = 0; i < MAX_LOCAL_APIC; i++)
 345                 if (__apicid_to_node[i] != NUMA_NO_NODE &&
 346                     fake_apicid_to_node[i] == NUMA_NO_NODE)
 347                         fake_apicid_to_node[i] = 0;
 348
 349         for (i = 0; i < num_nodes; i++)
 350                 __acpi_map_pxm_to_node(fake_node_to_pxm_map[i], i);
 351         memcpy(__apicid_to_node, fake_apicid_to_node, sizeof(__apicid_to_node));
 352
 353         nodes_clear(mem_nodes_parsed);
 354         for (i = 0; i < num_nodes; i++)
 355                 if (fake_nodes[i].start != fake_nodes[i].end)
 356                         node_set(i, mem_nodes_parsed);
 357 }
 358
 359 static int null_slit_node_compare(int a, int b)
 360 {
 361         return node_to_pxm(a) == node_to_pxm(b);
 362 }
 363 #else
 364 static int null_slit_node_compare(int a, int b)
 365 {
 366         return a == b;
 367 }
 368 #endif /* CONFIG_NUMA_EMU */
 369
 370 int __node_distance(int a, int b)
 371 {
 372         int index;
 373
 374         if (!acpi_slit)
 375                 return null_slit_node_compare(a, b) ? LOCAL_DISTANCE :
 376                                                       REMOTE_DISTANCE;
 377         index = acpi_slit->locality_count * node_to_pxm(a);
 378         return acpi_slit->entry[index + node_to_pxm(b)];
 379 }
 380
 381 EXPORT_SYMBOL(__node_distance);
 382
 383 #if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) || defined(CONFIG_ACPI_HOTPLUG_MEMORY)
 384 int memory_add_physaddr_to_nid(u64 start)
 385 {
 386         int i, ret = 0;
 387
 388         for_each_node(i)
 389                 if (nodes_add[i].start <= start && nodes_add[i].end > start)
 390                         ret = i;
 391
 392         return ret;
 393 }
 394 EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
 395 #endif