2 * Written by: Patricia Gaughen <gone@us.ibm.com>, IBM Corporation
3 * August 2002: added remote node KVA remap - Martin J. Bligh
5 * Copyright (C) 2002, IBM Corp.
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
14 * This program is distributed in the hope that it will be useful, but
15 * WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
17 * NON INFRINGEMENT. See the GNU General Public License for more
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26 #include <linux/bootmem.h>
27 #include <linux/memblock.h>
28 #include <linux/mmzone.h>
29 #include <linux/highmem.h>
30 #include <linux/initrd.h>
31 #include <linux/nodemask.h>
32 #include <linux/module.h>
33 #include <linux/kexec.h>
34 #include <linux/pfn.h>
35 #include <linux/swap.h>
36 #include <linux/acpi.h>
39 #include <asm/setup.h>
40 #include <asm/mmzone.h>
41 #include <asm/bios_ebda.h>
42 #include <asm/proto.h>
44 struct pglist_data
*node_data
[MAX_NUMNODES
] __read_mostly
;
45 EXPORT_SYMBOL(node_data
);
48 * numa interface - we expect the numa architecture specific code to have
49 * populated the following initialisation.
51 * 1) node_online_map - the map of all nodes configured (online) in the system
52 * 2) node_start_pfn - the starting page frame number for a node
53 * 3) node_end_pfn - the ending page fram number for a node
55 unsigned long node_start_pfn
[MAX_NUMNODES
] __read_mostly
;
56 unsigned long node_end_pfn
[MAX_NUMNODES
] __read_mostly
;
59 #ifdef CONFIG_DISCONTIGMEM
61 * 4) physnode_map - the mapping between a pfn and owning node
62 * physnode_map keeps track of the physical memory layout of a generic
63 * numa node on a 64Mb break (each element of the array will
64 * represent 64Mb of memory and will be marked by the node id. so,
65 * if the first gig is on node 0, and the second gig is on node 1
66 * physnode_map will contain:
68 * physnode_map[0-15] = 0;
69 * physnode_map[16-31] = 1;
70 * physnode_map[32- ] = -1;
72 s8 physnode_map
[MAX_ELEMENTS
] __read_mostly
= { [0 ... (MAX_ELEMENTS
- 1)] = -1};
73 EXPORT_SYMBOL(physnode_map
);
75 void memory_present(int nid
, unsigned long start
, unsigned long end
)
79 printk(KERN_INFO
"Node: %d, start_pfn: %lx, end_pfn: %lx\n",
81 printk(KERN_DEBUG
" Setting physnode_map array to node %d for pfns:\n", nid
);
82 printk(KERN_DEBUG
" ");
83 for (pfn
= start
; pfn
< end
; pfn
+= PAGES_PER_ELEMENT
) {
84 physnode_map
[pfn
/ PAGES_PER_ELEMENT
] = nid
;
85 printk(KERN_CONT
"%lx ", pfn
);
87 printk(KERN_CONT
"\n");
90 unsigned long node_memmap_size_bytes(int nid
, unsigned long start_pfn
,
91 unsigned long end_pfn
)
93 unsigned long nr_pages
= end_pfn
- start_pfn
;
98 return (nr_pages
+ 1) * sizeof(struct page
);
102 extern unsigned long find_max_low_pfn(void);
103 extern unsigned long highend_pfn
, highstart_pfn
;
105 #define LARGE_PAGE_BYTES (PTRS_PER_PTE * PAGE_SIZE)
107 static void *node_remap_start_vaddr
[MAX_NUMNODES
];
108 void set_pmd_pfn(unsigned long vaddr
, unsigned long pfn
, pgprot_t flags
);
111 * FLAT - support for basic PC memory model with discontig enabled, essentially
112 * a single node with all available processors in it with a flat
115 static int __init
get_memcfg_numa_flat(void)
117 printk(KERN_DEBUG
"NUMA - single node, flat memory mode\n");
119 node_start_pfn
[0] = 0;
120 node_end_pfn
[0] = max_pfn
;
121 memblock_x86_register_active_regions(0, 0, max_pfn
);
123 /* Indicate there is one node available. */
124 nodes_clear(node_online_map
);
130 * Find the highest page frame number we have available for the node
132 static void __init
propagate_e820_map_node(int nid
)
134 if (node_end_pfn
[nid
] > max_pfn
)
135 node_end_pfn
[nid
] = max_pfn
;
137 * if a user has given mem=XXXX, then we need to make sure
138 * that the node _starts_ before that, too, not just ends
140 if (node_start_pfn
[nid
] > max_pfn
)
141 node_start_pfn
[nid
] = max_pfn
;
142 BUG_ON(node_start_pfn
[nid
] > node_end_pfn
[nid
]);
146 * Allocate memory for the pg_data_t for this node via a crude pre-bootmem
147 * method. For node zero take this from the bottom of memory, for
148 * subsequent nodes place them at node_remap_start_vaddr which contains
149 * node local data in physically node local memory. See setup_memory()
152 static void __init
allocate_pgdat(int nid
)
156 NODE_DATA(nid
) = alloc_remap(nid
, ALIGN(sizeof(pg_data_t
), PAGE_SIZE
));
157 if (!NODE_DATA(nid
)) {
158 unsigned long pgdat_phys
;
159 pgdat_phys
= memblock_find_in_range(min_low_pfn
<<PAGE_SHIFT
,
160 max_pfn_mapped
<<PAGE_SHIFT
,
163 NODE_DATA(nid
) = (pg_data_t
*)(pfn_to_kaddr(pgdat_phys
>>PAGE_SHIFT
));
164 memset(buf
, 0, sizeof(buf
));
165 sprintf(buf
, "NODE_DATA %d", nid
);
166 memblock_x86_reserve_range(pgdat_phys
, pgdat_phys
+ sizeof(pg_data_t
), buf
);
168 printk(KERN_DEBUG
"allocate_pgdat: node %d NODE_DATA %08lx\n",
169 nid
, (unsigned long)NODE_DATA(nid
));
173 * Remap memory allocator
175 static unsigned long node_remap_start_pfn
[MAX_NUMNODES
];
176 static void *node_remap_end_vaddr
[MAX_NUMNODES
];
177 static void *node_remap_alloc_vaddr
[MAX_NUMNODES
];
180 * alloc_remap - Allocate remapped memory
181 * @nid: NUMA node to allocate memory from
182 * @size: The size of allocation
184 * Allocate @size bytes from the remap area of NUMA node @nid. The
185 * size of the remap area is predetermined by init_alloc_remap() and
186 * only the callers considered there should call this function. For
187 * more info, please read the comment on top of init_alloc_remap().
189 * The caller must be ready to handle allocation failure from this
190 * function and fall back to regular memory allocator in such cases.
193 * Single CPU early boot context.
196 * Pointer to the allocated memory on success, %NULL on failure.
198 void *alloc_remap(int nid
, unsigned long size
)
200 void *allocation
= node_remap_alloc_vaddr
[nid
];
202 size
= ALIGN(size
, L1_CACHE_BYTES
);
204 if (!allocation
|| (allocation
+ size
) > node_remap_end_vaddr
[nid
])
207 node_remap_alloc_vaddr
[nid
] += size
;
208 memset(allocation
, 0, size
);
213 #ifdef CONFIG_HIBERNATION
215 * resume_map_numa_kva - add KVA mapping to the temporary page tables created
216 * during resume from hibernation
217 * @pgd_base - temporary resume page directory
219 void resume_map_numa_kva(pgd_t
*pgd_base
)
223 for_each_online_node(node
) {
224 unsigned long start_va
, start_pfn
, nr_pages
, pfn
;
226 start_va
= (unsigned long)node_remap_start_vaddr
[node
];
227 start_pfn
= node_remap_start_pfn
[node
];
228 nr_pages
= (node_remap_end_vaddr
[node
] -
229 node_remap_start_vaddr
[node
]) >> PAGE_SHIFT
;
231 printk(KERN_DEBUG
"%s: node %d\n", __func__
, node
);
233 for (pfn
= 0; pfn
< nr_pages
; pfn
+= PTRS_PER_PTE
) {
234 unsigned long vaddr
= start_va
+ (pfn
<< PAGE_SHIFT
);
235 pgd_t
*pgd
= pgd_base
+ pgd_index(vaddr
);
236 pud_t
*pud
= pud_offset(pgd
, vaddr
);
237 pmd_t
*pmd
= pmd_offset(pud
, vaddr
);
239 set_pmd(pmd
, pfn_pmd(start_pfn
+ pfn
,
240 PAGE_KERNEL_LARGE_EXEC
));
242 printk(KERN_DEBUG
"%s: %08lx -> pfn %08lx\n",
243 __func__
, vaddr
, start_pfn
+ pfn
);
250 * init_alloc_remap - Initialize remap allocator for a NUMA node
251 * @nid: NUMA node to initizlie remap allocator for
253 * NUMA nodes may end up without any lowmem. As allocating pgdat and
254 * memmap on a different node with lowmem is inefficient, a special
255 * remap allocator is implemented which can be used by alloc_remap().
257 * For each node, the amount of memory which will be necessary for
258 * pgdat and memmap is calculated and two memory areas of the size are
259 * allocated - one in the node and the other in lowmem; then, the area
260 * in the node is remapped to the lowmem area.
262 * As pgdat and memmap must be allocated in lowmem anyway, this
263 * doesn't waste lowmem address space; however, the actual lowmem
264 * which gets remapped over is wasted. The amount shouldn't be
265 * problematic on machines this feature will be used.
267 * Initialization failure isn't fatal. alloc_remap() is used
268 * opportunistically and the callers will fall back to other memory
269 * allocation mechanisms on failure.
271 static __init
void init_alloc_remap(int nid
)
273 unsigned long size
, pfn
;
274 u64 node_pa
, remap_pa
;
278 * The acpi/srat node info can show hot-add memroy zones where
279 * memory could be added but not currently present.
281 printk(KERN_DEBUG
"node %d pfn: [%lx - %lx]\n",
282 nid
, node_start_pfn
[nid
], node_end_pfn
[nid
]);
283 if (node_start_pfn
[nid
] > max_pfn
)
285 if (!node_end_pfn
[nid
])
287 if (node_end_pfn
[nid
] > max_pfn
)
288 node_end_pfn
[nid
] = max_pfn
;
290 /* calculate the necessary space aligned to large page size */
291 size
= node_memmap_size_bytes(nid
, node_start_pfn
[nid
],
292 min(node_end_pfn
[nid
], max_pfn
));
293 size
+= ALIGN(sizeof(pg_data_t
), PAGE_SIZE
);
294 size
= ALIGN(size
, LARGE_PAGE_BYTES
);
296 /* allocate node memory and the lowmem remap area */
297 node_pa
= memblock_find_in_range(node_start_pfn
[nid
] << PAGE_SHIFT
,
298 (u64
)node_end_pfn
[nid
] << PAGE_SHIFT
,
299 size
, LARGE_PAGE_BYTES
);
300 if (node_pa
== MEMBLOCK_ERROR
) {
301 pr_warning("remap_alloc: failed to allocate %lu bytes for node %d\n",
305 memblock_x86_reserve_range(node_pa
, node_pa
+ size
, "KVA RAM");
307 remap_pa
= memblock_find_in_range(min_low_pfn
<< PAGE_SHIFT
,
308 max_low_pfn
<< PAGE_SHIFT
,
309 size
, LARGE_PAGE_BYTES
);
310 if (remap_pa
== MEMBLOCK_ERROR
) {
311 pr_warning("remap_alloc: failed to allocate %lu bytes remap area for node %d\n",
313 memblock_x86_free_range(node_pa
, node_pa
+ size
);
316 memblock_x86_reserve_range(remap_pa
, remap_pa
+ size
, "KVA PG");
317 remap_va
= phys_to_virt(remap_pa
);
319 /* perform actual remap */
320 for (pfn
= 0; pfn
< size
>> PAGE_SHIFT
; pfn
+= PTRS_PER_PTE
)
321 set_pmd_pfn((unsigned long)remap_va
+ (pfn
<< PAGE_SHIFT
),
322 (node_pa
>> PAGE_SHIFT
) + pfn
,
325 /* initialize remap allocator parameters */
326 node_remap_start_pfn
[nid
] = node_pa
>> PAGE_SHIFT
;
327 node_remap_start_vaddr
[nid
] = remap_va
;
328 node_remap_end_vaddr
[nid
] = remap_va
+ size
;
329 node_remap_alloc_vaddr
[nid
] = remap_va
;
331 printk(KERN_DEBUG
"remap_alloc: node %d [%08llx-%08llx) -> [%p-%p)\n",
332 nid
, node_pa
, node_pa
+ size
, remap_va
, remap_va
+ size
);
335 static int get_memcfg_numaq(void)
337 #ifdef CONFIG_X86_NUMAQ
343 if (numaq_numa_init() < 0) {
344 nodes_clear(numa_nodes_parsed
);
345 remove_all_active_ranges();
349 for_each_node_mask(nid
, numa_nodes_parsed
)
350 node_set_online(nid
);
358 static int get_memcfg_from_srat(void)
360 #ifdef CONFIG_ACPI_NUMA
366 if (x86_acpi_numa_init() < 0) {
367 nodes_clear(numa_nodes_parsed
);
368 remove_all_active_ranges();
372 for_each_node_mask(nid
, numa_nodes_parsed
)
373 node_set_online(nid
);
381 static void get_memcfg_numa(void)
383 if (get_memcfg_numaq())
385 if (get_memcfg_from_srat())
387 get_memcfg_numa_flat();
390 void __init
initmem_init(void)
397 for_each_online_node(nid
)
398 init_alloc_remap(nid
);
400 #ifdef CONFIG_HIGHMEM
401 highstart_pfn
= highend_pfn
= max_pfn
;
402 if (max_pfn
> max_low_pfn
)
403 highstart_pfn
= max_low_pfn
;
404 printk(KERN_NOTICE
"%ldMB HIGHMEM available.\n",
405 pages_to_mb(highend_pfn
- highstart_pfn
));
406 num_physpages
= highend_pfn
;
407 high_memory
= (void *) __va(highstart_pfn
* PAGE_SIZE
- 1) + 1;
409 num_physpages
= max_low_pfn
;
410 high_memory
= (void *) __va(max_low_pfn
* PAGE_SIZE
- 1) + 1;
412 printk(KERN_NOTICE
"%ldMB LOWMEM available.\n",
413 pages_to_mb(max_low_pfn
));
414 printk(KERN_DEBUG
"max_low_pfn = %lx, highstart_pfn = %lx\n",
415 max_low_pfn
, highstart_pfn
);
417 printk(KERN_DEBUG
"Low memory ends at vaddr %08lx\n",
418 (ulong
) pfn_to_kaddr(max_low_pfn
));
419 for_each_online_node(nid
)
422 printk(KERN_DEBUG
"High memory starts at vaddr %08lx\n",
423 (ulong
) pfn_to_kaddr(highstart_pfn
));
424 for_each_online_node(nid
)
425 propagate_e820_map_node(nid
);
427 for_each_online_node(nid
) {
428 memset(NODE_DATA(nid
), 0, sizeof(struct pglist_data
));
429 NODE_DATA(nid
)->node_id
= nid
;
432 setup_bootmem_allocator();
435 #ifdef CONFIG_MEMORY_HOTPLUG
436 static int paddr_to_nid(u64 addr
)
439 unsigned long pfn
= PFN_DOWN(addr
);
442 if (node_start_pfn
[nid
] <= pfn
&&
443 pfn
< node_end_pfn
[nid
])
450 * This function is used to ask node id BEFORE memmap and mem_section's
451 * initialization (pfn_to_nid() can't be used yet).
452 * If _PXM is not defined on ACPI's DSDT, node id must be found by this.
454 int memory_add_physaddr_to_nid(u64 addr
)
456 int nid
= paddr_to_nid(addr
);
457 return (nid
>= 0) ? nid
: 0;
460 EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid
);
463 /* temporary shim, will go away soon */
464 int __init
numa_add_memblk(int nid
, u64 start
, u64 end
)
466 unsigned long start_pfn
= start
>> PAGE_SHIFT
;
467 unsigned long end_pfn
= end
>> PAGE_SHIFT
;
469 printk(KERN_DEBUG
"nid %d start_pfn %08lx end_pfn %08lx\n",
470 nid
, start_pfn
, end_pfn
);
472 if (start
>= (u64
)max_pfn
<< PAGE_SHIFT
) {
473 printk(KERN_INFO
"Ignoring SRAT pfns: %08lx - %08lx\n",
478 node_set_online(nid
);
479 memblock_x86_register_active_regions(nid
, start_pfn
,
480 min(end_pfn
, max_pfn
));
482 if (!node_has_online_mem(nid
)) {
483 node_start_pfn
[nid
] = start_pfn
;
484 node_end_pfn
[nid
] = end_pfn
;
486 node_start_pfn
[nid
] = min(node_start_pfn
[nid
], start_pfn
);
487 node_end_pfn
[nid
] = max(node_end_pfn
[nid
], end_pfn
);
492 /* temporary shim, will go away soon */
493 void __init
numa_set_distance(int from
, int to
, int distance
)