2 * Memory subsystem support
4 * Written by Matt Tolentino <matthew.e.tolentino@intel.com>
5 * Dave Hansen <haveblue@us.ibm.com>
7 * This file provides the necessary infrastructure to represent
8 * a SPARSEMEM-memory-model system's physical memory in /sysfs.
9 * All arch-independent code that assumes MEMORY_HOTPLUG requires
10 * SPARSEMEM should be contained here, or in mm/memory_hotplug.c.
13 #include <linux/module.h>
14 #include <linux/init.h>
15 #include <linux/topology.h>
16 #include <linux/capability.h>
17 #include <linux/device.h>
18 #include <linux/memory.h>
19 #include <linux/kobject.h>
20 #include <linux/memory_hotplug.h>
22 #include <linux/mutex.h>
23 #include <linux/stat.h>
24 #include <linux/slab.h>
26 #include <linux/atomic.h>
27 #include <asm/uaccess.h>
29 static DEFINE_MUTEX(mem_sysfs_mutex
);
31 #define MEMORY_CLASS_NAME "memory"
33 static int sections_per_block
;
35 static inline int base_memory_block_id(int section_nr
)
37 return section_nr
/ sections_per_block
;
40 static int memory_subsys_online(struct device
*dev
);
41 static int memory_subsys_offline(struct device
*dev
);
43 static struct bus_type memory_subsys
= {
44 .name
= MEMORY_CLASS_NAME
,
45 .dev_name
= MEMORY_CLASS_NAME
,
46 .online
= memory_subsys_online
,
47 .offline
= memory_subsys_offline
,
50 static BLOCKING_NOTIFIER_HEAD(memory_chain
);
52 int register_memory_notifier(struct notifier_block
*nb
)
54 return blocking_notifier_chain_register(&memory_chain
, nb
);
56 EXPORT_SYMBOL(register_memory_notifier
);
58 void unregister_memory_notifier(struct notifier_block
*nb
)
60 blocking_notifier_chain_unregister(&memory_chain
, nb
);
62 EXPORT_SYMBOL(unregister_memory_notifier
);
64 static ATOMIC_NOTIFIER_HEAD(memory_isolate_chain
);
66 int register_memory_isolate_notifier(struct notifier_block
*nb
)
68 return atomic_notifier_chain_register(&memory_isolate_chain
, nb
);
70 EXPORT_SYMBOL(register_memory_isolate_notifier
);
72 void unregister_memory_isolate_notifier(struct notifier_block
*nb
)
74 atomic_notifier_chain_unregister(&memory_isolate_chain
, nb
);
76 EXPORT_SYMBOL(unregister_memory_isolate_notifier
);
78 static void memory_block_release(struct device
*dev
)
80 struct memory_block
*mem
= container_of(dev
, struct memory_block
, dev
);
85 unsigned long __weak
memory_block_size_bytes(void)
87 return MIN_MEMORY_BLOCK_SIZE
;
90 static unsigned long get_memory_block_size(void)
92 unsigned long block_sz
;
94 block_sz
= memory_block_size_bytes();
96 /* Validate blk_sz is a power of 2 and not less than section size */
97 if ((block_sz
& (block_sz
- 1)) || (block_sz
< MIN_MEMORY_BLOCK_SIZE
)) {
99 block_sz
= MIN_MEMORY_BLOCK_SIZE
;
106 * use this as the physical section index that this memsection
110 static ssize_t
show_mem_start_phys_index(struct device
*dev
,
111 struct device_attribute
*attr
, char *buf
)
113 struct memory_block
*mem
=
114 container_of(dev
, struct memory_block
, dev
);
115 unsigned long phys_index
;
117 phys_index
= mem
->start_section_nr
/ sections_per_block
;
118 return sprintf(buf
, "%08lx\n", phys_index
);
121 static ssize_t
show_mem_end_phys_index(struct device
*dev
,
122 struct device_attribute
*attr
, char *buf
)
124 struct memory_block
*mem
=
125 container_of(dev
, struct memory_block
, dev
);
126 unsigned long phys_index
;
128 phys_index
= mem
->end_section_nr
/ sections_per_block
;
129 return sprintf(buf
, "%08lx\n", phys_index
);
133 * Show whether the section of memory is likely to be hot-removable
135 static ssize_t
show_mem_removable(struct device
*dev
,
136 struct device_attribute
*attr
, char *buf
)
138 unsigned long i
, pfn
;
140 struct memory_block
*mem
=
141 container_of(dev
, struct memory_block
, dev
);
143 for (i
= 0; i
< sections_per_block
; i
++) {
144 pfn
= section_nr_to_pfn(mem
->start_section_nr
+ i
);
145 ret
&= is_mem_section_removable(pfn
, PAGES_PER_SECTION
);
148 return sprintf(buf
, "%d\n", ret
);
152 * online, offline, going offline, etc.
154 static ssize_t
show_mem_state(struct device
*dev
,
155 struct device_attribute
*attr
, char *buf
)
157 struct memory_block
*mem
=
158 container_of(dev
, struct memory_block
, dev
);
162 * We can probably put these states in a nice little array
163 * so that they're not open-coded
165 switch (mem
->state
) {
167 len
= sprintf(buf
, "online\n");
170 len
= sprintf(buf
, "offline\n");
172 case MEM_GOING_OFFLINE
:
173 len
= sprintf(buf
, "going-offline\n");
176 len
= sprintf(buf
, "ERROR-UNKNOWN-%ld\n",
185 int memory_notify(unsigned long val
, void *v
)
187 return blocking_notifier_call_chain(&memory_chain
, val
, v
);
190 int memory_isolate_notify(unsigned long val
, void *v
)
192 return atomic_notifier_call_chain(&memory_isolate_chain
, val
, v
);
196 * The probe routines leave the pages reserved, just as the bootmem code does.
197 * Make sure they're still that way.
199 static bool pages_correctly_reserved(unsigned long start_pfn
)
203 unsigned long pfn
= start_pfn
;
206 * memmap between sections is not contiguous except with
207 * SPARSEMEM_VMEMMAP. We lookup the page once per section
208 * and assume memmap is contiguous within each section
210 for (i
= 0; i
< sections_per_block
; i
++, pfn
+= PAGES_PER_SECTION
) {
211 if (WARN_ON_ONCE(!pfn_valid(pfn
)))
213 page
= pfn_to_page(pfn
);
215 for (j
= 0; j
< PAGES_PER_SECTION
; j
++) {
216 if (PageReserved(page
+ j
))
219 printk(KERN_WARNING
"section number %ld page number %d "
220 "not reserved, was it already online?\n",
221 pfn_to_section_nr(pfn
), j
);
231 * MEMORY_HOTPLUG depends on SPARSEMEM in mm/Kconfig, so it is
232 * OK to have direct references to sparsemem variables in here.
235 memory_block_action(unsigned long phys_index
, unsigned long action
, int online_type
)
237 unsigned long start_pfn
;
238 unsigned long nr_pages
= PAGES_PER_SECTION
* sections_per_block
;
239 struct page
*first_page
;
242 first_page
= pfn_to_page(phys_index
<< PFN_SECTION_SHIFT
);
243 start_pfn
= page_to_pfn(first_page
);
247 if (!pages_correctly_reserved(start_pfn
))
250 ret
= online_pages(start_pfn
, nr_pages
, online_type
);
253 ret
= offline_pages(start_pfn
, nr_pages
);
256 WARN(1, KERN_WARNING
"%s(%ld, %ld) unknown action: "
257 "%ld\n", __func__
, phys_index
, action
, action
);
264 static int __memory_block_change_state(struct memory_block
*mem
,
265 unsigned long to_state
, unsigned long from_state_req
,
270 if (mem
->state
!= from_state_req
)
273 if (to_state
== MEM_OFFLINE
)
274 mem
->state
= MEM_GOING_OFFLINE
;
276 ret
= memory_block_action(mem
->start_section_nr
, to_state
, online_type
);
277 mem
->state
= ret
? from_state_req
: to_state
;
281 static int memory_subsys_online(struct device
*dev
)
283 struct memory_block
*mem
= container_of(dev
, struct memory_block
, dev
);
286 mutex_lock(&mem
->state_mutex
);
288 ret
= mem
->state
== MEM_ONLINE
? 0 :
289 __memory_block_change_state(mem
, MEM_ONLINE
, MEM_OFFLINE
,
292 mutex_unlock(&mem
->state_mutex
);
296 static int memory_subsys_offline(struct device
*dev
)
298 struct memory_block
*mem
= container_of(dev
, struct memory_block
, dev
);
301 mutex_lock(&mem
->state_mutex
);
303 ret
= mem
->state
== MEM_OFFLINE
? 0 :
304 __memory_block_change_state(mem
, MEM_OFFLINE
, MEM_ONLINE
, -1);
306 mutex_unlock(&mem
->state_mutex
);
310 static int __memory_block_change_state_uevent(struct memory_block
*mem
,
311 unsigned long to_state
, unsigned long from_state_req
,
314 int ret
= __memory_block_change_state(mem
, to_state
, from_state_req
,
317 switch (mem
->state
) {
319 kobject_uevent(&mem
->dev
.kobj
, KOBJ_OFFLINE
);
322 kobject_uevent(&mem
->dev
.kobj
, KOBJ_ONLINE
);
331 static int memory_block_change_state(struct memory_block
*mem
,
332 unsigned long to_state
, unsigned long from_state_req
,
337 mutex_lock(&mem
->state_mutex
);
338 ret
= __memory_block_change_state_uevent(mem
, to_state
, from_state_req
,
340 mutex_unlock(&mem
->state_mutex
);
345 store_mem_state(struct device
*dev
,
346 struct device_attribute
*attr
, const char *buf
, size_t count
)
348 struct memory_block
*mem
;
352 mem
= container_of(dev
, struct memory_block
, dev
);
354 lock_device_hotplug();
356 if (!strncmp(buf
, "online_kernel", min_t(int, count
, 13))) {
358 ret
= memory_block_change_state(mem
, MEM_ONLINE
,
359 MEM_OFFLINE
, ONLINE_KERNEL
);
360 } else if (!strncmp(buf
, "online_movable", min_t(int, count
, 14))) {
362 ret
= memory_block_change_state(mem
, MEM_ONLINE
,
363 MEM_OFFLINE
, ONLINE_MOVABLE
);
364 } else if (!strncmp(buf
, "online", min_t(int, count
, 6))) {
366 ret
= memory_block_change_state(mem
, MEM_ONLINE
,
367 MEM_OFFLINE
, ONLINE_KEEP
);
368 } else if(!strncmp(buf
, "offline", min_t(int, count
, 7))) {
370 ret
= memory_block_change_state(mem
, MEM_OFFLINE
,
374 dev
->offline
= offline
;
376 unlock_device_hotplug();
384 * phys_device is a bad name for this. What I really want
385 * is a way to differentiate between memory ranges that
386 * are part of physical devices that constitute
387 * a complete removable unit or fru.
388 * i.e. do these ranges belong to the same physical device,
389 * s.t. if I offline all of these sections I can then
390 * remove the physical device?
392 static ssize_t
show_phys_device(struct device
*dev
,
393 struct device_attribute
*attr
, char *buf
)
395 struct memory_block
*mem
=
396 container_of(dev
, struct memory_block
, dev
);
397 return sprintf(buf
, "%d\n", mem
->phys_device
);
400 static DEVICE_ATTR(phys_index
, 0444, show_mem_start_phys_index
, NULL
);
401 static DEVICE_ATTR(end_phys_index
, 0444, show_mem_end_phys_index
, NULL
);
402 static DEVICE_ATTR(state
, 0644, show_mem_state
, store_mem_state
);
403 static DEVICE_ATTR(phys_device
, 0444, show_phys_device
, NULL
);
404 static DEVICE_ATTR(removable
, 0444, show_mem_removable
, NULL
);
407 * Block size attribute stuff
410 print_block_size(struct device
*dev
, struct device_attribute
*attr
,
413 return sprintf(buf
, "%lx\n", get_memory_block_size());
416 static DEVICE_ATTR(block_size_bytes
, 0444, print_block_size
, NULL
);
419 * Some architectures will have custom drivers to do this, and
420 * will not need to do it from userspace. The fake hot-add code
421 * as well as ppc64 will do all of their discovery in userspace
422 * and will require this interface.
424 #ifdef CONFIG_ARCH_MEMORY_PROBE
426 memory_probe_store(struct device
*dev
, struct device_attribute
*attr
,
427 const char *buf
, size_t count
)
432 unsigned long pages_per_block
= PAGES_PER_SECTION
* sections_per_block
;
434 phys_addr
= simple_strtoull(buf
, NULL
, 0);
436 if (phys_addr
& ((pages_per_block
<< PAGE_SHIFT
) - 1))
439 for (i
= 0; i
< sections_per_block
; i
++) {
440 nid
= memory_add_physaddr_to_nid(phys_addr
);
441 ret
= add_memory(nid
, phys_addr
,
442 PAGES_PER_SECTION
<< PAGE_SHIFT
);
446 phys_addr
+= MIN_MEMORY_BLOCK_SIZE
;
454 static DEVICE_ATTR(probe
, S_IWUSR
, NULL
, memory_probe_store
);
457 #ifdef CONFIG_MEMORY_FAILURE
459 * Support for offlining pages of memory
462 /* Soft offline a page */
464 store_soft_offline_page(struct device
*dev
,
465 struct device_attribute
*attr
,
466 const char *buf
, size_t count
)
470 if (!capable(CAP_SYS_ADMIN
))
472 if (strict_strtoull(buf
, 0, &pfn
) < 0)
477 ret
= soft_offline_page(pfn_to_page(pfn
), 0);
478 return ret
== 0 ? count
: ret
;
481 /* Forcibly offline a page, including killing processes. */
483 store_hard_offline_page(struct device
*dev
,
484 struct device_attribute
*attr
,
485 const char *buf
, size_t count
)
489 if (!capable(CAP_SYS_ADMIN
))
491 if (strict_strtoull(buf
, 0, &pfn
) < 0)
494 ret
= memory_failure(pfn
, 0, 0);
495 return ret
? ret
: count
;
498 static DEVICE_ATTR(soft_offline_page
, S_IWUSR
, NULL
, store_soft_offline_page
);
499 static DEVICE_ATTR(hard_offline_page
, S_IWUSR
, NULL
, store_hard_offline_page
);
503 * Note that phys_device is optional. It is here to allow for
504 * differentiation between which *physical* devices each
505 * section belongs to...
507 int __weak
arch_get_memory_phys_device(unsigned long start_pfn
)
513 * A reference for the returned object is held and the reference for the
514 * hinted object is released.
516 struct memory_block
*find_memory_block_hinted(struct mem_section
*section
,
517 struct memory_block
*hint
)
519 int block_id
= base_memory_block_id(__section_nr(section
));
520 struct device
*hintdev
= hint
? &hint
->dev
: NULL
;
523 dev
= subsys_find_device_by_id(&memory_subsys
, block_id
, hintdev
);
525 put_device(&hint
->dev
);
528 return container_of(dev
, struct memory_block
, dev
);
532 * For now, we have a linear search to go find the appropriate
533 * memory_block corresponding to a particular phys_index. If
534 * this gets to be a real problem, we can always use a radix
535 * tree or something here.
537 * This could be made generic for all device subsystems.
539 struct memory_block
*find_memory_block(struct mem_section
*section
)
541 return find_memory_block_hinted(section
, NULL
);
544 static struct attribute
*memory_memblk_attrs
[] = {
545 &dev_attr_phys_index
.attr
,
546 &dev_attr_end_phys_index
.attr
,
547 &dev_attr_state
.attr
,
548 &dev_attr_phys_device
.attr
,
549 &dev_attr_removable
.attr
,
553 static struct attribute_group memory_memblk_attr_group
= {
554 .attrs
= memory_memblk_attrs
,
557 static const struct attribute_group
*memory_memblk_attr_groups
[] = {
558 &memory_memblk_attr_group
,
563 * register_memory - Setup a sysfs device for a memory block
566 int register_memory(struct memory_block
*memory
)
570 memory
->dev
.bus
= &memory_subsys
;
571 memory
->dev
.id
= memory
->start_section_nr
/ sections_per_block
;
572 memory
->dev
.release
= memory_block_release
;
573 memory
->dev
.groups
= memory_memblk_attr_groups
;
574 memory
->dev
.offline
= memory
->state
== MEM_OFFLINE
;
576 error
= device_register(&memory
->dev
);
580 static int init_memory_block(struct memory_block
**memory
,
581 struct mem_section
*section
, unsigned long state
)
583 struct memory_block
*mem
;
584 unsigned long start_pfn
;
588 mem
= kzalloc(sizeof(*mem
), GFP_KERNEL
);
592 scn_nr
= __section_nr(section
);
593 mem
->start_section_nr
=
594 base_memory_block_id(scn_nr
) * sections_per_block
;
595 mem
->end_section_nr
= mem
->start_section_nr
+ sections_per_block
- 1;
597 mem
->section_count
++;
598 mutex_init(&mem
->state_mutex
);
599 start_pfn
= section_nr_to_pfn(mem
->start_section_nr
);
600 mem
->phys_device
= arch_get_memory_phys_device(start_pfn
);
602 ret
= register_memory(mem
);
608 static int add_memory_section(int nid
, struct mem_section
*section
,
609 struct memory_block
**mem_p
,
610 unsigned long state
, enum mem_add_context context
)
612 struct memory_block
*mem
= NULL
;
613 int scn_nr
= __section_nr(section
);
616 mutex_lock(&mem_sysfs_mutex
);
618 if (context
== BOOT
) {
619 /* same memory block ? */
621 if (scn_nr
>= (*mem_p
)->start_section_nr
&&
622 scn_nr
<= (*mem_p
)->end_section_nr
) {
624 kobject_get(&mem
->dev
.kobj
);
627 mem
= find_memory_block(section
);
630 mem
->section_count
++;
631 kobject_put(&mem
->dev
.kobj
);
633 ret
= init_memory_block(&mem
, section
, state
);
634 /* store memory_block pointer for next loop */
635 if (!ret
&& context
== BOOT
)
641 if (context
== HOTPLUG
&&
642 mem
->section_count
== sections_per_block
)
643 ret
= register_mem_sect_under_node(mem
, nid
);
646 mutex_unlock(&mem_sysfs_mutex
);
651 * need an interface for the VM to add new memory regions,
652 * but without onlining it.
654 int register_new_memory(int nid
, struct mem_section
*section
)
656 return add_memory_section(nid
, section
, NULL
, MEM_OFFLINE
, HOTPLUG
);
659 #ifdef CONFIG_MEMORY_HOTREMOVE
661 unregister_memory(struct memory_block
*memory
)
663 BUG_ON(memory
->dev
.bus
!= &memory_subsys
);
665 /* drop the ref. we got in remove_memory_block() */
666 kobject_put(&memory
->dev
.kobj
);
667 device_unregister(&memory
->dev
);
670 static int remove_memory_block(unsigned long node_id
,
671 struct mem_section
*section
, int phys_device
)
673 struct memory_block
*mem
;
675 mutex_lock(&mem_sysfs_mutex
);
676 mem
= find_memory_block(section
);
677 unregister_mem_sect_under_nodes(mem
, __section_nr(section
));
679 mem
->section_count
--;
680 if (mem
->section_count
== 0)
681 unregister_memory(mem
);
683 kobject_put(&mem
->dev
.kobj
);
685 mutex_unlock(&mem_sysfs_mutex
);
689 int unregister_memory_section(struct mem_section
*section
)
691 if (!present_section(section
))
694 return remove_memory_block(0, section
, 0);
696 #endif /* CONFIG_MEMORY_HOTREMOVE */
698 /* return true if the memory block is offlined, otherwise, return false */
699 bool is_memblock_offlined(struct memory_block
*mem
)
701 return mem
->state
== MEM_OFFLINE
;
704 static struct attribute
*memory_root_attrs
[] = {
705 #ifdef CONFIG_ARCH_MEMORY_PROBE
706 &dev_attr_probe
.attr
,
709 #ifdef CONFIG_MEMORY_FAILURE
710 &dev_attr_soft_offline_page
.attr
,
711 &dev_attr_hard_offline_page
.attr
,
714 &dev_attr_block_size_bytes
.attr
,
718 static struct attribute_group memory_root_attr_group
= {
719 .attrs
= memory_root_attrs
,
722 static const struct attribute_group
*memory_root_attr_groups
[] = {
723 &memory_root_attr_group
,
728 * Initialize the sysfs support for memory devices...
730 int __init
memory_dev_init(void)
735 unsigned long block_sz
;
736 struct memory_block
*mem
= NULL
;
738 ret
= subsys_system_register(&memory_subsys
, memory_root_attr_groups
);
742 block_sz
= get_memory_block_size();
743 sections_per_block
= block_sz
/ MIN_MEMORY_BLOCK_SIZE
;
746 * Create entries for memory sections that were found
747 * during boot and have been initialized
749 for (i
= 0; i
< NR_MEM_SECTIONS
; i
++) {
750 if (!present_section_nr(i
))
752 /* don't need to reuse memory_block if only one per block */
753 err
= add_memory_section(0, __nr_to_section(i
),
754 (sections_per_block
== 1) ? NULL
: &mem
,
763 printk(KERN_ERR
"%s() failed: %d\n", __func__
, ret
);