1 // SPDX-License-Identifier: GPL-2.0
3 * sparse memory mappings.
6 #include <linux/slab.h>
7 #include <linux/mmzone.h>
8 #include <linux/bootmem.h>
9 #include <linux/compiler.h>
10 #include <linux/highmem.h>
11 #include <linux/export.h>
12 #include <linux/spinlock.h>
13 #include <linux/vmalloc.h>
17 #include <asm/pgalloc.h>
18 #include <asm/pgtable.h>
21 * Permanent SPARSEMEM data:
23 * 1) mem_section - memory sections, mem_map's for valid memory
25 #ifdef CONFIG_SPARSEMEM_EXTREME
26 struct mem_section
*mem_section
[NR_SECTION_ROOTS
]
27 ____cacheline_internodealigned_in_smp
;
29 struct mem_section mem_section
[NR_SECTION_ROOTS
][SECTIONS_PER_ROOT
]
30 ____cacheline_internodealigned_in_smp
;
32 EXPORT_SYMBOL(mem_section
);
34 #ifdef NODE_NOT_IN_PAGE_FLAGS
36 * If we did not store the node number in the page then we have to
37 * do a lookup in the section_to_node_table in order to find which
38 * node the page belongs to.
40 #if MAX_NUMNODES <= 256
41 static u8 section_to_node_table
[NR_MEM_SECTIONS
] __cacheline_aligned
;
43 static u16 section_to_node_table
[NR_MEM_SECTIONS
] __cacheline_aligned
;
46 int page_to_nid(const struct page
*page
)
48 return section_to_node_table
[page_to_section(page
)];
50 EXPORT_SYMBOL(page_to_nid
);
52 static void set_section_nid(unsigned long section_nr
, int nid
)
54 section_to_node_table
[section_nr
] = nid
;
56 #else /* !NODE_NOT_IN_PAGE_FLAGS */
57 static inline void set_section_nid(unsigned long section_nr
, int nid
)
62 #ifdef CONFIG_SPARSEMEM_EXTREME
63 static noinline
struct mem_section __ref
*sparse_index_alloc(int nid
)
65 struct mem_section
*section
= NULL
;
66 unsigned long array_size
= SECTIONS_PER_ROOT
*
67 sizeof(struct mem_section
);
69 if (slab_is_available())
70 section
= kzalloc_node(array_size
, GFP_KERNEL
, nid
);
72 section
= memblock_virt_alloc_node(array_size
, nid
);
77 static int __meminit
sparse_index_init(unsigned long section_nr
, int nid
)
79 unsigned long root
= SECTION_NR_TO_ROOT(section_nr
);
80 struct mem_section
*section
;
82 if (mem_section
[root
])
85 section
= sparse_index_alloc(nid
);
89 mem_section
[root
] = section
;
93 #else /* !SPARSEMEM_EXTREME */
94 static inline int sparse_index_init(unsigned long section_nr
, int nid
)
100 #ifdef CONFIG_SPARSEMEM_EXTREME
101 int __section_nr(struct mem_section
* ms
)
103 unsigned long root_nr
;
104 struct mem_section
* root
;
106 for (root_nr
= 0; root_nr
< NR_SECTION_ROOTS
; root_nr
++) {
107 root
= __nr_to_section(root_nr
* SECTIONS_PER_ROOT
);
111 if ((ms
>= root
) && (ms
< (root
+ SECTIONS_PER_ROOT
)))
115 VM_BUG_ON(root_nr
== NR_SECTION_ROOTS
);
117 return (root_nr
* SECTIONS_PER_ROOT
) + (ms
- root
);
120 int __section_nr(struct mem_section
* ms
)
122 return (int)(ms
- mem_section
[0]);
127 * During early boot, before section_mem_map is used for an actual
128 * mem_map, we use section_mem_map to store the section's NUMA
129 * node. This keeps us from having to use another data structure. The
130 * node information is cleared just before we store the real mem_map.
132 static inline unsigned long sparse_encode_early_nid(int nid
)
134 return (nid
<< SECTION_NID_SHIFT
);
137 static inline int sparse_early_nid(struct mem_section
*section
)
139 return (section
->section_mem_map
>> SECTION_NID_SHIFT
);
142 /* Validate the physical addressing limitations of the model */
143 void __meminit
mminit_validate_memmodel_limits(unsigned long *start_pfn
,
144 unsigned long *end_pfn
)
146 unsigned long max_sparsemem_pfn
= 1UL << (MAX_PHYSMEM_BITS
-PAGE_SHIFT
);
149 * Sanity checks - do not allow an architecture to pass
150 * in larger pfns than the maximum scope of sparsemem:
152 if (*start_pfn
> max_sparsemem_pfn
) {
153 mminit_dprintk(MMINIT_WARNING
, "pfnvalidation",
154 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
155 *start_pfn
, *end_pfn
, max_sparsemem_pfn
);
157 *start_pfn
= max_sparsemem_pfn
;
158 *end_pfn
= max_sparsemem_pfn
;
159 } else if (*end_pfn
> max_sparsemem_pfn
) {
160 mminit_dprintk(MMINIT_WARNING
, "pfnvalidation",
161 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
162 *start_pfn
, *end_pfn
, max_sparsemem_pfn
);
164 *end_pfn
= max_sparsemem_pfn
;
169 * There are a number of times that we loop over NR_MEM_SECTIONS,
170 * looking for section_present() on each. But, when we have very
171 * large physical address spaces, NR_MEM_SECTIONS can also be
172 * very large which makes the loops quite long.
174 * Keeping track of this gives us an easy way to break out of
177 int __highest_present_section_nr
;
178 static void section_mark_present(struct mem_section
*ms
)
180 int section_nr
= __section_nr(ms
);
182 if (section_nr
> __highest_present_section_nr
)
183 __highest_present_section_nr
= section_nr
;
185 ms
->section_mem_map
|= SECTION_MARKED_PRESENT
;
188 static inline int next_present_section_nr(int section_nr
)
192 if (present_section_nr(section_nr
))
194 } while ((section_nr
< NR_MEM_SECTIONS
) &&
195 (section_nr
<= __highest_present_section_nr
));
199 #define for_each_present_section_nr(start, section_nr) \
200 for (section_nr = next_present_section_nr(start-1); \
201 ((section_nr >= 0) && \
202 (section_nr < NR_MEM_SECTIONS) && \
203 (section_nr <= __highest_present_section_nr)); \
204 section_nr = next_present_section_nr(section_nr))
206 /* Record a memory area against a node. */
207 void __init
memory_present(int nid
, unsigned long start
, unsigned long end
)
211 start
&= PAGE_SECTION_MASK
;
212 mminit_validate_memmodel_limits(&start
, &end
);
213 for (pfn
= start
; pfn
< end
; pfn
+= PAGES_PER_SECTION
) {
214 unsigned long section
= pfn_to_section_nr(pfn
);
215 struct mem_section
*ms
;
217 sparse_index_init(section
, nid
);
218 set_section_nid(section
, nid
);
220 ms
= __nr_to_section(section
);
221 if (!ms
->section_mem_map
) {
222 ms
->section_mem_map
= sparse_encode_early_nid(nid
) |
224 section_mark_present(ms
);
230 * Only used by the i386 NUMA architecures, but relatively
233 unsigned long __init
node_memmap_size_bytes(int nid
, unsigned long start_pfn
,
234 unsigned long end_pfn
)
237 unsigned long nr_pages
= 0;
239 mminit_validate_memmodel_limits(&start_pfn
, &end_pfn
);
240 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
241 if (nid
!= early_pfn_to_nid(pfn
))
244 if (pfn_present(pfn
))
245 nr_pages
+= PAGES_PER_SECTION
;
248 return nr_pages
* sizeof(struct page
);
252 * Subtle, we encode the real pfn into the mem_map such that
253 * the identity pfn - section_mem_map will return the actual
254 * physical page frame number.
256 static unsigned long sparse_encode_mem_map(struct page
*mem_map
, unsigned long pnum
)
258 return (unsigned long)(mem_map
- (section_nr_to_pfn(pnum
)));
262 * Decode mem_map from the coded memmap
264 struct page
*sparse_decode_mem_map(unsigned long coded_mem_map
, unsigned long pnum
)
266 /* mask off the extra low bits of information */
267 coded_mem_map
&= SECTION_MAP_MASK
;
268 return ((struct page
*)coded_mem_map
) + section_nr_to_pfn(pnum
);
271 static int __meminit
sparse_init_one_section(struct mem_section
*ms
,
272 unsigned long pnum
, struct page
*mem_map
,
273 unsigned long *pageblock_bitmap
)
275 if (!present_section(ms
))
278 ms
->section_mem_map
&= ~SECTION_MAP_MASK
;
279 ms
->section_mem_map
|= sparse_encode_mem_map(mem_map
, pnum
) |
281 ms
->pageblock_flags
= pageblock_bitmap
;
286 unsigned long usemap_size(void)
288 return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS
) * sizeof(unsigned long);
291 #ifdef CONFIG_MEMORY_HOTPLUG
292 static unsigned long *__kmalloc_section_usemap(void)
294 return kmalloc(usemap_size(), GFP_KERNEL
);
296 #endif /* CONFIG_MEMORY_HOTPLUG */
298 #ifdef CONFIG_MEMORY_HOTREMOVE
299 static unsigned long * __init
300 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data
*pgdat
,
303 unsigned long goal
, limit
;
307 * A page may contain usemaps for other sections preventing the
308 * page being freed and making a section unremovable while
309 * other sections referencing the usemap remain active. Similarly,
310 * a pgdat can prevent a section being removed. If section A
311 * contains a pgdat and section B contains the usemap, both
312 * sections become inter-dependent. This allocates usemaps
313 * from the same section as the pgdat where possible to avoid
316 goal
= __pa(pgdat
) & (PAGE_SECTION_MASK
<< PAGE_SHIFT
);
317 limit
= goal
+ (1UL << PA_SECTION_SHIFT
);
318 nid
= early_pfn_to_nid(goal
>> PAGE_SHIFT
);
320 p
= memblock_virt_alloc_try_nid_nopanic(size
,
321 SMP_CACHE_BYTES
, goal
, limit
,
330 static void __init
check_usemap_section_nr(int nid
, unsigned long *usemap
)
332 unsigned long usemap_snr
, pgdat_snr
;
333 static unsigned long old_usemap_snr
= NR_MEM_SECTIONS
;
334 static unsigned long old_pgdat_snr
= NR_MEM_SECTIONS
;
335 struct pglist_data
*pgdat
= NODE_DATA(nid
);
338 usemap_snr
= pfn_to_section_nr(__pa(usemap
) >> PAGE_SHIFT
);
339 pgdat_snr
= pfn_to_section_nr(__pa(pgdat
) >> PAGE_SHIFT
);
340 if (usemap_snr
== pgdat_snr
)
343 if (old_usemap_snr
== usemap_snr
&& old_pgdat_snr
== pgdat_snr
)
344 /* skip redundant message */
347 old_usemap_snr
= usemap_snr
;
348 old_pgdat_snr
= pgdat_snr
;
350 usemap_nid
= sparse_early_nid(__nr_to_section(usemap_snr
));
351 if (usemap_nid
!= nid
) {
352 pr_info("node %d must be removed before remove section %ld\n",
357 * There is a circular dependency.
358 * Some platforms allow un-removable section because they will just
359 * gather other removable sections for dynamic partitioning.
360 * Just notify un-removable section's number here.
362 pr_info("Section %ld and %ld (node %d) have a circular dependency on usemap and pgdat allocations\n",
363 usemap_snr
, pgdat_snr
, nid
);
366 static unsigned long * __init
367 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data
*pgdat
,
370 return memblock_virt_alloc_node_nopanic(size
, pgdat
->node_id
);
373 static void __init
check_usemap_section_nr(int nid
, unsigned long *usemap
)
376 #endif /* CONFIG_MEMORY_HOTREMOVE */
378 static void __init
sparse_early_usemaps_alloc_node(void *data
,
379 unsigned long pnum_begin
,
380 unsigned long pnum_end
,
381 unsigned long usemap_count
, int nodeid
)
385 unsigned long **usemap_map
= (unsigned long **)data
;
386 int size
= usemap_size();
388 usemap
= sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid
),
389 size
* usemap_count
);
391 pr_warn("%s: allocation failed\n", __func__
);
395 for (pnum
= pnum_begin
; pnum
< pnum_end
; pnum
++) {
396 if (!present_section_nr(pnum
))
398 usemap_map
[pnum
] = usemap
;
400 check_usemap_section_nr(nodeid
, usemap_map
[pnum
]);
404 #ifndef CONFIG_SPARSEMEM_VMEMMAP
405 struct page __init
*sparse_mem_map_populate(unsigned long pnum
, int nid
)
410 map
= alloc_remap(nid
, sizeof(struct page
) * PAGES_PER_SECTION
);
414 size
= PAGE_ALIGN(sizeof(struct page
) * PAGES_PER_SECTION
);
415 map
= memblock_virt_alloc_try_nid(size
,
416 PAGE_SIZE
, __pa(MAX_DMA_ADDRESS
),
417 BOOTMEM_ALLOC_ACCESSIBLE
, nid
);
420 void __init
sparse_mem_maps_populate_node(struct page
**map_map
,
421 unsigned long pnum_begin
,
422 unsigned long pnum_end
,
423 unsigned long map_count
, int nodeid
)
427 unsigned long size
= sizeof(struct page
) * PAGES_PER_SECTION
;
429 map
= alloc_remap(nodeid
, size
* map_count
);
431 for (pnum
= pnum_begin
; pnum
< pnum_end
; pnum
++) {
432 if (!present_section_nr(pnum
))
440 size
= PAGE_ALIGN(size
);
441 map
= memblock_virt_alloc_try_nid(size
* map_count
,
442 PAGE_SIZE
, __pa(MAX_DMA_ADDRESS
),
443 BOOTMEM_ALLOC_ACCESSIBLE
, nodeid
);
445 for (pnum
= pnum_begin
; pnum
< pnum_end
; pnum
++) {
446 if (!present_section_nr(pnum
))
455 for (pnum
= pnum_begin
; pnum
< pnum_end
; pnum
++) {
456 struct mem_section
*ms
;
458 if (!present_section_nr(pnum
))
460 map_map
[pnum
] = sparse_mem_map_populate(pnum
, nodeid
);
463 ms
= __nr_to_section(pnum
);
464 pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",
466 ms
->section_mem_map
= 0;
469 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
471 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
472 static void __init
sparse_early_mem_maps_alloc_node(void *data
,
473 unsigned long pnum_begin
,
474 unsigned long pnum_end
,
475 unsigned long map_count
, int nodeid
)
477 struct page
**map_map
= (struct page
**)data
;
478 sparse_mem_maps_populate_node(map_map
, pnum_begin
, pnum_end
,
482 static struct page __init
*sparse_early_mem_map_alloc(unsigned long pnum
)
485 struct mem_section
*ms
= __nr_to_section(pnum
);
486 int nid
= sparse_early_nid(ms
);
488 map
= sparse_mem_map_populate(pnum
, nid
);
492 pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",
494 ms
->section_mem_map
= 0;
499 void __weak __meminit
vmemmap_populate_print_last(void)
504 * alloc_usemap_and_memmap - memory alloction for pageblock flags and vmemmap
505 * @map: usemap_map for pageblock flags or mmap_map for vmemmap
507 static void __init
alloc_usemap_and_memmap(void (*alloc_func
)
508 (void *, unsigned long, unsigned long,
509 unsigned long, int), void *data
)
512 unsigned long map_count
;
513 int nodeid_begin
= 0;
514 unsigned long pnum_begin
= 0;
516 for_each_present_section_nr(0, pnum
) {
517 struct mem_section
*ms
;
519 ms
= __nr_to_section(pnum
);
520 nodeid_begin
= sparse_early_nid(ms
);
525 for_each_present_section_nr(pnum_begin
+ 1, pnum
) {
526 struct mem_section
*ms
;
529 ms
= __nr_to_section(pnum
);
530 nodeid
= sparse_early_nid(ms
);
531 if (nodeid
== nodeid_begin
) {
535 /* ok, we need to take cake of from pnum_begin to pnum - 1*/
536 alloc_func(data
, pnum_begin
, pnum
,
537 map_count
, nodeid_begin
);
538 /* new start, update count etc*/
539 nodeid_begin
= nodeid
;
544 alloc_func(data
, pnum_begin
, NR_MEM_SECTIONS
,
545 map_count
, nodeid_begin
);
549 * Allocate the accumulated non-linear sections, allocate a mem_map
550 * for each and record the physical to section mapping.
552 void __init
sparse_init(void)
556 unsigned long *usemap
;
557 unsigned long **usemap_map
;
559 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
561 struct page
**map_map
;
564 /* see include/linux/mmzone.h 'struct mem_section' definition */
565 BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section
)));
567 /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */
568 set_pageblock_order();
571 * map is using big page (aka 2M in x86 64 bit)
572 * usemap is less one page (aka 24 bytes)
573 * so alloc 2M (with 2M align) and 24 bytes in turn will
574 * make next 2M slip to one more 2M later.
575 * then in big system, the memory will have a lot of holes...
576 * here try to allocate 2M pages continuously.
578 * powerpc need to call sparse_init_one_section right after each
579 * sparse_early_mem_map_alloc, so allocate usemap_map at first.
581 size
= sizeof(unsigned long *) * NR_MEM_SECTIONS
;
582 usemap_map
= memblock_virt_alloc(size
, 0);
584 panic("can not allocate usemap_map\n");
585 alloc_usemap_and_memmap(sparse_early_usemaps_alloc_node
,
588 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
589 size2
= sizeof(struct page
*) * NR_MEM_SECTIONS
;
590 map_map
= memblock_virt_alloc(size2
, 0);
592 panic("can not allocate map_map\n");
593 alloc_usemap_and_memmap(sparse_early_mem_maps_alloc_node
,
597 for_each_present_section_nr(0, pnum
) {
598 usemap
= usemap_map
[pnum
];
602 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
605 map
= sparse_early_mem_map_alloc(pnum
);
610 sparse_init_one_section(__nr_to_section(pnum
), pnum
, map
,
614 vmemmap_populate_print_last();
616 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
617 memblock_free_early(__pa(map_map
), size2
);
619 memblock_free_early(__pa(usemap_map
), size
);
622 #ifdef CONFIG_MEMORY_HOTPLUG
624 /* Mark all memory sections within the pfn range as online */
625 void online_mem_sections(unsigned long start_pfn
, unsigned long end_pfn
)
629 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
630 unsigned long section_nr
= pfn_to_section_nr(pfn
);
631 struct mem_section
*ms
;
633 /* onlining code should never touch invalid ranges */
634 if (WARN_ON(!valid_section_nr(section_nr
)))
637 ms
= __nr_to_section(section_nr
);
638 ms
->section_mem_map
|= SECTION_IS_ONLINE
;
642 #ifdef CONFIG_MEMORY_HOTREMOVE
643 /* Mark all memory sections within the pfn range as online */
644 void offline_mem_sections(unsigned long start_pfn
, unsigned long end_pfn
)
648 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
649 unsigned long section_nr
= pfn_to_section_nr(start_pfn
);
650 struct mem_section
*ms
;
653 * TODO this needs some double checking. Offlining code makes
654 * sure to check pfn_valid but those checks might be just bogus
656 if (WARN_ON(!valid_section_nr(section_nr
)))
659 ms
= __nr_to_section(section_nr
);
660 ms
->section_mem_map
&= ~SECTION_IS_ONLINE
;
665 #ifdef CONFIG_SPARSEMEM_VMEMMAP
666 static inline struct page
*kmalloc_section_memmap(unsigned long pnum
, int nid
)
668 /* This will make the necessary allocations eventually. */
669 return sparse_mem_map_populate(pnum
, nid
);
671 static void __kfree_section_memmap(struct page
*memmap
)
673 unsigned long start
= (unsigned long)memmap
;
674 unsigned long end
= (unsigned long)(memmap
+ PAGES_PER_SECTION
);
676 vmemmap_free(start
, end
);
678 #ifdef CONFIG_MEMORY_HOTREMOVE
679 static void free_map_bootmem(struct page
*memmap
)
681 unsigned long start
= (unsigned long)memmap
;
682 unsigned long end
= (unsigned long)(memmap
+ PAGES_PER_SECTION
);
684 vmemmap_free(start
, end
);
686 #endif /* CONFIG_MEMORY_HOTREMOVE */
688 static struct page
*__kmalloc_section_memmap(void)
690 struct page
*page
, *ret
;
691 unsigned long memmap_size
= sizeof(struct page
) * PAGES_PER_SECTION
;
693 page
= alloc_pages(GFP_KERNEL
|__GFP_NOWARN
, get_order(memmap_size
));
697 ret
= vmalloc(memmap_size
);
703 ret
= (struct page
*)pfn_to_kaddr(page_to_pfn(page
));
709 static inline struct page
*kmalloc_section_memmap(unsigned long pnum
, int nid
)
711 return __kmalloc_section_memmap();
714 static void __kfree_section_memmap(struct page
*memmap
)
716 if (is_vmalloc_addr(memmap
))
719 free_pages((unsigned long)memmap
,
720 get_order(sizeof(struct page
) * PAGES_PER_SECTION
));
723 #ifdef CONFIG_MEMORY_HOTREMOVE
724 static void free_map_bootmem(struct page
*memmap
)
726 unsigned long maps_section_nr
, removing_section_nr
, i
;
727 unsigned long magic
, nr_pages
;
728 struct page
*page
= virt_to_page(memmap
);
730 nr_pages
= PAGE_ALIGN(PAGES_PER_SECTION
* sizeof(struct page
))
733 for (i
= 0; i
< nr_pages
; i
++, page
++) {
734 magic
= (unsigned long) page
->freelist
;
736 BUG_ON(magic
== NODE_INFO
);
738 maps_section_nr
= pfn_to_section_nr(page_to_pfn(page
));
739 removing_section_nr
= page_private(page
);
742 * When this function is called, the removing section is
743 * logical offlined state. This means all pages are isolated
744 * from page allocator. If removing section's memmap is placed
745 * on the same section, it must not be freed.
746 * If it is freed, page allocator may allocate it which will
747 * be removed physically soon.
749 if (maps_section_nr
!= removing_section_nr
)
750 put_page_bootmem(page
);
753 #endif /* CONFIG_MEMORY_HOTREMOVE */
754 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
757 * returns the number of sections whose mem_maps were properly
758 * set. If this is <=0, then that means that the passed-in
759 * map was not consumed and must be freed.
761 int __meminit
sparse_add_one_section(struct pglist_data
*pgdat
, unsigned long start_pfn
)
763 unsigned long section_nr
= pfn_to_section_nr(start_pfn
);
764 struct mem_section
*ms
;
766 unsigned long *usemap
;
771 * no locking for this, because it does its own
772 * plus, it does a kmalloc
774 ret
= sparse_index_init(section_nr
, pgdat
->node_id
);
775 if (ret
< 0 && ret
!= -EEXIST
)
777 memmap
= kmalloc_section_memmap(section_nr
, pgdat
->node_id
);
780 usemap
= __kmalloc_section_usemap();
782 __kfree_section_memmap(memmap
);
786 pgdat_resize_lock(pgdat
, &flags
);
788 ms
= __pfn_to_section(start_pfn
);
789 if (ms
->section_mem_map
& SECTION_MARKED_PRESENT
) {
794 memset(memmap
, 0, sizeof(struct page
) * PAGES_PER_SECTION
);
796 section_mark_present(ms
);
798 ret
= sparse_init_one_section(ms
, section_nr
, memmap
, usemap
);
801 pgdat_resize_unlock(pgdat
, &flags
);
804 __kfree_section_memmap(memmap
);
809 #ifdef CONFIG_MEMORY_HOTREMOVE
810 #ifdef CONFIG_MEMORY_FAILURE
811 static void clear_hwpoisoned_pages(struct page
*memmap
, int nr_pages
)
818 for (i
= 0; i
< nr_pages
; i
++) {
819 if (PageHWPoison(&memmap
[i
])) {
820 atomic_long_sub(1, &num_poisoned_pages
);
821 ClearPageHWPoison(&memmap
[i
]);
826 static inline void clear_hwpoisoned_pages(struct page
*memmap
, int nr_pages
)
831 static void free_section_usemap(struct page
*memmap
, unsigned long *usemap
)
833 struct page
*usemap_page
;
838 usemap_page
= virt_to_page(usemap
);
840 * Check to see if allocation came from hot-plug-add
842 if (PageSlab(usemap_page
) || PageCompound(usemap_page
)) {
845 __kfree_section_memmap(memmap
);
850 * The usemap came from bootmem. This is packed with other usemaps
851 * on the section which has pgdat at boot time. Just keep it as is now.
855 free_map_bootmem(memmap
);
858 void sparse_remove_one_section(struct zone
*zone
, struct mem_section
*ms
,
859 unsigned long map_offset
)
861 struct page
*memmap
= NULL
;
862 unsigned long *usemap
= NULL
, flags
;
863 struct pglist_data
*pgdat
= zone
->zone_pgdat
;
865 pgdat_resize_lock(pgdat
, &flags
);
866 if (ms
->section_mem_map
) {
867 usemap
= ms
->pageblock_flags
;
868 memmap
= sparse_decode_mem_map(ms
->section_mem_map
,
870 ms
->section_mem_map
= 0;
871 ms
->pageblock_flags
= NULL
;
873 pgdat_resize_unlock(pgdat
, &flags
);
875 clear_hwpoisoned_pages(memmap
+ map_offset
,
876 PAGES_PER_SECTION
- map_offset
);
877 free_section_usemap(memmap
, usemap
);
879 #endif /* CONFIG_MEMORY_HOTREMOVE */
880 #endif /* CONFIG_MEMORY_HOTPLUG */