1 // SPDX-License-Identifier: GPL-2.0
3 * sparse memory mappings.
6 #include <linux/slab.h>
7 #include <linux/mmzone.h>
8 #include <linux/memblock.h>
9 #include <linux/compiler.h>
10 #include <linux/highmem.h>
11 #include <linux/export.h>
12 #include <linux/spinlock.h>
13 #include <linux/vmalloc.h>
14 #include <linux/swap.h>
15 #include <linux/swapops.h>
16 #include <linux/bootmem_info.h>
17 #include <linux/vmstat.h>
22 * Permanent SPARSEMEM data:
24 * 1) mem_section - memory sections, mem_map's for valid memory
26 #ifdef CONFIG_SPARSEMEM_EXTREME
27 struct mem_section
**mem_section
;
29 struct mem_section mem_section
[NR_SECTION_ROOTS
][SECTIONS_PER_ROOT
]
30 ____cacheline_internodealigned_in_smp
;
32 EXPORT_SYMBOL(mem_section
);
34 #ifdef NODE_NOT_IN_PAGE_FLAGS
36 * If we did not store the node number in the page then we have to
37 * do a lookup in the section_to_node_table in order to find which
38 * node the page belongs to.
40 #if MAX_NUMNODES <= 256
41 static u8 section_to_node_table
[NR_MEM_SECTIONS
] __cacheline_aligned
;
43 static u16 section_to_node_table
[NR_MEM_SECTIONS
] __cacheline_aligned
;
46 int page_to_nid(const struct page
*page
)
48 return section_to_node_table
[page_to_section(page
)];
50 EXPORT_SYMBOL(page_to_nid
);
52 static void set_section_nid(unsigned long section_nr
, int nid
)
54 section_to_node_table
[section_nr
] = nid
;
56 #else /* !NODE_NOT_IN_PAGE_FLAGS */
57 static inline void set_section_nid(unsigned long section_nr
, int nid
)
62 #ifdef CONFIG_SPARSEMEM_EXTREME
63 static noinline
struct mem_section __ref
*sparse_index_alloc(int nid
)
65 struct mem_section
*section
= NULL
;
66 unsigned long array_size
= SECTIONS_PER_ROOT
*
67 sizeof(struct mem_section
);
69 if (slab_is_available()) {
70 section
= kzalloc_node(array_size
, GFP_KERNEL
, nid
);
72 section
= memblock_alloc_node(array_size
, SMP_CACHE_BYTES
,
75 panic("%s: Failed to allocate %lu bytes nid=%d\n",
76 __func__
, array_size
, nid
);
82 static int __meminit
sparse_index_init(unsigned long section_nr
, int nid
)
84 unsigned long root
= SECTION_NR_TO_ROOT(section_nr
);
85 struct mem_section
*section
;
88 * An existing section is possible in the sub-section hotplug
89 * case. First hot-add instantiates, follow-on hot-add reuses
90 * the existing section.
92 * The mem_hotplug_lock resolves the apparent race below.
94 if (mem_section
[root
])
97 section
= sparse_index_alloc(nid
);
101 mem_section
[root
] = section
;
105 #else /* !SPARSEMEM_EXTREME */
106 static inline int sparse_index_init(unsigned long section_nr
, int nid
)
113 * During early boot, before section_mem_map is used for an actual
114 * mem_map, we use section_mem_map to store the section's NUMA
115 * node. This keeps us from having to use another data structure. The
116 * node information is cleared just before we store the real mem_map.
118 static inline unsigned long sparse_encode_early_nid(int nid
)
120 return ((unsigned long)nid
<< SECTION_NID_SHIFT
);
123 static inline int sparse_early_nid(struct mem_section
*section
)
125 return (section
->section_mem_map
>> SECTION_NID_SHIFT
);
128 /* Validate the physical addressing limitations of the model */
129 static void __meminit
mminit_validate_memmodel_limits(unsigned long *start_pfn
,
130 unsigned long *end_pfn
)
132 unsigned long max_sparsemem_pfn
= 1UL << (MAX_PHYSMEM_BITS
-PAGE_SHIFT
);
135 * Sanity checks - do not allow an architecture to pass
136 * in larger pfns than the maximum scope of sparsemem:
138 if (*start_pfn
> max_sparsemem_pfn
) {
139 mminit_dprintk(MMINIT_WARNING
, "pfnvalidation",
140 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
141 *start_pfn
, *end_pfn
, max_sparsemem_pfn
);
143 *start_pfn
= max_sparsemem_pfn
;
144 *end_pfn
= max_sparsemem_pfn
;
145 } else if (*end_pfn
> max_sparsemem_pfn
) {
146 mminit_dprintk(MMINIT_WARNING
, "pfnvalidation",
147 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
148 *start_pfn
, *end_pfn
, max_sparsemem_pfn
);
150 *end_pfn
= max_sparsemem_pfn
;
155 * There are a number of times that we loop over NR_MEM_SECTIONS,
156 * looking for section_present() on each. But, when we have very
157 * large physical address spaces, NR_MEM_SECTIONS can also be
158 * very large which makes the loops quite long.
160 * Keeping track of this gives us an easy way to break out of
163 unsigned long __highest_present_section_nr
;
164 static void __section_mark_present(struct mem_section
*ms
,
165 unsigned long section_nr
)
167 if (section_nr
> __highest_present_section_nr
)
168 __highest_present_section_nr
= section_nr
;
170 ms
->section_mem_map
|= SECTION_MARKED_PRESENT
;
173 #define for_each_present_section_nr(start, section_nr) \
174 for (section_nr = next_present_section_nr(start-1); \
176 section_nr = next_present_section_nr(section_nr))
178 static inline unsigned long first_present_section_nr(void)
180 return next_present_section_nr(-1);
183 #ifdef CONFIG_SPARSEMEM_VMEMMAP
184 static void subsection_mask_set(unsigned long *map
, unsigned long pfn
,
185 unsigned long nr_pages
)
187 int idx
= subsection_map_index(pfn
);
188 int end
= subsection_map_index(pfn
+ nr_pages
- 1);
190 bitmap_set(map
, idx
, end
- idx
+ 1);
193 void __init
subsection_map_init(unsigned long pfn
, unsigned long nr_pages
)
195 int end_sec_nr
= pfn_to_section_nr(pfn
+ nr_pages
- 1);
196 unsigned long nr
, start_sec_nr
= pfn_to_section_nr(pfn
);
198 for (nr
= start_sec_nr
; nr
<= end_sec_nr
; nr
++) {
199 struct mem_section
*ms
;
202 pfns
= min(nr_pages
, PAGES_PER_SECTION
203 - (pfn
& ~PAGE_SECTION_MASK
));
204 ms
= __nr_to_section(nr
);
205 subsection_mask_set(ms
->usage
->subsection_map
, pfn
, pfns
);
207 pr_debug("%s: sec: %lu pfns: %lu set(%d, %d)\n", __func__
, nr
,
208 pfns
, subsection_map_index(pfn
),
209 subsection_map_index(pfn
+ pfns
- 1));
216 void __init
subsection_map_init(unsigned long pfn
, unsigned long nr_pages
)
221 /* Record a memory area against a node. */
222 static void __init
memory_present(int nid
, unsigned long start
, unsigned long end
)
226 start
&= PAGE_SECTION_MASK
;
227 mminit_validate_memmodel_limits(&start
, &end
);
228 for (pfn
= start
; pfn
< end
; pfn
+= PAGES_PER_SECTION
) {
229 unsigned long section_nr
= pfn_to_section_nr(pfn
);
230 struct mem_section
*ms
;
232 sparse_index_init(section_nr
, nid
);
233 set_section_nid(section_nr
, nid
);
235 ms
= __nr_to_section(section_nr
);
236 if (!ms
->section_mem_map
) {
237 ms
->section_mem_map
= sparse_encode_early_nid(nid
) |
239 __section_mark_present(ms
, section_nr
);
245 * Mark all memblocks as present using memory_present().
246 * This is a convenience function that is useful to mark all of the systems
247 * memory as present during initialization.
249 static void __init
memblocks_present(void)
251 unsigned long start
, end
;
254 #ifdef CONFIG_SPARSEMEM_EXTREME
255 if (unlikely(!mem_section
)) {
256 unsigned long size
, align
;
258 size
= sizeof(struct mem_section
*) * NR_SECTION_ROOTS
;
259 align
= 1 << (INTERNODE_CACHE_SHIFT
);
260 mem_section
= memblock_alloc(size
, align
);
262 panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
263 __func__
, size
, align
);
267 for_each_mem_pfn_range(i
, MAX_NUMNODES
, &start
, &end
, &nid
)
268 memory_present(nid
, start
, end
);
272 * Subtle, we encode the real pfn into the mem_map such that
273 * the identity pfn - section_mem_map will return the actual
274 * physical page frame number.
276 static unsigned long sparse_encode_mem_map(struct page
*mem_map
, unsigned long pnum
)
278 unsigned long coded_mem_map
=
279 (unsigned long)(mem_map
- (section_nr_to_pfn(pnum
)));
280 BUILD_BUG_ON(SECTION_MAP_LAST_BIT
> PFN_SECTION_SHIFT
);
281 BUG_ON(coded_mem_map
& ~SECTION_MAP_MASK
);
282 return coded_mem_map
;
285 #ifdef CONFIG_MEMORY_HOTPLUG
287 * Decode mem_map from the coded memmap
289 struct page
*sparse_decode_mem_map(unsigned long coded_mem_map
, unsigned long pnum
)
291 /* mask off the extra low bits of information */
292 coded_mem_map
&= SECTION_MAP_MASK
;
293 return ((struct page
*)coded_mem_map
) + section_nr_to_pfn(pnum
);
295 #endif /* CONFIG_MEMORY_HOTPLUG */
297 static void __meminit
sparse_init_one_section(struct mem_section
*ms
,
298 unsigned long pnum
, struct page
*mem_map
,
299 struct mem_section_usage
*usage
, unsigned long flags
)
301 ms
->section_mem_map
&= ~SECTION_MAP_MASK
;
302 ms
->section_mem_map
|= sparse_encode_mem_map(mem_map
, pnum
)
303 | SECTION_HAS_MEM_MAP
| flags
;
307 static unsigned long usemap_size(void)
309 return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS
) * sizeof(unsigned long);
312 size_t mem_section_usage_size(void)
314 return sizeof(struct mem_section_usage
) + usemap_size();
317 #ifdef CONFIG_MEMORY_HOTREMOVE
318 static inline phys_addr_t
pgdat_to_phys(struct pglist_data
*pgdat
)
321 VM_BUG_ON(pgdat
!= &contig_page_data
);
322 return __pa_symbol(&contig_page_data
);
328 static struct mem_section_usage
* __init
329 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data
*pgdat
,
332 struct mem_section_usage
*usage
;
333 unsigned long goal
, limit
;
336 * A page may contain usemaps for other sections preventing the
337 * page being freed and making a section unremovable while
338 * other sections referencing the usemap remain active. Similarly,
339 * a pgdat can prevent a section being removed. If section A
340 * contains a pgdat and section B contains the usemap, both
341 * sections become inter-dependent. This allocates usemaps
342 * from the same section as the pgdat where possible to avoid
345 goal
= pgdat_to_phys(pgdat
) & (PAGE_SECTION_MASK
<< PAGE_SHIFT
);
346 limit
= goal
+ (1UL << PA_SECTION_SHIFT
);
347 nid
= early_pfn_to_nid(goal
>> PAGE_SHIFT
);
349 usage
= memblock_alloc_try_nid(size
, SMP_CACHE_BYTES
, goal
, limit
, nid
);
350 if (!usage
&& limit
) {
351 limit
= MEMBLOCK_ALLOC_ACCESSIBLE
;
357 static void __init
check_usemap_section_nr(int nid
,
358 struct mem_section_usage
*usage
)
360 unsigned long usemap_snr
, pgdat_snr
;
361 static unsigned long old_usemap_snr
;
362 static unsigned long old_pgdat_snr
;
363 struct pglist_data
*pgdat
= NODE_DATA(nid
);
367 if (!old_usemap_snr
) {
368 old_usemap_snr
= NR_MEM_SECTIONS
;
369 old_pgdat_snr
= NR_MEM_SECTIONS
;
372 usemap_snr
= pfn_to_section_nr(__pa(usage
) >> PAGE_SHIFT
);
373 pgdat_snr
= pfn_to_section_nr(pgdat_to_phys(pgdat
) >> PAGE_SHIFT
);
374 if (usemap_snr
== pgdat_snr
)
377 if (old_usemap_snr
== usemap_snr
&& old_pgdat_snr
== pgdat_snr
)
378 /* skip redundant message */
381 old_usemap_snr
= usemap_snr
;
382 old_pgdat_snr
= pgdat_snr
;
384 usemap_nid
= sparse_early_nid(__nr_to_section(usemap_snr
));
385 if (usemap_nid
!= nid
) {
386 pr_info("node %d must be removed before remove section %ld\n",
391 * There is a circular dependency.
392 * Some platforms allow un-removable section because they will just
393 * gather other removable sections for dynamic partitioning.
394 * Just notify un-removable section's number here.
396 pr_info("Section %ld and %ld (node %d) have a circular dependency on usemap and pgdat allocations\n",
397 usemap_snr
, pgdat_snr
, nid
);
400 static struct mem_section_usage
* __init
401 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data
*pgdat
,
404 return memblock_alloc_node(size
, SMP_CACHE_BYTES
, pgdat
->node_id
);
407 static void __init
check_usemap_section_nr(int nid
,
408 struct mem_section_usage
*usage
)
411 #endif /* CONFIG_MEMORY_HOTREMOVE */
413 #ifdef CONFIG_SPARSEMEM_VMEMMAP
414 static unsigned long __init
section_map_size(void)
416 return ALIGN(sizeof(struct page
) * PAGES_PER_SECTION
, PMD_SIZE
);
420 static unsigned long __init
section_map_size(void)
422 return PAGE_ALIGN(sizeof(struct page
) * PAGES_PER_SECTION
);
425 struct page __init
*__populate_section_memmap(unsigned long pfn
,
426 unsigned long nr_pages
, int nid
, struct vmem_altmap
*altmap
,
427 struct dev_pagemap
*pgmap
)
429 unsigned long size
= section_map_size();
430 struct page
*map
= sparse_buffer_alloc(size
);
431 phys_addr_t addr
= __pa(MAX_DMA_ADDRESS
);
436 map
= memmap_alloc(size
, size
, addr
, nid
, false);
438 panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n",
439 __func__
, size
, PAGE_SIZE
, nid
, &addr
);
443 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
445 static void *sparsemap_buf __meminitdata
;
446 static void *sparsemap_buf_end __meminitdata
;
448 static inline void __meminit
sparse_buffer_free(unsigned long size
)
450 WARN_ON(!sparsemap_buf
|| size
== 0);
451 memblock_free(sparsemap_buf
, size
);
454 static void __init
sparse_buffer_init(unsigned long size
, int nid
)
456 phys_addr_t addr
= __pa(MAX_DMA_ADDRESS
);
457 WARN_ON(sparsemap_buf
); /* forgot to call sparse_buffer_fini()? */
459 * Pre-allocated buffer is mainly used by __populate_section_memmap
460 * and we want it to be properly aligned to the section size - this is
461 * especially the case for VMEMMAP which maps memmap to PMDs
463 sparsemap_buf
= memmap_alloc(size
, section_map_size(), addr
, nid
, true);
464 sparsemap_buf_end
= sparsemap_buf
+ size
;
465 #ifndef CONFIG_SPARSEMEM_VMEMMAP
466 mod_node_early_perpage_metadata(nid
, DIV_ROUND_UP(size
, PAGE_SIZE
));
470 static void __init
sparse_buffer_fini(void)
472 unsigned long size
= sparsemap_buf_end
- sparsemap_buf
;
474 if (sparsemap_buf
&& size
> 0)
475 sparse_buffer_free(size
);
476 sparsemap_buf
= NULL
;
479 void * __meminit
sparse_buffer_alloc(unsigned long size
)
484 ptr
= (void *) roundup((unsigned long)sparsemap_buf
, size
);
485 if (ptr
+ size
> sparsemap_buf_end
)
488 /* Free redundant aligned space */
489 if ((unsigned long)(ptr
- sparsemap_buf
) > 0)
490 sparse_buffer_free((unsigned long)(ptr
- sparsemap_buf
));
491 sparsemap_buf
= ptr
+ size
;
497 void __weak __meminit
vmemmap_populate_print_last(void)
502 * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end)
503 * And number of present sections in this node is map_count.
505 static void __init
sparse_init_nid(int nid
, unsigned long pnum_begin
,
506 unsigned long pnum_end
,
507 unsigned long map_count
)
509 struct mem_section_usage
*usage
;
513 usage
= sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nid
),
514 mem_section_usage_size() * map_count
);
516 pr_err("%s: node[%d] usemap allocation failed", __func__
, nid
);
519 sparse_buffer_init(map_count
* section_map_size(), nid
);
520 for_each_present_section_nr(pnum_begin
, pnum
) {
521 unsigned long pfn
= section_nr_to_pfn(pnum
);
523 if (pnum
>= pnum_end
)
526 map
= __populate_section_memmap(pfn
, PAGES_PER_SECTION
,
529 pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.",
532 sparse_buffer_fini();
535 check_usemap_section_nr(nid
, usage
);
536 sparse_init_one_section(__nr_to_section(pnum
), pnum
, map
, usage
,
538 usage
= (void *) usage
+ mem_section_usage_size();
540 sparse_buffer_fini();
543 /* We failed to allocate, mark all the following pnums as not present */
544 for_each_present_section_nr(pnum_begin
, pnum
) {
545 struct mem_section
*ms
;
547 if (pnum
>= pnum_end
)
549 ms
= __nr_to_section(pnum
);
550 ms
->section_mem_map
= 0;
555 * Allocate the accumulated non-linear sections, allocate a mem_map
556 * for each and record the physical to section mapping.
558 void __init
sparse_init(void)
560 unsigned long pnum_end
, pnum_begin
, map_count
= 1;
563 /* see include/linux/mmzone.h 'struct mem_section' definition */
564 BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section
)));
567 pnum_begin
= first_present_section_nr();
568 nid_begin
= sparse_early_nid(__nr_to_section(pnum_begin
));
570 /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */
571 set_pageblock_order();
573 for_each_present_section_nr(pnum_begin
+ 1, pnum_end
) {
574 int nid
= sparse_early_nid(__nr_to_section(pnum_end
));
576 if (nid
== nid_begin
) {
580 /* Init node with sections in range [pnum_begin, pnum_end) */
581 sparse_init_nid(nid_begin
, pnum_begin
, pnum_end
, map_count
);
583 pnum_begin
= pnum_end
;
586 /* cover the last node */
587 sparse_init_nid(nid_begin
, pnum_begin
, pnum_end
, map_count
);
588 vmemmap_populate_print_last();
591 #ifdef CONFIG_MEMORY_HOTPLUG
593 /* Mark all memory sections within the pfn range as online */
594 void online_mem_sections(unsigned long start_pfn
, unsigned long end_pfn
)
598 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
599 unsigned long section_nr
= pfn_to_section_nr(pfn
);
600 struct mem_section
*ms
;
602 /* onlining code should never touch invalid ranges */
603 if (WARN_ON(!valid_section_nr(section_nr
)))
606 ms
= __nr_to_section(section_nr
);
607 ms
->section_mem_map
|= SECTION_IS_ONLINE
;
611 /* Mark all memory sections within the pfn range as offline */
612 void offline_mem_sections(unsigned long start_pfn
, unsigned long end_pfn
)
616 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
617 unsigned long section_nr
= pfn_to_section_nr(pfn
);
618 struct mem_section
*ms
;
621 * TODO this needs some double checking. Offlining code makes
622 * sure to check pfn_valid but those checks might be just bogus
624 if (WARN_ON(!valid_section_nr(section_nr
)))
627 ms
= __nr_to_section(section_nr
);
628 ms
->section_mem_map
&= ~SECTION_IS_ONLINE
;
632 #ifdef CONFIG_SPARSEMEM_VMEMMAP
633 static struct page
* __meminit
populate_section_memmap(unsigned long pfn
,
634 unsigned long nr_pages
, int nid
, struct vmem_altmap
*altmap
,
635 struct dev_pagemap
*pgmap
)
637 return __populate_section_memmap(pfn
, nr_pages
, nid
, altmap
, pgmap
);
640 static void depopulate_section_memmap(unsigned long pfn
, unsigned long nr_pages
,
641 struct vmem_altmap
*altmap
)
643 unsigned long start
= (unsigned long) pfn_to_page(pfn
);
644 unsigned long end
= start
+ nr_pages
* sizeof(struct page
);
646 mod_node_page_state(page_pgdat(pfn_to_page(pfn
)), NR_MEMMAP
,
647 -1L * (DIV_ROUND_UP(end
- start
, PAGE_SIZE
)));
648 vmemmap_free(start
, end
, altmap
);
650 static void free_map_bootmem(struct page
*memmap
)
652 unsigned long start
= (unsigned long)memmap
;
653 unsigned long end
= (unsigned long)(memmap
+ PAGES_PER_SECTION
);
655 vmemmap_free(start
, end
, NULL
);
658 static int clear_subsection_map(unsigned long pfn
, unsigned long nr_pages
)
660 DECLARE_BITMAP(map
, SUBSECTIONS_PER_SECTION
) = { 0 };
661 DECLARE_BITMAP(tmp
, SUBSECTIONS_PER_SECTION
) = { 0 };
662 struct mem_section
*ms
= __pfn_to_section(pfn
);
663 unsigned long *subsection_map
= ms
->usage
664 ? &ms
->usage
->subsection_map
[0] : NULL
;
666 subsection_mask_set(map
, pfn
, nr_pages
);
668 bitmap_and(tmp
, map
, subsection_map
, SUBSECTIONS_PER_SECTION
);
670 if (WARN(!subsection_map
|| !bitmap_equal(tmp
, map
, SUBSECTIONS_PER_SECTION
),
671 "section already deactivated (%#lx + %ld)\n",
675 bitmap_xor(subsection_map
, map
, subsection_map
, SUBSECTIONS_PER_SECTION
);
679 static bool is_subsection_map_empty(struct mem_section
*ms
)
681 return bitmap_empty(&ms
->usage
->subsection_map
[0],
682 SUBSECTIONS_PER_SECTION
);
685 static int fill_subsection_map(unsigned long pfn
, unsigned long nr_pages
)
687 struct mem_section
*ms
= __pfn_to_section(pfn
);
688 DECLARE_BITMAP(map
, SUBSECTIONS_PER_SECTION
) = { 0 };
689 unsigned long *subsection_map
;
692 subsection_mask_set(map
, pfn
, nr_pages
);
694 subsection_map
= &ms
->usage
->subsection_map
[0];
696 if (bitmap_empty(map
, SUBSECTIONS_PER_SECTION
))
698 else if (bitmap_intersects(map
, subsection_map
, SUBSECTIONS_PER_SECTION
))
701 bitmap_or(subsection_map
, map
, subsection_map
,
702 SUBSECTIONS_PER_SECTION
);
707 static struct page
* __meminit
populate_section_memmap(unsigned long pfn
,
708 unsigned long nr_pages
, int nid
, struct vmem_altmap
*altmap
,
709 struct dev_pagemap
*pgmap
)
711 return kvmalloc_node(array_size(sizeof(struct page
),
712 PAGES_PER_SECTION
), GFP_KERNEL
, nid
);
715 static void depopulate_section_memmap(unsigned long pfn
, unsigned long nr_pages
,
716 struct vmem_altmap
*altmap
)
718 kvfree(pfn_to_page(pfn
));
721 static void free_map_bootmem(struct page
*memmap
)
723 unsigned long maps_section_nr
, removing_section_nr
, i
;
724 unsigned long magic
, nr_pages
;
725 struct page
*page
= virt_to_page(memmap
);
727 nr_pages
= PAGE_ALIGN(PAGES_PER_SECTION
* sizeof(struct page
))
730 for (i
= 0; i
< nr_pages
; i
++, page
++) {
733 BUG_ON(magic
== NODE_INFO
);
735 maps_section_nr
= pfn_to_section_nr(page_to_pfn(page
));
736 removing_section_nr
= page_private(page
);
739 * When this function is called, the removing section is
740 * logical offlined state. This means all pages are isolated
741 * from page allocator. If removing section's memmap is placed
742 * on the same section, it must not be freed.
743 * If it is freed, page allocator may allocate it which will
744 * be removed physically soon.
746 if (maps_section_nr
!= removing_section_nr
)
747 put_page_bootmem(page
);
751 static int clear_subsection_map(unsigned long pfn
, unsigned long nr_pages
)
756 static bool is_subsection_map_empty(struct mem_section
*ms
)
761 static int fill_subsection_map(unsigned long pfn
, unsigned long nr_pages
)
765 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
768 * To deactivate a memory region, there are 3 cases to handle across
769 * two configurations (SPARSEMEM_VMEMMAP={y,n}):
771 * 1. deactivation of a partial hot-added section (only possible in
772 * the SPARSEMEM_VMEMMAP=y case).
773 * a) section was present at memory init.
774 * b) section was hot-added post memory init.
775 * 2. deactivation of a complete hot-added section.
776 * 3. deactivation of a complete section from memory init.
778 * For 1, when subsection_map does not empty we will not be freeing the
779 * usage map, but still need to free the vmemmap range.
781 * For 2 and 3, the SPARSEMEM_VMEMMAP={y,n} cases are unified
783 static void section_deactivate(unsigned long pfn
, unsigned long nr_pages
,
784 struct vmem_altmap
*altmap
)
786 struct mem_section
*ms
= __pfn_to_section(pfn
);
787 bool section_is_early
= early_section(ms
);
788 struct page
*memmap
= NULL
;
791 if (clear_subsection_map(pfn
, nr_pages
))
794 empty
= is_subsection_map_empty(ms
);
796 unsigned long section_nr
= pfn_to_section_nr(pfn
);
799 * Mark the section invalid so that valid_section()
800 * return false. This prevents code from dereferencing
803 ms
->section_mem_map
&= ~SECTION_HAS_MEM_MAP
;
806 * When removing an early section, the usage map is kept (as the
807 * usage maps of other sections fall into the same page). It
808 * will be re-used when re-adding the section - which is then no
809 * longer an early section. If the usage map is PageReserved, it
810 * was allocated during boot.
812 if (!PageReserved(virt_to_page(ms
->usage
))) {
813 kfree_rcu(ms
->usage
, rcu
);
814 WRITE_ONCE(ms
->usage
, NULL
);
816 memmap
= sparse_decode_mem_map(ms
->section_mem_map
, section_nr
);
820 * The memmap of early sections is always fully populated. See
821 * section_activate() and pfn_valid() .
823 if (!section_is_early
)
824 depopulate_section_memmap(pfn
, nr_pages
, altmap
);
826 free_map_bootmem(memmap
);
829 ms
->section_mem_map
= (unsigned long)NULL
;
832 static struct page
* __meminit
section_activate(int nid
, unsigned long pfn
,
833 unsigned long nr_pages
, struct vmem_altmap
*altmap
,
834 struct dev_pagemap
*pgmap
)
836 struct mem_section
*ms
= __pfn_to_section(pfn
);
837 struct mem_section_usage
*usage
= NULL
;
842 usage
= kzalloc(mem_section_usage_size(), GFP_KERNEL
);
844 return ERR_PTR(-ENOMEM
);
848 rc
= fill_subsection_map(pfn
, nr_pages
);
857 * The early init code does not consider partially populated
858 * initial sections, it simply assumes that memory will never be
859 * referenced. If we hot-add memory into such a section then we
860 * do not need to populate the memmap and can simply reuse what
863 if (nr_pages
< PAGES_PER_SECTION
&& early_section(ms
))
864 return pfn_to_page(pfn
);
866 memmap
= populate_section_memmap(pfn
, nr_pages
, nid
, altmap
, pgmap
);
868 section_deactivate(pfn
, nr_pages
, altmap
);
869 return ERR_PTR(-ENOMEM
);
876 * sparse_add_section - add a memory section, or populate an existing one
877 * @nid: The node to add section on
878 * @start_pfn: start pfn of the memory range
879 * @nr_pages: number of pfns to add in the section
880 * @altmap: alternate pfns to allocate the memmap backing store
881 * @pgmap: alternate compound page geometry for devmap mappings
883 * This is only intended for hotplug.
885 * Note that only VMEMMAP supports sub-section aligned hotplug,
886 * the proper alignment and size are gated by check_pfn_span().
891 * * -EEXIST - Section has been present.
892 * * -ENOMEM - Out of memory.
894 int __meminit
sparse_add_section(int nid
, unsigned long start_pfn
,
895 unsigned long nr_pages
, struct vmem_altmap
*altmap
,
896 struct dev_pagemap
*pgmap
)
898 unsigned long section_nr
= pfn_to_section_nr(start_pfn
);
899 struct mem_section
*ms
;
903 ret
= sparse_index_init(section_nr
, nid
);
907 memmap
= section_activate(nid
, start_pfn
, nr_pages
, altmap
, pgmap
);
909 return PTR_ERR(memmap
);
912 * Poison uninitialized struct pages in order to catch invalid flags
915 if (!altmap
|| !altmap
->inaccessible
)
916 page_init_poison(memmap
, sizeof(struct page
) * nr_pages
);
918 ms
= __nr_to_section(section_nr
);
919 set_section_nid(section_nr
, nid
);
920 __section_mark_present(ms
, section_nr
);
922 /* Align memmap to section boundary in the subsection case */
923 if (section_nr_to_pfn(section_nr
) != start_pfn
)
924 memmap
= pfn_to_page(section_nr_to_pfn(section_nr
));
925 sparse_init_one_section(ms
, section_nr
, memmap
, ms
->usage
, 0);
930 void sparse_remove_section(unsigned long pfn
, unsigned long nr_pages
,
931 struct vmem_altmap
*altmap
)
933 struct mem_section
*ms
= __pfn_to_section(pfn
);
935 if (WARN_ON_ONCE(!valid_section(ms
)))
938 section_deactivate(pfn
, nr_pages
, altmap
);
940 #endif /* CONFIG_MEMORY_HOTPLUG */