2 * sparse memory mappings.
4 #include <linux/config.h>
6 #include <linux/mmzone.h>
7 #include <linux/bootmem.h>
8 #include <linux/highmem.h>
9 #include <linux/module.h>
10 #include <linux/spinlock.h>
11 #include <linux/vmalloc.h>
15 * Permanent SPARSEMEM data:
17 * 1) mem_section - memory sections, mem_map's for valid memory
19 #ifdef CONFIG_SPARSEMEM_EXTREME
20 struct mem_section
*mem_section
[NR_SECTION_ROOTS
]
21 ____cacheline_internodealigned_in_smp
;
23 struct mem_section mem_section
[NR_SECTION_ROOTS
][SECTIONS_PER_ROOT
]
24 ____cacheline_internodealigned_in_smp
;
26 EXPORT_SYMBOL(mem_section
);
28 #ifdef CONFIG_SPARSEMEM_EXTREME
29 static struct mem_section
*sparse_index_alloc(int nid
)
31 struct mem_section
*section
= NULL
;
32 unsigned long array_size
= SECTIONS_PER_ROOT
*
33 sizeof(struct mem_section
);
35 if (slab_is_available())
36 section
= kmalloc_node(array_size
, GFP_KERNEL
, nid
);
38 section
= alloc_bootmem_node(NODE_DATA(nid
), array_size
);
41 memset(section
, 0, array_size
);
46 static int sparse_index_init(unsigned long section_nr
, int nid
)
48 static spinlock_t index_init_lock
= SPIN_LOCK_UNLOCKED
;
49 unsigned long root
= SECTION_NR_TO_ROOT(section_nr
);
50 struct mem_section
*section
;
53 if (mem_section
[root
])
56 section
= sparse_index_alloc(nid
);
58 * This lock keeps two different sections from
59 * reallocating for the same index
61 spin_lock(&index_init_lock
);
63 if (mem_section
[root
]) {
68 mem_section
[root
] = section
;
70 spin_unlock(&index_init_lock
);
73 #else /* !SPARSEMEM_EXTREME */
74 static inline int sparse_index_init(unsigned long section_nr
, int nid
)
81 * Although written for the SPARSEMEM_EXTREME case, this happens
82 * to also work for the flat array case becase
83 * NR_SECTION_ROOTS==NR_MEM_SECTIONS.
85 int __section_nr(struct mem_section
* ms
)
87 unsigned long root_nr
;
88 struct mem_section
* root
;
90 for (root_nr
= 0; root_nr
< NR_SECTION_ROOTS
; root_nr
++) {
91 root
= __nr_to_section(root_nr
* SECTIONS_PER_ROOT
);
95 if ((ms
>= root
) && (ms
< (root
+ SECTIONS_PER_ROOT
)))
99 return (root_nr
* SECTIONS_PER_ROOT
) + (ms
- root
);
102 /* Record a memory area against a node. */
103 void memory_present(int nid
, unsigned long start
, unsigned long end
)
107 start
&= PAGE_SECTION_MASK
;
108 for (pfn
= start
; pfn
< end
; pfn
+= PAGES_PER_SECTION
) {
109 unsigned long section
= pfn_to_section_nr(pfn
);
110 struct mem_section
*ms
;
112 sparse_index_init(section
, nid
);
114 ms
= __nr_to_section(section
);
115 if (!ms
->section_mem_map
)
116 ms
->section_mem_map
= SECTION_MARKED_PRESENT
;
121 * Only used by the i386 NUMA architecures, but relatively
124 unsigned long __init
node_memmap_size_bytes(int nid
, unsigned long start_pfn
,
125 unsigned long end_pfn
)
128 unsigned long nr_pages
= 0;
130 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
131 if (nid
!= early_pfn_to_nid(pfn
))
135 nr_pages
+= PAGES_PER_SECTION
;
138 return nr_pages
* sizeof(struct page
);
142 * Subtle, we encode the real pfn into the mem_map such that
143 * the identity pfn - section_mem_map will return the actual
144 * physical page frame number.
146 static unsigned long sparse_encode_mem_map(struct page
*mem_map
, unsigned long pnum
)
148 return (unsigned long)(mem_map
- (section_nr_to_pfn(pnum
)));
152 * We need this if we ever free the mem_maps. While not implemented yet,
153 * this function is included for parity with its sibling.
155 static __attribute((unused
))
156 struct page
*sparse_decode_mem_map(unsigned long coded_mem_map
, unsigned long pnum
)
158 return ((struct page
*)coded_mem_map
) + section_nr_to_pfn(pnum
);
161 static int sparse_init_one_section(struct mem_section
*ms
,
162 unsigned long pnum
, struct page
*mem_map
)
164 if (!valid_section(ms
))
167 ms
->section_mem_map
|= sparse_encode_mem_map(mem_map
, pnum
);
172 static struct page
*sparse_early_mem_map_alloc(unsigned long pnum
)
175 int nid
= early_pfn_to_nid(section_nr_to_pfn(pnum
));
176 struct mem_section
*ms
= __nr_to_section(pnum
);
178 map
= alloc_remap(nid
, sizeof(struct page
) * PAGES_PER_SECTION
);
182 map
= alloc_bootmem_node(NODE_DATA(nid
),
183 sizeof(struct page
) * PAGES_PER_SECTION
);
187 printk(KERN_WARNING
"%s: allocation failed\n", __FUNCTION__
);
188 ms
->section_mem_map
= 0;
192 static struct page
*__kmalloc_section_memmap(unsigned long nr_pages
)
194 struct page
*page
, *ret
;
195 unsigned long memmap_size
= sizeof(struct page
) * nr_pages
;
197 page
= alloc_pages(GFP_KERNEL
, get_order(memmap_size
));
201 ret
= vmalloc(memmap_size
);
207 ret
= (struct page
*)pfn_to_kaddr(page_to_pfn(page
));
209 memset(ret
, 0, memmap_size
);
214 static int vaddr_in_vmalloc_area(void *addr
)
216 if (addr
>= (void *)VMALLOC_START
&&
217 addr
< (void *)VMALLOC_END
)
222 static void __kfree_section_memmap(struct page
*memmap
, unsigned long nr_pages
)
224 if (vaddr_in_vmalloc_area(memmap
))
227 free_pages((unsigned long)memmap
,
228 get_order(sizeof(struct page
) * nr_pages
));
232 * Allocate the accumulated non-linear sections, allocate a mem_map
233 * for each and record the physical to section mapping.
235 void sparse_init(void)
240 for (pnum
= 0; pnum
< NR_MEM_SECTIONS
; pnum
++) {
241 if (!valid_section_nr(pnum
))
244 map
= sparse_early_mem_map_alloc(pnum
);
247 sparse_init_one_section(__nr_to_section(pnum
), pnum
, map
);
252 * returns the number of sections whose mem_maps were properly
253 * set. If this is <=0, then that means that the passed-in
254 * map was not consumed and must be freed.
256 int sparse_add_one_section(struct zone
*zone
, unsigned long start_pfn
,
259 unsigned long section_nr
= pfn_to_section_nr(start_pfn
);
260 struct pglist_data
*pgdat
= zone
->zone_pgdat
;
261 struct mem_section
*ms
;
267 * no locking for this, because it does its own
268 * plus, it does a kmalloc
270 sparse_index_init(section_nr
, pgdat
->node_id
);
271 memmap
= __kmalloc_section_memmap(nr_pages
);
273 pgdat_resize_lock(pgdat
, &flags
);
275 ms
= __pfn_to_section(start_pfn
);
276 if (ms
->section_mem_map
& SECTION_MARKED_PRESENT
) {
280 ms
->section_mem_map
|= SECTION_MARKED_PRESENT
;
282 ret
= sparse_init_one_section(ms
, section_nr
, memmap
);
285 pgdat_resize_unlock(pgdat
, &flags
);
287 __kfree_section_memmap(memmap
, nr_pages
);