2 * mm/percpu-vm.c - vmalloc area based chunk allocation
4 * Copyright (C) 2010 SUSE Linux Products GmbH
5 * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
7 * This file is released under the GPLv2.
9 * Chunks are mapped into vmalloc areas and populated page by page.
10 * This is the default chunk allocator.
13 static struct page
*pcpu_chunk_page(struct pcpu_chunk
*chunk
,
14 unsigned int cpu
, int page_idx
)
16 /* must not be used on pre-mapped chunk */
17 WARN_ON(chunk
->immutable
);
19 return vmalloc_to_page((void *)pcpu_chunk_addr(chunk
, cpu
, page_idx
));
23 * pcpu_get_pages_and_bitmap - get temp pages array and bitmap
24 * @chunk: chunk of interest
25 * @bitmapp: output parameter for bitmap
26 * @may_alloc: may allocate the array
28 * Returns pointer to array of pointers to struct page and bitmap,
29 * both of which can be indexed with pcpu_page_idx(). The returned
30 * array is cleared to zero and *@bitmapp is copied from
31 * @chunk->populated. Note that there is only one array and bitmap
32 * and access exclusion is the caller's responsibility.
35 * pcpu_alloc_mutex and does GFP_KERNEL allocation if @may_alloc.
36 * Otherwise, don't care.
39 * Pointer to temp pages array on success, NULL on failure.
41 static struct page
**pcpu_get_pages_and_bitmap(struct pcpu_chunk
*chunk
,
42 unsigned long **bitmapp
,
45 static struct page
**pages
;
46 static unsigned long *bitmap
;
47 size_t pages_size
= pcpu_nr_units
* pcpu_unit_pages
* sizeof(pages
[0]);
48 size_t bitmap_size
= BITS_TO_LONGS(pcpu_unit_pages
) *
49 sizeof(unsigned long);
51 if (!pages
|| !bitmap
) {
52 if (may_alloc
&& !pages
)
53 pages
= pcpu_mem_zalloc(pages_size
);
54 if (may_alloc
&& !bitmap
)
55 bitmap
= pcpu_mem_zalloc(bitmap_size
);
56 if (!pages
|| !bitmap
)
60 bitmap_copy(bitmap
, chunk
->populated
, pcpu_unit_pages
);
67 * pcpu_free_pages - free pages which were allocated for @chunk
68 * @chunk: chunk pages were allocated for
69 * @pages: array of pages to be freed, indexed by pcpu_page_idx()
70 * @populated: populated bitmap
71 * @page_start: page index of the first page to be freed
72 * @page_end: page index of the last page to be freed + 1
74 * Free pages [@page_start and @page_end) in @pages for all units.
75 * The pages were allocated for @chunk.
77 static void pcpu_free_pages(struct pcpu_chunk
*chunk
,
78 struct page
**pages
, unsigned long *populated
,
79 int page_start
, int page_end
)
84 for_each_possible_cpu(cpu
) {
85 for (i
= page_start
; i
< page_end
; i
++) {
86 struct page
*page
= pages
[pcpu_page_idx(cpu
, i
)];
95 * pcpu_alloc_pages - allocates pages for @chunk
96 * @chunk: target chunk
97 * @pages: array to put the allocated pages into, indexed by pcpu_page_idx()
98 * @populated: populated bitmap
99 * @page_start: page index of the first page to be allocated
100 * @page_end: page index of the last page to be allocated + 1
102 * Allocate pages [@page_start,@page_end) into @pages for all units.
103 * The allocation is for @chunk. Percpu core doesn't care about the
104 * content of @pages and will pass it verbatim to pcpu_map_pages().
106 static int pcpu_alloc_pages(struct pcpu_chunk
*chunk
,
107 struct page
**pages
, unsigned long *populated
,
108 int page_start
, int page_end
)
110 const gfp_t gfp
= GFP_KERNEL
| __GFP_HIGHMEM
| __GFP_COLD
;
114 for_each_possible_cpu(cpu
) {
115 for (i
= page_start
; i
< page_end
; i
++) {
116 struct page
**pagep
= &pages
[pcpu_page_idx(cpu
, i
)];
118 *pagep
= alloc_pages_node(cpu_to_node(cpu
), gfp
, 0);
120 pcpu_free_pages(chunk
, pages
, populated
,
121 page_start
, page_end
);
130 * pcpu_pre_unmap_flush - flush cache prior to unmapping
131 * @chunk: chunk the regions to be flushed belongs to
132 * @page_start: page index of the first page to be flushed
133 * @page_end: page index of the last page to be flushed + 1
135 * Pages in [@page_start,@page_end) of @chunk are about to be
136 * unmapped. Flush cache. As each flushing trial can be very
137 * expensive, issue flush on the whole region at once rather than
138 * doing it for each cpu. This could be an overkill but is more
141 static void pcpu_pre_unmap_flush(struct pcpu_chunk
*chunk
,
142 int page_start
, int page_end
)
145 pcpu_chunk_addr(chunk
, pcpu_low_unit_cpu
, page_start
),
146 pcpu_chunk_addr(chunk
, pcpu_high_unit_cpu
, page_end
));
149 static void __pcpu_unmap_pages(unsigned long addr
, int nr_pages
)
151 unmap_kernel_range_noflush(addr
, nr_pages
<< PAGE_SHIFT
);
155 * pcpu_unmap_pages - unmap pages out of a pcpu_chunk
156 * @chunk: chunk of interest
157 * @pages: pages array which can be used to pass information to free
158 * @populated: populated bitmap
159 * @page_start: page index of the first page to unmap
160 * @page_end: page index of the last page to unmap + 1
162 * For each cpu, unmap pages [@page_start,@page_end) out of @chunk.
163 * Corresponding elements in @pages were cleared by the caller and can
164 * be used to carry information to pcpu_free_pages() which will be
165 * called after all unmaps are finished. The caller should call
166 * proper pre/post flush functions.
168 static void pcpu_unmap_pages(struct pcpu_chunk
*chunk
,
169 struct page
**pages
, unsigned long *populated
,
170 int page_start
, int page_end
)
175 for_each_possible_cpu(cpu
) {
176 for (i
= page_start
; i
< page_end
; i
++) {
179 page
= pcpu_chunk_page(chunk
, cpu
, i
);
181 pages
[pcpu_page_idx(cpu
, i
)] = page
;
183 __pcpu_unmap_pages(pcpu_chunk_addr(chunk
, cpu
, page_start
),
184 page_end
- page_start
);
187 bitmap_clear(populated
, page_start
, page_end
- page_start
);
191 * pcpu_post_unmap_tlb_flush - flush TLB after unmapping
192 * @chunk: pcpu_chunk the regions to be flushed belong to
193 * @page_start: page index of the first page to be flushed
194 * @page_end: page index of the last page to be flushed + 1
196 * Pages [@page_start,@page_end) of @chunk have been unmapped. Flush
197 * TLB for the regions. This can be skipped if the area is to be
198 * returned to vmalloc as vmalloc will handle TLB flushing lazily.
200 * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
201 * for the whole region.
203 static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk
*chunk
,
204 int page_start
, int page_end
)
206 flush_tlb_kernel_range(
207 pcpu_chunk_addr(chunk
, pcpu_low_unit_cpu
, page_start
),
208 pcpu_chunk_addr(chunk
, pcpu_high_unit_cpu
, page_end
));
211 static int __pcpu_map_pages(unsigned long addr
, struct page
**pages
,
214 return map_kernel_range_noflush(addr
, nr_pages
<< PAGE_SHIFT
,
219 * pcpu_map_pages - map pages into a pcpu_chunk
220 * @chunk: chunk of interest
221 * @pages: pages array containing pages to be mapped
222 * @populated: populated bitmap
223 * @page_start: page index of the first page to map
224 * @page_end: page index of the last page to map + 1
226 * For each cpu, map pages [@page_start,@page_end) into @chunk. The
227 * caller is responsible for calling pcpu_post_map_flush() after all
228 * mappings are complete.
230 * This function is responsible for setting corresponding bits in
231 * @chunk->populated bitmap and whatever is necessary for reverse
232 * lookup (addr -> chunk).
234 static int pcpu_map_pages(struct pcpu_chunk
*chunk
,
235 struct page
**pages
, unsigned long *populated
,
236 int page_start
, int page_end
)
238 unsigned int cpu
, tcpu
;
241 for_each_possible_cpu(cpu
) {
242 err
= __pcpu_map_pages(pcpu_chunk_addr(chunk
, cpu
, page_start
),
243 &pages
[pcpu_page_idx(cpu
, page_start
)],
244 page_end
- page_start
);
249 /* mapping successful, link chunk and mark populated */
250 for (i
= page_start
; i
< page_end
; i
++) {
251 for_each_possible_cpu(cpu
)
252 pcpu_set_page_chunk(pages
[pcpu_page_idx(cpu
, i
)],
254 __set_bit(i
, populated
);
260 for_each_possible_cpu(tcpu
) {
263 __pcpu_unmap_pages(pcpu_chunk_addr(chunk
, tcpu
, page_start
),
264 page_end
- page_start
);
270 * pcpu_post_map_flush - flush cache after mapping
271 * @chunk: pcpu_chunk the regions to be flushed belong to
272 * @page_start: page index of the first page to be flushed
273 * @page_end: page index of the last page to be flushed + 1
275 * Pages [@page_start,@page_end) of @chunk have been mapped. Flush
278 * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
279 * for the whole region.
281 static void pcpu_post_map_flush(struct pcpu_chunk
*chunk
,
282 int page_start
, int page_end
)
285 pcpu_chunk_addr(chunk
, pcpu_low_unit_cpu
, page_start
),
286 pcpu_chunk_addr(chunk
, pcpu_high_unit_cpu
, page_end
));
290 * pcpu_populate_chunk - populate and map an area of a pcpu_chunk
291 * @chunk: chunk of interest
292 * @off: offset to the area to populate
293 * @size: size of the area to populate in bytes
295 * For each cpu, populate and map pages [@page_start,@page_end) into
296 * @chunk. The area is cleared on return.
299 * pcpu_alloc_mutex, does GFP_KERNEL allocation.
301 static int pcpu_populate_chunk(struct pcpu_chunk
*chunk
, int off
, int size
)
303 int page_start
= PFN_DOWN(off
);
304 int page_end
= PFN_UP(off
+ size
);
305 int free_end
= page_start
, unmap_end
= page_start
;
307 unsigned long *populated
;
311 /* quick path, check whether all pages are already there */
313 pcpu_next_pop(chunk
, &rs
, &re
, page_end
);
314 if (rs
== page_start
&& re
== page_end
)
317 /* need to allocate and map pages, this chunk can't be immutable */
318 WARN_ON(chunk
->immutable
);
320 pages
= pcpu_get_pages_and_bitmap(chunk
, &populated
, true);
325 pcpu_for_each_unpop_region(chunk
, rs
, re
, page_start
, page_end
) {
326 rc
= pcpu_alloc_pages(chunk
, pages
, populated
, rs
, re
);
332 pcpu_for_each_unpop_region(chunk
, rs
, re
, page_start
, page_end
) {
333 rc
= pcpu_map_pages(chunk
, pages
, populated
, rs
, re
);
338 pcpu_post_map_flush(chunk
, page_start
, page_end
);
340 /* commit new bitmap */
341 bitmap_copy(chunk
->populated
, populated
, pcpu_unit_pages
);
343 for_each_possible_cpu(cpu
)
344 memset((void *)pcpu_chunk_addr(chunk
, cpu
, 0) + off
, 0, size
);
348 pcpu_pre_unmap_flush(chunk
, page_start
, unmap_end
);
349 pcpu_for_each_unpop_region(chunk
, rs
, re
, page_start
, unmap_end
)
350 pcpu_unmap_pages(chunk
, pages
, populated
, rs
, re
);
351 pcpu_post_unmap_tlb_flush(chunk
, page_start
, unmap_end
);
353 pcpu_for_each_unpop_region(chunk
, rs
, re
, page_start
, free_end
)
354 pcpu_free_pages(chunk
, pages
, populated
, rs
, re
);
359 * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk
360 * @chunk: chunk to depopulate
361 * @off: offset to the area to depopulate
362 * @size: size of the area to depopulate in bytes
364 * For each cpu, depopulate and unmap pages [@page_start,@page_end)
365 * from @chunk. If @flush is true, vcache is flushed before unmapping
371 static void pcpu_depopulate_chunk(struct pcpu_chunk
*chunk
, int off
, int size
)
373 int page_start
= PFN_DOWN(off
);
374 int page_end
= PFN_UP(off
+ size
);
376 unsigned long *populated
;
379 /* quick path, check whether it's empty already */
381 pcpu_next_unpop(chunk
, &rs
, &re
, page_end
);
382 if (rs
== page_start
&& re
== page_end
)
385 /* immutable chunks can't be depopulated */
386 WARN_ON(chunk
->immutable
);
389 * If control reaches here, there must have been at least one
390 * successful population attempt so the temp pages array must
393 pages
= pcpu_get_pages_and_bitmap(chunk
, &populated
, false);
397 pcpu_pre_unmap_flush(chunk
, page_start
, page_end
);
399 pcpu_for_each_pop_region(chunk
, rs
, re
, page_start
, page_end
)
400 pcpu_unmap_pages(chunk
, pages
, populated
, rs
, re
);
402 /* no need to flush tlb, vmalloc will handle it lazily */
404 pcpu_for_each_pop_region(chunk
, rs
, re
, page_start
, page_end
)
405 pcpu_free_pages(chunk
, pages
, populated
, rs
, re
);
407 /* commit new bitmap */
408 bitmap_copy(chunk
->populated
, populated
, pcpu_unit_pages
);
411 static struct pcpu_chunk
*pcpu_create_chunk(void)
413 struct pcpu_chunk
*chunk
;
414 struct vm_struct
**vms
;
416 chunk
= pcpu_alloc_chunk();
420 vms
= pcpu_get_vm_areas(pcpu_group_offsets
, pcpu_group_sizes
,
421 pcpu_nr_groups
, pcpu_atom_size
);
423 pcpu_free_chunk(chunk
);
428 chunk
->base_addr
= vms
[0]->addr
- pcpu_group_offsets
[0];
432 static void pcpu_destroy_chunk(struct pcpu_chunk
*chunk
)
434 if (chunk
&& chunk
->data
)
435 pcpu_free_vm_areas(chunk
->data
, pcpu_nr_groups
);
436 pcpu_free_chunk(chunk
);
439 static struct page
*pcpu_addr_to_page(void *addr
)
441 return vmalloc_to_page(addr
);
444 static int __init
pcpu_verify_alloc_info(const struct pcpu_alloc_info
*ai
)
446 /* no extra restriction */