2 * Simple NUMA memory policy for the Linux kernel.
4 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
5 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
6 * Subject to the GNU Public License, version 2.
8 * NUMA policy allows the user to give hints in which node(s) memory should
11 * Support four policies per VMA and per process:
13 * The VMA policy has priority over the process policy for a page fault.
15 * interleave Allocate memory interleaved over a set of nodes,
16 * with normal fallback if it fails.
17 * For VMA based allocations this interleaves based on the
18 * offset into the backing object or offset into the mapping
19 * for anonymous memory. For process policy an process counter
22 * bind Only allocate memory on a specific set of nodes,
24 * FIXME: memory is allocated starting with the first node
25 * to the last. It would be better if bind would truly restrict
26 * the allocation to memory nodes instead
28 * preferred Try a specific node first before normal fallback.
29 * As a special case NUMA_NO_NODE here means do the allocation
30 * on the local CPU. This is normally identical to default,
31 * but useful to set in a VMA when you have a non default
34 * default Allocate on the local node first, or when on a VMA
35 * use the process policy. This is what Linux always did
36 * in a NUMA aware kernel and still does by, ahem, default.
38 * The process policy is applied for most non interrupt memory allocations
39 * in that process' context. Interrupts ignore the policies and always
40 * try to allocate on the local CPU. The VMA policy is only applied for memory
41 * allocations for a VMA in the VM.
43 * Currently there are a few corner cases in swapping where the policy
44 * is not applied, but the majority should be handled. When process policy
45 * is used it is not remembered over swap outs/swap ins.
47 * Only the highest zone in the zone hierarchy gets policied. Allocations
48 * requesting a lower zone just use default policy. This implies that
49 * on systems with highmem kernel lowmem allocation don't get policied.
50 * Same with GFP_DMA allocations.
52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
53 * all users and remembered even when nobody has memory mapped.
57 fix mmap readahead to honour policy and enable policy for any page cache
59 statistics for bigpages
60 global policy for page cache? currently it uses process policy. Requires
62 handle mremap for shared memory (currently ignored for the policy)
64 make bind policy root only? It can trigger oom much faster and the
65 kernel is not always grateful with that.
68 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
70 #include <linux/mempolicy.h>
72 #include <linux/highmem.h>
73 #include <linux/hugetlb.h>
74 #include <linux/kernel.h>
75 #include <linux/sched.h>
76 #include <linux/nodemask.h>
77 #include <linux/cpuset.h>
78 #include <linux/slab.h>
79 #include <linux/string.h>
80 #include <linux/export.h>
81 #include <linux/nsproxy.h>
82 #include <linux/interrupt.h>
83 #include <linux/init.h>
84 #include <linux/compat.h>
85 #include <linux/swap.h>
86 #include <linux/seq_file.h>
87 #include <linux/proc_fs.h>
88 #include <linux/migrate.h>
89 #include <linux/ksm.h>
90 #include <linux/rmap.h>
91 #include <linux/security.h>
92 #include <linux/syscalls.h>
93 #include <linux/ctype.h>
94 #include <linux/mm_inline.h>
95 #include <linux/mmu_notifier.h>
96 #include <linux/printk.h>
98 #include <asm/tlbflush.h>
99 #include <asm/uaccess.h>
101 #include "internal.h"
104 #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
105 #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
107 static struct kmem_cache
*policy_cache
;
108 static struct kmem_cache
*sn_cache
;
110 /* Highest zone. An specific allocation for a zone below that is not
112 enum zone_type policy_zone
= 0;
115 * run-time system-wide default policy => local allocation
117 static struct mempolicy default_policy
= {
118 .refcnt
= ATOMIC_INIT(1), /* never free it */
119 .mode
= MPOL_PREFERRED
,
120 .flags
= MPOL_F_LOCAL
,
123 static struct mempolicy preferred_node_policy
[MAX_NUMNODES
];
125 struct mempolicy
*get_task_policy(struct task_struct
*p
)
127 struct mempolicy
*pol
= p
->mempolicy
;
133 node
= numa_node_id();
134 if (node
!= NUMA_NO_NODE
) {
135 pol
= &preferred_node_policy
[node
];
136 /* preferred_node_policy is not initialised early in boot */
141 return &default_policy
;
144 static const struct mempolicy_operations
{
145 int (*create
)(struct mempolicy
*pol
, const nodemask_t
*nodes
);
147 * If read-side task has no lock to protect task->mempolicy, write-side
148 * task will rebind the task->mempolicy by two step. The first step is
149 * setting all the newly nodes, and the second step is cleaning all the
150 * disallowed nodes. In this way, we can avoid finding no node to alloc
152 * If we have a lock to protect task->mempolicy in read-side, we do
156 * MPOL_REBIND_ONCE - do rebind work at once
157 * MPOL_REBIND_STEP1 - set all the newly nodes
158 * MPOL_REBIND_STEP2 - clean all the disallowed nodes
160 void (*rebind
)(struct mempolicy
*pol
, const nodemask_t
*nodes
,
161 enum mpol_rebind_step step
);
162 } mpol_ops
[MPOL_MAX
];
164 static inline int mpol_store_user_nodemask(const struct mempolicy
*pol
)
166 return pol
->flags
& MPOL_MODE_FLAGS
;
169 static void mpol_relative_nodemask(nodemask_t
*ret
, const nodemask_t
*orig
,
170 const nodemask_t
*rel
)
173 nodes_fold(tmp
, *orig
, nodes_weight(*rel
));
174 nodes_onto(*ret
, tmp
, *rel
);
177 static int mpol_new_interleave(struct mempolicy
*pol
, const nodemask_t
*nodes
)
179 if (nodes_empty(*nodes
))
181 pol
->v
.nodes
= *nodes
;
185 static int mpol_new_preferred(struct mempolicy
*pol
, const nodemask_t
*nodes
)
188 pol
->flags
|= MPOL_F_LOCAL
; /* local allocation */
189 else if (nodes_empty(*nodes
))
190 return -EINVAL
; /* no allowed nodes */
192 pol
->v
.preferred_node
= first_node(*nodes
);
196 static int mpol_new_bind(struct mempolicy
*pol
, const nodemask_t
*nodes
)
198 if (nodes_empty(*nodes
))
200 pol
->v
.nodes
= *nodes
;
205 * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if
206 * any, for the new policy. mpol_new() has already validated the nodes
207 * parameter with respect to the policy mode and flags. But, we need to
208 * handle an empty nodemask with MPOL_PREFERRED here.
210 * Must be called holding task's alloc_lock to protect task's mems_allowed
211 * and mempolicy. May also be called holding the mmap_semaphore for write.
213 static int mpol_set_nodemask(struct mempolicy
*pol
,
214 const nodemask_t
*nodes
, struct nodemask_scratch
*nsc
)
218 /* if mode is MPOL_DEFAULT, pol is NULL. This is right. */
222 nodes_and(nsc
->mask1
,
223 cpuset_current_mems_allowed
, node_states
[N_MEMORY
]);
226 if (pol
->mode
== MPOL_PREFERRED
&& nodes_empty(*nodes
))
227 nodes
= NULL
; /* explicit local allocation */
229 if (pol
->flags
& MPOL_F_RELATIVE_NODES
)
230 mpol_relative_nodemask(&nsc
->mask2
, nodes
, &nsc
->mask1
);
232 nodes_and(nsc
->mask2
, *nodes
, nsc
->mask1
);
234 if (mpol_store_user_nodemask(pol
))
235 pol
->w
.user_nodemask
= *nodes
;
237 pol
->w
.cpuset_mems_allowed
=
238 cpuset_current_mems_allowed
;
242 ret
= mpol_ops
[pol
->mode
].create(pol
, &nsc
->mask2
);
244 ret
= mpol_ops
[pol
->mode
].create(pol
, NULL
);
249 * This function just creates a new policy, does some check and simple
250 * initialization. You must invoke mpol_set_nodemask() to set nodes.
252 static struct mempolicy
*mpol_new(unsigned short mode
, unsigned short flags
,
255 struct mempolicy
*policy
;
257 pr_debug("setting mode %d flags %d nodes[0] %lx\n",
258 mode
, flags
, nodes
? nodes_addr(*nodes
)[0] : NUMA_NO_NODE
);
260 if (mode
== MPOL_DEFAULT
) {
261 if (nodes
&& !nodes_empty(*nodes
))
262 return ERR_PTR(-EINVAL
);
268 * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
269 * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
270 * All other modes require a valid pointer to a non-empty nodemask.
272 if (mode
== MPOL_PREFERRED
) {
273 if (nodes_empty(*nodes
)) {
274 if (((flags
& MPOL_F_STATIC_NODES
) ||
275 (flags
& MPOL_F_RELATIVE_NODES
)))
276 return ERR_PTR(-EINVAL
);
278 } else if (mode
== MPOL_LOCAL
) {
279 if (!nodes_empty(*nodes
))
280 return ERR_PTR(-EINVAL
);
281 mode
= MPOL_PREFERRED
;
282 } else if (nodes_empty(*nodes
))
283 return ERR_PTR(-EINVAL
);
284 policy
= kmem_cache_alloc(policy_cache
, GFP_KERNEL
);
286 return ERR_PTR(-ENOMEM
);
287 atomic_set(&policy
->refcnt
, 1);
289 policy
->flags
= flags
;
294 /* Slow path of a mpol destructor. */
295 void __mpol_put(struct mempolicy
*p
)
297 if (!atomic_dec_and_test(&p
->refcnt
))
299 kmem_cache_free(policy_cache
, p
);
302 static void mpol_rebind_default(struct mempolicy
*pol
, const nodemask_t
*nodes
,
303 enum mpol_rebind_step step
)
309 * MPOL_REBIND_ONCE - do rebind work at once
310 * MPOL_REBIND_STEP1 - set all the newly nodes
311 * MPOL_REBIND_STEP2 - clean all the disallowed nodes
313 static void mpol_rebind_nodemask(struct mempolicy
*pol
, const nodemask_t
*nodes
,
314 enum mpol_rebind_step step
)
318 if (pol
->flags
& MPOL_F_STATIC_NODES
)
319 nodes_and(tmp
, pol
->w
.user_nodemask
, *nodes
);
320 else if (pol
->flags
& MPOL_F_RELATIVE_NODES
)
321 mpol_relative_nodemask(&tmp
, &pol
->w
.user_nodemask
, nodes
);
324 * if step == 1, we use ->w.cpuset_mems_allowed to cache the
327 if (step
== MPOL_REBIND_ONCE
|| step
== MPOL_REBIND_STEP1
) {
328 nodes_remap(tmp
, pol
->v
.nodes
,
329 pol
->w
.cpuset_mems_allowed
, *nodes
);
330 pol
->w
.cpuset_mems_allowed
= step
? tmp
: *nodes
;
331 } else if (step
== MPOL_REBIND_STEP2
) {
332 tmp
= pol
->w
.cpuset_mems_allowed
;
333 pol
->w
.cpuset_mems_allowed
= *nodes
;
338 if (nodes_empty(tmp
))
341 if (step
== MPOL_REBIND_STEP1
)
342 nodes_or(pol
->v
.nodes
, pol
->v
.nodes
, tmp
);
343 else if (step
== MPOL_REBIND_ONCE
|| step
== MPOL_REBIND_STEP2
)
348 if (!node_isset(current
->il_next
, tmp
)) {
349 current
->il_next
= next_node_in(current
->il_next
, tmp
);
350 if (current
->il_next
>= MAX_NUMNODES
)
351 current
->il_next
= numa_node_id();
355 static void mpol_rebind_preferred(struct mempolicy
*pol
,
356 const nodemask_t
*nodes
,
357 enum mpol_rebind_step step
)
361 if (pol
->flags
& MPOL_F_STATIC_NODES
) {
362 int node
= first_node(pol
->w
.user_nodemask
);
364 if (node_isset(node
, *nodes
)) {
365 pol
->v
.preferred_node
= node
;
366 pol
->flags
&= ~MPOL_F_LOCAL
;
368 pol
->flags
|= MPOL_F_LOCAL
;
369 } else if (pol
->flags
& MPOL_F_RELATIVE_NODES
) {
370 mpol_relative_nodemask(&tmp
, &pol
->w
.user_nodemask
, nodes
);
371 pol
->v
.preferred_node
= first_node(tmp
);
372 } else if (!(pol
->flags
& MPOL_F_LOCAL
)) {
373 pol
->v
.preferred_node
= node_remap(pol
->v
.preferred_node
,
374 pol
->w
.cpuset_mems_allowed
,
376 pol
->w
.cpuset_mems_allowed
= *nodes
;
381 * mpol_rebind_policy - Migrate a policy to a different set of nodes
383 * If read-side task has no lock to protect task->mempolicy, write-side
384 * task will rebind the task->mempolicy by two step. The first step is
385 * setting all the newly nodes, and the second step is cleaning all the
386 * disallowed nodes. In this way, we can avoid finding no node to alloc
388 * If we have a lock to protect task->mempolicy in read-side, we do
392 * MPOL_REBIND_ONCE - do rebind work at once
393 * MPOL_REBIND_STEP1 - set all the newly nodes
394 * MPOL_REBIND_STEP2 - clean all the disallowed nodes
396 static void mpol_rebind_policy(struct mempolicy
*pol
, const nodemask_t
*newmask
,
397 enum mpol_rebind_step step
)
401 if (!mpol_store_user_nodemask(pol
) && step
== MPOL_REBIND_ONCE
&&
402 nodes_equal(pol
->w
.cpuset_mems_allowed
, *newmask
))
405 if (step
== MPOL_REBIND_STEP1
&& (pol
->flags
& MPOL_F_REBINDING
))
408 if (step
== MPOL_REBIND_STEP2
&& !(pol
->flags
& MPOL_F_REBINDING
))
411 if (step
== MPOL_REBIND_STEP1
)
412 pol
->flags
|= MPOL_F_REBINDING
;
413 else if (step
== MPOL_REBIND_STEP2
)
414 pol
->flags
&= ~MPOL_F_REBINDING
;
415 else if (step
>= MPOL_REBIND_NSTEP
)
418 mpol_ops
[pol
->mode
].rebind(pol
, newmask
, step
);
422 * Wrapper for mpol_rebind_policy() that just requires task
423 * pointer, and updates task mempolicy.
425 * Called with task's alloc_lock held.
428 void mpol_rebind_task(struct task_struct
*tsk
, const nodemask_t
*new,
429 enum mpol_rebind_step step
)
431 mpol_rebind_policy(tsk
->mempolicy
, new, step
);
435 * Rebind each vma in mm to new nodemask.
437 * Call holding a reference to mm. Takes mm->mmap_sem during call.
440 void mpol_rebind_mm(struct mm_struct
*mm
, nodemask_t
*new)
442 struct vm_area_struct
*vma
;
444 down_write(&mm
->mmap_sem
);
445 for (vma
= mm
->mmap
; vma
; vma
= vma
->vm_next
)
446 mpol_rebind_policy(vma
->vm_policy
, new, MPOL_REBIND_ONCE
);
447 up_write(&mm
->mmap_sem
);
450 static const struct mempolicy_operations mpol_ops
[MPOL_MAX
] = {
452 .rebind
= mpol_rebind_default
,
454 [MPOL_INTERLEAVE
] = {
455 .create
= mpol_new_interleave
,
456 .rebind
= mpol_rebind_nodemask
,
459 .create
= mpol_new_preferred
,
460 .rebind
= mpol_rebind_preferred
,
463 .create
= mpol_new_bind
,
464 .rebind
= mpol_rebind_nodemask
,
468 static void migrate_page_add(struct page
*page
, struct list_head
*pagelist
,
469 unsigned long flags
);
472 struct list_head
*pagelist
;
475 struct vm_area_struct
*prev
;
479 * Scan through pages checking if pages follow certain conditions,
480 * and move them to the pagelist if they do.
482 static int queue_pages_pte_range(pmd_t
*pmd
, unsigned long addr
,
483 unsigned long end
, struct mm_walk
*walk
)
485 struct vm_area_struct
*vma
= walk
->vma
;
487 struct queue_pages
*qp
= walk
->private;
488 unsigned long flags
= qp
->flags
;
493 if (pmd_trans_huge(*pmd
)) {
494 ptl
= pmd_lock(walk
->mm
, pmd
);
495 if (pmd_trans_huge(*pmd
)) {
496 page
= pmd_page(*pmd
);
497 if (is_huge_zero_page(page
)) {
499 split_huge_pmd(vma
, pmd
, addr
);
504 ret
= split_huge_page(page
);
516 pte
= pte_offset_map_lock(walk
->mm
, pmd
, addr
, &ptl
);
517 for (; addr
!= end
; pte
++, addr
+= PAGE_SIZE
) {
518 if (!pte_present(*pte
))
520 page
= vm_normal_page(vma
, addr
, *pte
);
524 * vm_normal_page() filters out zero pages, but there might
525 * still be PageReserved pages to skip, perhaps in a VDSO.
527 if (PageReserved(page
))
529 nid
= page_to_nid(page
);
530 if (node_isset(nid
, *qp
->nmask
) == !!(flags
& MPOL_MF_INVERT
))
532 if (PageTransCompound(page
) && PageAnon(page
)) {
534 pte_unmap_unlock(pte
, ptl
);
536 ret
= split_huge_page(page
);
539 /* Failed to split -- skip. */
541 pte
= pte_offset_map_lock(walk
->mm
, pmd
,
548 migrate_page_add(page
, qp
->pagelist
, flags
);
550 pte_unmap_unlock(pte
- 1, ptl
);
555 static int queue_pages_hugetlb(pte_t
*pte
, unsigned long hmask
,
556 unsigned long addr
, unsigned long end
,
557 struct mm_walk
*walk
)
559 #ifdef CONFIG_HUGETLB_PAGE
560 struct queue_pages
*qp
= walk
->private;
561 unsigned long flags
= qp
->flags
;
567 ptl
= huge_pte_lock(hstate_vma(walk
->vma
), walk
->mm
, pte
);
568 entry
= huge_ptep_get(pte
);
569 if (!pte_present(entry
))
571 page
= pte_page(entry
);
572 nid
= page_to_nid(page
);
573 if (node_isset(nid
, *qp
->nmask
) == !!(flags
& MPOL_MF_INVERT
))
575 /* With MPOL_MF_MOVE, we migrate only unshared hugepage. */
576 if (flags
& (MPOL_MF_MOVE_ALL
) ||
577 (flags
& MPOL_MF_MOVE
&& page_mapcount(page
) == 1))
578 isolate_huge_page(page
, qp
->pagelist
);
587 #ifdef CONFIG_NUMA_BALANCING
589 * This is used to mark a range of virtual addresses to be inaccessible.
590 * These are later cleared by a NUMA hinting fault. Depending on these
591 * faults, pages may be migrated for better NUMA placement.
593 * This is assuming that NUMA faults are handled using PROT_NONE. If
594 * an architecture makes a different choice, it will need further
595 * changes to the core.
597 unsigned long change_prot_numa(struct vm_area_struct
*vma
,
598 unsigned long addr
, unsigned long end
)
602 nr_updated
= change_protection(vma
, addr
, end
, PAGE_NONE
, 0, 1);
604 count_vm_numa_events(NUMA_PTE_UPDATES
, nr_updated
);
609 static unsigned long change_prot_numa(struct vm_area_struct
*vma
,
610 unsigned long addr
, unsigned long end
)
614 #endif /* CONFIG_NUMA_BALANCING */
616 static int queue_pages_test_walk(unsigned long start
, unsigned long end
,
617 struct mm_walk
*walk
)
619 struct vm_area_struct
*vma
= walk
->vma
;
620 struct queue_pages
*qp
= walk
->private;
621 unsigned long endvma
= vma
->vm_end
;
622 unsigned long flags
= qp
->flags
;
624 if (!vma_migratable(vma
))
629 if (vma
->vm_start
> start
)
630 start
= vma
->vm_start
;
632 if (!(flags
& MPOL_MF_DISCONTIG_OK
)) {
633 if (!vma
->vm_next
&& vma
->vm_end
< end
)
635 if (qp
->prev
&& qp
->prev
->vm_end
< vma
->vm_start
)
641 if (flags
& MPOL_MF_LAZY
) {
642 /* Similar to task_numa_work, skip inaccessible VMAs */
643 if (!is_vm_hugetlb_page(vma
) &&
644 (vma
->vm_flags
& (VM_READ
| VM_EXEC
| VM_WRITE
)) &&
645 !(vma
->vm_flags
& VM_MIXEDMAP
))
646 change_prot_numa(vma
, start
, endvma
);
650 /* queue pages from current vma */
651 if (flags
& (MPOL_MF_MOVE
| MPOL_MF_MOVE_ALL
))
657 * Walk through page tables and collect pages to be migrated.
659 * If pages found in a given range are on a set of nodes (determined by
660 * @nodes and @flags,) it's isolated and queued to the pagelist which is
661 * passed via @private.)
664 queue_pages_range(struct mm_struct
*mm
, unsigned long start
, unsigned long end
,
665 nodemask_t
*nodes
, unsigned long flags
,
666 struct list_head
*pagelist
)
668 struct queue_pages qp
= {
669 .pagelist
= pagelist
,
674 struct mm_walk queue_pages_walk
= {
675 .hugetlb_entry
= queue_pages_hugetlb
,
676 .pmd_entry
= queue_pages_pte_range
,
677 .test_walk
= queue_pages_test_walk
,
682 return walk_page_range(start
, end
, &queue_pages_walk
);
686 * Apply policy to a single VMA
687 * This must be called with the mmap_sem held for writing.
689 static int vma_replace_policy(struct vm_area_struct
*vma
,
690 struct mempolicy
*pol
)
693 struct mempolicy
*old
;
694 struct mempolicy
*new;
696 pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
697 vma
->vm_start
, vma
->vm_end
, vma
->vm_pgoff
,
698 vma
->vm_ops
, vma
->vm_file
,
699 vma
->vm_ops
? vma
->vm_ops
->set_policy
: NULL
);
705 if (vma
->vm_ops
&& vma
->vm_ops
->set_policy
) {
706 err
= vma
->vm_ops
->set_policy(vma
, new);
711 old
= vma
->vm_policy
;
712 vma
->vm_policy
= new; /* protected by mmap_sem */
721 /* Step 2: apply policy to a range and do splits. */
722 static int mbind_range(struct mm_struct
*mm
, unsigned long start
,
723 unsigned long end
, struct mempolicy
*new_pol
)
725 struct vm_area_struct
*next
;
726 struct vm_area_struct
*prev
;
727 struct vm_area_struct
*vma
;
730 unsigned long vmstart
;
733 vma
= find_vma(mm
, start
);
734 if (!vma
|| vma
->vm_start
> start
)
738 if (start
> vma
->vm_start
)
741 for (; vma
&& vma
->vm_start
< end
; prev
= vma
, vma
= next
) {
743 vmstart
= max(start
, vma
->vm_start
);
744 vmend
= min(end
, vma
->vm_end
);
746 if (mpol_equal(vma_policy(vma
), new_pol
))
749 pgoff
= vma
->vm_pgoff
+
750 ((vmstart
- vma
->vm_start
) >> PAGE_SHIFT
);
751 prev
= vma_merge(mm
, prev
, vmstart
, vmend
, vma
->vm_flags
,
752 vma
->anon_vma
, vma
->vm_file
, pgoff
,
753 new_pol
, vma
->vm_userfaultfd_ctx
);
757 if (mpol_equal(vma_policy(vma
), new_pol
))
759 /* vma_merge() joined vma && vma->next, case 8 */
762 if (vma
->vm_start
!= vmstart
) {
763 err
= split_vma(vma
->vm_mm
, vma
, vmstart
, 1);
767 if (vma
->vm_end
!= vmend
) {
768 err
= split_vma(vma
->vm_mm
, vma
, vmend
, 0);
773 err
= vma_replace_policy(vma
, new_pol
);
782 /* Set the process memory policy */
783 static long do_set_mempolicy(unsigned short mode
, unsigned short flags
,
786 struct mempolicy
*new, *old
;
787 NODEMASK_SCRATCH(scratch
);
793 new = mpol_new(mode
, flags
, nodes
);
800 ret
= mpol_set_nodemask(new, nodes
, scratch
);
802 task_unlock(current
);
806 old
= current
->mempolicy
;
807 current
->mempolicy
= new;
808 if (new && new->mode
== MPOL_INTERLEAVE
&&
809 nodes_weight(new->v
.nodes
))
810 current
->il_next
= first_node(new->v
.nodes
);
811 task_unlock(current
);
815 NODEMASK_SCRATCH_FREE(scratch
);
820 * Return nodemask for policy for get_mempolicy() query
822 * Called with task's alloc_lock held
824 static void get_policy_nodemask(struct mempolicy
*p
, nodemask_t
*nodes
)
827 if (p
== &default_policy
)
833 case MPOL_INTERLEAVE
:
837 if (!(p
->flags
& MPOL_F_LOCAL
))
838 node_set(p
->v
.preferred_node
, *nodes
);
839 /* else return empty node mask for local allocation */
846 static int lookup_node(unsigned long addr
)
851 err
= get_user_pages(addr
& PAGE_MASK
, 1, 0, 0, &p
, NULL
);
853 err
= page_to_nid(p
);
859 /* Retrieve NUMA policy */
860 static long do_get_mempolicy(int *policy
, nodemask_t
*nmask
,
861 unsigned long addr
, unsigned long flags
)
864 struct mm_struct
*mm
= current
->mm
;
865 struct vm_area_struct
*vma
= NULL
;
866 struct mempolicy
*pol
= current
->mempolicy
;
869 ~(unsigned long)(MPOL_F_NODE
|MPOL_F_ADDR
|MPOL_F_MEMS_ALLOWED
))
872 if (flags
& MPOL_F_MEMS_ALLOWED
) {
873 if (flags
& (MPOL_F_NODE
|MPOL_F_ADDR
))
875 *policy
= 0; /* just so it's initialized */
877 *nmask
= cpuset_current_mems_allowed
;
878 task_unlock(current
);
882 if (flags
& MPOL_F_ADDR
) {
884 * Do NOT fall back to task policy if the
885 * vma/shared policy at addr is NULL. We
886 * want to return MPOL_DEFAULT in this case.
888 down_read(&mm
->mmap_sem
);
889 vma
= find_vma_intersection(mm
, addr
, addr
+1);
891 up_read(&mm
->mmap_sem
);
894 if (vma
->vm_ops
&& vma
->vm_ops
->get_policy
)
895 pol
= vma
->vm_ops
->get_policy(vma
, addr
);
897 pol
= vma
->vm_policy
;
902 pol
= &default_policy
; /* indicates default behavior */
904 if (flags
& MPOL_F_NODE
) {
905 if (flags
& MPOL_F_ADDR
) {
906 err
= lookup_node(addr
);
910 } else if (pol
== current
->mempolicy
&&
911 pol
->mode
== MPOL_INTERLEAVE
) {
912 *policy
= current
->il_next
;
918 *policy
= pol
== &default_policy
? MPOL_DEFAULT
:
921 * Internal mempolicy flags must be masked off before exposing
922 * the policy to userspace.
924 *policy
|= (pol
->flags
& MPOL_MODE_FLAGS
);
928 up_read(¤t
->mm
->mmap_sem
);
934 if (mpol_store_user_nodemask(pol
)) {
935 *nmask
= pol
->w
.user_nodemask
;
938 get_policy_nodemask(pol
, nmask
);
939 task_unlock(current
);
946 up_read(¤t
->mm
->mmap_sem
);
950 #ifdef CONFIG_MIGRATION
954 static void migrate_page_add(struct page
*page
, struct list_head
*pagelist
,
958 * Avoid migrating a page that is shared with others.
960 if ((flags
& MPOL_MF_MOVE_ALL
) || page_mapcount(page
) == 1) {
961 if (!isolate_lru_page(page
)) {
962 list_add_tail(&page
->lru
, pagelist
);
963 inc_zone_page_state(page
, NR_ISOLATED_ANON
+
964 page_is_file_cache(page
));
969 static struct page
*new_node_page(struct page
*page
, unsigned long node
, int **x
)
972 return alloc_huge_page_node(page_hstate(compound_head(page
)),
975 return __alloc_pages_node(node
, GFP_HIGHUSER_MOVABLE
|
980 * Migrate pages from one node to a target node.
981 * Returns error or the number of pages not migrated.
983 static int migrate_to_node(struct mm_struct
*mm
, int source
, int dest
,
991 node_set(source
, nmask
);
994 * This does not "check" the range but isolates all pages that
995 * need migration. Between passing in the full user address
996 * space range and MPOL_MF_DISCONTIG_OK, this call can not fail.
998 VM_BUG_ON(!(flags
& (MPOL_MF_MOVE
| MPOL_MF_MOVE_ALL
)));
999 queue_pages_range(mm
, mm
->mmap
->vm_start
, mm
->task_size
, &nmask
,
1000 flags
| MPOL_MF_DISCONTIG_OK
, &pagelist
);
1002 if (!list_empty(&pagelist
)) {
1003 err
= migrate_pages(&pagelist
, new_node_page
, NULL
, dest
,
1004 MIGRATE_SYNC
, MR_SYSCALL
);
1006 putback_movable_pages(&pagelist
);
1013 * Move pages between the two nodesets so as to preserve the physical
1014 * layout as much as possible.
1016 * Returns the number of page that could not be moved.
1018 int do_migrate_pages(struct mm_struct
*mm
, const nodemask_t
*from
,
1019 const nodemask_t
*to
, int flags
)
1025 err
= migrate_prep();
1029 down_read(&mm
->mmap_sem
);
1032 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
1033 * bit in 'to' is not also set in 'tmp'. Clear the found 'source'
1034 * bit in 'tmp', and return that <source, dest> pair for migration.
1035 * The pair of nodemasks 'to' and 'from' define the map.
1037 * If no pair of bits is found that way, fallback to picking some
1038 * pair of 'source' and 'dest' bits that are not the same. If the
1039 * 'source' and 'dest' bits are the same, this represents a node
1040 * that will be migrating to itself, so no pages need move.
1042 * If no bits are left in 'tmp', or if all remaining bits left
1043 * in 'tmp' correspond to the same bit in 'to', return false
1044 * (nothing left to migrate).
1046 * This lets us pick a pair of nodes to migrate between, such that
1047 * if possible the dest node is not already occupied by some other
1048 * source node, minimizing the risk of overloading the memory on a
1049 * node that would happen if we migrated incoming memory to a node
1050 * before migrating outgoing memory source that same node.
1052 * A single scan of tmp is sufficient. As we go, we remember the
1053 * most recent <s, d> pair that moved (s != d). If we find a pair
1054 * that not only moved, but what's better, moved to an empty slot
1055 * (d is not set in tmp), then we break out then, with that pair.
1056 * Otherwise when we finish scanning from_tmp, we at least have the
1057 * most recent <s, d> pair that moved. If we get all the way through
1058 * the scan of tmp without finding any node that moved, much less
1059 * moved to an empty node, then there is nothing left worth migrating.
1063 while (!nodes_empty(tmp
)) {
1065 int source
= NUMA_NO_NODE
;
1068 for_each_node_mask(s
, tmp
) {
1071 * do_migrate_pages() tries to maintain the relative
1072 * node relationship of the pages established between
1073 * threads and memory areas.
1075 * However if the number of source nodes is not equal to
1076 * the number of destination nodes we can not preserve
1077 * this node relative relationship. In that case, skip
1078 * copying memory from a node that is in the destination
1081 * Example: [2,3,4] -> [3,4,5] moves everything.
1082 * [0-7] - > [3,4,5] moves only 0,1,2,6,7.
1085 if ((nodes_weight(*from
) != nodes_weight(*to
)) &&
1086 (node_isset(s
, *to
)))
1089 d
= node_remap(s
, *from
, *to
);
1093 source
= s
; /* Node moved. Memorize */
1096 /* dest not in remaining from nodes? */
1097 if (!node_isset(dest
, tmp
))
1100 if (source
== NUMA_NO_NODE
)
1103 node_clear(source
, tmp
);
1104 err
= migrate_to_node(mm
, source
, dest
, flags
);
1110 up_read(&mm
->mmap_sem
);
1118 * Allocate a new page for page migration based on vma policy.
1119 * Start by assuming the page is mapped by the same vma as contains @start.
1120 * Search forward from there, if not. N.B., this assumes that the
1121 * list of pages handed to migrate_pages()--which is how we get here--
1122 * is in virtual address order.
1124 static struct page
*new_page(struct page
*page
, unsigned long start
, int **x
)
1126 struct vm_area_struct
*vma
;
1127 unsigned long uninitialized_var(address
);
1129 vma
= find_vma(current
->mm
, start
);
1131 address
= page_address_in_vma(page
, vma
);
1132 if (address
!= -EFAULT
)
1137 if (PageHuge(page
)) {
1139 return alloc_huge_page_noerr(vma
, address
, 1);
1142 * if !vma, alloc_page_vma() will use task or system default policy
1144 return alloc_page_vma(GFP_HIGHUSER_MOVABLE
, vma
, address
);
1148 static void migrate_page_add(struct page
*page
, struct list_head
*pagelist
,
1149 unsigned long flags
)
1153 int do_migrate_pages(struct mm_struct
*mm
, const nodemask_t
*from
,
1154 const nodemask_t
*to
, int flags
)
1159 static struct page
*new_page(struct page
*page
, unsigned long start
, int **x
)
1165 static long do_mbind(unsigned long start
, unsigned long len
,
1166 unsigned short mode
, unsigned short mode_flags
,
1167 nodemask_t
*nmask
, unsigned long flags
)
1169 struct mm_struct
*mm
= current
->mm
;
1170 struct mempolicy
*new;
1173 LIST_HEAD(pagelist
);
1175 if (flags
& ~(unsigned long)MPOL_MF_VALID
)
1177 if ((flags
& MPOL_MF_MOVE_ALL
) && !capable(CAP_SYS_NICE
))
1180 if (start
& ~PAGE_MASK
)
1183 if (mode
== MPOL_DEFAULT
)
1184 flags
&= ~MPOL_MF_STRICT
;
1186 len
= (len
+ PAGE_SIZE
- 1) & PAGE_MASK
;
1194 new = mpol_new(mode
, mode_flags
, nmask
);
1196 return PTR_ERR(new);
1198 if (flags
& MPOL_MF_LAZY
)
1199 new->flags
|= MPOL_F_MOF
;
1202 * If we are using the default policy then operation
1203 * on discontinuous address spaces is okay after all
1206 flags
|= MPOL_MF_DISCONTIG_OK
;
1208 pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",
1209 start
, start
+ len
, mode
, mode_flags
,
1210 nmask
? nodes_addr(*nmask
)[0] : NUMA_NO_NODE
);
1212 if (flags
& (MPOL_MF_MOVE
| MPOL_MF_MOVE_ALL
)) {
1214 err
= migrate_prep();
1219 NODEMASK_SCRATCH(scratch
);
1221 down_write(&mm
->mmap_sem
);
1223 err
= mpol_set_nodemask(new, nmask
, scratch
);
1224 task_unlock(current
);
1226 up_write(&mm
->mmap_sem
);
1229 NODEMASK_SCRATCH_FREE(scratch
);
1234 err
= queue_pages_range(mm
, start
, end
, nmask
,
1235 flags
| MPOL_MF_INVERT
, &pagelist
);
1237 err
= mbind_range(mm
, start
, end
, new);
1242 if (!list_empty(&pagelist
)) {
1243 WARN_ON_ONCE(flags
& MPOL_MF_LAZY
);
1244 nr_failed
= migrate_pages(&pagelist
, new_page
, NULL
,
1245 start
, MIGRATE_SYNC
, MR_MEMPOLICY_MBIND
);
1247 putback_movable_pages(&pagelist
);
1250 if (nr_failed
&& (flags
& MPOL_MF_STRICT
))
1253 putback_movable_pages(&pagelist
);
1255 up_write(&mm
->mmap_sem
);
1262 * User space interface with variable sized bitmaps for nodelists.
1265 /* Copy a node mask from user space. */
1266 static int get_nodes(nodemask_t
*nodes
, const unsigned long __user
*nmask
,
1267 unsigned long maxnode
)
1270 unsigned long nlongs
;
1271 unsigned long endmask
;
1274 nodes_clear(*nodes
);
1275 if (maxnode
== 0 || !nmask
)
1277 if (maxnode
> PAGE_SIZE
*BITS_PER_BYTE
)
1280 nlongs
= BITS_TO_LONGS(maxnode
);
1281 if ((maxnode
% BITS_PER_LONG
) == 0)
1284 endmask
= (1UL << (maxnode
% BITS_PER_LONG
)) - 1;
1286 /* When the user specified more nodes than supported just check
1287 if the non supported part is all zero. */
1288 if (nlongs
> BITS_TO_LONGS(MAX_NUMNODES
)) {
1289 if (nlongs
> PAGE_SIZE
/sizeof(long))
1291 for (k
= BITS_TO_LONGS(MAX_NUMNODES
); k
< nlongs
; k
++) {
1293 if (get_user(t
, nmask
+ k
))
1295 if (k
== nlongs
- 1) {
1301 nlongs
= BITS_TO_LONGS(MAX_NUMNODES
);
1305 if (copy_from_user(nodes_addr(*nodes
), nmask
, nlongs
*sizeof(unsigned long)))
1307 nodes_addr(*nodes
)[nlongs
-1] &= endmask
;
1311 /* Copy a kernel node mask to user space */
1312 static int copy_nodes_to_user(unsigned long __user
*mask
, unsigned long maxnode
,
1315 unsigned long copy
= ALIGN(maxnode
-1, 64) / 8;
1316 const int nbytes
= BITS_TO_LONGS(MAX_NUMNODES
) * sizeof(long);
1318 if (copy
> nbytes
) {
1319 if (copy
> PAGE_SIZE
)
1321 if (clear_user((char __user
*)mask
+ nbytes
, copy
- nbytes
))
1325 return copy_to_user(mask
, nodes_addr(*nodes
), copy
) ? -EFAULT
: 0;
1328 SYSCALL_DEFINE6(mbind
, unsigned long, start
, unsigned long, len
,
1329 unsigned long, mode
, const unsigned long __user
*, nmask
,
1330 unsigned long, maxnode
, unsigned, flags
)
1334 unsigned short mode_flags
;
1336 mode_flags
= mode
& MPOL_MODE_FLAGS
;
1337 mode
&= ~MPOL_MODE_FLAGS
;
1338 if (mode
>= MPOL_MAX
)
1340 if ((mode_flags
& MPOL_F_STATIC_NODES
) &&
1341 (mode_flags
& MPOL_F_RELATIVE_NODES
))
1343 err
= get_nodes(&nodes
, nmask
, maxnode
);
1346 return do_mbind(start
, len
, mode
, mode_flags
, &nodes
, flags
);
1349 /* Set the process memory policy */
1350 SYSCALL_DEFINE3(set_mempolicy
, int, mode
, const unsigned long __user
*, nmask
,
1351 unsigned long, maxnode
)
1355 unsigned short flags
;
1357 flags
= mode
& MPOL_MODE_FLAGS
;
1358 mode
&= ~MPOL_MODE_FLAGS
;
1359 if ((unsigned int)mode
>= MPOL_MAX
)
1361 if ((flags
& MPOL_F_STATIC_NODES
) && (flags
& MPOL_F_RELATIVE_NODES
))
1363 err
= get_nodes(&nodes
, nmask
, maxnode
);
1366 return do_set_mempolicy(mode
, flags
, &nodes
);
1369 SYSCALL_DEFINE4(migrate_pages
, pid_t
, pid
, unsigned long, maxnode
,
1370 const unsigned long __user
*, old_nodes
,
1371 const unsigned long __user
*, new_nodes
)
1373 const struct cred
*cred
= current_cred(), *tcred
;
1374 struct mm_struct
*mm
= NULL
;
1375 struct task_struct
*task
;
1376 nodemask_t task_nodes
;
1380 NODEMASK_SCRATCH(scratch
);
1385 old
= &scratch
->mask1
;
1386 new = &scratch
->mask2
;
1388 err
= get_nodes(old
, old_nodes
, maxnode
);
1392 err
= get_nodes(new, new_nodes
, maxnode
);
1396 /* Find the mm_struct */
1398 task
= pid
? find_task_by_vpid(pid
) : current
;
1404 get_task_struct(task
);
1409 * Check if this process has the right to modify the specified
1410 * process. The right exists if the process has administrative
1411 * capabilities, superuser privileges or the same
1412 * userid as the target process.
1414 tcred
= __task_cred(task
);
1415 if (!uid_eq(cred
->euid
, tcred
->suid
) && !uid_eq(cred
->euid
, tcred
->uid
) &&
1416 !uid_eq(cred
->uid
, tcred
->suid
) && !uid_eq(cred
->uid
, tcred
->uid
) &&
1417 !capable(CAP_SYS_NICE
)) {
1424 task_nodes
= cpuset_mems_allowed(task
);
1425 /* Is the user allowed to access the target nodes? */
1426 if (!nodes_subset(*new, task_nodes
) && !capable(CAP_SYS_NICE
)) {
1431 if (!nodes_subset(*new, node_states
[N_MEMORY
])) {
1436 err
= security_task_movememory(task
);
1440 mm
= get_task_mm(task
);
1441 put_task_struct(task
);
1448 err
= do_migrate_pages(mm
, old
, new,
1449 capable(CAP_SYS_NICE
) ? MPOL_MF_MOVE_ALL
: MPOL_MF_MOVE
);
1453 NODEMASK_SCRATCH_FREE(scratch
);
1458 put_task_struct(task
);
1464 /* Retrieve NUMA policy */
1465 SYSCALL_DEFINE5(get_mempolicy
, int __user
*, policy
,
1466 unsigned long __user
*, nmask
, unsigned long, maxnode
,
1467 unsigned long, addr
, unsigned long, flags
)
1470 int uninitialized_var(pval
);
1473 if (nmask
!= NULL
&& maxnode
< MAX_NUMNODES
)
1476 err
= do_get_mempolicy(&pval
, &nodes
, addr
, flags
);
1481 if (policy
&& put_user(pval
, policy
))
1485 err
= copy_nodes_to_user(nmask
, maxnode
, &nodes
);
1490 #ifdef CONFIG_COMPAT
1492 COMPAT_SYSCALL_DEFINE5(get_mempolicy
, int __user
*, policy
,
1493 compat_ulong_t __user
*, nmask
,
1494 compat_ulong_t
, maxnode
,
1495 compat_ulong_t
, addr
, compat_ulong_t
, flags
)
1498 unsigned long __user
*nm
= NULL
;
1499 unsigned long nr_bits
, alloc_size
;
1500 DECLARE_BITMAP(bm
, MAX_NUMNODES
);
1502 nr_bits
= min_t(unsigned long, maxnode
-1, MAX_NUMNODES
);
1503 alloc_size
= ALIGN(nr_bits
, BITS_PER_LONG
) / 8;
1506 nm
= compat_alloc_user_space(alloc_size
);
1508 err
= sys_get_mempolicy(policy
, nm
, nr_bits
+1, addr
, flags
);
1510 if (!err
&& nmask
) {
1511 unsigned long copy_size
;
1512 copy_size
= min_t(unsigned long, sizeof(bm
), alloc_size
);
1513 err
= copy_from_user(bm
, nm
, copy_size
);
1514 /* ensure entire bitmap is zeroed */
1515 err
|= clear_user(nmask
, ALIGN(maxnode
-1, 8) / 8);
1516 err
|= compat_put_bitmap(nmask
, bm
, nr_bits
);
1522 COMPAT_SYSCALL_DEFINE3(set_mempolicy
, int, mode
, compat_ulong_t __user
*, nmask
,
1523 compat_ulong_t
, maxnode
)
1526 unsigned long __user
*nm
= NULL
;
1527 unsigned long nr_bits
, alloc_size
;
1528 DECLARE_BITMAP(bm
, MAX_NUMNODES
);
1530 nr_bits
= min_t(unsigned long, maxnode
-1, MAX_NUMNODES
);
1531 alloc_size
= ALIGN(nr_bits
, BITS_PER_LONG
) / 8;
1534 err
= compat_get_bitmap(bm
, nmask
, nr_bits
);
1535 nm
= compat_alloc_user_space(alloc_size
);
1536 err
|= copy_to_user(nm
, bm
, alloc_size
);
1542 return sys_set_mempolicy(mode
, nm
, nr_bits
+1);
1545 COMPAT_SYSCALL_DEFINE6(mbind
, compat_ulong_t
, start
, compat_ulong_t
, len
,
1546 compat_ulong_t
, mode
, compat_ulong_t __user
*, nmask
,
1547 compat_ulong_t
, maxnode
, compat_ulong_t
, flags
)
1550 unsigned long __user
*nm
= NULL
;
1551 unsigned long nr_bits
, alloc_size
;
1554 nr_bits
= min_t(unsigned long, maxnode
-1, MAX_NUMNODES
);
1555 alloc_size
= ALIGN(nr_bits
, BITS_PER_LONG
) / 8;
1558 err
= compat_get_bitmap(nodes_addr(bm
), nmask
, nr_bits
);
1559 nm
= compat_alloc_user_space(alloc_size
);
1560 err
|= copy_to_user(nm
, nodes_addr(bm
), alloc_size
);
1566 return sys_mbind(start
, len
, mode
, nm
, nr_bits
+1, flags
);
1571 struct mempolicy
*__get_vma_policy(struct vm_area_struct
*vma
,
1574 struct mempolicy
*pol
= NULL
;
1577 if (vma
->vm_ops
&& vma
->vm_ops
->get_policy
) {
1578 pol
= vma
->vm_ops
->get_policy(vma
, addr
);
1579 } else if (vma
->vm_policy
) {
1580 pol
= vma
->vm_policy
;
1583 * shmem_alloc_page() passes MPOL_F_SHARED policy with
1584 * a pseudo vma whose vma->vm_ops=NULL. Take a reference
1585 * count on these policies which will be dropped by
1586 * mpol_cond_put() later
1588 if (mpol_needs_cond_ref(pol
))
1597 * get_vma_policy(@vma, @addr)
1598 * @vma: virtual memory area whose policy is sought
1599 * @addr: address in @vma for shared policy lookup
1601 * Returns effective policy for a VMA at specified address.
1602 * Falls back to current->mempolicy or system default policy, as necessary.
1603 * Shared policies [those marked as MPOL_F_SHARED] require an extra reference
1604 * count--added by the get_policy() vm_op, as appropriate--to protect against
1605 * freeing by another task. It is the caller's responsibility to free the
1606 * extra reference for shared policies.
1608 static struct mempolicy
*get_vma_policy(struct vm_area_struct
*vma
,
1611 struct mempolicy
*pol
= __get_vma_policy(vma
, addr
);
1614 pol
= get_task_policy(current
);
1619 bool vma_policy_mof(struct vm_area_struct
*vma
)
1621 struct mempolicy
*pol
;
1623 if (vma
->vm_ops
&& vma
->vm_ops
->get_policy
) {
1626 pol
= vma
->vm_ops
->get_policy(vma
, vma
->vm_start
);
1627 if (pol
&& (pol
->flags
& MPOL_F_MOF
))
1634 pol
= vma
->vm_policy
;
1636 pol
= get_task_policy(current
);
1638 return pol
->flags
& MPOL_F_MOF
;
1641 static int apply_policy_zone(struct mempolicy
*policy
, enum zone_type zone
)
1643 enum zone_type dynamic_policy_zone
= policy_zone
;
1645 BUG_ON(dynamic_policy_zone
== ZONE_MOVABLE
);
1648 * if policy->v.nodes has movable memory only,
1649 * we apply policy when gfp_zone(gfp) = ZONE_MOVABLE only.
1651 * policy->v.nodes is intersect with node_states[N_MEMORY].
1652 * so if the following test faile, it implies
1653 * policy->v.nodes has movable memory only.
1655 if (!nodes_intersects(policy
->v
.nodes
, node_states
[N_HIGH_MEMORY
]))
1656 dynamic_policy_zone
= ZONE_MOVABLE
;
1658 return zone
>= dynamic_policy_zone
;
1662 * Return a nodemask representing a mempolicy for filtering nodes for
1665 static nodemask_t
*policy_nodemask(gfp_t gfp
, struct mempolicy
*policy
)
1667 /* Lower zones don't get a nodemask applied for MPOL_BIND */
1668 if (unlikely(policy
->mode
== MPOL_BIND
) &&
1669 apply_policy_zone(policy
, gfp_zone(gfp
)) &&
1670 cpuset_nodemask_valid_mems_allowed(&policy
->v
.nodes
))
1671 return &policy
->v
.nodes
;
1676 /* Return a zonelist indicated by gfp for node representing a mempolicy */
1677 static struct zonelist
*policy_zonelist(gfp_t gfp
, struct mempolicy
*policy
,
1680 switch (policy
->mode
) {
1681 case MPOL_PREFERRED
:
1682 if (!(policy
->flags
& MPOL_F_LOCAL
))
1683 nd
= policy
->v
.preferred_node
;
1687 * Normally, MPOL_BIND allocations are node-local within the
1688 * allowed nodemask. However, if __GFP_THISNODE is set and the
1689 * current node isn't part of the mask, we use the zonelist for
1690 * the first node in the mask instead.
1692 if (unlikely(gfp
& __GFP_THISNODE
) &&
1693 unlikely(!node_isset(nd
, policy
->v
.nodes
)))
1694 nd
= first_node(policy
->v
.nodes
);
1699 return node_zonelist(nd
, gfp
);
1702 /* Do dynamic interleaving for a process */
1703 static unsigned interleave_nodes(struct mempolicy
*policy
)
1706 struct task_struct
*me
= current
;
1709 next
= next_node_in(nid
, policy
->v
.nodes
);
1710 if (next
< MAX_NUMNODES
)
1716 * Depending on the memory policy provide a node from which to allocate the
1719 unsigned int mempolicy_slab_node(void)
1721 struct mempolicy
*policy
;
1722 int node
= numa_mem_id();
1727 policy
= current
->mempolicy
;
1728 if (!policy
|| policy
->flags
& MPOL_F_LOCAL
)
1731 switch (policy
->mode
) {
1732 case MPOL_PREFERRED
:
1734 * handled MPOL_F_LOCAL above
1736 return policy
->v
.preferred_node
;
1738 case MPOL_INTERLEAVE
:
1739 return interleave_nodes(policy
);
1745 * Follow bind policy behavior and start allocation at the
1748 struct zonelist
*zonelist
;
1749 enum zone_type highest_zoneidx
= gfp_zone(GFP_KERNEL
);
1750 zonelist
= &NODE_DATA(node
)->node_zonelists
[0];
1751 z
= first_zones_zonelist(zonelist
, highest_zoneidx
,
1753 return z
->zone
? z
->zone
->node
: node
;
1762 * Do static interleaving for a VMA with known offset @n. Returns the n'th
1763 * node in pol->v.nodes (starting from n=0), wrapping around if n exceeds the
1764 * number of present nodes.
1766 static unsigned offset_il_node(struct mempolicy
*pol
,
1767 struct vm_area_struct
*vma
, unsigned long n
)
1769 unsigned nnodes
= nodes_weight(pol
->v
.nodes
);
1775 return numa_node_id();
1776 target
= (unsigned int)n
% nnodes
;
1777 nid
= first_node(pol
->v
.nodes
);
1778 for (i
= 0; i
< target
; i
++)
1779 nid
= next_node(nid
, pol
->v
.nodes
);
1783 /* Determine a node number for interleave */
1784 static inline unsigned interleave_nid(struct mempolicy
*pol
,
1785 struct vm_area_struct
*vma
, unsigned long addr
, int shift
)
1791 * for small pages, there is no difference between
1792 * shift and PAGE_SHIFT, so the bit-shift is safe.
1793 * for huge pages, since vm_pgoff is in units of small
1794 * pages, we need to shift off the always 0 bits to get
1797 BUG_ON(shift
< PAGE_SHIFT
);
1798 off
= vma
->vm_pgoff
>> (shift
- PAGE_SHIFT
);
1799 off
+= (addr
- vma
->vm_start
) >> shift
;
1800 return offset_il_node(pol
, vma
, off
);
1802 return interleave_nodes(pol
);
1805 #ifdef CONFIG_HUGETLBFS
1807 * huge_zonelist(@vma, @addr, @gfp_flags, @mpol)
1808 * @vma: virtual memory area whose policy is sought
1809 * @addr: address in @vma for shared policy lookup and interleave policy
1810 * @gfp_flags: for requested zone
1811 * @mpol: pointer to mempolicy pointer for reference counted mempolicy
1812 * @nodemask: pointer to nodemask pointer for MPOL_BIND nodemask
1814 * Returns a zonelist suitable for a huge page allocation and a pointer
1815 * to the struct mempolicy for conditional unref after allocation.
1816 * If the effective policy is 'BIND, returns a pointer to the mempolicy's
1817 * @nodemask for filtering the zonelist.
1819 * Must be protected by read_mems_allowed_begin()
1821 struct zonelist
*huge_zonelist(struct vm_area_struct
*vma
, unsigned long addr
,
1822 gfp_t gfp_flags
, struct mempolicy
**mpol
,
1823 nodemask_t
**nodemask
)
1825 struct zonelist
*zl
;
1827 *mpol
= get_vma_policy(vma
, addr
);
1828 *nodemask
= NULL
; /* assume !MPOL_BIND */
1830 if (unlikely((*mpol
)->mode
== MPOL_INTERLEAVE
)) {
1831 zl
= node_zonelist(interleave_nid(*mpol
, vma
, addr
,
1832 huge_page_shift(hstate_vma(vma
))), gfp_flags
);
1834 zl
= policy_zonelist(gfp_flags
, *mpol
, numa_node_id());
1835 if ((*mpol
)->mode
== MPOL_BIND
)
1836 *nodemask
= &(*mpol
)->v
.nodes
;
1842 * init_nodemask_of_mempolicy
1844 * If the current task's mempolicy is "default" [NULL], return 'false'
1845 * to indicate default policy. Otherwise, extract the policy nodemask
1846 * for 'bind' or 'interleave' policy into the argument nodemask, or
1847 * initialize the argument nodemask to contain the single node for
1848 * 'preferred' or 'local' policy and return 'true' to indicate presence
1849 * of non-default mempolicy.
1851 * We don't bother with reference counting the mempolicy [mpol_get/put]
1852 * because the current task is examining it's own mempolicy and a task's
1853 * mempolicy is only ever changed by the task itself.
1855 * N.B., it is the caller's responsibility to free a returned nodemask.
1857 bool init_nodemask_of_mempolicy(nodemask_t
*mask
)
1859 struct mempolicy
*mempolicy
;
1862 if (!(mask
&& current
->mempolicy
))
1866 mempolicy
= current
->mempolicy
;
1867 switch (mempolicy
->mode
) {
1868 case MPOL_PREFERRED
:
1869 if (mempolicy
->flags
& MPOL_F_LOCAL
)
1870 nid
= numa_node_id();
1872 nid
= mempolicy
->v
.preferred_node
;
1873 init_nodemask_of_node(mask
, nid
);
1878 case MPOL_INTERLEAVE
:
1879 *mask
= mempolicy
->v
.nodes
;
1885 task_unlock(current
);
1892 * mempolicy_nodemask_intersects
1894 * If tsk's mempolicy is "default" [NULL], return 'true' to indicate default
1895 * policy. Otherwise, check for intersection between mask and the policy
1896 * nodemask for 'bind' or 'interleave' policy. For 'perferred' or 'local'
1897 * policy, always return true since it may allocate elsewhere on fallback.
1899 * Takes task_lock(tsk) to prevent freeing of its mempolicy.
1901 bool mempolicy_nodemask_intersects(struct task_struct
*tsk
,
1902 const nodemask_t
*mask
)
1904 struct mempolicy
*mempolicy
;
1910 mempolicy
= tsk
->mempolicy
;
1914 switch (mempolicy
->mode
) {
1915 case MPOL_PREFERRED
:
1917 * MPOL_PREFERRED and MPOL_F_LOCAL are only preferred nodes to
1918 * allocate from, they may fallback to other nodes when oom.
1919 * Thus, it's possible for tsk to have allocated memory from
1924 case MPOL_INTERLEAVE
:
1925 ret
= nodes_intersects(mempolicy
->v
.nodes
, *mask
);
1935 /* Allocate a page in interleaved policy.
1936 Own path because it needs to do special accounting. */
1937 static struct page
*alloc_page_interleave(gfp_t gfp
, unsigned order
,
1940 struct zonelist
*zl
;
1943 zl
= node_zonelist(nid
, gfp
);
1944 page
= __alloc_pages(gfp
, order
, zl
);
1945 if (page
&& page_zone(page
) == zonelist_zone(&zl
->_zonerefs
[0]))
1946 inc_zone_page_state(page
, NUMA_INTERLEAVE_HIT
);
1951 * alloc_pages_vma - Allocate a page for a VMA.
1954 * %GFP_USER user allocation.
1955 * %GFP_KERNEL kernel allocations,
1956 * %GFP_HIGHMEM highmem/user allocations,
1957 * %GFP_FS allocation should not call back into a file system.
1958 * %GFP_ATOMIC don't sleep.
1960 * @order:Order of the GFP allocation.
1961 * @vma: Pointer to VMA or NULL if not available.
1962 * @addr: Virtual Address of the allocation. Must be inside the VMA.
1963 * @node: Which node to prefer for allocation (modulo policy).
1964 * @hugepage: for hugepages try only the preferred node if possible
1966 * This function allocates a page from the kernel page pool and applies
1967 * a NUMA policy associated with the VMA or the current process.
1968 * When VMA is not NULL caller must hold down_read on the mmap_sem of the
1969 * mm_struct of the VMA to prevent it from going away. Should be used for
1970 * all allocations for pages that will be mapped into user space. Returns
1971 * NULL when no page can be allocated.
1974 alloc_pages_vma(gfp_t gfp
, int order
, struct vm_area_struct
*vma
,
1975 unsigned long addr
, int node
, bool hugepage
)
1977 struct mempolicy
*pol
;
1979 unsigned int cpuset_mems_cookie
;
1980 struct zonelist
*zl
;
1984 pol
= get_vma_policy(vma
, addr
);
1985 cpuset_mems_cookie
= read_mems_allowed_begin();
1987 if (pol
->mode
== MPOL_INTERLEAVE
) {
1990 nid
= interleave_nid(pol
, vma
, addr
, PAGE_SHIFT
+ order
);
1992 page
= alloc_page_interleave(gfp
, order
, nid
);
1996 if (unlikely(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE
) && hugepage
)) {
1997 int hpage_node
= node
;
2000 * For hugepage allocation and non-interleave policy which
2001 * allows the current node (or other explicitly preferred
2002 * node) we only try to allocate from the current/preferred
2003 * node and don't fall back to other nodes, as the cost of
2004 * remote accesses would likely offset THP benefits.
2006 * If the policy is interleave, or does not allow the current
2007 * node in its nodemask, we allocate the standard way.
2009 if (pol
->mode
== MPOL_PREFERRED
&&
2010 !(pol
->flags
& MPOL_F_LOCAL
))
2011 hpage_node
= pol
->v
.preferred_node
;
2013 nmask
= policy_nodemask(gfp
, pol
);
2014 if (!nmask
|| node_isset(hpage_node
, *nmask
)) {
2016 page
= __alloc_pages_node(hpage_node
,
2017 gfp
| __GFP_THISNODE
, order
);
2022 nmask
= policy_nodemask(gfp
, pol
);
2023 zl
= policy_zonelist(gfp
, pol
, node
);
2025 page
= __alloc_pages_nodemask(gfp
, order
, zl
, nmask
);
2027 if (unlikely(!page
&& read_mems_allowed_retry(cpuset_mems_cookie
)))
2033 * alloc_pages_current - Allocate pages.
2036 * %GFP_USER user allocation,
2037 * %GFP_KERNEL kernel allocation,
2038 * %GFP_HIGHMEM highmem allocation,
2039 * %GFP_FS don't call back into a file system.
2040 * %GFP_ATOMIC don't sleep.
2041 * @order: Power of two of allocation size in pages. 0 is a single page.
2043 * Allocate a page from the kernel page pool. When not in
2044 * interrupt context and apply the current process NUMA policy.
2045 * Returns NULL when no page can be allocated.
2047 * Don't call cpuset_update_task_memory_state() unless
2048 * 1) it's ok to take cpuset_sem (can WAIT), and
2049 * 2) allocating for current task (not interrupt).
2051 struct page
*alloc_pages_current(gfp_t gfp
, unsigned order
)
2053 struct mempolicy
*pol
= &default_policy
;
2055 unsigned int cpuset_mems_cookie
;
2057 if (!in_interrupt() && !(gfp
& __GFP_THISNODE
))
2058 pol
= get_task_policy(current
);
2061 cpuset_mems_cookie
= read_mems_allowed_begin();
2064 * No reference counting needed for current->mempolicy
2065 * nor system default_policy
2067 if (pol
->mode
== MPOL_INTERLEAVE
)
2068 page
= alloc_page_interleave(gfp
, order
, interleave_nodes(pol
));
2070 page
= __alloc_pages_nodemask(gfp
, order
,
2071 policy_zonelist(gfp
, pol
, numa_node_id()),
2072 policy_nodemask(gfp
, pol
));
2074 if (unlikely(!page
&& read_mems_allowed_retry(cpuset_mems_cookie
)))
2079 EXPORT_SYMBOL(alloc_pages_current
);
2081 int vma_dup_policy(struct vm_area_struct
*src
, struct vm_area_struct
*dst
)
2083 struct mempolicy
*pol
= mpol_dup(vma_policy(src
));
2086 return PTR_ERR(pol
);
2087 dst
->vm_policy
= pol
;
2092 * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
2093 * rebinds the mempolicy its copying by calling mpol_rebind_policy()
2094 * with the mems_allowed returned by cpuset_mems_allowed(). This
2095 * keeps mempolicies cpuset relative after its cpuset moves. See
2096 * further kernel/cpuset.c update_nodemask().
2098 * current's mempolicy may be rebinded by the other task(the task that changes
2099 * cpuset's mems), so we needn't do rebind work for current task.
2102 /* Slow path of a mempolicy duplicate */
2103 struct mempolicy
*__mpol_dup(struct mempolicy
*old
)
2105 struct mempolicy
*new = kmem_cache_alloc(policy_cache
, GFP_KERNEL
);
2108 return ERR_PTR(-ENOMEM
);
2110 /* task's mempolicy is protected by alloc_lock */
2111 if (old
== current
->mempolicy
) {
2114 task_unlock(current
);
2118 if (current_cpuset_is_being_rebound()) {
2119 nodemask_t mems
= cpuset_mems_allowed(current
);
2120 if (new->flags
& MPOL_F_REBINDING
)
2121 mpol_rebind_policy(new, &mems
, MPOL_REBIND_STEP2
);
2123 mpol_rebind_policy(new, &mems
, MPOL_REBIND_ONCE
);
2125 atomic_set(&new->refcnt
, 1);
2129 /* Slow path of a mempolicy comparison */
2130 bool __mpol_equal(struct mempolicy
*a
, struct mempolicy
*b
)
2134 if (a
->mode
!= b
->mode
)
2136 if (a
->flags
!= b
->flags
)
2138 if (mpol_store_user_nodemask(a
))
2139 if (!nodes_equal(a
->w
.user_nodemask
, b
->w
.user_nodemask
))
2145 case MPOL_INTERLEAVE
:
2146 return !!nodes_equal(a
->v
.nodes
, b
->v
.nodes
);
2147 case MPOL_PREFERRED
:
2148 return a
->v
.preferred_node
== b
->v
.preferred_node
;
2156 * Shared memory backing store policy support.
2158 * Remember policies even when nobody has shared memory mapped.
2159 * The policies are kept in Red-Black tree linked from the inode.
2160 * They are protected by the sp->lock rwlock, which should be held
2161 * for any accesses to the tree.
2165 * lookup first element intersecting start-end. Caller holds sp->lock for
2166 * reading or for writing
2168 static struct sp_node
*
2169 sp_lookup(struct shared_policy
*sp
, unsigned long start
, unsigned long end
)
2171 struct rb_node
*n
= sp
->root
.rb_node
;
2174 struct sp_node
*p
= rb_entry(n
, struct sp_node
, nd
);
2176 if (start
>= p
->end
)
2178 else if (end
<= p
->start
)
2186 struct sp_node
*w
= NULL
;
2187 struct rb_node
*prev
= rb_prev(n
);
2190 w
= rb_entry(prev
, struct sp_node
, nd
);
2191 if (w
->end
<= start
)
2195 return rb_entry(n
, struct sp_node
, nd
);
2199 * Insert a new shared policy into the list. Caller holds sp->lock for
2202 static void sp_insert(struct shared_policy
*sp
, struct sp_node
*new)
2204 struct rb_node
**p
= &sp
->root
.rb_node
;
2205 struct rb_node
*parent
= NULL
;
2210 nd
= rb_entry(parent
, struct sp_node
, nd
);
2211 if (new->start
< nd
->start
)
2213 else if (new->end
> nd
->end
)
2214 p
= &(*p
)->rb_right
;
2218 rb_link_node(&new->nd
, parent
, p
);
2219 rb_insert_color(&new->nd
, &sp
->root
);
2220 pr_debug("inserting %lx-%lx: %d\n", new->start
, new->end
,
2221 new->policy
? new->policy
->mode
: 0);
2224 /* Find shared policy intersecting idx */
2226 mpol_shared_policy_lookup(struct shared_policy
*sp
, unsigned long idx
)
2228 struct mempolicy
*pol
= NULL
;
2231 if (!sp
->root
.rb_node
)
2233 read_lock(&sp
->lock
);
2234 sn
= sp_lookup(sp
, idx
, idx
+1);
2236 mpol_get(sn
->policy
);
2239 read_unlock(&sp
->lock
);
2243 static void sp_free(struct sp_node
*n
)
2245 mpol_put(n
->policy
);
2246 kmem_cache_free(sn_cache
, n
);
2250 * mpol_misplaced - check whether current page node is valid in policy
2252 * @page: page to be checked
2253 * @vma: vm area where page mapped
2254 * @addr: virtual address where page mapped
2256 * Lookup current policy node id for vma,addr and "compare to" page's
2260 * -1 - not misplaced, page is in the right node
2261 * node - node id where the page should be
2263 * Policy determination "mimics" alloc_page_vma().
2264 * Called from fault path where we know the vma and faulting address.
2266 int mpol_misplaced(struct page
*page
, struct vm_area_struct
*vma
, unsigned long addr
)
2268 struct mempolicy
*pol
;
2270 int curnid
= page_to_nid(page
);
2271 unsigned long pgoff
;
2272 int thiscpu
= raw_smp_processor_id();
2273 int thisnid
= cpu_to_node(thiscpu
);
2279 pol
= get_vma_policy(vma
, addr
);
2280 if (!(pol
->flags
& MPOL_F_MOF
))
2283 switch (pol
->mode
) {
2284 case MPOL_INTERLEAVE
:
2285 BUG_ON(addr
>= vma
->vm_end
);
2286 BUG_ON(addr
< vma
->vm_start
);
2288 pgoff
= vma
->vm_pgoff
;
2289 pgoff
+= (addr
- vma
->vm_start
) >> PAGE_SHIFT
;
2290 polnid
= offset_il_node(pol
, vma
, pgoff
);
2293 case MPOL_PREFERRED
:
2294 if (pol
->flags
& MPOL_F_LOCAL
)
2295 polnid
= numa_node_id();
2297 polnid
= pol
->v
.preferred_node
;
2303 * allows binding to multiple nodes.
2304 * use current page if in policy nodemask,
2305 * else select nearest allowed node, if any.
2306 * If no allowed nodes, use current [!misplaced].
2308 if (node_isset(curnid
, pol
->v
.nodes
))
2310 z
= first_zones_zonelist(
2311 node_zonelist(numa_node_id(), GFP_HIGHUSER
),
2312 gfp_zone(GFP_HIGHUSER
),
2314 polnid
= z
->zone
->node
;
2321 /* Migrate the page towards the node whose CPU is referencing it */
2322 if (pol
->flags
& MPOL_F_MORON
) {
2325 if (!should_numa_migrate_memory(current
, page
, curnid
, thiscpu
))
2329 if (curnid
!= polnid
)
2337 static void sp_delete(struct shared_policy
*sp
, struct sp_node
*n
)
2339 pr_debug("deleting %lx-l%lx\n", n
->start
, n
->end
);
2340 rb_erase(&n
->nd
, &sp
->root
);
2344 static void sp_node_init(struct sp_node
*node
, unsigned long start
,
2345 unsigned long end
, struct mempolicy
*pol
)
2347 node
->start
= start
;
2352 static struct sp_node
*sp_alloc(unsigned long start
, unsigned long end
,
2353 struct mempolicy
*pol
)
2356 struct mempolicy
*newpol
;
2358 n
= kmem_cache_alloc(sn_cache
, GFP_KERNEL
);
2362 newpol
= mpol_dup(pol
);
2363 if (IS_ERR(newpol
)) {
2364 kmem_cache_free(sn_cache
, n
);
2367 newpol
->flags
|= MPOL_F_SHARED
;
2368 sp_node_init(n
, start
, end
, newpol
);
2373 /* Replace a policy range. */
2374 static int shared_policy_replace(struct shared_policy
*sp
, unsigned long start
,
2375 unsigned long end
, struct sp_node
*new)
2378 struct sp_node
*n_new
= NULL
;
2379 struct mempolicy
*mpol_new
= NULL
;
2383 write_lock(&sp
->lock
);
2384 n
= sp_lookup(sp
, start
, end
);
2385 /* Take care of old policies in the same range. */
2386 while (n
&& n
->start
< end
) {
2387 struct rb_node
*next
= rb_next(&n
->nd
);
2388 if (n
->start
>= start
) {
2394 /* Old policy spanning whole new range. */
2399 *mpol_new
= *n
->policy
;
2400 atomic_set(&mpol_new
->refcnt
, 1);
2401 sp_node_init(n_new
, end
, n
->end
, mpol_new
);
2403 sp_insert(sp
, n_new
);
2412 n
= rb_entry(next
, struct sp_node
, nd
);
2416 write_unlock(&sp
->lock
);
2423 kmem_cache_free(sn_cache
, n_new
);
2428 write_unlock(&sp
->lock
);
2430 n_new
= kmem_cache_alloc(sn_cache
, GFP_KERNEL
);
2433 mpol_new
= kmem_cache_alloc(policy_cache
, GFP_KERNEL
);
2440 * mpol_shared_policy_init - initialize shared policy for inode
2441 * @sp: pointer to inode shared policy
2442 * @mpol: struct mempolicy to install
2444 * Install non-NULL @mpol in inode's shared policy rb-tree.
2445 * On entry, the current task has a reference on a non-NULL @mpol.
2446 * This must be released on exit.
2447 * This is called at get_inode() calls and we can use GFP_KERNEL.
2449 void mpol_shared_policy_init(struct shared_policy
*sp
, struct mempolicy
*mpol
)
2453 sp
->root
= RB_ROOT
; /* empty tree == default mempolicy */
2454 rwlock_init(&sp
->lock
);
2457 struct vm_area_struct pvma
;
2458 struct mempolicy
*new;
2459 NODEMASK_SCRATCH(scratch
);
2463 /* contextualize the tmpfs mount point mempolicy */
2464 new = mpol_new(mpol
->mode
, mpol
->flags
, &mpol
->w
.user_nodemask
);
2466 goto free_scratch
; /* no valid nodemask intersection */
2469 ret
= mpol_set_nodemask(new, &mpol
->w
.user_nodemask
, scratch
);
2470 task_unlock(current
);
2474 /* Create pseudo-vma that contains just the policy */
2475 memset(&pvma
, 0, sizeof(struct vm_area_struct
));
2476 pvma
.vm_end
= TASK_SIZE
; /* policy covers entire file */
2477 mpol_set_shared_policy(sp
, &pvma
, new); /* adds ref */
2480 mpol_put(new); /* drop initial ref */
2482 NODEMASK_SCRATCH_FREE(scratch
);
2484 mpol_put(mpol
); /* drop our incoming ref on sb mpol */
2488 int mpol_set_shared_policy(struct shared_policy
*info
,
2489 struct vm_area_struct
*vma
, struct mempolicy
*npol
)
2492 struct sp_node
*new = NULL
;
2493 unsigned long sz
= vma_pages(vma
);
2495 pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n",
2497 sz
, npol
? npol
->mode
: -1,
2498 npol
? npol
->flags
: -1,
2499 npol
? nodes_addr(npol
->v
.nodes
)[0] : NUMA_NO_NODE
);
2502 new = sp_alloc(vma
->vm_pgoff
, vma
->vm_pgoff
+ sz
, npol
);
2506 err
= shared_policy_replace(info
, vma
->vm_pgoff
, vma
->vm_pgoff
+sz
, new);
2512 /* Free a backing policy store on inode delete. */
2513 void mpol_free_shared_policy(struct shared_policy
*p
)
2516 struct rb_node
*next
;
2518 if (!p
->root
.rb_node
)
2520 write_lock(&p
->lock
);
2521 next
= rb_first(&p
->root
);
2523 n
= rb_entry(next
, struct sp_node
, nd
);
2524 next
= rb_next(&n
->nd
);
2527 write_unlock(&p
->lock
);
2530 #ifdef CONFIG_NUMA_BALANCING
2531 static int __initdata numabalancing_override
;
2533 static void __init
check_numabalancing_enable(void)
2535 bool numabalancing_default
= false;
2537 if (IS_ENABLED(CONFIG_NUMA_BALANCING_DEFAULT_ENABLED
))
2538 numabalancing_default
= true;
2540 /* Parsed by setup_numabalancing. override == 1 enables, -1 disables */
2541 if (numabalancing_override
)
2542 set_numabalancing_state(numabalancing_override
== 1);
2544 if (num_online_nodes() > 1 && !numabalancing_override
) {
2545 pr_info("%s automatic NUMA balancing. Configure with numa_balancing= or the kernel.numa_balancing sysctl\n",
2546 numabalancing_default
? "Enabling" : "Disabling");
2547 set_numabalancing_state(numabalancing_default
);
2551 static int __init
setup_numabalancing(char *str
)
2557 if (!strcmp(str
, "enable")) {
2558 numabalancing_override
= 1;
2560 } else if (!strcmp(str
, "disable")) {
2561 numabalancing_override
= -1;
2566 pr_warn("Unable to parse numa_balancing=\n");
2570 __setup("numa_balancing=", setup_numabalancing
);
2572 static inline void __init
check_numabalancing_enable(void)
2575 #endif /* CONFIG_NUMA_BALANCING */
2577 /* assumes fs == KERNEL_DS */
2578 void __init
numa_policy_init(void)
2580 nodemask_t interleave_nodes
;
2581 unsigned long largest
= 0;
2582 int nid
, prefer
= 0;
2584 policy_cache
= kmem_cache_create("numa_policy",
2585 sizeof(struct mempolicy
),
2586 0, SLAB_PANIC
, NULL
);
2588 sn_cache
= kmem_cache_create("shared_policy_node",
2589 sizeof(struct sp_node
),
2590 0, SLAB_PANIC
, NULL
);
2592 for_each_node(nid
) {
2593 preferred_node_policy
[nid
] = (struct mempolicy
) {
2594 .refcnt
= ATOMIC_INIT(1),
2595 .mode
= MPOL_PREFERRED
,
2596 .flags
= MPOL_F_MOF
| MPOL_F_MORON
,
2597 .v
= { .preferred_node
= nid
, },
2602 * Set interleaving policy for system init. Interleaving is only
2603 * enabled across suitably sized nodes (default is >= 16MB), or
2604 * fall back to the largest node if they're all smaller.
2606 nodes_clear(interleave_nodes
);
2607 for_each_node_state(nid
, N_MEMORY
) {
2608 unsigned long total_pages
= node_present_pages(nid
);
2610 /* Preserve the largest node */
2611 if (largest
< total_pages
) {
2612 largest
= total_pages
;
2616 /* Interleave this node? */
2617 if ((total_pages
<< PAGE_SHIFT
) >= (16 << 20))
2618 node_set(nid
, interleave_nodes
);
2621 /* All too small, use the largest */
2622 if (unlikely(nodes_empty(interleave_nodes
)))
2623 node_set(prefer
, interleave_nodes
);
2625 if (do_set_mempolicy(MPOL_INTERLEAVE
, 0, &interleave_nodes
))
2626 pr_err("%s: interleaving failed\n", __func__
);
2628 check_numabalancing_enable();
2631 /* Reset policy of current process to default */
2632 void numa_default_policy(void)
2634 do_set_mempolicy(MPOL_DEFAULT
, 0, NULL
);
2638 * Parse and format mempolicy from/to strings
2642 * "local" is implemented internally by MPOL_PREFERRED with MPOL_F_LOCAL flag.
2644 static const char * const policy_modes
[] =
2646 [MPOL_DEFAULT
] = "default",
2647 [MPOL_PREFERRED
] = "prefer",
2648 [MPOL_BIND
] = "bind",
2649 [MPOL_INTERLEAVE
] = "interleave",
2650 [MPOL_LOCAL
] = "local",
2656 * mpol_parse_str - parse string to mempolicy, for tmpfs mpol mount option.
2657 * @str: string containing mempolicy to parse
2658 * @mpol: pointer to struct mempolicy pointer, returned on success.
2661 * <mode>[=<flags>][:<nodelist>]
2663 * On success, returns 0, else 1
2665 int mpol_parse_str(char *str
, struct mempolicy
**mpol
)
2667 struct mempolicy
*new = NULL
;
2668 unsigned short mode
;
2669 unsigned short mode_flags
;
2671 char *nodelist
= strchr(str
, ':');
2672 char *flags
= strchr(str
, '=');
2676 /* NUL-terminate mode or flags string */
2678 if (nodelist_parse(nodelist
, nodes
))
2680 if (!nodes_subset(nodes
, node_states
[N_MEMORY
]))
2686 *flags
++ = '\0'; /* terminate mode string */
2688 for (mode
= 0; mode
< MPOL_MAX
; mode
++) {
2689 if (!strcmp(str
, policy_modes
[mode
])) {
2693 if (mode
>= MPOL_MAX
)
2697 case MPOL_PREFERRED
:
2699 * Insist on a nodelist of one node only
2702 char *rest
= nodelist
;
2703 while (isdigit(*rest
))
2709 case MPOL_INTERLEAVE
:
2711 * Default to online nodes with memory if no nodelist
2714 nodes
= node_states
[N_MEMORY
];
2718 * Don't allow a nodelist; mpol_new() checks flags
2722 mode
= MPOL_PREFERRED
;
2726 * Insist on a empty nodelist
2733 * Insist on a nodelist
2742 * Currently, we only support two mutually exclusive
2745 if (!strcmp(flags
, "static"))
2746 mode_flags
|= MPOL_F_STATIC_NODES
;
2747 else if (!strcmp(flags
, "relative"))
2748 mode_flags
|= MPOL_F_RELATIVE_NODES
;
2753 new = mpol_new(mode
, mode_flags
, &nodes
);
2758 * Save nodes for mpol_to_str() to show the tmpfs mount options
2759 * for /proc/mounts, /proc/pid/mounts and /proc/pid/mountinfo.
2761 if (mode
!= MPOL_PREFERRED
)
2762 new->v
.nodes
= nodes
;
2764 new->v
.preferred_node
= first_node(nodes
);
2766 new->flags
|= MPOL_F_LOCAL
;
2769 * Save nodes for contextualization: this will be used to "clone"
2770 * the mempolicy in a specific context [cpuset] at a later time.
2772 new->w
.user_nodemask
= nodes
;
2777 /* Restore string for error message */
2786 #endif /* CONFIG_TMPFS */
2789 * mpol_to_str - format a mempolicy structure for printing
2790 * @buffer: to contain formatted mempolicy string
2791 * @maxlen: length of @buffer
2792 * @pol: pointer to mempolicy to be formatted
2794 * Convert @pol into a string. If @buffer is too short, truncate the string.
2795 * Recommend a @maxlen of at least 32 for the longest mode, "interleave", the
2796 * longest flag, "relative", and to display at least a few node ids.
2798 void mpol_to_str(char *buffer
, int maxlen
, struct mempolicy
*pol
)
2801 nodemask_t nodes
= NODE_MASK_NONE
;
2802 unsigned short mode
= MPOL_DEFAULT
;
2803 unsigned short flags
= 0;
2805 if (pol
&& pol
!= &default_policy
&& !(pol
->flags
& MPOL_F_MORON
)) {
2813 case MPOL_PREFERRED
:
2814 if (flags
& MPOL_F_LOCAL
)
2817 node_set(pol
->v
.preferred_node
, nodes
);
2820 case MPOL_INTERLEAVE
:
2821 nodes
= pol
->v
.nodes
;
2825 snprintf(p
, maxlen
, "unknown");
2829 p
+= snprintf(p
, maxlen
, "%s", policy_modes
[mode
]);
2831 if (flags
& MPOL_MODE_FLAGS
) {
2832 p
+= snprintf(p
, buffer
+ maxlen
- p
, "=");
2835 * Currently, the only defined flags are mutually exclusive
2837 if (flags
& MPOL_F_STATIC_NODES
)
2838 p
+= snprintf(p
, buffer
+ maxlen
- p
, "static");
2839 else if (flags
& MPOL_F_RELATIVE_NODES
)
2840 p
+= snprintf(p
, buffer
+ maxlen
- p
, "relative");
2843 if (!nodes_empty(nodes
))
2844 p
+= scnprintf(p
, buffer
+ maxlen
- p
, ":%*pbl",
2845 nodemask_pr_args(&nodes
));