4 * Processor and Memory placement constraints for sets of tasks.
6 * Copyright (C) 2003 BULL SA.
7 * Copyright (C) 2004-2006 Silicon Graphics, Inc.
9 * Portions derived from Patrick Mochel's sysfs code.
10 * sysfs is Copyright (c) 2001-3 Patrick Mochel
12 * 2003-10-10 Written by Simon Derr.
13 * 2003-10-22 Updates by Stephen Hemminger.
14 * 2004 May-July Rework by Paul Jackson.
16 * This file is subject to the terms and conditions of the GNU General Public
17 * License. See the file COPYING in the main directory of the Linux
18 * distribution for more details.
21 #include <linux/config.h>
22 #include <linux/cpu.h>
23 #include <linux/cpumask.h>
24 #include <linux/cpuset.h>
25 #include <linux/err.h>
26 #include <linux/errno.h>
27 #include <linux/file.h>
29 #include <linux/init.h>
30 #include <linux/interrupt.h>
31 #include <linux/kernel.h>
32 #include <linux/kmod.h>
33 #include <linux/list.h>
34 #include <linux/mempolicy.h>
36 #include <linux/module.h>
37 #include <linux/mount.h>
38 #include <linux/namei.h>
39 #include <linux/pagemap.h>
40 #include <linux/proc_fs.h>
41 #include <linux/rcupdate.h>
42 #include <linux/sched.h>
43 #include <linux/seq_file.h>
44 #include <linux/slab.h>
45 #include <linux/smp_lock.h>
46 #include <linux/spinlock.h>
47 #include <linux/stat.h>
48 #include <linux/string.h>
49 #include <linux/time.h>
50 #include <linux/backing-dev.h>
51 #include <linux/sort.h>
53 #include <asm/uaccess.h>
54 #include <asm/atomic.h>
55 #include <linux/mutex.h>
57 #define CPUSET_SUPER_MAGIC 0x27e0eb
60 * Tracks how many cpusets are currently defined in system.
61 * When there is only one cpuset (the root cpuset) we can
62 * short circuit some hooks.
64 int number_of_cpusets __read_mostly
;
66 /* See "Frequency meter" comments, below. */
69 int cnt
; /* unprocessed events count */
70 int val
; /* most recent output value */
71 time_t time
; /* clock (secs) when val computed */
72 spinlock_t lock
; /* guards read or write of above */
76 unsigned long flags
; /* "unsigned long" so bitops work */
77 cpumask_t cpus_allowed
; /* CPUs allowed to tasks in cpuset */
78 nodemask_t mems_allowed
; /* Memory Nodes allowed to tasks */
81 * Count is atomic so can incr (fork) or decr (exit) without a lock.
83 atomic_t count
; /* count tasks using this cpuset */
86 * We link our 'sibling' struct into our parents 'children'.
87 * Our children link their 'sibling' into our 'children'.
89 struct list_head sibling
; /* my parents children */
90 struct list_head children
; /* my children */
92 struct cpuset
*parent
; /* my parent */
93 struct dentry
*dentry
; /* cpuset fs entry */
96 * Copy of global cpuset_mems_generation as of the most
97 * recent time this cpuset changed its mems_allowed.
101 struct fmeter fmeter
; /* memory_pressure filter */
104 /* bits in struct cpuset flags field */
110 CS_NOTIFY_ON_RELEASE
,
115 /* convenient tests for these bits */
116 static inline int is_cpu_exclusive(const struct cpuset
*cs
)
118 return test_bit(CS_CPU_EXCLUSIVE
, &cs
->flags
);
121 static inline int is_mem_exclusive(const struct cpuset
*cs
)
123 return test_bit(CS_MEM_EXCLUSIVE
, &cs
->flags
);
126 static inline int is_removed(const struct cpuset
*cs
)
128 return test_bit(CS_REMOVED
, &cs
->flags
);
131 static inline int notify_on_release(const struct cpuset
*cs
)
133 return test_bit(CS_NOTIFY_ON_RELEASE
, &cs
->flags
);
136 static inline int is_memory_migrate(const struct cpuset
*cs
)
138 return test_bit(CS_MEMORY_MIGRATE
, &cs
->flags
);
141 static inline int is_spread_page(const struct cpuset
*cs
)
143 return test_bit(CS_SPREAD_PAGE
, &cs
->flags
);
146 static inline int is_spread_slab(const struct cpuset
*cs
)
148 return test_bit(CS_SPREAD_SLAB
, &cs
->flags
);
152 * Increment this atomic integer everytime any cpuset changes its
153 * mems_allowed value. Users of cpusets can track this generation
154 * number, and avoid having to lock and reload mems_allowed unless
155 * the cpuset they're using changes generation.
157 * A single, global generation is needed because attach_task() could
158 * reattach a task to a different cpuset, which must not have its
159 * generation numbers aliased with those of that tasks previous cpuset.
161 * Generations are needed for mems_allowed because one task cannot
162 * modify anothers memory placement. So we must enable every task,
163 * on every visit to __alloc_pages(), to efficiently check whether
164 * its current->cpuset->mems_allowed has changed, requiring an update
165 * of its current->mems_allowed.
167 static atomic_t cpuset_mems_generation
= ATOMIC_INIT(1);
169 static struct cpuset top_cpuset
= {
170 .flags
= ((1 << CS_CPU_EXCLUSIVE
) | (1 << CS_MEM_EXCLUSIVE
)),
171 .cpus_allowed
= CPU_MASK_ALL
,
172 .mems_allowed
= NODE_MASK_ALL
,
173 .count
= ATOMIC_INIT(0),
174 .sibling
= LIST_HEAD_INIT(top_cpuset
.sibling
),
175 .children
= LIST_HEAD_INIT(top_cpuset
.children
),
178 static struct vfsmount
*cpuset_mount
;
179 static struct super_block
*cpuset_sb
;
182 * We have two global cpuset mutexes below. They can nest.
183 * It is ok to first take manage_mutex, then nest callback_mutex. We also
184 * require taking task_lock() when dereferencing a tasks cpuset pointer.
185 * See "The task_lock() exception", at the end of this comment.
187 * A task must hold both mutexes to modify cpusets. If a task
188 * holds manage_mutex, then it blocks others wanting that mutex,
189 * ensuring that it is the only task able to also acquire callback_mutex
190 * and be able to modify cpusets. It can perform various checks on
191 * the cpuset structure first, knowing nothing will change. It can
192 * also allocate memory while just holding manage_mutex. While it is
193 * performing these checks, various callback routines can briefly
194 * acquire callback_mutex to query cpusets. Once it is ready to make
195 * the changes, it takes callback_mutex, blocking everyone else.
197 * Calls to the kernel memory allocator can not be made while holding
198 * callback_mutex, as that would risk double tripping on callback_mutex
199 * from one of the callbacks into the cpuset code from within
202 * If a task is only holding callback_mutex, then it has read-only
205 * The task_struct fields mems_allowed and mems_generation may only
206 * be accessed in the context of that task, so require no locks.
208 * Any task can increment and decrement the count field without lock.
209 * So in general, code holding manage_mutex or callback_mutex can't rely
210 * on the count field not changing. However, if the count goes to
211 * zero, then only attach_task(), which holds both mutexes, can
212 * increment it again. Because a count of zero means that no tasks
213 * are currently attached, therefore there is no way a task attached
214 * to that cpuset can fork (the other way to increment the count).
215 * So code holding manage_mutex or callback_mutex can safely assume that
216 * if the count is zero, it will stay zero. Similarly, if a task
217 * holds manage_mutex or callback_mutex on a cpuset with zero count, it
218 * knows that the cpuset won't be removed, as cpuset_rmdir() needs
219 * both of those mutexes.
221 * The cpuset_common_file_write handler for operations that modify
222 * the cpuset hierarchy holds manage_mutex across the entire operation,
223 * single threading all such cpuset modifications across the system.
225 * The cpuset_common_file_read() handlers only hold callback_mutex across
226 * small pieces of code, such as when reading out possibly multi-word
227 * cpumasks and nodemasks.
229 * The fork and exit callbacks cpuset_fork() and cpuset_exit(), don't
230 * (usually) take either mutex. These are the two most performance
231 * critical pieces of code here. The exception occurs on cpuset_exit(),
232 * when a task in a notify_on_release cpuset exits. Then manage_mutex
233 * is taken, and if the cpuset count is zero, a usermode call made
234 * to /sbin/cpuset_release_agent with the name of the cpuset (path
235 * relative to the root of cpuset file system) as the argument.
237 * A cpuset can only be deleted if both its 'count' of using tasks
238 * is zero, and its list of 'children' cpusets is empty. Since all
239 * tasks in the system use _some_ cpuset, and since there is always at
240 * least one task in the system (init, pid == 1), therefore, top_cpuset
241 * always has either children cpusets and/or using tasks. So we don't
242 * need a special hack to ensure that top_cpuset cannot be deleted.
244 * The above "Tale of Two Semaphores" would be complete, but for:
246 * The task_lock() exception
248 * The need for this exception arises from the action of attach_task(),
249 * which overwrites one tasks cpuset pointer with another. It does
250 * so using both mutexes, however there are several performance
251 * critical places that need to reference task->cpuset without the
252 * expense of grabbing a system global mutex. Therefore except as
253 * noted below, when dereferencing or, as in attach_task(), modifying
254 * a tasks cpuset pointer we use task_lock(), which acts on a spinlock
255 * (task->alloc_lock) already in the task_struct routinely used for
258 * P.S. One more locking exception. RCU is used to guard the
259 * update of a tasks cpuset pointer by attach_task() and the
260 * access of task->cpuset->mems_generation via that pointer in
261 * the routine cpuset_update_task_memory_state().
264 static DEFINE_MUTEX(manage_mutex
);
265 static DEFINE_MUTEX(callback_mutex
);
268 * A couple of forward declarations required, due to cyclic reference loop:
269 * cpuset_mkdir -> cpuset_create -> cpuset_populate_dir -> cpuset_add_file
270 * -> cpuset_create_file -> cpuset_dir_inode_operations -> cpuset_mkdir.
273 static int cpuset_mkdir(struct inode
*dir
, struct dentry
*dentry
, int mode
);
274 static int cpuset_rmdir(struct inode
*unused_dir
, struct dentry
*dentry
);
276 static struct backing_dev_info cpuset_backing_dev_info
= {
277 .ra_pages
= 0, /* No readahead */
278 .capabilities
= BDI_CAP_NO_ACCT_DIRTY
| BDI_CAP_NO_WRITEBACK
,
281 static struct inode
*cpuset_new_inode(mode_t mode
)
283 struct inode
*inode
= new_inode(cpuset_sb
);
286 inode
->i_mode
= mode
;
287 inode
->i_uid
= current
->fsuid
;
288 inode
->i_gid
= current
->fsgid
;
289 inode
->i_blksize
= PAGE_CACHE_SIZE
;
291 inode
->i_atime
= inode
->i_mtime
= inode
->i_ctime
= CURRENT_TIME
;
292 inode
->i_mapping
->backing_dev_info
= &cpuset_backing_dev_info
;
297 static void cpuset_diput(struct dentry
*dentry
, struct inode
*inode
)
299 /* is dentry a directory ? if so, kfree() associated cpuset */
300 if (S_ISDIR(inode
->i_mode
)) {
301 struct cpuset
*cs
= dentry
->d_fsdata
;
302 BUG_ON(!(is_removed(cs
)));
308 static struct dentry_operations cpuset_dops
= {
309 .d_iput
= cpuset_diput
,
312 static struct dentry
*cpuset_get_dentry(struct dentry
*parent
, const char *name
)
314 struct dentry
*d
= lookup_one_len(name
, parent
, strlen(name
));
316 d
->d_op
= &cpuset_dops
;
320 static void remove_dir(struct dentry
*d
)
322 struct dentry
*parent
= dget(d
->d_parent
);
325 simple_rmdir(parent
->d_inode
, d
);
330 * NOTE : the dentry must have been dget()'ed
332 static void cpuset_d_remove_dir(struct dentry
*dentry
)
334 struct list_head
*node
;
336 spin_lock(&dcache_lock
);
337 node
= dentry
->d_subdirs
.next
;
338 while (node
!= &dentry
->d_subdirs
) {
339 struct dentry
*d
= list_entry(node
, struct dentry
, d_u
.d_child
);
343 spin_unlock(&dcache_lock
);
345 simple_unlink(dentry
->d_inode
, d
);
347 spin_lock(&dcache_lock
);
349 node
= dentry
->d_subdirs
.next
;
351 list_del_init(&dentry
->d_u
.d_child
);
352 spin_unlock(&dcache_lock
);
356 static struct super_operations cpuset_ops
= {
357 .statfs
= simple_statfs
,
358 .drop_inode
= generic_delete_inode
,
361 static int cpuset_fill_super(struct super_block
*sb
, void *unused_data
,
367 sb
->s_blocksize
= PAGE_CACHE_SIZE
;
368 sb
->s_blocksize_bits
= PAGE_CACHE_SHIFT
;
369 sb
->s_magic
= CPUSET_SUPER_MAGIC
;
370 sb
->s_op
= &cpuset_ops
;
373 inode
= cpuset_new_inode(S_IFDIR
| S_IRUGO
| S_IXUGO
| S_IWUSR
);
375 inode
->i_op
= &simple_dir_inode_operations
;
376 inode
->i_fop
= &simple_dir_operations
;
377 /* directories start off with i_nlink == 2 (for "." entry) */
383 root
= d_alloc_root(inode
);
392 static struct super_block
*cpuset_get_sb(struct file_system_type
*fs_type
,
393 int flags
, const char *unused_dev_name
,
396 return get_sb_single(fs_type
, flags
, data
, cpuset_fill_super
);
399 static struct file_system_type cpuset_fs_type
= {
401 .get_sb
= cpuset_get_sb
,
402 .kill_sb
= kill_litter_super
,
407 * The files in the cpuset filesystem mostly have a very simple read/write
408 * handling, some common function will take care of it. Nevertheless some cases
409 * (read tasks) are special and therefore I define this structure for every
413 * When reading/writing to a file:
414 * - the cpuset to use in file->f_dentry->d_parent->d_fsdata
415 * - the 'cftype' of the file is file->f_dentry->d_fsdata
421 int (*open
) (struct inode
*inode
, struct file
*file
);
422 ssize_t (*read
) (struct file
*file
, char __user
*buf
, size_t nbytes
,
424 int (*write
) (struct file
*file
, const char __user
*buf
, size_t nbytes
,
426 int (*release
) (struct inode
*inode
, struct file
*file
);
429 static inline struct cpuset
*__d_cs(struct dentry
*dentry
)
431 return dentry
->d_fsdata
;
434 static inline struct cftype
*__d_cft(struct dentry
*dentry
)
436 return dentry
->d_fsdata
;
440 * Call with manage_mutex held. Writes path of cpuset into buf.
441 * Returns 0 on success, -errno on error.
444 static int cpuset_path(const struct cpuset
*cs
, char *buf
, int buflen
)
448 start
= buf
+ buflen
;
452 int len
= cs
->dentry
->d_name
.len
;
453 if ((start
-= len
) < buf
)
454 return -ENAMETOOLONG
;
455 memcpy(start
, cs
->dentry
->d_name
.name
, len
);
462 return -ENAMETOOLONG
;
465 memmove(buf
, start
, buf
+ buflen
- start
);
470 * Notify userspace when a cpuset is released, by running
471 * /sbin/cpuset_release_agent with the name of the cpuset (path
472 * relative to the root of cpuset file system) as the argument.
474 * Most likely, this user command will try to rmdir this cpuset.
476 * This races with the possibility that some other task will be
477 * attached to this cpuset before it is removed, or that some other
478 * user task will 'mkdir' a child cpuset of this cpuset. That's ok.
479 * The presumed 'rmdir' will fail quietly if this cpuset is no longer
480 * unused, and this cpuset will be reprieved from its death sentence,
481 * to continue to serve a useful existence. Next time it's released,
482 * we will get notified again, if it still has 'notify_on_release' set.
484 * The final arg to call_usermodehelper() is 0, which means don't
485 * wait. The separate /sbin/cpuset_release_agent task is forked by
486 * call_usermodehelper(), then control in this thread returns here,
487 * without waiting for the release agent task. We don't bother to
488 * wait because the caller of this routine has no use for the exit
489 * status of the /sbin/cpuset_release_agent task, so no sense holding
490 * our caller up for that.
492 * When we had only one cpuset mutex, we had to call this
493 * without holding it, to avoid deadlock when call_usermodehelper()
494 * allocated memory. With two locks, we could now call this while
495 * holding manage_mutex, but we still don't, so as to minimize
496 * the time manage_mutex is held.
499 static void cpuset_release_agent(const char *pathbuf
)
501 char *argv
[3], *envp
[3];
508 argv
[i
++] = "/sbin/cpuset_release_agent";
509 argv
[i
++] = (char *)pathbuf
;
513 /* minimal command environment */
514 envp
[i
++] = "HOME=/";
515 envp
[i
++] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
518 call_usermodehelper(argv
[0], argv
, envp
, 0);
523 * Either cs->count of using tasks transitioned to zero, or the
524 * cs->children list of child cpusets just became empty. If this
525 * cs is notify_on_release() and now both the user count is zero and
526 * the list of children is empty, prepare cpuset path in a kmalloc'd
527 * buffer, to be returned via ppathbuf, so that the caller can invoke
528 * cpuset_release_agent() with it later on, once manage_mutex is dropped.
529 * Call here with manage_mutex held.
531 * This check_for_release() routine is responsible for kmalloc'ing
532 * pathbuf. The above cpuset_release_agent() is responsible for
533 * kfree'ing pathbuf. The caller of these routines is responsible
534 * for providing a pathbuf pointer, initialized to NULL, then
535 * calling check_for_release() with manage_mutex held and the address
536 * of the pathbuf pointer, then dropping manage_mutex, then calling
537 * cpuset_release_agent() with pathbuf, as set by check_for_release().
540 static void check_for_release(struct cpuset
*cs
, char **ppathbuf
)
542 if (notify_on_release(cs
) && atomic_read(&cs
->count
) == 0 &&
543 list_empty(&cs
->children
)) {
546 buf
= kmalloc(PAGE_SIZE
, GFP_KERNEL
);
549 if (cpuset_path(cs
, buf
, PAGE_SIZE
) < 0)
557 * Return in *pmask the portion of a cpusets's cpus_allowed that
558 * are online. If none are online, walk up the cpuset hierarchy
559 * until we find one that does have some online cpus. If we get
560 * all the way to the top and still haven't found any online cpus,
561 * return cpu_online_map. Or if passed a NULL cs from an exit'ing
562 * task, return cpu_online_map.
564 * One way or another, we guarantee to return some non-empty subset
567 * Call with callback_mutex held.
570 static void guarantee_online_cpus(const struct cpuset
*cs
, cpumask_t
*pmask
)
572 while (cs
&& !cpus_intersects(cs
->cpus_allowed
, cpu_online_map
))
575 cpus_and(*pmask
, cs
->cpus_allowed
, cpu_online_map
);
577 *pmask
= cpu_online_map
;
578 BUG_ON(!cpus_intersects(*pmask
, cpu_online_map
));
582 * Return in *pmask the portion of a cpusets's mems_allowed that
583 * are online. If none are online, walk up the cpuset hierarchy
584 * until we find one that does have some online mems. If we get
585 * all the way to the top and still haven't found any online mems,
586 * return node_online_map.
588 * One way or another, we guarantee to return some non-empty subset
589 * of node_online_map.
591 * Call with callback_mutex held.
594 static void guarantee_online_mems(const struct cpuset
*cs
, nodemask_t
*pmask
)
596 while (cs
&& !nodes_intersects(cs
->mems_allowed
, node_online_map
))
599 nodes_and(*pmask
, cs
->mems_allowed
, node_online_map
);
601 *pmask
= node_online_map
;
602 BUG_ON(!nodes_intersects(*pmask
, node_online_map
));
606 * cpuset_update_task_memory_state - update task memory placement
608 * If the current tasks cpusets mems_allowed changed behind our
609 * backs, update current->mems_allowed, mems_generation and task NUMA
610 * mempolicy to the new value.
612 * Task mempolicy is updated by rebinding it relative to the
613 * current->cpuset if a task has its memory placement changed.
614 * Do not call this routine if in_interrupt().
616 * Call without callback_mutex or task_lock() held. May be called
617 * with or without manage_mutex held. Doesn't need task_lock to guard
618 * against another task changing a non-NULL cpuset pointer to NULL,
619 * as that is only done by a task on itself, and if the current task
620 * is here, it is not simultaneously in the exit code NULL'ing its
621 * cpuset pointer. This routine also might acquire callback_mutex and
622 * current->mm->mmap_sem during call.
624 * Reading current->cpuset->mems_generation doesn't need task_lock
625 * to guard the current->cpuset derefence, because it is guarded
626 * from concurrent freeing of current->cpuset by attach_task(),
629 * The rcu_dereference() is technically probably not needed,
630 * as I don't actually mind if I see a new cpuset pointer but
631 * an old value of mems_generation. However this really only
632 * matters on alpha systems using cpusets heavily. If I dropped
633 * that rcu_dereference(), it would save them a memory barrier.
634 * For all other arch's, rcu_dereference is a no-op anyway, and for
635 * alpha systems not using cpusets, another planned optimization,
636 * avoiding the rcu critical section for tasks in the root cpuset
637 * which is statically allocated, so can't vanish, will make this
638 * irrelevant. Better to use RCU as intended, than to engage in
639 * some cute trick to save a memory barrier that is impossible to
640 * test, for alpha systems using cpusets heavily, which might not
643 * This routine is needed to update the per-task mems_allowed data,
644 * within the tasks context, when it is trying to allocate memory
645 * (in various mm/mempolicy.c routines) and notices that some other
646 * task has been modifying its cpuset.
649 void cpuset_update_task_memory_state(void)
651 int my_cpusets_mem_gen
;
652 struct task_struct
*tsk
= current
;
655 if (tsk
->cpuset
== &top_cpuset
) {
656 /* Don't need rcu for top_cpuset. It's never freed. */
657 my_cpusets_mem_gen
= top_cpuset
.mems_generation
;
660 cs
= rcu_dereference(tsk
->cpuset
);
661 my_cpusets_mem_gen
= cs
->mems_generation
;
665 if (my_cpusets_mem_gen
!= tsk
->cpuset_mems_generation
) {
666 mutex_lock(&callback_mutex
);
668 cs
= tsk
->cpuset
; /* Maybe changed when task not locked */
669 guarantee_online_mems(cs
, &tsk
->mems_allowed
);
670 tsk
->cpuset_mems_generation
= cs
->mems_generation
;
671 if (is_spread_page(cs
))
672 tsk
->flags
|= PF_SPREAD_PAGE
;
674 tsk
->flags
&= ~PF_SPREAD_PAGE
;
675 if (is_spread_slab(cs
))
676 tsk
->flags
|= PF_SPREAD_SLAB
;
678 tsk
->flags
&= ~PF_SPREAD_SLAB
;
680 mutex_unlock(&callback_mutex
);
681 mpol_rebind_task(tsk
, &tsk
->mems_allowed
);
686 * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q?
688 * One cpuset is a subset of another if all its allowed CPUs and
689 * Memory Nodes are a subset of the other, and its exclusive flags
690 * are only set if the other's are set. Call holding manage_mutex.
693 static int is_cpuset_subset(const struct cpuset
*p
, const struct cpuset
*q
)
695 return cpus_subset(p
->cpus_allowed
, q
->cpus_allowed
) &&
696 nodes_subset(p
->mems_allowed
, q
->mems_allowed
) &&
697 is_cpu_exclusive(p
) <= is_cpu_exclusive(q
) &&
698 is_mem_exclusive(p
) <= is_mem_exclusive(q
);
702 * validate_change() - Used to validate that any proposed cpuset change
703 * follows the structural rules for cpusets.
705 * If we replaced the flag and mask values of the current cpuset
706 * (cur) with those values in the trial cpuset (trial), would
707 * our various subset and exclusive rules still be valid? Presumes
710 * 'cur' is the address of an actual, in-use cpuset. Operations
711 * such as list traversal that depend on the actual address of the
712 * cpuset in the list must use cur below, not trial.
714 * 'trial' is the address of bulk structure copy of cur, with
715 * perhaps one or more of the fields cpus_allowed, mems_allowed,
716 * or flags changed to new, trial values.
718 * Return 0 if valid, -errno if not.
721 static int validate_change(const struct cpuset
*cur
, const struct cpuset
*trial
)
723 struct cpuset
*c
, *par
;
725 /* Each of our child cpusets must be a subset of us */
726 list_for_each_entry(c
, &cur
->children
, sibling
) {
727 if (!is_cpuset_subset(c
, trial
))
731 /* Remaining checks don't apply to root cpuset */
732 if ((par
= cur
->parent
) == NULL
)
735 /* We must be a subset of our parent cpuset */
736 if (!is_cpuset_subset(trial
, par
))
739 /* If either I or some sibling (!= me) is exclusive, we can't overlap */
740 list_for_each_entry(c
, &par
->children
, sibling
) {
741 if ((is_cpu_exclusive(trial
) || is_cpu_exclusive(c
)) &&
743 cpus_intersects(trial
->cpus_allowed
, c
->cpus_allowed
))
745 if ((is_mem_exclusive(trial
) || is_mem_exclusive(c
)) &&
747 nodes_intersects(trial
->mems_allowed
, c
->mems_allowed
))
755 * For a given cpuset cur, partition the system as follows
756 * a. All cpus in the parent cpuset's cpus_allowed that are not part of any
757 * exclusive child cpusets
758 * b. All cpus in the current cpuset's cpus_allowed that are not part of any
759 * exclusive child cpusets
760 * Build these two partitions by calling partition_sched_domains
762 * Call with manage_mutex held. May nest a call to the
763 * lock_cpu_hotplug()/unlock_cpu_hotplug() pair.
766 static void update_cpu_domains(struct cpuset
*cur
)
768 struct cpuset
*c
, *par
= cur
->parent
;
769 cpumask_t pspan
, cspan
;
771 if (par
== NULL
|| cpus_empty(cur
->cpus_allowed
))
775 * Get all cpus from parent's cpus_allowed not part of exclusive
778 pspan
= par
->cpus_allowed
;
779 list_for_each_entry(c
, &par
->children
, sibling
) {
780 if (is_cpu_exclusive(c
))
781 cpus_andnot(pspan
, pspan
, c
->cpus_allowed
);
783 if (is_removed(cur
) || !is_cpu_exclusive(cur
)) {
784 cpus_or(pspan
, pspan
, cur
->cpus_allowed
);
785 if (cpus_equal(pspan
, cur
->cpus_allowed
))
787 cspan
= CPU_MASK_NONE
;
789 if (cpus_empty(pspan
))
791 cspan
= cur
->cpus_allowed
;
793 * Get all cpus from current cpuset's cpus_allowed not part
794 * of exclusive children
796 list_for_each_entry(c
, &cur
->children
, sibling
) {
797 if (is_cpu_exclusive(c
))
798 cpus_andnot(cspan
, cspan
, c
->cpus_allowed
);
803 partition_sched_domains(&pspan
, &cspan
);
804 unlock_cpu_hotplug();
808 * Call with manage_mutex held. May take callback_mutex during call.
811 static int update_cpumask(struct cpuset
*cs
, char *buf
)
813 struct cpuset trialcs
;
814 int retval
, cpus_unchanged
;
817 retval
= cpulist_parse(buf
, trialcs
.cpus_allowed
);
820 cpus_and(trialcs
.cpus_allowed
, trialcs
.cpus_allowed
, cpu_online_map
);
821 if (cpus_empty(trialcs
.cpus_allowed
))
823 retval
= validate_change(cs
, &trialcs
);
826 cpus_unchanged
= cpus_equal(cs
->cpus_allowed
, trialcs
.cpus_allowed
);
827 mutex_lock(&callback_mutex
);
828 cs
->cpus_allowed
= trialcs
.cpus_allowed
;
829 mutex_unlock(&callback_mutex
);
830 if (is_cpu_exclusive(cs
) && !cpus_unchanged
)
831 update_cpu_domains(cs
);
836 * Handle user request to change the 'mems' memory placement
837 * of a cpuset. Needs to validate the request, update the
838 * cpusets mems_allowed and mems_generation, and for each
839 * task in the cpuset, rebind any vma mempolicies and if
840 * the cpuset is marked 'memory_migrate', migrate the tasks
841 * pages to the new memory.
843 * Call with manage_mutex held. May take callback_mutex during call.
844 * Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
845 * lock each such tasks mm->mmap_sem, scan its vma's and rebind
846 * their mempolicies to the cpusets new mems_allowed.
849 static int update_nodemask(struct cpuset
*cs
, char *buf
)
851 struct cpuset trialcs
;
853 struct task_struct
*g
, *p
;
854 struct mm_struct
**mmarray
;
861 retval
= nodelist_parse(buf
, trialcs
.mems_allowed
);
864 nodes_and(trialcs
.mems_allowed
, trialcs
.mems_allowed
, node_online_map
);
865 oldmem
= cs
->mems_allowed
;
866 if (nodes_equal(oldmem
, trialcs
.mems_allowed
)) {
867 retval
= 0; /* Too easy - nothing to do */
870 if (nodes_empty(trialcs
.mems_allowed
)) {
874 retval
= validate_change(cs
, &trialcs
);
878 mutex_lock(&callback_mutex
);
879 cs
->mems_allowed
= trialcs
.mems_allowed
;
880 cs
->mems_generation
= atomic_inc_return(&cpuset_mems_generation
);
881 mutex_unlock(&callback_mutex
);
883 set_cpuset_being_rebound(cs
); /* causes mpol_copy() rebind */
885 fudge
= 10; /* spare mmarray[] slots */
886 fudge
+= cpus_weight(cs
->cpus_allowed
); /* imagine one fork-bomb/cpu */
890 * Allocate mmarray[] to hold mm reference for each task
891 * in cpuset cs. Can't kmalloc GFP_KERNEL while holding
892 * tasklist_lock. We could use GFP_ATOMIC, but with a
893 * few more lines of code, we can retry until we get a big
894 * enough mmarray[] w/o using GFP_ATOMIC.
897 ntasks
= atomic_read(&cs
->count
); /* guess */
899 mmarray
= kmalloc(ntasks
* sizeof(*mmarray
), GFP_KERNEL
);
902 write_lock_irq(&tasklist_lock
); /* block fork */
903 if (atomic_read(&cs
->count
) <= ntasks
)
904 break; /* got enough */
905 write_unlock_irq(&tasklist_lock
); /* try again */
911 /* Load up mmarray[] with mm reference for each task in cpuset. */
912 do_each_thread(g
, p
) {
913 struct mm_struct
*mm
;
917 "Cpuset mempolicy rebind incomplete.\n");
926 } while_each_thread(g
, p
);
927 write_unlock_irq(&tasklist_lock
);
930 * Now that we've dropped the tasklist spinlock, we can
931 * rebind the vma mempolicies of each mm in mmarray[] to their
932 * new cpuset, and release that mm. The mpol_rebind_mm()
933 * call takes mmap_sem, which we couldn't take while holding
934 * tasklist_lock. Forks can happen again now - the mpol_copy()
935 * cpuset_being_rebound check will catch such forks, and rebind
936 * their vma mempolicies too. Because we still hold the global
937 * cpuset manage_mutex, we know that no other rebind effort will
938 * be contending for the global variable cpuset_being_rebound.
939 * It's ok if we rebind the same mm twice; mpol_rebind_mm()
940 * is idempotent. Also migrate pages in each mm to new nodes.
942 migrate
= is_memory_migrate(cs
);
943 for (i
= 0; i
< n
; i
++) {
944 struct mm_struct
*mm
= mmarray
[i
];
946 mpol_rebind_mm(mm
, &cs
->mems_allowed
);
948 do_migrate_pages(mm
, &oldmem
, &cs
->mems_allowed
,
954 /* We're done rebinding vma's to this cpusets new mems_allowed. */
956 set_cpuset_being_rebound(NULL
);
963 * Call with manage_mutex held.
966 static int update_memory_pressure_enabled(struct cpuset
*cs
, char *buf
)
968 if (simple_strtoul(buf
, NULL
, 10) != 0)
969 cpuset_memory_pressure_enabled
= 1;
971 cpuset_memory_pressure_enabled
= 0;
976 * update_flag - read a 0 or a 1 in a file and update associated flag
977 * bit: the bit to update (CS_CPU_EXCLUSIVE, CS_MEM_EXCLUSIVE,
978 * CS_NOTIFY_ON_RELEASE, CS_MEMORY_MIGRATE,
979 * CS_SPREAD_PAGE, CS_SPREAD_SLAB)
980 * cs: the cpuset to update
981 * buf: the buffer where we read the 0 or 1
983 * Call with manage_mutex held.
986 static int update_flag(cpuset_flagbits_t bit
, struct cpuset
*cs
, char *buf
)
989 struct cpuset trialcs
;
990 int err
, cpu_exclusive_changed
;
992 turning_on
= (simple_strtoul(buf
, NULL
, 10) != 0);
996 set_bit(bit
, &trialcs
.flags
);
998 clear_bit(bit
, &trialcs
.flags
);
1000 err
= validate_change(cs
, &trialcs
);
1003 cpu_exclusive_changed
=
1004 (is_cpu_exclusive(cs
) != is_cpu_exclusive(&trialcs
));
1005 mutex_lock(&callback_mutex
);
1007 set_bit(bit
, &cs
->flags
);
1009 clear_bit(bit
, &cs
->flags
);
1010 mutex_unlock(&callback_mutex
);
1012 if (cpu_exclusive_changed
)
1013 update_cpu_domains(cs
);
1018 * Frequency meter - How fast is some event occuring?
1020 * These routines manage a digitally filtered, constant time based,
1021 * event frequency meter. There are four routines:
1022 * fmeter_init() - initialize a frequency meter.
1023 * fmeter_markevent() - called each time the event happens.
1024 * fmeter_getrate() - returns the recent rate of such events.
1025 * fmeter_update() - internal routine used to update fmeter.
1027 * A common data structure is passed to each of these routines,
1028 * which is used to keep track of the state required to manage the
1029 * frequency meter and its digital filter.
1031 * The filter works on the number of events marked per unit time.
1032 * The filter is single-pole low-pass recursive (IIR). The time unit
1033 * is 1 second. Arithmetic is done using 32-bit integers scaled to
1034 * simulate 3 decimal digits of precision (multiplied by 1000).
1036 * With an FM_COEF of 933, and a time base of 1 second, the filter
1037 * has a half-life of 10 seconds, meaning that if the events quit
1038 * happening, then the rate returned from the fmeter_getrate()
1039 * will be cut in half each 10 seconds, until it converges to zero.
1041 * It is not worth doing a real infinitely recursive filter. If more
1042 * than FM_MAXTICKS ticks have elapsed since the last filter event,
1043 * just compute FM_MAXTICKS ticks worth, by which point the level
1046 * Limit the count of unprocessed events to FM_MAXCNT, so as to avoid
1047 * arithmetic overflow in the fmeter_update() routine.
1049 * Given the simple 32 bit integer arithmetic used, this meter works
1050 * best for reporting rates between one per millisecond (msec) and
1051 * one per 32 (approx) seconds. At constant rates faster than one
1052 * per msec it maxes out at values just under 1,000,000. At constant
1053 * rates between one per msec, and one per second it will stabilize
1054 * to a value N*1000, where N is the rate of events per second.
1055 * At constant rates between one per second and one per 32 seconds,
1056 * it will be choppy, moving up on the seconds that have an event,
1057 * and then decaying until the next event. At rates slower than
1058 * about one in 32 seconds, it decays all the way back to zero between
1062 #define FM_COEF 933 /* coefficient for half-life of 10 secs */
1063 #define FM_MAXTICKS ((time_t)99) /* useless computing more ticks than this */
1064 #define FM_MAXCNT 1000000 /* limit cnt to avoid overflow */
1065 #define FM_SCALE 1000 /* faux fixed point scale */
1067 /* Initialize a frequency meter */
1068 static void fmeter_init(struct fmeter
*fmp
)
1073 spin_lock_init(&fmp
->lock
);
1076 /* Internal meter update - process cnt events and update value */
1077 static void fmeter_update(struct fmeter
*fmp
)
1079 time_t now
= get_seconds();
1080 time_t ticks
= now
- fmp
->time
;
1085 ticks
= min(FM_MAXTICKS
, ticks
);
1087 fmp
->val
= (FM_COEF
* fmp
->val
) / FM_SCALE
;
1090 fmp
->val
+= ((FM_SCALE
- FM_COEF
) * fmp
->cnt
) / FM_SCALE
;
1094 /* Process any previous ticks, then bump cnt by one (times scale). */
1095 static void fmeter_markevent(struct fmeter
*fmp
)
1097 spin_lock(&fmp
->lock
);
1099 fmp
->cnt
= min(FM_MAXCNT
, fmp
->cnt
+ FM_SCALE
);
1100 spin_unlock(&fmp
->lock
);
1103 /* Process any previous ticks, then return current value. */
1104 static int fmeter_getrate(struct fmeter
*fmp
)
1108 spin_lock(&fmp
->lock
);
1111 spin_unlock(&fmp
->lock
);
1116 * Attack task specified by pid in 'pidbuf' to cpuset 'cs', possibly
1117 * writing the path of the old cpuset in 'ppathbuf' if it needs to be
1118 * notified on release.
1120 * Call holding manage_mutex. May take callback_mutex and task_lock of
1121 * the task 'pid' during call.
1124 static int attach_task(struct cpuset
*cs
, char *pidbuf
, char **ppathbuf
)
1127 struct task_struct
*tsk
;
1128 struct cpuset
*oldcs
;
1130 nodemask_t from
, to
;
1131 struct mm_struct
*mm
;
1133 if (sscanf(pidbuf
, "%d", &pid
) != 1)
1135 if (cpus_empty(cs
->cpus_allowed
) || nodes_empty(cs
->mems_allowed
))
1139 read_lock(&tasklist_lock
);
1141 tsk
= find_task_by_pid(pid
);
1142 if (!tsk
|| tsk
->flags
& PF_EXITING
) {
1143 read_unlock(&tasklist_lock
);
1147 get_task_struct(tsk
);
1148 read_unlock(&tasklist_lock
);
1150 if ((current
->euid
) && (current
->euid
!= tsk
->uid
)
1151 && (current
->euid
!= tsk
->suid
)) {
1152 put_task_struct(tsk
);
1157 get_task_struct(tsk
);
1160 mutex_lock(&callback_mutex
);
1163 oldcs
= tsk
->cpuset
;
1166 mutex_unlock(&callback_mutex
);
1167 put_task_struct(tsk
);
1170 atomic_inc(&cs
->count
);
1171 rcu_assign_pointer(tsk
->cpuset
, cs
);
1174 guarantee_online_cpus(cs
, &cpus
);
1175 set_cpus_allowed(tsk
, cpus
);
1177 from
= oldcs
->mems_allowed
;
1178 to
= cs
->mems_allowed
;
1180 mutex_unlock(&callback_mutex
);
1182 mm
= get_task_mm(tsk
);
1184 mpol_rebind_mm(mm
, &to
);
1188 if (is_memory_migrate(cs
))
1189 do_migrate_pages(tsk
->mm
, &from
, &to
, MPOL_MF_MOVE_ALL
);
1190 put_task_struct(tsk
);
1192 if (atomic_dec_and_test(&oldcs
->count
))
1193 check_for_release(oldcs
, ppathbuf
);
1197 /* The various types of files and directories in a cpuset file system */
1202 FILE_MEMORY_MIGRATE
,
1207 FILE_NOTIFY_ON_RELEASE
,
1208 FILE_MEMORY_PRESSURE_ENABLED
,
1209 FILE_MEMORY_PRESSURE
,
1213 } cpuset_filetype_t
;
1215 static ssize_t
cpuset_common_file_write(struct file
*file
, const char __user
*userbuf
,
1216 size_t nbytes
, loff_t
*unused_ppos
)
1218 struct cpuset
*cs
= __d_cs(file
->f_dentry
->d_parent
);
1219 struct cftype
*cft
= __d_cft(file
->f_dentry
);
1220 cpuset_filetype_t type
= cft
->private;
1222 char *pathbuf
= NULL
;
1225 /* Crude upper limit on largest legitimate cpulist user might write. */
1226 if (nbytes
> 100 + 6 * NR_CPUS
)
1229 /* +1 for nul-terminator */
1230 if ((buffer
= kmalloc(nbytes
+ 1, GFP_KERNEL
)) == 0)
1233 if (copy_from_user(buffer
, userbuf
, nbytes
)) {
1237 buffer
[nbytes
] = 0; /* nul-terminate */
1239 mutex_lock(&manage_mutex
);
1241 if (is_removed(cs
)) {
1248 retval
= update_cpumask(cs
, buffer
);
1251 retval
= update_nodemask(cs
, buffer
);
1253 case FILE_CPU_EXCLUSIVE
:
1254 retval
= update_flag(CS_CPU_EXCLUSIVE
, cs
, buffer
);
1256 case FILE_MEM_EXCLUSIVE
:
1257 retval
= update_flag(CS_MEM_EXCLUSIVE
, cs
, buffer
);
1259 case FILE_NOTIFY_ON_RELEASE
:
1260 retval
= update_flag(CS_NOTIFY_ON_RELEASE
, cs
, buffer
);
1262 case FILE_MEMORY_MIGRATE
:
1263 retval
= update_flag(CS_MEMORY_MIGRATE
, cs
, buffer
);
1265 case FILE_MEMORY_PRESSURE_ENABLED
:
1266 retval
= update_memory_pressure_enabled(cs
, buffer
);
1268 case FILE_MEMORY_PRESSURE
:
1271 case FILE_SPREAD_PAGE
:
1272 retval
= update_flag(CS_SPREAD_PAGE
, cs
, buffer
);
1273 cs
->mems_generation
= atomic_inc_return(&cpuset_mems_generation
);
1275 case FILE_SPREAD_SLAB
:
1276 retval
= update_flag(CS_SPREAD_SLAB
, cs
, buffer
);
1277 cs
->mems_generation
= atomic_inc_return(&cpuset_mems_generation
);
1280 retval
= attach_task(cs
, buffer
, &pathbuf
);
1290 mutex_unlock(&manage_mutex
);
1291 cpuset_release_agent(pathbuf
);
1297 static ssize_t
cpuset_file_write(struct file
*file
, const char __user
*buf
,
1298 size_t nbytes
, loff_t
*ppos
)
1301 struct cftype
*cft
= __d_cft(file
->f_dentry
);
1305 /* special function ? */
1307 retval
= cft
->write(file
, buf
, nbytes
, ppos
);
1309 retval
= cpuset_common_file_write(file
, buf
, nbytes
, ppos
);
1315 * These ascii lists should be read in a single call, by using a user
1316 * buffer large enough to hold the entire map. If read in smaller
1317 * chunks, there is no guarantee of atomicity. Since the display format
1318 * used, list of ranges of sequential numbers, is variable length,
1319 * and since these maps can change value dynamically, one could read
1320 * gibberish by doing partial reads while a list was changing.
1321 * A single large read to a buffer that crosses a page boundary is
1322 * ok, because the result being copied to user land is not recomputed
1323 * across a page fault.
1326 static int cpuset_sprintf_cpulist(char *page
, struct cpuset
*cs
)
1330 mutex_lock(&callback_mutex
);
1331 mask
= cs
->cpus_allowed
;
1332 mutex_unlock(&callback_mutex
);
1334 return cpulist_scnprintf(page
, PAGE_SIZE
, mask
);
1337 static int cpuset_sprintf_memlist(char *page
, struct cpuset
*cs
)
1341 mutex_lock(&callback_mutex
);
1342 mask
= cs
->mems_allowed
;
1343 mutex_unlock(&callback_mutex
);
1345 return nodelist_scnprintf(page
, PAGE_SIZE
, mask
);
1348 static ssize_t
cpuset_common_file_read(struct file
*file
, char __user
*buf
,
1349 size_t nbytes
, loff_t
*ppos
)
1351 struct cftype
*cft
= __d_cft(file
->f_dentry
);
1352 struct cpuset
*cs
= __d_cs(file
->f_dentry
->d_parent
);
1353 cpuset_filetype_t type
= cft
->private;
1358 if (!(page
= (char *)__get_free_page(GFP_KERNEL
)))
1365 s
+= cpuset_sprintf_cpulist(s
, cs
);
1368 s
+= cpuset_sprintf_memlist(s
, cs
);
1370 case FILE_CPU_EXCLUSIVE
:
1371 *s
++ = is_cpu_exclusive(cs
) ? '1' : '0';
1373 case FILE_MEM_EXCLUSIVE
:
1374 *s
++ = is_mem_exclusive(cs
) ? '1' : '0';
1376 case FILE_NOTIFY_ON_RELEASE
:
1377 *s
++ = notify_on_release(cs
) ? '1' : '0';
1379 case FILE_MEMORY_MIGRATE
:
1380 *s
++ = is_memory_migrate(cs
) ? '1' : '0';
1382 case FILE_MEMORY_PRESSURE_ENABLED
:
1383 *s
++ = cpuset_memory_pressure_enabled
? '1' : '0';
1385 case FILE_MEMORY_PRESSURE
:
1386 s
+= sprintf(s
, "%d", fmeter_getrate(&cs
->fmeter
));
1388 case FILE_SPREAD_PAGE
:
1389 *s
++ = is_spread_page(cs
) ? '1' : '0';
1391 case FILE_SPREAD_SLAB
:
1392 *s
++ = is_spread_slab(cs
) ? '1' : '0';
1400 retval
= simple_read_from_buffer(buf
, nbytes
, ppos
, page
, s
- page
);
1402 free_page((unsigned long)page
);
1406 static ssize_t
cpuset_file_read(struct file
*file
, char __user
*buf
, size_t nbytes
,
1410 struct cftype
*cft
= __d_cft(file
->f_dentry
);
1414 /* special function ? */
1416 retval
= cft
->read(file
, buf
, nbytes
, ppos
);
1418 retval
= cpuset_common_file_read(file
, buf
, nbytes
, ppos
);
1423 static int cpuset_file_open(struct inode
*inode
, struct file
*file
)
1428 err
= generic_file_open(inode
, file
);
1432 cft
= __d_cft(file
->f_dentry
);
1436 err
= cft
->open(inode
, file
);
1443 static int cpuset_file_release(struct inode
*inode
, struct file
*file
)
1445 struct cftype
*cft
= __d_cft(file
->f_dentry
);
1447 return cft
->release(inode
, file
);
1452 * cpuset_rename - Only allow simple rename of directories in place.
1454 static int cpuset_rename(struct inode
*old_dir
, struct dentry
*old_dentry
,
1455 struct inode
*new_dir
, struct dentry
*new_dentry
)
1457 if (!S_ISDIR(old_dentry
->d_inode
->i_mode
))
1459 if (new_dentry
->d_inode
)
1461 if (old_dir
!= new_dir
)
1463 return simple_rename(old_dir
, old_dentry
, new_dir
, new_dentry
);
1466 static struct file_operations cpuset_file_operations
= {
1467 .read
= cpuset_file_read
,
1468 .write
= cpuset_file_write
,
1469 .llseek
= generic_file_llseek
,
1470 .open
= cpuset_file_open
,
1471 .release
= cpuset_file_release
,
1474 static struct inode_operations cpuset_dir_inode_operations
= {
1475 .lookup
= simple_lookup
,
1476 .mkdir
= cpuset_mkdir
,
1477 .rmdir
= cpuset_rmdir
,
1478 .rename
= cpuset_rename
,
1481 static int cpuset_create_file(struct dentry
*dentry
, int mode
)
1483 struct inode
*inode
;
1487 if (dentry
->d_inode
)
1490 inode
= cpuset_new_inode(mode
);
1494 if (S_ISDIR(mode
)) {
1495 inode
->i_op
= &cpuset_dir_inode_operations
;
1496 inode
->i_fop
= &simple_dir_operations
;
1498 /* start off with i_nlink == 2 (for "." entry) */
1500 } else if (S_ISREG(mode
)) {
1502 inode
->i_fop
= &cpuset_file_operations
;
1505 d_instantiate(dentry
, inode
);
1506 dget(dentry
); /* Extra count - pin the dentry in core */
1511 * cpuset_create_dir - create a directory for an object.
1512 * cs: the cpuset we create the directory for.
1513 * It must have a valid ->parent field
1514 * And we are going to fill its ->dentry field.
1515 * name: The name to give to the cpuset directory. Will be copied.
1516 * mode: mode to set on new directory.
1519 static int cpuset_create_dir(struct cpuset
*cs
, const char *name
, int mode
)
1521 struct dentry
*dentry
= NULL
;
1522 struct dentry
*parent
;
1525 parent
= cs
->parent
->dentry
;
1526 dentry
= cpuset_get_dentry(parent
, name
);
1528 return PTR_ERR(dentry
);
1529 error
= cpuset_create_file(dentry
, S_IFDIR
| mode
);
1531 dentry
->d_fsdata
= cs
;
1532 parent
->d_inode
->i_nlink
++;
1533 cs
->dentry
= dentry
;
1540 static int cpuset_add_file(struct dentry
*dir
, const struct cftype
*cft
)
1542 struct dentry
*dentry
;
1545 mutex_lock(&dir
->d_inode
->i_mutex
);
1546 dentry
= cpuset_get_dentry(dir
, cft
->name
);
1547 if (!IS_ERR(dentry
)) {
1548 error
= cpuset_create_file(dentry
, 0644 | S_IFREG
);
1550 dentry
->d_fsdata
= (void *)cft
;
1553 error
= PTR_ERR(dentry
);
1554 mutex_unlock(&dir
->d_inode
->i_mutex
);
1559 * Stuff for reading the 'tasks' file.
1561 * Reading this file can return large amounts of data if a cpuset has
1562 * *lots* of attached tasks. So it may need several calls to read(),
1563 * but we cannot guarantee that the information we produce is correct
1564 * unless we produce it entirely atomically.
1566 * Upon tasks file open(), a struct ctr_struct is allocated, that
1567 * will have a pointer to an array (also allocated here). The struct
1568 * ctr_struct * is stored in file->private_data. Its resources will
1569 * be freed by release() when the file is closed. The array is used
1570 * to sprintf the PIDs and then used by read().
1573 /* cpusets_tasks_read array */
1581 * Load into 'pidarray' up to 'npids' of the tasks using cpuset 'cs'.
1582 * Return actual number of pids loaded. No need to task_lock(p)
1583 * when reading out p->cpuset, as we don't really care if it changes
1584 * on the next cycle, and we are not going to try to dereference it.
1586 static int pid_array_load(pid_t
*pidarray
, int npids
, struct cpuset
*cs
)
1589 struct task_struct
*g
, *p
;
1591 read_lock(&tasklist_lock
);
1593 do_each_thread(g
, p
) {
1594 if (p
->cpuset
== cs
) {
1595 pidarray
[n
++] = p
->pid
;
1596 if (unlikely(n
== npids
))
1599 } while_each_thread(g
, p
);
1602 read_unlock(&tasklist_lock
);
1606 static int cmppid(const void *a
, const void *b
)
1608 return *(pid_t
*)a
- *(pid_t
*)b
;
1612 * Convert array 'a' of 'npids' pid_t's to a string of newline separated
1613 * decimal pids in 'buf'. Don't write more than 'sz' chars, but return
1614 * count 'cnt' of how many chars would be written if buf were large enough.
1616 static int pid_array_to_buf(char *buf
, int sz
, pid_t
*a
, int npids
)
1621 for (i
= 0; i
< npids
; i
++)
1622 cnt
+= snprintf(buf
+ cnt
, max(sz
- cnt
, 0), "%d\n", a
[i
]);
1627 * Handle an open on 'tasks' file. Prepare a buffer listing the
1628 * process id's of tasks currently attached to the cpuset being opened.
1630 * Does not require any specific cpuset mutexes, and does not take any.
1632 static int cpuset_tasks_open(struct inode
*unused
, struct file
*file
)
1634 struct cpuset
*cs
= __d_cs(file
->f_dentry
->d_parent
);
1635 struct ctr_struct
*ctr
;
1640 if (!(file
->f_mode
& FMODE_READ
))
1643 ctr
= kmalloc(sizeof(*ctr
), GFP_KERNEL
);
1648 * If cpuset gets more users after we read count, we won't have
1649 * enough space - tough. This race is indistinguishable to the
1650 * caller from the case that the additional cpuset users didn't
1651 * show up until sometime later on.
1653 npids
= atomic_read(&cs
->count
);
1654 pidarray
= kmalloc(npids
* sizeof(pid_t
), GFP_KERNEL
);
1658 npids
= pid_array_load(pidarray
, npids
, cs
);
1659 sort(pidarray
, npids
, sizeof(pid_t
), cmppid
, NULL
);
1661 /* Call pid_array_to_buf() twice, first just to get bufsz */
1662 ctr
->bufsz
= pid_array_to_buf(&c
, sizeof(c
), pidarray
, npids
) + 1;
1663 ctr
->buf
= kmalloc(ctr
->bufsz
, GFP_KERNEL
);
1666 ctr
->bufsz
= pid_array_to_buf(ctr
->buf
, ctr
->bufsz
, pidarray
, npids
);
1669 file
->private_data
= ctr
;
1680 static ssize_t
cpuset_tasks_read(struct file
*file
, char __user
*buf
,
1681 size_t nbytes
, loff_t
*ppos
)
1683 struct ctr_struct
*ctr
= file
->private_data
;
1685 if (*ppos
+ nbytes
> ctr
->bufsz
)
1686 nbytes
= ctr
->bufsz
- *ppos
;
1687 if (copy_to_user(buf
, ctr
->buf
+ *ppos
, nbytes
))
1693 static int cpuset_tasks_release(struct inode
*unused_inode
, struct file
*file
)
1695 struct ctr_struct
*ctr
;
1697 if (file
->f_mode
& FMODE_READ
) {
1698 ctr
= file
->private_data
;
1706 * for the common functions, 'private' gives the type of file
1709 static struct cftype cft_tasks
= {
1711 .open
= cpuset_tasks_open
,
1712 .read
= cpuset_tasks_read
,
1713 .release
= cpuset_tasks_release
,
1714 .private = FILE_TASKLIST
,
1717 static struct cftype cft_cpus
= {
1719 .private = FILE_CPULIST
,
1722 static struct cftype cft_mems
= {
1724 .private = FILE_MEMLIST
,
1727 static struct cftype cft_cpu_exclusive
= {
1728 .name
= "cpu_exclusive",
1729 .private = FILE_CPU_EXCLUSIVE
,
1732 static struct cftype cft_mem_exclusive
= {
1733 .name
= "mem_exclusive",
1734 .private = FILE_MEM_EXCLUSIVE
,
1737 static struct cftype cft_notify_on_release
= {
1738 .name
= "notify_on_release",
1739 .private = FILE_NOTIFY_ON_RELEASE
,
1742 static struct cftype cft_memory_migrate
= {
1743 .name
= "memory_migrate",
1744 .private = FILE_MEMORY_MIGRATE
,
1747 static struct cftype cft_memory_pressure_enabled
= {
1748 .name
= "memory_pressure_enabled",
1749 .private = FILE_MEMORY_PRESSURE_ENABLED
,
1752 static struct cftype cft_memory_pressure
= {
1753 .name
= "memory_pressure",
1754 .private = FILE_MEMORY_PRESSURE
,
1757 static struct cftype cft_spread_page
= {
1758 .name
= "memory_spread_page",
1759 .private = FILE_SPREAD_PAGE
,
1762 static struct cftype cft_spread_slab
= {
1763 .name
= "memory_spread_slab",
1764 .private = FILE_SPREAD_SLAB
,
1767 static int cpuset_populate_dir(struct dentry
*cs_dentry
)
1771 if ((err
= cpuset_add_file(cs_dentry
, &cft_cpus
)) < 0)
1773 if ((err
= cpuset_add_file(cs_dentry
, &cft_mems
)) < 0)
1775 if ((err
= cpuset_add_file(cs_dentry
, &cft_cpu_exclusive
)) < 0)
1777 if ((err
= cpuset_add_file(cs_dentry
, &cft_mem_exclusive
)) < 0)
1779 if ((err
= cpuset_add_file(cs_dentry
, &cft_notify_on_release
)) < 0)
1781 if ((err
= cpuset_add_file(cs_dentry
, &cft_memory_migrate
)) < 0)
1783 if ((err
= cpuset_add_file(cs_dentry
, &cft_memory_pressure
)) < 0)
1785 if ((err
= cpuset_add_file(cs_dentry
, &cft_spread_page
)) < 0)
1787 if ((err
= cpuset_add_file(cs_dentry
, &cft_spread_slab
)) < 0)
1789 if ((err
= cpuset_add_file(cs_dentry
, &cft_tasks
)) < 0)
1795 * cpuset_create - create a cpuset
1796 * parent: cpuset that will be parent of the new cpuset.
1797 * name: name of the new cpuset. Will be strcpy'ed.
1798 * mode: mode to set on new inode
1800 * Must be called with the mutex on the parent inode held
1803 static long cpuset_create(struct cpuset
*parent
, const char *name
, int mode
)
1808 cs
= kmalloc(sizeof(*cs
), GFP_KERNEL
);
1812 mutex_lock(&manage_mutex
);
1813 cpuset_update_task_memory_state();
1815 if (notify_on_release(parent
))
1816 set_bit(CS_NOTIFY_ON_RELEASE
, &cs
->flags
);
1817 if (is_spread_page(parent
))
1818 set_bit(CS_SPREAD_PAGE
, &cs
->flags
);
1819 if (is_spread_slab(parent
))
1820 set_bit(CS_SPREAD_SLAB
, &cs
->flags
);
1821 cs
->cpus_allowed
= CPU_MASK_NONE
;
1822 cs
->mems_allowed
= NODE_MASK_NONE
;
1823 atomic_set(&cs
->count
, 0);
1824 INIT_LIST_HEAD(&cs
->sibling
);
1825 INIT_LIST_HEAD(&cs
->children
);
1826 cs
->mems_generation
= atomic_inc_return(&cpuset_mems_generation
);
1827 fmeter_init(&cs
->fmeter
);
1829 cs
->parent
= parent
;
1831 mutex_lock(&callback_mutex
);
1832 list_add(&cs
->sibling
, &cs
->parent
->children
);
1833 number_of_cpusets
++;
1834 mutex_unlock(&callback_mutex
);
1836 err
= cpuset_create_dir(cs
, name
, mode
);
1841 * Release manage_mutex before cpuset_populate_dir() because it
1842 * will down() this new directory's i_mutex and if we race with
1843 * another mkdir, we might deadlock.
1845 mutex_unlock(&manage_mutex
);
1847 err
= cpuset_populate_dir(cs
->dentry
);
1848 /* If err < 0, we have a half-filled directory - oh well ;) */
1851 list_del(&cs
->sibling
);
1852 mutex_unlock(&manage_mutex
);
1857 static int cpuset_mkdir(struct inode
*dir
, struct dentry
*dentry
, int mode
)
1859 struct cpuset
*c_parent
= dentry
->d_parent
->d_fsdata
;
1861 /* the vfs holds inode->i_mutex already */
1862 return cpuset_create(c_parent
, dentry
->d_name
.name
, mode
| S_IFDIR
);
1865 static int cpuset_rmdir(struct inode
*unused_dir
, struct dentry
*dentry
)
1867 struct cpuset
*cs
= dentry
->d_fsdata
;
1869 struct cpuset
*parent
;
1870 char *pathbuf
= NULL
;
1872 /* the vfs holds both inode->i_mutex already */
1874 mutex_lock(&manage_mutex
);
1875 cpuset_update_task_memory_state();
1876 if (atomic_read(&cs
->count
) > 0) {
1877 mutex_unlock(&manage_mutex
);
1880 if (!list_empty(&cs
->children
)) {
1881 mutex_unlock(&manage_mutex
);
1884 parent
= cs
->parent
;
1885 mutex_lock(&callback_mutex
);
1886 set_bit(CS_REMOVED
, &cs
->flags
);
1887 if (is_cpu_exclusive(cs
))
1888 update_cpu_domains(cs
);
1889 list_del(&cs
->sibling
); /* delete my sibling from parent->children */
1890 spin_lock(&cs
->dentry
->d_lock
);
1891 d
= dget(cs
->dentry
);
1893 spin_unlock(&d
->d_lock
);
1894 cpuset_d_remove_dir(d
);
1896 number_of_cpusets
--;
1897 mutex_unlock(&callback_mutex
);
1898 if (list_empty(&parent
->children
))
1899 check_for_release(parent
, &pathbuf
);
1900 mutex_unlock(&manage_mutex
);
1901 cpuset_release_agent(pathbuf
);
1906 * cpuset_init_early - just enough so that the calls to
1907 * cpuset_update_task_memory_state() in early init code
1911 int __init
cpuset_init_early(void)
1913 struct task_struct
*tsk
= current
;
1915 tsk
->cpuset
= &top_cpuset
;
1916 tsk
->cpuset
->mems_generation
= atomic_inc_return(&cpuset_mems_generation
);
1921 * cpuset_init - initialize cpusets at system boot
1923 * Description: Initialize top_cpuset and the cpuset internal file system,
1926 int __init
cpuset_init(void)
1928 struct dentry
*root
;
1931 top_cpuset
.cpus_allowed
= CPU_MASK_ALL
;
1932 top_cpuset
.mems_allowed
= NODE_MASK_ALL
;
1934 fmeter_init(&top_cpuset
.fmeter
);
1935 top_cpuset
.mems_generation
= atomic_inc_return(&cpuset_mems_generation
);
1937 init_task
.cpuset
= &top_cpuset
;
1939 err
= register_filesystem(&cpuset_fs_type
);
1942 cpuset_mount
= kern_mount(&cpuset_fs_type
);
1943 if (IS_ERR(cpuset_mount
)) {
1944 printk(KERN_ERR
"cpuset: could not mount!\n");
1945 err
= PTR_ERR(cpuset_mount
);
1946 cpuset_mount
= NULL
;
1949 root
= cpuset_mount
->mnt_sb
->s_root
;
1950 root
->d_fsdata
= &top_cpuset
;
1951 root
->d_inode
->i_nlink
++;
1952 top_cpuset
.dentry
= root
;
1953 root
->d_inode
->i_op
= &cpuset_dir_inode_operations
;
1954 number_of_cpusets
= 1;
1955 err
= cpuset_populate_dir(root
);
1956 /* memory_pressure_enabled is in root cpuset only */
1958 err
= cpuset_add_file(root
, &cft_memory_pressure_enabled
);
1964 * cpuset_init_smp - initialize cpus_allowed
1966 * Description: Finish top cpuset after cpu, node maps are initialized
1969 void __init
cpuset_init_smp(void)
1971 top_cpuset
.cpus_allowed
= cpu_online_map
;
1972 top_cpuset
.mems_allowed
= node_online_map
;
1976 * cpuset_fork - attach newly forked task to its parents cpuset.
1977 * @tsk: pointer to task_struct of forking parent process.
1979 * Description: A task inherits its parent's cpuset at fork().
1981 * A pointer to the shared cpuset was automatically copied in fork.c
1982 * by dup_task_struct(). However, we ignore that copy, since it was
1983 * not made under the protection of task_lock(), so might no longer be
1984 * a valid cpuset pointer. attach_task() might have already changed
1985 * current->cpuset, allowing the previously referenced cpuset to
1986 * be removed and freed. Instead, we task_lock(current) and copy
1987 * its present value of current->cpuset for our freshly forked child.
1989 * At the point that cpuset_fork() is called, 'current' is the parent
1990 * task, and the passed argument 'child' points to the child task.
1993 void cpuset_fork(struct task_struct
*child
)
1996 child
->cpuset
= current
->cpuset
;
1997 atomic_inc(&child
->cpuset
->count
);
1998 task_unlock(current
);
2002 * cpuset_exit - detach cpuset from exiting task
2003 * @tsk: pointer to task_struct of exiting process
2005 * Description: Detach cpuset from @tsk and release it.
2007 * Note that cpusets marked notify_on_release force every task in
2008 * them to take the global manage_mutex mutex when exiting.
2009 * This could impact scaling on very large systems. Be reluctant to
2010 * use notify_on_release cpusets where very high task exit scaling
2011 * is required on large systems.
2013 * Don't even think about derefencing 'cs' after the cpuset use count
2014 * goes to zero, except inside a critical section guarded by manage_mutex
2015 * or callback_mutex. Otherwise a zero cpuset use count is a license to
2016 * any other task to nuke the cpuset immediately, via cpuset_rmdir().
2018 * This routine has to take manage_mutex, not callback_mutex, because
2019 * it is holding that mutex while calling check_for_release(),
2020 * which calls kmalloc(), so can't be called holding callback_mutex().
2022 * We don't need to task_lock() this reference to tsk->cpuset,
2023 * because tsk is already marked PF_EXITING, so attach_task() won't
2024 * mess with it, or task is a failed fork, never visible to attach_task.
2028 * Set the exiting tasks cpuset to the root cpuset (top_cpuset).
2030 * Don't leave a task unable to allocate memory, as that is an
2031 * accident waiting to happen should someone add a callout in
2032 * do_exit() after the cpuset_exit() call that might allocate.
2033 * If a task tries to allocate memory with an invalid cpuset,
2034 * it will oops in cpuset_update_task_memory_state().
2036 * We call cpuset_exit() while the task is still competent to
2037 * handle notify_on_release(), then leave the task attached to
2038 * the root cpuset (top_cpuset) for the remainder of its exit.
2040 * To do this properly, we would increment the reference count on
2041 * top_cpuset, and near the very end of the kernel/exit.c do_exit()
2042 * code we would add a second cpuset function call, to drop that
2043 * reference. This would just create an unnecessary hot spot on
2044 * the top_cpuset reference count, to no avail.
2046 * Normally, holding a reference to a cpuset without bumping its
2047 * count is unsafe. The cpuset could go away, or someone could
2048 * attach us to a different cpuset, decrementing the count on
2049 * the first cpuset that we never incremented. But in this case,
2050 * top_cpuset isn't going away, and either task has PF_EXITING set,
2051 * which wards off any attach_task() attempts, or task is a failed
2052 * fork, never visible to attach_task.
2054 * Another way to do this would be to set the cpuset pointer
2055 * to NULL here, and check in cpuset_update_task_memory_state()
2056 * for a NULL pointer. This hack avoids that NULL check, for no
2057 * cost (other than this way too long comment ;).
2060 void cpuset_exit(struct task_struct
*tsk
)
2065 tsk
->cpuset
= &top_cpuset
; /* Hack - see comment above */
2067 if (notify_on_release(cs
)) {
2068 char *pathbuf
= NULL
;
2070 mutex_lock(&manage_mutex
);
2071 if (atomic_dec_and_test(&cs
->count
))
2072 check_for_release(cs
, &pathbuf
);
2073 mutex_unlock(&manage_mutex
);
2074 cpuset_release_agent(pathbuf
);
2076 atomic_dec(&cs
->count
);
2081 * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.
2082 * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
2084 * Description: Returns the cpumask_t cpus_allowed of the cpuset
2085 * attached to the specified @tsk. Guaranteed to return some non-empty
2086 * subset of cpu_online_map, even if this means going outside the
2090 cpumask_t
cpuset_cpus_allowed(struct task_struct
*tsk
)
2094 mutex_lock(&callback_mutex
);
2096 guarantee_online_cpus(tsk
->cpuset
, &mask
);
2098 mutex_unlock(&callback_mutex
);
2103 void cpuset_init_current_mems_allowed(void)
2105 current
->mems_allowed
= NODE_MASK_ALL
;
2109 * cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset.
2110 * @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed.
2112 * Description: Returns the nodemask_t mems_allowed of the cpuset
2113 * attached to the specified @tsk. Guaranteed to return some non-empty
2114 * subset of node_online_map, even if this means going outside the
2118 nodemask_t
cpuset_mems_allowed(struct task_struct
*tsk
)
2122 mutex_lock(&callback_mutex
);
2124 guarantee_online_mems(tsk
->cpuset
, &mask
);
2126 mutex_unlock(&callback_mutex
);
2132 * cpuset_zonelist_valid_mems_allowed - check zonelist vs. curremt mems_allowed
2133 * @zl: the zonelist to be checked
2135 * Are any of the nodes on zonelist zl allowed in current->mems_allowed?
2137 int cpuset_zonelist_valid_mems_allowed(struct zonelist
*zl
)
2141 for (i
= 0; zl
->zones
[i
]; i
++) {
2142 int nid
= zl
->zones
[i
]->zone_pgdat
->node_id
;
2144 if (node_isset(nid
, current
->mems_allowed
))
2151 * nearest_exclusive_ancestor() - Returns the nearest mem_exclusive
2152 * ancestor to the specified cpuset. Call holding callback_mutex.
2153 * If no ancestor is mem_exclusive (an unusual configuration), then
2154 * returns the root cpuset.
2156 static const struct cpuset
*nearest_exclusive_ancestor(const struct cpuset
*cs
)
2158 while (!is_mem_exclusive(cs
) && cs
->parent
)
2164 * cpuset_zone_allowed - Can we allocate memory on zone z's memory node?
2165 * @z: is this zone on an allowed node?
2166 * @gfp_mask: memory allocation flags (we use __GFP_HARDWALL)
2168 * If we're in interrupt, yes, we can always allocate. If zone
2169 * z's node is in our tasks mems_allowed, yes. If it's not a
2170 * __GFP_HARDWALL request and this zone's nodes is in the nearest
2171 * mem_exclusive cpuset ancestor to this tasks cpuset, yes.
2174 * GFP_USER allocations are marked with the __GFP_HARDWALL bit,
2175 * and do not allow allocations outside the current tasks cpuset.
2176 * GFP_KERNEL allocations are not so marked, so can escape to the
2177 * nearest mem_exclusive ancestor cpuset.
2179 * Scanning up parent cpusets requires callback_mutex. The __alloc_pages()
2180 * routine only calls here with __GFP_HARDWALL bit _not_ set if
2181 * it's a GFP_KERNEL allocation, and all nodes in the current tasks
2182 * mems_allowed came up empty on the first pass over the zonelist.
2183 * So only GFP_KERNEL allocations, if all nodes in the cpuset are
2184 * short of memory, might require taking the callback_mutex mutex.
2186 * The first loop over the zonelist in mm/page_alloc.c:__alloc_pages()
2187 * calls here with __GFP_HARDWALL always set in gfp_mask, enforcing
2188 * hardwall cpusets - no allocation on a node outside the cpuset is
2189 * allowed (unless in interrupt, of course).
2191 * The second loop doesn't even call here for GFP_ATOMIC requests
2192 * (if the __alloc_pages() local variable 'wait' is set). That check
2193 * and the checks below have the combined affect in the second loop of
2194 * the __alloc_pages() routine that:
2195 * in_interrupt - any node ok (current task context irrelevant)
2196 * GFP_ATOMIC - any node ok
2197 * GFP_KERNEL - any node in enclosing mem_exclusive cpuset ok
2198 * GFP_USER - only nodes in current tasks mems allowed ok.
2201 int __cpuset_zone_allowed(struct zone
*z
, gfp_t gfp_mask
)
2203 int node
; /* node that zone z is on */
2204 const struct cpuset
*cs
; /* current cpuset ancestors */
2205 int allowed
= 1; /* is allocation in zone z allowed? */
2209 node
= z
->zone_pgdat
->node_id
;
2210 if (node_isset(node
, current
->mems_allowed
))
2212 if (gfp_mask
& __GFP_HARDWALL
) /* If hardwall request, stop here */
2215 if (current
->flags
& PF_EXITING
) /* Let dying task have memory */
2218 /* Not hardwall and node outside mems_allowed: scan up cpusets */
2219 mutex_lock(&callback_mutex
);
2222 cs
= nearest_exclusive_ancestor(current
->cpuset
);
2223 task_unlock(current
);
2225 allowed
= node_isset(node
, cs
->mems_allowed
);
2226 mutex_unlock(&callback_mutex
);
2231 * cpuset_lock - lock out any changes to cpuset structures
2233 * The out of memory (oom) code needs to mutex_lock cpusets
2234 * from being changed while it scans the tasklist looking for a
2235 * task in an overlapping cpuset. Expose callback_mutex via this
2236 * cpuset_lock() routine, so the oom code can lock it, before
2237 * locking the task list. The tasklist_lock is a spinlock, so
2238 * must be taken inside callback_mutex.
2241 void cpuset_lock(void)
2243 mutex_lock(&callback_mutex
);
2247 * cpuset_unlock - release lock on cpuset changes
2249 * Undo the lock taken in a previous cpuset_lock() call.
2252 void cpuset_unlock(void)
2254 mutex_unlock(&callback_mutex
);
2258 * cpuset_mem_spread_node() - On which node to begin search for a page
2260 * If a task is marked PF_SPREAD_PAGE or PF_SPREAD_SLAB (as for
2261 * tasks in a cpuset with is_spread_page or is_spread_slab set),
2262 * and if the memory allocation used cpuset_mem_spread_node()
2263 * to determine on which node to start looking, as it will for
2264 * certain page cache or slab cache pages such as used for file
2265 * system buffers and inode caches, then instead of starting on the
2266 * local node to look for a free page, rather spread the starting
2267 * node around the tasks mems_allowed nodes.
2269 * We don't have to worry about the returned node being offline
2270 * because "it can't happen", and even if it did, it would be ok.
2272 * The routines calling guarantee_online_mems() are careful to
2273 * only set nodes in task->mems_allowed that are online. So it
2274 * should not be possible for the following code to return an
2275 * offline node. But if it did, that would be ok, as this routine
2276 * is not returning the node where the allocation must be, only
2277 * the node where the search should start. The zonelist passed to
2278 * __alloc_pages() will include all nodes. If the slab allocator
2279 * is passed an offline node, it will fall back to the local node.
2280 * See kmem_cache_alloc_node().
2283 int cpuset_mem_spread_node(void)
2287 node
= next_node(current
->cpuset_mem_spread_rotor
, current
->mems_allowed
);
2288 if (node
== MAX_NUMNODES
)
2289 node
= first_node(current
->mems_allowed
);
2290 current
->cpuset_mem_spread_rotor
= node
;
2293 EXPORT_SYMBOL_GPL(cpuset_mem_spread_node
);
2296 * cpuset_excl_nodes_overlap - Do we overlap @p's mem_exclusive ancestors?
2297 * @p: pointer to task_struct of some other task.
2299 * Description: Return true if the nearest mem_exclusive ancestor
2300 * cpusets of tasks @p and current overlap. Used by oom killer to
2301 * determine if task @p's memory usage might impact the memory
2302 * available to the current task.
2304 * Call while holding callback_mutex.
2307 int cpuset_excl_nodes_overlap(const struct task_struct
*p
)
2309 const struct cpuset
*cs1
, *cs2
; /* my and p's cpuset ancestors */
2310 int overlap
= 0; /* do cpusets overlap? */
2313 if (current
->flags
& PF_EXITING
) {
2314 task_unlock(current
);
2317 cs1
= nearest_exclusive_ancestor(current
->cpuset
);
2318 task_unlock(current
);
2320 task_lock((struct task_struct
*)p
);
2321 if (p
->flags
& PF_EXITING
) {
2322 task_unlock((struct task_struct
*)p
);
2325 cs2
= nearest_exclusive_ancestor(p
->cpuset
);
2326 task_unlock((struct task_struct
*)p
);
2328 overlap
= nodes_intersects(cs1
->mems_allowed
, cs2
->mems_allowed
);
2334 * Collection of memory_pressure is suppressed unless
2335 * this flag is enabled by writing "1" to the special
2336 * cpuset file 'memory_pressure_enabled' in the root cpuset.
2339 int cpuset_memory_pressure_enabled __read_mostly
;
2342 * cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims.
2344 * Keep a running average of the rate of synchronous (direct)
2345 * page reclaim efforts initiated by tasks in each cpuset.
2347 * This represents the rate at which some task in the cpuset
2348 * ran low on memory on all nodes it was allowed to use, and
2349 * had to enter the kernels page reclaim code in an effort to
2350 * create more free memory by tossing clean pages or swapping
2351 * or writing dirty pages.
2353 * Display to user space in the per-cpuset read-only file
2354 * "memory_pressure". Value displayed is an integer
2355 * representing the recent rate of entry into the synchronous
2356 * (direct) page reclaim by any task attached to the cpuset.
2359 void __cpuset_memory_pressure_bump(void)
2364 cs
= current
->cpuset
;
2365 fmeter_markevent(&cs
->fmeter
);
2366 task_unlock(current
);
2370 * proc_cpuset_show()
2371 * - Print tasks cpuset path into seq_file.
2372 * - Used for /proc/<pid>/cpuset.
2373 * - No need to task_lock(tsk) on this tsk->cpuset reference, as it
2374 * doesn't really matter if tsk->cpuset changes after we read it,
2375 * and we take manage_mutex, keeping attach_task() from changing it
2379 static int proc_cpuset_show(struct seq_file
*m
, void *v
)
2382 struct task_struct
*tsk
;
2386 buf
= kmalloc(PAGE_SIZE
, GFP_KERNEL
);
2391 mutex_lock(&manage_mutex
);
2398 retval
= cpuset_path(cs
, buf
, PAGE_SIZE
);
2404 mutex_unlock(&manage_mutex
);
2409 static int cpuset_open(struct inode
*inode
, struct file
*file
)
2411 struct task_struct
*tsk
= PROC_I(inode
)->task
;
2412 return single_open(file
, proc_cpuset_show
, tsk
);
2415 struct file_operations proc_cpuset_operations
= {
2416 .open
= cpuset_open
,
2418 .llseek
= seq_lseek
,
2419 .release
= single_release
,
2422 /* Display task cpus_allowed, mems_allowed in /proc/<pid>/status file. */
2423 char *cpuset_task_status_allowed(struct task_struct
*task
, char *buffer
)
2425 buffer
+= sprintf(buffer
, "Cpus_allowed:\t");
2426 buffer
+= cpumask_scnprintf(buffer
, PAGE_SIZE
, task
->cpus_allowed
);
2427 buffer
+= sprintf(buffer
, "\n");
2428 buffer
+= sprintf(buffer
, "Mems_allowed:\t");
2429 buffer
+= nodemask_scnprintf(buffer
, PAGE_SIZE
, task
->mems_allowed
);
2430 buffer
+= sprintf(buffer
, "\n");