1 // SPDX-License-Identifier: GPL-2.0-or-later
3 #include <linux/memcontrol.h>
4 #include <linux/swap.h>
5 #include <linux/mm_inline.h>
6 #include <linux/pagewalk.h>
7 #include <linux/backing-dev.h>
8 #include <linux/swap_cgroup.h>
9 #include <linux/eventfd.h>
10 #include <linux/poll.h>
11 #include <linux/sort.h>
12 #include <linux/file.h>
13 #include <linux/seq_buf.h>
17 #include "memcontrol-v1.h"
20 * Cgroups above their limits are maintained in a RB-Tree, independent of
21 * their hierarchy representation
24 struct mem_cgroup_tree_per_node
{
25 struct rb_root rb_root
;
26 struct rb_node
*rb_rightmost
;
30 struct mem_cgroup_tree
{
31 struct mem_cgroup_tree_per_node
*rb_tree_per_node
[MAX_NUMNODES
];
34 static struct mem_cgroup_tree soft_limit_tree __read_mostly
;
37 * Maximum loops in mem_cgroup_soft_reclaim(), used for soft
38 * limit reclaim to prevent infinite loops, if they ever occur.
40 #define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
41 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
43 /* Stuffs for move charges at task migration. */
45 * Types of charges to be moved.
47 #define MOVE_ANON 0x1ULL
48 #define MOVE_FILE 0x2ULL
49 #define MOVE_MASK (MOVE_ANON | MOVE_FILE)
51 /* "mc" and its members are protected by cgroup_mutex */
52 static struct move_charge_struct
{
53 spinlock_t lock
; /* for from, to */
55 struct mem_cgroup
*from
;
56 struct mem_cgroup
*to
;
58 unsigned long precharge
;
59 unsigned long moved_charge
;
60 unsigned long moved_swap
;
61 struct task_struct
*moving_task
; /* a task moving charges */
62 wait_queue_head_t waitq
; /* a waitq for other context */
64 .lock
= __SPIN_LOCK_UNLOCKED(mc
.lock
),
65 .waitq
= __WAIT_QUEUE_HEAD_INITIALIZER(mc
.waitq
),
69 struct mem_cgroup_eventfd_list
{
70 struct list_head list
;
71 struct eventfd_ctx
*eventfd
;
75 * cgroup_event represents events which userspace want to receive.
77 struct mem_cgroup_event
{
79 * memcg which the event belongs to.
81 struct mem_cgroup
*memcg
;
83 * eventfd to signal userspace about the event.
85 struct eventfd_ctx
*eventfd
;
87 * Each of these stored in a list by the cgroup.
89 struct list_head list
;
91 * register_event() callback will be used to add new userspace
92 * waiter for changes related to this event. Use eventfd_signal()
93 * on eventfd to send notification to userspace.
95 int (*register_event
)(struct mem_cgroup
*memcg
,
96 struct eventfd_ctx
*eventfd
, const char *args
);
98 * unregister_event() callback will be called when userspace closes
99 * the eventfd or on cgroup removing. This callback must be set,
100 * if you want provide notification functionality.
102 void (*unregister_event
)(struct mem_cgroup
*memcg
,
103 struct eventfd_ctx
*eventfd
);
105 * All fields below needed to unregister event when
106 * userspace closes eventfd.
109 wait_queue_head_t
*wqh
;
110 wait_queue_entry_t wait
;
111 struct work_struct remove
;
114 #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
115 #define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
116 #define MEMFILE_ATTR(val) ((val) & 0xffff)
126 #ifdef CONFIG_LOCKDEP
127 static struct lockdep_map memcg_oom_lock_dep_map
= {
128 .name
= "memcg_oom_lock",
132 DEFINE_SPINLOCK(memcg_oom_lock
);
134 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node
*mz
,
135 struct mem_cgroup_tree_per_node
*mctz
,
136 unsigned long new_usage_in_excess
)
138 struct rb_node
**p
= &mctz
->rb_root
.rb_node
;
139 struct rb_node
*parent
= NULL
;
140 struct mem_cgroup_per_node
*mz_node
;
141 bool rightmost
= true;
146 mz
->usage_in_excess
= new_usage_in_excess
;
147 if (!mz
->usage_in_excess
)
151 mz_node
= rb_entry(parent
, struct mem_cgroup_per_node
,
153 if (mz
->usage_in_excess
< mz_node
->usage_in_excess
) {
162 mctz
->rb_rightmost
= &mz
->tree_node
;
164 rb_link_node(&mz
->tree_node
, parent
, p
);
165 rb_insert_color(&mz
->tree_node
, &mctz
->rb_root
);
169 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node
*mz
,
170 struct mem_cgroup_tree_per_node
*mctz
)
175 if (&mz
->tree_node
== mctz
->rb_rightmost
)
176 mctz
->rb_rightmost
= rb_prev(&mz
->tree_node
);
178 rb_erase(&mz
->tree_node
, &mctz
->rb_root
);
182 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node
*mz
,
183 struct mem_cgroup_tree_per_node
*mctz
)
187 spin_lock_irqsave(&mctz
->lock
, flags
);
188 __mem_cgroup_remove_exceeded(mz
, mctz
);
189 spin_unlock_irqrestore(&mctz
->lock
, flags
);
192 static unsigned long soft_limit_excess(struct mem_cgroup
*memcg
)
194 unsigned long nr_pages
= page_counter_read(&memcg
->memory
);
195 unsigned long soft_limit
= READ_ONCE(memcg
->soft_limit
);
196 unsigned long excess
= 0;
198 if (nr_pages
> soft_limit
)
199 excess
= nr_pages
- soft_limit
;
204 static void memcg1_update_tree(struct mem_cgroup
*memcg
, int nid
)
206 unsigned long excess
;
207 struct mem_cgroup_per_node
*mz
;
208 struct mem_cgroup_tree_per_node
*mctz
;
210 if (lru_gen_enabled()) {
211 if (soft_limit_excess(memcg
))
212 lru_gen_soft_reclaim(memcg
, nid
);
216 mctz
= soft_limit_tree
.rb_tree_per_node
[nid
];
220 * Necessary to update all ancestors when hierarchy is used.
221 * because their event counter is not touched.
223 for (; memcg
; memcg
= parent_mem_cgroup(memcg
)) {
224 mz
= memcg
->nodeinfo
[nid
];
225 excess
= soft_limit_excess(memcg
);
227 * We have to update the tree if mz is on RB-tree or
228 * mem is over its softlimit.
230 if (excess
|| mz
->on_tree
) {
233 spin_lock_irqsave(&mctz
->lock
, flags
);
234 /* if on-tree, remove it */
236 __mem_cgroup_remove_exceeded(mz
, mctz
);
238 * Insert again. mz->usage_in_excess will be updated.
239 * If excess is 0, no tree ops.
241 __mem_cgroup_insert_exceeded(mz
, mctz
, excess
);
242 spin_unlock_irqrestore(&mctz
->lock
, flags
);
247 void memcg1_remove_from_trees(struct mem_cgroup
*memcg
)
249 struct mem_cgroup_tree_per_node
*mctz
;
250 struct mem_cgroup_per_node
*mz
;
254 mz
= memcg
->nodeinfo
[nid
];
255 mctz
= soft_limit_tree
.rb_tree_per_node
[nid
];
257 mem_cgroup_remove_exceeded(mz
, mctz
);
261 static struct mem_cgroup_per_node
*
262 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node
*mctz
)
264 struct mem_cgroup_per_node
*mz
;
268 if (!mctz
->rb_rightmost
)
269 goto done
; /* Nothing to reclaim from */
271 mz
= rb_entry(mctz
->rb_rightmost
,
272 struct mem_cgroup_per_node
, tree_node
);
274 * Remove the node now but someone else can add it back,
275 * we will to add it back at the end of reclaim to its correct
276 * position in the tree.
278 __mem_cgroup_remove_exceeded(mz
, mctz
);
279 if (!soft_limit_excess(mz
->memcg
) ||
280 !css_tryget(&mz
->memcg
->css
))
286 static struct mem_cgroup_per_node
*
287 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node
*mctz
)
289 struct mem_cgroup_per_node
*mz
;
291 spin_lock_irq(&mctz
->lock
);
292 mz
= __mem_cgroup_largest_soft_limit_node(mctz
);
293 spin_unlock_irq(&mctz
->lock
);
297 static int mem_cgroup_soft_reclaim(struct mem_cgroup
*root_memcg
,
300 unsigned long *total_scanned
)
302 struct mem_cgroup
*victim
= NULL
;
305 unsigned long excess
;
306 unsigned long nr_scanned
;
307 struct mem_cgroup_reclaim_cookie reclaim
= {
311 excess
= soft_limit_excess(root_memcg
);
314 victim
= mem_cgroup_iter(root_memcg
, victim
, &reclaim
);
319 * If we have not been able to reclaim
320 * anything, it might because there are
321 * no reclaimable pages under this hierarchy
326 * We want to do more targeted reclaim.
327 * excess >> 2 is not to excessive so as to
328 * reclaim too much, nor too less that we keep
329 * coming back to reclaim from this cgroup
331 if (total
>= (excess
>> 2) ||
332 (loop
> MEM_CGROUP_MAX_RECLAIM_LOOPS
))
337 total
+= mem_cgroup_shrink_node(victim
, gfp_mask
, false,
339 *total_scanned
+= nr_scanned
;
340 if (!soft_limit_excess(root_memcg
))
343 mem_cgroup_iter_break(root_memcg
, victim
);
347 unsigned long memcg1_soft_limit_reclaim(pg_data_t
*pgdat
, int order
,
349 unsigned long *total_scanned
)
351 unsigned long nr_reclaimed
= 0;
352 struct mem_cgroup_per_node
*mz
, *next_mz
= NULL
;
353 unsigned long reclaimed
;
355 struct mem_cgroup_tree_per_node
*mctz
;
356 unsigned long excess
;
358 if (lru_gen_enabled())
364 mctz
= soft_limit_tree
.rb_tree_per_node
[pgdat
->node_id
];
367 * Do not even bother to check the largest node if the root
368 * is empty. Do it lockless to prevent lock bouncing. Races
369 * are acceptable as soft limit is best effort anyway.
371 if (!mctz
|| RB_EMPTY_ROOT(&mctz
->rb_root
))
375 * This loop can run a while, specially if mem_cgroup's continuously
376 * keep exceeding their soft limit and putting the system under
383 mz
= mem_cgroup_largest_soft_limit_node(mctz
);
387 reclaimed
= mem_cgroup_soft_reclaim(mz
->memcg
, pgdat
,
388 gfp_mask
, total_scanned
);
389 nr_reclaimed
+= reclaimed
;
390 spin_lock_irq(&mctz
->lock
);
393 * If we failed to reclaim anything from this memory cgroup
394 * it is time to move on to the next cgroup
398 next_mz
= __mem_cgroup_largest_soft_limit_node(mctz
);
400 excess
= soft_limit_excess(mz
->memcg
);
402 * One school of thought says that we should not add
403 * back the node to the tree if reclaim returns 0.
404 * But our reclaim could return 0, simply because due
405 * to priority we are exposing a smaller subset of
406 * memory to reclaim from. Consider this as a longer
409 /* If excess == 0, no tree ops */
410 __mem_cgroup_insert_exceeded(mz
, mctz
, excess
);
411 spin_unlock_irq(&mctz
->lock
);
412 css_put(&mz
->memcg
->css
);
415 * Could not reclaim anything and there are no more
416 * mem cgroups to try or we seem to be looping without
417 * reclaiming anything.
421 loop
> MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS
))
423 } while (!nr_reclaimed
);
425 css_put(&next_mz
->memcg
->css
);
430 * A routine for checking "mem" is under move_account() or not.
432 * Checking a cgroup is mc.from or mc.to or under hierarchy of
433 * moving cgroups. This is for waiting at high-memory pressure
436 static bool mem_cgroup_under_move(struct mem_cgroup
*memcg
)
438 struct mem_cgroup
*from
;
439 struct mem_cgroup
*to
;
442 * Unlike task_move routines, we access mc.to, mc.from not under
443 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
451 ret
= mem_cgroup_is_descendant(from
, memcg
) ||
452 mem_cgroup_is_descendant(to
, memcg
);
454 spin_unlock(&mc
.lock
);
458 bool memcg1_wait_acct_move(struct mem_cgroup
*memcg
)
460 if (mc
.moving_task
&& current
!= mc
.moving_task
) {
461 if (mem_cgroup_under_move(memcg
)) {
463 prepare_to_wait(&mc
.waitq
, &wait
, TASK_INTERRUPTIBLE
);
464 /* moving charge context might have finished. */
467 finish_wait(&mc
.waitq
, &wait
);
475 * folio_memcg_lock - Bind a folio to its memcg.
478 * This function prevents unlocked LRU folios from being moved to
481 * It ensures lifetime of the bound memcg. The caller is responsible
482 * for the lifetime of the folio.
484 void folio_memcg_lock(struct folio
*folio
)
486 struct mem_cgroup
*memcg
;
490 * The RCU lock is held throughout the transaction. The fast
491 * path can get away without acquiring the memcg->move_lock
492 * because page moving starts with an RCU grace period.
496 if (mem_cgroup_disabled())
499 memcg
= folio_memcg(folio
);
500 if (unlikely(!memcg
))
503 #ifdef CONFIG_PROVE_LOCKING
504 local_irq_save(flags
);
505 might_lock(&memcg
->move_lock
);
506 local_irq_restore(flags
);
509 if (atomic_read(&memcg
->moving_account
) <= 0)
512 spin_lock_irqsave(&memcg
->move_lock
, flags
);
513 if (memcg
!= folio_memcg(folio
)) {
514 spin_unlock_irqrestore(&memcg
->move_lock
, flags
);
519 * When charge migration first begins, we can have multiple
520 * critical sections holding the fast-path RCU lock and one
521 * holding the slowpath move_lock. Track the task who has the
522 * move_lock for folio_memcg_unlock().
524 memcg
->move_lock_task
= current
;
525 memcg
->move_lock_flags
= flags
;
528 static void __folio_memcg_unlock(struct mem_cgroup
*memcg
)
530 if (memcg
&& memcg
->move_lock_task
== current
) {
531 unsigned long flags
= memcg
->move_lock_flags
;
533 memcg
->move_lock_task
= NULL
;
534 memcg
->move_lock_flags
= 0;
536 spin_unlock_irqrestore(&memcg
->move_lock
, flags
);
543 * folio_memcg_unlock - Release the binding between a folio and its memcg.
546 * This releases the binding created by folio_memcg_lock(). This does
547 * not change the accounting of this folio to its memcg, but it does
548 * permit others to change it.
550 void folio_memcg_unlock(struct folio
*folio
)
552 __folio_memcg_unlock(folio_memcg(folio
));
557 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
558 * @entry: swap entry to be moved
559 * @from: mem_cgroup which the entry is moved from
560 * @to: mem_cgroup which the entry is moved to
562 * It succeeds only when the swap_cgroup's record for this entry is the same
563 * as the mem_cgroup's id of @from.
565 * Returns 0 on success, -EINVAL on failure.
567 * The caller must have charged to @to, IOW, called page_counter_charge() about
568 * both res and memsw, and called css_get().
570 static int mem_cgroup_move_swap_account(swp_entry_t entry
,
571 struct mem_cgroup
*from
, struct mem_cgroup
*to
)
573 unsigned short old_id
, new_id
;
575 old_id
= mem_cgroup_id(from
);
576 new_id
= mem_cgroup_id(to
);
578 if (swap_cgroup_cmpxchg(entry
, old_id
, new_id
) == old_id
) {
579 mod_memcg_state(from
, MEMCG_SWAP
, -1);
580 mod_memcg_state(to
, MEMCG_SWAP
, 1);
586 static inline int mem_cgroup_move_swap_account(swp_entry_t entry
,
587 struct mem_cgroup
*from
, struct mem_cgroup
*to
)
593 static u64
mem_cgroup_move_charge_read(struct cgroup_subsys_state
*css
,
596 return mem_cgroup_from_css(css
)->move_charge_at_immigrate
;
600 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state
*css
,
601 struct cftype
*cft
, u64 val
)
603 struct mem_cgroup
*memcg
= mem_cgroup_from_css(css
);
605 pr_warn_once("Cgroup memory moving (move_charge_at_immigrate) is deprecated. "
606 "Please report your usecase to linux-mm@kvack.org if you "
607 "depend on this functionality.\n");
609 if (val
& ~MOVE_MASK
)
613 * No kind of locking is needed in here, because ->can_attach() will
614 * check this value once in the beginning of the process, and then carry
615 * on with stale data. This means that changes to this value will only
616 * affect task migrations starting after the change.
618 memcg
->move_charge_at_immigrate
= val
;
622 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state
*css
,
623 struct cftype
*cft
, u64 val
)
630 /* Handlers for move charge at task migration. */
631 static int mem_cgroup_do_precharge(unsigned long count
)
635 /* Try a single bulk charge without reclaim first, kswapd may wake */
636 ret
= try_charge(mc
.to
, GFP_KERNEL
& ~__GFP_DIRECT_RECLAIM
, count
);
638 mc
.precharge
+= count
;
642 /* Try charges one by one with reclaim, but do not retry */
644 ret
= try_charge(mc
.to
, GFP_KERNEL
| __GFP_NORETRY
, 1);
658 enum mc_target_type
{
665 static struct page
*mc_handle_present_pte(struct vm_area_struct
*vma
,
666 unsigned long addr
, pte_t ptent
)
668 struct page
*page
= vm_normal_page(vma
, addr
, ptent
);
672 if (PageAnon(page
)) {
673 if (!(mc
.flags
& MOVE_ANON
))
676 if (!(mc
.flags
& MOVE_FILE
))
684 #if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
685 static struct page
*mc_handle_swap_pte(struct vm_area_struct
*vma
,
686 pte_t ptent
, swp_entry_t
*entry
)
688 struct page
*page
= NULL
;
689 swp_entry_t ent
= pte_to_swp_entry(ptent
);
691 if (!(mc
.flags
& MOVE_ANON
))
695 * Handle device private pages that are not accessible by the CPU, but
696 * stored as special swap entries in the page table.
698 if (is_device_private_entry(ent
)) {
699 page
= pfn_swap_entry_to_page(ent
);
700 if (!get_page_unless_zero(page
))
705 if (non_swap_entry(ent
))
709 * Because swap_cache_get_folio() updates some statistics counter,
710 * we call find_get_page() with swapper_space directly.
712 page
= find_get_page(swap_address_space(ent
), swap_cache_index(ent
));
713 entry
->val
= ent
.val
;
718 static struct page
*mc_handle_swap_pte(struct vm_area_struct
*vma
,
719 pte_t ptent
, swp_entry_t
*entry
)
725 static struct page
*mc_handle_file_pte(struct vm_area_struct
*vma
,
726 unsigned long addr
, pte_t ptent
)
731 if (!vma
->vm_file
) /* anonymous vma */
733 if (!(mc
.flags
& MOVE_FILE
))
736 /* folio is moved even if it's not RSS of this task(page-faulted). */
737 /* shmem/tmpfs may report page out on swap: account for that too. */
738 index
= linear_page_index(vma
, addr
);
739 folio
= filemap_get_incore_folio(vma
->vm_file
->f_mapping
, index
);
742 return folio_file_page(folio
, index
);
746 * mem_cgroup_move_account - move account of the folio
748 * @compound: charge the page as compound or small page
749 * @from: mem_cgroup which the folio is moved from.
750 * @to: mem_cgroup which the folio is moved to. @from != @to.
752 * The folio must be locked and not on the LRU.
754 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
757 static int mem_cgroup_move_account(struct folio
*folio
,
759 struct mem_cgroup
*from
,
760 struct mem_cgroup
*to
)
762 struct lruvec
*from_vec
, *to_vec
;
763 struct pglist_data
*pgdat
;
764 unsigned int nr_pages
= compound
? folio_nr_pages(folio
) : 1;
767 VM_BUG_ON(from
== to
);
768 VM_BUG_ON_FOLIO(!folio_test_locked(folio
), folio
);
769 VM_BUG_ON_FOLIO(folio_test_lru(folio
), folio
);
770 VM_BUG_ON(compound
&& !folio_test_large(folio
));
773 if (folio_memcg(folio
) != from
)
776 pgdat
= folio_pgdat(folio
);
777 from_vec
= mem_cgroup_lruvec(from
, pgdat
);
778 to_vec
= mem_cgroup_lruvec(to
, pgdat
);
780 folio_memcg_lock(folio
);
782 if (folio_test_anon(folio
)) {
783 if (folio_mapped(folio
)) {
784 __mod_lruvec_state(from_vec
, NR_ANON_MAPPED
, -nr_pages
);
785 __mod_lruvec_state(to_vec
, NR_ANON_MAPPED
, nr_pages
);
786 if (folio_test_pmd_mappable(folio
)) {
787 __mod_lruvec_state(from_vec
, NR_ANON_THPS
,
789 __mod_lruvec_state(to_vec
, NR_ANON_THPS
,
794 __mod_lruvec_state(from_vec
, NR_FILE_PAGES
, -nr_pages
);
795 __mod_lruvec_state(to_vec
, NR_FILE_PAGES
, nr_pages
);
797 if (folio_test_swapbacked(folio
)) {
798 __mod_lruvec_state(from_vec
, NR_SHMEM
, -nr_pages
);
799 __mod_lruvec_state(to_vec
, NR_SHMEM
, nr_pages
);
802 if (folio_mapped(folio
)) {
803 __mod_lruvec_state(from_vec
, NR_FILE_MAPPED
, -nr_pages
);
804 __mod_lruvec_state(to_vec
, NR_FILE_MAPPED
, nr_pages
);
807 if (folio_test_dirty(folio
)) {
808 struct address_space
*mapping
= folio_mapping(folio
);
810 if (mapping_can_writeback(mapping
)) {
811 __mod_lruvec_state(from_vec
, NR_FILE_DIRTY
,
813 __mod_lruvec_state(to_vec
, NR_FILE_DIRTY
,
820 if (folio_test_swapcache(folio
)) {
821 __mod_lruvec_state(from_vec
, NR_SWAPCACHE
, -nr_pages
);
822 __mod_lruvec_state(to_vec
, NR_SWAPCACHE
, nr_pages
);
825 if (folio_test_writeback(folio
)) {
826 __mod_lruvec_state(from_vec
, NR_WRITEBACK
, -nr_pages
);
827 __mod_lruvec_state(to_vec
, NR_WRITEBACK
, nr_pages
);
831 * All state has been migrated, let's switch to the new memcg.
833 * It is safe to change page's memcg here because the page
834 * is referenced, charged, isolated, and locked: we can't race
835 * with (un)charging, migration, LRU putback, or anything else
836 * that would rely on a stable page's memory cgroup.
838 * Note that folio_memcg_lock is a memcg lock, not a page lock,
839 * to save space. As soon as we switch page's memory cgroup to a
840 * new memcg that isn't locked, the above state can change
841 * concurrently again. Make sure we're truly done with it.
848 folio
->memcg_data
= (unsigned long)to
;
850 __folio_memcg_unlock(from
);
853 nid
= folio_nid(folio
);
856 mem_cgroup_charge_statistics(to
, nr_pages
);
857 memcg1_check_events(to
, nid
);
858 mem_cgroup_charge_statistics(from
, -nr_pages
);
859 memcg1_check_events(from
, nid
);
866 * get_mctgt_type - get target type of moving charge
867 * @vma: the vma the pte to be checked belongs
868 * @addr: the address corresponding to the pte to be checked
869 * @ptent: the pte to be checked
870 * @target: the pointer the target page or swap ent will be stored(can be NULL)
872 * Context: Called with pte lock held.
874 * * MC_TARGET_NONE - If the pte is not a target for move charge.
875 * * MC_TARGET_PAGE - If the page corresponding to this pte is a target for
876 * move charge. If @target is not NULL, the folio is stored in target->folio
877 * with extra refcnt taken (Caller should release it).
878 * * MC_TARGET_SWAP - If the swap entry corresponding to this pte is a
879 * target for charge migration. If @target is not NULL, the entry is
880 * stored in target->ent.
881 * * MC_TARGET_DEVICE - Like MC_TARGET_PAGE but page is device memory and
882 * thus not on the lru. For now such page is charged like a regular page
883 * would be as it is just special memory taking the place of a regular page.
884 * See Documentations/vm/hmm.txt and include/linux/hmm.h
886 static enum mc_target_type
get_mctgt_type(struct vm_area_struct
*vma
,
887 unsigned long addr
, pte_t ptent
, union mc_target
*target
)
889 struct page
*page
= NULL
;
891 enum mc_target_type ret
= MC_TARGET_NONE
;
892 swp_entry_t ent
= { .val
= 0 };
894 if (pte_present(ptent
))
895 page
= mc_handle_present_pte(vma
, addr
, ptent
);
896 else if (pte_none_mostly(ptent
))
898 * PTE markers should be treated as a none pte here, separated
899 * from other swap handling below.
901 page
= mc_handle_file_pte(vma
, addr
, ptent
);
902 else if (is_swap_pte(ptent
))
903 page
= mc_handle_swap_pte(vma
, ptent
, &ent
);
906 folio
= page_folio(page
);
907 if (target
&& page
) {
908 if (!folio_trylock(folio
)) {
913 * page_mapped() must be stable during the move. This
914 * pte is locked, so if it's present, the page cannot
915 * become unmapped. If it isn't, we have only partial
916 * control over the mapped state: the page lock will
917 * prevent new faults against pagecache and swapcache,
918 * so an unmapped page cannot become mapped. However,
919 * if the page is already mapped elsewhere, it can
920 * unmap, and there is nothing we can do about it.
921 * Alas, skip moving the page in this case.
923 if (!pte_present(ptent
) && page_mapped(page
)) {
930 if (!page
&& !ent
.val
)
934 * Do only loose check w/o serialization.
935 * mem_cgroup_move_account() checks the page is valid or
936 * not under LRU exclusion.
938 if (folio_memcg(folio
) == mc
.from
) {
939 ret
= MC_TARGET_PAGE
;
940 if (folio_is_device_private(folio
) ||
941 folio_is_device_coherent(folio
))
942 ret
= MC_TARGET_DEVICE
;
944 target
->folio
= folio
;
946 if (!ret
|| !target
) {
953 * There is a swap entry and a page doesn't exist or isn't charged.
954 * But we cannot move a tail-page in a THP.
956 if (ent
.val
&& !ret
&& (!page
|| !PageTransCompound(page
)) &&
957 mem_cgroup_id(mc
.from
) == lookup_swap_cgroup_id(ent
)) {
958 ret
= MC_TARGET_SWAP
;
965 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
967 * We don't consider PMD mapped swapping or file mapped pages because THP does
968 * not support them for now.
969 * Caller should make sure that pmd_trans_huge(pmd) is true.
971 static enum mc_target_type
get_mctgt_type_thp(struct vm_area_struct
*vma
,
972 unsigned long addr
, pmd_t pmd
, union mc_target
*target
)
974 struct page
*page
= NULL
;
976 enum mc_target_type ret
= MC_TARGET_NONE
;
978 if (unlikely(is_swap_pmd(pmd
))) {
979 VM_BUG_ON(thp_migration_supported() &&
980 !is_pmd_migration_entry(pmd
));
983 page
= pmd_page(pmd
);
984 VM_BUG_ON_PAGE(!page
|| !PageHead(page
), page
);
985 folio
= page_folio(page
);
986 if (!(mc
.flags
& MOVE_ANON
))
988 if (folio_memcg(folio
) == mc
.from
) {
989 ret
= MC_TARGET_PAGE
;
992 if (!folio_trylock(folio
)) {
994 return MC_TARGET_NONE
;
996 target
->folio
= folio
;
1002 static inline enum mc_target_type
get_mctgt_type_thp(struct vm_area_struct
*vma
,
1003 unsigned long addr
, pmd_t pmd
, union mc_target
*target
)
1005 return MC_TARGET_NONE
;
1009 static int mem_cgroup_count_precharge_pte_range(pmd_t
*pmd
,
1010 unsigned long addr
, unsigned long end
,
1011 struct mm_walk
*walk
)
1013 struct vm_area_struct
*vma
= walk
->vma
;
1017 ptl
= pmd_trans_huge_lock(pmd
, vma
);
1020 * Note their can not be MC_TARGET_DEVICE for now as we do not
1021 * support transparent huge page with MEMORY_DEVICE_PRIVATE but
1022 * this might change.
1024 if (get_mctgt_type_thp(vma
, addr
, *pmd
, NULL
) == MC_TARGET_PAGE
)
1025 mc
.precharge
+= HPAGE_PMD_NR
;
1030 pte
= pte_offset_map_lock(vma
->vm_mm
, pmd
, addr
, &ptl
);
1033 for (; addr
!= end
; pte
++, addr
+= PAGE_SIZE
)
1034 if (get_mctgt_type(vma
, addr
, ptep_get(pte
), NULL
))
1035 mc
.precharge
++; /* increment precharge temporarily */
1036 pte_unmap_unlock(pte
- 1, ptl
);
1042 static const struct mm_walk_ops precharge_walk_ops
= {
1043 .pmd_entry
= mem_cgroup_count_precharge_pte_range
,
1044 .walk_lock
= PGWALK_RDLOCK
,
1047 static unsigned long mem_cgroup_count_precharge(struct mm_struct
*mm
)
1049 unsigned long precharge
;
1052 walk_page_range(mm
, 0, ULONG_MAX
, &precharge_walk_ops
, NULL
);
1053 mmap_read_unlock(mm
);
1055 precharge
= mc
.precharge
;
1061 static int mem_cgroup_precharge_mc(struct mm_struct
*mm
)
1063 unsigned long precharge
= mem_cgroup_count_precharge(mm
);
1065 VM_BUG_ON(mc
.moving_task
);
1066 mc
.moving_task
= current
;
1067 return mem_cgroup_do_precharge(precharge
);
1070 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
1071 static void __mem_cgroup_clear_mc(void)
1073 struct mem_cgroup
*from
= mc
.from
;
1074 struct mem_cgroup
*to
= mc
.to
;
1076 /* we must uncharge all the leftover precharges from mc.to */
1078 mem_cgroup_cancel_charge(mc
.to
, mc
.precharge
);
1082 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
1083 * we must uncharge here.
1085 if (mc
.moved_charge
) {
1086 mem_cgroup_cancel_charge(mc
.from
, mc
.moved_charge
);
1087 mc
.moved_charge
= 0;
1089 /* we must fixup refcnts and charges */
1090 if (mc
.moved_swap
) {
1091 /* uncharge swap account from the old cgroup */
1092 if (!mem_cgroup_is_root(mc
.from
))
1093 page_counter_uncharge(&mc
.from
->memsw
, mc
.moved_swap
);
1095 mem_cgroup_id_put_many(mc
.from
, mc
.moved_swap
);
1098 * we charged both to->memory and to->memsw, so we
1099 * should uncharge to->memory.
1101 if (!mem_cgroup_is_root(mc
.to
))
1102 page_counter_uncharge(&mc
.to
->memory
, mc
.moved_swap
);
1106 memcg1_oom_recover(from
);
1107 memcg1_oom_recover(to
);
1108 wake_up_all(&mc
.waitq
);
1111 static void mem_cgroup_clear_mc(void)
1113 struct mm_struct
*mm
= mc
.mm
;
1116 * we must clear moving_task before waking up waiters at the end of
1119 mc
.moving_task
= NULL
;
1120 __mem_cgroup_clear_mc();
1121 spin_lock(&mc
.lock
);
1125 spin_unlock(&mc
.lock
);
1130 int memcg1_can_attach(struct cgroup_taskset
*tset
)
1132 struct cgroup_subsys_state
*css
;
1133 struct mem_cgroup
*memcg
= NULL
; /* unneeded init to make gcc happy */
1134 struct mem_cgroup
*from
;
1135 struct task_struct
*leader
, *p
;
1136 struct mm_struct
*mm
;
1137 unsigned long move_flags
;
1140 /* charge immigration isn't supported on the default hierarchy */
1141 if (cgroup_subsys_on_dfl(memory_cgrp_subsys
))
1145 * Multi-process migrations only happen on the default hierarchy
1146 * where charge immigration is not used. Perform charge
1147 * immigration if @tset contains a leader and whine if there are
1151 cgroup_taskset_for_each_leader(leader
, css
, tset
) {
1154 memcg
= mem_cgroup_from_css(css
);
1160 * We are now committed to this value whatever it is. Changes in this
1161 * tunable will only affect upcoming migrations, not the current one.
1162 * So we need to save it, and keep it going.
1164 move_flags
= READ_ONCE(memcg
->move_charge_at_immigrate
);
1168 from
= mem_cgroup_from_task(p
);
1170 VM_BUG_ON(from
== memcg
);
1172 mm
= get_task_mm(p
);
1175 /* We move charges only when we move a owner of the mm */
1176 if (mm
->owner
== p
) {
1179 VM_BUG_ON(mc
.precharge
);
1180 VM_BUG_ON(mc
.moved_charge
);
1181 VM_BUG_ON(mc
.moved_swap
);
1183 spin_lock(&mc
.lock
);
1187 mc
.flags
= move_flags
;
1188 spin_unlock(&mc
.lock
);
1189 /* We set mc.moving_task later */
1191 ret
= mem_cgroup_precharge_mc(mm
);
1193 mem_cgroup_clear_mc();
1200 void memcg1_cancel_attach(struct cgroup_taskset
*tset
)
1203 mem_cgroup_clear_mc();
1206 static int mem_cgroup_move_charge_pte_range(pmd_t
*pmd
,
1207 unsigned long addr
, unsigned long end
,
1208 struct mm_walk
*walk
)
1211 struct vm_area_struct
*vma
= walk
->vma
;
1214 enum mc_target_type target_type
;
1215 union mc_target target
;
1216 struct folio
*folio
;
1218 ptl
= pmd_trans_huge_lock(pmd
, vma
);
1220 if (mc
.precharge
< HPAGE_PMD_NR
) {
1224 target_type
= get_mctgt_type_thp(vma
, addr
, *pmd
, &target
);
1225 if (target_type
== MC_TARGET_PAGE
) {
1226 folio
= target
.folio
;
1227 if (folio_isolate_lru(folio
)) {
1228 if (!mem_cgroup_move_account(folio
, true,
1230 mc
.precharge
-= HPAGE_PMD_NR
;
1231 mc
.moved_charge
+= HPAGE_PMD_NR
;
1233 folio_putback_lru(folio
);
1235 folio_unlock(folio
);
1237 } else if (target_type
== MC_TARGET_DEVICE
) {
1238 folio
= target
.folio
;
1239 if (!mem_cgroup_move_account(folio
, true,
1241 mc
.precharge
-= HPAGE_PMD_NR
;
1242 mc
.moved_charge
+= HPAGE_PMD_NR
;
1244 folio_unlock(folio
);
1252 pte
= pte_offset_map_lock(vma
->vm_mm
, pmd
, addr
, &ptl
);
1255 for (; addr
!= end
; addr
+= PAGE_SIZE
) {
1256 pte_t ptent
= ptep_get(pte
++);
1257 bool device
= false;
1263 switch (get_mctgt_type(vma
, addr
, ptent
, &target
)) {
1264 case MC_TARGET_DEVICE
:
1267 case MC_TARGET_PAGE
:
1268 folio
= target
.folio
;
1270 * We can have a part of the split pmd here. Moving it
1271 * can be done but it would be too convoluted so simply
1272 * ignore such a partial THP and keep it in original
1273 * memcg. There should be somebody mapping the head.
1275 if (folio_test_large(folio
))
1277 if (!device
&& !folio_isolate_lru(folio
))
1279 if (!mem_cgroup_move_account(folio
, false,
1282 /* we uncharge from mc.from later. */
1286 folio_putback_lru(folio
);
1287 put
: /* get_mctgt_type() gets & locks the page */
1288 folio_unlock(folio
);
1291 case MC_TARGET_SWAP
:
1293 if (!mem_cgroup_move_swap_account(ent
, mc
.from
, mc
.to
)) {
1295 mem_cgroup_id_get_many(mc
.to
, 1);
1296 /* we fixup other refcnts and charges later. */
1304 pte_unmap_unlock(pte
- 1, ptl
);
1309 * We have consumed all precharges we got in can_attach().
1310 * We try charge one by one, but don't do any additional
1311 * charges to mc.to if we have failed in charge once in attach()
1314 ret
= mem_cgroup_do_precharge(1);
1322 static const struct mm_walk_ops charge_walk_ops
= {
1323 .pmd_entry
= mem_cgroup_move_charge_pte_range
,
1324 .walk_lock
= PGWALK_RDLOCK
,
1327 static void mem_cgroup_move_charge(void)
1329 lru_add_drain_all();
1331 * Signal folio_memcg_lock() to take the memcg's move_lock
1332 * while we're moving its pages to another memcg. Then wait
1333 * for already started RCU-only updates to finish.
1335 atomic_inc(&mc
.from
->moving_account
);
1338 if (unlikely(!mmap_read_trylock(mc
.mm
))) {
1340 * Someone who are holding the mmap_lock might be waiting in
1341 * waitq. So we cancel all extra charges, wake up all waiters,
1342 * and retry. Because we cancel precharges, we might not be able
1343 * to move enough charges, but moving charge is a best-effort
1344 * feature anyway, so it wouldn't be a big problem.
1346 __mem_cgroup_clear_mc();
1351 * When we have consumed all precharges and failed in doing
1352 * additional charge, the page walk just aborts.
1354 walk_page_range(mc
.mm
, 0, ULONG_MAX
, &charge_walk_ops
, NULL
);
1355 mmap_read_unlock(mc
.mm
);
1356 atomic_dec(&mc
.from
->moving_account
);
1359 void memcg1_move_task(void)
1362 mem_cgroup_move_charge();
1363 mem_cgroup_clear_mc();
1367 #else /* !CONFIG_MMU */
1368 int memcg1_can_attach(struct cgroup_taskset
*tset
)
1372 void memcg1_cancel_attach(struct cgroup_taskset
*tset
)
1375 void memcg1_move_task(void)
1380 static void __mem_cgroup_threshold(struct mem_cgroup
*memcg
, bool swap
)
1382 struct mem_cgroup_threshold_ary
*t
;
1383 unsigned long usage
;
1388 t
= rcu_dereference(memcg
->thresholds
.primary
);
1390 t
= rcu_dereference(memcg
->memsw_thresholds
.primary
);
1395 usage
= mem_cgroup_usage(memcg
, swap
);
1398 * current_threshold points to threshold just below or equal to usage.
1399 * If it's not true, a threshold was crossed after last
1400 * call of __mem_cgroup_threshold().
1402 i
= t
->current_threshold
;
1405 * Iterate backward over array of thresholds starting from
1406 * current_threshold and check if a threshold is crossed.
1407 * If none of thresholds below usage is crossed, we read
1408 * only one element of the array here.
1410 for (; i
>= 0 && unlikely(t
->entries
[i
].threshold
> usage
); i
--)
1411 eventfd_signal(t
->entries
[i
].eventfd
);
1413 /* i = current_threshold + 1 */
1417 * Iterate forward over array of thresholds starting from
1418 * current_threshold+1 and check if a threshold is crossed.
1419 * If none of thresholds above usage is crossed, we read
1420 * only one element of the array here.
1422 for (; i
< t
->size
&& unlikely(t
->entries
[i
].threshold
<= usage
); i
++)
1423 eventfd_signal(t
->entries
[i
].eventfd
);
1425 /* Update current_threshold */
1426 t
->current_threshold
= i
- 1;
1431 static void mem_cgroup_threshold(struct mem_cgroup
*memcg
)
1434 __mem_cgroup_threshold(memcg
, false);
1435 if (do_memsw_account())
1436 __mem_cgroup_threshold(memcg
, true);
1438 memcg
= parent_mem_cgroup(memcg
);
1443 * Check events in order.
1446 void memcg1_check_events(struct mem_cgroup
*memcg
, int nid
)
1448 if (IS_ENABLED(CONFIG_PREEMPT_RT
))
1451 /* threshold event is triggered in finer grain than soft limit */
1452 if (unlikely(mem_cgroup_event_ratelimit(memcg
,
1453 MEM_CGROUP_TARGET_THRESH
))) {
1456 do_softlimit
= mem_cgroup_event_ratelimit(memcg
,
1457 MEM_CGROUP_TARGET_SOFTLIMIT
);
1458 mem_cgroup_threshold(memcg
);
1459 if (unlikely(do_softlimit
))
1460 memcg1_update_tree(memcg
, nid
);
1464 static int compare_thresholds(const void *a
, const void *b
)
1466 const struct mem_cgroup_threshold
*_a
= a
;
1467 const struct mem_cgroup_threshold
*_b
= b
;
1469 if (_a
->threshold
> _b
->threshold
)
1472 if (_a
->threshold
< _b
->threshold
)
1478 static int mem_cgroup_oom_notify_cb(struct mem_cgroup
*memcg
)
1480 struct mem_cgroup_eventfd_list
*ev
;
1482 spin_lock(&memcg_oom_lock
);
1484 list_for_each_entry(ev
, &memcg
->oom_notify
, list
)
1485 eventfd_signal(ev
->eventfd
);
1487 spin_unlock(&memcg_oom_lock
);
1491 static void mem_cgroup_oom_notify(struct mem_cgroup
*memcg
)
1493 struct mem_cgroup
*iter
;
1495 for_each_mem_cgroup_tree(iter
, memcg
)
1496 mem_cgroup_oom_notify_cb(iter
);
1499 static int __mem_cgroup_usage_register_event(struct mem_cgroup
*memcg
,
1500 struct eventfd_ctx
*eventfd
, const char *args
, enum res_type type
)
1502 struct mem_cgroup_thresholds
*thresholds
;
1503 struct mem_cgroup_threshold_ary
*new;
1504 unsigned long threshold
;
1505 unsigned long usage
;
1508 ret
= page_counter_memparse(args
, "-1", &threshold
);
1512 mutex_lock(&memcg
->thresholds_lock
);
1515 thresholds
= &memcg
->thresholds
;
1516 usage
= mem_cgroup_usage(memcg
, false);
1517 } else if (type
== _MEMSWAP
) {
1518 thresholds
= &memcg
->memsw_thresholds
;
1519 usage
= mem_cgroup_usage(memcg
, true);
1523 /* Check if a threshold crossed before adding a new one */
1524 if (thresholds
->primary
)
1525 __mem_cgroup_threshold(memcg
, type
== _MEMSWAP
);
1527 size
= thresholds
->primary
? thresholds
->primary
->size
+ 1 : 1;
1529 /* Allocate memory for new array of thresholds */
1530 new = kmalloc(struct_size(new, entries
, size
), GFP_KERNEL
);
1537 /* Copy thresholds (if any) to new array */
1538 if (thresholds
->primary
)
1539 memcpy(new->entries
, thresholds
->primary
->entries
,
1540 flex_array_size(new, entries
, size
- 1));
1542 /* Add new threshold */
1543 new->entries
[size
- 1].eventfd
= eventfd
;
1544 new->entries
[size
- 1].threshold
= threshold
;
1546 /* Sort thresholds. Registering of new threshold isn't time-critical */
1547 sort(new->entries
, size
, sizeof(*new->entries
),
1548 compare_thresholds
, NULL
);
1550 /* Find current threshold */
1551 new->current_threshold
= -1;
1552 for (i
= 0; i
< size
; i
++) {
1553 if (new->entries
[i
].threshold
<= usage
) {
1555 * new->current_threshold will not be used until
1556 * rcu_assign_pointer(), so it's safe to increment
1559 ++new->current_threshold
;
1564 /* Free old spare buffer and save old primary buffer as spare */
1565 kfree(thresholds
->spare
);
1566 thresholds
->spare
= thresholds
->primary
;
1568 rcu_assign_pointer(thresholds
->primary
, new);
1570 /* To be sure that nobody uses thresholds */
1574 mutex_unlock(&memcg
->thresholds_lock
);
1579 static int mem_cgroup_usage_register_event(struct mem_cgroup
*memcg
,
1580 struct eventfd_ctx
*eventfd
, const char *args
)
1582 return __mem_cgroup_usage_register_event(memcg
, eventfd
, args
, _MEM
);
1585 static int memsw_cgroup_usage_register_event(struct mem_cgroup
*memcg
,
1586 struct eventfd_ctx
*eventfd
, const char *args
)
1588 return __mem_cgroup_usage_register_event(memcg
, eventfd
, args
, _MEMSWAP
);
1591 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup
*memcg
,
1592 struct eventfd_ctx
*eventfd
, enum res_type type
)
1594 struct mem_cgroup_thresholds
*thresholds
;
1595 struct mem_cgroup_threshold_ary
*new;
1596 unsigned long usage
;
1597 int i
, j
, size
, entries
;
1599 mutex_lock(&memcg
->thresholds_lock
);
1602 thresholds
= &memcg
->thresholds
;
1603 usage
= mem_cgroup_usage(memcg
, false);
1604 } else if (type
== _MEMSWAP
) {
1605 thresholds
= &memcg
->memsw_thresholds
;
1606 usage
= mem_cgroup_usage(memcg
, true);
1610 if (!thresholds
->primary
)
1613 /* Check if a threshold crossed before removing */
1614 __mem_cgroup_threshold(memcg
, type
== _MEMSWAP
);
1616 /* Calculate new number of threshold */
1618 for (i
= 0; i
< thresholds
->primary
->size
; i
++) {
1619 if (thresholds
->primary
->entries
[i
].eventfd
!= eventfd
)
1625 new = thresholds
->spare
;
1627 /* If no items related to eventfd have been cleared, nothing to do */
1631 /* Set thresholds array to NULL if we don't have thresholds */
1640 /* Copy thresholds and find current threshold */
1641 new->current_threshold
= -1;
1642 for (i
= 0, j
= 0; i
< thresholds
->primary
->size
; i
++) {
1643 if (thresholds
->primary
->entries
[i
].eventfd
== eventfd
)
1646 new->entries
[j
] = thresholds
->primary
->entries
[i
];
1647 if (new->entries
[j
].threshold
<= usage
) {
1649 * new->current_threshold will not be used
1650 * until rcu_assign_pointer(), so it's safe to increment
1653 ++new->current_threshold
;
1659 /* Swap primary and spare array */
1660 thresholds
->spare
= thresholds
->primary
;
1662 rcu_assign_pointer(thresholds
->primary
, new);
1664 /* To be sure that nobody uses thresholds */
1667 /* If all events are unregistered, free the spare array */
1669 kfree(thresholds
->spare
);
1670 thresholds
->spare
= NULL
;
1673 mutex_unlock(&memcg
->thresholds_lock
);
1676 static void mem_cgroup_usage_unregister_event(struct mem_cgroup
*memcg
,
1677 struct eventfd_ctx
*eventfd
)
1679 return __mem_cgroup_usage_unregister_event(memcg
, eventfd
, _MEM
);
1682 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup
*memcg
,
1683 struct eventfd_ctx
*eventfd
)
1685 return __mem_cgroup_usage_unregister_event(memcg
, eventfd
, _MEMSWAP
);
1688 static int mem_cgroup_oom_register_event(struct mem_cgroup
*memcg
,
1689 struct eventfd_ctx
*eventfd
, const char *args
)
1691 struct mem_cgroup_eventfd_list
*event
;
1693 event
= kmalloc(sizeof(*event
), GFP_KERNEL
);
1697 spin_lock(&memcg_oom_lock
);
1699 event
->eventfd
= eventfd
;
1700 list_add(&event
->list
, &memcg
->oom_notify
);
1702 /* already in OOM ? */
1703 if (memcg
->under_oom
)
1704 eventfd_signal(eventfd
);
1705 spin_unlock(&memcg_oom_lock
);
1710 static void mem_cgroup_oom_unregister_event(struct mem_cgroup
*memcg
,
1711 struct eventfd_ctx
*eventfd
)
1713 struct mem_cgroup_eventfd_list
*ev
, *tmp
;
1715 spin_lock(&memcg_oom_lock
);
1717 list_for_each_entry_safe(ev
, tmp
, &memcg
->oom_notify
, list
) {
1718 if (ev
->eventfd
== eventfd
) {
1719 list_del(&ev
->list
);
1724 spin_unlock(&memcg_oom_lock
);
1728 * DO NOT USE IN NEW FILES.
1730 * "cgroup.event_control" implementation.
1732 * This is way over-engineered. It tries to support fully configurable
1733 * events for each user. Such level of flexibility is completely
1734 * unnecessary especially in the light of the planned unified hierarchy.
1736 * Please deprecate this and replace with something simpler if at all
1741 * Unregister event and free resources.
1743 * Gets called from workqueue.
1745 static void memcg_event_remove(struct work_struct
*work
)
1747 struct mem_cgroup_event
*event
=
1748 container_of(work
, struct mem_cgroup_event
, remove
);
1749 struct mem_cgroup
*memcg
= event
->memcg
;
1751 remove_wait_queue(event
->wqh
, &event
->wait
);
1753 event
->unregister_event(memcg
, event
->eventfd
);
1755 /* Notify userspace the event is going away. */
1756 eventfd_signal(event
->eventfd
);
1758 eventfd_ctx_put(event
->eventfd
);
1760 css_put(&memcg
->css
);
1764 * Gets called on EPOLLHUP on eventfd when user closes it.
1766 * Called with wqh->lock held and interrupts disabled.
1768 static int memcg_event_wake(wait_queue_entry_t
*wait
, unsigned mode
,
1769 int sync
, void *key
)
1771 struct mem_cgroup_event
*event
=
1772 container_of(wait
, struct mem_cgroup_event
, wait
);
1773 struct mem_cgroup
*memcg
= event
->memcg
;
1774 __poll_t flags
= key_to_poll(key
);
1776 if (flags
& EPOLLHUP
) {
1778 * If the event has been detached at cgroup removal, we
1779 * can simply return knowing the other side will cleanup
1782 * We can't race against event freeing since the other
1783 * side will require wqh->lock via remove_wait_queue(),
1786 spin_lock(&memcg
->event_list_lock
);
1787 if (!list_empty(&event
->list
)) {
1788 list_del_init(&event
->list
);
1790 * We are in atomic context, but cgroup_event_remove()
1791 * may sleep, so we have to call it in workqueue.
1793 schedule_work(&event
->remove
);
1795 spin_unlock(&memcg
->event_list_lock
);
1801 static void memcg_event_ptable_queue_proc(struct file
*file
,
1802 wait_queue_head_t
*wqh
, poll_table
*pt
)
1804 struct mem_cgroup_event
*event
=
1805 container_of(pt
, struct mem_cgroup_event
, pt
);
1808 add_wait_queue(wqh
, &event
->wait
);
1812 * DO NOT USE IN NEW FILES.
1814 * Parse input and register new cgroup event handler.
1816 * Input must be in format '<event_fd> <control_fd> <args>'.
1817 * Interpretation of args is defined by control file implementation.
1819 static ssize_t
memcg_write_event_control(struct kernfs_open_file
*of
,
1820 char *buf
, size_t nbytes
, loff_t off
)
1822 struct cgroup_subsys_state
*css
= of_css(of
);
1823 struct mem_cgroup
*memcg
= mem_cgroup_from_css(css
);
1824 struct mem_cgroup_event
*event
;
1825 struct cgroup_subsys_state
*cfile_css
;
1826 unsigned int efd
, cfd
;
1829 struct dentry
*cdentry
;
1834 if (IS_ENABLED(CONFIG_PREEMPT_RT
))
1837 buf
= strstrip(buf
);
1839 efd
= simple_strtoul(buf
, &endp
, 10);
1844 cfd
= simple_strtoul(buf
, &endp
, 10);
1845 if ((*endp
!= ' ') && (*endp
!= '\0'))
1849 event
= kzalloc(sizeof(*event
), GFP_KERNEL
);
1853 event
->memcg
= memcg
;
1854 INIT_LIST_HEAD(&event
->list
);
1855 init_poll_funcptr(&event
->pt
, memcg_event_ptable_queue_proc
);
1856 init_waitqueue_func_entry(&event
->wait
, memcg_event_wake
);
1857 INIT_WORK(&event
->remove
, memcg_event_remove
);
1865 event
->eventfd
= eventfd_ctx_fileget(efile
.file
);
1866 if (IS_ERR(event
->eventfd
)) {
1867 ret
= PTR_ERR(event
->eventfd
);
1874 goto out_put_eventfd
;
1877 /* the process need read permission on control file */
1878 /* AV: shouldn't we check that it's been opened for read instead? */
1879 ret
= file_permission(cfile
.file
, MAY_READ
);
1884 * The control file must be a regular cgroup1 file. As a regular cgroup
1885 * file can't be renamed, it's safe to access its name afterwards.
1887 cdentry
= cfile
.file
->f_path
.dentry
;
1888 if (cdentry
->d_sb
->s_type
!= &cgroup_fs_type
|| !d_is_reg(cdentry
)) {
1894 * Determine the event callbacks and set them in @event. This used
1895 * to be done via struct cftype but cgroup core no longer knows
1896 * about these events. The following is crude but the whole thing
1897 * is for compatibility anyway.
1899 * DO NOT ADD NEW FILES.
1901 name
= cdentry
->d_name
.name
;
1903 if (!strcmp(name
, "memory.usage_in_bytes")) {
1904 event
->register_event
= mem_cgroup_usage_register_event
;
1905 event
->unregister_event
= mem_cgroup_usage_unregister_event
;
1906 } else if (!strcmp(name
, "memory.oom_control")) {
1907 event
->register_event
= mem_cgroup_oom_register_event
;
1908 event
->unregister_event
= mem_cgroup_oom_unregister_event
;
1909 } else if (!strcmp(name
, "memory.pressure_level")) {
1910 event
->register_event
= vmpressure_register_event
;
1911 event
->unregister_event
= vmpressure_unregister_event
;
1912 } else if (!strcmp(name
, "memory.memsw.usage_in_bytes")) {
1913 event
->register_event
= memsw_cgroup_usage_register_event
;
1914 event
->unregister_event
= memsw_cgroup_usage_unregister_event
;
1921 * Verify @cfile should belong to @css. Also, remaining events are
1922 * automatically removed on cgroup destruction but the removal is
1923 * asynchronous, so take an extra ref on @css.
1925 cfile_css
= css_tryget_online_from_dir(cdentry
->d_parent
,
1926 &memory_cgrp_subsys
);
1928 if (IS_ERR(cfile_css
))
1930 if (cfile_css
!= css
) {
1935 ret
= event
->register_event(memcg
, event
->eventfd
, buf
);
1939 vfs_poll(efile
.file
, &event
->pt
);
1941 spin_lock_irq(&memcg
->event_list_lock
);
1942 list_add(&event
->list
, &memcg
->event_list
);
1943 spin_unlock_irq(&memcg
->event_list_lock
);
1955 eventfd_ctx_put(event
->eventfd
);
1964 void memcg1_memcg_init(struct mem_cgroup
*memcg
)
1966 INIT_LIST_HEAD(&memcg
->oom_notify
);
1967 mutex_init(&memcg
->thresholds_lock
);
1968 spin_lock_init(&memcg
->move_lock
);
1969 INIT_LIST_HEAD(&memcg
->event_list
);
1970 spin_lock_init(&memcg
->event_list_lock
);
1973 void memcg1_css_offline(struct mem_cgroup
*memcg
)
1975 struct mem_cgroup_event
*event
, *tmp
;
1978 * Unregister events and notify userspace.
1979 * Notify userspace about cgroup removing only after rmdir of cgroup
1980 * directory to avoid race between userspace and kernelspace.
1982 spin_lock_irq(&memcg
->event_list_lock
);
1983 list_for_each_entry_safe(event
, tmp
, &memcg
->event_list
, list
) {
1984 list_del_init(&event
->list
);
1985 schedule_work(&event
->remove
);
1987 spin_unlock_irq(&memcg
->event_list_lock
);
1991 * Check OOM-Killer is already running under our hierarchy.
1992 * If someone is running, return false.
1994 static bool mem_cgroup_oom_trylock(struct mem_cgroup
*memcg
)
1996 struct mem_cgroup
*iter
, *failed
= NULL
;
1998 spin_lock(&memcg_oom_lock
);
2000 for_each_mem_cgroup_tree(iter
, memcg
) {
2001 if (iter
->oom_lock
) {
2003 * this subtree of our hierarchy is already locked
2004 * so we cannot give a lock.
2007 mem_cgroup_iter_break(memcg
, iter
);
2010 iter
->oom_lock
= true;
2015 * OK, we failed to lock the whole subtree so we have
2016 * to clean up what we set up to the failing subtree
2018 for_each_mem_cgroup_tree(iter
, memcg
) {
2019 if (iter
== failed
) {
2020 mem_cgroup_iter_break(memcg
, iter
);
2023 iter
->oom_lock
= false;
2026 mutex_acquire(&memcg_oom_lock_dep_map
, 0, 1, _RET_IP_
);
2028 spin_unlock(&memcg_oom_lock
);
2033 static void mem_cgroup_oom_unlock(struct mem_cgroup
*memcg
)
2035 struct mem_cgroup
*iter
;
2037 spin_lock(&memcg_oom_lock
);
2038 mutex_release(&memcg_oom_lock_dep_map
, _RET_IP_
);
2039 for_each_mem_cgroup_tree(iter
, memcg
)
2040 iter
->oom_lock
= false;
2041 spin_unlock(&memcg_oom_lock
);
2044 static void mem_cgroup_mark_under_oom(struct mem_cgroup
*memcg
)
2046 struct mem_cgroup
*iter
;
2048 spin_lock(&memcg_oom_lock
);
2049 for_each_mem_cgroup_tree(iter
, memcg
)
2051 spin_unlock(&memcg_oom_lock
);
2054 static void mem_cgroup_unmark_under_oom(struct mem_cgroup
*memcg
)
2056 struct mem_cgroup
*iter
;
2059 * Be careful about under_oom underflows because a child memcg
2060 * could have been added after mem_cgroup_mark_under_oom.
2062 spin_lock(&memcg_oom_lock
);
2063 for_each_mem_cgroup_tree(iter
, memcg
)
2064 if (iter
->under_oom
> 0)
2066 spin_unlock(&memcg_oom_lock
);
2069 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq
);
2071 struct oom_wait_info
{
2072 struct mem_cgroup
*memcg
;
2073 wait_queue_entry_t wait
;
2076 static int memcg_oom_wake_function(wait_queue_entry_t
*wait
,
2077 unsigned mode
, int sync
, void *arg
)
2079 struct mem_cgroup
*wake_memcg
= (struct mem_cgroup
*)arg
;
2080 struct mem_cgroup
*oom_wait_memcg
;
2081 struct oom_wait_info
*oom_wait_info
;
2083 oom_wait_info
= container_of(wait
, struct oom_wait_info
, wait
);
2084 oom_wait_memcg
= oom_wait_info
->memcg
;
2086 if (!mem_cgroup_is_descendant(wake_memcg
, oom_wait_memcg
) &&
2087 !mem_cgroup_is_descendant(oom_wait_memcg
, wake_memcg
))
2089 return autoremove_wake_function(wait
, mode
, sync
, arg
);
2092 void memcg1_oom_recover(struct mem_cgroup
*memcg
)
2095 * For the following lockless ->under_oom test, the only required
2096 * guarantee is that it must see the state asserted by an OOM when
2097 * this function is called as a result of userland actions
2098 * triggered by the notification of the OOM. This is trivially
2099 * achieved by invoking mem_cgroup_mark_under_oom() before
2100 * triggering notification.
2102 if (memcg
&& memcg
->under_oom
)
2103 __wake_up(&memcg_oom_waitq
, TASK_NORMAL
, 0, memcg
);
2107 * mem_cgroup_oom_synchronize - complete memcg OOM handling
2108 * @handle: actually kill/wait or just clean up the OOM state
2110 * This has to be called at the end of a page fault if the memcg OOM
2111 * handler was enabled.
2113 * Memcg supports userspace OOM handling where failed allocations must
2114 * sleep on a waitqueue until the userspace task resolves the
2115 * situation. Sleeping directly in the charge context with all kinds
2116 * of locks held is not a good idea, instead we remember an OOM state
2117 * in the task and mem_cgroup_oom_synchronize() has to be called at
2118 * the end of the page fault to complete the OOM handling.
2120 * Returns %true if an ongoing memcg OOM situation was detected and
2121 * completed, %false otherwise.
2123 bool mem_cgroup_oom_synchronize(bool handle
)
2125 struct mem_cgroup
*memcg
= current
->memcg_in_oom
;
2126 struct oom_wait_info owait
;
2129 /* OOM is global, do not handle */
2136 owait
.memcg
= memcg
;
2137 owait
.wait
.flags
= 0;
2138 owait
.wait
.func
= memcg_oom_wake_function
;
2139 owait
.wait
.private = current
;
2140 INIT_LIST_HEAD(&owait
.wait
.entry
);
2142 prepare_to_wait(&memcg_oom_waitq
, &owait
.wait
, TASK_KILLABLE
);
2143 mem_cgroup_mark_under_oom(memcg
);
2145 locked
= mem_cgroup_oom_trylock(memcg
);
2148 mem_cgroup_oom_notify(memcg
);
2151 mem_cgroup_unmark_under_oom(memcg
);
2152 finish_wait(&memcg_oom_waitq
, &owait
.wait
);
2155 mem_cgroup_oom_unlock(memcg
);
2157 current
->memcg_in_oom
= NULL
;
2158 css_put(&memcg
->css
);
2163 bool memcg1_oom_prepare(struct mem_cgroup
*memcg
, bool *locked
)
2166 * We are in the middle of the charge context here, so we
2167 * don't want to block when potentially sitting on a callstack
2168 * that holds all kinds of filesystem and mm locks.
2170 * cgroup1 allows disabling the OOM killer and waiting for outside
2171 * handling until the charge can succeed; remember the context and put
2172 * the task to sleep at the end of the page fault when all locks are
2175 * On the other hand, in-kernel OOM killer allows for an async victim
2176 * memory reclaim (oom_reaper) and that means that we are not solely
2177 * relying on the oom victim to make a forward progress and we can
2178 * invoke the oom killer here.
2180 * Please note that mem_cgroup_out_of_memory might fail to find a
2181 * victim and then we have to bail out from the charge path.
2183 if (READ_ONCE(memcg
->oom_kill_disable
)) {
2184 if (current
->in_user_fault
) {
2185 css_get(&memcg
->css
);
2186 current
->memcg_in_oom
= memcg
;
2191 mem_cgroup_mark_under_oom(memcg
);
2193 *locked
= mem_cgroup_oom_trylock(memcg
);
2196 mem_cgroup_oom_notify(memcg
);
2198 mem_cgroup_unmark_under_oom(memcg
);
2203 void memcg1_oom_finish(struct mem_cgroup
*memcg
, bool locked
)
2206 mem_cgroup_oom_unlock(memcg
);
2209 static DEFINE_MUTEX(memcg_max_mutex
);
2211 static int mem_cgroup_resize_max(struct mem_cgroup
*memcg
,
2212 unsigned long max
, bool memsw
)
2214 bool enlarge
= false;
2215 bool drained
= false;
2217 bool limits_invariant
;
2218 struct page_counter
*counter
= memsw
? &memcg
->memsw
: &memcg
->memory
;
2221 if (signal_pending(current
)) {
2226 mutex_lock(&memcg_max_mutex
);
2228 * Make sure that the new limit (memsw or memory limit) doesn't
2229 * break our basic invariant rule memory.max <= memsw.max.
2231 limits_invariant
= memsw
? max
>= READ_ONCE(memcg
->memory
.max
) :
2232 max
<= memcg
->memsw
.max
;
2233 if (!limits_invariant
) {
2234 mutex_unlock(&memcg_max_mutex
);
2238 if (max
> counter
->max
)
2240 ret
= page_counter_set_max(counter
, max
);
2241 mutex_unlock(&memcg_max_mutex
);
2247 drain_all_stock(memcg
);
2252 if (!try_to_free_mem_cgroup_pages(memcg
, 1, GFP_KERNEL
,
2253 memsw
? 0 : MEMCG_RECLAIM_MAY_SWAP
, NULL
)) {
2259 if (!ret
&& enlarge
)
2260 memcg1_oom_recover(memcg
);
2266 * Reclaims as many pages from the given memcg as possible.
2268 * Caller is responsible for holding css reference for memcg.
2270 static int mem_cgroup_force_empty(struct mem_cgroup
*memcg
)
2272 int nr_retries
= MAX_RECLAIM_RETRIES
;
2274 /* we call try-to-free pages for make this cgroup empty */
2275 lru_add_drain_all();
2277 drain_all_stock(memcg
);
2279 /* try to free all pages in this cgroup */
2280 while (nr_retries
&& page_counter_read(&memcg
->memory
)) {
2281 if (signal_pending(current
))
2284 if (!try_to_free_mem_cgroup_pages(memcg
, 1, GFP_KERNEL
,
2285 MEMCG_RECLAIM_MAY_SWAP
, NULL
))
2292 static ssize_t
mem_cgroup_force_empty_write(struct kernfs_open_file
*of
,
2293 char *buf
, size_t nbytes
,
2296 struct mem_cgroup
*memcg
= mem_cgroup_from_css(of_css(of
));
2298 if (mem_cgroup_is_root(memcg
))
2300 return mem_cgroup_force_empty(memcg
) ?: nbytes
;
2303 static u64
mem_cgroup_hierarchy_read(struct cgroup_subsys_state
*css
,
2309 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state
*css
,
2310 struct cftype
*cft
, u64 val
)
2315 pr_warn_once("Non-hierarchical mode is deprecated. "
2316 "Please report your usecase to linux-mm@kvack.org if you "
2317 "depend on this functionality.\n");
2322 static u64
mem_cgroup_read_u64(struct cgroup_subsys_state
*css
,
2325 struct mem_cgroup
*memcg
= mem_cgroup_from_css(css
);
2326 struct page_counter
*counter
;
2328 switch (MEMFILE_TYPE(cft
->private)) {
2330 counter
= &memcg
->memory
;
2333 counter
= &memcg
->memsw
;
2336 counter
= &memcg
->kmem
;
2339 counter
= &memcg
->tcpmem
;
2345 switch (MEMFILE_ATTR(cft
->private)) {
2347 if (counter
== &memcg
->memory
)
2348 return (u64
)mem_cgroup_usage(memcg
, false) * PAGE_SIZE
;
2349 if (counter
== &memcg
->memsw
)
2350 return (u64
)mem_cgroup_usage(memcg
, true) * PAGE_SIZE
;
2351 return (u64
)page_counter_read(counter
) * PAGE_SIZE
;
2353 return (u64
)counter
->max
* PAGE_SIZE
;
2355 return (u64
)counter
->watermark
* PAGE_SIZE
;
2357 return counter
->failcnt
;
2358 case RES_SOFT_LIMIT
:
2359 return (u64
)READ_ONCE(memcg
->soft_limit
) * PAGE_SIZE
;
2366 * This function doesn't do anything useful. Its only job is to provide a read
2367 * handler for a file so that cgroup_file_mode() will add read permissions.
2369 static int mem_cgroup_dummy_seq_show(__always_unused
struct seq_file
*m
,
2370 __always_unused
void *v
)
2375 static int memcg_update_tcp_max(struct mem_cgroup
*memcg
, unsigned long max
)
2379 mutex_lock(&memcg_max_mutex
);
2381 ret
= page_counter_set_max(&memcg
->tcpmem
, max
);
2385 if (!memcg
->tcpmem_active
) {
2387 * The active flag needs to be written after the static_key
2388 * update. This is what guarantees that the socket activation
2389 * function is the last one to run. See mem_cgroup_sk_alloc()
2390 * for details, and note that we don't mark any socket as
2391 * belonging to this memcg until that flag is up.
2393 * We need to do this, because static_keys will span multiple
2394 * sites, but we can't control their order. If we mark a socket
2395 * as accounted, but the accounting functions are not patched in
2396 * yet, we'll lose accounting.
2398 * We never race with the readers in mem_cgroup_sk_alloc(),
2399 * because when this value change, the code to process it is not
2402 static_branch_inc(&memcg_sockets_enabled_key
);
2403 memcg
->tcpmem_active
= true;
2406 mutex_unlock(&memcg_max_mutex
);
2411 * The user of this function is...
2414 static ssize_t
mem_cgroup_write(struct kernfs_open_file
*of
,
2415 char *buf
, size_t nbytes
, loff_t off
)
2417 struct mem_cgroup
*memcg
= mem_cgroup_from_css(of_css(of
));
2418 unsigned long nr_pages
;
2421 buf
= strstrip(buf
);
2422 ret
= page_counter_memparse(buf
, "-1", &nr_pages
);
2426 switch (MEMFILE_ATTR(of_cft(of
)->private)) {
2428 if (mem_cgroup_is_root(memcg
)) { /* Can't set limit on root */
2432 switch (MEMFILE_TYPE(of_cft(of
)->private)) {
2434 ret
= mem_cgroup_resize_max(memcg
, nr_pages
, false);
2437 ret
= mem_cgroup_resize_max(memcg
, nr_pages
, true);
2440 pr_warn_once("kmem.limit_in_bytes is deprecated and will be removed. "
2441 "Writing any value to this file has no effect. "
2442 "Please report your usecase to linux-mm@kvack.org if you "
2443 "depend on this functionality.\n");
2447 ret
= memcg_update_tcp_max(memcg
, nr_pages
);
2451 case RES_SOFT_LIMIT
:
2452 if (IS_ENABLED(CONFIG_PREEMPT_RT
)) {
2455 WRITE_ONCE(memcg
->soft_limit
, nr_pages
);
2460 return ret
?: nbytes
;
2463 static ssize_t
mem_cgroup_reset(struct kernfs_open_file
*of
, char *buf
,
2464 size_t nbytes
, loff_t off
)
2466 struct mem_cgroup
*memcg
= mem_cgroup_from_css(of_css(of
));
2467 struct page_counter
*counter
;
2469 switch (MEMFILE_TYPE(of_cft(of
)->private)) {
2471 counter
= &memcg
->memory
;
2474 counter
= &memcg
->memsw
;
2477 counter
= &memcg
->kmem
;
2480 counter
= &memcg
->tcpmem
;
2486 switch (MEMFILE_ATTR(of_cft(of
)->private)) {
2488 page_counter_reset_watermark(counter
);
2491 counter
->failcnt
= 0;
2502 #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
2503 #define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
2504 #define LRU_ALL ((1 << NR_LRU_LISTS) - 1)
2506 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup
*memcg
,
2507 int nid
, unsigned int lru_mask
, bool tree
)
2509 struct lruvec
*lruvec
= mem_cgroup_lruvec(memcg
, NODE_DATA(nid
));
2510 unsigned long nr
= 0;
2513 VM_BUG_ON((unsigned)nid
>= nr_node_ids
);
2516 if (!(BIT(lru
) & lru_mask
))
2519 nr
+= lruvec_page_state(lruvec
, NR_LRU_BASE
+ lru
);
2521 nr
+= lruvec_page_state_local(lruvec
, NR_LRU_BASE
+ lru
);
2526 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup
*memcg
,
2527 unsigned int lru_mask
,
2530 unsigned long nr
= 0;
2534 if (!(BIT(lru
) & lru_mask
))
2537 nr
+= memcg_page_state(memcg
, NR_LRU_BASE
+ lru
);
2539 nr
+= memcg_page_state_local(memcg
, NR_LRU_BASE
+ lru
);
2544 static int memcg_numa_stat_show(struct seq_file
*m
, void *v
)
2548 unsigned int lru_mask
;
2551 static const struct numa_stat stats
[] = {
2552 { "total", LRU_ALL
},
2553 { "file", LRU_ALL_FILE
},
2554 { "anon", LRU_ALL_ANON
},
2555 { "unevictable", BIT(LRU_UNEVICTABLE
) },
2557 const struct numa_stat
*stat
;
2559 struct mem_cgroup
*memcg
= mem_cgroup_from_seq(m
);
2561 mem_cgroup_flush_stats(memcg
);
2563 for (stat
= stats
; stat
< stats
+ ARRAY_SIZE(stats
); stat
++) {
2564 seq_printf(m
, "%s=%lu", stat
->name
,
2565 mem_cgroup_nr_lru_pages(memcg
, stat
->lru_mask
,
2567 for_each_node_state(nid
, N_MEMORY
)
2568 seq_printf(m
, " N%d=%lu", nid
,
2569 mem_cgroup_node_nr_lru_pages(memcg
, nid
,
2570 stat
->lru_mask
, false));
2574 for (stat
= stats
; stat
< stats
+ ARRAY_SIZE(stats
); stat
++) {
2576 seq_printf(m
, "hierarchical_%s=%lu", stat
->name
,
2577 mem_cgroup_nr_lru_pages(memcg
, stat
->lru_mask
,
2579 for_each_node_state(nid
, N_MEMORY
)
2580 seq_printf(m
, " N%d=%lu", nid
,
2581 mem_cgroup_node_nr_lru_pages(memcg
, nid
,
2582 stat
->lru_mask
, true));
2588 #endif /* CONFIG_NUMA */
2590 static const unsigned int memcg1_stats
[] = {
2593 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2600 WORKINGSET_REFAULT_ANON
,
2601 WORKINGSET_REFAULT_FILE
,
2608 static const char *const memcg1_stat_names
[] = {
2611 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2618 "workingset_refault_anon",
2619 "workingset_refault_file",
2626 /* Universal VM events cgroup1 shows, original sort order */
2627 static const unsigned int memcg1_events
[] = {
2634 void memcg1_stat_format(struct mem_cgroup
*memcg
, struct seq_buf
*s
)
2636 unsigned long memory
, memsw
;
2637 struct mem_cgroup
*mi
;
2640 BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names
) != ARRAY_SIZE(memcg1_stats
));
2642 mem_cgroup_flush_stats(memcg
);
2644 for (i
= 0; i
< ARRAY_SIZE(memcg1_stats
); i
++) {
2647 nr
= memcg_page_state_local_output(memcg
, memcg1_stats
[i
]);
2648 seq_buf_printf(s
, "%s %lu\n", memcg1_stat_names
[i
], nr
);
2651 for (i
= 0; i
< ARRAY_SIZE(memcg1_events
); i
++)
2652 seq_buf_printf(s
, "%s %lu\n", vm_event_name(memcg1_events
[i
]),
2653 memcg_events_local(memcg
, memcg1_events
[i
]));
2655 for (i
= 0; i
< NR_LRU_LISTS
; i
++)
2656 seq_buf_printf(s
, "%s %lu\n", lru_list_name(i
),
2657 memcg_page_state_local(memcg
, NR_LRU_BASE
+ i
) *
2660 /* Hierarchical information */
2661 memory
= memsw
= PAGE_COUNTER_MAX
;
2662 for (mi
= memcg
; mi
; mi
= parent_mem_cgroup(mi
)) {
2663 memory
= min(memory
, READ_ONCE(mi
->memory
.max
));
2664 memsw
= min(memsw
, READ_ONCE(mi
->memsw
.max
));
2666 seq_buf_printf(s
, "hierarchical_memory_limit %llu\n",
2667 (u64
)memory
* PAGE_SIZE
);
2668 seq_buf_printf(s
, "hierarchical_memsw_limit %llu\n",
2669 (u64
)memsw
* PAGE_SIZE
);
2671 for (i
= 0; i
< ARRAY_SIZE(memcg1_stats
); i
++) {
2674 nr
= memcg_page_state_output(memcg
, memcg1_stats
[i
]);
2675 seq_buf_printf(s
, "total_%s %llu\n", memcg1_stat_names
[i
],
2679 for (i
= 0; i
< ARRAY_SIZE(memcg1_events
); i
++)
2680 seq_buf_printf(s
, "total_%s %llu\n",
2681 vm_event_name(memcg1_events
[i
]),
2682 (u64
)memcg_events(memcg
, memcg1_events
[i
]));
2684 for (i
= 0; i
< NR_LRU_LISTS
; i
++)
2685 seq_buf_printf(s
, "total_%s %llu\n", lru_list_name(i
),
2686 (u64
)memcg_page_state(memcg
, NR_LRU_BASE
+ i
) *
2689 #ifdef CONFIG_DEBUG_VM
2692 struct mem_cgroup_per_node
*mz
;
2693 unsigned long anon_cost
= 0;
2694 unsigned long file_cost
= 0;
2696 for_each_online_pgdat(pgdat
) {
2697 mz
= memcg
->nodeinfo
[pgdat
->node_id
];
2699 anon_cost
+= mz
->lruvec
.anon_cost
;
2700 file_cost
+= mz
->lruvec
.file_cost
;
2702 seq_buf_printf(s
, "anon_cost %lu\n", anon_cost
);
2703 seq_buf_printf(s
, "file_cost %lu\n", file_cost
);
2708 static u64
mem_cgroup_swappiness_read(struct cgroup_subsys_state
*css
,
2711 struct mem_cgroup
*memcg
= mem_cgroup_from_css(css
);
2713 return mem_cgroup_swappiness(memcg
);
2716 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state
*css
,
2717 struct cftype
*cft
, u64 val
)
2719 struct mem_cgroup
*memcg
= mem_cgroup_from_css(css
);
2721 if (val
> MAX_SWAPPINESS
)
2724 if (!mem_cgroup_is_root(memcg
))
2725 WRITE_ONCE(memcg
->swappiness
, val
);
2727 WRITE_ONCE(vm_swappiness
, val
);
2732 static int mem_cgroup_oom_control_read(struct seq_file
*sf
, void *v
)
2734 struct mem_cgroup
*memcg
= mem_cgroup_from_seq(sf
);
2736 seq_printf(sf
, "oom_kill_disable %d\n", READ_ONCE(memcg
->oom_kill_disable
));
2737 seq_printf(sf
, "under_oom %d\n", (bool)memcg
->under_oom
);
2738 seq_printf(sf
, "oom_kill %lu\n",
2739 atomic_long_read(&memcg
->memory_events
[MEMCG_OOM_KILL
]));
2743 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state
*css
,
2744 struct cftype
*cft
, u64 val
)
2746 struct mem_cgroup
*memcg
= mem_cgroup_from_css(css
);
2748 /* cannot set to root cgroup and only 0 and 1 are allowed */
2749 if (mem_cgroup_is_root(memcg
) || !((val
== 0) || (val
== 1)))
2752 WRITE_ONCE(memcg
->oom_kill_disable
, val
);
2754 memcg1_oom_recover(memcg
);
2759 #ifdef CONFIG_SLUB_DEBUG
2760 static int mem_cgroup_slab_show(struct seq_file
*m
, void *p
)
2764 * Please, take a look at tools/cgroup/memcg_slabinfo.py .
2770 struct cftype mem_cgroup_legacy_files
[] = {
2772 .name
= "usage_in_bytes",
2773 .private = MEMFILE_PRIVATE(_MEM
, RES_USAGE
),
2774 .read_u64
= mem_cgroup_read_u64
,
2777 .name
= "max_usage_in_bytes",
2778 .private = MEMFILE_PRIVATE(_MEM
, RES_MAX_USAGE
),
2779 .write
= mem_cgroup_reset
,
2780 .read_u64
= mem_cgroup_read_u64
,
2783 .name
= "limit_in_bytes",
2784 .private = MEMFILE_PRIVATE(_MEM
, RES_LIMIT
),
2785 .write
= mem_cgroup_write
,
2786 .read_u64
= mem_cgroup_read_u64
,
2789 .name
= "soft_limit_in_bytes",
2790 .private = MEMFILE_PRIVATE(_MEM
, RES_SOFT_LIMIT
),
2791 .write
= mem_cgroup_write
,
2792 .read_u64
= mem_cgroup_read_u64
,
2796 .private = MEMFILE_PRIVATE(_MEM
, RES_FAILCNT
),
2797 .write
= mem_cgroup_reset
,
2798 .read_u64
= mem_cgroup_read_u64
,
2802 .seq_show
= memory_stat_show
,
2805 .name
= "force_empty",
2806 .write
= mem_cgroup_force_empty_write
,
2809 .name
= "use_hierarchy",
2810 .write_u64
= mem_cgroup_hierarchy_write
,
2811 .read_u64
= mem_cgroup_hierarchy_read
,
2814 .name
= "cgroup.event_control", /* XXX: for compat */
2815 .write
= memcg_write_event_control
,
2816 .flags
= CFTYPE_NO_PREFIX
| CFTYPE_WORLD_WRITABLE
,
2819 .name
= "swappiness",
2820 .read_u64
= mem_cgroup_swappiness_read
,
2821 .write_u64
= mem_cgroup_swappiness_write
,
2824 .name
= "move_charge_at_immigrate",
2825 .read_u64
= mem_cgroup_move_charge_read
,
2826 .write_u64
= mem_cgroup_move_charge_write
,
2829 .name
= "oom_control",
2830 .seq_show
= mem_cgroup_oom_control_read
,
2831 .write_u64
= mem_cgroup_oom_control_write
,
2834 .name
= "pressure_level",
2835 .seq_show
= mem_cgroup_dummy_seq_show
,
2839 .name
= "numa_stat",
2840 .seq_show
= memcg_numa_stat_show
,
2844 .name
= "kmem.limit_in_bytes",
2845 .private = MEMFILE_PRIVATE(_KMEM
, RES_LIMIT
),
2846 .write
= mem_cgroup_write
,
2847 .read_u64
= mem_cgroup_read_u64
,
2850 .name
= "kmem.usage_in_bytes",
2851 .private = MEMFILE_PRIVATE(_KMEM
, RES_USAGE
),
2852 .read_u64
= mem_cgroup_read_u64
,
2855 .name
= "kmem.failcnt",
2856 .private = MEMFILE_PRIVATE(_KMEM
, RES_FAILCNT
),
2857 .write
= mem_cgroup_reset
,
2858 .read_u64
= mem_cgroup_read_u64
,
2861 .name
= "kmem.max_usage_in_bytes",
2862 .private = MEMFILE_PRIVATE(_KMEM
, RES_MAX_USAGE
),
2863 .write
= mem_cgroup_reset
,
2864 .read_u64
= mem_cgroup_read_u64
,
2866 #ifdef CONFIG_SLUB_DEBUG
2868 .name
= "kmem.slabinfo",
2869 .seq_show
= mem_cgroup_slab_show
,
2873 .name
= "kmem.tcp.limit_in_bytes",
2874 .private = MEMFILE_PRIVATE(_TCP
, RES_LIMIT
),
2875 .write
= mem_cgroup_write
,
2876 .read_u64
= mem_cgroup_read_u64
,
2879 .name
= "kmem.tcp.usage_in_bytes",
2880 .private = MEMFILE_PRIVATE(_TCP
, RES_USAGE
),
2881 .read_u64
= mem_cgroup_read_u64
,
2884 .name
= "kmem.tcp.failcnt",
2885 .private = MEMFILE_PRIVATE(_TCP
, RES_FAILCNT
),
2886 .write
= mem_cgroup_reset
,
2887 .read_u64
= mem_cgroup_read_u64
,
2890 .name
= "kmem.tcp.max_usage_in_bytes",
2891 .private = MEMFILE_PRIVATE(_TCP
, RES_MAX_USAGE
),
2892 .write
= mem_cgroup_reset
,
2893 .read_u64
= mem_cgroup_read_u64
,
2895 { }, /* terminate */
2898 struct cftype memsw_files
[] = {
2900 .name
= "memsw.usage_in_bytes",
2901 .private = MEMFILE_PRIVATE(_MEMSWAP
, RES_USAGE
),
2902 .read_u64
= mem_cgroup_read_u64
,
2905 .name
= "memsw.max_usage_in_bytes",
2906 .private = MEMFILE_PRIVATE(_MEMSWAP
, RES_MAX_USAGE
),
2907 .write
= mem_cgroup_reset
,
2908 .read_u64
= mem_cgroup_read_u64
,
2911 .name
= "memsw.limit_in_bytes",
2912 .private = MEMFILE_PRIVATE(_MEMSWAP
, RES_LIMIT
),
2913 .write
= mem_cgroup_write
,
2914 .read_u64
= mem_cgroup_read_u64
,
2917 .name
= "memsw.failcnt",
2918 .private = MEMFILE_PRIVATE(_MEMSWAP
, RES_FAILCNT
),
2919 .write
= mem_cgroup_reset
,
2920 .read_u64
= mem_cgroup_read_u64
,
2922 { }, /* terminate */
2925 void memcg1_account_kmem(struct mem_cgroup
*memcg
, int nr_pages
)
2927 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys
)) {
2929 page_counter_charge(&memcg
->kmem
, nr_pages
);
2931 page_counter_uncharge(&memcg
->kmem
, -nr_pages
);
2935 bool memcg1_charge_skmem(struct mem_cgroup
*memcg
, unsigned int nr_pages
,
2938 struct page_counter
*fail
;
2940 if (page_counter_try_charge(&memcg
->tcpmem
, nr_pages
, &fail
)) {
2941 memcg
->tcpmem_pressure
= 0;
2944 memcg
->tcpmem_pressure
= 1;
2945 if (gfp_mask
& __GFP_NOFAIL
) {
2946 page_counter_charge(&memcg
->tcpmem
, nr_pages
);
2952 static int __init
memcg1_init(void)
2956 for_each_node(node
) {
2957 struct mem_cgroup_tree_per_node
*rtpn
;
2959 rtpn
= kzalloc_node(sizeof(*rtpn
), GFP_KERNEL
, node
);
2961 rtpn
->rb_root
= RB_ROOT
;
2962 rtpn
->rb_rightmost
= NULL
;
2963 spin_lock_init(&rtpn
->lock
);
2964 soft_limit_tree
.rb_tree_per_node
[node
] = rtpn
;
2969 subsys_initcall(memcg1_init
);