1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
9 * This file contains the default values for the operation of the
10 * Linux VM subsystem. Fine-tuning documentation can be found in
11 * Documentation/admin-guide/sysctl/vm.rst.
13 * Swap aging added 23.2.95, Stephen Tweedie.
14 * Buffermem limits added 12.3.98, Rik van Riel.
18 #include <linux/sched.h>
19 #include <linux/kernel_stat.h>
20 #include <linux/swap.h>
21 #include <linux/mman.h>
22 #include <linux/pagemap.h>
23 #include <linux/pagevec.h>
24 #include <linux/init.h>
25 #include <linux/export.h>
26 #include <linux/mm_inline.h>
27 #include <linux/percpu_counter.h>
28 #include <linux/memremap.h>
29 #include <linux/percpu.h>
30 #include <linux/cpu.h>
31 #include <linux/notifier.h>
32 #include <linux/backing-dev.h>
33 #include <linux/memcontrol.h>
34 #include <linux/gfp.h>
35 #include <linux/uio.h>
36 #include <linux/hugetlb.h>
37 #include <linux/page_idle.h>
38 #include <linux/local_lock.h>
39 #include <linux/buffer_head.h>
43 #define CREATE_TRACE_POINTS
44 #include <trace/events/pagemap.h>
46 /* How many pages do we try to swap or page in/out together? As a power of 2 */
48 const int page_cluster_max
= 31;
50 /* Protecting only lru_rotate.fbatch which requires disabling interrupts */
53 struct folio_batch fbatch
;
55 static DEFINE_PER_CPU(struct lru_rotate
, lru_rotate
) = {
56 .lock
= INIT_LOCAL_LOCK(lock
),
60 * The following folio batches are grouped together because they are protected
61 * by disabling preemption (and interrupts remain enabled).
65 struct folio_batch lru_add
;
66 struct folio_batch lru_deactivate_file
;
67 struct folio_batch lru_deactivate
;
68 struct folio_batch lru_lazyfree
;
70 struct folio_batch activate
;
73 static DEFINE_PER_CPU(struct cpu_fbatches
, cpu_fbatches
) = {
74 .lock
= INIT_LOCAL_LOCK(lock
),
77 static void __page_cache_release(struct folio
*folio
, struct lruvec
**lruvecp
,
78 unsigned long *flagsp
)
80 if (folio_test_lru(folio
)) {
81 folio_lruvec_relock_irqsave(folio
, lruvecp
, flagsp
);
82 lruvec_del_folio(*lruvecp
, folio
);
83 __folio_clear_lru_flags(folio
);
87 * In rare cases, when truncation or holepunching raced with
88 * munlock after VM_LOCKED was cleared, Mlocked may still be
89 * found set here. This does not indicate a problem, unless
90 * "unevictable_pgs_cleared" appears worryingly large.
92 if (unlikely(folio_test_mlocked(folio
))) {
93 long nr_pages
= folio_nr_pages(folio
);
95 __folio_clear_mlocked(folio
);
96 zone_stat_mod_folio(folio
, NR_MLOCK
, -nr_pages
);
97 count_vm_events(UNEVICTABLE_PGCLEARED
, nr_pages
);
102 * This path almost never happens for VM activity - pages are normally freed
103 * in batches. But it gets used by networking - and for compound pages.
105 static void page_cache_release(struct folio
*folio
)
107 struct lruvec
*lruvec
= NULL
;
110 __page_cache_release(folio
, &lruvec
, &flags
);
112 unlock_page_lruvec_irqrestore(lruvec
, flags
);
115 void __folio_put(struct folio
*folio
)
117 if (unlikely(folio_is_zone_device(folio
))) {
118 free_zone_device_folio(folio
);
120 } else if (folio_test_hugetlb(folio
)) {
121 free_huge_folio(folio
);
125 page_cache_release(folio
);
126 folio_undo_large_rmappable(folio
);
127 mem_cgroup_uncharge(folio
);
128 free_unref_page(&folio
->page
, folio_order(folio
));
130 EXPORT_SYMBOL(__folio_put
);
133 * put_pages_list() - release a list of pages
134 * @pages: list of pages threaded on page->lru
136 * Release a list of pages which are strung together on page.lru.
138 void put_pages_list(struct list_head
*pages
)
140 struct folio_batch fbatch
;
141 struct folio
*folio
, *next
;
143 folio_batch_init(&fbatch
);
144 list_for_each_entry_safe(folio
, next
, pages
, lru
) {
145 if (!folio_put_testzero(folio
))
147 if (folio_test_hugetlb(folio
)) {
148 free_huge_folio(folio
);
151 /* LRU flag must be clear because it's passed using the lru */
152 if (folio_batch_add(&fbatch
, folio
) > 0)
154 free_unref_folios(&fbatch
);
158 free_unref_folios(&fbatch
);
159 INIT_LIST_HEAD(pages
);
161 EXPORT_SYMBOL(put_pages_list
);
163 typedef void (*move_fn_t
)(struct lruvec
*lruvec
, struct folio
*folio
);
165 static void lru_add_fn(struct lruvec
*lruvec
, struct folio
*folio
)
167 int was_unevictable
= folio_test_clear_unevictable(folio
);
168 long nr_pages
= folio_nr_pages(folio
);
170 VM_BUG_ON_FOLIO(folio_test_lru(folio
), folio
);
173 * Is an smp_mb__after_atomic() still required here, before
174 * folio_evictable() tests the mlocked flag, to rule out the possibility
175 * of stranding an evictable folio on an unevictable LRU? I think
176 * not, because __munlock_folio() only clears the mlocked flag
177 * while the LRU lock is held.
179 * (That is not true of __page_cache_release(), and not necessarily
180 * true of folios_put(): but those only clear the mlocked flag after
181 * folio_put_testzero() has excluded any other users of the folio.)
183 if (folio_evictable(folio
)) {
185 __count_vm_events(UNEVICTABLE_PGRESCUED
, nr_pages
);
187 folio_clear_active(folio
);
188 folio_set_unevictable(folio
);
190 * folio->mlock_count = !!folio_test_mlocked(folio)?
191 * But that leaves __mlock_folio() in doubt whether another
192 * actor has already counted the mlock or not. Err on the
193 * safe side, underestimate, let page reclaim fix it, rather
194 * than leaving a page on the unevictable LRU indefinitely.
196 folio
->mlock_count
= 0;
197 if (!was_unevictable
)
198 __count_vm_events(UNEVICTABLE_PGCULLED
, nr_pages
);
201 lruvec_add_folio(lruvec
, folio
);
202 trace_mm_lru_insertion(folio
);
205 static void folio_batch_move_lru(struct folio_batch
*fbatch
, move_fn_t move_fn
)
208 struct lruvec
*lruvec
= NULL
;
209 unsigned long flags
= 0;
211 for (i
= 0; i
< folio_batch_count(fbatch
); i
++) {
212 struct folio
*folio
= fbatch
->folios
[i
];
214 folio_lruvec_relock_irqsave(folio
, &lruvec
, &flags
);
215 move_fn(lruvec
, folio
);
217 folio_set_lru(folio
);
221 unlock_page_lruvec_irqrestore(lruvec
, flags
);
225 static void folio_batch_add_and_move(struct folio_batch
*fbatch
,
226 struct folio
*folio
, move_fn_t move_fn
)
228 if (folio_batch_add(fbatch
, folio
) && !folio_test_large(folio
) &&
229 !lru_cache_disabled())
231 folio_batch_move_lru(fbatch
, move_fn
);
234 static void lru_move_tail_fn(struct lruvec
*lruvec
, struct folio
*folio
)
236 if (!folio_test_unevictable(folio
)) {
237 lruvec_del_folio(lruvec
, folio
);
238 folio_clear_active(folio
);
239 lruvec_add_folio_tail(lruvec
, folio
);
240 __count_vm_events(PGROTATED
, folio_nr_pages(folio
));
245 * Writeback is about to end against a folio which has been marked for
246 * immediate reclaim. If it still appears to be reclaimable, move it
247 * to the tail of the inactive list.
249 * folio_rotate_reclaimable() must disable IRQs, to prevent nasty races.
251 void folio_rotate_reclaimable(struct folio
*folio
)
253 if (!folio_test_locked(folio
) && !folio_test_dirty(folio
) &&
254 !folio_test_unevictable(folio
)) {
255 struct folio_batch
*fbatch
;
259 if (!folio_test_clear_lru(folio
)) {
264 local_lock_irqsave(&lru_rotate
.lock
, flags
);
265 fbatch
= this_cpu_ptr(&lru_rotate
.fbatch
);
266 folio_batch_add_and_move(fbatch
, folio
, lru_move_tail_fn
);
267 local_unlock_irqrestore(&lru_rotate
.lock
, flags
);
271 void lru_note_cost(struct lruvec
*lruvec
, bool file
,
272 unsigned int nr_io
, unsigned int nr_rotated
)
277 * Reflect the relative cost of incurring IO and spending CPU
278 * time on rotations. This doesn't attempt to make a precise
279 * comparison, it just says: if reloads are about comparable
280 * between the LRU lists, or rotations are overwhelmingly
281 * different between them, adjust scan balance for CPU work.
283 cost
= nr_io
* SWAP_CLUSTER_MAX
+ nr_rotated
;
286 unsigned long lrusize
;
289 * Hold lruvec->lru_lock is safe here, since
290 * 1) The pinned lruvec in reclaim, or
291 * 2) From a pre-LRU page during refault (which also holds the
292 * rcu lock, so would be safe even if the page was on the LRU
293 * and could move simultaneously to a new lruvec).
295 spin_lock_irq(&lruvec
->lru_lock
);
296 /* Record cost event */
298 lruvec
->file_cost
+= cost
;
300 lruvec
->anon_cost
+= cost
;
303 * Decay previous events
305 * Because workloads change over time (and to avoid
306 * overflow) we keep these statistics as a floating
307 * average, which ends up weighing recent refaults
308 * more than old ones.
310 lrusize
= lruvec_page_state(lruvec
, NR_INACTIVE_ANON
) +
311 lruvec_page_state(lruvec
, NR_ACTIVE_ANON
) +
312 lruvec_page_state(lruvec
, NR_INACTIVE_FILE
) +
313 lruvec_page_state(lruvec
, NR_ACTIVE_FILE
);
315 if (lruvec
->file_cost
+ lruvec
->anon_cost
> lrusize
/ 4) {
316 lruvec
->file_cost
/= 2;
317 lruvec
->anon_cost
/= 2;
319 spin_unlock_irq(&lruvec
->lru_lock
);
320 } while ((lruvec
= parent_lruvec(lruvec
)));
323 void lru_note_cost_refault(struct folio
*folio
)
325 lru_note_cost(folio_lruvec(folio
), folio_is_file_lru(folio
),
326 folio_nr_pages(folio
), 0);
329 static void folio_activate_fn(struct lruvec
*lruvec
, struct folio
*folio
)
331 if (!folio_test_active(folio
) && !folio_test_unevictable(folio
)) {
332 long nr_pages
= folio_nr_pages(folio
);
334 lruvec_del_folio(lruvec
, folio
);
335 folio_set_active(folio
);
336 lruvec_add_folio(lruvec
, folio
);
337 trace_mm_lru_activate(folio
);
339 __count_vm_events(PGACTIVATE
, nr_pages
);
340 __count_memcg_events(lruvec_memcg(lruvec
), PGACTIVATE
,
346 static void folio_activate_drain(int cpu
)
348 struct folio_batch
*fbatch
= &per_cpu(cpu_fbatches
.activate
, cpu
);
350 if (folio_batch_count(fbatch
))
351 folio_batch_move_lru(fbatch
, folio_activate_fn
);
354 void folio_activate(struct folio
*folio
)
356 if (!folio_test_active(folio
) && !folio_test_unevictable(folio
)) {
357 struct folio_batch
*fbatch
;
360 if (!folio_test_clear_lru(folio
)) {
365 local_lock(&cpu_fbatches
.lock
);
366 fbatch
= this_cpu_ptr(&cpu_fbatches
.activate
);
367 folio_batch_add_and_move(fbatch
, folio
, folio_activate_fn
);
368 local_unlock(&cpu_fbatches
.lock
);
373 static inline void folio_activate_drain(int cpu
)
377 void folio_activate(struct folio
*folio
)
379 struct lruvec
*lruvec
;
381 if (folio_test_clear_lru(folio
)) {
382 lruvec
= folio_lruvec_lock_irq(folio
);
383 folio_activate_fn(lruvec
, folio
);
384 unlock_page_lruvec_irq(lruvec
);
385 folio_set_lru(folio
);
390 static void __lru_cache_activate_folio(struct folio
*folio
)
392 struct folio_batch
*fbatch
;
395 local_lock(&cpu_fbatches
.lock
);
396 fbatch
= this_cpu_ptr(&cpu_fbatches
.lru_add
);
399 * Search backwards on the optimistic assumption that the folio being
400 * activated has just been added to this batch. Note that only
401 * the local batch is examined as a !LRU folio could be in the
402 * process of being released, reclaimed, migrated or on a remote
403 * batch that is currently being drained. Furthermore, marking
404 * a remote batch's folio active potentially hits a race where
405 * a folio is marked active just after it is added to the inactive
406 * list causing accounting errors and BUG_ON checks to trigger.
408 for (i
= folio_batch_count(fbatch
) - 1; i
>= 0; i
--) {
409 struct folio
*batch_folio
= fbatch
->folios
[i
];
411 if (batch_folio
== folio
) {
412 folio_set_active(folio
);
417 local_unlock(&cpu_fbatches
.lock
);
420 #ifdef CONFIG_LRU_GEN
421 static void folio_inc_refs(struct folio
*folio
)
423 unsigned long new_flags
, old_flags
= READ_ONCE(folio
->flags
);
425 if (folio_test_unevictable(folio
))
428 if (!folio_test_referenced(folio
)) {
429 folio_set_referenced(folio
);
433 if (!folio_test_workingset(folio
)) {
434 folio_set_workingset(folio
);
438 /* see the comment on MAX_NR_TIERS */
440 new_flags
= old_flags
& LRU_REFS_MASK
;
441 if (new_flags
== LRU_REFS_MASK
)
444 new_flags
+= BIT(LRU_REFS_PGOFF
);
445 new_flags
|= old_flags
& ~LRU_REFS_MASK
;
446 } while (!try_cmpxchg(&folio
->flags
, &old_flags
, new_flags
));
449 static void folio_inc_refs(struct folio
*folio
)
452 #endif /* CONFIG_LRU_GEN */
455 * folio_mark_accessed - Mark a folio as having seen activity.
456 * @folio: The folio to mark.
458 * This function will perform one of the following transitions:
460 * * inactive,unreferenced -> inactive,referenced
461 * * inactive,referenced -> active,unreferenced
462 * * active,unreferenced -> active,referenced
464 * When a newly allocated folio is not yet visible, so safe for non-atomic ops,
465 * __folio_set_referenced() may be substituted for folio_mark_accessed().
467 void folio_mark_accessed(struct folio
*folio
)
469 if (lru_gen_enabled()) {
470 folio_inc_refs(folio
);
474 if (!folio_test_referenced(folio
)) {
475 folio_set_referenced(folio
);
476 } else if (folio_test_unevictable(folio
)) {
478 * Unevictable pages are on the "LRU_UNEVICTABLE" list. But,
479 * this list is never rotated or maintained, so marking an
480 * unevictable page accessed has no effect.
482 } else if (!folio_test_active(folio
)) {
484 * If the folio is on the LRU, queue it for activation via
485 * cpu_fbatches.activate. Otherwise, assume the folio is in a
486 * folio_batch, mark it active and it'll be moved to the active
487 * LRU on the next drain.
489 if (folio_test_lru(folio
))
490 folio_activate(folio
);
492 __lru_cache_activate_folio(folio
);
493 folio_clear_referenced(folio
);
494 workingset_activation(folio
);
496 if (folio_test_idle(folio
))
497 folio_clear_idle(folio
);
499 EXPORT_SYMBOL(folio_mark_accessed
);
502 * folio_add_lru - Add a folio to an LRU list.
503 * @folio: The folio to be added to the LRU.
505 * Queue the folio for addition to the LRU. The decision on whether
506 * to add the page to the [in]active [file|anon] list is deferred until the
507 * folio_batch is drained. This gives a chance for the caller of folio_add_lru()
508 * have the folio added to the active list using folio_mark_accessed().
510 void folio_add_lru(struct folio
*folio
)
512 struct folio_batch
*fbatch
;
514 VM_BUG_ON_FOLIO(folio_test_active(folio
) &&
515 folio_test_unevictable(folio
), folio
);
516 VM_BUG_ON_FOLIO(folio_test_lru(folio
), folio
);
518 /* see the comment in lru_gen_add_folio() */
519 if (lru_gen_enabled() && !folio_test_unevictable(folio
) &&
520 lru_gen_in_fault() && !(current
->flags
& PF_MEMALLOC
))
521 folio_set_active(folio
);
524 local_lock(&cpu_fbatches
.lock
);
525 fbatch
= this_cpu_ptr(&cpu_fbatches
.lru_add
);
526 folio_batch_add_and_move(fbatch
, folio
, lru_add_fn
);
527 local_unlock(&cpu_fbatches
.lock
);
529 EXPORT_SYMBOL(folio_add_lru
);
532 * folio_add_lru_vma() - Add a folio to the appropate LRU list for this VMA.
533 * @folio: The folio to be added to the LRU.
534 * @vma: VMA in which the folio is mapped.
536 * If the VMA is mlocked, @folio is added to the unevictable list.
537 * Otherwise, it is treated the same way as folio_add_lru().
539 void folio_add_lru_vma(struct folio
*folio
, struct vm_area_struct
*vma
)
541 VM_BUG_ON_FOLIO(folio_test_lru(folio
), folio
);
543 if (unlikely((vma
->vm_flags
& (VM_LOCKED
| VM_SPECIAL
)) == VM_LOCKED
))
544 mlock_new_folio(folio
);
546 folio_add_lru(folio
);
550 * If the folio cannot be invalidated, it is moved to the
551 * inactive list to speed up its reclaim. It is moved to the
552 * head of the list, rather than the tail, to give the flusher
553 * threads some time to write it out, as this is much more
554 * effective than the single-page writeout from reclaim.
556 * If the folio isn't mapped and dirty/writeback, the folio
557 * could be reclaimed asap using the reclaim flag.
559 * 1. active, mapped folio -> none
560 * 2. active, dirty/writeback folio -> inactive, head, reclaim
561 * 3. inactive, mapped folio -> none
562 * 4. inactive, dirty/writeback folio -> inactive, head, reclaim
563 * 5. inactive, clean -> inactive, tail
566 * In 4, it moves to the head of the inactive list so the folio is
567 * written out by flusher threads as this is much more efficient
568 * than the single-page writeout from reclaim.
570 static void lru_deactivate_file_fn(struct lruvec
*lruvec
, struct folio
*folio
)
572 bool active
= folio_test_active(folio
);
573 long nr_pages
= folio_nr_pages(folio
);
575 if (folio_test_unevictable(folio
))
578 /* Some processes are using the folio */
579 if (folio_mapped(folio
))
582 lruvec_del_folio(lruvec
, folio
);
583 folio_clear_active(folio
);
584 folio_clear_referenced(folio
);
586 if (folio_test_writeback(folio
) || folio_test_dirty(folio
)) {
588 * Setting the reclaim flag could race with
589 * folio_end_writeback() and confuse readahead. But the
590 * race window is _really_ small and it's not a critical
593 lruvec_add_folio(lruvec
, folio
);
594 folio_set_reclaim(folio
);
597 * The folio's writeback ended while it was in the batch.
598 * We move that folio to the tail of the inactive list.
600 lruvec_add_folio_tail(lruvec
, folio
);
601 __count_vm_events(PGROTATED
, nr_pages
);
605 __count_vm_events(PGDEACTIVATE
, nr_pages
);
606 __count_memcg_events(lruvec_memcg(lruvec
), PGDEACTIVATE
,
611 static void lru_deactivate_fn(struct lruvec
*lruvec
, struct folio
*folio
)
613 if (!folio_test_unevictable(folio
) && (folio_test_active(folio
) || lru_gen_enabled())) {
614 long nr_pages
= folio_nr_pages(folio
);
616 lruvec_del_folio(lruvec
, folio
);
617 folio_clear_active(folio
);
618 folio_clear_referenced(folio
);
619 lruvec_add_folio(lruvec
, folio
);
621 __count_vm_events(PGDEACTIVATE
, nr_pages
);
622 __count_memcg_events(lruvec_memcg(lruvec
), PGDEACTIVATE
,
627 static void lru_lazyfree_fn(struct lruvec
*lruvec
, struct folio
*folio
)
629 if (folio_test_anon(folio
) && folio_test_swapbacked(folio
) &&
630 !folio_test_swapcache(folio
) && !folio_test_unevictable(folio
)) {
631 long nr_pages
= folio_nr_pages(folio
);
633 lruvec_del_folio(lruvec
, folio
);
634 folio_clear_active(folio
);
635 folio_clear_referenced(folio
);
637 * Lazyfree folios are clean anonymous folios. They have
638 * the swapbacked flag cleared, to distinguish them from normal
641 folio_clear_swapbacked(folio
);
642 lruvec_add_folio(lruvec
, folio
);
644 __count_vm_events(PGLAZYFREE
, nr_pages
);
645 __count_memcg_events(lruvec_memcg(lruvec
), PGLAZYFREE
,
651 * Drain pages out of the cpu's folio_batch.
652 * Either "cpu" is the current CPU, and preemption has already been
653 * disabled; or "cpu" is being hot-unplugged, and is already dead.
655 void lru_add_drain_cpu(int cpu
)
657 struct cpu_fbatches
*fbatches
= &per_cpu(cpu_fbatches
, cpu
);
658 struct folio_batch
*fbatch
= &fbatches
->lru_add
;
660 if (folio_batch_count(fbatch
))
661 folio_batch_move_lru(fbatch
, lru_add_fn
);
663 fbatch
= &per_cpu(lru_rotate
.fbatch
, cpu
);
664 /* Disabling interrupts below acts as a compiler barrier. */
665 if (data_race(folio_batch_count(fbatch
))) {
668 /* No harm done if a racing interrupt already did this */
669 local_lock_irqsave(&lru_rotate
.lock
, flags
);
670 folio_batch_move_lru(fbatch
, lru_move_tail_fn
);
671 local_unlock_irqrestore(&lru_rotate
.lock
, flags
);
674 fbatch
= &fbatches
->lru_deactivate_file
;
675 if (folio_batch_count(fbatch
))
676 folio_batch_move_lru(fbatch
, lru_deactivate_file_fn
);
678 fbatch
= &fbatches
->lru_deactivate
;
679 if (folio_batch_count(fbatch
))
680 folio_batch_move_lru(fbatch
, lru_deactivate_fn
);
682 fbatch
= &fbatches
->lru_lazyfree
;
683 if (folio_batch_count(fbatch
))
684 folio_batch_move_lru(fbatch
, lru_lazyfree_fn
);
686 folio_activate_drain(cpu
);
690 * deactivate_file_folio() - Deactivate a file folio.
691 * @folio: Folio to deactivate.
693 * This function hints to the VM that @folio is a good reclaim candidate,
694 * for example if its invalidation fails due to the folio being dirty
695 * or under writeback.
697 * Context: Caller holds a reference on the folio.
699 void deactivate_file_folio(struct folio
*folio
)
701 struct folio_batch
*fbatch
;
703 /* Deactivating an unevictable folio will not accelerate reclaim */
704 if (folio_test_unevictable(folio
))
708 if (!folio_test_clear_lru(folio
)) {
713 local_lock(&cpu_fbatches
.lock
);
714 fbatch
= this_cpu_ptr(&cpu_fbatches
.lru_deactivate_file
);
715 folio_batch_add_and_move(fbatch
, folio
, lru_deactivate_file_fn
);
716 local_unlock(&cpu_fbatches
.lock
);
720 * folio_deactivate - deactivate a folio
721 * @folio: folio to deactivate
723 * folio_deactivate() moves @folio to the inactive list if @folio was on the
724 * active list and was not unevictable. This is done to accelerate the
727 void folio_deactivate(struct folio
*folio
)
729 if (!folio_test_unevictable(folio
) && (folio_test_active(folio
) ||
730 lru_gen_enabled())) {
731 struct folio_batch
*fbatch
;
734 if (!folio_test_clear_lru(folio
)) {
739 local_lock(&cpu_fbatches
.lock
);
740 fbatch
= this_cpu_ptr(&cpu_fbatches
.lru_deactivate
);
741 folio_batch_add_and_move(fbatch
, folio
, lru_deactivate_fn
);
742 local_unlock(&cpu_fbatches
.lock
);
747 * folio_mark_lazyfree - make an anon folio lazyfree
748 * @folio: folio to deactivate
750 * folio_mark_lazyfree() moves @folio to the inactive file list.
751 * This is done to accelerate the reclaim of @folio.
753 void folio_mark_lazyfree(struct folio
*folio
)
755 if (folio_test_anon(folio
) && folio_test_swapbacked(folio
) &&
756 !folio_test_swapcache(folio
) && !folio_test_unevictable(folio
)) {
757 struct folio_batch
*fbatch
;
760 if (!folio_test_clear_lru(folio
)) {
765 local_lock(&cpu_fbatches
.lock
);
766 fbatch
= this_cpu_ptr(&cpu_fbatches
.lru_lazyfree
);
767 folio_batch_add_and_move(fbatch
, folio
, lru_lazyfree_fn
);
768 local_unlock(&cpu_fbatches
.lock
);
772 void lru_add_drain(void)
774 local_lock(&cpu_fbatches
.lock
);
775 lru_add_drain_cpu(smp_processor_id());
776 local_unlock(&cpu_fbatches
.lock
);
781 * It's called from per-cpu workqueue context in SMP case so
782 * lru_add_drain_cpu and invalidate_bh_lrus_cpu should run on
783 * the same cpu. It shouldn't be a problem in !SMP case since
784 * the core is only one and the locks will disable preemption.
786 static void lru_add_and_bh_lrus_drain(void)
788 local_lock(&cpu_fbatches
.lock
);
789 lru_add_drain_cpu(smp_processor_id());
790 local_unlock(&cpu_fbatches
.lock
);
791 invalidate_bh_lrus_cpu();
795 void lru_add_drain_cpu_zone(struct zone
*zone
)
797 local_lock(&cpu_fbatches
.lock
);
798 lru_add_drain_cpu(smp_processor_id());
799 drain_local_pages(zone
);
800 local_unlock(&cpu_fbatches
.lock
);
806 static DEFINE_PER_CPU(struct work_struct
, lru_add_drain_work
);
808 static void lru_add_drain_per_cpu(struct work_struct
*dummy
)
810 lru_add_and_bh_lrus_drain();
813 static bool cpu_needs_drain(unsigned int cpu
)
815 struct cpu_fbatches
*fbatches
= &per_cpu(cpu_fbatches
, cpu
);
817 /* Check these in order of likelihood that they're not zero */
818 return folio_batch_count(&fbatches
->lru_add
) ||
819 data_race(folio_batch_count(&per_cpu(lru_rotate
.fbatch
, cpu
))) ||
820 folio_batch_count(&fbatches
->lru_deactivate_file
) ||
821 folio_batch_count(&fbatches
->lru_deactivate
) ||
822 folio_batch_count(&fbatches
->lru_lazyfree
) ||
823 folio_batch_count(&fbatches
->activate
) ||
824 need_mlock_drain(cpu
) ||
825 has_bh_in_lru(cpu
, NULL
);
829 * Doesn't need any cpu hotplug locking because we do rely on per-cpu
830 * kworkers being shut down before our page_alloc_cpu_dead callback is
831 * executed on the offlined cpu.
832 * Calling this function with cpu hotplug locks held can actually lead
833 * to obscure indirect dependencies via WQ context.
835 static inline void __lru_add_drain_all(bool force_all_cpus
)
838 * lru_drain_gen - Global pages generation number
840 * (A) Definition: global lru_drain_gen = x implies that all generations
841 * 0 < n <= x are already *scheduled* for draining.
843 * This is an optimization for the highly-contended use case where a
844 * user space workload keeps constantly generating a flow of pages for
847 static unsigned int lru_drain_gen
;
848 static struct cpumask has_work
;
849 static DEFINE_MUTEX(lock
);
850 unsigned cpu
, this_gen
;
853 * Make sure nobody triggers this path before mm_percpu_wq is fully
856 if (WARN_ON(!mm_percpu_wq
))
860 * Guarantee folio_batch counter stores visible by this CPU
861 * are visible to other CPUs before loading the current drain
867 * (B) Locally cache global LRU draining generation number
869 * The read barrier ensures that the counter is loaded before the mutex
870 * is taken. It pairs with smp_mb() inside the mutex critical section
873 this_gen
= smp_load_acquire(&lru_drain_gen
);
878 * (C) Exit the draining operation if a newer generation, from another
879 * lru_add_drain_all(), was already scheduled for draining. Check (A).
881 if (unlikely(this_gen
!= lru_drain_gen
&& !force_all_cpus
))
885 * (D) Increment global generation number
887 * Pairs with smp_load_acquire() at (B), outside of the critical
888 * section. Use a full memory barrier to guarantee that the
889 * new global drain generation number is stored before loading
890 * folio_batch counters.
892 * This pairing must be done here, before the for_each_online_cpu loop
893 * below which drains the page vectors.
895 * Let x, y, and z represent some system CPU numbers, where x < y < z.
896 * Assume CPU #z is in the middle of the for_each_online_cpu loop
897 * below and has already reached CPU #y's per-cpu data. CPU #x comes
898 * along, adds some pages to its per-cpu vectors, then calls
899 * lru_add_drain_all().
901 * If the paired barrier is done at any later step, e.g. after the
902 * loop, CPU #x will just exit at (C) and miss flushing out all of its
905 WRITE_ONCE(lru_drain_gen
, lru_drain_gen
+ 1);
908 cpumask_clear(&has_work
);
909 for_each_online_cpu(cpu
) {
910 struct work_struct
*work
= &per_cpu(lru_add_drain_work
, cpu
);
912 if (cpu_needs_drain(cpu
)) {
913 INIT_WORK(work
, lru_add_drain_per_cpu
);
914 queue_work_on(cpu
, mm_percpu_wq
, work
);
915 __cpumask_set_cpu(cpu
, &has_work
);
919 for_each_cpu(cpu
, &has_work
)
920 flush_work(&per_cpu(lru_add_drain_work
, cpu
));
926 void lru_add_drain_all(void)
928 __lru_add_drain_all(false);
931 void lru_add_drain_all(void)
935 #endif /* CONFIG_SMP */
937 atomic_t lru_disable_count
= ATOMIC_INIT(0);
940 * lru_cache_disable() needs to be called before we start compiling
941 * a list of pages to be migrated using isolate_lru_page().
942 * It drains pages on LRU cache and then disable on all cpus until
943 * lru_cache_enable is called.
945 * Must be paired with a call to lru_cache_enable().
947 void lru_cache_disable(void)
949 atomic_inc(&lru_disable_count
);
951 * Readers of lru_disable_count are protected by either disabling
952 * preemption or rcu_read_lock:
954 * preempt_disable, local_irq_disable [bh_lru_lock()]
955 * rcu_read_lock [rt_spin_lock CONFIG_PREEMPT_RT]
956 * preempt_disable [local_lock !CONFIG_PREEMPT_RT]
958 * Since v5.1 kernel, synchronize_rcu() is guaranteed to wait on
959 * preempt_disable() regions of code. So any CPU which sees
960 * lru_disable_count = 0 will have exited the critical
961 * section when synchronize_rcu() returns.
963 synchronize_rcu_expedited();
965 __lru_add_drain_all(true);
967 lru_add_and_bh_lrus_drain();
972 * folios_put_refs - Reduce the reference count on a batch of folios.
973 * @folios: The folios.
974 * @refs: The number of refs to subtract from each folio.
976 * Like folio_put(), but for a batch of folios. This is more efficient
977 * than writing the loop yourself as it will optimise the locks which need
978 * to be taken if the folios are freed. The folios batch is returned
979 * empty and ready to be reused for another batch; there is no need
980 * to reinitialise it. If @refs is NULL, we subtract one from each
983 * Context: May be called in process or interrupt context, but not in NMI
984 * context. May be called while holding a spinlock.
986 void folios_put_refs(struct folio_batch
*folios
, unsigned int *refs
)
989 struct lruvec
*lruvec
= NULL
;
990 unsigned long flags
= 0;
992 for (i
= 0, j
= 0; i
< folios
->nr
; i
++) {
993 struct folio
*folio
= folios
->folios
[i
];
994 unsigned int nr_refs
= refs
? refs
[i
] : 1;
996 if (is_huge_zero_folio(folio
))
999 if (folio_is_zone_device(folio
)) {
1001 unlock_page_lruvec_irqrestore(lruvec
, flags
);
1004 if (put_devmap_managed_folio_refs(folio
, nr_refs
))
1006 if (folio_ref_sub_and_test(folio
, nr_refs
))
1007 free_zone_device_folio(folio
);
1011 if (!folio_ref_sub_and_test(folio
, nr_refs
))
1014 /* hugetlb has its own memcg */
1015 if (folio_test_hugetlb(folio
)) {
1017 unlock_page_lruvec_irqrestore(lruvec
, flags
);
1020 free_huge_folio(folio
);
1023 folio_undo_large_rmappable(folio
);
1024 __page_cache_release(folio
, &lruvec
, &flags
);
1027 folios
->folios
[j
] = folio
;
1031 unlock_page_lruvec_irqrestore(lruvec
, flags
);
1033 folio_batch_reinit(folios
);
1038 mem_cgroup_uncharge_folios(folios
);
1039 free_unref_folios(folios
);
1041 EXPORT_SYMBOL(folios_put_refs
);
1044 * release_pages - batched put_page()
1045 * @arg: array of pages to release
1046 * @nr: number of pages
1048 * Decrement the reference count on all the pages in @arg. If it
1049 * fell to zero, remove the page from the LRU and free it.
1051 * Note that the argument can be an array of pages, encoded pages,
1052 * or folio pointers. We ignore any encoded bits, and turn any of
1053 * them into just a folio that gets free'd.
1055 void release_pages(release_pages_arg arg
, int nr
)
1057 struct folio_batch fbatch
;
1058 int refs
[PAGEVEC_SIZE
];
1059 struct encoded_page
**encoded
= arg
.encoded_pages
;
1062 folio_batch_init(&fbatch
);
1063 for (i
= 0; i
< nr
; i
++) {
1064 /* Turn any of the argument types into a folio */
1065 struct folio
*folio
= page_folio(encoded_page_ptr(encoded
[i
]));
1067 /* Is our next entry actually "nr_pages" -> "nr_refs" ? */
1068 refs
[fbatch
.nr
] = 1;
1069 if (unlikely(encoded_page_flags(encoded
[i
]) &
1070 ENCODED_PAGE_BIT_NR_PAGES_NEXT
))
1071 refs
[fbatch
.nr
] = encoded_nr_pages(encoded
[++i
]);
1073 if (folio_batch_add(&fbatch
, folio
) > 0)
1075 folios_put_refs(&fbatch
, refs
);
1079 folios_put_refs(&fbatch
, refs
);
1081 EXPORT_SYMBOL(release_pages
);
1084 * The folios which we're about to release may be in the deferred lru-addition
1085 * queues. That would prevent them from really being freed right now. That's
1086 * OK from a correctness point of view but is inefficient - those folios may be
1087 * cache-warm and we want to give them back to the page allocator ASAP.
1089 * So __folio_batch_release() will drain those queues here.
1090 * folio_batch_move_lru() calls folios_put() directly to avoid
1093 void __folio_batch_release(struct folio_batch
*fbatch
)
1095 if (!fbatch
->percpu_pvec_drained
) {
1097 fbatch
->percpu_pvec_drained
= true;
1101 EXPORT_SYMBOL(__folio_batch_release
);
1104 * folio_batch_remove_exceptionals() - Prune non-folios from a batch.
1105 * @fbatch: The batch to prune
1107 * find_get_entries() fills a batch with both folios and shadow/swap/DAX
1108 * entries. This function prunes all the non-folio entries from @fbatch
1109 * without leaving holes, so that it can be passed on to folio-only batch
1112 void folio_batch_remove_exceptionals(struct folio_batch
*fbatch
)
1116 for (i
= 0, j
= 0; i
< folio_batch_count(fbatch
); i
++) {
1117 struct folio
*folio
= fbatch
->folios
[i
];
1118 if (!xa_is_value(folio
))
1119 fbatch
->folios
[j
++] = folio
;
1125 * Perform any setup for the swap system
1127 void __init
swap_setup(void)
1129 unsigned long megs
= totalram_pages() >> (20 - PAGE_SHIFT
);
1131 /* Use a smaller cluster for small-memory machines */
1137 * Right now other parts of the system means that we
1138 * _really_ don't want to cluster much more