1 // SPDX-License-Identifier: GPL-2.0
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
7 * Swap reorganised 29.12.95,
8 * Asynchronous swapping added 30.12.95. Stephen Tweedie
9 * Removed race in async swapping. 14.4.1996. Bruno Haible
10 * Add swap of shared pages through the page cache. 20.2.1998. Stephen Tweedie
11 * Always use brw_page, life becomes simpler. 12 May 1998 Eric Biederman
15 #include <linux/kernel_stat.h>
16 #include <linux/gfp.h>
17 #include <linux/pagemap.h>
18 #include <linux/swap.h>
19 #include <linux/bio.h>
20 #include <linux/swapops.h>
21 #include <linux/writeback.h>
22 #include <linux/blkdev.h>
23 #include <linux/psi.h>
24 #include <linux/uio.h>
25 #include <linux/sched/task.h>
26 #include <linux/delayacct.h>
27 #include <linux/zswap.h>
30 static void __end_swap_bio_write(struct bio
*bio
)
32 struct folio
*folio
= bio_first_folio_all(bio
);
36 * We failed to write the page out to swap-space.
37 * Re-dirty the page in order to avoid it being reclaimed.
38 * Also print a dire warning that things will go BAD (tm)
41 * Also clear PG_reclaim to avoid folio_rotate_reclaimable()
43 folio_mark_dirty(folio
);
44 pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n",
45 MAJOR(bio_dev(bio
)), MINOR(bio_dev(bio
)),
46 (unsigned long long)bio
->bi_iter
.bi_sector
);
47 folio_clear_reclaim(folio
);
49 folio_end_writeback(folio
);
52 static void end_swap_bio_write(struct bio
*bio
)
54 __end_swap_bio_write(bio
);
58 static void __end_swap_bio_read(struct bio
*bio
)
60 struct folio
*folio
= bio_first_folio_all(bio
);
63 pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n",
64 MAJOR(bio_dev(bio
)), MINOR(bio_dev(bio
)),
65 (unsigned long long)bio
->bi_iter
.bi_sector
);
67 folio_mark_uptodate(folio
);
72 static void end_swap_bio_read(struct bio
*bio
)
74 __end_swap_bio_read(bio
);
78 int generic_swapfile_activate(struct swap_info_struct
*sis
,
79 struct file
*swap_file
,
82 struct address_space
*mapping
= swap_file
->f_mapping
;
83 struct inode
*inode
= mapping
->host
;
84 unsigned blocks_per_page
;
85 unsigned long page_no
;
89 sector_t lowest_block
= -1;
90 sector_t highest_block
= 0;
94 blkbits
= inode
->i_blkbits
;
95 blocks_per_page
= PAGE_SIZE
>> blkbits
;
98 * Map all the blocks into the extent tree. This code doesn't try
103 last_block
= i_size_read(inode
) >> blkbits
;
104 while ((probe_block
+ blocks_per_page
) <= last_block
&&
105 page_no
< sis
->max
) {
106 unsigned block_in_page
;
107 sector_t first_block
;
111 first_block
= probe_block
;
112 ret
= bmap(inode
, &first_block
);
113 if (ret
|| !first_block
)
117 * It must be PAGE_SIZE aligned on-disk
119 if (first_block
& (blocks_per_page
- 1)) {
124 for (block_in_page
= 1; block_in_page
< blocks_per_page
;
128 block
= probe_block
+ block_in_page
;
129 ret
= bmap(inode
, &block
);
133 if (block
!= first_block
+ block_in_page
) {
140 first_block
>>= (PAGE_SHIFT
- blkbits
);
141 if (page_no
) { /* exclude the header page */
142 if (first_block
< lowest_block
)
143 lowest_block
= first_block
;
144 if (first_block
> highest_block
)
145 highest_block
= first_block
;
149 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
151 ret
= add_swap_extent(sis
, page_no
, 1, first_block
);
156 probe_block
+= blocks_per_page
;
161 *span
= 1 + highest_block
- lowest_block
;
163 page_no
= 1; /* force Empty message */
165 sis
->pages
= page_no
- 1;
166 sis
->highest_bit
= page_no
- 1;
170 pr_err("swapon: swapfile has holes\n");
176 * We may have stale swap cache pages in memory: notice
177 * them here and get rid of the unnecessary final write.
179 int swap_writepage(struct page
*page
, struct writeback_control
*wbc
)
181 struct folio
*folio
= page_folio(page
);
184 if (folio_free_swap(folio
)) {
189 * Arch code may have to preserve more data than just the page
190 * contents, e.g. memory tags.
192 ret
= arch_prepare_to_swap(folio
);
194 folio_mark_dirty(folio
);
198 if (zswap_store(folio
)) {
202 if (!mem_cgroup_zswap_writeback_enabled(folio_memcg(folio
))) {
203 folio_mark_dirty(folio
);
204 return AOP_WRITEPAGE_ACTIVATE
;
207 __swap_writepage(folio
, wbc
);
211 static inline void count_swpout_vm_event(struct folio
*folio
)
213 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
214 if (unlikely(folio_test_pmd_mappable(folio
))) {
215 count_memcg_folio_events(folio
, THP_SWPOUT
, 1);
216 count_vm_event(THP_SWPOUT
);
218 count_mthp_stat(folio_order(folio
), MTHP_STAT_SWPOUT
);
220 count_vm_events(PSWPOUT
, folio_nr_pages(folio
));
223 #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
224 static void bio_associate_blkg_from_page(struct bio
*bio
, struct folio
*folio
)
226 struct cgroup_subsys_state
*css
;
227 struct mem_cgroup
*memcg
;
229 memcg
= folio_memcg(folio
);
234 css
= cgroup_e_css(memcg
->css
.cgroup
, &io_cgrp_subsys
);
235 bio_associate_blkg_from_css(bio
, css
);
239 #define bio_associate_blkg_from_page(bio, folio) do { } while (0)
240 #endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */
244 struct bio_vec bvec
[SWAP_CLUSTER_MAX
];
248 static mempool_t
*sio_pool
;
250 int sio_pool_init(void)
253 mempool_t
*pool
= mempool_create_kmalloc_pool(
254 SWAP_CLUSTER_MAX
, sizeof(struct swap_iocb
));
255 if (cmpxchg(&sio_pool
, NULL
, pool
))
256 mempool_destroy(pool
);
263 static void sio_write_complete(struct kiocb
*iocb
, long ret
)
265 struct swap_iocb
*sio
= container_of(iocb
, struct swap_iocb
, iocb
);
266 struct page
*page
= sio
->bvec
[0].bv_page
;
269 if (ret
!= sio
->len
) {
271 * In the case of swap-over-nfs, this can be a
272 * temporary failure if the system has limited
273 * memory for allocating transmit buffers.
274 * Mark the page dirty and avoid
275 * folio_rotate_reclaimable but rate-limit the
276 * messages but do not flag PageError like
277 * the normal direct-to-bio case as it could
280 pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n",
281 ret
, swap_dev_pos(page_swap_entry(page
)));
282 for (p
= 0; p
< sio
->pages
; p
++) {
283 page
= sio
->bvec
[p
].bv_page
;
284 set_page_dirty(page
);
285 ClearPageReclaim(page
);
289 for (p
= 0; p
< sio
->pages
; p
++)
290 end_page_writeback(sio
->bvec
[p
].bv_page
);
292 mempool_free(sio
, sio_pool
);
295 static void swap_writepage_fs(struct folio
*folio
, struct writeback_control
*wbc
)
297 struct swap_iocb
*sio
= NULL
;
298 struct swap_info_struct
*sis
= swp_swap_info(folio
->swap
);
299 struct file
*swap_file
= sis
->swap_file
;
300 loff_t pos
= swap_dev_pos(folio
->swap
);
302 count_swpout_vm_event(folio
);
303 folio_start_writeback(folio
);
306 sio
= *wbc
->swap_plug
;
308 if (sio
->iocb
.ki_filp
!= swap_file
||
309 sio
->iocb
.ki_pos
+ sio
->len
!= pos
) {
310 swap_write_unplug(sio
);
315 sio
= mempool_alloc(sio_pool
, GFP_NOIO
);
316 init_sync_kiocb(&sio
->iocb
, swap_file
);
317 sio
->iocb
.ki_complete
= sio_write_complete
;
318 sio
->iocb
.ki_pos
= pos
;
322 bvec_set_folio(&sio
->bvec
[sio
->pages
], folio
, folio_size(folio
), 0);
323 sio
->len
+= folio_size(folio
);
325 if (sio
->pages
== ARRAY_SIZE(sio
->bvec
) || !wbc
->swap_plug
) {
326 swap_write_unplug(sio
);
330 *wbc
->swap_plug
= sio
;
333 static void swap_writepage_bdev_sync(struct folio
*folio
,
334 struct writeback_control
*wbc
, struct swap_info_struct
*sis
)
339 bio_init(&bio
, sis
->bdev
, &bv
, 1,
340 REQ_OP_WRITE
| REQ_SWAP
| wbc_to_write_flags(wbc
));
341 bio
.bi_iter
.bi_sector
= swap_folio_sector(folio
);
342 bio_add_folio_nofail(&bio
, folio
, folio_size(folio
), 0);
344 bio_associate_blkg_from_page(&bio
, folio
);
345 count_swpout_vm_event(folio
);
347 folio_start_writeback(folio
);
350 submit_bio_wait(&bio
);
351 __end_swap_bio_write(&bio
);
354 static void swap_writepage_bdev_async(struct folio
*folio
,
355 struct writeback_control
*wbc
, struct swap_info_struct
*sis
)
359 bio
= bio_alloc(sis
->bdev
, 1,
360 REQ_OP_WRITE
| REQ_SWAP
| wbc_to_write_flags(wbc
),
362 bio
->bi_iter
.bi_sector
= swap_folio_sector(folio
);
363 bio
->bi_end_io
= end_swap_bio_write
;
364 bio_add_folio_nofail(bio
, folio
, folio_size(folio
), 0);
366 bio_associate_blkg_from_page(bio
, folio
);
367 count_swpout_vm_event(folio
);
368 folio_start_writeback(folio
);
373 void __swap_writepage(struct folio
*folio
, struct writeback_control
*wbc
)
375 struct swap_info_struct
*sis
= swp_swap_info(folio
->swap
);
377 VM_BUG_ON_FOLIO(!folio_test_swapcache(folio
), folio
);
379 * ->flags can be updated non-atomicially (scan_swap_map_slots),
380 * but that will never affect SWP_FS_OPS, so the data_race
383 if (data_race(sis
->flags
& SWP_FS_OPS
))
384 swap_writepage_fs(folio
, wbc
);
386 * ->flags can be updated non-atomicially (scan_swap_map_slots),
387 * but that will never affect SWP_SYNCHRONOUS_IO, so the data_race
390 else if (data_race(sis
->flags
& SWP_SYNCHRONOUS_IO
))
391 swap_writepage_bdev_sync(folio
, wbc
, sis
);
393 swap_writepage_bdev_async(folio
, wbc
, sis
);
396 void swap_write_unplug(struct swap_iocb
*sio
)
398 struct iov_iter from
;
399 struct address_space
*mapping
= sio
->iocb
.ki_filp
->f_mapping
;
402 iov_iter_bvec(&from
, ITER_SOURCE
, sio
->bvec
, sio
->pages
, sio
->len
);
403 ret
= mapping
->a_ops
->swap_rw(&sio
->iocb
, &from
);
404 if (ret
!= -EIOCBQUEUED
)
405 sio_write_complete(&sio
->iocb
, ret
);
408 static void sio_read_complete(struct kiocb
*iocb
, long ret
)
410 struct swap_iocb
*sio
= container_of(iocb
, struct swap_iocb
, iocb
);
413 if (ret
== sio
->len
) {
414 for (p
= 0; p
< sio
->pages
; p
++) {
415 struct folio
*folio
= page_folio(sio
->bvec
[p
].bv_page
);
417 folio_mark_uptodate(folio
);
420 count_vm_events(PSWPIN
, sio
->pages
);
422 for (p
= 0; p
< sio
->pages
; p
++) {
423 struct folio
*folio
= page_folio(sio
->bvec
[p
].bv_page
);
427 pr_alert_ratelimited("Read-error on swap-device\n");
429 mempool_free(sio
, sio_pool
);
432 static void swap_read_folio_fs(struct folio
*folio
, struct swap_iocb
**plug
)
434 struct swap_info_struct
*sis
= swp_swap_info(folio
->swap
);
435 struct swap_iocb
*sio
= NULL
;
436 loff_t pos
= swap_dev_pos(folio
->swap
);
441 if (sio
->iocb
.ki_filp
!= sis
->swap_file
||
442 sio
->iocb
.ki_pos
+ sio
->len
!= pos
) {
443 swap_read_unplug(sio
);
448 sio
= mempool_alloc(sio_pool
, GFP_KERNEL
);
449 init_sync_kiocb(&sio
->iocb
, sis
->swap_file
);
450 sio
->iocb
.ki_pos
= pos
;
451 sio
->iocb
.ki_complete
= sio_read_complete
;
455 bvec_set_folio(&sio
->bvec
[sio
->pages
], folio
, folio_size(folio
), 0);
456 sio
->len
+= folio_size(folio
);
458 if (sio
->pages
== ARRAY_SIZE(sio
->bvec
) || !plug
) {
459 swap_read_unplug(sio
);
466 static void swap_read_folio_bdev_sync(struct folio
*folio
,
467 struct swap_info_struct
*sis
)
472 bio_init(&bio
, sis
->bdev
, &bv
, 1, REQ_OP_READ
);
473 bio
.bi_iter
.bi_sector
= swap_folio_sector(folio
);
474 bio_add_folio_nofail(&bio
, folio
, folio_size(folio
), 0);
476 * Keep this task valid during swap readpage because the oom killer may
477 * attempt to access it in the page fault retry time check.
479 get_task_struct(current
);
480 count_vm_event(PSWPIN
);
481 submit_bio_wait(&bio
);
482 __end_swap_bio_read(&bio
);
483 put_task_struct(current
);
486 static void swap_read_folio_bdev_async(struct folio
*folio
,
487 struct swap_info_struct
*sis
)
491 bio
= bio_alloc(sis
->bdev
, 1, REQ_OP_READ
, GFP_KERNEL
);
492 bio
->bi_iter
.bi_sector
= swap_folio_sector(folio
);
493 bio
->bi_end_io
= end_swap_bio_read
;
494 bio_add_folio_nofail(bio
, folio
, folio_size(folio
), 0);
495 count_vm_event(PSWPIN
);
499 void swap_read_folio(struct folio
*folio
, struct swap_iocb
**plug
)
501 struct swap_info_struct
*sis
= swp_swap_info(folio
->swap
);
502 bool synchronous
= sis
->flags
& SWP_SYNCHRONOUS_IO
;
503 bool workingset
= folio_test_workingset(folio
);
504 unsigned long pflags
;
507 VM_BUG_ON_FOLIO(!folio_test_swapcache(folio
) && !synchronous
, folio
);
508 VM_BUG_ON_FOLIO(!folio_test_locked(folio
), folio
);
509 VM_BUG_ON_FOLIO(folio_test_uptodate(folio
), folio
);
512 * Count submission time as memory stall and delay. When the device
513 * is congested, or the submitting cgroup IO-throttled, submission
514 * can be a significant part of overall IO time.
517 delayacct_thrashing_start(&in_thrashing
);
518 psi_memstall_enter(&pflags
);
520 delayacct_swapin_start();
522 if (zswap_load(folio
)) {
524 } else if (data_race(sis
->flags
& SWP_FS_OPS
)) {
525 swap_read_folio_fs(folio
, plug
);
526 } else if (synchronous
) {
527 swap_read_folio_bdev_sync(folio
, sis
);
529 swap_read_folio_bdev_async(folio
, sis
);
533 delayacct_thrashing_end(&in_thrashing
);
534 psi_memstall_leave(&pflags
);
536 delayacct_swapin_end();
539 void __swap_read_unplug(struct swap_iocb
*sio
)
541 struct iov_iter from
;
542 struct address_space
*mapping
= sio
->iocb
.ki_filp
->f_mapping
;
545 iov_iter_bvec(&from
, ITER_DEST
, sio
->bvec
, sio
->pages
, sio
->len
);
546 ret
= mapping
->a_ops
->swap_rw(&sio
->iocb
, &from
);
547 if (ret
!= -EIOCBQUEUED
)
548 sio_read_complete(&sio
->iocb
, ret
);