1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 1991, 1992 Linus Torvalds
4 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
5 * Copyright (C) 2016 - 2020 Christoph Hellwig
7 #include <linux/init.h>
9 #include <linux/blkdev.h>
10 #include <linux/buffer_head.h>
11 #include <linux/mpage.h>
12 #include <linux/uio.h>
13 #include <linux/namei.h>
14 #include <linux/task_io_accounting_ops.h>
15 #include <linux/falloc.h>
16 #include <linux/suspend.h>
18 #include <linux/iomap.h>
19 #include <linux/module.h>
22 static inline struct inode
*bdev_file_inode(struct file
*file
)
24 return file
->f_mapping
->host
;
27 static blk_opf_t
dio_bio_write_op(struct kiocb
*iocb
)
29 blk_opf_t opf
= REQ_OP_WRITE
| REQ_SYNC
| REQ_IDLE
;
31 /* avoid the need for a I/O completion work item */
32 if (iocb_is_dsync(iocb
))
37 static bool blkdev_dio_invalid(struct block_device
*bdev
, loff_t pos
,
38 struct iov_iter
*iter
, bool is_atomic
)
40 if (is_atomic
&& !generic_atomic_write_valid(iter
, pos
))
43 return pos
& (bdev_logical_block_size(bdev
) - 1) ||
44 !bdev_iter_is_aligned(bdev
, iter
);
47 #define DIO_INLINE_BIO_VECS 4
49 static ssize_t
__blkdev_direct_IO_simple(struct kiocb
*iocb
,
50 struct iov_iter
*iter
, struct block_device
*bdev
,
51 unsigned int nr_pages
)
53 struct bio_vec inline_vecs
[DIO_INLINE_BIO_VECS
], *vecs
;
54 loff_t pos
= iocb
->ki_pos
;
55 bool should_dirty
= false;
59 if (nr_pages
<= DIO_INLINE_BIO_VECS
)
62 vecs
= kmalloc_array(nr_pages
, sizeof(struct bio_vec
),
68 if (iov_iter_rw(iter
) == READ
) {
69 bio_init(&bio
, bdev
, vecs
, nr_pages
, REQ_OP_READ
);
70 if (user_backed_iter(iter
))
73 bio_init(&bio
, bdev
, vecs
, nr_pages
, dio_bio_write_op(iocb
));
75 bio
.bi_iter
.bi_sector
= pos
>> SECTOR_SHIFT
;
76 bio
.bi_write_hint
= file_inode(iocb
->ki_filp
)->i_write_hint
;
77 bio
.bi_ioprio
= iocb
->ki_ioprio
;
78 if (iocb
->ki_flags
& IOCB_ATOMIC
)
79 bio
.bi_opf
|= REQ_ATOMIC
;
81 ret
= bio_iov_iter_get_pages(&bio
, iter
);
84 ret
= bio
.bi_iter
.bi_size
;
86 if (iov_iter_rw(iter
) == WRITE
)
87 task_io_account_write(ret
);
89 if (iocb
->ki_flags
& IOCB_NOWAIT
)
90 bio
.bi_opf
|= REQ_NOWAIT
;
92 submit_bio_wait(&bio
);
94 bio_release_pages(&bio
, should_dirty
);
95 if (unlikely(bio
.bi_status
))
96 ret
= blk_status_to_errno(bio
.bi_status
);
99 if (vecs
!= inline_vecs
)
108 DIO_SHOULD_DIRTY
= 1,
115 struct task_struct
*waiter
;
120 struct bio bio ____cacheline_aligned_in_smp
;
123 static struct bio_set blkdev_dio_pool
;
125 static void blkdev_bio_end_io(struct bio
*bio
)
127 struct blkdev_dio
*dio
= bio
->bi_private
;
128 bool should_dirty
= dio
->flags
& DIO_SHOULD_DIRTY
;
130 if (bio
->bi_status
&& !dio
->bio
.bi_status
)
131 dio
->bio
.bi_status
= bio
->bi_status
;
133 if (atomic_dec_and_test(&dio
->ref
)) {
134 if (!(dio
->flags
& DIO_IS_SYNC
)) {
135 struct kiocb
*iocb
= dio
->iocb
;
138 WRITE_ONCE(iocb
->private, NULL
);
140 if (likely(!dio
->bio
.bi_status
)) {
144 ret
= blk_status_to_errno(dio
->bio
.bi_status
);
147 dio
->iocb
->ki_complete(iocb
, ret
);
150 struct task_struct
*waiter
= dio
->waiter
;
152 WRITE_ONCE(dio
->waiter
, NULL
);
153 blk_wake_io_task(waiter
);
158 bio_check_pages_dirty(bio
);
160 bio_release_pages(bio
, false);
165 static ssize_t
__blkdev_direct_IO(struct kiocb
*iocb
, struct iov_iter
*iter
,
166 struct block_device
*bdev
, unsigned int nr_pages
)
168 struct blk_plug plug
;
169 struct blkdev_dio
*dio
;
171 bool is_read
= (iov_iter_rw(iter
) == READ
), is_sync
;
172 blk_opf_t opf
= is_read
? REQ_OP_READ
: dio_bio_write_op(iocb
);
173 loff_t pos
= iocb
->ki_pos
;
176 if (iocb
->ki_flags
& IOCB_ALLOC_CACHE
)
177 opf
|= REQ_ALLOC_CACHE
;
178 bio
= bio_alloc_bioset(bdev
, nr_pages
, opf
, GFP_KERNEL
,
180 dio
= container_of(bio
, struct blkdev_dio
, bio
);
181 atomic_set(&dio
->ref
, 1);
183 * Grab an extra reference to ensure the dio structure which is embedded
184 * into the first bio stays around.
188 is_sync
= is_sync_kiocb(iocb
);
190 dio
->flags
= DIO_IS_SYNC
;
191 dio
->waiter
= current
;
198 if (is_read
&& user_backed_iter(iter
))
199 dio
->flags
|= DIO_SHOULD_DIRTY
;
201 blk_start_plug(&plug
);
204 bio
->bi_iter
.bi_sector
= pos
>> SECTOR_SHIFT
;
205 bio
->bi_write_hint
= file_inode(iocb
->ki_filp
)->i_write_hint
;
206 bio
->bi_private
= dio
;
207 bio
->bi_end_io
= blkdev_bio_end_io
;
208 bio
->bi_ioprio
= iocb
->ki_ioprio
;
210 ret
= bio_iov_iter_get_pages(bio
, iter
);
212 bio
->bi_status
= BLK_STS_IOERR
;
216 if (iocb
->ki_flags
& IOCB_NOWAIT
) {
218 * This is nonblocking IO, and we need to allocate
219 * another bio if we have data left to map. As we
220 * cannot guarantee that one of the sub bios will not
221 * fail getting issued FOR NOWAIT and as error results
222 * are coalesced across all of them, be safe and ask for
223 * a retry of this from blocking context.
225 if (unlikely(iov_iter_count(iter
))) {
226 bio_release_pages(bio
, false);
227 bio_clear_flag(bio
, BIO_REFFED
);
229 blk_finish_plug(&plug
);
232 bio
->bi_opf
|= REQ_NOWAIT
;
236 if (dio
->flags
& DIO_SHOULD_DIRTY
)
237 bio_set_pages_dirty(bio
);
239 task_io_account_write(bio
->bi_iter
.bi_size
);
241 dio
->size
+= bio
->bi_iter
.bi_size
;
242 pos
+= bio
->bi_iter
.bi_size
;
244 nr_pages
= bio_iov_vecs_to_alloc(iter
, BIO_MAX_VECS
);
249 atomic_inc(&dio
->ref
);
251 bio
= bio_alloc(bdev
, nr_pages
, opf
, GFP_KERNEL
);
254 blk_finish_plug(&plug
);
260 set_current_state(TASK_UNINTERRUPTIBLE
);
261 if (!READ_ONCE(dio
->waiter
))
265 __set_current_state(TASK_RUNNING
);
268 ret
= blk_status_to_errno(dio
->bio
.bi_status
);
276 static void blkdev_bio_end_io_async(struct bio
*bio
)
278 struct blkdev_dio
*dio
= container_of(bio
, struct blkdev_dio
, bio
);
279 struct kiocb
*iocb
= dio
->iocb
;
282 WRITE_ONCE(iocb
->private, NULL
);
284 if (likely(!bio
->bi_status
)) {
288 ret
= blk_status_to_errno(bio
->bi_status
);
291 iocb
->ki_complete(iocb
, ret
);
293 if (dio
->flags
& DIO_SHOULD_DIRTY
) {
294 bio_check_pages_dirty(bio
);
296 bio_release_pages(bio
, false);
301 static ssize_t
__blkdev_direct_IO_async(struct kiocb
*iocb
,
302 struct iov_iter
*iter
,
303 struct block_device
*bdev
,
304 unsigned int nr_pages
)
306 bool is_read
= iov_iter_rw(iter
) == READ
;
307 blk_opf_t opf
= is_read
? REQ_OP_READ
: dio_bio_write_op(iocb
);
308 struct blkdev_dio
*dio
;
310 loff_t pos
= iocb
->ki_pos
;
313 if (iocb
->ki_flags
& IOCB_ALLOC_CACHE
)
314 opf
|= REQ_ALLOC_CACHE
;
315 bio
= bio_alloc_bioset(bdev
, nr_pages
, opf
, GFP_KERNEL
,
317 dio
= container_of(bio
, struct blkdev_dio
, bio
);
320 bio
->bi_iter
.bi_sector
= pos
>> SECTOR_SHIFT
;
321 bio
->bi_write_hint
= file_inode(iocb
->ki_filp
)->i_write_hint
;
322 bio
->bi_end_io
= blkdev_bio_end_io_async
;
323 bio
->bi_ioprio
= iocb
->ki_ioprio
;
325 if (iov_iter_is_bvec(iter
)) {
327 * Users don't rely on the iterator being in any particular
328 * state for async I/O returning -EIOCBQUEUED, hence we can
329 * avoid expensive iov_iter_advance(). Bypass
330 * bio_iov_iter_get_pages() and set the bvec directly.
332 bio_iov_bvec_set(bio
, iter
);
334 ret
= bio_iov_iter_get_pages(bio
, iter
);
340 dio
->size
= bio
->bi_iter
.bi_size
;
343 if (user_backed_iter(iter
)) {
344 dio
->flags
|= DIO_SHOULD_DIRTY
;
345 bio_set_pages_dirty(bio
);
348 task_io_account_write(bio
->bi_iter
.bi_size
);
351 if (iocb
->ki_flags
& IOCB_ATOMIC
)
352 bio
->bi_opf
|= REQ_ATOMIC
;
354 if (iocb
->ki_flags
& IOCB_NOWAIT
)
355 bio
->bi_opf
|= REQ_NOWAIT
;
357 if (iocb
->ki_flags
& IOCB_HIPRI
) {
358 bio
->bi_opf
|= REQ_POLLED
;
360 WRITE_ONCE(iocb
->private, bio
);
367 static ssize_t
blkdev_direct_IO(struct kiocb
*iocb
, struct iov_iter
*iter
)
369 struct block_device
*bdev
= I_BDEV(iocb
->ki_filp
->f_mapping
->host
);
370 bool is_atomic
= iocb
->ki_flags
& IOCB_ATOMIC
;
371 unsigned int nr_pages
;
373 if (!iov_iter_count(iter
))
376 if (blkdev_dio_invalid(bdev
, iocb
->ki_pos
, iter
, is_atomic
))
379 nr_pages
= bio_iov_vecs_to_alloc(iter
, BIO_MAX_VECS
+ 1);
380 if (likely(nr_pages
<= BIO_MAX_VECS
)) {
381 if (is_sync_kiocb(iocb
))
382 return __blkdev_direct_IO_simple(iocb
, iter
, bdev
,
384 return __blkdev_direct_IO_async(iocb
, iter
, bdev
, nr_pages
);
385 } else if (is_atomic
) {
388 return __blkdev_direct_IO(iocb
, iter
, bdev
, bio_max_segs(nr_pages
));
391 static int blkdev_iomap_begin(struct inode
*inode
, loff_t offset
, loff_t length
,
392 unsigned int flags
, struct iomap
*iomap
, struct iomap
*srcmap
)
394 struct block_device
*bdev
= I_BDEV(inode
);
395 loff_t isize
= i_size_read(inode
);
401 iomap
->offset
= ALIGN_DOWN(offset
, bdev_logical_block_size(bdev
));
402 iomap
->type
= IOMAP_MAPPED
;
403 iomap
->addr
= iomap
->offset
;
404 iomap
->length
= isize
- iomap
->offset
;
405 iomap
->flags
|= IOMAP_F_BUFFER_HEAD
; /* noop for !CONFIG_BUFFER_HEAD */
409 static const struct iomap_ops blkdev_iomap_ops
= {
410 .iomap_begin
= blkdev_iomap_begin
,
413 #ifdef CONFIG_BUFFER_HEAD
414 static int blkdev_get_block(struct inode
*inode
, sector_t iblock
,
415 struct buffer_head
*bh
, int create
)
417 bh
->b_bdev
= I_BDEV(inode
);
418 bh
->b_blocknr
= iblock
;
419 set_buffer_mapped(bh
);
424 * We cannot call mpage_writepages() as it does not take the buffer lock.
425 * We must use block_write_full_folio() directly which holds the buffer
426 * lock. The buffer lock provides the synchronisation with writeback
427 * that filesystems rely on when they use the blockdev's mapping.
429 static int blkdev_writepages(struct address_space
*mapping
,
430 struct writeback_control
*wbc
)
432 struct blk_plug plug
;
435 blk_start_plug(&plug
);
436 err
= write_cache_pages(mapping
, wbc
, block_write_full_folio
,
438 blk_finish_plug(&plug
);
443 static int blkdev_read_folio(struct file
*file
, struct folio
*folio
)
445 return block_read_full_folio(folio
, blkdev_get_block
);
448 static void blkdev_readahead(struct readahead_control
*rac
)
450 mpage_readahead(rac
, blkdev_get_block
);
453 static int blkdev_write_begin(struct file
*file
, struct address_space
*mapping
,
454 loff_t pos
, unsigned len
, struct page
**pagep
, void **fsdata
)
456 return block_write_begin(mapping
, pos
, len
, pagep
, blkdev_get_block
);
459 static int blkdev_write_end(struct file
*file
, struct address_space
*mapping
,
460 loff_t pos
, unsigned len
, unsigned copied
, struct page
*page
,
464 ret
= block_write_end(file
, mapping
, pos
, len
, copied
, page
, fsdata
);
472 const struct address_space_operations def_blk_aops
= {
473 .dirty_folio
= block_dirty_folio
,
474 .invalidate_folio
= block_invalidate_folio
,
475 .read_folio
= blkdev_read_folio
,
476 .readahead
= blkdev_readahead
,
477 .writepages
= blkdev_writepages
,
478 .write_begin
= blkdev_write_begin
,
479 .write_end
= blkdev_write_end
,
480 .migrate_folio
= buffer_migrate_folio_norefs
,
481 .is_dirty_writeback
= buffer_check_dirty_writeback
,
483 #else /* CONFIG_BUFFER_HEAD */
484 static int blkdev_read_folio(struct file
*file
, struct folio
*folio
)
486 return iomap_read_folio(folio
, &blkdev_iomap_ops
);
489 static void blkdev_readahead(struct readahead_control
*rac
)
491 iomap_readahead(rac
, &blkdev_iomap_ops
);
494 static int blkdev_map_blocks(struct iomap_writepage_ctx
*wpc
,
495 struct inode
*inode
, loff_t offset
, unsigned int len
)
497 loff_t isize
= i_size_read(inode
);
499 if (WARN_ON_ONCE(offset
>= isize
))
501 if (offset
>= wpc
->iomap
.offset
&&
502 offset
< wpc
->iomap
.offset
+ wpc
->iomap
.length
)
504 return blkdev_iomap_begin(inode
, offset
, isize
- offset
,
505 IOMAP_WRITE
, &wpc
->iomap
, NULL
);
508 static const struct iomap_writeback_ops blkdev_writeback_ops
= {
509 .map_blocks
= blkdev_map_blocks
,
512 static int blkdev_writepages(struct address_space
*mapping
,
513 struct writeback_control
*wbc
)
515 struct iomap_writepage_ctx wpc
= { };
517 return iomap_writepages(mapping
, wbc
, &wpc
, &blkdev_writeback_ops
);
520 const struct address_space_operations def_blk_aops
= {
521 .dirty_folio
= filemap_dirty_folio
,
522 .release_folio
= iomap_release_folio
,
523 .invalidate_folio
= iomap_invalidate_folio
,
524 .read_folio
= blkdev_read_folio
,
525 .readahead
= blkdev_readahead
,
526 .writepages
= blkdev_writepages
,
527 .is_partially_uptodate
= iomap_is_partially_uptodate
,
528 .error_remove_folio
= generic_error_remove_folio
,
529 .migrate_folio
= filemap_migrate_folio
,
531 #endif /* CONFIG_BUFFER_HEAD */
534 * for a block special file file_inode(file)->i_size is zero
535 * so we compute the size by hand (just as in block_read/write above)
537 static loff_t
blkdev_llseek(struct file
*file
, loff_t offset
, int whence
)
539 struct inode
*bd_inode
= bdev_file_inode(file
);
542 inode_lock(bd_inode
);
543 retval
= fixed_size_llseek(file
, offset
, whence
, i_size_read(bd_inode
));
544 inode_unlock(bd_inode
);
548 static int blkdev_fsync(struct file
*filp
, loff_t start
, loff_t end
,
551 struct block_device
*bdev
= I_BDEV(filp
->f_mapping
->host
);
554 error
= file_write_and_wait_range(filp
, start
, end
);
559 * There is no need to serialise calls to blkdev_issue_flush with
560 * i_mutex and doing so causes performance issues with concurrent
561 * O_SYNC writers to a block device.
563 error
= blkdev_issue_flush(bdev
);
564 if (error
== -EOPNOTSUPP
)
571 * file_to_blk_mode - get block open flags from file flags
572 * @file: file whose open flags should be converted
574 * Look at file open flags and generate corresponding block open flags from
575 * them. The function works both for file just being open (e.g. during ->open
576 * callback) and for file that is already open. This is actually non-trivial
577 * (see comment in the function).
579 blk_mode_t
file_to_blk_mode(struct file
*file
)
583 if (file
->f_mode
& FMODE_READ
)
584 mode
|= BLK_OPEN_READ
;
585 if (file
->f_mode
& FMODE_WRITE
)
586 mode
|= BLK_OPEN_WRITE
;
588 * do_dentry_open() clears O_EXCL from f_flags, use file->private_data
589 * to determine whether the open was exclusive for already open files.
591 if (file
->private_data
)
592 mode
|= BLK_OPEN_EXCL
;
593 else if (file
->f_flags
& O_EXCL
)
594 mode
|= BLK_OPEN_EXCL
;
595 if (file
->f_flags
& O_NDELAY
)
596 mode
|= BLK_OPEN_NDELAY
;
599 * If all bits in O_ACCMODE set (aka O_RDWR | O_WRONLY), the floppy
600 * driver has historically allowed ioctls as if the file was opened for
601 * writing, but does not allow and actual reads or writes.
603 if ((file
->f_flags
& O_ACCMODE
) == (O_RDWR
| O_WRONLY
))
604 mode
|= BLK_OPEN_WRITE_IOCTL
;
609 static int blkdev_open(struct inode
*inode
, struct file
*filp
)
611 struct block_device
*bdev
;
615 mode
= file_to_blk_mode(filp
);
616 /* Use the file as the holder. */
617 if (mode
& BLK_OPEN_EXCL
)
618 filp
->private_data
= filp
;
619 ret
= bdev_permission(inode
->i_rdev
, mode
, filp
->private_data
);
623 bdev
= blkdev_get_no_open(inode
->i_rdev
);
627 if (bdev_can_atomic_write(bdev
) && filp
->f_flags
& O_DIRECT
)
628 filp
->f_mode
|= FMODE_CAN_ATOMIC_WRITE
;
630 ret
= bdev_open(bdev
, mode
, filp
->private_data
, NULL
, filp
);
632 blkdev_put_no_open(bdev
);
636 static int blkdev_release(struct inode
*inode
, struct file
*filp
)
643 blkdev_direct_write(struct kiocb
*iocb
, struct iov_iter
*from
)
645 size_t count
= iov_iter_count(from
);
648 written
= kiocb_invalidate_pages(iocb
, count
);
650 if (written
== -EBUSY
)
655 written
= blkdev_direct_IO(iocb
, from
);
657 kiocb_invalidate_post_direct_write(iocb
, count
);
658 iocb
->ki_pos
+= written
;
661 if (written
!= -EIOCBQUEUED
)
662 iov_iter_revert(from
, count
- iov_iter_count(from
));
666 static ssize_t
blkdev_buffered_write(struct kiocb
*iocb
, struct iov_iter
*from
)
668 return iomap_file_buffered_write(iocb
, from
, &blkdev_iomap_ops
);
672 * Write data to the block device. Only intended for the block device itself
673 * and the raw driver which basically is a fake block device.
675 * Does not take i_mutex for the write and thus is not for general purpose
678 static ssize_t
blkdev_write_iter(struct kiocb
*iocb
, struct iov_iter
*from
)
680 struct file
*file
= iocb
->ki_filp
;
681 struct inode
*bd_inode
= bdev_file_inode(file
);
682 struct block_device
*bdev
= I_BDEV(bd_inode
);
683 loff_t size
= bdev_nr_bytes(bdev
);
687 if (bdev_read_only(bdev
))
690 if (IS_SWAPFILE(bd_inode
) && !is_hibernate_resume_dev(bd_inode
->i_rdev
))
693 if (!iov_iter_count(from
))
696 if (iocb
->ki_pos
>= size
)
699 if ((iocb
->ki_flags
& (IOCB_NOWAIT
| IOCB_DIRECT
)) == IOCB_NOWAIT
)
702 size
-= iocb
->ki_pos
;
703 if (iov_iter_count(from
) > size
) {
704 shorted
= iov_iter_count(from
) - size
;
705 iov_iter_truncate(from
, size
);
708 ret
= file_update_time(file
);
712 if (iocb
->ki_flags
& IOCB_DIRECT
) {
713 ret
= blkdev_direct_write(iocb
, from
);
714 if (ret
>= 0 && iov_iter_count(from
))
715 ret
= direct_write_fallback(iocb
, from
, ret
,
716 blkdev_buffered_write(iocb
, from
));
718 ret
= blkdev_buffered_write(iocb
, from
);
722 ret
= generic_write_sync(iocb
, ret
);
723 iov_iter_reexpand(from
, iov_iter_count(from
) + shorted
);
727 static ssize_t
blkdev_read_iter(struct kiocb
*iocb
, struct iov_iter
*to
)
729 struct block_device
*bdev
= I_BDEV(iocb
->ki_filp
->f_mapping
->host
);
730 loff_t size
= bdev_nr_bytes(bdev
);
731 loff_t pos
= iocb
->ki_pos
;
736 if (unlikely(pos
+ iov_iter_count(to
) > size
)) {
740 shorted
= iov_iter_count(to
) - size
;
741 iov_iter_truncate(to
, size
);
744 count
= iov_iter_count(to
);
746 goto reexpand
; /* skip atime */
748 if (iocb
->ki_flags
& IOCB_DIRECT
) {
749 ret
= kiocb_write_and_wait(iocb
, count
);
752 file_accessed(iocb
->ki_filp
);
754 ret
= blkdev_direct_IO(iocb
, to
);
759 iov_iter_revert(to
, count
- iov_iter_count(to
));
760 if (ret
< 0 || !count
)
764 ret
= filemap_read(iocb
, to
, ret
);
767 if (unlikely(shorted
))
768 iov_iter_reexpand(to
, iov_iter_count(to
) + shorted
);
772 #define BLKDEV_FALLOC_FL_SUPPORTED \
773 (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | \
774 FALLOC_FL_ZERO_RANGE | FALLOC_FL_NO_HIDE_STALE)
776 static long blkdev_fallocate(struct file
*file
, int mode
, loff_t start
,
779 struct inode
*inode
= bdev_file_inode(file
);
780 struct block_device
*bdev
= I_BDEV(inode
);
781 loff_t end
= start
+ len
- 1;
785 /* Fail if we don't recognize the flags. */
786 if (mode
& ~BLKDEV_FALLOC_FL_SUPPORTED
)
789 /* Don't go off the end of the device. */
790 isize
= bdev_nr_bytes(bdev
);
794 if (mode
& FALLOC_FL_KEEP_SIZE
) {
796 end
= start
+ len
- 1;
802 * Don't allow IO that isn't aligned to logical block size.
804 if ((start
| len
) & (bdev_logical_block_size(bdev
) - 1))
807 filemap_invalidate_lock(inode
->i_mapping
);
810 * Invalidate the page cache, including dirty pages, for valid
811 * de-allocate mode calls to fallocate().
814 case FALLOC_FL_ZERO_RANGE
:
815 case FALLOC_FL_ZERO_RANGE
| FALLOC_FL_KEEP_SIZE
:
816 error
= truncate_bdev_range(bdev
, file_to_blk_mode(file
), start
, end
);
820 error
= blkdev_issue_zeroout(bdev
, start
>> SECTOR_SHIFT
,
821 len
>> SECTOR_SHIFT
, GFP_KERNEL
,
822 BLKDEV_ZERO_NOUNMAP
);
824 case FALLOC_FL_PUNCH_HOLE
| FALLOC_FL_KEEP_SIZE
:
825 error
= truncate_bdev_range(bdev
, file_to_blk_mode(file
), start
, end
);
829 error
= blkdev_issue_zeroout(bdev
, start
>> SECTOR_SHIFT
,
830 len
>> SECTOR_SHIFT
, GFP_KERNEL
,
831 BLKDEV_ZERO_NOFALLBACK
);
833 case FALLOC_FL_PUNCH_HOLE
| FALLOC_FL_KEEP_SIZE
| FALLOC_FL_NO_HIDE_STALE
:
834 error
= truncate_bdev_range(bdev
, file_to_blk_mode(file
), start
, end
);
838 error
= blkdev_issue_discard(bdev
, start
>> SECTOR_SHIFT
,
839 len
>> SECTOR_SHIFT
, GFP_KERNEL
);
846 filemap_invalidate_unlock(inode
->i_mapping
);
850 static int blkdev_mmap(struct file
*file
, struct vm_area_struct
*vma
)
852 struct inode
*bd_inode
= bdev_file_inode(file
);
854 if (bdev_read_only(I_BDEV(bd_inode
)))
855 return generic_file_readonly_mmap(file
, vma
);
857 return generic_file_mmap(file
, vma
);
860 const struct file_operations def_blk_fops
= {
862 .release
= blkdev_release
,
863 .llseek
= blkdev_llseek
,
864 .read_iter
= blkdev_read_iter
,
865 .write_iter
= blkdev_write_iter
,
866 .iopoll
= iocb_bio_iopoll
,
868 .fsync
= blkdev_fsync
,
869 .unlocked_ioctl
= blkdev_ioctl
,
871 .compat_ioctl
= compat_blkdev_ioctl
,
873 .splice_read
= filemap_splice_read
,
874 .splice_write
= iter_file_splice_write
,
875 .fallocate
= blkdev_fallocate
,
876 .fop_flags
= FOP_BUFFER_RASYNC
,
879 static __init
int blkdev_init(void)
881 return bioset_init(&blkdev_dio_pool
, 4,
882 offsetof(struct blkdev_dio
, bio
),
883 BIOSET_NEED_BVECS
|BIOSET_PERCPU_CACHE
);
885 module_init(blkdev_init
);