1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 1991, 1992 Linus Torvalds
4 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
5 * Copyright (C) 2016 - 2020 Christoph Hellwig
8 #include <linux/init.h>
10 #include <linux/slab.h>
11 #include <linux/kmod.h>
12 #include <linux/major.h>
13 #include <linux/device_cgroup.h>
14 #include <linux/blkdev.h>
15 #include <linux/blk-integrity.h>
16 #include <linux/backing-dev.h>
17 #include <linux/module.h>
18 #include <linux/blkpg.h>
19 #include <linux/magic.h>
20 #include <linux/buffer_head.h>
21 #include <linux/swap.h>
22 #include <linux/writeback.h>
23 #include <linux/mount.h>
24 #include <linux/pseudo_fs.h>
25 #include <linux/uio.h>
26 #include <linux/namei.h>
27 #include <linux/part_stat.h>
28 #include <linux/uaccess.h>
29 #include <linux/stat.h>
30 #include "../fs/internal.h"
33 /* Should we allow writing to mounted block devices? */
34 static bool bdev_allow_write_mounted
= IS_ENABLED(CONFIG_BLK_DEV_WRITE_MOUNTED
);
37 struct block_device bdev
;
38 struct inode vfs_inode
;
41 static inline struct bdev_inode
*BDEV_I(struct inode
*inode
)
43 return container_of(inode
, struct bdev_inode
, vfs_inode
);
46 static inline struct inode
*BD_INODE(struct block_device
*bdev
)
48 return &container_of(bdev
, struct bdev_inode
, bdev
)->vfs_inode
;
51 struct block_device
*I_BDEV(struct inode
*inode
)
53 return &BDEV_I(inode
)->bdev
;
55 EXPORT_SYMBOL(I_BDEV
);
57 struct block_device
*file_bdev(struct file
*bdev_file
)
59 return I_BDEV(bdev_file
->f_mapping
->host
);
61 EXPORT_SYMBOL(file_bdev
);
63 static void bdev_write_inode(struct block_device
*bdev
)
65 struct inode
*inode
= BD_INODE(bdev
);
68 spin_lock(&inode
->i_lock
);
69 while (inode
->i_state
& I_DIRTY
) {
70 spin_unlock(&inode
->i_lock
);
71 ret
= write_inode_now(inode
, true);
74 "VFS: Dirty inode writeback failed for block device %pg (err=%d).\n",
76 spin_lock(&inode
->i_lock
);
78 spin_unlock(&inode
->i_lock
);
81 /* Kill _all_ buffers and pagecache , dirty or not.. */
82 static void kill_bdev(struct block_device
*bdev
)
84 struct address_space
*mapping
= bdev
->bd_mapping
;
86 if (mapping_empty(mapping
))
90 truncate_inode_pages(mapping
, 0);
93 /* Invalidate clean unused buffers and pagecache. */
94 void invalidate_bdev(struct block_device
*bdev
)
96 struct address_space
*mapping
= bdev
->bd_mapping
;
98 if (mapping
->nrpages
) {
100 lru_add_drain_all(); /* make sure all lru add caches are flushed */
101 invalidate_mapping_pages(mapping
, 0, -1);
104 EXPORT_SYMBOL(invalidate_bdev
);
107 * Drop all buffers & page cache for given bdev range. This function bails
108 * with error if bdev has other exclusive owner (such as filesystem).
110 int truncate_bdev_range(struct block_device
*bdev
, blk_mode_t mode
,
111 loff_t lstart
, loff_t lend
)
114 * If we don't hold exclusive handle for the device, upgrade to it
115 * while we discard the buffer cache to avoid discarding buffers
116 * under live filesystem.
118 if (!(mode
& BLK_OPEN_EXCL
)) {
119 int err
= bd_prepare_to_claim(bdev
, truncate_bdev_range
, NULL
);
124 truncate_inode_pages_range(bdev
->bd_mapping
, lstart
, lend
);
125 if (!(mode
& BLK_OPEN_EXCL
))
126 bd_abort_claiming(bdev
, truncate_bdev_range
);
131 * Someone else has handle exclusively open. Try invalidating instead.
132 * The 'end' argument is inclusive so the rounding is safe.
134 return invalidate_inode_pages2_range(bdev
->bd_mapping
,
135 lstart
>> PAGE_SHIFT
,
139 static void set_init_blocksize(struct block_device
*bdev
)
141 unsigned int bsize
= bdev_logical_block_size(bdev
);
142 loff_t size
= i_size_read(BD_INODE(bdev
));
144 while (bsize
< PAGE_SIZE
) {
149 BD_INODE(bdev
)->i_blkbits
= blksize_bits(bsize
);
152 int set_blocksize(struct file
*file
, int size
)
154 struct inode
*inode
= file
->f_mapping
->host
;
155 struct block_device
*bdev
= I_BDEV(inode
);
157 /* Size must be a power of two, and between 512 and PAGE_SIZE */
158 if (size
> PAGE_SIZE
|| size
< 512 || !is_power_of_2(size
))
161 /* Size cannot be smaller than the size supported by the device */
162 if (size
< bdev_logical_block_size(bdev
))
165 if (!file
->private_data
)
168 /* Don't change the size if it is same as current */
169 if (inode
->i_blkbits
!= blksize_bits(size
)) {
171 inode
->i_blkbits
= blksize_bits(size
);
177 EXPORT_SYMBOL(set_blocksize
);
179 int sb_set_blocksize(struct super_block
*sb
, int size
)
181 if (set_blocksize(sb
->s_bdev_file
, size
))
183 /* If we get here, we know size is power of two
184 * and it's value is between 512 and PAGE_SIZE */
185 sb
->s_blocksize
= size
;
186 sb
->s_blocksize_bits
= blksize_bits(size
);
187 return sb
->s_blocksize
;
190 EXPORT_SYMBOL(sb_set_blocksize
);
192 int sb_min_blocksize(struct super_block
*sb
, int size
)
194 int minsize
= bdev_logical_block_size(sb
->s_bdev
);
197 return sb_set_blocksize(sb
, size
);
200 EXPORT_SYMBOL(sb_min_blocksize
);
202 int sync_blockdev_nowait(struct block_device
*bdev
)
206 return filemap_flush(bdev
->bd_mapping
);
208 EXPORT_SYMBOL_GPL(sync_blockdev_nowait
);
211 * Write out and wait upon all the dirty data associated with a block
212 * device via its mapping. Does not take the superblock lock.
214 int sync_blockdev(struct block_device
*bdev
)
218 return filemap_write_and_wait(bdev
->bd_mapping
);
220 EXPORT_SYMBOL(sync_blockdev
);
222 int sync_blockdev_range(struct block_device
*bdev
, loff_t lstart
, loff_t lend
)
224 return filemap_write_and_wait_range(bdev
->bd_mapping
,
227 EXPORT_SYMBOL(sync_blockdev_range
);
230 * bdev_freeze - lock a filesystem and force it into a consistent state
231 * @bdev: blockdevice to lock
233 * If a superblock is found on this device, we take the s_umount semaphore
234 * on it to make sure nobody unmounts until the snapshot creation is done.
235 * The reference counter (bd_fsfreeze_count) guarantees that only the last
236 * unfreeze process can unfreeze the frozen filesystem actually when multiple
237 * freeze requests arrive simultaneously. It counts up in bdev_freeze() and
238 * count down in bdev_thaw(). When it becomes 0, thaw_bdev() will unfreeze
241 * Return: On success zero is returned, negative error code on failure.
243 int bdev_freeze(struct block_device
*bdev
)
247 mutex_lock(&bdev
->bd_fsfreeze_mutex
);
249 if (atomic_inc_return(&bdev
->bd_fsfreeze_count
) > 1) {
250 mutex_unlock(&bdev
->bd_fsfreeze_mutex
);
254 mutex_lock(&bdev
->bd_holder_lock
);
255 if (bdev
->bd_holder_ops
&& bdev
->bd_holder_ops
->freeze
) {
256 error
= bdev
->bd_holder_ops
->freeze(bdev
);
257 lockdep_assert_not_held(&bdev
->bd_holder_lock
);
259 mutex_unlock(&bdev
->bd_holder_lock
);
260 error
= sync_blockdev(bdev
);
264 atomic_dec(&bdev
->bd_fsfreeze_count
);
266 mutex_unlock(&bdev
->bd_fsfreeze_mutex
);
269 EXPORT_SYMBOL(bdev_freeze
);
272 * bdev_thaw - unlock filesystem
273 * @bdev: blockdevice to unlock
275 * Unlocks the filesystem and marks it writeable again after bdev_freeze().
277 * Return: On success zero is returned, negative error code on failure.
279 int bdev_thaw(struct block_device
*bdev
)
281 int error
= -EINVAL
, nr_freeze
;
283 mutex_lock(&bdev
->bd_fsfreeze_mutex
);
286 * If this returns < 0 it means that @bd_fsfreeze_count was
287 * already 0 and no decrement was performed.
289 nr_freeze
= atomic_dec_if_positive(&bdev
->bd_fsfreeze_count
);
297 mutex_lock(&bdev
->bd_holder_lock
);
298 if (bdev
->bd_holder_ops
&& bdev
->bd_holder_ops
->thaw
) {
299 error
= bdev
->bd_holder_ops
->thaw(bdev
);
300 lockdep_assert_not_held(&bdev
->bd_holder_lock
);
302 mutex_unlock(&bdev
->bd_holder_lock
);
306 atomic_inc(&bdev
->bd_fsfreeze_count
);
308 mutex_unlock(&bdev
->bd_fsfreeze_mutex
);
311 EXPORT_SYMBOL(bdev_thaw
);
317 static __cacheline_aligned_in_smp
DEFINE_MUTEX(bdev_lock
);
318 static struct kmem_cache
*bdev_cachep __ro_after_init
;
320 static struct inode
*bdev_alloc_inode(struct super_block
*sb
)
322 struct bdev_inode
*ei
= alloc_inode_sb(sb
, bdev_cachep
, GFP_KERNEL
);
326 memset(&ei
->bdev
, 0, sizeof(ei
->bdev
));
327 return &ei
->vfs_inode
;
330 static void bdev_free_inode(struct inode
*inode
)
332 struct block_device
*bdev
= I_BDEV(inode
);
334 free_percpu(bdev
->bd_stats
);
335 kfree(bdev
->bd_meta_info
);
337 if (!bdev_is_partition(bdev
)) {
338 if (bdev
->bd_disk
&& bdev
->bd_disk
->bdi
)
339 bdi_put(bdev
->bd_disk
->bdi
);
340 kfree(bdev
->bd_disk
);
343 if (MAJOR(bdev
->bd_dev
) == BLOCK_EXT_MAJOR
)
344 blk_free_ext_minor(MINOR(bdev
->bd_dev
));
346 kmem_cache_free(bdev_cachep
, BDEV_I(inode
));
349 static void init_once(void *data
)
351 struct bdev_inode
*ei
= data
;
353 inode_init_once(&ei
->vfs_inode
);
356 static void bdev_evict_inode(struct inode
*inode
)
358 truncate_inode_pages_final(&inode
->i_data
);
359 invalidate_inode_buffers(inode
); /* is it needed here? */
363 static const struct super_operations bdev_sops
= {
364 .statfs
= simple_statfs
,
365 .alloc_inode
= bdev_alloc_inode
,
366 .free_inode
= bdev_free_inode
,
367 .drop_inode
= generic_delete_inode
,
368 .evict_inode
= bdev_evict_inode
,
371 static int bd_init_fs_context(struct fs_context
*fc
)
373 struct pseudo_fs_context
*ctx
= init_pseudo(fc
, BDEVFS_MAGIC
);
376 fc
->s_iflags
|= SB_I_CGROUPWB
;
377 ctx
->ops
= &bdev_sops
;
381 static struct file_system_type bd_type
= {
383 .init_fs_context
= bd_init_fs_context
,
384 .kill_sb
= kill_anon_super
,
387 struct super_block
*blockdev_superblock __ro_after_init
;
388 static struct vfsmount
*blockdev_mnt __ro_after_init
;
389 EXPORT_SYMBOL_GPL(blockdev_superblock
);
391 void __init
bdev_cache_init(void)
395 bdev_cachep
= kmem_cache_create("bdev_cache", sizeof(struct bdev_inode
),
396 0, (SLAB_HWCACHE_ALIGN
|SLAB_RECLAIM_ACCOUNT
|
397 SLAB_ACCOUNT
|SLAB_PANIC
),
399 err
= register_filesystem(&bd_type
);
401 panic("Cannot register bdev pseudo-fs");
402 blockdev_mnt
= kern_mount(&bd_type
);
403 if (IS_ERR(blockdev_mnt
))
404 panic("Cannot create bdev pseudo-fs");
405 blockdev_superblock
= blockdev_mnt
->mnt_sb
; /* For writeback */
408 struct block_device
*bdev_alloc(struct gendisk
*disk
, u8 partno
)
410 struct block_device
*bdev
;
413 inode
= new_inode(blockdev_superblock
);
416 inode
->i_mode
= S_IFBLK
;
418 inode
->i_data
.a_ops
= &def_blk_aops
;
419 mapping_set_gfp_mask(&inode
->i_data
, GFP_USER
);
421 bdev
= I_BDEV(inode
);
422 mutex_init(&bdev
->bd_fsfreeze_mutex
);
423 spin_lock_init(&bdev
->bd_size_lock
);
424 mutex_init(&bdev
->bd_holder_lock
);
425 atomic_set(&bdev
->__bd_flags
, partno
);
426 bdev
->bd_mapping
= &inode
->i_data
;
427 bdev
->bd_queue
= disk
->queue
;
428 if (partno
&& bdev_test_flag(disk
->part0
, BD_HAS_SUBMIT_BIO
))
429 bdev_set_flag(bdev
, BD_HAS_SUBMIT_BIO
);
430 bdev
->bd_stats
= alloc_percpu(struct disk_stats
);
431 if (!bdev
->bd_stats
) {
435 bdev
->bd_disk
= disk
;
439 void bdev_set_nr_sectors(struct block_device
*bdev
, sector_t sectors
)
441 spin_lock(&bdev
->bd_size_lock
);
442 i_size_write(BD_INODE(bdev
), (loff_t
)sectors
<< SECTOR_SHIFT
);
443 bdev
->bd_nr_sectors
= sectors
;
444 spin_unlock(&bdev
->bd_size_lock
);
447 void bdev_add(struct block_device
*bdev
, dev_t dev
)
449 struct inode
*inode
= BD_INODE(bdev
);
450 if (bdev_stable_writes(bdev
))
451 mapping_set_stable_writes(bdev
->bd_mapping
);
455 insert_inode_hash(inode
);
458 void bdev_unhash(struct block_device
*bdev
)
460 remove_inode_hash(BD_INODE(bdev
));
463 void bdev_drop(struct block_device
*bdev
)
465 iput(BD_INODE(bdev
));
468 long nr_blockdev_pages(void)
473 spin_lock(&blockdev_superblock
->s_inode_list_lock
);
474 list_for_each_entry(inode
, &blockdev_superblock
->s_inodes
, i_sb_list
)
475 ret
+= inode
->i_mapping
->nrpages
;
476 spin_unlock(&blockdev_superblock
->s_inode_list_lock
);
482 * bd_may_claim - test whether a block device can be claimed
483 * @bdev: block device of interest
484 * @holder: holder trying to claim @bdev
487 * Test whether @bdev can be claimed by @holder.
490 * %true if @bdev can be claimed, %false otherwise.
492 static bool bd_may_claim(struct block_device
*bdev
, void *holder
,
493 const struct blk_holder_ops
*hops
)
495 struct block_device
*whole
= bdev_whole(bdev
);
497 lockdep_assert_held(&bdev_lock
);
499 if (bdev
->bd_holder
) {
501 * The same holder can always re-claim.
503 if (bdev
->bd_holder
== holder
) {
504 if (WARN_ON_ONCE(bdev
->bd_holder_ops
!= hops
))
512 * If the whole devices holder is set to bd_may_claim, a partition on
513 * the device is claimed, but not the whole device.
516 whole
->bd_holder
&& whole
->bd_holder
!= bd_may_claim
)
522 * bd_prepare_to_claim - claim a block device
523 * @bdev: block device of interest
524 * @holder: holder trying to claim @bdev
527 * Claim @bdev. This function fails if @bdev is already claimed by another
528 * holder and waits if another claiming is in progress. return, the caller
529 * has ownership of bd_claiming and bd_holder[s].
532 * 0 if @bdev can be claimed, -EBUSY otherwise.
534 int bd_prepare_to_claim(struct block_device
*bdev
, void *holder
,
535 const struct blk_holder_ops
*hops
)
537 struct block_device
*whole
= bdev_whole(bdev
);
539 if (WARN_ON_ONCE(!holder
))
542 mutex_lock(&bdev_lock
);
543 /* if someone else claimed, fail */
544 if (!bd_may_claim(bdev
, holder
, hops
)) {
545 mutex_unlock(&bdev_lock
);
549 /* if claiming is already in progress, wait for it to finish */
550 if (whole
->bd_claiming
) {
551 wait_queue_head_t
*wq
= bit_waitqueue(&whole
->bd_claiming
, 0);
554 prepare_to_wait(wq
, &wait
, TASK_UNINTERRUPTIBLE
);
555 mutex_unlock(&bdev_lock
);
557 finish_wait(wq
, &wait
);
562 whole
->bd_claiming
= holder
;
563 mutex_unlock(&bdev_lock
);
566 EXPORT_SYMBOL_GPL(bd_prepare_to_claim
); /* only for the loop driver */
568 static void bd_clear_claiming(struct block_device
*whole
, void *holder
)
570 lockdep_assert_held(&bdev_lock
);
571 /* tell others that we're done */
572 BUG_ON(whole
->bd_claiming
!= holder
);
573 whole
->bd_claiming
= NULL
;
574 wake_up_bit(&whole
->bd_claiming
, 0);
578 * bd_finish_claiming - finish claiming of a block device
579 * @bdev: block device of interest
580 * @holder: holder that has claimed @bdev
581 * @hops: block device holder operations
583 * Finish exclusive open of a block device. Mark the device as exlusively
584 * open by the holder and wake up all waiters for exclusive open to finish.
586 static void bd_finish_claiming(struct block_device
*bdev
, void *holder
,
587 const struct blk_holder_ops
*hops
)
589 struct block_device
*whole
= bdev_whole(bdev
);
591 mutex_lock(&bdev_lock
);
592 BUG_ON(!bd_may_claim(bdev
, holder
, hops
));
594 * Note that for a whole device bd_holders will be incremented twice,
595 * and bd_holder will be set to bd_may_claim before being set to holder
598 whole
->bd_holder
= bd_may_claim
;
600 mutex_lock(&bdev
->bd_holder_lock
);
601 bdev
->bd_holder
= holder
;
602 bdev
->bd_holder_ops
= hops
;
603 mutex_unlock(&bdev
->bd_holder_lock
);
604 bd_clear_claiming(whole
, holder
);
605 mutex_unlock(&bdev_lock
);
609 * bd_abort_claiming - abort claiming of a block device
610 * @bdev: block device of interest
611 * @holder: holder that has claimed @bdev
613 * Abort claiming of a block device when the exclusive open failed. This can be
614 * also used when exclusive open is not actually desired and we just needed
615 * to block other exclusive openers for a while.
617 void bd_abort_claiming(struct block_device
*bdev
, void *holder
)
619 mutex_lock(&bdev_lock
);
620 bd_clear_claiming(bdev_whole(bdev
), holder
);
621 mutex_unlock(&bdev_lock
);
623 EXPORT_SYMBOL(bd_abort_claiming
);
625 static void bd_end_claim(struct block_device
*bdev
, void *holder
)
627 struct block_device
*whole
= bdev_whole(bdev
);
628 bool unblock
= false;
631 * Release a claim on the device. The holder fields are protected with
632 * bdev_lock. open_mutex is used to synchronize disk_holder unlinking.
634 mutex_lock(&bdev_lock
);
635 WARN_ON_ONCE(bdev
->bd_holder
!= holder
);
636 WARN_ON_ONCE(--bdev
->bd_holders
< 0);
637 WARN_ON_ONCE(--whole
->bd_holders
< 0);
638 if (!bdev
->bd_holders
) {
639 mutex_lock(&bdev
->bd_holder_lock
);
640 bdev
->bd_holder
= NULL
;
641 bdev
->bd_holder_ops
= NULL
;
642 mutex_unlock(&bdev
->bd_holder_lock
);
643 if (bdev_test_flag(bdev
, BD_WRITE_HOLDER
))
646 if (!whole
->bd_holders
)
647 whole
->bd_holder
= NULL
;
648 mutex_unlock(&bdev_lock
);
651 * If this was the last claim, remove holder link and unblock evpoll if
652 * it was a write holder.
655 disk_unblock_events(bdev
->bd_disk
);
656 bdev_clear_flag(bdev
, BD_WRITE_HOLDER
);
660 static void blkdev_flush_mapping(struct block_device
*bdev
)
662 WARN_ON_ONCE(bdev
->bd_holders
);
665 bdev_write_inode(bdev
);
668 static void blkdev_put_whole(struct block_device
*bdev
)
670 if (atomic_dec_and_test(&bdev
->bd_openers
))
671 blkdev_flush_mapping(bdev
);
672 if (bdev
->bd_disk
->fops
->release
)
673 bdev
->bd_disk
->fops
->release(bdev
->bd_disk
);
676 static int blkdev_get_whole(struct block_device
*bdev
, blk_mode_t mode
)
678 struct gendisk
*disk
= bdev
->bd_disk
;
681 if (disk
->fops
->open
) {
682 ret
= disk
->fops
->open(disk
, mode
);
684 /* avoid ghost partitions on a removed medium */
685 if (ret
== -ENOMEDIUM
&&
686 test_bit(GD_NEED_PART_SCAN
, &disk
->state
))
687 bdev_disk_changed(disk
, true);
692 if (!atomic_read(&bdev
->bd_openers
))
693 set_init_blocksize(bdev
);
694 atomic_inc(&bdev
->bd_openers
);
695 if (test_bit(GD_NEED_PART_SCAN
, &disk
->state
)) {
697 * Only return scanning errors if we are called from contexts
698 * that explicitly want them, e.g. the BLKRRPART ioctl.
700 ret
= bdev_disk_changed(disk
, false);
701 if (ret
&& (mode
& BLK_OPEN_STRICT_SCAN
)) {
702 blkdev_put_whole(bdev
);
709 static int blkdev_get_part(struct block_device
*part
, blk_mode_t mode
)
711 struct gendisk
*disk
= part
->bd_disk
;
714 ret
= blkdev_get_whole(bdev_whole(part
), mode
);
719 if (!bdev_nr_sectors(part
))
722 if (!atomic_read(&part
->bd_openers
)) {
723 disk
->open_partitions
++;
724 set_init_blocksize(part
);
726 atomic_inc(&part
->bd_openers
);
730 blkdev_put_whole(bdev_whole(part
));
734 int bdev_permission(dev_t dev
, blk_mode_t mode
, void *holder
)
738 ret
= devcgroup_check_permission(DEVCG_DEV_BLOCK
,
739 MAJOR(dev
), MINOR(dev
),
740 ((mode
& BLK_OPEN_READ
) ? DEVCG_ACC_READ
: 0) |
741 ((mode
& BLK_OPEN_WRITE
) ? DEVCG_ACC_WRITE
: 0));
745 /* Blocking writes requires exclusive opener */
746 if (mode
& BLK_OPEN_RESTRICT_WRITES
&& !holder
)
750 * We're using error pointers to indicate to ->release() when we
751 * failed to open that block device. Also this doesn't make sense.
753 if (WARN_ON_ONCE(IS_ERR(holder
)))
759 static void blkdev_put_part(struct block_device
*part
)
761 struct block_device
*whole
= bdev_whole(part
);
763 if (atomic_dec_and_test(&part
->bd_openers
)) {
764 blkdev_flush_mapping(part
);
765 whole
->bd_disk
->open_partitions
--;
767 blkdev_put_whole(whole
);
770 struct block_device
*blkdev_get_no_open(dev_t dev
)
772 struct block_device
*bdev
;
775 inode
= ilookup(blockdev_superblock
, dev
);
776 if (!inode
&& IS_ENABLED(CONFIG_BLOCK_LEGACY_AUTOLOAD
)) {
777 blk_request_module(dev
);
778 inode
= ilookup(blockdev_superblock
, dev
);
781 "block device autoloading is deprecated and will be removed.\n");
786 /* switch from the inode reference to a device mode one: */
787 bdev
= &BDEV_I(inode
)->bdev
;
788 if (!kobject_get_unless_zero(&bdev
->bd_device
.kobj
))
794 void blkdev_put_no_open(struct block_device
*bdev
)
796 put_device(&bdev
->bd_device
);
799 static bool bdev_writes_blocked(struct block_device
*bdev
)
801 return bdev
->bd_writers
< 0;
804 static void bdev_block_writes(struct block_device
*bdev
)
809 static void bdev_unblock_writes(struct block_device
*bdev
)
814 static bool bdev_may_open(struct block_device
*bdev
, blk_mode_t mode
)
816 if (bdev_allow_write_mounted
)
818 /* Writes blocked? */
819 if (mode
& BLK_OPEN_WRITE
&& bdev_writes_blocked(bdev
))
821 if (mode
& BLK_OPEN_RESTRICT_WRITES
&& bdev
->bd_writers
> 0)
826 static void bdev_claim_write_access(struct block_device
*bdev
, blk_mode_t mode
)
828 if (bdev_allow_write_mounted
)
831 /* Claim exclusive or shared write access. */
832 if (mode
& BLK_OPEN_RESTRICT_WRITES
)
833 bdev_block_writes(bdev
);
834 else if (mode
& BLK_OPEN_WRITE
)
838 static inline bool bdev_unclaimed(const struct file
*bdev_file
)
840 return bdev_file
->private_data
== BDEV_I(bdev_file
->f_mapping
->host
);
843 static void bdev_yield_write_access(struct file
*bdev_file
)
845 struct block_device
*bdev
;
847 if (bdev_allow_write_mounted
)
850 if (bdev_unclaimed(bdev_file
))
853 bdev
= file_bdev(bdev_file
);
855 if (bdev_file
->f_mode
& FMODE_WRITE_RESTRICTED
)
856 bdev_unblock_writes(bdev
);
857 else if (bdev_file
->f_mode
& FMODE_WRITE
)
862 * bdev_open - open a block device
863 * @bdev: block device to open
864 * @mode: open mode (BLK_OPEN_*)
865 * @holder: exclusive holder identifier
866 * @hops: holder operations
867 * @bdev_file: file for the block device
869 * Open the block device. If @holder is not %NULL, the block device is opened
870 * with exclusive access. Exclusive opens may nest for the same @holder.
876 * zero on success, -errno on failure.
878 int bdev_open(struct block_device
*bdev
, blk_mode_t mode
, void *holder
,
879 const struct blk_holder_ops
*hops
, struct file
*bdev_file
)
881 bool unblock_events
= true;
882 struct gendisk
*disk
= bdev
->bd_disk
;
886 mode
|= BLK_OPEN_EXCL
;
887 ret
= bd_prepare_to_claim(bdev
, holder
, hops
);
891 if (WARN_ON_ONCE(mode
& BLK_OPEN_EXCL
))
895 disk_block_events(disk
);
897 mutex_lock(&disk
->open_mutex
);
899 if (!disk_live(disk
))
901 if (!try_module_get(disk
->fops
->owner
))
904 if (!bdev_may_open(bdev
, mode
))
906 if (bdev_is_partition(bdev
))
907 ret
= blkdev_get_part(bdev
, mode
);
909 ret
= blkdev_get_whole(bdev
, mode
);
912 bdev_claim_write_access(bdev
, mode
);
914 bd_finish_claiming(bdev
, holder
, hops
);
917 * Block event polling for write claims if requested. Any write
918 * holder makes the write_holder state stick until all are
919 * released. This is good enough and tracking individual
920 * writeable reference is too fragile given the way @mode is
921 * used in blkdev_get/put().
923 if ((mode
& BLK_OPEN_WRITE
) &&
924 !bdev_test_flag(bdev
, BD_WRITE_HOLDER
) &&
925 (disk
->event_flags
& DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE
)) {
926 bdev_set_flag(bdev
, BD_WRITE_HOLDER
);
927 unblock_events
= false;
930 mutex_unlock(&disk
->open_mutex
);
933 disk_unblock_events(disk
);
935 bdev_file
->f_flags
|= O_LARGEFILE
;
936 bdev_file
->f_mode
|= FMODE_CAN_ODIRECT
;
937 if (bdev_nowait(bdev
))
938 bdev_file
->f_mode
|= FMODE_NOWAIT
;
939 if (mode
& BLK_OPEN_RESTRICT_WRITES
)
940 bdev_file
->f_mode
|= FMODE_WRITE_RESTRICTED
;
941 bdev_file
->f_mapping
= bdev
->bd_mapping
;
942 bdev_file
->f_wb_err
= filemap_sample_wb_err(bdev_file
->f_mapping
);
943 bdev_file
->private_data
= holder
;
947 module_put(disk
->fops
->owner
);
950 bd_abort_claiming(bdev
, holder
);
951 mutex_unlock(&disk
->open_mutex
);
952 disk_unblock_events(disk
);
957 * If BLK_OPEN_WRITE_IOCTL is set then this is a historical quirk
958 * associated with the floppy driver where it has allowed ioctls if the
959 * file was opened for writing, but does not allow reads or writes.
960 * Make sure that this quirk is reflected in @f_flags.
962 * It can also happen if a block device is opened as O_RDWR | O_WRONLY.
964 static unsigned blk_to_file_flags(blk_mode_t mode
)
966 unsigned int flags
= 0;
968 if ((mode
& (BLK_OPEN_READ
| BLK_OPEN_WRITE
)) ==
969 (BLK_OPEN_READ
| BLK_OPEN_WRITE
))
971 else if (mode
& BLK_OPEN_WRITE_IOCTL
)
972 flags
|= O_RDWR
| O_WRONLY
;
973 else if (mode
& BLK_OPEN_WRITE
)
975 else if (mode
& BLK_OPEN_READ
)
976 flags
|= O_RDONLY
; /* homeopathic, because O_RDONLY is 0 */
980 if (mode
& BLK_OPEN_NDELAY
)
986 struct file
*bdev_file_open_by_dev(dev_t dev
, blk_mode_t mode
, void *holder
,
987 const struct blk_holder_ops
*hops
)
989 struct file
*bdev_file
;
990 struct block_device
*bdev
;
994 ret
= bdev_permission(dev
, mode
, holder
);
998 bdev
= blkdev_get_no_open(dev
);
1000 return ERR_PTR(-ENXIO
);
1002 flags
= blk_to_file_flags(mode
);
1003 bdev_file
= alloc_file_pseudo_noaccount(BD_INODE(bdev
),
1004 blockdev_mnt
, "", flags
| O_LARGEFILE
, &def_blk_fops
);
1005 if (IS_ERR(bdev_file
)) {
1006 blkdev_put_no_open(bdev
);
1009 ihold(BD_INODE(bdev
));
1011 ret
= bdev_open(bdev
, mode
, holder
, hops
, bdev_file
);
1013 /* We failed to open the block device. Let ->release() know. */
1014 bdev_file
->private_data
= ERR_PTR(ret
);
1016 return ERR_PTR(ret
);
1020 EXPORT_SYMBOL(bdev_file_open_by_dev
);
1022 struct file
*bdev_file_open_by_path(const char *path
, blk_mode_t mode
,
1024 const struct blk_holder_ops
*hops
)
1030 error
= lookup_bdev(path
, &dev
);
1032 return ERR_PTR(error
);
1034 file
= bdev_file_open_by_dev(dev
, mode
, holder
, hops
);
1035 if (!IS_ERR(file
) && (mode
& BLK_OPEN_WRITE
)) {
1036 if (bdev_read_only(file_bdev(file
))) {
1038 file
= ERR_PTR(-EACCES
);
1044 EXPORT_SYMBOL(bdev_file_open_by_path
);
1046 static inline void bd_yield_claim(struct file
*bdev_file
)
1048 struct block_device
*bdev
= file_bdev(bdev_file
);
1049 void *holder
= bdev_file
->private_data
;
1051 lockdep_assert_held(&bdev
->bd_disk
->open_mutex
);
1053 if (WARN_ON_ONCE(IS_ERR_OR_NULL(holder
)))
1056 if (!bdev_unclaimed(bdev_file
))
1057 bd_end_claim(bdev
, holder
);
1060 void bdev_release(struct file
*bdev_file
)
1062 struct block_device
*bdev
= file_bdev(bdev_file
);
1063 void *holder
= bdev_file
->private_data
;
1064 struct gendisk
*disk
= bdev
->bd_disk
;
1066 /* We failed to open that block device. */
1071 * Sync early if it looks like we're the last one. If someone else
1072 * opens the block device between now and the decrement of bd_openers
1073 * then we did a sync that we didn't need to, but that's not the end
1074 * of the world and we want to avoid long (could be several minute)
1075 * syncs while holding the mutex.
1077 if (atomic_read(&bdev
->bd_openers
) == 1)
1078 sync_blockdev(bdev
);
1080 mutex_lock(&disk
->open_mutex
);
1081 bdev_yield_write_access(bdev_file
);
1084 bd_yield_claim(bdev_file
);
1087 * Trigger event checking and tell drivers to flush MEDIA_CHANGE
1088 * event. This is to ensure detection of media removal commanded
1089 * from userland - e.g. eject(1).
1091 disk_flush_events(disk
, DISK_EVENT_MEDIA_CHANGE
);
1093 if (bdev_is_partition(bdev
))
1094 blkdev_put_part(bdev
);
1096 blkdev_put_whole(bdev
);
1097 mutex_unlock(&disk
->open_mutex
);
1099 module_put(disk
->fops
->owner
);
1101 blkdev_put_no_open(bdev
);
1105 * bdev_fput - yield claim to the block device and put the file
1106 * @bdev_file: open block device
1108 * Yield claim on the block device and put the file. Ensure that the
1109 * block device can be reclaimed before the file is closed which is a
1110 * deferred operation.
1112 void bdev_fput(struct file
*bdev_file
)
1114 if (WARN_ON_ONCE(bdev_file
->f_op
!= &def_blk_fops
))
1117 if (bdev_file
->private_data
) {
1118 struct block_device
*bdev
= file_bdev(bdev_file
);
1119 struct gendisk
*disk
= bdev
->bd_disk
;
1121 mutex_lock(&disk
->open_mutex
);
1122 bdev_yield_write_access(bdev_file
);
1123 bd_yield_claim(bdev_file
);
1125 * Tell release we already gave up our hold on the
1126 * device and if write restrictions are available that
1127 * we already gave up write access to the device.
1129 bdev_file
->private_data
= BDEV_I(bdev_file
->f_mapping
->host
);
1130 mutex_unlock(&disk
->open_mutex
);
1135 EXPORT_SYMBOL(bdev_fput
);
1138 * lookup_bdev() - Look up a struct block_device by name.
1139 * @pathname: Name of the block device in the filesystem.
1140 * @dev: Pointer to the block device's dev_t, if found.
1142 * Lookup the block device's dev_t at @pathname in the current
1143 * namespace if possible and return it in @dev.
1145 * Context: May sleep.
1146 * Return: 0 if succeeded, negative errno otherwise.
1148 int lookup_bdev(const char *pathname
, dev_t
*dev
)
1150 struct inode
*inode
;
1154 if (!pathname
|| !*pathname
)
1157 error
= kern_path(pathname
, LOOKUP_FOLLOW
, &path
);
1161 inode
= d_backing_inode(path
.dentry
);
1163 if (!S_ISBLK(inode
->i_mode
))
1166 if (!may_open_dev(&path
))
1169 *dev
= inode
->i_rdev
;
1175 EXPORT_SYMBOL(lookup_bdev
);
1178 * bdev_mark_dead - mark a block device as dead
1179 * @bdev: block device to operate on
1180 * @surprise: indicate a surprise removal
1182 * Tell the file system that this devices or media is dead. If @surprise is set
1183 * to %true the device or media is already gone, if not we are preparing for an
1186 * This calls into the file system, which then typicall syncs out all dirty data
1187 * and writes back inodes and then invalidates any cached data in the inodes on
1188 * the file system. In addition we also invalidate the block device mapping.
1190 void bdev_mark_dead(struct block_device
*bdev
, bool surprise
)
1192 mutex_lock(&bdev
->bd_holder_lock
);
1193 if (bdev
->bd_holder_ops
&& bdev
->bd_holder_ops
->mark_dead
)
1194 bdev
->bd_holder_ops
->mark_dead(bdev
, surprise
);
1196 mutex_unlock(&bdev
->bd_holder_lock
);
1197 sync_blockdev(bdev
);
1200 invalidate_bdev(bdev
);
1203 * New drivers should not use this directly. There are some drivers however
1204 * that needs this for historical reasons. For example, the DASD driver has
1205 * historically had a shutdown to offline mode that doesn't actually remove the
1206 * gendisk that otherwise looks a lot like a safe device removal.
1208 EXPORT_SYMBOL_GPL(bdev_mark_dead
);
1210 void sync_bdevs(bool wait
)
1212 struct inode
*inode
, *old_inode
= NULL
;
1214 spin_lock(&blockdev_superblock
->s_inode_list_lock
);
1215 list_for_each_entry(inode
, &blockdev_superblock
->s_inodes
, i_sb_list
) {
1216 struct address_space
*mapping
= inode
->i_mapping
;
1217 struct block_device
*bdev
;
1219 spin_lock(&inode
->i_lock
);
1220 if (inode
->i_state
& (I_FREEING
|I_WILL_FREE
|I_NEW
) ||
1221 mapping
->nrpages
== 0) {
1222 spin_unlock(&inode
->i_lock
);
1226 spin_unlock(&inode
->i_lock
);
1227 spin_unlock(&blockdev_superblock
->s_inode_list_lock
);
1229 * We hold a reference to 'inode' so it couldn't have been
1230 * removed from s_inodes list while we dropped the
1231 * s_inode_list_lock We cannot iput the inode now as we can
1232 * be holding the last reference and we cannot iput it under
1233 * s_inode_list_lock. So we keep the reference and iput it
1238 bdev
= I_BDEV(inode
);
1240 mutex_lock(&bdev
->bd_disk
->open_mutex
);
1241 if (!atomic_read(&bdev
->bd_openers
)) {
1245 * We keep the error status of individual mapping so
1246 * that applications can catch the writeback error using
1247 * fsync(2). See filemap_fdatawait_keep_errors() for
1250 filemap_fdatawait_keep_errors(inode
->i_mapping
);
1252 filemap_fdatawrite(inode
->i_mapping
);
1254 mutex_unlock(&bdev
->bd_disk
->open_mutex
);
1256 spin_lock(&blockdev_superblock
->s_inode_list_lock
);
1258 spin_unlock(&blockdev_superblock
->s_inode_list_lock
);
1263 * Handle STATX_{DIOALIGN, WRITE_ATOMIC} for block devices.
1265 void bdev_statx(struct path
*path
, struct kstat
*stat
,
1268 struct inode
*backing_inode
;
1269 struct block_device
*bdev
;
1271 if (!(request_mask
& (STATX_DIOALIGN
| STATX_WRITE_ATOMIC
)))
1274 backing_inode
= d_backing_inode(path
->dentry
);
1277 * Note that backing_inode is the inode of a block device node file,
1278 * not the block device's internal inode. Therefore it is *not* valid
1279 * to use I_BDEV() here; the block device has to be looked up by i_rdev
1282 bdev
= blkdev_get_no_open(backing_inode
->i_rdev
);
1286 if (request_mask
& STATX_DIOALIGN
) {
1287 stat
->dio_mem_align
= bdev_dma_alignment(bdev
) + 1;
1288 stat
->dio_offset_align
= bdev_logical_block_size(bdev
);
1289 stat
->result_mask
|= STATX_DIOALIGN
;
1292 if (request_mask
& STATX_WRITE_ATOMIC
&& bdev_can_atomic_write(bdev
)) {
1293 struct request_queue
*bd_queue
= bdev
->bd_queue
;
1295 generic_fill_statx_atomic_writes(stat
,
1296 queue_atomic_write_unit_min_bytes(bd_queue
),
1297 queue_atomic_write_unit_max_bytes(bd_queue
));
1300 blkdev_put_no_open(bdev
);
1303 bool disk_live(struct gendisk
*disk
)
1305 return !inode_unhashed(BD_INODE(disk
->part0
));
1307 EXPORT_SYMBOL_GPL(disk_live
);
1309 unsigned int block_size(struct block_device
*bdev
)
1311 return 1 << BD_INODE(bdev
)->i_blkbits
;
1313 EXPORT_SYMBOL_GPL(block_size
);
1315 static int __init
setup_bdev_allow_write_mounted(char *str
)
1317 if (kstrtobool(str
, &bdev_allow_write_mounted
))
1318 pr_warn("Invalid option string for bdev_allow_write_mounted:"
1322 __setup("bdev_allow_write_mounted=", setup_bdev_allow_write_mounted
);