1 // SPDX-License-Identifier: GPL-2.0
5 * Portions Copyright (C) 2020 Christoph Hellwig
8 #include <linux/module.h>
9 #include <linux/ctype.h>
11 #include <linux/kdev_t.h>
12 #include <linux/kernel.h>
13 #include <linux/blkdev.h>
14 #include <linux/backing-dev.h>
15 #include <linux/init.h>
16 #include <linux/spinlock.h>
17 #include <linux/proc_fs.h>
18 #include <linux/seq_file.h>
19 #include <linux/slab.h>
20 #include <linux/kmod.h>
21 #include <linux/major.h>
22 #include <linux/mutex.h>
23 #include <linux/idr.h>
24 #include <linux/log2.h>
25 #include <linux/pm_runtime.h>
26 #include <linux/badblocks.h>
27 #include <linux/part_stat.h>
28 #include <linux/blktrace_api.h>
30 #include "blk-throttle.h"
32 #include "blk-mq-sched.h"
33 #include "blk-rq-qos.h"
34 #include "blk-cgroup.h"
36 static struct kobject
*block_depr
;
39 * Unique, monotonically increasing sequential number associated with block
40 * devices instances (i.e. incremented each time a device is attached).
41 * Associating uevents with block devices in userspace is difficult and racy:
42 * the uevent netlink socket is lossy, and on slow and overloaded systems has
43 * a very high latency.
44 * Block devices do not have exclusive owners in userspace, any process can set
45 * one up (e.g. loop devices). Moreover, device names can be reused (e.g. loop0
46 * can be reused again and again).
47 * A userspace process setting up a block device and watching for its events
48 * cannot thus reliably tell whether an event relates to the device it just set
49 * up or another earlier instance with the same name.
50 * This sequential number allows userspace processes to solve this problem, and
51 * uniquely associate an uevent to the lifetime to a device.
53 static atomic64_t diskseq
;
55 /* for extended dynamic devt allocation, currently only one major is used */
56 #define NR_EXT_DEVT (1 << MINORBITS)
57 static DEFINE_IDA(ext_devt_ida
);
59 void set_capacity(struct gendisk
*disk
, sector_t sectors
)
61 bdev_set_nr_sectors(disk
->part0
, sectors
);
63 EXPORT_SYMBOL(set_capacity
);
66 * Set disk capacity and notify if the size is not currently zero and will not
67 * be set to zero. Returns true if a uevent was sent, otherwise false.
69 bool set_capacity_and_notify(struct gendisk
*disk
, sector_t size
)
71 sector_t capacity
= get_capacity(disk
);
72 char *envp
[] = { "RESIZE=1", NULL
};
74 set_capacity(disk
, size
);
77 * Only print a message and send a uevent if the gendisk is user visible
78 * and alive. This avoids spamming the log and udev when setting the
79 * initial capacity during probing.
81 if (size
== capacity
||
83 (disk
->flags
& GENHD_FL_HIDDEN
))
86 pr_info("%s: detected capacity change from %lld to %lld\n",
87 disk
->disk_name
, capacity
, size
);
90 * Historically we did not send a uevent for changes to/from an empty
93 if (!capacity
|| !size
)
95 kobject_uevent_env(&disk_to_dev(disk
)->kobj
, KOBJ_CHANGE
, envp
);
98 EXPORT_SYMBOL_GPL(set_capacity_and_notify
);
100 static void part_stat_read_all(struct block_device
*part
,
101 struct disk_stats
*stat
)
105 memset(stat
, 0, sizeof(struct disk_stats
));
106 for_each_possible_cpu(cpu
) {
107 struct disk_stats
*ptr
= per_cpu_ptr(part
->bd_stats
, cpu
);
110 for (group
= 0; group
< NR_STAT_GROUPS
; group
++) {
111 stat
->nsecs
[group
] += ptr
->nsecs
[group
];
112 stat
->sectors
[group
] += ptr
->sectors
[group
];
113 stat
->ios
[group
] += ptr
->ios
[group
];
114 stat
->merges
[group
] += ptr
->merges
[group
];
117 stat
->io_ticks
+= ptr
->io_ticks
;
121 unsigned int part_in_flight(struct block_device
*part
)
123 unsigned int inflight
= 0;
126 for_each_possible_cpu(cpu
) {
127 inflight
+= part_stat_local_read_cpu(part
, in_flight
[0], cpu
) +
128 part_stat_local_read_cpu(part
, in_flight
[1], cpu
);
130 if ((int)inflight
< 0)
136 static void part_in_flight_rw(struct block_device
*part
,
137 unsigned int inflight
[2])
143 for_each_possible_cpu(cpu
) {
144 inflight
[0] += part_stat_local_read_cpu(part
, in_flight
[0], cpu
);
145 inflight
[1] += part_stat_local_read_cpu(part
, in_flight
[1], cpu
);
147 if ((int)inflight
[0] < 0)
149 if ((int)inflight
[1] < 0)
154 * Can be deleted altogether. Later.
157 #define BLKDEV_MAJOR_HASH_SIZE 255
158 static struct blk_major_name
{
159 struct blk_major_name
*next
;
162 #ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
163 void (*probe
)(dev_t devt
);
165 } *major_names
[BLKDEV_MAJOR_HASH_SIZE
];
166 static DEFINE_MUTEX(major_names_lock
);
167 static DEFINE_SPINLOCK(major_names_spinlock
);
169 /* index in the above - for now: assume no multimajor ranges */
170 static inline int major_to_index(unsigned major
)
172 return major
% BLKDEV_MAJOR_HASH_SIZE
;
175 #ifdef CONFIG_PROC_FS
176 void blkdev_show(struct seq_file
*seqf
, off_t offset
)
178 struct blk_major_name
*dp
;
180 spin_lock(&major_names_spinlock
);
181 for (dp
= major_names
[major_to_index(offset
)]; dp
; dp
= dp
->next
)
182 if (dp
->major
== offset
)
183 seq_printf(seqf
, "%3d %s\n", dp
->major
, dp
->name
);
184 spin_unlock(&major_names_spinlock
);
186 #endif /* CONFIG_PROC_FS */
189 * __register_blkdev - register a new block device
191 * @major: the requested major device number [1..BLKDEV_MAJOR_MAX-1]. If
192 * @major = 0, try to allocate any unused major number.
193 * @name: the name of the new block device as a zero terminated string
194 * @probe: pre-devtmpfs / pre-udev callback used to create disks when their
195 * pre-created device node is accessed. When a probe call uses
196 * add_disk() and it fails the driver must cleanup resources. This
197 * interface may soon be removed.
199 * The @name must be unique within the system.
201 * The return value depends on the @major input parameter:
203 * - if a major device number was requested in range [1..BLKDEV_MAJOR_MAX-1]
204 * then the function returns zero on success, or a negative error code
205 * - if any unused major number was requested with @major = 0 parameter
206 * then the return value is the allocated major number in range
207 * [1..BLKDEV_MAJOR_MAX-1] or a negative error code otherwise
209 * See Documentation/admin-guide/devices.txt for the list of allocated
212 * Use register_blkdev instead for any new code.
214 int __register_blkdev(unsigned int major
, const char *name
,
215 void (*probe
)(dev_t devt
))
217 struct blk_major_name
**n
, *p
;
220 mutex_lock(&major_names_lock
);
224 for (index
= ARRAY_SIZE(major_names
)-1; index
> 0; index
--) {
225 if (major_names
[index
] == NULL
)
230 printk("%s: failed to get major for %s\n",
239 if (major
>= BLKDEV_MAJOR_MAX
) {
240 pr_err("%s: major requested (%u) is greater than the maximum (%u) for %s\n",
241 __func__
, major
, BLKDEV_MAJOR_MAX
-1, name
);
247 p
= kmalloc(sizeof(struct blk_major_name
), GFP_KERNEL
);
254 #ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
257 strscpy(p
->name
, name
, sizeof(p
->name
));
259 index
= major_to_index(major
);
261 spin_lock(&major_names_spinlock
);
262 for (n
= &major_names
[index
]; *n
; n
= &(*n
)->next
) {
263 if ((*n
)->major
== major
)
270 spin_unlock(&major_names_spinlock
);
273 printk("register_blkdev: cannot get major %u for %s\n",
278 mutex_unlock(&major_names_lock
);
281 EXPORT_SYMBOL(__register_blkdev
);
283 void unregister_blkdev(unsigned int major
, const char *name
)
285 struct blk_major_name
**n
;
286 struct blk_major_name
*p
= NULL
;
287 int index
= major_to_index(major
);
289 mutex_lock(&major_names_lock
);
290 spin_lock(&major_names_spinlock
);
291 for (n
= &major_names
[index
]; *n
; n
= &(*n
)->next
)
292 if ((*n
)->major
== major
)
294 if (!*n
|| strcmp((*n
)->name
, name
)) {
300 spin_unlock(&major_names_spinlock
);
301 mutex_unlock(&major_names_lock
);
305 EXPORT_SYMBOL(unregister_blkdev
);
307 int blk_alloc_ext_minor(void)
311 idx
= ida_alloc_range(&ext_devt_ida
, 0, NR_EXT_DEVT
- 1, GFP_KERNEL
);
317 void blk_free_ext_minor(unsigned int minor
)
319 ida_free(&ext_devt_ida
, minor
);
322 void disk_uevent(struct gendisk
*disk
, enum kobject_action action
)
324 struct block_device
*part
;
328 xa_for_each(&disk
->part_tbl
, idx
, part
) {
329 if (bdev_is_partition(part
) && !bdev_nr_sectors(part
))
331 if (!kobject_get_unless_zero(&part
->bd_device
.kobj
))
335 kobject_uevent(bdev_kobj(part
), action
);
336 put_device(&part
->bd_device
);
341 EXPORT_SYMBOL_GPL(disk_uevent
);
343 int disk_scan_partitions(struct gendisk
*disk
, blk_mode_t mode
)
348 if (!disk_has_partscan(disk
))
350 if (disk
->open_partitions
)
354 * If the device is opened exclusively by current thread already, it's
355 * safe to scan partitons, otherwise, use bd_prepare_to_claim() to
356 * synchronize with other exclusive openers and other partition
359 if (!(mode
& BLK_OPEN_EXCL
)) {
360 ret
= bd_prepare_to_claim(disk
->part0
, disk_scan_partitions
,
366 set_bit(GD_NEED_PART_SCAN
, &disk
->state
);
367 file
= bdev_file_open_by_dev(disk_devt(disk
), mode
& ~BLK_OPEN_EXCL
,
375 * If blkdev_get_by_dev() failed early, GD_NEED_PART_SCAN is still set,
376 * and this will cause that re-assemble partitioned raid device will
377 * creat partition for underlying disk.
379 clear_bit(GD_NEED_PART_SCAN
, &disk
->state
);
380 if (!(mode
& BLK_OPEN_EXCL
))
381 bd_abort_claiming(disk
->part0
, disk_scan_partitions
);
386 * device_add_disk - add disk information to kernel list
387 * @parent: parent device for the disk
388 * @disk: per-device partitioning information
389 * @groups: Additional per-device sysfs groups
391 * This function registers the partitioning information in @disk
394 int __must_check
device_add_disk(struct device
*parent
, struct gendisk
*disk
,
395 const struct attribute_group
**groups
)
398 struct device
*ddev
= disk_to_dev(disk
);
401 /* Only makes sense for bio-based to set ->poll_bio */
402 if (queue_is_mq(disk
->queue
) && disk
->fops
->poll_bio
)
406 * The disk queue should now be all set with enough information about
407 * the device for the elevator code to pick an adequate default
408 * elevator if one is needed, that is, for devices requesting queue
411 elevator_init_mq(disk
->queue
);
413 /* Mark bdev as having a submit_bio, if needed */
414 if (disk
->fops
->submit_bio
)
415 bdev_set_flag(disk
->part0
, BD_HAS_SUBMIT_BIO
);
418 * If the driver provides an explicit major number it also must provide
419 * the number of minors numbers supported, and those will be used to
421 * Otherwise just allocate the device numbers for both the whole device
422 * and all partitions from the extended dev_t space.
426 if (WARN_ON(!disk
->minors
))
427 goto out_exit_elevator
;
429 if (disk
->minors
> DISK_MAX_PARTS
) {
430 pr_err("block: can't allocate more than %d partitions\n",
432 disk
->minors
= DISK_MAX_PARTS
;
434 if (disk
->first_minor
> MINORMASK
||
435 disk
->minors
> MINORMASK
+ 1 ||
436 disk
->first_minor
+ disk
->minors
> MINORMASK
+ 1)
437 goto out_exit_elevator
;
439 if (WARN_ON(disk
->minors
))
440 goto out_exit_elevator
;
442 ret
= blk_alloc_ext_minor();
444 goto out_exit_elevator
;
445 disk
->major
= BLOCK_EXT_MAJOR
;
446 disk
->first_minor
= ret
;
449 /* delay uevents, until we scanned partition table */
450 dev_set_uevent_suppress(ddev
, 1);
452 ddev
->parent
= parent
;
453 ddev
->groups
= groups
;
454 dev_set_name(ddev
, "%s", disk
->disk_name
);
455 if (!(disk
->flags
& GENHD_FL_HIDDEN
))
456 ddev
->devt
= MKDEV(disk
->major
, disk
->first_minor
);
457 ret
= device_add(ddev
);
459 goto out_free_ext_minor
;
461 ret
= disk_alloc_events(disk
);
465 ret
= sysfs_create_link(block_depr
, &ddev
->kobj
,
466 kobject_name(&ddev
->kobj
));
471 * avoid probable deadlock caused by allocating memory with
472 * GFP_KERNEL in runtime_resume callback of its all ancestor
475 pm_runtime_set_memalloc_noio(ddev
, true);
477 disk
->part0
->bd_holder_dir
=
478 kobject_create_and_add("holders", &ddev
->kobj
);
479 if (!disk
->part0
->bd_holder_dir
) {
481 goto out_del_block_link
;
483 disk
->slave_dir
= kobject_create_and_add("slaves", &ddev
->kobj
);
484 if (!disk
->slave_dir
) {
486 goto out_put_holder_dir
;
489 ret
= blk_register_queue(disk
);
491 goto out_put_slave_dir
;
493 if (!(disk
->flags
& GENHD_FL_HIDDEN
)) {
494 ret
= bdi_register(disk
->bdi
, "%u:%u",
495 disk
->major
, disk
->first_minor
);
497 goto out_unregister_queue
;
498 bdi_set_owner(disk
->bdi
, ddev
);
499 ret
= sysfs_create_link(&ddev
->kobj
,
500 &disk
->bdi
->dev
->kobj
, "bdi");
502 goto out_unregister_bdi
;
504 /* Make sure the first partition scan will be proceed */
505 if (get_capacity(disk
) && disk_has_partscan(disk
))
506 set_bit(GD_NEED_PART_SCAN
, &disk
->state
);
508 bdev_add(disk
->part0
, ddev
->devt
);
509 if (get_capacity(disk
))
510 disk_scan_partitions(disk
, BLK_OPEN_READ
);
513 * Announce the disk and partitions after all partitions are
514 * created. (for hidden disks uevents remain suppressed forever)
516 dev_set_uevent_suppress(ddev
, 0);
517 disk_uevent(disk
, KOBJ_ADD
);
520 * Even if the block_device for a hidden gendisk is not
521 * registered, it needs to have a valid bd_dev so that the
522 * freeing of the dynamic major works.
524 disk
->part0
->bd_dev
= MKDEV(disk
->major
, disk
->first_minor
);
527 blk_apply_bdi_limits(disk
->bdi
, &disk
->queue
->limits
);
528 disk_add_events(disk
);
529 set_bit(GD_ADDED
, &disk
->state
);
533 if (!(disk
->flags
& GENHD_FL_HIDDEN
))
534 bdi_unregister(disk
->bdi
);
535 out_unregister_queue
:
536 blk_unregister_queue(disk
);
537 rq_qos_exit(disk
->queue
);
539 kobject_put(disk
->slave_dir
);
540 disk
->slave_dir
= NULL
;
542 kobject_put(disk
->part0
->bd_holder_dir
);
544 sysfs_remove_link(block_depr
, dev_name(ddev
));
545 pm_runtime_set_memalloc_noio(ddev
, false);
549 if (disk
->major
== BLOCK_EXT_MAJOR
)
550 blk_free_ext_minor(disk
->first_minor
);
552 if (disk
->queue
->elevator
)
553 elevator_exit(disk
->queue
);
556 EXPORT_SYMBOL(device_add_disk
);
558 static void blk_report_disk_dead(struct gendisk
*disk
, bool surprise
)
560 struct block_device
*bdev
;
564 * On surprise disk removal, bdev_mark_dead() may call into file
565 * systems below. Make it clear that we're expecting to not hold
568 lockdep_assert_not_held(&disk
->open_mutex
);
571 xa_for_each(&disk
->part_tbl
, idx
, bdev
) {
572 if (!kobject_get_unless_zero(&bdev
->bd_device
.kobj
))
576 bdev_mark_dead(bdev
, surprise
);
578 put_device(&bdev
->bd_device
);
584 static void __blk_mark_disk_dead(struct gendisk
*disk
)
589 if (test_and_set_bit(GD_DEAD
, &disk
->state
))
592 if (test_bit(GD_OWNS_QUEUE
, &disk
->state
))
593 blk_queue_flag_set(QUEUE_FLAG_DYING
, disk
->queue
);
596 * Stop buffered writers from dirtying pages that can't be written out.
598 set_capacity(disk
, 0);
601 * Prevent new I/O from crossing bio_queue_enter().
603 blk_queue_start_drain(disk
->queue
);
607 * blk_mark_disk_dead - mark a disk as dead
608 * @disk: disk to mark as dead
610 * Mark as disk as dead (e.g. surprise removed) and don't accept any new I/O
613 void blk_mark_disk_dead(struct gendisk
*disk
)
615 __blk_mark_disk_dead(disk
);
616 blk_report_disk_dead(disk
, true);
618 EXPORT_SYMBOL_GPL(blk_mark_disk_dead
);
621 * del_gendisk - remove the gendisk
622 * @disk: the struct gendisk to remove
624 * Removes the gendisk and all its associated resources. This deletes the
625 * partitions associated with the gendisk, and unregisters the associated
628 * This is the counter to the respective __device_add_disk() call.
630 * The final removal of the struct gendisk happens when its refcount reaches 0
631 * with put_disk(), which should be called after del_gendisk(), if
632 * __device_add_disk() was used.
634 * Drivers exist which depend on the release of the gendisk to be synchronous,
635 * it should not be deferred.
639 void del_gendisk(struct gendisk
*disk
)
641 struct request_queue
*q
= disk
->queue
;
642 struct block_device
*part
;
647 if (WARN_ON_ONCE(!disk_live(disk
) && !(disk
->flags
& GENHD_FL_HIDDEN
)))
650 disk_del_events(disk
);
653 * Prevent new openers by unlinked the bdev inode.
655 mutex_lock(&disk
->open_mutex
);
656 xa_for_each(&disk
->part_tbl
, idx
, part
)
658 mutex_unlock(&disk
->open_mutex
);
661 * Tell the file system to write back all dirty data and shut down if
662 * it hasn't been notified earlier.
664 if (!test_bit(GD_DEAD
, &disk
->state
))
665 blk_report_disk_dead(disk
, false);
668 * Drop all partitions now that the disk is marked dead.
670 mutex_lock(&disk
->open_mutex
);
671 __blk_mark_disk_dead(disk
);
672 xa_for_each_start(&disk
->part_tbl
, idx
, part
, 1)
673 drop_partition(part
);
674 mutex_unlock(&disk
->open_mutex
);
676 if (!(disk
->flags
& GENHD_FL_HIDDEN
)) {
677 sysfs_remove_link(&disk_to_dev(disk
)->kobj
, "bdi");
680 * Unregister bdi before releasing device numbers (as they can
681 * get reused and we'd get clashes in sysfs).
683 bdi_unregister(disk
->bdi
);
686 blk_unregister_queue(disk
);
688 kobject_put(disk
->part0
->bd_holder_dir
);
689 kobject_put(disk
->slave_dir
);
690 disk
->slave_dir
= NULL
;
692 part_stat_set_all(disk
->part0
, 0);
693 disk
->part0
->bd_stamp
= 0;
694 sysfs_remove_link(block_depr
, dev_name(disk_to_dev(disk
)));
695 pm_runtime_set_memalloc_noio(disk_to_dev(disk
), false);
696 device_del(disk_to_dev(disk
));
698 blk_mq_freeze_queue_wait(q
);
700 blk_throtl_cancel_bios(disk
);
703 blk_flush_integrity();
706 blk_mq_cancel_work_sync(q
);
708 blk_mq_quiesce_queue(q
);
710 mutex_lock(&q
->sysfs_lock
);
712 mutex_unlock(&q
->sysfs_lock
);
715 blk_mq_unquiesce_queue(q
);
718 * If the disk does not own the queue, allow using passthrough requests
719 * again. Else leave the queue frozen to fail all I/O.
721 if (!test_bit(GD_OWNS_QUEUE
, &disk
->state
)) {
722 blk_queue_flag_clear(QUEUE_FLAG_INIT_DONE
, q
);
723 __blk_mq_unfreeze_queue(q
, true);
726 blk_mq_exit_queue(q
);
729 EXPORT_SYMBOL(del_gendisk
);
732 * invalidate_disk - invalidate the disk
733 * @disk: the struct gendisk to invalidate
735 * A helper to invalidates the disk. It will clean the disk's associated
736 * buffer/page caches and reset its internal states so that the disk
737 * can be reused by the drivers.
741 void invalidate_disk(struct gendisk
*disk
)
743 struct block_device
*bdev
= disk
->part0
;
745 invalidate_bdev(bdev
);
746 bdev
->bd_mapping
->wb_err
= 0;
747 set_capacity(disk
, 0);
749 EXPORT_SYMBOL(invalidate_disk
);
751 /* sysfs access to bad-blocks list. */
752 static ssize_t
disk_badblocks_show(struct device
*dev
,
753 struct device_attribute
*attr
,
756 struct gendisk
*disk
= dev_to_disk(dev
);
759 return sprintf(page
, "\n");
761 return badblocks_show(disk
->bb
, page
, 0);
764 static ssize_t
disk_badblocks_store(struct device
*dev
,
765 struct device_attribute
*attr
,
766 const char *page
, size_t len
)
768 struct gendisk
*disk
= dev_to_disk(dev
);
773 return badblocks_store(disk
->bb
, page
, len
, 0);
776 #ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
777 void blk_request_module(dev_t devt
)
779 unsigned int major
= MAJOR(devt
);
780 struct blk_major_name
**n
;
782 mutex_lock(&major_names_lock
);
783 for (n
= &major_names
[major_to_index(major
)]; *n
; n
= &(*n
)->next
) {
784 if ((*n
)->major
== major
&& (*n
)->probe
) {
786 mutex_unlock(&major_names_lock
);
790 mutex_unlock(&major_names_lock
);
792 if (request_module("block-major-%d-%d", MAJOR(devt
), MINOR(devt
)) > 0)
793 /* Make old-style 2.4 aliases work */
794 request_module("block-major-%d", MAJOR(devt
));
796 #endif /* CONFIG_BLOCK_LEGACY_AUTOLOAD */
798 #ifdef CONFIG_PROC_FS
800 static void *disk_seqf_start(struct seq_file
*seqf
, loff_t
*pos
)
803 struct class_dev_iter
*iter
;
806 iter
= kmalloc(sizeof(*iter
), GFP_KERNEL
);
808 return ERR_PTR(-ENOMEM
);
810 seqf
->private = iter
;
811 class_dev_iter_init(iter
, &block_class
, NULL
, &disk_type
);
813 dev
= class_dev_iter_next(iter
);
818 return dev_to_disk(dev
);
821 static void *disk_seqf_next(struct seq_file
*seqf
, void *v
, loff_t
*pos
)
826 dev
= class_dev_iter_next(seqf
->private);
828 return dev_to_disk(dev
);
833 static void disk_seqf_stop(struct seq_file
*seqf
, void *v
)
835 struct class_dev_iter
*iter
= seqf
->private;
837 /* stop is called even after start failed :-( */
839 class_dev_iter_exit(iter
);
841 seqf
->private = NULL
;
845 static void *show_partition_start(struct seq_file
*seqf
, loff_t
*pos
)
849 p
= disk_seqf_start(seqf
, pos
);
850 if (!IS_ERR_OR_NULL(p
) && !*pos
)
851 seq_puts(seqf
, "major minor #blocks name\n\n");
855 static int show_partition(struct seq_file
*seqf
, void *v
)
857 struct gendisk
*sgp
= v
;
858 struct block_device
*part
;
861 if (!get_capacity(sgp
) || (sgp
->flags
& GENHD_FL_HIDDEN
))
865 xa_for_each(&sgp
->part_tbl
, idx
, part
) {
866 if (!bdev_nr_sectors(part
))
868 seq_printf(seqf
, "%4d %7d %10llu %pg\n",
869 MAJOR(part
->bd_dev
), MINOR(part
->bd_dev
),
870 bdev_nr_sectors(part
) >> 1, part
);
876 static const struct seq_operations partitions_op
= {
877 .start
= show_partition_start
,
878 .next
= disk_seqf_next
,
879 .stop
= disk_seqf_stop
,
880 .show
= show_partition
884 static int __init
genhd_device_init(void)
888 error
= class_register(&block_class
);
893 register_blkdev(BLOCK_EXT_MAJOR
, "blkext");
895 /* create top-level block dir */
896 block_depr
= kobject_create_and_add("block", NULL
);
900 subsys_initcall(genhd_device_init
);
902 static ssize_t
disk_range_show(struct device
*dev
,
903 struct device_attribute
*attr
, char *buf
)
905 struct gendisk
*disk
= dev_to_disk(dev
);
907 return sprintf(buf
, "%d\n", disk
->minors
);
910 static ssize_t
disk_ext_range_show(struct device
*dev
,
911 struct device_attribute
*attr
, char *buf
)
913 struct gendisk
*disk
= dev_to_disk(dev
);
915 return sprintf(buf
, "%d\n",
916 (disk
->flags
& GENHD_FL_NO_PART
) ? 1 : DISK_MAX_PARTS
);
919 static ssize_t
disk_removable_show(struct device
*dev
,
920 struct device_attribute
*attr
, char *buf
)
922 struct gendisk
*disk
= dev_to_disk(dev
);
924 return sprintf(buf
, "%d\n",
925 (disk
->flags
& GENHD_FL_REMOVABLE
? 1 : 0));
928 static ssize_t
disk_hidden_show(struct device
*dev
,
929 struct device_attribute
*attr
, char *buf
)
931 struct gendisk
*disk
= dev_to_disk(dev
);
933 return sprintf(buf
, "%d\n",
934 (disk
->flags
& GENHD_FL_HIDDEN
? 1 : 0));
937 static ssize_t
disk_ro_show(struct device
*dev
,
938 struct device_attribute
*attr
, char *buf
)
940 struct gendisk
*disk
= dev_to_disk(dev
);
942 return sprintf(buf
, "%d\n", get_disk_ro(disk
) ? 1 : 0);
945 ssize_t
part_size_show(struct device
*dev
,
946 struct device_attribute
*attr
, char *buf
)
948 return sprintf(buf
, "%llu\n", bdev_nr_sectors(dev_to_bdev(dev
)));
951 ssize_t
part_stat_show(struct device
*dev
,
952 struct device_attribute
*attr
, char *buf
)
954 struct block_device
*bdev
= dev_to_bdev(dev
);
955 struct disk_stats stat
;
956 unsigned int inflight
;
958 inflight
= part_in_flight(bdev
);
961 update_io_ticks(bdev
, jiffies
, true);
964 part_stat_read_all(bdev
, &stat
);
966 "%8lu %8lu %8llu %8u "
967 "%8lu %8lu %8llu %8u "
969 "%8lu %8lu %8llu %8u "
973 stat
.merges
[STAT_READ
],
974 (unsigned long long)stat
.sectors
[STAT_READ
],
975 (unsigned int)div_u64(stat
.nsecs
[STAT_READ
], NSEC_PER_MSEC
),
976 stat
.ios
[STAT_WRITE
],
977 stat
.merges
[STAT_WRITE
],
978 (unsigned long long)stat
.sectors
[STAT_WRITE
],
979 (unsigned int)div_u64(stat
.nsecs
[STAT_WRITE
], NSEC_PER_MSEC
),
981 jiffies_to_msecs(stat
.io_ticks
),
982 (unsigned int)div_u64(stat
.nsecs
[STAT_READ
] +
983 stat
.nsecs
[STAT_WRITE
] +
984 stat
.nsecs
[STAT_DISCARD
] +
985 stat
.nsecs
[STAT_FLUSH
],
987 stat
.ios
[STAT_DISCARD
],
988 stat
.merges
[STAT_DISCARD
],
989 (unsigned long long)stat
.sectors
[STAT_DISCARD
],
990 (unsigned int)div_u64(stat
.nsecs
[STAT_DISCARD
], NSEC_PER_MSEC
),
991 stat
.ios
[STAT_FLUSH
],
992 (unsigned int)div_u64(stat
.nsecs
[STAT_FLUSH
], NSEC_PER_MSEC
));
995 ssize_t
part_inflight_show(struct device
*dev
, struct device_attribute
*attr
,
998 struct block_device
*bdev
= dev_to_bdev(dev
);
999 struct request_queue
*q
= bdev_get_queue(bdev
);
1000 unsigned int inflight
[2];
1003 blk_mq_in_flight_rw(q
, bdev
, inflight
);
1005 part_in_flight_rw(bdev
, inflight
);
1007 return sprintf(buf
, "%8u %8u\n", inflight
[0], inflight
[1]);
1010 static ssize_t
disk_capability_show(struct device
*dev
,
1011 struct device_attribute
*attr
, char *buf
)
1013 dev_warn_once(dev
, "the capability attribute has been deprecated.\n");
1014 return sprintf(buf
, "0\n");
1017 static ssize_t
disk_alignment_offset_show(struct device
*dev
,
1018 struct device_attribute
*attr
,
1021 struct gendisk
*disk
= dev_to_disk(dev
);
1023 return sprintf(buf
, "%d\n", bdev_alignment_offset(disk
->part0
));
1026 static ssize_t
disk_discard_alignment_show(struct device
*dev
,
1027 struct device_attribute
*attr
,
1030 struct gendisk
*disk
= dev_to_disk(dev
);
1032 return sprintf(buf
, "%d\n", bdev_alignment_offset(disk
->part0
));
1035 static ssize_t
diskseq_show(struct device
*dev
,
1036 struct device_attribute
*attr
, char *buf
)
1038 struct gendisk
*disk
= dev_to_disk(dev
);
1040 return sprintf(buf
, "%llu\n", disk
->diskseq
);
1043 static ssize_t
partscan_show(struct device
*dev
,
1044 struct device_attribute
*attr
, char *buf
)
1046 return sprintf(buf
, "%u\n", disk_has_partscan(dev_to_disk(dev
)));
1049 static DEVICE_ATTR(range
, 0444, disk_range_show
, NULL
);
1050 static DEVICE_ATTR(ext_range
, 0444, disk_ext_range_show
, NULL
);
1051 static DEVICE_ATTR(removable
, 0444, disk_removable_show
, NULL
);
1052 static DEVICE_ATTR(hidden
, 0444, disk_hidden_show
, NULL
);
1053 static DEVICE_ATTR(ro
, 0444, disk_ro_show
, NULL
);
1054 static DEVICE_ATTR(size
, 0444, part_size_show
, NULL
);
1055 static DEVICE_ATTR(alignment_offset
, 0444, disk_alignment_offset_show
, NULL
);
1056 static DEVICE_ATTR(discard_alignment
, 0444, disk_discard_alignment_show
, NULL
);
1057 static DEVICE_ATTR(capability
, 0444, disk_capability_show
, NULL
);
1058 static DEVICE_ATTR(stat
, 0444, part_stat_show
, NULL
);
1059 static DEVICE_ATTR(inflight
, 0444, part_inflight_show
, NULL
);
1060 static DEVICE_ATTR(badblocks
, 0644, disk_badblocks_show
, disk_badblocks_store
);
1061 static DEVICE_ATTR(diskseq
, 0444, diskseq_show
, NULL
);
1062 static DEVICE_ATTR(partscan
, 0444, partscan_show
, NULL
);
1064 #ifdef CONFIG_FAIL_MAKE_REQUEST
1065 ssize_t
part_fail_show(struct device
*dev
,
1066 struct device_attribute
*attr
, char *buf
)
1068 return sprintf(buf
, "%d\n",
1069 bdev_test_flag(dev_to_bdev(dev
), BD_MAKE_IT_FAIL
));
1072 ssize_t
part_fail_store(struct device
*dev
,
1073 struct device_attribute
*attr
,
1074 const char *buf
, size_t count
)
1078 if (count
> 0 && sscanf(buf
, "%d", &i
) > 0) {
1080 bdev_set_flag(dev_to_bdev(dev
), BD_MAKE_IT_FAIL
);
1082 bdev_clear_flag(dev_to_bdev(dev
), BD_MAKE_IT_FAIL
);
1087 static struct device_attribute dev_attr_fail
=
1088 __ATTR(make
-it
-fail
, 0644, part_fail_show
, part_fail_store
);
1089 #endif /* CONFIG_FAIL_MAKE_REQUEST */
1091 #ifdef CONFIG_FAIL_IO_TIMEOUT
1092 static struct device_attribute dev_attr_fail_timeout
=
1093 __ATTR(io
-timeout
-fail
, 0644, part_timeout_show
, part_timeout_store
);
1096 static struct attribute
*disk_attrs
[] = {
1097 &dev_attr_range
.attr
,
1098 &dev_attr_ext_range
.attr
,
1099 &dev_attr_removable
.attr
,
1100 &dev_attr_hidden
.attr
,
1102 &dev_attr_size
.attr
,
1103 &dev_attr_alignment_offset
.attr
,
1104 &dev_attr_discard_alignment
.attr
,
1105 &dev_attr_capability
.attr
,
1106 &dev_attr_stat
.attr
,
1107 &dev_attr_inflight
.attr
,
1108 &dev_attr_badblocks
.attr
,
1109 &dev_attr_events
.attr
,
1110 &dev_attr_events_async
.attr
,
1111 &dev_attr_events_poll_msecs
.attr
,
1112 &dev_attr_diskseq
.attr
,
1113 &dev_attr_partscan
.attr
,
1114 #ifdef CONFIG_FAIL_MAKE_REQUEST
1115 &dev_attr_fail
.attr
,
1117 #ifdef CONFIG_FAIL_IO_TIMEOUT
1118 &dev_attr_fail_timeout
.attr
,
1123 static umode_t
disk_visible(struct kobject
*kobj
, struct attribute
*a
, int n
)
1125 struct device
*dev
= container_of(kobj
, typeof(*dev
), kobj
);
1126 struct gendisk
*disk
= dev_to_disk(dev
);
1128 if (a
== &dev_attr_badblocks
.attr
&& !disk
->bb
)
1133 static struct attribute_group disk_attr_group
= {
1134 .attrs
= disk_attrs
,
1135 .is_visible
= disk_visible
,
1138 static const struct attribute_group
*disk_attr_groups
[] = {
1140 #ifdef CONFIG_BLK_DEV_IO_TRACE
1141 &blk_trace_attr_group
,
1143 #ifdef CONFIG_BLK_DEV_INTEGRITY
1144 &blk_integrity_attr_group
,
1150 * disk_release - releases all allocated resources of the gendisk
1151 * @dev: the device representing this disk
1153 * This function releases all allocated resources of the gendisk.
1155 * Drivers which used __device_add_disk() have a gendisk with a request_queue
1156 * assigned. Since the request_queue sits on top of the gendisk for these
1157 * drivers we also call blk_put_queue() for them, and we expect the
1158 * request_queue refcount to reach 0 at this point, and so the request_queue
1159 * will also be freed prior to the disk.
1161 * Context: can sleep
1163 static void disk_release(struct device
*dev
)
1165 struct gendisk
*disk
= dev_to_disk(dev
);
1168 WARN_ON_ONCE(disk_live(disk
));
1170 blk_trace_remove(disk
->queue
);
1173 * To undo the all initialization from blk_mq_init_allocated_queue in
1174 * case of a probe failure where add_disk is never called we have to
1175 * call blk_mq_exit_queue here. We can't do this for the more common
1176 * teardown case (yet) as the tagset can be gone by the time the disk
1177 * is released once it was added.
1179 if (queue_is_mq(disk
->queue
) &&
1180 test_bit(GD_OWNS_QUEUE
, &disk
->state
) &&
1181 !test_bit(GD_ADDED
, &disk
->state
))
1182 blk_mq_exit_queue(disk
->queue
);
1184 blkcg_exit_disk(disk
);
1186 bioset_exit(&disk
->bio_split
);
1188 disk_release_events(disk
);
1189 kfree(disk
->random
);
1190 disk_free_zone_resources(disk
);
1191 xa_destroy(&disk
->part_tbl
);
1193 disk
->queue
->disk
= NULL
;
1194 blk_put_queue(disk
->queue
);
1196 if (test_bit(GD_ADDED
, &disk
->state
) && disk
->fops
->free_disk
)
1197 disk
->fops
->free_disk(disk
);
1199 bdev_drop(disk
->part0
); /* frees the disk */
1202 static int block_uevent(const struct device
*dev
, struct kobj_uevent_env
*env
)
1204 const struct gendisk
*disk
= dev_to_disk(dev
);
1206 return add_uevent_var(env
, "DISKSEQ=%llu", disk
->diskseq
);
1209 const struct class block_class
= {
1211 .dev_uevent
= block_uevent
,
1214 static char *block_devnode(const struct device
*dev
, umode_t
*mode
,
1215 kuid_t
*uid
, kgid_t
*gid
)
1217 struct gendisk
*disk
= dev_to_disk(dev
);
1219 if (disk
->fops
->devnode
)
1220 return disk
->fops
->devnode(disk
, mode
);
1224 const struct device_type disk_type
= {
1226 .groups
= disk_attr_groups
,
1227 .release
= disk_release
,
1228 .devnode
= block_devnode
,
1231 #ifdef CONFIG_PROC_FS
1233 * aggregate disk stat collector. Uses the same stats that the sysfs
1234 * entries do, above, but makes them available through one seq_file.
1236 * The output looks suspiciously like /proc/partitions with a bunch of
1239 static int diskstats_show(struct seq_file
*seqf
, void *v
)
1241 struct gendisk
*gp
= v
;
1242 struct block_device
*hd
;
1243 unsigned int inflight
;
1244 struct disk_stats stat
;
1248 if (&disk_to_dev(gp)->kobj.entry == block_class.devices.next)
1249 seq_puts(seqf, "major minor name"
1250 " rio rmerge rsect ruse wio wmerge "
1251 "wsect wuse running use aveq"
1256 xa_for_each(&gp
->part_tbl
, idx
, hd
) {
1257 if (bdev_is_partition(hd
) && !bdev_nr_sectors(hd
))
1260 inflight
= part_in_flight(hd
);
1263 update_io_ticks(hd
, jiffies
, true);
1266 part_stat_read_all(hd
, &stat
);
1267 seq_printf(seqf
, "%4d %7d %pg "
1274 MAJOR(hd
->bd_dev
), MINOR(hd
->bd_dev
), hd
,
1275 stat
.ios
[STAT_READ
],
1276 stat
.merges
[STAT_READ
],
1277 stat
.sectors
[STAT_READ
],
1278 (unsigned int)div_u64(stat
.nsecs
[STAT_READ
],
1280 stat
.ios
[STAT_WRITE
],
1281 stat
.merges
[STAT_WRITE
],
1282 stat
.sectors
[STAT_WRITE
],
1283 (unsigned int)div_u64(stat
.nsecs
[STAT_WRITE
],
1286 jiffies_to_msecs(stat
.io_ticks
),
1287 (unsigned int)div_u64(stat
.nsecs
[STAT_READ
] +
1288 stat
.nsecs
[STAT_WRITE
] +
1289 stat
.nsecs
[STAT_DISCARD
] +
1290 stat
.nsecs
[STAT_FLUSH
],
1292 stat
.ios
[STAT_DISCARD
],
1293 stat
.merges
[STAT_DISCARD
],
1294 stat
.sectors
[STAT_DISCARD
],
1295 (unsigned int)div_u64(stat
.nsecs
[STAT_DISCARD
],
1297 stat
.ios
[STAT_FLUSH
],
1298 (unsigned int)div_u64(stat
.nsecs
[STAT_FLUSH
],
1307 static const struct seq_operations diskstats_op
= {
1308 .start
= disk_seqf_start
,
1309 .next
= disk_seqf_next
,
1310 .stop
= disk_seqf_stop
,
1311 .show
= diskstats_show
1314 static int __init
proc_genhd_init(void)
1316 proc_create_seq("diskstats", 0, NULL
, &diskstats_op
);
1317 proc_create_seq("partitions", 0, NULL
, &partitions_op
);
1320 module_init(proc_genhd_init
);
1321 #endif /* CONFIG_PROC_FS */
1323 dev_t
part_devt(struct gendisk
*disk
, u8 partno
)
1325 struct block_device
*part
;
1329 part
= xa_load(&disk
->part_tbl
, partno
);
1331 devt
= part
->bd_dev
;
1337 struct gendisk
*__alloc_disk_node(struct request_queue
*q
, int node_id
,
1338 struct lock_class_key
*lkclass
)
1340 struct gendisk
*disk
;
1342 disk
= kzalloc_node(sizeof(struct gendisk
), GFP_KERNEL
, node_id
);
1346 if (bioset_init(&disk
->bio_split
, BIO_POOL_SIZE
, 0, 0))
1349 disk
->bdi
= bdi_alloc(node_id
);
1351 goto out_free_bioset
;
1353 /* bdev_alloc() might need the queue, set before the first call */
1356 disk
->part0
= bdev_alloc(disk
, 0);
1360 disk
->node_id
= node_id
;
1361 mutex_init(&disk
->open_mutex
);
1362 xa_init(&disk
->part_tbl
);
1363 if (xa_insert(&disk
->part_tbl
, 0, disk
->part0
, GFP_KERNEL
))
1364 goto out_destroy_part_tbl
;
1366 if (blkcg_init_disk(disk
))
1367 goto out_erase_part0
;
1369 disk_init_zone_resources(disk
);
1370 rand_initialize_disk(disk
);
1371 disk_to_dev(disk
)->class = &block_class
;
1372 disk_to_dev(disk
)->type
= &disk_type
;
1373 device_initialize(disk_to_dev(disk
));
1376 lockdep_init_map(&disk
->lockdep_map
, "(bio completion)", lkclass
, 0);
1377 #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED
1378 INIT_LIST_HEAD(&disk
->slave_bdevs
);
1383 xa_erase(&disk
->part_tbl
, 0);
1384 out_destroy_part_tbl
:
1385 xa_destroy(&disk
->part_tbl
);
1386 disk
->part0
->bd_disk
= NULL
;
1387 bdev_drop(disk
->part0
);
1391 bioset_exit(&disk
->bio_split
);
1397 struct gendisk
*__blk_alloc_disk(struct queue_limits
*lim
, int node
,
1398 struct lock_class_key
*lkclass
)
1400 struct queue_limits default_lim
= { };
1401 struct request_queue
*q
;
1402 struct gendisk
*disk
;
1404 q
= blk_alloc_queue(lim
? lim
: &default_lim
, node
);
1408 disk
= __alloc_disk_node(q
, node
, lkclass
);
1411 return ERR_PTR(-ENOMEM
);
1413 set_bit(GD_OWNS_QUEUE
, &disk
->state
);
1416 EXPORT_SYMBOL(__blk_alloc_disk
);
1419 * put_disk - decrements the gendisk refcount
1420 * @disk: the struct gendisk to decrement the refcount for
1422 * This decrements the refcount for the struct gendisk. When this reaches 0
1423 * we'll have disk_release() called.
1425 * Note: for blk-mq disk put_disk must be called before freeing the tag_set
1426 * when handling probe errors (that is before add_disk() is called).
1428 * Context: Any context, but the last reference must not be dropped from
1431 void put_disk(struct gendisk
*disk
)
1434 put_device(disk_to_dev(disk
));
1436 EXPORT_SYMBOL(put_disk
);
1438 static void set_disk_ro_uevent(struct gendisk
*gd
, int ro
)
1440 char event
[] = "DISK_RO=1";
1441 char *envp
[] = { event
, NULL
};
1445 kobject_uevent_env(&disk_to_dev(gd
)->kobj
, KOBJ_CHANGE
, envp
);
1449 * set_disk_ro - set a gendisk read-only
1450 * @disk: gendisk to operate on
1451 * @read_only: %true to set the disk read-only, %false set the disk read/write
1453 * This function is used to indicate whether a given disk device should have its
1454 * read-only flag set. set_disk_ro() is typically used by device drivers to
1455 * indicate whether the underlying physical device is write-protected.
1457 void set_disk_ro(struct gendisk
*disk
, bool read_only
)
1460 if (test_and_set_bit(GD_READ_ONLY
, &disk
->state
))
1463 if (!test_and_clear_bit(GD_READ_ONLY
, &disk
->state
))
1466 set_disk_ro_uevent(disk
, read_only
);
1468 EXPORT_SYMBOL(set_disk_ro
);
1470 void inc_diskseq(struct gendisk
*disk
)
1472 disk
->diskseq
= atomic64_inc_return(&diskseq
);