2 #include <linux/spinlock.h>
3 #include <linux/slab.h>
4 #include <linux/blkdev.h>
5 #include <linux/hdreg.h>
6 #include <linux/module.h>
7 #include <linux/mutex.h>
8 #include <linux/virtio.h>
9 #include <linux/virtio_blk.h>
10 #include <linux/scatterlist.h>
11 #include <linux/string_helpers.h>
12 #include <scsi/scsi_cmnd.h>
13 #include <linux/idr.h>
18 static DEFINE_IDA(vd_index_ida
);
20 struct workqueue_struct
*virtblk_wq
;
24 struct virtio_device
*vdev
;
27 /* The disk structure for the kernel. */
32 /* Process context for config space updates */
33 struct work_struct config_work
;
35 /* Lock for config space updates */
36 struct mutex config_lock
;
38 /* enable config space updates */
41 /* What host tells us, plus 2 for header & tailer. */
42 unsigned int sg_elems
;
44 /* Ida index - used to track minor number allocations. */
47 /* Scatterlist: can be too big for stack. */
48 struct scatterlist sg
[/*sg_elems*/];
54 struct virtio_blk_outhdr out_hdr
;
55 struct virtio_scsi_inhdr in_hdr
;
59 static void blk_done(struct virtqueue
*vq
)
61 struct virtio_blk
*vblk
= vq
->vdev
->priv
;
62 struct virtblk_req
*vbr
;
66 spin_lock_irqsave(vblk
->disk
->queue
->queue_lock
, flags
);
67 while ((vbr
= virtqueue_get_buf(vblk
->vq
, &len
)) != NULL
) {
70 switch (vbr
->status
) {
74 case VIRTIO_BLK_S_UNSUPP
:
82 switch (vbr
->req
->cmd_type
) {
83 case REQ_TYPE_BLOCK_PC
:
84 vbr
->req
->resid_len
= vbr
->in_hdr
.residual
;
85 vbr
->req
->sense_len
= vbr
->in_hdr
.sense_len
;
86 vbr
->req
->errors
= vbr
->in_hdr
.errors
;
88 case REQ_TYPE_SPECIAL
:
89 vbr
->req
->errors
= (error
!= 0);
95 __blk_end_request_all(vbr
->req
, error
);
96 mempool_free(vbr
, vblk
->pool
);
98 /* In case queue is stopped waiting for more buffers. */
99 blk_start_queue(vblk
->disk
->queue
);
100 spin_unlock_irqrestore(vblk
->disk
->queue
->queue_lock
, flags
);
103 static bool do_req(struct request_queue
*q
, struct virtio_blk
*vblk
,
106 unsigned long num
, out
= 0, in
= 0;
107 struct virtblk_req
*vbr
;
109 vbr
= mempool_alloc(vblk
->pool
, GFP_ATOMIC
);
111 /* When another request finishes we'll try again. */
116 if (req
->cmd_flags
& REQ_FLUSH
) {
117 vbr
->out_hdr
.type
= VIRTIO_BLK_T_FLUSH
;
118 vbr
->out_hdr
.sector
= 0;
119 vbr
->out_hdr
.ioprio
= req_get_ioprio(vbr
->req
);
121 switch (req
->cmd_type
) {
123 vbr
->out_hdr
.type
= 0;
124 vbr
->out_hdr
.sector
= blk_rq_pos(vbr
->req
);
125 vbr
->out_hdr
.ioprio
= req_get_ioprio(vbr
->req
);
127 case REQ_TYPE_BLOCK_PC
:
128 vbr
->out_hdr
.type
= VIRTIO_BLK_T_SCSI_CMD
;
129 vbr
->out_hdr
.sector
= 0;
130 vbr
->out_hdr
.ioprio
= req_get_ioprio(vbr
->req
);
132 case REQ_TYPE_SPECIAL
:
133 vbr
->out_hdr
.type
= VIRTIO_BLK_T_GET_ID
;
134 vbr
->out_hdr
.sector
= 0;
135 vbr
->out_hdr
.ioprio
= req_get_ioprio(vbr
->req
);
138 /* We don't put anything else in the queue. */
143 sg_set_buf(&vblk
->sg
[out
++], &vbr
->out_hdr
, sizeof(vbr
->out_hdr
));
146 * If this is a packet command we need a couple of additional headers.
147 * Behind the normal outhdr we put a segment with the scsi command
148 * block, and before the normal inhdr we put the sense data and the
149 * inhdr with additional status information before the normal inhdr.
151 if (vbr
->req
->cmd_type
== REQ_TYPE_BLOCK_PC
)
152 sg_set_buf(&vblk
->sg
[out
++], vbr
->req
->cmd
, vbr
->req
->cmd_len
);
154 num
= blk_rq_map_sg(q
, vbr
->req
, vblk
->sg
+ out
);
156 if (vbr
->req
->cmd_type
== REQ_TYPE_BLOCK_PC
) {
157 sg_set_buf(&vblk
->sg
[num
+ out
+ in
++], vbr
->req
->sense
, SCSI_SENSE_BUFFERSIZE
);
158 sg_set_buf(&vblk
->sg
[num
+ out
+ in
++], &vbr
->in_hdr
,
159 sizeof(vbr
->in_hdr
));
162 sg_set_buf(&vblk
->sg
[num
+ out
+ in
++], &vbr
->status
,
163 sizeof(vbr
->status
));
166 if (rq_data_dir(vbr
->req
) == WRITE
) {
167 vbr
->out_hdr
.type
|= VIRTIO_BLK_T_OUT
;
170 vbr
->out_hdr
.type
|= VIRTIO_BLK_T_IN
;
175 if (virtqueue_add_buf(vblk
->vq
, vblk
->sg
, out
, in
, vbr
, GFP_ATOMIC
)<0) {
176 mempool_free(vbr
, vblk
->pool
);
183 static void do_virtblk_request(struct request_queue
*q
)
185 struct virtio_blk
*vblk
= q
->queuedata
;
187 unsigned int issued
= 0;
189 while ((req
= blk_peek_request(q
)) != NULL
) {
190 BUG_ON(req
->nr_phys_segments
+ 2 > vblk
->sg_elems
);
192 /* If this request fails, stop queue and wait for something to
193 finish to restart it. */
194 if (!do_req(q
, vblk
, req
)) {
198 blk_start_request(req
);
203 virtqueue_kick(vblk
->vq
);
206 /* return id (s/n) string for *disk to *id_str
208 static int virtblk_get_id(struct gendisk
*disk
, char *id_str
)
210 struct virtio_blk
*vblk
= disk
->private_data
;
215 bio
= bio_map_kern(vblk
->disk
->queue
, id_str
, VIRTIO_BLK_ID_BYTES
,
220 req
= blk_make_request(vblk
->disk
->queue
, bio
, GFP_KERNEL
);
226 req
->cmd_type
= REQ_TYPE_SPECIAL
;
227 err
= blk_execute_rq(vblk
->disk
->queue
, vblk
->disk
, req
, false);
228 blk_put_request(req
);
233 static int virtblk_ioctl(struct block_device
*bdev
, fmode_t mode
,
234 unsigned int cmd
, unsigned long data
)
236 struct gendisk
*disk
= bdev
->bd_disk
;
237 struct virtio_blk
*vblk
= disk
->private_data
;
240 * Only allow the generic SCSI ioctls if the host can support it.
242 if (!virtio_has_feature(vblk
->vdev
, VIRTIO_BLK_F_SCSI
))
245 return scsi_cmd_blk_ioctl(bdev
, mode
, cmd
,
246 (void __user
*)data
);
249 /* We provide getgeo only to please some old bootloader/partitioning tools */
250 static int virtblk_getgeo(struct block_device
*bd
, struct hd_geometry
*geo
)
252 struct virtio_blk
*vblk
= bd
->bd_disk
->private_data
;
253 struct virtio_blk_geometry vgeo
;
256 /* see if the host passed in geometry config */
257 err
= virtio_config_val(vblk
->vdev
, VIRTIO_BLK_F_GEOMETRY
,
258 offsetof(struct virtio_blk_config
, geometry
),
262 geo
->heads
= vgeo
.heads
;
263 geo
->sectors
= vgeo
.sectors
;
264 geo
->cylinders
= vgeo
.cylinders
;
266 /* some standard values, similar to sd */
268 geo
->sectors
= 1 << 5;
269 geo
->cylinders
= get_capacity(bd
->bd_disk
) >> 11;
274 static const struct block_device_operations virtblk_fops
= {
275 .ioctl
= virtblk_ioctl
,
276 .owner
= THIS_MODULE
,
277 .getgeo
= virtblk_getgeo
,
280 static int index_to_minor(int index
)
282 return index
<< PART_BITS
;
285 static int minor_to_index(int minor
)
287 return minor
>> PART_BITS
;
290 static ssize_t
virtblk_serial_show(struct device
*dev
,
291 struct device_attribute
*attr
, char *buf
)
293 struct gendisk
*disk
= dev_to_disk(dev
);
296 /* sysfs gives us a PAGE_SIZE buffer */
297 BUILD_BUG_ON(PAGE_SIZE
< VIRTIO_BLK_ID_BYTES
);
299 buf
[VIRTIO_BLK_ID_BYTES
] = '\0';
300 err
= virtblk_get_id(disk
, buf
);
304 if (err
== -EIO
) /* Unsupported? Make it empty. */
309 DEVICE_ATTR(serial
, S_IRUGO
, virtblk_serial_show
, NULL
);
311 static void virtblk_config_changed_work(struct work_struct
*work
)
313 struct virtio_blk
*vblk
=
314 container_of(work
, struct virtio_blk
, config_work
);
315 struct virtio_device
*vdev
= vblk
->vdev
;
316 struct request_queue
*q
= vblk
->disk
->queue
;
317 char cap_str_2
[10], cap_str_10
[10];
320 mutex_lock(&vblk
->config_lock
);
321 if (!vblk
->config_enable
)
324 /* Host must always specify the capacity. */
325 vdev
->config
->get(vdev
, offsetof(struct virtio_blk_config
, capacity
),
326 &capacity
, sizeof(capacity
));
328 /* If capacity is too big, truncate with warning. */
329 if ((sector_t
)capacity
!= capacity
) {
330 dev_warn(&vdev
->dev
, "Capacity %llu too large: truncating\n",
331 (unsigned long long)capacity
);
332 capacity
= (sector_t
)-1;
335 size
= capacity
* queue_logical_block_size(q
);
336 string_get_size(size
, STRING_UNITS_2
, cap_str_2
, sizeof(cap_str_2
));
337 string_get_size(size
, STRING_UNITS_10
, cap_str_10
, sizeof(cap_str_10
));
339 dev_notice(&vdev
->dev
,
340 "new size: %llu %d-byte logical blocks (%s/%s)\n",
341 (unsigned long long)capacity
,
342 queue_logical_block_size(q
),
343 cap_str_10
, cap_str_2
);
345 set_capacity(vblk
->disk
, capacity
);
346 revalidate_disk(vblk
->disk
);
348 mutex_unlock(&vblk
->config_lock
);
351 static void virtblk_config_changed(struct virtio_device
*vdev
)
353 struct virtio_blk
*vblk
= vdev
->priv
;
355 queue_work(virtblk_wq
, &vblk
->config_work
);
358 static int init_vq(struct virtio_blk
*vblk
)
362 /* We expect one virtqueue, for output. */
363 vblk
->vq
= virtio_find_single_vq(vblk
->vdev
, blk_done
, "requests");
364 if (IS_ERR(vblk
->vq
))
365 err
= PTR_ERR(vblk
->vq
);
371 * Legacy naming scheme used for virtio devices. We are stuck with it for
372 * virtio blk but don't ever use it for any new driver.
374 static int virtblk_name_format(char *prefix
, int index
, char *buf
, int buflen
)
376 const int base
= 'z' - 'a' + 1;
377 char *begin
= buf
+ strlen(prefix
);
378 char *end
= buf
+ buflen
;
388 *--p
= 'a' + (index
% unit
);
389 index
= (index
/ unit
) - 1;
390 } while (index
>= 0);
392 memmove(begin
, p
, end
- p
);
393 memcpy(buf
, prefix
, strlen(prefix
));
398 static int virtblk_get_cache_mode(struct virtio_device
*vdev
)
403 err
= virtio_config_val(vdev
, VIRTIO_BLK_F_CONFIG_WCE
,
404 offsetof(struct virtio_blk_config
, wce
),
407 writeback
= virtio_has_feature(vdev
, VIRTIO_BLK_F_WCE
);
412 static void virtblk_update_cache_mode(struct virtio_device
*vdev
)
414 u8 writeback
= virtblk_get_cache_mode(vdev
);
415 struct virtio_blk
*vblk
= vdev
->priv
;
418 blk_queue_flush(vblk
->disk
->queue
, REQ_FLUSH
);
420 blk_queue_flush(vblk
->disk
->queue
, 0);
422 revalidate_disk(vblk
->disk
);
425 static const char *const virtblk_cache_types
[] = {
426 "write through", "write back"
430 virtblk_cache_type_store(struct device
*dev
, struct device_attribute
*attr
,
431 const char *buf
, size_t count
)
433 struct gendisk
*disk
= dev_to_disk(dev
);
434 struct virtio_blk
*vblk
= disk
->private_data
;
435 struct virtio_device
*vdev
= vblk
->vdev
;
439 BUG_ON(!virtio_has_feature(vblk
->vdev
, VIRTIO_BLK_F_CONFIG_WCE
));
440 for (i
= ARRAY_SIZE(virtblk_cache_types
); --i
>= 0; )
441 if (sysfs_streq(buf
, virtblk_cache_types
[i
]))
448 vdev
->config
->set(vdev
,
449 offsetof(struct virtio_blk_config
, wce
),
450 &writeback
, sizeof(writeback
));
452 virtblk_update_cache_mode(vdev
);
457 virtblk_cache_type_show(struct device
*dev
, struct device_attribute
*attr
,
460 struct gendisk
*disk
= dev_to_disk(dev
);
461 struct virtio_blk
*vblk
= disk
->private_data
;
462 u8 writeback
= virtblk_get_cache_mode(vblk
->vdev
);
464 BUG_ON(writeback
>= ARRAY_SIZE(virtblk_cache_types
));
465 return snprintf(buf
, 40, "%s\n", virtblk_cache_types
[writeback
]);
468 static const struct device_attribute dev_attr_cache_type_ro
=
469 __ATTR(cache_type
, S_IRUGO
,
470 virtblk_cache_type_show
, NULL
);
471 static const struct device_attribute dev_attr_cache_type_rw
=
472 __ATTR(cache_type
, S_IRUGO
|S_IWUSR
,
473 virtblk_cache_type_show
, virtblk_cache_type_store
);
475 static int __devinit
virtblk_probe(struct virtio_device
*vdev
)
477 struct virtio_blk
*vblk
;
478 struct request_queue
*q
;
481 u32 v
, blk_size
, sg_elems
, opt_io_size
;
483 u8 physical_block_exp
, alignment_offset
;
485 err
= ida_simple_get(&vd_index_ida
, 0, minor_to_index(1 << MINORBITS
),
491 /* We need to know how many segments before we allocate. */
492 err
= virtio_config_val(vdev
, VIRTIO_BLK_F_SEG_MAX
,
493 offsetof(struct virtio_blk_config
, seg_max
),
496 /* We need at least one SG element, whatever they say. */
497 if (err
|| !sg_elems
)
500 /* We need an extra sg elements at head and tail. */
502 vdev
->priv
= vblk
= kmalloc(sizeof(*vblk
) +
503 sizeof(vblk
->sg
[0]) * sg_elems
, GFP_KERNEL
);
510 vblk
->sg_elems
= sg_elems
;
511 sg_init_table(vblk
->sg
, vblk
->sg_elems
);
512 mutex_init(&vblk
->config_lock
);
513 INIT_WORK(&vblk
->config_work
, virtblk_config_changed_work
);
514 vblk
->config_enable
= true;
520 vblk
->pool
= mempool_create_kmalloc_pool(1,sizeof(struct virtblk_req
));
526 /* FIXME: How many partitions? How long is a piece of string? */
527 vblk
->disk
= alloc_disk(1 << PART_BITS
);
533 q
= vblk
->disk
->queue
= blk_init_queue(do_virtblk_request
, NULL
);
541 virtblk_name_format("vd", index
, vblk
->disk
->disk_name
, DISK_NAME_LEN
);
543 vblk
->disk
->major
= major
;
544 vblk
->disk
->first_minor
= index_to_minor(index
);
545 vblk
->disk
->private_data
= vblk
;
546 vblk
->disk
->fops
= &virtblk_fops
;
547 vblk
->disk
->driverfs_dev
= &vdev
->dev
;
550 /* configure queue flush support */
551 virtblk_update_cache_mode(vdev
);
553 /* If disk is read-only in the host, the guest should obey */
554 if (virtio_has_feature(vdev
, VIRTIO_BLK_F_RO
))
555 set_disk_ro(vblk
->disk
, 1);
557 /* Host must always specify the capacity. */
558 vdev
->config
->get(vdev
, offsetof(struct virtio_blk_config
, capacity
),
561 /* If capacity is too big, truncate with warning. */
562 if ((sector_t
)cap
!= cap
) {
563 dev_warn(&vdev
->dev
, "Capacity %llu too large: truncating\n",
564 (unsigned long long)cap
);
567 set_capacity(vblk
->disk
, cap
);
569 /* We can handle whatever the host told us to handle. */
570 blk_queue_max_segments(q
, vblk
->sg_elems
-2);
572 /* No need to bounce any requests */
573 blk_queue_bounce_limit(q
, BLK_BOUNCE_ANY
);
575 /* No real sector limit. */
576 blk_queue_max_hw_sectors(q
, -1U);
578 /* Host can optionally specify maximum segment size and number of
580 err
= virtio_config_val(vdev
, VIRTIO_BLK_F_SIZE_MAX
,
581 offsetof(struct virtio_blk_config
, size_max
),
584 blk_queue_max_segment_size(q
, v
);
586 blk_queue_max_segment_size(q
, -1U);
588 /* Host can optionally specify the block size of the device */
589 err
= virtio_config_val(vdev
, VIRTIO_BLK_F_BLK_SIZE
,
590 offsetof(struct virtio_blk_config
, blk_size
),
593 blk_queue_logical_block_size(q
, blk_size
);
595 blk_size
= queue_logical_block_size(q
);
597 /* Use topology information if available */
598 err
= virtio_config_val(vdev
, VIRTIO_BLK_F_TOPOLOGY
,
599 offsetof(struct virtio_blk_config
, physical_block_exp
),
600 &physical_block_exp
);
601 if (!err
&& physical_block_exp
)
602 blk_queue_physical_block_size(q
,
603 blk_size
* (1 << physical_block_exp
));
605 err
= virtio_config_val(vdev
, VIRTIO_BLK_F_TOPOLOGY
,
606 offsetof(struct virtio_blk_config
, alignment_offset
),
608 if (!err
&& alignment_offset
)
609 blk_queue_alignment_offset(q
, blk_size
* alignment_offset
);
611 err
= virtio_config_val(vdev
, VIRTIO_BLK_F_TOPOLOGY
,
612 offsetof(struct virtio_blk_config
, min_io_size
),
614 if (!err
&& min_io_size
)
615 blk_queue_io_min(q
, blk_size
* min_io_size
);
617 err
= virtio_config_val(vdev
, VIRTIO_BLK_F_TOPOLOGY
,
618 offsetof(struct virtio_blk_config
, opt_io_size
),
620 if (!err
&& opt_io_size
)
621 blk_queue_io_opt(q
, blk_size
* opt_io_size
);
624 add_disk(vblk
->disk
);
625 err
= device_create_file(disk_to_dev(vblk
->disk
), &dev_attr_serial
);
629 if (virtio_has_feature(vdev
, VIRTIO_BLK_F_CONFIG_WCE
))
630 err
= device_create_file(disk_to_dev(vblk
->disk
),
631 &dev_attr_cache_type_rw
);
633 err
= device_create_file(disk_to_dev(vblk
->disk
),
634 &dev_attr_cache_type_ro
);
640 del_gendisk(vblk
->disk
);
641 blk_cleanup_queue(vblk
->disk
->queue
);
643 put_disk(vblk
->disk
);
645 mempool_destroy(vblk
->pool
);
647 vdev
->config
->del_vqs(vdev
);
651 ida_simple_remove(&vd_index_ida
, index
);
656 static void __devexit
virtblk_remove(struct virtio_device
*vdev
)
658 struct virtio_blk
*vblk
= vdev
->priv
;
659 int index
= vblk
->index
;
661 /* Prevent config work handler from accessing the device. */
662 mutex_lock(&vblk
->config_lock
);
663 vblk
->config_enable
= false;
664 mutex_unlock(&vblk
->config_lock
);
666 del_gendisk(vblk
->disk
);
667 blk_cleanup_queue(vblk
->disk
->queue
);
669 /* Stop all the virtqueues. */
670 vdev
->config
->reset(vdev
);
672 flush_work(&vblk
->config_work
);
674 put_disk(vblk
->disk
);
675 mempool_destroy(vblk
->pool
);
676 vdev
->config
->del_vqs(vdev
);
678 ida_simple_remove(&vd_index_ida
, index
);
682 static int virtblk_freeze(struct virtio_device
*vdev
)
684 struct virtio_blk
*vblk
= vdev
->priv
;
686 /* Ensure we don't receive any more interrupts */
687 vdev
->config
->reset(vdev
);
689 /* Prevent config work handler from accessing the device. */
690 mutex_lock(&vblk
->config_lock
);
691 vblk
->config_enable
= false;
692 mutex_unlock(&vblk
->config_lock
);
694 flush_work(&vblk
->config_work
);
696 spin_lock_irq(vblk
->disk
->queue
->queue_lock
);
697 blk_stop_queue(vblk
->disk
->queue
);
698 spin_unlock_irq(vblk
->disk
->queue
->queue_lock
);
699 blk_sync_queue(vblk
->disk
->queue
);
701 vdev
->config
->del_vqs(vdev
);
705 static int virtblk_restore(struct virtio_device
*vdev
)
707 struct virtio_blk
*vblk
= vdev
->priv
;
710 vblk
->config_enable
= true;
711 ret
= init_vq(vdev
->priv
);
713 spin_lock_irq(vblk
->disk
->queue
->queue_lock
);
714 blk_start_queue(vblk
->disk
->queue
);
715 spin_unlock_irq(vblk
->disk
->queue
->queue_lock
);
721 static const struct virtio_device_id id_table
[] = {
722 { VIRTIO_ID_BLOCK
, VIRTIO_DEV_ANY_ID
},
726 static unsigned int features
[] = {
727 VIRTIO_BLK_F_SEG_MAX
, VIRTIO_BLK_F_SIZE_MAX
, VIRTIO_BLK_F_GEOMETRY
,
728 VIRTIO_BLK_F_RO
, VIRTIO_BLK_F_BLK_SIZE
, VIRTIO_BLK_F_SCSI
,
729 VIRTIO_BLK_F_WCE
, VIRTIO_BLK_F_TOPOLOGY
, VIRTIO_BLK_F_CONFIG_WCE
733 * virtio_blk causes spurious section mismatch warning by
734 * simultaneously referring to a __devinit and a __devexit function.
735 * Use __refdata to avoid this warning.
737 static struct virtio_driver __refdata virtio_blk
= {
738 .feature_table
= features
,
739 .feature_table_size
= ARRAY_SIZE(features
),
740 .driver
.name
= KBUILD_MODNAME
,
741 .driver
.owner
= THIS_MODULE
,
742 .id_table
= id_table
,
743 .probe
= virtblk_probe
,
744 .remove
= __devexit_p(virtblk_remove
),
745 .config_changed
= virtblk_config_changed
,
747 .freeze
= virtblk_freeze
,
748 .restore
= virtblk_restore
,
752 static int __init
init(void)
756 virtblk_wq
= alloc_workqueue("virtio-blk", 0, 0);
760 major
= register_blkdev(0, "virtblk");
763 goto out_destroy_workqueue
;
766 error
= register_virtio_driver(&virtio_blk
);
768 goto out_unregister_blkdev
;
771 out_unregister_blkdev
:
772 unregister_blkdev(major
, "virtblk");
773 out_destroy_workqueue
:
774 destroy_workqueue(virtblk_wq
);
778 static void __exit
fini(void)
780 unregister_blkdev(major
, "virtblk");
781 unregister_virtio_driver(&virtio_blk
);
782 destroy_workqueue(virtblk_wq
);
787 MODULE_DEVICE_TABLE(virtio
, id_table
);
788 MODULE_DESCRIPTION("Virtio block driver");
789 MODULE_LICENSE("GPL");