2 * Dedicated thread for virtio-blk I/O processing
4 * Copyright 2012 IBM, Corp.
5 * Copyright 2012 Red Hat, Inc. and/or its affiliates
8 * Stefan Hajnoczi <stefanha@redhat.com>
10 * This work is licensed under the terms of the GNU GPL, version 2 or later.
11 * See the COPYING file in the top-level directory.
17 #include "qemu/thread.h"
18 #include "qemu/error-report.h"
19 #include "hw/virtio/dataplane/vring.h"
20 #include "block/block.h"
21 #include "hw/virtio/virtio-blk.h"
22 #include "virtio-blk.h"
23 #include "block/aio.h"
24 #include "hw/virtio/virtio-bus.h"
25 #include "qom/object_interfaces.h"
27 struct VirtIOBlockDataPlane
{
36 Vring vring
; /* virtqueue vring */
37 EventNotifier
*guest_notifier
; /* irq */
38 QEMUBH
*bh
; /* bh for guest notification */
40 /* Note that these EventNotifiers are assigned by value. This is
41 * fine as long as you do not call event_notifier_cleanup on them
42 * (because you don't own the file descriptor or handle; you just
46 IOThread internal_iothread_obj
;
48 EventNotifier host_notifier
; /* doorbell */
50 /* Operation blocker on BDS */
52 void (*saved_complete_request
)(struct VirtIOBlockReq
*req
,
53 unsigned char status
);
56 /* Raise an interrupt to signal guest, if necessary */
57 static void notify_guest(VirtIOBlockDataPlane
*s
)
59 if (!vring_should_notify(s
->vdev
, &s
->vring
)) {
63 event_notifier_set(s
->guest_notifier
);
66 static void notify_guest_bh(void *opaque
)
68 VirtIOBlockDataPlane
*s
= opaque
;
73 static void complete_request_vring(VirtIOBlockReq
*req
, unsigned char status
)
75 VirtIOBlockDataPlane
*s
= req
->dev
->dataplane
;
76 stb_p(&req
->in
->status
, status
);
78 vring_push(&req
->dev
->dataplane
->vring
, &req
->elem
,
79 req
->qiov
.size
+ sizeof(*req
->in
));
81 /* Suppress notification to guest by BH and its scheduled
82 * flag because requests are completed as a batch after io
83 * plug & unplug is introduced, and the BH can still be
84 * executed in dataplane aio context even after it is
85 * stopped, so needn't worry about notification loss with BH.
87 qemu_bh_schedule(s
->bh
);
90 static void handle_notify(EventNotifier
*e
)
92 VirtIOBlockDataPlane
*s
= container_of(e
, VirtIOBlockDataPlane
,
94 VirtIOBlock
*vblk
= VIRTIO_BLK(s
->vdev
);
96 event_notifier_test_and_clear(&s
->host_notifier
);
97 bdrv_io_plug(s
->blk
->conf
.bs
);
99 MultiReqBuffer mrb
= {
104 /* Disable guest->host notifies to avoid unnecessary vmexits */
105 vring_disable_notification(s
->vdev
, &s
->vring
);
108 VirtIOBlockReq
*req
= virtio_blk_alloc_request(vblk
);
110 ret
= vring_pop(s
->vdev
, &s
->vring
, &req
->elem
);
112 virtio_blk_free_request(req
);
113 break; /* no more requests */
116 trace_virtio_blk_data_plane_process_request(s
, req
->elem
.out_num
,
120 virtio_blk_handle_request(req
, &mrb
);
123 virtio_submit_multiwrite(s
->blk
->conf
.bs
, &mrb
);
125 if (likely(ret
== -EAGAIN
)) { /* vring emptied */
126 /* Re-enable guest->host notifies and stop processing the vring.
127 * But if the guest has snuck in more descriptors, keep processing.
129 if (vring_enable_notification(s
->vdev
, &s
->vring
)) {
132 } else { /* fatal error */
136 bdrv_io_unplug(s
->blk
->conf
.bs
);
139 /* Context: QEMU global mutex held */
140 void virtio_blk_data_plane_create(VirtIODevice
*vdev
, VirtIOBlkConf
*blk
,
141 VirtIOBlockDataPlane
**dataplane
,
144 VirtIOBlockDataPlane
*s
;
145 Error
*local_err
= NULL
;
146 BusState
*qbus
= BUS(qdev_get_parent_bus(DEVICE(vdev
)));
147 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
151 if (!blk
->data_plane
&& !blk
->iothread
) {
155 /* Don't try if transport does not support notifiers. */
156 if (!k
->set_guest_notifiers
|| !k
->set_host_notifier
) {
158 "device is incompatible with x-data-plane "
159 "(transport does not support notifiers)");
163 /* If dataplane is (re-)enabled while the guest is running there could be
164 * block jobs that can conflict.
166 if (bdrv_op_is_blocked(blk
->conf
.bs
, BLOCK_OP_TYPE_DATAPLANE
, &local_err
)) {
167 error_report("cannot start dataplane thread: %s",
168 error_get_pretty(local_err
));
169 error_free(local_err
);
173 s
= g_new0(VirtIOBlockDataPlane
, 1);
178 s
->iothread
= blk
->iothread
;
179 object_ref(OBJECT(s
->iothread
));
181 /* Create per-device IOThread if none specified. This is for
182 * x-data-plane option compatibility. If x-data-plane is removed we
185 object_initialize(&s
->internal_iothread_obj
,
186 sizeof(s
->internal_iothread_obj
),
188 user_creatable_complete(OBJECT(&s
->internal_iothread_obj
), &error_abort
);
189 s
->iothread
= &s
->internal_iothread_obj
;
191 s
->ctx
= iothread_get_aio_context(s
->iothread
);
192 s
->bh
= aio_bh_new(s
->ctx
, notify_guest_bh
, s
);
194 error_setg(&s
->blocker
, "block device is in use by data plane");
195 bdrv_op_block_all(blk
->conf
.bs
, s
->blocker
);
200 /* Context: QEMU global mutex held */
201 void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane
*s
)
207 virtio_blk_data_plane_stop(s
);
208 bdrv_op_unblock_all(s
->blk
->conf
.bs
, s
->blocker
);
209 error_free(s
->blocker
);
210 object_unref(OBJECT(s
->iothread
));
211 qemu_bh_delete(s
->bh
);
215 /* Context: QEMU global mutex held */
216 void virtio_blk_data_plane_start(VirtIOBlockDataPlane
*s
)
218 BusState
*qbus
= BUS(qdev_get_parent_bus(DEVICE(s
->vdev
)));
219 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
220 VirtIOBlock
*vblk
= VIRTIO_BLK(s
->vdev
);
224 if (s
->started
|| s
->disabled
) {
234 vq
= virtio_get_queue(s
->vdev
, 0);
235 if (!vring_setup(&s
->vring
, s
->vdev
, 0)) {
239 /* Set up guest notifier (irq) */
240 r
= k
->set_guest_notifiers(qbus
->parent
, 1, true);
242 fprintf(stderr
, "virtio-blk failed to set guest notifier (%d), "
243 "ensure -enable-kvm is set\n", r
);
244 goto fail_guest_notifiers
;
246 s
->guest_notifier
= virtio_queue_get_guest_notifier(vq
);
248 /* Set up virtqueue notify */
249 r
= k
->set_host_notifier(qbus
->parent
, 0, true);
251 fprintf(stderr
, "virtio-blk failed to set host notifier (%d)\n", r
);
252 goto fail_host_notifier
;
254 s
->host_notifier
= *virtio_queue_get_host_notifier(vq
);
256 s
->saved_complete_request
= vblk
->complete_request
;
257 vblk
->complete_request
= complete_request_vring
;
261 trace_virtio_blk_data_plane_start(s
);
263 bdrv_set_aio_context(s
->blk
->conf
.bs
, s
->ctx
);
265 /* Kick right away to begin processing requests already in vring */
266 event_notifier_set(virtio_queue_get_host_notifier(vq
));
268 /* Get this show started by hooking up our callbacks */
269 aio_context_acquire(s
->ctx
);
270 aio_set_event_notifier(s
->ctx
, &s
->host_notifier
, handle_notify
);
271 aio_context_release(s
->ctx
);
275 k
->set_guest_notifiers(qbus
->parent
, 1, false);
276 fail_guest_notifiers
:
277 vring_teardown(&s
->vring
, s
->vdev
, 0);
283 /* Context: QEMU global mutex held */
284 void virtio_blk_data_plane_stop(VirtIOBlockDataPlane
*s
)
286 BusState
*qbus
= BUS(qdev_get_parent_bus(DEVICE(s
->vdev
)));
287 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
288 VirtIOBlock
*vblk
= VIRTIO_BLK(s
->vdev
);
291 /* Better luck next time. */
296 if (!s
->started
|| s
->stopping
) {
300 vblk
->complete_request
= s
->saved_complete_request
;
301 trace_virtio_blk_data_plane_stop(s
);
303 aio_context_acquire(s
->ctx
);
305 /* Stop notifications for new requests from guest */
306 aio_set_event_notifier(s
->ctx
, &s
->host_notifier
, NULL
);
308 /* Drain and switch bs back to the QEMU main loop */
309 bdrv_set_aio_context(s
->blk
->conf
.bs
, qemu_get_aio_context());
311 aio_context_release(s
->ctx
);
313 /* Sync vring state back to virtqueue so that non-dataplane request
314 * processing can continue when we disable the host notifier below.
316 vring_teardown(&s
->vring
, s
->vdev
, 0);
318 k
->set_host_notifier(qbus
->parent
, 0, false);
320 /* Clean up guest notifier (irq) */
321 k
->set_guest_notifiers(qbus
->parent
, 1, false);