2 * Dedicated thread for virtio-blk I/O processing
4 * Copyright 2012 IBM, Corp.
5 * Copyright 2012 Red Hat, Inc. and/or its affiliates
8 * Stefan Hajnoczi <stefanha@redhat.com>
10 * This work is licensed under the terms of the GNU GPL, version 2 or later.
11 * See the COPYING file in the top-level directory.
17 #include "qemu/thread.h"
18 #include "qemu/error-report.h"
19 #include "hw/virtio/dataplane/vring.h"
20 #include "sysemu/block-backend.h"
21 #include "hw/virtio/virtio-blk.h"
22 #include "virtio-blk.h"
23 #include "block/aio.h"
24 #include "hw/virtio/virtio-bus.h"
25 #include "qom/object_interfaces.h"
27 struct VirtIOBlockDataPlane
{
36 Vring vring
; /* virtqueue vring */
37 EventNotifier
*guest_notifier
; /* irq */
38 QEMUBH
*bh
; /* bh for guest notification */
40 /* Note that these EventNotifiers are assigned by value. This is
41 * fine as long as you do not call event_notifier_cleanup on them
42 * (because you don't own the file descriptor or handle; you just
46 IOThread internal_iothread_obj
;
48 EventNotifier host_notifier
; /* doorbell */
50 /* Operation blocker on BDS */
52 void (*saved_complete_request
)(struct VirtIOBlockReq
*req
,
53 unsigned char status
);
56 /* Raise an interrupt to signal guest, if necessary */
57 static void notify_guest(VirtIOBlockDataPlane
*s
)
59 if (!vring_should_notify(s
->vdev
, &s
->vring
)) {
63 event_notifier_set(s
->guest_notifier
);
66 static void notify_guest_bh(void *opaque
)
68 VirtIOBlockDataPlane
*s
= opaque
;
73 static void complete_request_vring(VirtIOBlockReq
*req
, unsigned char status
)
75 VirtIOBlockDataPlane
*s
= req
->dev
->dataplane
;
76 stb_p(&req
->in
->status
, status
);
78 vring_push(&req
->dev
->dataplane
->vring
, &req
->elem
,
79 req
->qiov
.size
+ sizeof(*req
->in
));
81 /* Suppress notification to guest by BH and its scheduled
82 * flag because requests are completed as a batch after io
83 * plug & unplug is introduced, and the BH can still be
84 * executed in dataplane aio context even after it is
85 * stopped, so needn't worry about notification loss with BH.
87 qemu_bh_schedule(s
->bh
);
90 static void handle_notify(EventNotifier
*e
)
92 VirtIOBlockDataPlane
*s
= container_of(e
, VirtIOBlockDataPlane
,
94 VirtIOBlock
*vblk
= VIRTIO_BLK(s
->vdev
);
96 event_notifier_test_and_clear(&s
->host_notifier
);
97 blk_io_plug(s
->conf
->conf
.blk
);
99 MultiReqBuffer mrb
= {
104 /* Disable guest->host notifies to avoid unnecessary vmexits */
105 vring_disable_notification(s
->vdev
, &s
->vring
);
108 VirtIOBlockReq
*req
= virtio_blk_alloc_request(vblk
);
110 ret
= vring_pop(s
->vdev
, &s
->vring
, &req
->elem
);
112 virtio_blk_free_request(req
);
113 break; /* no more requests */
116 trace_virtio_blk_data_plane_process_request(s
, req
->elem
.out_num
,
120 virtio_blk_handle_request(req
, &mrb
);
123 virtio_submit_multiwrite(s
->conf
->conf
.blk
, &mrb
);
125 if (likely(ret
== -EAGAIN
)) { /* vring emptied */
126 /* Re-enable guest->host notifies and stop processing the vring.
127 * But if the guest has snuck in more descriptors, keep processing.
129 if (vring_enable_notification(s
->vdev
, &s
->vring
)) {
132 } else { /* fatal error */
136 blk_io_unplug(s
->conf
->conf
.blk
);
139 /* Context: QEMU global mutex held */
140 void virtio_blk_data_plane_create(VirtIODevice
*vdev
, VirtIOBlkConf
*conf
,
141 VirtIOBlockDataPlane
**dataplane
,
144 VirtIOBlockDataPlane
*s
;
145 Error
*local_err
= NULL
;
146 BusState
*qbus
= BUS(qdev_get_parent_bus(DEVICE(vdev
)));
147 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
151 if (!conf
->data_plane
&& !conf
->iothread
) {
155 /* Don't try if transport does not support notifiers. */
156 if (!k
->set_guest_notifiers
|| !k
->set_host_notifier
) {
158 "device is incompatible with x-data-plane "
159 "(transport does not support notifiers)");
163 /* If dataplane is (re-)enabled while the guest is running there could be
164 * block jobs that can conflict.
166 if (blk_op_is_blocked(conf
->conf
.blk
, BLOCK_OP_TYPE_DATAPLANE
,
168 error_setg(errp
, "cannot start dataplane thread: %s",
169 error_get_pretty(local_err
));
170 error_free(local_err
);
174 s
= g_new0(VirtIOBlockDataPlane
, 1);
178 if (conf
->iothread
) {
179 s
->iothread
= conf
->iothread
;
180 object_ref(OBJECT(s
->iothread
));
182 /* Create per-device IOThread if none specified. This is for
183 * x-data-plane option compatibility. If x-data-plane is removed we
186 object_initialize(&s
->internal_iothread_obj
,
187 sizeof(s
->internal_iothread_obj
),
189 user_creatable_complete(OBJECT(&s
->internal_iothread_obj
), &error_abort
);
190 s
->iothread
= &s
->internal_iothread_obj
;
192 s
->ctx
= iothread_get_aio_context(s
->iothread
);
193 s
->bh
= aio_bh_new(s
->ctx
, notify_guest_bh
, s
);
195 error_setg(&s
->blocker
, "block device is in use by data plane");
196 blk_op_block_all(conf
->conf
.blk
, s
->blocker
);
197 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_RESIZE
, s
->blocker
);
198 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_DRIVE_DEL
, s
->blocker
);
199 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_BACKUP_SOURCE
, s
->blocker
);
200 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_COMMIT
, s
->blocker
);
201 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_EJECT
, s
->blocker
);
202 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT_DELETE
,
204 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_MIRROR
, s
->blocker
);
205 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_STREAM
, s
->blocker
);
206 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_REPLACE
, s
->blocker
);
211 /* Context: QEMU global mutex held */
212 void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane
*s
)
218 virtio_blk_data_plane_stop(s
);
219 blk_op_unblock_all(s
->conf
->conf
.blk
, s
->blocker
);
220 error_free(s
->blocker
);
221 object_unref(OBJECT(s
->iothread
));
222 qemu_bh_delete(s
->bh
);
226 /* Context: QEMU global mutex held */
227 void virtio_blk_data_plane_start(VirtIOBlockDataPlane
*s
)
229 BusState
*qbus
= BUS(qdev_get_parent_bus(DEVICE(s
->vdev
)));
230 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
231 VirtIOBlock
*vblk
= VIRTIO_BLK(s
->vdev
);
235 if (s
->started
|| s
->disabled
) {
245 vq
= virtio_get_queue(s
->vdev
, 0);
246 if (!vring_setup(&s
->vring
, s
->vdev
, 0)) {
250 /* Set up guest notifier (irq) */
251 r
= k
->set_guest_notifiers(qbus
->parent
, 1, true);
253 fprintf(stderr
, "virtio-blk failed to set guest notifier (%d), "
254 "ensure -enable-kvm is set\n", r
);
255 goto fail_guest_notifiers
;
257 s
->guest_notifier
= virtio_queue_get_guest_notifier(vq
);
259 /* Set up virtqueue notify */
260 r
= k
->set_host_notifier(qbus
->parent
, 0, true);
262 fprintf(stderr
, "virtio-blk failed to set host notifier (%d)\n", r
);
263 goto fail_host_notifier
;
265 s
->host_notifier
= *virtio_queue_get_host_notifier(vq
);
267 s
->saved_complete_request
= vblk
->complete_request
;
268 vblk
->complete_request
= complete_request_vring
;
272 trace_virtio_blk_data_plane_start(s
);
274 blk_set_aio_context(s
->conf
->conf
.blk
, s
->ctx
);
276 /* Kick right away to begin processing requests already in vring */
277 event_notifier_set(virtio_queue_get_host_notifier(vq
));
279 /* Get this show started by hooking up our callbacks */
280 aio_context_acquire(s
->ctx
);
281 aio_set_event_notifier(s
->ctx
, &s
->host_notifier
, handle_notify
);
282 aio_context_release(s
->ctx
);
286 k
->set_guest_notifiers(qbus
->parent
, 1, false);
287 fail_guest_notifiers
:
288 vring_teardown(&s
->vring
, s
->vdev
, 0);
294 /* Context: QEMU global mutex held */
295 void virtio_blk_data_plane_stop(VirtIOBlockDataPlane
*s
)
297 BusState
*qbus
= BUS(qdev_get_parent_bus(DEVICE(s
->vdev
)));
298 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
299 VirtIOBlock
*vblk
= VIRTIO_BLK(s
->vdev
);
302 /* Better luck next time. */
307 if (!s
->started
|| s
->stopping
) {
311 vblk
->complete_request
= s
->saved_complete_request
;
312 trace_virtio_blk_data_plane_stop(s
);
314 aio_context_acquire(s
->ctx
);
316 /* Stop notifications for new requests from guest */
317 aio_set_event_notifier(s
->ctx
, &s
->host_notifier
, NULL
);
319 /* Drain and switch bs back to the QEMU main loop */
320 blk_set_aio_context(s
->conf
->conf
.blk
, qemu_get_aio_context());
322 aio_context_release(s
->ctx
);
324 /* Sync vring state back to virtqueue so that non-dataplane request
325 * processing can continue when we disable the host notifier below.
327 vring_teardown(&s
->vring
, s
->vdev
, 0);
329 k
->set_host_notifier(qbus
->parent
, 0, false);
331 /* Clean up guest notifier (irq) */
332 k
->set_guest_notifiers(qbus
->parent
, 1, false);