2 * Dedicated thread for virtio-blk I/O processing
4 * Copyright 2012 IBM, Corp.
5 * Copyright 2012 Red Hat, Inc. and/or its affiliates
8 * Stefan Hajnoczi <stefanha@redhat.com>
10 * This work is licensed under the terms of the GNU GPL, version 2 or later.
11 * See the COPYING file in the top-level directory.
17 #include "qemu/thread.h"
18 #include "qemu/error-report.h"
19 #include "hw/virtio/virtio-access.h"
20 #include "hw/virtio/dataplane/vring.h"
21 #include "hw/virtio/dataplane/vring-accessors.h"
22 #include "sysemu/block-backend.h"
23 #include "hw/virtio/virtio-blk.h"
24 #include "virtio-blk.h"
25 #include "block/aio.h"
26 #include "hw/virtio/virtio-bus.h"
27 #include "qom/object_interfaces.h"
29 struct VirtIOBlockDataPlane
{
38 Vring vring
; /* virtqueue vring */
39 EventNotifier
*guest_notifier
; /* irq */
40 QEMUBH
*bh
; /* bh for guest notification */
42 /* Note that these EventNotifiers are assigned by value. This is
43 * fine as long as you do not call event_notifier_cleanup on them
44 * (because you don't own the file descriptor or handle; you just
49 EventNotifier host_notifier
; /* doorbell */
51 /* Operation blocker on BDS */
53 void (*saved_complete_request
)(struct VirtIOBlockReq
*req
,
54 unsigned char status
);
57 /* Raise an interrupt to signal guest, if necessary */
58 static void notify_guest(VirtIOBlockDataPlane
*s
)
60 if (!vring_should_notify(s
->vdev
, &s
->vring
)) {
64 event_notifier_set(s
->guest_notifier
);
67 static void notify_guest_bh(void *opaque
)
69 VirtIOBlockDataPlane
*s
= opaque
;
74 static void complete_request_vring(VirtIOBlockReq
*req
, unsigned char status
)
76 VirtIOBlockDataPlane
*s
= req
->dev
->dataplane
;
77 stb_p(&req
->in
->status
, status
);
79 vring_push(s
->vdev
, &req
->dev
->dataplane
->vring
, &req
->elem
, req
->in_len
);
81 /* Suppress notification to guest by BH and its scheduled
82 * flag because requests are completed as a batch after io
83 * plug & unplug is introduced, and the BH can still be
84 * executed in dataplane aio context even after it is
85 * stopped, so needn't worry about notification loss with BH.
87 qemu_bh_schedule(s
->bh
);
90 static void handle_notify(EventNotifier
*e
)
92 VirtIOBlockDataPlane
*s
= container_of(e
, VirtIOBlockDataPlane
,
94 VirtIOBlock
*vblk
= VIRTIO_BLK(s
->vdev
);
96 event_notifier_test_and_clear(&s
->host_notifier
);
97 blk_io_plug(s
->conf
->conf
.blk
);
99 MultiReqBuffer mrb
= {};
102 /* Disable guest->host notifies to avoid unnecessary vmexits */
103 vring_disable_notification(s
->vdev
, &s
->vring
);
106 VirtIOBlockReq
*req
= virtio_blk_alloc_request(vblk
);
108 ret
= vring_pop(s
->vdev
, &s
->vring
, &req
->elem
);
110 virtio_blk_free_request(req
);
111 break; /* no more requests */
114 trace_virtio_blk_data_plane_process_request(s
, req
->elem
.out_num
,
118 virtio_blk_handle_request(req
, &mrb
);
122 virtio_blk_submit_multireq(s
->conf
->conf
.blk
, &mrb
);
125 if (likely(ret
== -EAGAIN
)) { /* vring emptied */
126 /* Re-enable guest->host notifies and stop processing the vring.
127 * But if the guest has snuck in more descriptors, keep processing.
129 if (vring_enable_notification(s
->vdev
, &s
->vring
)) {
132 } else { /* fatal error */
136 blk_io_unplug(s
->conf
->conf
.blk
);
139 /* Context: QEMU global mutex held */
140 void virtio_blk_data_plane_create(VirtIODevice
*vdev
, VirtIOBlkConf
*conf
,
141 VirtIOBlockDataPlane
**dataplane
,
144 VirtIOBlockDataPlane
*s
;
145 Error
*local_err
= NULL
;
146 BusState
*qbus
= BUS(qdev_get_parent_bus(DEVICE(vdev
)));
147 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
151 if (!conf
->iothread
) {
155 /* Don't try if transport does not support notifiers. */
156 if (!k
->set_guest_notifiers
|| !k
->set_host_notifier
) {
158 "device is incompatible with dataplane "
159 "(transport does not support notifiers)");
163 /* If dataplane is (re-)enabled while the guest is running there could be
164 * block jobs that can conflict.
166 if (blk_op_is_blocked(conf
->conf
.blk
, BLOCK_OP_TYPE_DATAPLANE
,
168 error_setg(errp
, "cannot start dataplane thread: %s",
169 error_get_pretty(local_err
));
170 error_free(local_err
);
174 s
= g_new0(VirtIOBlockDataPlane
, 1);
178 if (conf
->iothread
) {
179 s
->iothread
= conf
->iothread
;
180 object_ref(OBJECT(s
->iothread
));
182 s
->ctx
= iothread_get_aio_context(s
->iothread
);
183 s
->bh
= aio_bh_new(s
->ctx
, notify_guest_bh
, s
);
185 error_setg(&s
->blocker
, "block device is in use by data plane");
186 blk_op_block_all(conf
->conf
.blk
, s
->blocker
);
187 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_RESIZE
, s
->blocker
);
188 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_DRIVE_DEL
, s
->blocker
);
189 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_BACKUP_SOURCE
, s
->blocker
);
190 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_CHANGE
, s
->blocker
);
191 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_COMMIT_SOURCE
, s
->blocker
);
192 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_COMMIT_TARGET
, s
->blocker
);
193 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_EJECT
, s
->blocker
);
194 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT
, s
->blocker
);
195 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT
, s
->blocker
);
196 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT_DELETE
,
198 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_MIRROR_SOURCE
, s
->blocker
);
199 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_STREAM
, s
->blocker
);
200 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_REPLACE
, s
->blocker
);
205 /* Context: QEMU global mutex held */
206 void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane
*s
)
212 virtio_blk_data_plane_stop(s
);
213 blk_op_unblock_all(s
->conf
->conf
.blk
, s
->blocker
);
214 error_free(s
->blocker
);
215 qemu_bh_delete(s
->bh
);
216 object_unref(OBJECT(s
->iothread
));
220 /* Context: QEMU global mutex held */
221 void virtio_blk_data_plane_start(VirtIOBlockDataPlane
*s
)
223 BusState
*qbus
= BUS(qdev_get_parent_bus(DEVICE(s
->vdev
)));
224 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
225 VirtIOBlock
*vblk
= VIRTIO_BLK(s
->vdev
);
229 if (s
->started
|| s
->disabled
) {
239 vq
= virtio_get_queue(s
->vdev
, 0);
240 if (!vring_setup(&s
->vring
, s
->vdev
, 0)) {
244 /* Set up guest notifier (irq) */
245 r
= k
->set_guest_notifiers(qbus
->parent
, 1, true);
247 fprintf(stderr
, "virtio-blk failed to set guest notifier (%d), "
248 "ensure -enable-kvm is set\n", r
);
249 goto fail_guest_notifiers
;
251 s
->guest_notifier
= virtio_queue_get_guest_notifier(vq
);
253 /* Set up virtqueue notify */
254 r
= k
->set_host_notifier(qbus
->parent
, 0, true);
256 fprintf(stderr
, "virtio-blk failed to set host notifier (%d)\n", r
);
257 goto fail_host_notifier
;
259 s
->host_notifier
= *virtio_queue_get_host_notifier(vq
);
261 s
->saved_complete_request
= vblk
->complete_request
;
262 vblk
->complete_request
= complete_request_vring
;
266 trace_virtio_blk_data_plane_start(s
);
268 blk_set_aio_context(s
->conf
->conf
.blk
, s
->ctx
);
270 /* Kick right away to begin processing requests already in vring */
271 event_notifier_set(virtio_queue_get_host_notifier(vq
));
273 /* Get this show started by hooking up our callbacks */
274 aio_context_acquire(s
->ctx
);
275 aio_set_event_notifier(s
->ctx
, &s
->host_notifier
, true,
277 aio_context_release(s
->ctx
);
281 k
->set_guest_notifiers(qbus
->parent
, 1, false);
282 fail_guest_notifiers
:
283 vring_teardown(&s
->vring
, s
->vdev
, 0);
289 /* Context: QEMU global mutex held */
290 void virtio_blk_data_plane_stop(VirtIOBlockDataPlane
*s
)
292 BusState
*qbus
= BUS(qdev_get_parent_bus(DEVICE(s
->vdev
)));
293 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
294 VirtIOBlock
*vblk
= VIRTIO_BLK(s
->vdev
);
297 /* Better luck next time. */
302 if (!s
->started
|| s
->stopping
) {
306 vblk
->complete_request
= s
->saved_complete_request
;
307 trace_virtio_blk_data_plane_stop(s
);
309 aio_context_acquire(s
->ctx
);
311 /* Stop notifications for new requests from guest */
312 aio_set_event_notifier(s
->ctx
, &s
->host_notifier
, true, NULL
);
314 /* Drain and switch bs back to the QEMU main loop */
315 blk_set_aio_context(s
->conf
->conf
.blk
, qemu_get_aio_context());
317 aio_context_release(s
->ctx
);
319 /* Sync vring state back to virtqueue so that non-dataplane request
320 * processing can continue when we disable the host notifier below.
322 vring_teardown(&s
->vring
, s
->vdev
, 0);
324 k
->set_host_notifier(qbus
->parent
, 0, false);
326 /* Clean up guest notifier (irq) */
327 k
->set_guest_notifiers(qbus
->parent
, 1, false);