2 * Dedicated thread for virtio-blk I/O processing
4 * Copyright 2012 IBM, Corp.
5 * Copyright 2012 Red Hat, Inc. and/or its affiliates
8 * Stefan Hajnoczi <stefanha@redhat.com>
10 * This work is licensed under the terms of the GNU GPL, version 2 or later.
11 * See the COPYING file in the top-level directory.
15 #include "qemu/osdep.h"
18 #include "qemu/thread.h"
19 #include "qemu/error-report.h"
20 #include "hw/virtio/virtio-access.h"
21 #include "hw/virtio/dataplane/vring.h"
22 #include "hw/virtio/dataplane/vring-accessors.h"
23 #include "sysemu/block-backend.h"
24 #include "hw/virtio/virtio-blk.h"
25 #include "virtio-blk.h"
26 #include "block/aio.h"
27 #include "hw/virtio/virtio-bus.h"
28 #include "qom/object_interfaces.h"
30 struct VirtIOBlockDataPlane
{
39 Vring vring
; /* virtqueue vring */
40 EventNotifier
*guest_notifier
; /* irq */
41 QEMUBH
*bh
; /* bh for guest notification */
43 Notifier insert_notifier
, remove_notifier
;
45 /* Note that these EventNotifiers are assigned by value. This is
46 * fine as long as you do not call event_notifier_cleanup on them
47 * (because you don't own the file descriptor or handle; you just
52 EventNotifier host_notifier
; /* doorbell */
54 /* Operation blocker on BDS */
56 void (*saved_complete_request
)(struct VirtIOBlockReq
*req
,
57 unsigned char status
);
60 /* Raise an interrupt to signal guest, if necessary */
61 static void notify_guest(VirtIOBlockDataPlane
*s
)
63 if (!vring_should_notify(s
->vdev
, &s
->vring
)) {
67 event_notifier_set(s
->guest_notifier
);
70 static void notify_guest_bh(void *opaque
)
72 VirtIOBlockDataPlane
*s
= opaque
;
77 static void complete_request_vring(VirtIOBlockReq
*req
, unsigned char status
)
79 VirtIOBlockDataPlane
*s
= req
->dev
->dataplane
;
80 stb_p(&req
->in
->status
, status
);
82 vring_push(s
->vdev
, &req
->dev
->dataplane
->vring
, &req
->elem
, req
->in_len
);
84 /* Suppress notification to guest by BH and its scheduled
85 * flag because requests are completed as a batch after io
86 * plug & unplug is introduced, and the BH can still be
87 * executed in dataplane aio context even after it is
88 * stopped, so needn't worry about notification loss with BH.
90 qemu_bh_schedule(s
->bh
);
93 static void handle_notify(EventNotifier
*e
)
95 VirtIOBlockDataPlane
*s
= container_of(e
, VirtIOBlockDataPlane
,
97 VirtIOBlock
*vblk
= VIRTIO_BLK(s
->vdev
);
99 event_notifier_test_and_clear(&s
->host_notifier
);
100 blk_io_plug(s
->conf
->conf
.blk
);
102 MultiReqBuffer mrb
= {};
104 /* Disable guest->host notifies to avoid unnecessary vmexits */
105 vring_disable_notification(s
->vdev
, &s
->vring
);
108 VirtIOBlockReq
*req
= vring_pop(s
->vdev
, &s
->vring
,
109 sizeof(VirtIOBlockReq
));
112 break; /* no more requests */
115 virtio_blk_init_request(vblk
, req
);
116 trace_virtio_blk_data_plane_process_request(s
, req
->elem
.out_num
,
120 virtio_blk_handle_request(req
, &mrb
);
124 virtio_blk_submit_multireq(s
->conf
->conf
.blk
, &mrb
);
127 if (likely(!vring_more_avail(s
->vdev
, &s
->vring
))) { /* vring emptied */
128 /* Re-enable guest->host notifies and stop processing the vring.
129 * But if the guest has snuck in more descriptors, keep processing.
131 if (vring_enable_notification(s
->vdev
, &s
->vring
)) {
134 } else { /* fatal error */
138 blk_io_unplug(s
->conf
->conf
.blk
);
141 static void data_plane_set_up_op_blockers(VirtIOBlockDataPlane
*s
)
144 error_setg(&s
->blocker
, "block device is in use by data plane");
145 blk_op_block_all(s
->conf
->conf
.blk
, s
->blocker
);
146 blk_op_unblock(s
->conf
->conf
.blk
, BLOCK_OP_TYPE_RESIZE
, s
->blocker
);
147 blk_op_unblock(s
->conf
->conf
.blk
, BLOCK_OP_TYPE_DRIVE_DEL
, s
->blocker
);
148 blk_op_unblock(s
->conf
->conf
.blk
, BLOCK_OP_TYPE_BACKUP_SOURCE
, s
->blocker
);
149 blk_op_unblock(s
->conf
->conf
.blk
, BLOCK_OP_TYPE_CHANGE
, s
->blocker
);
150 blk_op_unblock(s
->conf
->conf
.blk
, BLOCK_OP_TYPE_COMMIT_SOURCE
, s
->blocker
);
151 blk_op_unblock(s
->conf
->conf
.blk
, BLOCK_OP_TYPE_COMMIT_TARGET
, s
->blocker
);
152 blk_op_unblock(s
->conf
->conf
.blk
, BLOCK_OP_TYPE_EJECT
, s
->blocker
);
153 blk_op_unblock(s
->conf
->conf
.blk
, BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT
,
155 blk_op_unblock(s
->conf
->conf
.blk
, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT
,
157 blk_op_unblock(s
->conf
->conf
.blk
, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT_DELETE
,
159 blk_op_unblock(s
->conf
->conf
.blk
, BLOCK_OP_TYPE_MIRROR_SOURCE
, s
->blocker
);
160 blk_op_unblock(s
->conf
->conf
.blk
, BLOCK_OP_TYPE_STREAM
, s
->blocker
);
161 blk_op_unblock(s
->conf
->conf
.blk
, BLOCK_OP_TYPE_REPLACE
, s
->blocker
);
164 static void data_plane_remove_op_blockers(VirtIOBlockDataPlane
*s
)
167 blk_op_unblock_all(s
->conf
->conf
.blk
, s
->blocker
);
168 error_free(s
->blocker
);
173 static void data_plane_blk_insert_notifier(Notifier
*n
, void *data
)
175 VirtIOBlockDataPlane
*s
= container_of(n
, VirtIOBlockDataPlane
,
177 assert(s
->conf
->conf
.blk
== data
);
178 data_plane_set_up_op_blockers(s
);
181 static void data_plane_blk_remove_notifier(Notifier
*n
, void *data
)
183 VirtIOBlockDataPlane
*s
= container_of(n
, VirtIOBlockDataPlane
,
185 assert(s
->conf
->conf
.blk
== data
);
186 data_plane_remove_op_blockers(s
);
189 /* Context: QEMU global mutex held */
190 void virtio_blk_data_plane_create(VirtIODevice
*vdev
, VirtIOBlkConf
*conf
,
191 VirtIOBlockDataPlane
**dataplane
,
194 VirtIOBlockDataPlane
*s
;
195 BusState
*qbus
= BUS(qdev_get_parent_bus(DEVICE(vdev
)));
196 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
200 if (!conf
->iothread
) {
204 /* Don't try if transport does not support notifiers. */
205 if (!k
->set_guest_notifiers
|| !k
->set_host_notifier
) {
207 "device is incompatible with dataplane "
208 "(transport does not support notifiers)");
212 /* If dataplane is (re-)enabled while the guest is running there could be
213 * block jobs that can conflict.
215 if (blk_op_is_blocked(conf
->conf
.blk
, BLOCK_OP_TYPE_DATAPLANE
, errp
)) {
216 error_prepend(errp
, "cannot start dataplane thread: ");
220 s
= g_new0(VirtIOBlockDataPlane
, 1);
224 if (conf
->iothread
) {
225 s
->iothread
= conf
->iothread
;
226 object_ref(OBJECT(s
->iothread
));
228 s
->ctx
= iothread_get_aio_context(s
->iothread
);
229 s
->bh
= aio_bh_new(s
->ctx
, notify_guest_bh
, s
);
231 s
->insert_notifier
.notify
= data_plane_blk_insert_notifier
;
232 s
->remove_notifier
.notify
= data_plane_blk_remove_notifier
;
233 blk_add_insert_bs_notifier(conf
->conf
.blk
, &s
->insert_notifier
);
234 blk_add_remove_bs_notifier(conf
->conf
.blk
, &s
->remove_notifier
);
236 data_plane_set_up_op_blockers(s
);
241 /* Context: QEMU global mutex held */
242 void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane
*s
)
248 virtio_blk_data_plane_stop(s
);
249 data_plane_remove_op_blockers(s
);
250 notifier_remove(&s
->insert_notifier
);
251 notifier_remove(&s
->remove_notifier
);
252 qemu_bh_delete(s
->bh
);
253 object_unref(OBJECT(s
->iothread
));
257 /* Context: QEMU global mutex held */
258 void virtio_blk_data_plane_start(VirtIOBlockDataPlane
*s
)
260 BusState
*qbus
= BUS(qdev_get_parent_bus(DEVICE(s
->vdev
)));
261 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
262 VirtIOBlock
*vblk
= VIRTIO_BLK(s
->vdev
);
266 if (s
->started
|| s
->disabled
) {
276 vq
= virtio_get_queue(s
->vdev
, 0);
277 if (!vring_setup(&s
->vring
, s
->vdev
, 0)) {
281 /* Set up guest notifier (irq) */
282 r
= k
->set_guest_notifiers(qbus
->parent
, 1, true);
284 fprintf(stderr
, "virtio-blk failed to set guest notifier (%d), "
285 "ensure -enable-kvm is set\n", r
);
286 goto fail_guest_notifiers
;
288 s
->guest_notifier
= virtio_queue_get_guest_notifier(vq
);
290 /* Set up virtqueue notify */
291 r
= k
->set_host_notifier(qbus
->parent
, 0, true);
293 fprintf(stderr
, "virtio-blk failed to set host notifier (%d)\n", r
);
294 goto fail_host_notifier
;
296 s
->host_notifier
= *virtio_queue_get_host_notifier(vq
);
298 s
->saved_complete_request
= vblk
->complete_request
;
299 vblk
->complete_request
= complete_request_vring
;
303 trace_virtio_blk_data_plane_start(s
);
305 blk_set_aio_context(s
->conf
->conf
.blk
, s
->ctx
);
307 /* Kick right away to begin processing requests already in vring */
308 event_notifier_set(virtio_queue_get_host_notifier(vq
));
310 /* Get this show started by hooking up our callbacks */
311 aio_context_acquire(s
->ctx
);
312 aio_set_event_notifier(s
->ctx
, &s
->host_notifier
, true,
314 aio_context_release(s
->ctx
);
318 k
->set_guest_notifiers(qbus
->parent
, 1, false);
319 fail_guest_notifiers
:
320 vring_teardown(&s
->vring
, s
->vdev
, 0);
326 /* Context: QEMU global mutex held */
327 void virtio_blk_data_plane_stop(VirtIOBlockDataPlane
*s
)
329 BusState
*qbus
= BUS(qdev_get_parent_bus(DEVICE(s
->vdev
)));
330 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
331 VirtIOBlock
*vblk
= VIRTIO_BLK(s
->vdev
);
334 /* Better luck next time. */
339 if (!s
->started
|| s
->stopping
) {
343 vblk
->complete_request
= s
->saved_complete_request
;
344 trace_virtio_blk_data_plane_stop(s
);
346 aio_context_acquire(s
->ctx
);
348 /* Stop notifications for new requests from guest */
349 aio_set_event_notifier(s
->ctx
, &s
->host_notifier
, true, NULL
);
351 /* Drain and switch bs back to the QEMU main loop */
352 blk_set_aio_context(s
->conf
->conf
.blk
, qemu_get_aio_context());
354 aio_context_release(s
->ctx
);
356 /* Sync vring state back to virtqueue so that non-dataplane request
357 * processing can continue when we disable the host notifier below.
359 vring_teardown(&s
->vring
, s
->vdev
, 0);
361 k
->set_host_notifier(qbus
->parent
, 0, false);
363 /* Clean up guest notifier (irq) */
364 k
->set_guest_notifiers(qbus
->parent
, 1, false);