hw/block/dataplane/virtio-blk.c

   1 /*
   2  * Dedicated thread for virtio-blk I/O processing
   3  *
   4  * Copyright 2012 IBM, Corp.
   5  * Copyright 2012 Red Hat, Inc. and/or its affiliates
   6  *
   7  * Authors:
   8  *   Stefan Hajnoczi <stefanha@redhat.com>
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  11  * See the COPYING file in the top-level directory.
  12  *
  13  */
  14
  15 #include "qemu/osdep.h"
  16 #include "trace.h"
  17 #include "qemu/iov.h"
  18 #include "qemu/thread.h"
  19 #include "qemu/error-report.h"
  20 #include "hw/virtio/virtio-access.h"
  21 #include "hw/virtio/dataplane/vring.h"
  22 #include "hw/virtio/dataplane/vring-accessors.h"
  23 #include "sysemu/block-backend.h"
  24 #include "hw/virtio/virtio-blk.h"
  25 #include "virtio-blk.h"
  26 #include "block/aio.h"
  27 #include "hw/virtio/virtio-bus.h"
  28 #include "qom/object_interfaces.h"
  29
  30 struct VirtIOBlockDataPlane {
  31     bool started;
  32     bool starting;
  33     bool stopping;
  34     bool disabled;
  35
  36     VirtIOBlkConf *conf;
  37
  38     VirtIODevice *vdev;
  39     Vring vring;                    /* virtqueue vring */
  40     EventNotifier *guest_notifier;  /* irq */
  41     QEMUBH *bh;                     /* bh for guest notification */
  42
  43     Notifier insert_notifier, remove_notifier;
  44
  45     /* Note that these EventNotifiers are assigned by value.  This is
  46      * fine as long as you do not call event_notifier_cleanup on them
  47      * (because you don't own the file descriptor or handle; you just
  48      * use it).
  49      */
  50     IOThread *iothread;
  51     AioContext *ctx;
  52     EventNotifier host_notifier;    /* doorbell */
  53
  54     /* Operation blocker on BDS */
  55     Error *blocker;
  56     void (*saved_complete_request)(struct VirtIOBlockReq *req,
  57                                    unsigned char status);
  58 };
  59
  60 /* Raise an interrupt to signal guest, if necessary */
  61 static void notify_guest(VirtIOBlockDataPlane *s)
  62 {
  63     if (!vring_should_notify(s->vdev, &s->vring)) {
  64         return;
  65     }
  66
  67     event_notifier_set(s->guest_notifier);
  68 }
  69
  70 static void notify_guest_bh(void *opaque)
  71 {
  72     VirtIOBlockDataPlane *s = opaque;
  73
  74     notify_guest(s);
  75 }
  76
  77 static void complete_request_vring(VirtIOBlockReq *req, unsigned char status)
  78 {
  79     VirtIOBlockDataPlane *s = req->dev->dataplane;
  80     stb_p(&req->in->status, status);
  81
  82     vring_push(s->vdev, &req->dev->dataplane->vring, &req->elem, req->in_len);
  83
  84     /* Suppress notification to guest by BH and its scheduled
  85      * flag because requests are completed as a batch after io
  86      * plug & unplug is introduced, and the BH can still be
  87      * executed in dataplane aio context even after it is
  88      * stopped, so needn't worry about notification loss with BH.
  89      */
  90     qemu_bh_schedule(s->bh);
  91 }
  92
  93 static void handle_notify(EventNotifier *e)
  94 {
  95     VirtIOBlockDataPlane *s = container_of(e, VirtIOBlockDataPlane,
  96                                            host_notifier);
  97     VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
  98
  99     event_notifier_test_and_clear(&s->host_notifier);
 100     blk_io_plug(s->conf->conf.blk);
 101     for (;;) {
 102         MultiReqBuffer mrb = {};
 103
 104         /* Disable guest->host notifies to avoid unnecessary vmexits */
 105         vring_disable_notification(s->vdev, &s->vring);
 106
 107         for (;;) {
 108             VirtIOBlockReq *req = vring_pop(s->vdev, &s->vring,
 109                                             sizeof(VirtIOBlockReq));
 110
 111             if (req == NULL) {
 112                 break; /* no more requests */
 113             }
 114
 115             virtio_blk_init_request(vblk, req);
 116             trace_virtio_blk_data_plane_process_request(s, req->elem.out_num,
 117                                                         req->elem.in_num,
 118                                                         req->elem.index);
 119
 120             virtio_blk_handle_request(req, &mrb);
 121         }
 122
 123         if (mrb.num_reqs) {
 124             virtio_blk_submit_multireq(s->conf->conf.blk, &mrb);
 125         }
 126
 127         if (likely(!vring_more_avail(s->vdev, &s->vring))) { /* vring emptied */
 128             /* Re-enable guest->host notifies and stop processing the vring.
 129              * But if the guest has snuck in more descriptors, keep processing.
 130              */
 131             if (vring_enable_notification(s->vdev, &s->vring)) {
 132                 break;
 133             }
 134         } else { /* fatal error */
 135             break;
 136         }
 137     }
 138     blk_io_unplug(s->conf->conf.blk);
 139 }
 140
 141 static void data_plane_set_up_op_blockers(VirtIOBlockDataPlane *s)
 142 {
 143     assert(!s->blocker);
 144     error_setg(&s->blocker, "block device is in use by data plane");
 145     blk_op_block_all(s->conf->conf.blk, s->blocker);
 146     blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_RESIZE, s->blocker);
 147     blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_DRIVE_DEL, s->blocker);
 148     blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_BACKUP_SOURCE, s->blocker);
 149     blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_CHANGE, s->blocker);
 150     blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_COMMIT_SOURCE, s->blocker);
 151     blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_COMMIT_TARGET, s->blocker);
 152     blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_EJECT, s->blocker);
 153     blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT,
 154                    s->blocker);
 155     blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT,
 156                    s->blocker);
 157     blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT_DELETE,
 158                    s->blocker);
 159     blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_MIRROR_SOURCE, s->blocker);
 160     blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_STREAM, s->blocker);
 161     blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_REPLACE, s->blocker);
 162 }
 163
 164 static void data_plane_remove_op_blockers(VirtIOBlockDataPlane *s)
 165 {
 166     if (s->blocker) {
 167         blk_op_unblock_all(s->conf->conf.blk, s->blocker);
 168         error_free(s->blocker);
 169         s->blocker = NULL;
 170     }
 171 }
 172
 173 static void data_plane_blk_insert_notifier(Notifier *n, void *data)
 174 {
 175     VirtIOBlockDataPlane *s = container_of(n, VirtIOBlockDataPlane,
 176                                            insert_notifier);
 177     assert(s->conf->conf.blk == data);
 178     data_plane_set_up_op_blockers(s);
 179 }
 180
 181 static void data_plane_blk_remove_notifier(Notifier *n, void *data)
 182 {
 183     VirtIOBlockDataPlane *s = container_of(n, VirtIOBlockDataPlane,
 184                                            remove_notifier);
 185     assert(s->conf->conf.blk == data);
 186     data_plane_remove_op_blockers(s);
 187 }
 188
 189 /* Context: QEMU global mutex held */
 190 void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
 191                                   VirtIOBlockDataPlane **dataplane,
 192                                   Error **errp)
 193 {
 194     VirtIOBlockDataPlane *s;
 195     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
 196     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
 197
 198     *dataplane = NULL;
 199
 200     if (!conf->iothread) {
 201         return;
 202     }
 203
 204     /* Don't try if transport does not support notifiers. */
 205     if (!k->set_guest_notifiers || !k->set_host_notifier) {
 206         error_setg(errp,
 207                    "device is incompatible with dataplane "
 208                    "(transport does not support notifiers)");
 209         return;
 210     }
 211
 212     /* If dataplane is (re-)enabled while the guest is running there could be
 213      * block jobs that can conflict.
 214      */
 215     if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) {
 216         error_prepend(errp, "cannot start dataplane thread: ");
 217         return;
 218     }
 219
 220     s = g_new0(VirtIOBlockDataPlane, 1);
 221     s->vdev = vdev;
 222     s->conf = conf;
 223
 224     if (conf->iothread) {
 225         s->iothread = conf->iothread;
 226         object_ref(OBJECT(s->iothread));
 227     }
 228     s->ctx = iothread_get_aio_context(s->iothread);
 229     s->bh = aio_bh_new(s->ctx, notify_guest_bh, s);
 230
 231     s->insert_notifier.notify = data_plane_blk_insert_notifier;
 232     s->remove_notifier.notify = data_plane_blk_remove_notifier;
 233     blk_add_insert_bs_notifier(conf->conf.blk, &s->insert_notifier);
 234     blk_add_remove_bs_notifier(conf->conf.blk, &s->remove_notifier);
 235
 236     data_plane_set_up_op_blockers(s);
 237
 238     *dataplane = s;
 239 }
 240
 241 /* Context: QEMU global mutex held */
 242 void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s)
 243 {
 244     if (!s) {
 245         return;
 246     }
 247
 248     virtio_blk_data_plane_stop(s);
 249     data_plane_remove_op_blockers(s);
 250     notifier_remove(&s->insert_notifier);
 251     notifier_remove(&s->remove_notifier);
 252     qemu_bh_delete(s->bh);
 253     object_unref(OBJECT(s->iothread));
 254     g_free(s);
 255 }
 256
 257 /* Context: QEMU global mutex held */
 258 void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s)
 259 {
 260     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s->vdev)));
 261     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
 262     VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
 263     VirtQueue *vq;
 264     int r;
 265
 266     if (s->started || s->disabled) {
 267         return;
 268     }
 269
 270     if (s->starting) {
 271         return;
 272     }
 273
 274     s->starting = true;
 275
 276     vq = virtio_get_queue(s->vdev, 0);
 277     if (!vring_setup(&s->vring, s->vdev, 0)) {
 278         goto fail_vring;
 279     }
 280
 281     /* Set up guest notifier (irq) */
 282     r = k->set_guest_notifiers(qbus->parent, 1, true);
 283     if (r != 0) {
 284         fprintf(stderr, "virtio-blk failed to set guest notifier (%d), "
 285                 "ensure -enable-kvm is set\n", r);
 286         goto fail_guest_notifiers;
 287     }
 288     s->guest_notifier = virtio_queue_get_guest_notifier(vq);
 289
 290     /* Set up virtqueue notify */
 291     r = k->set_host_notifier(qbus->parent, 0, true);
 292     if (r != 0) {
 293         fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r);
 294         goto fail_host_notifier;
 295     }
 296     s->host_notifier = *virtio_queue_get_host_notifier(vq);
 297
 298     s->saved_complete_request = vblk->complete_request;
 299     vblk->complete_request = complete_request_vring;
 300
 301     s->starting = false;
 302     s->started = true;
 303     trace_virtio_blk_data_plane_start(s);
 304
 305     blk_set_aio_context(s->conf->conf.blk, s->ctx);
 306
 307     /* Kick right away to begin processing requests already in vring */
 308     event_notifier_set(virtio_queue_get_host_notifier(vq));
 309
 310     /* Get this show started by hooking up our callbacks */
 311     aio_context_acquire(s->ctx);
 312     aio_set_event_notifier(s->ctx, &s->host_notifier, true,
 313                            handle_notify);
 314     aio_context_release(s->ctx);
 315     return;
 316
 317   fail_host_notifier:
 318     k->set_guest_notifiers(qbus->parent, 1, false);
 319   fail_guest_notifiers:
 320     vring_teardown(&s->vring, s->vdev, 0);
 321     s->disabled = true;
 322   fail_vring:
 323     s->starting = false;
 324 }
 325
 326 /* Context: QEMU global mutex held */
 327 void virtio_blk_data_plane_stop(VirtIOBlockDataPlane *s)
 328 {
 329     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s->vdev)));
 330     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
 331     VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
 332
 333
 334     /* Better luck next time. */
 335     if (s->disabled) {
 336         s->disabled = false;
 337         return;
 338     }
 339     if (!s->started || s->stopping) {
 340         return;
 341     }
 342     s->stopping = true;
 343     vblk->complete_request = s->saved_complete_request;
 344     trace_virtio_blk_data_plane_stop(s);
 345
 346     aio_context_acquire(s->ctx);
 347
 348     /* Stop notifications for new requests from guest */
 349     aio_set_event_notifier(s->ctx, &s->host_notifier, true, NULL);
 350
 351     /* Drain and switch bs back to the QEMU main loop */
 352     blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context());
 353
 354     aio_context_release(s->ctx);
 355
 356     /* Sync vring state back to virtqueue so that non-dataplane request
 357      * processing can continue when we disable the host notifier below.
 358      */
 359     vring_teardown(&s->vring, s->vdev, 0);
 360
 361     k->set_host_notifier(qbus->parent, 0, false);
 362
 363     /* Clean up guest notifier (irq) */
 364     k->set_guest_notifiers(qbus->parent, 1, false);
 365
 366     s->started = false;
 367     s->stopping = false;
 368 }