block/block-backend.c

   1 /*
   2  * QEMU Block backends
   3  *
   4  * Copyright (C) 2014-2016 Red Hat, Inc.
   5  *
   6  * Authors:
   7  *  Markus Armbruster <armbru@redhat.com>,
   8  *
   9  * This work is licensed under the terms of the GNU LGPL, version 2.1
  10  * or later.  See the COPYING.LIB file in the top-level directory.
  11  */
  12
  13 #include "qemu/osdep.h"
  14 #include "sysemu/block-backend.h"
  15 #include "block/block_int.h"
  16 #include "block/blockjob.h"
  17 #include "block/coroutines.h"
  18 #include "block/throttle-groups.h"
  19 #include "hw/qdev-core.h"
  20 #include "sysemu/blockdev.h"
  21 #include "sysemu/runstate.h"
  22 #include "sysemu/replay.h"
  23 #include "qapi/error.h"
  24 #include "qapi/qapi-events-block.h"
  25 #include "qemu/id.h"
  26 #include "qemu/main-loop.h"
  27 #include "qemu/option.h"
  28 #include "trace.h"
  29 #include "migration/misc.h"
  30
  31 /* Number of coroutines to reserve per attached device model */
  32 #define COROUTINE_POOL_RESERVATION 64
  33
  34 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
  35
  36 static AioContext *blk_aiocb_get_aio_context(BlockAIOCB *acb);
  37
  38 typedef struct BlockBackendAioNotifier {
  39     void (*attached_aio_context)(AioContext *new_context, void *opaque);
  40     void (*detach_aio_context)(void *opaque);
  41     void *opaque;
  42     QLIST_ENTRY(BlockBackendAioNotifier) list;
  43 } BlockBackendAioNotifier;
  44
  45 struct BlockBackend {
  46     char *name;
  47     int refcnt;
  48     BdrvChild *root;
  49     AioContext *ctx;
  50     DriveInfo *legacy_dinfo;    /* null unless created by drive_new() */
  51     QTAILQ_ENTRY(BlockBackend) link;         /* for block_backends */
  52     QTAILQ_ENTRY(BlockBackend) monitor_link; /* for monitor_block_backends */
  53     BlockBackendPublic public;
  54
  55     DeviceState *dev;           /* attached device model, if any */
  56     const BlockDevOps *dev_ops;
  57     void *dev_opaque;
  58
  59     /* If the BDS tree is removed, some of its options are stored here (which
  60      * can be used to restore those options in the new BDS on insert) */
  61     BlockBackendRootState root_state;
  62
  63     bool enable_write_cache;
  64
  65     /* I/O stats (display with "info blockstats"). */
  66     BlockAcctStats stats;
  67
  68     BlockdevOnError on_read_error, on_write_error;
  69     bool iostatus_enabled;
  70     BlockDeviceIoStatus iostatus;
  71
  72     uint64_t perm;
  73     uint64_t shared_perm;
  74     bool disable_perm;
  75
  76     bool allow_aio_context_change;
  77     bool allow_write_beyond_eof;
  78
  79     /* Protected by BQL */
  80     NotifierList remove_bs_notifiers, insert_bs_notifiers;
  81     QLIST_HEAD(, BlockBackendAioNotifier) aio_notifiers;
  82
  83     int quiesce_counter; /* atomic: written under BQL, read by other threads */
  84     QemuMutex queued_requests_lock; /* protects queued_requests */
  85     CoQueue queued_requests;
  86     bool disable_request_queuing; /* atomic */
  87
  88     VMChangeStateEntry *vmsh;
  89     bool force_allow_inactivate;
  90
  91     /* Number of in-flight aio requests.  BlockDriverState also counts
  92      * in-flight requests but aio requests can exist even when blk->root is
  93      * NULL, so we cannot rely on its counter for that case.
  94      * Accessed with atomic ops.
  95      */
  96     unsigned int in_flight;
  97 };
  98
  99 typedef struct BlockBackendAIOCB {
 100     BlockAIOCB common;
 101     BlockBackend *blk;
 102     int ret;
 103 } BlockBackendAIOCB;
 104
 105 static const AIOCBInfo block_backend_aiocb_info = {
 106     .get_aio_context = blk_aiocb_get_aio_context,
 107     .aiocb_size = sizeof(BlockBackendAIOCB),
 108 };
 109
 110 static void drive_info_del(DriveInfo *dinfo);
 111 static BlockBackend *bdrv_first_blk(BlockDriverState *bs);
 112
 113 /* All BlockBackends. Protected by BQL. */
 114 static QTAILQ_HEAD(, BlockBackend) block_backends =
 115     QTAILQ_HEAD_INITIALIZER(block_backends);
 116
 117 /*
 118  * All BlockBackends referenced by the monitor and which are iterated through by
 119  * blk_next(). Protected by BQL.
 120  */
 121 static QTAILQ_HEAD(, BlockBackend) monitor_block_backends =
 122     QTAILQ_HEAD_INITIALIZER(monitor_block_backends);
 123
 124 static int coroutine_mixed_fn GRAPH_RDLOCK
 125 blk_set_perm_locked(BlockBackend *blk, uint64_t perm, uint64_t shared_perm,
 126                     Error **errp);
 127
 128 static void blk_root_inherit_options(BdrvChildRole role, bool parent_is_format,
 129                                      int *child_flags, QDict *child_options,
 130                                      int parent_flags, QDict *parent_options)
 131 {
 132     /* We're not supposed to call this function for root nodes */
 133     abort();
 134 }
 135 static void blk_root_drained_begin(BdrvChild *child);
 136 static bool blk_root_drained_poll(BdrvChild *child);
 137 static void blk_root_drained_end(BdrvChild *child);
 138
 139 static void blk_root_change_media(BdrvChild *child, bool load);
 140 static void blk_root_resize(BdrvChild *child);
 141
 142 static bool blk_root_change_aio_ctx(BdrvChild *child, AioContext *ctx,
 143                                     GHashTable *visited, Transaction *tran,
 144                                     Error **errp);
 145
 146 static char *blk_root_get_parent_desc(BdrvChild *child)
 147 {
 148     BlockBackend *blk = child->opaque;
 149     g_autofree char *dev_id = NULL;
 150
 151     if (blk->name) {
 152         return g_strdup_printf("block device '%s'", blk->name);
 153     }
 154
 155     dev_id = blk_get_attached_dev_id(blk);
 156     if (*dev_id) {
 157         return g_strdup_printf("block device '%s'", dev_id);
 158     } else {
 159         /* TODO Callback into the BB owner for something more detailed */
 160         return g_strdup("an unnamed block device");
 161     }
 162 }
 163
 164 static const char *blk_root_get_name(BdrvChild *child)
 165 {
 166     return blk_name(child->opaque);
 167 }
 168
 169 static void blk_vm_state_changed(void *opaque, bool running, RunState state)
 170 {
 171     Error *local_err = NULL;
 172     BlockBackend *blk = opaque;
 173
 174     if (state == RUN_STATE_INMIGRATE) {
 175         return;
 176     }
 177
 178     qemu_del_vm_change_state_handler(blk->vmsh);
 179     blk->vmsh = NULL;
 180     blk_set_perm(blk, blk->perm, blk->shared_perm, &local_err);
 181     if (local_err) {
 182         error_report_err(local_err);
 183     }
 184 }
 185
 186 /*
 187  * Notifies the user of the BlockBackend that migration has completed. qdev
 188  * devices can tighten their permissions in response (specifically revoke
 189  * shared write permissions that we needed for storage migration).
 190  *
 191  * If an error is returned, the VM cannot be allowed to be resumed.
 192  */
 193 static void GRAPH_RDLOCK blk_root_activate(BdrvChild *child, Error **errp)
 194 {
 195     BlockBackend *blk = child->opaque;
 196     Error *local_err = NULL;
 197     uint64_t saved_shared_perm;
 198
 199     if (!blk->disable_perm) {
 200         return;
 201     }
 202
 203     blk->disable_perm = false;
 204
 205     /*
 206      * blk->shared_perm contains the permissions we want to share once
 207      * migration is really completely done.  For now, we need to share
 208      * all; but we also need to retain blk->shared_perm, which is
 209      * overwritten by a successful blk_set_perm() call.  Save it and
 210      * restore it below.
 211      */
 212     saved_shared_perm = blk->shared_perm;
 213
 214     blk_set_perm_locked(blk, blk->perm, BLK_PERM_ALL, &local_err);
 215     if (local_err) {
 216         error_propagate(errp, local_err);
 217         blk->disable_perm = true;
 218         return;
 219     }
 220     blk->shared_perm = saved_shared_perm;
 221
 222     if (runstate_check(RUN_STATE_INMIGRATE)) {
 223         /* Activation can happen when migration process is still active, for
 224          * example when nbd_server_add is called during non-shared storage
 225          * migration. Defer the shared_perm update to migration completion. */
 226         if (!blk->vmsh) {
 227             blk->vmsh = qemu_add_vm_change_state_handler(blk_vm_state_changed,
 228                                                          blk);
 229         }
 230         return;
 231     }
 232
 233     blk_set_perm_locked(blk, blk->perm, blk->shared_perm, &local_err);
 234     if (local_err) {
 235         error_propagate(errp, local_err);
 236         blk->disable_perm = true;
 237         return;
 238     }
 239 }
 240
 241 void blk_set_force_allow_inactivate(BlockBackend *blk)
 242 {
 243     GLOBAL_STATE_CODE();
 244     blk->force_allow_inactivate = true;
 245 }
 246
 247 static bool blk_can_inactivate(BlockBackend *blk)
 248 {
 249     /* If it is a guest device, inactivate is ok. */
 250     if (blk->dev || blk_name(blk)[0]) {
 251         return true;
 252     }
 253
 254     /* Inactivating means no more writes to the image can be done,
 255      * even if those writes would be changes invisible to the
 256      * guest.  For block job BBs that satisfy this, we can just allow
 257      * it.  This is the case for mirror job source, which is required
 258      * by libvirt non-shared block migration. */
 259     if (!(blk->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED))) {
 260         return true;
 261     }
 262
 263     return blk->force_allow_inactivate;
 264 }
 265
 266 static int GRAPH_RDLOCK blk_root_inactivate(BdrvChild *child)
 267 {
 268     BlockBackend *blk = child->opaque;
 269
 270     if (blk->disable_perm) {
 271         return 0;
 272     }
 273
 274     if (!blk_can_inactivate(blk)) {
 275         return -EPERM;
 276     }
 277
 278     blk->disable_perm = true;
 279     if (blk->root) {
 280         bdrv_child_try_set_perm(blk->root, 0, BLK_PERM_ALL, &error_abort);
 281     }
 282
 283     return 0;
 284 }
 285
 286 static void blk_root_attach(BdrvChild *child)
 287 {
 288     BlockBackend *blk = child->opaque;
 289     BlockBackendAioNotifier *notifier;
 290
 291     trace_blk_root_attach(child, blk, child->bs);
 292
 293     QLIST_FOREACH(notifier, &blk->aio_notifiers, list) {
 294         bdrv_add_aio_context_notifier(child->bs,
 295                 notifier->attached_aio_context,
 296                 notifier->detach_aio_context,
 297                 notifier->opaque);
 298     }
 299 }
 300
 301 static void blk_root_detach(BdrvChild *child)
 302 {
 303     BlockBackend *blk = child->opaque;
 304     BlockBackendAioNotifier *notifier;
 305
 306     trace_blk_root_detach(child, blk, child->bs);
 307
 308     QLIST_FOREACH(notifier, &blk->aio_notifiers, list) {
 309         bdrv_remove_aio_context_notifier(child->bs,
 310                 notifier->attached_aio_context,
 311                 notifier->detach_aio_context,
 312                 notifier->opaque);
 313     }
 314 }
 315
 316 static AioContext *blk_root_get_parent_aio_context(BdrvChild *c)
 317 {
 318     BlockBackend *blk = c->opaque;
 319     IO_CODE();
 320
 321     return blk_get_aio_context(blk);
 322 }
 323
 324 static const BdrvChildClass child_root = {
 325     .inherit_options    = blk_root_inherit_options,
 326
 327     .change_media       = blk_root_change_media,
 328     .resize             = blk_root_resize,
 329     .get_name           = blk_root_get_name,
 330     .get_parent_desc    = blk_root_get_parent_desc,
 331
 332     .drained_begin      = blk_root_drained_begin,
 333     .drained_poll       = blk_root_drained_poll,
 334     .drained_end        = blk_root_drained_end,
 335
 336     .activate           = blk_root_activate,
 337     .inactivate         = blk_root_inactivate,
 338
 339     .attach             = blk_root_attach,
 340     .detach             = blk_root_detach,
 341
 342     .change_aio_ctx     = blk_root_change_aio_ctx,
 343
 344     .get_parent_aio_context = blk_root_get_parent_aio_context,
 345 };
 346
 347 /*
 348  * Create a new BlockBackend with a reference count of one.
 349  *
 350  * @perm is a bitmasks of BLK_PERM_* constants which describes the permissions
 351  * to request for a block driver node that is attached to this BlockBackend.
 352  * @shared_perm is a bitmask which describes which permissions may be granted
 353  * to other users of the attached node.
 354  * Both sets of permissions can be changed later using blk_set_perm().
 355  *
 356  * Return the new BlockBackend on success, null on failure.
 357  */
 358 BlockBackend *blk_new(AioContext *ctx, uint64_t perm, uint64_t shared_perm)
 359 {
 360     BlockBackend *blk;
 361
 362     GLOBAL_STATE_CODE();
 363
 364     blk = g_new0(BlockBackend, 1);
 365     blk->refcnt = 1;
 366     blk->ctx = ctx;
 367     blk->perm = perm;
 368     blk->shared_perm = shared_perm;
 369     blk_set_enable_write_cache(blk, true);
 370
 371     blk->on_read_error = BLOCKDEV_ON_ERROR_REPORT;
 372     blk->on_write_error = BLOCKDEV_ON_ERROR_ENOSPC;
 373
 374     block_acct_init(&blk->stats);
 375
 376     qemu_mutex_init(&blk->queued_requests_lock);
 377     qemu_co_queue_init(&blk->queued_requests);
 378     notifier_list_init(&blk->remove_bs_notifiers);
 379     notifier_list_init(&blk->insert_bs_notifiers);
 380     QLIST_INIT(&blk->aio_notifiers);
 381
 382     QTAILQ_INSERT_TAIL(&block_backends, blk, link);
 383     return blk;
 384 }
 385
 386 /*
 387  * Create a new BlockBackend connected to an existing BlockDriverState.
 388  *
 389  * @perm is a bitmasks of BLK_PERM_* constants which describes the
 390  * permissions to request for @bs that is attached to this
 391  * BlockBackend.  @shared_perm is a bitmask which describes which
 392  * permissions may be granted to other users of the attached node.
 393  * Both sets of permissions can be changed later using blk_set_perm().
 394  *
 395  * Return the new BlockBackend on success, null on failure.
 396  *
 397  * Callers must hold the AioContext lock of @bs.
 398  */
 399 BlockBackend *blk_new_with_bs(BlockDriverState *bs, uint64_t perm,
 400                               uint64_t shared_perm, Error **errp)
 401 {
 402     BlockBackend *blk = blk_new(bdrv_get_aio_context(bs), perm, shared_perm);
 403
 404     GLOBAL_STATE_CODE();
 405
 406     if (blk_insert_bs(blk, bs, errp) < 0) {
 407         blk_unref(blk);
 408         return NULL;
 409     }
 410     return blk;
 411 }
 412
 413 /*
 414  * Creates a new BlockBackend, opens a new BlockDriverState, and connects both.
 415  * By default, the new BlockBackend is in the main AioContext, but if the
 416  * parameters connect it with any existing node in a different AioContext, it
 417  * may end up there instead.
 418  *
 419  * Just as with bdrv_open(), after having called this function the reference to
 420  * @options belongs to the block layer (even on failure).
 421  *
 422  * Called without holding an AioContext lock.
 423  *
 424  * TODO: Remove @filename and @flags; it should be possible to specify a whole
 425  * BDS tree just by specifying the @options QDict (or @reference,
 426  * alternatively). At the time of adding this function, this is not possible,
 427  * though, so callers of this function have to be able to specify @filename and
 428  * @flags.
 429  */
 430 BlockBackend *blk_new_open(const char *filename, const char *reference,
 431                            QDict *options, int flags, Error **errp)
 432 {
 433     BlockBackend *blk;
 434     BlockDriverState *bs;
 435     AioContext *ctx;
 436     uint64_t perm = 0;
 437     uint64_t shared = BLK_PERM_ALL;
 438
 439     GLOBAL_STATE_CODE();
 440
 441     /*
 442      * blk_new_open() is mainly used in .bdrv_create implementations and the
 443      * tools where sharing isn't a major concern because the BDS stays private
 444      * and the file is generally not supposed to be used by a second process,
 445      * so we just request permission according to the flags.
 446      *
 447      * The exceptions are xen_disk and blockdev_init(); in these cases, the
 448      * caller of blk_new_open() doesn't make use of the permissions, but they
 449      * shouldn't hurt either. We can still share everything here because the
 450      * guest devices will add their own blockers if they can't share.
 451      */
 452     if ((flags & BDRV_O_NO_IO) == 0) {
 453         perm |= BLK_PERM_CONSISTENT_READ;
 454         if (flags & BDRV_O_RDWR) {
 455             perm |= BLK_PERM_WRITE;
 456         }
 457     }
 458     if (flags & BDRV_O_RESIZE) {
 459         perm |= BLK_PERM_RESIZE;
 460     }
 461     if (flags & BDRV_O_NO_SHARE) {
 462         shared = BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED;
 463     }
 464
 465     aio_context_acquire(qemu_get_aio_context());
 466     bs = bdrv_open(filename, reference, options, flags, errp);
 467     aio_context_release(qemu_get_aio_context());
 468     if (!bs) {
 469         return NULL;
 470     }
 471
 472     /* bdrv_open() could have moved bs to a different AioContext */
 473     ctx = bdrv_get_aio_context(bs);
 474     blk = blk_new(bdrv_get_aio_context(bs), perm, shared);
 475     blk->perm = perm;
 476     blk->shared_perm = shared;
 477
 478     aio_context_acquire(ctx);
 479     blk_insert_bs(blk, bs, errp);
 480     bdrv_unref(bs);
 481     aio_context_release(ctx);
 482
 483     if (!blk->root) {
 484         blk_unref(blk);
 485         return NULL;
 486     }
 487
 488     return blk;
 489 }
 490
 491 static void blk_delete(BlockBackend *blk)
 492 {
 493     assert(!blk->refcnt);
 494     assert(!blk->name);
 495     assert(!blk->dev);
 496     if (blk->public.throttle_group_member.throttle_state) {
 497         blk_io_limits_disable(blk);
 498     }
 499     if (blk->root) {
 500         blk_remove_bs(blk);
 501     }
 502     if (blk->vmsh) {
 503         qemu_del_vm_change_state_handler(blk->vmsh);
 504         blk->vmsh = NULL;
 505     }
 506     assert(QLIST_EMPTY(&blk->remove_bs_notifiers.notifiers));
 507     assert(QLIST_EMPTY(&blk->insert_bs_notifiers.notifiers));
 508     assert(QLIST_EMPTY(&blk->aio_notifiers));
 509     assert(qemu_co_queue_empty(&blk->queued_requests));
 510     qemu_mutex_destroy(&blk->queued_requests_lock);
 511     QTAILQ_REMOVE(&block_backends, blk, link);
 512     drive_info_del(blk->legacy_dinfo);
 513     block_acct_cleanup(&blk->stats);
 514     g_free(blk);
 515 }
 516
 517 static void drive_info_del(DriveInfo *dinfo)
 518 {
 519     if (!dinfo) {
 520         return;
 521     }
 522     qemu_opts_del(dinfo->opts);
 523     g_free(dinfo);
 524 }
 525
 526 int blk_get_refcnt(BlockBackend *blk)
 527 {
 528     GLOBAL_STATE_CODE();
 529     return blk ? blk->refcnt : 0;
 530 }
 531
 532 /*
 533  * Increment @blk's reference count.
 534  * @blk must not be null.
 535  */
 536 void blk_ref(BlockBackend *blk)
 537 {
 538     assert(blk->refcnt > 0);
 539     GLOBAL_STATE_CODE();
 540     blk->refcnt++;
 541 }
 542
 543 /*
 544  * Decrement @blk's reference count.
 545  * If this drops it to zero, destroy @blk.
 546  * For convenience, do nothing if @blk is null.
 547  */
 548 void blk_unref(BlockBackend *blk)
 549 {
 550     GLOBAL_STATE_CODE();
 551     if (blk) {
 552         assert(blk->refcnt > 0);
 553         if (blk->refcnt > 1) {
 554             blk->refcnt--;
 555         } else {
 556             blk_drain(blk);
 557             /* blk_drain() cannot resurrect blk, nobody held a reference */
 558             assert(blk->refcnt == 1);
 559             blk->refcnt = 0;
 560             blk_delete(blk);
 561         }
 562     }
 563 }
 564
 565 /*
 566  * Behaves similarly to blk_next() but iterates over all BlockBackends, even the
 567  * ones which are hidden (i.e. are not referenced by the monitor).
 568  */
 569 BlockBackend *blk_all_next(BlockBackend *blk)
 570 {
 571     GLOBAL_STATE_CODE();
 572     return blk ? QTAILQ_NEXT(blk, link)
 573                : QTAILQ_FIRST(&block_backends);
 574 }
 575
 576 void blk_remove_all_bs(void)
 577 {
 578     BlockBackend *blk = NULL;
 579
 580     GLOBAL_STATE_CODE();
 581
 582     while ((blk = blk_all_next(blk)) != NULL) {
 583         AioContext *ctx = blk_get_aio_context(blk);
 584
 585         aio_context_acquire(ctx);
 586         if (blk->root) {
 587             blk_remove_bs(blk);
 588         }
 589         aio_context_release(ctx);
 590     }
 591 }
 592
 593 /*
 594  * Return the monitor-owned BlockBackend after @blk.
 595  * If @blk is null, return the first one.
 596  * Else, return @blk's next sibling, which may be null.
 597  *
 598  * To iterate over all BlockBackends, do
 599  * for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
 600  *     ...
 601  * }
 602  */
 603 BlockBackend *blk_next(BlockBackend *blk)
 604 {
 605     GLOBAL_STATE_CODE();
 606     return blk ? QTAILQ_NEXT(blk, monitor_link)
 607                : QTAILQ_FIRST(&monitor_block_backends);
 608 }
 609
 610 /* Iterates over all top-level BlockDriverStates, i.e. BDSs that are owned by
 611  * the monitor or attached to a BlockBackend */
 612 BlockDriverState *bdrv_next(BdrvNextIterator *it)
 613 {
 614     BlockDriverState *bs, *old_bs;
 615
 616     /* Must be called from the main loop */
 617     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
 618
 619     /* First, return all root nodes of BlockBackends. In order to avoid
 620      * returning a BDS twice when multiple BBs refer to it, we only return it
 621      * if the BB is the first one in the parent list of the BDS. */
 622     if (it->phase == BDRV_NEXT_BACKEND_ROOTS) {
 623         BlockBackend *old_blk = it->blk;
 624
 625         old_bs = old_blk ? blk_bs(old_blk) : NULL;
 626
 627         do {
 628             it->blk = blk_all_next(it->blk);
 629             bs = it->blk ? blk_bs(it->blk) : NULL;
 630         } while (it->blk && (bs == NULL || bdrv_first_blk(bs) != it->blk));
 631
 632         if (it->blk) {
 633             blk_ref(it->blk);
 634         }
 635         blk_unref(old_blk);
 636
 637         if (bs) {
 638             bdrv_ref(bs);
 639             bdrv_unref(old_bs);
 640             return bs;
 641         }
 642         it->phase = BDRV_NEXT_MONITOR_OWNED;
 643     } else {
 644         old_bs = it->bs;
 645     }
 646
 647     /* Then return the monitor-owned BDSes without a BB attached. Ignore all
 648      * BDSes that are attached to a BlockBackend here; they have been handled
 649      * by the above block already */
 650     do {
 651         it->bs = bdrv_next_monitor_owned(it->bs);
 652         bs = it->bs;
 653     } while (bs && bdrv_has_blk(bs));
 654
 655     if (bs) {
 656         bdrv_ref(bs);
 657     }
 658     bdrv_unref(old_bs);
 659
 660     return bs;
 661 }
 662
 663 static void bdrv_next_reset(BdrvNextIterator *it)
 664 {
 665     *it = (BdrvNextIterator) {
 666         .phase = BDRV_NEXT_BACKEND_ROOTS,
 667     };
 668 }
 669
 670 BlockDriverState *bdrv_first(BdrvNextIterator *it)
 671 {
 672     GLOBAL_STATE_CODE();
 673     bdrv_next_reset(it);
 674     return bdrv_next(it);
 675 }
 676
 677 /* Must be called when aborting a bdrv_next() iteration before
 678  * bdrv_next() returns NULL */
 679 void bdrv_next_cleanup(BdrvNextIterator *it)
 680 {
 681     /* Must be called from the main loop */
 682     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
 683
 684     if (it->phase == BDRV_NEXT_BACKEND_ROOTS) {
 685         if (it->blk) {
 686             bdrv_unref(blk_bs(it->blk));
 687             blk_unref(it->blk);
 688         }
 689     } else {
 690         bdrv_unref(it->bs);
 691     }
 692
 693     bdrv_next_reset(it);
 694 }
 695
 696 /*
 697  * Add a BlockBackend into the list of backends referenced by the monitor, with
 698  * the given @name acting as the handle for the monitor.
 699  * Strictly for use by blockdev.c.
 700  *
 701  * @name must not be null or empty.
 702  *
 703  * Returns true on success and false on failure. In the latter case, an Error
 704  * object is returned through @errp.
 705  */
 706 bool monitor_add_blk(BlockBackend *blk, const char *name, Error **errp)
 707 {
 708     assert(!blk->name);
 709     assert(name && name[0]);
 710     GLOBAL_STATE_CODE();
 711
 712     if (!id_wellformed(name)) {
 713         error_setg(errp, "Invalid device name");
 714         return false;
 715     }
 716     if (blk_by_name(name)) {
 717         error_setg(errp, "Device with id '%s' already exists", name);
 718         return false;
 719     }
 720     if (bdrv_find_node(name)) {
 721         error_setg(errp,
 722                    "Device name '%s' conflicts with an existing node name",
 723                    name);
 724         return false;
 725     }
 726
 727     blk->name = g_strdup(name);
 728     QTAILQ_INSERT_TAIL(&monitor_block_backends, blk, monitor_link);
 729     return true;
 730 }
 731
 732 /*
 733  * Remove a BlockBackend from the list of backends referenced by the monitor.
 734  * Strictly for use by blockdev.c.
 735  */
 736 void monitor_remove_blk(BlockBackend *blk)
 737 {
 738     GLOBAL_STATE_CODE();
 739
 740     if (!blk->name) {
 741         return;
 742     }
 743
 744     QTAILQ_REMOVE(&monitor_block_backends, blk, monitor_link);
 745     g_free(blk->name);
 746     blk->name = NULL;
 747 }
 748
 749 /*
 750  * Return @blk's name, a non-null string.
 751  * Returns an empty string iff @blk is not referenced by the monitor.
 752  */
 753 const char *blk_name(const BlockBackend *blk)
 754 {
 755     IO_CODE();
 756     return blk->name ?: "";
 757 }
 758
 759 /*
 760  * Return the BlockBackend with name @name if it exists, else null.
 761  * @name must not be null.
 762  */
 763 BlockBackend *blk_by_name(const char *name)
 764 {
 765     BlockBackend *blk = NULL;
 766
 767     GLOBAL_STATE_CODE();
 768     assert(name);
 769     while ((blk = blk_next(blk)) != NULL) {
 770         if (!strcmp(name, blk->name)) {
 771             return blk;
 772         }
 773     }
 774     return NULL;
 775 }
 776
 777 /*
 778  * Return the BlockDriverState attached to @blk if any, else null.
 779  */
 780 BlockDriverState *blk_bs(BlockBackend *blk)
 781 {
 782     IO_CODE();
 783     return blk->root ? blk->root->bs : NULL;
 784 }
 785
 786 static BlockBackend *bdrv_first_blk(BlockDriverState *bs)
 787 {
 788     BdrvChild *child;
 789
 790     GLOBAL_STATE_CODE();
 791
 792     QLIST_FOREACH(child, &bs->parents, next_parent) {
 793         if (child->klass == &child_root) {
 794             return child->opaque;
 795         }
 796     }
 797
 798     return NULL;
 799 }
 800
 801 /*
 802  * Returns true if @bs has an associated BlockBackend.
 803  */
 804 bool bdrv_has_blk(BlockDriverState *bs)
 805 {
 806     GLOBAL_STATE_CODE();
 807     return bdrv_first_blk(bs) != NULL;
 808 }
 809
 810 /*
 811  * Returns true if @bs has only BlockBackends as parents.
 812  */
 813 bool bdrv_is_root_node(BlockDriverState *bs)
 814 {
 815     BdrvChild *c;
 816
 817     GLOBAL_STATE_CODE();
 818     QLIST_FOREACH(c, &bs->parents, next_parent) {
 819         if (c->klass != &child_root) {
 820             return false;
 821         }
 822     }
 823
 824     return true;
 825 }
 826
 827 /*
 828  * Return @blk's DriveInfo if any, else null.
 829  */
 830 DriveInfo *blk_legacy_dinfo(BlockBackend *blk)
 831 {
 832     GLOBAL_STATE_CODE();
 833     return blk->legacy_dinfo;
 834 }
 835
 836 /*
 837  * Set @blk's DriveInfo to @dinfo, and return it.
 838  * @blk must not have a DriveInfo set already.
 839  * No other BlockBackend may have the same DriveInfo set.
 840  */
 841 DriveInfo *blk_set_legacy_dinfo(BlockBackend *blk, DriveInfo *dinfo)
 842 {
 843     assert(!blk->legacy_dinfo);
 844     GLOBAL_STATE_CODE();
 845     return blk->legacy_dinfo = dinfo;
 846 }
 847
 848 /*
 849  * Return the BlockBackend with DriveInfo @dinfo.
 850  * It must exist.
 851  */
 852 BlockBackend *blk_by_legacy_dinfo(DriveInfo *dinfo)
 853 {
 854     BlockBackend *blk = NULL;
 855     GLOBAL_STATE_CODE();
 856
 857     while ((blk = blk_next(blk)) != NULL) {
 858         if (blk->legacy_dinfo == dinfo) {
 859             return blk;
 860         }
 861     }
 862     abort();
 863 }
 864
 865 /*
 866  * Returns a pointer to the publicly accessible fields of @blk.
 867  */
 868 BlockBackendPublic *blk_get_public(BlockBackend *blk)
 869 {
 870     GLOBAL_STATE_CODE();
 871     return &blk->public;
 872 }
 873
 874 /*
 875  * Returns a BlockBackend given the associated @public fields.
 876  */
 877 BlockBackend *blk_by_public(BlockBackendPublic *public)
 878 {
 879     GLOBAL_STATE_CODE();
 880     return container_of(public, BlockBackend, public);
 881 }
 882
 883 /*
 884  * Disassociates the currently associated BlockDriverState from @blk.
 885  */
 886 void blk_remove_bs(BlockBackend *blk)
 887 {
 888     ThrottleGroupMember *tgm = &blk->public.throttle_group_member;
 889     BdrvChild *root;
 890
 891     GLOBAL_STATE_CODE();
 892
 893     notifier_list_notify(&blk->remove_bs_notifiers, blk);
 894     if (tgm->throttle_state) {
 895         BlockDriverState *bs = blk_bs(blk);
 896
 897         /*
 898          * Take a ref in case blk_bs() changes across bdrv_drained_begin(), for
 899          * example, if a temporary filter node is removed by a blockjob.
 900          */
 901         bdrv_ref(bs);
 902         bdrv_drained_begin(bs);
 903         throttle_group_detach_aio_context(tgm);
 904         throttle_group_attach_aio_context(tgm, qemu_get_aio_context());
 905         bdrv_drained_end(bs);
 906         bdrv_unref(bs);
 907     }
 908
 909     blk_update_root_state(blk);
 910
 911     /* bdrv_root_unref_child() will cause blk->root to become stale and may
 912      * switch to a completion coroutine later on. Let's drain all I/O here
 913      * to avoid that and a potential QEMU crash.
 914      */
 915     blk_drain(blk);
 916     root = blk->root;
 917     blk->root = NULL;
 918
 919     bdrv_graph_wrlock(NULL);
 920     bdrv_root_unref_child(root);
 921     bdrv_graph_wrunlock();
 922 }
 923
 924 /*
 925  * Associates a new BlockDriverState with @blk.
 926  *
 927  * Callers must hold the AioContext lock of @bs.
 928  */
 929 int blk_insert_bs(BlockBackend *blk, BlockDriverState *bs, Error **errp)
 930 {
 931     ThrottleGroupMember *tgm = &blk->public.throttle_group_member;
 932     GLOBAL_STATE_CODE();
 933     bdrv_ref(bs);
 934     blk->root = bdrv_root_attach_child(bs, "root", &child_root,
 935                                        BDRV_CHILD_FILTERED | BDRV_CHILD_PRIMARY,
 936                                        blk->perm, blk->shared_perm,
 937                                        blk, errp);
 938     if (blk->root == NULL) {
 939         return -EPERM;
 940     }
 941
 942     notifier_list_notify(&blk->insert_bs_notifiers, blk);
 943     if (tgm->throttle_state) {
 944         throttle_group_detach_aio_context(tgm);
 945         throttle_group_attach_aio_context(tgm, bdrv_get_aio_context(bs));
 946     }
 947
 948     return 0;
 949 }
 950
 951 /*
 952  * Change BlockDriverState associated with @blk.
 953  */
 954 int blk_replace_bs(BlockBackend *blk, BlockDriverState *new_bs, Error **errp)
 955 {
 956     GLOBAL_STATE_CODE();
 957     return bdrv_replace_child_bs(blk->root, new_bs, errp);
 958 }
 959
 960 /*
 961  * Sets the permission bitmasks that the user of the BlockBackend needs.
 962  */
 963 static int coroutine_mixed_fn GRAPH_RDLOCK
 964 blk_set_perm_locked(BlockBackend *blk, uint64_t perm, uint64_t shared_perm,
 965                     Error **errp)
 966 {
 967     int ret;
 968     GLOBAL_STATE_CODE();
 969
 970     if (blk->root && !blk->disable_perm) {
 971         ret = bdrv_child_try_set_perm(blk->root, perm, shared_perm, errp);
 972         if (ret < 0) {
 973             return ret;
 974         }
 975     }
 976
 977     blk->perm = perm;
 978     blk->shared_perm = shared_perm;
 979
 980     return 0;
 981 }
 982
 983 int blk_set_perm(BlockBackend *blk, uint64_t perm, uint64_t shared_perm,
 984                  Error **errp)
 985 {
 986     GLOBAL_STATE_CODE();
 987     GRAPH_RDLOCK_GUARD_MAINLOOP();
 988
 989     return blk_set_perm_locked(blk, perm, shared_perm, errp);
 990 }
 991
 992 void blk_get_perm(BlockBackend *blk, uint64_t *perm, uint64_t *shared_perm)
 993 {
 994     GLOBAL_STATE_CODE();
 995     *perm = blk->perm;
 996     *shared_perm = blk->shared_perm;
 997 }
 998
 999 /*
1000  * Attach device model @dev to @blk.
1001  * Return 0 on success, -EBUSY when a device model is attached already.
1002  */
1003 int blk_attach_dev(BlockBackend *blk, DeviceState *dev)
1004 {
1005     GLOBAL_STATE_CODE();
1006     if (blk->dev) {
1007         return -EBUSY;
1008     }
1009
1010     /* While migration is still incoming, we don't need to apply the
1011      * permissions of guest device BlockBackends. We might still have a block
1012      * job or NBD server writing to the image for storage migration. */
1013     if (runstate_check(RUN_STATE_INMIGRATE)) {
1014         blk->disable_perm = true;
1015     }
1016
1017     blk_ref(blk);
1018     blk->dev = dev;
1019     blk_iostatus_reset(blk);
1020
1021     return 0;
1022 }
1023
1024 /*
1025  * Detach device model @dev from @blk.
1026  * @dev must be currently attached to @blk.
1027  */
1028 void blk_detach_dev(BlockBackend *blk, DeviceState *dev)
1029 {
1030     assert(blk->dev == dev);
1031     GLOBAL_STATE_CODE();
1032     blk->dev = NULL;
1033     blk->dev_ops = NULL;
1034     blk->dev_opaque = NULL;
1035     blk_set_perm(blk, 0, BLK_PERM_ALL, &error_abort);
1036     blk_unref(blk);
1037 }
1038
1039 /*
1040  * Return the device model attached to @blk if any, else null.
1041  */
1042 DeviceState *blk_get_attached_dev(BlockBackend *blk)
1043 {
1044     GLOBAL_STATE_CODE();
1045     return blk->dev;
1046 }
1047
1048 /* Return the qdev ID, or if no ID is assigned the QOM path, of the block
1049  * device attached to the BlockBackend. */
1050 char *blk_get_attached_dev_id(BlockBackend *blk)
1051 {
1052     DeviceState *dev = blk->dev;
1053     IO_CODE();
1054
1055     if (!dev) {
1056         return g_strdup("");
1057     } else if (dev->id) {
1058         return g_strdup(dev->id);
1059     }
1060
1061     return object_get_canonical_path(OBJECT(dev)) ?: g_strdup("");
1062 }
1063
1064 /*
1065  * Return the BlockBackend which has the device model @dev attached if it
1066  * exists, else null.
1067  *
1068  * @dev must not be null.
1069  */
1070 BlockBackend *blk_by_dev(void *dev)
1071 {
1072     BlockBackend *blk = NULL;
1073
1074     GLOBAL_STATE_CODE();
1075
1076     assert(dev != NULL);
1077     while ((blk = blk_all_next(blk)) != NULL) {
1078         if (blk->dev == dev) {
1079             return blk;
1080         }
1081     }
1082     return NULL;
1083 }
1084
1085 /*
1086  * Set @blk's device model callbacks to @ops.
1087  * @opaque is the opaque argument to pass to the callbacks.
1088  * This is for use by device models.
1089  */
1090 void blk_set_dev_ops(BlockBackend *blk, const BlockDevOps *ops,
1091                      void *opaque)
1092 {
1093     GLOBAL_STATE_CODE();
1094     blk->dev_ops = ops;
1095     blk->dev_opaque = opaque;
1096
1097     /* Are we currently quiesced? Should we enforce this right now? */
1098     if (qatomic_read(&blk->quiesce_counter) && ops && ops->drained_begin) {
1099         ops->drained_begin(opaque);
1100     }
1101 }
1102
1103 /*
1104  * Notify @blk's attached device model of media change.
1105  *
1106  * If @load is true, notify of media load. This action can fail, meaning that
1107  * the medium cannot be loaded. @errp is set then.
1108  *
1109  * If @load is false, notify of media eject. This can never fail.
1110  *
1111  * Also send DEVICE_TRAY_MOVED events as appropriate.
1112  */
1113 void blk_dev_change_media_cb(BlockBackend *blk, bool load, Error **errp)
1114 {
1115     GLOBAL_STATE_CODE();
1116     if (blk->dev_ops && blk->dev_ops->change_media_cb) {
1117         bool tray_was_open, tray_is_open;
1118         Error *local_err = NULL;
1119
1120         tray_was_open = blk_dev_is_tray_open(blk);
1121         blk->dev_ops->change_media_cb(blk->dev_opaque, load, &local_err);
1122         if (local_err) {
1123             assert(load == true);
1124             error_propagate(errp, local_err);
1125             return;
1126         }
1127         tray_is_open = blk_dev_is_tray_open(blk);
1128
1129         if (tray_was_open != tray_is_open) {
1130             char *id = blk_get_attached_dev_id(blk);
1131             qapi_event_send_device_tray_moved(blk_name(blk), id, tray_is_open);
1132             g_free(id);
1133         }
1134     }
1135 }
1136
1137 static void blk_root_change_media(BdrvChild *child, bool load)
1138 {
1139     blk_dev_change_media_cb(child->opaque, load, NULL);
1140 }
1141
1142 /*
1143  * Does @blk's attached device model have removable media?
1144  * %true if no device model is attached.
1145  */
1146 bool blk_dev_has_removable_media(BlockBackend *blk)
1147 {
1148     GLOBAL_STATE_CODE();
1149     return !blk->dev || (blk->dev_ops && blk->dev_ops->change_media_cb);
1150 }
1151
1152 /*
1153  * Does @blk's attached device model have a tray?
1154  */
1155 bool blk_dev_has_tray(BlockBackend *blk)
1156 {
1157     IO_CODE();
1158     return blk->dev_ops && blk->dev_ops->is_tray_open;
1159 }
1160
1161 /*
1162  * Notify @blk's attached device model of a media eject request.
1163  * If @force is true, the medium is about to be yanked out forcefully.
1164  */
1165 void blk_dev_eject_request(BlockBackend *blk, bool force)
1166 {
1167     GLOBAL_STATE_CODE();
1168     if (blk->dev_ops && blk->dev_ops->eject_request_cb) {
1169         blk->dev_ops->eject_request_cb(blk->dev_opaque, force);
1170     }
1171 }
1172
1173 /*
1174  * Does @blk's attached device model have a tray, and is it open?
1175  */
1176 bool blk_dev_is_tray_open(BlockBackend *blk)
1177 {
1178     IO_CODE();
1179     if (blk_dev_has_tray(blk)) {
1180         return blk->dev_ops->is_tray_open(blk->dev_opaque);
1181     }
1182     return false;
1183 }
1184
1185 /*
1186  * Does @blk's attached device model have the medium locked?
1187  * %false if the device model has no such lock.
1188  */
1189 bool blk_dev_is_medium_locked(BlockBackend *blk)
1190 {
1191     GLOBAL_STATE_CODE();
1192     if (blk->dev_ops && blk->dev_ops->is_medium_locked) {
1193         return blk->dev_ops->is_medium_locked(blk->dev_opaque);
1194     }
1195     return false;
1196 }
1197
1198 /*
1199  * Notify @blk's attached device model of a backend size change.
1200  */
1201 static void blk_root_resize(BdrvChild *child)
1202 {
1203     BlockBackend *blk = child->opaque;
1204
1205     if (blk->dev_ops && blk->dev_ops->resize_cb) {
1206         blk->dev_ops->resize_cb(blk->dev_opaque);
1207     }
1208 }
1209
1210 void blk_iostatus_enable(BlockBackend *blk)
1211 {
1212     GLOBAL_STATE_CODE();
1213     blk->iostatus_enabled = true;
1214     blk->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
1215 }
1216
1217 /* The I/O status is only enabled if the drive explicitly
1218  * enables it _and_ the VM is configured to stop on errors */
1219 bool blk_iostatus_is_enabled(const BlockBackend *blk)
1220 {
1221     IO_CODE();
1222     return (blk->iostatus_enabled &&
1223            (blk->on_write_error == BLOCKDEV_ON_ERROR_ENOSPC ||
1224             blk->on_write_error == BLOCKDEV_ON_ERROR_STOP   ||
1225             blk->on_read_error == BLOCKDEV_ON_ERROR_STOP));
1226 }
1227
1228 BlockDeviceIoStatus blk_iostatus(const BlockBackend *blk)
1229 {
1230     GLOBAL_STATE_CODE();
1231     return blk->iostatus;
1232 }
1233
1234 void blk_iostatus_disable(BlockBackend *blk)
1235 {
1236     GLOBAL_STATE_CODE();
1237     blk->iostatus_enabled = false;
1238 }
1239
1240 void blk_iostatus_reset(BlockBackend *blk)
1241 {
1242     GLOBAL_STATE_CODE();
1243     if (blk_iostatus_is_enabled(blk)) {
1244         blk->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
1245     }
1246 }
1247
1248 void blk_iostatus_set_err(BlockBackend *blk, int error)
1249 {
1250     IO_CODE();
1251     assert(blk_iostatus_is_enabled(blk));
1252     if (blk->iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
1253         blk->iostatus = error == ENOSPC ? BLOCK_DEVICE_IO_STATUS_NOSPACE :
1254                                           BLOCK_DEVICE_IO_STATUS_FAILED;
1255     }
1256 }
1257
1258 void blk_set_allow_write_beyond_eof(BlockBackend *blk, bool allow)
1259 {
1260     IO_CODE();
1261     blk->allow_write_beyond_eof = allow;
1262 }
1263
1264 void blk_set_allow_aio_context_change(BlockBackend *blk, bool allow)
1265 {
1266     IO_CODE();
1267     blk->allow_aio_context_change = allow;
1268 }
1269
1270 void blk_set_disable_request_queuing(BlockBackend *blk, bool disable)
1271 {
1272     IO_CODE();
1273     qatomic_set(&blk->disable_request_queuing, disable);
1274 }
1275
1276 static int coroutine_fn GRAPH_RDLOCK
1277 blk_check_byte_request(BlockBackend *blk, int64_t offset, int64_t bytes)
1278 {
1279     int64_t len;
1280
1281     if (bytes < 0) {
1282         return -EIO;
1283     }
1284
1285     if (!blk_co_is_available(blk)) {
1286         return -ENOMEDIUM;
1287     }
1288
1289     if (offset < 0) {
1290         return -EIO;
1291     }
1292
1293     if (!blk->allow_write_beyond_eof) {
1294         len = bdrv_co_getlength(blk_bs(blk));
1295         if (len < 0) {
1296             return len;
1297         }
1298
1299         if (offset > len || len - offset < bytes) {
1300             return -EIO;
1301         }
1302     }
1303
1304     return 0;
1305 }
1306
1307 /* Are we currently in a drained section? */
1308 bool blk_in_drain(BlockBackend *blk)
1309 {
1310     GLOBAL_STATE_CODE(); /* change to IO_OR_GS_CODE(), if necessary */
1311     return qatomic_read(&blk->quiesce_counter);
1312 }
1313
1314 /* To be called between exactly one pair of blk_inc/dec_in_flight() */
1315 static void coroutine_fn blk_wait_while_drained(BlockBackend *blk)
1316 {
1317     assert(blk->in_flight > 0);
1318
1319     if (qatomic_read(&blk->quiesce_counter) &&
1320         !qatomic_read(&blk->disable_request_queuing)) {
1321         /*
1322          * Take lock before decrementing in flight counter so main loop thread
1323          * waits for us to enqueue ourselves before it can leave the drained
1324          * section.
1325          */
1326         qemu_mutex_lock(&blk->queued_requests_lock);
1327         blk_dec_in_flight(blk);
1328         qemu_co_queue_wait(&blk->queued_requests, &blk->queued_requests_lock);
1329         blk_inc_in_flight(blk);
1330         qemu_mutex_unlock(&blk->queued_requests_lock);
1331     }
1332 }
1333
1334 /* To be called between exactly one pair of blk_inc/dec_in_flight() */
1335 static int coroutine_fn
1336 blk_co_do_preadv_part(BlockBackend *blk, int64_t offset, int64_t bytes,
1337                       QEMUIOVector *qiov, size_t qiov_offset,
1338                       BdrvRequestFlags flags)
1339 {
1340     int ret;
1341     BlockDriverState *bs;
1342     IO_CODE();
1343
1344     blk_wait_while_drained(blk);
1345     GRAPH_RDLOCK_GUARD();
1346
1347     /* Call blk_bs() only after waiting, the graph may have changed */
1348     bs = blk_bs(blk);
1349     trace_blk_co_preadv(blk, bs, offset, bytes, flags);
1350
1351     ret = blk_check_byte_request(blk, offset, bytes);
1352     if (ret < 0) {
1353         return ret;
1354     }
1355
1356     bdrv_inc_in_flight(bs);
1357
1358     /* throttling disk I/O */
1359     if (blk->public.throttle_group_member.throttle_state) {
1360         throttle_group_co_io_limits_intercept(&blk->public.throttle_group_member,
1361                 bytes, false);
1362     }
1363
1364     ret = bdrv_co_preadv_part(blk->root, offset, bytes, qiov, qiov_offset,
1365                               flags);
1366     bdrv_dec_in_flight(bs);
1367     return ret;
1368 }
1369
1370 int coroutine_fn blk_co_pread(BlockBackend *blk, int64_t offset, int64_t bytes,
1371                               void *buf, BdrvRequestFlags flags)
1372 {
1373     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes);
1374     IO_OR_GS_CODE();
1375
1376     assert(bytes <= SIZE_MAX);
1377
1378     return blk_co_preadv(blk, offset, bytes, &qiov, flags);
1379 }
1380
1381 int coroutine_fn blk_co_preadv(BlockBackend *blk, int64_t offset,
1382                                int64_t bytes, QEMUIOVector *qiov,
1383                                BdrvRequestFlags flags)
1384 {
1385     int ret;
1386     IO_OR_GS_CODE();
1387
1388     blk_inc_in_flight(blk);
1389     ret = blk_co_do_preadv_part(blk, offset, bytes, qiov, 0, flags);
1390     blk_dec_in_flight(blk);
1391
1392     return ret;
1393 }
1394
1395 int coroutine_fn blk_co_preadv_part(BlockBackend *blk, int64_t offset,
1396                                     int64_t bytes, QEMUIOVector *qiov,
1397                                     size_t qiov_offset, BdrvRequestFlags flags)
1398 {
1399     int ret;
1400     IO_OR_GS_CODE();
1401
1402     blk_inc_in_flight(blk);
1403     ret = blk_co_do_preadv_part(blk, offset, bytes, qiov, qiov_offset, flags);
1404     blk_dec_in_flight(blk);
1405
1406     return ret;
1407 }
1408
1409 /* To be called between exactly one pair of blk_inc/dec_in_flight() */
1410 static int coroutine_fn
1411 blk_co_do_pwritev_part(BlockBackend *blk, int64_t offset, int64_t bytes,
1412                        QEMUIOVector *qiov, size_t qiov_offset,
1413                        BdrvRequestFlags flags)
1414 {
1415     int ret;
1416     BlockDriverState *bs;
1417     IO_CODE();
1418
1419     blk_wait_while_drained(blk);
1420     GRAPH_RDLOCK_GUARD();
1421
1422     /* Call blk_bs() only after waiting, the graph may have changed */
1423     bs = blk_bs(blk);
1424     trace_blk_co_pwritev(blk, bs, offset, bytes, flags);
1425
1426     ret = blk_check_byte_request(blk, offset, bytes);
1427     if (ret < 0) {
1428         return ret;
1429     }
1430
1431     bdrv_inc_in_flight(bs);
1432     /* throttling disk I/O */
1433     if (blk->public.throttle_group_member.throttle_state) {
1434         throttle_group_co_io_limits_intercept(&blk->public.throttle_group_member,
1435                 bytes, true);
1436     }
1437
1438     if (!blk->enable_write_cache) {
1439         flags |= BDRV_REQ_FUA;
1440     }
1441
1442     ret = bdrv_co_pwritev_part(blk->root, offset, bytes, qiov, qiov_offset,
1443                                flags);
1444     bdrv_dec_in_flight(bs);
1445     return ret;
1446 }
1447
1448 int coroutine_fn blk_co_pwritev_part(BlockBackend *blk, int64_t offset,
1449                                      int64_t bytes,
1450                                      QEMUIOVector *qiov, size_t qiov_offset,
1451                                      BdrvRequestFlags flags)
1452 {
1453     int ret;
1454     IO_OR_GS_CODE();
1455
1456     blk_inc_in_flight(blk);
1457     ret = blk_co_do_pwritev_part(blk, offset, bytes, qiov, qiov_offset, flags);
1458     blk_dec_in_flight(blk);
1459
1460     return ret;
1461 }
1462
1463 int coroutine_fn blk_co_pwrite(BlockBackend *blk, int64_t offset, int64_t bytes,
1464                                const void *buf, BdrvRequestFlags flags)
1465 {
1466     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes);
1467     IO_OR_GS_CODE();
1468
1469     assert(bytes <= SIZE_MAX);
1470
1471     return blk_co_pwritev(blk, offset, bytes, &qiov, flags);
1472 }
1473
1474 int coroutine_fn blk_co_pwritev(BlockBackend *blk, int64_t offset,
1475                                 int64_t bytes, QEMUIOVector *qiov,
1476                                 BdrvRequestFlags flags)
1477 {
1478     IO_OR_GS_CODE();
1479     return blk_co_pwritev_part(blk, offset, bytes, qiov, 0, flags);
1480 }
1481
1482 int coroutine_fn blk_co_block_status_above(BlockBackend *blk,
1483                                            BlockDriverState *base,
1484                                            int64_t offset, int64_t bytes,
1485                                            int64_t *pnum, int64_t *map,
1486                                            BlockDriverState **file)
1487 {
1488     IO_CODE();
1489     GRAPH_RDLOCK_GUARD();
1490     return bdrv_co_block_status_above(blk_bs(blk), base, offset, bytes, pnum,
1491                                       map, file);
1492 }
1493
1494 int coroutine_fn blk_co_is_allocated_above(BlockBackend *blk,
1495                                            BlockDriverState *base,
1496                                            bool include_base, int64_t offset,
1497                                            int64_t bytes, int64_t *pnum)
1498 {
1499     IO_CODE();
1500     GRAPH_RDLOCK_GUARD();
1501     return bdrv_co_is_allocated_above(blk_bs(blk), base, include_base, offset,
1502                                       bytes, pnum);
1503 }
1504
1505 typedef struct BlkRwCo {
1506     BlockBackend *blk;
1507     int64_t offset;
1508     void *iobuf;
1509     int ret;
1510     BdrvRequestFlags flags;
1511 } BlkRwCo;
1512
1513 int blk_make_zero(BlockBackend *blk, BdrvRequestFlags flags)
1514 {
1515     GLOBAL_STATE_CODE();
1516     return bdrv_make_zero(blk->root, flags);
1517 }
1518
1519 void blk_inc_in_flight(BlockBackend *blk)
1520 {
1521     IO_CODE();
1522     qatomic_inc(&blk->in_flight);
1523 }
1524
1525 void blk_dec_in_flight(BlockBackend *blk)
1526 {
1527     IO_CODE();
1528     qatomic_dec(&blk->in_flight);
1529     aio_wait_kick();
1530 }
1531
1532 static void error_callback_bh(void *opaque)
1533 {
1534     struct BlockBackendAIOCB *acb = opaque;
1535
1536     blk_dec_in_flight(acb->blk);
1537     acb->common.cb(acb->common.opaque, acb->ret);
1538     qemu_aio_unref(acb);
1539 }
1540
1541 BlockAIOCB *blk_abort_aio_request(BlockBackend *blk,
1542                                   BlockCompletionFunc *cb,
1543                                   void *opaque, int ret)
1544 {
1545     struct BlockBackendAIOCB *acb;
1546     IO_CODE();
1547
1548     blk_inc_in_flight(blk);
1549     acb = blk_aio_get(&block_backend_aiocb_info, blk, cb, opaque);
1550     acb->blk = blk;
1551     acb->ret = ret;
1552
1553     replay_bh_schedule_oneshot_event(blk_get_aio_context(blk),
1554                                      error_callback_bh, acb);
1555     return &acb->common;
1556 }
1557
1558 typedef struct BlkAioEmAIOCB {
1559     BlockAIOCB common;
1560     BlkRwCo rwco;
1561     int64_t bytes;
1562     bool has_returned;
1563 } BlkAioEmAIOCB;
1564
1565 static AioContext *blk_aio_em_aiocb_get_aio_context(BlockAIOCB *acb_)
1566 {
1567     BlkAioEmAIOCB *acb = container_of(acb_, BlkAioEmAIOCB, common);
1568
1569     return blk_get_aio_context(acb->rwco.blk);
1570 }
1571
1572 static const AIOCBInfo blk_aio_em_aiocb_info = {
1573     .aiocb_size         = sizeof(BlkAioEmAIOCB),
1574     .get_aio_context    = blk_aio_em_aiocb_get_aio_context,
1575 };
1576
1577 static void blk_aio_complete(BlkAioEmAIOCB *acb)
1578 {
1579     if (acb->has_returned) {
1580         acb->common.cb(acb->common.opaque, acb->rwco.ret);
1581         blk_dec_in_flight(acb->rwco.blk);
1582         qemu_aio_unref(acb);
1583     }
1584 }
1585
1586 static void blk_aio_complete_bh(void *opaque)
1587 {
1588     BlkAioEmAIOCB *acb = opaque;
1589     assert(acb->has_returned);
1590     blk_aio_complete(acb);
1591 }
1592
1593 static BlockAIOCB *blk_aio_prwv(BlockBackend *blk, int64_t offset,
1594                                 int64_t bytes,
1595                                 void *iobuf, CoroutineEntry co_entry,
1596                                 BdrvRequestFlags flags,
1597                                 BlockCompletionFunc *cb, void *opaque)
1598 {
1599     BlkAioEmAIOCB *acb;
1600     Coroutine *co;
1601
1602     blk_inc_in_flight(blk);
1603     acb = blk_aio_get(&blk_aio_em_aiocb_info, blk, cb, opaque);
1604     acb->rwco = (BlkRwCo) {
1605         .blk    = blk,
1606         .offset = offset,
1607         .iobuf  = iobuf,
1608         .flags  = flags,
1609         .ret    = NOT_DONE,
1610     };
1611     acb->bytes = bytes;
1612     acb->has_returned = false;
1613
1614     co = qemu_coroutine_create(co_entry, acb);
1615     aio_co_enter(blk_get_aio_context(blk), co);
1616
1617     acb->has_returned = true;
1618     if (acb->rwco.ret != NOT_DONE) {
1619         replay_bh_schedule_oneshot_event(blk_get_aio_context(blk),
1620                                          blk_aio_complete_bh, acb);
1621     }
1622
1623     return &acb->common;
1624 }
1625
1626 static void coroutine_fn blk_aio_read_entry(void *opaque)
1627 {
1628     BlkAioEmAIOCB *acb = opaque;
1629     BlkRwCo *rwco = &acb->rwco;
1630     QEMUIOVector *qiov = rwco->iobuf;
1631
1632     assert(qiov->size == acb->bytes);
1633     rwco->ret = blk_co_do_preadv_part(rwco->blk, rwco->offset, acb->bytes, qiov,
1634                                       0, rwco->flags);
1635     blk_aio_complete(acb);
1636 }
1637
1638 static void coroutine_fn blk_aio_write_entry(void *opaque)
1639 {
1640     BlkAioEmAIOCB *acb = opaque;
1641     BlkRwCo *rwco = &acb->rwco;
1642     QEMUIOVector *qiov = rwco->iobuf;
1643
1644     assert(!qiov || qiov->size == acb->bytes);
1645     rwco->ret = blk_co_do_pwritev_part(rwco->blk, rwco->offset, acb->bytes,
1646                                        qiov, 0, rwco->flags);
1647     blk_aio_complete(acb);
1648 }
1649
1650 BlockAIOCB *blk_aio_pwrite_zeroes(BlockBackend *blk, int64_t offset,
1651                                   int64_t bytes, BdrvRequestFlags flags,
1652                                   BlockCompletionFunc *cb, void *opaque)
1653 {
1654     IO_CODE();
1655     return blk_aio_prwv(blk, offset, bytes, NULL, blk_aio_write_entry,
1656                         flags | BDRV_REQ_ZERO_WRITE, cb, opaque);
1657 }
1658
1659 int64_t coroutine_fn blk_co_getlength(BlockBackend *blk)
1660 {
1661     IO_CODE();
1662     GRAPH_RDLOCK_GUARD();
1663
1664     if (!blk_co_is_available(blk)) {
1665         return -ENOMEDIUM;
1666     }
1667
1668     return bdrv_co_getlength(blk_bs(blk));
1669 }
1670
1671 int64_t coroutine_fn blk_co_nb_sectors(BlockBackend *blk)
1672 {
1673     BlockDriverState *bs = blk_bs(blk);
1674
1675     IO_CODE();
1676     GRAPH_RDLOCK_GUARD();
1677
1678     if (!bs) {
1679         return -ENOMEDIUM;
1680     } else {
1681         return bdrv_co_nb_sectors(bs);
1682     }
1683 }
1684
1685 /*
1686  * This wrapper is written by hand because this function is in the hot I/O path,
1687  * via blk_get_geometry.
1688  */
1689 int64_t coroutine_mixed_fn blk_nb_sectors(BlockBackend *blk)
1690 {
1691     BlockDriverState *bs = blk_bs(blk);
1692
1693     IO_CODE();
1694
1695     if (!bs) {
1696         return -ENOMEDIUM;
1697     } else {
1698         return bdrv_nb_sectors(bs);
1699     }
1700 }
1701
1702 /* return 0 as number of sectors if no device present or error */
1703 void coroutine_fn blk_co_get_geometry(BlockBackend *blk,
1704                                       uint64_t *nb_sectors_ptr)
1705 {
1706     int64_t ret = blk_co_nb_sectors(blk);
1707     *nb_sectors_ptr = ret < 0 ? 0 : ret;
1708 }
1709
1710 /*
1711  * This wrapper is written by hand because this function is in the hot I/O path.
1712  */
1713 void coroutine_mixed_fn blk_get_geometry(BlockBackend *blk,
1714                                          uint64_t *nb_sectors_ptr)
1715 {
1716     int64_t ret = blk_nb_sectors(blk);
1717     *nb_sectors_ptr = ret < 0 ? 0 : ret;
1718 }
1719
1720 BlockAIOCB *blk_aio_preadv(BlockBackend *blk, int64_t offset,
1721                            QEMUIOVector *qiov, BdrvRequestFlags flags,
1722                            BlockCompletionFunc *cb, void *opaque)
1723 {
1724     IO_CODE();
1725     assert((uint64_t)qiov->size <= INT64_MAX);
1726     return blk_aio_prwv(blk, offset, qiov->size, qiov,
1727                         blk_aio_read_entry, flags, cb, opaque);
1728 }
1729
1730 BlockAIOCB *blk_aio_pwritev(BlockBackend *blk, int64_t offset,
1731                             QEMUIOVector *qiov, BdrvRequestFlags flags,
1732                             BlockCompletionFunc *cb, void *opaque)
1733 {
1734     IO_CODE();
1735     assert((uint64_t)qiov->size <= INT64_MAX);
1736     return blk_aio_prwv(blk, offset, qiov->size, qiov,
1737                         blk_aio_write_entry, flags, cb, opaque);
1738 }
1739
1740 void blk_aio_cancel(BlockAIOCB *acb)
1741 {
1742     GLOBAL_STATE_CODE();
1743     bdrv_aio_cancel(acb);
1744 }
1745
1746 void blk_aio_cancel_async(BlockAIOCB *acb)
1747 {
1748     IO_CODE();
1749     bdrv_aio_cancel_async(acb);
1750 }
1751
1752 /* To be called between exactly one pair of blk_inc/dec_in_flight() */
1753 static int coroutine_fn
1754 blk_co_do_ioctl(BlockBackend *blk, unsigned long int req, void *buf)
1755 {
1756     IO_CODE();
1757
1758     blk_wait_while_drained(blk);
1759     GRAPH_RDLOCK_GUARD();
1760
1761     if (!blk_co_is_available(blk)) {
1762         return -ENOMEDIUM;
1763     }
1764
1765     return bdrv_co_ioctl(blk_bs(blk), req, buf);
1766 }
1767
1768 int coroutine_fn blk_co_ioctl(BlockBackend *blk, unsigned long int req,
1769                               void *buf)
1770 {
1771     int ret;
1772     IO_OR_GS_CODE();
1773
1774     blk_inc_in_flight(blk);
1775     ret = blk_co_do_ioctl(blk, req, buf);
1776     blk_dec_in_flight(blk);
1777
1778     return ret;
1779 }
1780
1781 static void coroutine_fn blk_aio_ioctl_entry(void *opaque)
1782 {
1783     BlkAioEmAIOCB *acb = opaque;
1784     BlkRwCo *rwco = &acb->rwco;
1785
1786     rwco->ret = blk_co_do_ioctl(rwco->blk, rwco->offset, rwco->iobuf);
1787
1788     blk_aio_complete(acb);
1789 }
1790
1791 BlockAIOCB *blk_aio_ioctl(BlockBackend *blk, unsigned long int req, void *buf,
1792                           BlockCompletionFunc *cb, void *opaque)
1793 {
1794     IO_CODE();
1795     return blk_aio_prwv(blk, req, 0, buf, blk_aio_ioctl_entry, 0, cb, opaque);
1796 }
1797
1798 /* To be called between exactly one pair of blk_inc/dec_in_flight() */
1799 static int coroutine_fn
1800 blk_co_do_pdiscard(BlockBackend *blk, int64_t offset, int64_t bytes)
1801 {
1802     int ret;
1803     IO_CODE();
1804
1805     blk_wait_while_drained(blk);
1806     GRAPH_RDLOCK_GUARD();
1807
1808     ret = blk_check_byte_request(blk, offset, bytes);
1809     if (ret < 0) {
1810         return ret;
1811     }
1812
1813     return bdrv_co_pdiscard(blk->root, offset, bytes);
1814 }
1815
1816 static void coroutine_fn blk_aio_pdiscard_entry(void *opaque)
1817 {
1818     BlkAioEmAIOCB *acb = opaque;
1819     BlkRwCo *rwco = &acb->rwco;
1820
1821     rwco->ret = blk_co_do_pdiscard(rwco->blk, rwco->offset, acb->bytes);
1822     blk_aio_complete(acb);
1823 }
1824
1825 BlockAIOCB *blk_aio_pdiscard(BlockBackend *blk,
1826                              int64_t offset, int64_t bytes,
1827                              BlockCompletionFunc *cb, void *opaque)
1828 {
1829     IO_CODE();
1830     return blk_aio_prwv(blk, offset, bytes, NULL, blk_aio_pdiscard_entry, 0,
1831                         cb, opaque);
1832 }
1833
1834 int coroutine_fn blk_co_pdiscard(BlockBackend *blk, int64_t offset,
1835                                  int64_t bytes)
1836 {
1837     int ret;
1838     IO_OR_GS_CODE();
1839
1840     blk_inc_in_flight(blk);
1841     ret = blk_co_do_pdiscard(blk, offset, bytes);
1842     blk_dec_in_flight(blk);
1843
1844     return ret;
1845 }
1846
1847 /* To be called between exactly one pair of blk_inc/dec_in_flight() */
1848 static int coroutine_fn blk_co_do_flush(BlockBackend *blk)
1849 {
1850     IO_CODE();
1851     blk_wait_while_drained(blk);
1852     GRAPH_RDLOCK_GUARD();
1853
1854     if (!blk_co_is_available(blk)) {
1855         return -ENOMEDIUM;
1856     }
1857
1858     return bdrv_co_flush(blk_bs(blk));
1859 }
1860
1861 static void coroutine_fn blk_aio_flush_entry(void *opaque)
1862 {
1863     BlkAioEmAIOCB *acb = opaque;
1864     BlkRwCo *rwco = &acb->rwco;
1865
1866     rwco->ret = blk_co_do_flush(rwco->blk);
1867     blk_aio_complete(acb);
1868 }
1869
1870 BlockAIOCB *blk_aio_flush(BlockBackend *blk,
1871                           BlockCompletionFunc *cb, void *opaque)
1872 {
1873     IO_CODE();
1874     return blk_aio_prwv(blk, 0, 0, NULL, blk_aio_flush_entry, 0, cb, opaque);
1875 }
1876
1877 int coroutine_fn blk_co_flush(BlockBackend *blk)
1878 {
1879     int ret;
1880     IO_OR_GS_CODE();
1881
1882     blk_inc_in_flight(blk);
1883     ret = blk_co_do_flush(blk);
1884     blk_dec_in_flight(blk);
1885
1886     return ret;
1887 }
1888
1889 static void coroutine_fn blk_aio_zone_report_entry(void *opaque)
1890 {
1891     BlkAioEmAIOCB *acb = opaque;
1892     BlkRwCo *rwco = &acb->rwco;
1893
1894     rwco->ret = blk_co_zone_report(rwco->blk, rwco->offset,
1895                                    (unsigned int*)(uintptr_t)acb->bytes,
1896                                    rwco->iobuf);
1897     blk_aio_complete(acb);
1898 }
1899
1900 BlockAIOCB *blk_aio_zone_report(BlockBackend *blk, int64_t offset,
1901                                 unsigned int *nr_zones,
1902                                 BlockZoneDescriptor  *zones,
1903                                 BlockCompletionFunc *cb, void *opaque)
1904 {
1905     BlkAioEmAIOCB *acb;
1906     Coroutine *co;
1907     IO_CODE();
1908
1909     blk_inc_in_flight(blk);
1910     acb = blk_aio_get(&blk_aio_em_aiocb_info, blk, cb, opaque);
1911     acb->rwco = (BlkRwCo) {
1912         .blk    = blk,
1913         .offset = offset,
1914         .iobuf  = zones,
1915         .ret    = NOT_DONE,
1916     };
1917     acb->bytes = (int64_t)(uintptr_t)nr_zones,
1918     acb->has_returned = false;
1919
1920     co = qemu_coroutine_create(blk_aio_zone_report_entry, acb);
1921     aio_co_enter(blk_get_aio_context(blk), co);
1922
1923     acb->has_returned = true;
1924     if (acb->rwco.ret != NOT_DONE) {
1925         replay_bh_schedule_oneshot_event(blk_get_aio_context(blk),
1926                                          blk_aio_complete_bh, acb);
1927     }
1928
1929     return &acb->common;
1930 }
1931
1932 static void coroutine_fn blk_aio_zone_mgmt_entry(void *opaque)
1933 {
1934     BlkAioEmAIOCB *acb = opaque;
1935     BlkRwCo *rwco = &acb->rwco;
1936
1937     rwco->ret = blk_co_zone_mgmt(rwco->blk,
1938                                  (BlockZoneOp)(uintptr_t)rwco->iobuf,
1939                                  rwco->offset, acb->bytes);
1940     blk_aio_complete(acb);
1941 }
1942
1943 BlockAIOCB *blk_aio_zone_mgmt(BlockBackend *blk, BlockZoneOp op,
1944                               int64_t offset, int64_t len,
1945                               BlockCompletionFunc *cb, void *opaque) {
1946     BlkAioEmAIOCB *acb;
1947     Coroutine *co;
1948     IO_CODE();
1949
1950     blk_inc_in_flight(blk);
1951     acb = blk_aio_get(&blk_aio_em_aiocb_info, blk, cb, opaque);
1952     acb->rwco = (BlkRwCo) {
1953         .blk    = blk,
1954         .offset = offset,
1955         .iobuf  = (void *)(uintptr_t)op,
1956         .ret    = NOT_DONE,
1957     };
1958     acb->bytes = len;
1959     acb->has_returned = false;
1960
1961     co = qemu_coroutine_create(blk_aio_zone_mgmt_entry, acb);
1962     aio_co_enter(blk_get_aio_context(blk), co);
1963
1964     acb->has_returned = true;
1965     if (acb->rwco.ret != NOT_DONE) {
1966         replay_bh_schedule_oneshot_event(blk_get_aio_context(blk),
1967                                          blk_aio_complete_bh, acb);
1968     }
1969
1970     return &acb->common;
1971 }
1972
1973 static void coroutine_fn blk_aio_zone_append_entry(void *opaque)
1974 {
1975     BlkAioEmAIOCB *acb = opaque;
1976     BlkRwCo *rwco = &acb->rwco;
1977
1978     rwco->ret = blk_co_zone_append(rwco->blk, (int64_t *)(uintptr_t)acb->bytes,
1979                                    rwco->iobuf, rwco->flags);
1980     blk_aio_complete(acb);
1981 }
1982
1983 BlockAIOCB *blk_aio_zone_append(BlockBackend *blk, int64_t *offset,
1984                                 QEMUIOVector *qiov, BdrvRequestFlags flags,
1985                                 BlockCompletionFunc *cb, void *opaque) {
1986     BlkAioEmAIOCB *acb;
1987     Coroutine *co;
1988     IO_CODE();
1989
1990     blk_inc_in_flight(blk);
1991     acb = blk_aio_get(&blk_aio_em_aiocb_info, blk, cb, opaque);
1992     acb->rwco = (BlkRwCo) {
1993         .blk    = blk,
1994         .ret    = NOT_DONE,
1995         .flags  = flags,
1996         .iobuf  = qiov,
1997     };
1998     acb->bytes = (int64_t)(uintptr_t)offset;
1999     acb->has_returned = false;
2000
2001     co = qemu_coroutine_create(blk_aio_zone_append_entry, acb);
2002     aio_co_enter(blk_get_aio_context(blk), co);
2003     acb->has_returned = true;
2004     if (acb->rwco.ret != NOT_DONE) {
2005         replay_bh_schedule_oneshot_event(blk_get_aio_context(blk),
2006                                          blk_aio_complete_bh, acb);
2007     }
2008
2009     return &acb->common;
2010 }
2011
2012 /*
2013  * Send a zone_report command.
2014  * offset is a byte offset from the start of the device. No alignment
2015  * required for offset.
2016  * nr_zones represents IN maximum and OUT actual.
2017  */
2018 int coroutine_fn blk_co_zone_report(BlockBackend *blk, int64_t offset,
2019                                     unsigned int *nr_zones,
2020                                     BlockZoneDescriptor *zones)
2021 {
2022     int ret;
2023     IO_CODE();
2024
2025     blk_inc_in_flight(blk); /* increase before waiting */
2026     blk_wait_while_drained(blk);
2027     GRAPH_RDLOCK_GUARD();
2028     if (!blk_is_available(blk)) {
2029         blk_dec_in_flight(blk);
2030         return -ENOMEDIUM;
2031     }
2032     ret = bdrv_co_zone_report(blk_bs(blk), offset, nr_zones, zones);
2033     blk_dec_in_flight(blk);
2034     return ret;
2035 }
2036
2037 /*
2038  * Send a zone_management command.
2039  * op is the zone operation;
2040  * offset is the byte offset from the start of the zoned device;
2041  * len is the maximum number of bytes the command should operate on. It
2042  * should be aligned with the device zone size.
2043  */
2044 int coroutine_fn blk_co_zone_mgmt(BlockBackend *blk, BlockZoneOp op,
2045         int64_t offset, int64_t len)
2046 {
2047     int ret;
2048     IO_CODE();
2049
2050     blk_inc_in_flight(blk);
2051     blk_wait_while_drained(blk);
2052     GRAPH_RDLOCK_GUARD();
2053
2054     ret = blk_check_byte_request(blk, offset, len);
2055     if (ret < 0) {
2056         blk_dec_in_flight(blk);
2057         return ret;
2058     }
2059
2060     ret = bdrv_co_zone_mgmt(blk_bs(blk), op, offset, len);
2061     blk_dec_in_flight(blk);
2062     return ret;
2063 }
2064
2065 /*
2066  * Send a zone_append command.
2067  */
2068 int coroutine_fn blk_co_zone_append(BlockBackend *blk, int64_t *offset,
2069         QEMUIOVector *qiov, BdrvRequestFlags flags)
2070 {
2071     int ret;
2072     IO_CODE();
2073
2074     blk_inc_in_flight(blk);
2075     blk_wait_while_drained(blk);
2076     GRAPH_RDLOCK_GUARD();
2077     if (!blk_is_available(blk)) {
2078         blk_dec_in_flight(blk);
2079         return -ENOMEDIUM;
2080     }
2081
2082     ret = bdrv_co_zone_append(blk_bs(blk), offset, qiov, flags);
2083     blk_dec_in_flight(blk);
2084     return ret;
2085 }
2086
2087 void blk_drain(BlockBackend *blk)
2088 {
2089     BlockDriverState *bs = blk_bs(blk);
2090     GLOBAL_STATE_CODE();
2091
2092     if (bs) {
2093         bdrv_ref(bs);
2094         bdrv_drained_begin(bs);
2095     }
2096
2097     /* We may have -ENOMEDIUM completions in flight */
2098     AIO_WAIT_WHILE(blk_get_aio_context(blk),
2099                    qatomic_read(&blk->in_flight) > 0);
2100
2101     if (bs) {
2102         bdrv_drained_end(bs);
2103         bdrv_unref(bs);
2104     }
2105 }
2106
2107 void blk_drain_all(void)
2108 {
2109     BlockBackend *blk = NULL;
2110
2111     GLOBAL_STATE_CODE();
2112
2113     bdrv_drain_all_begin();
2114
2115     while ((blk = blk_all_next(blk)) != NULL) {
2116         /* We may have -ENOMEDIUM completions in flight */
2117         AIO_WAIT_WHILE_UNLOCKED(NULL, qatomic_read(&blk->in_flight) > 0);
2118     }
2119
2120     bdrv_drain_all_end();
2121 }
2122
2123 void blk_set_on_error(BlockBackend *blk, BlockdevOnError on_read_error,
2124                       BlockdevOnError on_write_error)
2125 {
2126     GLOBAL_STATE_CODE();
2127     blk->on_read_error = on_read_error;
2128     blk->on_write_error = on_write_error;
2129 }
2130
2131 BlockdevOnError blk_get_on_error(BlockBackend *blk, bool is_read)
2132 {
2133     IO_CODE();
2134     return is_read ? blk->on_read_error : blk->on_write_error;
2135 }
2136
2137 BlockErrorAction blk_get_error_action(BlockBackend *blk, bool is_read,
2138                                       int error)
2139 {
2140     BlockdevOnError on_err = blk_get_on_error(blk, is_read);
2141     IO_CODE();
2142
2143     switch (on_err) {
2144     case BLOCKDEV_ON_ERROR_ENOSPC:
2145         return (error == ENOSPC) ?
2146                BLOCK_ERROR_ACTION_STOP : BLOCK_ERROR_ACTION_REPORT;
2147     case BLOCKDEV_ON_ERROR_STOP:
2148         return BLOCK_ERROR_ACTION_STOP;
2149     case BLOCKDEV_ON_ERROR_REPORT:
2150         return BLOCK_ERROR_ACTION_REPORT;
2151     case BLOCKDEV_ON_ERROR_IGNORE:
2152         return BLOCK_ERROR_ACTION_IGNORE;
2153     case BLOCKDEV_ON_ERROR_AUTO:
2154     default:
2155         abort();
2156     }
2157 }
2158
2159 static void send_qmp_error_event(BlockBackend *blk,
2160                                  BlockErrorAction action,
2161                                  bool is_read, int error)
2162 {
2163     IoOperationType optype;
2164     BlockDriverState *bs = blk_bs(blk);
2165
2166     optype = is_read ? IO_OPERATION_TYPE_READ : IO_OPERATION_TYPE_WRITE;
2167     qapi_event_send_block_io_error(blk_name(blk),
2168                                    bs ? bdrv_get_node_name(bs) : NULL, optype,
2169                                    action, blk_iostatus_is_enabled(blk),
2170                                    error == ENOSPC, strerror(error));
2171 }
2172
2173 /* This is done by device models because, while the block layer knows
2174  * about the error, it does not know whether an operation comes from
2175  * the device or the block layer (from a job, for example).
2176  */
2177 void blk_error_action(BlockBackend *blk, BlockErrorAction action,
2178                       bool is_read, int error)
2179 {
2180     assert(error >= 0);
2181     IO_CODE();
2182
2183     if (action == BLOCK_ERROR_ACTION_STOP) {
2184         /* First set the iostatus, so that "info block" returns an iostatus
2185          * that matches the events raised so far (an additional error iostatus
2186          * is fine, but not a lost one).
2187          */
2188         blk_iostatus_set_err(blk, error);
2189
2190         /* Then raise the request to stop the VM and the event.
2191          * qemu_system_vmstop_request_prepare has two effects.  First,
2192          * it ensures that the STOP event always comes after the
2193          * BLOCK_IO_ERROR event.  Second, it ensures that even if management
2194          * can observe the STOP event and do a "cont" before the STOP
2195          * event is issued, the VM will not stop.  In this case, vm_start()
2196          * also ensures that the STOP/RESUME pair of events is emitted.
2197          */
2198         qemu_system_vmstop_request_prepare();
2199         send_qmp_error_event(blk, action, is_read, error);
2200         qemu_system_vmstop_request(RUN_STATE_IO_ERROR);
2201     } else {
2202         send_qmp_error_event(blk, action, is_read, error);
2203     }
2204 }
2205
2206 /*
2207  * Returns true if the BlockBackend can support taking write permissions
2208  * (because its root node is not read-only).
2209  */
2210 bool blk_supports_write_perm(BlockBackend *blk)
2211 {
2212     BlockDriverState *bs = blk_bs(blk);
2213     GLOBAL_STATE_CODE();
2214
2215     if (bs) {
2216         return !bdrv_is_read_only(bs);
2217     } else {
2218         return blk->root_state.open_flags & BDRV_O_RDWR;
2219     }
2220 }
2221
2222 /*
2223  * Returns true if the BlockBackend can be written to in its current
2224  * configuration (i.e. if write permission have been requested)
2225  */
2226 bool blk_is_writable(BlockBackend *blk)
2227 {
2228     IO_CODE();
2229     return blk->perm & BLK_PERM_WRITE;
2230 }
2231
2232 bool blk_is_sg(BlockBackend *blk)
2233 {
2234     BlockDriverState *bs = blk_bs(blk);
2235     GLOBAL_STATE_CODE();
2236
2237     if (!bs) {
2238         return false;
2239     }
2240
2241     return bdrv_is_sg(bs);
2242 }
2243
2244 bool blk_enable_write_cache(BlockBackend *blk)
2245 {
2246     IO_CODE();
2247     return blk->enable_write_cache;
2248 }
2249
2250 void blk_set_enable_write_cache(BlockBackend *blk, bool wce)
2251 {
2252     IO_CODE();
2253     blk->enable_write_cache = wce;
2254 }
2255
2256 void blk_activate(BlockBackend *blk, Error **errp)
2257 {
2258     BlockDriverState *bs = blk_bs(blk);
2259     GLOBAL_STATE_CODE();
2260
2261     if (!bs) {
2262         error_setg(errp, "Device '%s' has no medium", blk->name);
2263         return;
2264     }
2265
2266     /*
2267      * Migration code can call this function in coroutine context, so leave
2268      * coroutine context if necessary.
2269      */
2270     if (qemu_in_coroutine()) {
2271         bdrv_co_activate(bs, errp);
2272     } else {
2273         bdrv_activate(bs, errp);
2274     }
2275 }
2276
2277 bool coroutine_fn blk_co_is_inserted(BlockBackend *blk)
2278 {
2279     BlockDriverState *bs = blk_bs(blk);
2280     IO_CODE();
2281     assert_bdrv_graph_readable();
2282
2283     return bs && bdrv_co_is_inserted(bs);
2284 }
2285
2286 bool coroutine_fn blk_co_is_available(BlockBackend *blk)
2287 {
2288     IO_CODE();
2289     return blk_co_is_inserted(blk) && !blk_dev_is_tray_open(blk);
2290 }
2291
2292 void coroutine_fn blk_co_lock_medium(BlockBackend *blk, bool locked)
2293 {
2294     BlockDriverState *bs = blk_bs(blk);
2295     IO_CODE();
2296     GRAPH_RDLOCK_GUARD();
2297
2298     if (bs) {
2299         bdrv_co_lock_medium(bs, locked);
2300     }
2301 }
2302
2303 void coroutine_fn blk_co_eject(BlockBackend *blk, bool eject_flag)
2304 {
2305     BlockDriverState *bs = blk_bs(blk);
2306     char *id;
2307     IO_CODE();
2308     GRAPH_RDLOCK_GUARD();
2309
2310     if (bs) {
2311         bdrv_co_eject(bs, eject_flag);
2312     }
2313
2314     /* Whether or not we ejected on the backend,
2315      * the frontend experienced a tray event. */
2316     id = blk_get_attached_dev_id(blk);
2317     qapi_event_send_device_tray_moved(blk_name(blk), id,
2318                                       eject_flag);
2319     g_free(id);
2320 }
2321
2322 int blk_get_flags(BlockBackend *blk)
2323 {
2324     BlockDriverState *bs = blk_bs(blk);
2325     GLOBAL_STATE_CODE();
2326
2327     if (bs) {
2328         return bdrv_get_flags(bs);
2329     } else {
2330         return blk->root_state.open_flags;
2331     }
2332 }
2333
2334 /* Returns the minimum request alignment, in bytes; guaranteed nonzero */
2335 uint32_t blk_get_request_alignment(BlockBackend *blk)
2336 {
2337     BlockDriverState *bs = blk_bs(blk);
2338     IO_CODE();
2339     return bs ? bs->bl.request_alignment : BDRV_SECTOR_SIZE;
2340 }
2341
2342 /* Returns the maximum hardware transfer length, in bytes; guaranteed nonzero */
2343 uint64_t blk_get_max_hw_transfer(BlockBackend *blk)
2344 {
2345     BlockDriverState *bs = blk_bs(blk);
2346     uint64_t max = INT_MAX;
2347     IO_CODE();
2348
2349     if (bs) {
2350         max = MIN_NON_ZERO(max, bs->bl.max_hw_transfer);
2351         max = MIN_NON_ZERO(max, bs->bl.max_transfer);
2352     }
2353     return ROUND_DOWN(max, blk_get_request_alignment(blk));
2354 }
2355
2356 /* Returns the maximum transfer length, in bytes; guaranteed nonzero */
2357 uint32_t blk_get_max_transfer(BlockBackend *blk)
2358 {
2359     BlockDriverState *bs = blk_bs(blk);
2360     uint32_t max = INT_MAX;
2361     IO_CODE();
2362
2363     if (bs) {
2364         max = MIN_NON_ZERO(max, bs->bl.max_transfer);
2365     }
2366     return ROUND_DOWN(max, blk_get_request_alignment(blk));
2367 }
2368
2369 int blk_get_max_hw_iov(BlockBackend *blk)
2370 {
2371     IO_CODE();
2372     return MIN_NON_ZERO(blk->root->bs->bl.max_hw_iov,
2373                         blk->root->bs->bl.max_iov);
2374 }
2375
2376 int blk_get_max_iov(BlockBackend *blk)
2377 {
2378     IO_CODE();
2379     return blk->root->bs->bl.max_iov;
2380 }
2381
2382 void *blk_try_blockalign(BlockBackend *blk, size_t size)
2383 {
2384     IO_CODE();
2385     return qemu_try_blockalign(blk ? blk_bs(blk) : NULL, size);
2386 }
2387
2388 void *blk_blockalign(BlockBackend *blk, size_t size)
2389 {
2390     IO_CODE();
2391     return qemu_blockalign(blk ? blk_bs(blk) : NULL, size);
2392 }
2393
2394 bool blk_op_is_blocked(BlockBackend *blk, BlockOpType op, Error **errp)
2395 {
2396     BlockDriverState *bs = blk_bs(blk);
2397     GLOBAL_STATE_CODE();
2398
2399     if (!bs) {
2400         return false;
2401     }
2402
2403     return bdrv_op_is_blocked(bs, op, errp);
2404 }
2405
2406 void blk_op_unblock(BlockBackend *blk, BlockOpType op, Error *reason)
2407 {
2408     BlockDriverState *bs = blk_bs(blk);
2409     GLOBAL_STATE_CODE();
2410
2411     if (bs) {
2412         bdrv_op_unblock(bs, op, reason);
2413     }
2414 }
2415
2416 void blk_op_block_all(BlockBackend *blk, Error *reason)
2417 {
2418     BlockDriverState *bs = blk_bs(blk);
2419     GLOBAL_STATE_CODE();
2420
2421     if (bs) {
2422         bdrv_op_block_all(bs, reason);
2423     }
2424 }
2425
2426 void blk_op_unblock_all(BlockBackend *blk, Error *reason)
2427 {
2428     BlockDriverState *bs = blk_bs(blk);
2429     GLOBAL_STATE_CODE();
2430
2431     if (bs) {
2432         bdrv_op_unblock_all(bs, reason);
2433     }
2434 }
2435
2436 AioContext *blk_get_aio_context(BlockBackend *blk)
2437 {
2438     BlockDriverState *bs;
2439     IO_CODE();
2440
2441     if (!blk) {
2442         return qemu_get_aio_context();
2443     }
2444
2445     bs = blk_bs(blk);
2446     if (bs) {
2447         AioContext *ctx = bdrv_get_aio_context(blk_bs(blk));
2448         assert(ctx == blk->ctx);
2449     }
2450
2451     return blk->ctx;
2452 }
2453
2454 static AioContext *blk_aiocb_get_aio_context(BlockAIOCB *acb)
2455 {
2456     BlockBackendAIOCB *blk_acb = DO_UPCAST(BlockBackendAIOCB, common, acb);
2457     return blk_get_aio_context(blk_acb->blk);
2458 }
2459
2460 int blk_set_aio_context(BlockBackend *blk, AioContext *new_context,
2461                         Error **errp)
2462 {
2463     bool old_allow_change;
2464     BlockDriverState *bs = blk_bs(blk);
2465     int ret;
2466
2467     GLOBAL_STATE_CODE();
2468
2469     if (!bs) {
2470         blk->ctx = new_context;
2471         return 0;
2472     }
2473
2474     bdrv_ref(bs);
2475
2476     old_allow_change = blk->allow_aio_context_change;
2477     blk->allow_aio_context_change = true;
2478
2479     ret = bdrv_try_change_aio_context(bs, new_context, NULL, errp);
2480
2481     blk->allow_aio_context_change = old_allow_change;
2482
2483     bdrv_unref(bs);
2484     return ret;
2485 }
2486
2487 typedef struct BdrvStateBlkRootContext {
2488     AioContext *new_ctx;
2489     BlockBackend *blk;
2490 } BdrvStateBlkRootContext;
2491
2492 static void blk_root_set_aio_ctx_commit(void *opaque)
2493 {
2494     BdrvStateBlkRootContext *s = opaque;
2495     BlockBackend *blk = s->blk;
2496     AioContext *new_context = s->new_ctx;
2497     ThrottleGroupMember *tgm = &blk->public.throttle_group_member;
2498
2499     blk->ctx = new_context;
2500     if (tgm->throttle_state) {
2501         throttle_group_detach_aio_context(tgm);
2502         throttle_group_attach_aio_context(tgm, new_context);
2503     }
2504 }
2505
2506 static TransactionActionDrv set_blk_root_context = {
2507     .commit = blk_root_set_aio_ctx_commit,
2508     .clean = g_free,
2509 };
2510
2511 static bool blk_root_change_aio_ctx(BdrvChild *child, AioContext *ctx,
2512                                     GHashTable *visited, Transaction *tran,
2513                                     Error **errp)
2514 {
2515     BlockBackend *blk = child->opaque;
2516     BdrvStateBlkRootContext *s;
2517
2518     if (!blk->allow_aio_context_change) {
2519         /*
2520          * Manually created BlockBackends (those with a name) that are not
2521          * attached to anything can change their AioContext without updating
2522          * their user; return an error for others.
2523          */
2524         if (!blk->name || blk->dev) {
2525             /* TODO Add BB name/QOM path */
2526             error_setg(errp, "Cannot change iothread of active block backend");
2527             return false;
2528         }
2529     }
2530
2531     s = g_new(BdrvStateBlkRootContext, 1);
2532     *s = (BdrvStateBlkRootContext) {
2533         .new_ctx = ctx,
2534         .blk = blk,
2535     };
2536
2537     tran_add(tran, &set_blk_root_context, s);
2538     return true;
2539 }
2540
2541 void blk_add_aio_context_notifier(BlockBackend *blk,
2542         void (*attached_aio_context)(AioContext *new_context, void *opaque),
2543         void (*detach_aio_context)(void *opaque), void *opaque)
2544 {
2545     BlockBackendAioNotifier *notifier;
2546     BlockDriverState *bs = blk_bs(blk);
2547     GLOBAL_STATE_CODE();
2548
2549     notifier = g_new(BlockBackendAioNotifier, 1);
2550     notifier->attached_aio_context = attached_aio_context;
2551     notifier->detach_aio_context = detach_aio_context;
2552     notifier->opaque = opaque;
2553     QLIST_INSERT_HEAD(&blk->aio_notifiers, notifier, list);
2554
2555     if (bs) {
2556         bdrv_add_aio_context_notifier(bs, attached_aio_context,
2557                                       detach_aio_context, opaque);
2558     }
2559 }
2560
2561 void blk_remove_aio_context_notifier(BlockBackend *blk,
2562                                      void (*attached_aio_context)(AioContext *,
2563                                                                   void *),
2564                                      void (*detach_aio_context)(void *),
2565                                      void *opaque)
2566 {
2567     BlockBackendAioNotifier *notifier;
2568     BlockDriverState *bs = blk_bs(blk);
2569
2570     GLOBAL_STATE_CODE();
2571
2572     if (bs) {
2573         bdrv_remove_aio_context_notifier(bs, attached_aio_context,
2574                                          detach_aio_context, opaque);
2575     }
2576
2577     QLIST_FOREACH(notifier, &blk->aio_notifiers, list) {
2578         if (notifier->attached_aio_context == attached_aio_context &&
2579             notifier->detach_aio_context == detach_aio_context &&
2580             notifier->opaque == opaque) {
2581             QLIST_REMOVE(notifier, list);
2582             g_free(notifier);
2583             return;
2584         }
2585     }
2586
2587     abort();
2588 }
2589
2590 void blk_add_remove_bs_notifier(BlockBackend *blk, Notifier *notify)
2591 {
2592     GLOBAL_STATE_CODE();
2593     notifier_list_add(&blk->remove_bs_notifiers, notify);
2594 }
2595
2596 void blk_add_insert_bs_notifier(BlockBackend *blk, Notifier *notify)
2597 {
2598     GLOBAL_STATE_CODE();
2599     notifier_list_add(&blk->insert_bs_notifiers, notify);
2600 }
2601
2602 BlockAcctStats *blk_get_stats(BlockBackend *blk)
2603 {
2604     IO_CODE();
2605     return &blk->stats;
2606 }
2607
2608 void *blk_aio_get(const AIOCBInfo *aiocb_info, BlockBackend *blk,
2609                   BlockCompletionFunc *cb, void *opaque)
2610 {
2611     IO_CODE();
2612     return qemu_aio_get(aiocb_info, blk_bs(blk), cb, opaque);
2613 }
2614
2615 int coroutine_fn blk_co_pwrite_zeroes(BlockBackend *blk, int64_t offset,
2616                                       int64_t bytes, BdrvRequestFlags flags)
2617 {
2618     IO_OR_GS_CODE();
2619     return blk_co_pwritev(blk, offset, bytes, NULL,
2620                           flags | BDRV_REQ_ZERO_WRITE);
2621 }
2622
2623 int coroutine_fn blk_co_pwrite_compressed(BlockBackend *blk, int64_t offset,
2624                                           int64_t bytes, const void *buf)
2625 {
2626     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes);
2627     IO_OR_GS_CODE();
2628     return blk_co_pwritev_part(blk, offset, bytes, &qiov, 0,
2629                                BDRV_REQ_WRITE_COMPRESSED);
2630 }
2631
2632 int coroutine_fn blk_co_truncate(BlockBackend *blk, int64_t offset, bool exact,
2633                                  PreallocMode prealloc, BdrvRequestFlags flags,
2634                                  Error **errp)
2635 {
2636     IO_OR_GS_CODE();
2637     GRAPH_RDLOCK_GUARD();
2638     if (!blk_co_is_available(blk)) {
2639         error_setg(errp, "No medium inserted");
2640         return -ENOMEDIUM;
2641     }
2642
2643     return bdrv_co_truncate(blk->root, offset, exact, prealloc, flags, errp);
2644 }
2645
2646 int blk_save_vmstate(BlockBackend *blk, const uint8_t *buf,
2647                      int64_t pos, int size)
2648 {
2649     int ret;
2650     GLOBAL_STATE_CODE();
2651
2652     if (!blk_is_available(blk)) {
2653         return -ENOMEDIUM;
2654     }
2655
2656     ret = bdrv_save_vmstate(blk_bs(blk), buf, pos, size);
2657     if (ret < 0) {
2658         return ret;
2659     }
2660
2661     if (ret == size && !blk->enable_write_cache) {
2662         ret = bdrv_flush(blk_bs(blk));
2663     }
2664
2665     return ret < 0 ? ret : size;
2666 }
2667
2668 int blk_load_vmstate(BlockBackend *blk, uint8_t *buf, int64_t pos, int size)
2669 {
2670     GLOBAL_STATE_CODE();
2671     if (!blk_is_available(blk)) {
2672         return -ENOMEDIUM;
2673     }
2674
2675     return bdrv_load_vmstate(blk_bs(blk), buf, pos, size);
2676 }
2677
2678 int blk_probe_blocksizes(BlockBackend *blk, BlockSizes *bsz)
2679 {
2680     GLOBAL_STATE_CODE();
2681     if (!blk_is_available(blk)) {
2682         return -ENOMEDIUM;
2683     }
2684
2685     return bdrv_probe_blocksizes(blk_bs(blk), bsz);
2686 }
2687
2688 int blk_probe_geometry(BlockBackend *blk, HDGeometry *geo)
2689 {
2690     GLOBAL_STATE_CODE();
2691     if (!blk_is_available(blk)) {
2692         return -ENOMEDIUM;
2693     }
2694
2695     return bdrv_probe_geometry(blk_bs(blk), geo);
2696 }
2697
2698 /*
2699  * Updates the BlockBackendRootState object with data from the currently
2700  * attached BlockDriverState.
2701  */
2702 void blk_update_root_state(BlockBackend *blk)
2703 {
2704     GLOBAL_STATE_CODE();
2705     assert(blk->root);
2706
2707     blk->root_state.open_flags    = blk->root->bs->open_flags;
2708     blk->root_state.detect_zeroes = blk->root->bs->detect_zeroes;
2709 }
2710
2711 /*
2712  * Returns the detect-zeroes setting to be used for bdrv_open() of a
2713  * BlockDriverState which is supposed to inherit the root state.
2714  */
2715 bool blk_get_detect_zeroes_from_root_state(BlockBackend *blk)
2716 {
2717     GLOBAL_STATE_CODE();
2718     return blk->root_state.detect_zeroes;
2719 }
2720
2721 /*
2722  * Returns the flags to be used for bdrv_open() of a BlockDriverState which is
2723  * supposed to inherit the root state.
2724  */
2725 int blk_get_open_flags_from_root_state(BlockBackend *blk)
2726 {
2727     GLOBAL_STATE_CODE();
2728     return blk->root_state.open_flags;
2729 }
2730
2731 BlockBackendRootState *blk_get_root_state(BlockBackend *blk)
2732 {
2733     GLOBAL_STATE_CODE();
2734     return &blk->root_state;
2735 }
2736
2737 int blk_commit_all(void)
2738 {
2739     BlockBackend *blk = NULL;
2740     GLOBAL_STATE_CODE();
2741
2742     while ((blk = blk_all_next(blk)) != NULL) {
2743         AioContext *aio_context = blk_get_aio_context(blk);
2744         BlockDriverState *unfiltered_bs = bdrv_skip_filters(blk_bs(blk));
2745
2746         aio_context_acquire(aio_context);
2747         if (blk_is_inserted(blk) && bdrv_cow_child(unfiltered_bs)) {
2748             int ret;
2749
2750             ret = bdrv_commit(unfiltered_bs);
2751             if (ret < 0) {
2752                 aio_context_release(aio_context);
2753                 return ret;
2754             }
2755         }
2756         aio_context_release(aio_context);
2757     }
2758     return 0;
2759 }
2760
2761
2762 /* throttling disk I/O limits */
2763 void blk_set_io_limits(BlockBackend *blk, ThrottleConfig *cfg)
2764 {
2765     GLOBAL_STATE_CODE();
2766     throttle_group_config(&blk->public.throttle_group_member, cfg);
2767 }
2768
2769 void blk_io_limits_disable(BlockBackend *blk)
2770 {
2771     BlockDriverState *bs = blk_bs(blk);
2772     ThrottleGroupMember *tgm = &blk->public.throttle_group_member;
2773     assert(tgm->throttle_state);
2774     GLOBAL_STATE_CODE();
2775     if (bs) {
2776         bdrv_ref(bs);
2777         bdrv_drained_begin(bs);
2778     }
2779     throttle_group_unregister_tgm(tgm);
2780     if (bs) {
2781         bdrv_drained_end(bs);
2782         bdrv_unref(bs);
2783     }
2784 }
2785
2786 /* should be called before blk_set_io_limits if a limit is set */
2787 void blk_io_limits_enable(BlockBackend *blk, const char *group)
2788 {
2789     assert(!blk->public.throttle_group_member.throttle_state);
2790     GLOBAL_STATE_CODE();
2791     throttle_group_register_tgm(&blk->public.throttle_group_member,
2792                                 group, blk_get_aio_context(blk));
2793 }
2794
2795 void blk_io_limits_update_group(BlockBackend *blk, const char *group)
2796 {
2797     GLOBAL_STATE_CODE();
2798     /* this BB is not part of any group */
2799     if (!blk->public.throttle_group_member.throttle_state) {
2800         return;
2801     }
2802
2803     /* this BB is a part of the same group than the one we want */
2804     if (!g_strcmp0(throttle_group_get_name(&blk->public.throttle_group_member),
2805                 group)) {
2806         return;
2807     }
2808
2809     /* need to change the group this bs belong to */
2810     blk_io_limits_disable(blk);
2811     blk_io_limits_enable(blk, group);
2812 }
2813
2814 static void blk_root_drained_begin(BdrvChild *child)
2815 {
2816     BlockBackend *blk = child->opaque;
2817     ThrottleGroupMember *tgm = &blk->public.throttle_group_member;
2818
2819     if (qatomic_fetch_inc(&blk->quiesce_counter) == 0) {
2820         if (blk->dev_ops && blk->dev_ops->drained_begin) {
2821             blk->dev_ops->drained_begin(blk->dev_opaque);
2822         }
2823     }
2824
2825     /* Note that blk->root may not be accessible here yet if we are just
2826      * attaching to a BlockDriverState that is drained. Use child instead. */
2827
2828     if (qatomic_fetch_inc(&tgm->io_limits_disabled) == 0) {
2829         throttle_group_restart_tgm(tgm);
2830     }
2831 }
2832
2833 static bool blk_root_drained_poll(BdrvChild *child)
2834 {
2835     BlockBackend *blk = child->opaque;
2836     bool busy = false;
2837     assert(qatomic_read(&blk->quiesce_counter));
2838
2839     if (blk->dev_ops && blk->dev_ops->drained_poll) {
2840         busy = blk->dev_ops->drained_poll(blk->dev_opaque);
2841     }
2842     return busy || !!blk->in_flight;
2843 }
2844
2845 static void blk_root_drained_end(BdrvChild *child)
2846 {
2847     BlockBackend *blk = child->opaque;
2848     assert(qatomic_read(&blk->quiesce_counter));
2849
2850     assert(blk->public.throttle_group_member.io_limits_disabled);
2851     qatomic_dec(&blk->public.throttle_group_member.io_limits_disabled);
2852
2853     if (qatomic_fetch_dec(&blk->quiesce_counter) == 1) {
2854         if (blk->dev_ops && blk->dev_ops->drained_end) {
2855             blk->dev_ops->drained_end(blk->dev_opaque);
2856         }
2857         qemu_mutex_lock(&blk->queued_requests_lock);
2858         while (qemu_co_enter_next(&blk->queued_requests,
2859                                   &blk->queued_requests_lock)) {
2860             /* Resume all queued requests */
2861         }
2862         qemu_mutex_unlock(&blk->queued_requests_lock);
2863     }
2864 }
2865
2866 bool blk_register_buf(BlockBackend *blk, void *host, size_t size, Error **errp)
2867 {
2868     BlockDriverState *bs = blk_bs(blk);
2869
2870     GLOBAL_STATE_CODE();
2871
2872     if (bs) {
2873         return bdrv_register_buf(bs, host, size, errp);
2874     }
2875     return true;
2876 }
2877
2878 void blk_unregister_buf(BlockBackend *blk, void *host, size_t size)
2879 {
2880     BlockDriverState *bs = blk_bs(blk);
2881
2882     GLOBAL_STATE_CODE();
2883
2884     if (bs) {
2885         bdrv_unregister_buf(bs, host, size);
2886     }
2887 }
2888
2889 int coroutine_fn blk_co_copy_range(BlockBackend *blk_in, int64_t off_in,
2890                                    BlockBackend *blk_out, int64_t off_out,
2891                                    int64_t bytes, BdrvRequestFlags read_flags,
2892                                    BdrvRequestFlags write_flags)
2893 {
2894     int r;
2895     IO_CODE();
2896     GRAPH_RDLOCK_GUARD();
2897
2898     r = blk_check_byte_request(blk_in, off_in, bytes);
2899     if (r) {
2900         return r;
2901     }
2902     r = blk_check_byte_request(blk_out, off_out, bytes);
2903     if (r) {
2904         return r;
2905     }
2906
2907     return bdrv_co_copy_range(blk_in->root, off_in,
2908                               blk_out->root, off_out,
2909                               bytes, read_flags, write_flags);
2910 }
2911
2912 const BdrvChild *blk_root(BlockBackend *blk)
2913 {
2914     GLOBAL_STATE_CODE();
2915     return blk->root;
2916 }
2917
2918 int blk_make_empty(BlockBackend *blk, Error **errp)
2919 {
2920     GLOBAL_STATE_CODE();
2921     if (!blk_is_available(blk)) {
2922         error_setg(errp, "No medium inserted");
2923         return -ENOMEDIUM;
2924     }
2925
2926     return bdrv_make_empty(blk->root, errp);
2927 }