block/block-backend.c

   1 /*
   2  * QEMU Block backends
   3  *
   4  * Copyright (C) 2014-2016 Red Hat, Inc.
   5  *
   6  * Authors:
   7  *  Markus Armbruster <armbru@redhat.com>,
   8  *
   9  * This work is licensed under the terms of the GNU LGPL, version 2.1
  10  * or later.  See the COPYING.LIB file in the top-level directory.
  11  */
  12
  13 #include "qemu/osdep.h"
  14 #include "sysemu/block-backend.h"
  15 #include "block/block_int.h"
  16 #include "block/blockjob.h"
  17 #include "block/coroutines.h"
  18 #include "block/throttle-groups.h"
  19 #include "hw/qdev-core.h"
  20 #include "sysemu/blockdev.h"
  21 #include "sysemu/runstate.h"
  22 #include "sysemu/replay.h"
  23 #include "qapi/error.h"
  24 #include "qapi/qapi-events-block.h"
  25 #include "qemu/id.h"
  26 #include "qemu/main-loop.h"
  27 #include "qemu/option.h"
  28 #include "trace.h"
  29 #include "migration/misc.h"
  30
  31 /* Number of coroutines to reserve per attached device model */
  32 #define COROUTINE_POOL_RESERVATION 64
  33
  34 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
  35
  36 typedef struct BlockBackendAioNotifier {
  37     void (*attached_aio_context)(AioContext *new_context, void *opaque);
  38     void (*detach_aio_context)(void *opaque);
  39     void *opaque;
  40     QLIST_ENTRY(BlockBackendAioNotifier) list;
  41 } BlockBackendAioNotifier;
  42
  43 struct BlockBackend {
  44     char *name;
  45     int refcnt;
  46     BdrvChild *root;
  47     AioContext *ctx;
  48     DriveInfo *legacy_dinfo;    /* null unless created by drive_new() */
  49     QTAILQ_ENTRY(BlockBackend) link;         /* for block_backends */
  50     QTAILQ_ENTRY(BlockBackend) monitor_link; /* for monitor_block_backends */
  51     BlockBackendPublic public;
  52
  53     DeviceState *dev;           /* attached device model, if any */
  54     const BlockDevOps *dev_ops;
  55     void *dev_opaque;
  56
  57     /* If the BDS tree is removed, some of its options are stored here (which
  58      * can be used to restore those options in the new BDS on insert) */
  59     BlockBackendRootState root_state;
  60
  61     bool enable_write_cache;
  62
  63     /* I/O stats (display with "info blockstats"). */
  64     BlockAcctStats stats;
  65
  66     BlockdevOnError on_read_error, on_write_error;
  67     bool iostatus_enabled;
  68     BlockDeviceIoStatus iostatus;
  69
  70     uint64_t perm;
  71     uint64_t shared_perm;
  72     bool disable_perm;
  73
  74     bool allow_aio_context_change;
  75     bool allow_write_beyond_eof;
  76
  77     /* Protected by BQL */
  78     NotifierList remove_bs_notifiers, insert_bs_notifiers;
  79     QLIST_HEAD(, BlockBackendAioNotifier) aio_notifiers;
  80
  81     int quiesce_counter; /* atomic: written under BQL, read by other threads */
  82     QemuMutex queued_requests_lock; /* protects queued_requests */
  83     CoQueue queued_requests;
  84     bool disable_request_queuing; /* atomic */
  85
  86     VMChangeStateEntry *vmsh;
  87     bool force_allow_inactivate;
  88
  89     /* Number of in-flight aio requests.  BlockDriverState also counts
  90      * in-flight requests but aio requests can exist even when blk->root is
  91      * NULL, so we cannot rely on its counter for that case.
  92      * Accessed with atomic ops.
  93      */
  94     unsigned int in_flight;
  95 };
  96
  97 typedef struct BlockBackendAIOCB {
  98     BlockAIOCB common;
  99     BlockBackend *blk;
 100     int ret;
 101 } BlockBackendAIOCB;
 102
 103 static const AIOCBInfo block_backend_aiocb_info = {
 104     .aiocb_size = sizeof(BlockBackendAIOCB),
 105 };
 106
 107 static void drive_info_del(DriveInfo *dinfo);
 108 static BlockBackend *bdrv_first_blk(BlockDriverState *bs);
 109
 110 /* All BlockBackends. Protected by BQL. */
 111 static QTAILQ_HEAD(, BlockBackend) block_backends =
 112     QTAILQ_HEAD_INITIALIZER(block_backends);
 113
 114 /*
 115  * All BlockBackends referenced by the monitor and which are iterated through by
 116  * blk_next(). Protected by BQL.
 117  */
 118 static QTAILQ_HEAD(, BlockBackend) monitor_block_backends =
 119     QTAILQ_HEAD_INITIALIZER(monitor_block_backends);
 120
 121 static int coroutine_mixed_fn GRAPH_RDLOCK
 122 blk_set_perm_locked(BlockBackend *blk, uint64_t perm, uint64_t shared_perm,
 123                     Error **errp);
 124
 125 static void blk_root_inherit_options(BdrvChildRole role, bool parent_is_format,
 126                                      int *child_flags, QDict *child_options,
 127                                      int parent_flags, QDict *parent_options)
 128 {
 129     /* We're not supposed to call this function for root nodes */
 130     abort();
 131 }
 132 static void blk_root_drained_begin(BdrvChild *child);
 133 static bool blk_root_drained_poll(BdrvChild *child);
 134 static void blk_root_drained_end(BdrvChild *child);
 135
 136 static void blk_root_change_media(BdrvChild *child, bool load);
 137 static void blk_root_resize(BdrvChild *child);
 138
 139 static bool blk_root_change_aio_ctx(BdrvChild *child, AioContext *ctx,
 140                                     GHashTable *visited, Transaction *tran,
 141                                     Error **errp);
 142
 143 static char *blk_root_get_parent_desc(BdrvChild *child)
 144 {
 145     BlockBackend *blk = child->opaque;
 146     g_autofree char *dev_id = NULL;
 147
 148     if (blk->name) {
 149         return g_strdup_printf("block device '%s'", blk->name);
 150     }
 151
 152     dev_id = blk_get_attached_dev_id(blk);
 153     if (*dev_id) {
 154         return g_strdup_printf("block device '%s'", dev_id);
 155     } else {
 156         /* TODO Callback into the BB owner for something more detailed */
 157         return g_strdup("an unnamed block device");
 158     }
 159 }
 160
 161 static const char *blk_root_get_name(BdrvChild *child)
 162 {
 163     return blk_name(child->opaque);
 164 }
 165
 166 static void blk_vm_state_changed(void *opaque, bool running, RunState state)
 167 {
 168     Error *local_err = NULL;
 169     BlockBackend *blk = opaque;
 170
 171     if (state == RUN_STATE_INMIGRATE) {
 172         return;
 173     }
 174
 175     qemu_del_vm_change_state_handler(blk->vmsh);
 176     blk->vmsh = NULL;
 177     blk_set_perm(blk, blk->perm, blk->shared_perm, &local_err);
 178     if (local_err) {
 179         error_report_err(local_err);
 180     }
 181 }
 182
 183 /*
 184  * Notifies the user of the BlockBackend that migration has completed. qdev
 185  * devices can tighten their permissions in response (specifically revoke
 186  * shared write permissions that we needed for storage migration).
 187  *
 188  * If an error is returned, the VM cannot be allowed to be resumed.
 189  */
 190 static void GRAPH_RDLOCK blk_root_activate(BdrvChild *child, Error **errp)
 191 {
 192     BlockBackend *blk = child->opaque;
 193     Error *local_err = NULL;
 194     uint64_t saved_shared_perm;
 195
 196     if (!blk->disable_perm) {
 197         return;
 198     }
 199
 200     blk->disable_perm = false;
 201
 202     /*
 203      * blk->shared_perm contains the permissions we want to share once
 204      * migration is really completely done.  For now, we need to share
 205      * all; but we also need to retain blk->shared_perm, which is
 206      * overwritten by a successful blk_set_perm() call.  Save it and
 207      * restore it below.
 208      */
 209     saved_shared_perm = blk->shared_perm;
 210
 211     blk_set_perm_locked(blk, blk->perm, BLK_PERM_ALL, &local_err);
 212     if (local_err) {
 213         error_propagate(errp, local_err);
 214         blk->disable_perm = true;
 215         return;
 216     }
 217     blk->shared_perm = saved_shared_perm;
 218
 219     if (runstate_check(RUN_STATE_INMIGRATE)) {
 220         /* Activation can happen when migration process is still active, for
 221          * example when nbd_server_add is called during non-shared storage
 222          * migration. Defer the shared_perm update to migration completion. */
 223         if (!blk->vmsh) {
 224             blk->vmsh = qemu_add_vm_change_state_handler(blk_vm_state_changed,
 225                                                          blk);
 226         }
 227         return;
 228     }
 229
 230     blk_set_perm_locked(blk, blk->perm, blk->shared_perm, &local_err);
 231     if (local_err) {
 232         error_propagate(errp, local_err);
 233         blk->disable_perm = true;
 234         return;
 235     }
 236 }
 237
 238 void blk_set_force_allow_inactivate(BlockBackend *blk)
 239 {
 240     GLOBAL_STATE_CODE();
 241     blk->force_allow_inactivate = true;
 242 }
 243
 244 static bool blk_can_inactivate(BlockBackend *blk)
 245 {
 246     /* If it is a guest device, inactivate is ok. */
 247     if (blk->dev || blk_name(blk)[0]) {
 248         return true;
 249     }
 250
 251     /* Inactivating means no more writes to the image can be done,
 252      * even if those writes would be changes invisible to the
 253      * guest.  For block job BBs that satisfy this, we can just allow
 254      * it.  This is the case for mirror job source, which is required
 255      * by libvirt non-shared block migration. */
 256     if (!(blk->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED))) {
 257         return true;
 258     }
 259
 260     return blk->force_allow_inactivate;
 261 }
 262
 263 static int GRAPH_RDLOCK blk_root_inactivate(BdrvChild *child)
 264 {
 265     BlockBackend *blk = child->opaque;
 266
 267     if (blk->disable_perm) {
 268         return 0;
 269     }
 270
 271     if (!blk_can_inactivate(blk)) {
 272         return -EPERM;
 273     }
 274
 275     blk->disable_perm = true;
 276     if (blk->root) {
 277         bdrv_child_try_set_perm(blk->root, 0, BLK_PERM_ALL, &error_abort);
 278     }
 279
 280     return 0;
 281 }
 282
 283 static void blk_root_attach(BdrvChild *child)
 284 {
 285     BlockBackend *blk = child->opaque;
 286     BlockBackendAioNotifier *notifier;
 287
 288     trace_blk_root_attach(child, blk, child->bs);
 289
 290     QLIST_FOREACH(notifier, &blk->aio_notifiers, list) {
 291         bdrv_add_aio_context_notifier(child->bs,
 292                 notifier->attached_aio_context,
 293                 notifier->detach_aio_context,
 294                 notifier->opaque);
 295     }
 296 }
 297
 298 static void blk_root_detach(BdrvChild *child)
 299 {
 300     BlockBackend *blk = child->opaque;
 301     BlockBackendAioNotifier *notifier;
 302
 303     trace_blk_root_detach(child, blk, child->bs);
 304
 305     QLIST_FOREACH(notifier, &blk->aio_notifiers, list) {
 306         bdrv_remove_aio_context_notifier(child->bs,
 307                 notifier->attached_aio_context,
 308                 notifier->detach_aio_context,
 309                 notifier->opaque);
 310     }
 311 }
 312
 313 static AioContext *blk_root_get_parent_aio_context(BdrvChild *c)
 314 {
 315     BlockBackend *blk = c->opaque;
 316     IO_CODE();
 317
 318     return blk_get_aio_context(blk);
 319 }
 320
 321 static const BdrvChildClass child_root = {
 322     .inherit_options    = blk_root_inherit_options,
 323
 324     .change_media       = blk_root_change_media,
 325     .resize             = blk_root_resize,
 326     .get_name           = blk_root_get_name,
 327     .get_parent_desc    = blk_root_get_parent_desc,
 328
 329     .drained_begin      = blk_root_drained_begin,
 330     .drained_poll       = blk_root_drained_poll,
 331     .drained_end        = blk_root_drained_end,
 332
 333     .activate           = blk_root_activate,
 334     .inactivate         = blk_root_inactivate,
 335
 336     .attach             = blk_root_attach,
 337     .detach             = blk_root_detach,
 338
 339     .change_aio_ctx     = blk_root_change_aio_ctx,
 340
 341     .get_parent_aio_context = blk_root_get_parent_aio_context,
 342 };
 343
 344 /*
 345  * Create a new BlockBackend with a reference count of one.
 346  *
 347  * @perm is a bitmasks of BLK_PERM_* constants which describes the permissions
 348  * to request for a block driver node that is attached to this BlockBackend.
 349  * @shared_perm is a bitmask which describes which permissions may be granted
 350  * to other users of the attached node.
 351  * Both sets of permissions can be changed later using blk_set_perm().
 352  *
 353  * Return the new BlockBackend on success, null on failure.
 354  */
 355 BlockBackend *blk_new(AioContext *ctx, uint64_t perm, uint64_t shared_perm)
 356 {
 357     BlockBackend *blk;
 358
 359     GLOBAL_STATE_CODE();
 360
 361     blk = g_new0(BlockBackend, 1);
 362     blk->refcnt = 1;
 363     blk->ctx = ctx;
 364     blk->perm = perm;
 365     blk->shared_perm = shared_perm;
 366     blk_set_enable_write_cache(blk, true);
 367
 368     blk->on_read_error = BLOCKDEV_ON_ERROR_REPORT;
 369     blk->on_write_error = BLOCKDEV_ON_ERROR_ENOSPC;
 370
 371     block_acct_init(&blk->stats);
 372
 373     qemu_mutex_init(&blk->queued_requests_lock);
 374     qemu_co_queue_init(&blk->queued_requests);
 375     notifier_list_init(&blk->remove_bs_notifiers);
 376     notifier_list_init(&blk->insert_bs_notifiers);
 377     QLIST_INIT(&blk->aio_notifiers);
 378
 379     QTAILQ_INSERT_TAIL(&block_backends, blk, link);
 380     return blk;
 381 }
 382
 383 /*
 384  * Create a new BlockBackend connected to an existing BlockDriverState.
 385  *
 386  * @perm is a bitmasks of BLK_PERM_* constants which describes the
 387  * permissions to request for @bs that is attached to this
 388  * BlockBackend.  @shared_perm is a bitmask which describes which
 389  * permissions may be granted to other users of the attached node.
 390  * Both sets of permissions can be changed later using blk_set_perm().
 391  *
 392  * Return the new BlockBackend on success, null on failure.
 393  *
 394  * Callers must hold the AioContext lock of @bs.
 395  */
 396 BlockBackend *blk_new_with_bs(BlockDriverState *bs, uint64_t perm,
 397                               uint64_t shared_perm, Error **errp)
 398 {
 399     BlockBackend *blk = blk_new(bdrv_get_aio_context(bs), perm, shared_perm);
 400
 401     GLOBAL_STATE_CODE();
 402
 403     if (blk_insert_bs(blk, bs, errp) < 0) {
 404         blk_unref(blk);
 405         return NULL;
 406     }
 407     return blk;
 408 }
 409
 410 /*
 411  * Creates a new BlockBackend, opens a new BlockDriverState, and connects both.
 412  * By default, the new BlockBackend is in the main AioContext, but if the
 413  * parameters connect it with any existing node in a different AioContext, it
 414  * may end up there instead.
 415  *
 416  * Just as with bdrv_open(), after having called this function the reference to
 417  * @options belongs to the block layer (even on failure).
 418  *
 419  * Called without holding an AioContext lock.
 420  *
 421  * TODO: Remove @filename and @flags; it should be possible to specify a whole
 422  * BDS tree just by specifying the @options QDict (or @reference,
 423  * alternatively). At the time of adding this function, this is not possible,
 424  * though, so callers of this function have to be able to specify @filename and
 425  * @flags.
 426  */
 427 BlockBackend *blk_new_open(const char *filename, const char *reference,
 428                            QDict *options, int flags, Error **errp)
 429 {
 430     BlockBackend *blk;
 431     BlockDriverState *bs;
 432     AioContext *ctx;
 433     uint64_t perm = 0;
 434     uint64_t shared = BLK_PERM_ALL;
 435
 436     GLOBAL_STATE_CODE();
 437
 438     /*
 439      * blk_new_open() is mainly used in .bdrv_create implementations and the
 440      * tools where sharing isn't a major concern because the BDS stays private
 441      * and the file is generally not supposed to be used by a second process,
 442      * so we just request permission according to the flags.
 443      *
 444      * The exceptions are xen_disk and blockdev_init(); in these cases, the
 445      * caller of blk_new_open() doesn't make use of the permissions, but they
 446      * shouldn't hurt either. We can still share everything here because the
 447      * guest devices will add their own blockers if they can't share.
 448      */
 449     if ((flags & BDRV_O_NO_IO) == 0) {
 450         perm |= BLK_PERM_CONSISTENT_READ;
 451         if (flags & BDRV_O_RDWR) {
 452             perm |= BLK_PERM_WRITE;
 453         }
 454     }
 455     if (flags & BDRV_O_RESIZE) {
 456         perm |= BLK_PERM_RESIZE;
 457     }
 458     if (flags & BDRV_O_NO_SHARE) {
 459         shared = BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED;
 460     }
 461
 462     aio_context_acquire(qemu_get_aio_context());
 463     bs = bdrv_open(filename, reference, options, flags, errp);
 464     aio_context_release(qemu_get_aio_context());
 465     if (!bs) {
 466         return NULL;
 467     }
 468
 469     /* bdrv_open() could have moved bs to a different AioContext */
 470     ctx = bdrv_get_aio_context(bs);
 471     blk = blk_new(bdrv_get_aio_context(bs), perm, shared);
 472     blk->perm = perm;
 473     blk->shared_perm = shared;
 474
 475     aio_context_acquire(ctx);
 476     blk_insert_bs(blk, bs, errp);
 477     bdrv_unref(bs);
 478     aio_context_release(ctx);
 479
 480     if (!blk->root) {
 481         blk_unref(blk);
 482         return NULL;
 483     }
 484
 485     return blk;
 486 }
 487
 488 static void blk_delete(BlockBackend *blk)
 489 {
 490     assert(!blk->refcnt);
 491     assert(!blk->name);
 492     assert(!blk->dev);
 493     if (blk->public.throttle_group_member.throttle_state) {
 494         blk_io_limits_disable(blk);
 495     }
 496     if (blk->root) {
 497         blk_remove_bs(blk);
 498     }
 499     if (blk->vmsh) {
 500         qemu_del_vm_change_state_handler(blk->vmsh);
 501         blk->vmsh = NULL;
 502     }
 503     assert(QLIST_EMPTY(&blk->remove_bs_notifiers.notifiers));
 504     assert(QLIST_EMPTY(&blk->insert_bs_notifiers.notifiers));
 505     assert(QLIST_EMPTY(&blk->aio_notifiers));
 506     assert(qemu_co_queue_empty(&blk->queued_requests));
 507     qemu_mutex_destroy(&blk->queued_requests_lock);
 508     QTAILQ_REMOVE(&block_backends, blk, link);
 509     drive_info_del(blk->legacy_dinfo);
 510     block_acct_cleanup(&blk->stats);
 511     g_free(blk);
 512 }
 513
 514 static void drive_info_del(DriveInfo *dinfo)
 515 {
 516     if (!dinfo) {
 517         return;
 518     }
 519     qemu_opts_del(dinfo->opts);
 520     g_free(dinfo);
 521 }
 522
 523 int blk_get_refcnt(BlockBackend *blk)
 524 {
 525     GLOBAL_STATE_CODE();
 526     return blk ? blk->refcnt : 0;
 527 }
 528
 529 /*
 530  * Increment @blk's reference count.
 531  * @blk must not be null.
 532  */
 533 void blk_ref(BlockBackend *blk)
 534 {
 535     assert(blk->refcnt > 0);
 536     GLOBAL_STATE_CODE();
 537     blk->refcnt++;
 538 }
 539
 540 /*
 541  * Decrement @blk's reference count.
 542  * If this drops it to zero, destroy @blk.
 543  * For convenience, do nothing if @blk is null.
 544  */
 545 void blk_unref(BlockBackend *blk)
 546 {
 547     GLOBAL_STATE_CODE();
 548     if (blk) {
 549         assert(blk->refcnt > 0);
 550         if (blk->refcnt > 1) {
 551             blk->refcnt--;
 552         } else {
 553             blk_drain(blk);
 554             /* blk_drain() cannot resurrect blk, nobody held a reference */
 555             assert(blk->refcnt == 1);
 556             blk->refcnt = 0;
 557             blk_delete(blk);
 558         }
 559     }
 560 }
 561
 562 /*
 563  * Behaves similarly to blk_next() but iterates over all BlockBackends, even the
 564  * ones which are hidden (i.e. are not referenced by the monitor).
 565  */
 566 BlockBackend *blk_all_next(BlockBackend *blk)
 567 {
 568     GLOBAL_STATE_CODE();
 569     return blk ? QTAILQ_NEXT(blk, link)
 570                : QTAILQ_FIRST(&block_backends);
 571 }
 572
 573 void blk_remove_all_bs(void)
 574 {
 575     BlockBackend *blk = NULL;
 576
 577     GLOBAL_STATE_CODE();
 578
 579     while ((blk = blk_all_next(blk)) != NULL) {
 580         AioContext *ctx = blk_get_aio_context(blk);
 581
 582         aio_context_acquire(ctx);
 583         if (blk->root) {
 584             blk_remove_bs(blk);
 585         }
 586         aio_context_release(ctx);
 587     }
 588 }
 589
 590 /*
 591  * Return the monitor-owned BlockBackend after @blk.
 592  * If @blk is null, return the first one.
 593  * Else, return @blk's next sibling, which may be null.
 594  *
 595  * To iterate over all BlockBackends, do
 596  * for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
 597  *     ...
 598  * }
 599  */
 600 BlockBackend *blk_next(BlockBackend *blk)
 601 {
 602     GLOBAL_STATE_CODE();
 603     return blk ? QTAILQ_NEXT(blk, monitor_link)
 604                : QTAILQ_FIRST(&monitor_block_backends);
 605 }
 606
 607 /* Iterates over all top-level BlockDriverStates, i.e. BDSs that are owned by
 608  * the monitor or attached to a BlockBackend */
 609 BlockDriverState *bdrv_next(BdrvNextIterator *it)
 610 {
 611     BlockDriverState *bs, *old_bs;
 612
 613     /* Must be called from the main loop */
 614     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
 615
 616     /* First, return all root nodes of BlockBackends. In order to avoid
 617      * returning a BDS twice when multiple BBs refer to it, we only return it
 618      * if the BB is the first one in the parent list of the BDS. */
 619     if (it->phase == BDRV_NEXT_BACKEND_ROOTS) {
 620         BlockBackend *old_blk = it->blk;
 621
 622         old_bs = old_blk ? blk_bs(old_blk) : NULL;
 623
 624         do {
 625             it->blk = blk_all_next(it->blk);
 626             bs = it->blk ? blk_bs(it->blk) : NULL;
 627         } while (it->blk && (bs == NULL || bdrv_first_blk(bs) != it->blk));
 628
 629         if (it->blk) {
 630             blk_ref(it->blk);
 631         }
 632         blk_unref(old_blk);
 633
 634         if (bs) {
 635             bdrv_ref(bs);
 636             bdrv_unref(old_bs);
 637             return bs;
 638         }
 639         it->phase = BDRV_NEXT_MONITOR_OWNED;
 640     } else {
 641         old_bs = it->bs;
 642     }
 643
 644     /* Then return the monitor-owned BDSes without a BB attached. Ignore all
 645      * BDSes that are attached to a BlockBackend here; they have been handled
 646      * by the above block already */
 647     do {
 648         it->bs = bdrv_next_monitor_owned(it->bs);
 649         bs = it->bs;
 650     } while (bs && bdrv_has_blk(bs));
 651
 652     if (bs) {
 653         bdrv_ref(bs);
 654     }
 655     bdrv_unref(old_bs);
 656
 657     return bs;
 658 }
 659
 660 static void bdrv_next_reset(BdrvNextIterator *it)
 661 {
 662     *it = (BdrvNextIterator) {
 663         .phase = BDRV_NEXT_BACKEND_ROOTS,
 664     };
 665 }
 666
 667 BlockDriverState *bdrv_first(BdrvNextIterator *it)
 668 {
 669     GLOBAL_STATE_CODE();
 670     bdrv_next_reset(it);
 671     return bdrv_next(it);
 672 }
 673
 674 /* Must be called when aborting a bdrv_next() iteration before
 675  * bdrv_next() returns NULL */
 676 void bdrv_next_cleanup(BdrvNextIterator *it)
 677 {
 678     /* Must be called from the main loop */
 679     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
 680
 681     if (it->phase == BDRV_NEXT_BACKEND_ROOTS) {
 682         if (it->blk) {
 683             bdrv_unref(blk_bs(it->blk));
 684             blk_unref(it->blk);
 685         }
 686     } else {
 687         bdrv_unref(it->bs);
 688     }
 689
 690     bdrv_next_reset(it);
 691 }
 692
 693 /*
 694  * Add a BlockBackend into the list of backends referenced by the monitor, with
 695  * the given @name acting as the handle for the monitor.
 696  * Strictly for use by blockdev.c.
 697  *
 698  * @name must not be null or empty.
 699  *
 700  * Returns true on success and false on failure. In the latter case, an Error
 701  * object is returned through @errp.
 702  */
 703 bool monitor_add_blk(BlockBackend *blk, const char *name, Error **errp)
 704 {
 705     assert(!blk->name);
 706     assert(name && name[0]);
 707     GLOBAL_STATE_CODE();
 708
 709     if (!id_wellformed(name)) {
 710         error_setg(errp, "Invalid device name");
 711         return false;
 712     }
 713     if (blk_by_name(name)) {
 714         error_setg(errp, "Device with id '%s' already exists", name);
 715         return false;
 716     }
 717     if (bdrv_find_node(name)) {
 718         error_setg(errp,
 719                    "Device name '%s' conflicts with an existing node name",
 720                    name);
 721         return false;
 722     }
 723
 724     blk->name = g_strdup(name);
 725     QTAILQ_INSERT_TAIL(&monitor_block_backends, blk, monitor_link);
 726     return true;
 727 }
 728
 729 /*
 730  * Remove a BlockBackend from the list of backends referenced by the monitor.
 731  * Strictly for use by blockdev.c.
 732  */
 733 void monitor_remove_blk(BlockBackend *blk)
 734 {
 735     GLOBAL_STATE_CODE();
 736
 737     if (!blk->name) {
 738         return;
 739     }
 740
 741     QTAILQ_REMOVE(&monitor_block_backends, blk, monitor_link);
 742     g_free(blk->name);
 743     blk->name = NULL;
 744 }
 745
 746 /*
 747  * Return @blk's name, a non-null string.
 748  * Returns an empty string iff @blk is not referenced by the monitor.
 749  */
 750 const char *blk_name(const BlockBackend *blk)
 751 {
 752     IO_CODE();
 753     return blk->name ?: "";
 754 }
 755
 756 /*
 757  * Return the BlockBackend with name @name if it exists, else null.
 758  * @name must not be null.
 759  */
 760 BlockBackend *blk_by_name(const char *name)
 761 {
 762     BlockBackend *blk = NULL;
 763
 764     GLOBAL_STATE_CODE();
 765     assert(name);
 766     while ((blk = blk_next(blk)) != NULL) {
 767         if (!strcmp(name, blk->name)) {
 768             return blk;
 769         }
 770     }
 771     return NULL;
 772 }
 773
 774 /*
 775  * Return the BlockDriverState attached to @blk if any, else null.
 776  */
 777 BlockDriverState *blk_bs(BlockBackend *blk)
 778 {
 779     IO_CODE();
 780     return blk->root ? blk->root->bs : NULL;
 781 }
 782
 783 static BlockBackend *bdrv_first_blk(BlockDriverState *bs)
 784 {
 785     BdrvChild *child;
 786
 787     GLOBAL_STATE_CODE();
 788
 789     QLIST_FOREACH(child, &bs->parents, next_parent) {
 790         if (child->klass == &child_root) {
 791             return child->opaque;
 792         }
 793     }
 794
 795     return NULL;
 796 }
 797
 798 /*
 799  * Returns true if @bs has an associated BlockBackend.
 800  */
 801 bool bdrv_has_blk(BlockDriverState *bs)
 802 {
 803     GLOBAL_STATE_CODE();
 804     return bdrv_first_blk(bs) != NULL;
 805 }
 806
 807 /*
 808  * Returns true if @bs has only BlockBackends as parents.
 809  */
 810 bool bdrv_is_root_node(BlockDriverState *bs)
 811 {
 812     BdrvChild *c;
 813
 814     GLOBAL_STATE_CODE();
 815     QLIST_FOREACH(c, &bs->parents, next_parent) {
 816         if (c->klass != &child_root) {
 817             return false;
 818         }
 819     }
 820
 821     return true;
 822 }
 823
 824 /*
 825  * Return @blk's DriveInfo if any, else null.
 826  */
 827 DriveInfo *blk_legacy_dinfo(BlockBackend *blk)
 828 {
 829     GLOBAL_STATE_CODE();
 830     return blk->legacy_dinfo;
 831 }
 832
 833 /*
 834  * Set @blk's DriveInfo to @dinfo, and return it.
 835  * @blk must not have a DriveInfo set already.
 836  * No other BlockBackend may have the same DriveInfo set.
 837  */
 838 DriveInfo *blk_set_legacy_dinfo(BlockBackend *blk, DriveInfo *dinfo)
 839 {
 840     assert(!blk->legacy_dinfo);
 841     GLOBAL_STATE_CODE();
 842     return blk->legacy_dinfo = dinfo;
 843 }
 844
 845 /*
 846  * Return the BlockBackend with DriveInfo @dinfo.
 847  * It must exist.
 848  */
 849 BlockBackend *blk_by_legacy_dinfo(DriveInfo *dinfo)
 850 {
 851     BlockBackend *blk = NULL;
 852     GLOBAL_STATE_CODE();
 853
 854     while ((blk = blk_next(blk)) != NULL) {
 855         if (blk->legacy_dinfo == dinfo) {
 856             return blk;
 857         }
 858     }
 859     abort();
 860 }
 861
 862 /*
 863  * Returns a pointer to the publicly accessible fields of @blk.
 864  */
 865 BlockBackendPublic *blk_get_public(BlockBackend *blk)
 866 {
 867     GLOBAL_STATE_CODE();
 868     return &blk->public;
 869 }
 870
 871 /*
 872  * Returns a BlockBackend given the associated @public fields.
 873  */
 874 BlockBackend *blk_by_public(BlockBackendPublic *public)
 875 {
 876     GLOBAL_STATE_CODE();
 877     return container_of(public, BlockBackend, public);
 878 }
 879
 880 /*
 881  * Disassociates the currently associated BlockDriverState from @blk.
 882  */
 883 void blk_remove_bs(BlockBackend *blk)
 884 {
 885     ThrottleGroupMember *tgm = &blk->public.throttle_group_member;
 886     BdrvChild *root;
 887
 888     GLOBAL_STATE_CODE();
 889
 890     notifier_list_notify(&blk->remove_bs_notifiers, blk);
 891     if (tgm->throttle_state) {
 892         BlockDriverState *bs = blk_bs(blk);
 893
 894         /*
 895          * Take a ref in case blk_bs() changes across bdrv_drained_begin(), for
 896          * example, if a temporary filter node is removed by a blockjob.
 897          */
 898         bdrv_ref(bs);
 899         bdrv_drained_begin(bs);
 900         throttle_group_detach_aio_context(tgm);
 901         throttle_group_attach_aio_context(tgm, qemu_get_aio_context());
 902         bdrv_drained_end(bs);
 903         bdrv_unref(bs);
 904     }
 905
 906     blk_update_root_state(blk);
 907
 908     /* bdrv_root_unref_child() will cause blk->root to become stale and may
 909      * switch to a completion coroutine later on. Let's drain all I/O here
 910      * to avoid that and a potential QEMU crash.
 911      */
 912     blk_drain(blk);
 913     root = blk->root;
 914     blk->root = NULL;
 915
 916     bdrv_graph_wrlock(NULL);
 917     bdrv_root_unref_child(root);
 918     bdrv_graph_wrunlock();
 919 }
 920
 921 /*
 922  * Associates a new BlockDriverState with @blk.
 923  *
 924  * Callers must hold the AioContext lock of @bs.
 925  */
 926 int blk_insert_bs(BlockBackend *blk, BlockDriverState *bs, Error **errp)
 927 {
 928     ThrottleGroupMember *tgm = &blk->public.throttle_group_member;
 929     GLOBAL_STATE_CODE();
 930     bdrv_ref(bs);
 931     blk->root = bdrv_root_attach_child(bs, "root", &child_root,
 932                                        BDRV_CHILD_FILTERED | BDRV_CHILD_PRIMARY,
 933                                        blk->perm, blk->shared_perm,
 934                                        blk, errp);
 935     if (blk->root == NULL) {
 936         return -EPERM;
 937     }
 938
 939     notifier_list_notify(&blk->insert_bs_notifiers, blk);
 940     if (tgm->throttle_state) {
 941         throttle_group_detach_aio_context(tgm);
 942         throttle_group_attach_aio_context(tgm, bdrv_get_aio_context(bs));
 943     }
 944
 945     return 0;
 946 }
 947
 948 /*
 949  * Change BlockDriverState associated with @blk.
 950  */
 951 int blk_replace_bs(BlockBackend *blk, BlockDriverState *new_bs, Error **errp)
 952 {
 953     GLOBAL_STATE_CODE();
 954     return bdrv_replace_child_bs(blk->root, new_bs, errp);
 955 }
 956
 957 /*
 958  * Sets the permission bitmasks that the user of the BlockBackend needs.
 959  */
 960 static int coroutine_mixed_fn GRAPH_RDLOCK
 961 blk_set_perm_locked(BlockBackend *blk, uint64_t perm, uint64_t shared_perm,
 962                     Error **errp)
 963 {
 964     int ret;
 965     GLOBAL_STATE_CODE();
 966
 967     if (blk->root && !blk->disable_perm) {
 968         ret = bdrv_child_try_set_perm(blk->root, perm, shared_perm, errp);
 969         if (ret < 0) {
 970             return ret;
 971         }
 972     }
 973
 974     blk->perm = perm;
 975     blk->shared_perm = shared_perm;
 976
 977     return 0;
 978 }
 979
 980 int blk_set_perm(BlockBackend *blk, uint64_t perm, uint64_t shared_perm,
 981                  Error **errp)
 982 {
 983     GLOBAL_STATE_CODE();
 984     GRAPH_RDLOCK_GUARD_MAINLOOP();
 985
 986     return blk_set_perm_locked(blk, perm, shared_perm, errp);
 987 }
 988
 989 void blk_get_perm(BlockBackend *blk, uint64_t *perm, uint64_t *shared_perm)
 990 {
 991     GLOBAL_STATE_CODE();
 992     *perm = blk->perm;
 993     *shared_perm = blk->shared_perm;
 994 }
 995
 996 /*
 997  * Attach device model @dev to @blk.
 998  * Return 0 on success, -EBUSY when a device model is attached already.
 999  */
1000 int blk_attach_dev(BlockBackend *blk, DeviceState *dev)
1001 {
1002     GLOBAL_STATE_CODE();
1003     if (blk->dev) {
1004         return -EBUSY;
1005     }
1006
1007     /* While migration is still incoming, we don't need to apply the
1008      * permissions of guest device BlockBackends. We might still have a block
1009      * job or NBD server writing to the image for storage migration. */
1010     if (runstate_check(RUN_STATE_INMIGRATE)) {
1011         blk->disable_perm = true;
1012     }
1013
1014     blk_ref(blk);
1015     blk->dev = dev;
1016     blk_iostatus_reset(blk);
1017
1018     return 0;
1019 }
1020
1021 /*
1022  * Detach device model @dev from @blk.
1023  * @dev must be currently attached to @blk.
1024  */
1025 void blk_detach_dev(BlockBackend *blk, DeviceState *dev)
1026 {
1027     assert(blk->dev == dev);
1028     GLOBAL_STATE_CODE();
1029     blk->dev = NULL;
1030     blk->dev_ops = NULL;
1031     blk->dev_opaque = NULL;
1032     blk_set_perm(blk, 0, BLK_PERM_ALL, &error_abort);
1033     blk_unref(blk);
1034 }
1035
1036 /*
1037  * Return the device model attached to @blk if any, else null.
1038  */
1039 DeviceState *blk_get_attached_dev(BlockBackend *blk)
1040 {
1041     GLOBAL_STATE_CODE();
1042     return blk->dev;
1043 }
1044
1045 /* Return the qdev ID, or if no ID is assigned the QOM path, of the block
1046  * device attached to the BlockBackend. */
1047 char *blk_get_attached_dev_id(BlockBackend *blk)
1048 {
1049     DeviceState *dev = blk->dev;
1050     IO_CODE();
1051
1052     if (!dev) {
1053         return g_strdup("");
1054     } else if (dev->id) {
1055         return g_strdup(dev->id);
1056     }
1057
1058     return object_get_canonical_path(OBJECT(dev)) ?: g_strdup("");
1059 }
1060
1061 /*
1062  * Return the BlockBackend which has the device model @dev attached if it
1063  * exists, else null.
1064  *
1065  * @dev must not be null.
1066  */
1067 BlockBackend *blk_by_dev(void *dev)
1068 {
1069     BlockBackend *blk = NULL;
1070
1071     GLOBAL_STATE_CODE();
1072
1073     assert(dev != NULL);
1074     while ((blk = blk_all_next(blk)) != NULL) {
1075         if (blk->dev == dev) {
1076             return blk;
1077         }
1078     }
1079     return NULL;
1080 }
1081
1082 /*
1083  * Set @blk's device model callbacks to @ops.
1084  * @opaque is the opaque argument to pass to the callbacks.
1085  * This is for use by device models.
1086  */
1087 void blk_set_dev_ops(BlockBackend *blk, const BlockDevOps *ops,
1088                      void *opaque)
1089 {
1090     GLOBAL_STATE_CODE();
1091     blk->dev_ops = ops;
1092     blk->dev_opaque = opaque;
1093
1094     /* Are we currently quiesced? Should we enforce this right now? */
1095     if (qatomic_read(&blk->quiesce_counter) && ops && ops->drained_begin) {
1096         ops->drained_begin(opaque);
1097     }
1098 }
1099
1100 /*
1101  * Notify @blk's attached device model of media change.
1102  *
1103  * If @load is true, notify of media load. This action can fail, meaning that
1104  * the medium cannot be loaded. @errp is set then.
1105  *
1106  * If @load is false, notify of media eject. This can never fail.
1107  *
1108  * Also send DEVICE_TRAY_MOVED events as appropriate.
1109  */
1110 void blk_dev_change_media_cb(BlockBackend *blk, bool load, Error **errp)
1111 {
1112     GLOBAL_STATE_CODE();
1113     if (blk->dev_ops && blk->dev_ops->change_media_cb) {
1114         bool tray_was_open, tray_is_open;
1115         Error *local_err = NULL;
1116
1117         tray_was_open = blk_dev_is_tray_open(blk);
1118         blk->dev_ops->change_media_cb(blk->dev_opaque, load, &local_err);
1119         if (local_err) {
1120             assert(load == true);
1121             error_propagate(errp, local_err);
1122             return;
1123         }
1124         tray_is_open = blk_dev_is_tray_open(blk);
1125
1126         if (tray_was_open != tray_is_open) {
1127             char *id = blk_get_attached_dev_id(blk);
1128             qapi_event_send_device_tray_moved(blk_name(blk), id, tray_is_open);
1129             g_free(id);
1130         }
1131     }
1132 }
1133
1134 static void blk_root_change_media(BdrvChild *child, bool load)
1135 {
1136     blk_dev_change_media_cb(child->opaque, load, NULL);
1137 }
1138
1139 /*
1140  * Does @blk's attached device model have removable media?
1141  * %true if no device model is attached.
1142  */
1143 bool blk_dev_has_removable_media(BlockBackend *blk)
1144 {
1145     GLOBAL_STATE_CODE();
1146     return !blk->dev || (blk->dev_ops && blk->dev_ops->change_media_cb);
1147 }
1148
1149 /*
1150  * Does @blk's attached device model have a tray?
1151  */
1152 bool blk_dev_has_tray(BlockBackend *blk)
1153 {
1154     IO_CODE();
1155     return blk->dev_ops && blk->dev_ops->is_tray_open;
1156 }
1157
1158 /*
1159  * Notify @blk's attached device model of a media eject request.
1160  * If @force is true, the medium is about to be yanked out forcefully.
1161  */
1162 void blk_dev_eject_request(BlockBackend *blk, bool force)
1163 {
1164     GLOBAL_STATE_CODE();
1165     if (blk->dev_ops && blk->dev_ops->eject_request_cb) {
1166         blk->dev_ops->eject_request_cb(blk->dev_opaque, force);
1167     }
1168 }
1169
1170 /*
1171  * Does @blk's attached device model have a tray, and is it open?
1172  */
1173 bool blk_dev_is_tray_open(BlockBackend *blk)
1174 {
1175     IO_CODE();
1176     if (blk_dev_has_tray(blk)) {
1177         return blk->dev_ops->is_tray_open(blk->dev_opaque);
1178     }
1179     return false;
1180 }
1181
1182 /*
1183  * Does @blk's attached device model have the medium locked?
1184  * %false if the device model has no such lock.
1185  */
1186 bool blk_dev_is_medium_locked(BlockBackend *blk)
1187 {
1188     GLOBAL_STATE_CODE();
1189     if (blk->dev_ops && blk->dev_ops->is_medium_locked) {
1190         return blk->dev_ops->is_medium_locked(blk->dev_opaque);
1191     }
1192     return false;
1193 }
1194
1195 /*
1196  * Notify @blk's attached device model of a backend size change.
1197  */
1198 static void blk_root_resize(BdrvChild *child)
1199 {
1200     BlockBackend *blk = child->opaque;
1201
1202     if (blk->dev_ops && blk->dev_ops->resize_cb) {
1203         blk->dev_ops->resize_cb(blk->dev_opaque);
1204     }
1205 }
1206
1207 void blk_iostatus_enable(BlockBackend *blk)
1208 {
1209     GLOBAL_STATE_CODE();
1210     blk->iostatus_enabled = true;
1211     blk->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
1212 }
1213
1214 /* The I/O status is only enabled if the drive explicitly
1215  * enables it _and_ the VM is configured to stop on errors */
1216 bool blk_iostatus_is_enabled(const BlockBackend *blk)
1217 {
1218     IO_CODE();
1219     return (blk->iostatus_enabled &&
1220            (blk->on_write_error == BLOCKDEV_ON_ERROR_ENOSPC ||
1221             blk->on_write_error == BLOCKDEV_ON_ERROR_STOP   ||
1222             blk->on_read_error == BLOCKDEV_ON_ERROR_STOP));
1223 }
1224
1225 BlockDeviceIoStatus blk_iostatus(const BlockBackend *blk)
1226 {
1227     GLOBAL_STATE_CODE();
1228     return blk->iostatus;
1229 }
1230
1231 void blk_iostatus_disable(BlockBackend *blk)
1232 {
1233     GLOBAL_STATE_CODE();
1234     blk->iostatus_enabled = false;
1235 }
1236
1237 void blk_iostatus_reset(BlockBackend *blk)
1238 {
1239     GLOBAL_STATE_CODE();
1240     if (blk_iostatus_is_enabled(blk)) {
1241         blk->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
1242     }
1243 }
1244
1245 void blk_iostatus_set_err(BlockBackend *blk, int error)
1246 {
1247     IO_CODE();
1248     assert(blk_iostatus_is_enabled(blk));
1249     if (blk->iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
1250         blk->iostatus = error == ENOSPC ? BLOCK_DEVICE_IO_STATUS_NOSPACE :
1251                                           BLOCK_DEVICE_IO_STATUS_FAILED;
1252     }
1253 }
1254
1255 void blk_set_allow_write_beyond_eof(BlockBackend *blk, bool allow)
1256 {
1257     IO_CODE();
1258     blk->allow_write_beyond_eof = allow;
1259 }
1260
1261 void blk_set_allow_aio_context_change(BlockBackend *blk, bool allow)
1262 {
1263     IO_CODE();
1264     blk->allow_aio_context_change = allow;
1265 }
1266
1267 void blk_set_disable_request_queuing(BlockBackend *blk, bool disable)
1268 {
1269     IO_CODE();
1270     qatomic_set(&blk->disable_request_queuing, disable);
1271 }
1272
1273 static int coroutine_fn GRAPH_RDLOCK
1274 blk_check_byte_request(BlockBackend *blk, int64_t offset, int64_t bytes)
1275 {
1276     int64_t len;
1277
1278     if (bytes < 0) {
1279         return -EIO;
1280     }
1281
1282     if (!blk_co_is_available(blk)) {
1283         return -ENOMEDIUM;
1284     }
1285
1286     if (offset < 0) {
1287         return -EIO;
1288     }
1289
1290     if (!blk->allow_write_beyond_eof) {
1291         len = bdrv_co_getlength(blk_bs(blk));
1292         if (len < 0) {
1293             return len;
1294         }
1295
1296         if (offset > len || len - offset < bytes) {
1297             return -EIO;
1298         }
1299     }
1300
1301     return 0;
1302 }
1303
1304 /* Are we currently in a drained section? */
1305 bool blk_in_drain(BlockBackend *blk)
1306 {
1307     GLOBAL_STATE_CODE(); /* change to IO_OR_GS_CODE(), if necessary */
1308     return qatomic_read(&blk->quiesce_counter);
1309 }
1310
1311 /* To be called between exactly one pair of blk_inc/dec_in_flight() */
1312 static void coroutine_fn blk_wait_while_drained(BlockBackend *blk)
1313 {
1314     assert(blk->in_flight > 0);
1315
1316     if (qatomic_read(&blk->quiesce_counter) &&
1317         !qatomic_read(&blk->disable_request_queuing)) {
1318         /*
1319          * Take lock before decrementing in flight counter so main loop thread
1320          * waits for us to enqueue ourselves before it can leave the drained
1321          * section.
1322          */
1323         qemu_mutex_lock(&blk->queued_requests_lock);
1324         blk_dec_in_flight(blk);
1325         qemu_co_queue_wait(&blk->queued_requests, &blk->queued_requests_lock);
1326         blk_inc_in_flight(blk);
1327         qemu_mutex_unlock(&blk->queued_requests_lock);
1328     }
1329 }
1330
1331 /* To be called between exactly one pair of blk_inc/dec_in_flight() */
1332 static int coroutine_fn
1333 blk_co_do_preadv_part(BlockBackend *blk, int64_t offset, int64_t bytes,
1334                       QEMUIOVector *qiov, size_t qiov_offset,
1335                       BdrvRequestFlags flags)
1336 {
1337     int ret;
1338     BlockDriverState *bs;
1339     IO_CODE();
1340
1341     blk_wait_while_drained(blk);
1342     GRAPH_RDLOCK_GUARD();
1343
1344     /* Call blk_bs() only after waiting, the graph may have changed */
1345     bs = blk_bs(blk);
1346     trace_blk_co_preadv(blk, bs, offset, bytes, flags);
1347
1348     ret = blk_check_byte_request(blk, offset, bytes);
1349     if (ret < 0) {
1350         return ret;
1351     }
1352
1353     bdrv_inc_in_flight(bs);
1354
1355     /* throttling disk I/O */
1356     if (blk->public.throttle_group_member.throttle_state) {
1357         throttle_group_co_io_limits_intercept(&blk->public.throttle_group_member,
1358                 bytes, THROTTLE_READ);
1359     }
1360
1361     ret = bdrv_co_preadv_part(blk->root, offset, bytes, qiov, qiov_offset,
1362                               flags);
1363     bdrv_dec_in_flight(bs);
1364     return ret;
1365 }
1366
1367 int coroutine_fn blk_co_pread(BlockBackend *blk, int64_t offset, int64_t bytes,
1368                               void *buf, BdrvRequestFlags flags)
1369 {
1370     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes);
1371     IO_OR_GS_CODE();
1372
1373     assert(bytes <= SIZE_MAX);
1374
1375     return blk_co_preadv(blk, offset, bytes, &qiov, flags);
1376 }
1377
1378 int coroutine_fn blk_co_preadv(BlockBackend *blk, int64_t offset,
1379                                int64_t bytes, QEMUIOVector *qiov,
1380                                BdrvRequestFlags flags)
1381 {
1382     int ret;
1383     IO_OR_GS_CODE();
1384
1385     blk_inc_in_flight(blk);
1386     ret = blk_co_do_preadv_part(blk, offset, bytes, qiov, 0, flags);
1387     blk_dec_in_flight(blk);
1388
1389     return ret;
1390 }
1391
1392 int coroutine_fn blk_co_preadv_part(BlockBackend *blk, int64_t offset,
1393                                     int64_t bytes, QEMUIOVector *qiov,
1394                                     size_t qiov_offset, BdrvRequestFlags flags)
1395 {
1396     int ret;
1397     IO_OR_GS_CODE();
1398
1399     blk_inc_in_flight(blk);
1400     ret = blk_co_do_preadv_part(blk, offset, bytes, qiov, qiov_offset, flags);
1401     blk_dec_in_flight(blk);
1402
1403     return ret;
1404 }
1405
1406 /* To be called between exactly one pair of blk_inc/dec_in_flight() */
1407 static int coroutine_fn
1408 blk_co_do_pwritev_part(BlockBackend *blk, int64_t offset, int64_t bytes,
1409                        QEMUIOVector *qiov, size_t qiov_offset,
1410                        BdrvRequestFlags flags)
1411 {
1412     int ret;
1413     BlockDriverState *bs;
1414     IO_CODE();
1415
1416     blk_wait_while_drained(blk);
1417     GRAPH_RDLOCK_GUARD();
1418
1419     /* Call blk_bs() only after waiting, the graph may have changed */
1420     bs = blk_bs(blk);
1421     trace_blk_co_pwritev(blk, bs, offset, bytes, flags);
1422
1423     ret = blk_check_byte_request(blk, offset, bytes);
1424     if (ret < 0) {
1425         return ret;
1426     }
1427
1428     bdrv_inc_in_flight(bs);
1429     /* throttling disk I/O */
1430     if (blk->public.throttle_group_member.throttle_state) {
1431         throttle_group_co_io_limits_intercept(&blk->public.throttle_group_member,
1432                 bytes, THROTTLE_WRITE);
1433     }
1434
1435     if (!blk->enable_write_cache) {
1436         flags |= BDRV_REQ_FUA;
1437     }
1438
1439     ret = bdrv_co_pwritev_part(blk->root, offset, bytes, qiov, qiov_offset,
1440                                flags);
1441     bdrv_dec_in_flight(bs);
1442     return ret;
1443 }
1444
1445 int coroutine_fn blk_co_pwritev_part(BlockBackend *blk, int64_t offset,
1446                                      int64_t bytes,
1447                                      QEMUIOVector *qiov, size_t qiov_offset,
1448                                      BdrvRequestFlags flags)
1449 {
1450     int ret;
1451     IO_OR_GS_CODE();
1452
1453     blk_inc_in_flight(blk);
1454     ret = blk_co_do_pwritev_part(blk, offset, bytes, qiov, qiov_offset, flags);
1455     blk_dec_in_flight(blk);
1456
1457     return ret;
1458 }
1459
1460 int coroutine_fn blk_co_pwrite(BlockBackend *blk, int64_t offset, int64_t bytes,
1461                                const void *buf, BdrvRequestFlags flags)
1462 {
1463     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes);
1464     IO_OR_GS_CODE();
1465
1466     assert(bytes <= SIZE_MAX);
1467
1468     return blk_co_pwritev(blk, offset, bytes, &qiov, flags);
1469 }
1470
1471 int coroutine_fn blk_co_pwritev(BlockBackend *blk, int64_t offset,
1472                                 int64_t bytes, QEMUIOVector *qiov,
1473                                 BdrvRequestFlags flags)
1474 {
1475     IO_OR_GS_CODE();
1476     return blk_co_pwritev_part(blk, offset, bytes, qiov, 0, flags);
1477 }
1478
1479 int coroutine_fn blk_co_block_status_above(BlockBackend *blk,
1480                                            BlockDriverState *base,
1481                                            int64_t offset, int64_t bytes,
1482                                            int64_t *pnum, int64_t *map,
1483                                            BlockDriverState **file)
1484 {
1485     IO_CODE();
1486     GRAPH_RDLOCK_GUARD();
1487     return bdrv_co_block_status_above(blk_bs(blk), base, offset, bytes, pnum,
1488                                       map, file);
1489 }
1490
1491 int coroutine_fn blk_co_is_allocated_above(BlockBackend *blk,
1492                                            BlockDriverState *base,
1493                                            bool include_base, int64_t offset,
1494                                            int64_t bytes, int64_t *pnum)
1495 {
1496     IO_CODE();
1497     GRAPH_RDLOCK_GUARD();
1498     return bdrv_co_is_allocated_above(blk_bs(blk), base, include_base, offset,
1499                                       bytes, pnum);
1500 }
1501
1502 typedef struct BlkRwCo {
1503     BlockBackend *blk;
1504     int64_t offset;
1505     void *iobuf;
1506     int ret;
1507     BdrvRequestFlags flags;
1508 } BlkRwCo;
1509
1510 int blk_make_zero(BlockBackend *blk, BdrvRequestFlags flags)
1511 {
1512     GLOBAL_STATE_CODE();
1513     return bdrv_make_zero(blk->root, flags);
1514 }
1515
1516 void blk_inc_in_flight(BlockBackend *blk)
1517 {
1518     IO_CODE();
1519     qatomic_inc(&blk->in_flight);
1520 }
1521
1522 void blk_dec_in_flight(BlockBackend *blk)
1523 {
1524     IO_CODE();
1525     qatomic_dec(&blk->in_flight);
1526     aio_wait_kick();
1527 }
1528
1529 static void error_callback_bh(void *opaque)
1530 {
1531     struct BlockBackendAIOCB *acb = opaque;
1532
1533     blk_dec_in_flight(acb->blk);
1534     acb->common.cb(acb->common.opaque, acb->ret);
1535     qemu_aio_unref(acb);
1536 }
1537
1538 BlockAIOCB *blk_abort_aio_request(BlockBackend *blk,
1539                                   BlockCompletionFunc *cb,
1540                                   void *opaque, int ret)
1541 {
1542     struct BlockBackendAIOCB *acb;
1543     IO_CODE();
1544
1545     blk_inc_in_flight(blk);
1546     acb = blk_aio_get(&block_backend_aiocb_info, blk, cb, opaque);
1547     acb->blk = blk;
1548     acb->ret = ret;
1549
1550     replay_bh_schedule_oneshot_event(qemu_get_current_aio_context(),
1551                                      error_callback_bh, acb);
1552     return &acb->common;
1553 }
1554
1555 typedef struct BlkAioEmAIOCB {
1556     BlockAIOCB common;
1557     BlkRwCo rwco;
1558     int64_t bytes;
1559     bool has_returned;
1560 } BlkAioEmAIOCB;
1561
1562 static const AIOCBInfo blk_aio_em_aiocb_info = {
1563     .aiocb_size         = sizeof(BlkAioEmAIOCB),
1564 };
1565
1566 static void blk_aio_complete(BlkAioEmAIOCB *acb)
1567 {
1568     if (acb->has_returned) {
1569         acb->common.cb(acb->common.opaque, acb->rwco.ret);
1570         blk_dec_in_flight(acb->rwco.blk);
1571         qemu_aio_unref(acb);
1572     }
1573 }
1574
1575 static void blk_aio_complete_bh(void *opaque)
1576 {
1577     BlkAioEmAIOCB *acb = opaque;
1578     assert(acb->has_returned);
1579     blk_aio_complete(acb);
1580 }
1581
1582 static BlockAIOCB *blk_aio_prwv(BlockBackend *blk, int64_t offset,
1583                                 int64_t bytes,
1584                                 void *iobuf, CoroutineEntry co_entry,
1585                                 BdrvRequestFlags flags,
1586                                 BlockCompletionFunc *cb, void *opaque)
1587 {
1588     BlkAioEmAIOCB *acb;
1589     Coroutine *co;
1590
1591     blk_inc_in_flight(blk);
1592     acb = blk_aio_get(&blk_aio_em_aiocb_info, blk, cb, opaque);
1593     acb->rwco = (BlkRwCo) {
1594         .blk    = blk,
1595         .offset = offset,
1596         .iobuf  = iobuf,
1597         .flags  = flags,
1598         .ret    = NOT_DONE,
1599     };
1600     acb->bytes = bytes;
1601     acb->has_returned = false;
1602
1603     co = qemu_coroutine_create(co_entry, acb);
1604     aio_co_enter(qemu_get_current_aio_context(), co);
1605
1606     acb->has_returned = true;
1607     if (acb->rwco.ret != NOT_DONE) {
1608         replay_bh_schedule_oneshot_event(qemu_get_current_aio_context(),
1609                                          blk_aio_complete_bh, acb);
1610     }
1611
1612     return &acb->common;
1613 }
1614
1615 static void coroutine_fn blk_aio_read_entry(void *opaque)
1616 {
1617     BlkAioEmAIOCB *acb = opaque;
1618     BlkRwCo *rwco = &acb->rwco;
1619     QEMUIOVector *qiov = rwco->iobuf;
1620
1621     assert(qiov->size == acb->bytes);
1622     rwco->ret = blk_co_do_preadv_part(rwco->blk, rwco->offset, acb->bytes, qiov,
1623                                       0, rwco->flags);
1624     blk_aio_complete(acb);
1625 }
1626
1627 static void coroutine_fn blk_aio_write_entry(void *opaque)
1628 {
1629     BlkAioEmAIOCB *acb = opaque;
1630     BlkRwCo *rwco = &acb->rwco;
1631     QEMUIOVector *qiov = rwco->iobuf;
1632
1633     assert(!qiov || qiov->size == acb->bytes);
1634     rwco->ret = blk_co_do_pwritev_part(rwco->blk, rwco->offset, acb->bytes,
1635                                        qiov, 0, rwco->flags);
1636     blk_aio_complete(acb);
1637 }
1638
1639 BlockAIOCB *blk_aio_pwrite_zeroes(BlockBackend *blk, int64_t offset,
1640                                   int64_t bytes, BdrvRequestFlags flags,
1641                                   BlockCompletionFunc *cb, void *opaque)
1642 {
1643     IO_CODE();
1644     return blk_aio_prwv(blk, offset, bytes, NULL, blk_aio_write_entry,
1645                         flags | BDRV_REQ_ZERO_WRITE, cb, opaque);
1646 }
1647
1648 int64_t coroutine_fn blk_co_getlength(BlockBackend *blk)
1649 {
1650     IO_CODE();
1651     GRAPH_RDLOCK_GUARD();
1652
1653     if (!blk_co_is_available(blk)) {
1654         return -ENOMEDIUM;
1655     }
1656
1657     return bdrv_co_getlength(blk_bs(blk));
1658 }
1659
1660 int64_t coroutine_fn blk_co_nb_sectors(BlockBackend *blk)
1661 {
1662     BlockDriverState *bs = blk_bs(blk);
1663
1664     IO_CODE();
1665     GRAPH_RDLOCK_GUARD();
1666
1667     if (!bs) {
1668         return -ENOMEDIUM;
1669     } else {
1670         return bdrv_co_nb_sectors(bs);
1671     }
1672 }
1673
1674 /*
1675  * This wrapper is written by hand because this function is in the hot I/O path,
1676  * via blk_get_geometry.
1677  */
1678 int64_t coroutine_mixed_fn blk_nb_sectors(BlockBackend *blk)
1679 {
1680     BlockDriverState *bs = blk_bs(blk);
1681
1682     IO_CODE();
1683
1684     if (!bs) {
1685         return -ENOMEDIUM;
1686     } else {
1687         return bdrv_nb_sectors(bs);
1688     }
1689 }
1690
1691 /* return 0 as number of sectors if no device present or error */
1692 void coroutine_fn blk_co_get_geometry(BlockBackend *blk,
1693                                       uint64_t *nb_sectors_ptr)
1694 {
1695     int64_t ret = blk_co_nb_sectors(blk);
1696     *nb_sectors_ptr = ret < 0 ? 0 : ret;
1697 }
1698
1699 /*
1700  * This wrapper is written by hand because this function is in the hot I/O path.
1701  */
1702 void coroutine_mixed_fn blk_get_geometry(BlockBackend *blk,
1703                                          uint64_t *nb_sectors_ptr)
1704 {
1705     int64_t ret = blk_nb_sectors(blk);
1706     *nb_sectors_ptr = ret < 0 ? 0 : ret;
1707 }
1708
1709 BlockAIOCB *blk_aio_preadv(BlockBackend *blk, int64_t offset,
1710                            QEMUIOVector *qiov, BdrvRequestFlags flags,
1711                            BlockCompletionFunc *cb, void *opaque)
1712 {
1713     IO_CODE();
1714     assert((uint64_t)qiov->size <= INT64_MAX);
1715     return blk_aio_prwv(blk, offset, qiov->size, qiov,
1716                         blk_aio_read_entry, flags, cb, opaque);
1717 }
1718
1719 BlockAIOCB *blk_aio_pwritev(BlockBackend *blk, int64_t offset,
1720                             QEMUIOVector *qiov, BdrvRequestFlags flags,
1721                             BlockCompletionFunc *cb, void *opaque)
1722 {
1723     IO_CODE();
1724     assert((uint64_t)qiov->size <= INT64_MAX);
1725     return blk_aio_prwv(blk, offset, qiov->size, qiov,
1726                         blk_aio_write_entry, flags, cb, opaque);
1727 }
1728
1729 void blk_aio_cancel(BlockAIOCB *acb)
1730 {
1731     GLOBAL_STATE_CODE();
1732     bdrv_aio_cancel(acb);
1733 }
1734
1735 void blk_aio_cancel_async(BlockAIOCB *acb)
1736 {
1737     IO_CODE();
1738     bdrv_aio_cancel_async(acb);
1739 }
1740
1741 /* To be called between exactly one pair of blk_inc/dec_in_flight() */
1742 static int coroutine_fn
1743 blk_co_do_ioctl(BlockBackend *blk, unsigned long int req, void *buf)
1744 {
1745     IO_CODE();
1746
1747     blk_wait_while_drained(blk);
1748     GRAPH_RDLOCK_GUARD();
1749
1750     if (!blk_co_is_available(blk)) {
1751         return -ENOMEDIUM;
1752     }
1753
1754     return bdrv_co_ioctl(blk_bs(blk), req, buf);
1755 }
1756
1757 int coroutine_fn blk_co_ioctl(BlockBackend *blk, unsigned long int req,
1758                               void *buf)
1759 {
1760     int ret;
1761     IO_OR_GS_CODE();
1762
1763     blk_inc_in_flight(blk);
1764     ret = blk_co_do_ioctl(blk, req, buf);
1765     blk_dec_in_flight(blk);
1766
1767     return ret;
1768 }
1769
1770 static void coroutine_fn blk_aio_ioctl_entry(void *opaque)
1771 {
1772     BlkAioEmAIOCB *acb = opaque;
1773     BlkRwCo *rwco = &acb->rwco;
1774
1775     rwco->ret = blk_co_do_ioctl(rwco->blk, rwco->offset, rwco->iobuf);
1776
1777     blk_aio_complete(acb);
1778 }
1779
1780 BlockAIOCB *blk_aio_ioctl(BlockBackend *blk, unsigned long int req, void *buf,
1781                           BlockCompletionFunc *cb, void *opaque)
1782 {
1783     IO_CODE();
1784     return blk_aio_prwv(blk, req, 0, buf, blk_aio_ioctl_entry, 0, cb, opaque);
1785 }
1786
1787 /* To be called between exactly one pair of blk_inc/dec_in_flight() */
1788 static int coroutine_fn
1789 blk_co_do_pdiscard(BlockBackend *blk, int64_t offset, int64_t bytes)
1790 {
1791     int ret;
1792     IO_CODE();
1793
1794     blk_wait_while_drained(blk);
1795     GRAPH_RDLOCK_GUARD();
1796
1797     ret = blk_check_byte_request(blk, offset, bytes);
1798     if (ret < 0) {
1799         return ret;
1800     }
1801
1802     return bdrv_co_pdiscard(blk->root, offset, bytes);
1803 }
1804
1805 static void coroutine_fn blk_aio_pdiscard_entry(void *opaque)
1806 {
1807     BlkAioEmAIOCB *acb = opaque;
1808     BlkRwCo *rwco = &acb->rwco;
1809
1810     rwco->ret = blk_co_do_pdiscard(rwco->blk, rwco->offset, acb->bytes);
1811     blk_aio_complete(acb);
1812 }
1813
1814 BlockAIOCB *blk_aio_pdiscard(BlockBackend *blk,
1815                              int64_t offset, int64_t bytes,
1816                              BlockCompletionFunc *cb, void *opaque)
1817 {
1818     IO_CODE();
1819     return blk_aio_prwv(blk, offset, bytes, NULL, blk_aio_pdiscard_entry, 0,
1820                         cb, opaque);
1821 }
1822
1823 int coroutine_fn blk_co_pdiscard(BlockBackend *blk, int64_t offset,
1824                                  int64_t bytes)
1825 {
1826     int ret;
1827     IO_OR_GS_CODE();
1828
1829     blk_inc_in_flight(blk);
1830     ret = blk_co_do_pdiscard(blk, offset, bytes);
1831     blk_dec_in_flight(blk);
1832
1833     return ret;
1834 }
1835
1836 /* To be called between exactly one pair of blk_inc/dec_in_flight() */
1837 static int coroutine_fn blk_co_do_flush(BlockBackend *blk)
1838 {
1839     IO_CODE();
1840     blk_wait_while_drained(blk);
1841     GRAPH_RDLOCK_GUARD();
1842
1843     if (!blk_co_is_available(blk)) {
1844         return -ENOMEDIUM;
1845     }
1846
1847     return bdrv_co_flush(blk_bs(blk));
1848 }
1849
1850 static void coroutine_fn blk_aio_flush_entry(void *opaque)
1851 {
1852     BlkAioEmAIOCB *acb = opaque;
1853     BlkRwCo *rwco = &acb->rwco;
1854
1855     rwco->ret = blk_co_do_flush(rwco->blk);
1856     blk_aio_complete(acb);
1857 }
1858
1859 BlockAIOCB *blk_aio_flush(BlockBackend *blk,
1860                           BlockCompletionFunc *cb, void *opaque)
1861 {
1862     IO_CODE();
1863     return blk_aio_prwv(blk, 0, 0, NULL, blk_aio_flush_entry, 0, cb, opaque);
1864 }
1865
1866 int coroutine_fn blk_co_flush(BlockBackend *blk)
1867 {
1868     int ret;
1869     IO_OR_GS_CODE();
1870
1871     blk_inc_in_flight(blk);
1872     ret = blk_co_do_flush(blk);
1873     blk_dec_in_flight(blk);
1874
1875     return ret;
1876 }
1877
1878 static void coroutine_fn blk_aio_zone_report_entry(void *opaque)
1879 {
1880     BlkAioEmAIOCB *acb = opaque;
1881     BlkRwCo *rwco = &acb->rwco;
1882
1883     rwco->ret = blk_co_zone_report(rwco->blk, rwco->offset,
1884                                    (unsigned int*)(uintptr_t)acb->bytes,
1885                                    rwco->iobuf);
1886     blk_aio_complete(acb);
1887 }
1888
1889 BlockAIOCB *blk_aio_zone_report(BlockBackend *blk, int64_t offset,
1890                                 unsigned int *nr_zones,
1891                                 BlockZoneDescriptor  *zones,
1892                                 BlockCompletionFunc *cb, void *opaque)
1893 {
1894     BlkAioEmAIOCB *acb;
1895     Coroutine *co;
1896     IO_CODE();
1897
1898     blk_inc_in_flight(blk);
1899     acb = blk_aio_get(&blk_aio_em_aiocb_info, blk, cb, opaque);
1900     acb->rwco = (BlkRwCo) {
1901         .blk    = blk,
1902         .offset = offset,
1903         .iobuf  = zones,
1904         .ret    = NOT_DONE,
1905     };
1906     acb->bytes = (int64_t)(uintptr_t)nr_zones,
1907     acb->has_returned = false;
1908
1909     co = qemu_coroutine_create(blk_aio_zone_report_entry, acb);
1910     aio_co_enter(qemu_get_current_aio_context(), co);
1911
1912     acb->has_returned = true;
1913     if (acb->rwco.ret != NOT_DONE) {
1914         replay_bh_schedule_oneshot_event(qemu_get_current_aio_context(),
1915                                          blk_aio_complete_bh, acb);
1916     }
1917
1918     return &acb->common;
1919 }
1920
1921 static void coroutine_fn blk_aio_zone_mgmt_entry(void *opaque)
1922 {
1923     BlkAioEmAIOCB *acb = opaque;
1924     BlkRwCo *rwco = &acb->rwco;
1925
1926     rwco->ret = blk_co_zone_mgmt(rwco->blk,
1927                                  (BlockZoneOp)(uintptr_t)rwco->iobuf,
1928                                  rwco->offset, acb->bytes);
1929     blk_aio_complete(acb);
1930 }
1931
1932 BlockAIOCB *blk_aio_zone_mgmt(BlockBackend *blk, BlockZoneOp op,
1933                               int64_t offset, int64_t len,
1934                               BlockCompletionFunc *cb, void *opaque) {
1935     BlkAioEmAIOCB *acb;
1936     Coroutine *co;
1937     IO_CODE();
1938
1939     blk_inc_in_flight(blk);
1940     acb = blk_aio_get(&blk_aio_em_aiocb_info, blk, cb, opaque);
1941     acb->rwco = (BlkRwCo) {
1942         .blk    = blk,
1943         .offset = offset,
1944         .iobuf  = (void *)(uintptr_t)op,
1945         .ret    = NOT_DONE,
1946     };
1947     acb->bytes = len;
1948     acb->has_returned = false;
1949
1950     co = qemu_coroutine_create(blk_aio_zone_mgmt_entry, acb);
1951     aio_co_enter(qemu_get_current_aio_context(), co);
1952
1953     acb->has_returned = true;
1954     if (acb->rwco.ret != NOT_DONE) {
1955         replay_bh_schedule_oneshot_event(qemu_get_current_aio_context(),
1956                                          blk_aio_complete_bh, acb);
1957     }
1958
1959     return &acb->common;
1960 }
1961
1962 static void coroutine_fn blk_aio_zone_append_entry(void *opaque)
1963 {
1964     BlkAioEmAIOCB *acb = opaque;
1965     BlkRwCo *rwco = &acb->rwco;
1966
1967     rwco->ret = blk_co_zone_append(rwco->blk, (int64_t *)(uintptr_t)acb->bytes,
1968                                    rwco->iobuf, rwco->flags);
1969     blk_aio_complete(acb);
1970 }
1971
1972 BlockAIOCB *blk_aio_zone_append(BlockBackend *blk, int64_t *offset,
1973                                 QEMUIOVector *qiov, BdrvRequestFlags flags,
1974                                 BlockCompletionFunc *cb, void *opaque) {
1975     BlkAioEmAIOCB *acb;
1976     Coroutine *co;
1977     IO_CODE();
1978
1979     blk_inc_in_flight(blk);
1980     acb = blk_aio_get(&blk_aio_em_aiocb_info, blk, cb, opaque);
1981     acb->rwco = (BlkRwCo) {
1982         .blk    = blk,
1983         .ret    = NOT_DONE,
1984         .flags  = flags,
1985         .iobuf  = qiov,
1986     };
1987     acb->bytes = (int64_t)(uintptr_t)offset;
1988     acb->has_returned = false;
1989
1990     co = qemu_coroutine_create(blk_aio_zone_append_entry, acb);
1991     aio_co_enter(qemu_get_current_aio_context(), co);
1992     acb->has_returned = true;
1993     if (acb->rwco.ret != NOT_DONE) {
1994         replay_bh_schedule_oneshot_event(qemu_get_current_aio_context(),
1995                                          blk_aio_complete_bh, acb);
1996     }
1997
1998     return &acb->common;
1999 }
2000
2001 /*
2002  * Send a zone_report command.
2003  * offset is a byte offset from the start of the device. No alignment
2004  * required for offset.
2005  * nr_zones represents IN maximum and OUT actual.
2006  */
2007 int coroutine_fn blk_co_zone_report(BlockBackend *blk, int64_t offset,
2008                                     unsigned int *nr_zones,
2009                                     BlockZoneDescriptor *zones)
2010 {
2011     int ret;
2012     IO_CODE();
2013
2014     blk_inc_in_flight(blk); /* increase before waiting */
2015     blk_wait_while_drained(blk);
2016     GRAPH_RDLOCK_GUARD();
2017     if (!blk_is_available(blk)) {
2018         blk_dec_in_flight(blk);
2019         return -ENOMEDIUM;
2020     }
2021     ret = bdrv_co_zone_report(blk_bs(blk), offset, nr_zones, zones);
2022     blk_dec_in_flight(blk);
2023     return ret;
2024 }
2025
2026 /*
2027  * Send a zone_management command.
2028  * op is the zone operation;
2029  * offset is the byte offset from the start of the zoned device;
2030  * len is the maximum number of bytes the command should operate on. It
2031  * should be aligned with the device zone size.
2032  */
2033 int coroutine_fn blk_co_zone_mgmt(BlockBackend *blk, BlockZoneOp op,
2034         int64_t offset, int64_t len)
2035 {
2036     int ret;
2037     IO_CODE();
2038
2039     blk_inc_in_flight(blk);
2040     blk_wait_while_drained(blk);
2041     GRAPH_RDLOCK_GUARD();
2042
2043     ret = blk_check_byte_request(blk, offset, len);
2044     if (ret < 0) {
2045         blk_dec_in_flight(blk);
2046         return ret;
2047     }
2048
2049     ret = bdrv_co_zone_mgmt(blk_bs(blk), op, offset, len);
2050     blk_dec_in_flight(blk);
2051     return ret;
2052 }
2053
2054 /*
2055  * Send a zone_append command.
2056  */
2057 int coroutine_fn blk_co_zone_append(BlockBackend *blk, int64_t *offset,
2058         QEMUIOVector *qiov, BdrvRequestFlags flags)
2059 {
2060     int ret;
2061     IO_CODE();
2062
2063     blk_inc_in_flight(blk);
2064     blk_wait_while_drained(blk);
2065     GRAPH_RDLOCK_GUARD();
2066     if (!blk_is_available(blk)) {
2067         blk_dec_in_flight(blk);
2068         return -ENOMEDIUM;
2069     }
2070
2071     ret = bdrv_co_zone_append(blk_bs(blk), offset, qiov, flags);
2072     blk_dec_in_flight(blk);
2073     return ret;
2074 }
2075
2076 void blk_drain(BlockBackend *blk)
2077 {
2078     BlockDriverState *bs = blk_bs(blk);
2079     GLOBAL_STATE_CODE();
2080
2081     if (bs) {
2082         bdrv_ref(bs);
2083         bdrv_drained_begin(bs);
2084     }
2085
2086     /* We may have -ENOMEDIUM completions in flight */
2087     AIO_WAIT_WHILE(blk_get_aio_context(blk),
2088                    qatomic_read(&blk->in_flight) > 0);
2089
2090     if (bs) {
2091         bdrv_drained_end(bs);
2092         bdrv_unref(bs);
2093     }
2094 }
2095
2096 void blk_drain_all(void)
2097 {
2098     BlockBackend *blk = NULL;
2099
2100     GLOBAL_STATE_CODE();
2101
2102     bdrv_drain_all_begin();
2103
2104     while ((blk = blk_all_next(blk)) != NULL) {
2105         /* We may have -ENOMEDIUM completions in flight */
2106         AIO_WAIT_WHILE_UNLOCKED(NULL, qatomic_read(&blk->in_flight) > 0);
2107     }
2108
2109     bdrv_drain_all_end();
2110 }
2111
2112 void blk_set_on_error(BlockBackend *blk, BlockdevOnError on_read_error,
2113                       BlockdevOnError on_write_error)
2114 {
2115     GLOBAL_STATE_CODE();
2116     blk->on_read_error = on_read_error;
2117     blk->on_write_error = on_write_error;
2118 }
2119
2120 BlockdevOnError blk_get_on_error(BlockBackend *blk, bool is_read)
2121 {
2122     IO_CODE();
2123     return is_read ? blk->on_read_error : blk->on_write_error;
2124 }
2125
2126 BlockErrorAction blk_get_error_action(BlockBackend *blk, bool is_read,
2127                                       int error)
2128 {
2129     BlockdevOnError on_err = blk_get_on_error(blk, is_read);
2130     IO_CODE();
2131
2132     switch (on_err) {
2133     case BLOCKDEV_ON_ERROR_ENOSPC:
2134         return (error == ENOSPC) ?
2135                BLOCK_ERROR_ACTION_STOP : BLOCK_ERROR_ACTION_REPORT;
2136     case BLOCKDEV_ON_ERROR_STOP:
2137         return BLOCK_ERROR_ACTION_STOP;
2138     case BLOCKDEV_ON_ERROR_REPORT:
2139         return BLOCK_ERROR_ACTION_REPORT;
2140     case BLOCKDEV_ON_ERROR_IGNORE:
2141         return BLOCK_ERROR_ACTION_IGNORE;
2142     case BLOCKDEV_ON_ERROR_AUTO:
2143     default:
2144         abort();
2145     }
2146 }
2147
2148 static void send_qmp_error_event(BlockBackend *blk,
2149                                  BlockErrorAction action,
2150                                  bool is_read, int error)
2151 {
2152     IoOperationType optype;
2153     BlockDriverState *bs = blk_bs(blk);
2154
2155     optype = is_read ? IO_OPERATION_TYPE_READ : IO_OPERATION_TYPE_WRITE;
2156     qapi_event_send_block_io_error(blk_name(blk),
2157                                    bs ? bdrv_get_node_name(bs) : NULL, optype,
2158                                    action, blk_iostatus_is_enabled(blk),
2159                                    error == ENOSPC, strerror(error));
2160 }
2161
2162 /* This is done by device models because, while the block layer knows
2163  * about the error, it does not know whether an operation comes from
2164  * the device or the block layer (from a job, for example).
2165  */
2166 void blk_error_action(BlockBackend *blk, BlockErrorAction action,
2167                       bool is_read, int error)
2168 {
2169     assert(error >= 0);
2170     IO_CODE();
2171
2172     if (action == BLOCK_ERROR_ACTION_STOP) {
2173         /* First set the iostatus, so that "info block" returns an iostatus
2174          * that matches the events raised so far (an additional error iostatus
2175          * is fine, but not a lost one).
2176          */
2177         blk_iostatus_set_err(blk, error);
2178
2179         /* Then raise the request to stop the VM and the event.
2180          * qemu_system_vmstop_request_prepare has two effects.  First,
2181          * it ensures that the STOP event always comes after the
2182          * BLOCK_IO_ERROR event.  Second, it ensures that even if management
2183          * can observe the STOP event and do a "cont" before the STOP
2184          * event is issued, the VM will not stop.  In this case, vm_start()
2185          * also ensures that the STOP/RESUME pair of events is emitted.
2186          */
2187         qemu_system_vmstop_request_prepare();
2188         send_qmp_error_event(blk, action, is_read, error);
2189         qemu_system_vmstop_request(RUN_STATE_IO_ERROR);
2190     } else {
2191         send_qmp_error_event(blk, action, is_read, error);
2192     }
2193 }
2194
2195 /*
2196  * Returns true if the BlockBackend can support taking write permissions
2197  * (because its root node is not read-only).
2198  */
2199 bool blk_supports_write_perm(BlockBackend *blk)
2200 {
2201     BlockDriverState *bs = blk_bs(blk);
2202     GLOBAL_STATE_CODE();
2203
2204     if (bs) {
2205         return !bdrv_is_read_only(bs);
2206     } else {
2207         return blk->root_state.open_flags & BDRV_O_RDWR;
2208     }
2209 }
2210
2211 /*
2212  * Returns true if the BlockBackend can be written to in its current
2213  * configuration (i.e. if write permission have been requested)
2214  */
2215 bool blk_is_writable(BlockBackend *blk)
2216 {
2217     IO_CODE();
2218     return blk->perm & BLK_PERM_WRITE;
2219 }
2220
2221 bool blk_is_sg(BlockBackend *blk)
2222 {
2223     BlockDriverState *bs = blk_bs(blk);
2224     GLOBAL_STATE_CODE();
2225
2226     if (!bs) {
2227         return false;
2228     }
2229
2230     return bdrv_is_sg(bs);
2231 }
2232
2233 bool blk_enable_write_cache(BlockBackend *blk)
2234 {
2235     IO_CODE();
2236     return blk->enable_write_cache;
2237 }
2238
2239 void blk_set_enable_write_cache(BlockBackend *blk, bool wce)
2240 {
2241     IO_CODE();
2242     blk->enable_write_cache = wce;
2243 }
2244
2245 void blk_activate(BlockBackend *blk, Error **errp)
2246 {
2247     BlockDriverState *bs = blk_bs(blk);
2248     GLOBAL_STATE_CODE();
2249
2250     if (!bs) {
2251         error_setg(errp, "Device '%s' has no medium", blk->name);
2252         return;
2253     }
2254
2255     /*
2256      * Migration code can call this function in coroutine context, so leave
2257      * coroutine context if necessary.
2258      */
2259     if (qemu_in_coroutine()) {
2260         bdrv_co_activate(bs, errp);
2261     } else {
2262         bdrv_activate(bs, errp);
2263     }
2264 }
2265
2266 bool coroutine_fn blk_co_is_inserted(BlockBackend *blk)
2267 {
2268     BlockDriverState *bs = blk_bs(blk);
2269     IO_CODE();
2270     assert_bdrv_graph_readable();
2271
2272     return bs && bdrv_co_is_inserted(bs);
2273 }
2274
2275 bool coroutine_fn blk_co_is_available(BlockBackend *blk)
2276 {
2277     IO_CODE();
2278     return blk_co_is_inserted(blk) && !blk_dev_is_tray_open(blk);
2279 }
2280
2281 void coroutine_fn blk_co_lock_medium(BlockBackend *blk, bool locked)
2282 {
2283     BlockDriverState *bs = blk_bs(blk);
2284     IO_CODE();
2285     GRAPH_RDLOCK_GUARD();
2286
2287     if (bs) {
2288         bdrv_co_lock_medium(bs, locked);
2289     }
2290 }
2291
2292 void coroutine_fn blk_co_eject(BlockBackend *blk, bool eject_flag)
2293 {
2294     BlockDriverState *bs = blk_bs(blk);
2295     char *id;
2296     IO_CODE();
2297     GRAPH_RDLOCK_GUARD();
2298
2299     if (bs) {
2300         bdrv_co_eject(bs, eject_flag);
2301     }
2302
2303     /* Whether or not we ejected on the backend,
2304      * the frontend experienced a tray event. */
2305     id = blk_get_attached_dev_id(blk);
2306     qapi_event_send_device_tray_moved(blk_name(blk), id,
2307                                       eject_flag);
2308     g_free(id);
2309 }
2310
2311 int blk_get_flags(BlockBackend *blk)
2312 {
2313     BlockDriverState *bs = blk_bs(blk);
2314     GLOBAL_STATE_CODE();
2315
2316     if (bs) {
2317         return bdrv_get_flags(bs);
2318     } else {
2319         return blk->root_state.open_flags;
2320     }
2321 }
2322
2323 /* Returns the minimum request alignment, in bytes; guaranteed nonzero */
2324 uint32_t blk_get_request_alignment(BlockBackend *blk)
2325 {
2326     BlockDriverState *bs = blk_bs(blk);
2327     IO_CODE();
2328     return bs ? bs->bl.request_alignment : BDRV_SECTOR_SIZE;
2329 }
2330
2331 /* Returns the maximum hardware transfer length, in bytes; guaranteed nonzero */
2332 uint64_t blk_get_max_hw_transfer(BlockBackend *blk)
2333 {
2334     BlockDriverState *bs = blk_bs(blk);
2335     uint64_t max = INT_MAX;
2336     IO_CODE();
2337
2338     if (bs) {
2339         max = MIN_NON_ZERO(max, bs->bl.max_hw_transfer);
2340         max = MIN_NON_ZERO(max, bs->bl.max_transfer);
2341     }
2342     return ROUND_DOWN(max, blk_get_request_alignment(blk));
2343 }
2344
2345 /* Returns the maximum transfer length, in bytes; guaranteed nonzero */
2346 uint32_t blk_get_max_transfer(BlockBackend *blk)
2347 {
2348     BlockDriverState *bs = blk_bs(blk);
2349     uint32_t max = INT_MAX;
2350     IO_CODE();
2351
2352     if (bs) {
2353         max = MIN_NON_ZERO(max, bs->bl.max_transfer);
2354     }
2355     return ROUND_DOWN(max, blk_get_request_alignment(blk));
2356 }
2357
2358 int blk_get_max_hw_iov(BlockBackend *blk)
2359 {
2360     IO_CODE();
2361     return MIN_NON_ZERO(blk->root->bs->bl.max_hw_iov,
2362                         blk->root->bs->bl.max_iov);
2363 }
2364
2365 int blk_get_max_iov(BlockBackend *blk)
2366 {
2367     IO_CODE();
2368     return blk->root->bs->bl.max_iov;
2369 }
2370
2371 void *blk_try_blockalign(BlockBackend *blk, size_t size)
2372 {
2373     IO_CODE();
2374     return qemu_try_blockalign(blk ? blk_bs(blk) : NULL, size);
2375 }
2376
2377 void *blk_blockalign(BlockBackend *blk, size_t size)
2378 {
2379     IO_CODE();
2380     return qemu_blockalign(blk ? blk_bs(blk) : NULL, size);
2381 }
2382
2383 bool blk_op_is_blocked(BlockBackend *blk, BlockOpType op, Error **errp)
2384 {
2385     BlockDriverState *bs = blk_bs(blk);
2386     GLOBAL_STATE_CODE();
2387
2388     if (!bs) {
2389         return false;
2390     }
2391
2392     return bdrv_op_is_blocked(bs, op, errp);
2393 }
2394
2395 void blk_op_unblock(BlockBackend *blk, BlockOpType op, Error *reason)
2396 {
2397     BlockDriverState *bs = blk_bs(blk);
2398     GLOBAL_STATE_CODE();
2399
2400     if (bs) {
2401         bdrv_op_unblock(bs, op, reason);
2402     }
2403 }
2404
2405 void blk_op_block_all(BlockBackend *blk, Error *reason)
2406 {
2407     BlockDriverState *bs = blk_bs(blk);
2408     GLOBAL_STATE_CODE();
2409
2410     if (bs) {
2411         bdrv_op_block_all(bs, reason);
2412     }
2413 }
2414
2415 void blk_op_unblock_all(BlockBackend *blk, Error *reason)
2416 {
2417     BlockDriverState *bs = blk_bs(blk);
2418     GLOBAL_STATE_CODE();
2419
2420     if (bs) {
2421         bdrv_op_unblock_all(bs, reason);
2422     }
2423 }
2424
2425 AioContext *blk_get_aio_context(BlockBackend *blk)
2426 {
2427     BlockDriverState *bs;
2428     IO_CODE();
2429
2430     if (!blk) {
2431         return qemu_get_aio_context();
2432     }
2433
2434     bs = blk_bs(blk);
2435     if (bs) {
2436         AioContext *ctx = bdrv_get_aio_context(blk_bs(blk));
2437         assert(ctx == blk->ctx);
2438     }
2439
2440     return blk->ctx;
2441 }
2442
2443 int blk_set_aio_context(BlockBackend *blk, AioContext *new_context,
2444                         Error **errp)
2445 {
2446     bool old_allow_change;
2447     BlockDriverState *bs = blk_bs(blk);
2448     int ret;
2449
2450     GLOBAL_STATE_CODE();
2451
2452     if (!bs) {
2453         blk->ctx = new_context;
2454         return 0;
2455     }
2456
2457     bdrv_ref(bs);
2458
2459     old_allow_change = blk->allow_aio_context_change;
2460     blk->allow_aio_context_change = true;
2461
2462     ret = bdrv_try_change_aio_context(bs, new_context, NULL, errp);
2463
2464     blk->allow_aio_context_change = old_allow_change;
2465
2466     bdrv_unref(bs);
2467     return ret;
2468 }
2469
2470 typedef struct BdrvStateBlkRootContext {
2471     AioContext *new_ctx;
2472     BlockBackend *blk;
2473 } BdrvStateBlkRootContext;
2474
2475 static void blk_root_set_aio_ctx_commit(void *opaque)
2476 {
2477     BdrvStateBlkRootContext *s = opaque;
2478     BlockBackend *blk = s->blk;
2479     AioContext *new_context = s->new_ctx;
2480     ThrottleGroupMember *tgm = &blk->public.throttle_group_member;
2481
2482     blk->ctx = new_context;
2483     if (tgm->throttle_state) {
2484         throttle_group_detach_aio_context(tgm);
2485         throttle_group_attach_aio_context(tgm, new_context);
2486     }
2487 }
2488
2489 static TransactionActionDrv set_blk_root_context = {
2490     .commit = blk_root_set_aio_ctx_commit,
2491     .clean = g_free,
2492 };
2493
2494 static bool blk_root_change_aio_ctx(BdrvChild *child, AioContext *ctx,
2495                                     GHashTable *visited, Transaction *tran,
2496                                     Error **errp)
2497 {
2498     BlockBackend *blk = child->opaque;
2499     BdrvStateBlkRootContext *s;
2500
2501     if (!blk->allow_aio_context_change) {
2502         /*
2503          * Manually created BlockBackends (those with a name) that are not
2504          * attached to anything can change their AioContext without updating
2505          * their user; return an error for others.
2506          */
2507         if (!blk->name || blk->dev) {
2508             /* TODO Add BB name/QOM path */
2509             error_setg(errp, "Cannot change iothread of active block backend");
2510             return false;
2511         }
2512     }
2513
2514     s = g_new(BdrvStateBlkRootContext, 1);
2515     *s = (BdrvStateBlkRootContext) {
2516         .new_ctx = ctx,
2517         .blk = blk,
2518     };
2519
2520     tran_add(tran, &set_blk_root_context, s);
2521     return true;
2522 }
2523
2524 void blk_add_aio_context_notifier(BlockBackend *blk,
2525         void (*attached_aio_context)(AioContext *new_context, void *opaque),
2526         void (*detach_aio_context)(void *opaque), void *opaque)
2527 {
2528     BlockBackendAioNotifier *notifier;
2529     BlockDriverState *bs = blk_bs(blk);
2530     GLOBAL_STATE_CODE();
2531
2532     notifier = g_new(BlockBackendAioNotifier, 1);
2533     notifier->attached_aio_context = attached_aio_context;
2534     notifier->detach_aio_context = detach_aio_context;
2535     notifier->opaque = opaque;
2536     QLIST_INSERT_HEAD(&blk->aio_notifiers, notifier, list);
2537
2538     if (bs) {
2539         bdrv_add_aio_context_notifier(bs, attached_aio_context,
2540                                       detach_aio_context, opaque);
2541     }
2542 }
2543
2544 void blk_remove_aio_context_notifier(BlockBackend *blk,
2545                                      void (*attached_aio_context)(AioContext *,
2546                                                                   void *),
2547                                      void (*detach_aio_context)(void *),
2548                                      void *opaque)
2549 {
2550     BlockBackendAioNotifier *notifier;
2551     BlockDriverState *bs = blk_bs(blk);
2552
2553     GLOBAL_STATE_CODE();
2554
2555     if (bs) {
2556         bdrv_remove_aio_context_notifier(bs, attached_aio_context,
2557                                          detach_aio_context, opaque);
2558     }
2559
2560     QLIST_FOREACH(notifier, &blk->aio_notifiers, list) {
2561         if (notifier->attached_aio_context == attached_aio_context &&
2562             notifier->detach_aio_context == detach_aio_context &&
2563             notifier->opaque == opaque) {
2564             QLIST_REMOVE(notifier, list);
2565             g_free(notifier);
2566             return;
2567         }
2568     }
2569
2570     abort();
2571 }
2572
2573 void blk_add_remove_bs_notifier(BlockBackend *blk, Notifier *notify)
2574 {
2575     GLOBAL_STATE_CODE();
2576     notifier_list_add(&blk->remove_bs_notifiers, notify);
2577 }
2578
2579 void blk_add_insert_bs_notifier(BlockBackend *blk, Notifier *notify)
2580 {
2581     GLOBAL_STATE_CODE();
2582     notifier_list_add(&blk->insert_bs_notifiers, notify);
2583 }
2584
2585 BlockAcctStats *blk_get_stats(BlockBackend *blk)
2586 {
2587     IO_CODE();
2588     return &blk->stats;
2589 }
2590
2591 void *blk_aio_get(const AIOCBInfo *aiocb_info, BlockBackend *blk,
2592                   BlockCompletionFunc *cb, void *opaque)
2593 {
2594     IO_CODE();
2595     return qemu_aio_get(aiocb_info, blk_bs(blk), cb, opaque);
2596 }
2597
2598 int coroutine_fn blk_co_pwrite_zeroes(BlockBackend *blk, int64_t offset,
2599                                       int64_t bytes, BdrvRequestFlags flags)
2600 {
2601     IO_OR_GS_CODE();
2602     return blk_co_pwritev(blk, offset, bytes, NULL,
2603                           flags | BDRV_REQ_ZERO_WRITE);
2604 }
2605
2606 int coroutine_fn blk_co_pwrite_compressed(BlockBackend *blk, int64_t offset,
2607                                           int64_t bytes, const void *buf)
2608 {
2609     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes);
2610     IO_OR_GS_CODE();
2611     return blk_co_pwritev_part(blk, offset, bytes, &qiov, 0,
2612                                BDRV_REQ_WRITE_COMPRESSED);
2613 }
2614
2615 int coroutine_fn blk_co_truncate(BlockBackend *blk, int64_t offset, bool exact,
2616                                  PreallocMode prealloc, BdrvRequestFlags flags,
2617                                  Error **errp)
2618 {
2619     IO_OR_GS_CODE();
2620     GRAPH_RDLOCK_GUARD();
2621     if (!blk_co_is_available(blk)) {
2622         error_setg(errp, "No medium inserted");
2623         return -ENOMEDIUM;
2624     }
2625
2626     return bdrv_co_truncate(blk->root, offset, exact, prealloc, flags, errp);
2627 }
2628
2629 int blk_save_vmstate(BlockBackend *blk, const uint8_t *buf,
2630                      int64_t pos, int size)
2631 {
2632     int ret;
2633     GLOBAL_STATE_CODE();
2634
2635     if (!blk_is_available(blk)) {
2636         return -ENOMEDIUM;
2637     }
2638
2639     ret = bdrv_save_vmstate(blk_bs(blk), buf, pos, size);
2640     if (ret < 0) {
2641         return ret;
2642     }
2643
2644     if (ret == size && !blk->enable_write_cache) {
2645         ret = bdrv_flush(blk_bs(blk));
2646     }
2647
2648     return ret < 0 ? ret : size;
2649 }
2650
2651 int blk_load_vmstate(BlockBackend *blk, uint8_t *buf, int64_t pos, int size)
2652 {
2653     GLOBAL_STATE_CODE();
2654     if (!blk_is_available(blk)) {
2655         return -ENOMEDIUM;
2656     }
2657
2658     return bdrv_load_vmstate(blk_bs(blk), buf, pos, size);
2659 }
2660
2661 int blk_probe_blocksizes(BlockBackend *blk, BlockSizes *bsz)
2662 {
2663     GLOBAL_STATE_CODE();
2664     if (!blk_is_available(blk)) {
2665         return -ENOMEDIUM;
2666     }
2667
2668     return bdrv_probe_blocksizes(blk_bs(blk), bsz);
2669 }
2670
2671 int blk_probe_geometry(BlockBackend *blk, HDGeometry *geo)
2672 {
2673     GLOBAL_STATE_CODE();
2674     if (!blk_is_available(blk)) {
2675         return -ENOMEDIUM;
2676     }
2677
2678     return bdrv_probe_geometry(blk_bs(blk), geo);
2679 }
2680
2681 /*
2682  * Updates the BlockBackendRootState object with data from the currently
2683  * attached BlockDriverState.
2684  */
2685 void blk_update_root_state(BlockBackend *blk)
2686 {
2687     GLOBAL_STATE_CODE();
2688     assert(blk->root);
2689
2690     blk->root_state.open_flags    = blk->root->bs->open_flags;
2691     blk->root_state.detect_zeroes = blk->root->bs->detect_zeroes;
2692 }
2693
2694 /*
2695  * Returns the detect-zeroes setting to be used for bdrv_open() of a
2696  * BlockDriverState which is supposed to inherit the root state.
2697  */
2698 bool blk_get_detect_zeroes_from_root_state(BlockBackend *blk)
2699 {
2700     GLOBAL_STATE_CODE();
2701     return blk->root_state.detect_zeroes;
2702 }
2703
2704 /*
2705  * Returns the flags to be used for bdrv_open() of a BlockDriverState which is
2706  * supposed to inherit the root state.
2707  */
2708 int blk_get_open_flags_from_root_state(BlockBackend *blk)
2709 {
2710     GLOBAL_STATE_CODE();
2711     return blk->root_state.open_flags;
2712 }
2713
2714 BlockBackendRootState *blk_get_root_state(BlockBackend *blk)
2715 {
2716     GLOBAL_STATE_CODE();
2717     return &blk->root_state;
2718 }
2719
2720 int blk_commit_all(void)
2721 {
2722     BlockBackend *blk = NULL;
2723     GLOBAL_STATE_CODE();
2724
2725     while ((blk = blk_all_next(blk)) != NULL) {
2726         AioContext *aio_context = blk_get_aio_context(blk);
2727         BlockDriverState *unfiltered_bs = bdrv_skip_filters(blk_bs(blk));
2728
2729         aio_context_acquire(aio_context);
2730         if (blk_is_inserted(blk) && bdrv_cow_child(unfiltered_bs)) {
2731             int ret;
2732
2733             ret = bdrv_commit(unfiltered_bs);
2734             if (ret < 0) {
2735                 aio_context_release(aio_context);
2736                 return ret;
2737             }
2738         }
2739         aio_context_release(aio_context);
2740     }
2741     return 0;
2742 }
2743
2744
2745 /* throttling disk I/O limits */
2746 void blk_set_io_limits(BlockBackend *blk, ThrottleConfig *cfg)
2747 {
2748     GLOBAL_STATE_CODE();
2749     throttle_group_config(&blk->public.throttle_group_member, cfg);
2750 }
2751
2752 void blk_io_limits_disable(BlockBackend *blk)
2753 {
2754     BlockDriverState *bs = blk_bs(blk);
2755     ThrottleGroupMember *tgm = &blk->public.throttle_group_member;
2756     assert(tgm->throttle_state);
2757     GLOBAL_STATE_CODE();
2758     if (bs) {
2759         bdrv_ref(bs);
2760         bdrv_drained_begin(bs);
2761     }
2762     throttle_group_unregister_tgm(tgm);
2763     if (bs) {
2764         bdrv_drained_end(bs);
2765         bdrv_unref(bs);
2766     }
2767 }
2768
2769 /* should be called before blk_set_io_limits if a limit is set */
2770 void blk_io_limits_enable(BlockBackend *blk, const char *group)
2771 {
2772     assert(!blk->public.throttle_group_member.throttle_state);
2773     GLOBAL_STATE_CODE();
2774     throttle_group_register_tgm(&blk->public.throttle_group_member,
2775                                 group, blk_get_aio_context(blk));
2776 }
2777
2778 void blk_io_limits_update_group(BlockBackend *blk, const char *group)
2779 {
2780     GLOBAL_STATE_CODE();
2781     /* this BB is not part of any group */
2782     if (!blk->public.throttle_group_member.throttle_state) {
2783         return;
2784     }
2785
2786     /* this BB is a part of the same group than the one we want */
2787     if (!g_strcmp0(throttle_group_get_name(&blk->public.throttle_group_member),
2788                 group)) {
2789         return;
2790     }
2791
2792     /* need to change the group this bs belong to */
2793     blk_io_limits_disable(blk);
2794     blk_io_limits_enable(blk, group);
2795 }
2796
2797 static void blk_root_drained_begin(BdrvChild *child)
2798 {
2799     BlockBackend *blk = child->opaque;
2800     ThrottleGroupMember *tgm = &blk->public.throttle_group_member;
2801
2802     if (qatomic_fetch_inc(&blk->quiesce_counter) == 0) {
2803         if (blk->dev_ops && blk->dev_ops->drained_begin) {
2804             blk->dev_ops->drained_begin(blk->dev_opaque);
2805         }
2806     }
2807
2808     /* Note that blk->root may not be accessible here yet if we are just
2809      * attaching to a BlockDriverState that is drained. Use child instead. */
2810
2811     if (qatomic_fetch_inc(&tgm->io_limits_disabled) == 0) {
2812         throttle_group_restart_tgm(tgm);
2813     }
2814 }
2815
2816 static bool blk_root_drained_poll(BdrvChild *child)
2817 {
2818     BlockBackend *blk = child->opaque;
2819     bool busy = false;
2820     assert(qatomic_read(&blk->quiesce_counter));
2821
2822     if (blk->dev_ops && blk->dev_ops->drained_poll) {
2823         busy = blk->dev_ops->drained_poll(blk->dev_opaque);
2824     }
2825     return busy || !!blk->in_flight;
2826 }
2827
2828 static void blk_root_drained_end(BdrvChild *child)
2829 {
2830     BlockBackend *blk = child->opaque;
2831     assert(qatomic_read(&blk->quiesce_counter));
2832
2833     assert(blk->public.throttle_group_member.io_limits_disabled);
2834     qatomic_dec(&blk->public.throttle_group_member.io_limits_disabled);
2835
2836     if (qatomic_fetch_dec(&blk->quiesce_counter) == 1) {
2837         if (blk->dev_ops && blk->dev_ops->drained_end) {
2838             blk->dev_ops->drained_end(blk->dev_opaque);
2839         }
2840         qemu_mutex_lock(&blk->queued_requests_lock);
2841         while (qemu_co_enter_next(&blk->queued_requests,
2842                                   &blk->queued_requests_lock)) {
2843             /* Resume all queued requests */
2844         }
2845         qemu_mutex_unlock(&blk->queued_requests_lock);
2846     }
2847 }
2848
2849 bool blk_register_buf(BlockBackend *blk, void *host, size_t size, Error **errp)
2850 {
2851     BlockDriverState *bs = blk_bs(blk);
2852
2853     GLOBAL_STATE_CODE();
2854
2855     if (bs) {
2856         return bdrv_register_buf(bs, host, size, errp);
2857     }
2858     return true;
2859 }
2860
2861 void blk_unregister_buf(BlockBackend *blk, void *host, size_t size)
2862 {
2863     BlockDriverState *bs = blk_bs(blk);
2864
2865     GLOBAL_STATE_CODE();
2866
2867     if (bs) {
2868         bdrv_unregister_buf(bs, host, size);
2869     }
2870 }
2871
2872 int coroutine_fn blk_co_copy_range(BlockBackend *blk_in, int64_t off_in,
2873                                    BlockBackend *blk_out, int64_t off_out,
2874                                    int64_t bytes, BdrvRequestFlags read_flags,
2875                                    BdrvRequestFlags write_flags)
2876 {
2877     int r;
2878     IO_CODE();
2879     GRAPH_RDLOCK_GUARD();
2880
2881     r = blk_check_byte_request(blk_in, off_in, bytes);
2882     if (r) {
2883         return r;
2884     }
2885     r = blk_check_byte_request(blk_out, off_out, bytes);
2886     if (r) {
2887         return r;
2888     }
2889
2890     return bdrv_co_copy_range(blk_in->root, off_in,
2891                               blk_out->root, off_out,
2892                               bytes, read_flags, write_flags);
2893 }
2894
2895 const BdrvChild *blk_root(BlockBackend *blk)
2896 {
2897     GLOBAL_STATE_CODE();
2898     return blk->root;
2899 }
2900
2901 int blk_make_empty(BlockBackend *blk, Error **errp)
2902 {
2903     GLOBAL_STATE_CODE();
2904     if (!blk_is_available(blk)) {
2905         error_setg(errp, "No medium inserted");
2906         return -ENOMEDIUM;
2907     }
2908
2909     return bdrv_make_empty(blk->root, errp);
2910 }