block/io.c

   1 /*
   2  * Block layer I/O functions
   3  *
   4  * Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24
  25 #include "qemu/osdep.h"
  26 #include "trace.h"
  27 #include "sysemu/block-backend.h"
  28 #include "block/aio-wait.h"
  29 #include "block/blockjob.h"
  30 #include "block/blockjob_int.h"
  31 #include "block/block_int.h"
  32 #include "block/coroutines.h"
  33 #include "block/dirty-bitmap.h"
  34 #include "block/write-threshold.h"
  35 #include "qemu/cutils.h"
  36 #include "qemu/memalign.h"
  37 #include "qapi/error.h"
  38 #include "qemu/error-report.h"
  39 #include "qemu/main-loop.h"
  40 #include "sysemu/replay.h"
  41
  42 /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */
  43 #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS)
  44
  45 static void bdrv_parent_cb_resize(BlockDriverState *bs);
  46 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
  47     int64_t offset, int64_t bytes, BdrvRequestFlags flags);
  48
  49 static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore)
  50 {
  51     BdrvChild *c, *next;
  52
  53     QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
  54         if (c == ignore) {
  55             continue;
  56         }
  57         bdrv_parent_drained_begin_single(c);
  58     }
  59 }
  60
  61 void bdrv_parent_drained_end_single(BdrvChild *c)
  62 {
  63     IO_OR_GS_CODE();
  64
  65     assert(c->quiesced_parent);
  66     c->quiesced_parent = false;
  67
  68     if (c->klass->drained_end) {
  69         c->klass->drained_end(c);
  70     }
  71 }
  72
  73 static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore)
  74 {
  75     BdrvChild *c;
  76
  77     QLIST_FOREACH(c, &bs->parents, next_parent) {
  78         if (c == ignore) {
  79             continue;
  80         }
  81         bdrv_parent_drained_end_single(c);
  82     }
  83 }
  84
  85 bool bdrv_parent_drained_poll_single(BdrvChild *c)
  86 {
  87     if (c->klass->drained_poll) {
  88         return c->klass->drained_poll(c);
  89     }
  90     return false;
  91 }
  92
  93 static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore,
  94                                      bool ignore_bds_parents)
  95 {
  96     BdrvChild *c, *next;
  97     bool busy = false;
  98
  99     QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
 100         if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
 101             continue;
 102         }
 103         busy |= bdrv_parent_drained_poll_single(c);
 104     }
 105
 106     return busy;
 107 }
 108
 109 void bdrv_parent_drained_begin_single(BdrvChild *c)
 110 {
 111     IO_OR_GS_CODE();
 112
 113     assert(!c->quiesced_parent);
 114     c->quiesced_parent = true;
 115
 116     if (c->klass->drained_begin) {
 117         c->klass->drained_begin(c);
 118     }
 119 }
 120
 121 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src)
 122 {
 123     dst->pdiscard_alignment = MAX(dst->pdiscard_alignment,
 124                                   src->pdiscard_alignment);
 125     dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer);
 126     dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer);
 127     dst->max_hw_transfer = MIN_NON_ZERO(dst->max_hw_transfer,
 128                                         src->max_hw_transfer);
 129     dst->opt_mem_alignment = MAX(dst->opt_mem_alignment,
 130                                  src->opt_mem_alignment);
 131     dst->min_mem_alignment = MAX(dst->min_mem_alignment,
 132                                  src->min_mem_alignment);
 133     dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov);
 134     dst->max_hw_iov = MIN_NON_ZERO(dst->max_hw_iov, src->max_hw_iov);
 135 }
 136
 137 typedef struct BdrvRefreshLimitsState {
 138     BlockDriverState *bs;
 139     BlockLimits old_bl;
 140 } BdrvRefreshLimitsState;
 141
 142 static void bdrv_refresh_limits_abort(void *opaque)
 143 {
 144     BdrvRefreshLimitsState *s = opaque;
 145
 146     s->bs->bl = s->old_bl;
 147 }
 148
 149 static TransactionActionDrv bdrv_refresh_limits_drv = {
 150     .abort = bdrv_refresh_limits_abort,
 151     .clean = g_free,
 152 };
 153
 154 /* @tran is allowed to be NULL, in this case no rollback is possible. */
 155 void bdrv_refresh_limits(BlockDriverState *bs, Transaction *tran, Error **errp)
 156 {
 157     ERRP_GUARD();
 158     BlockDriver *drv = bs->drv;
 159     BdrvChild *c;
 160     bool have_limits;
 161
 162     GLOBAL_STATE_CODE();
 163
 164     if (tran) {
 165         BdrvRefreshLimitsState *s = g_new(BdrvRefreshLimitsState, 1);
 166         *s = (BdrvRefreshLimitsState) {
 167             .bs = bs,
 168             .old_bl = bs->bl,
 169         };
 170         tran_add(tran, &bdrv_refresh_limits_drv, s);
 171     }
 172
 173     memset(&bs->bl, 0, sizeof(bs->bl));
 174
 175     if (!drv) {
 176         return;
 177     }
 178
 179     /* Default alignment based on whether driver has byte interface */
 180     bs->bl.request_alignment = (drv->bdrv_co_preadv ||
 181                                 drv->bdrv_aio_preadv ||
 182                                 drv->bdrv_co_preadv_part) ? 1 : 512;
 183
 184     /* Take some limits from the children as a default */
 185     have_limits = false;
 186     QLIST_FOREACH(c, &bs->children, next) {
 187         if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW))
 188         {
 189             bdrv_merge_limits(&bs->bl, &c->bs->bl);
 190             have_limits = true;
 191         }
 192     }
 193
 194     if (!have_limits) {
 195         bs->bl.min_mem_alignment = 512;
 196         bs->bl.opt_mem_alignment = qemu_real_host_page_size();
 197
 198         /* Safe default since most protocols use readv()/writev()/etc */
 199         bs->bl.max_iov = IOV_MAX;
 200     }
 201
 202     /* Then let the driver override it */
 203     if (drv->bdrv_refresh_limits) {
 204         drv->bdrv_refresh_limits(bs, errp);
 205         if (*errp) {
 206             return;
 207         }
 208     }
 209
 210     if (bs->bl.request_alignment > BDRV_MAX_ALIGNMENT) {
 211         error_setg(errp, "Driver requires too large request alignment");
 212     }
 213 }
 214
 215 /**
 216  * The copy-on-read flag is actually a reference count so multiple users may
 217  * use the feature without worrying about clobbering its previous state.
 218  * Copy-on-read stays enabled until all users have called to disable it.
 219  */
 220 void bdrv_enable_copy_on_read(BlockDriverState *bs)
 221 {
 222     IO_CODE();
 223     qatomic_inc(&bs->copy_on_read);
 224 }
 225
 226 void bdrv_disable_copy_on_read(BlockDriverState *bs)
 227 {
 228     int old = qatomic_fetch_dec(&bs->copy_on_read);
 229     IO_CODE();
 230     assert(old >= 1);
 231 }
 232
 233 typedef struct {
 234     Coroutine *co;
 235     BlockDriverState *bs;
 236     bool done;
 237     bool begin;
 238     bool poll;
 239     BdrvChild *parent;
 240 } BdrvCoDrainData;
 241
 242 /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */
 243 bool bdrv_drain_poll(BlockDriverState *bs, BdrvChild *ignore_parent,
 244                      bool ignore_bds_parents)
 245 {
 246     IO_OR_GS_CODE();
 247
 248     if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) {
 249         return true;
 250     }
 251
 252     if (qatomic_read(&bs->in_flight)) {
 253         return true;
 254     }
 255
 256     return false;
 257 }
 258
 259 static bool bdrv_drain_poll_top_level(BlockDriverState *bs,
 260                                       BdrvChild *ignore_parent)
 261 {
 262     return bdrv_drain_poll(bs, ignore_parent, false);
 263 }
 264
 265 static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent,
 266                                   bool poll);
 267 static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent);
 268
 269 static void bdrv_co_drain_bh_cb(void *opaque)
 270 {
 271     BdrvCoDrainData *data = opaque;
 272     Coroutine *co = data->co;
 273     BlockDriverState *bs = data->bs;
 274
 275     if (bs) {
 276         AioContext *ctx = bdrv_get_aio_context(bs);
 277         aio_context_acquire(ctx);
 278         bdrv_dec_in_flight(bs);
 279         if (data->begin) {
 280             bdrv_do_drained_begin(bs, data->parent, data->poll);
 281         } else {
 282             assert(!data->poll);
 283             bdrv_do_drained_end(bs, data->parent);
 284         }
 285         aio_context_release(ctx);
 286     } else {
 287         assert(data->begin);
 288         bdrv_drain_all_begin();
 289     }
 290
 291     data->done = true;
 292     aio_co_wake(co);
 293 }
 294
 295 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
 296                                                 bool begin,
 297                                                 BdrvChild *parent,
 298                                                 bool poll)
 299 {
 300     BdrvCoDrainData data;
 301     Coroutine *self = qemu_coroutine_self();
 302     AioContext *ctx = bdrv_get_aio_context(bs);
 303     AioContext *co_ctx = qemu_coroutine_get_aio_context(self);
 304
 305     /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
 306      * other coroutines run if they were queued by aio_co_enter(). */
 307
 308     assert(qemu_in_coroutine());
 309     data = (BdrvCoDrainData) {
 310         .co = self,
 311         .bs = bs,
 312         .done = false,
 313         .begin = begin,
 314         .parent = parent,
 315         .poll = poll,
 316     };
 317
 318     if (bs) {
 319         bdrv_inc_in_flight(bs);
 320     }
 321
 322     /*
 323      * Temporarily drop the lock across yield or we would get deadlocks.
 324      * bdrv_co_drain_bh_cb() reaquires the lock as needed.
 325      *
 326      * When we yield below, the lock for the current context will be
 327      * released, so if this is actually the lock that protects bs, don't drop
 328      * it a second time.
 329      */
 330     if (ctx != co_ctx) {
 331         aio_context_release(ctx);
 332     }
 333     replay_bh_schedule_oneshot_event(ctx, bdrv_co_drain_bh_cb, &data);
 334
 335     qemu_coroutine_yield();
 336     /* If we are resumed from some other event (such as an aio completion or a
 337      * timer callback), it is a bug in the caller that should be fixed. */
 338     assert(data.done);
 339
 340     /* Reaquire the AioContext of bs if we dropped it */
 341     if (ctx != co_ctx) {
 342         aio_context_acquire(ctx);
 343     }
 344 }
 345
 346 static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent,
 347                                   bool poll)
 348 {
 349     IO_OR_GS_CODE();
 350
 351     if (qemu_in_coroutine()) {
 352         bdrv_co_yield_to_drain(bs, true, parent, poll);
 353         return;
 354     }
 355
 356     /* Stop things in parent-to-child order */
 357     if (qatomic_fetch_inc(&bs->quiesce_counter) == 0) {
 358         aio_disable_external(bdrv_get_aio_context(bs));
 359         bdrv_parent_drained_begin(bs, parent);
 360         if (bs->drv && bs->drv->bdrv_drain_begin) {
 361             bs->drv->bdrv_drain_begin(bs);
 362         }
 363     }
 364
 365     /*
 366      * Wait for drained requests to finish.
 367      *
 368      * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The
 369      * call is needed so things in this AioContext can make progress even
 370      * though we don't return to the main AioContext loop - this automatically
 371      * includes other nodes in the same AioContext and therefore all child
 372      * nodes.
 373      */
 374     if (poll) {
 375         BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, parent));
 376     }
 377 }
 378
 379 void bdrv_do_drained_begin_quiesce(BlockDriverState *bs, BdrvChild *parent)
 380 {
 381     bdrv_do_drained_begin(bs, parent, false);
 382 }
 383
 384 void bdrv_drained_begin(BlockDriverState *bs)
 385 {
 386     IO_OR_GS_CODE();
 387     bdrv_do_drained_begin(bs, NULL, true);
 388 }
 389
 390 /**
 391  * This function does not poll, nor must any of its recursively called
 392  * functions.
 393  */
 394 static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent)
 395 {
 396     int old_quiesce_counter;
 397
 398     if (qemu_in_coroutine()) {
 399         bdrv_co_yield_to_drain(bs, false, parent, false);
 400         return;
 401     }
 402     assert(bs->quiesce_counter > 0);
 403
 404     /* Re-enable things in child-to-parent order */
 405     old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter);
 406     if (old_quiesce_counter == 1) {
 407         if (bs->drv && bs->drv->bdrv_drain_end) {
 408             bs->drv->bdrv_drain_end(bs);
 409         }
 410         bdrv_parent_drained_end(bs, parent);
 411         aio_enable_external(bdrv_get_aio_context(bs));
 412     }
 413 }
 414
 415 void bdrv_drained_end(BlockDriverState *bs)
 416 {
 417     IO_OR_GS_CODE();
 418     bdrv_do_drained_end(bs, NULL);
 419 }
 420
 421 void bdrv_drain(BlockDriverState *bs)
 422 {
 423     IO_OR_GS_CODE();
 424     bdrv_drained_begin(bs);
 425     bdrv_drained_end(bs);
 426 }
 427
 428 static void bdrv_drain_assert_idle(BlockDriverState *bs)
 429 {
 430     BdrvChild *child, *next;
 431
 432     assert(qatomic_read(&bs->in_flight) == 0);
 433     QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
 434         bdrv_drain_assert_idle(child->bs);
 435     }
 436 }
 437
 438 unsigned int bdrv_drain_all_count = 0;
 439
 440 static bool bdrv_drain_all_poll(void)
 441 {
 442     BlockDriverState *bs = NULL;
 443     bool result = false;
 444     GLOBAL_STATE_CODE();
 445
 446     /* bdrv_drain_poll() can't make changes to the graph and we are holding the
 447      * main AioContext lock, so iterating bdrv_next_all_states() is safe. */
 448     while ((bs = bdrv_next_all_states(bs))) {
 449         AioContext *aio_context = bdrv_get_aio_context(bs);
 450         aio_context_acquire(aio_context);
 451         result |= bdrv_drain_poll(bs, NULL, true);
 452         aio_context_release(aio_context);
 453     }
 454
 455     return result;
 456 }
 457
 458 /*
 459  * Wait for pending requests to complete across all BlockDriverStates
 460  *
 461  * This function does not flush data to disk, use bdrv_flush_all() for that
 462  * after calling this function.
 463  *
 464  * This pauses all block jobs and disables external clients. It must
 465  * be paired with bdrv_drain_all_end().
 466  *
 467  * NOTE: no new block jobs or BlockDriverStates can be created between
 468  * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls.
 469  */
 470 void bdrv_drain_all_begin_nopoll(void)
 471 {
 472     BlockDriverState *bs = NULL;
 473     GLOBAL_STATE_CODE();
 474
 475     /*
 476      * bdrv queue is managed by record/replay,
 477      * waiting for finishing the I/O requests may
 478      * be infinite
 479      */
 480     if (replay_events_enabled()) {
 481         return;
 482     }
 483
 484     /* AIO_WAIT_WHILE() with a NULL context can only be called from the main
 485      * loop AioContext, so make sure we're in the main context. */
 486     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
 487     assert(bdrv_drain_all_count < INT_MAX);
 488     bdrv_drain_all_count++;
 489
 490     /* Quiesce all nodes, without polling in-flight requests yet. The graph
 491      * cannot change during this loop. */
 492     while ((bs = bdrv_next_all_states(bs))) {
 493         AioContext *aio_context = bdrv_get_aio_context(bs);
 494
 495         aio_context_acquire(aio_context);
 496         bdrv_do_drained_begin(bs, NULL, false);
 497         aio_context_release(aio_context);
 498     }
 499 }
 500
 501 void bdrv_drain_all_begin(void)
 502 {
 503     BlockDriverState *bs = NULL;
 504
 505     if (qemu_in_coroutine()) {
 506         bdrv_co_yield_to_drain(NULL, true, NULL, true);
 507         return;
 508     }
 509
 510     /*
 511      * bdrv queue is managed by record/replay,
 512      * waiting for finishing the I/O requests may
 513      * be infinite
 514      */
 515     if (replay_events_enabled()) {
 516         return;
 517     }
 518
 519     bdrv_drain_all_begin_nopoll();
 520
 521     /* Now poll the in-flight requests */
 522     AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll());
 523
 524     while ((bs = bdrv_next_all_states(bs))) {
 525         bdrv_drain_assert_idle(bs);
 526     }
 527 }
 528
 529 void bdrv_drain_all_end_quiesce(BlockDriverState *bs)
 530 {
 531     GLOBAL_STATE_CODE();
 532
 533     g_assert(bs->quiesce_counter > 0);
 534     g_assert(!bs->refcnt);
 535
 536     while (bs->quiesce_counter) {
 537         bdrv_do_drained_end(bs, NULL);
 538     }
 539 }
 540
 541 void bdrv_drain_all_end(void)
 542 {
 543     BlockDriverState *bs = NULL;
 544     GLOBAL_STATE_CODE();
 545
 546     /*
 547      * bdrv queue is managed by record/replay,
 548      * waiting for finishing the I/O requests may
 549      * be endless
 550      */
 551     if (replay_events_enabled()) {
 552         return;
 553     }
 554
 555     while ((bs = bdrv_next_all_states(bs))) {
 556         AioContext *aio_context = bdrv_get_aio_context(bs);
 557
 558         aio_context_acquire(aio_context);
 559         bdrv_do_drained_end(bs, NULL);
 560         aio_context_release(aio_context);
 561     }
 562
 563     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
 564     assert(bdrv_drain_all_count > 0);
 565     bdrv_drain_all_count--;
 566 }
 567
 568 void bdrv_drain_all(void)
 569 {
 570     GLOBAL_STATE_CODE();
 571     bdrv_drain_all_begin();
 572     bdrv_drain_all_end();
 573 }
 574
 575 /**
 576  * Remove an active request from the tracked requests list
 577  *
 578  * This function should be called when a tracked request is completing.
 579  */
 580 static void coroutine_fn tracked_request_end(BdrvTrackedRequest *req)
 581 {
 582     if (req->serialising) {
 583         qatomic_dec(&req->bs->serialising_in_flight);
 584     }
 585
 586     qemu_co_mutex_lock(&req->bs->reqs_lock);
 587     QLIST_REMOVE(req, list);
 588     qemu_co_queue_restart_all(&req->wait_queue);
 589     qemu_co_mutex_unlock(&req->bs->reqs_lock);
 590 }
 591
 592 /**
 593  * Add an active request to the tracked requests list
 594  */
 595 static void coroutine_fn tracked_request_begin(BdrvTrackedRequest *req,
 596                                                BlockDriverState *bs,
 597                                                int64_t offset,
 598                                                int64_t bytes,
 599                                                enum BdrvTrackedRequestType type)
 600 {
 601     bdrv_check_request(offset, bytes, &error_abort);
 602
 603     *req = (BdrvTrackedRequest){
 604         .bs = bs,
 605         .offset         = offset,
 606         .bytes          = bytes,
 607         .type           = type,
 608         .co             = qemu_coroutine_self(),
 609         .serialising    = false,
 610         .overlap_offset = offset,
 611         .overlap_bytes  = bytes,
 612     };
 613
 614     qemu_co_queue_init(&req->wait_queue);
 615
 616     qemu_co_mutex_lock(&bs->reqs_lock);
 617     QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
 618     qemu_co_mutex_unlock(&bs->reqs_lock);
 619 }
 620
 621 static bool tracked_request_overlaps(BdrvTrackedRequest *req,
 622                                      int64_t offset, int64_t bytes)
 623 {
 624     bdrv_check_request(offset, bytes, &error_abort);
 625
 626     /*        aaaa   bbbb */
 627     if (offset >= req->overlap_offset + req->overlap_bytes) {
 628         return false;
 629     }
 630     /* bbbb   aaaa        */
 631     if (req->overlap_offset >= offset + bytes) {
 632         return false;
 633     }
 634     return true;
 635 }
 636
 637 /* Called with self->bs->reqs_lock held */
 638 static coroutine_fn BdrvTrackedRequest *
 639 bdrv_find_conflicting_request(BdrvTrackedRequest *self)
 640 {
 641     BdrvTrackedRequest *req;
 642
 643     QLIST_FOREACH(req, &self->bs->tracked_requests, list) {
 644         if (req == self || (!req->serialising && !self->serialising)) {
 645             continue;
 646         }
 647         if (tracked_request_overlaps(req, self->overlap_offset,
 648                                      self->overlap_bytes))
 649         {
 650             /*
 651              * Hitting this means there was a reentrant request, for
 652              * example, a block driver issuing nested requests.  This must
 653              * never happen since it means deadlock.
 654              */
 655             assert(qemu_coroutine_self() != req->co);
 656
 657             /*
 658              * If the request is already (indirectly) waiting for us, or
 659              * will wait for us as soon as it wakes up, then just go on
 660              * (instead of producing a deadlock in the former case).
 661              */
 662             if (!req->waiting_for) {
 663                 return req;
 664             }
 665         }
 666     }
 667
 668     return NULL;
 669 }
 670
 671 /* Called with self->bs->reqs_lock held */
 672 static void coroutine_fn
 673 bdrv_wait_serialising_requests_locked(BdrvTrackedRequest *self)
 674 {
 675     BdrvTrackedRequest *req;
 676
 677     while ((req = bdrv_find_conflicting_request(self))) {
 678         self->waiting_for = req;
 679         qemu_co_queue_wait(&req->wait_queue, &self->bs->reqs_lock);
 680         self->waiting_for = NULL;
 681     }
 682 }
 683
 684 /* Called with req->bs->reqs_lock held */
 685 static void tracked_request_set_serialising(BdrvTrackedRequest *req,
 686                                             uint64_t align)
 687 {
 688     int64_t overlap_offset = req->offset & ~(align - 1);
 689     int64_t overlap_bytes =
 690         ROUND_UP(req->offset + req->bytes, align) - overlap_offset;
 691
 692     bdrv_check_request(req->offset, req->bytes, &error_abort);
 693
 694     if (!req->serialising) {
 695         qatomic_inc(&req->bs->serialising_in_flight);
 696         req->serialising = true;
 697     }
 698
 699     req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
 700     req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
 701 }
 702
 703 /**
 704  * Return the tracked request on @bs for the current coroutine, or
 705  * NULL if there is none.
 706  */
 707 BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs)
 708 {
 709     BdrvTrackedRequest *req;
 710     Coroutine *self = qemu_coroutine_self();
 711     IO_CODE();
 712
 713     QLIST_FOREACH(req, &bs->tracked_requests, list) {
 714         if (req->co == self) {
 715             return req;
 716         }
 717     }
 718
 719     return NULL;
 720 }
 721
 722 /**
 723  * Round a region to cluster boundaries
 724  */
 725 void coroutine_fn bdrv_round_to_clusters(BlockDriverState *bs,
 726                             int64_t offset, int64_t bytes,
 727                             int64_t *cluster_offset,
 728                             int64_t *cluster_bytes)
 729 {
 730     BlockDriverInfo bdi;
 731     IO_CODE();
 732     if (bdrv_co_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
 733         *cluster_offset = offset;
 734         *cluster_bytes = bytes;
 735     } else {
 736         int64_t c = bdi.cluster_size;
 737         *cluster_offset = QEMU_ALIGN_DOWN(offset, c);
 738         *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c);
 739     }
 740 }
 741
 742 static coroutine_fn int bdrv_get_cluster_size(BlockDriverState *bs)
 743 {
 744     BlockDriverInfo bdi;
 745     int ret;
 746
 747     ret = bdrv_co_get_info(bs, &bdi);
 748     if (ret < 0 || bdi.cluster_size == 0) {
 749         return bs->bl.request_alignment;
 750     } else {
 751         return bdi.cluster_size;
 752     }
 753 }
 754
 755 void bdrv_inc_in_flight(BlockDriverState *bs)
 756 {
 757     IO_CODE();
 758     qatomic_inc(&bs->in_flight);
 759 }
 760
 761 void bdrv_wakeup(BlockDriverState *bs)
 762 {
 763     IO_CODE();
 764     aio_wait_kick();
 765 }
 766
 767 void bdrv_dec_in_flight(BlockDriverState *bs)
 768 {
 769     IO_CODE();
 770     qatomic_dec(&bs->in_flight);
 771     bdrv_wakeup(bs);
 772 }
 773
 774 static void coroutine_fn
 775 bdrv_wait_serialising_requests(BdrvTrackedRequest *self)
 776 {
 777     BlockDriverState *bs = self->bs;
 778
 779     if (!qatomic_read(&bs->serialising_in_flight)) {
 780         return;
 781     }
 782
 783     qemu_co_mutex_lock(&bs->reqs_lock);
 784     bdrv_wait_serialising_requests_locked(self);
 785     qemu_co_mutex_unlock(&bs->reqs_lock);
 786 }
 787
 788 void coroutine_fn bdrv_make_request_serialising(BdrvTrackedRequest *req,
 789                                                 uint64_t align)
 790 {
 791     IO_CODE();
 792
 793     qemu_co_mutex_lock(&req->bs->reqs_lock);
 794
 795     tracked_request_set_serialising(req, align);
 796     bdrv_wait_serialising_requests_locked(req);
 797
 798     qemu_co_mutex_unlock(&req->bs->reqs_lock);
 799 }
 800
 801 int bdrv_check_qiov_request(int64_t offset, int64_t bytes,
 802                             QEMUIOVector *qiov, size_t qiov_offset,
 803                             Error **errp)
 804 {
 805     /*
 806      * Check generic offset/bytes correctness
 807      */
 808
 809     if (offset < 0) {
 810         error_setg(errp, "offset is negative: %" PRIi64, offset);
 811         return -EIO;
 812     }
 813
 814     if (bytes < 0) {
 815         error_setg(errp, "bytes is negative: %" PRIi64, bytes);
 816         return -EIO;
 817     }
 818
 819     if (bytes > BDRV_MAX_LENGTH) {
 820         error_setg(errp, "bytes(%" PRIi64 ") exceeds maximum(%" PRIi64 ")",
 821                    bytes, BDRV_MAX_LENGTH);
 822         return -EIO;
 823     }
 824
 825     if (offset > BDRV_MAX_LENGTH) {
 826         error_setg(errp, "offset(%" PRIi64 ") exceeds maximum(%" PRIi64 ")",
 827                    offset, BDRV_MAX_LENGTH);
 828         return -EIO;
 829     }
 830
 831     if (offset > BDRV_MAX_LENGTH - bytes) {
 832         error_setg(errp, "sum of offset(%" PRIi64 ") and bytes(%" PRIi64 ") "
 833                    "exceeds maximum(%" PRIi64 ")", offset, bytes,
 834                    BDRV_MAX_LENGTH);
 835         return -EIO;
 836     }
 837
 838     if (!qiov) {
 839         return 0;
 840     }
 841
 842     /*
 843      * Check qiov and qiov_offset
 844      */
 845
 846     if (qiov_offset > qiov->size) {
 847         error_setg(errp, "qiov_offset(%zu) overflow io vector size(%zu)",
 848                    qiov_offset, qiov->size);
 849         return -EIO;
 850     }
 851
 852     if (bytes > qiov->size - qiov_offset) {
 853         error_setg(errp, "bytes(%" PRIi64 ") + qiov_offset(%zu) overflow io "
 854                    "vector size(%zu)", bytes, qiov_offset, qiov->size);
 855         return -EIO;
 856     }
 857
 858     return 0;
 859 }
 860
 861 int bdrv_check_request(int64_t offset, int64_t bytes, Error **errp)
 862 {
 863     return bdrv_check_qiov_request(offset, bytes, NULL, 0, errp);
 864 }
 865
 866 static int bdrv_check_request32(int64_t offset, int64_t bytes,
 867                                 QEMUIOVector *qiov, size_t qiov_offset)
 868 {
 869     int ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL);
 870     if (ret < 0) {
 871         return ret;
 872     }
 873
 874     if (bytes > BDRV_REQUEST_MAX_BYTES) {
 875         return -EIO;
 876     }
 877
 878     return 0;
 879 }
 880
 881 /*
 882  * Completely zero out a block device with the help of bdrv_pwrite_zeroes.
 883  * The operation is sped up by checking the block status and only writing
 884  * zeroes to the device if they currently do not return zeroes. Optional
 885  * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
 886  * BDRV_REQ_FUA).
 887  *
 888  * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite().
 889  */
 890 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
 891 {
 892     int ret;
 893     int64_t target_size, bytes, offset = 0;
 894     BlockDriverState *bs = child->bs;
 895     IO_CODE();
 896
 897     target_size = bdrv_getlength(bs);
 898     if (target_size < 0) {
 899         return target_size;
 900     }
 901
 902     for (;;) {
 903         bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES);
 904         if (bytes <= 0) {
 905             return 0;
 906         }
 907         ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL);
 908         if (ret < 0) {
 909             return ret;
 910         }
 911         if (ret & BDRV_BLOCK_ZERO) {
 912             offset += bytes;
 913             continue;
 914         }
 915         ret = bdrv_pwrite_zeroes(child, offset, bytes, flags);
 916         if (ret < 0) {
 917             return ret;
 918         }
 919         offset += bytes;
 920     }
 921 }
 922
 923 /*
 924  * Writes to the file and ensures that no writes are reordered across this
 925  * request (acts as a barrier)
 926  *
 927  * Returns 0 on success, -errno in error cases.
 928  */
 929 int coroutine_fn bdrv_co_pwrite_sync(BdrvChild *child, int64_t offset,
 930                                      int64_t bytes, const void *buf,
 931                                      BdrvRequestFlags flags)
 932 {
 933     int ret;
 934     IO_CODE();
 935
 936     ret = bdrv_co_pwrite(child, offset, bytes, buf, flags);
 937     if (ret < 0) {
 938         return ret;
 939     }
 940
 941     ret = bdrv_co_flush(child->bs);
 942     if (ret < 0) {
 943         return ret;
 944     }
 945
 946     return 0;
 947 }
 948
 949 typedef struct CoroutineIOCompletion {
 950     Coroutine *coroutine;
 951     int ret;
 952 } CoroutineIOCompletion;
 953
 954 static void bdrv_co_io_em_complete(void *opaque, int ret)
 955 {
 956     CoroutineIOCompletion *co = opaque;
 957
 958     co->ret = ret;
 959     aio_co_wake(co->coroutine);
 960 }
 961
 962 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
 963                                            int64_t offset, int64_t bytes,
 964                                            QEMUIOVector *qiov,
 965                                            size_t qiov_offset, int flags)
 966 {
 967     BlockDriver *drv = bs->drv;
 968     int64_t sector_num;
 969     unsigned int nb_sectors;
 970     QEMUIOVector local_qiov;
 971     int ret;
 972
 973     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
 974     assert(!(flags & ~bs->supported_read_flags));
 975
 976     if (!drv) {
 977         return -ENOMEDIUM;
 978     }
 979
 980     if (drv->bdrv_co_preadv_part) {
 981         return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset,
 982                                         flags);
 983     }
 984
 985     if (qiov_offset > 0 || bytes != qiov->size) {
 986         qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
 987         qiov = &local_qiov;
 988     }
 989
 990     if (drv->bdrv_co_preadv) {
 991         ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
 992         goto out;
 993     }
 994
 995     if (drv->bdrv_aio_preadv) {
 996         BlockAIOCB *acb;
 997         CoroutineIOCompletion co = {
 998             .coroutine = qemu_coroutine_self(),
 999         };
1000
1001         acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags,
1002                                    bdrv_co_io_em_complete, &co);
1003         if (acb == NULL) {
1004             ret = -EIO;
1005             goto out;
1006         } else {
1007             qemu_coroutine_yield();
1008             ret = co.ret;
1009             goto out;
1010         }
1011     }
1012
1013     sector_num = offset >> BDRV_SECTOR_BITS;
1014     nb_sectors = bytes >> BDRV_SECTOR_BITS;
1015
1016     assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
1017     assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
1018     assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1019     assert(drv->bdrv_co_readv);
1020
1021     ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
1022
1023 out:
1024     if (qiov == &local_qiov) {
1025         qemu_iovec_destroy(&local_qiov);
1026     }
1027
1028     return ret;
1029 }
1030
1031 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
1032                                             int64_t offset, int64_t bytes,
1033                                             QEMUIOVector *qiov,
1034                                             size_t qiov_offset,
1035                                             BdrvRequestFlags flags)
1036 {
1037     BlockDriver *drv = bs->drv;
1038     bool emulate_fua = false;
1039     int64_t sector_num;
1040     unsigned int nb_sectors;
1041     QEMUIOVector local_qiov;
1042     int ret;
1043
1044     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1045
1046     if (!drv) {
1047         return -ENOMEDIUM;
1048     }
1049
1050     if ((flags & BDRV_REQ_FUA) &&
1051         (~bs->supported_write_flags & BDRV_REQ_FUA)) {
1052         flags &= ~BDRV_REQ_FUA;
1053         emulate_fua = true;
1054     }
1055
1056     flags &= bs->supported_write_flags;
1057
1058     if (drv->bdrv_co_pwritev_part) {
1059         ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset,
1060                                         flags);
1061         goto emulate_flags;
1062     }
1063
1064     if (qiov_offset > 0 || bytes != qiov->size) {
1065         qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1066         qiov = &local_qiov;
1067     }
1068
1069     if (drv->bdrv_co_pwritev) {
1070         ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, flags);
1071         goto emulate_flags;
1072     }
1073
1074     if (drv->bdrv_aio_pwritev) {
1075         BlockAIOCB *acb;
1076         CoroutineIOCompletion co = {
1077             .coroutine = qemu_coroutine_self(),
1078         };
1079
1080         acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov, flags,
1081                                     bdrv_co_io_em_complete, &co);
1082         if (acb == NULL) {
1083             ret = -EIO;
1084         } else {
1085             qemu_coroutine_yield();
1086             ret = co.ret;
1087         }
1088         goto emulate_flags;
1089     }
1090
1091     sector_num = offset >> BDRV_SECTOR_BITS;
1092     nb_sectors = bytes >> BDRV_SECTOR_BITS;
1093
1094     assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
1095     assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
1096     assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1097
1098     assert(drv->bdrv_co_writev);
1099     ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov, flags);
1100
1101 emulate_flags:
1102     if (ret == 0 && emulate_fua) {
1103         ret = bdrv_co_flush(bs);
1104     }
1105
1106     if (qiov == &local_qiov) {
1107         qemu_iovec_destroy(&local_qiov);
1108     }
1109
1110     return ret;
1111 }
1112
1113 static int coroutine_fn
1114 bdrv_driver_pwritev_compressed(BlockDriverState *bs, int64_t offset,
1115                                int64_t bytes, QEMUIOVector *qiov,
1116                                size_t qiov_offset)
1117 {
1118     BlockDriver *drv = bs->drv;
1119     QEMUIOVector local_qiov;
1120     int ret;
1121
1122     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1123
1124     if (!drv) {
1125         return -ENOMEDIUM;
1126     }
1127
1128     if (!block_driver_can_compress(drv)) {
1129         return -ENOTSUP;
1130     }
1131
1132     if (drv->bdrv_co_pwritev_compressed_part) {
1133         return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes,
1134                                                     qiov, qiov_offset);
1135     }
1136
1137     if (qiov_offset == 0) {
1138         return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov);
1139     }
1140
1141     qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1142     ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov);
1143     qemu_iovec_destroy(&local_qiov);
1144
1145     return ret;
1146 }
1147
1148 static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
1149         int64_t offset, int64_t bytes, QEMUIOVector *qiov,
1150         size_t qiov_offset, int flags)
1151 {
1152     BlockDriverState *bs = child->bs;
1153
1154     /* Perform I/O through a temporary buffer so that users who scribble over
1155      * their read buffer while the operation is in progress do not end up
1156      * modifying the image file.  This is critical for zero-copy guest I/O
1157      * where anything might happen inside guest memory.
1158      */
1159     void *bounce_buffer = NULL;
1160
1161     BlockDriver *drv = bs->drv;
1162     int64_t cluster_offset;
1163     int64_t cluster_bytes;
1164     int64_t skip_bytes;
1165     int ret;
1166     int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer,
1167                                     BDRV_REQUEST_MAX_BYTES);
1168     int64_t progress = 0;
1169     bool skip_write;
1170
1171     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1172
1173     if (!drv) {
1174         return -ENOMEDIUM;
1175     }
1176
1177     /*
1178      * Do not write anything when the BDS is inactive.  That is not
1179      * allowed, and it would not help.
1180      */
1181     skip_write = (bs->open_flags & BDRV_O_INACTIVE);
1182
1183     /* FIXME We cannot require callers to have write permissions when all they
1184      * are doing is a read request. If we did things right, write permissions
1185      * would be obtained anyway, but internally by the copy-on-read code. As
1186      * long as it is implemented here rather than in a separate filter driver,
1187      * the copy-on-read code doesn't have its own BdrvChild, however, for which
1188      * it could request permissions. Therefore we have to bypass the permission
1189      * system for the moment. */
1190     // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1191
1192     /* Cover entire cluster so no additional backing file I/O is required when
1193      * allocating cluster in the image file.  Note that this value may exceed
1194      * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which
1195      * is one reason we loop rather than doing it all at once.
1196      */
1197     bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes);
1198     skip_bytes = offset - cluster_offset;
1199
1200     trace_bdrv_co_do_copy_on_readv(bs, offset, bytes,
1201                                    cluster_offset, cluster_bytes);
1202
1203     while (cluster_bytes) {
1204         int64_t pnum;
1205
1206         if (skip_write) {
1207             ret = 1; /* "already allocated", so nothing will be copied */
1208             pnum = MIN(cluster_bytes, max_transfer);
1209         } else {
1210             ret = bdrv_is_allocated(bs, cluster_offset,
1211                                     MIN(cluster_bytes, max_transfer), &pnum);
1212             if (ret < 0) {
1213                 /*
1214                  * Safe to treat errors in querying allocation as if
1215                  * unallocated; we'll probably fail again soon on the
1216                  * read, but at least that will set a decent errno.
1217                  */
1218                 pnum = MIN(cluster_bytes, max_transfer);
1219             }
1220
1221             /* Stop at EOF if the image ends in the middle of the cluster */
1222             if (ret == 0 && pnum == 0) {
1223                 assert(progress >= bytes);
1224                 break;
1225             }
1226
1227             assert(skip_bytes < pnum);
1228         }
1229
1230         if (ret <= 0) {
1231             QEMUIOVector local_qiov;
1232
1233             /* Must copy-on-read; use the bounce buffer */
1234             pnum = MIN(pnum, MAX_BOUNCE_BUFFER);
1235             if (!bounce_buffer) {
1236                 int64_t max_we_need = MAX(pnum, cluster_bytes - pnum);
1237                 int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER);
1238                 int64_t bounce_buffer_len = MIN(max_we_need, max_allowed);
1239
1240                 bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len);
1241                 if (!bounce_buffer) {
1242                     ret = -ENOMEM;
1243                     goto err;
1244                 }
1245             }
1246             qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum);
1247
1248             ret = bdrv_driver_preadv(bs, cluster_offset, pnum,
1249                                      &local_qiov, 0, 0);
1250             if (ret < 0) {
1251                 goto err;
1252             }
1253
1254             bdrv_co_debug_event(bs, BLKDBG_COR_WRITE);
1255             if (drv->bdrv_co_pwrite_zeroes &&
1256                 buffer_is_zero(bounce_buffer, pnum)) {
1257                 /* FIXME: Should we (perhaps conditionally) be setting
1258                  * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy
1259                  * that still correctly reads as zero? */
1260                 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum,
1261                                                BDRV_REQ_WRITE_UNCHANGED);
1262             } else {
1263                 /* This does not change the data on the disk, it is not
1264                  * necessary to flush even in cache=writethrough mode.
1265                  */
1266                 ret = bdrv_driver_pwritev(bs, cluster_offset, pnum,
1267                                           &local_qiov, 0,
1268                                           BDRV_REQ_WRITE_UNCHANGED);
1269             }
1270
1271             if (ret < 0) {
1272                 /* It might be okay to ignore write errors for guest
1273                  * requests.  If this is a deliberate copy-on-read
1274                  * then we don't want to ignore the error.  Simply
1275                  * report it in all cases.
1276                  */
1277                 goto err;
1278             }
1279
1280             if (!(flags & BDRV_REQ_PREFETCH)) {
1281                 qemu_iovec_from_buf(qiov, qiov_offset + progress,
1282                                     bounce_buffer + skip_bytes,
1283                                     MIN(pnum - skip_bytes, bytes - progress));
1284             }
1285         } else if (!(flags & BDRV_REQ_PREFETCH)) {
1286             /* Read directly into the destination */
1287             ret = bdrv_driver_preadv(bs, offset + progress,
1288                                      MIN(pnum - skip_bytes, bytes - progress),
1289                                      qiov, qiov_offset + progress, 0);
1290             if (ret < 0) {
1291                 goto err;
1292             }
1293         }
1294
1295         cluster_offset += pnum;
1296         cluster_bytes -= pnum;
1297         progress += pnum - skip_bytes;
1298         skip_bytes = 0;
1299     }
1300     ret = 0;
1301
1302 err:
1303     qemu_vfree(bounce_buffer);
1304     return ret;
1305 }
1306
1307 /*
1308  * Forwards an already correctly aligned request to the BlockDriver. This
1309  * handles copy on read, zeroing after EOF, and fragmentation of large
1310  * reads; any other features must be implemented by the caller.
1311  */
1312 static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
1313     BdrvTrackedRequest *req, int64_t offset, int64_t bytes,
1314     int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags)
1315 {
1316     BlockDriverState *bs = child->bs;
1317     int64_t total_bytes, max_bytes;
1318     int ret = 0;
1319     int64_t bytes_remaining = bytes;
1320     int max_transfer;
1321
1322     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1323     assert(is_power_of_2(align));
1324     assert((offset & (align - 1)) == 0);
1325     assert((bytes & (align - 1)) == 0);
1326     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1327     max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
1328                                    align);
1329
1330     /*
1331      * TODO: We would need a per-BDS .supported_read_flags and
1332      * potential fallback support, if we ever implement any read flags
1333      * to pass through to drivers.  For now, there aren't any
1334      * passthrough flags except the BDRV_REQ_REGISTERED_BUF optimization hint.
1335      */
1336     assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH |
1337                        BDRV_REQ_REGISTERED_BUF)));
1338
1339     /* Handle Copy on Read and associated serialisation */
1340     if (flags & BDRV_REQ_COPY_ON_READ) {
1341         /* If we touch the same cluster it counts as an overlap.  This
1342          * guarantees that allocating writes will be serialized and not race
1343          * with each other for the same cluster.  For example, in copy-on-read
1344          * it ensures that the CoR read and write operations are atomic and
1345          * guest writes cannot interleave between them. */
1346         bdrv_make_request_serialising(req, bdrv_get_cluster_size(bs));
1347     } else {
1348         bdrv_wait_serialising_requests(req);
1349     }
1350
1351     if (flags & BDRV_REQ_COPY_ON_READ) {
1352         int64_t pnum;
1353
1354         /* The flag BDRV_REQ_COPY_ON_READ has reached its addressee */
1355         flags &= ~BDRV_REQ_COPY_ON_READ;
1356
1357         ret = bdrv_is_allocated(bs, offset, bytes, &pnum);
1358         if (ret < 0) {
1359             goto out;
1360         }
1361
1362         if (!ret || pnum != bytes) {
1363             ret = bdrv_co_do_copy_on_readv(child, offset, bytes,
1364                                            qiov, qiov_offset, flags);
1365             goto out;
1366         } else if (flags & BDRV_REQ_PREFETCH) {
1367             goto out;
1368         }
1369     }
1370
1371     /* Forward the request to the BlockDriver, possibly fragmenting it */
1372     total_bytes = bdrv_getlength(bs);
1373     if (total_bytes < 0) {
1374         ret = total_bytes;
1375         goto out;
1376     }
1377
1378     assert(!(flags & ~(bs->supported_read_flags | BDRV_REQ_REGISTERED_BUF)));
1379
1380     max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align);
1381     if (bytes <= max_bytes && bytes <= max_transfer) {
1382         ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, flags);
1383         goto out;
1384     }
1385
1386     while (bytes_remaining) {
1387         int64_t num;
1388
1389         if (max_bytes) {
1390             num = MIN(bytes_remaining, MIN(max_bytes, max_transfer));
1391             assert(num);
1392
1393             ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining,
1394                                      num, qiov,
1395                                      qiov_offset + bytes - bytes_remaining,
1396                                      flags);
1397             max_bytes -= num;
1398         } else {
1399             num = bytes_remaining;
1400             ret = qemu_iovec_memset(qiov, qiov_offset + bytes - bytes_remaining,
1401                                     0, bytes_remaining);
1402         }
1403         if (ret < 0) {
1404             goto out;
1405         }
1406         bytes_remaining -= num;
1407     }
1408
1409 out:
1410     return ret < 0 ? ret : 0;
1411 }
1412
1413 /*
1414  * Request padding
1415  *
1416  *  |<---- align ----->|                     |<----- align ---->|
1417  *  |<- head ->|<------------- bytes ------------->|<-- tail -->|
1418  *  |          |       |                     |     |            |
1419  * -*----------$-------*-------- ... --------*-----$------------*---
1420  *  |          |       |                     |     |            |
1421  *  |          offset  |                     |     end          |
1422  *  ALIGN_DOWN(offset) ALIGN_UP(offset)      ALIGN_DOWN(end)   ALIGN_UP(end)
1423  *  [buf   ... )                             [tail_buf          )
1424  *
1425  * @buf is an aligned allocation needed to store @head and @tail paddings. @head
1426  * is placed at the beginning of @buf and @tail at the @end.
1427  *
1428  * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk
1429  * around tail, if tail exists.
1430  *
1431  * @merge_reads is true for small requests,
1432  * if @buf_len == @head + bytes + @tail. In this case it is possible that both
1433  * head and tail exist but @buf_len == align and @tail_buf == @buf.
1434  */
1435 typedef struct BdrvRequestPadding {
1436     uint8_t *buf;
1437     size_t buf_len;
1438     uint8_t *tail_buf;
1439     size_t head;
1440     size_t tail;
1441     bool merge_reads;
1442     QEMUIOVector local_qiov;
1443 } BdrvRequestPadding;
1444
1445 static bool bdrv_init_padding(BlockDriverState *bs,
1446                               int64_t offset, int64_t bytes,
1447                               BdrvRequestPadding *pad)
1448 {
1449     int64_t align = bs->bl.request_alignment;
1450     int64_t sum;
1451
1452     bdrv_check_request(offset, bytes, &error_abort);
1453     assert(align <= INT_MAX); /* documented in block/block_int.h */
1454     assert(align <= SIZE_MAX / 2); /* so we can allocate the buffer */
1455
1456     memset(pad, 0, sizeof(*pad));
1457
1458     pad->head = offset & (align - 1);
1459     pad->tail = ((offset + bytes) & (align - 1));
1460     if (pad->tail) {
1461         pad->tail = align - pad->tail;
1462     }
1463
1464     if (!pad->head && !pad->tail) {
1465         return false;
1466     }
1467
1468     assert(bytes); /* Nothing good in aligning zero-length requests */
1469
1470     sum = pad->head + bytes + pad->tail;
1471     pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align;
1472     pad->buf = qemu_blockalign(bs, pad->buf_len);
1473     pad->merge_reads = sum == pad->buf_len;
1474     if (pad->tail) {
1475         pad->tail_buf = pad->buf + pad->buf_len - align;
1476     }
1477
1478     return true;
1479 }
1480
1481 static coroutine_fn int bdrv_padding_rmw_read(BdrvChild *child,
1482                                               BdrvTrackedRequest *req,
1483                                               BdrvRequestPadding *pad,
1484                                               bool zero_middle)
1485 {
1486     QEMUIOVector local_qiov;
1487     BlockDriverState *bs = child->bs;
1488     uint64_t align = bs->bl.request_alignment;
1489     int ret;
1490
1491     assert(req->serialising && pad->buf);
1492
1493     if (pad->head || pad->merge_reads) {
1494         int64_t bytes = pad->merge_reads ? pad->buf_len : align;
1495
1496         qemu_iovec_init_buf(&local_qiov, pad->buf, bytes);
1497
1498         if (pad->head) {
1499             bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1500         }
1501         if (pad->merge_reads && pad->tail) {
1502             bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1503         }
1504         ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes,
1505                                   align, &local_qiov, 0, 0);
1506         if (ret < 0) {
1507             return ret;
1508         }
1509         if (pad->head) {
1510             bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1511         }
1512         if (pad->merge_reads && pad->tail) {
1513             bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1514         }
1515
1516         if (pad->merge_reads) {
1517             goto zero_mem;
1518         }
1519     }
1520
1521     if (pad->tail) {
1522         qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align);
1523
1524         bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1525         ret = bdrv_aligned_preadv(
1526                 child, req,
1527                 req->overlap_offset + req->overlap_bytes - align,
1528                 align, align, &local_qiov, 0, 0);
1529         if (ret < 0) {
1530             return ret;
1531         }
1532         bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1533     }
1534
1535 zero_mem:
1536     if (zero_middle) {
1537         memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail);
1538     }
1539
1540     return 0;
1541 }
1542
1543 static void bdrv_padding_destroy(BdrvRequestPadding *pad)
1544 {
1545     if (pad->buf) {
1546         qemu_vfree(pad->buf);
1547         qemu_iovec_destroy(&pad->local_qiov);
1548     }
1549     memset(pad, 0, sizeof(*pad));
1550 }
1551
1552 /*
1553  * bdrv_pad_request
1554  *
1555  * Exchange request parameters with padded request if needed. Don't include RMW
1556  * read of padding, bdrv_padding_rmw_read() should be called separately if
1557  * needed.
1558  *
1559  * Request parameters (@qiov, &qiov_offset, &offset, &bytes) are in-out:
1560  *  - on function start they represent original request
1561  *  - on failure or when padding is not needed they are unchanged
1562  *  - on success when padding is needed they represent padded request
1563  */
1564 static int bdrv_pad_request(BlockDriverState *bs,
1565                             QEMUIOVector **qiov, size_t *qiov_offset,
1566                             int64_t *offset, int64_t *bytes,
1567                             BdrvRequestPadding *pad, bool *padded,
1568                             BdrvRequestFlags *flags)
1569 {
1570     int ret;
1571
1572     bdrv_check_qiov_request(*offset, *bytes, *qiov, *qiov_offset, &error_abort);
1573
1574     if (!bdrv_init_padding(bs, *offset, *bytes, pad)) {
1575         if (padded) {
1576             *padded = false;
1577         }
1578         return 0;
1579     }
1580
1581     ret = qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head,
1582                                    *qiov, *qiov_offset, *bytes,
1583                                    pad->buf + pad->buf_len - pad->tail,
1584                                    pad->tail);
1585     if (ret < 0) {
1586         bdrv_padding_destroy(pad);
1587         return ret;
1588     }
1589     *bytes += pad->head + pad->tail;
1590     *offset -= pad->head;
1591     *qiov = &pad->local_qiov;
1592     *qiov_offset = 0;
1593     if (padded) {
1594         *padded = true;
1595     }
1596     if (flags) {
1597         /* Can't use optimization hint with bounce buffer */
1598         *flags &= ~BDRV_REQ_REGISTERED_BUF;
1599     }
1600
1601     return 0;
1602 }
1603
1604 int coroutine_fn bdrv_co_preadv(BdrvChild *child,
1605     int64_t offset, int64_t bytes, QEMUIOVector *qiov,
1606     BdrvRequestFlags flags)
1607 {
1608     IO_CODE();
1609     return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags);
1610 }
1611
1612 int coroutine_fn bdrv_co_preadv_part(BdrvChild *child,
1613     int64_t offset, int64_t bytes,
1614     QEMUIOVector *qiov, size_t qiov_offset,
1615     BdrvRequestFlags flags)
1616 {
1617     BlockDriverState *bs = child->bs;
1618     BdrvTrackedRequest req;
1619     BdrvRequestPadding pad;
1620     int ret;
1621     IO_CODE();
1622
1623     trace_bdrv_co_preadv_part(bs, offset, bytes, flags);
1624
1625     if (!bdrv_co_is_inserted(bs)) {
1626         return -ENOMEDIUM;
1627     }
1628
1629     ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset);
1630     if (ret < 0) {
1631         return ret;
1632     }
1633
1634     if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
1635         /*
1636          * Aligning zero request is nonsense. Even if driver has special meaning
1637          * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
1638          * it to driver due to request_alignment.
1639          *
1640          * Still, no reason to return an error if someone do unaligned
1641          * zero-length read occasionally.
1642          */
1643         return 0;
1644     }
1645
1646     bdrv_inc_in_flight(bs);
1647
1648     /* Don't do copy-on-read if we read data before write operation */
1649     if (qatomic_read(&bs->copy_on_read)) {
1650         flags |= BDRV_REQ_COPY_ON_READ;
1651     }
1652
1653     ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad,
1654                            NULL, &flags);
1655     if (ret < 0) {
1656         goto fail;
1657     }
1658
1659     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
1660     ret = bdrv_aligned_preadv(child, &req, offset, bytes,
1661                               bs->bl.request_alignment,
1662                               qiov, qiov_offset, flags);
1663     tracked_request_end(&req);
1664     bdrv_padding_destroy(&pad);
1665
1666 fail:
1667     bdrv_dec_in_flight(bs);
1668
1669     return ret;
1670 }
1671
1672 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
1673     int64_t offset, int64_t bytes, BdrvRequestFlags flags)
1674 {
1675     BlockDriver *drv = bs->drv;
1676     QEMUIOVector qiov;
1677     void *buf = NULL;
1678     int ret = 0;
1679     bool need_flush = false;
1680     int head = 0;
1681     int tail = 0;
1682
1683     int64_t max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes,
1684                                             INT64_MAX);
1685     int alignment = MAX(bs->bl.pwrite_zeroes_alignment,
1686                         bs->bl.request_alignment);
1687     int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER);
1688
1689     bdrv_check_request(offset, bytes, &error_abort);
1690
1691     if (!drv) {
1692         return -ENOMEDIUM;
1693     }
1694
1695     if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) {
1696         return -ENOTSUP;
1697     }
1698
1699     /* By definition there is no user buffer so this flag doesn't make sense */
1700     if (flags & BDRV_REQ_REGISTERED_BUF) {
1701         return -EINVAL;
1702     }
1703
1704     /* Invalidate the cached block-status data range if this write overlaps */
1705     bdrv_bsc_invalidate_range(bs, offset, bytes);
1706
1707     assert(alignment % bs->bl.request_alignment == 0);
1708     head = offset % alignment;
1709     tail = (offset + bytes) % alignment;
1710     max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment);
1711     assert(max_write_zeroes >= bs->bl.request_alignment);
1712
1713     while (bytes > 0 && !ret) {
1714         int64_t num = bytes;
1715
1716         /* Align request.  Block drivers can expect the "bulk" of the request
1717          * to be aligned, and that unaligned requests do not cross cluster
1718          * boundaries.
1719          */
1720         if (head) {
1721             /* Make a small request up to the first aligned sector. For
1722              * convenience, limit this request to max_transfer even if
1723              * we don't need to fall back to writes.  */
1724             num = MIN(MIN(bytes, max_transfer), alignment - head);
1725             head = (head + num) % alignment;
1726             assert(num < max_write_zeroes);
1727         } else if (tail && num > alignment) {
1728             /* Shorten the request to the last aligned sector.  */
1729             num -= tail;
1730         }
1731
1732         /* limit request size */
1733         if (num > max_write_zeroes) {
1734             num = max_write_zeroes;
1735         }
1736
1737         ret = -ENOTSUP;
1738         /* First try the efficient write zeroes operation */
1739         if (drv->bdrv_co_pwrite_zeroes) {
1740             ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num,
1741                                              flags & bs->supported_zero_flags);
1742             if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1743                 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1744                 need_flush = true;
1745             }
1746         } else {
1747             assert(!bs->supported_zero_flags);
1748         }
1749
1750         if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) {
1751             /* Fall back to bounce buffer if write zeroes is unsupported */
1752             BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1753
1754             if ((flags & BDRV_REQ_FUA) &&
1755                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1756                 /* No need for bdrv_driver_pwrite() to do a fallback
1757                  * flush on each chunk; use just one at the end */
1758                 write_flags &= ~BDRV_REQ_FUA;
1759                 need_flush = true;
1760             }
1761             num = MIN(num, max_transfer);
1762             if (buf == NULL) {
1763                 buf = qemu_try_blockalign0(bs, num);
1764                 if (buf == NULL) {
1765                     ret = -ENOMEM;
1766                     goto fail;
1767                 }
1768             }
1769             qemu_iovec_init_buf(&qiov, buf, num);
1770
1771             ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags);
1772
1773             /* Keep bounce buffer around if it is big enough for all
1774              * all future requests.
1775              */
1776             if (num < max_transfer) {
1777                 qemu_vfree(buf);
1778                 buf = NULL;
1779             }
1780         }
1781
1782         offset += num;
1783         bytes -= num;
1784     }
1785
1786 fail:
1787     if (ret == 0 && need_flush) {
1788         ret = bdrv_co_flush(bs);
1789     }
1790     qemu_vfree(buf);
1791     return ret;
1792 }
1793
1794 static inline int coroutine_fn
1795 bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, int64_t bytes,
1796                           BdrvTrackedRequest *req, int flags)
1797 {
1798     BlockDriverState *bs = child->bs;
1799
1800     bdrv_check_request(offset, bytes, &error_abort);
1801
1802     if (bdrv_is_read_only(bs)) {
1803         return -EPERM;
1804     }
1805
1806     assert(!(bs->open_flags & BDRV_O_INACTIVE));
1807     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1808     assert(!(flags & ~BDRV_REQ_MASK));
1809     assert(!((flags & BDRV_REQ_NO_WAIT) && !(flags & BDRV_REQ_SERIALISING)));
1810
1811     if (flags & BDRV_REQ_SERIALISING) {
1812         QEMU_LOCK_GUARD(&bs->reqs_lock);
1813
1814         tracked_request_set_serialising(req, bdrv_get_cluster_size(bs));
1815
1816         if ((flags & BDRV_REQ_NO_WAIT) && bdrv_find_conflicting_request(req)) {
1817             return -EBUSY;
1818         }
1819
1820         bdrv_wait_serialising_requests_locked(req);
1821     } else {
1822         bdrv_wait_serialising_requests(req);
1823     }
1824
1825     assert(req->overlap_offset <= offset);
1826     assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1827     assert(offset + bytes <= bs->total_sectors * BDRV_SECTOR_SIZE ||
1828            child->perm & BLK_PERM_RESIZE);
1829
1830     switch (req->type) {
1831     case BDRV_TRACKED_WRITE:
1832     case BDRV_TRACKED_DISCARD:
1833         if (flags & BDRV_REQ_WRITE_UNCHANGED) {
1834             assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1835         } else {
1836             assert(child->perm & BLK_PERM_WRITE);
1837         }
1838         bdrv_write_threshold_check_write(bs, offset, bytes);
1839         return 0;
1840     case BDRV_TRACKED_TRUNCATE:
1841         assert(child->perm & BLK_PERM_RESIZE);
1842         return 0;
1843     default:
1844         abort();
1845     }
1846 }
1847
1848 static inline void coroutine_fn
1849 bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, int64_t bytes,
1850                          BdrvTrackedRequest *req, int ret)
1851 {
1852     int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
1853     BlockDriverState *bs = child->bs;
1854
1855     bdrv_check_request(offset, bytes, &error_abort);
1856
1857     qatomic_inc(&bs->write_gen);
1858
1859     /*
1860      * Discard cannot extend the image, but in error handling cases, such as
1861      * when reverting a qcow2 cluster allocation, the discarded range can pass
1862      * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD
1863      * here. Instead, just skip it, since semantically a discard request
1864      * beyond EOF cannot expand the image anyway.
1865      */
1866     if (ret == 0 &&
1867         (req->type == BDRV_TRACKED_TRUNCATE ||
1868          end_sector > bs->total_sectors) &&
1869         req->type != BDRV_TRACKED_DISCARD) {
1870         bs->total_sectors = end_sector;
1871         bdrv_parent_cb_resize(bs);
1872         bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS);
1873     }
1874     if (req->bytes) {
1875         switch (req->type) {
1876         case BDRV_TRACKED_WRITE:
1877             stat64_max(&bs->wr_highest_offset, offset + bytes);
1878             /* fall through, to set dirty bits */
1879         case BDRV_TRACKED_DISCARD:
1880             bdrv_set_dirty(bs, offset, bytes);
1881             break;
1882         default:
1883             break;
1884         }
1885     }
1886 }
1887
1888 /*
1889  * Forwards an already correctly aligned write request to the BlockDriver,
1890  * after possibly fragmenting it.
1891  */
1892 static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child,
1893     BdrvTrackedRequest *req, int64_t offset, int64_t bytes,
1894     int64_t align, QEMUIOVector *qiov, size_t qiov_offset,
1895     BdrvRequestFlags flags)
1896 {
1897     BlockDriverState *bs = child->bs;
1898     BlockDriver *drv = bs->drv;
1899     int ret;
1900
1901     int64_t bytes_remaining = bytes;
1902     int max_transfer;
1903
1904     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1905
1906     if (!drv) {
1907         return -ENOMEDIUM;
1908     }
1909
1910     if (bdrv_has_readonly_bitmaps(bs)) {
1911         return -EPERM;
1912     }
1913
1914     assert(is_power_of_2(align));
1915     assert((offset & (align - 1)) == 0);
1916     assert((bytes & (align - 1)) == 0);
1917     max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
1918                                    align);
1919
1920     ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags);
1921
1922     if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
1923         !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes &&
1924         qemu_iovec_is_zero(qiov, qiov_offset, bytes)) {
1925         flags |= BDRV_REQ_ZERO_WRITE;
1926         if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
1927             flags |= BDRV_REQ_MAY_UNMAP;
1928         }
1929
1930         /* Can't use optimization hint with bufferless zero write */
1931         flags &= ~BDRV_REQ_REGISTERED_BUF;
1932     }
1933
1934     if (ret < 0) {
1935         /* Do nothing, write notifier decided to fail this request */
1936     } else if (flags & BDRV_REQ_ZERO_WRITE) {
1937         bdrv_co_debug_event(bs, BLKDBG_PWRITEV_ZERO);
1938         ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags);
1939     } else if (flags & BDRV_REQ_WRITE_COMPRESSED) {
1940         ret = bdrv_driver_pwritev_compressed(bs, offset, bytes,
1941                                              qiov, qiov_offset);
1942     } else if (bytes <= max_transfer) {
1943         bdrv_co_debug_event(bs, BLKDBG_PWRITEV);
1944         ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags);
1945     } else {
1946         bdrv_co_debug_event(bs, BLKDBG_PWRITEV);
1947         while (bytes_remaining) {
1948             int num = MIN(bytes_remaining, max_transfer);
1949             int local_flags = flags;
1950
1951             assert(num);
1952             if (num < bytes_remaining && (flags & BDRV_REQ_FUA) &&
1953                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1954                 /* If FUA is going to be emulated by flush, we only
1955                  * need to flush on the last iteration */
1956                 local_flags &= ~BDRV_REQ_FUA;
1957             }
1958
1959             ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining,
1960                                       num, qiov,
1961                                       qiov_offset + bytes - bytes_remaining,
1962                                       local_flags);
1963             if (ret < 0) {
1964                 break;
1965             }
1966             bytes_remaining -= num;
1967         }
1968     }
1969     bdrv_co_debug_event(bs, BLKDBG_PWRITEV_DONE);
1970
1971     if (ret >= 0) {
1972         ret = 0;
1973     }
1974     bdrv_co_write_req_finish(child, offset, bytes, req, ret);
1975
1976     return ret;
1977 }
1978
1979 static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,
1980                                                 int64_t offset,
1981                                                 int64_t bytes,
1982                                                 BdrvRequestFlags flags,
1983                                                 BdrvTrackedRequest *req)
1984 {
1985     BlockDriverState *bs = child->bs;
1986     QEMUIOVector local_qiov;
1987     uint64_t align = bs->bl.request_alignment;
1988     int ret = 0;
1989     bool padding;
1990     BdrvRequestPadding pad;
1991
1992     /* This flag doesn't make sense for padding or zero writes */
1993     flags &= ~BDRV_REQ_REGISTERED_BUF;
1994
1995     padding = bdrv_init_padding(bs, offset, bytes, &pad);
1996     if (padding) {
1997         assert(!(flags & BDRV_REQ_NO_WAIT));
1998         bdrv_make_request_serialising(req, align);
1999
2000         bdrv_padding_rmw_read(child, req, &pad, true);
2001
2002         if (pad.head || pad.merge_reads) {
2003             int64_t aligned_offset = offset & ~(align - 1);
2004             int64_t write_bytes = pad.merge_reads ? pad.buf_len : align;
2005
2006             qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes);
2007             ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes,
2008                                        align, &local_qiov, 0,
2009                                        flags & ~BDRV_REQ_ZERO_WRITE);
2010             if (ret < 0 || pad.merge_reads) {
2011                 /* Error or all work is done */
2012                 goto out;
2013             }
2014             offset += write_bytes - pad.head;
2015             bytes -= write_bytes - pad.head;
2016         }
2017     }
2018
2019     assert(!bytes || (offset & (align - 1)) == 0);
2020     if (bytes >= align) {
2021         /* Write the aligned part in the middle. */
2022         int64_t aligned_bytes = bytes & ~(align - 1);
2023         ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align,
2024                                    NULL, 0, flags);
2025         if (ret < 0) {
2026             goto out;
2027         }
2028         bytes -= aligned_bytes;
2029         offset += aligned_bytes;
2030     }
2031
2032     assert(!bytes || (offset & (align - 1)) == 0);
2033     if (bytes) {
2034         assert(align == pad.tail + bytes);
2035
2036         qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align);
2037         ret = bdrv_aligned_pwritev(child, req, offset, align, align,
2038                                    &local_qiov, 0,
2039                                    flags & ~BDRV_REQ_ZERO_WRITE);
2040     }
2041
2042 out:
2043     bdrv_padding_destroy(&pad);
2044
2045     return ret;
2046 }
2047
2048 /*
2049  * Handle a write request in coroutine context
2050  */
2051 int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
2052     int64_t offset, int64_t bytes, QEMUIOVector *qiov,
2053     BdrvRequestFlags flags)
2054 {
2055     IO_CODE();
2056     return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags);
2057 }
2058
2059 int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child,
2060     int64_t offset, int64_t bytes, QEMUIOVector *qiov, size_t qiov_offset,
2061     BdrvRequestFlags flags)
2062 {
2063     BlockDriverState *bs = child->bs;
2064     BdrvTrackedRequest req;
2065     uint64_t align = bs->bl.request_alignment;
2066     BdrvRequestPadding pad;
2067     int ret;
2068     bool padded = false;
2069     IO_CODE();
2070
2071     trace_bdrv_co_pwritev_part(child->bs, offset, bytes, flags);
2072
2073     if (!bdrv_co_is_inserted(bs)) {
2074         return -ENOMEDIUM;
2075     }
2076
2077     if (flags & BDRV_REQ_ZERO_WRITE) {
2078         ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL);
2079     } else {
2080         ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset);
2081     }
2082     if (ret < 0) {
2083         return ret;
2084     }
2085
2086     /* If the request is misaligned then we can't make it efficient */
2087     if ((flags & BDRV_REQ_NO_FALLBACK) &&
2088         !QEMU_IS_ALIGNED(offset | bytes, align))
2089     {
2090         return -ENOTSUP;
2091     }
2092
2093     if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
2094         /*
2095          * Aligning zero request is nonsense. Even if driver has special meaning
2096          * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
2097          * it to driver due to request_alignment.
2098          *
2099          * Still, no reason to return an error if someone do unaligned
2100          * zero-length write occasionally.
2101          */
2102         return 0;
2103     }
2104
2105     if (!(flags & BDRV_REQ_ZERO_WRITE)) {
2106         /*
2107          * Pad request for following read-modify-write cycle.
2108          * bdrv_co_do_zero_pwritev() does aligning by itself, so, we do
2109          * alignment only if there is no ZERO flag.
2110          */
2111         ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad,
2112                                &padded, &flags);
2113         if (ret < 0) {
2114             return ret;
2115         }
2116     }
2117
2118     bdrv_inc_in_flight(bs);
2119     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
2120
2121     if (flags & BDRV_REQ_ZERO_WRITE) {
2122         assert(!padded);
2123         ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req);
2124         goto out;
2125     }
2126
2127     if (padded) {
2128         /*
2129          * Request was unaligned to request_alignment and therefore
2130          * padded.  We are going to do read-modify-write, and must
2131          * serialize the request to prevent interactions of the
2132          * widened region with other transactions.
2133          */
2134         assert(!(flags & BDRV_REQ_NO_WAIT));
2135         bdrv_make_request_serialising(&req, align);
2136         bdrv_padding_rmw_read(child, &req, &pad, false);
2137     }
2138
2139     ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align,
2140                                qiov, qiov_offset, flags);
2141
2142     bdrv_padding_destroy(&pad);
2143
2144 out:
2145     tracked_request_end(&req);
2146     bdrv_dec_in_flight(bs);
2147
2148     return ret;
2149 }
2150
2151 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset,
2152                                        int64_t bytes, BdrvRequestFlags flags)
2153 {
2154     IO_CODE();
2155     trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags);
2156
2157     if (!(child->bs->open_flags & BDRV_O_UNMAP)) {
2158         flags &= ~BDRV_REQ_MAY_UNMAP;
2159     }
2160
2161     return bdrv_co_pwritev(child, offset, bytes, NULL,
2162                            BDRV_REQ_ZERO_WRITE | flags);
2163 }
2164
2165 /*
2166  * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not.
2167  */
2168 int bdrv_flush_all(void)
2169 {
2170     BdrvNextIterator it;
2171     BlockDriverState *bs = NULL;
2172     int result = 0;
2173
2174     GLOBAL_STATE_CODE();
2175
2176     /*
2177      * bdrv queue is managed by record/replay,
2178      * creating new flush request for stopping
2179      * the VM may break the determinism
2180      */
2181     if (replay_events_enabled()) {
2182         return result;
2183     }
2184
2185     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
2186         AioContext *aio_context = bdrv_get_aio_context(bs);
2187         int ret;
2188
2189         aio_context_acquire(aio_context);
2190         ret = bdrv_flush(bs);
2191         if (ret < 0 && !result) {
2192             result = ret;
2193         }
2194         aio_context_release(aio_context);
2195     }
2196
2197     return result;
2198 }
2199
2200 /*
2201  * Returns the allocation status of the specified sectors.
2202  * Drivers not implementing the functionality are assumed to not support
2203  * backing files, hence all their sectors are reported as allocated.
2204  *
2205  * If 'want_zero' is true, the caller is querying for mapping
2206  * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and
2207  * _ZERO where possible; otherwise, the result favors larger 'pnum',
2208  * with a focus on accurate BDRV_BLOCK_ALLOCATED.
2209  *
2210  * If 'offset' is beyond the end of the disk image the return value is
2211  * BDRV_BLOCK_EOF and 'pnum' is set to 0.
2212  *
2213  * 'bytes' is the max value 'pnum' should be set to.  If bytes goes
2214  * beyond the end of the disk image it will be clamped; if 'pnum' is set to
2215  * the end of the image, then the returned value will include BDRV_BLOCK_EOF.
2216  *
2217  * 'pnum' is set to the number of bytes (including and immediately
2218  * following the specified offset) that are easily known to be in the
2219  * same allocated/unallocated state.  Note that a second call starting
2220  * at the original offset plus returned pnum may have the same status.
2221  * The returned value is non-zero on success except at end-of-file.
2222  *
2223  * Returns negative errno on failure.  Otherwise, if the
2224  * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are
2225  * set to the host mapping and BDS corresponding to the guest offset.
2226  */
2227 static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs,
2228                                              bool want_zero,
2229                                              int64_t offset, int64_t bytes,
2230                                              int64_t *pnum, int64_t *map,
2231                                              BlockDriverState **file)
2232 {
2233     int64_t total_size;
2234     int64_t n; /* bytes */
2235     int ret;
2236     int64_t local_map = 0;
2237     BlockDriverState *local_file = NULL;
2238     int64_t aligned_offset, aligned_bytes;
2239     uint32_t align;
2240     bool has_filtered_child;
2241
2242     assert(pnum);
2243     *pnum = 0;
2244     total_size = bdrv_getlength(bs);
2245     if (total_size < 0) {
2246         ret = total_size;
2247         goto early_out;
2248     }
2249
2250     if (offset >= total_size) {
2251         ret = BDRV_BLOCK_EOF;
2252         goto early_out;
2253     }
2254     if (!bytes) {
2255         ret = 0;
2256         goto early_out;
2257     }
2258
2259     n = total_size - offset;
2260     if (n < bytes) {
2261         bytes = n;
2262     }
2263
2264     /* Must be non-NULL or bdrv_getlength() would have failed */
2265     assert(bs->drv);
2266     has_filtered_child = bdrv_filter_child(bs);
2267     if (!bs->drv->bdrv_co_block_status && !has_filtered_child) {
2268         *pnum = bytes;
2269         ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
2270         if (offset + bytes == total_size) {
2271             ret |= BDRV_BLOCK_EOF;
2272         }
2273         if (bs->drv->protocol_name) {
2274             ret |= BDRV_BLOCK_OFFSET_VALID;
2275             local_map = offset;
2276             local_file = bs;
2277         }
2278         goto early_out;
2279     }
2280
2281     bdrv_inc_in_flight(bs);
2282
2283     /* Round out to request_alignment boundaries */
2284     align = bs->bl.request_alignment;
2285     aligned_offset = QEMU_ALIGN_DOWN(offset, align);
2286     aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset;
2287
2288     if (bs->drv->bdrv_co_block_status) {
2289         /*
2290          * Use the block-status cache only for protocol nodes: Format
2291          * drivers are generally quick to inquire the status, but protocol
2292          * drivers often need to get information from outside of qemu, so
2293          * we do not have control over the actual implementation.  There
2294          * have been cases where inquiring the status took an unreasonably
2295          * long time, and we can do nothing in qemu to fix it.
2296          * This is especially problematic for images with large data areas,
2297          * because finding the few holes in them and giving them special
2298          * treatment does not gain much performance.  Therefore, we try to
2299          * cache the last-identified data region.
2300          *
2301          * Second, limiting ourselves to protocol nodes allows us to assume
2302          * the block status for data regions to be DATA | OFFSET_VALID, and
2303          * that the host offset is the same as the guest offset.
2304          *
2305          * Note that it is possible that external writers zero parts of
2306          * the cached regions without the cache being invalidated, and so
2307          * we may report zeroes as data.  This is not catastrophic,
2308          * however, because reporting zeroes as data is fine.
2309          */
2310         if (QLIST_EMPTY(&bs->children) &&
2311             bdrv_bsc_is_data(bs, aligned_offset, pnum))
2312         {
2313             ret = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID;
2314             local_file = bs;
2315             local_map = aligned_offset;
2316         } else {
2317             ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset,
2318                                                 aligned_bytes, pnum, &local_map,
2319                                                 &local_file);
2320
2321             /*
2322              * Note that checking QLIST_EMPTY(&bs->children) is also done when
2323              * the cache is queried above.  Technically, we do not need to check
2324              * it here; the worst that can happen is that we fill the cache for
2325              * non-protocol nodes, and then it is never used.  However, filling
2326              * the cache requires an RCU update, so double check here to avoid
2327              * such an update if possible.
2328              *
2329              * Check want_zero, because we only want to update the cache when we
2330              * have accurate information about what is zero and what is data.
2331              */
2332             if (want_zero &&
2333                 ret == (BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID) &&
2334                 QLIST_EMPTY(&bs->children))
2335             {
2336                 /*
2337                  * When a protocol driver reports BLOCK_OFFSET_VALID, the
2338                  * returned local_map value must be the same as the offset we
2339                  * have passed (aligned_offset), and local_bs must be the node
2340                  * itself.
2341                  * Assert this, because we follow this rule when reading from
2342                  * the cache (see the `local_file = bs` and
2343                  * `local_map = aligned_offset` assignments above), and the
2344                  * result the cache delivers must be the same as the driver
2345                  * would deliver.
2346                  */
2347                 assert(local_file == bs);
2348                 assert(local_map == aligned_offset);
2349                 bdrv_bsc_fill(bs, aligned_offset, *pnum);
2350             }
2351         }
2352     } else {
2353         /* Default code for filters */
2354
2355         local_file = bdrv_filter_bs(bs);
2356         assert(local_file);
2357
2358         *pnum = aligned_bytes;
2359         local_map = aligned_offset;
2360         ret = BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID;
2361     }
2362     if (ret < 0) {
2363         *pnum = 0;
2364         goto out;
2365     }
2366
2367     /*
2368      * The driver's result must be a non-zero multiple of request_alignment.
2369      * Clamp pnum and adjust map to original request.
2370      */
2371     assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) &&
2372            align > offset - aligned_offset);
2373     if (ret & BDRV_BLOCK_RECURSE) {
2374         assert(ret & BDRV_BLOCK_DATA);
2375         assert(ret & BDRV_BLOCK_OFFSET_VALID);
2376         assert(!(ret & BDRV_BLOCK_ZERO));
2377     }
2378
2379     *pnum -= offset - aligned_offset;
2380     if (*pnum > bytes) {
2381         *pnum = bytes;
2382     }
2383     if (ret & BDRV_BLOCK_OFFSET_VALID) {
2384         local_map += offset - aligned_offset;
2385     }
2386
2387     if (ret & BDRV_BLOCK_RAW) {
2388         assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file);
2389         ret = bdrv_co_block_status(local_file, want_zero, local_map,
2390                                    *pnum, pnum, &local_map, &local_file);
2391         goto out;
2392     }
2393
2394     if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
2395         ret |= BDRV_BLOCK_ALLOCATED;
2396     } else if (bs->drv->supports_backing) {
2397         BlockDriverState *cow_bs = bdrv_cow_bs(bs);
2398
2399         if (!cow_bs) {
2400             ret |= BDRV_BLOCK_ZERO;
2401         } else if (want_zero) {
2402             int64_t size2 = bdrv_getlength(cow_bs);
2403
2404             if (size2 >= 0 && offset >= size2) {
2405                 ret |= BDRV_BLOCK_ZERO;
2406             }
2407         }
2408     }
2409
2410     if (want_zero && ret & BDRV_BLOCK_RECURSE &&
2411         local_file && local_file != bs &&
2412         (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
2413         (ret & BDRV_BLOCK_OFFSET_VALID)) {
2414         int64_t file_pnum;
2415         int ret2;
2416
2417         ret2 = bdrv_co_block_status(local_file, want_zero, local_map,
2418                                     *pnum, &file_pnum, NULL, NULL);
2419         if (ret2 >= 0) {
2420             /* Ignore errors.  This is just providing extra information, it
2421              * is useful but not necessary.
2422              */
2423             if (ret2 & BDRV_BLOCK_EOF &&
2424                 (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) {
2425                 /*
2426                  * It is valid for the format block driver to read
2427                  * beyond the end of the underlying file's current
2428                  * size; such areas read as zero.
2429                  */
2430                 ret |= BDRV_BLOCK_ZERO;
2431             } else {
2432                 /* Limit request to the range reported by the protocol driver */
2433                 *pnum = file_pnum;
2434                 ret |= (ret2 & BDRV_BLOCK_ZERO);
2435             }
2436         }
2437     }
2438
2439 out:
2440     bdrv_dec_in_flight(bs);
2441     if (ret >= 0 && offset + *pnum == total_size) {
2442         ret |= BDRV_BLOCK_EOF;
2443     }
2444 early_out:
2445     if (file) {
2446         *file = local_file;
2447     }
2448     if (map) {
2449         *map = local_map;
2450     }
2451     return ret;
2452 }
2453
2454 int coroutine_fn
2455 bdrv_co_common_block_status_above(BlockDriverState *bs,
2456                                   BlockDriverState *base,
2457                                   bool include_base,
2458                                   bool want_zero,
2459                                   int64_t offset,
2460                                   int64_t bytes,
2461                                   int64_t *pnum,
2462                                   int64_t *map,
2463                                   BlockDriverState **file,
2464                                   int *depth)
2465 {
2466     int ret;
2467     BlockDriverState *p;
2468     int64_t eof = 0;
2469     int dummy;
2470     IO_CODE();
2471
2472     assert(!include_base || base); /* Can't include NULL base */
2473
2474     if (!depth) {
2475         depth = &dummy;
2476     }
2477     *depth = 0;
2478
2479     if (!include_base && bs == base) {
2480         *pnum = bytes;
2481         return 0;
2482     }
2483
2484     ret = bdrv_co_block_status(bs, want_zero, offset, bytes, pnum, map, file);
2485     ++*depth;
2486     if (ret < 0 || *pnum == 0 || ret & BDRV_BLOCK_ALLOCATED || bs == base) {
2487         return ret;
2488     }
2489
2490     if (ret & BDRV_BLOCK_EOF) {
2491         eof = offset + *pnum;
2492     }
2493
2494     assert(*pnum <= bytes);
2495     bytes = *pnum;
2496
2497     for (p = bdrv_filter_or_cow_bs(bs); include_base || p != base;
2498          p = bdrv_filter_or_cow_bs(p))
2499     {
2500         ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map,
2501                                    file);
2502         ++*depth;
2503         if (ret < 0) {
2504             return ret;
2505         }
2506         if (*pnum == 0) {
2507             /*
2508              * The top layer deferred to this layer, and because this layer is
2509              * short, any zeroes that we synthesize beyond EOF behave as if they
2510              * were allocated at this layer.
2511              *
2512              * We don't include BDRV_BLOCK_EOF into ret, as upper layer may be
2513              * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see
2514              * below.
2515              */
2516             assert(ret & BDRV_BLOCK_EOF);
2517             *pnum = bytes;
2518             if (file) {
2519                 *file = p;
2520             }
2521             ret = BDRV_BLOCK_ZERO | BDRV_BLOCK_ALLOCATED;
2522             break;
2523         }
2524         if (ret & BDRV_BLOCK_ALLOCATED) {
2525             /*
2526              * We've found the node and the status, we must break.
2527              *
2528              * Drop BDRV_BLOCK_EOF, as it's not for upper layer, which may be
2529              * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see
2530              * below.
2531              */
2532             ret &= ~BDRV_BLOCK_EOF;
2533             break;
2534         }
2535
2536         if (p == base) {
2537             assert(include_base);
2538             break;
2539         }
2540
2541         /*
2542          * OK, [offset, offset + *pnum) region is unallocated on this layer,
2543          * let's continue the diving.
2544          */
2545         assert(*pnum <= bytes);
2546         bytes = *pnum;
2547     }
2548
2549     if (offset + *pnum == eof) {
2550         ret |= BDRV_BLOCK_EOF;
2551     }
2552
2553     return ret;
2554 }
2555
2556 int coroutine_fn bdrv_co_block_status_above(BlockDriverState *bs,
2557                                             BlockDriverState *base,
2558                                             int64_t offset, int64_t bytes,
2559                                             int64_t *pnum, int64_t *map,
2560                                             BlockDriverState **file)
2561 {
2562     IO_CODE();
2563     return bdrv_co_common_block_status_above(bs, base, false, true, offset,
2564                                              bytes, pnum, map, file, NULL);
2565 }
2566
2567 int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base,
2568                             int64_t offset, int64_t bytes, int64_t *pnum,
2569                             int64_t *map, BlockDriverState **file)
2570 {
2571     IO_CODE();
2572     return bdrv_common_block_status_above(bs, base, false, true, offset, bytes,
2573                                           pnum, map, file, NULL);
2574 }
2575
2576 int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes,
2577                       int64_t *pnum, int64_t *map, BlockDriverState **file)
2578 {
2579     IO_CODE();
2580     return bdrv_block_status_above(bs, bdrv_filter_or_cow_bs(bs),
2581                                    offset, bytes, pnum, map, file);
2582 }
2583
2584 /*
2585  * Check @bs (and its backing chain) to see if the range defined
2586  * by @offset and @bytes is known to read as zeroes.
2587  * Return 1 if that is the case, 0 otherwise and -errno on error.
2588  * This test is meant to be fast rather than accurate so returning 0
2589  * does not guarantee non-zero data.
2590  */
2591 int coroutine_fn bdrv_co_is_zero_fast(BlockDriverState *bs, int64_t offset,
2592                                       int64_t bytes)
2593 {
2594     int ret;
2595     int64_t pnum = bytes;
2596     IO_CODE();
2597
2598     if (!bytes) {
2599         return 1;
2600     }
2601
2602     ret = bdrv_co_common_block_status_above(bs, NULL, false, false, offset,
2603                                             bytes, &pnum, NULL, NULL, NULL);
2604
2605     if (ret < 0) {
2606         return ret;
2607     }
2608
2609     return (pnum == bytes) && (ret & BDRV_BLOCK_ZERO);
2610 }
2611
2612 int coroutine_fn bdrv_co_is_allocated(BlockDriverState *bs, int64_t offset,
2613                                       int64_t bytes, int64_t *pnum)
2614 {
2615     int ret;
2616     int64_t dummy;
2617     IO_CODE();
2618
2619     ret = bdrv_co_common_block_status_above(bs, bs, true, false, offset,
2620                                             bytes, pnum ? pnum : &dummy, NULL,
2621                                             NULL, NULL);
2622     if (ret < 0) {
2623         return ret;
2624     }
2625     return !!(ret & BDRV_BLOCK_ALLOCATED);
2626 }
2627
2628 int bdrv_is_allocated(BlockDriverState *bs, int64_t offset, int64_t bytes,
2629                       int64_t *pnum)
2630 {
2631     int ret;
2632     int64_t dummy;
2633     IO_CODE();
2634
2635     ret = bdrv_common_block_status_above(bs, bs, true, false, offset,
2636                                          bytes, pnum ? pnum : &dummy, NULL,
2637                                          NULL, NULL);
2638     if (ret < 0) {
2639         return ret;
2640     }
2641     return !!(ret & BDRV_BLOCK_ALLOCATED);
2642 }
2643
2644 /* See bdrv_is_allocated_above for documentation */
2645 int coroutine_fn bdrv_co_is_allocated_above(BlockDriverState *top,
2646                                             BlockDriverState *base,
2647                                             bool include_base, int64_t offset,
2648                                             int64_t bytes, int64_t *pnum)
2649 {
2650     int depth;
2651     int ret;
2652     IO_CODE();
2653
2654     ret = bdrv_co_common_block_status_above(top, base, include_base, false,
2655                                             offset, bytes, pnum, NULL, NULL,
2656                                             &depth);
2657     if (ret < 0) {
2658         return ret;
2659     }
2660
2661     if (ret & BDRV_BLOCK_ALLOCATED) {
2662         return depth;
2663     }
2664     return 0;
2665 }
2666
2667 /*
2668  * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
2669  *
2670  * Return a positive depth if (a prefix of) the given range is allocated
2671  * in any image between BASE and TOP (BASE is only included if include_base
2672  * is set).  Depth 1 is TOP, 2 is the first backing layer, and so forth.
2673  * BASE can be NULL to check if the given offset is allocated in any
2674  * image of the chain.  Return 0 otherwise, or negative errno on
2675  * failure.
2676  *
2677  * 'pnum' is set to the number of bytes (including and immediately
2678  * following the specified offset) that are known to be in the same
2679  * allocated/unallocated state.  Note that a subsequent call starting
2680  * at 'offset + *pnum' may return the same allocation status (in other
2681  * words, the result is not necessarily the maximum possible range);
2682  * but 'pnum' will only be 0 when end of file is reached.
2683  */
2684 int bdrv_is_allocated_above(BlockDriverState *top,
2685                             BlockDriverState *base,
2686                             bool include_base, int64_t offset,
2687                             int64_t bytes, int64_t *pnum)
2688 {
2689     int depth;
2690     int ret;
2691     IO_CODE();
2692
2693     ret = bdrv_common_block_status_above(top, base, include_base, false,
2694                                          offset, bytes, pnum, NULL, NULL,
2695                                          &depth);
2696     if (ret < 0) {
2697         return ret;
2698     }
2699
2700     if (ret & BDRV_BLOCK_ALLOCATED) {
2701         return depth;
2702     }
2703     return 0;
2704 }
2705
2706 int coroutine_fn
2707 bdrv_co_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2708 {
2709     BlockDriver *drv = bs->drv;
2710     BlockDriverState *child_bs = bdrv_primary_bs(bs);
2711     int ret;
2712     IO_CODE();
2713     assert_bdrv_graph_readable();
2714
2715     ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL);
2716     if (ret < 0) {
2717         return ret;
2718     }
2719
2720     if (!drv) {
2721         return -ENOMEDIUM;
2722     }
2723
2724     bdrv_inc_in_flight(bs);
2725
2726     if (drv->bdrv_co_load_vmstate) {
2727         ret = drv->bdrv_co_load_vmstate(bs, qiov, pos);
2728     } else if (child_bs) {
2729         ret = bdrv_co_readv_vmstate(child_bs, qiov, pos);
2730     } else {
2731         ret = -ENOTSUP;
2732     }
2733
2734     bdrv_dec_in_flight(bs);
2735
2736     return ret;
2737 }
2738
2739 int coroutine_fn
2740 bdrv_co_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2741 {
2742     BlockDriver *drv = bs->drv;
2743     BlockDriverState *child_bs = bdrv_primary_bs(bs);
2744     int ret;
2745     IO_CODE();
2746     assert_bdrv_graph_readable();
2747
2748     ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL);
2749     if (ret < 0) {
2750         return ret;
2751     }
2752
2753     if (!drv) {
2754         return -ENOMEDIUM;
2755     }
2756
2757     bdrv_inc_in_flight(bs);
2758
2759     if (drv->bdrv_co_save_vmstate) {
2760         ret = drv->bdrv_co_save_vmstate(bs, qiov, pos);
2761     } else if (child_bs) {
2762         ret = bdrv_co_writev_vmstate(child_bs, qiov, pos);
2763     } else {
2764         ret = -ENOTSUP;
2765     }
2766
2767     bdrv_dec_in_flight(bs);
2768
2769     return ret;
2770 }
2771
2772 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
2773                       int64_t pos, int size)
2774 {
2775     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2776     int ret = bdrv_writev_vmstate(bs, &qiov, pos);
2777     IO_CODE();
2778
2779     return ret < 0 ? ret : size;
2780 }
2781
2782 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
2783                       int64_t pos, int size)
2784 {
2785     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2786     int ret = bdrv_readv_vmstate(bs, &qiov, pos);
2787     IO_CODE();
2788
2789     return ret < 0 ? ret : size;
2790 }
2791
2792 /**************************************************************/
2793 /* async I/Os */
2794
2795 void bdrv_aio_cancel(BlockAIOCB *acb)
2796 {
2797     IO_CODE();
2798     qemu_aio_ref(acb);
2799     bdrv_aio_cancel_async(acb);
2800     while (acb->refcnt > 1) {
2801         if (acb->aiocb_info->get_aio_context) {
2802             aio_poll(acb->aiocb_info->get_aio_context(acb), true);
2803         } else if (acb->bs) {
2804             /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so
2805              * assert that we're not using an I/O thread.  Thread-safe
2806              * code should use bdrv_aio_cancel_async exclusively.
2807              */
2808             assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context());
2809             aio_poll(bdrv_get_aio_context(acb->bs), true);
2810         } else {
2811             abort();
2812         }
2813     }
2814     qemu_aio_unref(acb);
2815 }
2816
2817 /* Async version of aio cancel. The caller is not blocked if the acb implements
2818  * cancel_async, otherwise we do nothing and let the request normally complete.
2819  * In either case the completion callback must be called. */
2820 void bdrv_aio_cancel_async(BlockAIOCB *acb)
2821 {
2822     IO_CODE();
2823     if (acb->aiocb_info->cancel_async) {
2824         acb->aiocb_info->cancel_async(acb);
2825     }
2826 }
2827
2828 /**************************************************************/
2829 /* Coroutine block device emulation */
2830
2831 int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
2832 {
2833     BdrvChild *primary_child = bdrv_primary_child(bs);
2834     BdrvChild *child;
2835     int current_gen;
2836     int ret = 0;
2837     IO_CODE();
2838
2839     bdrv_inc_in_flight(bs);
2840
2841     if (!bdrv_co_is_inserted(bs) || bdrv_is_read_only(bs) ||
2842         bdrv_is_sg(bs)) {
2843         goto early_exit;
2844     }
2845
2846     qemu_co_mutex_lock(&bs->reqs_lock);
2847     current_gen = qatomic_read(&bs->write_gen);
2848
2849     /* Wait until any previous flushes are completed */
2850     while (bs->active_flush_req) {
2851         qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock);
2852     }
2853
2854     /* Flushes reach this point in nondecreasing current_gen order.  */
2855     bs->active_flush_req = true;
2856     qemu_co_mutex_unlock(&bs->reqs_lock);
2857
2858     /* Write back all layers by calling one driver function */
2859     if (bs->drv->bdrv_co_flush) {
2860         ret = bs->drv->bdrv_co_flush(bs);
2861         goto out;
2862     }
2863
2864     /* Write back cached data to the OS even with cache=unsafe */
2865     BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_OS);
2866     if (bs->drv->bdrv_co_flush_to_os) {
2867         ret = bs->drv->bdrv_co_flush_to_os(bs);
2868         if (ret < 0) {
2869             goto out;
2870         }
2871     }
2872
2873     /* But don't actually force it to the disk with cache=unsafe */
2874     if (bs->open_flags & BDRV_O_NO_FLUSH) {
2875         goto flush_children;
2876     }
2877
2878     /* Check if we really need to flush anything */
2879     if (bs->flushed_gen == current_gen) {
2880         goto flush_children;
2881     }
2882
2883     BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_DISK);
2884     if (!bs->drv) {
2885         /* bs->drv->bdrv_co_flush() might have ejected the BDS
2886          * (even in case of apparent success) */
2887         ret = -ENOMEDIUM;
2888         goto out;
2889     }
2890     if (bs->drv->bdrv_co_flush_to_disk) {
2891         ret = bs->drv->bdrv_co_flush_to_disk(bs);
2892     } else if (bs->drv->bdrv_aio_flush) {
2893         BlockAIOCB *acb;
2894         CoroutineIOCompletion co = {
2895             .coroutine = qemu_coroutine_self(),
2896         };
2897
2898         acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
2899         if (acb == NULL) {
2900             ret = -EIO;
2901         } else {
2902             qemu_coroutine_yield();
2903             ret = co.ret;
2904         }
2905     } else {
2906         /*
2907          * Some block drivers always operate in either writethrough or unsafe
2908          * mode and don't support bdrv_flush therefore. Usually qemu doesn't
2909          * know how the server works (because the behaviour is hardcoded or
2910          * depends on server-side configuration), so we can't ensure that
2911          * everything is safe on disk. Returning an error doesn't work because
2912          * that would break guests even if the server operates in writethrough
2913          * mode.
2914          *
2915          * Let's hope the user knows what he's doing.
2916          */
2917         ret = 0;
2918     }
2919
2920     if (ret < 0) {
2921         goto out;
2922     }
2923
2924     /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
2925      * in the case of cache=unsafe, so there are no useless flushes.
2926      */
2927 flush_children:
2928     ret = 0;
2929     QLIST_FOREACH(child, &bs->children, next) {
2930         if (child->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) {
2931             int this_child_ret = bdrv_co_flush(child->bs);
2932             if (!ret) {
2933                 ret = this_child_ret;
2934             }
2935         }
2936     }
2937
2938 out:
2939     /* Notify any pending flushes that we have completed */
2940     if (ret == 0) {
2941         bs->flushed_gen = current_gen;
2942     }
2943
2944     qemu_co_mutex_lock(&bs->reqs_lock);
2945     bs->active_flush_req = false;
2946     /* Return value is ignored - it's ok if wait queue is empty */
2947     qemu_co_queue_next(&bs->flush_queue);
2948     qemu_co_mutex_unlock(&bs->reqs_lock);
2949
2950 early_exit:
2951     bdrv_dec_in_flight(bs);
2952     return ret;
2953 }
2954
2955 int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset,
2956                                   int64_t bytes)
2957 {
2958     BdrvTrackedRequest req;
2959     int ret;
2960     int64_t max_pdiscard;
2961     int head, tail, align;
2962     BlockDriverState *bs = child->bs;
2963     IO_CODE();
2964
2965     if (!bs || !bs->drv || !bdrv_co_is_inserted(bs)) {
2966         return -ENOMEDIUM;
2967     }
2968
2969     if (bdrv_has_readonly_bitmaps(bs)) {
2970         return -EPERM;
2971     }
2972
2973     ret = bdrv_check_request(offset, bytes, NULL);
2974     if (ret < 0) {
2975         return ret;
2976     }
2977
2978     /* Do nothing if disabled.  */
2979     if (!(bs->open_flags & BDRV_O_UNMAP)) {
2980         return 0;
2981     }
2982
2983     if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) {
2984         return 0;
2985     }
2986
2987     /* Invalidate the cached block-status data range if this discard overlaps */
2988     bdrv_bsc_invalidate_range(bs, offset, bytes);
2989
2990     /* Discard is advisory, but some devices track and coalesce
2991      * unaligned requests, so we must pass everything down rather than
2992      * round here.  Still, most devices will just silently ignore
2993      * unaligned requests (by returning -ENOTSUP), so we must fragment
2994      * the request accordingly.  */
2995     align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment);
2996     assert(align % bs->bl.request_alignment == 0);
2997     head = offset % align;
2998     tail = (offset + bytes) % align;
2999
3000     bdrv_inc_in_flight(bs);
3001     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD);
3002
3003     ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0);
3004     if (ret < 0) {
3005         goto out;
3006     }
3007
3008     max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT64_MAX),
3009                                    align);
3010     assert(max_pdiscard >= bs->bl.request_alignment);
3011
3012     while (bytes > 0) {
3013         int64_t num = bytes;
3014
3015         if (head) {
3016             /* Make small requests to get to alignment boundaries. */
3017             num = MIN(bytes, align - head);
3018             if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) {
3019                 num %= bs->bl.request_alignment;
3020             }
3021             head = (head + num) % align;
3022             assert(num < max_pdiscard);
3023         } else if (tail) {
3024             if (num > align) {
3025                 /* Shorten the request to the last aligned cluster.  */
3026                 num -= tail;
3027             } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) &&
3028                        tail > bs->bl.request_alignment) {
3029                 tail %= bs->bl.request_alignment;
3030                 num -= tail;
3031             }
3032         }
3033         /* limit request size */
3034         if (num > max_pdiscard) {
3035             num = max_pdiscard;
3036         }
3037
3038         if (!bs->drv) {
3039             ret = -ENOMEDIUM;
3040             goto out;
3041         }
3042         if (bs->drv->bdrv_co_pdiscard) {
3043             ret = bs->drv->bdrv_co_pdiscard(bs, offset, num);
3044         } else {
3045             BlockAIOCB *acb;
3046             CoroutineIOCompletion co = {
3047                 .coroutine = qemu_coroutine_self(),
3048             };
3049
3050             acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num,
3051                                              bdrv_co_io_em_complete, &co);
3052             if (acb == NULL) {
3053                 ret = -EIO;
3054                 goto out;
3055             } else {
3056                 qemu_coroutine_yield();
3057                 ret = co.ret;
3058             }
3059         }
3060         if (ret && ret != -ENOTSUP) {
3061             goto out;
3062         }
3063
3064         offset += num;
3065         bytes -= num;
3066     }
3067     ret = 0;
3068 out:
3069     bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret);
3070     tracked_request_end(&req);
3071     bdrv_dec_in_flight(bs);
3072     return ret;
3073 }
3074
3075 int coroutine_fn bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf)
3076 {
3077     BlockDriver *drv = bs->drv;
3078     CoroutineIOCompletion co = {
3079         .coroutine = qemu_coroutine_self(),
3080     };
3081     BlockAIOCB *acb;
3082     IO_CODE();
3083
3084     bdrv_inc_in_flight(bs);
3085     if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) {
3086         co.ret = -ENOTSUP;
3087         goto out;
3088     }
3089
3090     if (drv->bdrv_co_ioctl) {
3091         co.ret = drv->bdrv_co_ioctl(bs, req, buf);
3092     } else {
3093         acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
3094         if (!acb) {
3095             co.ret = -ENOTSUP;
3096             goto out;
3097         }
3098         qemu_coroutine_yield();
3099     }
3100 out:
3101     bdrv_dec_in_flight(bs);
3102     return co.ret;
3103 }
3104
3105 void *qemu_blockalign(BlockDriverState *bs, size_t size)
3106 {
3107     IO_CODE();
3108     return qemu_memalign(bdrv_opt_mem_align(bs), size);
3109 }
3110
3111 void *qemu_blockalign0(BlockDriverState *bs, size_t size)
3112 {
3113     IO_CODE();
3114     return memset(qemu_blockalign(bs, size), 0, size);
3115 }
3116
3117 void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
3118 {
3119     size_t align = bdrv_opt_mem_align(bs);
3120     IO_CODE();
3121
3122     /* Ensure that NULL is never returned on success */
3123     assert(align > 0);
3124     if (size == 0) {
3125         size = align;
3126     }
3127
3128     return qemu_try_memalign(align, size);
3129 }
3130
3131 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
3132 {
3133     void *mem = qemu_try_blockalign(bs, size);
3134     IO_CODE();
3135
3136     if (mem) {
3137         memset(mem, 0, size);
3138     }
3139
3140     return mem;
3141 }
3142
3143 void coroutine_fn bdrv_co_io_plug(BlockDriverState *bs)
3144 {
3145     BdrvChild *child;
3146     IO_CODE();
3147
3148     QLIST_FOREACH(child, &bs->children, next) {
3149         bdrv_co_io_plug(child->bs);
3150     }
3151
3152     if (qatomic_fetch_inc(&bs->io_plugged) == 0) {
3153         BlockDriver *drv = bs->drv;
3154         if (drv && drv->bdrv_co_io_plug) {
3155             drv->bdrv_co_io_plug(bs);
3156         }
3157     }
3158 }
3159
3160 void coroutine_fn bdrv_co_io_unplug(BlockDriverState *bs)
3161 {
3162     BdrvChild *child;
3163     IO_CODE();
3164
3165     assert(bs->io_plugged);
3166     if (qatomic_fetch_dec(&bs->io_plugged) == 1) {
3167         BlockDriver *drv = bs->drv;
3168         if (drv && drv->bdrv_co_io_unplug) {
3169             drv->bdrv_co_io_unplug(bs);
3170         }
3171     }
3172
3173     QLIST_FOREACH(child, &bs->children, next) {
3174         bdrv_co_io_unplug(child->bs);
3175     }
3176 }
3177
3178 /* Helper that undoes bdrv_register_buf() when it fails partway through */
3179 static void bdrv_register_buf_rollback(BlockDriverState *bs,
3180                                        void *host,
3181                                        size_t size,
3182                                        BdrvChild *final_child)
3183 {
3184     BdrvChild *child;
3185
3186     QLIST_FOREACH(child, &bs->children, next) {
3187         if (child == final_child) {
3188             break;
3189         }
3190
3191         bdrv_unregister_buf(child->bs, host, size);
3192     }
3193
3194     if (bs->drv && bs->drv->bdrv_unregister_buf) {
3195         bs->drv->bdrv_unregister_buf(bs, host, size);
3196     }
3197 }
3198
3199 bool bdrv_register_buf(BlockDriverState *bs, void *host, size_t size,
3200                        Error **errp)
3201 {
3202     BdrvChild *child;
3203
3204     GLOBAL_STATE_CODE();
3205     if (bs->drv && bs->drv->bdrv_register_buf) {
3206         if (!bs->drv->bdrv_register_buf(bs, host, size, errp)) {
3207             return false;
3208         }
3209     }
3210     QLIST_FOREACH(child, &bs->children, next) {
3211         if (!bdrv_register_buf(child->bs, host, size, errp)) {
3212             bdrv_register_buf_rollback(bs, host, size, child);
3213             return false;
3214         }
3215     }
3216     return true;
3217 }
3218
3219 void bdrv_unregister_buf(BlockDriverState *bs, void *host, size_t size)
3220 {
3221     BdrvChild *child;
3222
3223     GLOBAL_STATE_CODE();
3224     if (bs->drv && bs->drv->bdrv_unregister_buf) {
3225         bs->drv->bdrv_unregister_buf(bs, host, size);
3226     }
3227     QLIST_FOREACH(child, &bs->children, next) {
3228         bdrv_unregister_buf(child->bs, host, size);
3229     }
3230 }
3231
3232 static int coroutine_fn bdrv_co_copy_range_internal(
3233         BdrvChild *src, int64_t src_offset, BdrvChild *dst,
3234         int64_t dst_offset, int64_t bytes,
3235         BdrvRequestFlags read_flags, BdrvRequestFlags write_flags,
3236         bool recurse_src)
3237 {
3238     BdrvTrackedRequest req;
3239     int ret;
3240
3241     /* TODO We can support BDRV_REQ_NO_FALLBACK here */
3242     assert(!(read_flags & BDRV_REQ_NO_FALLBACK));
3243     assert(!(write_flags & BDRV_REQ_NO_FALLBACK));
3244     assert(!(read_flags & BDRV_REQ_NO_WAIT));
3245     assert(!(write_flags & BDRV_REQ_NO_WAIT));
3246
3247     if (!dst || !dst->bs || !bdrv_co_is_inserted(dst->bs)) {
3248         return -ENOMEDIUM;
3249     }
3250     ret = bdrv_check_request32(dst_offset, bytes, NULL, 0);
3251     if (ret) {
3252         return ret;
3253     }
3254     if (write_flags & BDRV_REQ_ZERO_WRITE) {
3255         return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags);
3256     }
3257
3258     if (!src || !src->bs || !bdrv_co_is_inserted(src->bs)) {
3259         return -ENOMEDIUM;
3260     }
3261     ret = bdrv_check_request32(src_offset, bytes, NULL, 0);
3262     if (ret) {
3263         return ret;
3264     }
3265
3266     if (!src->bs->drv->bdrv_co_copy_range_from
3267         || !dst->bs->drv->bdrv_co_copy_range_to
3268         || src->bs->encrypted || dst->bs->encrypted) {
3269         return -ENOTSUP;
3270     }
3271
3272     if (recurse_src) {
3273         bdrv_inc_in_flight(src->bs);
3274         tracked_request_begin(&req, src->bs, src_offset, bytes,
3275                               BDRV_TRACKED_READ);
3276
3277         /* BDRV_REQ_SERIALISING is only for write operation */
3278         assert(!(read_flags & BDRV_REQ_SERIALISING));
3279         bdrv_wait_serialising_requests(&req);
3280
3281         ret = src->bs->drv->bdrv_co_copy_range_from(src->bs,
3282                                                     src, src_offset,
3283                                                     dst, dst_offset,
3284                                                     bytes,
3285                                                     read_flags, write_flags);
3286
3287         tracked_request_end(&req);
3288         bdrv_dec_in_flight(src->bs);
3289     } else {
3290         bdrv_inc_in_flight(dst->bs);
3291         tracked_request_begin(&req, dst->bs, dst_offset, bytes,
3292                               BDRV_TRACKED_WRITE);
3293         ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req,
3294                                         write_flags);
3295         if (!ret) {
3296             ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs,
3297                                                       src, src_offset,
3298                                                       dst, dst_offset,
3299                                                       bytes,
3300                                                       read_flags, write_flags);
3301         }
3302         bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret);
3303         tracked_request_end(&req);
3304         bdrv_dec_in_flight(dst->bs);
3305     }
3306
3307     return ret;
3308 }
3309
3310 /* Copy range from @src to @dst.
3311  *
3312  * See the comment of bdrv_co_copy_range for the parameter and return value
3313  * semantics. */
3314 int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, int64_t src_offset,
3315                                          BdrvChild *dst, int64_t dst_offset,
3316                                          int64_t bytes,
3317                                          BdrvRequestFlags read_flags,
3318                                          BdrvRequestFlags write_flags)
3319 {
3320     IO_CODE();
3321     trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes,
3322                                   read_flags, write_flags);
3323     return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
3324                                        bytes, read_flags, write_flags, true);
3325 }
3326
3327 /* Copy range from @src to @dst.
3328  *
3329  * See the comment of bdrv_co_copy_range for the parameter and return value
3330  * semantics. */
3331 int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, int64_t src_offset,
3332                                        BdrvChild *dst, int64_t dst_offset,
3333                                        int64_t bytes,
3334                                        BdrvRequestFlags read_flags,
3335                                        BdrvRequestFlags write_flags)
3336 {
3337     IO_CODE();
3338     trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes,
3339                                 read_flags, write_flags);
3340     return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
3341                                        bytes, read_flags, write_flags, false);
3342 }
3343
3344 int coroutine_fn bdrv_co_copy_range(BdrvChild *src, int64_t src_offset,
3345                                     BdrvChild *dst, int64_t dst_offset,
3346                                     int64_t bytes, BdrvRequestFlags read_flags,
3347                                     BdrvRequestFlags write_flags)
3348 {
3349     IO_CODE();
3350     return bdrv_co_copy_range_from(src, src_offset,
3351                                    dst, dst_offset,
3352                                    bytes, read_flags, write_flags);
3353 }
3354
3355 static void bdrv_parent_cb_resize(BlockDriverState *bs)
3356 {
3357     BdrvChild *c;
3358     QLIST_FOREACH(c, &bs->parents, next_parent) {
3359         if (c->klass->resize) {
3360             c->klass->resize(c);
3361         }
3362     }
3363 }
3364
3365 /**
3366  * Truncate file to 'offset' bytes (needed only for file protocols)
3367  *
3368  * If 'exact' is true, the file must be resized to exactly the given
3369  * 'offset'.  Otherwise, it is sufficient for the node to be at least
3370  * 'offset' bytes in length.
3371  */
3372 int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact,
3373                                   PreallocMode prealloc, BdrvRequestFlags flags,
3374                                   Error **errp)
3375 {
3376     BlockDriverState *bs = child->bs;
3377     BdrvChild *filtered, *backing;
3378     BlockDriver *drv = bs->drv;
3379     BdrvTrackedRequest req;
3380     int64_t old_size, new_bytes;
3381     int ret;
3382     IO_CODE();
3383
3384     /* if bs->drv == NULL, bs is closed, so there's nothing to do here */
3385     if (!drv) {
3386         error_setg(errp, "No medium inserted");
3387         return -ENOMEDIUM;
3388     }
3389     if (offset < 0) {
3390         error_setg(errp, "Image size cannot be negative");
3391         return -EINVAL;
3392     }
3393
3394     ret = bdrv_check_request(offset, 0, errp);
3395     if (ret < 0) {
3396         return ret;
3397     }
3398
3399     old_size = bdrv_getlength(bs);
3400     if (old_size < 0) {
3401         error_setg_errno(errp, -old_size, "Failed to get old image size");
3402         return old_size;
3403     }
3404
3405     if (bdrv_is_read_only(bs)) {
3406         error_setg(errp, "Image is read-only");
3407         return -EACCES;
3408     }
3409
3410     if (offset > old_size) {
3411         new_bytes = offset - old_size;
3412     } else {
3413         new_bytes = 0;
3414     }
3415
3416     bdrv_inc_in_flight(bs);
3417     tracked_request_begin(&req, bs, offset - new_bytes, new_bytes,
3418                           BDRV_TRACKED_TRUNCATE);
3419
3420     /* If we are growing the image and potentially using preallocation for the
3421      * new area, we need to make sure that no write requests are made to it
3422      * concurrently or they might be overwritten by preallocation. */
3423     if (new_bytes) {
3424         bdrv_make_request_serialising(&req, 1);
3425     }
3426     ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req,
3427                                     0);
3428     if (ret < 0) {
3429         error_setg_errno(errp, -ret,
3430                          "Failed to prepare request for truncation");
3431         goto out;
3432     }
3433
3434     filtered = bdrv_filter_child(bs);
3435     backing = bdrv_cow_child(bs);
3436
3437     /*
3438      * If the image has a backing file that is large enough that it would
3439      * provide data for the new area, we cannot leave it unallocated because
3440      * then the backing file content would become visible. Instead, zero-fill
3441      * the new area.
3442      *
3443      * Note that if the image has a backing file, but was opened without the
3444      * backing file, taking care of keeping things consistent with that backing
3445      * file is the user's responsibility.
3446      */
3447     if (new_bytes && backing) {
3448         int64_t backing_len;
3449
3450         backing_len = bdrv_co_getlength(backing->bs);
3451         if (backing_len < 0) {
3452             ret = backing_len;
3453             error_setg_errno(errp, -ret, "Could not get backing file size");
3454             goto out;
3455         }
3456
3457         if (backing_len > old_size) {
3458             flags |= BDRV_REQ_ZERO_WRITE;
3459         }
3460     }
3461
3462     if (drv->bdrv_co_truncate) {
3463         if (flags & ~bs->supported_truncate_flags) {
3464             error_setg(errp, "Block driver does not support requested flags");
3465             ret = -ENOTSUP;
3466             goto out;
3467         }
3468         ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, flags, errp);
3469     } else if (filtered) {
3470         ret = bdrv_co_truncate(filtered, offset, exact, prealloc, flags, errp);
3471     } else {
3472         error_setg(errp, "Image format driver does not support resize");
3473         ret = -ENOTSUP;
3474         goto out;
3475     }
3476     if (ret < 0) {
3477         goto out;
3478     }
3479
3480     ret = bdrv_co_refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
3481     if (ret < 0) {
3482         error_setg_errno(errp, -ret, "Could not refresh total sector count");
3483     } else {
3484         offset = bs->total_sectors * BDRV_SECTOR_SIZE;
3485     }
3486     /*
3487      * It's possible that truncation succeeded but bdrv_refresh_total_sectors
3488      * failed, but the latter doesn't affect how we should finish the request.
3489      * Pass 0 as the last parameter so that dirty bitmaps etc. are handled.
3490      */
3491     bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0);
3492
3493 out:
3494     tracked_request_end(&req);
3495     bdrv_dec_in_flight(bs);
3496
3497     return ret;
3498 }
3499
3500 void bdrv_cancel_in_flight(BlockDriverState *bs)
3501 {
3502     GLOBAL_STATE_CODE();
3503     if (!bs || !bs->drv) {
3504         return;
3505     }
3506
3507     if (bs->drv->bdrv_cancel_in_flight) {
3508         bs->drv->bdrv_cancel_in_flight(bs);
3509     }
3510 }
3511
3512 int coroutine_fn
3513 bdrv_co_preadv_snapshot(BdrvChild *child, int64_t offset, int64_t bytes,
3514                         QEMUIOVector *qiov, size_t qiov_offset)
3515 {
3516     BlockDriverState *bs = child->bs;
3517     BlockDriver *drv = bs->drv;
3518     int ret;
3519     IO_CODE();
3520
3521     if (!drv) {
3522         return -ENOMEDIUM;
3523     }
3524
3525     if (!drv->bdrv_co_preadv_snapshot) {
3526         return -ENOTSUP;
3527     }
3528
3529     bdrv_inc_in_flight(bs);
3530     ret = drv->bdrv_co_preadv_snapshot(bs, offset, bytes, qiov, qiov_offset);
3531     bdrv_dec_in_flight(bs);
3532
3533     return ret;
3534 }
3535
3536 int coroutine_fn
3537 bdrv_co_snapshot_block_status(BlockDriverState *bs,
3538                               bool want_zero, int64_t offset, int64_t bytes,
3539                               int64_t *pnum, int64_t *map,
3540                               BlockDriverState **file)
3541 {
3542     BlockDriver *drv = bs->drv;
3543     int ret;
3544     IO_CODE();
3545
3546     if (!drv) {
3547         return -ENOMEDIUM;
3548     }
3549
3550     if (!drv->bdrv_co_snapshot_block_status) {
3551         return -ENOTSUP;
3552     }
3553
3554     bdrv_inc_in_flight(bs);
3555     ret = drv->bdrv_co_snapshot_block_status(bs, want_zero, offset, bytes,
3556                                              pnum, map, file);
3557     bdrv_dec_in_flight(bs);
3558
3559     return ret;
3560 }
3561
3562 int coroutine_fn
3563 bdrv_co_pdiscard_snapshot(BlockDriverState *bs, int64_t offset, int64_t bytes)
3564 {
3565     BlockDriver *drv = bs->drv;
3566     int ret;
3567     IO_CODE();
3568
3569     if (!drv) {
3570         return -ENOMEDIUM;
3571     }
3572
3573     if (!drv->bdrv_co_pdiscard_snapshot) {
3574         return -ENOTSUP;
3575     }
3576
3577     bdrv_inc_in_flight(bs);
3578     ret = drv->bdrv_co_pdiscard_snapshot(bs, offset, bytes);
3579     bdrv_dec_in_flight(bs);
3580
3581     return ret;
3582 }