block/io.c

   1 /*
   2  * Block layer I/O functions
   3  *
   4  * Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24
  25 #include "qemu/osdep.h"
  26 #include "trace.h"
  27 #include "sysemu/block-backend.h"
  28 #include "block/aio-wait.h"
  29 #include "block/blockjob.h"
  30 #include "block/blockjob_int.h"
  31 #include "block/block_int.h"
  32 #include "block/coroutines.h"
  33 #include "block/dirty-bitmap.h"
  34 #include "block/write-threshold.h"
  35 #include "qemu/cutils.h"
  36 #include "qemu/memalign.h"
  37 #include "qapi/error.h"
  38 #include "qemu/error-report.h"
  39 #include "qemu/main-loop.h"
  40 #include "sysemu/replay.h"
  41
  42 /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */
  43 #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS)
  44
  45 static void bdrv_parent_cb_resize(BlockDriverState *bs);
  46 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
  47     int64_t offset, int64_t bytes, BdrvRequestFlags flags);
  48
  49 static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore)
  50 {
  51     BdrvChild *c, *next;
  52
  53     QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
  54         if (c == ignore) {
  55             continue;
  56         }
  57         bdrv_parent_drained_begin_single(c);
  58     }
  59 }
  60
  61 void bdrv_parent_drained_end_single(BdrvChild *c)
  62 {
  63     IO_OR_GS_CODE();
  64
  65     assert(c->quiesced_parent);
  66     c->quiesced_parent = false;
  67
  68     if (c->klass->drained_end) {
  69         c->klass->drained_end(c);
  70     }
  71 }
  72
  73 static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore)
  74 {
  75     BdrvChild *c;
  76
  77     QLIST_FOREACH(c, &bs->parents, next_parent) {
  78         if (c == ignore) {
  79             continue;
  80         }
  81         bdrv_parent_drained_end_single(c);
  82     }
  83 }
  84
  85 bool bdrv_parent_drained_poll_single(BdrvChild *c)
  86 {
  87     if (c->klass->drained_poll) {
  88         return c->klass->drained_poll(c);
  89     }
  90     return false;
  91 }
  92
  93 static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore,
  94                                      bool ignore_bds_parents)
  95 {
  96     BdrvChild *c, *next;
  97     bool busy = false;
  98
  99     QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
 100         if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
 101             continue;
 102         }
 103         busy |= bdrv_parent_drained_poll_single(c);
 104     }
 105
 106     return busy;
 107 }
 108
 109 void bdrv_parent_drained_begin_single(BdrvChild *c)
 110 {
 111     IO_OR_GS_CODE();
 112
 113     assert(!c->quiesced_parent);
 114     c->quiesced_parent = true;
 115
 116     if (c->klass->drained_begin) {
 117         c->klass->drained_begin(c);
 118     }
 119 }
 120
 121 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src)
 122 {
 123     dst->pdiscard_alignment = MAX(dst->pdiscard_alignment,
 124                                   src->pdiscard_alignment);
 125     dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer);
 126     dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer);
 127     dst->max_hw_transfer = MIN_NON_ZERO(dst->max_hw_transfer,
 128                                         src->max_hw_transfer);
 129     dst->opt_mem_alignment = MAX(dst->opt_mem_alignment,
 130                                  src->opt_mem_alignment);
 131     dst->min_mem_alignment = MAX(dst->min_mem_alignment,
 132                                  src->min_mem_alignment);
 133     dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov);
 134     dst->max_hw_iov = MIN_NON_ZERO(dst->max_hw_iov, src->max_hw_iov);
 135 }
 136
 137 typedef struct BdrvRefreshLimitsState {
 138     BlockDriverState *bs;
 139     BlockLimits old_bl;
 140 } BdrvRefreshLimitsState;
 141
 142 static void bdrv_refresh_limits_abort(void *opaque)
 143 {
 144     BdrvRefreshLimitsState *s = opaque;
 145
 146     s->bs->bl = s->old_bl;
 147 }
 148
 149 static TransactionActionDrv bdrv_refresh_limits_drv = {
 150     .abort = bdrv_refresh_limits_abort,
 151     .clean = g_free,
 152 };
 153
 154 /* @tran is allowed to be NULL, in this case no rollback is possible. */
 155 void bdrv_refresh_limits(BlockDriverState *bs, Transaction *tran, Error **errp)
 156 {
 157     ERRP_GUARD();
 158     BlockDriver *drv = bs->drv;
 159     BdrvChild *c;
 160     bool have_limits;
 161
 162     GLOBAL_STATE_CODE();
 163
 164     if (tran) {
 165         BdrvRefreshLimitsState *s = g_new(BdrvRefreshLimitsState, 1);
 166         *s = (BdrvRefreshLimitsState) {
 167             .bs = bs,
 168             .old_bl = bs->bl,
 169         };
 170         tran_add(tran, &bdrv_refresh_limits_drv, s);
 171     }
 172
 173     memset(&bs->bl, 0, sizeof(bs->bl));
 174
 175     if (!drv) {
 176         return;
 177     }
 178
 179     /* Default alignment based on whether driver has byte interface */
 180     bs->bl.request_alignment = (drv->bdrv_co_preadv ||
 181                                 drv->bdrv_aio_preadv ||
 182                                 drv->bdrv_co_preadv_part) ? 1 : 512;
 183
 184     /* Take some limits from the children as a default */
 185     have_limits = false;
 186     QLIST_FOREACH(c, &bs->children, next) {
 187         if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW))
 188         {
 189             bdrv_merge_limits(&bs->bl, &c->bs->bl);
 190             have_limits = true;
 191         }
 192     }
 193
 194     if (!have_limits) {
 195         bs->bl.min_mem_alignment = 512;
 196         bs->bl.opt_mem_alignment = qemu_real_host_page_size();
 197
 198         /* Safe default since most protocols use readv()/writev()/etc */
 199         bs->bl.max_iov = IOV_MAX;
 200     }
 201
 202     /* Then let the driver override it */
 203     if (drv->bdrv_refresh_limits) {
 204         drv->bdrv_refresh_limits(bs, errp);
 205         if (*errp) {
 206             return;
 207         }
 208     }
 209
 210     if (bs->bl.request_alignment > BDRV_MAX_ALIGNMENT) {
 211         error_setg(errp, "Driver requires too large request alignment");
 212     }
 213 }
 214
 215 /**
 216  * The copy-on-read flag is actually a reference count so multiple users may
 217  * use the feature without worrying about clobbering its previous state.
 218  * Copy-on-read stays enabled until all users have called to disable it.
 219  */
 220 void bdrv_enable_copy_on_read(BlockDriverState *bs)
 221 {
 222     IO_CODE();
 223     qatomic_inc(&bs->copy_on_read);
 224 }
 225
 226 void bdrv_disable_copy_on_read(BlockDriverState *bs)
 227 {
 228     int old = qatomic_fetch_dec(&bs->copy_on_read);
 229     IO_CODE();
 230     assert(old >= 1);
 231 }
 232
 233 typedef struct {
 234     Coroutine *co;
 235     BlockDriverState *bs;
 236     bool done;
 237     bool begin;
 238     bool poll;
 239     BdrvChild *parent;
 240 } BdrvCoDrainData;
 241
 242 /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */
 243 bool bdrv_drain_poll(BlockDriverState *bs, BdrvChild *ignore_parent,
 244                      bool ignore_bds_parents)
 245 {
 246     IO_OR_GS_CODE();
 247
 248     if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) {
 249         return true;
 250     }
 251
 252     if (qatomic_read(&bs->in_flight)) {
 253         return true;
 254     }
 255
 256     return false;
 257 }
 258
 259 static bool bdrv_drain_poll_top_level(BlockDriverState *bs,
 260                                       BdrvChild *ignore_parent)
 261 {
 262     return bdrv_drain_poll(bs, ignore_parent, false);
 263 }
 264
 265 static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent,
 266                                   bool poll);
 267 static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent);
 268
 269 static void bdrv_co_drain_bh_cb(void *opaque)
 270 {
 271     BdrvCoDrainData *data = opaque;
 272     Coroutine *co = data->co;
 273     BlockDriverState *bs = data->bs;
 274
 275     if (bs) {
 276         AioContext *ctx = bdrv_get_aio_context(bs);
 277         aio_context_acquire(ctx);
 278         bdrv_dec_in_flight(bs);
 279         if (data->begin) {
 280             bdrv_do_drained_begin(bs, data->parent, data->poll);
 281         } else {
 282             assert(!data->poll);
 283             bdrv_do_drained_end(bs, data->parent);
 284         }
 285         aio_context_release(ctx);
 286     } else {
 287         assert(data->begin);
 288         bdrv_drain_all_begin();
 289     }
 290
 291     data->done = true;
 292     aio_co_wake(co);
 293 }
 294
 295 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
 296                                                 bool begin,
 297                                                 BdrvChild *parent,
 298                                                 bool poll)
 299 {
 300     BdrvCoDrainData data;
 301     Coroutine *self = qemu_coroutine_self();
 302     AioContext *ctx = bdrv_get_aio_context(bs);
 303     AioContext *co_ctx = qemu_coroutine_get_aio_context(self);
 304
 305     /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
 306      * other coroutines run if they were queued by aio_co_enter(). */
 307
 308     assert(qemu_in_coroutine());
 309     data = (BdrvCoDrainData) {
 310         .co = self,
 311         .bs = bs,
 312         .done = false,
 313         .begin = begin,
 314         .parent = parent,
 315         .poll = poll,
 316     };
 317
 318     if (bs) {
 319         bdrv_inc_in_flight(bs);
 320     }
 321
 322     /*
 323      * Temporarily drop the lock across yield or we would get deadlocks.
 324      * bdrv_co_drain_bh_cb() reaquires the lock as needed.
 325      *
 326      * When we yield below, the lock for the current context will be
 327      * released, so if this is actually the lock that protects bs, don't drop
 328      * it a second time.
 329      */
 330     if (ctx != co_ctx) {
 331         aio_context_release(ctx);
 332     }
 333     replay_bh_schedule_oneshot_event(ctx, bdrv_co_drain_bh_cb, &data);
 334
 335     qemu_coroutine_yield();
 336     /* If we are resumed from some other event (such as an aio completion or a
 337      * timer callback), it is a bug in the caller that should be fixed. */
 338     assert(data.done);
 339
 340     /* Reaquire the AioContext of bs if we dropped it */
 341     if (ctx != co_ctx) {
 342         aio_context_acquire(ctx);
 343     }
 344 }
 345
 346 static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent,
 347                                   bool poll)
 348 {
 349     IO_OR_GS_CODE();
 350
 351     if (qemu_in_coroutine()) {
 352         bdrv_co_yield_to_drain(bs, true, parent, poll);
 353         return;
 354     }
 355
 356     /* Stop things in parent-to-child order */
 357     if (qatomic_fetch_inc(&bs->quiesce_counter) == 0) {
 358         aio_disable_external(bdrv_get_aio_context(bs));
 359         bdrv_parent_drained_begin(bs, parent);
 360         if (bs->drv && bs->drv->bdrv_drain_begin) {
 361             bs->drv->bdrv_drain_begin(bs);
 362         }
 363     }
 364
 365     /*
 366      * Wait for drained requests to finish.
 367      *
 368      * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The
 369      * call is needed so things in this AioContext can make progress even
 370      * though we don't return to the main AioContext loop - this automatically
 371      * includes other nodes in the same AioContext and therefore all child
 372      * nodes.
 373      */
 374     if (poll) {
 375         BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, parent));
 376     }
 377 }
 378
 379 void bdrv_do_drained_begin_quiesce(BlockDriverState *bs, BdrvChild *parent)
 380 {
 381     bdrv_do_drained_begin(bs, parent, false);
 382 }
 383
 384 void bdrv_drained_begin(BlockDriverState *bs)
 385 {
 386     IO_OR_GS_CODE();
 387     bdrv_do_drained_begin(bs, NULL, true);
 388 }
 389
 390 /**
 391  * This function does not poll, nor must any of its recursively called
 392  * functions.
 393  */
 394 static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent)
 395 {
 396     int old_quiesce_counter;
 397
 398     if (qemu_in_coroutine()) {
 399         bdrv_co_yield_to_drain(bs, false, parent, false);
 400         return;
 401     }
 402     assert(bs->quiesce_counter > 0);
 403
 404     /* Re-enable things in child-to-parent order */
 405     old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter);
 406     if (old_quiesce_counter == 1) {
 407         if (bs->drv && bs->drv->bdrv_drain_end) {
 408             bs->drv->bdrv_drain_end(bs);
 409         }
 410         bdrv_parent_drained_end(bs, parent);
 411         aio_enable_external(bdrv_get_aio_context(bs));
 412     }
 413 }
 414
 415 void bdrv_drained_end(BlockDriverState *bs)
 416 {
 417     IO_OR_GS_CODE();
 418     bdrv_do_drained_end(bs, NULL);
 419 }
 420
 421 void bdrv_drain(BlockDriverState *bs)
 422 {
 423     IO_OR_GS_CODE();
 424     bdrv_drained_begin(bs);
 425     bdrv_drained_end(bs);
 426 }
 427
 428 static void bdrv_drain_assert_idle(BlockDriverState *bs)
 429 {
 430     BdrvChild *child, *next;
 431
 432     assert(qatomic_read(&bs->in_flight) == 0);
 433     QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
 434         bdrv_drain_assert_idle(child->bs);
 435     }
 436 }
 437
 438 unsigned int bdrv_drain_all_count = 0;
 439
 440 static bool bdrv_drain_all_poll(void)
 441 {
 442     BlockDriverState *bs = NULL;
 443     bool result = false;
 444     GLOBAL_STATE_CODE();
 445
 446     /* bdrv_drain_poll() can't make changes to the graph and we are holding the
 447      * main AioContext lock, so iterating bdrv_next_all_states() is safe. */
 448     while ((bs = bdrv_next_all_states(bs))) {
 449         AioContext *aio_context = bdrv_get_aio_context(bs);
 450         aio_context_acquire(aio_context);
 451         result |= bdrv_drain_poll(bs, NULL, true);
 452         aio_context_release(aio_context);
 453     }
 454
 455     return result;
 456 }
 457
 458 /*
 459  * Wait for pending requests to complete across all BlockDriverStates
 460  *
 461  * This function does not flush data to disk, use bdrv_flush_all() for that
 462  * after calling this function.
 463  *
 464  * This pauses all block jobs and disables external clients. It must
 465  * be paired with bdrv_drain_all_end().
 466  *
 467  * NOTE: no new block jobs or BlockDriverStates can be created between
 468  * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls.
 469  */
 470 void bdrv_drain_all_begin_nopoll(void)
 471 {
 472     BlockDriverState *bs = NULL;
 473     GLOBAL_STATE_CODE();
 474
 475     /*
 476      * bdrv queue is managed by record/replay,
 477      * waiting for finishing the I/O requests may
 478      * be infinite
 479      */
 480     if (replay_events_enabled()) {
 481         return;
 482     }
 483
 484     /* AIO_WAIT_WHILE() with a NULL context can only be called from the main
 485      * loop AioContext, so make sure we're in the main context. */
 486     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
 487     assert(bdrv_drain_all_count < INT_MAX);
 488     bdrv_drain_all_count++;
 489
 490     /* Quiesce all nodes, without polling in-flight requests yet. The graph
 491      * cannot change during this loop. */
 492     while ((bs = bdrv_next_all_states(bs))) {
 493         AioContext *aio_context = bdrv_get_aio_context(bs);
 494
 495         aio_context_acquire(aio_context);
 496         bdrv_do_drained_begin(bs, NULL, false);
 497         aio_context_release(aio_context);
 498     }
 499 }
 500
 501 void bdrv_drain_all_begin(void)
 502 {
 503     BlockDriverState *bs = NULL;
 504
 505     if (qemu_in_coroutine()) {
 506         bdrv_co_yield_to_drain(NULL, true, NULL, true);
 507         return;
 508     }
 509
 510     /*
 511      * bdrv queue is managed by record/replay,
 512      * waiting for finishing the I/O requests may
 513      * be infinite
 514      */
 515     if (replay_events_enabled()) {
 516         return;
 517     }
 518
 519     bdrv_drain_all_begin_nopoll();
 520
 521     /* Now poll the in-flight requests */
 522     AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll());
 523
 524     while ((bs = bdrv_next_all_states(bs))) {
 525         bdrv_drain_assert_idle(bs);
 526     }
 527 }
 528
 529 void bdrv_drain_all_end_quiesce(BlockDriverState *bs)
 530 {
 531     GLOBAL_STATE_CODE();
 532
 533     g_assert(bs->quiesce_counter > 0);
 534     g_assert(!bs->refcnt);
 535
 536     while (bs->quiesce_counter) {
 537         bdrv_do_drained_end(bs, NULL);
 538     }
 539 }
 540
 541 void bdrv_drain_all_end(void)
 542 {
 543     BlockDriverState *bs = NULL;
 544     GLOBAL_STATE_CODE();
 545
 546     /*
 547      * bdrv queue is managed by record/replay,
 548      * waiting for finishing the I/O requests may
 549      * be endless
 550      */
 551     if (replay_events_enabled()) {
 552         return;
 553     }
 554
 555     while ((bs = bdrv_next_all_states(bs))) {
 556         AioContext *aio_context = bdrv_get_aio_context(bs);
 557
 558         aio_context_acquire(aio_context);
 559         bdrv_do_drained_end(bs, NULL);
 560         aio_context_release(aio_context);
 561     }
 562
 563     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
 564     assert(bdrv_drain_all_count > 0);
 565     bdrv_drain_all_count--;
 566 }
 567
 568 void bdrv_drain_all(void)
 569 {
 570     GLOBAL_STATE_CODE();
 571     bdrv_drain_all_begin();
 572     bdrv_drain_all_end();
 573 }
 574
 575 /**
 576  * Remove an active request from the tracked requests list
 577  *
 578  * This function should be called when a tracked request is completing.
 579  */
 580 static void coroutine_fn tracked_request_end(BdrvTrackedRequest *req)
 581 {
 582     if (req->serialising) {
 583         qatomic_dec(&req->bs->serialising_in_flight);
 584     }
 585
 586     qemu_co_mutex_lock(&req->bs->reqs_lock);
 587     QLIST_REMOVE(req, list);
 588     qemu_co_queue_restart_all(&req->wait_queue);
 589     qemu_co_mutex_unlock(&req->bs->reqs_lock);
 590 }
 591
 592 /**
 593  * Add an active request to the tracked requests list
 594  */
 595 static void coroutine_fn tracked_request_begin(BdrvTrackedRequest *req,
 596                                                BlockDriverState *bs,
 597                                                int64_t offset,
 598                                                int64_t bytes,
 599                                                enum BdrvTrackedRequestType type)
 600 {
 601     bdrv_check_request(offset, bytes, &error_abort);
 602
 603     *req = (BdrvTrackedRequest){
 604         .bs = bs,
 605         .offset         = offset,
 606         .bytes          = bytes,
 607         .type           = type,
 608         .co             = qemu_coroutine_self(),
 609         .serialising    = false,
 610         .overlap_offset = offset,
 611         .overlap_bytes  = bytes,
 612     };
 613
 614     qemu_co_queue_init(&req->wait_queue);
 615
 616     qemu_co_mutex_lock(&bs->reqs_lock);
 617     QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
 618     qemu_co_mutex_unlock(&bs->reqs_lock);
 619 }
 620
 621 static bool tracked_request_overlaps(BdrvTrackedRequest *req,
 622                                      int64_t offset, int64_t bytes)
 623 {
 624     bdrv_check_request(offset, bytes, &error_abort);
 625
 626     /*        aaaa   bbbb */
 627     if (offset >= req->overlap_offset + req->overlap_bytes) {
 628         return false;
 629     }
 630     /* bbbb   aaaa        */
 631     if (req->overlap_offset >= offset + bytes) {
 632         return false;
 633     }
 634     return true;
 635 }
 636
 637 /* Called with self->bs->reqs_lock held */
 638 static coroutine_fn BdrvTrackedRequest *
 639 bdrv_find_conflicting_request(BdrvTrackedRequest *self)
 640 {
 641     BdrvTrackedRequest *req;
 642
 643     QLIST_FOREACH(req, &self->bs->tracked_requests, list) {
 644         if (req == self || (!req->serialising && !self->serialising)) {
 645             continue;
 646         }
 647         if (tracked_request_overlaps(req, self->overlap_offset,
 648                                      self->overlap_bytes))
 649         {
 650             /*
 651              * Hitting this means there was a reentrant request, for
 652              * example, a block driver issuing nested requests.  This must
 653              * never happen since it means deadlock.
 654              */
 655             assert(qemu_coroutine_self() != req->co);
 656
 657             /*
 658              * If the request is already (indirectly) waiting for us, or
 659              * will wait for us as soon as it wakes up, then just go on
 660              * (instead of producing a deadlock in the former case).
 661              */
 662             if (!req->waiting_for) {
 663                 return req;
 664             }
 665         }
 666     }
 667
 668     return NULL;
 669 }
 670
 671 /* Called with self->bs->reqs_lock held */
 672 static void coroutine_fn
 673 bdrv_wait_serialising_requests_locked(BdrvTrackedRequest *self)
 674 {
 675     BdrvTrackedRequest *req;
 676
 677     while ((req = bdrv_find_conflicting_request(self))) {
 678         self->waiting_for = req;
 679         qemu_co_queue_wait(&req->wait_queue, &self->bs->reqs_lock);
 680         self->waiting_for = NULL;
 681     }
 682 }
 683
 684 /* Called with req->bs->reqs_lock held */
 685 static void tracked_request_set_serialising(BdrvTrackedRequest *req,
 686                                             uint64_t align)
 687 {
 688     int64_t overlap_offset = req->offset & ~(align - 1);
 689     int64_t overlap_bytes =
 690         ROUND_UP(req->offset + req->bytes, align) - overlap_offset;
 691
 692     bdrv_check_request(req->offset, req->bytes, &error_abort);
 693
 694     if (!req->serialising) {
 695         qatomic_inc(&req->bs->serialising_in_flight);
 696         req->serialising = true;
 697     }
 698
 699     req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
 700     req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
 701 }
 702
 703 /**
 704  * Return the tracked request on @bs for the current coroutine, or
 705  * NULL if there is none.
 706  */
 707 BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs)
 708 {
 709     BdrvTrackedRequest *req;
 710     Coroutine *self = qemu_coroutine_self();
 711     IO_CODE();
 712
 713     QLIST_FOREACH(req, &bs->tracked_requests, list) {
 714         if (req->co == self) {
 715             return req;
 716         }
 717     }
 718
 719     return NULL;
 720 }
 721
 722 /**
 723  * Round a region to cluster boundaries
 724  */
 725 void bdrv_round_to_clusters(BlockDriverState *bs,
 726                             int64_t offset, int64_t bytes,
 727                             int64_t *cluster_offset,
 728                             int64_t *cluster_bytes)
 729 {
 730     BlockDriverInfo bdi;
 731     IO_CODE();
 732     if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
 733         *cluster_offset = offset;
 734         *cluster_bytes = bytes;
 735     } else {
 736         int64_t c = bdi.cluster_size;
 737         *cluster_offset = QEMU_ALIGN_DOWN(offset, c);
 738         *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c);
 739     }
 740 }
 741
 742 static int bdrv_get_cluster_size(BlockDriverState *bs)
 743 {
 744     BlockDriverInfo bdi;
 745     int ret;
 746
 747     ret = bdrv_get_info(bs, &bdi);
 748     if (ret < 0 || bdi.cluster_size == 0) {
 749         return bs->bl.request_alignment;
 750     } else {
 751         return bdi.cluster_size;
 752     }
 753 }
 754
 755 void bdrv_inc_in_flight(BlockDriverState *bs)
 756 {
 757     IO_CODE();
 758     qatomic_inc(&bs->in_flight);
 759 }
 760
 761 void bdrv_wakeup(BlockDriverState *bs)
 762 {
 763     IO_CODE();
 764     aio_wait_kick();
 765 }
 766
 767 void bdrv_dec_in_flight(BlockDriverState *bs)
 768 {
 769     IO_CODE();
 770     qatomic_dec(&bs->in_flight);
 771     bdrv_wakeup(bs);
 772 }
 773
 774 static void coroutine_fn
 775 bdrv_wait_serialising_requests(BdrvTrackedRequest *self)
 776 {
 777     BlockDriverState *bs = self->bs;
 778
 779     if (!qatomic_read(&bs->serialising_in_flight)) {
 780         return;
 781     }
 782
 783     qemu_co_mutex_lock(&bs->reqs_lock);
 784     bdrv_wait_serialising_requests_locked(self);
 785     qemu_co_mutex_unlock(&bs->reqs_lock);
 786 }
 787
 788 void coroutine_fn bdrv_make_request_serialising(BdrvTrackedRequest *req,
 789                                                 uint64_t align)
 790 {
 791     IO_CODE();
 792
 793     qemu_co_mutex_lock(&req->bs->reqs_lock);
 794
 795     tracked_request_set_serialising(req, align);
 796     bdrv_wait_serialising_requests_locked(req);
 797
 798     qemu_co_mutex_unlock(&req->bs->reqs_lock);
 799 }
 800
 801 int bdrv_check_qiov_request(int64_t offset, int64_t bytes,
 802                             QEMUIOVector *qiov, size_t qiov_offset,
 803                             Error **errp)
 804 {
 805     /*
 806      * Check generic offset/bytes correctness
 807      */
 808
 809     if (offset < 0) {
 810         error_setg(errp, "offset is negative: %" PRIi64, offset);
 811         return -EIO;
 812     }
 813
 814     if (bytes < 0) {
 815         error_setg(errp, "bytes is negative: %" PRIi64, bytes);
 816         return -EIO;
 817     }
 818
 819     if (bytes > BDRV_MAX_LENGTH) {
 820         error_setg(errp, "bytes(%" PRIi64 ") exceeds maximum(%" PRIi64 ")",
 821                    bytes, BDRV_MAX_LENGTH);
 822         return -EIO;
 823     }
 824
 825     if (offset > BDRV_MAX_LENGTH) {
 826         error_setg(errp, "offset(%" PRIi64 ") exceeds maximum(%" PRIi64 ")",
 827                    offset, BDRV_MAX_LENGTH);
 828         return -EIO;
 829     }
 830
 831     if (offset > BDRV_MAX_LENGTH - bytes) {
 832         error_setg(errp, "sum of offset(%" PRIi64 ") and bytes(%" PRIi64 ") "
 833                    "exceeds maximum(%" PRIi64 ")", offset, bytes,
 834                    BDRV_MAX_LENGTH);
 835         return -EIO;
 836     }
 837
 838     if (!qiov) {
 839         return 0;
 840     }
 841
 842     /*
 843      * Check qiov and qiov_offset
 844      */
 845
 846     if (qiov_offset > qiov->size) {
 847         error_setg(errp, "qiov_offset(%zu) overflow io vector size(%zu)",
 848                    qiov_offset, qiov->size);
 849         return -EIO;
 850     }
 851
 852     if (bytes > qiov->size - qiov_offset) {
 853         error_setg(errp, "bytes(%" PRIi64 ") + qiov_offset(%zu) overflow io "
 854                    "vector size(%zu)", bytes, qiov_offset, qiov->size);
 855         return -EIO;
 856     }
 857
 858     return 0;
 859 }
 860
 861 int bdrv_check_request(int64_t offset, int64_t bytes, Error **errp)
 862 {
 863     return bdrv_check_qiov_request(offset, bytes, NULL, 0, errp);
 864 }
 865
 866 static int bdrv_check_request32(int64_t offset, int64_t bytes,
 867                                 QEMUIOVector *qiov, size_t qiov_offset)
 868 {
 869     int ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL);
 870     if (ret < 0) {
 871         return ret;
 872     }
 873
 874     if (bytes > BDRV_REQUEST_MAX_BYTES) {
 875         return -EIO;
 876     }
 877
 878     return 0;
 879 }
 880
 881 /*
 882  * Completely zero out a block device with the help of bdrv_pwrite_zeroes.
 883  * The operation is sped up by checking the block status and only writing
 884  * zeroes to the device if they currently do not return zeroes. Optional
 885  * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
 886  * BDRV_REQ_FUA).
 887  *
 888  * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite().
 889  */
 890 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
 891 {
 892     int ret;
 893     int64_t target_size, bytes, offset = 0;
 894     BlockDriverState *bs = child->bs;
 895     IO_CODE();
 896
 897     target_size = bdrv_getlength(bs);
 898     if (target_size < 0) {
 899         return target_size;
 900     }
 901
 902     for (;;) {
 903         bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES);
 904         if (bytes <= 0) {
 905             return 0;
 906         }
 907         ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL);
 908         if (ret < 0) {
 909             return ret;
 910         }
 911         if (ret & BDRV_BLOCK_ZERO) {
 912             offset += bytes;
 913             continue;
 914         }
 915         ret = bdrv_pwrite_zeroes(child, offset, bytes, flags);
 916         if (ret < 0) {
 917             return ret;
 918         }
 919         offset += bytes;
 920     }
 921 }
 922
 923 /*
 924  * Writes to the file and ensures that no writes are reordered across this
 925  * request (acts as a barrier)
 926  *
 927  * Returns 0 on success, -errno in error cases.
 928  */
 929 int coroutine_fn bdrv_co_pwrite_sync(BdrvChild *child, int64_t offset,
 930                                      int64_t bytes, const void *buf,
 931                                      BdrvRequestFlags flags)
 932 {
 933     int ret;
 934     IO_CODE();
 935
 936     ret = bdrv_co_pwrite(child, offset, bytes, buf, flags);
 937     if (ret < 0) {
 938         return ret;
 939     }
 940
 941     ret = bdrv_co_flush(child->bs);
 942     if (ret < 0) {
 943         return ret;
 944     }
 945
 946     return 0;
 947 }
 948
 949 typedef struct CoroutineIOCompletion {
 950     Coroutine *coroutine;
 951     int ret;
 952 } CoroutineIOCompletion;
 953
 954 static void bdrv_co_io_em_complete(void *opaque, int ret)
 955 {
 956     CoroutineIOCompletion *co = opaque;
 957
 958     co->ret = ret;
 959     aio_co_wake(co->coroutine);
 960 }
 961
 962 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
 963                                            int64_t offset, int64_t bytes,
 964                                            QEMUIOVector *qiov,
 965                                            size_t qiov_offset, int flags)
 966 {
 967     BlockDriver *drv = bs->drv;
 968     int64_t sector_num;
 969     unsigned int nb_sectors;
 970     QEMUIOVector local_qiov;
 971     int ret;
 972
 973     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
 974     assert(!(flags & ~bs->supported_read_flags));
 975
 976     if (!drv) {
 977         return -ENOMEDIUM;
 978     }
 979
 980     if (drv->bdrv_co_preadv_part) {
 981         return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset,
 982                                         flags);
 983     }
 984
 985     if (qiov_offset > 0 || bytes != qiov->size) {
 986         qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
 987         qiov = &local_qiov;
 988     }
 989
 990     if (drv->bdrv_co_preadv) {
 991         ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
 992         goto out;
 993     }
 994
 995     if (drv->bdrv_aio_preadv) {
 996         BlockAIOCB *acb;
 997         CoroutineIOCompletion co = {
 998             .coroutine = qemu_coroutine_self(),
 999         };
1000
1001         acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags,
1002                                    bdrv_co_io_em_complete, &co);
1003         if (acb == NULL) {
1004             ret = -EIO;
1005             goto out;
1006         } else {
1007             qemu_coroutine_yield();
1008             ret = co.ret;
1009             goto out;
1010         }
1011     }
1012
1013     sector_num = offset >> BDRV_SECTOR_BITS;
1014     nb_sectors = bytes >> BDRV_SECTOR_BITS;
1015
1016     assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
1017     assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
1018     assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1019     assert(drv->bdrv_co_readv);
1020
1021     ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
1022
1023 out:
1024     if (qiov == &local_qiov) {
1025         qemu_iovec_destroy(&local_qiov);
1026     }
1027
1028     return ret;
1029 }
1030
1031 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
1032                                             int64_t offset, int64_t bytes,
1033                                             QEMUIOVector *qiov,
1034                                             size_t qiov_offset,
1035                                             BdrvRequestFlags flags)
1036 {
1037     BlockDriver *drv = bs->drv;
1038     bool emulate_fua = false;
1039     int64_t sector_num;
1040     unsigned int nb_sectors;
1041     QEMUIOVector local_qiov;
1042     int ret;
1043
1044     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1045
1046     if (!drv) {
1047         return -ENOMEDIUM;
1048     }
1049
1050     if ((flags & BDRV_REQ_FUA) &&
1051         (~bs->supported_write_flags & BDRV_REQ_FUA)) {
1052         flags &= ~BDRV_REQ_FUA;
1053         emulate_fua = true;
1054     }
1055
1056     flags &= bs->supported_write_flags;
1057
1058     if (drv->bdrv_co_pwritev_part) {
1059         ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset,
1060                                         flags);
1061         goto emulate_flags;
1062     }
1063
1064     if (qiov_offset > 0 || bytes != qiov->size) {
1065         qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1066         qiov = &local_qiov;
1067     }
1068
1069     if (drv->bdrv_co_pwritev) {
1070         ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, flags);
1071         goto emulate_flags;
1072     }
1073
1074     if (drv->bdrv_aio_pwritev) {
1075         BlockAIOCB *acb;
1076         CoroutineIOCompletion co = {
1077             .coroutine = qemu_coroutine_self(),
1078         };
1079
1080         acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov, flags,
1081                                     bdrv_co_io_em_complete, &co);
1082         if (acb == NULL) {
1083             ret = -EIO;
1084         } else {
1085             qemu_coroutine_yield();
1086             ret = co.ret;
1087         }
1088         goto emulate_flags;
1089     }
1090
1091     sector_num = offset >> BDRV_SECTOR_BITS;
1092     nb_sectors = bytes >> BDRV_SECTOR_BITS;
1093
1094     assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
1095     assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
1096     assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1097
1098     assert(drv->bdrv_co_writev);
1099     ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov, flags);
1100
1101 emulate_flags:
1102     if (ret == 0 && emulate_fua) {
1103         ret = bdrv_co_flush(bs);
1104     }
1105
1106     if (qiov == &local_qiov) {
1107         qemu_iovec_destroy(&local_qiov);
1108     }
1109
1110     return ret;
1111 }
1112
1113 static int coroutine_fn
1114 bdrv_driver_pwritev_compressed(BlockDriverState *bs, int64_t offset,
1115                                int64_t bytes, QEMUIOVector *qiov,
1116                                size_t qiov_offset)
1117 {
1118     BlockDriver *drv = bs->drv;
1119     QEMUIOVector local_qiov;
1120     int ret;
1121
1122     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1123
1124     if (!drv) {
1125         return -ENOMEDIUM;
1126     }
1127
1128     if (!block_driver_can_compress(drv)) {
1129         return -ENOTSUP;
1130     }
1131
1132     if (drv->bdrv_co_pwritev_compressed_part) {
1133         return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes,
1134                                                     qiov, qiov_offset);
1135     }
1136
1137     if (qiov_offset == 0) {
1138         return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov);
1139     }
1140
1141     qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1142     ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov);
1143     qemu_iovec_destroy(&local_qiov);
1144
1145     return ret;
1146 }
1147
1148 static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
1149         int64_t offset, int64_t bytes, QEMUIOVector *qiov,
1150         size_t qiov_offset, int flags)
1151 {
1152     BlockDriverState *bs = child->bs;
1153
1154     /* Perform I/O through a temporary buffer so that users who scribble over
1155      * their read buffer while the operation is in progress do not end up
1156      * modifying the image file.  This is critical for zero-copy guest I/O
1157      * where anything might happen inside guest memory.
1158      */
1159     void *bounce_buffer = NULL;
1160
1161     BlockDriver *drv = bs->drv;
1162     int64_t cluster_offset;
1163     int64_t cluster_bytes;
1164     int64_t skip_bytes;
1165     int ret;
1166     int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer,
1167                                     BDRV_REQUEST_MAX_BYTES);
1168     int64_t progress = 0;
1169     bool skip_write;
1170
1171     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1172
1173     if (!drv) {
1174         return -ENOMEDIUM;
1175     }
1176
1177     /*
1178      * Do not write anything when the BDS is inactive.  That is not
1179      * allowed, and it would not help.
1180      */
1181     skip_write = (bs->open_flags & BDRV_O_INACTIVE);
1182
1183     /* FIXME We cannot require callers to have write permissions when all they
1184      * are doing is a read request. If we did things right, write permissions
1185      * would be obtained anyway, but internally by the copy-on-read code. As
1186      * long as it is implemented here rather than in a separate filter driver,
1187      * the copy-on-read code doesn't have its own BdrvChild, however, for which
1188      * it could request permissions. Therefore we have to bypass the permission
1189      * system for the moment. */
1190     // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1191
1192     /* Cover entire cluster so no additional backing file I/O is required when
1193      * allocating cluster in the image file.  Note that this value may exceed
1194      * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which
1195      * is one reason we loop rather than doing it all at once.
1196      */
1197     bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes);
1198     skip_bytes = offset - cluster_offset;
1199
1200     trace_bdrv_co_do_copy_on_readv(bs, offset, bytes,
1201                                    cluster_offset, cluster_bytes);
1202
1203     while (cluster_bytes) {
1204         int64_t pnum;
1205
1206         if (skip_write) {
1207             ret = 1; /* "already allocated", so nothing will be copied */
1208             pnum = MIN(cluster_bytes, max_transfer);
1209         } else {
1210             ret = bdrv_is_allocated(bs, cluster_offset,
1211                                     MIN(cluster_bytes, max_transfer), &pnum);
1212             if (ret < 0) {
1213                 /*
1214                  * Safe to treat errors in querying allocation as if
1215                  * unallocated; we'll probably fail again soon on the
1216                  * read, but at least that will set a decent errno.
1217                  */
1218                 pnum = MIN(cluster_bytes, max_transfer);
1219             }
1220
1221             /* Stop at EOF if the image ends in the middle of the cluster */
1222             if (ret == 0 && pnum == 0) {
1223                 assert(progress >= bytes);
1224                 break;
1225             }
1226
1227             assert(skip_bytes < pnum);
1228         }
1229
1230         if (ret <= 0) {
1231             QEMUIOVector local_qiov;
1232
1233             /* Must copy-on-read; use the bounce buffer */
1234             pnum = MIN(pnum, MAX_BOUNCE_BUFFER);
1235             if (!bounce_buffer) {
1236                 int64_t max_we_need = MAX(pnum, cluster_bytes - pnum);
1237                 int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER);
1238                 int64_t bounce_buffer_len = MIN(max_we_need, max_allowed);
1239
1240                 bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len);
1241                 if (!bounce_buffer) {
1242                     ret = -ENOMEM;
1243                     goto err;
1244                 }
1245             }
1246             qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum);
1247
1248             ret = bdrv_driver_preadv(bs, cluster_offset, pnum,
1249                                      &local_qiov, 0, 0);
1250             if (ret < 0) {
1251                 goto err;
1252             }
1253
1254             bdrv_debug_event(bs, BLKDBG_COR_WRITE);
1255             if (drv->bdrv_co_pwrite_zeroes &&
1256                 buffer_is_zero(bounce_buffer, pnum)) {
1257                 /* FIXME: Should we (perhaps conditionally) be setting
1258                  * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy
1259                  * that still correctly reads as zero? */
1260                 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum,
1261                                                BDRV_REQ_WRITE_UNCHANGED);
1262             } else {
1263                 /* This does not change the data on the disk, it is not
1264                  * necessary to flush even in cache=writethrough mode.
1265                  */
1266                 ret = bdrv_driver_pwritev(bs, cluster_offset, pnum,
1267                                           &local_qiov, 0,
1268                                           BDRV_REQ_WRITE_UNCHANGED);
1269             }
1270
1271             if (ret < 0) {
1272                 /* It might be okay to ignore write errors for guest
1273                  * requests.  If this is a deliberate copy-on-read
1274                  * then we don't want to ignore the error.  Simply
1275                  * report it in all cases.
1276                  */
1277                 goto err;
1278             }
1279
1280             if (!(flags & BDRV_REQ_PREFETCH)) {
1281                 qemu_iovec_from_buf(qiov, qiov_offset + progress,
1282                                     bounce_buffer + skip_bytes,
1283                                     MIN(pnum - skip_bytes, bytes - progress));
1284             }
1285         } else if (!(flags & BDRV_REQ_PREFETCH)) {
1286             /* Read directly into the destination */
1287             ret = bdrv_driver_preadv(bs, offset + progress,
1288                                      MIN(pnum - skip_bytes, bytes - progress),
1289                                      qiov, qiov_offset + progress, 0);
1290             if (ret < 0) {
1291                 goto err;
1292             }
1293         }
1294
1295         cluster_offset += pnum;
1296         cluster_bytes -= pnum;
1297         progress += pnum - skip_bytes;
1298         skip_bytes = 0;
1299     }
1300     ret = 0;
1301
1302 err:
1303     qemu_vfree(bounce_buffer);
1304     return ret;
1305 }
1306
1307 /*
1308  * Forwards an already correctly aligned request to the BlockDriver. This
1309  * handles copy on read, zeroing after EOF, and fragmentation of large
1310  * reads; any other features must be implemented by the caller.
1311  */
1312 static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
1313     BdrvTrackedRequest *req, int64_t offset, int64_t bytes,
1314     int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags)
1315 {
1316     BlockDriverState *bs = child->bs;
1317     int64_t total_bytes, max_bytes;
1318     int ret = 0;
1319     int64_t bytes_remaining = bytes;
1320     int max_transfer;
1321
1322     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1323     assert(is_power_of_2(align));
1324     assert((offset & (align - 1)) == 0);
1325     assert((bytes & (align - 1)) == 0);
1326     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1327     max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
1328                                    align);
1329
1330     /*
1331      * TODO: We would need a per-BDS .supported_read_flags and
1332      * potential fallback support, if we ever implement any read flags
1333      * to pass through to drivers.  For now, there aren't any
1334      * passthrough flags except the BDRV_REQ_REGISTERED_BUF optimization hint.
1335      */
1336     assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH |
1337                        BDRV_REQ_REGISTERED_BUF)));
1338
1339     /* Handle Copy on Read and associated serialisation */
1340     if (flags & BDRV_REQ_COPY_ON_READ) {
1341         /* If we touch the same cluster it counts as an overlap.  This
1342          * guarantees that allocating writes will be serialized and not race
1343          * with each other for the same cluster.  For example, in copy-on-read
1344          * it ensures that the CoR read and write operations are atomic and
1345          * guest writes cannot interleave between them. */
1346         bdrv_make_request_serialising(req, bdrv_get_cluster_size(bs));
1347     } else {
1348         bdrv_wait_serialising_requests(req);
1349     }
1350
1351     if (flags & BDRV_REQ_COPY_ON_READ) {
1352         int64_t pnum;
1353
1354         /* The flag BDRV_REQ_COPY_ON_READ has reached its addressee */
1355         flags &= ~BDRV_REQ_COPY_ON_READ;
1356
1357         ret = bdrv_is_allocated(bs, offset, bytes, &pnum);
1358         if (ret < 0) {
1359             goto out;
1360         }
1361
1362         if (!ret || pnum != bytes) {
1363             ret = bdrv_co_do_copy_on_readv(child, offset, bytes,
1364                                            qiov, qiov_offset, flags);
1365             goto out;
1366         } else if (flags & BDRV_REQ_PREFETCH) {
1367             goto out;
1368         }
1369     }
1370
1371     /* Forward the request to the BlockDriver, possibly fragmenting it */
1372     total_bytes = bdrv_getlength(bs);
1373     if (total_bytes < 0) {
1374         ret = total_bytes;
1375         goto out;
1376     }
1377
1378     assert(!(flags & ~(bs->supported_read_flags | BDRV_REQ_REGISTERED_BUF)));
1379
1380     max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align);
1381     if (bytes <= max_bytes && bytes <= max_transfer) {
1382         ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, flags);
1383         goto out;
1384     }
1385
1386     while (bytes_remaining) {
1387         int64_t num;
1388
1389         if (max_bytes) {
1390             num = MIN(bytes_remaining, MIN(max_bytes, max_transfer));
1391             assert(num);
1392
1393             ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining,
1394                                      num, qiov,
1395                                      qiov_offset + bytes - bytes_remaining,
1396                                      flags);
1397             max_bytes -= num;
1398         } else {
1399             num = bytes_remaining;
1400             ret = qemu_iovec_memset(qiov, qiov_offset + bytes - bytes_remaining,
1401                                     0, bytes_remaining);
1402         }
1403         if (ret < 0) {
1404             goto out;
1405         }
1406         bytes_remaining -= num;
1407     }
1408
1409 out:
1410     return ret < 0 ? ret : 0;
1411 }
1412
1413 /*
1414  * Request padding
1415  *
1416  *  |<---- align ----->|                     |<----- align ---->|
1417  *  |<- head ->|<------------- bytes ------------->|<-- tail -->|
1418  *  |          |       |                     |     |            |
1419  * -*----------$-------*-------- ... --------*-----$------------*---
1420  *  |          |       |                     |     |            |
1421  *  |          offset  |                     |     end          |
1422  *  ALIGN_DOWN(offset) ALIGN_UP(offset)      ALIGN_DOWN(end)   ALIGN_UP(end)
1423  *  [buf   ... )                             [tail_buf          )
1424  *
1425  * @buf is an aligned allocation needed to store @head and @tail paddings. @head
1426  * is placed at the beginning of @buf and @tail at the @end.
1427  *
1428  * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk
1429  * around tail, if tail exists.
1430  *
1431  * @merge_reads is true for small requests,
1432  * if @buf_len == @head + bytes + @tail. In this case it is possible that both
1433  * head and tail exist but @buf_len == align and @tail_buf == @buf.
1434  */
1435 typedef struct BdrvRequestPadding {
1436     uint8_t *buf;
1437     size_t buf_len;
1438     uint8_t *tail_buf;
1439     size_t head;
1440     size_t tail;
1441     bool merge_reads;
1442     QEMUIOVector local_qiov;
1443 } BdrvRequestPadding;
1444
1445 static bool bdrv_init_padding(BlockDriverState *bs,
1446                               int64_t offset, int64_t bytes,
1447                               BdrvRequestPadding *pad)
1448 {
1449     int64_t align = bs->bl.request_alignment;
1450     int64_t sum;
1451
1452     bdrv_check_request(offset, bytes, &error_abort);
1453     assert(align <= INT_MAX); /* documented in block/block_int.h */
1454     assert(align <= SIZE_MAX / 2); /* so we can allocate the buffer */
1455
1456     memset(pad, 0, sizeof(*pad));
1457
1458     pad->head = offset & (align - 1);
1459     pad->tail = ((offset + bytes) & (align - 1));
1460     if (pad->tail) {
1461         pad->tail = align - pad->tail;
1462     }
1463
1464     if (!pad->head && !pad->tail) {
1465         return false;
1466     }
1467
1468     assert(bytes); /* Nothing good in aligning zero-length requests */
1469
1470     sum = pad->head + bytes + pad->tail;
1471     pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align;
1472     pad->buf = qemu_blockalign(bs, pad->buf_len);
1473     pad->merge_reads = sum == pad->buf_len;
1474     if (pad->tail) {
1475         pad->tail_buf = pad->buf + pad->buf_len - align;
1476     }
1477
1478     return true;
1479 }
1480
1481 static coroutine_fn int bdrv_padding_rmw_read(BdrvChild *child,
1482                                               BdrvTrackedRequest *req,
1483                                               BdrvRequestPadding *pad,
1484                                               bool zero_middle)
1485 {
1486     QEMUIOVector local_qiov;
1487     BlockDriverState *bs = child->bs;
1488     uint64_t align = bs->bl.request_alignment;
1489     int ret;
1490
1491     assert(req->serialising && pad->buf);
1492
1493     if (pad->head || pad->merge_reads) {
1494         int64_t bytes = pad->merge_reads ? pad->buf_len : align;
1495
1496         qemu_iovec_init_buf(&local_qiov, pad->buf, bytes);
1497
1498         if (pad->head) {
1499             bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1500         }
1501         if (pad->merge_reads && pad->tail) {
1502             bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1503         }
1504         ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes,
1505                                   align, &local_qiov, 0, 0);
1506         if (ret < 0) {
1507             return ret;
1508         }
1509         if (pad->head) {
1510             bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1511         }
1512         if (pad->merge_reads && pad->tail) {
1513             bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1514         }
1515
1516         if (pad->merge_reads) {
1517             goto zero_mem;
1518         }
1519     }
1520
1521     if (pad->tail) {
1522         qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align);
1523
1524         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1525         ret = bdrv_aligned_preadv(
1526                 child, req,
1527                 req->overlap_offset + req->overlap_bytes - align,
1528                 align, align, &local_qiov, 0, 0);
1529         if (ret < 0) {
1530             return ret;
1531         }
1532         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1533     }
1534
1535 zero_mem:
1536     if (zero_middle) {
1537         memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail);
1538     }
1539
1540     return 0;
1541 }
1542
1543 static void bdrv_padding_destroy(BdrvRequestPadding *pad)
1544 {
1545     if (pad->buf) {
1546         qemu_vfree(pad->buf);
1547         qemu_iovec_destroy(&pad->local_qiov);
1548     }
1549     memset(pad, 0, sizeof(*pad));
1550 }
1551
1552 /*
1553  * bdrv_pad_request
1554  *
1555  * Exchange request parameters with padded request if needed. Don't include RMW
1556  * read of padding, bdrv_padding_rmw_read() should be called separately if
1557  * needed.
1558  *
1559  * Request parameters (@qiov, &qiov_offset, &offset, &bytes) are in-out:
1560  *  - on function start they represent original request
1561  *  - on failure or when padding is not needed they are unchanged
1562  *  - on success when padding is needed they represent padded request
1563  */
1564 static int bdrv_pad_request(BlockDriverState *bs,
1565                             QEMUIOVector **qiov, size_t *qiov_offset,
1566                             int64_t *offset, int64_t *bytes,
1567                             BdrvRequestPadding *pad, bool *padded,
1568                             BdrvRequestFlags *flags)
1569 {
1570     int ret;
1571
1572     bdrv_check_qiov_request(*offset, *bytes, *qiov, *qiov_offset, &error_abort);
1573
1574     if (!bdrv_init_padding(bs, *offset, *bytes, pad)) {
1575         if (padded) {
1576             *padded = false;
1577         }
1578         return 0;
1579     }
1580
1581     ret = qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head,
1582                                    *qiov, *qiov_offset, *bytes,
1583                                    pad->buf + pad->buf_len - pad->tail,
1584                                    pad->tail);
1585     if (ret < 0) {
1586         bdrv_padding_destroy(pad);
1587         return ret;
1588     }
1589     *bytes += pad->head + pad->tail;
1590     *offset -= pad->head;
1591     *qiov = &pad->local_qiov;
1592     *qiov_offset = 0;
1593     if (padded) {
1594         *padded = true;
1595     }
1596     if (flags) {
1597         /* Can't use optimization hint with bounce buffer */
1598         *flags &= ~BDRV_REQ_REGISTERED_BUF;
1599     }
1600
1601     return 0;
1602 }
1603
1604 int coroutine_fn bdrv_co_preadv(BdrvChild *child,
1605     int64_t offset, int64_t bytes, QEMUIOVector *qiov,
1606     BdrvRequestFlags flags)
1607 {
1608     IO_CODE();
1609     return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags);
1610 }
1611
1612 int coroutine_fn bdrv_co_preadv_part(BdrvChild *child,
1613     int64_t offset, int64_t bytes,
1614     QEMUIOVector *qiov, size_t qiov_offset,
1615     BdrvRequestFlags flags)
1616 {
1617     BlockDriverState *bs = child->bs;
1618     BdrvTrackedRequest req;
1619     BdrvRequestPadding pad;
1620     int ret;
1621     IO_CODE();
1622
1623     trace_bdrv_co_preadv_part(bs, offset, bytes, flags);
1624
1625     if (!bdrv_is_inserted(bs)) {
1626         return -ENOMEDIUM;
1627     }
1628
1629     ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset);
1630     if (ret < 0) {
1631         return ret;
1632     }
1633
1634     if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
1635         /*
1636          * Aligning zero request is nonsense. Even if driver has special meaning
1637          * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
1638          * it to driver due to request_alignment.
1639          *
1640          * Still, no reason to return an error if someone do unaligned
1641          * zero-length read occasionally.
1642          */
1643         return 0;
1644     }
1645
1646     bdrv_inc_in_flight(bs);
1647
1648     /* Don't do copy-on-read if we read data before write operation */
1649     if (qatomic_read(&bs->copy_on_read)) {
1650         flags |= BDRV_REQ_COPY_ON_READ;
1651     }
1652
1653     ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad,
1654                            NULL, &flags);
1655     if (ret < 0) {
1656         goto fail;
1657     }
1658
1659     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
1660     ret = bdrv_aligned_preadv(child, &req, offset, bytes,
1661                               bs->bl.request_alignment,
1662                               qiov, qiov_offset, flags);
1663     tracked_request_end(&req);
1664     bdrv_padding_destroy(&pad);
1665
1666 fail:
1667     bdrv_dec_in_flight(bs);
1668
1669     return ret;
1670 }
1671
1672 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
1673     int64_t offset, int64_t bytes, BdrvRequestFlags flags)
1674 {
1675     BlockDriver *drv = bs->drv;
1676     QEMUIOVector qiov;
1677     void *buf = NULL;
1678     int ret = 0;
1679     bool need_flush = false;
1680     int head = 0;
1681     int tail = 0;
1682
1683     int64_t max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes,
1684                                             INT64_MAX);
1685     int alignment = MAX(bs->bl.pwrite_zeroes_alignment,
1686                         bs->bl.request_alignment);
1687     int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER);
1688
1689     bdrv_check_request(offset, bytes, &error_abort);
1690
1691     if (!drv) {
1692         return -ENOMEDIUM;
1693     }
1694
1695     if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) {
1696         return -ENOTSUP;
1697     }
1698
1699     /* By definition there is no user buffer so this flag doesn't make sense */
1700     if (flags & BDRV_REQ_REGISTERED_BUF) {
1701         return -EINVAL;
1702     }
1703
1704     /* Invalidate the cached block-status data range if this write overlaps */
1705     bdrv_bsc_invalidate_range(bs, offset, bytes);
1706
1707     assert(alignment % bs->bl.request_alignment == 0);
1708     head = offset % alignment;
1709     tail = (offset + bytes) % alignment;
1710     max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment);
1711     assert(max_write_zeroes >= bs->bl.request_alignment);
1712
1713     while (bytes > 0 && !ret) {
1714         int64_t num = bytes;
1715
1716         /* Align request.  Block drivers can expect the "bulk" of the request
1717          * to be aligned, and that unaligned requests do not cross cluster
1718          * boundaries.
1719          */
1720         if (head) {
1721             /* Make a small request up to the first aligned sector. For
1722              * convenience, limit this request to max_transfer even if
1723              * we don't need to fall back to writes.  */
1724             num = MIN(MIN(bytes, max_transfer), alignment - head);
1725             head = (head + num) % alignment;
1726             assert(num < max_write_zeroes);
1727         } else if (tail && num > alignment) {
1728             /* Shorten the request to the last aligned sector.  */
1729             num -= tail;
1730         }
1731
1732         /* limit request size */
1733         if (num > max_write_zeroes) {
1734             num = max_write_zeroes;
1735         }
1736
1737         ret = -ENOTSUP;
1738         /* First try the efficient write zeroes operation */
1739         if (drv->bdrv_co_pwrite_zeroes) {
1740             ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num,
1741                                              flags & bs->supported_zero_flags);
1742             if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1743                 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1744                 need_flush = true;
1745             }
1746         } else {
1747             assert(!bs->supported_zero_flags);
1748         }
1749
1750         if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) {
1751             /* Fall back to bounce buffer if write zeroes is unsupported */
1752             BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1753
1754             if ((flags & BDRV_REQ_FUA) &&
1755                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1756                 /* No need for bdrv_driver_pwrite() to do a fallback
1757                  * flush on each chunk; use just one at the end */
1758                 write_flags &= ~BDRV_REQ_FUA;
1759                 need_flush = true;
1760             }
1761             num = MIN(num, max_transfer);
1762             if (buf == NULL) {
1763                 buf = qemu_try_blockalign0(bs, num);
1764                 if (buf == NULL) {
1765                     ret = -ENOMEM;
1766                     goto fail;
1767                 }
1768             }
1769             qemu_iovec_init_buf(&qiov, buf, num);
1770
1771             ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags);
1772
1773             /* Keep bounce buffer around if it is big enough for all
1774              * all future requests.
1775              */
1776             if (num < max_transfer) {
1777                 qemu_vfree(buf);
1778                 buf = NULL;
1779             }
1780         }
1781
1782         offset += num;
1783         bytes -= num;
1784     }
1785
1786 fail:
1787     if (ret == 0 && need_flush) {
1788         ret = bdrv_co_flush(bs);
1789     }
1790     qemu_vfree(buf);
1791     return ret;
1792 }
1793
1794 static inline int coroutine_fn
1795 bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, int64_t bytes,
1796                           BdrvTrackedRequest *req, int flags)
1797 {
1798     BlockDriverState *bs = child->bs;
1799
1800     bdrv_check_request(offset, bytes, &error_abort);
1801
1802     if (bdrv_is_read_only(bs)) {
1803         return -EPERM;
1804     }
1805
1806     assert(!(bs->open_flags & BDRV_O_INACTIVE));
1807     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1808     assert(!(flags & ~BDRV_REQ_MASK));
1809     assert(!((flags & BDRV_REQ_NO_WAIT) && !(flags & BDRV_REQ_SERIALISING)));
1810
1811     if (flags & BDRV_REQ_SERIALISING) {
1812         QEMU_LOCK_GUARD(&bs->reqs_lock);
1813
1814         tracked_request_set_serialising(req, bdrv_get_cluster_size(bs));
1815
1816         if ((flags & BDRV_REQ_NO_WAIT) && bdrv_find_conflicting_request(req)) {
1817             return -EBUSY;
1818         }
1819
1820         bdrv_wait_serialising_requests_locked(req);
1821     } else {
1822         bdrv_wait_serialising_requests(req);
1823     }
1824
1825     assert(req->overlap_offset <= offset);
1826     assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1827     assert(offset + bytes <= bs->total_sectors * BDRV_SECTOR_SIZE ||
1828            child->perm & BLK_PERM_RESIZE);
1829
1830     switch (req->type) {
1831     case BDRV_TRACKED_WRITE:
1832     case BDRV_TRACKED_DISCARD:
1833         if (flags & BDRV_REQ_WRITE_UNCHANGED) {
1834             assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1835         } else {
1836             assert(child->perm & BLK_PERM_WRITE);
1837         }
1838         bdrv_write_threshold_check_write(bs, offset, bytes);
1839         return 0;
1840     case BDRV_TRACKED_TRUNCATE:
1841         assert(child->perm & BLK_PERM_RESIZE);
1842         return 0;
1843     default:
1844         abort();
1845     }
1846 }
1847
1848 static inline void coroutine_fn
1849 bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, int64_t bytes,
1850                          BdrvTrackedRequest *req, int ret)
1851 {
1852     int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
1853     BlockDriverState *bs = child->bs;
1854
1855     bdrv_check_request(offset, bytes, &error_abort);
1856
1857     qatomic_inc(&bs->write_gen);
1858
1859     /*
1860      * Discard cannot extend the image, but in error handling cases, such as
1861      * when reverting a qcow2 cluster allocation, the discarded range can pass
1862      * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD
1863      * here. Instead, just skip it, since semantically a discard request
1864      * beyond EOF cannot expand the image anyway.
1865      */
1866     if (ret == 0 &&
1867         (req->type == BDRV_TRACKED_TRUNCATE ||
1868          end_sector > bs->total_sectors) &&
1869         req->type != BDRV_TRACKED_DISCARD) {
1870         bs->total_sectors = end_sector;
1871         bdrv_parent_cb_resize(bs);
1872         bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS);
1873     }
1874     if (req->bytes) {
1875         switch (req->type) {
1876         case BDRV_TRACKED_WRITE:
1877             stat64_max(&bs->wr_highest_offset, offset + bytes);
1878             /* fall through, to set dirty bits */
1879         case BDRV_TRACKED_DISCARD:
1880             bdrv_set_dirty(bs, offset, bytes);
1881             break;
1882         default:
1883             break;
1884         }
1885     }
1886 }
1887
1888 /*
1889  * Forwards an already correctly aligned write request to the BlockDriver,
1890  * after possibly fragmenting it.
1891  */
1892 static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child,
1893     BdrvTrackedRequest *req, int64_t offset, int64_t bytes,
1894     int64_t align, QEMUIOVector *qiov, size_t qiov_offset,
1895     BdrvRequestFlags flags)
1896 {
1897     BlockDriverState *bs = child->bs;
1898     BlockDriver *drv = bs->drv;
1899     int ret;
1900
1901     int64_t bytes_remaining = bytes;
1902     int max_transfer;
1903
1904     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1905
1906     if (!drv) {
1907         return -ENOMEDIUM;
1908     }
1909
1910     if (bdrv_has_readonly_bitmaps(bs)) {
1911         return -EPERM;
1912     }
1913
1914     assert(is_power_of_2(align));
1915     assert((offset & (align - 1)) == 0);
1916     assert((bytes & (align - 1)) == 0);
1917     max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
1918                                    align);
1919
1920     ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags);
1921
1922     if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
1923         !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes &&
1924         qemu_iovec_is_zero(qiov, qiov_offset, bytes)) {
1925         flags |= BDRV_REQ_ZERO_WRITE;
1926         if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
1927             flags |= BDRV_REQ_MAY_UNMAP;
1928         }
1929     }
1930
1931     if (ret < 0) {
1932         /* Do nothing, write notifier decided to fail this request */
1933     } else if (flags & BDRV_REQ_ZERO_WRITE) {
1934         bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
1935         ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags);
1936     } else if (flags & BDRV_REQ_WRITE_COMPRESSED) {
1937         ret = bdrv_driver_pwritev_compressed(bs, offset, bytes,
1938                                              qiov, qiov_offset);
1939     } else if (bytes <= max_transfer) {
1940         bdrv_debug_event(bs, BLKDBG_PWRITEV);
1941         ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags);
1942     } else {
1943         bdrv_debug_event(bs, BLKDBG_PWRITEV);
1944         while (bytes_remaining) {
1945             int num = MIN(bytes_remaining, max_transfer);
1946             int local_flags = flags;
1947
1948             assert(num);
1949             if (num < bytes_remaining && (flags & BDRV_REQ_FUA) &&
1950                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1951                 /* If FUA is going to be emulated by flush, we only
1952                  * need to flush on the last iteration */
1953                 local_flags &= ~BDRV_REQ_FUA;
1954             }
1955
1956             ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining,
1957                                       num, qiov,
1958                                       qiov_offset + bytes - bytes_remaining,
1959                                       local_flags);
1960             if (ret < 0) {
1961                 break;
1962             }
1963             bytes_remaining -= num;
1964         }
1965     }
1966     bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
1967
1968     if (ret >= 0) {
1969         ret = 0;
1970     }
1971     bdrv_co_write_req_finish(child, offset, bytes, req, ret);
1972
1973     return ret;
1974 }
1975
1976 static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,
1977                                                 int64_t offset,
1978                                                 int64_t bytes,
1979                                                 BdrvRequestFlags flags,
1980                                                 BdrvTrackedRequest *req)
1981 {
1982     BlockDriverState *bs = child->bs;
1983     QEMUIOVector local_qiov;
1984     uint64_t align = bs->bl.request_alignment;
1985     int ret = 0;
1986     bool padding;
1987     BdrvRequestPadding pad;
1988
1989     /* This flag doesn't make sense for padding or zero writes */
1990     flags &= ~BDRV_REQ_REGISTERED_BUF;
1991
1992     padding = bdrv_init_padding(bs, offset, bytes, &pad);
1993     if (padding) {
1994         assert(!(flags & BDRV_REQ_NO_WAIT));
1995         bdrv_make_request_serialising(req, align);
1996
1997         bdrv_padding_rmw_read(child, req, &pad, true);
1998
1999         if (pad.head || pad.merge_reads) {
2000             int64_t aligned_offset = offset & ~(align - 1);
2001             int64_t write_bytes = pad.merge_reads ? pad.buf_len : align;
2002
2003             qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes);
2004             ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes,
2005                                        align, &local_qiov, 0,
2006                                        flags & ~BDRV_REQ_ZERO_WRITE);
2007             if (ret < 0 || pad.merge_reads) {
2008                 /* Error or all work is done */
2009                 goto out;
2010             }
2011             offset += write_bytes - pad.head;
2012             bytes -= write_bytes - pad.head;
2013         }
2014     }
2015
2016     assert(!bytes || (offset & (align - 1)) == 0);
2017     if (bytes >= align) {
2018         /* Write the aligned part in the middle. */
2019         int64_t aligned_bytes = bytes & ~(align - 1);
2020         ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align,
2021                                    NULL, 0, flags);
2022         if (ret < 0) {
2023             goto out;
2024         }
2025         bytes -= aligned_bytes;
2026         offset += aligned_bytes;
2027     }
2028
2029     assert(!bytes || (offset & (align - 1)) == 0);
2030     if (bytes) {
2031         assert(align == pad.tail + bytes);
2032
2033         qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align);
2034         ret = bdrv_aligned_pwritev(child, req, offset, align, align,
2035                                    &local_qiov, 0,
2036                                    flags & ~BDRV_REQ_ZERO_WRITE);
2037     }
2038
2039 out:
2040     bdrv_padding_destroy(&pad);
2041
2042     return ret;
2043 }
2044
2045 /*
2046  * Handle a write request in coroutine context
2047  */
2048 int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
2049     int64_t offset, int64_t bytes, QEMUIOVector *qiov,
2050     BdrvRequestFlags flags)
2051 {
2052     IO_CODE();
2053     return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags);
2054 }
2055
2056 int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child,
2057     int64_t offset, int64_t bytes, QEMUIOVector *qiov, size_t qiov_offset,
2058     BdrvRequestFlags flags)
2059 {
2060     BlockDriverState *bs = child->bs;
2061     BdrvTrackedRequest req;
2062     uint64_t align = bs->bl.request_alignment;
2063     BdrvRequestPadding pad;
2064     int ret;
2065     bool padded = false;
2066     IO_CODE();
2067
2068     trace_bdrv_co_pwritev_part(child->bs, offset, bytes, flags);
2069
2070     if (!bdrv_is_inserted(bs)) {
2071         return -ENOMEDIUM;
2072     }
2073
2074     if (flags & BDRV_REQ_ZERO_WRITE) {
2075         ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL);
2076     } else {
2077         ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset);
2078     }
2079     if (ret < 0) {
2080         return ret;
2081     }
2082
2083     /* If the request is misaligned then we can't make it efficient */
2084     if ((flags & BDRV_REQ_NO_FALLBACK) &&
2085         !QEMU_IS_ALIGNED(offset | bytes, align))
2086     {
2087         return -ENOTSUP;
2088     }
2089
2090     if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
2091         /*
2092          * Aligning zero request is nonsense. Even if driver has special meaning
2093          * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
2094          * it to driver due to request_alignment.
2095          *
2096          * Still, no reason to return an error if someone do unaligned
2097          * zero-length write occasionally.
2098          */
2099         return 0;
2100     }
2101
2102     if (!(flags & BDRV_REQ_ZERO_WRITE)) {
2103         /*
2104          * Pad request for following read-modify-write cycle.
2105          * bdrv_co_do_zero_pwritev() does aligning by itself, so, we do
2106          * alignment only if there is no ZERO flag.
2107          */
2108         ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad,
2109                                &padded, &flags);
2110         if (ret < 0) {
2111             return ret;
2112         }
2113     }
2114
2115     bdrv_inc_in_flight(bs);
2116     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
2117
2118     if (flags & BDRV_REQ_ZERO_WRITE) {
2119         assert(!padded);
2120         ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req);
2121         goto out;
2122     }
2123
2124     if (padded) {
2125         /*
2126          * Request was unaligned to request_alignment and therefore
2127          * padded.  We are going to do read-modify-write, and must
2128          * serialize the request to prevent interactions of the
2129          * widened region with other transactions.
2130          */
2131         assert(!(flags & BDRV_REQ_NO_WAIT));
2132         bdrv_make_request_serialising(&req, align);
2133         bdrv_padding_rmw_read(child, &req, &pad, false);
2134     }
2135
2136     ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align,
2137                                qiov, qiov_offset, flags);
2138
2139     bdrv_padding_destroy(&pad);
2140
2141 out:
2142     tracked_request_end(&req);
2143     bdrv_dec_in_flight(bs);
2144
2145     return ret;
2146 }
2147
2148 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset,
2149                                        int64_t bytes, BdrvRequestFlags flags)
2150 {
2151     IO_CODE();
2152     trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags);
2153
2154     if (!(child->bs->open_flags & BDRV_O_UNMAP)) {
2155         flags &= ~BDRV_REQ_MAY_UNMAP;
2156     }
2157
2158     return bdrv_co_pwritev(child, offset, bytes, NULL,
2159                            BDRV_REQ_ZERO_WRITE | flags);
2160 }
2161
2162 /*
2163  * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not.
2164  */
2165 int bdrv_flush_all(void)
2166 {
2167     BdrvNextIterator it;
2168     BlockDriverState *bs = NULL;
2169     int result = 0;
2170
2171     GLOBAL_STATE_CODE();
2172
2173     /*
2174      * bdrv queue is managed by record/replay,
2175      * creating new flush request for stopping
2176      * the VM may break the determinism
2177      */
2178     if (replay_events_enabled()) {
2179         return result;
2180     }
2181
2182     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
2183         AioContext *aio_context = bdrv_get_aio_context(bs);
2184         int ret;
2185
2186         aio_context_acquire(aio_context);
2187         ret = bdrv_flush(bs);
2188         if (ret < 0 && !result) {
2189             result = ret;
2190         }
2191         aio_context_release(aio_context);
2192     }
2193
2194     return result;
2195 }
2196
2197 /*
2198  * Returns the allocation status of the specified sectors.
2199  * Drivers not implementing the functionality are assumed to not support
2200  * backing files, hence all their sectors are reported as allocated.
2201  *
2202  * If 'want_zero' is true, the caller is querying for mapping
2203  * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and
2204  * _ZERO where possible; otherwise, the result favors larger 'pnum',
2205  * with a focus on accurate BDRV_BLOCK_ALLOCATED.
2206  *
2207  * If 'offset' is beyond the end of the disk image the return value is
2208  * BDRV_BLOCK_EOF and 'pnum' is set to 0.
2209  *
2210  * 'bytes' is the max value 'pnum' should be set to.  If bytes goes
2211  * beyond the end of the disk image it will be clamped; if 'pnum' is set to
2212  * the end of the image, then the returned value will include BDRV_BLOCK_EOF.
2213  *
2214  * 'pnum' is set to the number of bytes (including and immediately
2215  * following the specified offset) that are easily known to be in the
2216  * same allocated/unallocated state.  Note that a second call starting
2217  * at the original offset plus returned pnum may have the same status.
2218  * The returned value is non-zero on success except at end-of-file.
2219  *
2220  * Returns negative errno on failure.  Otherwise, if the
2221  * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are
2222  * set to the host mapping and BDS corresponding to the guest offset.
2223  */
2224 static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs,
2225                                              bool want_zero,
2226                                              int64_t offset, int64_t bytes,
2227                                              int64_t *pnum, int64_t *map,
2228                                              BlockDriverState **file)
2229 {
2230     int64_t total_size;
2231     int64_t n; /* bytes */
2232     int ret;
2233     int64_t local_map = 0;
2234     BlockDriverState *local_file = NULL;
2235     int64_t aligned_offset, aligned_bytes;
2236     uint32_t align;
2237     bool has_filtered_child;
2238
2239     assert(pnum);
2240     *pnum = 0;
2241     total_size = bdrv_getlength(bs);
2242     if (total_size < 0) {
2243         ret = total_size;
2244         goto early_out;
2245     }
2246
2247     if (offset >= total_size) {
2248         ret = BDRV_BLOCK_EOF;
2249         goto early_out;
2250     }
2251     if (!bytes) {
2252         ret = 0;
2253         goto early_out;
2254     }
2255
2256     n = total_size - offset;
2257     if (n < bytes) {
2258         bytes = n;
2259     }
2260
2261     /* Must be non-NULL or bdrv_getlength() would have failed */
2262     assert(bs->drv);
2263     has_filtered_child = bdrv_filter_child(bs);
2264     if (!bs->drv->bdrv_co_block_status && !has_filtered_child) {
2265         *pnum = bytes;
2266         ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
2267         if (offset + bytes == total_size) {
2268             ret |= BDRV_BLOCK_EOF;
2269         }
2270         if (bs->drv->protocol_name) {
2271             ret |= BDRV_BLOCK_OFFSET_VALID;
2272             local_map = offset;
2273             local_file = bs;
2274         }
2275         goto early_out;
2276     }
2277
2278     bdrv_inc_in_flight(bs);
2279
2280     /* Round out to request_alignment boundaries */
2281     align = bs->bl.request_alignment;
2282     aligned_offset = QEMU_ALIGN_DOWN(offset, align);
2283     aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset;
2284
2285     if (bs->drv->bdrv_co_block_status) {
2286         /*
2287          * Use the block-status cache only for protocol nodes: Format
2288          * drivers are generally quick to inquire the status, but protocol
2289          * drivers often need to get information from outside of qemu, so
2290          * we do not have control over the actual implementation.  There
2291          * have been cases where inquiring the status took an unreasonably
2292          * long time, and we can do nothing in qemu to fix it.
2293          * This is especially problematic for images with large data areas,
2294          * because finding the few holes in them and giving them special
2295          * treatment does not gain much performance.  Therefore, we try to
2296          * cache the last-identified data region.
2297          *
2298          * Second, limiting ourselves to protocol nodes allows us to assume
2299          * the block status for data regions to be DATA | OFFSET_VALID, and
2300          * that the host offset is the same as the guest offset.
2301          *
2302          * Note that it is possible that external writers zero parts of
2303          * the cached regions without the cache being invalidated, and so
2304          * we may report zeroes as data.  This is not catastrophic,
2305          * however, because reporting zeroes as data is fine.
2306          */
2307         if (QLIST_EMPTY(&bs->children) &&
2308             bdrv_bsc_is_data(bs, aligned_offset, pnum))
2309         {
2310             ret = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID;
2311             local_file = bs;
2312             local_map = aligned_offset;
2313         } else {
2314             ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset,
2315                                                 aligned_bytes, pnum, &local_map,
2316                                                 &local_file);
2317
2318             /*
2319              * Note that checking QLIST_EMPTY(&bs->children) is also done when
2320              * the cache is queried above.  Technically, we do not need to check
2321              * it here; the worst that can happen is that we fill the cache for
2322              * non-protocol nodes, and then it is never used.  However, filling
2323              * the cache requires an RCU update, so double check here to avoid
2324              * such an update if possible.
2325              *
2326              * Check want_zero, because we only want to update the cache when we
2327              * have accurate information about what is zero and what is data.
2328              */
2329             if (want_zero &&
2330                 ret == (BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID) &&
2331                 QLIST_EMPTY(&bs->children))
2332             {
2333                 /*
2334                  * When a protocol driver reports BLOCK_OFFSET_VALID, the
2335                  * returned local_map value must be the same as the offset we
2336                  * have passed (aligned_offset), and local_bs must be the node
2337                  * itself.
2338                  * Assert this, because we follow this rule when reading from
2339                  * the cache (see the `local_file = bs` and
2340                  * `local_map = aligned_offset` assignments above), and the
2341                  * result the cache delivers must be the same as the driver
2342                  * would deliver.
2343                  */
2344                 assert(local_file == bs);
2345                 assert(local_map == aligned_offset);
2346                 bdrv_bsc_fill(bs, aligned_offset, *pnum);
2347             }
2348         }
2349     } else {
2350         /* Default code for filters */
2351
2352         local_file = bdrv_filter_bs(bs);
2353         assert(local_file);
2354
2355         *pnum = aligned_bytes;
2356         local_map = aligned_offset;
2357         ret = BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID;
2358     }
2359     if (ret < 0) {
2360         *pnum = 0;
2361         goto out;
2362     }
2363
2364     /*
2365      * The driver's result must be a non-zero multiple of request_alignment.
2366      * Clamp pnum and adjust map to original request.
2367      */
2368     assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) &&
2369            align > offset - aligned_offset);
2370     if (ret & BDRV_BLOCK_RECURSE) {
2371         assert(ret & BDRV_BLOCK_DATA);
2372         assert(ret & BDRV_BLOCK_OFFSET_VALID);
2373         assert(!(ret & BDRV_BLOCK_ZERO));
2374     }
2375
2376     *pnum -= offset - aligned_offset;
2377     if (*pnum > bytes) {
2378         *pnum = bytes;
2379     }
2380     if (ret & BDRV_BLOCK_OFFSET_VALID) {
2381         local_map += offset - aligned_offset;
2382     }
2383
2384     if (ret & BDRV_BLOCK_RAW) {
2385         assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file);
2386         ret = bdrv_co_block_status(local_file, want_zero, local_map,
2387                                    *pnum, pnum, &local_map, &local_file);
2388         goto out;
2389     }
2390
2391     if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
2392         ret |= BDRV_BLOCK_ALLOCATED;
2393     } else if (bs->drv->supports_backing) {
2394         BlockDriverState *cow_bs = bdrv_cow_bs(bs);
2395
2396         if (!cow_bs) {
2397             ret |= BDRV_BLOCK_ZERO;
2398         } else if (want_zero) {
2399             int64_t size2 = bdrv_getlength(cow_bs);
2400
2401             if (size2 >= 0 && offset >= size2) {
2402                 ret |= BDRV_BLOCK_ZERO;
2403             }
2404         }
2405     }
2406
2407     if (want_zero && ret & BDRV_BLOCK_RECURSE &&
2408         local_file && local_file != bs &&
2409         (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
2410         (ret & BDRV_BLOCK_OFFSET_VALID)) {
2411         int64_t file_pnum;
2412         int ret2;
2413
2414         ret2 = bdrv_co_block_status(local_file, want_zero, local_map,
2415                                     *pnum, &file_pnum, NULL, NULL);
2416         if (ret2 >= 0) {
2417             /* Ignore errors.  This is just providing extra information, it
2418              * is useful but not necessary.
2419              */
2420             if (ret2 & BDRV_BLOCK_EOF &&
2421                 (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) {
2422                 /*
2423                  * It is valid for the format block driver to read
2424                  * beyond the end of the underlying file's current
2425                  * size; such areas read as zero.
2426                  */
2427                 ret |= BDRV_BLOCK_ZERO;
2428             } else {
2429                 /* Limit request to the range reported by the protocol driver */
2430                 *pnum = file_pnum;
2431                 ret |= (ret2 & BDRV_BLOCK_ZERO);
2432             }
2433         }
2434     }
2435
2436 out:
2437     bdrv_dec_in_flight(bs);
2438     if (ret >= 0 && offset + *pnum == total_size) {
2439         ret |= BDRV_BLOCK_EOF;
2440     }
2441 early_out:
2442     if (file) {
2443         *file = local_file;
2444     }
2445     if (map) {
2446         *map = local_map;
2447     }
2448     return ret;
2449 }
2450
2451 int coroutine_fn
2452 bdrv_co_common_block_status_above(BlockDriverState *bs,
2453                                   BlockDriverState *base,
2454                                   bool include_base,
2455                                   bool want_zero,
2456                                   int64_t offset,
2457                                   int64_t bytes,
2458                                   int64_t *pnum,
2459                                   int64_t *map,
2460                                   BlockDriverState **file,
2461                                   int *depth)
2462 {
2463     int ret;
2464     BlockDriverState *p;
2465     int64_t eof = 0;
2466     int dummy;
2467     IO_CODE();
2468
2469     assert(!include_base || base); /* Can't include NULL base */
2470
2471     if (!depth) {
2472         depth = &dummy;
2473     }
2474     *depth = 0;
2475
2476     if (!include_base && bs == base) {
2477         *pnum = bytes;
2478         return 0;
2479     }
2480
2481     ret = bdrv_co_block_status(bs, want_zero, offset, bytes, pnum, map, file);
2482     ++*depth;
2483     if (ret < 0 || *pnum == 0 || ret & BDRV_BLOCK_ALLOCATED || bs == base) {
2484         return ret;
2485     }
2486
2487     if (ret & BDRV_BLOCK_EOF) {
2488         eof = offset + *pnum;
2489     }
2490
2491     assert(*pnum <= bytes);
2492     bytes = *pnum;
2493
2494     for (p = bdrv_filter_or_cow_bs(bs); include_base || p != base;
2495          p = bdrv_filter_or_cow_bs(p))
2496     {
2497         ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map,
2498                                    file);
2499         ++*depth;
2500         if (ret < 0) {
2501             return ret;
2502         }
2503         if (*pnum == 0) {
2504             /*
2505              * The top layer deferred to this layer, and because this layer is
2506              * short, any zeroes that we synthesize beyond EOF behave as if they
2507              * were allocated at this layer.
2508              *
2509              * We don't include BDRV_BLOCK_EOF into ret, as upper layer may be
2510              * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see
2511              * below.
2512              */
2513             assert(ret & BDRV_BLOCK_EOF);
2514             *pnum = bytes;
2515             if (file) {
2516                 *file = p;
2517             }
2518             ret = BDRV_BLOCK_ZERO | BDRV_BLOCK_ALLOCATED;
2519             break;
2520         }
2521         if (ret & BDRV_BLOCK_ALLOCATED) {
2522             /*
2523              * We've found the node and the status, we must break.
2524              *
2525              * Drop BDRV_BLOCK_EOF, as it's not for upper layer, which may be
2526              * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see
2527              * below.
2528              */
2529             ret &= ~BDRV_BLOCK_EOF;
2530             break;
2531         }
2532
2533         if (p == base) {
2534             assert(include_base);
2535             break;
2536         }
2537
2538         /*
2539          * OK, [offset, offset + *pnum) region is unallocated on this layer,
2540          * let's continue the diving.
2541          */
2542         assert(*pnum <= bytes);
2543         bytes = *pnum;
2544     }
2545
2546     if (offset + *pnum == eof) {
2547         ret |= BDRV_BLOCK_EOF;
2548     }
2549
2550     return ret;
2551 }
2552
2553 int coroutine_fn bdrv_co_block_status_above(BlockDriverState *bs,
2554                                             BlockDriverState *base,
2555                                             int64_t offset, int64_t bytes,
2556                                             int64_t *pnum, int64_t *map,
2557                                             BlockDriverState **file)
2558 {
2559     IO_CODE();
2560     return bdrv_co_common_block_status_above(bs, base, false, true, offset,
2561                                              bytes, pnum, map, file, NULL);
2562 }
2563
2564 int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base,
2565                             int64_t offset, int64_t bytes, int64_t *pnum,
2566                             int64_t *map, BlockDriverState **file)
2567 {
2568     IO_CODE();
2569     return bdrv_common_block_status_above(bs, base, false, true, offset, bytes,
2570                                           pnum, map, file, NULL);
2571 }
2572
2573 int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes,
2574                       int64_t *pnum, int64_t *map, BlockDriverState **file)
2575 {
2576     IO_CODE();
2577     return bdrv_block_status_above(bs, bdrv_filter_or_cow_bs(bs),
2578                                    offset, bytes, pnum, map, file);
2579 }
2580
2581 /*
2582  * Check @bs (and its backing chain) to see if the range defined
2583  * by @offset and @bytes is known to read as zeroes.
2584  * Return 1 if that is the case, 0 otherwise and -errno on error.
2585  * This test is meant to be fast rather than accurate so returning 0
2586  * does not guarantee non-zero data.
2587  */
2588 int coroutine_fn bdrv_co_is_zero_fast(BlockDriverState *bs, int64_t offset,
2589                                       int64_t bytes)
2590 {
2591     int ret;
2592     int64_t pnum = bytes;
2593     IO_CODE();
2594
2595     if (!bytes) {
2596         return 1;
2597     }
2598
2599     ret = bdrv_co_common_block_status_above(bs, NULL, false, false, offset,
2600                                             bytes, &pnum, NULL, NULL, NULL);
2601
2602     if (ret < 0) {
2603         return ret;
2604     }
2605
2606     return (pnum == bytes) && (ret & BDRV_BLOCK_ZERO);
2607 }
2608
2609 int coroutine_fn bdrv_co_is_allocated(BlockDriverState *bs, int64_t offset,
2610                                       int64_t bytes, int64_t *pnum)
2611 {
2612     int ret;
2613     int64_t dummy;
2614     IO_CODE();
2615
2616     ret = bdrv_co_common_block_status_above(bs, bs, true, false, offset,
2617                                             bytes, pnum ? pnum : &dummy, NULL,
2618                                             NULL, NULL);
2619     if (ret < 0) {
2620         return ret;
2621     }
2622     return !!(ret & BDRV_BLOCK_ALLOCATED);
2623 }
2624
2625 int bdrv_is_allocated(BlockDriverState *bs, int64_t offset, int64_t bytes,
2626                       int64_t *pnum)
2627 {
2628     int ret;
2629     int64_t dummy;
2630     IO_CODE();
2631
2632     ret = bdrv_common_block_status_above(bs, bs, true, false, offset,
2633                                          bytes, pnum ? pnum : &dummy, NULL,
2634                                          NULL, NULL);
2635     if (ret < 0) {
2636         return ret;
2637     }
2638     return !!(ret & BDRV_BLOCK_ALLOCATED);
2639 }
2640
2641 /* See bdrv_is_allocated_above for documentation */
2642 int coroutine_fn bdrv_co_is_allocated_above(BlockDriverState *top,
2643                                             BlockDriverState *base,
2644                                             bool include_base, int64_t offset,
2645                                             int64_t bytes, int64_t *pnum)
2646 {
2647     int depth;
2648     int ret;
2649     IO_CODE();
2650
2651     ret = bdrv_co_common_block_status_above(top, base, include_base, false,
2652                                             offset, bytes, pnum, NULL, NULL,
2653                                             &depth);
2654     if (ret < 0) {
2655         return ret;
2656     }
2657
2658     if (ret & BDRV_BLOCK_ALLOCATED) {
2659         return depth;
2660     }
2661     return 0;
2662 }
2663
2664 /*
2665  * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
2666  *
2667  * Return a positive depth if (a prefix of) the given range is allocated
2668  * in any image between BASE and TOP (BASE is only included if include_base
2669  * is set).  Depth 1 is TOP, 2 is the first backing layer, and so forth.
2670  * BASE can be NULL to check if the given offset is allocated in any
2671  * image of the chain.  Return 0 otherwise, or negative errno on
2672  * failure.
2673  *
2674  * 'pnum' is set to the number of bytes (including and immediately
2675  * following the specified offset) that are known to be in the same
2676  * allocated/unallocated state.  Note that a subsequent call starting
2677  * at 'offset + *pnum' may return the same allocation status (in other
2678  * words, the result is not necessarily the maximum possible range);
2679  * but 'pnum' will only be 0 when end of file is reached.
2680  */
2681 int bdrv_is_allocated_above(BlockDriverState *top,
2682                             BlockDriverState *base,
2683                             bool include_base, int64_t offset,
2684                             int64_t bytes, int64_t *pnum)
2685 {
2686     int depth;
2687     int ret;
2688     IO_CODE();
2689
2690     ret = bdrv_common_block_status_above(top, base, include_base, false,
2691                                          offset, bytes, pnum, NULL, NULL,
2692                                          &depth);
2693     if (ret < 0) {
2694         return ret;
2695     }
2696
2697     if (ret & BDRV_BLOCK_ALLOCATED) {
2698         return depth;
2699     }
2700     return 0;
2701 }
2702
2703 int coroutine_fn
2704 bdrv_co_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2705 {
2706     BlockDriver *drv = bs->drv;
2707     BlockDriverState *child_bs = bdrv_primary_bs(bs);
2708     int ret;
2709     IO_CODE();
2710     assert_bdrv_graph_readable();
2711
2712     ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL);
2713     if (ret < 0) {
2714         return ret;
2715     }
2716
2717     if (!drv) {
2718         return -ENOMEDIUM;
2719     }
2720
2721     bdrv_inc_in_flight(bs);
2722
2723     if (drv->bdrv_load_vmstate) {
2724         ret = drv->bdrv_load_vmstate(bs, qiov, pos);
2725     } else if (child_bs) {
2726         ret = bdrv_co_readv_vmstate(child_bs, qiov, pos);
2727     } else {
2728         ret = -ENOTSUP;
2729     }
2730
2731     bdrv_dec_in_flight(bs);
2732
2733     return ret;
2734 }
2735
2736 int coroutine_fn
2737 bdrv_co_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2738 {
2739     BlockDriver *drv = bs->drv;
2740     BlockDriverState *child_bs = bdrv_primary_bs(bs);
2741     int ret;
2742     IO_CODE();
2743     assert_bdrv_graph_readable();
2744
2745     ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL);
2746     if (ret < 0) {
2747         return ret;
2748     }
2749
2750     if (!drv) {
2751         return -ENOMEDIUM;
2752     }
2753
2754     bdrv_inc_in_flight(bs);
2755
2756     if (drv->bdrv_save_vmstate) {
2757         ret = drv->bdrv_save_vmstate(bs, qiov, pos);
2758     } else if (child_bs) {
2759         ret = bdrv_co_writev_vmstate(child_bs, qiov, pos);
2760     } else {
2761         ret = -ENOTSUP;
2762     }
2763
2764     bdrv_dec_in_flight(bs);
2765
2766     return ret;
2767 }
2768
2769 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
2770                       int64_t pos, int size)
2771 {
2772     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2773     int ret = bdrv_writev_vmstate(bs, &qiov, pos);
2774     IO_CODE();
2775
2776     return ret < 0 ? ret : size;
2777 }
2778
2779 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
2780                       int64_t pos, int size)
2781 {
2782     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2783     int ret = bdrv_readv_vmstate(bs, &qiov, pos);
2784     IO_CODE();
2785
2786     return ret < 0 ? ret : size;
2787 }
2788
2789 /**************************************************************/
2790 /* async I/Os */
2791
2792 void bdrv_aio_cancel(BlockAIOCB *acb)
2793 {
2794     IO_CODE();
2795     qemu_aio_ref(acb);
2796     bdrv_aio_cancel_async(acb);
2797     while (acb->refcnt > 1) {
2798         if (acb->aiocb_info->get_aio_context) {
2799             aio_poll(acb->aiocb_info->get_aio_context(acb), true);
2800         } else if (acb->bs) {
2801             /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so
2802              * assert that we're not using an I/O thread.  Thread-safe
2803              * code should use bdrv_aio_cancel_async exclusively.
2804              */
2805             assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context());
2806             aio_poll(bdrv_get_aio_context(acb->bs), true);
2807         } else {
2808             abort();
2809         }
2810     }
2811     qemu_aio_unref(acb);
2812 }
2813
2814 /* Async version of aio cancel. The caller is not blocked if the acb implements
2815  * cancel_async, otherwise we do nothing and let the request normally complete.
2816  * In either case the completion callback must be called. */
2817 void bdrv_aio_cancel_async(BlockAIOCB *acb)
2818 {
2819     IO_CODE();
2820     if (acb->aiocb_info->cancel_async) {
2821         acb->aiocb_info->cancel_async(acb);
2822     }
2823 }
2824
2825 /**************************************************************/
2826 /* Coroutine block device emulation */
2827
2828 int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
2829 {
2830     BdrvChild *primary_child = bdrv_primary_child(bs);
2831     BdrvChild *child;
2832     int current_gen;
2833     int ret = 0;
2834     IO_CODE();
2835
2836     bdrv_inc_in_flight(bs);
2837
2838     if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
2839         bdrv_is_sg(bs)) {
2840         goto early_exit;
2841     }
2842
2843     qemu_co_mutex_lock(&bs->reqs_lock);
2844     current_gen = qatomic_read(&bs->write_gen);
2845
2846     /* Wait until any previous flushes are completed */
2847     while (bs->active_flush_req) {
2848         qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock);
2849     }
2850
2851     /* Flushes reach this point in nondecreasing current_gen order.  */
2852     bs->active_flush_req = true;
2853     qemu_co_mutex_unlock(&bs->reqs_lock);
2854
2855     /* Write back all layers by calling one driver function */
2856     if (bs->drv->bdrv_co_flush) {
2857         ret = bs->drv->bdrv_co_flush(bs);
2858         goto out;
2859     }
2860
2861     /* Write back cached data to the OS even with cache=unsafe */
2862     BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_OS);
2863     if (bs->drv->bdrv_co_flush_to_os) {
2864         ret = bs->drv->bdrv_co_flush_to_os(bs);
2865         if (ret < 0) {
2866             goto out;
2867         }
2868     }
2869
2870     /* But don't actually force it to the disk with cache=unsafe */
2871     if (bs->open_flags & BDRV_O_NO_FLUSH) {
2872         goto flush_children;
2873     }
2874
2875     /* Check if we really need to flush anything */
2876     if (bs->flushed_gen == current_gen) {
2877         goto flush_children;
2878     }
2879
2880     BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_DISK);
2881     if (!bs->drv) {
2882         /* bs->drv->bdrv_co_flush() might have ejected the BDS
2883          * (even in case of apparent success) */
2884         ret = -ENOMEDIUM;
2885         goto out;
2886     }
2887     if (bs->drv->bdrv_co_flush_to_disk) {
2888         ret = bs->drv->bdrv_co_flush_to_disk(bs);
2889     } else if (bs->drv->bdrv_aio_flush) {
2890         BlockAIOCB *acb;
2891         CoroutineIOCompletion co = {
2892             .coroutine = qemu_coroutine_self(),
2893         };
2894
2895         acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
2896         if (acb == NULL) {
2897             ret = -EIO;
2898         } else {
2899             qemu_coroutine_yield();
2900             ret = co.ret;
2901         }
2902     } else {
2903         /*
2904          * Some block drivers always operate in either writethrough or unsafe
2905          * mode and don't support bdrv_flush therefore. Usually qemu doesn't
2906          * know how the server works (because the behaviour is hardcoded or
2907          * depends on server-side configuration), so we can't ensure that
2908          * everything is safe on disk. Returning an error doesn't work because
2909          * that would break guests even if the server operates in writethrough
2910          * mode.
2911          *
2912          * Let's hope the user knows what he's doing.
2913          */
2914         ret = 0;
2915     }
2916
2917     if (ret < 0) {
2918         goto out;
2919     }
2920
2921     /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
2922      * in the case of cache=unsafe, so there are no useless flushes.
2923      */
2924 flush_children:
2925     ret = 0;
2926     QLIST_FOREACH(child, &bs->children, next) {
2927         if (child->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) {
2928             int this_child_ret = bdrv_co_flush(child->bs);
2929             if (!ret) {
2930                 ret = this_child_ret;
2931             }
2932         }
2933     }
2934
2935 out:
2936     /* Notify any pending flushes that we have completed */
2937     if (ret == 0) {
2938         bs->flushed_gen = current_gen;
2939     }
2940
2941     qemu_co_mutex_lock(&bs->reqs_lock);
2942     bs->active_flush_req = false;
2943     /* Return value is ignored - it's ok if wait queue is empty */
2944     qemu_co_queue_next(&bs->flush_queue);
2945     qemu_co_mutex_unlock(&bs->reqs_lock);
2946
2947 early_exit:
2948     bdrv_dec_in_flight(bs);
2949     return ret;
2950 }
2951
2952 int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset,
2953                                   int64_t bytes)
2954 {
2955     BdrvTrackedRequest req;
2956     int ret;
2957     int64_t max_pdiscard;
2958     int head, tail, align;
2959     BlockDriverState *bs = child->bs;
2960     IO_CODE();
2961
2962     if (!bs || !bs->drv || !bdrv_is_inserted(bs)) {
2963         return -ENOMEDIUM;
2964     }
2965
2966     if (bdrv_has_readonly_bitmaps(bs)) {
2967         return -EPERM;
2968     }
2969
2970     ret = bdrv_check_request(offset, bytes, NULL);
2971     if (ret < 0) {
2972         return ret;
2973     }
2974
2975     /* Do nothing if disabled.  */
2976     if (!(bs->open_flags & BDRV_O_UNMAP)) {
2977         return 0;
2978     }
2979
2980     if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) {
2981         return 0;
2982     }
2983
2984     /* Invalidate the cached block-status data range if this discard overlaps */
2985     bdrv_bsc_invalidate_range(bs, offset, bytes);
2986
2987     /* Discard is advisory, but some devices track and coalesce
2988      * unaligned requests, so we must pass everything down rather than
2989      * round here.  Still, most devices will just silently ignore
2990      * unaligned requests (by returning -ENOTSUP), so we must fragment
2991      * the request accordingly.  */
2992     align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment);
2993     assert(align % bs->bl.request_alignment == 0);
2994     head = offset % align;
2995     tail = (offset + bytes) % align;
2996
2997     bdrv_inc_in_flight(bs);
2998     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD);
2999
3000     ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0);
3001     if (ret < 0) {
3002         goto out;
3003     }
3004
3005     max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT64_MAX),
3006                                    align);
3007     assert(max_pdiscard >= bs->bl.request_alignment);
3008
3009     while (bytes > 0) {
3010         int64_t num = bytes;
3011
3012         if (head) {
3013             /* Make small requests to get to alignment boundaries. */
3014             num = MIN(bytes, align - head);
3015             if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) {
3016                 num %= bs->bl.request_alignment;
3017             }
3018             head = (head + num) % align;
3019             assert(num < max_pdiscard);
3020         } else if (tail) {
3021             if (num > align) {
3022                 /* Shorten the request to the last aligned cluster.  */
3023                 num -= tail;
3024             } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) &&
3025                        tail > bs->bl.request_alignment) {
3026                 tail %= bs->bl.request_alignment;
3027                 num -= tail;
3028             }
3029         }
3030         /* limit request size */
3031         if (num > max_pdiscard) {
3032             num = max_pdiscard;
3033         }
3034
3035         if (!bs->drv) {
3036             ret = -ENOMEDIUM;
3037             goto out;
3038         }
3039         if (bs->drv->bdrv_co_pdiscard) {
3040             ret = bs->drv->bdrv_co_pdiscard(bs, offset, num);
3041         } else {
3042             BlockAIOCB *acb;
3043             CoroutineIOCompletion co = {
3044                 .coroutine = qemu_coroutine_self(),
3045             };
3046
3047             acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num,
3048                                              bdrv_co_io_em_complete, &co);
3049             if (acb == NULL) {
3050                 ret = -EIO;
3051                 goto out;
3052             } else {
3053                 qemu_coroutine_yield();
3054                 ret = co.ret;
3055             }
3056         }
3057         if (ret && ret != -ENOTSUP) {
3058             goto out;
3059         }
3060
3061         offset += num;
3062         bytes -= num;
3063     }
3064     ret = 0;
3065 out:
3066     bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret);
3067     tracked_request_end(&req);
3068     bdrv_dec_in_flight(bs);
3069     return ret;
3070 }
3071
3072 int coroutine_fn bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf)
3073 {
3074     BlockDriver *drv = bs->drv;
3075     CoroutineIOCompletion co = {
3076         .coroutine = qemu_coroutine_self(),
3077     };
3078     BlockAIOCB *acb;
3079     IO_CODE();
3080
3081     bdrv_inc_in_flight(bs);
3082     if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) {
3083         co.ret = -ENOTSUP;
3084         goto out;
3085     }
3086
3087     if (drv->bdrv_co_ioctl) {
3088         co.ret = drv->bdrv_co_ioctl(bs, req, buf);
3089     } else {
3090         acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
3091         if (!acb) {
3092             co.ret = -ENOTSUP;
3093             goto out;
3094         }
3095         qemu_coroutine_yield();
3096     }
3097 out:
3098     bdrv_dec_in_flight(bs);
3099     return co.ret;
3100 }
3101
3102 void *qemu_blockalign(BlockDriverState *bs, size_t size)
3103 {
3104     IO_CODE();
3105     return qemu_memalign(bdrv_opt_mem_align(bs), size);
3106 }
3107
3108 void *qemu_blockalign0(BlockDriverState *bs, size_t size)
3109 {
3110     IO_CODE();
3111     return memset(qemu_blockalign(bs, size), 0, size);
3112 }
3113
3114 void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
3115 {
3116     size_t align = bdrv_opt_mem_align(bs);
3117     IO_CODE();
3118
3119     /* Ensure that NULL is never returned on success */
3120     assert(align > 0);
3121     if (size == 0) {
3122         size = align;
3123     }
3124
3125     return qemu_try_memalign(align, size);
3126 }
3127
3128 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
3129 {
3130     void *mem = qemu_try_blockalign(bs, size);
3131     IO_CODE();
3132
3133     if (mem) {
3134         memset(mem, 0, size);
3135     }
3136
3137     return mem;
3138 }
3139
3140 void bdrv_io_plug(BlockDriverState *bs)
3141 {
3142     BdrvChild *child;
3143     IO_CODE();
3144
3145     QLIST_FOREACH(child, &bs->children, next) {
3146         bdrv_io_plug(child->bs);
3147     }
3148
3149     if (qatomic_fetch_inc(&bs->io_plugged) == 0) {
3150         BlockDriver *drv = bs->drv;
3151         if (drv && drv->bdrv_io_plug) {
3152             drv->bdrv_io_plug(bs);
3153         }
3154     }
3155 }
3156
3157 void bdrv_io_unplug(BlockDriverState *bs)
3158 {
3159     BdrvChild *child;
3160     IO_CODE();
3161
3162     assert(bs->io_plugged);
3163     if (qatomic_fetch_dec(&bs->io_plugged) == 1) {
3164         BlockDriver *drv = bs->drv;
3165         if (drv && drv->bdrv_io_unplug) {
3166             drv->bdrv_io_unplug(bs);
3167         }
3168     }
3169
3170     QLIST_FOREACH(child, &bs->children, next) {
3171         bdrv_io_unplug(child->bs);
3172     }
3173 }
3174
3175 /* Helper that undoes bdrv_register_buf() when it fails partway through */
3176 static void bdrv_register_buf_rollback(BlockDriverState *bs,
3177                                        void *host,
3178                                        size_t size,
3179                                        BdrvChild *final_child)
3180 {
3181     BdrvChild *child;
3182
3183     QLIST_FOREACH(child, &bs->children, next) {
3184         if (child == final_child) {
3185             break;
3186         }
3187
3188         bdrv_unregister_buf(child->bs, host, size);
3189     }
3190
3191     if (bs->drv && bs->drv->bdrv_unregister_buf) {
3192         bs->drv->bdrv_unregister_buf(bs, host, size);
3193     }
3194 }
3195
3196 bool bdrv_register_buf(BlockDriverState *bs, void *host, size_t size,
3197                        Error **errp)
3198 {
3199     BdrvChild *child;
3200
3201     GLOBAL_STATE_CODE();
3202     if (bs->drv && bs->drv->bdrv_register_buf) {
3203         if (!bs->drv->bdrv_register_buf(bs, host, size, errp)) {
3204             return false;
3205         }
3206     }
3207     QLIST_FOREACH(child, &bs->children, next) {
3208         if (!bdrv_register_buf(child->bs, host, size, errp)) {
3209             bdrv_register_buf_rollback(bs, host, size, child);
3210             return false;
3211         }
3212     }
3213     return true;
3214 }
3215
3216 void bdrv_unregister_buf(BlockDriverState *bs, void *host, size_t size)
3217 {
3218     BdrvChild *child;
3219
3220     GLOBAL_STATE_CODE();
3221     if (bs->drv && bs->drv->bdrv_unregister_buf) {
3222         bs->drv->bdrv_unregister_buf(bs, host, size);
3223     }
3224     QLIST_FOREACH(child, &bs->children, next) {
3225         bdrv_unregister_buf(child->bs, host, size);
3226     }
3227 }
3228
3229 static int coroutine_fn bdrv_co_copy_range_internal(
3230         BdrvChild *src, int64_t src_offset, BdrvChild *dst,
3231         int64_t dst_offset, int64_t bytes,
3232         BdrvRequestFlags read_flags, BdrvRequestFlags write_flags,
3233         bool recurse_src)
3234 {
3235     BdrvTrackedRequest req;
3236     int ret;
3237
3238     /* TODO We can support BDRV_REQ_NO_FALLBACK here */
3239     assert(!(read_flags & BDRV_REQ_NO_FALLBACK));
3240     assert(!(write_flags & BDRV_REQ_NO_FALLBACK));
3241     assert(!(read_flags & BDRV_REQ_NO_WAIT));
3242     assert(!(write_flags & BDRV_REQ_NO_WAIT));
3243
3244     if (!dst || !dst->bs || !bdrv_is_inserted(dst->bs)) {
3245         return -ENOMEDIUM;
3246     }
3247     ret = bdrv_check_request32(dst_offset, bytes, NULL, 0);
3248     if (ret) {
3249         return ret;
3250     }
3251     if (write_flags & BDRV_REQ_ZERO_WRITE) {
3252         return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags);
3253     }
3254
3255     if (!src || !src->bs || !bdrv_is_inserted(src->bs)) {
3256         return -ENOMEDIUM;
3257     }
3258     ret = bdrv_check_request32(src_offset, bytes, NULL, 0);
3259     if (ret) {
3260         return ret;
3261     }
3262
3263     if (!src->bs->drv->bdrv_co_copy_range_from
3264         || !dst->bs->drv->bdrv_co_copy_range_to
3265         || src->bs->encrypted || dst->bs->encrypted) {
3266         return -ENOTSUP;
3267     }
3268
3269     if (recurse_src) {
3270         bdrv_inc_in_flight(src->bs);
3271         tracked_request_begin(&req, src->bs, src_offset, bytes,
3272                               BDRV_TRACKED_READ);
3273
3274         /* BDRV_REQ_SERIALISING is only for write operation */
3275         assert(!(read_flags & BDRV_REQ_SERIALISING));
3276         bdrv_wait_serialising_requests(&req);
3277
3278         ret = src->bs->drv->bdrv_co_copy_range_from(src->bs,
3279                                                     src, src_offset,
3280                                                     dst, dst_offset,
3281                                                     bytes,
3282                                                     read_flags, write_flags);
3283
3284         tracked_request_end(&req);
3285         bdrv_dec_in_flight(src->bs);
3286     } else {
3287         bdrv_inc_in_flight(dst->bs);
3288         tracked_request_begin(&req, dst->bs, dst_offset, bytes,
3289                               BDRV_TRACKED_WRITE);
3290         ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req,
3291                                         write_flags);
3292         if (!ret) {
3293             ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs,
3294                                                       src, src_offset,
3295                                                       dst, dst_offset,
3296                                                       bytes,
3297                                                       read_flags, write_flags);
3298         }
3299         bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret);
3300         tracked_request_end(&req);
3301         bdrv_dec_in_flight(dst->bs);
3302     }
3303
3304     return ret;
3305 }
3306
3307 /* Copy range from @src to @dst.
3308  *
3309  * See the comment of bdrv_co_copy_range for the parameter and return value
3310  * semantics. */
3311 int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, int64_t src_offset,
3312                                          BdrvChild *dst, int64_t dst_offset,
3313                                          int64_t bytes,
3314                                          BdrvRequestFlags read_flags,
3315                                          BdrvRequestFlags write_flags)
3316 {
3317     IO_CODE();
3318     trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes,
3319                                   read_flags, write_flags);
3320     return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
3321                                        bytes, read_flags, write_flags, true);
3322 }
3323
3324 /* Copy range from @src to @dst.
3325  *
3326  * See the comment of bdrv_co_copy_range for the parameter and return value
3327  * semantics. */
3328 int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, int64_t src_offset,
3329                                        BdrvChild *dst, int64_t dst_offset,
3330                                        int64_t bytes,
3331                                        BdrvRequestFlags read_flags,
3332                                        BdrvRequestFlags write_flags)
3333 {
3334     IO_CODE();
3335     trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes,
3336                                 read_flags, write_flags);
3337     return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
3338                                        bytes, read_flags, write_flags, false);
3339 }
3340
3341 int coroutine_fn bdrv_co_copy_range(BdrvChild *src, int64_t src_offset,
3342                                     BdrvChild *dst, int64_t dst_offset,
3343                                     int64_t bytes, BdrvRequestFlags read_flags,
3344                                     BdrvRequestFlags write_flags)
3345 {
3346     IO_CODE();
3347     return bdrv_co_copy_range_from(src, src_offset,
3348                                    dst, dst_offset,
3349                                    bytes, read_flags, write_flags);
3350 }
3351
3352 static void bdrv_parent_cb_resize(BlockDriverState *bs)
3353 {
3354     BdrvChild *c;
3355     QLIST_FOREACH(c, &bs->parents, next_parent) {
3356         if (c->klass->resize) {
3357             c->klass->resize(c);
3358         }
3359     }
3360 }
3361
3362 /**
3363  * Truncate file to 'offset' bytes (needed only for file protocols)
3364  *
3365  * If 'exact' is true, the file must be resized to exactly the given
3366  * 'offset'.  Otherwise, it is sufficient for the node to be at least
3367  * 'offset' bytes in length.
3368  */
3369 int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact,
3370                                   PreallocMode prealloc, BdrvRequestFlags flags,
3371                                   Error **errp)
3372 {
3373     BlockDriverState *bs = child->bs;
3374     BdrvChild *filtered, *backing;
3375     BlockDriver *drv = bs->drv;
3376     BdrvTrackedRequest req;
3377     int64_t old_size, new_bytes;
3378     int ret;
3379     IO_CODE();
3380
3381     /* if bs->drv == NULL, bs is closed, so there's nothing to do here */
3382     if (!drv) {
3383         error_setg(errp, "No medium inserted");
3384         return -ENOMEDIUM;
3385     }
3386     if (offset < 0) {
3387         error_setg(errp, "Image size cannot be negative");
3388         return -EINVAL;
3389     }
3390
3391     ret = bdrv_check_request(offset, 0, errp);
3392     if (ret < 0) {
3393         return ret;
3394     }
3395
3396     old_size = bdrv_getlength(bs);
3397     if (old_size < 0) {
3398         error_setg_errno(errp, -old_size, "Failed to get old image size");
3399         return old_size;
3400     }
3401
3402     if (bdrv_is_read_only(bs)) {
3403         error_setg(errp, "Image is read-only");
3404         return -EACCES;
3405     }
3406
3407     if (offset > old_size) {
3408         new_bytes = offset - old_size;
3409     } else {
3410         new_bytes = 0;
3411     }
3412
3413     bdrv_inc_in_flight(bs);
3414     tracked_request_begin(&req, bs, offset - new_bytes, new_bytes,
3415                           BDRV_TRACKED_TRUNCATE);
3416
3417     /* If we are growing the image and potentially using preallocation for the
3418      * new area, we need to make sure that no write requests are made to it
3419      * concurrently or they might be overwritten by preallocation. */
3420     if (new_bytes) {
3421         bdrv_make_request_serialising(&req, 1);
3422     }
3423     ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req,
3424                                     0);
3425     if (ret < 0) {
3426         error_setg_errno(errp, -ret,
3427                          "Failed to prepare request for truncation");
3428         goto out;
3429     }
3430
3431     filtered = bdrv_filter_child(bs);
3432     backing = bdrv_cow_child(bs);
3433
3434     /*
3435      * If the image has a backing file that is large enough that it would
3436      * provide data for the new area, we cannot leave it unallocated because
3437      * then the backing file content would become visible. Instead, zero-fill
3438      * the new area.
3439      *
3440      * Note that if the image has a backing file, but was opened without the
3441      * backing file, taking care of keeping things consistent with that backing
3442      * file is the user's responsibility.
3443      */
3444     if (new_bytes && backing) {
3445         int64_t backing_len;
3446
3447         backing_len = bdrv_getlength(backing->bs);
3448         if (backing_len < 0) {
3449             ret = backing_len;
3450             error_setg_errno(errp, -ret, "Could not get backing file size");
3451             goto out;
3452         }
3453
3454         if (backing_len > old_size) {
3455             flags |= BDRV_REQ_ZERO_WRITE;
3456         }
3457     }
3458
3459     if (drv->bdrv_co_truncate) {
3460         if (flags & ~bs->supported_truncate_flags) {
3461             error_setg(errp, "Block driver does not support requested flags");
3462             ret = -ENOTSUP;
3463             goto out;
3464         }
3465         ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, flags, errp);
3466     } else if (filtered) {
3467         ret = bdrv_co_truncate(filtered, offset, exact, prealloc, flags, errp);
3468     } else {
3469         error_setg(errp, "Image format driver does not support resize");
3470         ret = -ENOTSUP;
3471         goto out;
3472     }
3473     if (ret < 0) {
3474         goto out;
3475     }
3476
3477     ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
3478     if (ret < 0) {
3479         error_setg_errno(errp, -ret, "Could not refresh total sector count");
3480     } else {
3481         offset = bs->total_sectors * BDRV_SECTOR_SIZE;
3482     }
3483     /* It's possible that truncation succeeded but refresh_total_sectors
3484      * failed, but the latter doesn't affect how we should finish the request.
3485      * Pass 0 as the last parameter so that dirty bitmaps etc. are handled. */
3486     bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0);
3487
3488 out:
3489     tracked_request_end(&req);
3490     bdrv_dec_in_flight(bs);
3491
3492     return ret;
3493 }
3494
3495 void bdrv_cancel_in_flight(BlockDriverState *bs)
3496 {
3497     GLOBAL_STATE_CODE();
3498     if (!bs || !bs->drv) {
3499         return;
3500     }
3501
3502     if (bs->drv->bdrv_cancel_in_flight) {
3503         bs->drv->bdrv_cancel_in_flight(bs);
3504     }
3505 }
3506
3507 int coroutine_fn
3508 bdrv_co_preadv_snapshot(BdrvChild *child, int64_t offset, int64_t bytes,
3509                         QEMUIOVector *qiov, size_t qiov_offset)
3510 {
3511     BlockDriverState *bs = child->bs;
3512     BlockDriver *drv = bs->drv;
3513     int ret;
3514     IO_CODE();
3515
3516     if (!drv) {
3517         return -ENOMEDIUM;
3518     }
3519
3520     if (!drv->bdrv_co_preadv_snapshot) {
3521         return -ENOTSUP;
3522     }
3523
3524     bdrv_inc_in_flight(bs);
3525     ret = drv->bdrv_co_preadv_snapshot(bs, offset, bytes, qiov, qiov_offset);
3526     bdrv_dec_in_flight(bs);
3527
3528     return ret;
3529 }
3530
3531 int coroutine_fn
3532 bdrv_co_snapshot_block_status(BlockDriverState *bs,
3533                               bool want_zero, int64_t offset, int64_t bytes,
3534                               int64_t *pnum, int64_t *map,
3535                               BlockDriverState **file)
3536 {
3537     BlockDriver *drv = bs->drv;
3538     int ret;
3539     IO_CODE();
3540
3541     if (!drv) {
3542         return -ENOMEDIUM;
3543     }
3544
3545     if (!drv->bdrv_co_snapshot_block_status) {
3546         return -ENOTSUP;
3547     }
3548
3549     bdrv_inc_in_flight(bs);
3550     ret = drv->bdrv_co_snapshot_block_status(bs, want_zero, offset, bytes,
3551                                              pnum, map, file);
3552     bdrv_dec_in_flight(bs);
3553
3554     return ret;
3555 }
3556
3557 int coroutine_fn
3558 bdrv_co_pdiscard_snapshot(BlockDriverState *bs, int64_t offset, int64_t bytes)
3559 {
3560     BlockDriver *drv = bs->drv;
3561     int ret;
3562     IO_CODE();
3563
3564     if (!drv) {
3565         return -ENOMEDIUM;
3566     }
3567
3568     if (!drv->bdrv_co_pdiscard_snapshot) {
3569         return -ENOTSUP;
3570     }
3571
3572     bdrv_inc_in_flight(bs);
3573     ret = drv->bdrv_co_pdiscard_snapshot(bs, offset, bytes);
3574     bdrv_dec_in_flight(bs);
3575
3576     return ret;
3577 }