block/io.c

   1 /*
   2  * Block layer I/O functions
   3  *
   4  * Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24
  25 #include "qemu/osdep.h"
  26 #include "trace.h"
  27 #include "sysemu/block-backend.h"
  28 #include "block/blockjob.h"
  29 #include "block/block_int.h"
  30 #include "qemu/cutils.h"
  31 #include "qapi/error.h"
  32 #include "qemu/error-report.h"
  33
  34 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
  35
  36 static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
  37                                          int64_t sector_num,
  38                                          QEMUIOVector *qiov,
  39                                          int nb_sectors,
  40                                          BdrvRequestFlags flags,
  41                                          BlockCompletionFunc *cb,
  42                                          void *opaque,
  43                                          bool is_write);
  44 static void coroutine_fn bdrv_co_do_rw(void *opaque);
  45 static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
  46     int64_t sector_num, int nb_sectors, BdrvRequestFlags flags);
  47
  48 static void bdrv_parent_drained_begin(BlockDriverState *bs)
  49 {
  50     BdrvChild *c;
  51
  52     QLIST_FOREACH(c, &bs->parents, next_parent) {
  53         if (c->role->drained_begin) {
  54             c->role->drained_begin(c);
  55         }
  56     }
  57 }
  58
  59 static void bdrv_parent_drained_end(BlockDriverState *bs)
  60 {
  61     BdrvChild *c;
  62
  63     QLIST_FOREACH(c, &bs->parents, next_parent) {
  64         if (c->role->drained_end) {
  65             c->role->drained_end(c);
  66         }
  67     }
  68 }
  69
  70 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
  71 {
  72     BlockDriver *drv = bs->drv;
  73     Error *local_err = NULL;
  74
  75     memset(&bs->bl, 0, sizeof(bs->bl));
  76
  77     if (!drv) {
  78         return;
  79     }
  80
  81     /* Take some limits from the children as a default */
  82     if (bs->file) {
  83         bdrv_refresh_limits(bs->file->bs, &local_err);
  84         if (local_err) {
  85             error_propagate(errp, local_err);
  86             return;
  87         }
  88         bs->bl.opt_transfer_length = bs->file->bs->bl.opt_transfer_length;
  89         bs->bl.max_transfer_length = bs->file->bs->bl.max_transfer_length;
  90         bs->bl.min_mem_alignment = bs->file->bs->bl.min_mem_alignment;
  91         bs->bl.opt_mem_alignment = bs->file->bs->bl.opt_mem_alignment;
  92         bs->bl.max_iov = bs->file->bs->bl.max_iov;
  93     } else {
  94         bs->bl.min_mem_alignment = 512;
  95         bs->bl.opt_mem_alignment = getpagesize();
  96
  97         /* Safe default since most protocols use readv()/writev()/etc */
  98         bs->bl.max_iov = IOV_MAX;
  99     }
 100
 101     if (bs->backing) {
 102         bdrv_refresh_limits(bs->backing->bs, &local_err);
 103         if (local_err) {
 104             error_propagate(errp, local_err);
 105             return;
 106         }
 107         bs->bl.opt_transfer_length =
 108             MAX(bs->bl.opt_transfer_length,
 109                 bs->backing->bs->bl.opt_transfer_length);
 110         bs->bl.max_transfer_length =
 111             MIN_NON_ZERO(bs->bl.max_transfer_length,
 112                          bs->backing->bs->bl.max_transfer_length);
 113         bs->bl.opt_mem_alignment =
 114             MAX(bs->bl.opt_mem_alignment,
 115                 bs->backing->bs->bl.opt_mem_alignment);
 116         bs->bl.min_mem_alignment =
 117             MAX(bs->bl.min_mem_alignment,
 118                 bs->backing->bs->bl.min_mem_alignment);
 119         bs->bl.max_iov =
 120             MIN(bs->bl.max_iov,
 121                 bs->backing->bs->bl.max_iov);
 122     }
 123
 124     /* Then let the driver override it */
 125     if (drv->bdrv_refresh_limits) {
 126         drv->bdrv_refresh_limits(bs, errp);
 127     }
 128 }
 129
 130 /**
 131  * The copy-on-read flag is actually a reference count so multiple users may
 132  * use the feature without worrying about clobbering its previous state.
 133  * Copy-on-read stays enabled until all users have called to disable it.
 134  */
 135 void bdrv_enable_copy_on_read(BlockDriverState *bs)
 136 {
 137     bs->copy_on_read++;
 138 }
 139
 140 void bdrv_disable_copy_on_read(BlockDriverState *bs)
 141 {
 142     assert(bs->copy_on_read > 0);
 143     bs->copy_on_read--;
 144 }
 145
 146 /* Check if any requests are in-flight (including throttled requests) */
 147 bool bdrv_requests_pending(BlockDriverState *bs)
 148 {
 149     BdrvChild *child;
 150
 151     if (!QLIST_EMPTY(&bs->tracked_requests)) {
 152         return true;
 153     }
 154
 155     QLIST_FOREACH(child, &bs->children, next) {
 156         if (bdrv_requests_pending(child->bs)) {
 157             return true;
 158         }
 159     }
 160
 161     return false;
 162 }
 163
 164 static void bdrv_drain_recurse(BlockDriverState *bs)
 165 {
 166     BdrvChild *child;
 167
 168     if (bs->drv && bs->drv->bdrv_drain) {
 169         bs->drv->bdrv_drain(bs);
 170     }
 171     QLIST_FOREACH(child, &bs->children, next) {
 172         bdrv_drain_recurse(child->bs);
 173     }
 174 }
 175
 176 typedef struct {
 177     Coroutine *co;
 178     BlockDriverState *bs;
 179     QEMUBH *bh;
 180     bool done;
 181 } BdrvCoDrainData;
 182
 183 static void bdrv_drain_poll(BlockDriverState *bs)
 184 {
 185     bool busy = true;
 186
 187     while (busy) {
 188         /* Keep iterating */
 189         busy = bdrv_requests_pending(bs);
 190         busy |= aio_poll(bdrv_get_aio_context(bs), busy);
 191     }
 192 }
 193
 194 static void bdrv_co_drain_bh_cb(void *opaque)
 195 {
 196     BdrvCoDrainData *data = opaque;
 197     Coroutine *co = data->co;
 198
 199     qemu_bh_delete(data->bh);
 200     bdrv_drain_poll(data->bs);
 201     data->done = true;
 202     qemu_coroutine_enter(co, NULL);
 203 }
 204
 205 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs)
 206 {
 207     BdrvCoDrainData data;
 208
 209     /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
 210      * other coroutines run if they were queued from
 211      * qemu_co_queue_run_restart(). */
 212
 213     assert(qemu_in_coroutine());
 214     data = (BdrvCoDrainData) {
 215         .co = qemu_coroutine_self(),
 216         .bs = bs,
 217         .done = false,
 218         .bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_drain_bh_cb, &data),
 219     };
 220     qemu_bh_schedule(data.bh);
 221
 222     qemu_coroutine_yield();
 223     /* If we are resumed from some other event (such as an aio completion or a
 224      * timer callback), it is a bug in the caller that should be fixed. */
 225     assert(data.done);
 226 }
 227
 228 void bdrv_drained_begin(BlockDriverState *bs)
 229 {
 230     if (!bs->quiesce_counter++) {
 231         aio_disable_external(bdrv_get_aio_context(bs));
 232         bdrv_parent_drained_begin(bs);
 233     }
 234
 235     bdrv_io_unplugged_begin(bs);
 236     bdrv_drain_recurse(bs);
 237     if (qemu_in_coroutine()) {
 238         bdrv_co_yield_to_drain(bs);
 239     } else {
 240         bdrv_drain_poll(bs);
 241     }
 242     bdrv_io_unplugged_end(bs);
 243 }
 244
 245 void bdrv_drained_end(BlockDriverState *bs)
 246 {
 247     assert(bs->quiesce_counter > 0);
 248     if (--bs->quiesce_counter > 0) {
 249         return;
 250     }
 251
 252     bdrv_parent_drained_end(bs);
 253     aio_enable_external(bdrv_get_aio_context(bs));
 254 }
 255
 256 /*
 257  * Wait for pending requests to complete on a single BlockDriverState subtree,
 258  * and suspend block driver's internal I/O until next request arrives.
 259  *
 260  * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
 261  * AioContext.
 262  *
 263  * Only this BlockDriverState's AioContext is run, so in-flight requests must
 264  * not depend on events in other AioContexts.  In that case, use
 265  * bdrv_drain_all() instead.
 266  */
 267 void coroutine_fn bdrv_co_drain(BlockDriverState *bs)
 268 {
 269     assert(qemu_in_coroutine());
 270     bdrv_drained_begin(bs);
 271     bdrv_drained_end(bs);
 272 }
 273
 274 void bdrv_drain(BlockDriverState *bs)
 275 {
 276     bdrv_drained_begin(bs);
 277     bdrv_drained_end(bs);
 278 }
 279
 280 /*
 281  * Wait for pending requests to complete across all BlockDriverStates
 282  *
 283  * This function does not flush data to disk, use bdrv_flush_all() for that
 284  * after calling this function.
 285  */
 286 void bdrv_drain_all(void)
 287 {
 288     /* Always run first iteration so any pending completion BHs run */
 289     bool busy = true;
 290     BlockDriverState *bs;
 291     BdrvNextIterator it;
 292     GSList *aio_ctxs = NULL, *ctx;
 293
 294     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
 295         AioContext *aio_context = bdrv_get_aio_context(bs);
 296
 297         aio_context_acquire(aio_context);
 298         if (bs->job) {
 299             block_job_pause(bs->job);
 300         }
 301         bdrv_parent_drained_begin(bs);
 302         bdrv_io_unplugged_begin(bs);
 303         bdrv_drain_recurse(bs);
 304         aio_context_release(aio_context);
 305
 306         if (!g_slist_find(aio_ctxs, aio_context)) {
 307             aio_ctxs = g_slist_prepend(aio_ctxs, aio_context);
 308         }
 309     }
 310
 311     /* Note that completion of an asynchronous I/O operation can trigger any
 312      * number of other I/O operations on other devices---for example a
 313      * coroutine can submit an I/O request to another device in response to
 314      * request completion.  Therefore we must keep looping until there was no
 315      * more activity rather than simply draining each device independently.
 316      */
 317     while (busy) {
 318         busy = false;
 319
 320         for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) {
 321             AioContext *aio_context = ctx->data;
 322
 323             aio_context_acquire(aio_context);
 324             for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
 325                 if (aio_context == bdrv_get_aio_context(bs)) {
 326                     if (bdrv_requests_pending(bs)) {
 327                         busy = true;
 328                         aio_poll(aio_context, busy);
 329                     }
 330                 }
 331             }
 332             busy |= aio_poll(aio_context, false);
 333             aio_context_release(aio_context);
 334         }
 335     }
 336
 337     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
 338         AioContext *aio_context = bdrv_get_aio_context(bs);
 339
 340         aio_context_acquire(aio_context);
 341         bdrv_io_unplugged_end(bs);
 342         bdrv_parent_drained_end(bs);
 343         if (bs->job) {
 344             block_job_resume(bs->job);
 345         }
 346         aio_context_release(aio_context);
 347     }
 348     g_slist_free(aio_ctxs);
 349 }
 350
 351 /**
 352  * Remove an active request from the tracked requests list
 353  *
 354  * This function should be called when a tracked request is completing.
 355  */
 356 static void tracked_request_end(BdrvTrackedRequest *req)
 357 {
 358     if (req->serialising) {
 359         req->bs->serialising_in_flight--;
 360     }
 361
 362     QLIST_REMOVE(req, list);
 363     qemu_co_queue_restart_all(&req->wait_queue);
 364 }
 365
 366 /**
 367  * Add an active request to the tracked requests list
 368  */
 369 static void tracked_request_begin(BdrvTrackedRequest *req,
 370                                   BlockDriverState *bs,
 371                                   int64_t offset,
 372                                   unsigned int bytes,
 373                                   enum BdrvTrackedRequestType type)
 374 {
 375     *req = (BdrvTrackedRequest){
 376         .bs = bs,
 377         .offset         = offset,
 378         .bytes          = bytes,
 379         .type           = type,
 380         .co             = qemu_coroutine_self(),
 381         .serialising    = false,
 382         .overlap_offset = offset,
 383         .overlap_bytes  = bytes,
 384     };
 385
 386     qemu_co_queue_init(&req->wait_queue);
 387
 388     QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
 389 }
 390
 391 static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
 392 {
 393     int64_t overlap_offset = req->offset & ~(align - 1);
 394     unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
 395                                - overlap_offset;
 396
 397     if (!req->serialising) {
 398         req->bs->serialising_in_flight++;
 399         req->serialising = true;
 400     }
 401
 402     req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
 403     req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
 404 }
 405
 406 /**
 407  * Round a region to cluster boundaries
 408  */
 409 void bdrv_round_to_clusters(BlockDriverState *bs,
 410                             int64_t sector_num, int nb_sectors,
 411                             int64_t *cluster_sector_num,
 412                             int *cluster_nb_sectors)
 413 {
 414     BlockDriverInfo bdi;
 415
 416     if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
 417         *cluster_sector_num = sector_num;
 418         *cluster_nb_sectors = nb_sectors;
 419     } else {
 420         int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE;
 421         *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c);
 422         *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num +
 423                                             nb_sectors, c);
 424     }
 425 }
 426
 427 static int bdrv_get_cluster_size(BlockDriverState *bs)
 428 {
 429     BlockDriverInfo bdi;
 430     int ret;
 431
 432     ret = bdrv_get_info(bs, &bdi);
 433     if (ret < 0 || bdi.cluster_size == 0) {
 434         return bs->request_alignment;
 435     } else {
 436         return bdi.cluster_size;
 437     }
 438 }
 439
 440 static bool tracked_request_overlaps(BdrvTrackedRequest *req,
 441                                      int64_t offset, unsigned int bytes)
 442 {
 443     /*        aaaa   bbbb */
 444     if (offset >= req->overlap_offset + req->overlap_bytes) {
 445         return false;
 446     }
 447     /* bbbb   aaaa        */
 448     if (req->overlap_offset >= offset + bytes) {
 449         return false;
 450     }
 451     return true;
 452 }
 453
 454 static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self)
 455 {
 456     BlockDriverState *bs = self->bs;
 457     BdrvTrackedRequest *req;
 458     bool retry;
 459     bool waited = false;
 460
 461     if (!bs->serialising_in_flight) {
 462         return false;
 463     }
 464
 465     do {
 466         retry = false;
 467         QLIST_FOREACH(req, &bs->tracked_requests, list) {
 468             if (req == self || (!req->serialising && !self->serialising)) {
 469                 continue;
 470             }
 471             if (tracked_request_overlaps(req, self->overlap_offset,
 472                                          self->overlap_bytes))
 473             {
 474                 /* Hitting this means there was a reentrant request, for
 475                  * example, a block driver issuing nested requests.  This must
 476                  * never happen since it means deadlock.
 477                  */
 478                 assert(qemu_coroutine_self() != req->co);
 479
 480                 /* If the request is already (indirectly) waiting for us, or
 481                  * will wait for us as soon as it wakes up, then just go on
 482                  * (instead of producing a deadlock in the former case). */
 483                 if (!req->waiting_for) {
 484                     self->waiting_for = req;
 485                     qemu_co_queue_wait(&req->wait_queue);
 486                     self->waiting_for = NULL;
 487                     retry = true;
 488                     waited = true;
 489                     break;
 490                 }
 491             }
 492         }
 493     } while (retry);
 494
 495     return waited;
 496 }
 497
 498 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
 499                                    size_t size)
 500 {
 501     if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) {
 502         return -EIO;
 503     }
 504
 505     if (!bdrv_is_inserted(bs)) {
 506         return -ENOMEDIUM;
 507     }
 508
 509     if (offset < 0) {
 510         return -EIO;
 511     }
 512
 513     return 0;
 514 }
 515
 516 static int bdrv_check_request(BlockDriverState *bs, int64_t sector_num,
 517                               int nb_sectors)
 518 {
 519     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
 520         return -EIO;
 521     }
 522
 523     return bdrv_check_byte_request(bs, sector_num * BDRV_SECTOR_SIZE,
 524                                    nb_sectors * BDRV_SECTOR_SIZE);
 525 }
 526
 527 typedef struct RwCo {
 528     BlockDriverState *bs;
 529     int64_t offset;
 530     QEMUIOVector *qiov;
 531     bool is_write;
 532     int ret;
 533     BdrvRequestFlags flags;
 534 } RwCo;
 535
 536 static void coroutine_fn bdrv_rw_co_entry(void *opaque)
 537 {
 538     RwCo *rwco = opaque;
 539
 540     if (!rwco->is_write) {
 541         rwco->ret = bdrv_co_preadv(rwco->bs, rwco->offset,
 542                                    rwco->qiov->size, rwco->qiov,
 543                                    rwco->flags);
 544     } else {
 545         rwco->ret = bdrv_co_pwritev(rwco->bs, rwco->offset,
 546                                     rwco->qiov->size, rwco->qiov,
 547                                     rwco->flags);
 548     }
 549 }
 550
 551 /*
 552  * Process a vectored synchronous request using coroutines
 553  */
 554 static int bdrv_prwv_co(BlockDriverState *bs, int64_t offset,
 555                         QEMUIOVector *qiov, bool is_write,
 556                         BdrvRequestFlags flags)
 557 {
 558     Coroutine *co;
 559     RwCo rwco = {
 560         .bs = bs,
 561         .offset = offset,
 562         .qiov = qiov,
 563         .is_write = is_write,
 564         .ret = NOT_DONE,
 565         .flags = flags,
 566     };
 567
 568     if (qemu_in_coroutine()) {
 569         /* Fast-path if already in coroutine context */
 570         bdrv_rw_co_entry(&rwco);
 571     } else {
 572         AioContext *aio_context = bdrv_get_aio_context(bs);
 573
 574         co = qemu_coroutine_create(bdrv_rw_co_entry);
 575         qemu_coroutine_enter(co, &rwco);
 576         while (rwco.ret == NOT_DONE) {
 577             aio_poll(aio_context, true);
 578         }
 579     }
 580     return rwco.ret;
 581 }
 582
 583 /*
 584  * Process a synchronous request using coroutines
 585  */
 586 static int bdrv_rw_co(BlockDriverState *bs, int64_t sector_num, uint8_t *buf,
 587                       int nb_sectors, bool is_write, BdrvRequestFlags flags)
 588 {
 589     QEMUIOVector qiov;
 590     struct iovec iov = {
 591         .iov_base = (void *)buf,
 592         .iov_len = nb_sectors * BDRV_SECTOR_SIZE,
 593     };
 594
 595     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
 596         return -EINVAL;
 597     }
 598
 599     qemu_iovec_init_external(&qiov, &iov, 1);
 600     return bdrv_prwv_co(bs, sector_num << BDRV_SECTOR_BITS,
 601                         &qiov, is_write, flags);
 602 }
 603
 604 /* return < 0 if error. See bdrv_write() for the return codes */
 605 int bdrv_read(BlockDriverState *bs, int64_t sector_num,
 606               uint8_t *buf, int nb_sectors)
 607 {
 608     return bdrv_rw_co(bs, sector_num, buf, nb_sectors, false, 0);
 609 }
 610
 611 /* Return < 0 if error. Important errors are:
 612   -EIO         generic I/O error (may happen for all errors)
 613   -ENOMEDIUM   No media inserted.
 614   -EINVAL      Invalid sector number or nb_sectors
 615   -EACCES      Trying to write a read-only device
 616 */
 617 int bdrv_write(BlockDriverState *bs, int64_t sector_num,
 618                const uint8_t *buf, int nb_sectors)
 619 {
 620     return bdrv_rw_co(bs, sector_num, (uint8_t *)buf, nb_sectors, true, 0);
 621 }
 622
 623 int bdrv_write_zeroes(BlockDriverState *bs, int64_t sector_num,
 624                       int nb_sectors, BdrvRequestFlags flags)
 625 {
 626     return bdrv_rw_co(bs, sector_num, NULL, nb_sectors, true,
 627                       BDRV_REQ_ZERO_WRITE | flags);
 628 }
 629
 630 /*
 631  * Completely zero out a block device with the help of bdrv_write_zeroes.
 632  * The operation is sped up by checking the block status and only writing
 633  * zeroes to the device if they currently do not return zeroes. Optional
 634  * flags are passed through to bdrv_write_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
 635  * BDRV_REQ_FUA).
 636  *
 637  * Returns < 0 on error, 0 on success. For error codes see bdrv_write().
 638  */
 639 int bdrv_make_zero(BlockDriverState *bs, BdrvRequestFlags flags)
 640 {
 641     int64_t target_sectors, ret, nb_sectors, sector_num = 0;
 642     BlockDriverState *file;
 643     int n;
 644
 645     target_sectors = bdrv_nb_sectors(bs);
 646     if (target_sectors < 0) {
 647         return target_sectors;
 648     }
 649
 650     for (;;) {
 651         nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS);
 652         if (nb_sectors <= 0) {
 653             return 0;
 654         }
 655         ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file);
 656         if (ret < 0) {
 657             error_report("error getting block status at sector %" PRId64 ": %s",
 658                          sector_num, strerror(-ret));
 659             return ret;
 660         }
 661         if (ret & BDRV_BLOCK_ZERO) {
 662             sector_num += n;
 663             continue;
 664         }
 665         ret = bdrv_write_zeroes(bs, sector_num, n, flags);
 666         if (ret < 0) {
 667             error_report("error writing zeroes at sector %" PRId64 ": %s",
 668                          sector_num, strerror(-ret));
 669             return ret;
 670         }
 671         sector_num += n;
 672     }
 673 }
 674
 675 int bdrv_pread(BlockDriverState *bs, int64_t offset, void *buf, int bytes)
 676 {
 677     QEMUIOVector qiov;
 678     struct iovec iov = {
 679         .iov_base = (void *)buf,
 680         .iov_len = bytes,
 681     };
 682     int ret;
 683
 684     if (bytes < 0) {
 685         return -EINVAL;
 686     }
 687
 688     qemu_iovec_init_external(&qiov, &iov, 1);
 689     ret = bdrv_prwv_co(bs, offset, &qiov, false, 0);
 690     if (ret < 0) {
 691         return ret;
 692     }
 693
 694     return bytes;
 695 }
 696
 697 int bdrv_pwritev(BlockDriverState *bs, int64_t offset, QEMUIOVector *qiov)
 698 {
 699     int ret;
 700
 701     ret = bdrv_prwv_co(bs, offset, qiov, true, 0);
 702     if (ret < 0) {
 703         return ret;
 704     }
 705
 706     return qiov->size;
 707 }
 708
 709 int bdrv_pwrite(BlockDriverState *bs, int64_t offset,
 710                 const void *buf, int bytes)
 711 {
 712     QEMUIOVector qiov;
 713     struct iovec iov = {
 714         .iov_base   = (void *) buf,
 715         .iov_len    = bytes,
 716     };
 717
 718     if (bytes < 0) {
 719         return -EINVAL;
 720     }
 721
 722     qemu_iovec_init_external(&qiov, &iov, 1);
 723     return bdrv_pwritev(bs, offset, &qiov);
 724 }
 725
 726 /*
 727  * Writes to the file and ensures that no writes are reordered across this
 728  * request (acts as a barrier)
 729  *
 730  * Returns 0 on success, -errno in error cases.
 731  */
 732 int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset,
 733     const void *buf, int count)
 734 {
 735     int ret;
 736
 737     ret = bdrv_pwrite(bs, offset, buf, count);
 738     if (ret < 0) {
 739         return ret;
 740     }
 741
 742     ret = bdrv_flush(bs);
 743     if (ret < 0) {
 744         return ret;
 745     }
 746
 747     return 0;
 748 }
 749
 750 typedef struct CoroutineIOCompletion {
 751     Coroutine *coroutine;
 752     int ret;
 753 } CoroutineIOCompletion;
 754
 755 static void bdrv_co_io_em_complete(void *opaque, int ret)
 756 {
 757     CoroutineIOCompletion *co = opaque;
 758
 759     co->ret = ret;
 760     qemu_coroutine_enter(co->coroutine, NULL);
 761 }
 762
 763 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
 764                                            uint64_t offset, uint64_t bytes,
 765                                            QEMUIOVector *qiov, int flags)
 766 {
 767     BlockDriver *drv = bs->drv;
 768     int64_t sector_num;
 769     unsigned int nb_sectors;
 770
 771     if (drv->bdrv_co_preadv) {
 772         return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
 773     }
 774
 775     sector_num = offset >> BDRV_SECTOR_BITS;
 776     nb_sectors = bytes >> BDRV_SECTOR_BITS;
 777
 778     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
 779     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
 780     assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
 781
 782     if (drv->bdrv_co_readv) {
 783         return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
 784     } else {
 785         BlockAIOCB *acb;
 786         CoroutineIOCompletion co = {
 787             .coroutine = qemu_coroutine_self(),
 788         };
 789
 790         acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors,
 791                                       bdrv_co_io_em_complete, &co);
 792         if (acb == NULL) {
 793             return -EIO;
 794         } else {
 795             qemu_coroutine_yield();
 796             return co.ret;
 797         }
 798     }
 799 }
 800
 801 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
 802                                             uint64_t offset, uint64_t bytes,
 803                                             QEMUIOVector *qiov, int flags)
 804 {
 805     BlockDriver *drv = bs->drv;
 806     int64_t sector_num;
 807     unsigned int nb_sectors;
 808     int ret;
 809
 810     if (drv->bdrv_co_pwritev) {
 811         ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, flags);
 812         goto emulate_flags;
 813     }
 814
 815     sector_num = offset >> BDRV_SECTOR_BITS;
 816     nb_sectors = bytes >> BDRV_SECTOR_BITS;
 817
 818     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
 819     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
 820     assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
 821
 822     if (drv->bdrv_co_writev_flags) {
 823         ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov,
 824                                         flags & bs->supported_write_flags);
 825         flags &= ~bs->supported_write_flags;
 826     } else if (drv->bdrv_co_writev) {
 827         assert(!bs->supported_write_flags);
 828         ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov);
 829     } else {
 830         BlockAIOCB *acb;
 831         CoroutineIOCompletion co = {
 832             .coroutine = qemu_coroutine_self(),
 833         };
 834
 835         acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors,
 836                                        bdrv_co_io_em_complete, &co);
 837         if (acb == NULL) {
 838             ret = -EIO;
 839         } else {
 840             qemu_coroutine_yield();
 841             ret = co.ret;
 842         }
 843     }
 844
 845 emulate_flags:
 846     if (ret == 0 && (flags & BDRV_REQ_FUA)) {
 847         ret = bdrv_co_flush(bs);
 848     }
 849
 850     return ret;
 851 }
 852
 853 static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs,
 854         int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
 855 {
 856     /* Perform I/O through a temporary buffer so that users who scribble over
 857      * their read buffer while the operation is in progress do not end up
 858      * modifying the image file.  This is critical for zero-copy guest I/O
 859      * where anything might happen inside guest memory.
 860      */
 861     void *bounce_buffer;
 862
 863     BlockDriver *drv = bs->drv;
 864     struct iovec iov;
 865     QEMUIOVector bounce_qiov;
 866     int64_t cluster_sector_num;
 867     int cluster_nb_sectors;
 868     size_t skip_bytes;
 869     int ret;
 870
 871     /* Cover entire cluster so no additional backing file I/O is required when
 872      * allocating cluster in the image file.
 873      */
 874     bdrv_round_to_clusters(bs, sector_num, nb_sectors,
 875                            &cluster_sector_num, &cluster_nb_sectors);
 876
 877     trace_bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors,
 878                                    cluster_sector_num, cluster_nb_sectors);
 879
 880     iov.iov_len = cluster_nb_sectors * BDRV_SECTOR_SIZE;
 881     iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len);
 882     if (bounce_buffer == NULL) {
 883         ret = -ENOMEM;
 884         goto err;
 885     }
 886
 887     qemu_iovec_init_external(&bounce_qiov, &iov, 1);
 888
 889     ret = bdrv_driver_preadv(bs, cluster_sector_num * BDRV_SECTOR_SIZE,
 890                              cluster_nb_sectors * BDRV_SECTOR_SIZE,
 891                              &bounce_qiov, 0);
 892     if (ret < 0) {
 893         goto err;
 894     }
 895
 896     if (drv->bdrv_co_write_zeroes &&
 897         buffer_is_zero(bounce_buffer, iov.iov_len)) {
 898         ret = bdrv_co_do_write_zeroes(bs, cluster_sector_num,
 899                                       cluster_nb_sectors, 0);
 900     } else {
 901         /* This does not change the data on the disk, it is not necessary
 902          * to flush even in cache=writethrough mode.
 903          */
 904         ret = bdrv_driver_pwritev(bs, cluster_sector_num * BDRV_SECTOR_SIZE,
 905                                   cluster_nb_sectors * BDRV_SECTOR_SIZE,
 906                                   &bounce_qiov, 0);
 907     }
 908
 909     if (ret < 0) {
 910         /* It might be okay to ignore write errors for guest requests.  If this
 911          * is a deliberate copy-on-read then we don't want to ignore the error.
 912          * Simply report it in all cases.
 913          */
 914         goto err;
 915     }
 916
 917     skip_bytes = (sector_num - cluster_sector_num) * BDRV_SECTOR_SIZE;
 918     qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes,
 919                         nb_sectors * BDRV_SECTOR_SIZE);
 920
 921 err:
 922     qemu_vfree(bounce_buffer);
 923     return ret;
 924 }
 925
 926 /*
 927  * Forwards an already correctly aligned request to the BlockDriver. This
 928  * handles copy on read and zeroing after EOF; any other features must be
 929  * implemented by the caller.
 930  */
 931 static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs,
 932     BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
 933     int64_t align, QEMUIOVector *qiov, int flags)
 934 {
 935     int ret;
 936
 937     int64_t sector_num = offset >> BDRV_SECTOR_BITS;
 938     unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS;
 939
 940     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
 941     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
 942     assert(!qiov || bytes == qiov->size);
 943     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
 944
 945     /* Handle Copy on Read and associated serialisation */
 946     if (flags & BDRV_REQ_COPY_ON_READ) {
 947         /* If we touch the same cluster it counts as an overlap.  This
 948          * guarantees that allocating writes will be serialized and not race
 949          * with each other for the same cluster.  For example, in copy-on-read
 950          * it ensures that the CoR read and write operations are atomic and
 951          * guest writes cannot interleave between them. */
 952         mark_request_serialising(req, bdrv_get_cluster_size(bs));
 953     }
 954
 955     if (!(flags & BDRV_REQ_NO_SERIALISING)) {
 956         wait_serialising_requests(req);
 957     }
 958
 959     if (flags & BDRV_REQ_COPY_ON_READ) {
 960         int pnum;
 961
 962         ret = bdrv_is_allocated(bs, sector_num, nb_sectors, &pnum);
 963         if (ret < 0) {
 964             goto out;
 965         }
 966
 967         if (!ret || pnum != nb_sectors) {
 968             ret = bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, qiov);
 969             goto out;
 970         }
 971     }
 972
 973     /* Forward the request to the BlockDriver */
 974     if (!bs->zero_beyond_eof) {
 975         ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
 976     } else {
 977         /* Read zeros after EOF */
 978         int64_t total_sectors, max_nb_sectors;
 979
 980         total_sectors = bdrv_nb_sectors(bs);
 981         if (total_sectors < 0) {
 982             ret = total_sectors;
 983             goto out;
 984         }
 985
 986         max_nb_sectors = ROUND_UP(MAX(0, total_sectors - sector_num),
 987                                   align >> BDRV_SECTOR_BITS);
 988         if (nb_sectors < max_nb_sectors) {
 989             ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
 990         } else if (max_nb_sectors > 0) {
 991             QEMUIOVector local_qiov;
 992
 993             qemu_iovec_init(&local_qiov, qiov->niov);
 994             qemu_iovec_concat(&local_qiov, qiov, 0,
 995                               max_nb_sectors * BDRV_SECTOR_SIZE);
 996
 997             ret = bdrv_driver_preadv(bs, offset,
 998                                      max_nb_sectors * BDRV_SECTOR_SIZE,
 999                                      &local_qiov, 0);
1000
1001             qemu_iovec_destroy(&local_qiov);
1002         } else {
1003             ret = 0;
1004         }
1005
1006         /* Reading beyond end of file is supposed to produce zeroes */
1007         if (ret == 0 && total_sectors < sector_num + nb_sectors) {
1008             uint64_t offset = MAX(0, total_sectors - sector_num);
1009             uint64_t bytes = (sector_num + nb_sectors - offset) *
1010                               BDRV_SECTOR_SIZE;
1011             qemu_iovec_memset(qiov, offset * BDRV_SECTOR_SIZE, 0, bytes);
1012         }
1013     }
1014
1015 out:
1016     return ret;
1017 }
1018
1019 /*
1020  * Handle a read request in coroutine context
1021  */
1022 int coroutine_fn bdrv_co_preadv(BlockDriverState *bs,
1023     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1024     BdrvRequestFlags flags)
1025 {
1026     BlockDriver *drv = bs->drv;
1027     BdrvTrackedRequest req;
1028
1029     /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */
1030     uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1031     uint8_t *head_buf = NULL;
1032     uint8_t *tail_buf = NULL;
1033     QEMUIOVector local_qiov;
1034     bool use_local_qiov = false;
1035     int ret;
1036
1037     if (!drv) {
1038         return -ENOMEDIUM;
1039     }
1040
1041     ret = bdrv_check_byte_request(bs, offset, bytes);
1042     if (ret < 0) {
1043         return ret;
1044     }
1045
1046     /* Don't do copy-on-read if we read data before write operation */
1047     if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) {
1048         flags |= BDRV_REQ_COPY_ON_READ;
1049     }
1050
1051     /* Align read if necessary by padding qiov */
1052     if (offset & (align - 1)) {
1053         head_buf = qemu_blockalign(bs, align);
1054         qemu_iovec_init(&local_qiov, qiov->niov + 2);
1055         qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
1056         qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1057         use_local_qiov = true;
1058
1059         bytes += offset & (align - 1);
1060         offset = offset & ~(align - 1);
1061     }
1062
1063     if ((offset + bytes) & (align - 1)) {
1064         if (!use_local_qiov) {
1065             qemu_iovec_init(&local_qiov, qiov->niov + 1);
1066             qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1067             use_local_qiov = true;
1068         }
1069         tail_buf = qemu_blockalign(bs, align);
1070         qemu_iovec_add(&local_qiov, tail_buf,
1071                        align - ((offset + bytes) & (align - 1)));
1072
1073         bytes = ROUND_UP(bytes, align);
1074     }
1075
1076     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
1077     ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align,
1078                               use_local_qiov ? &local_qiov : qiov,
1079                               flags);
1080     tracked_request_end(&req);
1081
1082     if (use_local_qiov) {
1083         qemu_iovec_destroy(&local_qiov);
1084         qemu_vfree(head_buf);
1085         qemu_vfree(tail_buf);
1086     }
1087
1088     return ret;
1089 }
1090
1091 static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs,
1092     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
1093     BdrvRequestFlags flags)
1094 {
1095     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
1096         return -EINVAL;
1097     }
1098
1099     return bdrv_co_preadv(bs, sector_num << BDRV_SECTOR_BITS,
1100                           nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
1101 }
1102
1103 int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num,
1104     int nb_sectors, QEMUIOVector *qiov)
1105 {
1106     trace_bdrv_co_readv(bs, sector_num, nb_sectors);
1107
1108     return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 0);
1109 }
1110
1111 #define MAX_WRITE_ZEROES_BOUNCE_BUFFER 32768
1112
1113 static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
1114     int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
1115 {
1116     BlockDriver *drv = bs->drv;
1117     QEMUIOVector qiov;
1118     struct iovec iov = {0};
1119     int ret = 0;
1120     bool need_flush = false;
1121
1122     int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_write_zeroes,
1123                                         BDRV_REQUEST_MAX_SECTORS);
1124
1125     while (nb_sectors > 0 && !ret) {
1126         int num = nb_sectors;
1127
1128         /* Align request.  Block drivers can expect the "bulk" of the request
1129          * to be aligned.
1130          */
1131         if (bs->bl.write_zeroes_alignment
1132             && num > bs->bl.write_zeroes_alignment) {
1133             if (sector_num % bs->bl.write_zeroes_alignment != 0) {
1134                 /* Make a small request up to the first aligned sector.  */
1135                 num = bs->bl.write_zeroes_alignment;
1136                 num -= sector_num % bs->bl.write_zeroes_alignment;
1137             } else if ((sector_num + num) % bs->bl.write_zeroes_alignment != 0) {
1138                 /* Shorten the request to the last aligned sector.  num cannot
1139                  * underflow because num > bs->bl.write_zeroes_alignment.
1140                  */
1141                 num -= (sector_num + num) % bs->bl.write_zeroes_alignment;
1142             }
1143         }
1144
1145         /* limit request size */
1146         if (num > max_write_zeroes) {
1147             num = max_write_zeroes;
1148         }
1149
1150         ret = -ENOTSUP;
1151         /* First try the efficient write zeroes operation */
1152         if (drv->bdrv_co_write_zeroes) {
1153             ret = drv->bdrv_co_write_zeroes(bs, sector_num, num,
1154                                             flags & bs->supported_zero_flags);
1155             if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1156                 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1157                 need_flush = true;
1158             }
1159         } else {
1160             assert(!bs->supported_zero_flags);
1161         }
1162
1163         if (ret == -ENOTSUP) {
1164             /* Fall back to bounce buffer if write zeroes is unsupported */
1165             int max_xfer_len = MIN_NON_ZERO(bs->bl.max_transfer_length,
1166                                             MAX_WRITE_ZEROES_BOUNCE_BUFFER);
1167             BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1168
1169             if ((flags & BDRV_REQ_FUA) &&
1170                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1171                 /* No need for bdrv_driver_pwrite() to do a fallback
1172                  * flush on each chunk; use just one at the end */
1173                 write_flags &= ~BDRV_REQ_FUA;
1174                 need_flush = true;
1175             }
1176             num = MIN(num, max_xfer_len);
1177             iov.iov_len = num * BDRV_SECTOR_SIZE;
1178             if (iov.iov_base == NULL) {
1179                 iov.iov_base = qemu_try_blockalign(bs, num * BDRV_SECTOR_SIZE);
1180                 if (iov.iov_base == NULL) {
1181                     ret = -ENOMEM;
1182                     goto fail;
1183                 }
1184                 memset(iov.iov_base, 0, num * BDRV_SECTOR_SIZE);
1185             }
1186             qemu_iovec_init_external(&qiov, &iov, 1);
1187
1188             ret = bdrv_driver_pwritev(bs, sector_num * BDRV_SECTOR_SIZE,
1189                                       num * BDRV_SECTOR_SIZE, &qiov,
1190                                       write_flags);
1191
1192             /* Keep bounce buffer around if it is big enough for all
1193              * all future requests.
1194              */
1195             if (num < max_xfer_len) {
1196                 qemu_vfree(iov.iov_base);
1197                 iov.iov_base = NULL;
1198             }
1199         }
1200
1201         sector_num += num;
1202         nb_sectors -= num;
1203     }
1204
1205 fail:
1206     if (ret == 0 && need_flush) {
1207         ret = bdrv_co_flush(bs);
1208     }
1209     qemu_vfree(iov.iov_base);
1210     return ret;
1211 }
1212
1213 /*
1214  * Forwards an already correctly aligned write request to the BlockDriver.
1215  */
1216 static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
1217     BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
1218     QEMUIOVector *qiov, int flags)
1219 {
1220     BlockDriver *drv = bs->drv;
1221     bool waited;
1222     int ret;
1223
1224     int64_t sector_num = offset >> BDRV_SECTOR_BITS;
1225     unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS;
1226
1227     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
1228     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
1229     assert(!qiov || bytes == qiov->size);
1230     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1231
1232     waited = wait_serialising_requests(req);
1233     assert(!waited || !req->serialising);
1234     assert(req->overlap_offset <= offset);
1235     assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1236
1237     ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req);
1238
1239     if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
1240         !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_write_zeroes &&
1241         qemu_iovec_is_zero(qiov)) {
1242         flags |= BDRV_REQ_ZERO_WRITE;
1243         if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
1244             flags |= BDRV_REQ_MAY_UNMAP;
1245         }
1246     }
1247
1248     if (ret < 0) {
1249         /* Do nothing, write notifier decided to fail this request */
1250     } else if (flags & BDRV_REQ_ZERO_WRITE) {
1251         bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
1252         ret = bdrv_co_do_write_zeroes(bs, sector_num, nb_sectors, flags);
1253     } else {
1254         bdrv_debug_event(bs, BLKDBG_PWRITEV);
1255         ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags);
1256     }
1257     bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
1258
1259     bdrv_set_dirty(bs, sector_num, nb_sectors);
1260
1261     if (bs->wr_highest_offset < offset + bytes) {
1262         bs->wr_highest_offset = offset + bytes;
1263     }
1264
1265     if (ret >= 0) {
1266         bs->total_sectors = MAX(bs->total_sectors, sector_num + nb_sectors);
1267     }
1268
1269     return ret;
1270 }
1271
1272 static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs,
1273                                                 int64_t offset,
1274                                                 unsigned int bytes,
1275                                                 BdrvRequestFlags flags,
1276                                                 BdrvTrackedRequest *req)
1277 {
1278     uint8_t *buf = NULL;
1279     QEMUIOVector local_qiov;
1280     struct iovec iov;
1281     uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1282     unsigned int head_padding_bytes, tail_padding_bytes;
1283     int ret = 0;
1284
1285     head_padding_bytes = offset & (align - 1);
1286     tail_padding_bytes = align - ((offset + bytes) & (align - 1));
1287
1288
1289     assert(flags & BDRV_REQ_ZERO_WRITE);
1290     if (head_padding_bytes || tail_padding_bytes) {
1291         buf = qemu_blockalign(bs, align);
1292         iov = (struct iovec) {
1293             .iov_base   = buf,
1294             .iov_len    = align,
1295         };
1296         qemu_iovec_init_external(&local_qiov, &iov, 1);
1297     }
1298     if (head_padding_bytes) {
1299         uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes);
1300
1301         /* RMW the unaligned part before head. */
1302         mark_request_serialising(req, align);
1303         wait_serialising_requests(req);
1304         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1305         ret = bdrv_aligned_preadv(bs, req, offset & ~(align - 1), align,
1306                                   align, &local_qiov, 0);
1307         if (ret < 0) {
1308             goto fail;
1309         }
1310         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1311
1312         memset(buf + head_padding_bytes, 0, zero_bytes);
1313         ret = bdrv_aligned_pwritev(bs, req, offset & ~(align - 1), align,
1314                                    &local_qiov,
1315                                    flags & ~BDRV_REQ_ZERO_WRITE);
1316         if (ret < 0) {
1317             goto fail;
1318         }
1319         offset += zero_bytes;
1320         bytes -= zero_bytes;
1321     }
1322
1323     assert(!bytes || (offset & (align - 1)) == 0);
1324     if (bytes >= align) {
1325         /* Write the aligned part in the middle. */
1326         uint64_t aligned_bytes = bytes & ~(align - 1);
1327         ret = bdrv_aligned_pwritev(bs, req, offset, aligned_bytes,
1328                                    NULL, flags);
1329         if (ret < 0) {
1330             goto fail;
1331         }
1332         bytes -= aligned_bytes;
1333         offset += aligned_bytes;
1334     }
1335
1336     assert(!bytes || (offset & (align - 1)) == 0);
1337     if (bytes) {
1338         assert(align == tail_padding_bytes + bytes);
1339         /* RMW the unaligned part after tail. */
1340         mark_request_serialising(req, align);
1341         wait_serialising_requests(req);
1342         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1343         ret = bdrv_aligned_preadv(bs, req, offset, align,
1344                                   align, &local_qiov, 0);
1345         if (ret < 0) {
1346             goto fail;
1347         }
1348         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1349
1350         memset(buf, 0, bytes);
1351         ret = bdrv_aligned_pwritev(bs, req, offset, align,
1352                                    &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE);
1353     }
1354 fail:
1355     qemu_vfree(buf);
1356     return ret;
1357
1358 }
1359
1360 /*
1361  * Handle a write request in coroutine context
1362  */
1363 int coroutine_fn bdrv_co_pwritev(BlockDriverState *bs,
1364     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1365     BdrvRequestFlags flags)
1366 {
1367     BdrvTrackedRequest req;
1368     /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */
1369     uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1370     uint8_t *head_buf = NULL;
1371     uint8_t *tail_buf = NULL;
1372     QEMUIOVector local_qiov;
1373     bool use_local_qiov = false;
1374     int ret;
1375
1376     if (!bs->drv) {
1377         return -ENOMEDIUM;
1378     }
1379     if (bs->read_only) {
1380         return -EPERM;
1381     }
1382     assert(!(bs->open_flags & BDRV_O_INACTIVE));
1383
1384     ret = bdrv_check_byte_request(bs, offset, bytes);
1385     if (ret < 0) {
1386         return ret;
1387     }
1388
1389     /*
1390      * Align write if necessary by performing a read-modify-write cycle.
1391      * Pad qiov with the read parts and be sure to have a tracked request not
1392      * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
1393      */
1394     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
1395
1396     if (!qiov) {
1397         ret = bdrv_co_do_zero_pwritev(bs, offset, bytes, flags, &req);
1398         goto out;
1399     }
1400
1401     if (offset & (align - 1)) {
1402         QEMUIOVector head_qiov;
1403         struct iovec head_iov;
1404
1405         mark_request_serialising(&req, align);
1406         wait_serialising_requests(&req);
1407
1408         head_buf = qemu_blockalign(bs, align);
1409         head_iov = (struct iovec) {
1410             .iov_base   = head_buf,
1411             .iov_len    = align,
1412         };
1413         qemu_iovec_init_external(&head_qiov, &head_iov, 1);
1414
1415         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1416         ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align,
1417                                   align, &head_qiov, 0);
1418         if (ret < 0) {
1419             goto fail;
1420         }
1421         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1422
1423         qemu_iovec_init(&local_qiov, qiov->niov + 2);
1424         qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
1425         qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1426         use_local_qiov = true;
1427
1428         bytes += offset & (align - 1);
1429         offset = offset & ~(align - 1);
1430     }
1431
1432     if ((offset + bytes) & (align - 1)) {
1433         QEMUIOVector tail_qiov;
1434         struct iovec tail_iov;
1435         size_t tail_bytes;
1436         bool waited;
1437
1438         mark_request_serialising(&req, align);
1439         waited = wait_serialising_requests(&req);
1440         assert(!waited || !use_local_qiov);
1441
1442         tail_buf = qemu_blockalign(bs, align);
1443         tail_iov = (struct iovec) {
1444             .iov_base   = tail_buf,
1445             .iov_len    = align,
1446         };
1447         qemu_iovec_init_external(&tail_qiov, &tail_iov, 1);
1448
1449         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1450         ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align,
1451                                   align, &tail_qiov, 0);
1452         if (ret < 0) {
1453             goto fail;
1454         }
1455         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1456
1457         if (!use_local_qiov) {
1458             qemu_iovec_init(&local_qiov, qiov->niov + 1);
1459             qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1460             use_local_qiov = true;
1461         }
1462
1463         tail_bytes = (offset + bytes) & (align - 1);
1464         qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes);
1465
1466         bytes = ROUND_UP(bytes, align);
1467     }
1468
1469     ret = bdrv_aligned_pwritev(bs, &req, offset, bytes,
1470                                use_local_qiov ? &local_qiov : qiov,
1471                                flags);
1472
1473 fail:
1474
1475     if (use_local_qiov) {
1476         qemu_iovec_destroy(&local_qiov);
1477     }
1478     qemu_vfree(head_buf);
1479     qemu_vfree(tail_buf);
1480 out:
1481     tracked_request_end(&req);
1482     return ret;
1483 }
1484
1485 static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs,
1486     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
1487     BdrvRequestFlags flags)
1488 {
1489     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
1490         return -EINVAL;
1491     }
1492
1493     return bdrv_co_pwritev(bs, sector_num << BDRV_SECTOR_BITS,
1494                            nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
1495 }
1496
1497 int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num,
1498     int nb_sectors, QEMUIOVector *qiov)
1499 {
1500     trace_bdrv_co_writev(bs, sector_num, nb_sectors);
1501
1502     return bdrv_co_do_writev(bs, sector_num, nb_sectors, qiov, 0);
1503 }
1504
1505 int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs,
1506                                       int64_t sector_num, int nb_sectors,
1507                                       BdrvRequestFlags flags)
1508 {
1509     trace_bdrv_co_write_zeroes(bs, sector_num, nb_sectors, flags);
1510
1511     if (!(bs->open_flags & BDRV_O_UNMAP)) {
1512         flags &= ~BDRV_REQ_MAY_UNMAP;
1513     }
1514
1515     return bdrv_co_do_writev(bs, sector_num, nb_sectors, NULL,
1516                              BDRV_REQ_ZERO_WRITE | flags);
1517 }
1518
1519 typedef struct BdrvCoGetBlockStatusData {
1520     BlockDriverState *bs;
1521     BlockDriverState *base;
1522     BlockDriverState **file;
1523     int64_t sector_num;
1524     int nb_sectors;
1525     int *pnum;
1526     int64_t ret;
1527     bool done;
1528 } BdrvCoGetBlockStatusData;
1529
1530 /*
1531  * Returns the allocation status of the specified sectors.
1532  * Drivers not implementing the functionality are assumed to not support
1533  * backing files, hence all their sectors are reported as allocated.
1534  *
1535  * If 'sector_num' is beyond the end of the disk image the return value is 0
1536  * and 'pnum' is set to 0.
1537  *
1538  * 'pnum' is set to the number of sectors (including and immediately following
1539  * the specified sector) that are known to be in the same
1540  * allocated/unallocated state.
1541  *
1542  * 'nb_sectors' is the max value 'pnum' should be set to.  If nb_sectors goes
1543  * beyond the end of the disk image it will be clamped.
1544  *
1545  * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file'
1546  * points to the BDS which the sector range is allocated in.
1547  */
1548 static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
1549                                                      int64_t sector_num,
1550                                                      int nb_sectors, int *pnum,
1551                                                      BlockDriverState **file)
1552 {
1553     int64_t total_sectors;
1554     int64_t n;
1555     int64_t ret, ret2;
1556
1557     total_sectors = bdrv_nb_sectors(bs);
1558     if (total_sectors < 0) {
1559         return total_sectors;
1560     }
1561
1562     if (sector_num >= total_sectors) {
1563         *pnum = 0;
1564         return 0;
1565     }
1566
1567     n = total_sectors - sector_num;
1568     if (n < nb_sectors) {
1569         nb_sectors = n;
1570     }
1571
1572     if (!bs->drv->bdrv_co_get_block_status) {
1573         *pnum = nb_sectors;
1574         ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
1575         if (bs->drv->protocol_name) {
1576             ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE);
1577         }
1578         return ret;
1579     }
1580
1581     *file = NULL;
1582     ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum,
1583                                             file);
1584     if (ret < 0) {
1585         *pnum = 0;
1586         return ret;
1587     }
1588
1589     if (ret & BDRV_BLOCK_RAW) {
1590         assert(ret & BDRV_BLOCK_OFFSET_VALID);
1591         return bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS,
1592                                      *pnum, pnum, file);
1593     }
1594
1595     if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
1596         ret |= BDRV_BLOCK_ALLOCATED;
1597     } else {
1598         if (bdrv_unallocated_blocks_are_zero(bs)) {
1599             ret |= BDRV_BLOCK_ZERO;
1600         } else if (bs->backing) {
1601             BlockDriverState *bs2 = bs->backing->bs;
1602             int64_t nb_sectors2 = bdrv_nb_sectors(bs2);
1603             if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) {
1604                 ret |= BDRV_BLOCK_ZERO;
1605             }
1606         }
1607     }
1608
1609     if (*file && *file != bs &&
1610         (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
1611         (ret & BDRV_BLOCK_OFFSET_VALID)) {
1612         BlockDriverState *file2;
1613         int file_pnum;
1614
1615         ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS,
1616                                         *pnum, &file_pnum, &file2);
1617         if (ret2 >= 0) {
1618             /* Ignore errors.  This is just providing extra information, it
1619              * is useful but not necessary.
1620              */
1621             if (!file_pnum) {
1622                 /* !file_pnum indicates an offset at or beyond the EOF; it is
1623                  * perfectly valid for the format block driver to point to such
1624                  * offsets, so catch it and mark everything as zero */
1625                 ret |= BDRV_BLOCK_ZERO;
1626             } else {
1627                 /* Limit request to the range reported by the protocol driver */
1628                 *pnum = file_pnum;
1629                 ret |= (ret2 & BDRV_BLOCK_ZERO);
1630             }
1631         }
1632     }
1633
1634     return ret;
1635 }
1636
1637 static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs,
1638         BlockDriverState *base,
1639         int64_t sector_num,
1640         int nb_sectors,
1641         int *pnum,
1642         BlockDriverState **file)
1643 {
1644     BlockDriverState *p;
1645     int64_t ret = 0;
1646
1647     assert(bs != base);
1648     for (p = bs; p != base; p = backing_bs(p)) {
1649         ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file);
1650         if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) {
1651             break;
1652         }
1653         /* [sector_num, pnum] unallocated on this layer, which could be only
1654          * the first part of [sector_num, nb_sectors].  */
1655         nb_sectors = MIN(nb_sectors, *pnum);
1656     }
1657     return ret;
1658 }
1659
1660 /* Coroutine wrapper for bdrv_get_block_status_above() */
1661 static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque)
1662 {
1663     BdrvCoGetBlockStatusData *data = opaque;
1664
1665     data->ret = bdrv_co_get_block_status_above(data->bs, data->base,
1666                                                data->sector_num,
1667                                                data->nb_sectors,
1668                                                data->pnum,
1669                                                data->file);
1670     data->done = true;
1671 }
1672
1673 /*
1674  * Synchronous wrapper around bdrv_co_get_block_status_above().
1675  *
1676  * See bdrv_co_get_block_status_above() for details.
1677  */
1678 int64_t bdrv_get_block_status_above(BlockDriverState *bs,
1679                                     BlockDriverState *base,
1680                                     int64_t sector_num,
1681                                     int nb_sectors, int *pnum,
1682                                     BlockDriverState **file)
1683 {
1684     Coroutine *co;
1685     BdrvCoGetBlockStatusData data = {
1686         .bs = bs,
1687         .base = base,
1688         .file = file,
1689         .sector_num = sector_num,
1690         .nb_sectors = nb_sectors,
1691         .pnum = pnum,
1692         .done = false,
1693     };
1694
1695     if (qemu_in_coroutine()) {
1696         /* Fast-path if already in coroutine context */
1697         bdrv_get_block_status_above_co_entry(&data);
1698     } else {
1699         AioContext *aio_context = bdrv_get_aio_context(bs);
1700
1701         co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry);
1702         qemu_coroutine_enter(co, &data);
1703         while (!data.done) {
1704             aio_poll(aio_context, true);
1705         }
1706     }
1707     return data.ret;
1708 }
1709
1710 int64_t bdrv_get_block_status(BlockDriverState *bs,
1711                               int64_t sector_num,
1712                               int nb_sectors, int *pnum,
1713                               BlockDriverState **file)
1714 {
1715     return bdrv_get_block_status_above(bs, backing_bs(bs),
1716                                        sector_num, nb_sectors, pnum, file);
1717 }
1718
1719 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num,
1720                                    int nb_sectors, int *pnum)
1721 {
1722     BlockDriverState *file;
1723     int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum,
1724                                         &file);
1725     if (ret < 0) {
1726         return ret;
1727     }
1728     return !!(ret & BDRV_BLOCK_ALLOCATED);
1729 }
1730
1731 /*
1732  * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
1733  *
1734  * Return true if the given sector is allocated in any image between
1735  * BASE and TOP (inclusive).  BASE can be NULL to check if the given
1736  * sector is allocated in any image of the chain.  Return false otherwise.
1737  *
1738  * 'pnum' is set to the number of sectors (including and immediately following
1739  *  the specified sector) that are known to be in the same
1740  *  allocated/unallocated state.
1741  *
1742  */
1743 int bdrv_is_allocated_above(BlockDriverState *top,
1744                             BlockDriverState *base,
1745                             int64_t sector_num,
1746                             int nb_sectors, int *pnum)
1747 {
1748     BlockDriverState *intermediate;
1749     int ret, n = nb_sectors;
1750
1751     intermediate = top;
1752     while (intermediate && intermediate != base) {
1753         int pnum_inter;
1754         ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors,
1755                                 &pnum_inter);
1756         if (ret < 0) {
1757             return ret;
1758         } else if (ret) {
1759             *pnum = pnum_inter;
1760             return 1;
1761         }
1762
1763         /*
1764          * [sector_num, nb_sectors] is unallocated on top but intermediate
1765          * might have
1766          *
1767          * [sector_num+x, nr_sectors] allocated.
1768          */
1769         if (n > pnum_inter &&
1770             (intermediate == top ||
1771              sector_num + pnum_inter < intermediate->total_sectors)) {
1772             n = pnum_inter;
1773         }
1774
1775         intermediate = backing_bs(intermediate);
1776     }
1777
1778     *pnum = n;
1779     return 0;
1780 }
1781
1782 int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
1783                           const uint8_t *buf, int nb_sectors)
1784 {
1785     BlockDriver *drv = bs->drv;
1786     int ret;
1787
1788     if (!drv) {
1789         return -ENOMEDIUM;
1790     }
1791     if (!drv->bdrv_write_compressed) {
1792         return -ENOTSUP;
1793     }
1794     ret = bdrv_check_request(bs, sector_num, nb_sectors);
1795     if (ret < 0) {
1796         return ret;
1797     }
1798
1799     assert(QLIST_EMPTY(&bs->dirty_bitmaps));
1800
1801     return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors);
1802 }
1803
1804 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
1805                       int64_t pos, int size)
1806 {
1807     QEMUIOVector qiov;
1808     struct iovec iov = {
1809         .iov_base   = (void *) buf,
1810         .iov_len    = size,
1811     };
1812
1813     qemu_iovec_init_external(&qiov, &iov, 1);
1814     return bdrv_writev_vmstate(bs, &qiov, pos);
1815 }
1816
1817 int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
1818 {
1819     BlockDriver *drv = bs->drv;
1820
1821     if (!drv) {
1822         return -ENOMEDIUM;
1823     } else if (drv->bdrv_save_vmstate) {
1824         return drv->bdrv_save_vmstate(bs, qiov, pos);
1825     } else if (bs->file) {
1826         return bdrv_writev_vmstate(bs->file->bs, qiov, pos);
1827     }
1828
1829     return -ENOTSUP;
1830 }
1831
1832 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
1833                       int64_t pos, int size)
1834 {
1835     BlockDriver *drv = bs->drv;
1836     if (!drv)
1837         return -ENOMEDIUM;
1838     if (drv->bdrv_load_vmstate)
1839         return drv->bdrv_load_vmstate(bs, buf, pos, size);
1840     if (bs->file)
1841         return bdrv_load_vmstate(bs->file->bs, buf, pos, size);
1842     return -ENOTSUP;
1843 }
1844
1845 /**************************************************************/
1846 /* async I/Os */
1847
1848 BlockAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
1849                            QEMUIOVector *qiov, int nb_sectors,
1850                            BlockCompletionFunc *cb, void *opaque)
1851 {
1852     trace_bdrv_aio_readv(bs, sector_num, nb_sectors, opaque);
1853
1854     return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0,
1855                                  cb, opaque, false);
1856 }
1857
1858 BlockAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
1859                             QEMUIOVector *qiov, int nb_sectors,
1860                             BlockCompletionFunc *cb, void *opaque)
1861 {
1862     trace_bdrv_aio_writev(bs, sector_num, nb_sectors, opaque);
1863
1864     return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0,
1865                                  cb, opaque, true);
1866 }
1867
1868 BlockAIOCB *bdrv_aio_write_zeroes(BlockDriverState *bs,
1869         int64_t sector_num, int nb_sectors, BdrvRequestFlags flags,
1870         BlockCompletionFunc *cb, void *opaque)
1871 {
1872     trace_bdrv_aio_write_zeroes(bs, sector_num, nb_sectors, flags, opaque);
1873
1874     return bdrv_co_aio_rw_vector(bs, sector_num, NULL, nb_sectors,
1875                                  BDRV_REQ_ZERO_WRITE | flags,
1876                                  cb, opaque, true);
1877 }
1878
1879 void bdrv_aio_cancel(BlockAIOCB *acb)
1880 {
1881     qemu_aio_ref(acb);
1882     bdrv_aio_cancel_async(acb);
1883     while (acb->refcnt > 1) {
1884         if (acb->aiocb_info->get_aio_context) {
1885             aio_poll(acb->aiocb_info->get_aio_context(acb), true);
1886         } else if (acb->bs) {
1887             aio_poll(bdrv_get_aio_context(acb->bs), true);
1888         } else {
1889             abort();
1890         }
1891     }
1892     qemu_aio_unref(acb);
1893 }
1894
1895 /* Async version of aio cancel. The caller is not blocked if the acb implements
1896  * cancel_async, otherwise we do nothing and let the request normally complete.
1897  * In either case the completion callback must be called. */
1898 void bdrv_aio_cancel_async(BlockAIOCB *acb)
1899 {
1900     if (acb->aiocb_info->cancel_async) {
1901         acb->aiocb_info->cancel_async(acb);
1902     }
1903 }
1904
1905 /**************************************************************/
1906 /* async block device emulation */
1907
1908 typedef struct BlockAIOCBCoroutine {
1909     BlockAIOCB common;
1910     BlockRequest req;
1911     bool is_write;
1912     bool need_bh;
1913     bool *done;
1914     QEMUBH* bh;
1915 } BlockAIOCBCoroutine;
1916
1917 static const AIOCBInfo bdrv_em_co_aiocb_info = {
1918     .aiocb_size         = sizeof(BlockAIOCBCoroutine),
1919 };
1920
1921 static void bdrv_co_complete(BlockAIOCBCoroutine *acb)
1922 {
1923     if (!acb->need_bh) {
1924         acb->common.cb(acb->common.opaque, acb->req.error);
1925         qemu_aio_unref(acb);
1926     }
1927 }
1928
1929 static void bdrv_co_em_bh(void *opaque)
1930 {
1931     BlockAIOCBCoroutine *acb = opaque;
1932
1933     assert(!acb->need_bh);
1934     qemu_bh_delete(acb->bh);
1935     bdrv_co_complete(acb);
1936 }
1937
1938 static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb)
1939 {
1940     acb->need_bh = false;
1941     if (acb->req.error != -EINPROGRESS) {
1942         BlockDriverState *bs = acb->common.bs;
1943
1944         acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb);
1945         qemu_bh_schedule(acb->bh);
1946     }
1947 }
1948
1949 /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */
1950 static void coroutine_fn bdrv_co_do_rw(void *opaque)
1951 {
1952     BlockAIOCBCoroutine *acb = opaque;
1953     BlockDriverState *bs = acb->common.bs;
1954
1955     if (!acb->is_write) {
1956         acb->req.error = bdrv_co_do_readv(bs, acb->req.sector,
1957             acb->req.nb_sectors, acb->req.qiov, acb->req.flags);
1958     } else {
1959         acb->req.error = bdrv_co_do_writev(bs, acb->req.sector,
1960             acb->req.nb_sectors, acb->req.qiov, acb->req.flags);
1961     }
1962
1963     bdrv_co_complete(acb);
1964 }
1965
1966 static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
1967                                          int64_t sector_num,
1968                                          QEMUIOVector *qiov,
1969                                          int nb_sectors,
1970                                          BdrvRequestFlags flags,
1971                                          BlockCompletionFunc *cb,
1972                                          void *opaque,
1973                                          bool is_write)
1974 {
1975     Coroutine *co;
1976     BlockAIOCBCoroutine *acb;
1977
1978     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
1979     acb->need_bh = true;
1980     acb->req.error = -EINPROGRESS;
1981     acb->req.sector = sector_num;
1982     acb->req.nb_sectors = nb_sectors;
1983     acb->req.qiov = qiov;
1984     acb->req.flags = flags;
1985     acb->is_write = is_write;
1986
1987     co = qemu_coroutine_create(bdrv_co_do_rw);
1988     qemu_coroutine_enter(co, acb);
1989
1990     bdrv_co_maybe_schedule_bh(acb);
1991     return &acb->common;
1992 }
1993
1994 static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque)
1995 {
1996     BlockAIOCBCoroutine *acb = opaque;
1997     BlockDriverState *bs = acb->common.bs;
1998
1999     acb->req.error = bdrv_co_flush(bs);
2000     bdrv_co_complete(acb);
2001 }
2002
2003 BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs,
2004         BlockCompletionFunc *cb, void *opaque)
2005 {
2006     trace_bdrv_aio_flush(bs, opaque);
2007
2008     Coroutine *co;
2009     BlockAIOCBCoroutine *acb;
2010
2011     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2012     acb->need_bh = true;
2013     acb->req.error = -EINPROGRESS;
2014
2015     co = qemu_coroutine_create(bdrv_aio_flush_co_entry);
2016     qemu_coroutine_enter(co, acb);
2017
2018     bdrv_co_maybe_schedule_bh(acb);
2019     return &acb->common;
2020 }
2021
2022 static void coroutine_fn bdrv_aio_discard_co_entry(void *opaque)
2023 {
2024     BlockAIOCBCoroutine *acb = opaque;
2025     BlockDriverState *bs = acb->common.bs;
2026
2027     acb->req.error = bdrv_co_discard(bs, acb->req.sector, acb->req.nb_sectors);
2028     bdrv_co_complete(acb);
2029 }
2030
2031 BlockAIOCB *bdrv_aio_discard(BlockDriverState *bs,
2032         int64_t sector_num, int nb_sectors,
2033         BlockCompletionFunc *cb, void *opaque)
2034 {
2035     Coroutine *co;
2036     BlockAIOCBCoroutine *acb;
2037
2038     trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque);
2039
2040     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2041     acb->need_bh = true;
2042     acb->req.error = -EINPROGRESS;
2043     acb->req.sector = sector_num;
2044     acb->req.nb_sectors = nb_sectors;
2045     co = qemu_coroutine_create(bdrv_aio_discard_co_entry);
2046     qemu_coroutine_enter(co, acb);
2047
2048     bdrv_co_maybe_schedule_bh(acb);
2049     return &acb->common;
2050 }
2051
2052 void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
2053                    BlockCompletionFunc *cb, void *opaque)
2054 {
2055     BlockAIOCB *acb;
2056
2057     acb = g_malloc(aiocb_info->aiocb_size);
2058     acb->aiocb_info = aiocb_info;
2059     acb->bs = bs;
2060     acb->cb = cb;
2061     acb->opaque = opaque;
2062     acb->refcnt = 1;
2063     return acb;
2064 }
2065
2066 void qemu_aio_ref(void *p)
2067 {
2068     BlockAIOCB *acb = p;
2069     acb->refcnt++;
2070 }
2071
2072 void qemu_aio_unref(void *p)
2073 {
2074     BlockAIOCB *acb = p;
2075     assert(acb->refcnt > 0);
2076     if (--acb->refcnt == 0) {
2077         g_free(acb);
2078     }
2079 }
2080
2081 /**************************************************************/
2082 /* Coroutine block device emulation */
2083
2084 static void coroutine_fn bdrv_flush_co_entry(void *opaque)
2085 {
2086     RwCo *rwco = opaque;
2087
2088     rwco->ret = bdrv_co_flush(rwco->bs);
2089 }
2090
2091 int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
2092 {
2093     int ret;
2094     BdrvTrackedRequest req;
2095
2096     if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
2097         bdrv_is_sg(bs)) {
2098         return 0;
2099     }
2100
2101     tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH);
2102
2103     /* Write back all layers by calling one driver function */
2104     if (bs->drv->bdrv_co_flush) {
2105         ret = bs->drv->bdrv_co_flush(bs);
2106         goto out;
2107     }
2108
2109     /* Write back cached data to the OS even with cache=unsafe */
2110     BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
2111     if (bs->drv->bdrv_co_flush_to_os) {
2112         ret = bs->drv->bdrv_co_flush_to_os(bs);
2113         if (ret < 0) {
2114             goto out;
2115         }
2116     }
2117
2118     /* But don't actually force it to the disk with cache=unsafe */
2119     if (bs->open_flags & BDRV_O_NO_FLUSH) {
2120         goto flush_parent;
2121     }
2122
2123     BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
2124     if (bs->drv->bdrv_co_flush_to_disk) {
2125         ret = bs->drv->bdrv_co_flush_to_disk(bs);
2126     } else if (bs->drv->bdrv_aio_flush) {
2127         BlockAIOCB *acb;
2128         CoroutineIOCompletion co = {
2129             .coroutine = qemu_coroutine_self(),
2130         };
2131
2132         acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
2133         if (acb == NULL) {
2134             ret = -EIO;
2135         } else {
2136             qemu_coroutine_yield();
2137             ret = co.ret;
2138         }
2139     } else {
2140         /*
2141          * Some block drivers always operate in either writethrough or unsafe
2142          * mode and don't support bdrv_flush therefore. Usually qemu doesn't
2143          * know how the server works (because the behaviour is hardcoded or
2144          * depends on server-side configuration), so we can't ensure that
2145          * everything is safe on disk. Returning an error doesn't work because
2146          * that would break guests even if the server operates in writethrough
2147          * mode.
2148          *
2149          * Let's hope the user knows what he's doing.
2150          */
2151         ret = 0;
2152     }
2153     if (ret < 0) {
2154         goto out;
2155     }
2156
2157     /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
2158      * in the case of cache=unsafe, so there are no useless flushes.
2159      */
2160 flush_parent:
2161     ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
2162 out:
2163     tracked_request_end(&req);
2164     return ret;
2165 }
2166
2167 int bdrv_flush(BlockDriverState *bs)
2168 {
2169     Coroutine *co;
2170     RwCo rwco = {
2171         .bs = bs,
2172         .ret = NOT_DONE,
2173     };
2174
2175     if (qemu_in_coroutine()) {
2176         /* Fast-path if already in coroutine context */
2177         bdrv_flush_co_entry(&rwco);
2178     } else {
2179         AioContext *aio_context = bdrv_get_aio_context(bs);
2180
2181         co = qemu_coroutine_create(bdrv_flush_co_entry);
2182         qemu_coroutine_enter(co, &rwco);
2183         while (rwco.ret == NOT_DONE) {
2184             aio_poll(aio_context, true);
2185         }
2186     }
2187
2188     return rwco.ret;
2189 }
2190
2191 typedef struct DiscardCo {
2192     BlockDriverState *bs;
2193     int64_t sector_num;
2194     int nb_sectors;
2195     int ret;
2196 } DiscardCo;
2197 static void coroutine_fn bdrv_discard_co_entry(void *opaque)
2198 {
2199     DiscardCo *rwco = opaque;
2200
2201     rwco->ret = bdrv_co_discard(rwco->bs, rwco->sector_num, rwco->nb_sectors);
2202 }
2203
2204 int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
2205                                  int nb_sectors)
2206 {
2207     BdrvTrackedRequest req;
2208     int max_discard, ret;
2209
2210     if (!bs->drv) {
2211         return -ENOMEDIUM;
2212     }
2213
2214     ret = bdrv_check_request(bs, sector_num, nb_sectors);
2215     if (ret < 0) {
2216         return ret;
2217     } else if (bs->read_only) {
2218         return -EPERM;
2219     }
2220     assert(!(bs->open_flags & BDRV_O_INACTIVE));
2221
2222     /* Do nothing if disabled.  */
2223     if (!(bs->open_flags & BDRV_O_UNMAP)) {
2224         return 0;
2225     }
2226
2227     if (!bs->drv->bdrv_co_discard && !bs->drv->bdrv_aio_discard) {
2228         return 0;
2229     }
2230
2231     tracked_request_begin(&req, bs, sector_num, nb_sectors,
2232                           BDRV_TRACKED_DISCARD);
2233     bdrv_set_dirty(bs, sector_num, nb_sectors);
2234
2235     max_discard = MIN_NON_ZERO(bs->bl.max_discard, BDRV_REQUEST_MAX_SECTORS);
2236     while (nb_sectors > 0) {
2237         int ret;
2238         int num = nb_sectors;
2239
2240         /* align request */
2241         if (bs->bl.discard_alignment &&
2242             num >= bs->bl.discard_alignment &&
2243             sector_num % bs->bl.discard_alignment) {
2244             if (num > bs->bl.discard_alignment) {
2245                 num = bs->bl.discard_alignment;
2246             }
2247             num -= sector_num % bs->bl.discard_alignment;
2248         }
2249
2250         /* limit request size */
2251         if (num > max_discard) {
2252             num = max_discard;
2253         }
2254
2255         if (bs->drv->bdrv_co_discard) {
2256             ret = bs->drv->bdrv_co_discard(bs, sector_num, num);
2257         } else {
2258             BlockAIOCB *acb;
2259             CoroutineIOCompletion co = {
2260                 .coroutine = qemu_coroutine_self(),
2261             };
2262
2263             acb = bs->drv->bdrv_aio_discard(bs, sector_num, nb_sectors,
2264                                             bdrv_co_io_em_complete, &co);
2265             if (acb == NULL) {
2266                 ret = -EIO;
2267                 goto out;
2268             } else {
2269                 qemu_coroutine_yield();
2270                 ret = co.ret;
2271             }
2272         }
2273         if (ret && ret != -ENOTSUP) {
2274             goto out;
2275         }
2276
2277         sector_num += num;
2278         nb_sectors -= num;
2279     }
2280     ret = 0;
2281 out:
2282     tracked_request_end(&req);
2283     return ret;
2284 }
2285
2286 int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors)
2287 {
2288     Coroutine *co;
2289     DiscardCo rwco = {
2290         .bs = bs,
2291         .sector_num = sector_num,
2292         .nb_sectors = nb_sectors,
2293         .ret = NOT_DONE,
2294     };
2295
2296     if (qemu_in_coroutine()) {
2297         /* Fast-path if already in coroutine context */
2298         bdrv_discard_co_entry(&rwco);
2299     } else {
2300         AioContext *aio_context = bdrv_get_aio_context(bs);
2301
2302         co = qemu_coroutine_create(bdrv_discard_co_entry);
2303         qemu_coroutine_enter(co, &rwco);
2304         while (rwco.ret == NOT_DONE) {
2305             aio_poll(aio_context, true);
2306         }
2307     }
2308
2309     return rwco.ret;
2310 }
2311
2312 typedef struct {
2313     CoroutineIOCompletion *co;
2314     QEMUBH *bh;
2315 } BdrvIoctlCompletionData;
2316
2317 static void bdrv_ioctl_bh_cb(void *opaque)
2318 {
2319     BdrvIoctlCompletionData *data = opaque;
2320
2321     bdrv_co_io_em_complete(data->co, -ENOTSUP);
2322     qemu_bh_delete(data->bh);
2323 }
2324
2325 static int bdrv_co_do_ioctl(BlockDriverState *bs, int req, void *buf)
2326 {
2327     BlockDriver *drv = bs->drv;
2328     BdrvTrackedRequest tracked_req;
2329     CoroutineIOCompletion co = {
2330         .coroutine = qemu_coroutine_self(),
2331     };
2332     BlockAIOCB *acb;
2333
2334     tracked_request_begin(&tracked_req, bs, 0, 0, BDRV_TRACKED_IOCTL);
2335     if (!drv || !drv->bdrv_aio_ioctl) {
2336         co.ret = -ENOTSUP;
2337         goto out;
2338     }
2339
2340     acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
2341     if (!acb) {
2342         BdrvIoctlCompletionData *data = g_new(BdrvIoctlCompletionData, 1);
2343         data->bh = aio_bh_new(bdrv_get_aio_context(bs),
2344                                 bdrv_ioctl_bh_cb, data);
2345         data->co = &co;
2346         qemu_bh_schedule(data->bh);
2347     }
2348     qemu_coroutine_yield();
2349 out:
2350     tracked_request_end(&tracked_req);
2351     return co.ret;
2352 }
2353
2354 typedef struct {
2355     BlockDriverState *bs;
2356     int req;
2357     void *buf;
2358     int ret;
2359 } BdrvIoctlCoData;
2360
2361 static void coroutine_fn bdrv_co_ioctl_entry(void *opaque)
2362 {
2363     BdrvIoctlCoData *data = opaque;
2364     data->ret = bdrv_co_do_ioctl(data->bs, data->req, data->buf);
2365 }
2366
2367 /* needed for generic scsi interface */
2368 int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
2369 {
2370     BdrvIoctlCoData data = {
2371         .bs = bs,
2372         .req = req,
2373         .buf = buf,
2374         .ret = -EINPROGRESS,
2375     };
2376
2377     if (qemu_in_coroutine()) {
2378         /* Fast-path if already in coroutine context */
2379         bdrv_co_ioctl_entry(&data);
2380     } else {
2381         Coroutine *co = qemu_coroutine_create(bdrv_co_ioctl_entry);
2382
2383         qemu_coroutine_enter(co, &data);
2384         while (data.ret == -EINPROGRESS) {
2385             aio_poll(bdrv_get_aio_context(bs), true);
2386         }
2387     }
2388     return data.ret;
2389 }
2390
2391 static void coroutine_fn bdrv_co_aio_ioctl_entry(void *opaque)
2392 {
2393     BlockAIOCBCoroutine *acb = opaque;
2394     acb->req.error = bdrv_co_do_ioctl(acb->common.bs,
2395                                       acb->req.req, acb->req.buf);
2396     bdrv_co_complete(acb);
2397 }
2398
2399 BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
2400         unsigned long int req, void *buf,
2401         BlockCompletionFunc *cb, void *opaque)
2402 {
2403     BlockAIOCBCoroutine *acb = qemu_aio_get(&bdrv_em_co_aiocb_info,
2404                                             bs, cb, opaque);
2405     Coroutine *co;
2406
2407     acb->need_bh = true;
2408     acb->req.error = -EINPROGRESS;
2409     acb->req.req = req;
2410     acb->req.buf = buf;
2411     co = qemu_coroutine_create(bdrv_co_aio_ioctl_entry);
2412     qemu_coroutine_enter(co, acb);
2413
2414     bdrv_co_maybe_schedule_bh(acb);
2415     return &acb->common;
2416 }
2417
2418 void *qemu_blockalign(BlockDriverState *bs, size_t size)
2419 {
2420     return qemu_memalign(bdrv_opt_mem_align(bs), size);
2421 }
2422
2423 void *qemu_blockalign0(BlockDriverState *bs, size_t size)
2424 {
2425     return memset(qemu_blockalign(bs, size), 0, size);
2426 }
2427
2428 void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
2429 {
2430     size_t align = bdrv_opt_mem_align(bs);
2431
2432     /* Ensure that NULL is never returned on success */
2433     assert(align > 0);
2434     if (size == 0) {
2435         size = align;
2436     }
2437
2438     return qemu_try_memalign(align, size);
2439 }
2440
2441 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
2442 {
2443     void *mem = qemu_try_blockalign(bs, size);
2444
2445     if (mem) {
2446         memset(mem, 0, size);
2447     }
2448
2449     return mem;
2450 }
2451
2452 /*
2453  * Check if all memory in this vector is sector aligned.
2454  */
2455 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
2456 {
2457     int i;
2458     size_t alignment = bdrv_min_mem_align(bs);
2459
2460     for (i = 0; i < qiov->niov; i++) {
2461         if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
2462             return false;
2463         }
2464         if (qiov->iov[i].iov_len % alignment) {
2465             return false;
2466         }
2467     }
2468
2469     return true;
2470 }
2471
2472 void bdrv_add_before_write_notifier(BlockDriverState *bs,
2473                                     NotifierWithReturn *notifier)
2474 {
2475     notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
2476 }
2477
2478 void bdrv_io_plug(BlockDriverState *bs)
2479 {
2480     BdrvChild *child;
2481
2482     QLIST_FOREACH(child, &bs->children, next) {
2483         bdrv_io_plug(child->bs);
2484     }
2485
2486     if (bs->io_plugged++ == 0 && bs->io_plug_disabled == 0) {
2487         BlockDriver *drv = bs->drv;
2488         if (drv && drv->bdrv_io_plug) {
2489             drv->bdrv_io_plug(bs);
2490         }
2491     }
2492 }
2493
2494 void bdrv_io_unplug(BlockDriverState *bs)
2495 {
2496     BdrvChild *child;
2497
2498     assert(bs->io_plugged);
2499     if (--bs->io_plugged == 0 && bs->io_plug_disabled == 0) {
2500         BlockDriver *drv = bs->drv;
2501         if (drv && drv->bdrv_io_unplug) {
2502             drv->bdrv_io_unplug(bs);
2503         }
2504     }
2505
2506     QLIST_FOREACH(child, &bs->children, next) {
2507         bdrv_io_unplug(child->bs);
2508     }
2509 }
2510
2511 void bdrv_io_unplugged_begin(BlockDriverState *bs)
2512 {
2513     BdrvChild *child;
2514
2515     if (bs->io_plug_disabled++ == 0 && bs->io_plugged > 0) {
2516         BlockDriver *drv = bs->drv;
2517         if (drv && drv->bdrv_io_unplug) {
2518             drv->bdrv_io_unplug(bs);
2519         }
2520     }
2521
2522     QLIST_FOREACH(child, &bs->children, next) {
2523         bdrv_io_unplugged_begin(child->bs);
2524     }
2525 }
2526
2527 void bdrv_io_unplugged_end(BlockDriverState *bs)
2528 {
2529     BdrvChild *child;
2530
2531     assert(bs->io_plug_disabled);
2532     QLIST_FOREACH(child, &bs->children, next) {
2533         bdrv_io_unplugged_end(child->bs);
2534     }
2535
2536     if (--bs->io_plug_disabled == 0 && bs->io_plugged > 0) {
2537         BlockDriver *drv = bs->drv;
2538         if (drv && drv->bdrv_io_plug) {
2539             drv->bdrv_io_plug(bs);
2540         }
2541     }
2542 }