block/io.c

   1 /*
   2  * Block layer I/O functions
   3  *
   4  * Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24
  25 #include "qemu/osdep.h"
  26 #include "trace.h"
  27 #include "sysemu/block-backend.h"
  28 #include "block/blockjob.h"
  29 #include "block/block_int.h"
  30 #include "qemu/cutils.h"
  31 #include "qapi/error.h"
  32 #include "qemu/error-report.h"
  33
  34 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
  35
  36 static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
  37                                          int64_t sector_num,
  38                                          QEMUIOVector *qiov,
  39                                          int nb_sectors,
  40                                          BdrvRequestFlags flags,
  41                                          BlockCompletionFunc *cb,
  42                                          void *opaque,
  43                                          bool is_write);
  44 static void coroutine_fn bdrv_co_do_rw(void *opaque);
  45 static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
  46     int64_t sector_num, int nb_sectors, BdrvRequestFlags flags);
  47
  48 static void bdrv_parent_drained_begin(BlockDriverState *bs)
  49 {
  50     BdrvChild *c;
  51
  52     QLIST_FOREACH(c, &bs->parents, next_parent) {
  53         if (c->role->drained_begin) {
  54             c->role->drained_begin(c);
  55         }
  56     }
  57 }
  58
  59 static void bdrv_parent_drained_end(BlockDriverState *bs)
  60 {
  61     BdrvChild *c;
  62
  63     QLIST_FOREACH(c, &bs->parents, next_parent) {
  64         if (c->role->drained_end) {
  65             c->role->drained_end(c);
  66         }
  67     }
  68 }
  69
  70 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
  71 {
  72     BlockDriver *drv = bs->drv;
  73     Error *local_err = NULL;
  74
  75     memset(&bs->bl, 0, sizeof(bs->bl));
  76
  77     if (!drv) {
  78         return;
  79     }
  80
  81     /* Take some limits from the children as a default */
  82     if (bs->file) {
  83         bdrv_refresh_limits(bs->file->bs, &local_err);
  84         if (local_err) {
  85             error_propagate(errp, local_err);
  86             return;
  87         }
  88         bs->bl.opt_transfer_length = bs->file->bs->bl.opt_transfer_length;
  89         bs->bl.max_transfer_length = bs->file->bs->bl.max_transfer_length;
  90         bs->bl.min_mem_alignment = bs->file->bs->bl.min_mem_alignment;
  91         bs->bl.opt_mem_alignment = bs->file->bs->bl.opt_mem_alignment;
  92         bs->bl.max_iov = bs->file->bs->bl.max_iov;
  93     } else {
  94         bs->bl.min_mem_alignment = 512;
  95         bs->bl.opt_mem_alignment = getpagesize();
  96
  97         /* Safe default since most protocols use readv()/writev()/etc */
  98         bs->bl.max_iov = IOV_MAX;
  99     }
 100
 101     if (bs->backing) {
 102         bdrv_refresh_limits(bs->backing->bs, &local_err);
 103         if (local_err) {
 104             error_propagate(errp, local_err);
 105             return;
 106         }
 107         bs->bl.opt_transfer_length =
 108             MAX(bs->bl.opt_transfer_length,
 109                 bs->backing->bs->bl.opt_transfer_length);
 110         bs->bl.max_transfer_length =
 111             MIN_NON_ZERO(bs->bl.max_transfer_length,
 112                          bs->backing->bs->bl.max_transfer_length);
 113         bs->bl.opt_mem_alignment =
 114             MAX(bs->bl.opt_mem_alignment,
 115                 bs->backing->bs->bl.opt_mem_alignment);
 116         bs->bl.min_mem_alignment =
 117             MAX(bs->bl.min_mem_alignment,
 118                 bs->backing->bs->bl.min_mem_alignment);
 119         bs->bl.max_iov =
 120             MIN(bs->bl.max_iov,
 121                 bs->backing->bs->bl.max_iov);
 122     }
 123
 124     /* Then let the driver override it */
 125     if (drv->bdrv_refresh_limits) {
 126         drv->bdrv_refresh_limits(bs, errp);
 127     }
 128 }
 129
 130 /**
 131  * The copy-on-read flag is actually a reference count so multiple users may
 132  * use the feature without worrying about clobbering its previous state.
 133  * Copy-on-read stays enabled until all users have called to disable it.
 134  */
 135 void bdrv_enable_copy_on_read(BlockDriverState *bs)
 136 {
 137     bs->copy_on_read++;
 138 }
 139
 140 void bdrv_disable_copy_on_read(BlockDriverState *bs)
 141 {
 142     assert(bs->copy_on_read > 0);
 143     bs->copy_on_read--;
 144 }
 145
 146 /* Check if any requests are in-flight (including throttled requests) */
 147 bool bdrv_requests_pending(BlockDriverState *bs)
 148 {
 149     BdrvChild *child;
 150
 151     if (!QLIST_EMPTY(&bs->tracked_requests)) {
 152         return true;
 153     }
 154
 155     QLIST_FOREACH(child, &bs->children, next) {
 156         if (bdrv_requests_pending(child->bs)) {
 157             return true;
 158         }
 159     }
 160
 161     return false;
 162 }
 163
 164 static void bdrv_drain_recurse(BlockDriverState *bs)
 165 {
 166     BdrvChild *child;
 167
 168     if (bs->drv && bs->drv->bdrv_drain) {
 169         bs->drv->bdrv_drain(bs);
 170     }
 171     QLIST_FOREACH(child, &bs->children, next) {
 172         bdrv_drain_recurse(child->bs);
 173     }
 174 }
 175
 176 typedef struct {
 177     Coroutine *co;
 178     BlockDriverState *bs;
 179     QEMUBH *bh;
 180     bool done;
 181 } BdrvCoDrainData;
 182
 183 static void bdrv_drain_poll(BlockDriverState *bs)
 184 {
 185     bool busy = true;
 186
 187     while (busy) {
 188         /* Keep iterating */
 189         busy = bdrv_requests_pending(bs);
 190         busy |= aio_poll(bdrv_get_aio_context(bs), busy);
 191     }
 192 }
 193
 194 static void bdrv_co_drain_bh_cb(void *opaque)
 195 {
 196     BdrvCoDrainData *data = opaque;
 197     Coroutine *co = data->co;
 198
 199     qemu_bh_delete(data->bh);
 200     bdrv_drain_poll(data->bs);
 201     data->done = true;
 202     qemu_coroutine_enter(co, NULL);
 203 }
 204
 205 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs)
 206 {
 207     BdrvCoDrainData data;
 208
 209     /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
 210      * other coroutines run if they were queued from
 211      * qemu_co_queue_run_restart(). */
 212
 213     assert(qemu_in_coroutine());
 214     data = (BdrvCoDrainData) {
 215         .co = qemu_coroutine_self(),
 216         .bs = bs,
 217         .done = false,
 218         .bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_drain_bh_cb, &data),
 219     };
 220     qemu_bh_schedule(data.bh);
 221
 222     qemu_coroutine_yield();
 223     /* If we are resumed from some other event (such as an aio completion or a
 224      * timer callback), it is a bug in the caller that should be fixed. */
 225     assert(data.done);
 226 }
 227
 228 /*
 229  * Wait for pending requests to complete on a single BlockDriverState subtree,
 230  * and suspend block driver's internal I/O until next request arrives.
 231  *
 232  * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
 233  * AioContext.
 234  *
 235  * Only this BlockDriverState's AioContext is run, so in-flight requests must
 236  * not depend on events in other AioContexts.  In that case, use
 237  * bdrv_drain_all() instead.
 238  */
 239 void coroutine_fn bdrv_co_drain(BlockDriverState *bs)
 240 {
 241     bdrv_parent_drained_begin(bs);
 242     bdrv_io_unplugged_begin(bs);
 243     bdrv_drain_recurse(bs);
 244     bdrv_co_yield_to_drain(bs);
 245     bdrv_io_unplugged_end(bs);
 246     bdrv_parent_drained_end(bs);
 247 }
 248
 249 void bdrv_drain(BlockDriverState *bs)
 250 {
 251     bdrv_parent_drained_begin(bs);
 252     bdrv_io_unplugged_begin(bs);
 253     bdrv_drain_recurse(bs);
 254     if (qemu_in_coroutine()) {
 255         bdrv_co_yield_to_drain(bs);
 256     } else {
 257         bdrv_drain_poll(bs);
 258     }
 259     bdrv_io_unplugged_end(bs);
 260     bdrv_parent_drained_end(bs);
 261 }
 262
 263 /*
 264  * Wait for pending requests to complete across all BlockDriverStates
 265  *
 266  * This function does not flush data to disk, use bdrv_flush_all() for that
 267  * after calling this function.
 268  */
 269 void bdrv_drain_all(void)
 270 {
 271     /* Always run first iteration so any pending completion BHs run */
 272     bool busy = true;
 273     BlockDriverState *bs;
 274     BdrvNextIterator *it = NULL;
 275     GSList *aio_ctxs = NULL, *ctx;
 276
 277     while ((it = bdrv_next(it, &bs))) {
 278         AioContext *aio_context = bdrv_get_aio_context(bs);
 279
 280         aio_context_acquire(aio_context);
 281         if (bs->job) {
 282             block_job_pause(bs->job);
 283         }
 284         bdrv_parent_drained_begin(bs);
 285         bdrv_io_unplugged_begin(bs);
 286         bdrv_drain_recurse(bs);
 287         aio_context_release(aio_context);
 288
 289         if (!g_slist_find(aio_ctxs, aio_context)) {
 290             aio_ctxs = g_slist_prepend(aio_ctxs, aio_context);
 291         }
 292     }
 293
 294     /* Note that completion of an asynchronous I/O operation can trigger any
 295      * number of other I/O operations on other devices---for example a
 296      * coroutine can submit an I/O request to another device in response to
 297      * request completion.  Therefore we must keep looping until there was no
 298      * more activity rather than simply draining each device independently.
 299      */
 300     while (busy) {
 301         busy = false;
 302
 303         for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) {
 304             AioContext *aio_context = ctx->data;
 305             it = NULL;
 306
 307             aio_context_acquire(aio_context);
 308             while ((it = bdrv_next(it, &bs))) {
 309                 if (aio_context == bdrv_get_aio_context(bs)) {
 310                     if (bdrv_requests_pending(bs)) {
 311                         busy = true;
 312                         aio_poll(aio_context, busy);
 313                     }
 314                 }
 315             }
 316             busy |= aio_poll(aio_context, false);
 317             aio_context_release(aio_context);
 318         }
 319     }
 320
 321     it = NULL;
 322     while ((it = bdrv_next(it, &bs))) {
 323         AioContext *aio_context = bdrv_get_aio_context(bs);
 324
 325         aio_context_acquire(aio_context);
 326         bdrv_io_unplugged_end(bs);
 327         bdrv_parent_drained_end(bs);
 328         if (bs->job) {
 329             block_job_resume(bs->job);
 330         }
 331         aio_context_release(aio_context);
 332     }
 333     g_slist_free(aio_ctxs);
 334 }
 335
 336 /**
 337  * Remove an active request from the tracked requests list
 338  *
 339  * This function should be called when a tracked request is completing.
 340  */
 341 static void tracked_request_end(BdrvTrackedRequest *req)
 342 {
 343     if (req->serialising) {
 344         req->bs->serialising_in_flight--;
 345     }
 346
 347     QLIST_REMOVE(req, list);
 348     qemu_co_queue_restart_all(&req->wait_queue);
 349 }
 350
 351 /**
 352  * Add an active request to the tracked requests list
 353  */
 354 static void tracked_request_begin(BdrvTrackedRequest *req,
 355                                   BlockDriverState *bs,
 356                                   int64_t offset,
 357                                   unsigned int bytes,
 358                                   enum BdrvTrackedRequestType type)
 359 {
 360     *req = (BdrvTrackedRequest){
 361         .bs = bs,
 362         .offset         = offset,
 363         .bytes          = bytes,
 364         .type           = type,
 365         .co             = qemu_coroutine_self(),
 366         .serialising    = false,
 367         .overlap_offset = offset,
 368         .overlap_bytes  = bytes,
 369     };
 370
 371     qemu_co_queue_init(&req->wait_queue);
 372
 373     QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
 374 }
 375
 376 static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
 377 {
 378     int64_t overlap_offset = req->offset & ~(align - 1);
 379     unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
 380                                - overlap_offset;
 381
 382     if (!req->serialising) {
 383         req->bs->serialising_in_flight++;
 384         req->serialising = true;
 385     }
 386
 387     req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
 388     req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
 389 }
 390
 391 /**
 392  * Round a region to cluster boundaries
 393  */
 394 void bdrv_round_to_clusters(BlockDriverState *bs,
 395                             int64_t sector_num, int nb_sectors,
 396                             int64_t *cluster_sector_num,
 397                             int *cluster_nb_sectors)
 398 {
 399     BlockDriverInfo bdi;
 400
 401     if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
 402         *cluster_sector_num = sector_num;
 403         *cluster_nb_sectors = nb_sectors;
 404     } else {
 405         int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE;
 406         *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c);
 407         *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num +
 408                                             nb_sectors, c);
 409     }
 410 }
 411
 412 static int bdrv_get_cluster_size(BlockDriverState *bs)
 413 {
 414     BlockDriverInfo bdi;
 415     int ret;
 416
 417     ret = bdrv_get_info(bs, &bdi);
 418     if (ret < 0 || bdi.cluster_size == 0) {
 419         return bs->request_alignment;
 420     } else {
 421         return bdi.cluster_size;
 422     }
 423 }
 424
 425 static bool tracked_request_overlaps(BdrvTrackedRequest *req,
 426                                      int64_t offset, unsigned int bytes)
 427 {
 428     /*        aaaa   bbbb */
 429     if (offset >= req->overlap_offset + req->overlap_bytes) {
 430         return false;
 431     }
 432     /* bbbb   aaaa        */
 433     if (req->overlap_offset >= offset + bytes) {
 434         return false;
 435     }
 436     return true;
 437 }
 438
 439 static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self)
 440 {
 441     BlockDriverState *bs = self->bs;
 442     BdrvTrackedRequest *req;
 443     bool retry;
 444     bool waited = false;
 445
 446     if (!bs->serialising_in_flight) {
 447         return false;
 448     }
 449
 450     do {
 451         retry = false;
 452         QLIST_FOREACH(req, &bs->tracked_requests, list) {
 453             if (req == self || (!req->serialising && !self->serialising)) {
 454                 continue;
 455             }
 456             if (tracked_request_overlaps(req, self->overlap_offset,
 457                                          self->overlap_bytes))
 458             {
 459                 /* Hitting this means there was a reentrant request, for
 460                  * example, a block driver issuing nested requests.  This must
 461                  * never happen since it means deadlock.
 462                  */
 463                 assert(qemu_coroutine_self() != req->co);
 464
 465                 /* If the request is already (indirectly) waiting for us, or
 466                  * will wait for us as soon as it wakes up, then just go on
 467                  * (instead of producing a deadlock in the former case). */
 468                 if (!req->waiting_for) {
 469                     self->waiting_for = req;
 470                     qemu_co_queue_wait(&req->wait_queue);
 471                     self->waiting_for = NULL;
 472                     retry = true;
 473                     waited = true;
 474                     break;
 475                 }
 476             }
 477         }
 478     } while (retry);
 479
 480     return waited;
 481 }
 482
 483 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
 484                                    size_t size)
 485 {
 486     if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) {
 487         return -EIO;
 488     }
 489
 490     if (!bdrv_is_inserted(bs)) {
 491         return -ENOMEDIUM;
 492     }
 493
 494     if (offset < 0) {
 495         return -EIO;
 496     }
 497
 498     return 0;
 499 }
 500
 501 static int bdrv_check_request(BlockDriverState *bs, int64_t sector_num,
 502                               int nb_sectors)
 503 {
 504     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
 505         return -EIO;
 506     }
 507
 508     return bdrv_check_byte_request(bs, sector_num * BDRV_SECTOR_SIZE,
 509                                    nb_sectors * BDRV_SECTOR_SIZE);
 510 }
 511
 512 typedef struct RwCo {
 513     BlockDriverState *bs;
 514     int64_t offset;
 515     QEMUIOVector *qiov;
 516     bool is_write;
 517     int ret;
 518     BdrvRequestFlags flags;
 519 } RwCo;
 520
 521 static void coroutine_fn bdrv_rw_co_entry(void *opaque)
 522 {
 523     RwCo *rwco = opaque;
 524
 525     if (!rwco->is_write) {
 526         rwco->ret = bdrv_co_preadv(rwco->bs, rwco->offset,
 527                                    rwco->qiov->size, rwco->qiov,
 528                                    rwco->flags);
 529     } else {
 530         rwco->ret = bdrv_co_pwritev(rwco->bs, rwco->offset,
 531                                     rwco->qiov->size, rwco->qiov,
 532                                     rwco->flags);
 533     }
 534 }
 535
 536 /*
 537  * Process a vectored synchronous request using coroutines
 538  */
 539 static int bdrv_prwv_co(BlockDriverState *bs, int64_t offset,
 540                         QEMUIOVector *qiov, bool is_write,
 541                         BdrvRequestFlags flags)
 542 {
 543     Coroutine *co;
 544     RwCo rwco = {
 545         .bs = bs,
 546         .offset = offset,
 547         .qiov = qiov,
 548         .is_write = is_write,
 549         .ret = NOT_DONE,
 550         .flags = flags,
 551     };
 552
 553     if (qemu_in_coroutine()) {
 554         /* Fast-path if already in coroutine context */
 555         bdrv_rw_co_entry(&rwco);
 556     } else {
 557         AioContext *aio_context = bdrv_get_aio_context(bs);
 558
 559         co = qemu_coroutine_create(bdrv_rw_co_entry);
 560         qemu_coroutine_enter(co, &rwco);
 561         while (rwco.ret == NOT_DONE) {
 562             aio_poll(aio_context, true);
 563         }
 564     }
 565     return rwco.ret;
 566 }
 567
 568 /*
 569  * Process a synchronous request using coroutines
 570  */
 571 static int bdrv_rw_co(BlockDriverState *bs, int64_t sector_num, uint8_t *buf,
 572                       int nb_sectors, bool is_write, BdrvRequestFlags flags)
 573 {
 574     QEMUIOVector qiov;
 575     struct iovec iov = {
 576         .iov_base = (void *)buf,
 577         .iov_len = nb_sectors * BDRV_SECTOR_SIZE,
 578     };
 579
 580     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
 581         return -EINVAL;
 582     }
 583
 584     qemu_iovec_init_external(&qiov, &iov, 1);
 585     return bdrv_prwv_co(bs, sector_num << BDRV_SECTOR_BITS,
 586                         &qiov, is_write, flags);
 587 }
 588
 589 /* return < 0 if error. See bdrv_write() for the return codes */
 590 int bdrv_read(BlockDriverState *bs, int64_t sector_num,
 591               uint8_t *buf, int nb_sectors)
 592 {
 593     return bdrv_rw_co(bs, sector_num, buf, nb_sectors, false, 0);
 594 }
 595
 596 /* Return < 0 if error. Important errors are:
 597   -EIO         generic I/O error (may happen for all errors)
 598   -ENOMEDIUM   No media inserted.
 599   -EINVAL      Invalid sector number or nb_sectors
 600   -EACCES      Trying to write a read-only device
 601 */
 602 int bdrv_write(BlockDriverState *bs, int64_t sector_num,
 603                const uint8_t *buf, int nb_sectors)
 604 {
 605     return bdrv_rw_co(bs, sector_num, (uint8_t *)buf, nb_sectors, true, 0);
 606 }
 607
 608 int bdrv_write_zeroes(BlockDriverState *bs, int64_t sector_num,
 609                       int nb_sectors, BdrvRequestFlags flags)
 610 {
 611     return bdrv_rw_co(bs, sector_num, NULL, nb_sectors, true,
 612                       BDRV_REQ_ZERO_WRITE | flags);
 613 }
 614
 615 /*
 616  * Completely zero out a block device with the help of bdrv_write_zeroes.
 617  * The operation is sped up by checking the block status and only writing
 618  * zeroes to the device if they currently do not return zeroes. Optional
 619  * flags are passed through to bdrv_write_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
 620  * BDRV_REQ_FUA).
 621  *
 622  * Returns < 0 on error, 0 on success. For error codes see bdrv_write().
 623  */
 624 int bdrv_make_zero(BlockDriverState *bs, BdrvRequestFlags flags)
 625 {
 626     int64_t target_sectors, ret, nb_sectors, sector_num = 0;
 627     BlockDriverState *file;
 628     int n;
 629
 630     target_sectors = bdrv_nb_sectors(bs);
 631     if (target_sectors < 0) {
 632         return target_sectors;
 633     }
 634
 635     for (;;) {
 636         nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS);
 637         if (nb_sectors <= 0) {
 638             return 0;
 639         }
 640         ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file);
 641         if (ret < 0) {
 642             error_report("error getting block status at sector %" PRId64 ": %s",
 643                          sector_num, strerror(-ret));
 644             return ret;
 645         }
 646         if (ret & BDRV_BLOCK_ZERO) {
 647             sector_num += n;
 648             continue;
 649         }
 650         ret = bdrv_write_zeroes(bs, sector_num, n, flags);
 651         if (ret < 0) {
 652             error_report("error writing zeroes at sector %" PRId64 ": %s",
 653                          sector_num, strerror(-ret));
 654             return ret;
 655         }
 656         sector_num += n;
 657     }
 658 }
 659
 660 int bdrv_pread(BlockDriverState *bs, int64_t offset, void *buf, int bytes)
 661 {
 662     QEMUIOVector qiov;
 663     struct iovec iov = {
 664         .iov_base = (void *)buf,
 665         .iov_len = bytes,
 666     };
 667     int ret;
 668
 669     if (bytes < 0) {
 670         return -EINVAL;
 671     }
 672
 673     qemu_iovec_init_external(&qiov, &iov, 1);
 674     ret = bdrv_prwv_co(bs, offset, &qiov, false, 0);
 675     if (ret < 0) {
 676         return ret;
 677     }
 678
 679     return bytes;
 680 }
 681
 682 int bdrv_pwritev(BlockDriverState *bs, int64_t offset, QEMUIOVector *qiov)
 683 {
 684     int ret;
 685
 686     ret = bdrv_prwv_co(bs, offset, qiov, true, 0);
 687     if (ret < 0) {
 688         return ret;
 689     }
 690
 691     return qiov->size;
 692 }
 693
 694 int bdrv_pwrite(BlockDriverState *bs, int64_t offset,
 695                 const void *buf, int bytes)
 696 {
 697     QEMUIOVector qiov;
 698     struct iovec iov = {
 699         .iov_base   = (void *) buf,
 700         .iov_len    = bytes,
 701     };
 702
 703     if (bytes < 0) {
 704         return -EINVAL;
 705     }
 706
 707     qemu_iovec_init_external(&qiov, &iov, 1);
 708     return bdrv_pwritev(bs, offset, &qiov);
 709 }
 710
 711 /*
 712  * Writes to the file and ensures that no writes are reordered across this
 713  * request (acts as a barrier)
 714  *
 715  * Returns 0 on success, -errno in error cases.
 716  */
 717 int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset,
 718     const void *buf, int count)
 719 {
 720     int ret;
 721
 722     ret = bdrv_pwrite(bs, offset, buf, count);
 723     if (ret < 0) {
 724         return ret;
 725     }
 726
 727     ret = bdrv_flush(bs);
 728     if (ret < 0) {
 729         return ret;
 730     }
 731
 732     return 0;
 733 }
 734
 735 typedef struct CoroutineIOCompletion {
 736     Coroutine *coroutine;
 737     int ret;
 738 } CoroutineIOCompletion;
 739
 740 static void bdrv_co_io_em_complete(void *opaque, int ret)
 741 {
 742     CoroutineIOCompletion *co = opaque;
 743
 744     co->ret = ret;
 745     qemu_coroutine_enter(co->coroutine, NULL);
 746 }
 747
 748 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
 749                                            uint64_t offset, uint64_t bytes,
 750                                            QEMUIOVector *qiov, int flags)
 751 {
 752     BlockDriver *drv = bs->drv;
 753     int64_t sector_num;
 754     unsigned int nb_sectors;
 755
 756     if (drv->bdrv_co_preadv) {
 757         return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
 758     }
 759
 760     sector_num = offset >> BDRV_SECTOR_BITS;
 761     nb_sectors = bytes >> BDRV_SECTOR_BITS;
 762
 763     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
 764     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
 765     assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
 766
 767     if (drv->bdrv_co_readv) {
 768         return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
 769     } else {
 770         BlockAIOCB *acb;
 771         CoroutineIOCompletion co = {
 772             .coroutine = qemu_coroutine_self(),
 773         };
 774
 775         acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors,
 776                                       bdrv_co_io_em_complete, &co);
 777         if (acb == NULL) {
 778             return -EIO;
 779         } else {
 780             qemu_coroutine_yield();
 781             return co.ret;
 782         }
 783     }
 784 }
 785
 786 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
 787                                             uint64_t offset, uint64_t bytes,
 788                                             QEMUIOVector *qiov, int flags)
 789 {
 790     BlockDriver *drv = bs->drv;
 791     int64_t sector_num;
 792     unsigned int nb_sectors;
 793     int ret;
 794
 795     if (drv->bdrv_co_pwritev) {
 796         ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, flags);
 797         goto emulate_flags;
 798     }
 799
 800     sector_num = offset >> BDRV_SECTOR_BITS;
 801     nb_sectors = bytes >> BDRV_SECTOR_BITS;
 802
 803     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
 804     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
 805     assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
 806
 807     if (drv->bdrv_co_writev_flags) {
 808         ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov,
 809                                         flags & bs->supported_write_flags);
 810         flags &= ~bs->supported_write_flags;
 811     } else if (drv->bdrv_co_writev) {
 812         assert(!bs->supported_write_flags);
 813         ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov);
 814     } else {
 815         BlockAIOCB *acb;
 816         CoroutineIOCompletion co = {
 817             .coroutine = qemu_coroutine_self(),
 818         };
 819
 820         acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors,
 821                                        bdrv_co_io_em_complete, &co);
 822         if (acb == NULL) {
 823             ret = -EIO;
 824         } else {
 825             qemu_coroutine_yield();
 826             ret = co.ret;
 827         }
 828     }
 829
 830 emulate_flags:
 831     if (ret == 0 && (flags & BDRV_REQ_FUA)) {
 832         ret = bdrv_co_flush(bs);
 833     }
 834
 835     return ret;
 836 }
 837
 838 static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs,
 839         int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
 840 {
 841     /* Perform I/O through a temporary buffer so that users who scribble over
 842      * their read buffer while the operation is in progress do not end up
 843      * modifying the image file.  This is critical for zero-copy guest I/O
 844      * where anything might happen inside guest memory.
 845      */
 846     void *bounce_buffer;
 847
 848     BlockDriver *drv = bs->drv;
 849     struct iovec iov;
 850     QEMUIOVector bounce_qiov;
 851     int64_t cluster_sector_num;
 852     int cluster_nb_sectors;
 853     size_t skip_bytes;
 854     int ret;
 855
 856     /* Cover entire cluster so no additional backing file I/O is required when
 857      * allocating cluster in the image file.
 858      */
 859     bdrv_round_to_clusters(bs, sector_num, nb_sectors,
 860                            &cluster_sector_num, &cluster_nb_sectors);
 861
 862     trace_bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors,
 863                                    cluster_sector_num, cluster_nb_sectors);
 864
 865     iov.iov_len = cluster_nb_sectors * BDRV_SECTOR_SIZE;
 866     iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len);
 867     if (bounce_buffer == NULL) {
 868         ret = -ENOMEM;
 869         goto err;
 870     }
 871
 872     qemu_iovec_init_external(&bounce_qiov, &iov, 1);
 873
 874     ret = bdrv_driver_preadv(bs, cluster_sector_num * BDRV_SECTOR_SIZE,
 875                              cluster_nb_sectors * BDRV_SECTOR_SIZE,
 876                              &bounce_qiov, 0);
 877     if (ret < 0) {
 878         goto err;
 879     }
 880
 881     if (drv->bdrv_co_write_zeroes &&
 882         buffer_is_zero(bounce_buffer, iov.iov_len)) {
 883         ret = bdrv_co_do_write_zeroes(bs, cluster_sector_num,
 884                                       cluster_nb_sectors, 0);
 885     } else {
 886         /* This does not change the data on the disk, it is not necessary
 887          * to flush even in cache=writethrough mode.
 888          */
 889         ret = bdrv_driver_pwritev(bs, cluster_sector_num * BDRV_SECTOR_SIZE,
 890                                   cluster_nb_sectors * BDRV_SECTOR_SIZE,
 891                                   &bounce_qiov, 0);
 892     }
 893
 894     if (ret < 0) {
 895         /* It might be okay to ignore write errors for guest requests.  If this
 896          * is a deliberate copy-on-read then we don't want to ignore the error.
 897          * Simply report it in all cases.
 898          */
 899         goto err;
 900     }
 901
 902     skip_bytes = (sector_num - cluster_sector_num) * BDRV_SECTOR_SIZE;
 903     qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes,
 904                         nb_sectors * BDRV_SECTOR_SIZE);
 905
 906 err:
 907     qemu_vfree(bounce_buffer);
 908     return ret;
 909 }
 910
 911 /*
 912  * Forwards an already correctly aligned request to the BlockDriver. This
 913  * handles copy on read and zeroing after EOF; any other features must be
 914  * implemented by the caller.
 915  */
 916 static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs,
 917     BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
 918     int64_t align, QEMUIOVector *qiov, int flags)
 919 {
 920     int ret;
 921
 922     int64_t sector_num = offset >> BDRV_SECTOR_BITS;
 923     unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS;
 924
 925     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
 926     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
 927     assert(!qiov || bytes == qiov->size);
 928     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
 929
 930     /* Handle Copy on Read and associated serialisation */
 931     if (flags & BDRV_REQ_COPY_ON_READ) {
 932         /* If we touch the same cluster it counts as an overlap.  This
 933          * guarantees that allocating writes will be serialized and not race
 934          * with each other for the same cluster.  For example, in copy-on-read
 935          * it ensures that the CoR read and write operations are atomic and
 936          * guest writes cannot interleave between them. */
 937         mark_request_serialising(req, bdrv_get_cluster_size(bs));
 938     }
 939
 940     if (!(flags & BDRV_REQ_NO_SERIALISING)) {
 941         wait_serialising_requests(req);
 942     }
 943
 944     if (flags & BDRV_REQ_COPY_ON_READ) {
 945         int pnum;
 946
 947         ret = bdrv_is_allocated(bs, sector_num, nb_sectors, &pnum);
 948         if (ret < 0) {
 949             goto out;
 950         }
 951
 952         if (!ret || pnum != nb_sectors) {
 953             ret = bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, qiov);
 954             goto out;
 955         }
 956     }
 957
 958     /* Forward the request to the BlockDriver */
 959     if (!bs->zero_beyond_eof) {
 960         ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
 961     } else {
 962         /* Read zeros after EOF */
 963         int64_t total_sectors, max_nb_sectors;
 964
 965         total_sectors = bdrv_nb_sectors(bs);
 966         if (total_sectors < 0) {
 967             ret = total_sectors;
 968             goto out;
 969         }
 970
 971         max_nb_sectors = ROUND_UP(MAX(0, total_sectors - sector_num),
 972                                   align >> BDRV_SECTOR_BITS);
 973         if (nb_sectors < max_nb_sectors) {
 974             ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
 975         } else if (max_nb_sectors > 0) {
 976             QEMUIOVector local_qiov;
 977
 978             qemu_iovec_init(&local_qiov, qiov->niov);
 979             qemu_iovec_concat(&local_qiov, qiov, 0,
 980                               max_nb_sectors * BDRV_SECTOR_SIZE);
 981
 982             ret = bdrv_driver_preadv(bs, offset,
 983                                      max_nb_sectors * BDRV_SECTOR_SIZE,
 984                                      &local_qiov, 0);
 985
 986             qemu_iovec_destroy(&local_qiov);
 987         } else {
 988             ret = 0;
 989         }
 990
 991         /* Reading beyond end of file is supposed to produce zeroes */
 992         if (ret == 0 && total_sectors < sector_num + nb_sectors) {
 993             uint64_t offset = MAX(0, total_sectors - sector_num);
 994             uint64_t bytes = (sector_num + nb_sectors - offset) *
 995                               BDRV_SECTOR_SIZE;
 996             qemu_iovec_memset(qiov, offset * BDRV_SECTOR_SIZE, 0, bytes);
 997         }
 998     }
 999
1000 out:
1001     return ret;
1002 }
1003
1004 /*
1005  * Handle a read request in coroutine context
1006  */
1007 int coroutine_fn bdrv_co_preadv(BlockDriverState *bs,
1008     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1009     BdrvRequestFlags flags)
1010 {
1011     BlockDriver *drv = bs->drv;
1012     BdrvTrackedRequest req;
1013
1014     /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */
1015     uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1016     uint8_t *head_buf = NULL;
1017     uint8_t *tail_buf = NULL;
1018     QEMUIOVector local_qiov;
1019     bool use_local_qiov = false;
1020     int ret;
1021
1022     if (!drv) {
1023         return -ENOMEDIUM;
1024     }
1025
1026     ret = bdrv_check_byte_request(bs, offset, bytes);
1027     if (ret < 0) {
1028         return ret;
1029     }
1030
1031     /* Don't do copy-on-read if we read data before write operation */
1032     if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) {
1033         flags |= BDRV_REQ_COPY_ON_READ;
1034     }
1035
1036     /* Align read if necessary by padding qiov */
1037     if (offset & (align - 1)) {
1038         head_buf = qemu_blockalign(bs, align);
1039         qemu_iovec_init(&local_qiov, qiov->niov + 2);
1040         qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
1041         qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1042         use_local_qiov = true;
1043
1044         bytes += offset & (align - 1);
1045         offset = offset & ~(align - 1);
1046     }
1047
1048     if ((offset + bytes) & (align - 1)) {
1049         if (!use_local_qiov) {
1050             qemu_iovec_init(&local_qiov, qiov->niov + 1);
1051             qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1052             use_local_qiov = true;
1053         }
1054         tail_buf = qemu_blockalign(bs, align);
1055         qemu_iovec_add(&local_qiov, tail_buf,
1056                        align - ((offset + bytes) & (align - 1)));
1057
1058         bytes = ROUND_UP(bytes, align);
1059     }
1060
1061     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
1062     ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align,
1063                               use_local_qiov ? &local_qiov : qiov,
1064                               flags);
1065     tracked_request_end(&req);
1066
1067     if (use_local_qiov) {
1068         qemu_iovec_destroy(&local_qiov);
1069         qemu_vfree(head_buf);
1070         qemu_vfree(tail_buf);
1071     }
1072
1073     return ret;
1074 }
1075
1076 static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs,
1077     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
1078     BdrvRequestFlags flags)
1079 {
1080     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
1081         return -EINVAL;
1082     }
1083
1084     return bdrv_co_preadv(bs, sector_num << BDRV_SECTOR_BITS,
1085                           nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
1086 }
1087
1088 int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num,
1089     int nb_sectors, QEMUIOVector *qiov)
1090 {
1091     trace_bdrv_co_readv(bs, sector_num, nb_sectors);
1092
1093     return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 0);
1094 }
1095
1096 int coroutine_fn bdrv_co_readv_no_serialising(BlockDriverState *bs,
1097     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
1098 {
1099     trace_bdrv_co_readv_no_serialising(bs, sector_num, nb_sectors);
1100
1101     return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov,
1102                             BDRV_REQ_NO_SERIALISING);
1103 }
1104
1105 int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs,
1106     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
1107 {
1108     trace_bdrv_co_copy_on_readv(bs, sector_num, nb_sectors);
1109
1110     return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov,
1111                             BDRV_REQ_COPY_ON_READ);
1112 }
1113
1114 #define MAX_WRITE_ZEROES_BOUNCE_BUFFER 32768
1115
1116 static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
1117     int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
1118 {
1119     BlockDriver *drv = bs->drv;
1120     QEMUIOVector qiov;
1121     struct iovec iov = {0};
1122     int ret = 0;
1123     bool need_flush = false;
1124
1125     int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_write_zeroes,
1126                                         BDRV_REQUEST_MAX_SECTORS);
1127
1128     while (nb_sectors > 0 && !ret) {
1129         int num = nb_sectors;
1130
1131         /* Align request.  Block drivers can expect the "bulk" of the request
1132          * to be aligned.
1133          */
1134         if (bs->bl.write_zeroes_alignment
1135             && num > bs->bl.write_zeroes_alignment) {
1136             if (sector_num % bs->bl.write_zeroes_alignment != 0) {
1137                 /* Make a small request up to the first aligned sector.  */
1138                 num = bs->bl.write_zeroes_alignment;
1139                 num -= sector_num % bs->bl.write_zeroes_alignment;
1140             } else if ((sector_num + num) % bs->bl.write_zeroes_alignment != 0) {
1141                 /* Shorten the request to the last aligned sector.  num cannot
1142                  * underflow because num > bs->bl.write_zeroes_alignment.
1143                  */
1144                 num -= (sector_num + num) % bs->bl.write_zeroes_alignment;
1145             }
1146         }
1147
1148         /* limit request size */
1149         if (num > max_write_zeroes) {
1150             num = max_write_zeroes;
1151         }
1152
1153         ret = -ENOTSUP;
1154         /* First try the efficient write zeroes operation */
1155         if (drv->bdrv_co_write_zeroes) {
1156             ret = drv->bdrv_co_write_zeroes(bs, sector_num, num,
1157                                             flags & bs->supported_zero_flags);
1158             if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1159                 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1160                 need_flush = true;
1161             }
1162         } else {
1163             assert(!bs->supported_zero_flags);
1164         }
1165
1166         if (ret == -ENOTSUP) {
1167             /* Fall back to bounce buffer if write zeroes is unsupported */
1168             int max_xfer_len = MIN_NON_ZERO(bs->bl.max_transfer_length,
1169                                             MAX_WRITE_ZEROES_BOUNCE_BUFFER);
1170             BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1171
1172             if ((flags & BDRV_REQ_FUA) &&
1173                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1174                 /* No need for bdrv_driver_pwrite() to do a fallback
1175                  * flush on each chunk; use just one at the end */
1176                 write_flags &= ~BDRV_REQ_FUA;
1177                 need_flush = true;
1178             }
1179             num = MIN(num, max_xfer_len);
1180             iov.iov_len = num * BDRV_SECTOR_SIZE;
1181             if (iov.iov_base == NULL) {
1182                 iov.iov_base = qemu_try_blockalign(bs, num * BDRV_SECTOR_SIZE);
1183                 if (iov.iov_base == NULL) {
1184                     ret = -ENOMEM;
1185                     goto fail;
1186                 }
1187                 memset(iov.iov_base, 0, num * BDRV_SECTOR_SIZE);
1188             }
1189             qemu_iovec_init_external(&qiov, &iov, 1);
1190
1191             ret = bdrv_driver_pwritev(bs, sector_num * BDRV_SECTOR_SIZE,
1192                                       num * BDRV_SECTOR_SIZE, &qiov,
1193                                       write_flags);
1194
1195             /* Keep bounce buffer around if it is big enough for all
1196              * all future requests.
1197              */
1198             if (num < max_xfer_len) {
1199                 qemu_vfree(iov.iov_base);
1200                 iov.iov_base = NULL;
1201             }
1202         }
1203
1204         sector_num += num;
1205         nb_sectors -= num;
1206     }
1207
1208 fail:
1209     if (ret == 0 && need_flush) {
1210         ret = bdrv_co_flush(bs);
1211     }
1212     qemu_vfree(iov.iov_base);
1213     return ret;
1214 }
1215
1216 /*
1217  * Forwards an already correctly aligned write request to the BlockDriver.
1218  */
1219 static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
1220     BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
1221     QEMUIOVector *qiov, int flags)
1222 {
1223     BlockDriver *drv = bs->drv;
1224     bool waited;
1225     int ret;
1226
1227     int64_t sector_num = offset >> BDRV_SECTOR_BITS;
1228     unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS;
1229
1230     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
1231     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
1232     assert(!qiov || bytes == qiov->size);
1233     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1234
1235     waited = wait_serialising_requests(req);
1236     assert(!waited || !req->serialising);
1237     assert(req->overlap_offset <= offset);
1238     assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1239
1240     ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req);
1241
1242     if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
1243         !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_write_zeroes &&
1244         qemu_iovec_is_zero(qiov)) {
1245         flags |= BDRV_REQ_ZERO_WRITE;
1246         if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
1247             flags |= BDRV_REQ_MAY_UNMAP;
1248         }
1249     }
1250
1251     if (ret < 0) {
1252         /* Do nothing, write notifier decided to fail this request */
1253     } else if (flags & BDRV_REQ_ZERO_WRITE) {
1254         bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
1255         ret = bdrv_co_do_write_zeroes(bs, sector_num, nb_sectors, flags);
1256     } else {
1257         bdrv_debug_event(bs, BLKDBG_PWRITEV);
1258         ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags);
1259     }
1260     bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
1261
1262     bdrv_set_dirty(bs, sector_num, nb_sectors);
1263
1264     if (bs->wr_highest_offset < offset + bytes) {
1265         bs->wr_highest_offset = offset + bytes;
1266     }
1267
1268     if (ret >= 0) {
1269         bs->total_sectors = MAX(bs->total_sectors, sector_num + nb_sectors);
1270     }
1271
1272     return ret;
1273 }
1274
1275 static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs,
1276                                                 int64_t offset,
1277                                                 unsigned int bytes,
1278                                                 BdrvRequestFlags flags,
1279                                                 BdrvTrackedRequest *req)
1280 {
1281     uint8_t *buf = NULL;
1282     QEMUIOVector local_qiov;
1283     struct iovec iov;
1284     uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1285     unsigned int head_padding_bytes, tail_padding_bytes;
1286     int ret = 0;
1287
1288     head_padding_bytes = offset & (align - 1);
1289     tail_padding_bytes = align - ((offset + bytes) & (align - 1));
1290
1291
1292     assert(flags & BDRV_REQ_ZERO_WRITE);
1293     if (head_padding_bytes || tail_padding_bytes) {
1294         buf = qemu_blockalign(bs, align);
1295         iov = (struct iovec) {
1296             .iov_base   = buf,
1297             .iov_len    = align,
1298         };
1299         qemu_iovec_init_external(&local_qiov, &iov, 1);
1300     }
1301     if (head_padding_bytes) {
1302         uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes);
1303
1304         /* RMW the unaligned part before head. */
1305         mark_request_serialising(req, align);
1306         wait_serialising_requests(req);
1307         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1308         ret = bdrv_aligned_preadv(bs, req, offset & ~(align - 1), align,
1309                                   align, &local_qiov, 0);
1310         if (ret < 0) {
1311             goto fail;
1312         }
1313         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1314
1315         memset(buf + head_padding_bytes, 0, zero_bytes);
1316         ret = bdrv_aligned_pwritev(bs, req, offset & ~(align - 1), align,
1317                                    &local_qiov,
1318                                    flags & ~BDRV_REQ_ZERO_WRITE);
1319         if (ret < 0) {
1320             goto fail;
1321         }
1322         offset += zero_bytes;
1323         bytes -= zero_bytes;
1324     }
1325
1326     assert(!bytes || (offset & (align - 1)) == 0);
1327     if (bytes >= align) {
1328         /* Write the aligned part in the middle. */
1329         uint64_t aligned_bytes = bytes & ~(align - 1);
1330         ret = bdrv_aligned_pwritev(bs, req, offset, aligned_bytes,
1331                                    NULL, flags);
1332         if (ret < 0) {
1333             goto fail;
1334         }
1335         bytes -= aligned_bytes;
1336         offset += aligned_bytes;
1337     }
1338
1339     assert(!bytes || (offset & (align - 1)) == 0);
1340     if (bytes) {
1341         assert(align == tail_padding_bytes + bytes);
1342         /* RMW the unaligned part after tail. */
1343         mark_request_serialising(req, align);
1344         wait_serialising_requests(req);
1345         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1346         ret = bdrv_aligned_preadv(bs, req, offset, align,
1347                                   align, &local_qiov, 0);
1348         if (ret < 0) {
1349             goto fail;
1350         }
1351         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1352
1353         memset(buf, 0, bytes);
1354         ret = bdrv_aligned_pwritev(bs, req, offset, align,
1355                                    &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE);
1356     }
1357 fail:
1358     qemu_vfree(buf);
1359     return ret;
1360
1361 }
1362
1363 /*
1364  * Handle a write request in coroutine context
1365  */
1366 int coroutine_fn bdrv_co_pwritev(BlockDriverState *bs,
1367     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1368     BdrvRequestFlags flags)
1369 {
1370     BdrvTrackedRequest req;
1371     /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */
1372     uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1373     uint8_t *head_buf = NULL;
1374     uint8_t *tail_buf = NULL;
1375     QEMUIOVector local_qiov;
1376     bool use_local_qiov = false;
1377     int ret;
1378
1379     if (!bs->drv) {
1380         return -ENOMEDIUM;
1381     }
1382     if (bs->read_only) {
1383         return -EPERM;
1384     }
1385     assert(!(bs->open_flags & BDRV_O_INACTIVE));
1386
1387     ret = bdrv_check_byte_request(bs, offset, bytes);
1388     if (ret < 0) {
1389         return ret;
1390     }
1391
1392     /*
1393      * Align write if necessary by performing a read-modify-write cycle.
1394      * Pad qiov with the read parts and be sure to have a tracked request not
1395      * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
1396      */
1397     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
1398
1399     if (!qiov) {
1400         ret = bdrv_co_do_zero_pwritev(bs, offset, bytes, flags, &req);
1401         goto out;
1402     }
1403
1404     if (offset & (align - 1)) {
1405         QEMUIOVector head_qiov;
1406         struct iovec head_iov;
1407
1408         mark_request_serialising(&req, align);
1409         wait_serialising_requests(&req);
1410
1411         head_buf = qemu_blockalign(bs, align);
1412         head_iov = (struct iovec) {
1413             .iov_base   = head_buf,
1414             .iov_len    = align,
1415         };
1416         qemu_iovec_init_external(&head_qiov, &head_iov, 1);
1417
1418         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1419         ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align,
1420                                   align, &head_qiov, 0);
1421         if (ret < 0) {
1422             goto fail;
1423         }
1424         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1425
1426         qemu_iovec_init(&local_qiov, qiov->niov + 2);
1427         qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
1428         qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1429         use_local_qiov = true;
1430
1431         bytes += offset & (align - 1);
1432         offset = offset & ~(align - 1);
1433     }
1434
1435     if ((offset + bytes) & (align - 1)) {
1436         QEMUIOVector tail_qiov;
1437         struct iovec tail_iov;
1438         size_t tail_bytes;
1439         bool waited;
1440
1441         mark_request_serialising(&req, align);
1442         waited = wait_serialising_requests(&req);
1443         assert(!waited || !use_local_qiov);
1444
1445         tail_buf = qemu_blockalign(bs, align);
1446         tail_iov = (struct iovec) {
1447             .iov_base   = tail_buf,
1448             .iov_len    = align,
1449         };
1450         qemu_iovec_init_external(&tail_qiov, &tail_iov, 1);
1451
1452         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1453         ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align,
1454                                   align, &tail_qiov, 0);
1455         if (ret < 0) {
1456             goto fail;
1457         }
1458         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1459
1460         if (!use_local_qiov) {
1461             qemu_iovec_init(&local_qiov, qiov->niov + 1);
1462             qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1463             use_local_qiov = true;
1464         }
1465
1466         tail_bytes = (offset + bytes) & (align - 1);
1467         qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes);
1468
1469         bytes = ROUND_UP(bytes, align);
1470     }
1471
1472     ret = bdrv_aligned_pwritev(bs, &req, offset, bytes,
1473                                use_local_qiov ? &local_qiov : qiov,
1474                                flags);
1475
1476 fail:
1477
1478     if (use_local_qiov) {
1479         qemu_iovec_destroy(&local_qiov);
1480     }
1481     qemu_vfree(head_buf);
1482     qemu_vfree(tail_buf);
1483 out:
1484     tracked_request_end(&req);
1485     return ret;
1486 }
1487
1488 static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs,
1489     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
1490     BdrvRequestFlags flags)
1491 {
1492     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
1493         return -EINVAL;
1494     }
1495
1496     return bdrv_co_pwritev(bs, sector_num << BDRV_SECTOR_BITS,
1497                            nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
1498 }
1499
1500 int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num,
1501     int nb_sectors, QEMUIOVector *qiov)
1502 {
1503     trace_bdrv_co_writev(bs, sector_num, nb_sectors);
1504
1505     return bdrv_co_do_writev(bs, sector_num, nb_sectors, qiov, 0);
1506 }
1507
1508 int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs,
1509                                       int64_t sector_num, int nb_sectors,
1510                                       BdrvRequestFlags flags)
1511 {
1512     trace_bdrv_co_write_zeroes(bs, sector_num, nb_sectors, flags);
1513
1514     if (!(bs->open_flags & BDRV_O_UNMAP)) {
1515         flags &= ~BDRV_REQ_MAY_UNMAP;
1516     }
1517
1518     return bdrv_co_do_writev(bs, sector_num, nb_sectors, NULL,
1519                              BDRV_REQ_ZERO_WRITE | flags);
1520 }
1521
1522 typedef struct BdrvCoGetBlockStatusData {
1523     BlockDriverState *bs;
1524     BlockDriverState *base;
1525     BlockDriverState **file;
1526     int64_t sector_num;
1527     int nb_sectors;
1528     int *pnum;
1529     int64_t ret;
1530     bool done;
1531 } BdrvCoGetBlockStatusData;
1532
1533 /*
1534  * Returns the allocation status of the specified sectors.
1535  * Drivers not implementing the functionality are assumed to not support
1536  * backing files, hence all their sectors are reported as allocated.
1537  *
1538  * If 'sector_num' is beyond the end of the disk image the return value is 0
1539  * and 'pnum' is set to 0.
1540  *
1541  * 'pnum' is set to the number of sectors (including and immediately following
1542  * the specified sector) that are known to be in the same
1543  * allocated/unallocated state.
1544  *
1545  * 'nb_sectors' is the max value 'pnum' should be set to.  If nb_sectors goes
1546  * beyond the end of the disk image it will be clamped.
1547  *
1548  * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file'
1549  * points to the BDS which the sector range is allocated in.
1550  */
1551 static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
1552                                                      int64_t sector_num,
1553                                                      int nb_sectors, int *pnum,
1554                                                      BlockDriverState **file)
1555 {
1556     int64_t total_sectors;
1557     int64_t n;
1558     int64_t ret, ret2;
1559
1560     total_sectors = bdrv_nb_sectors(bs);
1561     if (total_sectors < 0) {
1562         return total_sectors;
1563     }
1564
1565     if (sector_num >= total_sectors) {
1566         *pnum = 0;
1567         return 0;
1568     }
1569
1570     n = total_sectors - sector_num;
1571     if (n < nb_sectors) {
1572         nb_sectors = n;
1573     }
1574
1575     if (!bs->drv->bdrv_co_get_block_status) {
1576         *pnum = nb_sectors;
1577         ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
1578         if (bs->drv->protocol_name) {
1579             ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE);
1580         }
1581         return ret;
1582     }
1583
1584     *file = NULL;
1585     ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum,
1586                                             file);
1587     if (ret < 0) {
1588         *pnum = 0;
1589         return ret;
1590     }
1591
1592     if (ret & BDRV_BLOCK_RAW) {
1593         assert(ret & BDRV_BLOCK_OFFSET_VALID);
1594         return bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS,
1595                                      *pnum, pnum, file);
1596     }
1597
1598     if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
1599         ret |= BDRV_BLOCK_ALLOCATED;
1600     } else {
1601         if (bdrv_unallocated_blocks_are_zero(bs)) {
1602             ret |= BDRV_BLOCK_ZERO;
1603         } else if (bs->backing) {
1604             BlockDriverState *bs2 = bs->backing->bs;
1605             int64_t nb_sectors2 = bdrv_nb_sectors(bs2);
1606             if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) {
1607                 ret |= BDRV_BLOCK_ZERO;
1608             }
1609         }
1610     }
1611
1612     if (*file && *file != bs &&
1613         (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
1614         (ret & BDRV_BLOCK_OFFSET_VALID)) {
1615         BlockDriverState *file2;
1616         int file_pnum;
1617
1618         ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS,
1619                                         *pnum, &file_pnum, &file2);
1620         if (ret2 >= 0) {
1621             /* Ignore errors.  This is just providing extra information, it
1622              * is useful but not necessary.
1623              */
1624             if (!file_pnum) {
1625                 /* !file_pnum indicates an offset at or beyond the EOF; it is
1626                  * perfectly valid for the format block driver to point to such
1627                  * offsets, so catch it and mark everything as zero */
1628                 ret |= BDRV_BLOCK_ZERO;
1629             } else {
1630                 /* Limit request to the range reported by the protocol driver */
1631                 *pnum = file_pnum;
1632                 ret |= (ret2 & BDRV_BLOCK_ZERO);
1633             }
1634         }
1635     }
1636
1637     return ret;
1638 }
1639
1640 static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs,
1641         BlockDriverState *base,
1642         int64_t sector_num,
1643         int nb_sectors,
1644         int *pnum,
1645         BlockDriverState **file)
1646 {
1647     BlockDriverState *p;
1648     int64_t ret = 0;
1649
1650     assert(bs != base);
1651     for (p = bs; p != base; p = backing_bs(p)) {
1652         ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file);
1653         if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) {
1654             break;
1655         }
1656         /* [sector_num, pnum] unallocated on this layer, which could be only
1657          * the first part of [sector_num, nb_sectors].  */
1658         nb_sectors = MIN(nb_sectors, *pnum);
1659     }
1660     return ret;
1661 }
1662
1663 /* Coroutine wrapper for bdrv_get_block_status_above() */
1664 static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque)
1665 {
1666     BdrvCoGetBlockStatusData *data = opaque;
1667
1668     data->ret = bdrv_co_get_block_status_above(data->bs, data->base,
1669                                                data->sector_num,
1670                                                data->nb_sectors,
1671                                                data->pnum,
1672                                                data->file);
1673     data->done = true;
1674 }
1675
1676 /*
1677  * Synchronous wrapper around bdrv_co_get_block_status_above().
1678  *
1679  * See bdrv_co_get_block_status_above() for details.
1680  */
1681 int64_t bdrv_get_block_status_above(BlockDriverState *bs,
1682                                     BlockDriverState *base,
1683                                     int64_t sector_num,
1684                                     int nb_sectors, int *pnum,
1685                                     BlockDriverState **file)
1686 {
1687     Coroutine *co;
1688     BdrvCoGetBlockStatusData data = {
1689         .bs = bs,
1690         .base = base,
1691         .file = file,
1692         .sector_num = sector_num,
1693         .nb_sectors = nb_sectors,
1694         .pnum = pnum,
1695         .done = false,
1696     };
1697
1698     if (qemu_in_coroutine()) {
1699         /* Fast-path if already in coroutine context */
1700         bdrv_get_block_status_above_co_entry(&data);
1701     } else {
1702         AioContext *aio_context = bdrv_get_aio_context(bs);
1703
1704         co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry);
1705         qemu_coroutine_enter(co, &data);
1706         while (!data.done) {
1707             aio_poll(aio_context, true);
1708         }
1709     }
1710     return data.ret;
1711 }
1712
1713 int64_t bdrv_get_block_status(BlockDriverState *bs,
1714                               int64_t sector_num,
1715                               int nb_sectors, int *pnum,
1716                               BlockDriverState **file)
1717 {
1718     return bdrv_get_block_status_above(bs, backing_bs(bs),
1719                                        sector_num, nb_sectors, pnum, file);
1720 }
1721
1722 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num,
1723                                    int nb_sectors, int *pnum)
1724 {
1725     BlockDriverState *file;
1726     int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum,
1727                                         &file);
1728     if (ret < 0) {
1729         return ret;
1730     }
1731     return !!(ret & BDRV_BLOCK_ALLOCATED);
1732 }
1733
1734 /*
1735  * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
1736  *
1737  * Return true if the given sector is allocated in any image between
1738  * BASE and TOP (inclusive).  BASE can be NULL to check if the given
1739  * sector is allocated in any image of the chain.  Return false otherwise.
1740  *
1741  * 'pnum' is set to the number of sectors (including and immediately following
1742  *  the specified sector) that are known to be in the same
1743  *  allocated/unallocated state.
1744  *
1745  */
1746 int bdrv_is_allocated_above(BlockDriverState *top,
1747                             BlockDriverState *base,
1748                             int64_t sector_num,
1749                             int nb_sectors, int *pnum)
1750 {
1751     BlockDriverState *intermediate;
1752     int ret, n = nb_sectors;
1753
1754     intermediate = top;
1755     while (intermediate && intermediate != base) {
1756         int pnum_inter;
1757         ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors,
1758                                 &pnum_inter);
1759         if (ret < 0) {
1760             return ret;
1761         } else if (ret) {
1762             *pnum = pnum_inter;
1763             return 1;
1764         }
1765
1766         /*
1767          * [sector_num, nb_sectors] is unallocated on top but intermediate
1768          * might have
1769          *
1770          * [sector_num+x, nr_sectors] allocated.
1771          */
1772         if (n > pnum_inter &&
1773             (intermediate == top ||
1774              sector_num + pnum_inter < intermediate->total_sectors)) {
1775             n = pnum_inter;
1776         }
1777
1778         intermediate = backing_bs(intermediate);
1779     }
1780
1781     *pnum = n;
1782     return 0;
1783 }
1784
1785 int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
1786                           const uint8_t *buf, int nb_sectors)
1787 {
1788     BlockDriver *drv = bs->drv;
1789     int ret;
1790
1791     if (!drv) {
1792         return -ENOMEDIUM;
1793     }
1794     if (!drv->bdrv_write_compressed) {
1795         return -ENOTSUP;
1796     }
1797     ret = bdrv_check_request(bs, sector_num, nb_sectors);
1798     if (ret < 0) {
1799         return ret;
1800     }
1801
1802     assert(QLIST_EMPTY(&bs->dirty_bitmaps));
1803
1804     return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors);
1805 }
1806
1807 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
1808                       int64_t pos, int size)
1809 {
1810     QEMUIOVector qiov;
1811     struct iovec iov = {
1812         .iov_base   = (void *) buf,
1813         .iov_len    = size,
1814     };
1815
1816     qemu_iovec_init_external(&qiov, &iov, 1);
1817     return bdrv_writev_vmstate(bs, &qiov, pos);
1818 }
1819
1820 int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
1821 {
1822     BlockDriver *drv = bs->drv;
1823
1824     if (!drv) {
1825         return -ENOMEDIUM;
1826     } else if (drv->bdrv_save_vmstate) {
1827         return drv->bdrv_save_vmstate(bs, qiov, pos);
1828     } else if (bs->file) {
1829         return bdrv_writev_vmstate(bs->file->bs, qiov, pos);
1830     }
1831
1832     return -ENOTSUP;
1833 }
1834
1835 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
1836                       int64_t pos, int size)
1837 {
1838     BlockDriver *drv = bs->drv;
1839     if (!drv)
1840         return -ENOMEDIUM;
1841     if (drv->bdrv_load_vmstate)
1842         return drv->bdrv_load_vmstate(bs, buf, pos, size);
1843     if (bs->file)
1844         return bdrv_load_vmstate(bs->file->bs, buf, pos, size);
1845     return -ENOTSUP;
1846 }
1847
1848 /**************************************************************/
1849 /* async I/Os */
1850
1851 BlockAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
1852                            QEMUIOVector *qiov, int nb_sectors,
1853                            BlockCompletionFunc *cb, void *opaque)
1854 {
1855     trace_bdrv_aio_readv(bs, sector_num, nb_sectors, opaque);
1856
1857     return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0,
1858                                  cb, opaque, false);
1859 }
1860
1861 BlockAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
1862                             QEMUIOVector *qiov, int nb_sectors,
1863                             BlockCompletionFunc *cb, void *opaque)
1864 {
1865     trace_bdrv_aio_writev(bs, sector_num, nb_sectors, opaque);
1866
1867     return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0,
1868                                  cb, opaque, true);
1869 }
1870
1871 BlockAIOCB *bdrv_aio_write_zeroes(BlockDriverState *bs,
1872         int64_t sector_num, int nb_sectors, BdrvRequestFlags flags,
1873         BlockCompletionFunc *cb, void *opaque)
1874 {
1875     trace_bdrv_aio_write_zeroes(bs, sector_num, nb_sectors, flags, opaque);
1876
1877     return bdrv_co_aio_rw_vector(bs, sector_num, NULL, nb_sectors,
1878                                  BDRV_REQ_ZERO_WRITE | flags,
1879                                  cb, opaque, true);
1880 }
1881
1882 void bdrv_aio_cancel(BlockAIOCB *acb)
1883 {
1884     qemu_aio_ref(acb);
1885     bdrv_aio_cancel_async(acb);
1886     while (acb->refcnt > 1) {
1887         if (acb->aiocb_info->get_aio_context) {
1888             aio_poll(acb->aiocb_info->get_aio_context(acb), true);
1889         } else if (acb->bs) {
1890             aio_poll(bdrv_get_aio_context(acb->bs), true);
1891         } else {
1892             abort();
1893         }
1894     }
1895     qemu_aio_unref(acb);
1896 }
1897
1898 /* Async version of aio cancel. The caller is not blocked if the acb implements
1899  * cancel_async, otherwise we do nothing and let the request normally complete.
1900  * In either case the completion callback must be called. */
1901 void bdrv_aio_cancel_async(BlockAIOCB *acb)
1902 {
1903     if (acb->aiocb_info->cancel_async) {
1904         acb->aiocb_info->cancel_async(acb);
1905     }
1906 }
1907
1908 /**************************************************************/
1909 /* async block device emulation */
1910
1911 typedef struct BlockAIOCBCoroutine {
1912     BlockAIOCB common;
1913     BlockRequest req;
1914     bool is_write;
1915     bool need_bh;
1916     bool *done;
1917     QEMUBH* bh;
1918 } BlockAIOCBCoroutine;
1919
1920 static const AIOCBInfo bdrv_em_co_aiocb_info = {
1921     .aiocb_size         = sizeof(BlockAIOCBCoroutine),
1922 };
1923
1924 static void bdrv_co_complete(BlockAIOCBCoroutine *acb)
1925 {
1926     if (!acb->need_bh) {
1927         acb->common.cb(acb->common.opaque, acb->req.error);
1928         qemu_aio_unref(acb);
1929     }
1930 }
1931
1932 static void bdrv_co_em_bh(void *opaque)
1933 {
1934     BlockAIOCBCoroutine *acb = opaque;
1935
1936     assert(!acb->need_bh);
1937     qemu_bh_delete(acb->bh);
1938     bdrv_co_complete(acb);
1939 }
1940
1941 static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb)
1942 {
1943     acb->need_bh = false;
1944     if (acb->req.error != -EINPROGRESS) {
1945         BlockDriverState *bs = acb->common.bs;
1946
1947         acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb);
1948         qemu_bh_schedule(acb->bh);
1949     }
1950 }
1951
1952 /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */
1953 static void coroutine_fn bdrv_co_do_rw(void *opaque)
1954 {
1955     BlockAIOCBCoroutine *acb = opaque;
1956     BlockDriverState *bs = acb->common.bs;
1957
1958     if (!acb->is_write) {
1959         acb->req.error = bdrv_co_do_readv(bs, acb->req.sector,
1960             acb->req.nb_sectors, acb->req.qiov, acb->req.flags);
1961     } else {
1962         acb->req.error = bdrv_co_do_writev(bs, acb->req.sector,
1963             acb->req.nb_sectors, acb->req.qiov, acb->req.flags);
1964     }
1965
1966     bdrv_co_complete(acb);
1967 }
1968
1969 static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
1970                                          int64_t sector_num,
1971                                          QEMUIOVector *qiov,
1972                                          int nb_sectors,
1973                                          BdrvRequestFlags flags,
1974                                          BlockCompletionFunc *cb,
1975                                          void *opaque,
1976                                          bool is_write)
1977 {
1978     Coroutine *co;
1979     BlockAIOCBCoroutine *acb;
1980
1981     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
1982     acb->need_bh = true;
1983     acb->req.error = -EINPROGRESS;
1984     acb->req.sector = sector_num;
1985     acb->req.nb_sectors = nb_sectors;
1986     acb->req.qiov = qiov;
1987     acb->req.flags = flags;
1988     acb->is_write = is_write;
1989
1990     co = qemu_coroutine_create(bdrv_co_do_rw);
1991     qemu_coroutine_enter(co, acb);
1992
1993     bdrv_co_maybe_schedule_bh(acb);
1994     return &acb->common;
1995 }
1996
1997 static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque)
1998 {
1999     BlockAIOCBCoroutine *acb = opaque;
2000     BlockDriverState *bs = acb->common.bs;
2001
2002     acb->req.error = bdrv_co_flush(bs);
2003     bdrv_co_complete(acb);
2004 }
2005
2006 BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs,
2007         BlockCompletionFunc *cb, void *opaque)
2008 {
2009     trace_bdrv_aio_flush(bs, opaque);
2010
2011     Coroutine *co;
2012     BlockAIOCBCoroutine *acb;
2013
2014     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2015     acb->need_bh = true;
2016     acb->req.error = -EINPROGRESS;
2017
2018     co = qemu_coroutine_create(bdrv_aio_flush_co_entry);
2019     qemu_coroutine_enter(co, acb);
2020
2021     bdrv_co_maybe_schedule_bh(acb);
2022     return &acb->common;
2023 }
2024
2025 static void coroutine_fn bdrv_aio_discard_co_entry(void *opaque)
2026 {
2027     BlockAIOCBCoroutine *acb = opaque;
2028     BlockDriverState *bs = acb->common.bs;
2029
2030     acb->req.error = bdrv_co_discard(bs, acb->req.sector, acb->req.nb_sectors);
2031     bdrv_co_complete(acb);
2032 }
2033
2034 BlockAIOCB *bdrv_aio_discard(BlockDriverState *bs,
2035         int64_t sector_num, int nb_sectors,
2036         BlockCompletionFunc *cb, void *opaque)
2037 {
2038     Coroutine *co;
2039     BlockAIOCBCoroutine *acb;
2040
2041     trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque);
2042
2043     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2044     acb->need_bh = true;
2045     acb->req.error = -EINPROGRESS;
2046     acb->req.sector = sector_num;
2047     acb->req.nb_sectors = nb_sectors;
2048     co = qemu_coroutine_create(bdrv_aio_discard_co_entry);
2049     qemu_coroutine_enter(co, acb);
2050
2051     bdrv_co_maybe_schedule_bh(acb);
2052     return &acb->common;
2053 }
2054
2055 void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
2056                    BlockCompletionFunc *cb, void *opaque)
2057 {
2058     BlockAIOCB *acb;
2059
2060     acb = g_malloc(aiocb_info->aiocb_size);
2061     acb->aiocb_info = aiocb_info;
2062     acb->bs = bs;
2063     acb->cb = cb;
2064     acb->opaque = opaque;
2065     acb->refcnt = 1;
2066     return acb;
2067 }
2068
2069 void qemu_aio_ref(void *p)
2070 {
2071     BlockAIOCB *acb = p;
2072     acb->refcnt++;
2073 }
2074
2075 void qemu_aio_unref(void *p)
2076 {
2077     BlockAIOCB *acb = p;
2078     assert(acb->refcnt > 0);
2079     if (--acb->refcnt == 0) {
2080         g_free(acb);
2081     }
2082 }
2083
2084 /**************************************************************/
2085 /* Coroutine block device emulation */
2086
2087 static void coroutine_fn bdrv_flush_co_entry(void *opaque)
2088 {
2089     RwCo *rwco = opaque;
2090
2091     rwco->ret = bdrv_co_flush(rwco->bs);
2092 }
2093
2094 int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
2095 {
2096     int ret;
2097     BdrvTrackedRequest req;
2098
2099     if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
2100         bdrv_is_sg(bs)) {
2101         return 0;
2102     }
2103
2104     tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH);
2105
2106     /* Write back all layers by calling one driver function */
2107     if (bs->drv->bdrv_co_flush) {
2108         ret = bs->drv->bdrv_co_flush(bs);
2109         goto out;
2110     }
2111
2112     /* Write back cached data to the OS even with cache=unsafe */
2113     BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
2114     if (bs->drv->bdrv_co_flush_to_os) {
2115         ret = bs->drv->bdrv_co_flush_to_os(bs);
2116         if (ret < 0) {
2117             goto out;
2118         }
2119     }
2120
2121     /* But don't actually force it to the disk with cache=unsafe */
2122     if (bs->open_flags & BDRV_O_NO_FLUSH) {
2123         goto flush_parent;
2124     }
2125
2126     BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
2127     if (bs->drv->bdrv_co_flush_to_disk) {
2128         ret = bs->drv->bdrv_co_flush_to_disk(bs);
2129     } else if (bs->drv->bdrv_aio_flush) {
2130         BlockAIOCB *acb;
2131         CoroutineIOCompletion co = {
2132             .coroutine = qemu_coroutine_self(),
2133         };
2134
2135         acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
2136         if (acb == NULL) {
2137             ret = -EIO;
2138         } else {
2139             qemu_coroutine_yield();
2140             ret = co.ret;
2141         }
2142     } else {
2143         /*
2144          * Some block drivers always operate in either writethrough or unsafe
2145          * mode and don't support bdrv_flush therefore. Usually qemu doesn't
2146          * know how the server works (because the behaviour is hardcoded or
2147          * depends on server-side configuration), so we can't ensure that
2148          * everything is safe on disk. Returning an error doesn't work because
2149          * that would break guests even if the server operates in writethrough
2150          * mode.
2151          *
2152          * Let's hope the user knows what he's doing.
2153          */
2154         ret = 0;
2155     }
2156     if (ret < 0) {
2157         goto out;
2158     }
2159
2160     /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
2161      * in the case of cache=unsafe, so there are no useless flushes.
2162      */
2163 flush_parent:
2164     ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
2165 out:
2166     tracked_request_end(&req);
2167     return ret;
2168 }
2169
2170 int bdrv_flush(BlockDriverState *bs)
2171 {
2172     Coroutine *co;
2173     RwCo rwco = {
2174         .bs = bs,
2175         .ret = NOT_DONE,
2176     };
2177
2178     if (qemu_in_coroutine()) {
2179         /* Fast-path if already in coroutine context */
2180         bdrv_flush_co_entry(&rwco);
2181     } else {
2182         AioContext *aio_context = bdrv_get_aio_context(bs);
2183
2184         co = qemu_coroutine_create(bdrv_flush_co_entry);
2185         qemu_coroutine_enter(co, &rwco);
2186         while (rwco.ret == NOT_DONE) {
2187             aio_poll(aio_context, true);
2188         }
2189     }
2190
2191     return rwco.ret;
2192 }
2193
2194 typedef struct DiscardCo {
2195     BlockDriverState *bs;
2196     int64_t sector_num;
2197     int nb_sectors;
2198     int ret;
2199 } DiscardCo;
2200 static void coroutine_fn bdrv_discard_co_entry(void *opaque)
2201 {
2202     DiscardCo *rwco = opaque;
2203
2204     rwco->ret = bdrv_co_discard(rwco->bs, rwco->sector_num, rwco->nb_sectors);
2205 }
2206
2207 int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
2208                                  int nb_sectors)
2209 {
2210     BdrvTrackedRequest req;
2211     int max_discard, ret;
2212
2213     if (!bs->drv) {
2214         return -ENOMEDIUM;
2215     }
2216
2217     ret = bdrv_check_request(bs, sector_num, nb_sectors);
2218     if (ret < 0) {
2219         return ret;
2220     } else if (bs->read_only) {
2221         return -EPERM;
2222     }
2223     assert(!(bs->open_flags & BDRV_O_INACTIVE));
2224
2225     /* Do nothing if disabled.  */
2226     if (!(bs->open_flags & BDRV_O_UNMAP)) {
2227         return 0;
2228     }
2229
2230     if (!bs->drv->bdrv_co_discard && !bs->drv->bdrv_aio_discard) {
2231         return 0;
2232     }
2233
2234     tracked_request_begin(&req, bs, sector_num, nb_sectors,
2235                           BDRV_TRACKED_DISCARD);
2236     bdrv_set_dirty(bs, sector_num, nb_sectors);
2237
2238     max_discard = MIN_NON_ZERO(bs->bl.max_discard, BDRV_REQUEST_MAX_SECTORS);
2239     while (nb_sectors > 0) {
2240         int ret;
2241         int num = nb_sectors;
2242
2243         /* align request */
2244         if (bs->bl.discard_alignment &&
2245             num >= bs->bl.discard_alignment &&
2246             sector_num % bs->bl.discard_alignment) {
2247             if (num > bs->bl.discard_alignment) {
2248                 num = bs->bl.discard_alignment;
2249             }
2250             num -= sector_num % bs->bl.discard_alignment;
2251         }
2252
2253         /* limit request size */
2254         if (num > max_discard) {
2255             num = max_discard;
2256         }
2257
2258         if (bs->drv->bdrv_co_discard) {
2259             ret = bs->drv->bdrv_co_discard(bs, sector_num, num);
2260         } else {
2261             BlockAIOCB *acb;
2262             CoroutineIOCompletion co = {
2263                 .coroutine = qemu_coroutine_self(),
2264             };
2265
2266             acb = bs->drv->bdrv_aio_discard(bs, sector_num, nb_sectors,
2267                                             bdrv_co_io_em_complete, &co);
2268             if (acb == NULL) {
2269                 ret = -EIO;
2270                 goto out;
2271             } else {
2272                 qemu_coroutine_yield();
2273                 ret = co.ret;
2274             }
2275         }
2276         if (ret && ret != -ENOTSUP) {
2277             goto out;
2278         }
2279
2280         sector_num += num;
2281         nb_sectors -= num;
2282     }
2283     ret = 0;
2284 out:
2285     tracked_request_end(&req);
2286     return ret;
2287 }
2288
2289 int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors)
2290 {
2291     Coroutine *co;
2292     DiscardCo rwco = {
2293         .bs = bs,
2294         .sector_num = sector_num,
2295         .nb_sectors = nb_sectors,
2296         .ret = NOT_DONE,
2297     };
2298
2299     if (qemu_in_coroutine()) {
2300         /* Fast-path if already in coroutine context */
2301         bdrv_discard_co_entry(&rwco);
2302     } else {
2303         AioContext *aio_context = bdrv_get_aio_context(bs);
2304
2305         co = qemu_coroutine_create(bdrv_discard_co_entry);
2306         qemu_coroutine_enter(co, &rwco);
2307         while (rwco.ret == NOT_DONE) {
2308             aio_poll(aio_context, true);
2309         }
2310     }
2311
2312     return rwco.ret;
2313 }
2314
2315 typedef struct {
2316     CoroutineIOCompletion *co;
2317     QEMUBH *bh;
2318 } BdrvIoctlCompletionData;
2319
2320 static void bdrv_ioctl_bh_cb(void *opaque)
2321 {
2322     BdrvIoctlCompletionData *data = opaque;
2323
2324     bdrv_co_io_em_complete(data->co, -ENOTSUP);
2325     qemu_bh_delete(data->bh);
2326 }
2327
2328 static int bdrv_co_do_ioctl(BlockDriverState *bs, int req, void *buf)
2329 {
2330     BlockDriver *drv = bs->drv;
2331     BdrvTrackedRequest tracked_req;
2332     CoroutineIOCompletion co = {
2333         .coroutine = qemu_coroutine_self(),
2334     };
2335     BlockAIOCB *acb;
2336
2337     tracked_request_begin(&tracked_req, bs, 0, 0, BDRV_TRACKED_IOCTL);
2338     if (!drv || !drv->bdrv_aio_ioctl) {
2339         co.ret = -ENOTSUP;
2340         goto out;
2341     }
2342
2343     acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
2344     if (!acb) {
2345         BdrvIoctlCompletionData *data = g_new(BdrvIoctlCompletionData, 1);
2346         data->bh = aio_bh_new(bdrv_get_aio_context(bs),
2347                                 bdrv_ioctl_bh_cb, data);
2348         data->co = &co;
2349         qemu_bh_schedule(data->bh);
2350     }
2351     qemu_coroutine_yield();
2352 out:
2353     tracked_request_end(&tracked_req);
2354     return co.ret;
2355 }
2356
2357 typedef struct {
2358     BlockDriverState *bs;
2359     int req;
2360     void *buf;
2361     int ret;
2362 } BdrvIoctlCoData;
2363
2364 static void coroutine_fn bdrv_co_ioctl_entry(void *opaque)
2365 {
2366     BdrvIoctlCoData *data = opaque;
2367     data->ret = bdrv_co_do_ioctl(data->bs, data->req, data->buf);
2368 }
2369
2370 /* needed for generic scsi interface */
2371 int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
2372 {
2373     BdrvIoctlCoData data = {
2374         .bs = bs,
2375         .req = req,
2376         .buf = buf,
2377         .ret = -EINPROGRESS,
2378     };
2379
2380     if (qemu_in_coroutine()) {
2381         /* Fast-path if already in coroutine context */
2382         bdrv_co_ioctl_entry(&data);
2383     } else {
2384         Coroutine *co = qemu_coroutine_create(bdrv_co_ioctl_entry);
2385
2386         qemu_coroutine_enter(co, &data);
2387         while (data.ret == -EINPROGRESS) {
2388             aio_poll(bdrv_get_aio_context(bs), true);
2389         }
2390     }
2391     return data.ret;
2392 }
2393
2394 static void coroutine_fn bdrv_co_aio_ioctl_entry(void *opaque)
2395 {
2396     BlockAIOCBCoroutine *acb = opaque;
2397     acb->req.error = bdrv_co_do_ioctl(acb->common.bs,
2398                                       acb->req.req, acb->req.buf);
2399     bdrv_co_complete(acb);
2400 }
2401
2402 BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
2403         unsigned long int req, void *buf,
2404         BlockCompletionFunc *cb, void *opaque)
2405 {
2406     BlockAIOCBCoroutine *acb = qemu_aio_get(&bdrv_em_co_aiocb_info,
2407                                             bs, cb, opaque);
2408     Coroutine *co;
2409
2410     acb->need_bh = true;
2411     acb->req.error = -EINPROGRESS;
2412     acb->req.req = req;
2413     acb->req.buf = buf;
2414     co = qemu_coroutine_create(bdrv_co_aio_ioctl_entry);
2415     qemu_coroutine_enter(co, acb);
2416
2417     bdrv_co_maybe_schedule_bh(acb);
2418     return &acb->common;
2419 }
2420
2421 void *qemu_blockalign(BlockDriverState *bs, size_t size)
2422 {
2423     return qemu_memalign(bdrv_opt_mem_align(bs), size);
2424 }
2425
2426 void *qemu_blockalign0(BlockDriverState *bs, size_t size)
2427 {
2428     return memset(qemu_blockalign(bs, size), 0, size);
2429 }
2430
2431 void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
2432 {
2433     size_t align = bdrv_opt_mem_align(bs);
2434
2435     /* Ensure that NULL is never returned on success */
2436     assert(align > 0);
2437     if (size == 0) {
2438         size = align;
2439     }
2440
2441     return qemu_try_memalign(align, size);
2442 }
2443
2444 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
2445 {
2446     void *mem = qemu_try_blockalign(bs, size);
2447
2448     if (mem) {
2449         memset(mem, 0, size);
2450     }
2451
2452     return mem;
2453 }
2454
2455 /*
2456  * Check if all memory in this vector is sector aligned.
2457  */
2458 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
2459 {
2460     int i;
2461     size_t alignment = bdrv_min_mem_align(bs);
2462
2463     for (i = 0; i < qiov->niov; i++) {
2464         if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
2465             return false;
2466         }
2467         if (qiov->iov[i].iov_len % alignment) {
2468             return false;
2469         }
2470     }
2471
2472     return true;
2473 }
2474
2475 void bdrv_add_before_write_notifier(BlockDriverState *bs,
2476                                     NotifierWithReturn *notifier)
2477 {
2478     notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
2479 }
2480
2481 void bdrv_io_plug(BlockDriverState *bs)
2482 {
2483     BdrvChild *child;
2484
2485     QLIST_FOREACH(child, &bs->children, next) {
2486         bdrv_io_plug(child->bs);
2487     }
2488
2489     if (bs->io_plugged++ == 0 && bs->io_plug_disabled == 0) {
2490         BlockDriver *drv = bs->drv;
2491         if (drv && drv->bdrv_io_plug) {
2492             drv->bdrv_io_plug(bs);
2493         }
2494     }
2495 }
2496
2497 void bdrv_io_unplug(BlockDriverState *bs)
2498 {
2499     BdrvChild *child;
2500
2501     assert(bs->io_plugged);
2502     if (--bs->io_plugged == 0 && bs->io_plug_disabled == 0) {
2503         BlockDriver *drv = bs->drv;
2504         if (drv && drv->bdrv_io_unplug) {
2505             drv->bdrv_io_unplug(bs);
2506         }
2507     }
2508
2509     QLIST_FOREACH(child, &bs->children, next) {
2510         bdrv_io_unplug(child->bs);
2511     }
2512 }
2513
2514 void bdrv_io_unplugged_begin(BlockDriverState *bs)
2515 {
2516     BdrvChild *child;
2517
2518     if (bs->io_plug_disabled++ == 0 && bs->io_plugged > 0) {
2519         BlockDriver *drv = bs->drv;
2520         if (drv && drv->bdrv_io_unplug) {
2521             drv->bdrv_io_unplug(bs);
2522         }
2523     }
2524
2525     QLIST_FOREACH(child, &bs->children, next) {
2526         bdrv_io_unplugged_begin(child->bs);
2527     }
2528 }
2529
2530 void bdrv_io_unplugged_end(BlockDriverState *bs)
2531 {
2532     BdrvChild *child;
2533
2534     assert(bs->io_plug_disabled);
2535     QLIST_FOREACH(child, &bs->children, next) {
2536         bdrv_io_unplugged_end(child->bs);
2537     }
2538
2539     if (--bs->io_plug_disabled == 0 && bs->io_plugged > 0) {
2540         BlockDriver *drv = bs->drv;
2541         if (drv && drv->bdrv_io_plug) {
2542             drv->bdrv_io_plug(bs);
2543         }
2544     }
2545 }
2546
2547 void bdrv_drained_begin(BlockDriverState *bs)
2548 {
2549     if (!bs->quiesce_counter++) {
2550         aio_disable_external(bdrv_get_aio_context(bs));
2551     }
2552     bdrv_parent_drained_begin(bs);
2553     bdrv_drain(bs);
2554 }
2555
2556 void bdrv_drained_end(BlockDriverState *bs)
2557 {
2558     bdrv_parent_drained_end(bs);
2559
2560     assert(bs->quiesce_counter > 0);
2561     if (--bs->quiesce_counter > 0) {
2562         return;
2563     }
2564     aio_enable_external(bdrv_get_aio_context(bs));
2565 }