2 * Block layer I/O functions
4 * Copyright (c) 2003 Fabrice Bellard
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 #include "qemu/osdep.h"
27 #include "sysemu/block-backend.h"
28 #include "block/blockjob.h"
29 #include "block/blockjob_int.h"
30 #include "block/block_int.h"
31 #include "qemu/cutils.h"
32 #include "qapi/error.h"
33 #include "qemu/error-report.h"
35 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
37 static BlockAIOCB
*bdrv_co_aio_prw_vector(BdrvChild
*child
,
40 BdrvRequestFlags flags
,
41 BlockCompletionFunc
*cb
,
44 static void coroutine_fn
bdrv_co_do_rw(void *opaque
);
45 static int coroutine_fn
bdrv_co_do_pwrite_zeroes(BlockDriverState
*bs
,
46 int64_t offset
, int count
, BdrvRequestFlags flags
);
48 void bdrv_parent_drained_begin(BlockDriverState
*bs
)
52 QLIST_FOREACH(c
, &bs
->parents
, next_parent
) {
53 if (c
->role
->drained_begin
) {
54 c
->role
->drained_begin(c
);
59 void bdrv_parent_drained_end(BlockDriverState
*bs
)
63 QLIST_FOREACH(c
, &bs
->parents
, next_parent
) {
64 if (c
->role
->drained_end
) {
65 c
->role
->drained_end(c
);
70 static void bdrv_merge_limits(BlockLimits
*dst
, const BlockLimits
*src
)
72 dst
->opt_transfer
= MAX(dst
->opt_transfer
, src
->opt_transfer
);
73 dst
->max_transfer
= MIN_NON_ZERO(dst
->max_transfer
, src
->max_transfer
);
74 dst
->opt_mem_alignment
= MAX(dst
->opt_mem_alignment
,
75 src
->opt_mem_alignment
);
76 dst
->min_mem_alignment
= MAX(dst
->min_mem_alignment
,
77 src
->min_mem_alignment
);
78 dst
->max_iov
= MIN_NON_ZERO(dst
->max_iov
, src
->max_iov
);
81 void bdrv_refresh_limits(BlockDriverState
*bs
, Error
**errp
)
83 BlockDriver
*drv
= bs
->drv
;
84 Error
*local_err
= NULL
;
86 memset(&bs
->bl
, 0, sizeof(bs
->bl
));
92 /* Default alignment based on whether driver has byte interface */
93 bs
->bl
.request_alignment
= drv
->bdrv_co_preadv
? 1 : 512;
95 /* Take some limits from the children as a default */
97 bdrv_refresh_limits(bs
->file
->bs
, &local_err
);
99 error_propagate(errp
, local_err
);
102 bdrv_merge_limits(&bs
->bl
, &bs
->file
->bs
->bl
);
104 bs
->bl
.min_mem_alignment
= 512;
105 bs
->bl
.opt_mem_alignment
= getpagesize();
107 /* Safe default since most protocols use readv()/writev()/etc */
108 bs
->bl
.max_iov
= IOV_MAX
;
112 bdrv_refresh_limits(bs
->backing
->bs
, &local_err
);
114 error_propagate(errp
, local_err
);
117 bdrv_merge_limits(&bs
->bl
, &bs
->backing
->bs
->bl
);
120 /* Then let the driver override it */
121 if (drv
->bdrv_refresh_limits
) {
122 drv
->bdrv_refresh_limits(bs
, errp
);
127 * The copy-on-read flag is actually a reference count so multiple users may
128 * use the feature without worrying about clobbering its previous state.
129 * Copy-on-read stays enabled until all users have called to disable it.
131 void bdrv_enable_copy_on_read(BlockDriverState
*bs
)
136 void bdrv_disable_copy_on_read(BlockDriverState
*bs
)
138 assert(bs
->copy_on_read
> 0);
142 /* Check if any requests are in-flight (including throttled requests) */
143 bool bdrv_requests_pending(BlockDriverState
*bs
)
147 if (atomic_read(&bs
->in_flight
)) {
151 QLIST_FOREACH(child
, &bs
->children
, next
) {
152 if (bdrv_requests_pending(child
->bs
)) {
160 static bool bdrv_drain_recurse(BlockDriverState
*bs
)
162 BdrvChild
*child
, *tmp
;
165 waited
= BDRV_POLL_WHILE(bs
, atomic_read(&bs
->in_flight
) > 0);
167 if (bs
->drv
&& bs
->drv
->bdrv_drain
) {
168 bs
->drv
->bdrv_drain(bs
);
171 QLIST_FOREACH_SAFE(child
, &bs
->children
, next
, tmp
) {
172 BlockDriverState
*bs
= child
->bs
;
174 qemu_get_current_aio_context() == qemu_get_aio_context();
175 assert(bs
->refcnt
> 0);
177 /* In case the recursive bdrv_drain_recurse processes a
178 * block_job_defer_to_main_loop BH and modifies the graph,
179 * let's hold a reference to bs until we are done.
181 * IOThread doesn't have such a BH, and it is not safe to call
182 * bdrv_unref without BQL, so skip doing it there.
186 waited
|= bdrv_drain_recurse(bs
);
197 BlockDriverState
*bs
;
201 static void bdrv_co_drain_bh_cb(void *opaque
)
203 BdrvCoDrainData
*data
= opaque
;
204 Coroutine
*co
= data
->co
;
205 BlockDriverState
*bs
= data
->bs
;
207 bdrv_dec_in_flight(bs
);
208 bdrv_drained_begin(bs
);
213 static void coroutine_fn
bdrv_co_yield_to_drain(BlockDriverState
*bs
)
215 BdrvCoDrainData data
;
217 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
218 * other coroutines run if they were queued from
219 * qemu_co_queue_run_restart(). */
221 assert(qemu_in_coroutine());
222 data
= (BdrvCoDrainData
) {
223 .co
= qemu_coroutine_self(),
227 bdrv_inc_in_flight(bs
);
228 aio_bh_schedule_oneshot(bdrv_get_aio_context(bs
),
229 bdrv_co_drain_bh_cb
, &data
);
231 qemu_coroutine_yield();
232 /* If we are resumed from some other event (such as an aio completion or a
233 * timer callback), it is a bug in the caller that should be fixed. */
237 void bdrv_drained_begin(BlockDriverState
*bs
)
239 if (qemu_in_coroutine()) {
240 bdrv_co_yield_to_drain(bs
);
244 if (!bs
->quiesce_counter
++) {
245 aio_disable_external(bdrv_get_aio_context(bs
));
246 bdrv_parent_drained_begin(bs
);
249 bdrv_drain_recurse(bs
);
252 void bdrv_drained_end(BlockDriverState
*bs
)
254 assert(bs
->quiesce_counter
> 0);
255 if (--bs
->quiesce_counter
> 0) {
259 bdrv_parent_drained_end(bs
);
260 aio_enable_external(bdrv_get_aio_context(bs
));
264 * Wait for pending requests to complete on a single BlockDriverState subtree,
265 * and suspend block driver's internal I/O until next request arrives.
267 * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
270 * Only this BlockDriverState's AioContext is run, so in-flight requests must
271 * not depend on events in other AioContexts. In that case, use
272 * bdrv_drain_all() instead.
274 void coroutine_fn
bdrv_co_drain(BlockDriverState
*bs
)
276 assert(qemu_in_coroutine());
277 bdrv_drained_begin(bs
);
278 bdrv_drained_end(bs
);
281 void bdrv_drain(BlockDriverState
*bs
)
283 bdrv_drained_begin(bs
);
284 bdrv_drained_end(bs
);
288 * Wait for pending requests to complete across all BlockDriverStates
290 * This function does not flush data to disk, use bdrv_flush_all() for that
291 * after calling this function.
293 * This pauses all block jobs and disables external clients. It must
294 * be paired with bdrv_drain_all_end().
296 * NOTE: no new block jobs or BlockDriverStates can be created between
297 * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls.
299 void bdrv_drain_all_begin(void)
301 /* Always run first iteration so any pending completion BHs run */
303 BlockDriverState
*bs
;
305 GSList
*aio_ctxs
= NULL
, *ctx
;
307 block_job_pause_all();
309 for (bs
= bdrv_first(&it
); bs
; bs
= bdrv_next(&it
)) {
310 AioContext
*aio_context
= bdrv_get_aio_context(bs
);
312 aio_context_acquire(aio_context
);
313 bdrv_parent_drained_begin(bs
);
314 aio_disable_external(aio_context
);
315 aio_context_release(aio_context
);
317 if (!g_slist_find(aio_ctxs
, aio_context
)) {
318 aio_ctxs
= g_slist_prepend(aio_ctxs
, aio_context
);
322 /* Note that completion of an asynchronous I/O operation can trigger any
323 * number of other I/O operations on other devices---for example a
324 * coroutine can submit an I/O request to another device in response to
325 * request completion. Therefore we must keep looping until there was no
326 * more activity rather than simply draining each device independently.
331 for (ctx
= aio_ctxs
; ctx
!= NULL
; ctx
= ctx
->next
) {
332 AioContext
*aio_context
= ctx
->data
;
334 aio_context_acquire(aio_context
);
335 for (bs
= bdrv_first(&it
); bs
; bs
= bdrv_next(&it
)) {
336 if (aio_context
== bdrv_get_aio_context(bs
)) {
337 waited
|= bdrv_drain_recurse(bs
);
340 aio_context_release(aio_context
);
344 g_slist_free(aio_ctxs
);
347 void bdrv_drain_all_end(void)
349 BlockDriverState
*bs
;
352 for (bs
= bdrv_first(&it
); bs
; bs
= bdrv_next(&it
)) {
353 AioContext
*aio_context
= bdrv_get_aio_context(bs
);
355 aio_context_acquire(aio_context
);
356 aio_enable_external(aio_context
);
357 bdrv_parent_drained_end(bs
);
358 aio_context_release(aio_context
);
361 block_job_resume_all();
364 void bdrv_drain_all(void)
366 bdrv_drain_all_begin();
367 bdrv_drain_all_end();
371 * Remove an active request from the tracked requests list
373 * This function should be called when a tracked request is completing.
375 static void tracked_request_end(BdrvTrackedRequest
*req
)
377 if (req
->serialising
) {
378 req
->bs
->serialising_in_flight
--;
381 QLIST_REMOVE(req
, list
);
382 qemu_co_queue_restart_all(&req
->wait_queue
);
386 * Add an active request to the tracked requests list
388 static void tracked_request_begin(BdrvTrackedRequest
*req
,
389 BlockDriverState
*bs
,
392 enum BdrvTrackedRequestType type
)
394 *req
= (BdrvTrackedRequest
){
399 .co
= qemu_coroutine_self(),
400 .serialising
= false,
401 .overlap_offset
= offset
,
402 .overlap_bytes
= bytes
,
405 qemu_co_queue_init(&req
->wait_queue
);
407 QLIST_INSERT_HEAD(&bs
->tracked_requests
, req
, list
);
410 static void mark_request_serialising(BdrvTrackedRequest
*req
, uint64_t align
)
412 int64_t overlap_offset
= req
->offset
& ~(align
- 1);
413 unsigned int overlap_bytes
= ROUND_UP(req
->offset
+ req
->bytes
, align
)
416 if (!req
->serialising
) {
417 req
->bs
->serialising_in_flight
++;
418 req
->serialising
= true;
421 req
->overlap_offset
= MIN(req
->overlap_offset
, overlap_offset
);
422 req
->overlap_bytes
= MAX(req
->overlap_bytes
, overlap_bytes
);
426 * Round a region to cluster boundaries (sector-based)
428 void bdrv_round_sectors_to_clusters(BlockDriverState
*bs
,
429 int64_t sector_num
, int nb_sectors
,
430 int64_t *cluster_sector_num
,
431 int *cluster_nb_sectors
)
435 if (bdrv_get_info(bs
, &bdi
) < 0 || bdi
.cluster_size
== 0) {
436 *cluster_sector_num
= sector_num
;
437 *cluster_nb_sectors
= nb_sectors
;
439 int64_t c
= bdi
.cluster_size
/ BDRV_SECTOR_SIZE
;
440 *cluster_sector_num
= QEMU_ALIGN_DOWN(sector_num
, c
);
441 *cluster_nb_sectors
= QEMU_ALIGN_UP(sector_num
- *cluster_sector_num
+
447 * Round a region to cluster boundaries
449 void bdrv_round_to_clusters(BlockDriverState
*bs
,
450 int64_t offset
, unsigned int bytes
,
451 int64_t *cluster_offset
,
452 unsigned int *cluster_bytes
)
456 if (bdrv_get_info(bs
, &bdi
) < 0 || bdi
.cluster_size
== 0) {
457 *cluster_offset
= offset
;
458 *cluster_bytes
= bytes
;
460 int64_t c
= bdi
.cluster_size
;
461 *cluster_offset
= QEMU_ALIGN_DOWN(offset
, c
);
462 *cluster_bytes
= QEMU_ALIGN_UP(offset
- *cluster_offset
+ bytes
, c
);
466 static int bdrv_get_cluster_size(BlockDriverState
*bs
)
471 ret
= bdrv_get_info(bs
, &bdi
);
472 if (ret
< 0 || bdi
.cluster_size
== 0) {
473 return bs
->bl
.request_alignment
;
475 return bdi
.cluster_size
;
479 static bool tracked_request_overlaps(BdrvTrackedRequest
*req
,
480 int64_t offset
, unsigned int bytes
)
483 if (offset
>= req
->overlap_offset
+ req
->overlap_bytes
) {
487 if (req
->overlap_offset
>= offset
+ bytes
) {
493 void bdrv_inc_in_flight(BlockDriverState
*bs
)
495 atomic_inc(&bs
->in_flight
);
498 static void dummy_bh_cb(void *opaque
)
502 void bdrv_wakeup(BlockDriverState
*bs
)
505 aio_bh_schedule_oneshot(qemu_get_aio_context(), dummy_bh_cb
, NULL
);
509 void bdrv_dec_in_flight(BlockDriverState
*bs
)
511 atomic_dec(&bs
->in_flight
);
515 static bool coroutine_fn
wait_serialising_requests(BdrvTrackedRequest
*self
)
517 BlockDriverState
*bs
= self
->bs
;
518 BdrvTrackedRequest
*req
;
522 if (!bs
->serialising_in_flight
) {
528 QLIST_FOREACH(req
, &bs
->tracked_requests
, list
) {
529 if (req
== self
|| (!req
->serialising
&& !self
->serialising
)) {
532 if (tracked_request_overlaps(req
, self
->overlap_offset
,
533 self
->overlap_bytes
))
535 /* Hitting this means there was a reentrant request, for
536 * example, a block driver issuing nested requests. This must
537 * never happen since it means deadlock.
539 assert(qemu_coroutine_self() != req
->co
);
541 /* If the request is already (indirectly) waiting for us, or
542 * will wait for us as soon as it wakes up, then just go on
543 * (instead of producing a deadlock in the former case). */
544 if (!req
->waiting_for
) {
545 self
->waiting_for
= req
;
546 qemu_co_queue_wait(&req
->wait_queue
, NULL
);
547 self
->waiting_for
= NULL
;
559 static int bdrv_check_byte_request(BlockDriverState
*bs
, int64_t offset
,
562 if (size
> BDRV_REQUEST_MAX_SECTORS
<< BDRV_SECTOR_BITS
) {
566 if (!bdrv_is_inserted(bs
)) {
577 typedef struct RwCo
{
583 BdrvRequestFlags flags
;
586 static void coroutine_fn
bdrv_rw_co_entry(void *opaque
)
590 if (!rwco
->is_write
) {
591 rwco
->ret
= bdrv_co_preadv(rwco
->child
, rwco
->offset
,
592 rwco
->qiov
->size
, rwco
->qiov
,
595 rwco
->ret
= bdrv_co_pwritev(rwco
->child
, rwco
->offset
,
596 rwco
->qiov
->size
, rwco
->qiov
,
602 * Process a vectored synchronous request using coroutines
604 static int bdrv_prwv_co(BdrvChild
*child
, int64_t offset
,
605 QEMUIOVector
*qiov
, bool is_write
,
606 BdrvRequestFlags flags
)
613 .is_write
= is_write
,
618 if (qemu_in_coroutine()) {
619 /* Fast-path if already in coroutine context */
620 bdrv_rw_co_entry(&rwco
);
622 co
= qemu_coroutine_create(bdrv_rw_co_entry
, &rwco
);
623 bdrv_coroutine_enter(child
->bs
, co
);
624 BDRV_POLL_WHILE(child
->bs
, rwco
.ret
== NOT_DONE
);
630 * Process a synchronous request using coroutines
632 static int bdrv_rw_co(BdrvChild
*child
, int64_t sector_num
, uint8_t *buf
,
633 int nb_sectors
, bool is_write
, BdrvRequestFlags flags
)
637 .iov_base
= (void *)buf
,
638 .iov_len
= nb_sectors
* BDRV_SECTOR_SIZE
,
641 if (nb_sectors
< 0 || nb_sectors
> BDRV_REQUEST_MAX_SECTORS
) {
645 qemu_iovec_init_external(&qiov
, &iov
, 1);
646 return bdrv_prwv_co(child
, sector_num
<< BDRV_SECTOR_BITS
,
647 &qiov
, is_write
, flags
);
650 /* return < 0 if error. See bdrv_write() for the return codes */
651 int bdrv_read(BdrvChild
*child
, int64_t sector_num
,
652 uint8_t *buf
, int nb_sectors
)
654 return bdrv_rw_co(child
, sector_num
, buf
, nb_sectors
, false, 0);
657 /* Return < 0 if error. Important errors are:
658 -EIO generic I/O error (may happen for all errors)
659 -ENOMEDIUM No media inserted.
660 -EINVAL Invalid sector number or nb_sectors
661 -EACCES Trying to write a read-only device
663 int bdrv_write(BdrvChild
*child
, int64_t sector_num
,
664 const uint8_t *buf
, int nb_sectors
)
666 return bdrv_rw_co(child
, sector_num
, (uint8_t *)buf
, nb_sectors
, true, 0);
669 int bdrv_pwrite_zeroes(BdrvChild
*child
, int64_t offset
,
670 int count
, BdrvRequestFlags flags
)
678 qemu_iovec_init_external(&qiov
, &iov
, 1);
679 return bdrv_prwv_co(child
, offset
, &qiov
, true,
680 BDRV_REQ_ZERO_WRITE
| flags
);
684 * Completely zero out a block device with the help of bdrv_pwrite_zeroes.
685 * The operation is sped up by checking the block status and only writing
686 * zeroes to the device if they currently do not return zeroes. Optional
687 * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
690 * Returns < 0 on error, 0 on success. For error codes see bdrv_write().
692 int bdrv_make_zero(BdrvChild
*child
, BdrvRequestFlags flags
)
694 int64_t target_sectors
, ret
, nb_sectors
, sector_num
= 0;
695 BlockDriverState
*bs
= child
->bs
;
696 BlockDriverState
*file
;
699 target_sectors
= bdrv_nb_sectors(bs
);
700 if (target_sectors
< 0) {
701 return target_sectors
;
705 nb_sectors
= MIN(target_sectors
- sector_num
, BDRV_REQUEST_MAX_SECTORS
);
706 if (nb_sectors
<= 0) {
709 ret
= bdrv_get_block_status(bs
, sector_num
, nb_sectors
, &n
, &file
);
711 error_report("error getting block status at sector %" PRId64
": %s",
712 sector_num
, strerror(-ret
));
715 if (ret
& BDRV_BLOCK_ZERO
) {
719 ret
= bdrv_pwrite_zeroes(child
, sector_num
<< BDRV_SECTOR_BITS
,
720 n
<< BDRV_SECTOR_BITS
, flags
);
722 error_report("error writing zeroes at sector %" PRId64
": %s",
723 sector_num
, strerror(-ret
));
730 int bdrv_preadv(BdrvChild
*child
, int64_t offset
, QEMUIOVector
*qiov
)
734 ret
= bdrv_prwv_co(child
, offset
, qiov
, false, 0);
742 int bdrv_pread(BdrvChild
*child
, int64_t offset
, void *buf
, int bytes
)
746 .iov_base
= (void *)buf
,
754 qemu_iovec_init_external(&qiov
, &iov
, 1);
755 return bdrv_preadv(child
, offset
, &qiov
);
758 int bdrv_pwritev(BdrvChild
*child
, int64_t offset
, QEMUIOVector
*qiov
)
762 ret
= bdrv_prwv_co(child
, offset
, qiov
, true, 0);
770 int bdrv_pwrite(BdrvChild
*child
, int64_t offset
, const void *buf
, int bytes
)
774 .iov_base
= (void *) buf
,
782 qemu_iovec_init_external(&qiov
, &iov
, 1);
783 return bdrv_pwritev(child
, offset
, &qiov
);
787 * Writes to the file and ensures that no writes are reordered across this
788 * request (acts as a barrier)
790 * Returns 0 on success, -errno in error cases.
792 int bdrv_pwrite_sync(BdrvChild
*child
, int64_t offset
,
793 const void *buf
, int count
)
797 ret
= bdrv_pwrite(child
, offset
, buf
, count
);
802 ret
= bdrv_flush(child
->bs
);
810 typedef struct CoroutineIOCompletion
{
811 Coroutine
*coroutine
;
813 } CoroutineIOCompletion
;
815 static void bdrv_co_io_em_complete(void *opaque
, int ret
)
817 CoroutineIOCompletion
*co
= opaque
;
820 aio_co_wake(co
->coroutine
);
823 static int coroutine_fn
bdrv_driver_preadv(BlockDriverState
*bs
,
824 uint64_t offset
, uint64_t bytes
,
825 QEMUIOVector
*qiov
, int flags
)
827 BlockDriver
*drv
= bs
->drv
;
829 unsigned int nb_sectors
;
831 assert(!(flags
& ~BDRV_REQ_MASK
));
833 if (drv
->bdrv_co_preadv
) {
834 return drv
->bdrv_co_preadv(bs
, offset
, bytes
, qiov
, flags
);
837 sector_num
= offset
>> BDRV_SECTOR_BITS
;
838 nb_sectors
= bytes
>> BDRV_SECTOR_BITS
;
840 assert((offset
& (BDRV_SECTOR_SIZE
- 1)) == 0);
841 assert((bytes
& (BDRV_SECTOR_SIZE
- 1)) == 0);
842 assert((bytes
>> BDRV_SECTOR_BITS
) <= BDRV_REQUEST_MAX_SECTORS
);
844 if (drv
->bdrv_co_readv
) {
845 return drv
->bdrv_co_readv(bs
, sector_num
, nb_sectors
, qiov
);
848 CoroutineIOCompletion co
= {
849 .coroutine
= qemu_coroutine_self(),
852 acb
= bs
->drv
->bdrv_aio_readv(bs
, sector_num
, qiov
, nb_sectors
,
853 bdrv_co_io_em_complete
, &co
);
857 qemu_coroutine_yield();
863 static int coroutine_fn
bdrv_driver_pwritev(BlockDriverState
*bs
,
864 uint64_t offset
, uint64_t bytes
,
865 QEMUIOVector
*qiov
, int flags
)
867 BlockDriver
*drv
= bs
->drv
;
869 unsigned int nb_sectors
;
872 assert(!(flags
& ~BDRV_REQ_MASK
));
874 if (drv
->bdrv_co_pwritev
) {
875 ret
= drv
->bdrv_co_pwritev(bs
, offset
, bytes
, qiov
,
876 flags
& bs
->supported_write_flags
);
877 flags
&= ~bs
->supported_write_flags
;
881 sector_num
= offset
>> BDRV_SECTOR_BITS
;
882 nb_sectors
= bytes
>> BDRV_SECTOR_BITS
;
884 assert((offset
& (BDRV_SECTOR_SIZE
- 1)) == 0);
885 assert((bytes
& (BDRV_SECTOR_SIZE
- 1)) == 0);
886 assert((bytes
>> BDRV_SECTOR_BITS
) <= BDRV_REQUEST_MAX_SECTORS
);
888 if (drv
->bdrv_co_writev_flags
) {
889 ret
= drv
->bdrv_co_writev_flags(bs
, sector_num
, nb_sectors
, qiov
,
890 flags
& bs
->supported_write_flags
);
891 flags
&= ~bs
->supported_write_flags
;
892 } else if (drv
->bdrv_co_writev
) {
893 assert(!bs
->supported_write_flags
);
894 ret
= drv
->bdrv_co_writev(bs
, sector_num
, nb_sectors
, qiov
);
897 CoroutineIOCompletion co
= {
898 .coroutine
= qemu_coroutine_self(),
901 acb
= bs
->drv
->bdrv_aio_writev(bs
, sector_num
, qiov
, nb_sectors
,
902 bdrv_co_io_em_complete
, &co
);
906 qemu_coroutine_yield();
912 if (ret
== 0 && (flags
& BDRV_REQ_FUA
)) {
913 ret
= bdrv_co_flush(bs
);
919 static int coroutine_fn
920 bdrv_driver_pwritev_compressed(BlockDriverState
*bs
, uint64_t offset
,
921 uint64_t bytes
, QEMUIOVector
*qiov
)
923 BlockDriver
*drv
= bs
->drv
;
925 if (!drv
->bdrv_co_pwritev_compressed
) {
929 return drv
->bdrv_co_pwritev_compressed(bs
, offset
, bytes
, qiov
);
932 static int coroutine_fn
bdrv_co_do_copy_on_readv(BdrvChild
*child
,
933 int64_t offset
, unsigned int bytes
, QEMUIOVector
*qiov
)
935 BlockDriverState
*bs
= child
->bs
;
937 /* Perform I/O through a temporary buffer so that users who scribble over
938 * their read buffer while the operation is in progress do not end up
939 * modifying the image file. This is critical for zero-copy guest I/O
940 * where anything might happen inside guest memory.
944 BlockDriver
*drv
= bs
->drv
;
946 QEMUIOVector bounce_qiov
;
947 int64_t cluster_offset
;
948 unsigned int cluster_bytes
;
952 /* FIXME We cannot require callers to have write permissions when all they
953 * are doing is a read request. If we did things right, write permissions
954 * would be obtained anyway, but internally by the copy-on-read code. As
955 * long as it is implemented here rather than in a separat filter driver,
956 * the copy-on-read code doesn't have its own BdrvChild, however, for which
957 * it could request permissions. Therefore we have to bypass the permission
958 * system for the moment. */
959 // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
961 /* Cover entire cluster so no additional backing file I/O is required when
962 * allocating cluster in the image file.
964 bdrv_round_to_clusters(bs
, offset
, bytes
, &cluster_offset
, &cluster_bytes
);
966 trace_bdrv_co_do_copy_on_readv(bs
, offset
, bytes
,
967 cluster_offset
, cluster_bytes
);
969 iov
.iov_len
= cluster_bytes
;
970 iov
.iov_base
= bounce_buffer
= qemu_try_blockalign(bs
, iov
.iov_len
);
971 if (bounce_buffer
== NULL
) {
976 qemu_iovec_init_external(&bounce_qiov
, &iov
, 1);
978 ret
= bdrv_driver_preadv(bs
, cluster_offset
, cluster_bytes
,
984 if (drv
->bdrv_co_pwrite_zeroes
&&
985 buffer_is_zero(bounce_buffer
, iov
.iov_len
)) {
986 /* FIXME: Should we (perhaps conditionally) be setting
987 * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy
988 * that still correctly reads as zero? */
989 ret
= bdrv_co_do_pwrite_zeroes(bs
, cluster_offset
, cluster_bytes
, 0);
991 /* This does not change the data on the disk, it is not necessary
992 * to flush even in cache=writethrough mode.
994 ret
= bdrv_driver_pwritev(bs
, cluster_offset
, cluster_bytes
,
999 /* It might be okay to ignore write errors for guest requests. If this
1000 * is a deliberate copy-on-read then we don't want to ignore the error.
1001 * Simply report it in all cases.
1006 skip_bytes
= offset
- cluster_offset
;
1007 qemu_iovec_from_buf(qiov
, 0, bounce_buffer
+ skip_bytes
, bytes
);
1010 qemu_vfree(bounce_buffer
);
1015 * Forwards an already correctly aligned request to the BlockDriver. This
1016 * handles copy on read, zeroing after EOF, and fragmentation of large
1017 * reads; any other features must be implemented by the caller.
1019 static int coroutine_fn
bdrv_aligned_preadv(BdrvChild
*child
,
1020 BdrvTrackedRequest
*req
, int64_t offset
, unsigned int bytes
,
1021 int64_t align
, QEMUIOVector
*qiov
, int flags
)
1023 BlockDriverState
*bs
= child
->bs
;
1024 int64_t total_bytes
, max_bytes
;
1026 uint64_t bytes_remaining
= bytes
;
1029 assert(is_power_of_2(align
));
1030 assert((offset
& (align
- 1)) == 0);
1031 assert((bytes
& (align
- 1)) == 0);
1032 assert(!qiov
|| bytes
== qiov
->size
);
1033 assert((bs
->open_flags
& BDRV_O_NO_IO
) == 0);
1034 max_transfer
= QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs
->bl
.max_transfer
, INT_MAX
),
1037 /* TODO: We would need a per-BDS .supported_read_flags and
1038 * potential fallback support, if we ever implement any read flags
1039 * to pass through to drivers. For now, there aren't any
1040 * passthrough flags. */
1041 assert(!(flags
& ~(BDRV_REQ_NO_SERIALISING
| BDRV_REQ_COPY_ON_READ
)));
1043 /* Handle Copy on Read and associated serialisation */
1044 if (flags
& BDRV_REQ_COPY_ON_READ
) {
1045 /* If we touch the same cluster it counts as an overlap. This
1046 * guarantees that allocating writes will be serialized and not race
1047 * with each other for the same cluster. For example, in copy-on-read
1048 * it ensures that the CoR read and write operations are atomic and
1049 * guest writes cannot interleave between them. */
1050 mark_request_serialising(req
, bdrv_get_cluster_size(bs
));
1053 if (!(flags
& BDRV_REQ_NO_SERIALISING
)) {
1054 wait_serialising_requests(req
);
1057 if (flags
& BDRV_REQ_COPY_ON_READ
) {
1058 int64_t start_sector
= offset
>> BDRV_SECTOR_BITS
;
1059 int64_t end_sector
= DIV_ROUND_UP(offset
+ bytes
, BDRV_SECTOR_SIZE
);
1060 unsigned int nb_sectors
= end_sector
- start_sector
;
1063 ret
= bdrv_is_allocated(bs
, start_sector
, nb_sectors
, &pnum
);
1068 if (!ret
|| pnum
!= nb_sectors
) {
1069 ret
= bdrv_co_do_copy_on_readv(child
, offset
, bytes
, qiov
);
1074 /* Forward the request to the BlockDriver, possibly fragmenting it */
1075 total_bytes
= bdrv_getlength(bs
);
1076 if (total_bytes
< 0) {
1081 max_bytes
= ROUND_UP(MAX(0, total_bytes
- offset
), align
);
1082 if (bytes
<= max_bytes
&& bytes
<= max_transfer
) {
1083 ret
= bdrv_driver_preadv(bs
, offset
, bytes
, qiov
, 0);
1087 while (bytes_remaining
) {
1091 QEMUIOVector local_qiov
;
1093 num
= MIN(bytes_remaining
, MIN(max_bytes
, max_transfer
));
1095 qemu_iovec_init(&local_qiov
, qiov
->niov
);
1096 qemu_iovec_concat(&local_qiov
, qiov
, bytes
- bytes_remaining
, num
);
1098 ret
= bdrv_driver_preadv(bs
, offset
+ bytes
- bytes_remaining
,
1099 num
, &local_qiov
, 0);
1101 qemu_iovec_destroy(&local_qiov
);
1103 num
= bytes_remaining
;
1104 ret
= qemu_iovec_memset(qiov
, bytes
- bytes_remaining
, 0,
1110 bytes_remaining
-= num
;
1114 return ret
< 0 ? ret
: 0;
1118 * Handle a read request in coroutine context
1120 int coroutine_fn
bdrv_co_preadv(BdrvChild
*child
,
1121 int64_t offset
, unsigned int bytes
, QEMUIOVector
*qiov
,
1122 BdrvRequestFlags flags
)
1124 BlockDriverState
*bs
= child
->bs
;
1125 BlockDriver
*drv
= bs
->drv
;
1126 BdrvTrackedRequest req
;
1128 uint64_t align
= bs
->bl
.request_alignment
;
1129 uint8_t *head_buf
= NULL
;
1130 uint8_t *tail_buf
= NULL
;
1131 QEMUIOVector local_qiov
;
1132 bool use_local_qiov
= false;
1139 ret
= bdrv_check_byte_request(bs
, offset
, bytes
);
1144 bdrv_inc_in_flight(bs
);
1146 /* Don't do copy-on-read if we read data before write operation */
1147 if (bs
->copy_on_read
&& !(flags
& BDRV_REQ_NO_SERIALISING
)) {
1148 flags
|= BDRV_REQ_COPY_ON_READ
;
1151 /* Align read if necessary by padding qiov */
1152 if (offset
& (align
- 1)) {
1153 head_buf
= qemu_blockalign(bs
, align
);
1154 qemu_iovec_init(&local_qiov
, qiov
->niov
+ 2);
1155 qemu_iovec_add(&local_qiov
, head_buf
, offset
& (align
- 1));
1156 qemu_iovec_concat(&local_qiov
, qiov
, 0, qiov
->size
);
1157 use_local_qiov
= true;
1159 bytes
+= offset
& (align
- 1);
1160 offset
= offset
& ~(align
- 1);
1163 if ((offset
+ bytes
) & (align
- 1)) {
1164 if (!use_local_qiov
) {
1165 qemu_iovec_init(&local_qiov
, qiov
->niov
+ 1);
1166 qemu_iovec_concat(&local_qiov
, qiov
, 0, qiov
->size
);
1167 use_local_qiov
= true;
1169 tail_buf
= qemu_blockalign(bs
, align
);
1170 qemu_iovec_add(&local_qiov
, tail_buf
,
1171 align
- ((offset
+ bytes
) & (align
- 1)));
1173 bytes
= ROUND_UP(bytes
, align
);
1176 tracked_request_begin(&req
, bs
, offset
, bytes
, BDRV_TRACKED_READ
);
1177 ret
= bdrv_aligned_preadv(child
, &req
, offset
, bytes
, align
,
1178 use_local_qiov
? &local_qiov
: qiov
,
1180 tracked_request_end(&req
);
1181 bdrv_dec_in_flight(bs
);
1183 if (use_local_qiov
) {
1184 qemu_iovec_destroy(&local_qiov
);
1185 qemu_vfree(head_buf
);
1186 qemu_vfree(tail_buf
);
1192 static int coroutine_fn
bdrv_co_do_readv(BdrvChild
*child
,
1193 int64_t sector_num
, int nb_sectors
, QEMUIOVector
*qiov
,
1194 BdrvRequestFlags flags
)
1196 if (nb_sectors
< 0 || nb_sectors
> BDRV_REQUEST_MAX_SECTORS
) {
1200 return bdrv_co_preadv(child
, sector_num
<< BDRV_SECTOR_BITS
,
1201 nb_sectors
<< BDRV_SECTOR_BITS
, qiov
, flags
);
1204 int coroutine_fn
bdrv_co_readv(BdrvChild
*child
, int64_t sector_num
,
1205 int nb_sectors
, QEMUIOVector
*qiov
)
1207 trace_bdrv_co_readv(child
->bs
, sector_num
, nb_sectors
);
1209 return bdrv_co_do_readv(child
, sector_num
, nb_sectors
, qiov
, 0);
1212 /* Maximum buffer for write zeroes fallback, in bytes */
1213 #define MAX_WRITE_ZEROES_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS)
1215 static int coroutine_fn
bdrv_co_do_pwrite_zeroes(BlockDriverState
*bs
,
1216 int64_t offset
, int count
, BdrvRequestFlags flags
)
1218 BlockDriver
*drv
= bs
->drv
;
1220 struct iovec iov
= {0};
1222 bool need_flush
= false;
1226 int max_write_zeroes
= MIN_NON_ZERO(bs
->bl
.max_pwrite_zeroes
, INT_MAX
);
1227 int alignment
= MAX(bs
->bl
.pwrite_zeroes_alignment
,
1228 bs
->bl
.request_alignment
);
1229 int max_transfer
= MIN_NON_ZERO(bs
->bl
.max_transfer
,
1230 MAX_WRITE_ZEROES_BOUNCE_BUFFER
);
1232 assert(alignment
% bs
->bl
.request_alignment
== 0);
1233 head
= offset
% alignment
;
1234 tail
= (offset
+ count
) % alignment
;
1235 max_write_zeroes
= QEMU_ALIGN_DOWN(max_write_zeroes
, alignment
);
1236 assert(max_write_zeroes
>= bs
->bl
.request_alignment
);
1238 while (count
> 0 && !ret
) {
1241 /* Align request. Block drivers can expect the "bulk" of the request
1242 * to be aligned, and that unaligned requests do not cross cluster
1246 /* Make a small request up to the first aligned sector. For
1247 * convenience, limit this request to max_transfer even if
1248 * we don't need to fall back to writes. */
1249 num
= MIN(MIN(count
, max_transfer
), alignment
- head
);
1250 head
= (head
+ num
) % alignment
;
1251 assert(num
< max_write_zeroes
);
1252 } else if (tail
&& num
> alignment
) {
1253 /* Shorten the request to the last aligned sector. */
1257 /* limit request size */
1258 if (num
> max_write_zeroes
) {
1259 num
= max_write_zeroes
;
1263 /* First try the efficient write zeroes operation */
1264 if (drv
->bdrv_co_pwrite_zeroes
) {
1265 ret
= drv
->bdrv_co_pwrite_zeroes(bs
, offset
, num
,
1266 flags
& bs
->supported_zero_flags
);
1267 if (ret
!= -ENOTSUP
&& (flags
& BDRV_REQ_FUA
) &&
1268 !(bs
->supported_zero_flags
& BDRV_REQ_FUA
)) {
1272 assert(!bs
->supported_zero_flags
);
1275 if (ret
== -ENOTSUP
) {
1276 /* Fall back to bounce buffer if write zeroes is unsupported */
1277 BdrvRequestFlags write_flags
= flags
& ~BDRV_REQ_ZERO_WRITE
;
1279 if ((flags
& BDRV_REQ_FUA
) &&
1280 !(bs
->supported_write_flags
& BDRV_REQ_FUA
)) {
1281 /* No need for bdrv_driver_pwrite() to do a fallback
1282 * flush on each chunk; use just one at the end */
1283 write_flags
&= ~BDRV_REQ_FUA
;
1286 num
= MIN(num
, max_transfer
);
1288 if (iov
.iov_base
== NULL
) {
1289 iov
.iov_base
= qemu_try_blockalign(bs
, num
);
1290 if (iov
.iov_base
== NULL
) {
1294 memset(iov
.iov_base
, 0, num
);
1296 qemu_iovec_init_external(&qiov
, &iov
, 1);
1298 ret
= bdrv_driver_pwritev(bs
, offset
, num
, &qiov
, write_flags
);
1300 /* Keep bounce buffer around if it is big enough for all
1301 * all future requests.
1303 if (num
< max_transfer
) {
1304 qemu_vfree(iov
.iov_base
);
1305 iov
.iov_base
= NULL
;
1314 if (ret
== 0 && need_flush
) {
1315 ret
= bdrv_co_flush(bs
);
1317 qemu_vfree(iov
.iov_base
);
1322 * Forwards an already correctly aligned write request to the BlockDriver,
1323 * after possibly fragmenting it.
1325 static int coroutine_fn
bdrv_aligned_pwritev(BdrvChild
*child
,
1326 BdrvTrackedRequest
*req
, int64_t offset
, unsigned int bytes
,
1327 int64_t align
, QEMUIOVector
*qiov
, int flags
)
1329 BlockDriverState
*bs
= child
->bs
;
1330 BlockDriver
*drv
= bs
->drv
;
1334 int64_t start_sector
= offset
>> BDRV_SECTOR_BITS
;
1335 int64_t end_sector
= DIV_ROUND_UP(offset
+ bytes
, BDRV_SECTOR_SIZE
);
1336 uint64_t bytes_remaining
= bytes
;
1339 assert(is_power_of_2(align
));
1340 assert((offset
& (align
- 1)) == 0);
1341 assert((bytes
& (align
- 1)) == 0);
1342 assert(!qiov
|| bytes
== qiov
->size
);
1343 assert((bs
->open_flags
& BDRV_O_NO_IO
) == 0);
1344 assert(!(flags
& ~BDRV_REQ_MASK
));
1345 max_transfer
= QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs
->bl
.max_transfer
, INT_MAX
),
1348 waited
= wait_serialising_requests(req
);
1349 assert(!waited
|| !req
->serialising
);
1350 assert(req
->overlap_offset
<= offset
);
1351 assert(offset
+ bytes
<= req
->overlap_offset
+ req
->overlap_bytes
);
1352 assert(child
->perm
& BLK_PERM_WRITE
);
1353 assert(end_sector
<= bs
->total_sectors
|| child
->perm
& BLK_PERM_RESIZE
);
1355 ret
= notifier_with_return_list_notify(&bs
->before_write_notifiers
, req
);
1357 if (!ret
&& bs
->detect_zeroes
!= BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF
&&
1358 !(flags
& BDRV_REQ_ZERO_WRITE
) && drv
->bdrv_co_pwrite_zeroes
&&
1359 qemu_iovec_is_zero(qiov
)) {
1360 flags
|= BDRV_REQ_ZERO_WRITE
;
1361 if (bs
->detect_zeroes
== BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP
) {
1362 flags
|= BDRV_REQ_MAY_UNMAP
;
1367 /* Do nothing, write notifier decided to fail this request */
1368 } else if (flags
& BDRV_REQ_ZERO_WRITE
) {
1369 bdrv_debug_event(bs
, BLKDBG_PWRITEV_ZERO
);
1370 ret
= bdrv_co_do_pwrite_zeroes(bs
, offset
, bytes
, flags
);
1371 } else if (flags
& BDRV_REQ_WRITE_COMPRESSED
) {
1372 ret
= bdrv_driver_pwritev_compressed(bs
, offset
, bytes
, qiov
);
1373 } else if (bytes
<= max_transfer
) {
1374 bdrv_debug_event(bs
, BLKDBG_PWRITEV
);
1375 ret
= bdrv_driver_pwritev(bs
, offset
, bytes
, qiov
, flags
);
1377 bdrv_debug_event(bs
, BLKDBG_PWRITEV
);
1378 while (bytes_remaining
) {
1379 int num
= MIN(bytes_remaining
, max_transfer
);
1380 QEMUIOVector local_qiov
;
1381 int local_flags
= flags
;
1384 if (num
< bytes_remaining
&& (flags
& BDRV_REQ_FUA
) &&
1385 !(bs
->supported_write_flags
& BDRV_REQ_FUA
)) {
1386 /* If FUA is going to be emulated by flush, we only
1387 * need to flush on the last iteration */
1388 local_flags
&= ~BDRV_REQ_FUA
;
1390 qemu_iovec_init(&local_qiov
, qiov
->niov
);
1391 qemu_iovec_concat(&local_qiov
, qiov
, bytes
- bytes_remaining
, num
);
1393 ret
= bdrv_driver_pwritev(bs
, offset
+ bytes
- bytes_remaining
,
1394 num
, &local_qiov
, local_flags
);
1395 qemu_iovec_destroy(&local_qiov
);
1399 bytes_remaining
-= num
;
1402 bdrv_debug_event(bs
, BLKDBG_PWRITEV_DONE
);
1405 bdrv_set_dirty(bs
, start_sector
, end_sector
- start_sector
);
1407 if (bs
->wr_highest_offset
< offset
+ bytes
) {
1408 bs
->wr_highest_offset
= offset
+ bytes
;
1412 bs
->total_sectors
= MAX(bs
->total_sectors
, end_sector
);
1419 static int coroutine_fn
bdrv_co_do_zero_pwritev(BdrvChild
*child
,
1422 BdrvRequestFlags flags
,
1423 BdrvTrackedRequest
*req
)
1425 BlockDriverState
*bs
= child
->bs
;
1426 uint8_t *buf
= NULL
;
1427 QEMUIOVector local_qiov
;
1429 uint64_t align
= bs
->bl
.request_alignment
;
1430 unsigned int head_padding_bytes
, tail_padding_bytes
;
1433 head_padding_bytes
= offset
& (align
- 1);
1434 tail_padding_bytes
= (align
- (offset
+ bytes
)) & (align
- 1);
1437 assert(flags
& BDRV_REQ_ZERO_WRITE
);
1438 if (head_padding_bytes
|| tail_padding_bytes
) {
1439 buf
= qemu_blockalign(bs
, align
);
1440 iov
= (struct iovec
) {
1444 qemu_iovec_init_external(&local_qiov
, &iov
, 1);
1446 if (head_padding_bytes
) {
1447 uint64_t zero_bytes
= MIN(bytes
, align
- head_padding_bytes
);
1449 /* RMW the unaligned part before head. */
1450 mark_request_serialising(req
, align
);
1451 wait_serialising_requests(req
);
1452 bdrv_debug_event(bs
, BLKDBG_PWRITEV_RMW_HEAD
);
1453 ret
= bdrv_aligned_preadv(child
, req
, offset
& ~(align
- 1), align
,
1454 align
, &local_qiov
, 0);
1458 bdrv_debug_event(bs
, BLKDBG_PWRITEV_RMW_AFTER_HEAD
);
1460 memset(buf
+ head_padding_bytes
, 0, zero_bytes
);
1461 ret
= bdrv_aligned_pwritev(child
, req
, offset
& ~(align
- 1), align
,
1463 flags
& ~BDRV_REQ_ZERO_WRITE
);
1467 offset
+= zero_bytes
;
1468 bytes
-= zero_bytes
;
1471 assert(!bytes
|| (offset
& (align
- 1)) == 0);
1472 if (bytes
>= align
) {
1473 /* Write the aligned part in the middle. */
1474 uint64_t aligned_bytes
= bytes
& ~(align
- 1);
1475 ret
= bdrv_aligned_pwritev(child
, req
, offset
, aligned_bytes
, align
,
1480 bytes
-= aligned_bytes
;
1481 offset
+= aligned_bytes
;
1484 assert(!bytes
|| (offset
& (align
- 1)) == 0);
1486 assert(align
== tail_padding_bytes
+ bytes
);
1487 /* RMW the unaligned part after tail. */
1488 mark_request_serialising(req
, align
);
1489 wait_serialising_requests(req
);
1490 bdrv_debug_event(bs
, BLKDBG_PWRITEV_RMW_TAIL
);
1491 ret
= bdrv_aligned_preadv(child
, req
, offset
, align
,
1492 align
, &local_qiov
, 0);
1496 bdrv_debug_event(bs
, BLKDBG_PWRITEV_RMW_AFTER_TAIL
);
1498 memset(buf
, 0, bytes
);
1499 ret
= bdrv_aligned_pwritev(child
, req
, offset
, align
, align
,
1500 &local_qiov
, flags
& ~BDRV_REQ_ZERO_WRITE
);
1509 * Handle a write request in coroutine context
1511 int coroutine_fn
bdrv_co_pwritev(BdrvChild
*child
,
1512 int64_t offset
, unsigned int bytes
, QEMUIOVector
*qiov
,
1513 BdrvRequestFlags flags
)
1515 BlockDriverState
*bs
= child
->bs
;
1516 BdrvTrackedRequest req
;
1517 uint64_t align
= bs
->bl
.request_alignment
;
1518 uint8_t *head_buf
= NULL
;
1519 uint8_t *tail_buf
= NULL
;
1520 QEMUIOVector local_qiov
;
1521 bool use_local_qiov
= false;
1527 if (bs
->read_only
) {
1530 assert(!(bs
->open_flags
& BDRV_O_INACTIVE
));
1532 ret
= bdrv_check_byte_request(bs
, offset
, bytes
);
1537 bdrv_inc_in_flight(bs
);
1539 * Align write if necessary by performing a read-modify-write cycle.
1540 * Pad qiov with the read parts and be sure to have a tracked request not
1541 * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
1543 tracked_request_begin(&req
, bs
, offset
, bytes
, BDRV_TRACKED_WRITE
);
1546 ret
= bdrv_co_do_zero_pwritev(child
, offset
, bytes
, flags
, &req
);
1550 if (offset
& (align
- 1)) {
1551 QEMUIOVector head_qiov
;
1552 struct iovec head_iov
;
1554 mark_request_serialising(&req
, align
);
1555 wait_serialising_requests(&req
);
1557 head_buf
= qemu_blockalign(bs
, align
);
1558 head_iov
= (struct iovec
) {
1559 .iov_base
= head_buf
,
1562 qemu_iovec_init_external(&head_qiov
, &head_iov
, 1);
1564 bdrv_debug_event(bs
, BLKDBG_PWRITEV_RMW_HEAD
);
1565 ret
= bdrv_aligned_preadv(child
, &req
, offset
& ~(align
- 1), align
,
1566 align
, &head_qiov
, 0);
1570 bdrv_debug_event(bs
, BLKDBG_PWRITEV_RMW_AFTER_HEAD
);
1572 qemu_iovec_init(&local_qiov
, qiov
->niov
+ 2);
1573 qemu_iovec_add(&local_qiov
, head_buf
, offset
& (align
- 1));
1574 qemu_iovec_concat(&local_qiov
, qiov
, 0, qiov
->size
);
1575 use_local_qiov
= true;
1577 bytes
+= offset
& (align
- 1);
1578 offset
= offset
& ~(align
- 1);
1580 /* We have read the tail already if the request is smaller
1581 * than one aligned block.
1583 if (bytes
< align
) {
1584 qemu_iovec_add(&local_qiov
, head_buf
+ bytes
, align
- bytes
);
1589 if ((offset
+ bytes
) & (align
- 1)) {
1590 QEMUIOVector tail_qiov
;
1591 struct iovec tail_iov
;
1595 mark_request_serialising(&req
, align
);
1596 waited
= wait_serialising_requests(&req
);
1597 assert(!waited
|| !use_local_qiov
);
1599 tail_buf
= qemu_blockalign(bs
, align
);
1600 tail_iov
= (struct iovec
) {
1601 .iov_base
= tail_buf
,
1604 qemu_iovec_init_external(&tail_qiov
, &tail_iov
, 1);
1606 bdrv_debug_event(bs
, BLKDBG_PWRITEV_RMW_TAIL
);
1607 ret
= bdrv_aligned_preadv(child
, &req
, (offset
+ bytes
) & ~(align
- 1),
1608 align
, align
, &tail_qiov
, 0);
1612 bdrv_debug_event(bs
, BLKDBG_PWRITEV_RMW_AFTER_TAIL
);
1614 if (!use_local_qiov
) {
1615 qemu_iovec_init(&local_qiov
, qiov
->niov
+ 1);
1616 qemu_iovec_concat(&local_qiov
, qiov
, 0, qiov
->size
);
1617 use_local_qiov
= true;
1620 tail_bytes
= (offset
+ bytes
) & (align
- 1);
1621 qemu_iovec_add(&local_qiov
, tail_buf
+ tail_bytes
, align
- tail_bytes
);
1623 bytes
= ROUND_UP(bytes
, align
);
1626 ret
= bdrv_aligned_pwritev(child
, &req
, offset
, bytes
, align
,
1627 use_local_qiov
? &local_qiov
: qiov
,
1632 if (use_local_qiov
) {
1633 qemu_iovec_destroy(&local_qiov
);
1635 qemu_vfree(head_buf
);
1636 qemu_vfree(tail_buf
);
1638 tracked_request_end(&req
);
1639 bdrv_dec_in_flight(bs
);
1643 static int coroutine_fn
bdrv_co_do_writev(BdrvChild
*child
,
1644 int64_t sector_num
, int nb_sectors
, QEMUIOVector
*qiov
,
1645 BdrvRequestFlags flags
)
1647 if (nb_sectors
< 0 || nb_sectors
> BDRV_REQUEST_MAX_SECTORS
) {
1651 return bdrv_co_pwritev(child
, sector_num
<< BDRV_SECTOR_BITS
,
1652 nb_sectors
<< BDRV_SECTOR_BITS
, qiov
, flags
);
1655 int coroutine_fn
bdrv_co_writev(BdrvChild
*child
, int64_t sector_num
,
1656 int nb_sectors
, QEMUIOVector
*qiov
)
1658 trace_bdrv_co_writev(child
->bs
, sector_num
, nb_sectors
);
1660 return bdrv_co_do_writev(child
, sector_num
, nb_sectors
, qiov
, 0);
1663 int coroutine_fn
bdrv_co_pwrite_zeroes(BdrvChild
*child
, int64_t offset
,
1664 int count
, BdrvRequestFlags flags
)
1666 trace_bdrv_co_pwrite_zeroes(child
->bs
, offset
, count
, flags
);
1668 if (!(child
->bs
->open_flags
& BDRV_O_UNMAP
)) {
1669 flags
&= ~BDRV_REQ_MAY_UNMAP
;
1672 return bdrv_co_pwritev(child
, offset
, count
, NULL
,
1673 BDRV_REQ_ZERO_WRITE
| flags
);
1677 * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not.
1679 int bdrv_flush_all(void)
1681 BdrvNextIterator it
;
1682 BlockDriverState
*bs
= NULL
;
1685 for (bs
= bdrv_first(&it
); bs
; bs
= bdrv_next(&it
)) {
1686 AioContext
*aio_context
= bdrv_get_aio_context(bs
);
1689 aio_context_acquire(aio_context
);
1690 ret
= bdrv_flush(bs
);
1691 if (ret
< 0 && !result
) {
1694 aio_context_release(aio_context
);
1701 typedef struct BdrvCoGetBlockStatusData
{
1702 BlockDriverState
*bs
;
1703 BlockDriverState
*base
;
1704 BlockDriverState
**file
;
1710 } BdrvCoGetBlockStatusData
;
1713 * Returns the allocation status of the specified sectors.
1714 * Drivers not implementing the functionality are assumed to not support
1715 * backing files, hence all their sectors are reported as allocated.
1717 * If 'sector_num' is beyond the end of the disk image the return value is 0
1718 * and 'pnum' is set to 0.
1720 * 'pnum' is set to the number of sectors (including and immediately following
1721 * the specified sector) that are known to be in the same
1722 * allocated/unallocated state.
1724 * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes
1725 * beyond the end of the disk image it will be clamped.
1727 * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file'
1728 * points to the BDS which the sector range is allocated in.
1730 static int64_t coroutine_fn
bdrv_co_get_block_status(BlockDriverState
*bs
,
1732 int nb_sectors
, int *pnum
,
1733 BlockDriverState
**file
)
1735 int64_t total_sectors
;
1739 total_sectors
= bdrv_nb_sectors(bs
);
1740 if (total_sectors
< 0) {
1741 return total_sectors
;
1744 if (sector_num
>= total_sectors
) {
1749 n
= total_sectors
- sector_num
;
1750 if (n
< nb_sectors
) {
1754 if (!bs
->drv
->bdrv_co_get_block_status
) {
1756 ret
= BDRV_BLOCK_DATA
| BDRV_BLOCK_ALLOCATED
;
1757 if (bs
->drv
->protocol_name
) {
1758 ret
|= BDRV_BLOCK_OFFSET_VALID
| (sector_num
* BDRV_SECTOR_SIZE
);
1764 bdrv_inc_in_flight(bs
);
1765 ret
= bs
->drv
->bdrv_co_get_block_status(bs
, sector_num
, nb_sectors
, pnum
,
1772 if (ret
& BDRV_BLOCK_RAW
) {
1773 assert(ret
& BDRV_BLOCK_OFFSET_VALID
);
1774 ret
= bdrv_co_get_block_status(*file
, ret
>> BDRV_SECTOR_BITS
,
1779 if (ret
& (BDRV_BLOCK_DATA
| BDRV_BLOCK_ZERO
)) {
1780 ret
|= BDRV_BLOCK_ALLOCATED
;
1782 if (bdrv_unallocated_blocks_are_zero(bs
)) {
1783 ret
|= BDRV_BLOCK_ZERO
;
1784 } else if (bs
->backing
) {
1785 BlockDriverState
*bs2
= bs
->backing
->bs
;
1786 int64_t nb_sectors2
= bdrv_nb_sectors(bs2
);
1787 if (nb_sectors2
>= 0 && sector_num
>= nb_sectors2
) {
1788 ret
|= BDRV_BLOCK_ZERO
;
1793 if (*file
&& *file
!= bs
&&
1794 (ret
& BDRV_BLOCK_DATA
) && !(ret
& BDRV_BLOCK_ZERO
) &&
1795 (ret
& BDRV_BLOCK_OFFSET_VALID
)) {
1796 BlockDriverState
*file2
;
1799 ret2
= bdrv_co_get_block_status(*file
, ret
>> BDRV_SECTOR_BITS
,
1800 *pnum
, &file_pnum
, &file2
);
1802 /* Ignore errors. This is just providing extra information, it
1803 * is useful but not necessary.
1806 /* !file_pnum indicates an offset at or beyond the EOF; it is
1807 * perfectly valid for the format block driver to point to such
1808 * offsets, so catch it and mark everything as zero */
1809 ret
|= BDRV_BLOCK_ZERO
;
1811 /* Limit request to the range reported by the protocol driver */
1813 ret
|= (ret2
& BDRV_BLOCK_ZERO
);
1819 bdrv_dec_in_flight(bs
);
1823 static int64_t coroutine_fn
bdrv_co_get_block_status_above(BlockDriverState
*bs
,
1824 BlockDriverState
*base
,
1828 BlockDriverState
**file
)
1830 BlockDriverState
*p
;
1834 for (p
= bs
; p
!= base
; p
= backing_bs(p
)) {
1835 ret
= bdrv_co_get_block_status(p
, sector_num
, nb_sectors
, pnum
, file
);
1836 if (ret
< 0 || ret
& BDRV_BLOCK_ALLOCATED
) {
1839 /* [sector_num, pnum] unallocated on this layer, which could be only
1840 * the first part of [sector_num, nb_sectors]. */
1841 nb_sectors
= MIN(nb_sectors
, *pnum
);
1846 /* Coroutine wrapper for bdrv_get_block_status_above() */
1847 static void coroutine_fn
bdrv_get_block_status_above_co_entry(void *opaque
)
1849 BdrvCoGetBlockStatusData
*data
= opaque
;
1851 data
->ret
= bdrv_co_get_block_status_above(data
->bs
, data
->base
,
1860 * Synchronous wrapper around bdrv_co_get_block_status_above().
1862 * See bdrv_co_get_block_status_above() for details.
1864 int64_t bdrv_get_block_status_above(BlockDriverState
*bs
,
1865 BlockDriverState
*base
,
1867 int nb_sectors
, int *pnum
,
1868 BlockDriverState
**file
)
1871 BdrvCoGetBlockStatusData data
= {
1875 .sector_num
= sector_num
,
1876 .nb_sectors
= nb_sectors
,
1881 if (qemu_in_coroutine()) {
1882 /* Fast-path if already in coroutine context */
1883 bdrv_get_block_status_above_co_entry(&data
);
1885 co
= qemu_coroutine_create(bdrv_get_block_status_above_co_entry
,
1887 bdrv_coroutine_enter(bs
, co
);
1888 BDRV_POLL_WHILE(bs
, !data
.done
);
1893 int64_t bdrv_get_block_status(BlockDriverState
*bs
,
1895 int nb_sectors
, int *pnum
,
1896 BlockDriverState
**file
)
1898 return bdrv_get_block_status_above(bs
, backing_bs(bs
),
1899 sector_num
, nb_sectors
, pnum
, file
);
1902 int coroutine_fn
bdrv_is_allocated(BlockDriverState
*bs
, int64_t sector_num
,
1903 int nb_sectors
, int *pnum
)
1905 BlockDriverState
*file
;
1906 int64_t ret
= bdrv_get_block_status(bs
, sector_num
, nb_sectors
, pnum
,
1911 return !!(ret
& BDRV_BLOCK_ALLOCATED
);
1915 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
1917 * Return true if the given sector is allocated in any image between
1918 * BASE and TOP (inclusive). BASE can be NULL to check if the given
1919 * sector is allocated in any image of the chain. Return false otherwise.
1921 * 'pnum' is set to the number of sectors (including and immediately following
1922 * the specified sector) that are known to be in the same
1923 * allocated/unallocated state.
1926 int bdrv_is_allocated_above(BlockDriverState
*top
,
1927 BlockDriverState
*base
,
1929 int nb_sectors
, int *pnum
)
1931 BlockDriverState
*intermediate
;
1932 int ret
, n
= nb_sectors
;
1935 while (intermediate
&& intermediate
!= base
) {
1937 ret
= bdrv_is_allocated(intermediate
, sector_num
, nb_sectors
,
1947 * [sector_num, nb_sectors] is unallocated on top but intermediate
1950 * [sector_num+x, nr_sectors] allocated.
1952 if (n
> pnum_inter
&&
1953 (intermediate
== top
||
1954 sector_num
+ pnum_inter
< intermediate
->total_sectors
)) {
1958 intermediate
= backing_bs(intermediate
);
1965 typedef struct BdrvVmstateCo
{
1966 BlockDriverState
*bs
;
1973 static int coroutine_fn
1974 bdrv_co_rw_vmstate(BlockDriverState
*bs
, QEMUIOVector
*qiov
, int64_t pos
,
1977 BlockDriver
*drv
= bs
->drv
;
1981 } else if (drv
->bdrv_load_vmstate
) {
1982 return is_read
? drv
->bdrv_load_vmstate(bs
, qiov
, pos
)
1983 : drv
->bdrv_save_vmstate(bs
, qiov
, pos
);
1984 } else if (bs
->file
) {
1985 return bdrv_co_rw_vmstate(bs
->file
->bs
, qiov
, pos
, is_read
);
1991 static void coroutine_fn
bdrv_co_rw_vmstate_entry(void *opaque
)
1993 BdrvVmstateCo
*co
= opaque
;
1994 co
->ret
= bdrv_co_rw_vmstate(co
->bs
, co
->qiov
, co
->pos
, co
->is_read
);
1998 bdrv_rw_vmstate(BlockDriverState
*bs
, QEMUIOVector
*qiov
, int64_t pos
,
2001 if (qemu_in_coroutine()) {
2002 return bdrv_co_rw_vmstate(bs
, qiov
, pos
, is_read
);
2004 BdrvVmstateCo data
= {
2009 .ret
= -EINPROGRESS
,
2011 Coroutine
*co
= qemu_coroutine_create(bdrv_co_rw_vmstate_entry
, &data
);
2013 bdrv_coroutine_enter(bs
, co
);
2014 while (data
.ret
== -EINPROGRESS
) {
2015 aio_poll(bdrv_get_aio_context(bs
), true);
2021 int bdrv_save_vmstate(BlockDriverState
*bs
, const uint8_t *buf
,
2022 int64_t pos
, int size
)
2025 struct iovec iov
= {
2026 .iov_base
= (void *) buf
,
2031 qemu_iovec_init_external(&qiov
, &iov
, 1);
2033 ret
= bdrv_writev_vmstate(bs
, &qiov
, pos
);
2041 int bdrv_writev_vmstate(BlockDriverState
*bs
, QEMUIOVector
*qiov
, int64_t pos
)
2043 return bdrv_rw_vmstate(bs
, qiov
, pos
, false);
2046 int bdrv_load_vmstate(BlockDriverState
*bs
, uint8_t *buf
,
2047 int64_t pos
, int size
)
2050 struct iovec iov
= {
2056 qemu_iovec_init_external(&qiov
, &iov
, 1);
2057 ret
= bdrv_readv_vmstate(bs
, &qiov
, pos
);
2065 int bdrv_readv_vmstate(BlockDriverState
*bs
, QEMUIOVector
*qiov
, int64_t pos
)
2067 return bdrv_rw_vmstate(bs
, qiov
, pos
, true);
2070 /**************************************************************/
2073 BlockAIOCB
*bdrv_aio_readv(BdrvChild
*child
, int64_t sector_num
,
2074 QEMUIOVector
*qiov
, int nb_sectors
,
2075 BlockCompletionFunc
*cb
, void *opaque
)
2077 trace_bdrv_aio_readv(child
->bs
, sector_num
, nb_sectors
, opaque
);
2079 assert(nb_sectors
<< BDRV_SECTOR_BITS
== qiov
->size
);
2080 return bdrv_co_aio_prw_vector(child
, sector_num
<< BDRV_SECTOR_BITS
, qiov
,
2081 0, cb
, opaque
, false);
2084 BlockAIOCB
*bdrv_aio_writev(BdrvChild
*child
, int64_t sector_num
,
2085 QEMUIOVector
*qiov
, int nb_sectors
,
2086 BlockCompletionFunc
*cb
, void *opaque
)
2088 trace_bdrv_aio_writev(child
->bs
, sector_num
, nb_sectors
, opaque
);
2090 assert(nb_sectors
<< BDRV_SECTOR_BITS
== qiov
->size
);
2091 return bdrv_co_aio_prw_vector(child
, sector_num
<< BDRV_SECTOR_BITS
, qiov
,
2092 0, cb
, opaque
, true);
2095 void bdrv_aio_cancel(BlockAIOCB
*acb
)
2098 bdrv_aio_cancel_async(acb
);
2099 while (acb
->refcnt
> 1) {
2100 if (acb
->aiocb_info
->get_aio_context
) {
2101 aio_poll(acb
->aiocb_info
->get_aio_context(acb
), true);
2102 } else if (acb
->bs
) {
2103 /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so
2104 * assert that we're not using an I/O thread. Thread-safe
2105 * code should use bdrv_aio_cancel_async exclusively.
2107 assert(bdrv_get_aio_context(acb
->bs
) == qemu_get_aio_context());
2108 aio_poll(bdrv_get_aio_context(acb
->bs
), true);
2113 qemu_aio_unref(acb
);
2116 /* Async version of aio cancel. The caller is not blocked if the acb implements
2117 * cancel_async, otherwise we do nothing and let the request normally complete.
2118 * In either case the completion callback must be called. */
2119 void bdrv_aio_cancel_async(BlockAIOCB
*acb
)
2121 if (acb
->aiocb_info
->cancel_async
) {
2122 acb
->aiocb_info
->cancel_async(acb
);
2126 /**************************************************************/
2127 /* async block device emulation */
2129 typedef struct BlockRequest
{
2131 /* Used during read, write, trim */
2138 /* Used during ioctl */
2144 BlockCompletionFunc
*cb
;
2150 typedef struct BlockAIOCBCoroutine
{
2157 } BlockAIOCBCoroutine
;
2159 static const AIOCBInfo bdrv_em_co_aiocb_info
= {
2160 .aiocb_size
= sizeof(BlockAIOCBCoroutine
),
2163 static void bdrv_co_complete(BlockAIOCBCoroutine
*acb
)
2165 if (!acb
->need_bh
) {
2166 bdrv_dec_in_flight(acb
->common
.bs
);
2167 acb
->common
.cb(acb
->common
.opaque
, acb
->req
.error
);
2168 qemu_aio_unref(acb
);
2172 static void bdrv_co_em_bh(void *opaque
)
2174 BlockAIOCBCoroutine
*acb
= opaque
;
2176 assert(!acb
->need_bh
);
2177 bdrv_co_complete(acb
);
2180 static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine
*acb
)
2182 acb
->need_bh
= false;
2183 if (acb
->req
.error
!= -EINPROGRESS
) {
2184 BlockDriverState
*bs
= acb
->common
.bs
;
2186 aio_bh_schedule_oneshot(bdrv_get_aio_context(bs
), bdrv_co_em_bh
, acb
);
2190 /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */
2191 static void coroutine_fn
bdrv_co_do_rw(void *opaque
)
2193 BlockAIOCBCoroutine
*acb
= opaque
;
2195 if (!acb
->is_write
) {
2196 acb
->req
.error
= bdrv_co_preadv(acb
->child
, acb
->req
.offset
,
2197 acb
->req
.qiov
->size
, acb
->req
.qiov
, acb
->req
.flags
);
2199 acb
->req
.error
= bdrv_co_pwritev(acb
->child
, acb
->req
.offset
,
2200 acb
->req
.qiov
->size
, acb
->req
.qiov
, acb
->req
.flags
);
2203 bdrv_co_complete(acb
);
2206 static BlockAIOCB
*bdrv_co_aio_prw_vector(BdrvChild
*child
,
2209 BdrvRequestFlags flags
,
2210 BlockCompletionFunc
*cb
,
2215 BlockAIOCBCoroutine
*acb
;
2217 /* Matched by bdrv_co_complete's bdrv_dec_in_flight. */
2218 bdrv_inc_in_flight(child
->bs
);
2220 acb
= qemu_aio_get(&bdrv_em_co_aiocb_info
, child
->bs
, cb
, opaque
);
2222 acb
->need_bh
= true;
2223 acb
->req
.error
= -EINPROGRESS
;
2224 acb
->req
.offset
= offset
;
2225 acb
->req
.qiov
= qiov
;
2226 acb
->req
.flags
= flags
;
2227 acb
->is_write
= is_write
;
2229 co
= qemu_coroutine_create(bdrv_co_do_rw
, acb
);
2230 bdrv_coroutine_enter(child
->bs
, co
);
2232 bdrv_co_maybe_schedule_bh(acb
);
2233 return &acb
->common
;
2236 static void coroutine_fn
bdrv_aio_flush_co_entry(void *opaque
)
2238 BlockAIOCBCoroutine
*acb
= opaque
;
2239 BlockDriverState
*bs
= acb
->common
.bs
;
2241 acb
->req
.error
= bdrv_co_flush(bs
);
2242 bdrv_co_complete(acb
);
2245 BlockAIOCB
*bdrv_aio_flush(BlockDriverState
*bs
,
2246 BlockCompletionFunc
*cb
, void *opaque
)
2248 trace_bdrv_aio_flush(bs
, opaque
);
2251 BlockAIOCBCoroutine
*acb
;
2253 /* Matched by bdrv_co_complete's bdrv_dec_in_flight. */
2254 bdrv_inc_in_flight(bs
);
2256 acb
= qemu_aio_get(&bdrv_em_co_aiocb_info
, bs
, cb
, opaque
);
2257 acb
->need_bh
= true;
2258 acb
->req
.error
= -EINPROGRESS
;
2260 co
= qemu_coroutine_create(bdrv_aio_flush_co_entry
, acb
);
2261 bdrv_coroutine_enter(bs
, co
);
2263 bdrv_co_maybe_schedule_bh(acb
);
2264 return &acb
->common
;
2267 /**************************************************************/
2268 /* Coroutine block device emulation */
2270 typedef struct FlushCo
{
2271 BlockDriverState
*bs
;
2276 static void coroutine_fn
bdrv_flush_co_entry(void *opaque
)
2278 FlushCo
*rwco
= opaque
;
2280 rwco
->ret
= bdrv_co_flush(rwco
->bs
);
2283 int coroutine_fn
bdrv_co_flush(BlockDriverState
*bs
)
2288 bdrv_inc_in_flight(bs
);
2290 if (!bdrv_is_inserted(bs
) || bdrv_is_read_only(bs
) ||
2295 current_gen
= bs
->write_gen
;
2297 /* Wait until any previous flushes are completed */
2298 while (bs
->active_flush_req
) {
2299 qemu_co_queue_wait(&bs
->flush_queue
, NULL
);
2302 bs
->active_flush_req
= true;
2304 /* Write back all layers by calling one driver function */
2305 if (bs
->drv
->bdrv_co_flush
) {
2306 ret
= bs
->drv
->bdrv_co_flush(bs
);
2310 /* Write back cached data to the OS even with cache=unsafe */
2311 BLKDBG_EVENT(bs
->file
, BLKDBG_FLUSH_TO_OS
);
2312 if (bs
->drv
->bdrv_co_flush_to_os
) {
2313 ret
= bs
->drv
->bdrv_co_flush_to_os(bs
);
2319 /* But don't actually force it to the disk with cache=unsafe */
2320 if (bs
->open_flags
& BDRV_O_NO_FLUSH
) {
2324 /* Check if we really need to flush anything */
2325 if (bs
->flushed_gen
== current_gen
) {
2329 BLKDBG_EVENT(bs
->file
, BLKDBG_FLUSH_TO_DISK
);
2330 if (bs
->drv
->bdrv_co_flush_to_disk
) {
2331 ret
= bs
->drv
->bdrv_co_flush_to_disk(bs
);
2332 } else if (bs
->drv
->bdrv_aio_flush
) {
2334 CoroutineIOCompletion co
= {
2335 .coroutine
= qemu_coroutine_self(),
2338 acb
= bs
->drv
->bdrv_aio_flush(bs
, bdrv_co_io_em_complete
, &co
);
2342 qemu_coroutine_yield();
2347 * Some block drivers always operate in either writethrough or unsafe
2348 * mode and don't support bdrv_flush therefore. Usually qemu doesn't
2349 * know how the server works (because the behaviour is hardcoded or
2350 * depends on server-side configuration), so we can't ensure that
2351 * everything is safe on disk. Returning an error doesn't work because
2352 * that would break guests even if the server operates in writethrough
2355 * Let's hope the user knows what he's doing.
2364 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH
2365 * in the case of cache=unsafe, so there are no useless flushes.
2368 ret
= bs
->file
? bdrv_co_flush(bs
->file
->bs
) : 0;
2370 /* Notify any pending flushes that we have completed */
2372 bs
->flushed_gen
= current_gen
;
2374 bs
->active_flush_req
= false;
2375 /* Return value is ignored - it's ok if wait queue is empty */
2376 qemu_co_queue_next(&bs
->flush_queue
);
2379 bdrv_dec_in_flight(bs
);
2383 int bdrv_flush(BlockDriverState
*bs
)
2386 FlushCo flush_co
= {
2391 if (qemu_in_coroutine()) {
2392 /* Fast-path if already in coroutine context */
2393 bdrv_flush_co_entry(&flush_co
);
2395 co
= qemu_coroutine_create(bdrv_flush_co_entry
, &flush_co
);
2396 bdrv_coroutine_enter(bs
, co
);
2397 BDRV_POLL_WHILE(bs
, flush_co
.ret
== NOT_DONE
);
2400 return flush_co
.ret
;
2403 typedef struct DiscardCo
{
2404 BlockDriverState
*bs
;
2409 static void coroutine_fn
bdrv_pdiscard_co_entry(void *opaque
)
2411 DiscardCo
*rwco
= opaque
;
2413 rwco
->ret
= bdrv_co_pdiscard(rwco
->bs
, rwco
->offset
, rwco
->count
);
2416 int coroutine_fn
bdrv_co_pdiscard(BlockDriverState
*bs
, int64_t offset
,
2419 BdrvTrackedRequest req
;
2420 int max_pdiscard
, ret
;
2421 int head
, tail
, align
;
2427 ret
= bdrv_check_byte_request(bs
, offset
, count
);
2430 } else if (bs
->read_only
) {
2433 assert(!(bs
->open_flags
& BDRV_O_INACTIVE
));
2435 /* Do nothing if disabled. */
2436 if (!(bs
->open_flags
& BDRV_O_UNMAP
)) {
2440 if (!bs
->drv
->bdrv_co_pdiscard
&& !bs
->drv
->bdrv_aio_pdiscard
) {
2444 /* Discard is advisory, but some devices track and coalesce
2445 * unaligned requests, so we must pass everything down rather than
2446 * round here. Still, most devices will just silently ignore
2447 * unaligned requests (by returning -ENOTSUP), so we must fragment
2448 * the request accordingly. */
2449 align
= MAX(bs
->bl
.pdiscard_alignment
, bs
->bl
.request_alignment
);
2450 assert(align
% bs
->bl
.request_alignment
== 0);
2451 head
= offset
% align
;
2452 tail
= (offset
+ count
) % align
;
2454 bdrv_inc_in_flight(bs
);
2455 tracked_request_begin(&req
, bs
, offset
, count
, BDRV_TRACKED_DISCARD
);
2457 ret
= notifier_with_return_list_notify(&bs
->before_write_notifiers
, &req
);
2462 max_pdiscard
= QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs
->bl
.max_pdiscard
, INT_MAX
),
2464 assert(max_pdiscard
>= bs
->bl
.request_alignment
);
2471 /* Make small requests to get to alignment boundaries. */
2472 num
= MIN(count
, align
- head
);
2473 if (!QEMU_IS_ALIGNED(num
, bs
->bl
.request_alignment
)) {
2474 num
%= bs
->bl
.request_alignment
;
2476 head
= (head
+ num
) % align
;
2477 assert(num
< max_pdiscard
);
2480 /* Shorten the request to the last aligned cluster. */
2482 } else if (!QEMU_IS_ALIGNED(tail
, bs
->bl
.request_alignment
) &&
2483 tail
> bs
->bl
.request_alignment
) {
2484 tail
%= bs
->bl
.request_alignment
;
2488 /* limit request size */
2489 if (num
> max_pdiscard
) {
2493 if (bs
->drv
->bdrv_co_pdiscard
) {
2494 ret
= bs
->drv
->bdrv_co_pdiscard(bs
, offset
, num
);
2497 CoroutineIOCompletion co
= {
2498 .coroutine
= qemu_coroutine_self(),
2501 acb
= bs
->drv
->bdrv_aio_pdiscard(bs
, offset
, num
,
2502 bdrv_co_io_em_complete
, &co
);
2507 qemu_coroutine_yield();
2511 if (ret
&& ret
!= -ENOTSUP
) {
2521 bdrv_set_dirty(bs
, req
.offset
>> BDRV_SECTOR_BITS
,
2522 req
.bytes
>> BDRV_SECTOR_BITS
);
2523 tracked_request_end(&req
);
2524 bdrv_dec_in_flight(bs
);
2528 int bdrv_pdiscard(BlockDriverState
*bs
, int64_t offset
, int count
)
2538 if (qemu_in_coroutine()) {
2539 /* Fast-path if already in coroutine context */
2540 bdrv_pdiscard_co_entry(&rwco
);
2542 co
= qemu_coroutine_create(bdrv_pdiscard_co_entry
, &rwco
);
2543 bdrv_coroutine_enter(bs
, co
);
2544 BDRV_POLL_WHILE(bs
, rwco
.ret
== NOT_DONE
);
2550 int bdrv_co_ioctl(BlockDriverState
*bs
, int req
, void *buf
)
2552 BlockDriver
*drv
= bs
->drv
;
2553 CoroutineIOCompletion co
= {
2554 .coroutine
= qemu_coroutine_self(),
2558 bdrv_inc_in_flight(bs
);
2559 if (!drv
|| (!drv
->bdrv_aio_ioctl
&& !drv
->bdrv_co_ioctl
)) {
2564 if (drv
->bdrv_co_ioctl
) {
2565 co
.ret
= drv
->bdrv_co_ioctl(bs
, req
, buf
);
2567 acb
= drv
->bdrv_aio_ioctl(bs
, req
, buf
, bdrv_co_io_em_complete
, &co
);
2572 qemu_coroutine_yield();
2575 bdrv_dec_in_flight(bs
);
2579 void *qemu_blockalign(BlockDriverState
*bs
, size_t size
)
2581 return qemu_memalign(bdrv_opt_mem_align(bs
), size
);
2584 void *qemu_blockalign0(BlockDriverState
*bs
, size_t size
)
2586 return memset(qemu_blockalign(bs
, size
), 0, size
);
2589 void *qemu_try_blockalign(BlockDriverState
*bs
, size_t size
)
2591 size_t align
= bdrv_opt_mem_align(bs
);
2593 /* Ensure that NULL is never returned on success */
2599 return qemu_try_memalign(align
, size
);
2602 void *qemu_try_blockalign0(BlockDriverState
*bs
, size_t size
)
2604 void *mem
= qemu_try_blockalign(bs
, size
);
2607 memset(mem
, 0, size
);
2614 * Check if all memory in this vector is sector aligned.
2616 bool bdrv_qiov_is_aligned(BlockDriverState
*bs
, QEMUIOVector
*qiov
)
2619 size_t alignment
= bdrv_min_mem_align(bs
);
2621 for (i
= 0; i
< qiov
->niov
; i
++) {
2622 if ((uintptr_t) qiov
->iov
[i
].iov_base
% alignment
) {
2625 if (qiov
->iov
[i
].iov_len
% alignment
) {
2633 void bdrv_add_before_write_notifier(BlockDriverState
*bs
,
2634 NotifierWithReturn
*notifier
)
2636 notifier_with_return_list_add(&bs
->before_write_notifiers
, notifier
);
2639 void bdrv_io_plug(BlockDriverState
*bs
)
2643 QLIST_FOREACH(child
, &bs
->children
, next
) {
2644 bdrv_io_plug(child
->bs
);
2647 if (bs
->io_plugged
++ == 0) {
2648 BlockDriver
*drv
= bs
->drv
;
2649 if (drv
&& drv
->bdrv_io_plug
) {
2650 drv
->bdrv_io_plug(bs
);
2655 void bdrv_io_unplug(BlockDriverState
*bs
)
2659 assert(bs
->io_plugged
);
2660 if (--bs
->io_plugged
== 0) {
2661 BlockDriver
*drv
= bs
->drv
;
2662 if (drv
&& drv
->bdrv_io_unplug
) {
2663 drv
->bdrv_io_unplug(bs
);
2667 QLIST_FOREACH(child
, &bs
->children
, next
) {
2668 bdrv_io_unplug(child
->bs
);