4 * Copyright (C) 2013 Proxmox Server Solutions
7 * Dietmar Maurer (dietmar@proxmox.com)
9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
10 * See the COPYING file in the top-level directory.
14 #include "qemu/osdep.h"
17 #include "block/block.h"
18 #include "block/block_int.h"
19 #include "block/blockjob_int.h"
20 #include "block/block_backup.h"
21 #include "qapi/error.h"
22 #include "qapi/qmp/qerror.h"
23 #include "qemu/ratelimit.h"
24 #include "qemu/cutils.h"
25 #include "sysemu/block-backend.h"
26 #include "qemu/bitmap.h"
27 #include "qemu/error-report.h"
29 #define BACKUP_CLUSTER_SIZE_DEFAULT (1 << 16)
31 typedef struct CowRequest
{
34 QLIST_ENTRY(CowRequest
) list
;
35 CoQueue wait_queue
; /* coroutines blocked on this request */
38 typedef struct BackupBlockJob
{
42 BdrvDirtyBitmap
*sync_bitmap
;
43 BdrvDirtyBitmap
*copy_bitmap
;
45 MirrorSyncMode sync_mode
;
46 BitmapSyncMode bitmap_mode
;
47 BlockdevOnError on_source_error
;
48 BlockdevOnError on_target_error
;
49 CoRwlock flush_rwlock
;
54 NotifierWithReturn before_write
;
55 QLIST_HEAD(, CowRequest
) inflight_reqs
;
58 int64_t copy_range_size
;
60 bool serialize_target_writes
;
63 static const BlockJobDriver backup_job_driver
;
65 /* See if in-flight requests overlap and wait for them to complete */
66 static void coroutine_fn
wait_for_overlapping_requests(BackupBlockJob
*job
,
75 QLIST_FOREACH(req
, &job
->inflight_reqs
, list
) {
76 if (end
> req
->start_byte
&& start
< req
->end_byte
) {
77 qemu_co_queue_wait(&req
->wait_queue
, NULL
);
85 /* Keep track of an in-flight request */
86 static void cow_request_begin(CowRequest
*req
, BackupBlockJob
*job
,
87 int64_t start
, int64_t end
)
89 req
->start_byte
= start
;
91 qemu_co_queue_init(&req
->wait_queue
);
92 QLIST_INSERT_HEAD(&job
->inflight_reqs
, req
, list
);
95 /* Forget about a completed request */
96 static void cow_request_end(CowRequest
*req
)
98 QLIST_REMOVE(req
, list
);
99 qemu_co_queue_restart_all(&req
->wait_queue
);
102 /* Copy range to target with a bounce buffer and return the bytes copied. If
103 * error occurred, return a negative error number */
104 static int coroutine_fn
backup_cow_with_bounce_buffer(BackupBlockJob
*job
,
107 bool is_write_notifier
,
109 void **bounce_buffer
)
112 BlockBackend
*blk
= job
->common
.blk
;
114 int read_flags
= is_write_notifier
? BDRV_REQ_NO_SERIALISING
: 0;
115 int write_flags
= job
->serialize_target_writes
? BDRV_REQ_SERIALISING
: 0;
117 assert(QEMU_IS_ALIGNED(start
, job
->cluster_size
));
118 bdrv_reset_dirty_bitmap(job
->copy_bitmap
, start
, job
->cluster_size
);
119 nbytes
= MIN(job
->cluster_size
, job
->len
- start
);
120 if (!*bounce_buffer
) {
121 *bounce_buffer
= blk_blockalign(blk
, job
->cluster_size
);
124 ret
= blk_co_pread(blk
, start
, nbytes
, *bounce_buffer
, read_flags
);
126 trace_backup_do_cow_read_fail(job
, start
, ret
);
128 *error_is_read
= true;
133 if (buffer_is_zero(*bounce_buffer
, nbytes
)) {
134 ret
= blk_co_pwrite_zeroes(job
->target
, start
,
135 nbytes
, write_flags
| BDRV_REQ_MAY_UNMAP
);
137 ret
= blk_co_pwrite(job
->target
, start
,
138 nbytes
, *bounce_buffer
, write_flags
|
139 (job
->compress
? BDRV_REQ_WRITE_COMPRESSED
: 0));
142 trace_backup_do_cow_write_fail(job
, start
, ret
);
144 *error_is_read
= false;
151 bdrv_set_dirty_bitmap(job
->copy_bitmap
, start
, job
->cluster_size
);
156 /* Copy range to target and return the bytes copied. If error occurred, return a
157 * negative error number. */
158 static int coroutine_fn
backup_cow_with_offload(BackupBlockJob
*job
,
161 bool is_write_notifier
)
165 BlockBackend
*blk
= job
->common
.blk
;
167 int read_flags
= is_write_notifier
? BDRV_REQ_NO_SERIALISING
: 0;
168 int write_flags
= job
->serialize_target_writes
? BDRV_REQ_SERIALISING
: 0;
170 assert(QEMU_IS_ALIGNED(job
->copy_range_size
, job
->cluster_size
));
171 assert(QEMU_IS_ALIGNED(start
, job
->cluster_size
));
172 nbytes
= MIN(job
->copy_range_size
, end
- start
);
173 nr_clusters
= DIV_ROUND_UP(nbytes
, job
->cluster_size
);
174 bdrv_reset_dirty_bitmap(job
->copy_bitmap
, start
,
175 job
->cluster_size
* nr_clusters
);
176 ret
= blk_co_copy_range(blk
, start
, job
->target
, start
, nbytes
,
177 read_flags
, write_flags
);
179 trace_backup_do_cow_copy_range_fail(job
, start
, ret
);
180 bdrv_set_dirty_bitmap(job
->copy_bitmap
, start
,
181 job
->cluster_size
* nr_clusters
);
188 static int coroutine_fn
backup_do_cow(BackupBlockJob
*job
,
189 int64_t offset
, uint64_t bytes
,
191 bool is_write_notifier
)
193 CowRequest cow_request
;
195 int64_t start
, end
; /* bytes */
196 void *bounce_buffer
= NULL
;
198 qemu_co_rwlock_rdlock(&job
->flush_rwlock
);
200 start
= QEMU_ALIGN_DOWN(offset
, job
->cluster_size
);
201 end
= QEMU_ALIGN_UP(bytes
+ offset
, job
->cluster_size
);
203 trace_backup_do_cow_enter(job
, start
, offset
, bytes
);
205 wait_for_overlapping_requests(job
, start
, end
);
206 cow_request_begin(&cow_request
, job
, start
, end
);
208 while (start
< end
) {
211 if (!bdrv_dirty_bitmap_get(job
->copy_bitmap
, start
)) {
212 trace_backup_do_cow_skip(job
, start
);
213 start
+= job
->cluster_size
;
214 continue; /* already copied */
217 dirty_end
= bdrv_dirty_bitmap_next_zero(job
->copy_bitmap
, start
,
223 trace_backup_do_cow_process(job
, start
);
225 if (job
->use_copy_range
) {
226 ret
= backup_cow_with_offload(job
, start
, dirty_end
,
229 job
->use_copy_range
= false;
232 if (!job
->use_copy_range
) {
233 ret
= backup_cow_with_bounce_buffer(job
, start
, dirty_end
,
235 error_is_read
, &bounce_buffer
);
241 /* Publish progress, guest I/O counts as progress too. Note that the
242 * offset field is an opaque progress value, it is not a disk offset.
245 job
->bytes_read
+= ret
;
246 job_progress_update(&job
->common
.job
, ret
);
251 qemu_vfree(bounce_buffer
);
254 cow_request_end(&cow_request
);
256 trace_backup_do_cow_return(job
, offset
, bytes
, ret
);
258 qemu_co_rwlock_unlock(&job
->flush_rwlock
);
263 static int coroutine_fn
backup_before_write_notify(
264 NotifierWithReturn
*notifier
,
267 BackupBlockJob
*job
= container_of(notifier
, BackupBlockJob
, before_write
);
268 BdrvTrackedRequest
*req
= opaque
;
270 assert(req
->bs
== blk_bs(job
->common
.blk
));
271 assert(QEMU_IS_ALIGNED(req
->offset
, BDRV_SECTOR_SIZE
));
272 assert(QEMU_IS_ALIGNED(req
->bytes
, BDRV_SECTOR_SIZE
));
274 return backup_do_cow(job
, req
->offset
, req
->bytes
, NULL
, true);
277 static void backup_cleanup_sync_bitmap(BackupBlockJob
*job
, int ret
)
280 BlockDriverState
*bs
= blk_bs(job
->common
.blk
);
281 bool sync
= (((ret
== 0) || (job
->bitmap_mode
== BITMAP_SYNC_MODE_ALWAYS
)) \
282 && (job
->bitmap_mode
!= BITMAP_SYNC_MODE_NEVER
));
286 * We succeeded, or we always intended to sync the bitmap.
287 * Delete this bitmap and install the child.
289 bm
= bdrv_dirty_bitmap_abdicate(bs
, job
->sync_bitmap
, NULL
);
292 * We failed, or we never intended to sync the bitmap anyway.
293 * Merge the successor back into the parent, keeping all data.
295 bm
= bdrv_reclaim_dirty_bitmap(bs
, job
->sync_bitmap
, NULL
);
300 if (ret
< 0 && job
->bitmap_mode
== BITMAP_SYNC_MODE_ALWAYS
) {
301 /* If we failed and synced, merge in the bits we didn't copy: */
302 bdrv_dirty_bitmap_merge_internal(bm
, job
->copy_bitmap
,
307 static void backup_commit(Job
*job
)
309 BackupBlockJob
*s
= container_of(job
, BackupBlockJob
, common
.job
);
310 if (s
->sync_bitmap
) {
311 backup_cleanup_sync_bitmap(s
, 0);
315 static void backup_abort(Job
*job
)
317 BackupBlockJob
*s
= container_of(job
, BackupBlockJob
, common
.job
);
318 if (s
->sync_bitmap
) {
319 backup_cleanup_sync_bitmap(s
, -1);
323 static void backup_clean(Job
*job
)
325 BackupBlockJob
*s
= container_of(job
, BackupBlockJob
, common
.job
);
326 BlockDriverState
*bs
= blk_bs(s
->common
.blk
);
328 if (s
->copy_bitmap
) {
329 bdrv_release_dirty_bitmap(bs
, s
->copy_bitmap
);
330 s
->copy_bitmap
= NULL
;
334 blk_unref(s
->target
);
338 void backup_do_checkpoint(BlockJob
*job
, Error
**errp
)
340 BackupBlockJob
*backup_job
= container_of(job
, BackupBlockJob
, common
);
342 assert(block_job_driver(job
) == &backup_job_driver
);
344 if (backup_job
->sync_mode
!= MIRROR_SYNC_MODE_NONE
) {
345 error_setg(errp
, "The backup job only supports block checkpoint in"
350 bdrv_set_dirty_bitmap(backup_job
->copy_bitmap
, 0, backup_job
->len
);
353 static void backup_drain(BlockJob
*job
)
355 BackupBlockJob
*s
= container_of(job
, BackupBlockJob
, common
);
357 /* Need to keep a reference in case blk_drain triggers execution
358 * of backup_complete...
361 BlockBackend
*target
= s
->target
;
368 static BlockErrorAction
backup_error_action(BackupBlockJob
*job
,
369 bool read
, int error
)
372 return block_job_error_action(&job
->common
, job
->on_source_error
,
375 return block_job_error_action(&job
->common
, job
->on_target_error
,
380 static bool coroutine_fn
yield_and_check(BackupBlockJob
*job
)
384 if (job_is_cancelled(&job
->common
.job
)) {
388 /* We need to yield even for delay_ns = 0 so that bdrv_drain_all() can
389 * return. Without a yield, the VM would not reboot. */
390 delay_ns
= block_job_ratelimit_get_delay(&job
->common
, job
->bytes_read
);
392 job_sleep_ns(&job
->common
.job
, delay_ns
);
394 if (job_is_cancelled(&job
->common
.job
)) {
401 static bool bdrv_is_unallocated_range(BlockDriverState
*bs
,
402 int64_t offset
, int64_t bytes
)
404 int64_t end
= offset
+ bytes
;
406 while (offset
< end
&& !bdrv_is_allocated(bs
, offset
, bytes
, &bytes
)) {
411 bytes
= end
- offset
;
414 return offset
>= end
;
417 static int coroutine_fn
backup_loop(BackupBlockJob
*job
)
421 BdrvDirtyBitmapIter
*bdbi
;
422 BlockDriverState
*bs
= blk_bs(job
->common
.blk
);
425 bdbi
= bdrv_dirty_iter_new(job
->copy_bitmap
);
426 while ((offset
= bdrv_dirty_iter_next(bdbi
)) != -1) {
427 if (job
->sync_mode
== MIRROR_SYNC_MODE_TOP
&&
428 bdrv_is_unallocated_range(bs
, offset
, job
->cluster_size
))
430 bdrv_reset_dirty_bitmap(job
->copy_bitmap
, offset
,
436 if (yield_and_check(job
)) {
439 ret
= backup_do_cow(job
, offset
,
440 job
->cluster_size
, &error_is_read
, false);
441 if (ret
< 0 && backup_error_action(job
, error_is_read
, -ret
) ==
442 BLOCK_ERROR_ACTION_REPORT
)
450 bdrv_dirty_iter_free(bdbi
);
454 /* init copy_bitmap from sync_bitmap */
455 static void backup_incremental_init_copy_bitmap(BackupBlockJob
*job
)
457 bool ret
= bdrv_dirty_bitmap_merge_internal(job
->copy_bitmap
,
462 /* TODO job_progress_set_remaining() would make more sense */
463 job_progress_update(&job
->common
.job
,
464 job
->len
- bdrv_get_dirty_count(job
->copy_bitmap
));
467 static int coroutine_fn
backup_run(Job
*job
, Error
**errp
)
469 BackupBlockJob
*s
= container_of(job
, BackupBlockJob
, common
.job
);
470 BlockDriverState
*bs
= blk_bs(s
->common
.blk
);
473 QLIST_INIT(&s
->inflight_reqs
);
474 qemu_co_rwlock_init(&s
->flush_rwlock
);
476 job_progress_set_remaining(job
, s
->len
);
478 if (s
->sync_mode
== MIRROR_SYNC_MODE_BITMAP
) {
479 backup_incremental_init_copy_bitmap(s
);
481 bdrv_set_dirty_bitmap(s
->copy_bitmap
, 0, s
->len
);
484 s
->before_write
.notify
= backup_before_write_notify
;
485 bdrv_add_before_write_notifier(bs
, &s
->before_write
);
487 if (s
->sync_mode
== MIRROR_SYNC_MODE_NONE
) {
488 /* All bits are set in copy_bitmap to allow any cluster to be copied.
489 * This does not actually require them to be copied. */
490 while (!job_is_cancelled(job
)) {
491 /* Yield until the job is cancelled. We just let our before_write
492 * notify callback service CoW requests. */
496 ret
= backup_loop(s
);
499 notifier_with_return_remove(&s
->before_write
);
501 /* wait until pending backup_do_cow() calls have completed */
502 qemu_co_rwlock_wrlock(&s
->flush_rwlock
);
503 qemu_co_rwlock_unlock(&s
->flush_rwlock
);
508 static const BlockJobDriver backup_job_driver
= {
510 .instance_size
= sizeof(BackupBlockJob
),
511 .job_type
= JOB_TYPE_BACKUP
,
512 .free
= block_job_free
,
513 .user_resume
= block_job_user_resume
,
514 .drain
= block_job_drain
,
516 .commit
= backup_commit
,
517 .abort
= backup_abort
,
518 .clean
= backup_clean
,
520 .drain
= backup_drain
,
523 static int64_t backup_calculate_cluster_size(BlockDriverState
*target
,
530 * If there is no backing file on the target, we cannot rely on COW if our
531 * backup cluster size is smaller than the target cluster size. Even for
532 * targets with a backing file, try to avoid COW if possible.
534 ret
= bdrv_get_info(target
, &bdi
);
535 if (ret
== -ENOTSUP
&& !target
->backing
) {
536 /* Cluster size is not defined */
537 warn_report("The target block device doesn't provide "
538 "information about the block size and it doesn't have a "
539 "backing file. The default block size of %u bytes is "
540 "used. If the actual block size of the target exceeds "
541 "this default, the backup may be unusable",
542 BACKUP_CLUSTER_SIZE_DEFAULT
);
543 return BACKUP_CLUSTER_SIZE_DEFAULT
;
544 } else if (ret
< 0 && !target
->backing
) {
545 error_setg_errno(errp
, -ret
,
546 "Couldn't determine the cluster size of the target image, "
547 "which has no backing file");
548 error_append_hint(errp
,
549 "Aborting, since this may create an unusable destination image\n");
551 } else if (ret
< 0 && target
->backing
) {
552 /* Not fatal; just trudge on ahead. */
553 return BACKUP_CLUSTER_SIZE_DEFAULT
;
556 return MAX(BACKUP_CLUSTER_SIZE_DEFAULT
, bdi
.cluster_size
);
559 BlockJob
*backup_job_create(const char *job_id
, BlockDriverState
*bs
,
560 BlockDriverState
*target
, int64_t speed
,
561 MirrorSyncMode sync_mode
, BdrvDirtyBitmap
*sync_bitmap
,
562 BitmapSyncMode bitmap_mode
,
564 BlockdevOnError on_source_error
,
565 BlockdevOnError on_target_error
,
567 BlockCompletionFunc
*cb
, void *opaque
,
568 JobTxn
*txn
, Error
**errp
)
571 BackupBlockJob
*job
= NULL
;
573 int64_t cluster_size
;
574 BdrvDirtyBitmap
*copy_bitmap
= NULL
;
580 error_setg(errp
, "Source and target cannot be the same");
584 if (!bdrv_is_inserted(bs
)) {
585 error_setg(errp
, "Device is not inserted: %s",
586 bdrv_get_device_name(bs
));
590 if (!bdrv_is_inserted(target
)) {
591 error_setg(errp
, "Device is not inserted: %s",
592 bdrv_get_device_name(target
));
596 if (compress
&& target
->drv
->bdrv_co_pwritev_compressed
== NULL
) {
597 error_setg(errp
, "Compression is not supported for this drive %s",
598 bdrv_get_device_name(target
));
602 if (bdrv_op_is_blocked(bs
, BLOCK_OP_TYPE_BACKUP_SOURCE
, errp
)) {
606 if (bdrv_op_is_blocked(target
, BLOCK_OP_TYPE_BACKUP_TARGET
, errp
)) {
610 /* QMP interface should have handled translating this to bitmap mode */
611 assert(sync_mode
!= MIRROR_SYNC_MODE_INCREMENTAL
);
613 if (sync_mode
== MIRROR_SYNC_MODE_BITMAP
) {
615 error_setg(errp
, "must provide a valid bitmap name for "
616 "'%s' sync mode", MirrorSyncMode_str(sync_mode
));
620 /* Create a new bitmap, and freeze/disable this one. */
621 if (bdrv_dirty_bitmap_create_successor(bs
, sync_bitmap
, errp
) < 0) {
624 } else if (sync_bitmap
) {
626 "a bitmap was given to backup_job_create, "
627 "but it received an incompatible sync_mode (%s)",
628 MirrorSyncMode_str(sync_mode
));
632 len
= bdrv_getlength(bs
);
634 error_setg_errno(errp
, -len
, "unable to get length for '%s'",
635 bdrv_get_device_name(bs
));
639 cluster_size
= backup_calculate_cluster_size(target
, errp
);
640 if (cluster_size
< 0) {
644 copy_bitmap
= bdrv_create_dirty_bitmap(bs
, cluster_size
, NULL
, errp
);
648 bdrv_disable_dirty_bitmap(copy_bitmap
);
650 /* job->len is fixed, so we can't allow resize */
651 job
= block_job_create(job_id
, &backup_job_driver
, txn
, bs
,
652 BLK_PERM_CONSISTENT_READ
,
653 BLK_PERM_CONSISTENT_READ
| BLK_PERM_WRITE
|
654 BLK_PERM_WRITE_UNCHANGED
| BLK_PERM_GRAPH_MOD
,
655 speed
, creation_flags
, cb
, opaque
, errp
);
660 /* The target must match the source in size, so no resize here either */
661 job
->target
= blk_new(job
->common
.job
.aio_context
,
663 BLK_PERM_CONSISTENT_READ
| BLK_PERM_WRITE
|
664 BLK_PERM_WRITE_UNCHANGED
| BLK_PERM_GRAPH_MOD
);
665 ret
= blk_insert_bs(job
->target
, target
, errp
);
669 blk_set_disable_request_queuing(job
->target
, true);
671 job
->on_source_error
= on_source_error
;
672 job
->on_target_error
= on_target_error
;
673 job
->sync_mode
= sync_mode
;
674 job
->sync_bitmap
= sync_bitmap
;
675 job
->bitmap_mode
= bitmap_mode
;
676 job
->compress
= compress
;
678 /* Detect image-fleecing (and similar) schemes */
679 job
->serialize_target_writes
= bdrv_chain_contains(target
, bs
);
680 job
->cluster_size
= cluster_size
;
681 job
->copy_bitmap
= copy_bitmap
;
683 job
->use_copy_range
= !compress
; /* compression isn't supported for it */
684 job
->copy_range_size
= MIN_NON_ZERO(blk_get_max_transfer(job
->common
.blk
),
685 blk_get_max_transfer(job
->target
));
686 job
->copy_range_size
= MAX(job
->cluster_size
,
687 QEMU_ALIGN_UP(job
->copy_range_size
,
690 /* Required permissions are already taken with target's blk_new() */
691 block_job_add_bdrv(&job
->common
, "target", target
, 0, BLK_PERM_ALL
,
699 assert(!job
|| !job
->copy_bitmap
);
700 bdrv_release_dirty_bitmap(bs
, copy_bitmap
);
703 bdrv_reclaim_dirty_bitmap(bs
, sync_bitmap
, NULL
);
706 backup_clean(&job
->common
.job
);
707 job_early_fail(&job
->common
.job
);