Merge tag 'qemu-macppc-20230206' of https://github.com/mcayland/qemu into staging
[qemu.git] / migration / block-dirty-bitmap.c
blob15127d489a73044543671d6600433cc7aec516af
1 /*
2 * Block dirty bitmap postcopy migration
4 * Copyright IBM, Corp. 2009
5 * Copyright (c) 2016-2017 Virtuozzo International GmbH. All rights reserved.
7 * Authors:
8 * Liran Schour <lirans@il.ibm.com>
9 * Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
11 * This work is licensed under the terms of the GNU GPL, version 2. See
12 * the COPYING file in the top-level directory.
13 * This file is derived from migration/block.c, so it's author and IBM copyright
14 * are here, although content is quite different.
16 * Contributions after 2012-01-13 are licensed under the terms of the
17 * GNU GPL, version 2 or (at your option) any later version.
19 * ***
21 * Here postcopy migration of dirty bitmaps is realized. Only QMP-addressable
22 * bitmaps are migrated.
24 * Bitmap migration implies creating bitmap with the same name and granularity
25 * in destination QEMU. If the bitmap with the same name (for the same node)
26 * already exists on destination an error will be generated.
28 * format of migration:
30 * # Header (shared for different chunk types)
31 * 1, 2 or 4 bytes: flags (see qemu_{put,put}_flags)
32 * [ 1 byte: node alias size ] \ flags & DEVICE_NAME
33 * [ n bytes: node alias ] /
34 * [ 1 byte: bitmap alias size ] \ flags & BITMAP_NAME
35 * [ n bytes: bitmap alias ] /
37 * # Start of bitmap migration (flags & START)
38 * header
39 * be64: granularity
40 * 1 byte: bitmap flags (corresponds to BdrvDirtyBitmap)
41 * bit 0 - bitmap is enabled
42 * bit 1 - bitmap is persistent
43 * bit 2 - bitmap is autoloading
44 * bits 3-7 - reserved, must be zero
46 * # Complete of bitmap migration (flags & COMPLETE)
47 * header
49 * # Data chunk of bitmap migration
50 * header
51 * be64: start sector
52 * be32: number of sectors
53 * [ be64: buffer size ] \ ! (flags & ZEROES)
54 * [ n bytes: buffer ] /
56 * The last chunk in stream should contain flags & EOS. The chunk may skip
57 * device and/or bitmap names, assuming them to be the same with the previous
58 * chunk.
61 #include "qemu/osdep.h"
62 #include "block/block.h"
63 #include "block/block_int.h"
64 #include "block/dirty-bitmap.h"
65 #include "sysemu/block-backend.h"
66 #include "sysemu/runstate.h"
67 #include "qemu/main-loop.h"
68 #include "qemu/error-report.h"
69 #include "migration/misc.h"
70 #include "migration/migration.h"
71 #include "qemu-file.h"
72 #include "migration/vmstate.h"
73 #include "migration/register.h"
74 #include "qemu/hbitmap.h"
75 #include "qemu/cutils.h"
76 #include "qemu/id.h"
77 #include "qapi/error.h"
78 #include "qapi/qapi-commands-migration.h"
79 #include "qapi/qapi-visit-migration.h"
80 #include "qapi/clone-visitor.h"
81 #include "trace.h"
83 #define CHUNK_SIZE (1 << 10)
85 /* Flags occupy one, two or four bytes (Big Endian). The size is determined as
86 * follows:
87 * in first (most significant) byte bit 8 is clear --> one byte
88 * in first byte bit 8 is set --> two or four bytes, depending on second
89 * byte:
90 * | in second byte bit 8 is clear --> two bytes
91 * | in second byte bit 8 is set --> four bytes
93 #define DIRTY_BITMAP_MIG_FLAG_EOS 0x01
94 #define DIRTY_BITMAP_MIG_FLAG_ZEROES 0x02
95 #define DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME 0x04
96 #define DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME 0x08
97 #define DIRTY_BITMAP_MIG_FLAG_START 0x10
98 #define DIRTY_BITMAP_MIG_FLAG_COMPLETE 0x20
99 #define DIRTY_BITMAP_MIG_FLAG_BITS 0x40
101 #define DIRTY_BITMAP_MIG_EXTRA_FLAGS 0x80
103 #define DIRTY_BITMAP_MIG_START_FLAG_ENABLED 0x01
104 #define DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT 0x02
105 /* 0x04 was "AUTOLOAD" flags on older versions, now it is ignored */
106 #define DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK 0xf8
108 /* State of one bitmap during save process */
109 typedef struct SaveBitmapState {
110 /* Written during setup phase. */
111 BlockDriverState *bs;
112 char *node_alias;
113 char *bitmap_alias;
114 BdrvDirtyBitmap *bitmap;
115 uint64_t total_sectors;
116 uint64_t sectors_per_chunk;
117 QSIMPLEQ_ENTRY(SaveBitmapState) entry;
118 uint8_t flags;
120 /* For bulk phase. */
121 bool bulk_completed;
122 uint64_t cur_sector;
123 } SaveBitmapState;
125 /* State of the dirty bitmap migration (DBM) during save process */
126 typedef struct DBMSaveState {
127 QSIMPLEQ_HEAD(, SaveBitmapState) dbms_list;
129 bool bulk_completed;
130 bool no_bitmaps;
132 /* for send_bitmap_bits() */
133 BlockDriverState *prev_bs;
134 BdrvDirtyBitmap *prev_bitmap;
135 } DBMSaveState;
137 typedef struct LoadBitmapState {
138 BlockDriverState *bs;
139 BdrvDirtyBitmap *bitmap;
140 bool migrated;
141 bool enabled;
142 } LoadBitmapState;
144 /* State of the dirty bitmap migration (DBM) during load process */
145 typedef struct DBMLoadState {
146 uint32_t flags;
147 char node_alias[256];
148 char bitmap_alias[256];
149 char bitmap_name[BDRV_BITMAP_MAX_NAME_SIZE + 1];
150 BlockDriverState *bs;
151 BdrvDirtyBitmap *bitmap;
153 bool before_vm_start_handled; /* set in dirty_bitmap_mig_before_vm_start */
154 BitmapMigrationBitmapAlias *bmap_inner;
157 * cancelled
158 * Incoming migration is cancelled for some reason. That means that we
159 * still should read our chunks from migration stream, to not affect other
160 * migration objects (like RAM), but just ignore them and do not touch any
161 * bitmaps or nodes.
163 bool cancelled;
165 GSList *bitmaps;
166 QemuMutex lock; /* protect bitmaps */
167 } DBMLoadState;
169 typedef struct DBMState {
170 DBMSaveState save;
171 DBMLoadState load;
172 } DBMState;
174 static DBMState dbm_state;
176 /* For hash tables that map node/bitmap names to aliases */
177 typedef struct AliasMapInnerNode {
178 char *string;
179 GHashTable *subtree;
180 } AliasMapInnerNode;
182 static void free_alias_map_inner_node(void *amin_ptr)
184 AliasMapInnerNode *amin = amin_ptr;
186 g_free(amin->string);
187 g_hash_table_unref(amin->subtree);
188 g_free(amin);
192 * Construct an alias map based on the given QMP structure.
194 * (Note that we cannot store such maps in the MigrationParameters
195 * object, because that struct is defined by the QAPI schema, which
196 * makes it basically impossible to have dicts with arbitrary keys.
197 * Therefore, we instead have to construct these maps when migration
198 * starts.)
200 * @bbm is the block_bitmap_mapping from the migration parameters.
202 * If @name_to_alias is true, the returned hash table will map node
203 * and bitmap names to their respective aliases (for outgoing
204 * migration).
206 * If @name_to_alias is false, the returned hash table will map node
207 * and bitmap aliases to their respective names (for incoming
208 * migration).
210 * The hash table maps node names/aliases to AliasMapInnerNode
211 * objects, whose .string is the respective node alias/name, and whose
212 * .subtree table maps bitmap names/aliases to the respective bitmap
213 * alias/name.
215 static GHashTable *construct_alias_map(const BitmapMigrationNodeAliasList *bbm,
216 bool name_to_alias,
217 Error **errp)
219 GHashTable *alias_map;
220 size_t max_node_name_len = sizeof_field(BlockDriverState, node_name) - 1;
222 alias_map = g_hash_table_new_full(g_str_hash, g_str_equal,
223 g_free, free_alias_map_inner_node);
225 for (; bbm; bbm = bbm->next) {
226 const BitmapMigrationNodeAlias *bmna = bbm->value;
227 const BitmapMigrationBitmapAliasList *bmbal;
228 AliasMapInnerNode *amin;
229 GHashTable *bitmaps_map;
230 const char *node_map_from, *node_map_to;
231 GDestroyNotify gdn;
233 if (!id_wellformed(bmna->alias)) {
234 error_setg(errp, "The node alias '%s' is not well-formed",
235 bmna->alias);
236 goto fail;
239 if (strlen(bmna->alias) > UINT8_MAX) {
240 error_setg(errp, "The node alias '%s' is longer than %u bytes",
241 bmna->alias, UINT8_MAX);
242 goto fail;
245 if (strlen(bmna->node_name) > max_node_name_len) {
246 error_setg(errp, "The node name '%s' is longer than %zu bytes",
247 bmna->node_name, max_node_name_len);
248 goto fail;
251 if (name_to_alias) {
252 if (g_hash_table_contains(alias_map, bmna->node_name)) {
253 error_setg(errp, "The node name '%s' is mapped twice",
254 bmna->node_name);
255 goto fail;
258 node_map_from = bmna->node_name;
259 node_map_to = bmna->alias;
260 } else {
261 if (g_hash_table_contains(alias_map, bmna->alias)) {
262 error_setg(errp, "The node alias '%s' is used twice",
263 bmna->alias);
264 goto fail;
267 node_map_from = bmna->alias;
268 node_map_to = bmna->node_name;
271 gdn = (GDestroyNotify) qapi_free_BitmapMigrationBitmapAlias;
272 bitmaps_map = g_hash_table_new_full(g_str_hash, g_str_equal, g_free,
273 gdn);
275 amin = g_new(AliasMapInnerNode, 1);
276 *amin = (AliasMapInnerNode){
277 .string = g_strdup(node_map_to),
278 .subtree = bitmaps_map,
281 g_hash_table_insert(alias_map, g_strdup(node_map_from), amin);
283 for (bmbal = bmna->bitmaps; bmbal; bmbal = bmbal->next) {
284 const BitmapMigrationBitmapAlias *bmba = bmbal->value;
285 const char *bmap_map_from;
287 if (strlen(bmba->alias) > UINT8_MAX) {
288 error_setg(errp,
289 "The bitmap alias '%s' is longer than %u bytes",
290 bmba->alias, UINT8_MAX);
291 goto fail;
294 if (strlen(bmba->name) > BDRV_BITMAP_MAX_NAME_SIZE) {
295 error_setg(errp, "The bitmap name '%s' is longer than %d bytes",
296 bmba->name, BDRV_BITMAP_MAX_NAME_SIZE);
297 goto fail;
300 if (name_to_alias) {
301 bmap_map_from = bmba->name;
303 if (g_hash_table_contains(bitmaps_map, bmba->name)) {
304 error_setg(errp, "The bitmap '%s'/'%s' is mapped twice",
305 bmna->node_name, bmba->name);
306 goto fail;
308 } else {
309 bmap_map_from = bmba->alias;
311 if (g_hash_table_contains(bitmaps_map, bmba->alias)) {
312 error_setg(errp, "The bitmap alias '%s'/'%s' is used twice",
313 bmna->alias, bmba->alias);
314 goto fail;
318 g_hash_table_insert(bitmaps_map, g_strdup(bmap_map_from),
319 QAPI_CLONE(BitmapMigrationBitmapAlias, bmba));
323 return alias_map;
325 fail:
326 g_hash_table_destroy(alias_map);
327 return NULL;
331 * Run construct_alias_map() in both directions to check whether @bbm
332 * is valid.
333 * (This function is to be used by migration/migration.c to validate
334 * the user-specified block-bitmap-mapping migration parameter.)
336 * Returns true if and only if the mapping is valid.
338 bool check_dirty_bitmap_mig_alias_map(const BitmapMigrationNodeAliasList *bbm,
339 Error **errp)
341 GHashTable *alias_map;
343 alias_map = construct_alias_map(bbm, true, errp);
344 if (!alias_map) {
345 return false;
347 g_hash_table_destroy(alias_map);
349 alias_map = construct_alias_map(bbm, false, errp);
350 if (!alias_map) {
351 return false;
353 g_hash_table_destroy(alias_map);
355 return true;
358 static uint32_t qemu_get_bitmap_flags(QEMUFile *f)
360 uint8_t flags = qemu_get_byte(f);
361 if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) {
362 flags = flags << 8 | qemu_get_byte(f);
363 if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) {
364 flags = flags << 16 | qemu_get_be16(f);
368 return flags;
371 static void qemu_put_bitmap_flags(QEMUFile *f, uint32_t flags)
373 /* The code currently does not send flags as more than one byte */
374 assert(!(flags & (0xffffff00 | DIRTY_BITMAP_MIG_EXTRA_FLAGS)));
376 qemu_put_byte(f, flags);
379 static void send_bitmap_header(QEMUFile *f, DBMSaveState *s,
380 SaveBitmapState *dbms, uint32_t additional_flags)
382 BlockDriverState *bs = dbms->bs;
383 BdrvDirtyBitmap *bitmap = dbms->bitmap;
384 uint32_t flags = additional_flags;
385 trace_send_bitmap_header_enter();
387 if (bs != s->prev_bs) {
388 s->prev_bs = bs;
389 flags |= DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME;
392 if (bitmap != s->prev_bitmap) {
393 s->prev_bitmap = bitmap;
394 flags |= DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME;
397 qemu_put_bitmap_flags(f, flags);
399 if (flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
400 qemu_put_counted_string(f, dbms->node_alias);
403 if (flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
404 qemu_put_counted_string(f, dbms->bitmap_alias);
408 static void send_bitmap_start(QEMUFile *f, DBMSaveState *s,
409 SaveBitmapState *dbms)
411 send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_START);
412 qemu_put_be32(f, bdrv_dirty_bitmap_granularity(dbms->bitmap));
413 qemu_put_byte(f, dbms->flags);
416 static void send_bitmap_complete(QEMUFile *f, DBMSaveState *s,
417 SaveBitmapState *dbms)
419 send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_COMPLETE);
422 static void send_bitmap_bits(QEMUFile *f, DBMSaveState *s,
423 SaveBitmapState *dbms,
424 uint64_t start_sector, uint32_t nr_sectors)
426 /* align for buffer_is_zero() */
427 uint64_t align = 4 * sizeof(long);
428 uint64_t unaligned_size =
429 bdrv_dirty_bitmap_serialization_size(
430 dbms->bitmap, start_sector << BDRV_SECTOR_BITS,
431 (uint64_t)nr_sectors << BDRV_SECTOR_BITS);
432 uint64_t buf_size = QEMU_ALIGN_UP(unaligned_size, align);
433 uint8_t *buf = g_malloc0(buf_size);
434 uint32_t flags = DIRTY_BITMAP_MIG_FLAG_BITS;
436 bdrv_dirty_bitmap_serialize_part(
437 dbms->bitmap, buf, start_sector << BDRV_SECTOR_BITS,
438 (uint64_t)nr_sectors << BDRV_SECTOR_BITS);
440 if (buffer_is_zero(buf, buf_size)) {
441 g_free(buf);
442 buf = NULL;
443 flags |= DIRTY_BITMAP_MIG_FLAG_ZEROES;
446 trace_send_bitmap_bits(flags, start_sector, nr_sectors, buf_size);
448 send_bitmap_header(f, s, dbms, flags);
450 qemu_put_be64(f, start_sector);
451 qemu_put_be32(f, nr_sectors);
453 /* if a block is zero we need to flush here since the network
454 * bandwidth is now a lot higher than the storage device bandwidth.
455 * thus if we queue zero blocks we slow down the migration. */
456 if (flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) {
457 qemu_fflush(f);
458 } else {
459 qemu_put_be64(f, buf_size);
460 qemu_put_buffer(f, buf, buf_size);
463 g_free(buf);
466 /* Called with iothread lock taken. */
467 static void dirty_bitmap_do_save_cleanup(DBMSaveState *s)
469 SaveBitmapState *dbms;
471 while ((dbms = QSIMPLEQ_FIRST(&s->dbms_list)) != NULL) {
472 QSIMPLEQ_REMOVE_HEAD(&s->dbms_list, entry);
473 bdrv_dirty_bitmap_set_busy(dbms->bitmap, false);
474 bdrv_unref(dbms->bs);
475 g_free(dbms->node_alias);
476 g_free(dbms->bitmap_alias);
477 g_free(dbms);
481 /* Called with iothread lock taken. */
482 static int add_bitmaps_to_list(DBMSaveState *s, BlockDriverState *bs,
483 const char *bs_name, GHashTable *alias_map)
485 BdrvDirtyBitmap *bitmap;
486 SaveBitmapState *dbms;
487 GHashTable *bitmap_aliases;
488 const char *node_alias, *bitmap_name, *bitmap_alias;
489 Error *local_err = NULL;
491 /* When an alias map is given, @bs_name must be @bs's node name */
492 assert(!alias_map || !strcmp(bs_name, bdrv_get_node_name(bs)));
494 FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
495 if (bdrv_dirty_bitmap_name(bitmap)) {
496 break;
499 if (!bitmap) {
500 return 0;
503 bitmap_name = bdrv_dirty_bitmap_name(bitmap);
505 if (!bs_name || strcmp(bs_name, "") == 0) {
506 error_report("Bitmap '%s' in unnamed node can't be migrated",
507 bitmap_name);
508 return -1;
511 if (alias_map) {
512 const AliasMapInnerNode *amin = g_hash_table_lookup(alias_map, bs_name);
514 if (!amin) {
515 /* Skip bitmaps on nodes with no alias */
516 return 0;
519 node_alias = amin->string;
520 bitmap_aliases = amin->subtree;
521 } else {
522 node_alias = bs_name;
523 bitmap_aliases = NULL;
526 if (node_alias[0] == '#') {
527 error_report("Bitmap '%s' in a node with auto-generated "
528 "name '%s' can't be migrated",
529 bitmap_name, node_alias);
530 return -1;
533 FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
534 BitmapMigrationBitmapAliasTransform *bitmap_transform = NULL;
535 bitmap_name = bdrv_dirty_bitmap_name(bitmap);
536 if (!bitmap_name) {
537 continue;
540 if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_DEFAULT, &local_err)) {
541 error_report_err(local_err);
542 return -1;
545 if (bitmap_aliases) {
546 BitmapMigrationBitmapAlias *bmap_inner;
548 bmap_inner = g_hash_table_lookup(bitmap_aliases, bitmap_name);
549 if (!bmap_inner) {
550 /* Skip bitmaps with no alias */
551 continue;
554 bitmap_alias = bmap_inner->alias;
555 if (bmap_inner->transform) {
556 bitmap_transform = bmap_inner->transform;
558 } else {
559 if (strlen(bitmap_name) > UINT8_MAX) {
560 error_report("Cannot migrate bitmap '%s' on node '%s': "
561 "Name is longer than %u bytes",
562 bitmap_name, bs_name, UINT8_MAX);
563 return -1;
565 bitmap_alias = bitmap_name;
568 bdrv_ref(bs);
569 bdrv_dirty_bitmap_set_busy(bitmap, true);
571 dbms = g_new0(SaveBitmapState, 1);
572 dbms->bs = bs;
573 dbms->node_alias = g_strdup(node_alias);
574 dbms->bitmap_alias = g_strdup(bitmap_alias);
575 dbms->bitmap = bitmap;
576 dbms->total_sectors = bdrv_nb_sectors(bs);
577 dbms->sectors_per_chunk = CHUNK_SIZE * 8LLU *
578 (bdrv_dirty_bitmap_granularity(bitmap) >> BDRV_SECTOR_BITS);
579 assert(dbms->sectors_per_chunk != 0);
580 if (bdrv_dirty_bitmap_enabled(bitmap)) {
581 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_ENABLED;
583 if (bitmap_transform &&
584 bitmap_transform->has_persistent) {
585 if (bitmap_transform->persistent) {
586 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
588 } else {
589 if (bdrv_dirty_bitmap_get_persistence(bitmap)) {
590 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
594 QSIMPLEQ_INSERT_TAIL(&s->dbms_list, dbms, entry);
597 return 0;
600 /* Called with iothread lock taken. */
601 static int init_dirty_bitmap_migration(DBMSaveState *s)
603 BlockDriverState *bs;
604 SaveBitmapState *dbms;
605 GHashTable *handled_by_blk = g_hash_table_new(NULL, NULL);
606 BlockBackend *blk;
607 const MigrationParameters *mig_params = &migrate_get_current()->parameters;
608 GHashTable *alias_map = NULL;
610 if (mig_params->has_block_bitmap_mapping) {
611 alias_map = construct_alias_map(mig_params->block_bitmap_mapping, true,
612 &error_abort);
615 s->bulk_completed = false;
616 s->prev_bs = NULL;
617 s->prev_bitmap = NULL;
618 s->no_bitmaps = false;
620 if (!alias_map) {
622 * Use blockdevice name for direct (or filtered) children of named block
623 * backends.
625 for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
626 const char *name = blk_name(blk);
628 if (!name || strcmp(name, "") == 0) {
629 continue;
632 bs = blk_bs(blk);
634 /* Skip filters without bitmaps */
635 while (bs && bs->drv && bs->drv->is_filter &&
636 !bdrv_has_named_bitmaps(bs))
638 bs = bdrv_filter_bs(bs);
641 if (bs && bs->drv && !bs->drv->is_filter) {
642 if (add_bitmaps_to_list(s, bs, name, NULL)) {
643 goto fail;
645 g_hash_table_add(handled_by_blk, bs);
650 for (bs = bdrv_next_all_states(NULL); bs; bs = bdrv_next_all_states(bs)) {
651 if (g_hash_table_contains(handled_by_blk, bs)) {
652 continue;
655 if (add_bitmaps_to_list(s, bs, bdrv_get_node_name(bs), alias_map)) {
656 goto fail;
660 /* unset migration flags here, to not roll back it */
661 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
662 bdrv_dirty_bitmap_skip_store(dbms->bitmap, true);
665 if (QSIMPLEQ_EMPTY(&s->dbms_list)) {
666 s->no_bitmaps = true;
669 g_hash_table_destroy(handled_by_blk);
670 if (alias_map) {
671 g_hash_table_destroy(alias_map);
674 return 0;
676 fail:
677 g_hash_table_destroy(handled_by_blk);
678 if (alias_map) {
679 g_hash_table_destroy(alias_map);
681 dirty_bitmap_do_save_cleanup(s);
683 return -1;
686 /* Called with no lock taken. */
687 static void bulk_phase_send_chunk(QEMUFile *f, DBMSaveState *s,
688 SaveBitmapState *dbms)
690 uint32_t nr_sectors = MIN(dbms->total_sectors - dbms->cur_sector,
691 dbms->sectors_per_chunk);
693 send_bitmap_bits(f, s, dbms, dbms->cur_sector, nr_sectors);
695 dbms->cur_sector += nr_sectors;
696 if (dbms->cur_sector >= dbms->total_sectors) {
697 dbms->bulk_completed = true;
701 /* Called with no lock taken. */
702 static void bulk_phase(QEMUFile *f, DBMSaveState *s, bool limit)
704 SaveBitmapState *dbms;
706 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
707 while (!dbms->bulk_completed) {
708 bulk_phase_send_chunk(f, s, dbms);
709 if (limit && qemu_file_rate_limit(f)) {
710 return;
715 s->bulk_completed = true;
718 /* for SaveVMHandlers */
719 static void dirty_bitmap_save_cleanup(void *opaque)
721 DBMSaveState *s = &((DBMState *)opaque)->save;
723 dirty_bitmap_do_save_cleanup(s);
726 static int dirty_bitmap_save_iterate(QEMUFile *f, void *opaque)
728 DBMSaveState *s = &((DBMState *)opaque)->save;
730 trace_dirty_bitmap_save_iterate(migration_in_postcopy());
732 if (migration_in_postcopy() && !s->bulk_completed) {
733 bulk_phase(f, s, true);
736 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
738 return s->bulk_completed;
741 /* Called with iothread lock taken. */
743 static int dirty_bitmap_save_complete(QEMUFile *f, void *opaque)
745 DBMSaveState *s = &((DBMState *)opaque)->save;
746 SaveBitmapState *dbms;
747 trace_dirty_bitmap_save_complete_enter();
749 if (!s->bulk_completed) {
750 bulk_phase(f, s, false);
753 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
754 send_bitmap_complete(f, s, dbms);
757 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
759 trace_dirty_bitmap_save_complete_finish();
761 dirty_bitmap_save_cleanup(opaque);
762 return 0;
765 static void dirty_bitmap_save_pending(QEMUFile *f, void *opaque,
766 uint64_t max_size,
767 uint64_t *res_precopy_only,
768 uint64_t *res_compatible,
769 uint64_t *res_postcopy_only)
771 DBMSaveState *s = &((DBMState *)opaque)->save;
772 SaveBitmapState *dbms;
773 uint64_t pending = 0;
775 qemu_mutex_lock_iothread();
777 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
778 uint64_t gran = bdrv_dirty_bitmap_granularity(dbms->bitmap);
779 uint64_t sectors = dbms->bulk_completed ? 0 :
780 dbms->total_sectors - dbms->cur_sector;
782 pending += DIV_ROUND_UP(sectors * BDRV_SECTOR_SIZE, gran);
785 qemu_mutex_unlock_iothread();
787 trace_dirty_bitmap_save_pending(pending, max_size);
789 *res_postcopy_only += pending;
792 /* First occurrence of this bitmap. It should be created if doesn't exist */
793 static int dirty_bitmap_load_start(QEMUFile *f, DBMLoadState *s)
795 Error *local_err = NULL;
796 uint32_t granularity = qemu_get_be32(f);
797 uint8_t flags = qemu_get_byte(f);
798 LoadBitmapState *b;
799 bool persistent;
801 if (s->cancelled) {
802 return 0;
805 if (s->bitmap) {
806 error_report("Bitmap with the same name ('%s') already exists on "
807 "destination", bdrv_dirty_bitmap_name(s->bitmap));
808 return -EINVAL;
809 } else {
810 s->bitmap = bdrv_create_dirty_bitmap(s->bs, granularity,
811 s->bitmap_name, &local_err);
812 if (!s->bitmap) {
813 error_report_err(local_err);
814 return -EINVAL;
818 if (flags & DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK) {
819 error_report("Unknown flags in migrated dirty bitmap header: %x",
820 flags);
821 return -EINVAL;
824 if (s->bmap_inner &&
825 s->bmap_inner->transform &&
826 s->bmap_inner->transform->has_persistent) {
827 persistent = s->bmap_inner->transform->persistent;
828 } else {
829 persistent = flags & DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
832 if (persistent) {
833 bdrv_dirty_bitmap_set_persistence(s->bitmap, true);
836 bdrv_disable_dirty_bitmap(s->bitmap);
837 if (flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED) {
838 bdrv_dirty_bitmap_create_successor(s->bitmap, &local_err);
839 if (local_err) {
840 error_report_err(local_err);
841 return -EINVAL;
843 } else {
844 bdrv_dirty_bitmap_set_busy(s->bitmap, true);
847 b = g_new(LoadBitmapState, 1);
848 b->bs = s->bs;
849 b->bitmap = s->bitmap;
850 b->migrated = false;
851 b->enabled = flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED;
853 s->bitmaps = g_slist_prepend(s->bitmaps, b);
855 return 0;
859 * before_vm_start_handle_item
861 * g_slist_foreach helper
863 * item is LoadBitmapState*
864 * opaque is DBMLoadState*
866 static void before_vm_start_handle_item(void *item, void *opaque)
868 DBMLoadState *s = opaque;
869 LoadBitmapState *b = item;
871 if (b->enabled) {
872 if (b->migrated) {
873 bdrv_enable_dirty_bitmap(b->bitmap);
874 } else {
875 bdrv_dirty_bitmap_enable_successor(b->bitmap);
879 if (b->migrated) {
880 s->bitmaps = g_slist_remove(s->bitmaps, b);
881 g_free(b);
885 void dirty_bitmap_mig_before_vm_start(void)
887 DBMLoadState *s = &dbm_state.load;
888 qemu_mutex_lock(&s->lock);
890 assert(!s->before_vm_start_handled);
891 g_slist_foreach(s->bitmaps, before_vm_start_handle_item, s);
892 s->before_vm_start_handled = true;
894 qemu_mutex_unlock(&s->lock);
897 static void cancel_incoming_locked(DBMLoadState *s)
899 GSList *item;
901 if (s->cancelled) {
902 return;
905 s->cancelled = true;
906 s->bs = NULL;
907 s->bitmap = NULL;
909 /* Drop all unfinished bitmaps */
910 for (item = s->bitmaps; item; item = g_slist_next(item)) {
911 LoadBitmapState *b = item->data;
914 * Bitmap must be unfinished, as finished bitmaps should already be
915 * removed from the list.
917 assert(!s->before_vm_start_handled || !b->migrated);
918 if (bdrv_dirty_bitmap_has_successor(b->bitmap)) {
919 bdrv_reclaim_dirty_bitmap(b->bitmap, &error_abort);
920 } else {
921 bdrv_dirty_bitmap_set_busy(b->bitmap, false);
923 bdrv_release_dirty_bitmap(b->bitmap);
926 g_slist_free_full(s->bitmaps, g_free);
927 s->bitmaps = NULL;
930 void dirty_bitmap_mig_cancel_outgoing(void)
932 dirty_bitmap_do_save_cleanup(&dbm_state.save);
935 void dirty_bitmap_mig_cancel_incoming(void)
937 DBMLoadState *s = &dbm_state.load;
939 qemu_mutex_lock(&s->lock);
941 cancel_incoming_locked(s);
943 qemu_mutex_unlock(&s->lock);
946 static void dirty_bitmap_load_complete(QEMUFile *f, DBMLoadState *s)
948 GSList *item;
949 trace_dirty_bitmap_load_complete();
951 if (s->cancelled) {
952 return;
955 bdrv_dirty_bitmap_deserialize_finish(s->bitmap);
957 if (bdrv_dirty_bitmap_has_successor(s->bitmap)) {
958 bdrv_reclaim_dirty_bitmap(s->bitmap, &error_abort);
959 } else {
960 bdrv_dirty_bitmap_set_busy(s->bitmap, false);
963 for (item = s->bitmaps; item; item = g_slist_next(item)) {
964 LoadBitmapState *b = item->data;
966 if (b->bitmap == s->bitmap) {
967 b->migrated = true;
968 if (s->before_vm_start_handled) {
969 s->bitmaps = g_slist_remove(s->bitmaps, b);
970 g_free(b);
972 break;
977 static int dirty_bitmap_load_bits(QEMUFile *f, DBMLoadState *s)
979 uint64_t first_byte = qemu_get_be64(f) << BDRV_SECTOR_BITS;
980 uint64_t nr_bytes = (uint64_t)qemu_get_be32(f) << BDRV_SECTOR_BITS;
981 trace_dirty_bitmap_load_bits_enter(first_byte >> BDRV_SECTOR_BITS,
982 nr_bytes >> BDRV_SECTOR_BITS);
984 if (s->flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) {
985 trace_dirty_bitmap_load_bits_zeroes();
986 if (!s->cancelled) {
987 bdrv_dirty_bitmap_deserialize_zeroes(s->bitmap, first_byte,
988 nr_bytes, false);
990 } else {
991 size_t ret;
992 g_autofree uint8_t *buf = NULL;
993 uint64_t buf_size = qemu_get_be64(f);
994 uint64_t needed_size;
997 * The actual check for buf_size is done a bit later. We can't do it in
998 * cancelled mode as we don't have the bitmap to check the constraints
999 * (so, we allocate a buffer and read prior to the check). On the other
1000 * hand, we shouldn't blindly g_malloc the number from the stream.
1001 * Actually one chunk should not be larger than CHUNK_SIZE. Let's allow
1002 * a bit larger (which means that bitmap migration will fail anyway and
1003 * the whole migration will most probably fail soon due to broken
1004 * stream).
1006 if (buf_size > 10 * CHUNK_SIZE) {
1007 error_report("Bitmap migration stream buffer allocation request "
1008 "is too large");
1009 return -EIO;
1012 buf = g_malloc(buf_size);
1013 ret = qemu_get_buffer(f, buf, buf_size);
1014 if (ret != buf_size) {
1015 error_report("Failed to read bitmap bits");
1016 return -EIO;
1019 if (s->cancelled) {
1020 return 0;
1023 needed_size = bdrv_dirty_bitmap_serialization_size(s->bitmap,
1024 first_byte,
1025 nr_bytes);
1027 if (needed_size > buf_size ||
1028 buf_size > QEMU_ALIGN_UP(needed_size, 4 * sizeof(long))
1029 /* Here used same alignment as in send_bitmap_bits */
1031 error_report("Migrated bitmap granularity doesn't "
1032 "match the destination bitmap '%s' granularity",
1033 bdrv_dirty_bitmap_name(s->bitmap));
1034 cancel_incoming_locked(s);
1035 return 0;
1038 bdrv_dirty_bitmap_deserialize_part(s->bitmap, buf, first_byte, nr_bytes,
1039 false);
1042 return 0;
1045 static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s,
1046 GHashTable *alias_map)
1048 GHashTable *bitmap_alias_map = NULL;
1049 Error *local_err = NULL;
1050 bool nothing;
1051 s->flags = qemu_get_bitmap_flags(f);
1052 trace_dirty_bitmap_load_header(s->flags);
1054 nothing = s->flags == (s->flags & DIRTY_BITMAP_MIG_FLAG_EOS);
1056 if (s->flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
1057 if (!qemu_get_counted_string(f, s->node_alias)) {
1058 error_report("Unable to read node alias string");
1059 return -EINVAL;
1062 if (!s->cancelled) {
1063 if (alias_map) {
1064 const AliasMapInnerNode *amin;
1066 amin = g_hash_table_lookup(alias_map, s->node_alias);
1067 if (!amin) {
1068 error_setg(&local_err, "Error: Unknown node alias '%s'",
1069 s->node_alias);
1070 s->bs = NULL;
1071 } else {
1072 bitmap_alias_map = amin->subtree;
1073 s->bs = bdrv_lookup_bs(NULL, amin->string, &local_err);
1075 } else {
1076 s->bs = bdrv_lookup_bs(s->node_alias, s->node_alias,
1077 &local_err);
1079 if (!s->bs) {
1080 error_report_err(local_err);
1081 cancel_incoming_locked(s);
1084 } else if (s->bs) {
1085 if (alias_map) {
1086 const AliasMapInnerNode *amin;
1088 /* Must be present in the map, or s->bs would not be set */
1089 amin = g_hash_table_lookup(alias_map, s->node_alias);
1090 assert(amin != NULL);
1092 bitmap_alias_map = amin->subtree;
1094 } else if (!nothing && !s->cancelled) {
1095 error_report("Error: block device name is not set");
1096 cancel_incoming_locked(s);
1099 assert(nothing || s->cancelled || !!alias_map == !!bitmap_alias_map);
1101 if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
1102 const char *bitmap_name;
1104 if (!qemu_get_counted_string(f, s->bitmap_alias)) {
1105 error_report("Unable to read bitmap alias string");
1106 return -EINVAL;
1109 bitmap_name = s->bitmap_alias;
1110 if (!s->cancelled && bitmap_alias_map) {
1111 BitmapMigrationBitmapAlias *bmap_inner;
1113 bmap_inner = g_hash_table_lookup(bitmap_alias_map, s->bitmap_alias);
1114 if (!bmap_inner) {
1115 error_report("Error: Unknown bitmap alias '%s' on node "
1116 "'%s' (alias '%s')", s->bitmap_alias,
1117 s->bs->node_name, s->node_alias);
1118 cancel_incoming_locked(s);
1119 } else {
1120 bitmap_name = bmap_inner->name;
1123 s->bmap_inner = bmap_inner;
1126 if (!s->cancelled) {
1127 g_strlcpy(s->bitmap_name, bitmap_name, sizeof(s->bitmap_name));
1128 s->bitmap = bdrv_find_dirty_bitmap(s->bs, s->bitmap_name);
1131 * bitmap may be NULL here, it wouldn't be an error if it is the
1132 * first occurrence of the bitmap
1134 if (!s->bitmap && !(s->flags & DIRTY_BITMAP_MIG_FLAG_START)) {
1135 error_report("Error: unknown dirty bitmap "
1136 "'%s' for block device '%s'",
1137 s->bitmap_name, s->bs->node_name);
1138 cancel_incoming_locked(s);
1141 } else if (!s->bitmap && !nothing && !s->cancelled) {
1142 error_report("Error: block device name is not set");
1143 cancel_incoming_locked(s);
1146 return 0;
1150 * dirty_bitmap_load
1152 * Load sequence of dirty bitmap chunks. Return error only on fatal io stream
1153 * violations. On other errors just cancel bitmaps incoming migration and return
1154 * 0.
1156 * Note, than when incoming bitmap migration is canceled, we still must read all
1157 * our chunks (and just ignore them), to not affect other migration objects.
1159 static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
1161 GHashTable *alias_map = NULL;
1162 const MigrationParameters *mig_params = &migrate_get_current()->parameters;
1163 DBMLoadState *s = &((DBMState *)opaque)->load;
1164 int ret = 0;
1166 trace_dirty_bitmap_load_enter();
1168 if (version_id != 1) {
1169 QEMU_LOCK_GUARD(&s->lock);
1170 cancel_incoming_locked(s);
1171 return -EINVAL;
1174 if (mig_params->has_block_bitmap_mapping) {
1175 alias_map = construct_alias_map(mig_params->block_bitmap_mapping,
1176 false, &error_abort);
1179 do {
1180 QEMU_LOCK_GUARD(&s->lock);
1182 ret = dirty_bitmap_load_header(f, s, alias_map);
1183 if (ret < 0) {
1184 cancel_incoming_locked(s);
1185 goto fail;
1188 if (s->flags & DIRTY_BITMAP_MIG_FLAG_START) {
1189 ret = dirty_bitmap_load_start(f, s);
1190 } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_COMPLETE) {
1191 dirty_bitmap_load_complete(f, s);
1192 } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITS) {
1193 ret = dirty_bitmap_load_bits(f, s);
1196 if (!ret) {
1197 ret = qemu_file_get_error(f);
1200 if (ret) {
1201 cancel_incoming_locked(s);
1202 goto fail;
1204 } while (!(s->flags & DIRTY_BITMAP_MIG_FLAG_EOS));
1206 trace_dirty_bitmap_load_success();
1207 ret = 0;
1208 fail:
1209 if (alias_map) {
1210 g_hash_table_destroy(alias_map);
1212 return ret;
1215 static int dirty_bitmap_save_setup(QEMUFile *f, void *opaque)
1217 DBMSaveState *s = &((DBMState *)opaque)->save;
1218 SaveBitmapState *dbms = NULL;
1220 qemu_mutex_lock_iothread();
1221 if (init_dirty_bitmap_migration(s) < 0) {
1222 qemu_mutex_unlock_iothread();
1223 return -1;
1226 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
1227 send_bitmap_start(f, s, dbms);
1229 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
1230 qemu_mutex_unlock_iothread();
1231 return 0;
1234 static bool dirty_bitmap_is_active(void *opaque)
1236 DBMSaveState *s = &((DBMState *)opaque)->save;
1238 return migrate_dirty_bitmaps() && !s->no_bitmaps;
1241 static bool dirty_bitmap_is_active_iterate(void *opaque)
1243 return dirty_bitmap_is_active(opaque) && !runstate_is_running();
1246 static bool dirty_bitmap_has_postcopy(void *opaque)
1248 return true;
1251 static SaveVMHandlers savevm_dirty_bitmap_handlers = {
1252 .save_setup = dirty_bitmap_save_setup,
1253 .save_live_complete_postcopy = dirty_bitmap_save_complete,
1254 .save_live_complete_precopy = dirty_bitmap_save_complete,
1255 .has_postcopy = dirty_bitmap_has_postcopy,
1256 .save_live_pending = dirty_bitmap_save_pending,
1257 .save_live_iterate = dirty_bitmap_save_iterate,
1258 .is_active_iterate = dirty_bitmap_is_active_iterate,
1259 .load_state = dirty_bitmap_load,
1260 .save_cleanup = dirty_bitmap_save_cleanup,
1261 .is_active = dirty_bitmap_is_active,
1264 void dirty_bitmap_mig_init(void)
1266 QSIMPLEQ_INIT(&dbm_state.save.dbms_list);
1267 qemu_mutex_init(&dbm_state.load.lock);
1269 register_savevm_live("dirty-bitmap", 0, 1,
1270 &savevm_dirty_bitmap_handlers,
1271 &dbm_state);