migration/block-dirty-bitmap.c

   1 /*
   2  * Block dirty bitmap postcopy migration
   3  *
   4  * Copyright IBM, Corp. 2009
   5  * Copyright (c) 2016-2017 Virtuozzo International GmbH. All rights reserved.
   6  *
   7  * Authors:
   8  *  Liran Schour   <lirans@il.ibm.com>
   9  *  Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2.  See
  12  * the COPYING file in the top-level directory.
  13  * This file is derived from migration/block.c, so it's author and IBM copyright
  14  * are here, although content is quite different.
  15  *
  16  * Contributions after 2012-01-13 are licensed under the terms of the
  17  * GNU GPL, version 2 or (at your option) any later version.
  18  *
  19  *                                ***
  20  *
  21  * Here postcopy migration of dirty bitmaps is realized. Only QMP-addressable
  22  * bitmaps are migrated.
  23  *
  24  * Bitmap migration implies creating bitmap with the same name and granularity
  25  * in destination QEMU. If the bitmap with the same name (for the same node)
  26  * already exists on destination an error will be generated.
  27  *
  28  * format of migration:
  29  *
  30  * # Header (shared for different chunk types)
  31  * 1, 2 or 4 bytes: flags (see qemu_{put,put}_flags)
  32  * [ 1 byte: node name size ] \  flags & DEVICE_NAME
  33  * [ n bytes: node name     ] /
  34  * [ 1 byte: bitmap name size ] \  flags & BITMAP_NAME
  35  * [ n bytes: bitmap name     ] /
  36  *
  37  * # Start of bitmap migration (flags & START)
  38  * header
  39  * be64: granularity
  40  * 1 byte: bitmap flags (corresponds to BdrvDirtyBitmap)
  41  *   bit 0    -  bitmap is enabled
  42  *   bit 1    -  bitmap is persistent
  43  *   bit 2    -  bitmap is autoloading
  44  *   bits 3-7 - reserved, must be zero
  45  *
  46  * # Complete of bitmap migration (flags & COMPLETE)
  47  * header
  48  *
  49  * # Data chunk of bitmap migration
  50  * header
  51  * be64: start sector
  52  * be32: number of sectors
  53  * [ be64: buffer size  ] \ ! (flags & ZEROES)
  54  * [ n bytes: buffer    ] /
  55  *
  56  * The last chunk in stream should contain flags & EOS. The chunk may skip
  57  * device and/or bitmap names, assuming them to be the same with the previous
  58  * chunk.
  59  */
  60
  61 #include "qemu/osdep.h"
  62 #include "block/block.h"
  63 #include "block/block_int.h"
  64 #include "sysemu/block-backend.h"
  65 #include "sysemu/runstate.h"
  66 #include "qemu/main-loop.h"
  67 #include "qemu/error-report.h"
  68 #include "migration/misc.h"
  69 #include "migration/migration.h"
  70 #include "qemu-file.h"
  71 #include "migration/vmstate.h"
  72 #include "migration/register.h"
  73 #include "qemu/hbitmap.h"
  74 #include "qemu/cutils.h"
  75 #include "qapi/error.h"
  76 #include "trace.h"
  77
  78 #define CHUNK_SIZE     (1 << 10)
  79
  80 /* Flags occupy one, two or four bytes (Big Endian). The size is determined as
  81  * follows:
  82  * in first (most significant) byte bit 8 is clear  -->  one byte
  83  * in first byte bit 8 is set    -->  two or four bytes, depending on second
  84  *                                    byte:
  85  *    | in second byte bit 8 is clear  -->  two bytes
  86  *    | in second byte bit 8 is set    -->  four bytes
  87  */
  88 #define DIRTY_BITMAP_MIG_FLAG_EOS           0x01
  89 #define DIRTY_BITMAP_MIG_FLAG_ZEROES        0x02
  90 #define DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME   0x04
  91 #define DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME   0x08
  92 #define DIRTY_BITMAP_MIG_FLAG_START         0x10
  93 #define DIRTY_BITMAP_MIG_FLAG_COMPLETE      0x20
  94 #define DIRTY_BITMAP_MIG_FLAG_BITS          0x40
  95
  96 #define DIRTY_BITMAP_MIG_EXTRA_FLAGS        0x80
  97
  98 #define DIRTY_BITMAP_MIG_START_FLAG_ENABLED          0x01
  99 #define DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT       0x02
 100 /* 0x04 was "AUTOLOAD" flags on elder versions, no it is ignored */
 101 #define DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK    0xf8
 102
 103 typedef struct DirtyBitmapMigBitmapState {
 104     /* Written during setup phase. */
 105     BlockDriverState *bs;
 106     const char *node_name;
 107     BdrvDirtyBitmap *bitmap;
 108     uint64_t total_sectors;
 109     uint64_t sectors_per_chunk;
 110     QSIMPLEQ_ENTRY(DirtyBitmapMigBitmapState) entry;
 111     uint8_t flags;
 112
 113     /* For bulk phase. */
 114     bool bulk_completed;
 115     uint64_t cur_sector;
 116 } DirtyBitmapMigBitmapState;
 117
 118 typedef struct DirtyBitmapMigState {
 119     QSIMPLEQ_HEAD(, DirtyBitmapMigBitmapState) dbms_list;
 120
 121     bool bulk_completed;
 122     bool no_bitmaps;
 123
 124     /* for send_bitmap_bits() */
 125     BlockDriverState *prev_bs;
 126     BdrvDirtyBitmap *prev_bitmap;
 127 } DirtyBitmapMigState;
 128
 129 typedef struct DirtyBitmapLoadState {
 130     uint32_t flags;
 131     char node_name[256];
 132     char bitmap_name[256];
 133     BlockDriverState *bs;
 134     BdrvDirtyBitmap *bitmap;
 135 } DirtyBitmapLoadState;
 136
 137 static DirtyBitmapMigState dirty_bitmap_mig_state;
 138
 139 typedef struct DirtyBitmapLoadBitmapState {
 140     BlockDriverState *bs;
 141     BdrvDirtyBitmap *bitmap;
 142     bool migrated;
 143 } DirtyBitmapLoadBitmapState;
 144 static GSList *enabled_bitmaps;
 145 QemuMutex finish_lock;
 146
 147 void init_dirty_bitmap_incoming_migration(void)
 148 {
 149     qemu_mutex_init(&finish_lock);
 150 }
 151
 152 static uint32_t qemu_get_bitmap_flags(QEMUFile *f)
 153 {
 154     uint8_t flags = qemu_get_byte(f);
 155     if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) {
 156         flags = flags << 8 | qemu_get_byte(f);
 157         if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) {
 158             flags = flags << 16 | qemu_get_be16(f);
 159         }
 160     }
 161
 162     return flags;
 163 }
 164
 165 static void qemu_put_bitmap_flags(QEMUFile *f, uint32_t flags)
 166 {
 167     /* The code currently do not send flags more than one byte */
 168     assert(!(flags & (0xffffff00 | DIRTY_BITMAP_MIG_EXTRA_FLAGS)));
 169
 170     qemu_put_byte(f, flags);
 171 }
 172
 173 static void send_bitmap_header(QEMUFile *f, DirtyBitmapMigBitmapState *dbms,
 174                                uint32_t additional_flags)
 175 {
 176     BlockDriverState *bs = dbms->bs;
 177     BdrvDirtyBitmap *bitmap = dbms->bitmap;
 178     uint32_t flags = additional_flags;
 179     trace_send_bitmap_header_enter();
 180
 181     if (bs != dirty_bitmap_mig_state.prev_bs) {
 182         dirty_bitmap_mig_state.prev_bs = bs;
 183         flags |= DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME;
 184     }
 185
 186     if (bitmap != dirty_bitmap_mig_state.prev_bitmap) {
 187         dirty_bitmap_mig_state.prev_bitmap = bitmap;
 188         flags |= DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME;
 189     }
 190
 191     qemu_put_bitmap_flags(f, flags);
 192
 193     if (flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
 194         qemu_put_counted_string(f, dbms->node_name);
 195     }
 196
 197     if (flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
 198         qemu_put_counted_string(f, bdrv_dirty_bitmap_name(bitmap));
 199     }
 200 }
 201
 202 static void send_bitmap_start(QEMUFile *f, DirtyBitmapMigBitmapState *dbms)
 203 {
 204     send_bitmap_header(f, dbms, DIRTY_BITMAP_MIG_FLAG_START);
 205     qemu_put_be32(f, bdrv_dirty_bitmap_granularity(dbms->bitmap));
 206     qemu_put_byte(f, dbms->flags);
 207 }
 208
 209 static void send_bitmap_complete(QEMUFile *f, DirtyBitmapMigBitmapState *dbms)
 210 {
 211     send_bitmap_header(f, dbms, DIRTY_BITMAP_MIG_FLAG_COMPLETE);
 212 }
 213
 214 static void send_bitmap_bits(QEMUFile *f, DirtyBitmapMigBitmapState *dbms,
 215                              uint64_t start_sector, uint32_t nr_sectors)
 216 {
 217     /* align for buffer_is_zero() */
 218     uint64_t align = 4 * sizeof(long);
 219     uint64_t unaligned_size =
 220         bdrv_dirty_bitmap_serialization_size(
 221             dbms->bitmap, start_sector << BDRV_SECTOR_BITS,
 222             (uint64_t)nr_sectors << BDRV_SECTOR_BITS);
 223     uint64_t buf_size = QEMU_ALIGN_UP(unaligned_size, align);
 224     uint8_t *buf = g_malloc0(buf_size);
 225     uint32_t flags = DIRTY_BITMAP_MIG_FLAG_BITS;
 226
 227     bdrv_dirty_bitmap_serialize_part(
 228         dbms->bitmap, buf, start_sector << BDRV_SECTOR_BITS,
 229         (uint64_t)nr_sectors << BDRV_SECTOR_BITS);
 230
 231     if (buffer_is_zero(buf, buf_size)) {
 232         g_free(buf);
 233         buf = NULL;
 234         flags |= DIRTY_BITMAP_MIG_FLAG_ZEROES;
 235     }
 236
 237     trace_send_bitmap_bits(flags, start_sector, nr_sectors, buf_size);
 238
 239     send_bitmap_header(f, dbms, flags);
 240
 241     qemu_put_be64(f, start_sector);
 242     qemu_put_be32(f, nr_sectors);
 243
 244     /* if a block is zero we need to flush here since the network
 245      * bandwidth is now a lot higher than the storage device bandwidth.
 246      * thus if we queue zero blocks we slow down the migration. */
 247     if (flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) {
 248         qemu_fflush(f);
 249     } else {
 250         qemu_put_be64(f, buf_size);
 251         qemu_put_buffer(f, buf, buf_size);
 252     }
 253
 254     g_free(buf);
 255 }
 256
 257 /* Called with iothread lock taken.  */
 258 static void dirty_bitmap_mig_cleanup(void)
 259 {
 260     DirtyBitmapMigBitmapState *dbms;
 261
 262     while ((dbms = QSIMPLEQ_FIRST(&dirty_bitmap_mig_state.dbms_list)) != NULL) {
 263         QSIMPLEQ_REMOVE_HEAD(&dirty_bitmap_mig_state.dbms_list, entry);
 264         bdrv_dirty_bitmap_set_busy(dbms->bitmap, false);
 265         bdrv_unref(dbms->bs);
 266         g_free(dbms);
 267     }
 268 }
 269
 270 /* Called with iothread lock taken. */
 271 static int add_bitmaps_to_list(BlockDriverState *bs, const char *bs_name)
 272 {
 273     BdrvDirtyBitmap *bitmap;
 274     DirtyBitmapMigBitmapState *dbms;
 275     Error *local_err = NULL;
 276
 277     FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
 278         if (bdrv_dirty_bitmap_name(bitmap)) {
 279             break;
 280         }
 281     }
 282     if (!bitmap) {
 283         return 0;
 284     }
 285
 286     if (!bs_name || strcmp(bs_name, "") == 0) {
 287         error_report("Bitmap '%s' in unnamed node can't be migrated",
 288                      bdrv_dirty_bitmap_name(bitmap));
 289         return -1;
 290     }
 291
 292     if (bs_name[0] == '#') {
 293         error_report("Bitmap '%s' in a node with auto-generated "
 294                      "name '%s' can't be migrated",
 295                      bdrv_dirty_bitmap_name(bitmap), bs_name);
 296         return -1;
 297     }
 298
 299     FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
 300         if (!bdrv_dirty_bitmap_name(bitmap)) {
 301             continue;
 302         }
 303
 304         if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_DEFAULT, &local_err)) {
 305             error_report_err(local_err);
 306             return -1;
 307         }
 308
 309         bdrv_ref(bs);
 310         bdrv_dirty_bitmap_set_busy(bitmap, true);
 311
 312         dbms = g_new0(DirtyBitmapMigBitmapState, 1);
 313         dbms->bs = bs;
 314         dbms->node_name = bs_name;
 315         dbms->bitmap = bitmap;
 316         dbms->total_sectors = bdrv_nb_sectors(bs);
 317         dbms->sectors_per_chunk = CHUNK_SIZE * 8 *
 318             bdrv_dirty_bitmap_granularity(bitmap) >> BDRV_SECTOR_BITS;
 319         if (bdrv_dirty_bitmap_enabled(bitmap)) {
 320             dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_ENABLED;
 321         }
 322         if (bdrv_dirty_bitmap_get_persistence(bitmap)) {
 323             dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
 324         }
 325
 326         QSIMPLEQ_INSERT_TAIL(&dirty_bitmap_mig_state.dbms_list,
 327                              dbms, entry);
 328     }
 329
 330     return 0;
 331 }
 332
 333 /* Called with iothread lock taken. */
 334 static int init_dirty_bitmap_migration(void)
 335 {
 336     BlockDriverState *bs;
 337     DirtyBitmapMigBitmapState *dbms;
 338     GHashTable *handled_by_blk = g_hash_table_new(NULL, NULL);
 339     BlockBackend *blk;
 340
 341     dirty_bitmap_mig_state.bulk_completed = false;
 342     dirty_bitmap_mig_state.prev_bs = NULL;
 343     dirty_bitmap_mig_state.prev_bitmap = NULL;
 344     dirty_bitmap_mig_state.no_bitmaps = false;
 345
 346     /*
 347      * Use blockdevice name for direct (or filtered) children of named block
 348      * backends.
 349      */
 350     for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
 351         const char *name = blk_name(blk);
 352
 353         if (!name || strcmp(name, "") == 0) {
 354             continue;
 355         }
 356
 357         bs = blk_bs(blk);
 358
 359         /* Skip filters without bitmaps */
 360         while (bs && bs->drv && bs->drv->is_filter &&
 361                !bdrv_has_named_bitmaps(bs))
 362         {
 363             if (bs->backing) {
 364                 bs = bs->backing->bs;
 365             } else if (bs->file) {
 366                 bs = bs->file->bs;
 367             } else {
 368                 bs = NULL;
 369             }
 370         }
 371
 372         if (bs && bs->drv && !bs->drv->is_filter) {
 373             if (add_bitmaps_to_list(bs, name)) {
 374                 goto fail;
 375             }
 376             g_hash_table_add(handled_by_blk, bs);
 377         }
 378     }
 379
 380     for (bs = bdrv_next_all_states(NULL); bs; bs = bdrv_next_all_states(bs)) {
 381         if (g_hash_table_contains(handled_by_blk, bs)) {
 382             continue;
 383         }
 384
 385         if (add_bitmaps_to_list(bs, bdrv_get_node_name(bs))) {
 386             goto fail;
 387         }
 388     }
 389
 390     /* unset migration flags here, to not roll back it */
 391     QSIMPLEQ_FOREACH(dbms, &dirty_bitmap_mig_state.dbms_list, entry) {
 392         bdrv_dirty_bitmap_skip_store(dbms->bitmap, true);
 393     }
 394
 395     if (QSIMPLEQ_EMPTY(&dirty_bitmap_mig_state.dbms_list)) {
 396         dirty_bitmap_mig_state.no_bitmaps = true;
 397     }
 398
 399     g_hash_table_destroy(handled_by_blk);
 400
 401     return 0;
 402
 403 fail:
 404     g_hash_table_destroy(handled_by_blk);
 405     dirty_bitmap_mig_cleanup();
 406
 407     return -1;
 408 }
 409
 410 /* Called with no lock taken.  */
 411 static void bulk_phase_send_chunk(QEMUFile *f, DirtyBitmapMigBitmapState *dbms)
 412 {
 413     uint32_t nr_sectors = MIN(dbms->total_sectors - dbms->cur_sector,
 414                              dbms->sectors_per_chunk);
 415
 416     send_bitmap_bits(f, dbms, dbms->cur_sector, nr_sectors);
 417
 418     dbms->cur_sector += nr_sectors;
 419     if (dbms->cur_sector >= dbms->total_sectors) {
 420         dbms->bulk_completed = true;
 421     }
 422 }
 423
 424 /* Called with no lock taken.  */
 425 static void bulk_phase(QEMUFile *f, bool limit)
 426 {
 427     DirtyBitmapMigBitmapState *dbms;
 428
 429     QSIMPLEQ_FOREACH(dbms, &dirty_bitmap_mig_state.dbms_list, entry) {
 430         while (!dbms->bulk_completed) {
 431             bulk_phase_send_chunk(f, dbms);
 432             if (limit && qemu_file_rate_limit(f)) {
 433                 return;
 434             }
 435         }
 436     }
 437
 438     dirty_bitmap_mig_state.bulk_completed = true;
 439 }
 440
 441 /* for SaveVMHandlers */
 442 static void dirty_bitmap_save_cleanup(void *opaque)
 443 {
 444     dirty_bitmap_mig_cleanup();
 445 }
 446
 447 static int dirty_bitmap_save_iterate(QEMUFile *f, void *opaque)
 448 {
 449     trace_dirty_bitmap_save_iterate(migration_in_postcopy());
 450
 451     if (migration_in_postcopy() && !dirty_bitmap_mig_state.bulk_completed) {
 452         bulk_phase(f, true);
 453     }
 454
 455     qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
 456
 457     return dirty_bitmap_mig_state.bulk_completed;
 458 }
 459
 460 /* Called with iothread lock taken.  */
 461
 462 static int dirty_bitmap_save_complete(QEMUFile *f, void *opaque)
 463 {
 464     DirtyBitmapMigBitmapState *dbms;
 465     trace_dirty_bitmap_save_complete_enter();
 466
 467     if (!dirty_bitmap_mig_state.bulk_completed) {
 468         bulk_phase(f, false);
 469     }
 470
 471     QSIMPLEQ_FOREACH(dbms, &dirty_bitmap_mig_state.dbms_list, entry) {
 472         send_bitmap_complete(f, dbms);
 473     }
 474
 475     qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
 476
 477     trace_dirty_bitmap_save_complete_finish();
 478
 479     dirty_bitmap_mig_cleanup();
 480     return 0;
 481 }
 482
 483 static void dirty_bitmap_save_pending(QEMUFile *f, void *opaque,
 484                                       uint64_t max_size,
 485                                       uint64_t *res_precopy_only,
 486                                       uint64_t *res_compatible,
 487                                       uint64_t *res_postcopy_only)
 488 {
 489     DirtyBitmapMigBitmapState *dbms;
 490     uint64_t pending = 0;
 491
 492     qemu_mutex_lock_iothread();
 493
 494     QSIMPLEQ_FOREACH(dbms, &dirty_bitmap_mig_state.dbms_list, entry) {
 495         uint64_t gran = bdrv_dirty_bitmap_granularity(dbms->bitmap);
 496         uint64_t sectors = dbms->bulk_completed ? 0 :
 497                            dbms->total_sectors - dbms->cur_sector;
 498
 499         pending += DIV_ROUND_UP(sectors * BDRV_SECTOR_SIZE, gran);
 500     }
 501
 502     qemu_mutex_unlock_iothread();
 503
 504     trace_dirty_bitmap_save_pending(pending, max_size);
 505
 506     *res_postcopy_only += pending;
 507 }
 508
 509 /* First occurrence of this bitmap. It should be created if doesn't exist */
 510 static int dirty_bitmap_load_start(QEMUFile *f, DirtyBitmapLoadState *s)
 511 {
 512     Error *local_err = NULL;
 513     uint32_t granularity = qemu_get_be32(f);
 514     uint8_t flags = qemu_get_byte(f);
 515
 516     if (s->bitmap) {
 517         error_report("Bitmap with the same name ('%s') already exists on "
 518                      "destination", bdrv_dirty_bitmap_name(s->bitmap));
 519         return -EINVAL;
 520     } else {
 521         s->bitmap = bdrv_create_dirty_bitmap(s->bs, granularity,
 522                                              s->bitmap_name, &local_err);
 523         if (!s->bitmap) {
 524             error_report_err(local_err);
 525             return -EINVAL;
 526         }
 527     }
 528
 529     if (flags & DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK) {
 530         error_report("Unknown flags in migrated dirty bitmap header: %x",
 531                      flags);
 532         return -EINVAL;
 533     }
 534
 535     if (flags & DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT) {
 536         bdrv_dirty_bitmap_set_persistence(s->bitmap, true);
 537     }
 538
 539     bdrv_disable_dirty_bitmap(s->bitmap);
 540     if (flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED) {
 541         DirtyBitmapLoadBitmapState *b;
 542
 543         bdrv_dirty_bitmap_create_successor(s->bitmap, &local_err);
 544         if (local_err) {
 545             error_report_err(local_err);
 546             return -EINVAL;
 547         }
 548
 549         b = g_new(DirtyBitmapLoadBitmapState, 1);
 550         b->bs = s->bs;
 551         b->bitmap = s->bitmap;
 552         b->migrated = false;
 553         enabled_bitmaps = g_slist_prepend(enabled_bitmaps, b);
 554     }
 555
 556     return 0;
 557 }
 558
 559 void dirty_bitmap_mig_before_vm_start(void)
 560 {
 561     GSList *item;
 562
 563     qemu_mutex_lock(&finish_lock);
 564
 565     for (item = enabled_bitmaps; item; item = g_slist_next(item)) {
 566         DirtyBitmapLoadBitmapState *b = item->data;
 567
 568         if (b->migrated) {
 569             bdrv_enable_dirty_bitmap(b->bitmap);
 570         } else {
 571             bdrv_dirty_bitmap_enable_successor(b->bitmap);
 572         }
 573
 574         g_free(b);
 575     }
 576
 577     g_slist_free(enabled_bitmaps);
 578     enabled_bitmaps = NULL;
 579
 580     qemu_mutex_unlock(&finish_lock);
 581 }
 582
 583 static void dirty_bitmap_load_complete(QEMUFile *f, DirtyBitmapLoadState *s)
 584 {
 585     GSList *item;
 586     trace_dirty_bitmap_load_complete();
 587     bdrv_dirty_bitmap_deserialize_finish(s->bitmap);
 588
 589     qemu_mutex_lock(&finish_lock);
 590
 591     for (item = enabled_bitmaps; item; item = g_slist_next(item)) {
 592         DirtyBitmapLoadBitmapState *b = item->data;
 593
 594         if (b->bitmap == s->bitmap) {
 595             b->migrated = true;
 596             break;
 597         }
 598     }
 599
 600     if (bdrv_dirty_bitmap_has_successor(s->bitmap)) {
 601         bdrv_dirty_bitmap_lock(s->bitmap);
 602         if (enabled_bitmaps == NULL) {
 603             /* in postcopy */
 604             bdrv_reclaim_dirty_bitmap_locked(s->bitmap, &error_abort);
 605             bdrv_enable_dirty_bitmap_locked(s->bitmap);
 606         } else {
 607             /* target not started, successor must be empty */
 608             int64_t count = bdrv_get_dirty_count(s->bitmap);
 609             BdrvDirtyBitmap *ret = bdrv_reclaim_dirty_bitmap_locked(s->bitmap,
 610                                                                     NULL);
 611             /* bdrv_reclaim_dirty_bitmap can fail only on no successor (it
 612              * must be) or on merge fail, but merge can't fail when second
 613              * bitmap is empty
 614              */
 615             assert(ret == s->bitmap &&
 616                    count == bdrv_get_dirty_count(s->bitmap));
 617         }
 618         bdrv_dirty_bitmap_unlock(s->bitmap);
 619     }
 620
 621     qemu_mutex_unlock(&finish_lock);
 622 }
 623
 624 static int dirty_bitmap_load_bits(QEMUFile *f, DirtyBitmapLoadState *s)
 625 {
 626     uint64_t first_byte = qemu_get_be64(f) << BDRV_SECTOR_BITS;
 627     uint64_t nr_bytes = (uint64_t)qemu_get_be32(f) << BDRV_SECTOR_BITS;
 628     trace_dirty_bitmap_load_bits_enter(first_byte >> BDRV_SECTOR_BITS,
 629                                        nr_bytes >> BDRV_SECTOR_BITS);
 630
 631     if (s->flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) {
 632         trace_dirty_bitmap_load_bits_zeroes();
 633         bdrv_dirty_bitmap_deserialize_zeroes(s->bitmap, first_byte, nr_bytes,
 634                                              false);
 635     } else {
 636         size_t ret;
 637         uint8_t *buf;
 638         uint64_t buf_size = qemu_get_be64(f);
 639         uint64_t needed_size =
 640             bdrv_dirty_bitmap_serialization_size(s->bitmap,
 641                                                  first_byte, nr_bytes);
 642
 643         if (needed_size > buf_size ||
 644             buf_size > QEMU_ALIGN_UP(needed_size, 4 * sizeof(long))
 645              /* Here used same alignment as in send_bitmap_bits */
 646         ) {
 647             error_report("Migrated bitmap granularity doesn't "
 648                          "match the destination bitmap '%s' granularity",
 649                          bdrv_dirty_bitmap_name(s->bitmap));
 650             return -EINVAL;
 651         }
 652
 653         buf = g_malloc(buf_size);
 654         ret = qemu_get_buffer(f, buf, buf_size);
 655         if (ret != buf_size) {
 656             error_report("Failed to read bitmap bits");
 657             g_free(buf);
 658             return -EIO;
 659         }
 660
 661         bdrv_dirty_bitmap_deserialize_part(s->bitmap, buf, first_byte, nr_bytes,
 662                                            false);
 663         g_free(buf);
 664     }
 665
 666     return 0;
 667 }
 668
 669 static int dirty_bitmap_load_header(QEMUFile *f, DirtyBitmapLoadState *s)
 670 {
 671     Error *local_err = NULL;
 672     bool nothing;
 673     s->flags = qemu_get_bitmap_flags(f);
 674     trace_dirty_bitmap_load_header(s->flags);
 675
 676     nothing = s->flags == (s->flags & DIRTY_BITMAP_MIG_FLAG_EOS);
 677
 678     if (s->flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
 679         if (!qemu_get_counted_string(f, s->node_name)) {
 680             error_report("Unable to read node name string");
 681             return -EINVAL;
 682         }
 683         s->bs = bdrv_lookup_bs(s->node_name, s->node_name, &local_err);
 684         if (!s->bs) {
 685             error_report_err(local_err);
 686             return -EINVAL;
 687         }
 688     } else if (!s->bs && !nothing) {
 689         error_report("Error: block device name is not set");
 690         return -EINVAL;
 691     }
 692
 693     if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
 694         if (!qemu_get_counted_string(f, s->bitmap_name)) {
 695             error_report("Unable to read bitmap name string");
 696             return -EINVAL;
 697         }
 698         s->bitmap = bdrv_find_dirty_bitmap(s->bs, s->bitmap_name);
 699
 700         /* bitmap may be NULL here, it wouldn't be an error if it is the
 701          * first occurrence of the bitmap */
 702         if (!s->bitmap && !(s->flags & DIRTY_BITMAP_MIG_FLAG_START)) {
 703             error_report("Error: unknown dirty bitmap "
 704                          "'%s' for block device '%s'",
 705                          s->bitmap_name, s->node_name);
 706             return -EINVAL;
 707         }
 708     } else if (!s->bitmap && !nothing) {
 709         error_report("Error: block device name is not set");
 710         return -EINVAL;
 711     }
 712
 713     return 0;
 714 }
 715
 716 static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
 717 {
 718     static DirtyBitmapLoadState s;
 719     int ret = 0;
 720
 721     trace_dirty_bitmap_load_enter();
 722
 723     if (version_id != 1) {
 724         return -EINVAL;
 725     }
 726
 727     do {
 728         ret = dirty_bitmap_load_header(f, &s);
 729         if (ret < 0) {
 730             return ret;
 731         }
 732
 733         if (s.flags & DIRTY_BITMAP_MIG_FLAG_START) {
 734             ret = dirty_bitmap_load_start(f, &s);
 735         } else if (s.flags & DIRTY_BITMAP_MIG_FLAG_COMPLETE) {
 736             dirty_bitmap_load_complete(f, &s);
 737         } else if (s.flags & DIRTY_BITMAP_MIG_FLAG_BITS) {
 738             ret = dirty_bitmap_load_bits(f, &s);
 739         }
 740
 741         if (!ret) {
 742             ret = qemu_file_get_error(f);
 743         }
 744
 745         if (ret) {
 746             return ret;
 747         }
 748     } while (!(s.flags & DIRTY_BITMAP_MIG_FLAG_EOS));
 749
 750     trace_dirty_bitmap_load_success();
 751     return 0;
 752 }
 753
 754 static int dirty_bitmap_save_setup(QEMUFile *f, void *opaque)
 755 {
 756     DirtyBitmapMigBitmapState *dbms = NULL;
 757     if (init_dirty_bitmap_migration() < 0) {
 758         return -1;
 759     }
 760
 761     QSIMPLEQ_FOREACH(dbms, &dirty_bitmap_mig_state.dbms_list, entry) {
 762         send_bitmap_start(f, dbms);
 763     }
 764     qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
 765
 766     return 0;
 767 }
 768
 769 static bool dirty_bitmap_is_active(void *opaque)
 770 {
 771     return migrate_dirty_bitmaps() && !dirty_bitmap_mig_state.no_bitmaps;
 772 }
 773
 774 static bool dirty_bitmap_is_active_iterate(void *opaque)
 775 {
 776     return dirty_bitmap_is_active(opaque) && !runstate_is_running();
 777 }
 778
 779 static bool dirty_bitmap_has_postcopy(void *opaque)
 780 {
 781     return true;
 782 }
 783
 784 static SaveVMHandlers savevm_dirty_bitmap_handlers = {
 785     .save_setup = dirty_bitmap_save_setup,
 786     .save_live_complete_postcopy = dirty_bitmap_save_complete,
 787     .save_live_complete_precopy = dirty_bitmap_save_complete,
 788     .has_postcopy = dirty_bitmap_has_postcopy,
 789     .save_live_pending = dirty_bitmap_save_pending,
 790     .save_live_iterate = dirty_bitmap_save_iterate,
 791     .is_active_iterate = dirty_bitmap_is_active_iterate,
 792     .load_state = dirty_bitmap_load,
 793     .save_cleanup = dirty_bitmap_save_cleanup,
 794     .is_active = dirty_bitmap_is_active,
 795 };
 796
 797 void dirty_bitmap_mig_init(void)
 798 {
 799     QSIMPLEQ_INIT(&dirty_bitmap_mig_state.dbms_list);
 800
 801     register_savevm_live("dirty-bitmap", 0, 1,
 802                          &savevm_dirty_bitmap_handlers,
 803                          &dirty_bitmap_mig_state);
 804 }