migration/block.c

   1 /*
   2  * QEMU live block migration
   3  *
   4  * Copyright IBM, Corp. 2009
   5  *
   6  * Authors:
   7  *  Liran Schour   <lirans@il.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include "qemu-common.h"
  18 #include "block/block.h"
  19 #include "qemu/error-report.h"
  20 #include "qemu/main-loop.h"
  21 #include "hw/hw.h"
  22 #include "qemu/queue.h"
  23 #include "qemu/timer.h"
  24 #include "migration/block.h"
  25 #include "migration/migration.h"
  26 #include "sysemu/blockdev.h"
  27 #include "sysemu/block-backend.h"
  28
  29 #define BLOCK_SIZE                       (1 << 20)
  30 #define BDRV_SECTORS_PER_DIRTY_CHUNK     (BLOCK_SIZE >> BDRV_SECTOR_BITS)
  31
  32 #define BLK_MIG_FLAG_DEVICE_BLOCK       0x01
  33 #define BLK_MIG_FLAG_EOS                0x02
  34 #define BLK_MIG_FLAG_PROGRESS           0x04
  35 #define BLK_MIG_FLAG_ZERO_BLOCK         0x08
  36
  37 #define MAX_IS_ALLOCATED_SEARCH 65536
  38
  39 #define MAX_INFLIGHT_IO 512
  40
  41 //#define DEBUG_BLK_MIGRATION
  42
  43 #ifdef DEBUG_BLK_MIGRATION
  44 #define DPRINTF(fmt, ...) \
  45     do { printf("blk_migration: " fmt, ## __VA_ARGS__); } while (0)
  46 #else
  47 #define DPRINTF(fmt, ...) \
  48     do { } while (0)
  49 #endif
  50
  51 typedef struct BlkMigDevState {
  52     /* Written during setup phase.  Can be read without a lock.  */
  53     BlockDriverState *bs;
  54     int shared_base;
  55     int64_t total_sectors;
  56     QSIMPLEQ_ENTRY(BlkMigDevState) entry;
  57     Error *blocker;
  58
  59     /* Only used by migration thread.  Does not need a lock.  */
  60     int bulk_completed;
  61     int64_t cur_sector;
  62     int64_t cur_dirty;
  63
  64     /* Data in the aio_bitmap is protected by block migration lock.
  65      * Allocation and free happen during setup and cleanup respectively.
  66      */
  67     unsigned long *aio_bitmap;
  68
  69     /* Protected by block migration lock.  */
  70     int64_t completed_sectors;
  71
  72     /* During migration this is protected by iothread lock / AioContext.
  73      * Allocation and free happen during setup and cleanup respectively.
  74      */
  75     BdrvDirtyBitmap *dirty_bitmap;
  76 } BlkMigDevState;
  77
  78 typedef struct BlkMigBlock {
  79     /* Only used by migration thread.  */
  80     uint8_t *buf;
  81     BlkMigDevState *bmds;
  82     int64_t sector;
  83     int nr_sectors;
  84     struct iovec iov;
  85     QEMUIOVector qiov;
  86     BlockAIOCB *aiocb;
  87
  88     /* Protected by block migration lock.  */
  89     int ret;
  90     QSIMPLEQ_ENTRY(BlkMigBlock) entry;
  91 } BlkMigBlock;
  92
  93 typedef struct BlkMigState {
  94     /* Written during setup phase.  Can be read without a lock.  */
  95     int blk_enable;
  96     int shared_base;
  97     QSIMPLEQ_HEAD(bmds_list, BlkMigDevState) bmds_list;
  98     int64_t total_sector_sum;
  99     bool zero_blocks;
 100
 101     /* Protected by lock.  */
 102     QSIMPLEQ_HEAD(blk_list, BlkMigBlock) blk_list;
 103     int submitted;
 104     int read_done;
 105
 106     /* Only used by migration thread.  Does not need a lock.  */
 107     int transferred;
 108     int prev_progress;
 109     int bulk_completed;
 110
 111     /* Lock must be taken _inside_ the iothread lock and any AioContexts.  */
 112     QemuMutex lock;
 113 } BlkMigState;
 114
 115 static BlkMigState block_mig_state;
 116
 117 static void blk_mig_lock(void)
 118 {
 119     qemu_mutex_lock(&block_mig_state.lock);
 120 }
 121
 122 static void blk_mig_unlock(void)
 123 {
 124     qemu_mutex_unlock(&block_mig_state.lock);
 125 }
 126
 127 /* Must run outside of the iothread lock during the bulk phase,
 128  * or the VM will stall.
 129  */
 130
 131 static void blk_send(QEMUFile *f, BlkMigBlock * blk)
 132 {
 133     int len;
 134     uint64_t flags = BLK_MIG_FLAG_DEVICE_BLOCK;
 135
 136     if (block_mig_state.zero_blocks &&
 137         buffer_is_zero(blk->buf, BLOCK_SIZE)) {
 138         flags |= BLK_MIG_FLAG_ZERO_BLOCK;
 139     }
 140
 141     /* sector number and flags */
 142     qemu_put_be64(f, (blk->sector << BDRV_SECTOR_BITS)
 143                      | flags);
 144
 145     /* device name */
 146     len = strlen(bdrv_get_device_name(blk->bmds->bs));
 147     qemu_put_byte(f, len);
 148     qemu_put_buffer(f, (uint8_t *)bdrv_get_device_name(blk->bmds->bs), len);
 149
 150     /* if a block is zero we need to flush here since the network
 151      * bandwidth is now a lot higher than the storage device bandwidth.
 152      * thus if we queue zero blocks we slow down the migration */
 153     if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
 154         qemu_fflush(f);
 155         return;
 156     }
 157
 158     qemu_put_buffer(f, blk->buf, BLOCK_SIZE);
 159 }
 160
 161 int blk_mig_active(void)
 162 {
 163     return !QSIMPLEQ_EMPTY(&block_mig_state.bmds_list);
 164 }
 165
 166 uint64_t blk_mig_bytes_transferred(void)
 167 {
 168     BlkMigDevState *bmds;
 169     uint64_t sum = 0;
 170
 171     blk_mig_lock();
 172     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 173         sum += bmds->completed_sectors;
 174     }
 175     blk_mig_unlock();
 176     return sum << BDRV_SECTOR_BITS;
 177 }
 178
 179 uint64_t blk_mig_bytes_remaining(void)
 180 {
 181     return blk_mig_bytes_total() - blk_mig_bytes_transferred();
 182 }
 183
 184 uint64_t blk_mig_bytes_total(void)
 185 {
 186     BlkMigDevState *bmds;
 187     uint64_t sum = 0;
 188
 189     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 190         sum += bmds->total_sectors;
 191     }
 192     return sum << BDRV_SECTOR_BITS;
 193 }
 194
 195
 196 /* Called with migration lock held.  */
 197
 198 static int bmds_aio_inflight(BlkMigDevState *bmds, int64_t sector)
 199 {
 200     int64_t chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;
 201
 202     if (sector < bdrv_nb_sectors(bmds->bs)) {
 203         return !!(bmds->aio_bitmap[chunk / (sizeof(unsigned long) * 8)] &
 204             (1UL << (chunk % (sizeof(unsigned long) * 8))));
 205     } else {
 206         return 0;
 207     }
 208 }
 209
 210 /* Called with migration lock held.  */
 211
 212 static void bmds_set_aio_inflight(BlkMigDevState *bmds, int64_t sector_num,
 213                              int nb_sectors, int set)
 214 {
 215     int64_t start, end;
 216     unsigned long val, idx, bit;
 217
 218     start = sector_num / BDRV_SECTORS_PER_DIRTY_CHUNK;
 219     end = (sector_num + nb_sectors - 1) / BDRV_SECTORS_PER_DIRTY_CHUNK;
 220
 221     for (; start <= end; start++) {
 222         idx = start / (sizeof(unsigned long) * 8);
 223         bit = start % (sizeof(unsigned long) * 8);
 224         val = bmds->aio_bitmap[idx];
 225         if (set) {
 226             val |= 1UL << bit;
 227         } else {
 228             val &= ~(1UL << bit);
 229         }
 230         bmds->aio_bitmap[idx] = val;
 231     }
 232 }
 233
 234 static void alloc_aio_bitmap(BlkMigDevState *bmds)
 235 {
 236     BlockDriverState *bs = bmds->bs;
 237     int64_t bitmap_size;
 238
 239     bitmap_size = bdrv_nb_sectors(bs) + BDRV_SECTORS_PER_DIRTY_CHUNK * 8 - 1;
 240     bitmap_size /= BDRV_SECTORS_PER_DIRTY_CHUNK * 8;
 241
 242     bmds->aio_bitmap = g_malloc0(bitmap_size);
 243 }
 244
 245 /* Never hold migration lock when yielding to the main loop!  */
 246
 247 static void blk_mig_read_cb(void *opaque, int ret)
 248 {
 249     BlkMigBlock *blk = opaque;
 250
 251     blk_mig_lock();
 252     blk->ret = ret;
 253
 254     QSIMPLEQ_INSERT_TAIL(&block_mig_state.blk_list, blk, entry);
 255     bmds_set_aio_inflight(blk->bmds, blk->sector, blk->nr_sectors, 0);
 256
 257     block_mig_state.submitted--;
 258     block_mig_state.read_done++;
 259     assert(block_mig_state.submitted >= 0);
 260     blk_mig_unlock();
 261 }
 262
 263 /* Called with no lock taken.  */
 264
 265 static int mig_save_device_bulk(QEMUFile *f, BlkMigDevState *bmds)
 266 {
 267     int64_t total_sectors = bmds->total_sectors;
 268     int64_t cur_sector = bmds->cur_sector;
 269     BlockDriverState *bs = bmds->bs;
 270     BlkMigBlock *blk;
 271     int nr_sectors;
 272
 273     if (bmds->shared_base) {
 274         qemu_mutex_lock_iothread();
 275         aio_context_acquire(bdrv_get_aio_context(bs));
 276         while (cur_sector < total_sectors &&
 277                !bdrv_is_allocated(bs, cur_sector, MAX_IS_ALLOCATED_SEARCH,
 278                                   &nr_sectors)) {
 279             cur_sector += nr_sectors;
 280         }
 281         aio_context_release(bdrv_get_aio_context(bs));
 282         qemu_mutex_unlock_iothread();
 283     }
 284
 285     if (cur_sector >= total_sectors) {
 286         bmds->cur_sector = bmds->completed_sectors = total_sectors;
 287         return 1;
 288     }
 289
 290     bmds->completed_sectors = cur_sector;
 291
 292     cur_sector &= ~((int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK - 1);
 293
 294     /* we are going to transfer a full block even if it is not allocated */
 295     nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
 296
 297     if (total_sectors - cur_sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
 298         nr_sectors = total_sectors - cur_sector;
 299     }
 300
 301     blk = g_new(BlkMigBlock, 1);
 302     blk->buf = g_malloc(BLOCK_SIZE);
 303     blk->bmds = bmds;
 304     blk->sector = cur_sector;
 305     blk->nr_sectors = nr_sectors;
 306
 307     blk->iov.iov_base = blk->buf;
 308     blk->iov.iov_len = nr_sectors * BDRV_SECTOR_SIZE;
 309     qemu_iovec_init_external(&blk->qiov, &blk->iov, 1);
 310
 311     blk_mig_lock();
 312     block_mig_state.submitted++;
 313     blk_mig_unlock();
 314
 315     /* We do not know if bs is under the main thread (and thus does
 316      * not acquire the AioContext when doing AIO) or rather under
 317      * dataplane.  Thus acquire both the iothread mutex and the
 318      * AioContext.
 319      *
 320      * This is ugly and will disappear when we make bdrv_* thread-safe,
 321      * without the need to acquire the AioContext.
 322      */
 323     qemu_mutex_lock_iothread();
 324     aio_context_acquire(bdrv_get_aio_context(bmds->bs));
 325     blk->aiocb = bdrv_aio_readv(bs, cur_sector, &blk->qiov,
 326                                 nr_sectors, blk_mig_read_cb, blk);
 327
 328     bdrv_reset_dirty_bitmap(bmds->dirty_bitmap, cur_sector, nr_sectors);
 329     aio_context_release(bdrv_get_aio_context(bmds->bs));
 330     qemu_mutex_unlock_iothread();
 331
 332     bmds->cur_sector = cur_sector + nr_sectors;
 333     return (bmds->cur_sector >= total_sectors);
 334 }
 335
 336 /* Called with iothread lock taken.  */
 337
 338 static int set_dirty_tracking(void)
 339 {
 340     BlkMigDevState *bmds;
 341     int ret;
 342
 343     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 344         aio_context_acquire(bdrv_get_aio_context(bmds->bs));
 345         bmds->dirty_bitmap = bdrv_create_dirty_bitmap(bmds->bs, BLOCK_SIZE,
 346                                                       NULL, NULL);
 347         aio_context_release(bdrv_get_aio_context(bmds->bs));
 348         if (!bmds->dirty_bitmap) {
 349             ret = -errno;
 350             goto fail;
 351         }
 352     }
 353     return 0;
 354
 355 fail:
 356     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 357         if (bmds->dirty_bitmap) {
 358             aio_context_acquire(bdrv_get_aio_context(bmds->bs));
 359             bdrv_release_dirty_bitmap(bmds->bs, bmds->dirty_bitmap);
 360             aio_context_release(bdrv_get_aio_context(bmds->bs));
 361         }
 362     }
 363     return ret;
 364 }
 365
 366 /* Called with iothread lock taken.  */
 367
 368 static void unset_dirty_tracking(void)
 369 {
 370     BlkMigDevState *bmds;
 371
 372     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 373         aio_context_acquire(bdrv_get_aio_context(bmds->bs));
 374         bdrv_release_dirty_bitmap(bmds->bs, bmds->dirty_bitmap);
 375         aio_context_release(bdrv_get_aio_context(bmds->bs));
 376     }
 377 }
 378
 379 static void init_blk_migration(QEMUFile *f)
 380 {
 381     BlockDriverState *bs;
 382     BlkMigDevState *bmds;
 383     int64_t sectors;
 384
 385     block_mig_state.submitted = 0;
 386     block_mig_state.read_done = 0;
 387     block_mig_state.transferred = 0;
 388     block_mig_state.total_sector_sum = 0;
 389     block_mig_state.prev_progress = -1;
 390     block_mig_state.bulk_completed = 0;
 391     block_mig_state.zero_blocks = migrate_zero_blocks();
 392
 393     for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
 394         if (bdrv_is_read_only(bs)) {
 395             continue;
 396         }
 397
 398         sectors = bdrv_nb_sectors(bs);
 399         if (sectors <= 0) {
 400             return;
 401         }
 402
 403         bmds = g_new0(BlkMigDevState, 1);
 404         bmds->bs = bs;
 405         bmds->bulk_completed = 0;
 406         bmds->total_sectors = sectors;
 407         bmds->completed_sectors = 0;
 408         bmds->shared_base = block_mig_state.shared_base;
 409         alloc_aio_bitmap(bmds);
 410         error_setg(&bmds->blocker, "block device is in use by migration");
 411         bdrv_op_block_all(bs, bmds->blocker);
 412         bdrv_ref(bs);
 413
 414         block_mig_state.total_sector_sum += sectors;
 415
 416         if (bmds->shared_base) {
 417             DPRINTF("Start migration for %s with shared base image\n",
 418                     bdrv_get_device_name(bs));
 419         } else {
 420             DPRINTF("Start full migration for %s\n", bdrv_get_device_name(bs));
 421         }
 422
 423         QSIMPLEQ_INSERT_TAIL(&block_mig_state.bmds_list, bmds, entry);
 424     }
 425 }
 426
 427 /* Called with no lock taken.  */
 428
 429 static int blk_mig_save_bulked_block(QEMUFile *f)
 430 {
 431     int64_t completed_sector_sum = 0;
 432     BlkMigDevState *bmds;
 433     int progress;
 434     int ret = 0;
 435
 436     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 437         if (bmds->bulk_completed == 0) {
 438             if (mig_save_device_bulk(f, bmds) == 1) {
 439                 /* completed bulk section for this device */
 440                 bmds->bulk_completed = 1;
 441             }
 442             completed_sector_sum += bmds->completed_sectors;
 443             ret = 1;
 444             break;
 445         } else {
 446             completed_sector_sum += bmds->completed_sectors;
 447         }
 448     }
 449
 450     if (block_mig_state.total_sector_sum != 0) {
 451         progress = completed_sector_sum * 100 /
 452                    block_mig_state.total_sector_sum;
 453     } else {
 454         progress = 100;
 455     }
 456     if (progress != block_mig_state.prev_progress) {
 457         block_mig_state.prev_progress = progress;
 458         qemu_put_be64(f, (progress << BDRV_SECTOR_BITS)
 459                          | BLK_MIG_FLAG_PROGRESS);
 460         DPRINTF("Completed %d %%\r", progress);
 461     }
 462
 463     return ret;
 464 }
 465
 466 static void blk_mig_reset_dirty_cursor(void)
 467 {
 468     BlkMigDevState *bmds;
 469
 470     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 471         bmds->cur_dirty = 0;
 472     }
 473 }
 474
 475 /* Called with iothread lock and AioContext taken.  */
 476
 477 static int mig_save_device_dirty(QEMUFile *f, BlkMigDevState *bmds,
 478                                  int is_async)
 479 {
 480     BlkMigBlock *blk;
 481     int64_t total_sectors = bmds->total_sectors;
 482     int64_t sector;
 483     int nr_sectors;
 484     int ret = -EIO;
 485
 486     for (sector = bmds->cur_dirty; sector < bmds->total_sectors;) {
 487         blk_mig_lock();
 488         if (bmds_aio_inflight(bmds, sector)) {
 489             blk_mig_unlock();
 490             bdrv_drain(bmds->bs);
 491         } else {
 492             blk_mig_unlock();
 493         }
 494         if (bdrv_get_dirty(bmds->bs, bmds->dirty_bitmap, sector)) {
 495
 496             if (total_sectors - sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
 497                 nr_sectors = total_sectors - sector;
 498             } else {
 499                 nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
 500             }
 501             blk = g_new(BlkMigBlock, 1);
 502             blk->buf = g_malloc(BLOCK_SIZE);
 503             blk->bmds = bmds;
 504             blk->sector = sector;
 505             blk->nr_sectors = nr_sectors;
 506
 507             if (is_async) {
 508                 blk->iov.iov_base = blk->buf;
 509                 blk->iov.iov_len = nr_sectors * BDRV_SECTOR_SIZE;
 510                 qemu_iovec_init_external(&blk->qiov, &blk->iov, 1);
 511
 512                 blk->aiocb = bdrv_aio_readv(bmds->bs, sector, &blk->qiov,
 513                                             nr_sectors, blk_mig_read_cb, blk);
 514
 515                 blk_mig_lock();
 516                 block_mig_state.submitted++;
 517                 bmds_set_aio_inflight(bmds, sector, nr_sectors, 1);
 518                 blk_mig_unlock();
 519             } else {
 520                 ret = bdrv_read(bmds->bs, sector, blk->buf, nr_sectors);
 521                 if (ret < 0) {
 522                     goto error;
 523                 }
 524                 blk_send(f, blk);
 525
 526                 g_free(blk->buf);
 527                 g_free(blk);
 528             }
 529
 530             bdrv_reset_dirty_bitmap(bmds->dirty_bitmap, sector, nr_sectors);
 531             break;
 532         }
 533         sector += BDRV_SECTORS_PER_DIRTY_CHUNK;
 534         bmds->cur_dirty = sector;
 535     }
 536
 537     return (bmds->cur_dirty >= bmds->total_sectors);
 538
 539 error:
 540     DPRINTF("Error reading sector %" PRId64 "\n", sector);
 541     g_free(blk->buf);
 542     g_free(blk);
 543     return ret;
 544 }
 545
 546 /* Called with iothread lock taken.
 547  *
 548  * return value:
 549  * 0: too much data for max_downtime
 550  * 1: few enough data for max_downtime
 551 */
 552 static int blk_mig_save_dirty_block(QEMUFile *f, int is_async)
 553 {
 554     BlkMigDevState *bmds;
 555     int ret = 1;
 556
 557     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 558         aio_context_acquire(bdrv_get_aio_context(bmds->bs));
 559         ret = mig_save_device_dirty(f, bmds, is_async);
 560         aio_context_release(bdrv_get_aio_context(bmds->bs));
 561         if (ret <= 0) {
 562             break;
 563         }
 564     }
 565
 566     return ret;
 567 }
 568
 569 /* Called with no locks taken.  */
 570
 571 static int flush_blks(QEMUFile *f)
 572 {
 573     BlkMigBlock *blk;
 574     int ret = 0;
 575
 576     DPRINTF("%s Enter submitted %d read_done %d transferred %d\n",
 577             __FUNCTION__, block_mig_state.submitted, block_mig_state.read_done,
 578             block_mig_state.transferred);
 579
 580     blk_mig_lock();
 581     while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
 582         if (qemu_file_rate_limit(f)) {
 583             break;
 584         }
 585         if (blk->ret < 0) {
 586             ret = blk->ret;
 587             break;
 588         }
 589
 590         QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
 591         blk_mig_unlock();
 592         blk_send(f, blk);
 593         blk_mig_lock();
 594
 595         g_free(blk->buf);
 596         g_free(blk);
 597
 598         block_mig_state.read_done--;
 599         block_mig_state.transferred++;
 600         assert(block_mig_state.read_done >= 0);
 601     }
 602     blk_mig_unlock();
 603
 604     DPRINTF("%s Exit submitted %d read_done %d transferred %d\n", __FUNCTION__,
 605             block_mig_state.submitted, block_mig_state.read_done,
 606             block_mig_state.transferred);
 607     return ret;
 608 }
 609
 610 /* Called with iothread lock taken.  */
 611
 612 static int64_t get_remaining_dirty(void)
 613 {
 614     BlkMigDevState *bmds;
 615     int64_t dirty = 0;
 616
 617     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 618         aio_context_acquire(bdrv_get_aio_context(bmds->bs));
 619         dirty += bdrv_get_dirty_count(bmds->dirty_bitmap);
 620         aio_context_release(bdrv_get_aio_context(bmds->bs));
 621     }
 622
 623     return dirty << BDRV_SECTOR_BITS;
 624 }
 625
 626 /* Called with iothread lock taken.  */
 627
 628 static void block_migration_cleanup(void *opaque)
 629 {
 630     BlkMigDevState *bmds;
 631     BlkMigBlock *blk;
 632     AioContext *ctx;
 633
 634     bdrv_drain_all();
 635
 636     unset_dirty_tracking();
 637
 638     while ((bmds = QSIMPLEQ_FIRST(&block_mig_state.bmds_list)) != NULL) {
 639         QSIMPLEQ_REMOVE_HEAD(&block_mig_state.bmds_list, entry);
 640         bdrv_op_unblock_all(bmds->bs, bmds->blocker);
 641         error_free(bmds->blocker);
 642
 643         /* Save ctx, because bmds->bs can disappear during bdrv_unref.  */
 644         ctx = bdrv_get_aio_context(bmds->bs);
 645         aio_context_acquire(ctx);
 646         bdrv_unref(bmds->bs);
 647         aio_context_release(ctx);
 648
 649         g_free(bmds->aio_bitmap);
 650         g_free(bmds);
 651     }
 652
 653     blk_mig_lock();
 654     while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
 655         QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
 656         g_free(blk->buf);
 657         g_free(blk);
 658     }
 659     blk_mig_unlock();
 660 }
 661
 662 static int block_save_setup(QEMUFile *f, void *opaque)
 663 {
 664     int ret;
 665
 666     DPRINTF("Enter save live setup submitted %d transferred %d\n",
 667             block_mig_state.submitted, block_mig_state.transferred);
 668
 669     qemu_mutex_lock_iothread();
 670     init_blk_migration(f);
 671
 672     /* start track dirty blocks */
 673     ret = set_dirty_tracking();
 674
 675     qemu_mutex_unlock_iothread();
 676
 677     if (ret) {
 678         return ret;
 679     }
 680
 681     ret = flush_blks(f);
 682     blk_mig_reset_dirty_cursor();
 683     qemu_put_be64(f, BLK_MIG_FLAG_EOS);
 684
 685     return ret;
 686 }
 687
 688 static int block_save_iterate(QEMUFile *f, void *opaque)
 689 {
 690     int ret;
 691     int64_t last_ftell = qemu_ftell(f);
 692     int64_t delta_ftell;
 693
 694     DPRINTF("Enter save live iterate submitted %d transferred %d\n",
 695             block_mig_state.submitted, block_mig_state.transferred);
 696
 697     ret = flush_blks(f);
 698     if (ret) {
 699         return ret;
 700     }
 701
 702     blk_mig_reset_dirty_cursor();
 703
 704     /* control the rate of transfer */
 705     blk_mig_lock();
 706     while ((block_mig_state.submitted +
 707             block_mig_state.read_done) * BLOCK_SIZE <
 708            qemu_file_get_rate_limit(f) &&
 709            (block_mig_state.submitted +
 710             block_mig_state.read_done) <
 711            MAX_INFLIGHT_IO) {
 712         blk_mig_unlock();
 713         if (block_mig_state.bulk_completed == 0) {
 714             /* first finish the bulk phase */
 715             if (blk_mig_save_bulked_block(f) == 0) {
 716                 /* finished saving bulk on all devices */
 717                 block_mig_state.bulk_completed = 1;
 718             }
 719             ret = 0;
 720         } else {
 721             /* Always called with iothread lock taken for
 722              * simplicity, block_save_complete also calls it.
 723              */
 724             qemu_mutex_lock_iothread();
 725             ret = blk_mig_save_dirty_block(f, 1);
 726             qemu_mutex_unlock_iothread();
 727         }
 728         if (ret < 0) {
 729             return ret;
 730         }
 731         blk_mig_lock();
 732         if (ret != 0) {
 733             /* no more dirty blocks */
 734             break;
 735         }
 736     }
 737     blk_mig_unlock();
 738
 739     ret = flush_blks(f);
 740     if (ret) {
 741         return ret;
 742     }
 743
 744     qemu_put_be64(f, BLK_MIG_FLAG_EOS);
 745     delta_ftell = qemu_ftell(f) - last_ftell;
 746     if (delta_ftell > 0) {
 747         return 1;
 748     } else if (delta_ftell < 0) {
 749         return -1;
 750     } else {
 751         return 0;
 752     }
 753 }
 754
 755 /* Called with iothread lock taken.  */
 756
 757 static int block_save_complete(QEMUFile *f, void *opaque)
 758 {
 759     int ret;
 760
 761     DPRINTF("Enter save live complete submitted %d transferred %d\n",
 762             block_mig_state.submitted, block_mig_state.transferred);
 763
 764     ret = flush_blks(f);
 765     if (ret) {
 766         return ret;
 767     }
 768
 769     blk_mig_reset_dirty_cursor();
 770
 771     /* we know for sure that save bulk is completed and
 772        all async read completed */
 773     blk_mig_lock();
 774     assert(block_mig_state.submitted == 0);
 775     blk_mig_unlock();
 776
 777     do {
 778         ret = blk_mig_save_dirty_block(f, 0);
 779         if (ret < 0) {
 780             return ret;
 781         }
 782     } while (ret == 0);
 783
 784     /* report completion */
 785     qemu_put_be64(f, (100 << BDRV_SECTOR_BITS) | BLK_MIG_FLAG_PROGRESS);
 786
 787     DPRINTF("Block migration completed\n");
 788
 789     qemu_put_be64(f, BLK_MIG_FLAG_EOS);
 790
 791     return 0;
 792 }
 793
 794 static void block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size,
 795                                uint64_t *non_postcopiable_pending,
 796                                uint64_t *postcopiable_pending)
 797 {
 798     /* Estimate pending number of bytes to send */
 799     uint64_t pending;
 800
 801     qemu_mutex_lock_iothread();
 802     pending = get_remaining_dirty();
 803     qemu_mutex_unlock_iothread();
 804
 805     blk_mig_lock();
 806     pending += block_mig_state.submitted * BLOCK_SIZE +
 807                block_mig_state.read_done * BLOCK_SIZE;
 808     blk_mig_unlock();
 809
 810     /* Report at least one block pending during bulk phase */
 811     if (pending <= max_size && !block_mig_state.bulk_completed) {
 812         pending = max_size + BLOCK_SIZE;
 813     }
 814
 815     DPRINTF("Enter save live pending  %" PRIu64 "\n", pending);
 816     /* We don't do postcopy */
 817     *non_postcopiable_pending += pending;
 818 }
 819
 820 static int block_load(QEMUFile *f, void *opaque, int version_id)
 821 {
 822     static int banner_printed;
 823     int len, flags;
 824     char device_name[256];
 825     int64_t addr;
 826     BlockDriverState *bs, *bs_prev = NULL;
 827     BlockBackend *blk;
 828     Error *local_err = NULL;
 829     uint8_t *buf;
 830     int64_t total_sectors = 0;
 831     int nr_sectors;
 832     int ret;
 833
 834     do {
 835         addr = qemu_get_be64(f);
 836
 837         flags = addr & ~BDRV_SECTOR_MASK;
 838         addr >>= BDRV_SECTOR_BITS;
 839
 840         if (flags & BLK_MIG_FLAG_DEVICE_BLOCK) {
 841             /* get device name */
 842             len = qemu_get_byte(f);
 843             qemu_get_buffer(f, (uint8_t *)device_name, len);
 844             device_name[len] = '\0';
 845
 846             blk = blk_by_name(device_name);
 847             if (!blk) {
 848                 fprintf(stderr, "Error unknown block device %s\n",
 849                         device_name);
 850                 return -EINVAL;
 851             }
 852             bs = blk_bs(blk);
 853             if (!bs) {
 854                 fprintf(stderr, "Block device %s has no medium\n",
 855                         device_name);
 856                 return -EINVAL;
 857             }
 858
 859             if (bs != bs_prev) {
 860                 bs_prev = bs;
 861                 total_sectors = bdrv_nb_sectors(bs);
 862                 if (total_sectors <= 0) {
 863                     error_report("Error getting length of block device %s",
 864                                  device_name);
 865                     return -EINVAL;
 866                 }
 867
 868                 bdrv_invalidate_cache(bs, &local_err);
 869                 if (local_err) {
 870                     error_report_err(local_err);
 871                     return -EINVAL;
 872                 }
 873             }
 874
 875             if (total_sectors - addr < BDRV_SECTORS_PER_DIRTY_CHUNK) {
 876                 nr_sectors = total_sectors - addr;
 877             } else {
 878                 nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
 879             }
 880
 881             if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
 882                 ret = bdrv_write_zeroes(bs, addr, nr_sectors,
 883                                         BDRV_REQ_MAY_UNMAP);
 884             } else {
 885                 buf = g_malloc(BLOCK_SIZE);
 886                 qemu_get_buffer(f, buf, BLOCK_SIZE);
 887                 ret = bdrv_write(bs, addr, buf, nr_sectors);
 888                 g_free(buf);
 889             }
 890
 891             if (ret < 0) {
 892                 return ret;
 893             }
 894         } else if (flags & BLK_MIG_FLAG_PROGRESS) {
 895             if (!banner_printed) {
 896                 printf("Receiving block device images\n");
 897                 banner_printed = 1;
 898             }
 899             printf("Completed %d %%%c", (int)addr,
 900                    (addr == 100) ? '\n' : '\r');
 901             fflush(stdout);
 902         } else if (!(flags & BLK_MIG_FLAG_EOS)) {
 903             fprintf(stderr, "Unknown block migration flags: %#x\n", flags);
 904             return -EINVAL;
 905         }
 906         ret = qemu_file_get_error(f);
 907         if (ret != 0) {
 908             return ret;
 909         }
 910     } while (!(flags & BLK_MIG_FLAG_EOS));
 911
 912     return 0;
 913 }
 914
 915 static void block_set_params(const MigrationParams *params, void *opaque)
 916 {
 917     block_mig_state.blk_enable = params->blk;
 918     block_mig_state.shared_base = params->shared;
 919
 920     /* shared base means that blk_enable = 1 */
 921     block_mig_state.blk_enable |= params->shared;
 922 }
 923
 924 static bool block_is_active(void *opaque)
 925 {
 926     return block_mig_state.blk_enable == 1;
 927 }
 928
 929 static SaveVMHandlers savevm_block_handlers = {
 930     .set_params = block_set_params,
 931     .save_live_setup = block_save_setup,
 932     .save_live_iterate = block_save_iterate,
 933     .save_live_complete_precopy = block_save_complete,
 934     .save_live_pending = block_save_pending,
 935     .load_state = block_load,
 936     .cleanup = block_migration_cleanup,
 937     .is_active = block_is_active,
 938 };
 939
 940 void blk_mig_init(void)
 941 {
 942     QSIMPLEQ_INIT(&block_mig_state.bmds_list);
 943     QSIMPLEQ_INIT(&block_mig_state.blk_list);
 944     qemu_mutex_init(&block_mig_state.lock);
 945
 946     register_savevm_live(NULL, "block", 0, 1, &savevm_block_handlers,
 947                          &block_mig_state);
 948 }