block-migration.c

   1 /*
   2  * QEMU live block migration
   3  *
   4  * Copyright IBM, Corp. 2009
   5  *
   6  * Authors:
   7  *  Liran Schour   <lirans@il.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu-common.h"
  17 #include "block/block.h"
  18 #include "qemu/error-report.h"
  19 #include "qemu/main-loop.h"
  20 #include "hw/hw.h"
  21 #include "qemu/queue.h"
  22 #include "qemu/timer.h"
  23 #include "migration/block.h"
  24 #include "migration/migration.h"
  25 #include "sysemu/blockdev.h"
  26 #include <assert.h>
  27
  28 #define BLOCK_SIZE                       (1 << 20)
  29 #define BDRV_SECTORS_PER_DIRTY_CHUNK     (BLOCK_SIZE >> BDRV_SECTOR_BITS)
  30
  31 #define BLK_MIG_FLAG_DEVICE_BLOCK       0x01
  32 #define BLK_MIG_FLAG_EOS                0x02
  33 #define BLK_MIG_FLAG_PROGRESS           0x04
  34 #define BLK_MIG_FLAG_ZERO_BLOCK         0x08
  35
  36 #define MAX_IS_ALLOCATED_SEARCH 65536
  37
  38 //#define DEBUG_BLK_MIGRATION
  39
  40 #ifdef DEBUG_BLK_MIGRATION
  41 #define DPRINTF(fmt, ...) \
  42     do { printf("blk_migration: " fmt, ## __VA_ARGS__); } while (0)
  43 #else
  44 #define DPRINTF(fmt, ...) \
  45     do { } while (0)
  46 #endif
  47
  48 typedef struct BlkMigDevState {
  49     /* Written during setup phase.  Can be read without a lock.  */
  50     BlockDriverState *bs;
  51     int shared_base;
  52     int64_t total_sectors;
  53     QSIMPLEQ_ENTRY(BlkMigDevState) entry;
  54
  55     /* Only used by migration thread.  Does not need a lock.  */
  56     int bulk_completed;
  57     int64_t cur_sector;
  58     int64_t cur_dirty;
  59
  60     /* Protected by block migration lock.  */
  61     unsigned long *aio_bitmap;
  62     int64_t completed_sectors;
  63     BdrvDirtyBitmap *dirty_bitmap;
  64     Error *blocker;
  65 } BlkMigDevState;
  66
  67 typedef struct BlkMigBlock {
  68     /* Only used by migration thread.  */
  69     uint8_t *buf;
  70     BlkMigDevState *bmds;
  71     int64_t sector;
  72     int nr_sectors;
  73     struct iovec iov;
  74     QEMUIOVector qiov;
  75     BlockAIOCB *aiocb;
  76
  77     /* Protected by block migration lock.  */
  78     int ret;
  79     QSIMPLEQ_ENTRY(BlkMigBlock) entry;
  80 } BlkMigBlock;
  81
  82 typedef struct BlkMigState {
  83     /* Written during setup phase.  Can be read without a lock.  */
  84     int blk_enable;
  85     int shared_base;
  86     QSIMPLEQ_HEAD(bmds_list, BlkMigDevState) bmds_list;
  87     int64_t total_sector_sum;
  88     bool zero_blocks;
  89
  90     /* Protected by lock.  */
  91     QSIMPLEQ_HEAD(blk_list, BlkMigBlock) blk_list;
  92     int submitted;
  93     int read_done;
  94
  95     /* Only used by migration thread.  Does not need a lock.  */
  96     int transferred;
  97     int prev_progress;
  98     int bulk_completed;
  99
 100     /* Lock must be taken _inside_ the iothread lock.  */
 101     QemuMutex lock;
 102 } BlkMigState;
 103
 104 static BlkMigState block_mig_state;
 105
 106 static void blk_mig_lock(void)
 107 {
 108     qemu_mutex_lock(&block_mig_state.lock);
 109 }
 110
 111 static void blk_mig_unlock(void)
 112 {
 113     qemu_mutex_unlock(&block_mig_state.lock);
 114 }
 115
 116 /* Must run outside of the iothread lock during the bulk phase,
 117  * or the VM will stall.
 118  */
 119
 120 static void blk_send(QEMUFile *f, BlkMigBlock * blk)
 121 {
 122     int len;
 123     uint64_t flags = BLK_MIG_FLAG_DEVICE_BLOCK;
 124
 125     if (block_mig_state.zero_blocks &&
 126         buffer_is_zero(blk->buf, BLOCK_SIZE)) {
 127         flags |= BLK_MIG_FLAG_ZERO_BLOCK;
 128     }
 129
 130     /* sector number and flags */
 131     qemu_put_be64(f, (blk->sector << BDRV_SECTOR_BITS)
 132                      | flags);
 133
 134     /* device name */
 135     len = strlen(bdrv_get_device_name(blk->bmds->bs));
 136     qemu_put_byte(f, len);
 137     qemu_put_buffer(f, (uint8_t *)bdrv_get_device_name(blk->bmds->bs), len);
 138
 139     /* if a block is zero we need to flush here since the network
 140      * bandwidth is now a lot higher than the storage device bandwidth.
 141      * thus if we queue zero blocks we slow down the migration */
 142     if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
 143         qemu_fflush(f);
 144         return;
 145     }
 146
 147     qemu_put_buffer(f, blk->buf, BLOCK_SIZE);
 148 }
 149
 150 int blk_mig_active(void)
 151 {
 152     return !QSIMPLEQ_EMPTY(&block_mig_state.bmds_list);
 153 }
 154
 155 uint64_t blk_mig_bytes_transferred(void)
 156 {
 157     BlkMigDevState *bmds;
 158     uint64_t sum = 0;
 159
 160     blk_mig_lock();
 161     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 162         sum += bmds->completed_sectors;
 163     }
 164     blk_mig_unlock();
 165     return sum << BDRV_SECTOR_BITS;
 166 }
 167
 168 uint64_t blk_mig_bytes_remaining(void)
 169 {
 170     return blk_mig_bytes_total() - blk_mig_bytes_transferred();
 171 }
 172
 173 uint64_t blk_mig_bytes_total(void)
 174 {
 175     BlkMigDevState *bmds;
 176     uint64_t sum = 0;
 177
 178     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 179         sum += bmds->total_sectors;
 180     }
 181     return sum << BDRV_SECTOR_BITS;
 182 }
 183
 184
 185 /* Called with migration lock held.  */
 186
 187 static int bmds_aio_inflight(BlkMigDevState *bmds, int64_t sector)
 188 {
 189     int64_t chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;
 190
 191     if (sector < bdrv_nb_sectors(bmds->bs)) {
 192         return !!(bmds->aio_bitmap[chunk / (sizeof(unsigned long) * 8)] &
 193             (1UL << (chunk % (sizeof(unsigned long) * 8))));
 194     } else {
 195         return 0;
 196     }
 197 }
 198
 199 /* Called with migration lock held.  */
 200
 201 static void bmds_set_aio_inflight(BlkMigDevState *bmds, int64_t sector_num,
 202                              int nb_sectors, int set)
 203 {
 204     int64_t start, end;
 205     unsigned long val, idx, bit;
 206
 207     start = sector_num / BDRV_SECTORS_PER_DIRTY_CHUNK;
 208     end = (sector_num + nb_sectors - 1) / BDRV_SECTORS_PER_DIRTY_CHUNK;
 209
 210     for (; start <= end; start++) {
 211         idx = start / (sizeof(unsigned long) * 8);
 212         bit = start % (sizeof(unsigned long) * 8);
 213         val = bmds->aio_bitmap[idx];
 214         if (set) {
 215             val |= 1UL << bit;
 216         } else {
 217             val &= ~(1UL << bit);
 218         }
 219         bmds->aio_bitmap[idx] = val;
 220     }
 221 }
 222
 223 static void alloc_aio_bitmap(BlkMigDevState *bmds)
 224 {
 225     BlockDriverState *bs = bmds->bs;
 226     int64_t bitmap_size;
 227
 228     bitmap_size = bdrv_nb_sectors(bs) + BDRV_SECTORS_PER_DIRTY_CHUNK * 8 - 1;
 229     bitmap_size /= BDRV_SECTORS_PER_DIRTY_CHUNK * 8;
 230
 231     bmds->aio_bitmap = g_malloc0(bitmap_size);
 232 }
 233
 234 /* Never hold migration lock when yielding to the main loop!  */
 235
 236 static void blk_mig_read_cb(void *opaque, int ret)
 237 {
 238     BlkMigBlock *blk = opaque;
 239
 240     blk_mig_lock();
 241     blk->ret = ret;
 242
 243     QSIMPLEQ_INSERT_TAIL(&block_mig_state.blk_list, blk, entry);
 244     bmds_set_aio_inflight(blk->bmds, blk->sector, blk->nr_sectors, 0);
 245
 246     block_mig_state.submitted--;
 247     block_mig_state.read_done++;
 248     assert(block_mig_state.submitted >= 0);
 249     blk_mig_unlock();
 250 }
 251
 252 /* Called with no lock taken.  */
 253
 254 static int mig_save_device_bulk(QEMUFile *f, BlkMigDevState *bmds)
 255 {
 256     int64_t total_sectors = bmds->total_sectors;
 257     int64_t cur_sector = bmds->cur_sector;
 258     BlockDriverState *bs = bmds->bs;
 259     BlkMigBlock *blk;
 260     int nr_sectors;
 261
 262     if (bmds->shared_base) {
 263         qemu_mutex_lock_iothread();
 264         while (cur_sector < total_sectors &&
 265                !bdrv_is_allocated(bs, cur_sector, MAX_IS_ALLOCATED_SEARCH,
 266                                   &nr_sectors)) {
 267             cur_sector += nr_sectors;
 268         }
 269         qemu_mutex_unlock_iothread();
 270     }
 271
 272     if (cur_sector >= total_sectors) {
 273         bmds->cur_sector = bmds->completed_sectors = total_sectors;
 274         return 1;
 275     }
 276
 277     bmds->completed_sectors = cur_sector;
 278
 279     cur_sector &= ~((int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK - 1);
 280
 281     /* we are going to transfer a full block even if it is not allocated */
 282     nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
 283
 284     if (total_sectors - cur_sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
 285         nr_sectors = total_sectors - cur_sector;
 286     }
 287
 288     blk = g_new(BlkMigBlock, 1);
 289     blk->buf = g_malloc(BLOCK_SIZE);
 290     blk->bmds = bmds;
 291     blk->sector = cur_sector;
 292     blk->nr_sectors = nr_sectors;
 293
 294     blk->iov.iov_base = blk->buf;
 295     blk->iov.iov_len = nr_sectors * BDRV_SECTOR_SIZE;
 296     qemu_iovec_init_external(&blk->qiov, &blk->iov, 1);
 297
 298     blk_mig_lock();
 299     block_mig_state.submitted++;
 300     blk_mig_unlock();
 301
 302     qemu_mutex_lock_iothread();
 303     blk->aiocb = bdrv_aio_readv(bs, cur_sector, &blk->qiov,
 304                                 nr_sectors, blk_mig_read_cb, blk);
 305
 306     bdrv_reset_dirty(bs, cur_sector, nr_sectors);
 307     qemu_mutex_unlock_iothread();
 308
 309     bmds->cur_sector = cur_sector + nr_sectors;
 310     return (bmds->cur_sector >= total_sectors);
 311 }
 312
 313 /* Called with iothread lock taken.  */
 314
 315 static int set_dirty_tracking(void)
 316 {
 317     BlkMigDevState *bmds;
 318     int ret;
 319
 320     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 321         bmds->dirty_bitmap = bdrv_create_dirty_bitmap(bmds->bs, BLOCK_SIZE,
 322                                                       NULL);
 323         if (!bmds->dirty_bitmap) {
 324             ret = -errno;
 325             goto fail;
 326         }
 327     }
 328     return 0;
 329
 330 fail:
 331     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 332         if (bmds->dirty_bitmap) {
 333             bdrv_release_dirty_bitmap(bmds->bs, bmds->dirty_bitmap);
 334         }
 335     }
 336     return ret;
 337 }
 338
 339 static void unset_dirty_tracking(void)
 340 {
 341     BlkMigDevState *bmds;
 342
 343     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 344         bdrv_release_dirty_bitmap(bmds->bs, bmds->dirty_bitmap);
 345     }
 346 }
 347
 348 static void init_blk_migration(QEMUFile *f)
 349 {
 350     BlockDriverState *bs;
 351     BlkMigDevState *bmds;
 352     int64_t sectors;
 353
 354     block_mig_state.submitted = 0;
 355     block_mig_state.read_done = 0;
 356     block_mig_state.transferred = 0;
 357     block_mig_state.total_sector_sum = 0;
 358     block_mig_state.prev_progress = -1;
 359     block_mig_state.bulk_completed = 0;
 360     block_mig_state.zero_blocks = migrate_zero_blocks();
 361
 362     for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
 363         if (bdrv_is_read_only(bs)) {
 364             continue;
 365         }
 366
 367         sectors = bdrv_nb_sectors(bs);
 368         if (sectors <= 0) {
 369             return;
 370         }
 371
 372         bmds = g_new0(BlkMigDevState, 1);
 373         bmds->bs = bs;
 374         bmds->bulk_completed = 0;
 375         bmds->total_sectors = sectors;
 376         bmds->completed_sectors = 0;
 377         bmds->shared_base = block_mig_state.shared_base;
 378         alloc_aio_bitmap(bmds);
 379         error_setg(&bmds->blocker, "block device is in use by migration");
 380         bdrv_op_block_all(bs, bmds->blocker);
 381         bdrv_ref(bs);
 382
 383         block_mig_state.total_sector_sum += sectors;
 384
 385         if (bmds->shared_base) {
 386             DPRINTF("Start migration for %s with shared base image\n",
 387                     bdrv_get_device_name(bs));
 388         } else {
 389             DPRINTF("Start full migration for %s\n", bdrv_get_device_name(bs));
 390         }
 391
 392         QSIMPLEQ_INSERT_TAIL(&block_mig_state.bmds_list, bmds, entry);
 393     }
 394 }
 395
 396 /* Called with no lock taken.  */
 397
 398 static int blk_mig_save_bulked_block(QEMUFile *f)
 399 {
 400     int64_t completed_sector_sum = 0;
 401     BlkMigDevState *bmds;
 402     int progress;
 403     int ret = 0;
 404
 405     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 406         if (bmds->bulk_completed == 0) {
 407             if (mig_save_device_bulk(f, bmds) == 1) {
 408                 /* completed bulk section for this device */
 409                 bmds->bulk_completed = 1;
 410             }
 411             completed_sector_sum += bmds->completed_sectors;
 412             ret = 1;
 413             break;
 414         } else {
 415             completed_sector_sum += bmds->completed_sectors;
 416         }
 417     }
 418
 419     if (block_mig_state.total_sector_sum != 0) {
 420         progress = completed_sector_sum * 100 /
 421                    block_mig_state.total_sector_sum;
 422     } else {
 423         progress = 100;
 424     }
 425     if (progress != block_mig_state.prev_progress) {
 426         block_mig_state.prev_progress = progress;
 427         qemu_put_be64(f, (progress << BDRV_SECTOR_BITS)
 428                          | BLK_MIG_FLAG_PROGRESS);
 429         DPRINTF("Completed %d %%\r", progress);
 430     }
 431
 432     return ret;
 433 }
 434
 435 static void blk_mig_reset_dirty_cursor(void)
 436 {
 437     BlkMigDevState *bmds;
 438
 439     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 440         bmds->cur_dirty = 0;
 441     }
 442 }
 443
 444 /* Called with iothread lock taken.  */
 445
 446 static int mig_save_device_dirty(QEMUFile *f, BlkMigDevState *bmds,
 447                                  int is_async)
 448 {
 449     BlkMigBlock *blk;
 450     int64_t total_sectors = bmds->total_sectors;
 451     int64_t sector;
 452     int nr_sectors;
 453     int ret = -EIO;
 454
 455     for (sector = bmds->cur_dirty; sector < bmds->total_sectors;) {
 456         blk_mig_lock();
 457         if (bmds_aio_inflight(bmds, sector)) {
 458             blk_mig_unlock();
 459             bdrv_drain_all();
 460         } else {
 461             blk_mig_unlock();
 462         }
 463         if (bdrv_get_dirty(bmds->bs, bmds->dirty_bitmap, sector)) {
 464
 465             if (total_sectors - sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
 466                 nr_sectors = total_sectors - sector;
 467             } else {
 468                 nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
 469             }
 470             blk = g_new(BlkMigBlock, 1);
 471             blk->buf = g_malloc(BLOCK_SIZE);
 472             blk->bmds = bmds;
 473             blk->sector = sector;
 474             blk->nr_sectors = nr_sectors;
 475
 476             if (is_async) {
 477                 blk->iov.iov_base = blk->buf;
 478                 blk->iov.iov_len = nr_sectors * BDRV_SECTOR_SIZE;
 479                 qemu_iovec_init_external(&blk->qiov, &blk->iov, 1);
 480
 481                 blk->aiocb = bdrv_aio_readv(bmds->bs, sector, &blk->qiov,
 482                                             nr_sectors, blk_mig_read_cb, blk);
 483
 484                 blk_mig_lock();
 485                 block_mig_state.submitted++;
 486                 bmds_set_aio_inflight(bmds, sector, nr_sectors, 1);
 487                 blk_mig_unlock();
 488             } else {
 489                 ret = bdrv_read(bmds->bs, sector, blk->buf, nr_sectors);
 490                 if (ret < 0) {
 491                     goto error;
 492                 }
 493                 blk_send(f, blk);
 494
 495                 g_free(blk->buf);
 496                 g_free(blk);
 497             }
 498
 499             bdrv_reset_dirty(bmds->bs, sector, nr_sectors);
 500             break;
 501         }
 502         sector += BDRV_SECTORS_PER_DIRTY_CHUNK;
 503         bmds->cur_dirty = sector;
 504     }
 505
 506     return (bmds->cur_dirty >= bmds->total_sectors);
 507
 508 error:
 509     DPRINTF("Error reading sector %" PRId64 "\n", sector);
 510     g_free(blk->buf);
 511     g_free(blk);
 512     return ret;
 513 }
 514
 515 /* Called with iothread lock taken.
 516  *
 517  * return value:
 518  * 0: too much data for max_downtime
 519  * 1: few enough data for max_downtime
 520 */
 521 static int blk_mig_save_dirty_block(QEMUFile *f, int is_async)
 522 {
 523     BlkMigDevState *bmds;
 524     int ret = 1;
 525
 526     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 527         ret = mig_save_device_dirty(f, bmds, is_async);
 528         if (ret <= 0) {
 529             break;
 530         }
 531     }
 532
 533     return ret;
 534 }
 535
 536 /* Called with no locks taken.  */
 537
 538 static int flush_blks(QEMUFile *f)
 539 {
 540     BlkMigBlock *blk;
 541     int ret = 0;
 542
 543     DPRINTF("%s Enter submitted %d read_done %d transferred %d\n",
 544             __FUNCTION__, block_mig_state.submitted, block_mig_state.read_done,
 545             block_mig_state.transferred);
 546
 547     blk_mig_lock();
 548     while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
 549         if (qemu_file_rate_limit(f)) {
 550             break;
 551         }
 552         if (blk->ret < 0) {
 553             ret = blk->ret;
 554             break;
 555         }
 556
 557         QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
 558         blk_mig_unlock();
 559         blk_send(f, blk);
 560         blk_mig_lock();
 561
 562         g_free(blk->buf);
 563         g_free(blk);
 564
 565         block_mig_state.read_done--;
 566         block_mig_state.transferred++;
 567         assert(block_mig_state.read_done >= 0);
 568     }
 569     blk_mig_unlock();
 570
 571     DPRINTF("%s Exit submitted %d read_done %d transferred %d\n", __FUNCTION__,
 572             block_mig_state.submitted, block_mig_state.read_done,
 573             block_mig_state.transferred);
 574     return ret;
 575 }
 576
 577 /* Called with iothread lock taken.  */
 578
 579 static int64_t get_remaining_dirty(void)
 580 {
 581     BlkMigDevState *bmds;
 582     int64_t dirty = 0;
 583
 584     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 585         dirty += bdrv_get_dirty_count(bmds->bs, bmds->dirty_bitmap);
 586     }
 587
 588     return dirty << BDRV_SECTOR_BITS;
 589 }
 590
 591 /* Called with iothread lock taken.  */
 592
 593 static void blk_mig_cleanup(void)
 594 {
 595     BlkMigDevState *bmds;
 596     BlkMigBlock *blk;
 597
 598     bdrv_drain_all();
 599
 600     unset_dirty_tracking();
 601
 602     blk_mig_lock();
 603     while ((bmds = QSIMPLEQ_FIRST(&block_mig_state.bmds_list)) != NULL) {
 604         QSIMPLEQ_REMOVE_HEAD(&block_mig_state.bmds_list, entry);
 605         bdrv_op_unblock_all(bmds->bs, bmds->blocker);
 606         error_free(bmds->blocker);
 607         bdrv_unref(bmds->bs);
 608         g_free(bmds->aio_bitmap);
 609         g_free(bmds);
 610     }
 611
 612     while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
 613         QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
 614         g_free(blk->buf);
 615         g_free(blk);
 616     }
 617     blk_mig_unlock();
 618 }
 619
 620 static void block_migration_cancel(void *opaque)
 621 {
 622     blk_mig_cleanup();
 623 }
 624
 625 static int block_save_setup(QEMUFile *f, void *opaque)
 626 {
 627     int ret;
 628
 629     DPRINTF("Enter save live setup submitted %d transferred %d\n",
 630             block_mig_state.submitted, block_mig_state.transferred);
 631
 632     qemu_mutex_lock_iothread();
 633     init_blk_migration(f);
 634
 635     /* start track dirty blocks */
 636     ret = set_dirty_tracking();
 637
 638     if (ret) {
 639         qemu_mutex_unlock_iothread();
 640         return ret;
 641     }
 642
 643     qemu_mutex_unlock_iothread();
 644
 645     ret = flush_blks(f);
 646     blk_mig_reset_dirty_cursor();
 647     qemu_put_be64(f, BLK_MIG_FLAG_EOS);
 648
 649     return ret;
 650 }
 651
 652 static int block_save_iterate(QEMUFile *f, void *opaque)
 653 {
 654     int ret;
 655     int64_t last_ftell = qemu_ftell(f);
 656
 657     DPRINTF("Enter save live iterate submitted %d transferred %d\n",
 658             block_mig_state.submitted, block_mig_state.transferred);
 659
 660     ret = flush_blks(f);
 661     if (ret) {
 662         return ret;
 663     }
 664
 665     blk_mig_reset_dirty_cursor();
 666
 667     /* control the rate of transfer */
 668     blk_mig_lock();
 669     while ((block_mig_state.submitted +
 670             block_mig_state.read_done) * BLOCK_SIZE <
 671            qemu_file_get_rate_limit(f)) {
 672         blk_mig_unlock();
 673         if (block_mig_state.bulk_completed == 0) {
 674             /* first finish the bulk phase */
 675             if (blk_mig_save_bulked_block(f) == 0) {
 676                 /* finished saving bulk on all devices */
 677                 block_mig_state.bulk_completed = 1;
 678             }
 679             ret = 0;
 680         } else {
 681             /* Always called with iothread lock taken for
 682              * simplicity, block_save_complete also calls it.
 683              */
 684             qemu_mutex_lock_iothread();
 685             ret = blk_mig_save_dirty_block(f, 1);
 686             qemu_mutex_unlock_iothread();
 687         }
 688         if (ret < 0) {
 689             return ret;
 690         }
 691         blk_mig_lock();
 692         if (ret != 0) {
 693             /* no more dirty blocks */
 694             break;
 695         }
 696     }
 697     blk_mig_unlock();
 698
 699     ret = flush_blks(f);
 700     if (ret) {
 701         return ret;
 702     }
 703
 704     qemu_put_be64(f, BLK_MIG_FLAG_EOS);
 705     return qemu_ftell(f) - last_ftell;
 706 }
 707
 708 /* Called with iothread lock taken.  */
 709
 710 static int block_save_complete(QEMUFile *f, void *opaque)
 711 {
 712     int ret;
 713
 714     DPRINTF("Enter save live complete submitted %d transferred %d\n",
 715             block_mig_state.submitted, block_mig_state.transferred);
 716
 717     ret = flush_blks(f);
 718     if (ret) {
 719         return ret;
 720     }
 721
 722     blk_mig_reset_dirty_cursor();
 723
 724     /* we know for sure that save bulk is completed and
 725        all async read completed */
 726     blk_mig_lock();
 727     assert(block_mig_state.submitted == 0);
 728     blk_mig_unlock();
 729
 730     do {
 731         ret = blk_mig_save_dirty_block(f, 0);
 732         if (ret < 0) {
 733             return ret;
 734         }
 735     } while (ret == 0);
 736
 737     /* report completion */
 738     qemu_put_be64(f, (100 << BDRV_SECTOR_BITS) | BLK_MIG_FLAG_PROGRESS);
 739
 740     DPRINTF("Block migration completed\n");
 741
 742     qemu_put_be64(f, BLK_MIG_FLAG_EOS);
 743
 744     blk_mig_cleanup();
 745     return 0;
 746 }
 747
 748 static uint64_t block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size)
 749 {
 750     /* Estimate pending number of bytes to send */
 751     uint64_t pending;
 752
 753     qemu_mutex_lock_iothread();
 754     blk_mig_lock();
 755     pending = get_remaining_dirty() +
 756                        block_mig_state.submitted * BLOCK_SIZE +
 757                        block_mig_state.read_done * BLOCK_SIZE;
 758
 759     /* Report at least one block pending during bulk phase */
 760     if (pending == 0 && !block_mig_state.bulk_completed) {
 761         pending = BLOCK_SIZE;
 762     }
 763     blk_mig_unlock();
 764     qemu_mutex_unlock_iothread();
 765
 766     DPRINTF("Enter save live pending  %" PRIu64 "\n", pending);
 767     return pending;
 768 }
 769
 770 static int block_load(QEMUFile *f, void *opaque, int version_id)
 771 {
 772     static int banner_printed;
 773     int len, flags;
 774     char device_name[256];
 775     int64_t addr;
 776     BlockDriverState *bs, *bs_prev = NULL;
 777     uint8_t *buf;
 778     int64_t total_sectors = 0;
 779     int nr_sectors;
 780     int ret;
 781
 782     do {
 783         addr = qemu_get_be64(f);
 784
 785         flags = addr & ~BDRV_SECTOR_MASK;
 786         addr >>= BDRV_SECTOR_BITS;
 787
 788         if (flags & BLK_MIG_FLAG_DEVICE_BLOCK) {
 789             /* get device name */
 790             len = qemu_get_byte(f);
 791             qemu_get_buffer(f, (uint8_t *)device_name, len);
 792             device_name[len] = '\0';
 793
 794             bs = bdrv_find(device_name);
 795             if (!bs) {
 796                 fprintf(stderr, "Error unknown block device %s\n",
 797                         device_name);
 798                 return -EINVAL;
 799             }
 800
 801             if (bs != bs_prev) {
 802                 bs_prev = bs;
 803                 total_sectors = bdrv_nb_sectors(bs);
 804                 if (total_sectors <= 0) {
 805                     error_report("Error getting length of block device %s",
 806                                  device_name);
 807                     return -EINVAL;
 808                 }
 809             }
 810
 811             if (total_sectors - addr < BDRV_SECTORS_PER_DIRTY_CHUNK) {
 812                 nr_sectors = total_sectors - addr;
 813             } else {
 814                 nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
 815             }
 816
 817             if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
 818                 ret = bdrv_write_zeroes(bs, addr, nr_sectors,
 819                                         BDRV_REQ_MAY_UNMAP);
 820             } else {
 821                 buf = g_malloc(BLOCK_SIZE);
 822                 qemu_get_buffer(f, buf, BLOCK_SIZE);
 823                 ret = bdrv_write(bs, addr, buf, nr_sectors);
 824                 g_free(buf);
 825             }
 826
 827             if (ret < 0) {
 828                 return ret;
 829             }
 830         } else if (flags & BLK_MIG_FLAG_PROGRESS) {
 831             if (!banner_printed) {
 832                 printf("Receiving block device images\n");
 833                 banner_printed = 1;
 834             }
 835             printf("Completed %d %%%c", (int)addr,
 836                    (addr == 100) ? '\n' : '\r');
 837             fflush(stdout);
 838         } else if (!(flags & BLK_MIG_FLAG_EOS)) {
 839             fprintf(stderr, "Unknown block migration flags: %#x\n", flags);
 840             return -EINVAL;
 841         }
 842         ret = qemu_file_get_error(f);
 843         if (ret != 0) {
 844             return ret;
 845         }
 846     } while (!(flags & BLK_MIG_FLAG_EOS));
 847
 848     return 0;
 849 }
 850
 851 static void block_set_params(const MigrationParams *params, void *opaque)
 852 {
 853     block_mig_state.blk_enable = params->blk;
 854     block_mig_state.shared_base = params->shared;
 855
 856     /* shared base means that blk_enable = 1 */
 857     block_mig_state.blk_enable |= params->shared;
 858 }
 859
 860 static bool block_is_active(void *opaque)
 861 {
 862     return block_mig_state.blk_enable == 1;
 863 }
 864
 865 static SaveVMHandlers savevm_block_handlers = {
 866     .set_params = block_set_params,
 867     .save_live_setup = block_save_setup,
 868     .save_live_iterate = block_save_iterate,
 869     .save_live_complete = block_save_complete,
 870     .save_live_pending = block_save_pending,
 871     .load_state = block_load,
 872     .cancel = block_migration_cancel,
 873     .is_active = block_is_active,
 874 };
 875
 876 void blk_mig_init(void)
 877 {
 878     QSIMPLEQ_INIT(&block_mig_state.bmds_list);
 879     QSIMPLEQ_INIT(&block_mig_state.blk_list);
 880     qemu_mutex_init(&block_mig_state.lock);
 881
 882     register_savevm_live(NULL, "block", 0, 1, &savevm_block_handlers,
 883                          &block_mig_state);
 884 }