migration/block.c

   1 /*
   2  * QEMU live block migration
   3  *
   4  * Copyright IBM, Corp. 2009
   5  *
   6  * Authors:
   7  *  Liran Schour   <lirans@il.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu-common.h"
  17 #include "block/block.h"
  18 #include "qemu/error-report.h"
  19 #include "qemu/main-loop.h"
  20 #include "hw/hw.h"
  21 #include "qemu/queue.h"
  22 #include "qemu/timer.h"
  23 #include "migration/block.h"
  24 #include "migration/migration.h"
  25 #include "sysemu/blockdev.h"
  26 #include <assert.h>
  27
  28 #define BLOCK_SIZE                       (1 << 20)
  29 #define BDRV_SECTORS_PER_DIRTY_CHUNK     (BLOCK_SIZE >> BDRV_SECTOR_BITS)
  30
  31 #define BLK_MIG_FLAG_DEVICE_BLOCK       0x01
  32 #define BLK_MIG_FLAG_EOS                0x02
  33 #define BLK_MIG_FLAG_PROGRESS           0x04
  34 #define BLK_MIG_FLAG_ZERO_BLOCK         0x08
  35
  36 #define MAX_IS_ALLOCATED_SEARCH 65536
  37
  38 //#define DEBUG_BLK_MIGRATION
  39
  40 #ifdef DEBUG_BLK_MIGRATION
  41 #define DPRINTF(fmt, ...) \
  42     do { printf("blk_migration: " fmt, ## __VA_ARGS__); } while (0)
  43 #else
  44 #define DPRINTF(fmt, ...) \
  45     do { } while (0)
  46 #endif
  47
  48 typedef struct BlkMigDevState {
  49     /* Written during setup phase.  Can be read without a lock.  */
  50     BlockDriverState *bs;
  51     int shared_base;
  52     int64_t total_sectors;
  53     QSIMPLEQ_ENTRY(BlkMigDevState) entry;
  54
  55     /* Only used by migration thread.  Does not need a lock.  */
  56     int bulk_completed;
  57     int64_t cur_sector;
  58     int64_t cur_dirty;
  59
  60     /* Protected by block migration lock.  */
  61     unsigned long *aio_bitmap;
  62     int64_t completed_sectors;
  63     BdrvDirtyBitmap *dirty_bitmap;
  64     Error *blocker;
  65 } BlkMigDevState;
  66
  67 typedef struct BlkMigBlock {
  68     /* Only used by migration thread.  */
  69     uint8_t *buf;
  70     BlkMigDevState *bmds;
  71     int64_t sector;
  72     int nr_sectors;
  73     struct iovec iov;
  74     QEMUIOVector qiov;
  75     BlockAIOCB *aiocb;
  76
  77     /* Protected by block migration lock.  */
  78     int ret;
  79     QSIMPLEQ_ENTRY(BlkMigBlock) entry;
  80 } BlkMigBlock;
  81
  82 typedef struct BlkMigState {
  83     /* Written during setup phase.  Can be read without a lock.  */
  84     int blk_enable;
  85     int shared_base;
  86     QSIMPLEQ_HEAD(bmds_list, BlkMigDevState) bmds_list;
  87     int64_t total_sector_sum;
  88     bool zero_blocks;
  89
  90     /* Protected by lock.  */
  91     QSIMPLEQ_HEAD(blk_list, BlkMigBlock) blk_list;
  92     int submitted;
  93     int read_done;
  94
  95     /* Only used by migration thread.  Does not need a lock.  */
  96     int transferred;
  97     int prev_progress;
  98     int bulk_completed;
  99
 100     /* Lock must be taken _inside_ the iothread lock.  */
 101     QemuMutex lock;
 102 } BlkMigState;
 103
 104 static BlkMigState block_mig_state;
 105
 106 static void blk_mig_lock(void)
 107 {
 108     qemu_mutex_lock(&block_mig_state.lock);
 109 }
 110
 111 static void blk_mig_unlock(void)
 112 {
 113     qemu_mutex_unlock(&block_mig_state.lock);
 114 }
 115
 116 /* Must run outside of the iothread lock during the bulk phase,
 117  * or the VM will stall.
 118  */
 119
 120 static void blk_send(QEMUFile *f, BlkMigBlock * blk)
 121 {
 122     int len;
 123     uint64_t flags = BLK_MIG_FLAG_DEVICE_BLOCK;
 124
 125     if (block_mig_state.zero_blocks &&
 126         buffer_is_zero(blk->buf, BLOCK_SIZE)) {
 127         flags |= BLK_MIG_FLAG_ZERO_BLOCK;
 128     }
 129
 130     /* sector number and flags */
 131     qemu_put_be64(f, (blk->sector << BDRV_SECTOR_BITS)
 132                      | flags);
 133
 134     /* device name */
 135     len = strlen(bdrv_get_device_name(blk->bmds->bs));
 136     qemu_put_byte(f, len);
 137     qemu_put_buffer(f, (uint8_t *)bdrv_get_device_name(blk->bmds->bs), len);
 138
 139     /* if a block is zero we need to flush here since the network
 140      * bandwidth is now a lot higher than the storage device bandwidth.
 141      * thus if we queue zero blocks we slow down the migration */
 142     if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
 143         qemu_fflush(f);
 144         return;
 145     }
 146
 147     qemu_put_buffer(f, blk->buf, BLOCK_SIZE);
 148 }
 149
 150 int blk_mig_active(void)
 151 {
 152     return !QSIMPLEQ_EMPTY(&block_mig_state.bmds_list);
 153 }
 154
 155 uint64_t blk_mig_bytes_transferred(void)
 156 {
 157     BlkMigDevState *bmds;
 158     uint64_t sum = 0;
 159
 160     blk_mig_lock();
 161     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 162         sum += bmds->completed_sectors;
 163     }
 164     blk_mig_unlock();
 165     return sum << BDRV_SECTOR_BITS;
 166 }
 167
 168 uint64_t blk_mig_bytes_remaining(void)
 169 {
 170     return blk_mig_bytes_total() - blk_mig_bytes_transferred();
 171 }
 172
 173 uint64_t blk_mig_bytes_total(void)
 174 {
 175     BlkMigDevState *bmds;
 176     uint64_t sum = 0;
 177
 178     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 179         sum += bmds->total_sectors;
 180     }
 181     return sum << BDRV_SECTOR_BITS;
 182 }
 183
 184
 185 /* Called with migration lock held.  */
 186
 187 static int bmds_aio_inflight(BlkMigDevState *bmds, int64_t sector)
 188 {
 189     int64_t chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;
 190
 191     if (sector < bdrv_nb_sectors(bmds->bs)) {
 192         return !!(bmds->aio_bitmap[chunk / (sizeof(unsigned long) * 8)] &
 193             (1UL << (chunk % (sizeof(unsigned long) * 8))));
 194     } else {
 195         return 0;
 196     }
 197 }
 198
 199 /* Called with migration lock held.  */
 200
 201 static void bmds_set_aio_inflight(BlkMigDevState *bmds, int64_t sector_num,
 202                              int nb_sectors, int set)
 203 {
 204     int64_t start, end;
 205     unsigned long val, idx, bit;
 206
 207     start = sector_num / BDRV_SECTORS_PER_DIRTY_CHUNK;
 208     end = (sector_num + nb_sectors - 1) / BDRV_SECTORS_PER_DIRTY_CHUNK;
 209
 210     for (; start <= end; start++) {
 211         idx = start / (sizeof(unsigned long) * 8);
 212         bit = start % (sizeof(unsigned long) * 8);
 213         val = bmds->aio_bitmap[idx];
 214         if (set) {
 215             val |= 1UL << bit;
 216         } else {
 217             val &= ~(1UL << bit);
 218         }
 219         bmds->aio_bitmap[idx] = val;
 220     }
 221 }
 222
 223 static void alloc_aio_bitmap(BlkMigDevState *bmds)
 224 {
 225     BlockDriverState *bs = bmds->bs;
 226     int64_t bitmap_size;
 227
 228     bitmap_size = bdrv_nb_sectors(bs) + BDRV_SECTORS_PER_DIRTY_CHUNK * 8 - 1;
 229     bitmap_size /= BDRV_SECTORS_PER_DIRTY_CHUNK * 8;
 230
 231     bmds->aio_bitmap = g_malloc0(bitmap_size);
 232 }
 233
 234 /* Never hold migration lock when yielding to the main loop!  */
 235
 236 static void blk_mig_read_cb(void *opaque, int ret)
 237 {
 238     BlkMigBlock *blk = opaque;
 239
 240     blk_mig_lock();
 241     blk->ret = ret;
 242
 243     QSIMPLEQ_INSERT_TAIL(&block_mig_state.blk_list, blk, entry);
 244     bmds_set_aio_inflight(blk->bmds, blk->sector, blk->nr_sectors, 0);
 245
 246     block_mig_state.submitted--;
 247     block_mig_state.read_done++;
 248     assert(block_mig_state.submitted >= 0);
 249     blk_mig_unlock();
 250 }
 251
 252 /* Called with no lock taken.  */
 253
 254 static int mig_save_device_bulk(QEMUFile *f, BlkMigDevState *bmds)
 255 {
 256     int64_t total_sectors = bmds->total_sectors;
 257     int64_t cur_sector = bmds->cur_sector;
 258     BlockDriverState *bs = bmds->bs;
 259     BlkMigBlock *blk;
 260     int nr_sectors;
 261
 262     if (bmds->shared_base) {
 263         qemu_mutex_lock_iothread();
 264         while (cur_sector < total_sectors &&
 265                !bdrv_is_allocated(bs, cur_sector, MAX_IS_ALLOCATED_SEARCH,
 266                                   &nr_sectors)) {
 267             cur_sector += nr_sectors;
 268         }
 269         qemu_mutex_unlock_iothread();
 270     }
 271
 272     if (cur_sector >= total_sectors) {
 273         bmds->cur_sector = bmds->completed_sectors = total_sectors;
 274         return 1;
 275     }
 276
 277     bmds->completed_sectors = cur_sector;
 278
 279     cur_sector &= ~((int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK - 1);
 280
 281     /* we are going to transfer a full block even if it is not allocated */
 282     nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
 283
 284     if (total_sectors - cur_sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
 285         nr_sectors = total_sectors - cur_sector;
 286     }
 287
 288     blk = g_new(BlkMigBlock, 1);
 289     blk->buf = g_malloc(BLOCK_SIZE);
 290     blk->bmds = bmds;
 291     blk->sector = cur_sector;
 292     blk->nr_sectors = nr_sectors;
 293
 294     blk->iov.iov_base = blk->buf;
 295     blk->iov.iov_len = nr_sectors * BDRV_SECTOR_SIZE;
 296     qemu_iovec_init_external(&blk->qiov, &blk->iov, 1);
 297
 298     blk_mig_lock();
 299     block_mig_state.submitted++;
 300     blk_mig_unlock();
 301
 302     qemu_mutex_lock_iothread();
 303     blk->aiocb = bdrv_aio_readv(bs, cur_sector, &blk->qiov,
 304                                 nr_sectors, blk_mig_read_cb, blk);
 305
 306     bdrv_reset_dirty_bitmap(bs, bmds->dirty_bitmap, cur_sector, nr_sectors);
 307     qemu_mutex_unlock_iothread();
 308
 309     bmds->cur_sector = cur_sector + nr_sectors;
 310     return (bmds->cur_sector >= total_sectors);
 311 }
 312
 313 /* Called with iothread lock taken.  */
 314
 315 static int set_dirty_tracking(void)
 316 {
 317     BlkMigDevState *bmds;
 318     int ret;
 319
 320     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 321         bmds->dirty_bitmap = bdrv_create_dirty_bitmap(bmds->bs, BLOCK_SIZE,
 322                                                       NULL);
 323         if (!bmds->dirty_bitmap) {
 324             ret = -errno;
 325             goto fail;
 326         }
 327     }
 328     return 0;
 329
 330 fail:
 331     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 332         if (bmds->dirty_bitmap) {
 333             bdrv_release_dirty_bitmap(bmds->bs, bmds->dirty_bitmap);
 334         }
 335     }
 336     return ret;
 337 }
 338
 339 static void unset_dirty_tracking(void)
 340 {
 341     BlkMigDevState *bmds;
 342
 343     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 344         bdrv_release_dirty_bitmap(bmds->bs, bmds->dirty_bitmap);
 345     }
 346 }
 347
 348 static void init_blk_migration(QEMUFile *f)
 349 {
 350     BlockDriverState *bs;
 351     BlkMigDevState *bmds;
 352     int64_t sectors;
 353
 354     block_mig_state.submitted = 0;
 355     block_mig_state.read_done = 0;
 356     block_mig_state.transferred = 0;
 357     block_mig_state.total_sector_sum = 0;
 358     block_mig_state.prev_progress = -1;
 359     block_mig_state.bulk_completed = 0;
 360     block_mig_state.zero_blocks = migrate_zero_blocks();
 361
 362     for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
 363         if (bdrv_is_read_only(bs)) {
 364             continue;
 365         }
 366
 367         sectors = bdrv_nb_sectors(bs);
 368         if (sectors <= 0) {
 369             return;
 370         }
 371
 372         bmds = g_new0(BlkMigDevState, 1);
 373         bmds->bs = bs;
 374         bmds->bulk_completed = 0;
 375         bmds->total_sectors = sectors;
 376         bmds->completed_sectors = 0;
 377         bmds->shared_base = block_mig_state.shared_base;
 378         alloc_aio_bitmap(bmds);
 379         error_setg(&bmds->blocker, "block device is in use by migration");
 380         bdrv_op_block_all(bs, bmds->blocker);
 381         bdrv_ref(bs);
 382
 383         block_mig_state.total_sector_sum += sectors;
 384
 385         if (bmds->shared_base) {
 386             DPRINTF("Start migration for %s with shared base image\n",
 387                     bdrv_get_device_name(bs));
 388         } else {
 389             DPRINTF("Start full migration for %s\n", bdrv_get_device_name(bs));
 390         }
 391
 392         QSIMPLEQ_INSERT_TAIL(&block_mig_state.bmds_list, bmds, entry);
 393     }
 394 }
 395
 396 /* Called with no lock taken.  */
 397
 398 static int blk_mig_save_bulked_block(QEMUFile *f)
 399 {
 400     int64_t completed_sector_sum = 0;
 401     BlkMigDevState *bmds;
 402     int progress;
 403     int ret = 0;
 404
 405     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 406         if (bmds->bulk_completed == 0) {
 407             if (mig_save_device_bulk(f, bmds) == 1) {
 408                 /* completed bulk section for this device */
 409                 bmds->bulk_completed = 1;
 410             }
 411             completed_sector_sum += bmds->completed_sectors;
 412             ret = 1;
 413             break;
 414         } else {
 415             completed_sector_sum += bmds->completed_sectors;
 416         }
 417     }
 418
 419     if (block_mig_state.total_sector_sum != 0) {
 420         progress = completed_sector_sum * 100 /
 421                    block_mig_state.total_sector_sum;
 422     } else {
 423         progress = 100;
 424     }
 425     if (progress != block_mig_state.prev_progress) {
 426         block_mig_state.prev_progress = progress;
 427         qemu_put_be64(f, (progress << BDRV_SECTOR_BITS)
 428                          | BLK_MIG_FLAG_PROGRESS);
 429         DPRINTF("Completed %d %%\r", progress);
 430     }
 431
 432     return ret;
 433 }
 434
 435 static void blk_mig_reset_dirty_cursor(void)
 436 {
 437     BlkMigDevState *bmds;
 438
 439     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 440         bmds->cur_dirty = 0;
 441     }
 442 }
 443
 444 /* Called with iothread lock taken.  */
 445
 446 static int mig_save_device_dirty(QEMUFile *f, BlkMigDevState *bmds,
 447                                  int is_async)
 448 {
 449     BlkMigBlock *blk;
 450     int64_t total_sectors = bmds->total_sectors;
 451     int64_t sector;
 452     int nr_sectors;
 453     int ret = -EIO;
 454
 455     for (sector = bmds->cur_dirty; sector < bmds->total_sectors;) {
 456         blk_mig_lock();
 457         if (bmds_aio_inflight(bmds, sector)) {
 458             blk_mig_unlock();
 459             bdrv_drain_all();
 460         } else {
 461             blk_mig_unlock();
 462         }
 463         if (bdrv_get_dirty(bmds->bs, bmds->dirty_bitmap, sector)) {
 464
 465             if (total_sectors - sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
 466                 nr_sectors = total_sectors - sector;
 467             } else {
 468                 nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
 469             }
 470             blk = g_new(BlkMigBlock, 1);
 471             blk->buf = g_malloc(BLOCK_SIZE);
 472             blk->bmds = bmds;
 473             blk->sector = sector;
 474             blk->nr_sectors = nr_sectors;
 475
 476             if (is_async) {
 477                 blk->iov.iov_base = blk->buf;
 478                 blk->iov.iov_len = nr_sectors * BDRV_SECTOR_SIZE;
 479                 qemu_iovec_init_external(&blk->qiov, &blk->iov, 1);
 480
 481                 blk->aiocb = bdrv_aio_readv(bmds->bs, sector, &blk->qiov,
 482                                             nr_sectors, blk_mig_read_cb, blk);
 483
 484                 blk_mig_lock();
 485                 block_mig_state.submitted++;
 486                 bmds_set_aio_inflight(bmds, sector, nr_sectors, 1);
 487                 blk_mig_unlock();
 488             } else {
 489                 ret = bdrv_read(bmds->bs, sector, blk->buf, nr_sectors);
 490                 if (ret < 0) {
 491                     goto error;
 492                 }
 493                 blk_send(f, blk);
 494
 495                 g_free(blk->buf);
 496                 g_free(blk);
 497             }
 498
 499             bdrv_reset_dirty_bitmap(bmds->bs, bmds->dirty_bitmap, sector,
 500                                     nr_sectors);
 501             break;
 502         }
 503         sector += BDRV_SECTORS_PER_DIRTY_CHUNK;
 504         bmds->cur_dirty = sector;
 505     }
 506
 507     return (bmds->cur_dirty >= bmds->total_sectors);
 508
 509 error:
 510     DPRINTF("Error reading sector %" PRId64 "\n", sector);
 511     g_free(blk->buf);
 512     g_free(blk);
 513     return ret;
 514 }
 515
 516 /* Called with iothread lock taken.
 517  *
 518  * return value:
 519  * 0: too much data for max_downtime
 520  * 1: few enough data for max_downtime
 521 */
 522 static int blk_mig_save_dirty_block(QEMUFile *f, int is_async)
 523 {
 524     BlkMigDevState *bmds;
 525     int ret = 1;
 526
 527     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 528         ret = mig_save_device_dirty(f, bmds, is_async);
 529         if (ret <= 0) {
 530             break;
 531         }
 532     }
 533
 534     return ret;
 535 }
 536
 537 /* Called with no locks taken.  */
 538
 539 static int flush_blks(QEMUFile *f)
 540 {
 541     BlkMigBlock *blk;
 542     int ret = 0;
 543
 544     DPRINTF("%s Enter submitted %d read_done %d transferred %d\n",
 545             __FUNCTION__, block_mig_state.submitted, block_mig_state.read_done,
 546             block_mig_state.transferred);
 547
 548     blk_mig_lock();
 549     while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
 550         if (qemu_file_rate_limit(f)) {
 551             break;
 552         }
 553         if (blk->ret < 0) {
 554             ret = blk->ret;
 555             break;
 556         }
 557
 558         QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
 559         blk_mig_unlock();
 560         blk_send(f, blk);
 561         blk_mig_lock();
 562
 563         g_free(blk->buf);
 564         g_free(blk);
 565
 566         block_mig_state.read_done--;
 567         block_mig_state.transferred++;
 568         assert(block_mig_state.read_done >= 0);
 569     }
 570     blk_mig_unlock();
 571
 572     DPRINTF("%s Exit submitted %d read_done %d transferred %d\n", __FUNCTION__,
 573             block_mig_state.submitted, block_mig_state.read_done,
 574             block_mig_state.transferred);
 575     return ret;
 576 }
 577
 578 /* Called with iothread lock taken.  */
 579
 580 static int64_t get_remaining_dirty(void)
 581 {
 582     BlkMigDevState *bmds;
 583     int64_t dirty = 0;
 584
 585     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
 586         dirty += bdrv_get_dirty_count(bmds->bs, bmds->dirty_bitmap);
 587     }
 588
 589     return dirty << BDRV_SECTOR_BITS;
 590 }
 591
 592 /* Called with iothread lock taken.  */
 593
 594 static void blk_mig_cleanup(void)
 595 {
 596     BlkMigDevState *bmds;
 597     BlkMigBlock *blk;
 598
 599     bdrv_drain_all();
 600
 601     unset_dirty_tracking();
 602
 603     blk_mig_lock();
 604     while ((bmds = QSIMPLEQ_FIRST(&block_mig_state.bmds_list)) != NULL) {
 605         QSIMPLEQ_REMOVE_HEAD(&block_mig_state.bmds_list, entry);
 606         bdrv_op_unblock_all(bmds->bs, bmds->blocker);
 607         error_free(bmds->blocker);
 608         bdrv_unref(bmds->bs);
 609         g_free(bmds->aio_bitmap);
 610         g_free(bmds);
 611     }
 612
 613     while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
 614         QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
 615         g_free(blk->buf);
 616         g_free(blk);
 617     }
 618     blk_mig_unlock();
 619 }
 620
 621 static void block_migration_cancel(void *opaque)
 622 {
 623     blk_mig_cleanup();
 624 }
 625
 626 static int block_save_setup(QEMUFile *f, void *opaque)
 627 {
 628     int ret;
 629
 630     DPRINTF("Enter save live setup submitted %d transferred %d\n",
 631             block_mig_state.submitted, block_mig_state.transferred);
 632
 633     qemu_mutex_lock_iothread();
 634     init_blk_migration(f);
 635
 636     /* start track dirty blocks */
 637     ret = set_dirty_tracking();
 638
 639     if (ret) {
 640         qemu_mutex_unlock_iothread();
 641         return ret;
 642     }
 643
 644     qemu_mutex_unlock_iothread();
 645
 646     ret = flush_blks(f);
 647     blk_mig_reset_dirty_cursor();
 648     qemu_put_be64(f, BLK_MIG_FLAG_EOS);
 649
 650     return ret;
 651 }
 652
 653 static int block_save_iterate(QEMUFile *f, void *opaque)
 654 {
 655     int ret;
 656     int64_t last_ftell = qemu_ftell(f);
 657     int64_t delta_ftell;
 658
 659     DPRINTF("Enter save live iterate submitted %d transferred %d\n",
 660             block_mig_state.submitted, block_mig_state.transferred);
 661
 662     ret = flush_blks(f);
 663     if (ret) {
 664         return ret;
 665     }
 666
 667     blk_mig_reset_dirty_cursor();
 668
 669     /* control the rate of transfer */
 670     blk_mig_lock();
 671     while ((block_mig_state.submitted +
 672             block_mig_state.read_done) * BLOCK_SIZE <
 673            qemu_file_get_rate_limit(f)) {
 674         blk_mig_unlock();
 675         if (block_mig_state.bulk_completed == 0) {
 676             /* first finish the bulk phase */
 677             if (blk_mig_save_bulked_block(f) == 0) {
 678                 /* finished saving bulk on all devices */
 679                 block_mig_state.bulk_completed = 1;
 680             }
 681             ret = 0;
 682         } else {
 683             /* Always called with iothread lock taken for
 684              * simplicity, block_save_complete also calls it.
 685              */
 686             qemu_mutex_lock_iothread();
 687             ret = blk_mig_save_dirty_block(f, 1);
 688             qemu_mutex_unlock_iothread();
 689         }
 690         if (ret < 0) {
 691             return ret;
 692         }
 693         blk_mig_lock();
 694         if (ret != 0) {
 695             /* no more dirty blocks */
 696             break;
 697         }
 698     }
 699     blk_mig_unlock();
 700
 701     ret = flush_blks(f);
 702     if (ret) {
 703         return ret;
 704     }
 705
 706     qemu_put_be64(f, BLK_MIG_FLAG_EOS);
 707     delta_ftell = qemu_ftell(f) - last_ftell;
 708     if (delta_ftell > 0) {
 709         return 1;
 710     } else if (delta_ftell < 0) {
 711         return -1;
 712     } else {
 713         return 0;
 714     }
 715 }
 716
 717 /* Called with iothread lock taken.  */
 718
 719 static int block_save_complete(QEMUFile *f, void *opaque)
 720 {
 721     int ret;
 722
 723     DPRINTF("Enter save live complete submitted %d transferred %d\n",
 724             block_mig_state.submitted, block_mig_state.transferred);
 725
 726     ret = flush_blks(f);
 727     if (ret) {
 728         return ret;
 729     }
 730
 731     blk_mig_reset_dirty_cursor();
 732
 733     /* we know for sure that save bulk is completed and
 734        all async read completed */
 735     blk_mig_lock();
 736     assert(block_mig_state.submitted == 0);
 737     blk_mig_unlock();
 738
 739     do {
 740         ret = blk_mig_save_dirty_block(f, 0);
 741         if (ret < 0) {
 742             return ret;
 743         }
 744     } while (ret == 0);
 745
 746     /* report completion */
 747     qemu_put_be64(f, (100 << BDRV_SECTOR_BITS) | BLK_MIG_FLAG_PROGRESS);
 748
 749     DPRINTF("Block migration completed\n");
 750
 751     qemu_put_be64(f, BLK_MIG_FLAG_EOS);
 752
 753     blk_mig_cleanup();
 754     return 0;
 755 }
 756
 757 static uint64_t block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size)
 758 {
 759     /* Estimate pending number of bytes to send */
 760     uint64_t pending;
 761
 762     qemu_mutex_lock_iothread();
 763     blk_mig_lock();
 764     pending = get_remaining_dirty() +
 765                        block_mig_state.submitted * BLOCK_SIZE +
 766                        block_mig_state.read_done * BLOCK_SIZE;
 767
 768     /* Report at least one block pending during bulk phase */
 769     if (pending <= max_size && !block_mig_state.bulk_completed) {
 770         pending = max_size + BLOCK_SIZE;
 771     }
 772     blk_mig_unlock();
 773     qemu_mutex_unlock_iothread();
 774
 775     DPRINTF("Enter save live pending  %" PRIu64 "\n", pending);
 776     return pending;
 777 }
 778
 779 static int block_load(QEMUFile *f, void *opaque, int version_id)
 780 {
 781     static int banner_printed;
 782     int len, flags;
 783     char device_name[256];
 784     int64_t addr;
 785     BlockDriverState *bs, *bs_prev = NULL;
 786     uint8_t *buf;
 787     int64_t total_sectors = 0;
 788     int nr_sectors;
 789     int ret;
 790
 791     do {
 792         addr = qemu_get_be64(f);
 793
 794         flags = addr & ~BDRV_SECTOR_MASK;
 795         addr >>= BDRV_SECTOR_BITS;
 796
 797         if (flags & BLK_MIG_FLAG_DEVICE_BLOCK) {
 798             /* get device name */
 799             len = qemu_get_byte(f);
 800             qemu_get_buffer(f, (uint8_t *)device_name, len);
 801             device_name[len] = '\0';
 802
 803             bs = bdrv_find(device_name);
 804             if (!bs) {
 805                 fprintf(stderr, "Error unknown block device %s\n",
 806                         device_name);
 807                 return -EINVAL;
 808             }
 809
 810             if (bs != bs_prev) {
 811                 bs_prev = bs;
 812                 total_sectors = bdrv_nb_sectors(bs);
 813                 if (total_sectors <= 0) {
 814                     error_report("Error getting length of block device %s",
 815                                  device_name);
 816                     return -EINVAL;
 817                 }
 818             }
 819
 820             if (total_sectors - addr < BDRV_SECTORS_PER_DIRTY_CHUNK) {
 821                 nr_sectors = total_sectors - addr;
 822             } else {
 823                 nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
 824             }
 825
 826             if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
 827                 ret = bdrv_write_zeroes(bs, addr, nr_sectors,
 828                                         BDRV_REQ_MAY_UNMAP);
 829             } else {
 830                 buf = g_malloc(BLOCK_SIZE);
 831                 qemu_get_buffer(f, buf, BLOCK_SIZE);
 832                 ret = bdrv_write(bs, addr, buf, nr_sectors);
 833                 g_free(buf);
 834             }
 835
 836             if (ret < 0) {
 837                 return ret;
 838             }
 839         } else if (flags & BLK_MIG_FLAG_PROGRESS) {
 840             if (!banner_printed) {
 841                 printf("Receiving block device images\n");
 842                 banner_printed = 1;
 843             }
 844             printf("Completed %d %%%c", (int)addr,
 845                    (addr == 100) ? '\n' : '\r');
 846             fflush(stdout);
 847         } else if (!(flags & BLK_MIG_FLAG_EOS)) {
 848             fprintf(stderr, "Unknown block migration flags: %#x\n", flags);
 849             return -EINVAL;
 850         }
 851         ret = qemu_file_get_error(f);
 852         if (ret != 0) {
 853             return ret;
 854         }
 855     } while (!(flags & BLK_MIG_FLAG_EOS));
 856
 857     return 0;
 858 }
 859
 860 static void block_set_params(const MigrationParams *params, void *opaque)
 861 {
 862     block_mig_state.blk_enable = params->blk;
 863     block_mig_state.shared_base = params->shared;
 864
 865     /* shared base means that blk_enable = 1 */
 866     block_mig_state.blk_enable |= params->shared;
 867 }
 868
 869 static bool block_is_active(void *opaque)
 870 {
 871     return block_mig_state.blk_enable == 1;
 872 }
 873
 874 static SaveVMHandlers savevm_block_handlers = {
 875     .set_params = block_set_params,
 876     .save_live_setup = block_save_setup,
 877     .save_live_iterate = block_save_iterate,
 878     .save_live_complete = block_save_complete,
 879     .save_live_pending = block_save_pending,
 880     .load_state = block_load,
 881     .cancel = block_migration_cancel,
 882     .is_active = block_is_active,
 883 };
 884
 885 void blk_mig_init(void)
 886 {
 887     QSIMPLEQ_INIT(&block_mig_state.bmds_list);
 888     QSIMPLEQ_INIT(&block_mig_state.blk_list);
 889     qemu_mutex_init(&block_mig_state.lock);
 890
 891     register_savevm_live(NULL, "block", 0, 1, &savevm_block_handlers,
 892                          &block_mig_state);
 893 }