migration/migration.c

   1 /*
   2  * QEMU live migration
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *
   6  * Authors:
   7  *  Anthony Liguori   <aliguori@us.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include "qemu/cutils.h"
  18 #include "qemu/error-report.h"
  19 #include "qemu/main-loop.h"
  20 #include "migration/blocker.h"
  21 #include "exec.h"
  22 #include "fd.h"
  23 #include "socket.h"
  24 #include "sysemu/runstate.h"
  25 #include "sysemu/sysemu.h"
  26 #include "rdma.h"
  27 #include "ram.h"
  28 #include "migration/global_state.h"
  29 #include "migration/misc.h"
  30 #include "migration.h"
  31 #include "savevm.h"
  32 #include "qemu-file-channel.h"
  33 #include "qemu-file.h"
  34 #include "migration/vmstate.h"
  35 #include "block/block.h"
  36 #include "qapi/error.h"
  37 #include "qapi/clone-visitor.h"
  38 #include "qapi/qapi-visit-sockets.h"
  39 #include "qapi/qapi-commands-migration.h"
  40 #include "qapi/qapi-events-migration.h"
  41 #include "qapi/qmp/qerror.h"
  42 #include "qapi/qmp/qnull.h"
  43 #include "qemu/rcu.h"
  44 #include "block.h"
  45 #include "postcopy-ram.h"
  46 #include "qemu/thread.h"
  47 #include "trace.h"
  48 #include "exec/target_page.h"
  49 #include "io/channel-buffer.h"
  50 #include "migration/colo.h"
  51 #include "hw/boards.h"
  52 #include "hw/qdev-properties.h"
  53 #include "monitor/monitor.h"
  54 #include "net/announce.h"
  55 #include "qemu/queue.h"
  56 #include "multifd.h"
  57
  58 #define MAX_THROTTLE  (32 << 20)      /* Migration transfer speed throttling */
  59
  60 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
  61  * data. */
  62 #define BUFFER_DELAY     100
  63 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
  64
  65 /* Time in milliseconds we are allowed to stop the source,
  66  * for sending the last part */
  67 #define DEFAULT_MIGRATE_SET_DOWNTIME 300
  68
  69 /* Maximum migrate downtime set to 2000 seconds */
  70 #define MAX_MIGRATE_DOWNTIME_SECONDS 2000
  71 #define MAX_MIGRATE_DOWNTIME (MAX_MIGRATE_DOWNTIME_SECONDS * 1000)
  72
  73 /* Default compression thread count */
  74 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8
  75 /* Default decompression thread count, usually decompression is at
  76  * least 4 times as fast as compression.*/
  77 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2
  78 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */
  79 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1
  80 /* Define default autoconverge cpu throttle migration parameters */
  81 #define DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD 50
  82 #define DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL 20
  83 #define DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT 10
  84 #define DEFAULT_MIGRATE_MAX_CPU_THROTTLE 99
  85
  86 /* Migration XBZRLE default cache size */
  87 #define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024)
  88
  89 /* The delay time (in ms) between two COLO checkpoints */
  90 #define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100)
  91 #define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2
  92 #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE
  93 /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */
  94 #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1
  95 /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */
  96 #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1
  97
  98 /* Background transfer rate for postcopy, 0 means unlimited, note
  99  * that page requests can still exceed this limit.
 100  */
 101 #define DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH 0
 102
 103 /*
 104  * Parameters for self_announce_delay giving a stream of RARP/ARP
 105  * packets after migration.
 106  */
 107 #define DEFAULT_MIGRATE_ANNOUNCE_INITIAL  50
 108 #define DEFAULT_MIGRATE_ANNOUNCE_MAX     550
 109 #define DEFAULT_MIGRATE_ANNOUNCE_ROUNDS    5
 110 #define DEFAULT_MIGRATE_ANNOUNCE_STEP    100
 111
 112 static NotifierList migration_state_notifiers =
 113     NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
 114
 115 static bool deferred_incoming;
 116
 117 /* Messages sent on the return path from destination to source */
 118 enum mig_rp_message_type {
 119     MIG_RP_MSG_INVALID = 0,  /* Must be 0 */
 120     MIG_RP_MSG_SHUT,         /* sibling will not send any more RP messages */
 121     MIG_RP_MSG_PONG,         /* Response to a PING; data (seq: be32 ) */
 122
 123     MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
 124     MIG_RP_MSG_REQ_PAGES,    /* data (start: be64, len: be32) */
 125     MIG_RP_MSG_RECV_BITMAP,  /* send recved_bitmap back to source */
 126     MIG_RP_MSG_RESUME_ACK,   /* tell source that we are ready to resume */
 127
 128     MIG_RP_MSG_MAX
 129 };
 130
 131 /* When we add fault tolerance, we could have several
 132    migrations at once.  For now we don't need to add
 133    dynamic creation of migration */
 134
 135 static MigrationState *current_migration;
 136 static MigrationIncomingState *current_incoming;
 137
 138 static bool migration_object_check(MigrationState *ms, Error **errp);
 139 static int migration_maybe_pause(MigrationState *s,
 140                                  int *current_active_state,
 141                                  int new_state);
 142 static void migrate_fd_cancel(MigrationState *s);
 143
 144 void migration_object_init(void)
 145 {
 146     MachineState *ms = MACHINE(qdev_get_machine());
 147     Error *err = NULL;
 148
 149     /* This can only be called once. */
 150     assert(!current_migration);
 151     current_migration = MIGRATION_OBJ(object_new(TYPE_MIGRATION));
 152
 153     /*
 154      * Init the migrate incoming object as well no matter whether
 155      * we'll use it or not.
 156      */
 157     assert(!current_incoming);
 158     current_incoming = g_new0(MigrationIncomingState, 1);
 159     current_incoming->state = MIGRATION_STATUS_NONE;
 160     current_incoming->postcopy_remote_fds =
 161         g_array_new(FALSE, TRUE, sizeof(struct PostCopyFD));
 162     qemu_mutex_init(&current_incoming->rp_mutex);
 163     qemu_event_init(&current_incoming->main_thread_load_event, false);
 164     qemu_sem_init(&current_incoming->postcopy_pause_sem_dst, 0);
 165     qemu_sem_init(&current_incoming->postcopy_pause_sem_fault, 0);
 166
 167     init_dirty_bitmap_incoming_migration();
 168
 169     if (!migration_object_check(current_migration, &err)) {
 170         error_report_err(err);
 171         exit(1);
 172     }
 173
 174     /*
 175      * We cannot really do this in migration_instance_init() since at
 176      * that time global properties are not yet applied, then this
 177      * value will be definitely replaced by something else.
 178      */
 179     if (ms->enforce_config_section) {
 180         current_migration->send_configuration = true;
 181     }
 182 }
 183
 184 void migration_shutdown(void)
 185 {
 186     /*
 187      * Cancel the current migration - that will (eventually)
 188      * stop the migration using this structure
 189      */
 190     migrate_fd_cancel(current_migration);
 191     object_unref(OBJECT(current_migration));
 192 }
 193
 194 /* For outgoing */
 195 MigrationState *migrate_get_current(void)
 196 {
 197     /* This can only be called after the object created. */
 198     assert(current_migration);
 199     return current_migration;
 200 }
 201
 202 MigrationIncomingState *migration_incoming_get_current(void)
 203 {
 204     assert(current_incoming);
 205     return current_incoming;
 206 }
 207
 208 void migration_incoming_state_destroy(void)
 209 {
 210     struct MigrationIncomingState *mis = migration_incoming_get_current();
 211
 212     if (mis->to_src_file) {
 213         /* Tell source that we are done */
 214         migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
 215         qemu_fclose(mis->to_src_file);
 216         mis->to_src_file = NULL;
 217     }
 218
 219     if (mis->from_src_file) {
 220         qemu_fclose(mis->from_src_file);
 221         mis->from_src_file = NULL;
 222     }
 223     if (mis->postcopy_remote_fds) {
 224         g_array_free(mis->postcopy_remote_fds, TRUE);
 225         mis->postcopy_remote_fds = NULL;
 226     }
 227
 228     qemu_event_reset(&mis->main_thread_load_event);
 229
 230     if (mis->socket_address_list) {
 231         qapi_free_SocketAddressList(mis->socket_address_list);
 232         mis->socket_address_list = NULL;
 233     }
 234 }
 235
 236 static void migrate_generate_event(int new_state)
 237 {
 238     if (migrate_use_events()) {
 239         qapi_event_send_migration(new_state);
 240     }
 241 }
 242
 243 static bool migrate_late_block_activate(void)
 244 {
 245     MigrationState *s;
 246
 247     s = migrate_get_current();
 248
 249     return s->enabled_capabilities[
 250         MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE];
 251 }
 252
 253 /*
 254  * Called on -incoming with a defer: uri.
 255  * The migration can be started later after any parameters have been
 256  * changed.
 257  */
 258 static void deferred_incoming_migration(Error **errp)
 259 {
 260     if (deferred_incoming) {
 261         error_setg(errp, "Incoming migration already deferred");
 262     }
 263     deferred_incoming = true;
 264 }
 265
 266 /*
 267  * Send a message on the return channel back to the source
 268  * of the migration.
 269  */
 270 static int migrate_send_rp_message(MigrationIncomingState *mis,
 271                                    enum mig_rp_message_type message_type,
 272                                    uint16_t len, void *data)
 273 {
 274     int ret = 0;
 275
 276     trace_migrate_send_rp_message((int)message_type, len);
 277     qemu_mutex_lock(&mis->rp_mutex);
 278
 279     /*
 280      * It's possible that the file handle got lost due to network
 281      * failures.
 282      */
 283     if (!mis->to_src_file) {
 284         ret = -EIO;
 285         goto error;
 286     }
 287
 288     qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
 289     qemu_put_be16(mis->to_src_file, len);
 290     qemu_put_buffer(mis->to_src_file, data, len);
 291     qemu_fflush(mis->to_src_file);
 292
 293     /* It's possible that qemu file got error during sending */
 294     ret = qemu_file_get_error(mis->to_src_file);
 295
 296 error:
 297     qemu_mutex_unlock(&mis->rp_mutex);
 298     return ret;
 299 }
 300
 301 /* Request a range of pages from the source VM at the given
 302  * start address.
 303  *   rbname: Name of the RAMBlock to request the page in, if NULL it's the same
 304  *           as the last request (a name must have been given previously)
 305  *   Start: Address offset within the RB
 306  *   Len: Length in bytes required - must be a multiple of pagesize
 307  */
 308 int migrate_send_rp_req_pages(MigrationIncomingState *mis, const char *rbname,
 309                               ram_addr_t start, size_t len)
 310 {
 311     uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */
 312     size_t msglen = 12; /* start + len */
 313     enum mig_rp_message_type msg_type;
 314
 315     *(uint64_t *)bufc = cpu_to_be64((uint64_t)start);
 316     *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len);
 317
 318     if (rbname) {
 319         int rbname_len = strlen(rbname);
 320         assert(rbname_len < 256);
 321
 322         bufc[msglen++] = rbname_len;
 323         memcpy(bufc + msglen, rbname, rbname_len);
 324         msglen += rbname_len;
 325         msg_type = MIG_RP_MSG_REQ_PAGES_ID;
 326     } else {
 327         msg_type = MIG_RP_MSG_REQ_PAGES;
 328     }
 329
 330     return migrate_send_rp_message(mis, msg_type, msglen, bufc);
 331 }
 332
 333 static bool migration_colo_enabled;
 334 bool migration_incoming_colo_enabled(void)
 335 {
 336     return migration_colo_enabled;
 337 }
 338
 339 void migration_incoming_disable_colo(void)
 340 {
 341     ram_block_discard_disable(false);
 342     migration_colo_enabled = false;
 343 }
 344
 345 int migration_incoming_enable_colo(void)
 346 {
 347     if (ram_block_discard_disable(true)) {
 348         error_report("COLO: cannot disable RAM discard");
 349         return -EBUSY;
 350     }
 351     migration_colo_enabled = true;
 352     return 0;
 353 }
 354
 355 void migrate_add_address(SocketAddress *address)
 356 {
 357     MigrationIncomingState *mis = migration_incoming_get_current();
 358     SocketAddressList *addrs;
 359
 360     addrs = g_new0(SocketAddressList, 1);
 361     addrs->next = mis->socket_address_list;
 362     mis->socket_address_list = addrs;
 363     addrs->value = QAPI_CLONE(SocketAddress, address);
 364 }
 365
 366 void qemu_start_incoming_migration(const char *uri, Error **errp)
 367 {
 368     const char *p;
 369
 370     qapi_event_send_migration(MIGRATION_STATUS_SETUP);
 371     if (!strcmp(uri, "defer")) {
 372         deferred_incoming_migration(errp);
 373     } else if (strstart(uri, "tcp:", &p)) {
 374         tcp_start_incoming_migration(p, errp);
 375 #ifdef CONFIG_RDMA
 376     } else if (strstart(uri, "rdma:", &p)) {
 377         rdma_start_incoming_migration(p, errp);
 378 #endif
 379     } else if (strstart(uri, "exec:", &p)) {
 380         exec_start_incoming_migration(p, errp);
 381     } else if (strstart(uri, "unix:", &p)) {
 382         unix_start_incoming_migration(p, errp);
 383     } else if (strstart(uri, "fd:", &p)) {
 384         fd_start_incoming_migration(p, errp);
 385     } else {
 386         error_setg(errp, "unknown migration protocol: %s", uri);
 387     }
 388 }
 389
 390 static void process_incoming_migration_bh(void *opaque)
 391 {
 392     Error *local_err = NULL;
 393     MigrationIncomingState *mis = opaque;
 394
 395     /* If capability late_block_activate is set:
 396      * Only fire up the block code now if we're going to restart the
 397      * VM, else 'cont' will do it.
 398      * This causes file locking to happen; so we don't want it to happen
 399      * unless we really are starting the VM.
 400      */
 401     if (!migrate_late_block_activate() ||
 402          (autostart && (!global_state_received() ||
 403             global_state_get_runstate() == RUN_STATE_RUNNING))) {
 404         /* Make sure all file formats flush their mutable metadata.
 405          * If we get an error here, just don't restart the VM yet. */
 406         bdrv_invalidate_cache_all(&local_err);
 407         if (local_err) {
 408             error_report_err(local_err);
 409             local_err = NULL;
 410             autostart = false;
 411         }
 412     }
 413
 414     /*
 415      * This must happen after all error conditions are dealt with and
 416      * we're sure the VM is going to be running on this host.
 417      */
 418     qemu_announce_self(&mis->announce_timer, migrate_announce_params());
 419
 420     if (multifd_load_cleanup(&local_err) != 0) {
 421         error_report_err(local_err);
 422         autostart = false;
 423     }
 424     /* If global state section was not received or we are in running
 425        state, we need to obey autostart. Any other state is set with
 426        runstate_set. */
 427
 428     dirty_bitmap_mig_before_vm_start();
 429
 430     if (!global_state_received() ||
 431         global_state_get_runstate() == RUN_STATE_RUNNING) {
 432         if (autostart) {
 433             vm_start();
 434         } else {
 435             runstate_set(RUN_STATE_PAUSED);
 436         }
 437     } else if (migration_incoming_colo_enabled()) {
 438         migration_incoming_disable_colo();
 439         vm_start();
 440     } else {
 441         runstate_set(global_state_get_runstate());
 442     }
 443     /*
 444      * This must happen after any state changes since as soon as an external
 445      * observer sees this event they might start to prod at the VM assuming
 446      * it's ready to use.
 447      */
 448     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 449                       MIGRATION_STATUS_COMPLETED);
 450     qemu_bh_delete(mis->bh);
 451     migration_incoming_state_destroy();
 452 }
 453
 454 static void process_incoming_migration_co(void *opaque)
 455 {
 456     MigrationIncomingState *mis = migration_incoming_get_current();
 457     PostcopyState ps;
 458     int ret;
 459     Error *local_err = NULL;
 460
 461     assert(mis->from_src_file);
 462     mis->migration_incoming_co = qemu_coroutine_self();
 463     mis->largest_page_size = qemu_ram_pagesize_largest();
 464     postcopy_state_set(POSTCOPY_INCOMING_NONE);
 465     migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
 466                       MIGRATION_STATUS_ACTIVE);
 467     ret = qemu_loadvm_state(mis->from_src_file);
 468
 469     ps = postcopy_state_get();
 470     trace_process_incoming_migration_co_end(ret, ps);
 471     if (ps != POSTCOPY_INCOMING_NONE) {
 472         if (ps == POSTCOPY_INCOMING_ADVISE) {
 473             /*
 474              * Where a migration had postcopy enabled (and thus went to advise)
 475              * but managed to complete within the precopy period, we can use
 476              * the normal exit.
 477              */
 478             postcopy_ram_incoming_cleanup(mis);
 479         } else if (ret >= 0) {
 480             /*
 481              * Postcopy was started, cleanup should happen at the end of the
 482              * postcopy thread.
 483              */
 484             trace_process_incoming_migration_co_postcopy_end_main();
 485             return;
 486         }
 487         /* Else if something went wrong then just fall out of the normal exit */
 488     }
 489
 490     /* we get COLO info, and know if we are in COLO mode */
 491     if (!ret && migration_incoming_colo_enabled()) {
 492         /* Make sure all file formats flush their mutable metadata */
 493         bdrv_invalidate_cache_all(&local_err);
 494         if (local_err) {
 495             error_report_err(local_err);
 496             goto fail;
 497         }
 498
 499         qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
 500              colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
 501         mis->have_colo_incoming_thread = true;
 502         qemu_coroutine_yield();
 503
 504         /* Wait checkpoint incoming thread exit before free resource */
 505         qemu_thread_join(&mis->colo_incoming_thread);
 506         /* We hold the global iothread lock, so it is safe here */
 507         colo_release_ram_cache();
 508     }
 509
 510     if (ret < 0) {
 511         error_report("load of migration failed: %s", strerror(-ret));
 512         goto fail;
 513     }
 514     mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
 515     qemu_bh_schedule(mis->bh);
 516     mis->migration_incoming_co = NULL;
 517     return;
 518 fail:
 519     local_err = NULL;
 520     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 521                       MIGRATION_STATUS_FAILED);
 522     qemu_fclose(mis->from_src_file);
 523     if (multifd_load_cleanup(&local_err) != 0) {
 524         error_report_err(local_err);
 525     }
 526     exit(EXIT_FAILURE);
 527 }
 528
 529 /**
 530  * @migration_incoming_setup: Setup incoming migration
 531  *
 532  * Returns 0 for no error or 1 for error
 533  *
 534  * @f: file for main migration channel
 535  * @errp: where to put errors
 536  */
 537 static int migration_incoming_setup(QEMUFile *f, Error **errp)
 538 {
 539     MigrationIncomingState *mis = migration_incoming_get_current();
 540     Error *local_err = NULL;
 541
 542     if (multifd_load_setup(&local_err) != 0) {
 543         /* We haven't been able to create multifd threads
 544            nothing better to do */
 545         error_report_err(local_err);
 546         exit(EXIT_FAILURE);
 547     }
 548
 549     if (!mis->from_src_file) {
 550         mis->from_src_file = f;
 551     }
 552     qemu_file_set_blocking(f, false);
 553     return 0;
 554 }
 555
 556 void migration_incoming_process(void)
 557 {
 558     Coroutine *co = qemu_coroutine_create(process_incoming_migration_co, NULL);
 559     qemu_coroutine_enter(co);
 560 }
 561
 562 /* Returns true if recovered from a paused migration, otherwise false */
 563 static bool postcopy_try_recover(QEMUFile *f)
 564 {
 565     MigrationIncomingState *mis = migration_incoming_get_current();
 566
 567     if (mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
 568         /* Resumed from a paused postcopy migration */
 569
 570         mis->from_src_file = f;
 571         /* Postcopy has standalone thread to do vm load */
 572         qemu_file_set_blocking(f, true);
 573
 574         /* Re-configure the return path */
 575         mis->to_src_file = qemu_file_get_return_path(f);
 576
 577         migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
 578                           MIGRATION_STATUS_POSTCOPY_RECOVER);
 579
 580         /*
 581          * Here, we only wake up the main loading thread (while the
 582          * fault thread will still be waiting), so that we can receive
 583          * commands from source now, and answer it if needed. The
 584          * fault thread will be woken up afterwards until we are sure
 585          * that source is ready to reply to page requests.
 586          */
 587         qemu_sem_post(&mis->postcopy_pause_sem_dst);
 588         return true;
 589     }
 590
 591     return false;
 592 }
 593
 594 void migration_fd_process_incoming(QEMUFile *f, Error **errp)
 595 {
 596     Error *local_err = NULL;
 597
 598     if (postcopy_try_recover(f)) {
 599         return;
 600     }
 601
 602     if (migration_incoming_setup(f, &local_err)) {
 603         if (local_err) {
 604             error_propagate(errp, local_err);
 605         }
 606         return;
 607     }
 608     migration_incoming_process();
 609 }
 610
 611 void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp)
 612 {
 613     MigrationIncomingState *mis = migration_incoming_get_current();
 614     Error *local_err = NULL;
 615     bool start_migration;
 616
 617     if (!mis->from_src_file) {
 618         /* The first connection (multifd may have multiple) */
 619         QEMUFile *f = qemu_fopen_channel_input(ioc);
 620
 621         /* If it's a recovery, we're done */
 622         if (postcopy_try_recover(f)) {
 623             return;
 624         }
 625
 626         if (migration_incoming_setup(f, &local_err)) {
 627             if (local_err) {
 628                 error_propagate(errp, local_err);
 629             }
 630             return;
 631         }
 632
 633         /*
 634          * Common migration only needs one channel, so we can start
 635          * right now.  Multifd needs more than one channel, we wait.
 636          */
 637         start_migration = !migrate_use_multifd();
 638     } else {
 639         /* Multiple connections */
 640         assert(migrate_use_multifd());
 641         start_migration = multifd_recv_new_channel(ioc, &local_err);
 642         if (local_err) {
 643             error_propagate(errp, local_err);
 644             return;
 645         }
 646     }
 647
 648     if (start_migration) {
 649         migration_incoming_process();
 650     }
 651 }
 652
 653 /**
 654  * @migration_has_all_channels: We have received all channels that we need
 655  *
 656  * Returns true when we have got connections to all the channels that
 657  * we need for migration.
 658  */
 659 bool migration_has_all_channels(void)
 660 {
 661     MigrationIncomingState *mis = migration_incoming_get_current();
 662     bool all_channels;
 663
 664     all_channels = multifd_recv_all_channels_created();
 665
 666     return all_channels && mis->from_src_file != NULL;
 667 }
 668
 669 /*
 670  * Send a 'SHUT' message on the return channel with the given value
 671  * to indicate that we've finished with the RP.  Non-0 value indicates
 672  * error.
 673  */
 674 void migrate_send_rp_shut(MigrationIncomingState *mis,
 675                           uint32_t value)
 676 {
 677     uint32_t buf;
 678
 679     buf = cpu_to_be32(value);
 680     migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
 681 }
 682
 683 /*
 684  * Send a 'PONG' message on the return channel with the given value
 685  * (normally in response to a 'PING')
 686  */
 687 void migrate_send_rp_pong(MigrationIncomingState *mis,
 688                           uint32_t value)
 689 {
 690     uint32_t buf;
 691
 692     buf = cpu_to_be32(value);
 693     migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
 694 }
 695
 696 void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
 697                                  char *block_name)
 698 {
 699     char buf[512];
 700     int len;
 701     int64_t res;
 702
 703     /*
 704      * First, we send the header part. It contains only the len of
 705      * idstr, and the idstr itself.
 706      */
 707     len = strlen(block_name);
 708     buf[0] = len;
 709     memcpy(buf + 1, block_name, len);
 710
 711     if (mis->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
 712         error_report("%s: MSG_RP_RECV_BITMAP only used for recovery",
 713                      __func__);
 714         return;
 715     }
 716
 717     migrate_send_rp_message(mis, MIG_RP_MSG_RECV_BITMAP, len + 1, buf);
 718
 719     /*
 720      * Next, we dump the received bitmap to the stream.
 721      *
 722      * TODO: currently we are safe since we are the only one that is
 723      * using the to_src_file handle (fault thread is still paused),
 724      * and it's ok even not taking the mutex. However the best way is
 725      * to take the lock before sending the message header, and release
 726      * the lock after sending the bitmap.
 727      */
 728     qemu_mutex_lock(&mis->rp_mutex);
 729     res = ramblock_recv_bitmap_send(mis->to_src_file, block_name);
 730     qemu_mutex_unlock(&mis->rp_mutex);
 731
 732     trace_migrate_send_rp_recv_bitmap(block_name, res);
 733 }
 734
 735 void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value)
 736 {
 737     uint32_t buf;
 738
 739     buf = cpu_to_be32(value);
 740     migrate_send_rp_message(mis, MIG_RP_MSG_RESUME_ACK, sizeof(buf), &buf);
 741 }
 742
 743 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
 744 {
 745     MigrationCapabilityStatusList *head = NULL;
 746     MigrationCapabilityStatusList *caps;
 747     MigrationState *s = migrate_get_current();
 748     int i;
 749
 750     caps = NULL; /* silence compiler warning */
 751     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
 752 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
 753         if (i == MIGRATION_CAPABILITY_BLOCK) {
 754             continue;
 755         }
 756 #endif
 757         if (head == NULL) {
 758             head = g_malloc0(sizeof(*caps));
 759             caps = head;
 760         } else {
 761             caps->next = g_malloc0(sizeof(*caps));
 762             caps = caps->next;
 763         }
 764         caps->value =
 765             g_malloc(sizeof(*caps->value));
 766         caps->value->capability = i;
 767         caps->value->state = s->enabled_capabilities[i];
 768     }
 769
 770     return head;
 771 }
 772
 773 MigrationParameters *qmp_query_migrate_parameters(Error **errp)
 774 {
 775     MigrationParameters *params;
 776     MigrationState *s = migrate_get_current();
 777
 778     /* TODO use QAPI_CLONE() instead of duplicating it inline */
 779     params = g_malloc0(sizeof(*params));
 780     params->has_compress_level = true;
 781     params->compress_level = s->parameters.compress_level;
 782     params->has_compress_threads = true;
 783     params->compress_threads = s->parameters.compress_threads;
 784     params->has_compress_wait_thread = true;
 785     params->compress_wait_thread = s->parameters.compress_wait_thread;
 786     params->has_decompress_threads = true;
 787     params->decompress_threads = s->parameters.decompress_threads;
 788     params->has_throttle_trigger_threshold = true;
 789     params->throttle_trigger_threshold = s->parameters.throttle_trigger_threshold;
 790     params->has_cpu_throttle_initial = true;
 791     params->cpu_throttle_initial = s->parameters.cpu_throttle_initial;
 792     params->has_cpu_throttle_increment = true;
 793     params->cpu_throttle_increment = s->parameters.cpu_throttle_increment;
 794     params->has_cpu_throttle_tailslow = true;
 795     params->cpu_throttle_tailslow = s->parameters.cpu_throttle_tailslow;
 796     params->has_tls_creds = true;
 797     params->tls_creds = g_strdup(s->parameters.tls_creds);
 798     params->has_tls_hostname = true;
 799     params->tls_hostname = g_strdup(s->parameters.tls_hostname);
 800     params->has_tls_authz = true;
 801     params->tls_authz = g_strdup(s->parameters.tls_authz ?
 802                                  s->parameters.tls_authz : "");
 803     params->has_max_bandwidth = true;
 804     params->max_bandwidth = s->parameters.max_bandwidth;
 805     params->has_downtime_limit = true;
 806     params->downtime_limit = s->parameters.downtime_limit;
 807     params->has_x_checkpoint_delay = true;
 808     params->x_checkpoint_delay = s->parameters.x_checkpoint_delay;
 809     params->has_block_incremental = true;
 810     params->block_incremental = s->parameters.block_incremental;
 811     params->has_multifd_channels = true;
 812     params->multifd_channels = s->parameters.multifd_channels;
 813     params->has_multifd_compression = true;
 814     params->multifd_compression = s->parameters.multifd_compression;
 815     params->has_multifd_zlib_level = true;
 816     params->multifd_zlib_level = s->parameters.multifd_zlib_level;
 817     params->has_multifd_zstd_level = true;
 818     params->multifd_zstd_level = s->parameters.multifd_zstd_level;
 819     params->has_xbzrle_cache_size = true;
 820     params->xbzrle_cache_size = s->parameters.xbzrle_cache_size;
 821     params->has_max_postcopy_bandwidth = true;
 822     params->max_postcopy_bandwidth = s->parameters.max_postcopy_bandwidth;
 823     params->has_max_cpu_throttle = true;
 824     params->max_cpu_throttle = s->parameters.max_cpu_throttle;
 825     params->has_announce_initial = true;
 826     params->announce_initial = s->parameters.announce_initial;
 827     params->has_announce_max = true;
 828     params->announce_max = s->parameters.announce_max;
 829     params->has_announce_rounds = true;
 830     params->announce_rounds = s->parameters.announce_rounds;
 831     params->has_announce_step = true;
 832     params->announce_step = s->parameters.announce_step;
 833
 834     return params;
 835 }
 836
 837 AnnounceParameters *migrate_announce_params(void)
 838 {
 839     static AnnounceParameters ap;
 840
 841     MigrationState *s = migrate_get_current();
 842
 843     ap.initial = s->parameters.announce_initial;
 844     ap.max = s->parameters.announce_max;
 845     ap.rounds = s->parameters.announce_rounds;
 846     ap.step = s->parameters.announce_step;
 847
 848     return &ap;
 849 }
 850
 851 /*
 852  * Return true if we're already in the middle of a migration
 853  * (i.e. any of the active or setup states)
 854  */
 855 bool migration_is_setup_or_active(int state)
 856 {
 857     switch (state) {
 858     case MIGRATION_STATUS_ACTIVE:
 859     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 860     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 861     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 862     case MIGRATION_STATUS_SETUP:
 863     case MIGRATION_STATUS_PRE_SWITCHOVER:
 864     case MIGRATION_STATUS_DEVICE:
 865     case MIGRATION_STATUS_WAIT_UNPLUG:
 866     case MIGRATION_STATUS_COLO:
 867         return true;
 868
 869     default:
 870         return false;
 871
 872     }
 873 }
 874
 875 bool migration_is_running(int state)
 876 {
 877     switch (state) {
 878     case MIGRATION_STATUS_ACTIVE:
 879     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 880     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 881     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 882     case MIGRATION_STATUS_SETUP:
 883     case MIGRATION_STATUS_PRE_SWITCHOVER:
 884     case MIGRATION_STATUS_DEVICE:
 885     case MIGRATION_STATUS_WAIT_UNPLUG:
 886     case MIGRATION_STATUS_CANCELLING:
 887         return true;
 888
 889     default:
 890         return false;
 891
 892     }
 893 }
 894
 895 static void populate_time_info(MigrationInfo *info, MigrationState *s)
 896 {
 897     info->has_status = true;
 898     info->has_setup_time = true;
 899     info->setup_time = s->setup_time;
 900     if (s->state == MIGRATION_STATUS_COMPLETED) {
 901         info->has_total_time = true;
 902         info->total_time = s->total_time;
 903         info->has_downtime = true;
 904         info->downtime = s->downtime;
 905     } else {
 906         info->has_total_time = true;
 907         info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) -
 908                            s->start_time;
 909         info->has_expected_downtime = true;
 910         info->expected_downtime = s->expected_downtime;
 911     }
 912 }
 913
 914 static void populate_ram_info(MigrationInfo *info, MigrationState *s)
 915 {
 916     info->has_ram = true;
 917     info->ram = g_malloc0(sizeof(*info->ram));
 918     info->ram->transferred = ram_counters.transferred;
 919     info->ram->total = ram_bytes_total();
 920     info->ram->duplicate = ram_counters.duplicate;
 921     /* legacy value.  It is not used anymore */
 922     info->ram->skipped = 0;
 923     info->ram->normal = ram_counters.normal;
 924     info->ram->normal_bytes = ram_counters.normal *
 925         qemu_target_page_size();
 926     info->ram->mbps = s->mbps;
 927     info->ram->dirty_sync_count = ram_counters.dirty_sync_count;
 928     info->ram->postcopy_requests = ram_counters.postcopy_requests;
 929     info->ram->page_size = qemu_target_page_size();
 930     info->ram->multifd_bytes = ram_counters.multifd_bytes;
 931     info->ram->pages_per_second = s->pages_per_second;
 932
 933     if (migrate_use_xbzrle()) {
 934         info->has_xbzrle_cache = true;
 935         info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
 936         info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
 937         info->xbzrle_cache->bytes = xbzrle_counters.bytes;
 938         info->xbzrle_cache->pages = xbzrle_counters.pages;
 939         info->xbzrle_cache->cache_miss = xbzrle_counters.cache_miss;
 940         info->xbzrle_cache->cache_miss_rate = xbzrle_counters.cache_miss_rate;
 941         info->xbzrle_cache->encoding_rate = xbzrle_counters.encoding_rate;
 942         info->xbzrle_cache->overflow = xbzrle_counters.overflow;
 943     }
 944
 945     if (migrate_use_compression()) {
 946         info->has_compression = true;
 947         info->compression = g_malloc0(sizeof(*info->compression));
 948         info->compression->pages = compression_counters.pages;
 949         info->compression->busy = compression_counters.busy;
 950         info->compression->busy_rate = compression_counters.busy_rate;
 951         info->compression->compressed_size =
 952                                     compression_counters.compressed_size;
 953         info->compression->compression_rate =
 954                                     compression_counters.compression_rate;
 955     }
 956
 957     if (cpu_throttle_active()) {
 958         info->has_cpu_throttle_percentage = true;
 959         info->cpu_throttle_percentage = cpu_throttle_get_percentage();
 960     }
 961
 962     if (s->state != MIGRATION_STATUS_COMPLETED) {
 963         info->ram->remaining = ram_bytes_remaining();
 964         info->ram->dirty_pages_rate = ram_counters.dirty_pages_rate;
 965     }
 966 }
 967
 968 static void populate_disk_info(MigrationInfo *info)
 969 {
 970     if (blk_mig_active()) {
 971         info->has_disk = true;
 972         info->disk = g_malloc0(sizeof(*info->disk));
 973         info->disk->transferred = blk_mig_bytes_transferred();
 974         info->disk->remaining = blk_mig_bytes_remaining();
 975         info->disk->total = blk_mig_bytes_total();
 976     }
 977 }
 978
 979 static void fill_source_migration_info(MigrationInfo *info)
 980 {
 981     MigrationState *s = migrate_get_current();
 982
 983     switch (s->state) {
 984     case MIGRATION_STATUS_NONE:
 985         /* no migration has happened ever */
 986         /* do not overwrite destination migration status */
 987         return;
 988         break;
 989     case MIGRATION_STATUS_SETUP:
 990         info->has_status = true;
 991         info->has_total_time = false;
 992         break;
 993     case MIGRATION_STATUS_ACTIVE:
 994     case MIGRATION_STATUS_CANCELLING:
 995     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 996     case MIGRATION_STATUS_PRE_SWITCHOVER:
 997     case MIGRATION_STATUS_DEVICE:
 998     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 999     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1000         /* TODO add some postcopy stats */
1001         populate_time_info(info, s);
1002         populate_ram_info(info, s);
1003         populate_disk_info(info);
1004         break;
1005     case MIGRATION_STATUS_COLO:
1006         info->has_status = true;
1007         /* TODO: display COLO specific information (checkpoint info etc.) */
1008         break;
1009     case MIGRATION_STATUS_COMPLETED:
1010         populate_time_info(info, s);
1011         populate_ram_info(info, s);
1012         break;
1013     case MIGRATION_STATUS_FAILED:
1014         info->has_status = true;
1015         if (s->error) {
1016             info->has_error_desc = true;
1017             info->error_desc = g_strdup(error_get_pretty(s->error));
1018         }
1019         break;
1020     case MIGRATION_STATUS_CANCELLED:
1021         info->has_status = true;
1022         break;
1023     case MIGRATION_STATUS_WAIT_UNPLUG:
1024         info->has_status = true;
1025         break;
1026     }
1027     info->status = s->state;
1028 }
1029
1030 /**
1031  * @migration_caps_check - check capability validity
1032  *
1033  * @cap_list: old capability list, array of bool
1034  * @params: new capabilities to be applied soon
1035  * @errp: set *errp if the check failed, with reason
1036  *
1037  * Returns true if check passed, otherwise false.
1038  */
1039 static bool migrate_caps_check(bool *cap_list,
1040                                MigrationCapabilityStatusList *params,
1041                                Error **errp)
1042 {
1043     MigrationCapabilityStatusList *cap;
1044     bool old_postcopy_cap;
1045     MigrationIncomingState *mis = migration_incoming_get_current();
1046
1047     old_postcopy_cap = cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM];
1048
1049     for (cap = params; cap; cap = cap->next) {
1050         cap_list[cap->value->capability] = cap->value->state;
1051     }
1052
1053 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
1054     if (cap_list[MIGRATION_CAPABILITY_BLOCK]) {
1055         error_setg(errp, "QEMU compiled without old-style (blk/-b, inc/-i) "
1056                    "block migration");
1057         error_append_hint(errp, "Use drive_mirror+NBD instead.\n");
1058         return false;
1059     }
1060 #endif
1061
1062 #ifndef CONFIG_REPLICATION
1063     if (cap_list[MIGRATION_CAPABILITY_X_COLO]) {
1064         error_setg(errp, "QEMU compiled without replication module"
1065                    " can't enable COLO");
1066         error_append_hint(errp, "Please enable replication before COLO.\n");
1067         return false;
1068     }
1069 #endif
1070
1071     if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
1072         /* This check is reasonably expensive, so only when it's being
1073          * set the first time, also it's only the destination that needs
1074          * special support.
1075          */
1076         if (!old_postcopy_cap && runstate_check(RUN_STATE_INMIGRATE) &&
1077             !postcopy_ram_supported_by_host(mis)) {
1078             /* postcopy_ram_supported_by_host will have emitted a more
1079              * detailed message
1080              */
1081             error_setg(errp, "Postcopy is not supported");
1082             return false;
1083         }
1084
1085         if (cap_list[MIGRATION_CAPABILITY_X_IGNORE_SHARED]) {
1086             error_setg(errp, "Postcopy is not compatible with ignore-shared");
1087             return false;
1088         }
1089     }
1090
1091     return true;
1092 }
1093
1094 static void fill_destination_migration_info(MigrationInfo *info)
1095 {
1096     MigrationIncomingState *mis = migration_incoming_get_current();
1097
1098     if (mis->socket_address_list) {
1099         info->has_socket_address = true;
1100         info->socket_address =
1101             QAPI_CLONE(SocketAddressList, mis->socket_address_list);
1102     }
1103
1104     switch (mis->state) {
1105     case MIGRATION_STATUS_NONE:
1106         return;
1107         break;
1108     case MIGRATION_STATUS_SETUP:
1109     case MIGRATION_STATUS_CANCELLING:
1110     case MIGRATION_STATUS_CANCELLED:
1111     case MIGRATION_STATUS_ACTIVE:
1112     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1113     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1114     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1115     case MIGRATION_STATUS_FAILED:
1116     case MIGRATION_STATUS_COLO:
1117         info->has_status = true;
1118         break;
1119     case MIGRATION_STATUS_COMPLETED:
1120         info->has_status = true;
1121         fill_destination_postcopy_migration_info(info);
1122         break;
1123     }
1124     info->status = mis->state;
1125 }
1126
1127 MigrationInfo *qmp_query_migrate(Error **errp)
1128 {
1129     MigrationInfo *info = g_malloc0(sizeof(*info));
1130
1131     fill_destination_migration_info(info);
1132     fill_source_migration_info(info);
1133
1134     return info;
1135 }
1136
1137 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
1138                                   Error **errp)
1139 {
1140     MigrationState *s = migrate_get_current();
1141     MigrationCapabilityStatusList *cap;
1142     bool cap_list[MIGRATION_CAPABILITY__MAX];
1143
1144     if (migration_is_running(s->state)) {
1145         error_setg(errp, QERR_MIGRATION_ACTIVE);
1146         return;
1147     }
1148
1149     memcpy(cap_list, s->enabled_capabilities, sizeof(cap_list));
1150     if (!migrate_caps_check(cap_list, params, errp)) {
1151         return;
1152     }
1153
1154     for (cap = params; cap; cap = cap->next) {
1155         s->enabled_capabilities[cap->value->capability] = cap->value->state;
1156     }
1157 }
1158
1159 /*
1160  * Check whether the parameters are valid. Error will be put into errp
1161  * (if provided). Return true if valid, otherwise false.
1162  */
1163 static bool migrate_params_check(MigrationParameters *params, Error **errp)
1164 {
1165     if (params->has_compress_level &&
1166         (params->compress_level > 9)) {
1167         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level",
1168                    "is invalid, it should be in the range of 0 to 9");
1169         return false;
1170     }
1171
1172     if (params->has_compress_threads && (params->compress_threads < 1)) {
1173         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1174                    "compress_threads",
1175                    "is invalid, it should be in the range of 1 to 255");
1176         return false;
1177     }
1178
1179     if (params->has_decompress_threads && (params->decompress_threads < 1)) {
1180         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1181                    "decompress_threads",
1182                    "is invalid, it should be in the range of 1 to 255");
1183         return false;
1184     }
1185
1186     if (params->has_throttle_trigger_threshold &&
1187         (params->throttle_trigger_threshold < 1 ||
1188          params->throttle_trigger_threshold > 100)) {
1189         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1190                    "throttle_trigger_threshold",
1191                    "an integer in the range of 1 to 100");
1192         return false;
1193     }
1194
1195     if (params->has_cpu_throttle_initial &&
1196         (params->cpu_throttle_initial < 1 ||
1197          params->cpu_throttle_initial > 99)) {
1198         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1199                    "cpu_throttle_initial",
1200                    "an integer in the range of 1 to 99");
1201         return false;
1202     }
1203
1204     if (params->has_cpu_throttle_increment &&
1205         (params->cpu_throttle_increment < 1 ||
1206          params->cpu_throttle_increment > 99)) {
1207         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1208                    "cpu_throttle_increment",
1209                    "an integer in the range of 1 to 99");
1210         return false;
1211     }
1212
1213     if (params->has_max_bandwidth && (params->max_bandwidth > SIZE_MAX)) {
1214         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1215                    "max_bandwidth",
1216                    "an integer in the range of 0 to "stringify(SIZE_MAX)
1217                    " bytes/second");
1218         return false;
1219     }
1220
1221     if (params->has_downtime_limit &&
1222         (params->downtime_limit > MAX_MIGRATE_DOWNTIME)) {
1223         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1224                    "downtime_limit",
1225                    "an integer in the range of 0 to "
1226                     stringify(MAX_MIGRATE_DOWNTIME)" ms");
1227         return false;
1228     }
1229
1230     /* x_checkpoint_delay is now always positive */
1231
1232     if (params->has_multifd_channels && (params->multifd_channels < 1)) {
1233         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1234                    "multifd_channels",
1235                    "is invalid, it should be in the range of 1 to 255");
1236         return false;
1237     }
1238
1239     if (params->has_multifd_zlib_level &&
1240         (params->multifd_zlib_level > 9)) {
1241         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zlib_level",
1242                    "is invalid, it should be in the range of 0 to 9");
1243         return false;
1244     }
1245
1246     if (params->has_multifd_zstd_level &&
1247         (params->multifd_zstd_level > 20)) {
1248         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zstd_level",
1249                    "is invalid, it should be in the range of 0 to 20");
1250         return false;
1251     }
1252
1253     if (params->has_xbzrle_cache_size &&
1254         (params->xbzrle_cache_size < qemu_target_page_size() ||
1255          !is_power_of_2(params->xbzrle_cache_size))) {
1256         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1257                    "xbzrle_cache_size",
1258                    "is invalid, it should be bigger than target page size"
1259                    " and a power of 2");
1260         return false;
1261     }
1262
1263     if (params->has_max_cpu_throttle &&
1264         (params->max_cpu_throttle < params->cpu_throttle_initial ||
1265          params->max_cpu_throttle > 99)) {
1266         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1267                    "max_cpu_throttle",
1268                    "an integer in the range of cpu_throttle_initial to 99");
1269         return false;
1270     }
1271
1272     if (params->has_announce_initial &&
1273         params->announce_initial > 100000) {
1274         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1275                    "announce_initial",
1276                    "is invalid, it must be less than 100000 ms");
1277         return false;
1278     }
1279     if (params->has_announce_max &&
1280         params->announce_max > 100000) {
1281         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1282                    "announce_max",
1283                    "is invalid, it must be less than 100000 ms");
1284        return false;
1285     }
1286     if (params->has_announce_rounds &&
1287         params->announce_rounds > 1000) {
1288         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1289                    "announce_rounds",
1290                    "is invalid, it must be in the range of 0 to 1000");
1291        return false;
1292     }
1293     if (params->has_announce_step &&
1294         (params->announce_step < 1 ||
1295         params->announce_step > 10000)) {
1296         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1297                    "announce_step",
1298                    "is invalid, it must be in the range of 1 to 10000 ms");
1299        return false;
1300     }
1301     return true;
1302 }
1303
1304 static void migrate_params_test_apply(MigrateSetParameters *params,
1305                                       MigrationParameters *dest)
1306 {
1307     *dest = migrate_get_current()->parameters;
1308
1309     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1310
1311     if (params->has_compress_level) {
1312         dest->compress_level = params->compress_level;
1313     }
1314
1315     if (params->has_compress_threads) {
1316         dest->compress_threads = params->compress_threads;
1317     }
1318
1319     if (params->has_compress_wait_thread) {
1320         dest->compress_wait_thread = params->compress_wait_thread;
1321     }
1322
1323     if (params->has_decompress_threads) {
1324         dest->decompress_threads = params->decompress_threads;
1325     }
1326
1327     if (params->has_throttle_trigger_threshold) {
1328         dest->throttle_trigger_threshold = params->throttle_trigger_threshold;
1329     }
1330
1331     if (params->has_cpu_throttle_initial) {
1332         dest->cpu_throttle_initial = params->cpu_throttle_initial;
1333     }
1334
1335     if (params->has_cpu_throttle_increment) {
1336         dest->cpu_throttle_increment = params->cpu_throttle_increment;
1337     }
1338
1339     if (params->has_cpu_throttle_tailslow) {
1340         dest->cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1341     }
1342
1343     if (params->has_tls_creds) {
1344         assert(params->tls_creds->type == QTYPE_QSTRING);
1345         dest->tls_creds = g_strdup(params->tls_creds->u.s);
1346     }
1347
1348     if (params->has_tls_hostname) {
1349         assert(params->tls_hostname->type == QTYPE_QSTRING);
1350         dest->tls_hostname = g_strdup(params->tls_hostname->u.s);
1351     }
1352
1353     if (params->has_max_bandwidth) {
1354         dest->max_bandwidth = params->max_bandwidth;
1355     }
1356
1357     if (params->has_downtime_limit) {
1358         dest->downtime_limit = params->downtime_limit;
1359     }
1360
1361     if (params->has_x_checkpoint_delay) {
1362         dest->x_checkpoint_delay = params->x_checkpoint_delay;
1363     }
1364
1365     if (params->has_block_incremental) {
1366         dest->block_incremental = params->block_incremental;
1367     }
1368     if (params->has_multifd_channels) {
1369         dest->multifd_channels = params->multifd_channels;
1370     }
1371     if (params->has_multifd_compression) {
1372         dest->multifd_compression = params->multifd_compression;
1373     }
1374     if (params->has_xbzrle_cache_size) {
1375         dest->xbzrle_cache_size = params->xbzrle_cache_size;
1376     }
1377     if (params->has_max_postcopy_bandwidth) {
1378         dest->max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1379     }
1380     if (params->has_max_cpu_throttle) {
1381         dest->max_cpu_throttle = params->max_cpu_throttle;
1382     }
1383     if (params->has_announce_initial) {
1384         dest->announce_initial = params->announce_initial;
1385     }
1386     if (params->has_announce_max) {
1387         dest->announce_max = params->announce_max;
1388     }
1389     if (params->has_announce_rounds) {
1390         dest->announce_rounds = params->announce_rounds;
1391     }
1392     if (params->has_announce_step) {
1393         dest->announce_step = params->announce_step;
1394     }
1395 }
1396
1397 static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
1398 {
1399     MigrationState *s = migrate_get_current();
1400
1401     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1402
1403     if (params->has_compress_level) {
1404         s->parameters.compress_level = params->compress_level;
1405     }
1406
1407     if (params->has_compress_threads) {
1408         s->parameters.compress_threads = params->compress_threads;
1409     }
1410
1411     if (params->has_compress_wait_thread) {
1412         s->parameters.compress_wait_thread = params->compress_wait_thread;
1413     }
1414
1415     if (params->has_decompress_threads) {
1416         s->parameters.decompress_threads = params->decompress_threads;
1417     }
1418
1419     if (params->has_throttle_trigger_threshold) {
1420         s->parameters.throttle_trigger_threshold = params->throttle_trigger_threshold;
1421     }
1422
1423     if (params->has_cpu_throttle_initial) {
1424         s->parameters.cpu_throttle_initial = params->cpu_throttle_initial;
1425     }
1426
1427     if (params->has_cpu_throttle_increment) {
1428         s->parameters.cpu_throttle_increment = params->cpu_throttle_increment;
1429     }
1430
1431     if (params->has_cpu_throttle_tailslow) {
1432         s->parameters.cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1433     }
1434
1435     if (params->has_tls_creds) {
1436         g_free(s->parameters.tls_creds);
1437         assert(params->tls_creds->type == QTYPE_QSTRING);
1438         s->parameters.tls_creds = g_strdup(params->tls_creds->u.s);
1439     }
1440
1441     if (params->has_tls_hostname) {
1442         g_free(s->parameters.tls_hostname);
1443         assert(params->tls_hostname->type == QTYPE_QSTRING);
1444         s->parameters.tls_hostname = g_strdup(params->tls_hostname->u.s);
1445     }
1446
1447     if (params->has_tls_authz) {
1448         g_free(s->parameters.tls_authz);
1449         assert(params->tls_authz->type == QTYPE_QSTRING);
1450         s->parameters.tls_authz = g_strdup(params->tls_authz->u.s);
1451     }
1452
1453     if (params->has_max_bandwidth) {
1454         s->parameters.max_bandwidth = params->max_bandwidth;
1455         if (s->to_dst_file && !migration_in_postcopy()) {
1456             qemu_file_set_rate_limit(s->to_dst_file,
1457                                 s->parameters.max_bandwidth / XFER_LIMIT_RATIO);
1458         }
1459     }
1460
1461     if (params->has_downtime_limit) {
1462         s->parameters.downtime_limit = params->downtime_limit;
1463     }
1464
1465     if (params->has_x_checkpoint_delay) {
1466         s->parameters.x_checkpoint_delay = params->x_checkpoint_delay;
1467         if (migration_in_colo_state()) {
1468             colo_checkpoint_notify(s);
1469         }
1470     }
1471
1472     if (params->has_block_incremental) {
1473         s->parameters.block_incremental = params->block_incremental;
1474     }
1475     if (params->has_multifd_channels) {
1476         s->parameters.multifd_channels = params->multifd_channels;
1477     }
1478     if (params->has_multifd_compression) {
1479         s->parameters.multifd_compression = params->multifd_compression;
1480     }
1481     if (params->has_xbzrle_cache_size) {
1482         s->parameters.xbzrle_cache_size = params->xbzrle_cache_size;
1483         xbzrle_cache_resize(params->xbzrle_cache_size, errp);
1484     }
1485     if (params->has_max_postcopy_bandwidth) {
1486         s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1487         if (s->to_dst_file && migration_in_postcopy()) {
1488             qemu_file_set_rate_limit(s->to_dst_file,
1489                     s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO);
1490         }
1491     }
1492     if (params->has_max_cpu_throttle) {
1493         s->parameters.max_cpu_throttle = params->max_cpu_throttle;
1494     }
1495     if (params->has_announce_initial) {
1496         s->parameters.announce_initial = params->announce_initial;
1497     }
1498     if (params->has_announce_max) {
1499         s->parameters.announce_max = params->announce_max;
1500     }
1501     if (params->has_announce_rounds) {
1502         s->parameters.announce_rounds = params->announce_rounds;
1503     }
1504     if (params->has_announce_step) {
1505         s->parameters.announce_step = params->announce_step;
1506     }
1507 }
1508
1509 void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
1510 {
1511     MigrationParameters tmp;
1512
1513     /* TODO Rewrite "" to null instead */
1514     if (params->has_tls_creds
1515         && params->tls_creds->type == QTYPE_QNULL) {
1516         qobject_unref(params->tls_creds->u.n);
1517         params->tls_creds->type = QTYPE_QSTRING;
1518         params->tls_creds->u.s = strdup("");
1519     }
1520     /* TODO Rewrite "" to null instead */
1521     if (params->has_tls_hostname
1522         && params->tls_hostname->type == QTYPE_QNULL) {
1523         qobject_unref(params->tls_hostname->u.n);
1524         params->tls_hostname->type = QTYPE_QSTRING;
1525         params->tls_hostname->u.s = strdup("");
1526     }
1527
1528     migrate_params_test_apply(params, &tmp);
1529
1530     if (!migrate_params_check(&tmp, errp)) {
1531         /* Invalid parameter */
1532         return;
1533     }
1534
1535     migrate_params_apply(params, errp);
1536 }
1537
1538
1539 void qmp_migrate_start_postcopy(Error **errp)
1540 {
1541     MigrationState *s = migrate_get_current();
1542
1543     if (!migrate_postcopy()) {
1544         error_setg(errp, "Enable postcopy with migrate_set_capability before"
1545                          " the start of migration");
1546         return;
1547     }
1548
1549     if (s->state == MIGRATION_STATUS_NONE) {
1550         error_setg(errp, "Postcopy must be started after migration has been"
1551                          " started");
1552         return;
1553     }
1554     /*
1555      * we don't error if migration has finished since that would be racy
1556      * with issuing this command.
1557      */
1558     atomic_set(&s->start_postcopy, true);
1559 }
1560
1561 /* shared migration helpers */
1562
1563 void migrate_set_state(int *state, int old_state, int new_state)
1564 {
1565     assert(new_state < MIGRATION_STATUS__MAX);
1566     if (atomic_cmpxchg(state, old_state, new_state) == old_state) {
1567         trace_migrate_set_state(MigrationStatus_str(new_state));
1568         migrate_generate_event(new_state);
1569     }
1570 }
1571
1572 static MigrationCapabilityStatusList *migrate_cap_add(
1573     MigrationCapabilityStatusList *list,
1574     MigrationCapability index,
1575     bool state)
1576 {
1577     MigrationCapabilityStatusList *cap;
1578
1579     cap = g_new0(MigrationCapabilityStatusList, 1);
1580     cap->value = g_new0(MigrationCapabilityStatus, 1);
1581     cap->value->capability = index;
1582     cap->value->state = state;
1583     cap->next = list;
1584
1585     return cap;
1586 }
1587
1588 void migrate_set_block_enabled(bool value, Error **errp)
1589 {
1590     MigrationCapabilityStatusList *cap;
1591
1592     cap = migrate_cap_add(NULL, MIGRATION_CAPABILITY_BLOCK, value);
1593     qmp_migrate_set_capabilities(cap, errp);
1594     qapi_free_MigrationCapabilityStatusList(cap);
1595 }
1596
1597 static void migrate_set_block_incremental(MigrationState *s, bool value)
1598 {
1599     s->parameters.block_incremental = value;
1600 }
1601
1602 static void block_cleanup_parameters(MigrationState *s)
1603 {
1604     if (s->must_remove_block_options) {
1605         /* setting to false can never fail */
1606         migrate_set_block_enabled(false, &error_abort);
1607         migrate_set_block_incremental(s, false);
1608         s->must_remove_block_options = false;
1609     }
1610 }
1611
1612 static void migrate_fd_cleanup(MigrationState *s)
1613 {
1614     qemu_bh_delete(s->cleanup_bh);
1615     s->cleanup_bh = NULL;
1616
1617     qemu_savevm_state_cleanup();
1618
1619     if (s->to_dst_file) {
1620         QEMUFile *tmp;
1621
1622         trace_migrate_fd_cleanup();
1623         qemu_mutex_unlock_iothread();
1624         if (s->migration_thread_running) {
1625             qemu_thread_join(&s->thread);
1626             s->migration_thread_running = false;
1627         }
1628         qemu_mutex_lock_iothread();
1629
1630         multifd_save_cleanup();
1631         qemu_mutex_lock(&s->qemu_file_lock);
1632         tmp = s->to_dst_file;
1633         s->to_dst_file = NULL;
1634         qemu_mutex_unlock(&s->qemu_file_lock);
1635         /*
1636          * Close the file handle without the lock to make sure the
1637          * critical section won't block for long.
1638          */
1639         qemu_fclose(tmp);
1640     }
1641
1642     assert(!migration_is_active(s));
1643
1644     if (s->state == MIGRATION_STATUS_CANCELLING) {
1645         migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING,
1646                           MIGRATION_STATUS_CANCELLED);
1647     }
1648
1649     if (s->error) {
1650         /* It is used on info migrate.  We can't free it */
1651         error_report_err(error_copy(s->error));
1652     }
1653     notifier_list_notify(&migration_state_notifiers, s);
1654     block_cleanup_parameters(s);
1655 }
1656
1657 static void migrate_fd_cleanup_schedule(MigrationState *s)
1658 {
1659     /*
1660      * Ref the state for bh, because it may be called when
1661      * there're already no other refs
1662      */
1663     object_ref(OBJECT(s));
1664     qemu_bh_schedule(s->cleanup_bh);
1665 }
1666
1667 static void migrate_fd_cleanup_bh(void *opaque)
1668 {
1669     MigrationState *s = opaque;
1670     migrate_fd_cleanup(s);
1671     object_unref(OBJECT(s));
1672 }
1673
1674 void migrate_set_error(MigrationState *s, const Error *error)
1675 {
1676     QEMU_LOCK_GUARD(&s->error_mutex);
1677     if (!s->error) {
1678         s->error = error_copy(error);
1679     }
1680 }
1681
1682 void migrate_fd_error(MigrationState *s, const Error *error)
1683 {
1684     trace_migrate_fd_error(error_get_pretty(error));
1685     assert(s->to_dst_file == NULL);
1686     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
1687                       MIGRATION_STATUS_FAILED);
1688     migrate_set_error(s, error);
1689 }
1690
1691 static void migrate_fd_cancel(MigrationState *s)
1692 {
1693     int old_state ;
1694     QEMUFile *f = migrate_get_current()->to_dst_file;
1695     trace_migrate_fd_cancel();
1696
1697     if (s->rp_state.from_dst_file) {
1698         /* shutdown the rp socket, so causing the rp thread to shutdown */
1699         qemu_file_shutdown(s->rp_state.from_dst_file);
1700     }
1701
1702     do {
1703         old_state = s->state;
1704         if (!migration_is_running(old_state)) {
1705             break;
1706         }
1707         /* If the migration is paused, kick it out of the pause */
1708         if (old_state == MIGRATION_STATUS_PRE_SWITCHOVER) {
1709             qemu_sem_post(&s->pause_sem);
1710         }
1711         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING);
1712     } while (s->state != MIGRATION_STATUS_CANCELLING);
1713
1714     /*
1715      * If we're unlucky the migration code might be stuck somewhere in a
1716      * send/write while the network has failed and is waiting to timeout;
1717      * if we've got shutdown(2) available then we can force it to quit.
1718      * The outgoing qemu file gets closed in migrate_fd_cleanup that is
1719      * called in a bh, so there is no race against this cancel.
1720      */
1721     if (s->state == MIGRATION_STATUS_CANCELLING && f) {
1722         qemu_file_shutdown(f);
1723     }
1724     if (s->state == MIGRATION_STATUS_CANCELLING && s->block_inactive) {
1725         Error *local_err = NULL;
1726
1727         bdrv_invalidate_cache_all(&local_err);
1728         if (local_err) {
1729             error_report_err(local_err);
1730         } else {
1731             s->block_inactive = false;
1732         }
1733     }
1734 }
1735
1736 void add_migration_state_change_notifier(Notifier *notify)
1737 {
1738     notifier_list_add(&migration_state_notifiers, notify);
1739 }
1740
1741 void remove_migration_state_change_notifier(Notifier *notify)
1742 {
1743     notifier_remove(notify);
1744 }
1745
1746 bool migration_in_setup(MigrationState *s)
1747 {
1748     return s->state == MIGRATION_STATUS_SETUP;
1749 }
1750
1751 bool migration_has_finished(MigrationState *s)
1752 {
1753     return s->state == MIGRATION_STATUS_COMPLETED;
1754 }
1755
1756 bool migration_has_failed(MigrationState *s)
1757 {
1758     return (s->state == MIGRATION_STATUS_CANCELLED ||
1759             s->state == MIGRATION_STATUS_FAILED);
1760 }
1761
1762 bool migration_in_postcopy(void)
1763 {
1764     MigrationState *s = migrate_get_current();
1765
1766     switch (s->state) {
1767     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1768     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1769     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1770         return true;
1771     default:
1772         return false;
1773     }
1774 }
1775
1776 bool migration_in_postcopy_after_devices(MigrationState *s)
1777 {
1778     return migration_in_postcopy() && s->postcopy_after_devices;
1779 }
1780
1781 bool migration_in_incoming_postcopy(void)
1782 {
1783     PostcopyState ps = postcopy_state_get();
1784
1785     return ps >= POSTCOPY_INCOMING_DISCARD && ps < POSTCOPY_INCOMING_END;
1786 }
1787
1788 bool migration_is_idle(void)
1789 {
1790     MigrationState *s = current_migration;
1791
1792     if (!s) {
1793         return true;
1794     }
1795
1796     switch (s->state) {
1797     case MIGRATION_STATUS_NONE:
1798     case MIGRATION_STATUS_CANCELLED:
1799     case MIGRATION_STATUS_COMPLETED:
1800     case MIGRATION_STATUS_FAILED:
1801         return true;
1802     case MIGRATION_STATUS_SETUP:
1803     case MIGRATION_STATUS_CANCELLING:
1804     case MIGRATION_STATUS_ACTIVE:
1805     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1806     case MIGRATION_STATUS_COLO:
1807     case MIGRATION_STATUS_PRE_SWITCHOVER:
1808     case MIGRATION_STATUS_DEVICE:
1809     case MIGRATION_STATUS_WAIT_UNPLUG:
1810         return false;
1811     case MIGRATION_STATUS__MAX:
1812         g_assert_not_reached();
1813     }
1814
1815     return false;
1816 }
1817
1818 bool migration_is_active(MigrationState *s)
1819 {
1820     return (s->state == MIGRATION_STATUS_ACTIVE ||
1821             s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
1822 }
1823
1824 void migrate_init(MigrationState *s)
1825 {
1826     /*
1827      * Reinitialise all migration state, except
1828      * parameters/capabilities that the user set, and
1829      * locks.
1830      */
1831     s->cleanup_bh = 0;
1832     s->to_dst_file = NULL;
1833     s->state = MIGRATION_STATUS_NONE;
1834     s->rp_state.from_dst_file = NULL;
1835     s->rp_state.error = false;
1836     s->mbps = 0.0;
1837     s->pages_per_second = 0.0;
1838     s->downtime = 0;
1839     s->expected_downtime = 0;
1840     s->setup_time = 0;
1841     s->start_postcopy = false;
1842     s->postcopy_after_devices = false;
1843     s->migration_thread_running = false;
1844     error_free(s->error);
1845     s->error = NULL;
1846
1847     migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
1848
1849     s->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1850     s->total_time = 0;
1851     s->vm_was_running = false;
1852     s->iteration_initial_bytes = 0;
1853     s->threshold_size = 0;
1854 }
1855
1856 static GSList *migration_blockers;
1857
1858 int migrate_add_blocker(Error *reason, Error **errp)
1859 {
1860     if (only_migratable) {
1861         error_propagate_prepend(errp, error_copy(reason),
1862                                 "disallowing migration blocker "
1863                                 "(--only-migratable) for: ");
1864         return -EACCES;
1865     }
1866
1867     if (migration_is_idle()) {
1868         migration_blockers = g_slist_prepend(migration_blockers, reason);
1869         return 0;
1870     }
1871
1872     error_propagate_prepend(errp, error_copy(reason),
1873                             "disallowing migration blocker "
1874                             "(migration in progress) for: ");
1875     return -EBUSY;
1876 }
1877
1878 void migrate_del_blocker(Error *reason)
1879 {
1880     migration_blockers = g_slist_remove(migration_blockers, reason);
1881 }
1882
1883 void qmp_migrate_incoming(const char *uri, Error **errp)
1884 {
1885     Error *local_err = NULL;
1886     static bool once = true;
1887
1888     if (!deferred_incoming) {
1889         error_setg(errp, "For use with '-incoming defer'");
1890         return;
1891     }
1892     if (!once) {
1893         error_setg(errp, "The incoming migration has already been started");
1894         return;
1895     }
1896
1897     qemu_start_incoming_migration(uri, &local_err);
1898
1899     if (local_err) {
1900         error_propagate(errp, local_err);
1901         return;
1902     }
1903
1904     once = false;
1905 }
1906
1907 void qmp_migrate_recover(const char *uri, Error **errp)
1908 {
1909     MigrationIncomingState *mis = migration_incoming_get_current();
1910
1911     if (mis->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
1912         error_setg(errp, "Migrate recover can only be run "
1913                    "when postcopy is paused.");
1914         return;
1915     }
1916
1917     if (atomic_cmpxchg(&mis->postcopy_recover_triggered,
1918                        false, true) == true) {
1919         error_setg(errp, "Migrate recovery is triggered already");
1920         return;
1921     }
1922
1923     /*
1924      * Note that this call will never start a real migration; it will
1925      * only re-setup the migration stream and poke existing migration
1926      * to continue using that newly established channel.
1927      */
1928     qemu_start_incoming_migration(uri, errp);
1929 }
1930
1931 void qmp_migrate_pause(Error **errp)
1932 {
1933     MigrationState *ms = migrate_get_current();
1934     MigrationIncomingState *mis = migration_incoming_get_current();
1935     int ret;
1936
1937     if (ms->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
1938         /* Source side, during postcopy */
1939         qemu_mutex_lock(&ms->qemu_file_lock);
1940         ret = qemu_file_shutdown(ms->to_dst_file);
1941         qemu_mutex_unlock(&ms->qemu_file_lock);
1942         if (ret) {
1943             error_setg(errp, "Failed to pause source migration");
1944         }
1945         return;
1946     }
1947
1948     if (mis->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
1949         ret = qemu_file_shutdown(mis->from_src_file);
1950         if (ret) {
1951             error_setg(errp, "Failed to pause destination migration");
1952         }
1953         return;
1954     }
1955
1956     error_setg(errp, "migrate-pause is currently only supported "
1957                "during postcopy-active state");
1958 }
1959
1960 bool migration_is_blocked(Error **errp)
1961 {
1962     if (qemu_savevm_state_blocked(errp)) {
1963         return true;
1964     }
1965
1966     if (migration_blockers) {
1967         error_propagate(errp, error_copy(migration_blockers->data));
1968         return true;
1969     }
1970
1971     return false;
1972 }
1973
1974 /* Returns true if continue to migrate, or false if error detected */
1975 static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
1976                             bool resume, Error **errp)
1977 {
1978     Error *local_err = NULL;
1979
1980     if (resume) {
1981         if (s->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
1982             error_setg(errp, "Cannot resume if there is no "
1983                        "paused migration");
1984             return false;
1985         }
1986
1987         /*
1988          * Postcopy recovery won't work well with release-ram
1989          * capability since release-ram will drop the page buffer as
1990          * long as the page is put into the send buffer.  So if there
1991          * is a network failure happened, any page buffers that have
1992          * not yet reached the destination VM but have already been
1993          * sent from the source VM will be lost forever.  Let's refuse
1994          * the client from resuming such a postcopy migration.
1995          * Luckily release-ram was designed to only be used when src
1996          * and destination VMs are on the same host, so it should be
1997          * fine.
1998          */
1999         if (migrate_release_ram()) {
2000             error_setg(errp, "Postcopy recovery cannot work "
2001                        "when release-ram capability is set");
2002             return false;
2003         }
2004
2005         /* This is a resume, skip init status */
2006         return true;
2007     }
2008
2009     if (migration_is_running(s->state)) {
2010         error_setg(errp, QERR_MIGRATION_ACTIVE);
2011         return false;
2012     }
2013
2014     if (runstate_check(RUN_STATE_INMIGRATE)) {
2015         error_setg(errp, "Guest is waiting for an incoming migration");
2016         return false;
2017     }
2018
2019     if (migration_is_blocked(errp)) {
2020         return false;
2021     }
2022
2023     if (blk || blk_inc) {
2024         if (migrate_use_block() || migrate_use_block_incremental()) {
2025             error_setg(errp, "Command options are incompatible with "
2026                        "current migration capabilities");
2027             return false;
2028         }
2029         migrate_set_block_enabled(true, &local_err);
2030         if (local_err) {
2031             error_propagate(errp, local_err);
2032             return false;
2033         }
2034         s->must_remove_block_options = true;
2035     }
2036
2037     if (blk_inc) {
2038         migrate_set_block_incremental(s, true);
2039     }
2040
2041     migrate_init(s);
2042     /*
2043      * set ram_counters memory to zero for a
2044      * new migration
2045      */
2046     memset(&ram_counters, 0, sizeof(ram_counters));
2047
2048     return true;
2049 }
2050
2051 void qmp_migrate(const char *uri, bool has_blk, bool blk,
2052                  bool has_inc, bool inc, bool has_detach, bool detach,
2053                  bool has_resume, bool resume, Error **errp)
2054 {
2055     Error *local_err = NULL;
2056     MigrationState *s = migrate_get_current();
2057     const char *p;
2058
2059     if (!migrate_prepare(s, has_blk && blk, has_inc && inc,
2060                          has_resume && resume, errp)) {
2061         /* Error detected, put into errp */
2062         return;
2063     }
2064
2065     if (strstart(uri, "tcp:", &p)) {
2066         tcp_start_outgoing_migration(s, p, &local_err);
2067 #ifdef CONFIG_RDMA
2068     } else if (strstart(uri, "rdma:", &p)) {
2069         rdma_start_outgoing_migration(s, p, &local_err);
2070 #endif
2071     } else if (strstart(uri, "exec:", &p)) {
2072         exec_start_outgoing_migration(s, p, &local_err);
2073     } else if (strstart(uri, "unix:", &p)) {
2074         unix_start_outgoing_migration(s, p, &local_err);
2075     } else if (strstart(uri, "fd:", &p)) {
2076         fd_start_outgoing_migration(s, p, &local_err);
2077     } else {
2078         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri",
2079                    "a valid migration protocol");
2080         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
2081                           MIGRATION_STATUS_FAILED);
2082         block_cleanup_parameters(s);
2083         return;
2084     }
2085
2086     if (local_err) {
2087         migrate_fd_error(s, local_err);
2088         error_propagate(errp, local_err);
2089         return;
2090     }
2091 }
2092
2093 void qmp_migrate_cancel(Error **errp)
2094 {
2095     migrate_fd_cancel(migrate_get_current());
2096 }
2097
2098 void qmp_migrate_continue(MigrationStatus state, Error **errp)
2099 {
2100     MigrationState *s = migrate_get_current();
2101     if (s->state != state) {
2102         error_setg(errp,  "Migration not in expected state: %s",
2103                    MigrationStatus_str(s->state));
2104         return;
2105     }
2106     qemu_sem_post(&s->pause_sem);
2107 }
2108
2109 void qmp_migrate_set_cache_size(int64_t value, Error **errp)
2110 {
2111     MigrateSetParameters p = {
2112         .has_xbzrle_cache_size = true,
2113         .xbzrle_cache_size = value,
2114     };
2115
2116     qmp_migrate_set_parameters(&p, errp);
2117 }
2118
2119 int64_t qmp_query_migrate_cache_size(Error **errp)
2120 {
2121     return migrate_xbzrle_cache_size();
2122 }
2123
2124 void qmp_migrate_set_speed(int64_t value, Error **errp)
2125 {
2126     MigrateSetParameters p = {
2127         .has_max_bandwidth = true,
2128         .max_bandwidth = value,
2129     };
2130
2131     qmp_migrate_set_parameters(&p, errp);
2132 }
2133
2134 void qmp_migrate_set_downtime(double value, Error **errp)
2135 {
2136     if (value < 0 || value > MAX_MIGRATE_DOWNTIME_SECONDS) {
2137         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
2138                    "downtime_limit",
2139                    "an integer in the range of 0 to "
2140                     stringify(MAX_MIGRATE_DOWNTIME_SECONDS)" seconds");
2141         return;
2142     }
2143
2144     value *= 1000; /* Convert to milliseconds */
2145
2146     MigrateSetParameters p = {
2147         .has_downtime_limit = true,
2148         .downtime_limit = (int64_t)value,
2149     };
2150
2151     qmp_migrate_set_parameters(&p, errp);
2152 }
2153
2154 bool migrate_release_ram(void)
2155 {
2156     MigrationState *s;
2157
2158     s = migrate_get_current();
2159
2160     return s->enabled_capabilities[MIGRATION_CAPABILITY_RELEASE_RAM];
2161 }
2162
2163 bool migrate_postcopy_ram(void)
2164 {
2165     MigrationState *s;
2166
2167     s = migrate_get_current();
2168
2169     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM];
2170 }
2171
2172 bool migrate_postcopy(void)
2173 {
2174     return migrate_postcopy_ram() || migrate_dirty_bitmaps();
2175 }
2176
2177 bool migrate_auto_converge(void)
2178 {
2179     MigrationState *s;
2180
2181     s = migrate_get_current();
2182
2183     return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
2184 }
2185
2186 bool migrate_zero_blocks(void)
2187 {
2188     MigrationState *s;
2189
2190     s = migrate_get_current();
2191
2192     return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
2193 }
2194
2195 bool migrate_postcopy_blocktime(void)
2196 {
2197     MigrationState *s;
2198
2199     s = migrate_get_current();
2200
2201     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME];
2202 }
2203
2204 bool migrate_use_compression(void)
2205 {
2206     MigrationState *s;
2207
2208     s = migrate_get_current();
2209
2210     return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS];
2211 }
2212
2213 int migrate_compress_level(void)
2214 {
2215     MigrationState *s;
2216
2217     s = migrate_get_current();
2218
2219     return s->parameters.compress_level;
2220 }
2221
2222 int migrate_compress_threads(void)
2223 {
2224     MigrationState *s;
2225
2226     s = migrate_get_current();
2227
2228     return s->parameters.compress_threads;
2229 }
2230
2231 int migrate_compress_wait_thread(void)
2232 {
2233     MigrationState *s;
2234
2235     s = migrate_get_current();
2236
2237     return s->parameters.compress_wait_thread;
2238 }
2239
2240 int migrate_decompress_threads(void)
2241 {
2242     MigrationState *s;
2243
2244     s = migrate_get_current();
2245
2246     return s->parameters.decompress_threads;
2247 }
2248
2249 bool migrate_dirty_bitmaps(void)
2250 {
2251     MigrationState *s;
2252
2253     s = migrate_get_current();
2254
2255     return s->enabled_capabilities[MIGRATION_CAPABILITY_DIRTY_BITMAPS];
2256 }
2257
2258 bool migrate_ignore_shared(void)
2259 {
2260     MigrationState *s;
2261
2262     s = migrate_get_current();
2263
2264     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_IGNORE_SHARED];
2265 }
2266
2267 bool migrate_validate_uuid(void)
2268 {
2269     MigrationState *s;
2270
2271     s = migrate_get_current();
2272
2273     return s->enabled_capabilities[MIGRATION_CAPABILITY_VALIDATE_UUID];
2274 }
2275
2276 bool migrate_use_events(void)
2277 {
2278     MigrationState *s;
2279
2280     s = migrate_get_current();
2281
2282     return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS];
2283 }
2284
2285 bool migrate_use_multifd(void)
2286 {
2287     MigrationState *s;
2288
2289     s = migrate_get_current();
2290
2291     return s->enabled_capabilities[MIGRATION_CAPABILITY_MULTIFD];
2292 }
2293
2294 bool migrate_pause_before_switchover(void)
2295 {
2296     MigrationState *s;
2297
2298     s = migrate_get_current();
2299
2300     return s->enabled_capabilities[
2301         MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER];
2302 }
2303
2304 int migrate_multifd_channels(void)
2305 {
2306     MigrationState *s;
2307
2308     s = migrate_get_current();
2309
2310     return s->parameters.multifd_channels;
2311 }
2312
2313 MultiFDCompression migrate_multifd_compression(void)
2314 {
2315     MigrationState *s;
2316
2317     s = migrate_get_current();
2318
2319     return s->parameters.multifd_compression;
2320 }
2321
2322 int migrate_multifd_zlib_level(void)
2323 {
2324     MigrationState *s;
2325
2326     s = migrate_get_current();
2327
2328     return s->parameters.multifd_zlib_level;
2329 }
2330
2331 int migrate_multifd_zstd_level(void)
2332 {
2333     MigrationState *s;
2334
2335     s = migrate_get_current();
2336
2337     return s->parameters.multifd_zstd_level;
2338 }
2339
2340 int migrate_use_xbzrle(void)
2341 {
2342     MigrationState *s;
2343
2344     s = migrate_get_current();
2345
2346     return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
2347 }
2348
2349 int64_t migrate_xbzrle_cache_size(void)
2350 {
2351     MigrationState *s;
2352
2353     s = migrate_get_current();
2354
2355     return s->parameters.xbzrle_cache_size;
2356 }
2357
2358 static int64_t migrate_max_postcopy_bandwidth(void)
2359 {
2360     MigrationState *s;
2361
2362     s = migrate_get_current();
2363
2364     return s->parameters.max_postcopy_bandwidth;
2365 }
2366
2367 bool migrate_use_block(void)
2368 {
2369     MigrationState *s;
2370
2371     s = migrate_get_current();
2372
2373     return s->enabled_capabilities[MIGRATION_CAPABILITY_BLOCK];
2374 }
2375
2376 bool migrate_use_return_path(void)
2377 {
2378     MigrationState *s;
2379
2380     s = migrate_get_current();
2381
2382     return s->enabled_capabilities[MIGRATION_CAPABILITY_RETURN_PATH];
2383 }
2384
2385 bool migrate_use_block_incremental(void)
2386 {
2387     MigrationState *s;
2388
2389     s = migrate_get_current();
2390
2391     return s->parameters.block_incremental;
2392 }
2393
2394 /* migration thread support */
2395 /*
2396  * Something bad happened to the RP stream, mark an error
2397  * The caller shall print or trace something to indicate why
2398  */
2399 static void mark_source_rp_bad(MigrationState *s)
2400 {
2401     s->rp_state.error = true;
2402 }
2403
2404 static struct rp_cmd_args {
2405     ssize_t     len; /* -1 = variable */
2406     const char *name;
2407 } rp_cmd_args[] = {
2408     [MIG_RP_MSG_INVALID]        = { .len = -1, .name = "INVALID" },
2409     [MIG_RP_MSG_SHUT]           = { .len =  4, .name = "SHUT" },
2410     [MIG_RP_MSG_PONG]           = { .len =  4, .name = "PONG" },
2411     [MIG_RP_MSG_REQ_PAGES]      = { .len = 12, .name = "REQ_PAGES" },
2412     [MIG_RP_MSG_REQ_PAGES_ID]   = { .len = -1, .name = "REQ_PAGES_ID" },
2413     [MIG_RP_MSG_RECV_BITMAP]    = { .len = -1, .name = "RECV_BITMAP" },
2414     [MIG_RP_MSG_RESUME_ACK]     = { .len =  4, .name = "RESUME_ACK" },
2415     [MIG_RP_MSG_MAX]            = { .len = -1, .name = "MAX" },
2416 };
2417
2418 /*
2419  * Process a request for pages received on the return path,
2420  * We're allowed to send more than requested (e.g. to round to our page size)
2421  * and we don't need to send pages that have already been sent.
2422  */
2423 static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
2424                                        ram_addr_t start, size_t len)
2425 {
2426     long our_host_ps = qemu_real_host_page_size;
2427
2428     trace_migrate_handle_rp_req_pages(rbname, start, len);
2429
2430     /*
2431      * Since we currently insist on matching page sizes, just sanity check
2432      * we're being asked for whole host pages.
2433      */
2434     if (start & (our_host_ps-1) ||
2435        (len & (our_host_ps-1))) {
2436         error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
2437                      " len: %zd", __func__, start, len);
2438         mark_source_rp_bad(ms);
2439         return;
2440     }
2441
2442     if (ram_save_queue_pages(rbname, start, len)) {
2443         mark_source_rp_bad(ms);
2444     }
2445 }
2446
2447 /* Return true to retry, false to quit */
2448 static bool postcopy_pause_return_path_thread(MigrationState *s)
2449 {
2450     trace_postcopy_pause_return_path();
2451
2452     qemu_sem_wait(&s->postcopy_pause_rp_sem);
2453
2454     trace_postcopy_pause_return_path_continued();
2455
2456     return true;
2457 }
2458
2459 static int migrate_handle_rp_recv_bitmap(MigrationState *s, char *block_name)
2460 {
2461     RAMBlock *block = qemu_ram_block_by_name(block_name);
2462
2463     if (!block) {
2464         error_report("%s: invalid block name '%s'", __func__, block_name);
2465         return -EINVAL;
2466     }
2467
2468     /* Fetch the received bitmap and refresh the dirty bitmap */
2469     return ram_dirty_bitmap_reload(s, block);
2470 }
2471
2472 static int migrate_handle_rp_resume_ack(MigrationState *s, uint32_t value)
2473 {
2474     trace_source_return_path_thread_resume_ack(value);
2475
2476     if (value != MIGRATION_RESUME_ACK_VALUE) {
2477         error_report("%s: illegal resume_ack value %"PRIu32,
2478                      __func__, value);
2479         return -1;
2480     }
2481
2482     /* Now both sides are active. */
2483     migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_RECOVER,
2484                       MIGRATION_STATUS_POSTCOPY_ACTIVE);
2485
2486     /* Notify send thread that time to continue send pages */
2487     qemu_sem_post(&s->rp_state.rp_sem);
2488
2489     return 0;
2490 }
2491
2492 /*
2493  * Handles messages sent on the return path towards the source VM
2494  *
2495  */
2496 static void *source_return_path_thread(void *opaque)
2497 {
2498     MigrationState *ms = opaque;
2499     QEMUFile *rp = ms->rp_state.from_dst_file;
2500     uint16_t header_len, header_type;
2501     uint8_t buf[512];
2502     uint32_t tmp32, sibling_error;
2503     ram_addr_t start = 0; /* =0 to silence warning */
2504     size_t  len = 0, expected_len;
2505     int res;
2506
2507     trace_source_return_path_thread_entry();
2508     rcu_register_thread();
2509
2510 retry:
2511     while (!ms->rp_state.error && !qemu_file_get_error(rp) &&
2512            migration_is_setup_or_active(ms->state)) {
2513         trace_source_return_path_thread_loop_top();
2514         header_type = qemu_get_be16(rp);
2515         header_len = qemu_get_be16(rp);
2516
2517         if (qemu_file_get_error(rp)) {
2518             mark_source_rp_bad(ms);
2519             goto out;
2520         }
2521
2522         if (header_type >= MIG_RP_MSG_MAX ||
2523             header_type == MIG_RP_MSG_INVALID) {
2524             error_report("RP: Received invalid message 0x%04x length 0x%04x",
2525                          header_type, header_len);
2526             mark_source_rp_bad(ms);
2527             goto out;
2528         }
2529
2530         if ((rp_cmd_args[header_type].len != -1 &&
2531             header_len != rp_cmd_args[header_type].len) ||
2532             header_len > sizeof(buf)) {
2533             error_report("RP: Received '%s' message (0x%04x) with"
2534                          "incorrect length %d expecting %zu",
2535                          rp_cmd_args[header_type].name, header_type, header_len,
2536                          (size_t)rp_cmd_args[header_type].len);
2537             mark_source_rp_bad(ms);
2538             goto out;
2539         }
2540
2541         /* We know we've got a valid header by this point */
2542         res = qemu_get_buffer(rp, buf, header_len);
2543         if (res != header_len) {
2544             error_report("RP: Failed reading data for message 0x%04x"
2545                          " read %d expected %d",
2546                          header_type, res, header_len);
2547             mark_source_rp_bad(ms);
2548             goto out;
2549         }
2550
2551         /* OK, we have the message and the data */
2552         switch (header_type) {
2553         case MIG_RP_MSG_SHUT:
2554             sibling_error = ldl_be_p(buf);
2555             trace_source_return_path_thread_shut(sibling_error);
2556             if (sibling_error) {
2557                 error_report("RP: Sibling indicated error %d", sibling_error);
2558                 mark_source_rp_bad(ms);
2559             }
2560             /*
2561              * We'll let the main thread deal with closing the RP
2562              * we could do a shutdown(2) on it, but we're the only user
2563              * anyway, so there's nothing gained.
2564              */
2565             goto out;
2566
2567         case MIG_RP_MSG_PONG:
2568             tmp32 = ldl_be_p(buf);
2569             trace_source_return_path_thread_pong(tmp32);
2570             break;
2571
2572         case MIG_RP_MSG_REQ_PAGES:
2573             start = ldq_be_p(buf);
2574             len = ldl_be_p(buf + 8);
2575             migrate_handle_rp_req_pages(ms, NULL, start, len);
2576             break;
2577
2578         case MIG_RP_MSG_REQ_PAGES_ID:
2579             expected_len = 12 + 1; /* header + termination */
2580
2581             if (header_len >= expected_len) {
2582                 start = ldq_be_p(buf);
2583                 len = ldl_be_p(buf + 8);
2584                 /* Now we expect an idstr */
2585                 tmp32 = buf[12]; /* Length of the following idstr */
2586                 buf[13 + tmp32] = '\0';
2587                 expected_len += tmp32;
2588             }
2589             if (header_len != expected_len) {
2590                 error_report("RP: Req_Page_id with length %d expecting %zd",
2591                              header_len, expected_len);
2592                 mark_source_rp_bad(ms);
2593                 goto out;
2594             }
2595             migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len);
2596             break;
2597
2598         case MIG_RP_MSG_RECV_BITMAP:
2599             if (header_len < 1) {
2600                 error_report("%s: missing block name", __func__);
2601                 mark_source_rp_bad(ms);
2602                 goto out;
2603             }
2604             /* Format: len (1B) + idstr (<255B). This ends the idstr. */
2605             buf[buf[0] + 1] = '\0';
2606             if (migrate_handle_rp_recv_bitmap(ms, (char *)(buf + 1))) {
2607                 mark_source_rp_bad(ms);
2608                 goto out;
2609             }
2610             break;
2611
2612         case MIG_RP_MSG_RESUME_ACK:
2613             tmp32 = ldl_be_p(buf);
2614             if (migrate_handle_rp_resume_ack(ms, tmp32)) {
2615                 mark_source_rp_bad(ms);
2616                 goto out;
2617             }
2618             break;
2619
2620         default:
2621             break;
2622         }
2623     }
2624
2625 out:
2626     res = qemu_file_get_error(rp);
2627     if (res) {
2628         if (res == -EIO && migration_in_postcopy()) {
2629             /*
2630              * Maybe there is something we can do: it looks like a
2631              * network down issue, and we pause for a recovery.
2632              */
2633             if (postcopy_pause_return_path_thread(ms)) {
2634                 /* Reload rp, reset the rest */
2635                 if (rp != ms->rp_state.from_dst_file) {
2636                     qemu_fclose(rp);
2637                     rp = ms->rp_state.from_dst_file;
2638                 }
2639                 ms->rp_state.error = false;
2640                 goto retry;
2641             }
2642         }
2643
2644         trace_source_return_path_thread_bad_end();
2645         mark_source_rp_bad(ms);
2646     }
2647
2648     trace_source_return_path_thread_end();
2649     ms->rp_state.from_dst_file = NULL;
2650     qemu_fclose(rp);
2651     rcu_unregister_thread();
2652     return NULL;
2653 }
2654
2655 static int open_return_path_on_source(MigrationState *ms,
2656                                       bool create_thread)
2657 {
2658
2659     ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file);
2660     if (!ms->rp_state.from_dst_file) {
2661         return -1;
2662     }
2663
2664     trace_open_return_path_on_source();
2665
2666     if (!create_thread) {
2667         /* We're done */
2668         return 0;
2669     }
2670
2671     qemu_thread_create(&ms->rp_state.rp_thread, "return path",
2672                        source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
2673
2674     trace_open_return_path_on_source_continue();
2675
2676     return 0;
2677 }
2678
2679 /* Returns 0 if the RP was ok, otherwise there was an error on the RP */
2680 static int await_return_path_close_on_source(MigrationState *ms)
2681 {
2682     /*
2683      * If this is a normal exit then the destination will send a SHUT and the
2684      * rp_thread will exit, however if there's an error we need to cause
2685      * it to exit.
2686      */
2687     if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) {
2688         /*
2689          * shutdown(2), if we have it, will cause it to unblock if it's stuck
2690          * waiting for the destination.
2691          */
2692         qemu_file_shutdown(ms->rp_state.from_dst_file);
2693         mark_source_rp_bad(ms);
2694     }
2695     trace_await_return_path_close_on_source_joining();
2696     qemu_thread_join(&ms->rp_state.rp_thread);
2697     trace_await_return_path_close_on_source_close();
2698     return ms->rp_state.error;
2699 }
2700
2701 /*
2702  * Switch from normal iteration to postcopy
2703  * Returns non-0 on error
2704  */
2705 static int postcopy_start(MigrationState *ms)
2706 {
2707     int ret;
2708     QIOChannelBuffer *bioc;
2709     QEMUFile *fb;
2710     int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2711     int64_t bandwidth = migrate_max_postcopy_bandwidth();
2712     bool restart_block = false;
2713     int cur_state = MIGRATION_STATUS_ACTIVE;
2714     if (!migrate_pause_before_switchover()) {
2715         migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
2716                           MIGRATION_STATUS_POSTCOPY_ACTIVE);
2717     }
2718
2719     trace_postcopy_start();
2720     qemu_mutex_lock_iothread();
2721     trace_postcopy_start_set_run();
2722
2723     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
2724     global_state_store();
2725     ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
2726     if (ret < 0) {
2727         goto fail;
2728     }
2729
2730     ret = migration_maybe_pause(ms, &cur_state,
2731                                 MIGRATION_STATUS_POSTCOPY_ACTIVE);
2732     if (ret < 0) {
2733         goto fail;
2734     }
2735
2736     ret = bdrv_inactivate_all();
2737     if (ret < 0) {
2738         goto fail;
2739     }
2740     restart_block = true;
2741
2742     /*
2743      * Cause any non-postcopiable, but iterative devices to
2744      * send out their final data.
2745      */
2746     qemu_savevm_state_complete_precopy(ms->to_dst_file, true, false);
2747
2748     /*
2749      * in Finish migrate and with the io-lock held everything should
2750      * be quiet, but we've potentially still got dirty pages and we
2751      * need to tell the destination to throw any pages it's already received
2752      * that are dirty
2753      */
2754     if (migrate_postcopy_ram()) {
2755         if (ram_postcopy_send_discard_bitmap(ms)) {
2756             error_report("postcopy send discard bitmap failed");
2757             goto fail;
2758         }
2759     }
2760
2761     /*
2762      * send rest of state - note things that are doing postcopy
2763      * will notice we're in POSTCOPY_ACTIVE and not actually
2764      * wrap their state up here
2765      */
2766     /* 0 max-postcopy-bandwidth means unlimited */
2767     if (!bandwidth) {
2768         qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX);
2769     } else {
2770         qemu_file_set_rate_limit(ms->to_dst_file, bandwidth / XFER_LIMIT_RATIO);
2771     }
2772     if (migrate_postcopy_ram()) {
2773         /* Ping just for debugging, helps line traces up */
2774         qemu_savevm_send_ping(ms->to_dst_file, 2);
2775     }
2776
2777     /*
2778      * While loading the device state we may trigger page transfer
2779      * requests and the fd must be free to process those, and thus
2780      * the destination must read the whole device state off the fd before
2781      * it starts processing it.  Unfortunately the ad-hoc migration format
2782      * doesn't allow the destination to know the size to read without fully
2783      * parsing it through each devices load-state code (especially the open
2784      * coded devices that use get/put).
2785      * So we wrap the device state up in a package with a length at the start;
2786      * to do this we use a qemu_buf to hold the whole of the device state.
2787      */
2788     bioc = qio_channel_buffer_new(4096);
2789     qio_channel_set_name(QIO_CHANNEL(bioc), "migration-postcopy-buffer");
2790     fb = qemu_fopen_channel_output(QIO_CHANNEL(bioc));
2791     object_unref(OBJECT(bioc));
2792
2793     /*
2794      * Make sure the receiver can get incoming pages before we send the rest
2795      * of the state
2796      */
2797     qemu_savevm_send_postcopy_listen(fb);
2798
2799     qemu_savevm_state_complete_precopy(fb, false, false);
2800     if (migrate_postcopy_ram()) {
2801         qemu_savevm_send_ping(fb, 3);
2802     }
2803
2804     qemu_savevm_send_postcopy_run(fb);
2805
2806     /* <><> end of stuff going into the package */
2807
2808     /* Last point of recovery; as soon as we send the package the destination
2809      * can open devices and potentially start running.
2810      * Lets just check again we've not got any errors.
2811      */
2812     ret = qemu_file_get_error(ms->to_dst_file);
2813     if (ret) {
2814         error_report("postcopy_start: Migration stream errored (pre package)");
2815         goto fail_closefb;
2816     }
2817
2818     restart_block = false;
2819
2820     /* Now send that blob */
2821     if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
2822         goto fail_closefb;
2823     }
2824     qemu_fclose(fb);
2825
2826     /* Send a notify to give a chance for anything that needs to happen
2827      * at the transition to postcopy and after the device state; in particular
2828      * spice needs to trigger a transition now
2829      */
2830     ms->postcopy_after_devices = true;
2831     notifier_list_notify(&migration_state_notifiers, ms);
2832
2833     ms->downtime =  qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop;
2834
2835     qemu_mutex_unlock_iothread();
2836
2837     if (migrate_postcopy_ram()) {
2838         /*
2839          * Although this ping is just for debug, it could potentially be
2840          * used for getting a better measurement of downtime at the source.
2841          */
2842         qemu_savevm_send_ping(ms->to_dst_file, 4);
2843     }
2844
2845     if (migrate_release_ram()) {
2846         ram_postcopy_migrated_memory_release(ms);
2847     }
2848
2849     ret = qemu_file_get_error(ms->to_dst_file);
2850     if (ret) {
2851         error_report("postcopy_start: Migration stream errored");
2852         migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
2853                               MIGRATION_STATUS_FAILED);
2854     }
2855
2856     return ret;
2857
2858 fail_closefb:
2859     qemu_fclose(fb);
2860 fail:
2861     migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
2862                           MIGRATION_STATUS_FAILED);
2863     if (restart_block) {
2864         /* A failure happened early enough that we know the destination hasn't
2865          * accessed block devices, so we're safe to recover.
2866          */
2867         Error *local_err = NULL;
2868
2869         bdrv_invalidate_cache_all(&local_err);
2870         if (local_err) {
2871             error_report_err(local_err);
2872         }
2873     }
2874     qemu_mutex_unlock_iothread();
2875     return -1;
2876 }
2877
2878 /**
2879  * migration_maybe_pause: Pause if required to by
2880  * migrate_pause_before_switchover called with the iothread locked
2881  * Returns: 0 on success
2882  */
2883 static int migration_maybe_pause(MigrationState *s,
2884                                  int *current_active_state,
2885                                  int new_state)
2886 {
2887     if (!migrate_pause_before_switchover()) {
2888         return 0;
2889     }
2890
2891     /* Since leaving this state is not atomic with posting the semaphore
2892      * it's possible that someone could have issued multiple migrate_continue
2893      * and the semaphore is incorrectly positive at this point;
2894      * the docs say it's undefined to reinit a semaphore that's already
2895      * init'd, so use timedwait to eat up any existing posts.
2896      */
2897     while (qemu_sem_timedwait(&s->pause_sem, 1) == 0) {
2898         /* This block intentionally left blank */
2899     }
2900
2901     /*
2902      * If the migration is cancelled when it is in the completion phase,
2903      * the migration state is set to MIGRATION_STATUS_CANCELLING.
2904      * So we don't need to wait a semaphore, otherwise we would always
2905      * wait for the 'pause_sem' semaphore.
2906      */
2907     if (s->state != MIGRATION_STATUS_CANCELLING) {
2908         qemu_mutex_unlock_iothread();
2909         migrate_set_state(&s->state, *current_active_state,
2910                           MIGRATION_STATUS_PRE_SWITCHOVER);
2911         qemu_sem_wait(&s->pause_sem);
2912         migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER,
2913                           new_state);
2914         *current_active_state = new_state;
2915         qemu_mutex_lock_iothread();
2916     }
2917
2918     return s->state == new_state ? 0 : -EINVAL;
2919 }
2920
2921 /**
2922  * migration_completion: Used by migration_thread when there's not much left.
2923  *   The caller 'breaks' the loop when this returns.
2924  *
2925  * @s: Current migration state
2926  */
2927 static void migration_completion(MigrationState *s)
2928 {
2929     int ret;
2930     int current_active_state = s->state;
2931
2932     if (s->state == MIGRATION_STATUS_ACTIVE) {
2933         qemu_mutex_lock_iothread();
2934         s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2935         qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
2936         s->vm_was_running = runstate_is_running();
2937         ret = global_state_store();
2938
2939         if (!ret) {
2940             bool inactivate = !migrate_colo_enabled();
2941             ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
2942             if (ret >= 0) {
2943                 ret = migration_maybe_pause(s, &current_active_state,
2944                                             MIGRATION_STATUS_DEVICE);
2945             }
2946             if (ret >= 0) {
2947                 qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
2948                 ret = qemu_savevm_state_complete_precopy(s->to_dst_file, false,
2949                                                          inactivate);
2950             }
2951             if (inactivate && ret >= 0) {
2952                 s->block_inactive = true;
2953             }
2954         }
2955         qemu_mutex_unlock_iothread();
2956
2957         if (ret < 0) {
2958             goto fail;
2959         }
2960     } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2961         trace_migration_completion_postcopy_end();
2962
2963         qemu_savevm_state_complete_postcopy(s->to_dst_file);
2964         trace_migration_completion_postcopy_end_after_complete();
2965     }
2966
2967     /*
2968      * If rp was opened we must clean up the thread before
2969      * cleaning everything else up (since if there are no failures
2970      * it will wait for the destination to send it's status in
2971      * a SHUT command).
2972      */
2973     if (s->rp_state.from_dst_file) {
2974         int rp_error;
2975         trace_migration_return_path_end_before();
2976         rp_error = await_return_path_close_on_source(s);
2977         trace_migration_return_path_end_after(rp_error);
2978         if (rp_error) {
2979             goto fail_invalidate;
2980         }
2981     }
2982
2983     if (qemu_file_get_error(s->to_dst_file)) {
2984         trace_migration_completion_file_err();
2985         goto fail_invalidate;
2986     }
2987
2988     if (!migrate_colo_enabled()) {
2989         migrate_set_state(&s->state, current_active_state,
2990                           MIGRATION_STATUS_COMPLETED);
2991     }
2992
2993     return;
2994
2995 fail_invalidate:
2996     /* If not doing postcopy, vm_start() will be called: let's regain
2997      * control on images.
2998      */
2999     if (s->state == MIGRATION_STATUS_ACTIVE ||
3000         s->state == MIGRATION_STATUS_DEVICE) {
3001         Error *local_err = NULL;
3002
3003         qemu_mutex_lock_iothread();
3004         bdrv_invalidate_cache_all(&local_err);
3005         if (local_err) {
3006             error_report_err(local_err);
3007         } else {
3008             s->block_inactive = false;
3009         }
3010         qemu_mutex_unlock_iothread();
3011     }
3012
3013 fail:
3014     migrate_set_state(&s->state, current_active_state,
3015                       MIGRATION_STATUS_FAILED);
3016 }
3017
3018 bool migrate_colo_enabled(void)
3019 {
3020     MigrationState *s = migrate_get_current();
3021     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_COLO];
3022 }
3023
3024 typedef enum MigThrError {
3025     /* No error detected */
3026     MIG_THR_ERR_NONE = 0,
3027     /* Detected error, but resumed successfully */
3028     MIG_THR_ERR_RECOVERED = 1,
3029     /* Detected fatal error, need to exit */
3030     MIG_THR_ERR_FATAL = 2,
3031 } MigThrError;
3032
3033 static int postcopy_resume_handshake(MigrationState *s)
3034 {
3035     qemu_savevm_send_postcopy_resume(s->to_dst_file);
3036
3037     while (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3038         qemu_sem_wait(&s->rp_state.rp_sem);
3039     }
3040
3041     if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3042         return 0;
3043     }
3044
3045     return -1;
3046 }
3047
3048 /* Return zero if success, or <0 for error */
3049 static int postcopy_do_resume(MigrationState *s)
3050 {
3051     int ret;
3052
3053     /*
3054      * Call all the resume_prepare() hooks, so that modules can be
3055      * ready for the migration resume.
3056      */
3057     ret = qemu_savevm_state_resume_prepare(s);
3058     if (ret) {
3059         error_report("%s: resume_prepare() failure detected: %d",
3060                      __func__, ret);
3061         return ret;
3062     }
3063
3064     /*
3065      * Last handshake with destination on the resume (destination will
3066      * switch to postcopy-active afterwards)
3067      */
3068     ret = postcopy_resume_handshake(s);
3069     if (ret) {
3070         error_report("%s: handshake failed: %d", __func__, ret);
3071         return ret;
3072     }
3073
3074     return 0;
3075 }
3076
3077 /*
3078  * We don't return until we are in a safe state to continue current
3079  * postcopy migration.  Returns MIG_THR_ERR_RECOVERED if recovered, or
3080  * MIG_THR_ERR_FATAL if unrecovery failure happened.
3081  */
3082 static MigThrError postcopy_pause(MigrationState *s)
3083 {
3084     assert(s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
3085
3086     while (true) {
3087         QEMUFile *file;
3088
3089         migrate_set_state(&s->state, s->state,
3090                           MIGRATION_STATUS_POSTCOPY_PAUSED);
3091
3092         /* Current channel is possibly broken. Release it. */
3093         assert(s->to_dst_file);
3094         qemu_mutex_lock(&s->qemu_file_lock);
3095         file = s->to_dst_file;
3096         s->to_dst_file = NULL;
3097         qemu_mutex_unlock(&s->qemu_file_lock);
3098
3099         qemu_file_shutdown(file);
3100         qemu_fclose(file);
3101
3102         error_report("Detected IO failure for postcopy. "
3103                      "Migration paused.");
3104
3105         /*
3106          * We wait until things fixed up. Then someone will setup the
3107          * status back for us.
3108          */
3109         while (s->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
3110             qemu_sem_wait(&s->postcopy_pause_sem);
3111         }
3112
3113         if (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3114             /* Woken up by a recover procedure. Give it a shot */
3115
3116             /*
3117              * Firstly, let's wake up the return path now, with a new
3118              * return path channel.
3119              */
3120             qemu_sem_post(&s->postcopy_pause_rp_sem);
3121
3122             /* Do the resume logic */
3123             if (postcopy_do_resume(s) == 0) {
3124                 /* Let's continue! */
3125                 trace_postcopy_pause_continued();
3126                 return MIG_THR_ERR_RECOVERED;
3127             } else {
3128                 /*
3129                  * Something wrong happened during the recovery, let's
3130                  * pause again. Pause is always better than throwing
3131                  * data away.
3132                  */
3133                 continue;
3134             }
3135         } else {
3136             /* This is not right... Time to quit. */
3137             return MIG_THR_ERR_FATAL;
3138         }
3139     }
3140 }
3141
3142 static MigThrError migration_detect_error(MigrationState *s)
3143 {
3144     int ret;
3145     int state = s->state;
3146     Error *local_error = NULL;
3147
3148     if (state == MIGRATION_STATUS_CANCELLING ||
3149         state == MIGRATION_STATUS_CANCELLED) {
3150         /* End the migration, but don't set the state to failed */
3151         return MIG_THR_ERR_FATAL;
3152     }
3153
3154     /* Try to detect any file errors */
3155     ret = qemu_file_get_error_obj(s->to_dst_file, &local_error);
3156     if (!ret) {
3157         /* Everything is fine */
3158         assert(!local_error);
3159         return MIG_THR_ERR_NONE;
3160     }
3161
3162     if (local_error) {
3163         migrate_set_error(s, local_error);
3164         error_free(local_error);
3165     }
3166
3167     if (state == MIGRATION_STATUS_POSTCOPY_ACTIVE && ret == -EIO) {
3168         /*
3169          * For postcopy, we allow the network to be down for a
3170          * while. After that, it can be continued by a
3171          * recovery phase.
3172          */
3173         return postcopy_pause(s);
3174     } else {
3175         /*
3176          * For precopy (or postcopy with error outside IO), we fail
3177          * with no time.
3178          */
3179         migrate_set_state(&s->state, state, MIGRATION_STATUS_FAILED);
3180         trace_migration_thread_file_err();
3181
3182         /* Time to stop the migration, now. */
3183         return MIG_THR_ERR_FATAL;
3184     }
3185 }
3186
3187 /* How many bytes have we transferred since the beginning of the migration */
3188 static uint64_t migration_total_bytes(MigrationState *s)
3189 {
3190     return qemu_ftell(s->to_dst_file) + ram_counters.multifd_bytes;
3191 }
3192
3193 static void migration_calculate_complete(MigrationState *s)
3194 {
3195     uint64_t bytes = migration_total_bytes(s);
3196     int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3197     int64_t transfer_time;
3198
3199     s->total_time = end_time - s->start_time;
3200     if (!s->downtime) {
3201         /*
3202          * It's still not set, so we are precopy migration.  For
3203          * postcopy, downtime is calculated during postcopy_start().
3204          */
3205         s->downtime = end_time - s->downtime_start;
3206     }
3207
3208     transfer_time = s->total_time - s->setup_time;
3209     if (transfer_time) {
3210         s->mbps = ((double) bytes * 8.0) / transfer_time / 1000;
3211     }
3212 }
3213
3214 static void update_iteration_initial_status(MigrationState *s)
3215 {
3216     /*
3217      * Update these three fields at the same time to avoid mismatch info lead
3218      * wrong speed calculation.
3219      */
3220     s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3221     s->iteration_initial_bytes = migration_total_bytes(s);
3222     s->iteration_initial_pages = ram_get_total_transferred_pages();
3223 }
3224
3225 static void migration_update_counters(MigrationState *s,
3226                                       int64_t current_time)
3227 {
3228     uint64_t transferred, transferred_pages, time_spent;
3229     uint64_t current_bytes; /* bytes transferred since the beginning */
3230     double bandwidth;
3231
3232     if (current_time < s->iteration_start_time + BUFFER_DELAY) {
3233         return;
3234     }
3235
3236     current_bytes = migration_total_bytes(s);
3237     transferred = current_bytes - s->iteration_initial_bytes;
3238     time_spent = current_time - s->iteration_start_time;
3239     bandwidth = (double)transferred / time_spent;
3240     s->threshold_size = bandwidth * s->parameters.downtime_limit;
3241
3242     s->mbps = (((double) transferred * 8.0) /
3243                ((double) time_spent / 1000.0)) / 1000.0 / 1000.0;
3244
3245     transferred_pages = ram_get_total_transferred_pages() -
3246                             s->iteration_initial_pages;
3247     s->pages_per_second = (double) transferred_pages /
3248                              (((double) time_spent / 1000.0));
3249
3250     /*
3251      * if we haven't sent anything, we don't want to
3252      * recalculate. 10000 is a small enough number for our purposes
3253      */
3254     if (ram_counters.dirty_pages_rate && transferred > 10000) {
3255         s->expected_downtime = ram_counters.remaining / bandwidth;
3256     }
3257
3258     qemu_file_reset_rate_limit(s->to_dst_file);
3259
3260     update_iteration_initial_status(s);
3261
3262     trace_migrate_transferred(transferred, time_spent,
3263                               bandwidth, s->threshold_size);
3264 }
3265
3266 /* Migration thread iteration status */
3267 typedef enum {
3268     MIG_ITERATE_RESUME,         /* Resume current iteration */
3269     MIG_ITERATE_SKIP,           /* Skip current iteration */
3270     MIG_ITERATE_BREAK,          /* Break the loop */
3271 } MigIterateState;
3272
3273 /*
3274  * Return true if continue to the next iteration directly, false
3275  * otherwise.
3276  */
3277 static MigIterateState migration_iteration_run(MigrationState *s)
3278 {
3279     uint64_t pending_size, pend_pre, pend_compat, pend_post;
3280     bool in_postcopy = s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE;
3281
3282     qemu_savevm_state_pending(s->to_dst_file, s->threshold_size, &pend_pre,
3283                               &pend_compat, &pend_post);
3284     pending_size = pend_pre + pend_compat + pend_post;
3285
3286     trace_migrate_pending(pending_size, s->threshold_size,
3287                           pend_pre, pend_compat, pend_post);
3288
3289     if (pending_size && pending_size >= s->threshold_size) {
3290         /* Still a significant amount to transfer */
3291         if (!in_postcopy && pend_pre <= s->threshold_size &&
3292             atomic_read(&s->start_postcopy)) {
3293             if (postcopy_start(s)) {
3294                 error_report("%s: postcopy failed to start", __func__);
3295             }
3296             return MIG_ITERATE_SKIP;
3297         }
3298         /* Just another iteration step */
3299         qemu_savevm_state_iterate(s->to_dst_file, in_postcopy);
3300     } else {
3301         trace_migration_thread_low_pending(pending_size);
3302         migration_completion(s);
3303         return MIG_ITERATE_BREAK;
3304     }
3305
3306     return MIG_ITERATE_RESUME;
3307 }
3308
3309 static void migration_iteration_finish(MigrationState *s)
3310 {
3311     /* If we enabled cpu throttling for auto-converge, turn it off. */
3312     cpu_throttle_stop();
3313
3314     qemu_mutex_lock_iothread();
3315     switch (s->state) {
3316     case MIGRATION_STATUS_COMPLETED:
3317         migration_calculate_complete(s);
3318         runstate_set(RUN_STATE_POSTMIGRATE);
3319         break;
3320
3321     case MIGRATION_STATUS_ACTIVE:
3322         /*
3323          * We should really assert here, but since it's during
3324          * migration, let's try to reduce the usage of assertions.
3325          */
3326         if (!migrate_colo_enabled()) {
3327             error_report("%s: critical error: calling COLO code without "
3328                          "COLO enabled", __func__);
3329         }
3330         migrate_start_colo_process(s);
3331         /*
3332          * Fixme: we will run VM in COLO no matter its old running state.
3333          * After exited COLO, we will keep running.
3334          */
3335         s->vm_was_running = true;
3336         /* Fallthrough */
3337     case MIGRATION_STATUS_FAILED:
3338     case MIGRATION_STATUS_CANCELLED:
3339     case MIGRATION_STATUS_CANCELLING:
3340         if (s->vm_was_running) {
3341             vm_start();
3342         } else {
3343             if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
3344                 runstate_set(RUN_STATE_POSTMIGRATE);
3345             }
3346         }
3347         break;
3348
3349     default:
3350         /* Should not reach here, but if so, forgive the VM. */
3351         error_report("%s: Unknown ending state %d", __func__, s->state);
3352         break;
3353     }
3354     migrate_fd_cleanup_schedule(s);
3355     qemu_mutex_unlock_iothread();
3356 }
3357
3358 void migration_make_urgent_request(void)
3359 {
3360     qemu_sem_post(&migrate_get_current()->rate_limit_sem);
3361 }
3362
3363 void migration_consume_urgent_request(void)
3364 {
3365     qemu_sem_wait(&migrate_get_current()->rate_limit_sem);
3366 }
3367
3368 /* Returns true if the rate limiting was broken by an urgent request */
3369 bool migration_rate_limit(void)
3370 {
3371     int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3372     MigrationState *s = migrate_get_current();
3373
3374     bool urgent = false;
3375     migration_update_counters(s, now);
3376     if (qemu_file_rate_limit(s->to_dst_file)) {
3377
3378         if (qemu_file_get_error(s->to_dst_file)) {
3379             return false;
3380         }
3381         /*
3382          * Wait for a delay to do rate limiting OR
3383          * something urgent to post the semaphore.
3384          */
3385         int ms = s->iteration_start_time + BUFFER_DELAY - now;
3386         trace_migration_rate_limit_pre(ms);
3387         if (qemu_sem_timedwait(&s->rate_limit_sem, ms) == 0) {
3388             /*
3389              * We were woken by one or more urgent things but
3390              * the timedwait will have consumed one of them.
3391              * The service routine for the urgent wake will dec
3392              * the semaphore itself for each item it consumes,
3393              * so add this one we just eat back.
3394              */
3395             qemu_sem_post(&s->rate_limit_sem);
3396             urgent = true;
3397         }
3398         trace_migration_rate_limit_post(urgent);
3399     }
3400     return urgent;
3401 }
3402
3403 /*
3404  * Master migration thread on the source VM.
3405  * It drives the migration and pumps the data down the outgoing channel.
3406  */
3407 static void *migration_thread(void *opaque)
3408 {
3409     MigrationState *s = opaque;
3410     int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3411     MigThrError thr_error;
3412     bool urgent = false;
3413
3414     rcu_register_thread();
3415
3416     object_ref(OBJECT(s));
3417     update_iteration_initial_status(s);
3418
3419     qemu_savevm_state_header(s->to_dst_file);
3420
3421     /*
3422      * If we opened the return path, we need to make sure dst has it
3423      * opened as well.
3424      */
3425     if (s->rp_state.from_dst_file) {
3426         /* Now tell the dest that it should open its end so it can reply */
3427         qemu_savevm_send_open_return_path(s->to_dst_file);
3428
3429         /* And do a ping that will make stuff easier to debug */
3430         qemu_savevm_send_ping(s->to_dst_file, 1);
3431     }
3432
3433     if (migrate_postcopy()) {
3434         /*
3435          * Tell the destination that we *might* want to do postcopy later;
3436          * if the other end can't do postcopy it should fail now, nice and
3437          * early.
3438          */
3439         qemu_savevm_send_postcopy_advise(s->to_dst_file);
3440     }
3441
3442     if (migrate_colo_enabled()) {
3443         /* Notify migration destination that we enable COLO */
3444         qemu_savevm_send_colo_enable(s->to_dst_file);
3445     }
3446
3447     qemu_savevm_state_setup(s->to_dst_file);
3448
3449     if (qemu_savevm_state_guest_unplug_pending()) {
3450         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3451                           MIGRATION_STATUS_WAIT_UNPLUG);
3452
3453         while (s->state == MIGRATION_STATUS_WAIT_UNPLUG &&
3454                qemu_savevm_state_guest_unplug_pending()) {
3455             qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3456         }
3457
3458         migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG,
3459                 MIGRATION_STATUS_ACTIVE);
3460     }
3461
3462     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3463     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3464                       MIGRATION_STATUS_ACTIVE);
3465
3466     trace_migration_thread_setup_complete();
3467
3468     while (migration_is_active(s)) {
3469         if (urgent || !qemu_file_rate_limit(s->to_dst_file)) {
3470             MigIterateState iter_state = migration_iteration_run(s);
3471             if (iter_state == MIG_ITERATE_SKIP) {
3472                 continue;
3473             } else if (iter_state == MIG_ITERATE_BREAK) {
3474                 break;
3475             }
3476         }
3477
3478         /*
3479          * Try to detect any kind of failures, and see whether we
3480          * should stop the migration now.
3481          */
3482         thr_error = migration_detect_error(s);
3483         if (thr_error == MIG_THR_ERR_FATAL) {
3484             /* Stop migration */
3485             break;
3486         } else if (thr_error == MIG_THR_ERR_RECOVERED) {
3487             /*
3488              * Just recovered from a e.g. network failure, reset all
3489              * the local variables. This is important to avoid
3490              * breaking transferred_bytes and bandwidth calculation
3491              */
3492             update_iteration_initial_status(s);
3493         }
3494
3495         urgent = migration_rate_limit();
3496     }
3497
3498     trace_migration_thread_after_loop();
3499     migration_iteration_finish(s);
3500     object_unref(OBJECT(s));
3501     rcu_unregister_thread();
3502     return NULL;
3503 }
3504
3505 void migrate_fd_connect(MigrationState *s, Error *error_in)
3506 {
3507     Error *local_err = NULL;
3508     int64_t rate_limit;
3509     bool resume = s->state == MIGRATION_STATUS_POSTCOPY_PAUSED;
3510
3511     s->expected_downtime = s->parameters.downtime_limit;
3512     if (resume) {
3513         assert(s->cleanup_bh);
3514     } else {
3515         assert(!s->cleanup_bh);
3516         s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup_bh, s);
3517     }
3518     if (error_in) {
3519         migrate_fd_error(s, error_in);
3520         migrate_fd_cleanup(s);
3521         return;
3522     }
3523
3524     if (resume) {
3525         /* This is a resumed migration */
3526         rate_limit = s->parameters.max_postcopy_bandwidth /
3527             XFER_LIMIT_RATIO;
3528     } else {
3529         /* This is a fresh new migration */
3530         rate_limit = s->parameters.max_bandwidth / XFER_LIMIT_RATIO;
3531
3532         /* Notify before starting migration thread */
3533         notifier_list_notify(&migration_state_notifiers, s);
3534     }
3535
3536     qemu_file_set_rate_limit(s->to_dst_file, rate_limit);
3537     qemu_file_set_blocking(s->to_dst_file, true);
3538
3539     /*
3540      * Open the return path. For postcopy, it is used exclusively. For
3541      * precopy, only if user specified "return-path" capability would
3542      * QEMU uses the return path.
3543      */
3544     if (migrate_postcopy_ram() || migrate_use_return_path()) {
3545         if (open_return_path_on_source(s, !resume)) {
3546             error_report("Unable to open return-path for postcopy");
3547             migrate_set_state(&s->state, s->state, MIGRATION_STATUS_FAILED);
3548             migrate_fd_cleanup(s);
3549             return;
3550         }
3551     }
3552
3553     if (resume) {
3554         /* Wakeup the main migration thread to do the recovery */
3555         migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
3556                           MIGRATION_STATUS_POSTCOPY_RECOVER);
3557         qemu_sem_post(&s->postcopy_pause_sem);
3558         return;
3559     }
3560
3561     if (multifd_save_setup(&local_err) != 0) {
3562         error_report_err(local_err);
3563         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3564                           MIGRATION_STATUS_FAILED);
3565         migrate_fd_cleanup(s);
3566         return;
3567     }
3568     qemu_thread_create(&s->thread, "live_migration", migration_thread, s,
3569                        QEMU_THREAD_JOINABLE);
3570     s->migration_thread_running = true;
3571 }
3572
3573 void migration_global_dump(Monitor *mon)
3574 {
3575     MigrationState *ms = migrate_get_current();
3576
3577     monitor_printf(mon, "globals:\n");
3578     monitor_printf(mon, "store-global-state: %s\n",
3579                    ms->store_global_state ? "on" : "off");
3580     monitor_printf(mon, "only-migratable: %s\n",
3581                    only_migratable ? "on" : "off");
3582     monitor_printf(mon, "send-configuration: %s\n",
3583                    ms->send_configuration ? "on" : "off");
3584     monitor_printf(mon, "send-section-footer: %s\n",
3585                    ms->send_section_footer ? "on" : "off");
3586     monitor_printf(mon, "decompress-error-check: %s\n",
3587                    ms->decompress_error_check ? "on" : "off");
3588     monitor_printf(mon, "clear-bitmap-shift: %u\n",
3589                    ms->clear_bitmap_shift);
3590 }
3591
3592 #define DEFINE_PROP_MIG_CAP(name, x)             \
3593     DEFINE_PROP_BOOL(name, MigrationState, enabled_capabilities[x], false)
3594
3595 static Property migration_properties[] = {
3596     DEFINE_PROP_BOOL("store-global-state", MigrationState,
3597                      store_global_state, true),
3598     DEFINE_PROP_BOOL("send-configuration", MigrationState,
3599                      send_configuration, true),
3600     DEFINE_PROP_BOOL("send-section-footer", MigrationState,
3601                      send_section_footer, true),
3602     DEFINE_PROP_BOOL("decompress-error-check", MigrationState,
3603                       decompress_error_check, true),
3604     DEFINE_PROP_UINT8("x-clear-bitmap-shift", MigrationState,
3605                       clear_bitmap_shift, CLEAR_BITMAP_SHIFT_DEFAULT),
3606
3607     /* Migration parameters */
3608     DEFINE_PROP_UINT8("x-compress-level", MigrationState,
3609                       parameters.compress_level,
3610                       DEFAULT_MIGRATE_COMPRESS_LEVEL),
3611     DEFINE_PROP_UINT8("x-compress-threads", MigrationState,
3612                       parameters.compress_threads,
3613                       DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT),
3614     DEFINE_PROP_BOOL("x-compress-wait-thread", MigrationState,
3615                       parameters.compress_wait_thread, true),
3616     DEFINE_PROP_UINT8("x-decompress-threads", MigrationState,
3617                       parameters.decompress_threads,
3618                       DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT),
3619     DEFINE_PROP_UINT8("x-throttle-trigger-threshold", MigrationState,
3620                       parameters.throttle_trigger_threshold,
3621                       DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD),
3622     DEFINE_PROP_UINT8("x-cpu-throttle-initial", MigrationState,
3623                       parameters.cpu_throttle_initial,
3624                       DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL),
3625     DEFINE_PROP_UINT8("x-cpu-throttle-increment", MigrationState,
3626                       parameters.cpu_throttle_increment,
3627                       DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT),
3628     DEFINE_PROP_BOOL("x-cpu-throttle-tailslow", MigrationState,
3629                       parameters.cpu_throttle_tailslow, false),
3630     DEFINE_PROP_SIZE("x-max-bandwidth", MigrationState,
3631                       parameters.max_bandwidth, MAX_THROTTLE),
3632     DEFINE_PROP_UINT64("x-downtime-limit", MigrationState,
3633                       parameters.downtime_limit,
3634                       DEFAULT_MIGRATE_SET_DOWNTIME),
3635     DEFINE_PROP_UINT32("x-checkpoint-delay", MigrationState,
3636                       parameters.x_checkpoint_delay,
3637                       DEFAULT_MIGRATE_X_CHECKPOINT_DELAY),
3638     DEFINE_PROP_UINT8("multifd-channels", MigrationState,
3639                       parameters.multifd_channels,
3640                       DEFAULT_MIGRATE_MULTIFD_CHANNELS),
3641     DEFINE_PROP_MULTIFD_COMPRESSION("multifd-compression", MigrationState,
3642                       parameters.multifd_compression,
3643                       DEFAULT_MIGRATE_MULTIFD_COMPRESSION),
3644     DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState,
3645                       parameters.multifd_zlib_level,
3646                       DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL),
3647     DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState,
3648                       parameters.multifd_zstd_level,
3649                       DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL),
3650     DEFINE_PROP_SIZE("xbzrle-cache-size", MigrationState,
3651                       parameters.xbzrle_cache_size,
3652                       DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE),
3653     DEFINE_PROP_SIZE("max-postcopy-bandwidth", MigrationState,
3654                       parameters.max_postcopy_bandwidth,
3655                       DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH),
3656     DEFINE_PROP_UINT8("max-cpu-throttle", MigrationState,
3657                       parameters.max_cpu_throttle,
3658                       DEFAULT_MIGRATE_MAX_CPU_THROTTLE),
3659     DEFINE_PROP_SIZE("announce-initial", MigrationState,
3660                       parameters.announce_initial,
3661                       DEFAULT_MIGRATE_ANNOUNCE_INITIAL),
3662     DEFINE_PROP_SIZE("announce-max", MigrationState,
3663                       parameters.announce_max,
3664                       DEFAULT_MIGRATE_ANNOUNCE_MAX),
3665     DEFINE_PROP_SIZE("announce-rounds", MigrationState,
3666                       parameters.announce_rounds,
3667                       DEFAULT_MIGRATE_ANNOUNCE_ROUNDS),
3668     DEFINE_PROP_SIZE("announce-step", MigrationState,
3669                       parameters.announce_step,
3670                       DEFAULT_MIGRATE_ANNOUNCE_STEP),
3671
3672     /* Migration capabilities */
3673     DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
3674     DEFINE_PROP_MIG_CAP("x-rdma-pin-all", MIGRATION_CAPABILITY_RDMA_PIN_ALL),
3675     DEFINE_PROP_MIG_CAP("x-auto-converge", MIGRATION_CAPABILITY_AUTO_CONVERGE),
3676     DEFINE_PROP_MIG_CAP("x-zero-blocks", MIGRATION_CAPABILITY_ZERO_BLOCKS),
3677     DEFINE_PROP_MIG_CAP("x-compress", MIGRATION_CAPABILITY_COMPRESS),
3678     DEFINE_PROP_MIG_CAP("x-events", MIGRATION_CAPABILITY_EVENTS),
3679     DEFINE_PROP_MIG_CAP("x-postcopy-ram", MIGRATION_CAPABILITY_POSTCOPY_RAM),
3680     DEFINE_PROP_MIG_CAP("x-colo", MIGRATION_CAPABILITY_X_COLO),
3681     DEFINE_PROP_MIG_CAP("x-release-ram", MIGRATION_CAPABILITY_RELEASE_RAM),
3682     DEFINE_PROP_MIG_CAP("x-block", MIGRATION_CAPABILITY_BLOCK),
3683     DEFINE_PROP_MIG_CAP("x-return-path", MIGRATION_CAPABILITY_RETURN_PATH),
3684     DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD),
3685
3686     DEFINE_PROP_END_OF_LIST(),
3687 };
3688
3689 static void migration_class_init(ObjectClass *klass, void *data)
3690 {
3691     DeviceClass *dc = DEVICE_CLASS(klass);
3692
3693     dc->user_creatable = false;
3694     device_class_set_props(dc, migration_properties);
3695 }
3696
3697 static void migration_instance_finalize(Object *obj)
3698 {
3699     MigrationState *ms = MIGRATION_OBJ(obj);
3700     MigrationParameters *params = &ms->parameters;
3701
3702     qemu_mutex_destroy(&ms->error_mutex);
3703     qemu_mutex_destroy(&ms->qemu_file_lock);
3704     g_free(params->tls_hostname);
3705     g_free(params->tls_creds);
3706     qemu_sem_destroy(&ms->wait_unplug_sem);
3707     qemu_sem_destroy(&ms->rate_limit_sem);
3708     qemu_sem_destroy(&ms->pause_sem);
3709     qemu_sem_destroy(&ms->postcopy_pause_sem);
3710     qemu_sem_destroy(&ms->postcopy_pause_rp_sem);
3711     qemu_sem_destroy(&ms->rp_state.rp_sem);
3712     error_free(ms->error);
3713 }
3714
3715 static void migration_instance_init(Object *obj)
3716 {
3717     MigrationState *ms = MIGRATION_OBJ(obj);
3718     MigrationParameters *params = &ms->parameters;
3719
3720     ms->state = MIGRATION_STATUS_NONE;
3721     ms->mbps = -1;
3722     ms->pages_per_second = -1;
3723     qemu_sem_init(&ms->pause_sem, 0);
3724     qemu_mutex_init(&ms->error_mutex);
3725
3726     params->tls_hostname = g_strdup("");
3727     params->tls_creds = g_strdup("");
3728
3729     /* Set has_* up only for parameter checks */
3730     params->has_compress_level = true;
3731     params->has_compress_threads = true;
3732     params->has_decompress_threads = true;
3733     params->has_throttle_trigger_threshold = true;
3734     params->has_cpu_throttle_initial = true;
3735     params->has_cpu_throttle_increment = true;
3736     params->has_cpu_throttle_tailslow = true;
3737     params->has_max_bandwidth = true;
3738     params->has_downtime_limit = true;
3739     params->has_x_checkpoint_delay = true;
3740     params->has_block_incremental = true;
3741     params->has_multifd_channels = true;
3742     params->has_multifd_compression = true;
3743     params->has_multifd_zlib_level = true;
3744     params->has_multifd_zstd_level = true;
3745     params->has_xbzrle_cache_size = true;
3746     params->has_max_postcopy_bandwidth = true;
3747     params->has_max_cpu_throttle = true;
3748     params->has_announce_initial = true;
3749     params->has_announce_max = true;
3750     params->has_announce_rounds = true;
3751     params->has_announce_step = true;
3752
3753     qemu_sem_init(&ms->postcopy_pause_sem, 0);
3754     qemu_sem_init(&ms->postcopy_pause_rp_sem, 0);
3755     qemu_sem_init(&ms->rp_state.rp_sem, 0);
3756     qemu_sem_init(&ms->rate_limit_sem, 0);
3757     qemu_sem_init(&ms->wait_unplug_sem, 0);
3758     qemu_mutex_init(&ms->qemu_file_lock);
3759 }
3760
3761 /*
3762  * Return true if check pass, false otherwise. Error will be put
3763  * inside errp if provided.
3764  */
3765 static bool migration_object_check(MigrationState *ms, Error **errp)
3766 {
3767     MigrationCapabilityStatusList *head = NULL;
3768     /* Assuming all off */
3769     bool cap_list[MIGRATION_CAPABILITY__MAX] = { 0 }, ret;
3770     int i;
3771
3772     if (!migrate_params_check(&ms->parameters, errp)) {
3773         return false;
3774     }
3775
3776     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
3777         if (ms->enabled_capabilities[i]) {
3778             head = migrate_cap_add(head, i, true);
3779         }
3780     }
3781
3782     ret = migrate_caps_check(cap_list, head, errp);
3783
3784     /* It works with head == NULL */
3785     qapi_free_MigrationCapabilityStatusList(head);
3786
3787     return ret;
3788 }
3789
3790 static const TypeInfo migration_type = {
3791     .name = TYPE_MIGRATION,
3792     /*
3793      * NOTE: TYPE_MIGRATION is not really a device, as the object is
3794      * not created using qdev_new(), it is not attached to the qdev
3795      * device tree, and it is never realized.
3796      *
3797      * TODO: Make this TYPE_OBJECT once QOM provides something like
3798      * TYPE_DEVICE's "-global" properties.
3799      */
3800     .parent = TYPE_DEVICE,
3801     .class_init = migration_class_init,
3802     .class_size = sizeof(MigrationClass),
3803     .instance_size = sizeof(MigrationState),
3804     .instance_init = migration_instance_init,
3805     .instance_finalize = migration_instance_finalize,
3806 };
3807
3808 static void register_migration_types(void)
3809 {
3810     type_register_static(&migration_type);
3811 }
3812
3813 type_init(register_migration_types);