migration/migration.c

   1 /*
   2  * QEMU live migration
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *
   6  * Authors:
   7  *  Anthony Liguori   <aliguori@us.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include "qemu/cutils.h"
  18 #include "qemu/error-report.h"
  19 #include "qemu/main-loop.h"
  20 #include "migration/blocker.h"
  21 #include "exec.h"
  22 #include "fd.h"
  23 #include "socket.h"
  24 #include "sysemu/runstate.h"
  25 #include "sysemu/sysemu.h"
  26 #include "sysemu/cpu-throttle.h"
  27 #include "rdma.h"
  28 #include "ram.h"
  29 #include "migration/global_state.h"
  30 #include "migration/misc.h"
  31 #include "migration.h"
  32 #include "savevm.h"
  33 #include "qemu-file-channel.h"
  34 #include "qemu-file.h"
  35 #include "migration/vmstate.h"
  36 #include "block/block.h"
  37 #include "qapi/error.h"
  38 #include "qapi/clone-visitor.h"
  39 #include "qapi/qapi-visit-migration.h"
  40 #include "qapi/qapi-visit-sockets.h"
  41 #include "qapi/qapi-commands-migration.h"
  42 #include "qapi/qapi-events-migration.h"
  43 #include "qapi/qmp/qerror.h"
  44 #include "qapi/qmp/qnull.h"
  45 #include "qemu/rcu.h"
  46 #include "block.h"
  47 #include "postcopy-ram.h"
  48 #include "qemu/thread.h"
  49 #include "trace.h"
  50 #include "exec/target_page.h"
  51 #include "io/channel-buffer.h"
  52 #include "migration/colo.h"
  53 #include "hw/boards.h"
  54 #include "hw/qdev-properties.h"
  55 #include "hw/qdev-properties-system.h"
  56 #include "monitor/monitor.h"
  57 #include "net/announce.h"
  58 #include "qemu/queue.h"
  59 #include "multifd.h"
  60 #include "qemu/yank.h"
  61 #include "sysemu/cpus.h"
  62
  63 #ifdef CONFIG_VFIO
  64 #include "hw/vfio/vfio-common.h"
  65 #endif
  66
  67 #define MAX_THROTTLE  (128 << 20)      /* Migration transfer speed throttling */
  68
  69 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
  70  * data. */
  71 #define BUFFER_DELAY     100
  72 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
  73
  74 /* Time in milliseconds we are allowed to stop the source,
  75  * for sending the last part */
  76 #define DEFAULT_MIGRATE_SET_DOWNTIME 300
  77
  78 /* Maximum migrate downtime set to 2000 seconds */
  79 #define MAX_MIGRATE_DOWNTIME_SECONDS 2000
  80 #define MAX_MIGRATE_DOWNTIME (MAX_MIGRATE_DOWNTIME_SECONDS * 1000)
  81
  82 /* Default compression thread count */
  83 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8
  84 /* Default decompression thread count, usually decompression is at
  85  * least 4 times as fast as compression.*/
  86 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2
  87 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */
  88 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1
  89 /* Define default autoconverge cpu throttle migration parameters */
  90 #define DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD 50
  91 #define DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL 20
  92 #define DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT 10
  93 #define DEFAULT_MIGRATE_MAX_CPU_THROTTLE 99
  94
  95 /* Migration XBZRLE default cache size */
  96 #define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024)
  97
  98 /* The delay time (in ms) between two COLO checkpoints */
  99 #define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100)
 100 #define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2
 101 #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE
 102 /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */
 103 #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1
 104 /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */
 105 #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1
 106
 107 /* Background transfer rate for postcopy, 0 means unlimited, note
 108  * that page requests can still exceed this limit.
 109  */
 110 #define DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH 0
 111
 112 /*
 113  * Parameters for self_announce_delay giving a stream of RARP/ARP
 114  * packets after migration.
 115  */
 116 #define DEFAULT_MIGRATE_ANNOUNCE_INITIAL  50
 117 #define DEFAULT_MIGRATE_ANNOUNCE_MAX     550
 118 #define DEFAULT_MIGRATE_ANNOUNCE_ROUNDS    5
 119 #define DEFAULT_MIGRATE_ANNOUNCE_STEP    100
 120
 121 static NotifierList migration_state_notifiers =
 122     NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
 123
 124 /* Messages sent on the return path from destination to source */
 125 enum mig_rp_message_type {
 126     MIG_RP_MSG_INVALID = 0,  /* Must be 0 */
 127     MIG_RP_MSG_SHUT,         /* sibling will not send any more RP messages */
 128     MIG_RP_MSG_PONG,         /* Response to a PING; data (seq: be32 ) */
 129
 130     MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
 131     MIG_RP_MSG_REQ_PAGES,    /* data (start: be64, len: be32) */
 132     MIG_RP_MSG_RECV_BITMAP,  /* send recved_bitmap back to source */
 133     MIG_RP_MSG_RESUME_ACK,   /* tell source that we are ready to resume */
 134
 135     MIG_RP_MSG_MAX
 136 };
 137
 138 /* Migration capabilities set */
 139 struct MigrateCapsSet {
 140     int size;                       /* Capability set size */
 141     MigrationCapability caps[];     /* Variadic array of capabilities */
 142 };
 143 typedef struct MigrateCapsSet MigrateCapsSet;
 144
 145 /* Define and initialize MigrateCapsSet */
 146 #define INITIALIZE_MIGRATE_CAPS_SET(_name, ...)   \
 147     MigrateCapsSet _name = {    \
 148         .size = sizeof((int []) { __VA_ARGS__ }) / sizeof(int), \
 149         .caps = { __VA_ARGS__ } \
 150     }
 151
 152 /* Background-snapshot compatibility check list */
 153 static const
 154 INITIALIZE_MIGRATE_CAPS_SET(check_caps_background_snapshot,
 155     MIGRATION_CAPABILITY_POSTCOPY_RAM,
 156     MIGRATION_CAPABILITY_DIRTY_BITMAPS,
 157     MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME,
 158     MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE,
 159     MIGRATION_CAPABILITY_RETURN_PATH,
 160     MIGRATION_CAPABILITY_MULTIFD,
 161     MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER,
 162     MIGRATION_CAPABILITY_AUTO_CONVERGE,
 163     MIGRATION_CAPABILITY_RELEASE_RAM,
 164     MIGRATION_CAPABILITY_RDMA_PIN_ALL,
 165     MIGRATION_CAPABILITY_COMPRESS,
 166     MIGRATION_CAPABILITY_XBZRLE,
 167     MIGRATION_CAPABILITY_X_COLO,
 168     MIGRATION_CAPABILITY_VALIDATE_UUID);
 169
 170 /* When we add fault tolerance, we could have several
 171    migrations at once.  For now we don't need to add
 172    dynamic creation of migration */
 173
 174 static MigrationState *current_migration;
 175 static MigrationIncomingState *current_incoming;
 176
 177 static GSList *migration_blockers;
 178
 179 static bool migration_object_check(MigrationState *ms, Error **errp);
 180 static int migration_maybe_pause(MigrationState *s,
 181                                  int *current_active_state,
 182                                  int new_state);
 183 static void migrate_fd_cancel(MigrationState *s);
 184
 185 static gint page_request_addr_cmp(gconstpointer ap, gconstpointer bp)
 186 {
 187     uintptr_t a = (uintptr_t) ap, b = (uintptr_t) bp;
 188
 189     return (a > b) - (a < b);
 190 }
 191
 192 void migration_object_init(void)
 193 {
 194     Error *err = NULL;
 195
 196     /* This can only be called once. */
 197     assert(!current_migration);
 198     current_migration = MIGRATION_OBJ(object_new(TYPE_MIGRATION));
 199
 200     /*
 201      * Init the migrate incoming object as well no matter whether
 202      * we'll use it or not.
 203      */
 204     assert(!current_incoming);
 205     current_incoming = g_new0(MigrationIncomingState, 1);
 206     current_incoming->state = MIGRATION_STATUS_NONE;
 207     current_incoming->postcopy_remote_fds =
 208         g_array_new(FALSE, TRUE, sizeof(struct PostCopyFD));
 209     qemu_mutex_init(&current_incoming->rp_mutex);
 210     qemu_event_init(&current_incoming->main_thread_load_event, false);
 211     qemu_sem_init(&current_incoming->postcopy_pause_sem_dst, 0);
 212     qemu_sem_init(&current_incoming->postcopy_pause_sem_fault, 0);
 213     qemu_mutex_init(&current_incoming->page_request_mutex);
 214     current_incoming->page_requested = g_tree_new(page_request_addr_cmp);
 215
 216     if (!migration_object_check(current_migration, &err)) {
 217         error_report_err(err);
 218         exit(1);
 219     }
 220
 221     blk_mig_init();
 222     ram_mig_init();
 223     dirty_bitmap_mig_init();
 224 }
 225
 226 void migration_shutdown(void)
 227 {
 228     /*
 229      * Cancel the current migration - that will (eventually)
 230      * stop the migration using this structure
 231      */
 232     migrate_fd_cancel(current_migration);
 233     object_unref(OBJECT(current_migration));
 234
 235     /*
 236      * Cancel outgoing migration of dirty bitmaps. It should
 237      * at least unref used block nodes.
 238      */
 239     dirty_bitmap_mig_cancel_outgoing();
 240
 241     /*
 242      * Cancel incoming migration of dirty bitmaps. Dirty bitmaps
 243      * are non-critical data, and their loss never considered as
 244      * something serious.
 245      */
 246     dirty_bitmap_mig_cancel_incoming();
 247 }
 248
 249 /* For outgoing */
 250 MigrationState *migrate_get_current(void)
 251 {
 252     /* This can only be called after the object created. */
 253     assert(current_migration);
 254     return current_migration;
 255 }
 256
 257 MigrationIncomingState *migration_incoming_get_current(void)
 258 {
 259     assert(current_incoming);
 260     return current_incoming;
 261 }
 262
 263 void migration_incoming_state_destroy(void)
 264 {
 265     struct MigrationIncomingState *mis = migration_incoming_get_current();
 266
 267     if (mis->to_src_file) {
 268         /* Tell source that we are done */
 269         migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
 270         qemu_fclose(mis->to_src_file);
 271         mis->to_src_file = NULL;
 272     }
 273
 274     if (mis->from_src_file) {
 275         qemu_fclose(mis->from_src_file);
 276         mis->from_src_file = NULL;
 277     }
 278     if (mis->postcopy_remote_fds) {
 279         g_array_free(mis->postcopy_remote_fds, TRUE);
 280         mis->postcopy_remote_fds = NULL;
 281     }
 282
 283     qemu_event_reset(&mis->main_thread_load_event);
 284
 285     if (mis->page_requested) {
 286         g_tree_destroy(mis->page_requested);
 287         mis->page_requested = NULL;
 288     }
 289
 290     if (mis->socket_address_list) {
 291         qapi_free_SocketAddressList(mis->socket_address_list);
 292         mis->socket_address_list = NULL;
 293     }
 294
 295     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
 296 }
 297
 298 static void migrate_generate_event(int new_state)
 299 {
 300     if (migrate_use_events()) {
 301         qapi_event_send_migration(new_state);
 302     }
 303 }
 304
 305 static bool migrate_late_block_activate(void)
 306 {
 307     MigrationState *s;
 308
 309     s = migrate_get_current();
 310
 311     return s->enabled_capabilities[
 312         MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE];
 313 }
 314
 315 /*
 316  * Send a message on the return channel back to the source
 317  * of the migration.
 318  */
 319 static int migrate_send_rp_message(MigrationIncomingState *mis,
 320                                    enum mig_rp_message_type message_type,
 321                                    uint16_t len, void *data)
 322 {
 323     int ret = 0;
 324
 325     trace_migrate_send_rp_message((int)message_type, len);
 326     QEMU_LOCK_GUARD(&mis->rp_mutex);
 327
 328     /*
 329      * It's possible that the file handle got lost due to network
 330      * failures.
 331      */
 332     if (!mis->to_src_file) {
 333         ret = -EIO;
 334         return ret;
 335     }
 336
 337     qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
 338     qemu_put_be16(mis->to_src_file, len);
 339     qemu_put_buffer(mis->to_src_file, data, len);
 340     qemu_fflush(mis->to_src_file);
 341
 342     /* It's possible that qemu file got error during sending */
 343     ret = qemu_file_get_error(mis->to_src_file);
 344
 345     return ret;
 346 }
 347
 348 /* Request one page from the source VM at the given start address.
 349  *   rb: the RAMBlock to request the page in
 350  *   Start: Address offset within the RB
 351  *   Len: Length in bytes required - must be a multiple of pagesize
 352  */
 353 int migrate_send_rp_message_req_pages(MigrationIncomingState *mis,
 354                                       RAMBlock *rb, ram_addr_t start)
 355 {
 356     uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */
 357     size_t msglen = 12; /* start + len */
 358     size_t len = qemu_ram_pagesize(rb);
 359     enum mig_rp_message_type msg_type;
 360     const char *rbname;
 361     int rbname_len;
 362
 363     *(uint64_t *)bufc = cpu_to_be64((uint64_t)start);
 364     *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len);
 365
 366     /*
 367      * We maintain the last ramblock that we requested for page.  Note that we
 368      * don't need locking because this function will only be called within the
 369      * postcopy ram fault thread.
 370      */
 371     if (rb != mis->last_rb) {
 372         mis->last_rb = rb;
 373
 374         rbname = qemu_ram_get_idstr(rb);
 375         rbname_len = strlen(rbname);
 376
 377         assert(rbname_len < 256);
 378
 379         bufc[msglen++] = rbname_len;
 380         memcpy(bufc + msglen, rbname, rbname_len);
 381         msglen += rbname_len;
 382         msg_type = MIG_RP_MSG_REQ_PAGES_ID;
 383     } else {
 384         msg_type = MIG_RP_MSG_REQ_PAGES;
 385     }
 386
 387     return migrate_send_rp_message(mis, msg_type, msglen, bufc);
 388 }
 389
 390 int migrate_send_rp_req_pages(MigrationIncomingState *mis,
 391                               RAMBlock *rb, ram_addr_t start, uint64_t haddr)
 392 {
 393     void *aligned = (void *)(uintptr_t)(haddr & (-qemu_ram_pagesize(rb)));
 394     bool received = false;
 395
 396     WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) {
 397         received = ramblock_recv_bitmap_test_byte_offset(rb, start);
 398         if (!received && !g_tree_lookup(mis->page_requested, aligned)) {
 399             /*
 400              * The page has not been received, and it's not yet in the page
 401              * request list.  Queue it.  Set the value of element to 1, so that
 402              * things like g_tree_lookup() will return TRUE (1) when found.
 403              */
 404             g_tree_insert(mis->page_requested, aligned, (gpointer)1);
 405             mis->page_requested_count++;
 406             trace_postcopy_page_req_add(aligned, mis->page_requested_count);
 407         }
 408     }
 409
 410     /*
 411      * If the page is there, skip sending the message.  We don't even need the
 412      * lock because as long as the page arrived, it'll be there forever.
 413      */
 414     if (received) {
 415         return 0;
 416     }
 417
 418     return migrate_send_rp_message_req_pages(mis, rb, start);
 419 }
 420
 421 static bool migration_colo_enabled;
 422 bool migration_incoming_colo_enabled(void)
 423 {
 424     return migration_colo_enabled;
 425 }
 426
 427 void migration_incoming_disable_colo(void)
 428 {
 429     ram_block_discard_disable(false);
 430     migration_colo_enabled = false;
 431 }
 432
 433 int migration_incoming_enable_colo(void)
 434 {
 435     if (ram_block_discard_disable(true)) {
 436         error_report("COLO: cannot disable RAM discard");
 437         return -EBUSY;
 438     }
 439     migration_colo_enabled = true;
 440     return 0;
 441 }
 442
 443 void migrate_add_address(SocketAddress *address)
 444 {
 445     MigrationIncomingState *mis = migration_incoming_get_current();
 446
 447     QAPI_LIST_PREPEND(mis->socket_address_list,
 448                       QAPI_CLONE(SocketAddress, address));
 449 }
 450
 451 static void qemu_start_incoming_migration(const char *uri, Error **errp)
 452 {
 453     const char *p = NULL;
 454
 455     if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
 456         return;
 457     }
 458
 459     qapi_event_send_migration(MIGRATION_STATUS_SETUP);
 460     if (strstart(uri, "tcp:", &p) ||
 461         strstart(uri, "unix:", NULL) ||
 462         strstart(uri, "vsock:", NULL)) {
 463         socket_start_incoming_migration(p ? p : uri, errp);
 464 #ifdef CONFIG_RDMA
 465     } else if (strstart(uri, "rdma:", &p)) {
 466         rdma_start_incoming_migration(p, errp);
 467 #endif
 468     } else if (strstart(uri, "exec:", &p)) {
 469         exec_start_incoming_migration(p, errp);
 470     } else if (strstart(uri, "fd:", &p)) {
 471         fd_start_incoming_migration(p, errp);
 472     } else {
 473         yank_unregister_instance(MIGRATION_YANK_INSTANCE);
 474         error_setg(errp, "unknown migration protocol: %s", uri);
 475     }
 476 }
 477
 478 static void process_incoming_migration_bh(void *opaque)
 479 {
 480     Error *local_err = NULL;
 481     MigrationIncomingState *mis = opaque;
 482
 483     /* If capability late_block_activate is set:
 484      * Only fire up the block code now if we're going to restart the
 485      * VM, else 'cont' will do it.
 486      * This causes file locking to happen; so we don't want it to happen
 487      * unless we really are starting the VM.
 488      */
 489     if (!migrate_late_block_activate() ||
 490          (autostart && (!global_state_received() ||
 491             global_state_get_runstate() == RUN_STATE_RUNNING))) {
 492         /* Make sure all file formats flush their mutable metadata.
 493          * If we get an error here, just don't restart the VM yet. */
 494         bdrv_invalidate_cache_all(&local_err);
 495         if (local_err) {
 496             error_report_err(local_err);
 497             local_err = NULL;
 498             autostart = false;
 499         }
 500     }
 501
 502     /*
 503      * This must happen after all error conditions are dealt with and
 504      * we're sure the VM is going to be running on this host.
 505      */
 506     qemu_announce_self(&mis->announce_timer, migrate_announce_params());
 507
 508     if (multifd_load_cleanup(&local_err) != 0) {
 509         error_report_err(local_err);
 510         autostart = false;
 511     }
 512     /* If global state section was not received or we are in running
 513        state, we need to obey autostart. Any other state is set with
 514        runstate_set. */
 515
 516     dirty_bitmap_mig_before_vm_start();
 517
 518     if (!global_state_received() ||
 519         global_state_get_runstate() == RUN_STATE_RUNNING) {
 520         if (autostart) {
 521             vm_start();
 522         } else {
 523             runstate_set(RUN_STATE_PAUSED);
 524         }
 525     } else if (migration_incoming_colo_enabled()) {
 526         migration_incoming_disable_colo();
 527         vm_start();
 528     } else {
 529         runstate_set(global_state_get_runstate());
 530     }
 531     /*
 532      * This must happen after any state changes since as soon as an external
 533      * observer sees this event they might start to prod at the VM assuming
 534      * it's ready to use.
 535      */
 536     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 537                       MIGRATION_STATUS_COMPLETED);
 538     qemu_bh_delete(mis->bh);
 539     migration_incoming_state_destroy();
 540 }
 541
 542 static void process_incoming_migration_co(void *opaque)
 543 {
 544     MigrationIncomingState *mis = migration_incoming_get_current();
 545     PostcopyState ps;
 546     int ret;
 547     Error *local_err = NULL;
 548
 549     assert(mis->from_src_file);
 550     mis->migration_incoming_co = qemu_coroutine_self();
 551     mis->largest_page_size = qemu_ram_pagesize_largest();
 552     postcopy_state_set(POSTCOPY_INCOMING_NONE);
 553     migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
 554                       MIGRATION_STATUS_ACTIVE);
 555     ret = qemu_loadvm_state(mis->from_src_file);
 556
 557     ps = postcopy_state_get();
 558     trace_process_incoming_migration_co_end(ret, ps);
 559     if (ps != POSTCOPY_INCOMING_NONE) {
 560         if (ps == POSTCOPY_INCOMING_ADVISE) {
 561             /*
 562              * Where a migration had postcopy enabled (and thus went to advise)
 563              * but managed to complete within the precopy period, we can use
 564              * the normal exit.
 565              */
 566             postcopy_ram_incoming_cleanup(mis);
 567         } else if (ret >= 0) {
 568             /*
 569              * Postcopy was started, cleanup should happen at the end of the
 570              * postcopy thread.
 571              */
 572             trace_process_incoming_migration_co_postcopy_end_main();
 573             return;
 574         }
 575         /* Else if something went wrong then just fall out of the normal exit */
 576     }
 577
 578     /* we get COLO info, and know if we are in COLO mode */
 579     if (!ret && migration_incoming_colo_enabled()) {
 580         /* Make sure all file formats flush their mutable metadata */
 581         bdrv_invalidate_cache_all(&local_err);
 582         if (local_err) {
 583             error_report_err(local_err);
 584             goto fail;
 585         }
 586
 587         qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
 588              colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
 589         mis->have_colo_incoming_thread = true;
 590         qemu_coroutine_yield();
 591
 592         /* Wait checkpoint incoming thread exit before free resource */
 593         qemu_thread_join(&mis->colo_incoming_thread);
 594         /* We hold the global iothread lock, so it is safe here */
 595         colo_release_ram_cache();
 596     }
 597
 598     if (ret < 0) {
 599         error_report("load of migration failed: %s", strerror(-ret));
 600         goto fail;
 601     }
 602     mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
 603     qemu_bh_schedule(mis->bh);
 604     mis->migration_incoming_co = NULL;
 605     return;
 606 fail:
 607     local_err = NULL;
 608     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 609                       MIGRATION_STATUS_FAILED);
 610     qemu_fclose(mis->from_src_file);
 611     if (multifd_load_cleanup(&local_err) != 0) {
 612         error_report_err(local_err);
 613     }
 614     exit(EXIT_FAILURE);
 615 }
 616
 617 /**
 618  * @migration_incoming_setup: Setup incoming migration
 619  *
 620  * Returns 0 for no error or 1 for error
 621  *
 622  * @f: file for main migration channel
 623  * @errp: where to put errors
 624  */
 625 static int migration_incoming_setup(QEMUFile *f, Error **errp)
 626 {
 627     MigrationIncomingState *mis = migration_incoming_get_current();
 628     Error *local_err = NULL;
 629
 630     if (multifd_load_setup(&local_err) != 0) {
 631         /* We haven't been able to create multifd threads
 632            nothing better to do */
 633         error_report_err(local_err);
 634         exit(EXIT_FAILURE);
 635     }
 636
 637     if (!mis->from_src_file) {
 638         mis->from_src_file = f;
 639     }
 640     qemu_file_set_blocking(f, false);
 641     return 0;
 642 }
 643
 644 void migration_incoming_process(void)
 645 {
 646     Coroutine *co = qemu_coroutine_create(process_incoming_migration_co, NULL);
 647     qemu_coroutine_enter(co);
 648 }
 649
 650 /* Returns true if recovered from a paused migration, otherwise false */
 651 static bool postcopy_try_recover(QEMUFile *f)
 652 {
 653     MigrationIncomingState *mis = migration_incoming_get_current();
 654
 655     if (mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
 656         /* Resumed from a paused postcopy migration */
 657
 658         mis->from_src_file = f;
 659         /* Postcopy has standalone thread to do vm load */
 660         qemu_file_set_blocking(f, true);
 661
 662         /* Re-configure the return path */
 663         mis->to_src_file = qemu_file_get_return_path(f);
 664
 665         migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
 666                           MIGRATION_STATUS_POSTCOPY_RECOVER);
 667
 668         /*
 669          * Here, we only wake up the main loading thread (while the
 670          * fault thread will still be waiting), so that we can receive
 671          * commands from source now, and answer it if needed. The
 672          * fault thread will be woken up afterwards until we are sure
 673          * that source is ready to reply to page requests.
 674          */
 675         qemu_sem_post(&mis->postcopy_pause_sem_dst);
 676         return true;
 677     }
 678
 679     return false;
 680 }
 681
 682 void migration_fd_process_incoming(QEMUFile *f, Error **errp)
 683 {
 684     Error *local_err = NULL;
 685
 686     if (postcopy_try_recover(f)) {
 687         return;
 688     }
 689
 690     if (migration_incoming_setup(f, &local_err)) {
 691         error_propagate(errp, local_err);
 692         return;
 693     }
 694     migration_incoming_process();
 695 }
 696
 697 void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp)
 698 {
 699     MigrationIncomingState *mis = migration_incoming_get_current();
 700     Error *local_err = NULL;
 701     bool start_migration;
 702
 703     if (!mis->from_src_file) {
 704         /* The first connection (multifd may have multiple) */
 705         QEMUFile *f = qemu_fopen_channel_input(ioc);
 706
 707         /* If it's a recovery, we're done */
 708         if (postcopy_try_recover(f)) {
 709             return;
 710         }
 711
 712         if (migration_incoming_setup(f, &local_err)) {
 713             error_propagate(errp, local_err);
 714             return;
 715         }
 716
 717         /*
 718          * Common migration only needs one channel, so we can start
 719          * right now.  Multifd needs more than one channel, we wait.
 720          */
 721         start_migration = !migrate_use_multifd();
 722     } else {
 723         /* Multiple connections */
 724         assert(migrate_use_multifd());
 725         start_migration = multifd_recv_new_channel(ioc, &local_err);
 726         if (local_err) {
 727             error_propagate(errp, local_err);
 728             return;
 729         }
 730     }
 731
 732     if (start_migration) {
 733         migration_incoming_process();
 734     }
 735 }
 736
 737 /**
 738  * @migration_has_all_channels: We have received all channels that we need
 739  *
 740  * Returns true when we have got connections to all the channels that
 741  * we need for migration.
 742  */
 743 bool migration_has_all_channels(void)
 744 {
 745     MigrationIncomingState *mis = migration_incoming_get_current();
 746     bool all_channels;
 747
 748     all_channels = multifd_recv_all_channels_created();
 749
 750     return all_channels && mis->from_src_file != NULL;
 751 }
 752
 753 /*
 754  * Send a 'SHUT' message on the return channel with the given value
 755  * to indicate that we've finished with the RP.  Non-0 value indicates
 756  * error.
 757  */
 758 void migrate_send_rp_shut(MigrationIncomingState *mis,
 759                           uint32_t value)
 760 {
 761     uint32_t buf;
 762
 763     buf = cpu_to_be32(value);
 764     migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
 765 }
 766
 767 /*
 768  * Send a 'PONG' message on the return channel with the given value
 769  * (normally in response to a 'PING')
 770  */
 771 void migrate_send_rp_pong(MigrationIncomingState *mis,
 772                           uint32_t value)
 773 {
 774     uint32_t buf;
 775
 776     buf = cpu_to_be32(value);
 777     migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
 778 }
 779
 780 void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
 781                                  char *block_name)
 782 {
 783     char buf[512];
 784     int len;
 785     int64_t res;
 786
 787     /*
 788      * First, we send the header part. It contains only the len of
 789      * idstr, and the idstr itself.
 790      */
 791     len = strlen(block_name);
 792     buf[0] = len;
 793     memcpy(buf + 1, block_name, len);
 794
 795     if (mis->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
 796         error_report("%s: MSG_RP_RECV_BITMAP only used for recovery",
 797                      __func__);
 798         return;
 799     }
 800
 801     migrate_send_rp_message(mis, MIG_RP_MSG_RECV_BITMAP, len + 1, buf);
 802
 803     /*
 804      * Next, we dump the received bitmap to the stream.
 805      *
 806      * TODO: currently we are safe since we are the only one that is
 807      * using the to_src_file handle (fault thread is still paused),
 808      * and it's ok even not taking the mutex. However the best way is
 809      * to take the lock before sending the message header, and release
 810      * the lock after sending the bitmap.
 811      */
 812     qemu_mutex_lock(&mis->rp_mutex);
 813     res = ramblock_recv_bitmap_send(mis->to_src_file, block_name);
 814     qemu_mutex_unlock(&mis->rp_mutex);
 815
 816     trace_migrate_send_rp_recv_bitmap(block_name, res);
 817 }
 818
 819 void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value)
 820 {
 821     uint32_t buf;
 822
 823     buf = cpu_to_be32(value);
 824     migrate_send_rp_message(mis, MIG_RP_MSG_RESUME_ACK, sizeof(buf), &buf);
 825 }
 826
 827 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
 828 {
 829     MigrationCapabilityStatusList *head = NULL, **tail = &head;
 830     MigrationCapabilityStatus *caps;
 831     MigrationState *s = migrate_get_current();
 832     int i;
 833
 834     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
 835 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
 836         if (i == MIGRATION_CAPABILITY_BLOCK) {
 837             continue;
 838         }
 839 #endif
 840         caps = g_malloc0(sizeof(*caps));
 841         caps->capability = i;
 842         caps->state = s->enabled_capabilities[i];
 843         QAPI_LIST_APPEND(tail, caps);
 844     }
 845
 846     return head;
 847 }
 848
 849 MigrationParameters *qmp_query_migrate_parameters(Error **errp)
 850 {
 851     MigrationParameters *params;
 852     MigrationState *s = migrate_get_current();
 853
 854     /* TODO use QAPI_CLONE() instead of duplicating it inline */
 855     params = g_malloc0(sizeof(*params));
 856     params->has_compress_level = true;
 857     params->compress_level = s->parameters.compress_level;
 858     params->has_compress_threads = true;
 859     params->compress_threads = s->parameters.compress_threads;
 860     params->has_compress_wait_thread = true;
 861     params->compress_wait_thread = s->parameters.compress_wait_thread;
 862     params->has_decompress_threads = true;
 863     params->decompress_threads = s->parameters.decompress_threads;
 864     params->has_throttle_trigger_threshold = true;
 865     params->throttle_trigger_threshold = s->parameters.throttle_trigger_threshold;
 866     params->has_cpu_throttle_initial = true;
 867     params->cpu_throttle_initial = s->parameters.cpu_throttle_initial;
 868     params->has_cpu_throttle_increment = true;
 869     params->cpu_throttle_increment = s->parameters.cpu_throttle_increment;
 870     params->has_cpu_throttle_tailslow = true;
 871     params->cpu_throttle_tailslow = s->parameters.cpu_throttle_tailslow;
 872     params->has_tls_creds = true;
 873     params->tls_creds = g_strdup(s->parameters.tls_creds);
 874     params->has_tls_hostname = true;
 875     params->tls_hostname = g_strdup(s->parameters.tls_hostname);
 876     params->has_tls_authz = true;
 877     params->tls_authz = g_strdup(s->parameters.tls_authz ?
 878                                  s->parameters.tls_authz : "");
 879     params->has_max_bandwidth = true;
 880     params->max_bandwidth = s->parameters.max_bandwidth;
 881     params->has_downtime_limit = true;
 882     params->downtime_limit = s->parameters.downtime_limit;
 883     params->has_x_checkpoint_delay = true;
 884     params->x_checkpoint_delay = s->parameters.x_checkpoint_delay;
 885     params->has_block_incremental = true;
 886     params->block_incremental = s->parameters.block_incremental;
 887     params->has_multifd_channels = true;
 888     params->multifd_channels = s->parameters.multifd_channels;
 889     params->has_multifd_compression = true;
 890     params->multifd_compression = s->parameters.multifd_compression;
 891     params->has_multifd_zlib_level = true;
 892     params->multifd_zlib_level = s->parameters.multifd_zlib_level;
 893     params->has_multifd_zstd_level = true;
 894     params->multifd_zstd_level = s->parameters.multifd_zstd_level;
 895     params->has_xbzrle_cache_size = true;
 896     params->xbzrle_cache_size = s->parameters.xbzrle_cache_size;
 897     params->has_max_postcopy_bandwidth = true;
 898     params->max_postcopy_bandwidth = s->parameters.max_postcopy_bandwidth;
 899     params->has_max_cpu_throttle = true;
 900     params->max_cpu_throttle = s->parameters.max_cpu_throttle;
 901     params->has_announce_initial = true;
 902     params->announce_initial = s->parameters.announce_initial;
 903     params->has_announce_max = true;
 904     params->announce_max = s->parameters.announce_max;
 905     params->has_announce_rounds = true;
 906     params->announce_rounds = s->parameters.announce_rounds;
 907     params->has_announce_step = true;
 908     params->announce_step = s->parameters.announce_step;
 909
 910     if (s->parameters.has_block_bitmap_mapping) {
 911         params->has_block_bitmap_mapping = true;
 912         params->block_bitmap_mapping =
 913             QAPI_CLONE(BitmapMigrationNodeAliasList,
 914                        s->parameters.block_bitmap_mapping);
 915     }
 916
 917     return params;
 918 }
 919
 920 AnnounceParameters *migrate_announce_params(void)
 921 {
 922     static AnnounceParameters ap;
 923
 924     MigrationState *s = migrate_get_current();
 925
 926     ap.initial = s->parameters.announce_initial;
 927     ap.max = s->parameters.announce_max;
 928     ap.rounds = s->parameters.announce_rounds;
 929     ap.step = s->parameters.announce_step;
 930
 931     return &ap;
 932 }
 933
 934 /*
 935  * Return true if we're already in the middle of a migration
 936  * (i.e. any of the active or setup states)
 937  */
 938 bool migration_is_setup_or_active(int state)
 939 {
 940     switch (state) {
 941     case MIGRATION_STATUS_ACTIVE:
 942     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 943     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 944     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 945     case MIGRATION_STATUS_SETUP:
 946     case MIGRATION_STATUS_PRE_SWITCHOVER:
 947     case MIGRATION_STATUS_DEVICE:
 948     case MIGRATION_STATUS_WAIT_UNPLUG:
 949     case MIGRATION_STATUS_COLO:
 950         return true;
 951
 952     default:
 953         return false;
 954
 955     }
 956 }
 957
 958 bool migration_is_running(int state)
 959 {
 960     switch (state) {
 961     case MIGRATION_STATUS_ACTIVE:
 962     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 963     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 964     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 965     case MIGRATION_STATUS_SETUP:
 966     case MIGRATION_STATUS_PRE_SWITCHOVER:
 967     case MIGRATION_STATUS_DEVICE:
 968     case MIGRATION_STATUS_WAIT_UNPLUG:
 969     case MIGRATION_STATUS_CANCELLING:
 970         return true;
 971
 972     default:
 973         return false;
 974
 975     }
 976 }
 977
 978 static void populate_time_info(MigrationInfo *info, MigrationState *s)
 979 {
 980     info->has_status = true;
 981     info->has_setup_time = true;
 982     info->setup_time = s->setup_time;
 983     if (s->state == MIGRATION_STATUS_COMPLETED) {
 984         info->has_total_time = true;
 985         info->total_time = s->total_time;
 986         info->has_downtime = true;
 987         info->downtime = s->downtime;
 988     } else {
 989         info->has_total_time = true;
 990         info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) -
 991                            s->start_time;
 992         info->has_expected_downtime = true;
 993         info->expected_downtime = s->expected_downtime;
 994     }
 995 }
 996
 997 static void populate_ram_info(MigrationInfo *info, MigrationState *s)
 998 {
 999     info->has_ram = true;
1000     info->ram = g_malloc0(sizeof(*info->ram));
1001     info->ram->transferred = ram_counters.transferred;
1002     info->ram->total = ram_bytes_total();
1003     info->ram->duplicate = ram_counters.duplicate;
1004     /* legacy value.  It is not used anymore */
1005     info->ram->skipped = 0;
1006     info->ram->normal = ram_counters.normal;
1007     info->ram->normal_bytes = ram_counters.normal *
1008         qemu_target_page_size();
1009     info->ram->mbps = s->mbps;
1010     info->ram->dirty_sync_count = ram_counters.dirty_sync_count;
1011     info->ram->postcopy_requests = ram_counters.postcopy_requests;
1012     info->ram->page_size = qemu_target_page_size();
1013     info->ram->multifd_bytes = ram_counters.multifd_bytes;
1014     info->ram->pages_per_second = s->pages_per_second;
1015
1016     if (migrate_use_xbzrle()) {
1017         info->has_xbzrle_cache = true;
1018         info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
1019         info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
1020         info->xbzrle_cache->bytes = xbzrle_counters.bytes;
1021         info->xbzrle_cache->pages = xbzrle_counters.pages;
1022         info->xbzrle_cache->cache_miss = xbzrle_counters.cache_miss;
1023         info->xbzrle_cache->cache_miss_rate = xbzrle_counters.cache_miss_rate;
1024         info->xbzrle_cache->encoding_rate = xbzrle_counters.encoding_rate;
1025         info->xbzrle_cache->overflow = xbzrle_counters.overflow;
1026     }
1027
1028     if (migrate_use_compression()) {
1029         info->has_compression = true;
1030         info->compression = g_malloc0(sizeof(*info->compression));
1031         info->compression->pages = compression_counters.pages;
1032         info->compression->busy = compression_counters.busy;
1033         info->compression->busy_rate = compression_counters.busy_rate;
1034         info->compression->compressed_size =
1035                                     compression_counters.compressed_size;
1036         info->compression->compression_rate =
1037                                     compression_counters.compression_rate;
1038     }
1039
1040     if (cpu_throttle_active()) {
1041         info->has_cpu_throttle_percentage = true;
1042         info->cpu_throttle_percentage = cpu_throttle_get_percentage();
1043     }
1044
1045     if (s->state != MIGRATION_STATUS_COMPLETED) {
1046         info->ram->remaining = ram_bytes_remaining();
1047         info->ram->dirty_pages_rate = ram_counters.dirty_pages_rate;
1048     }
1049 }
1050
1051 static void populate_disk_info(MigrationInfo *info)
1052 {
1053     if (blk_mig_active()) {
1054         info->has_disk = true;
1055         info->disk = g_malloc0(sizeof(*info->disk));
1056         info->disk->transferred = blk_mig_bytes_transferred();
1057         info->disk->remaining = blk_mig_bytes_remaining();
1058         info->disk->total = blk_mig_bytes_total();
1059     }
1060 }
1061
1062 static void populate_vfio_info(MigrationInfo *info)
1063 {
1064 #ifdef CONFIG_VFIO
1065     if (vfio_mig_active()) {
1066         info->has_vfio = true;
1067         info->vfio = g_malloc0(sizeof(*info->vfio));
1068         info->vfio->transferred = vfio_mig_bytes_transferred();
1069     }
1070 #endif
1071 }
1072
1073 static void fill_source_migration_info(MigrationInfo *info)
1074 {
1075     MigrationState *s = migrate_get_current();
1076
1077     info->blocked = migration_is_blocked(NULL);
1078     info->has_blocked_reasons = info->blocked;
1079     info->blocked_reasons = NULL;
1080     if (info->blocked) {
1081         GSList *cur_blocker = migration_blockers;
1082
1083         /*
1084          * There are two types of reasons a migration might be blocked;
1085          * a) devices marked in VMState as non-migratable, and
1086          * b) Explicit migration blockers
1087          * We need to add both of them here.
1088          */
1089         qemu_savevm_non_migratable_list(&info->blocked_reasons);
1090
1091         while (cur_blocker) {
1092             QAPI_LIST_PREPEND(info->blocked_reasons,
1093                               g_strdup(error_get_pretty(cur_blocker->data)));
1094             cur_blocker = g_slist_next(cur_blocker);
1095         }
1096     }
1097
1098     switch (s->state) {
1099     case MIGRATION_STATUS_NONE:
1100         /* no migration has happened ever */
1101         /* do not overwrite destination migration status */
1102         return;
1103     case MIGRATION_STATUS_SETUP:
1104         info->has_status = true;
1105         info->has_total_time = false;
1106         break;
1107     case MIGRATION_STATUS_ACTIVE:
1108     case MIGRATION_STATUS_CANCELLING:
1109     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1110     case MIGRATION_STATUS_PRE_SWITCHOVER:
1111     case MIGRATION_STATUS_DEVICE:
1112     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1113     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1114         /* TODO add some postcopy stats */
1115         populate_time_info(info, s);
1116         populate_ram_info(info, s);
1117         populate_disk_info(info);
1118         populate_vfio_info(info);
1119         break;
1120     case MIGRATION_STATUS_COLO:
1121         info->has_status = true;
1122         /* TODO: display COLO specific information (checkpoint info etc.) */
1123         break;
1124     case MIGRATION_STATUS_COMPLETED:
1125         populate_time_info(info, s);
1126         populate_ram_info(info, s);
1127         populate_vfio_info(info);
1128         break;
1129     case MIGRATION_STATUS_FAILED:
1130         info->has_status = true;
1131         if (s->error) {
1132             info->has_error_desc = true;
1133             info->error_desc = g_strdup(error_get_pretty(s->error));
1134         }
1135         break;
1136     case MIGRATION_STATUS_CANCELLED:
1137         info->has_status = true;
1138         break;
1139     case MIGRATION_STATUS_WAIT_UNPLUG:
1140         info->has_status = true;
1141         break;
1142     }
1143     info->status = s->state;
1144 }
1145
1146 typedef enum WriteTrackingSupport {
1147     WT_SUPPORT_UNKNOWN = 0,
1148     WT_SUPPORT_ABSENT,
1149     WT_SUPPORT_AVAILABLE,
1150     WT_SUPPORT_COMPATIBLE
1151 } WriteTrackingSupport;
1152
1153 static
1154 WriteTrackingSupport migrate_query_write_tracking(void)
1155 {
1156     /* Check if kernel supports required UFFD features */
1157     if (!ram_write_tracking_available()) {
1158         return WT_SUPPORT_ABSENT;
1159     }
1160     /*
1161      * Check if current memory configuration is
1162      * compatible with required UFFD features.
1163      */
1164     if (!ram_write_tracking_compatible()) {
1165         return WT_SUPPORT_AVAILABLE;
1166     }
1167
1168     return WT_SUPPORT_COMPATIBLE;
1169 }
1170
1171 /**
1172  * @migration_caps_check - check capability validity
1173  *
1174  * @cap_list: old capability list, array of bool
1175  * @params: new capabilities to be applied soon
1176  * @errp: set *errp if the check failed, with reason
1177  *
1178  * Returns true if check passed, otherwise false.
1179  */
1180 static bool migrate_caps_check(bool *cap_list,
1181                                MigrationCapabilityStatusList *params,
1182                                Error **errp)
1183 {
1184     MigrationCapabilityStatusList *cap;
1185     bool old_postcopy_cap;
1186     MigrationIncomingState *mis = migration_incoming_get_current();
1187
1188     old_postcopy_cap = cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM];
1189
1190     for (cap = params; cap; cap = cap->next) {
1191         cap_list[cap->value->capability] = cap->value->state;
1192     }
1193
1194 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
1195     if (cap_list[MIGRATION_CAPABILITY_BLOCK]) {
1196         error_setg(errp, "QEMU compiled without old-style (blk/-b, inc/-i) "
1197                    "block migration");
1198         error_append_hint(errp, "Use drive_mirror+NBD instead.\n");
1199         return false;
1200     }
1201 #endif
1202
1203 #ifndef CONFIG_REPLICATION
1204     if (cap_list[MIGRATION_CAPABILITY_X_COLO]) {
1205         error_setg(errp, "QEMU compiled without replication module"
1206                    " can't enable COLO");
1207         error_append_hint(errp, "Please enable replication before COLO.\n");
1208         return false;
1209     }
1210 #endif
1211
1212     if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
1213         /* This check is reasonably expensive, so only when it's being
1214          * set the first time, also it's only the destination that needs
1215          * special support.
1216          */
1217         if (!old_postcopy_cap && runstate_check(RUN_STATE_INMIGRATE) &&
1218             !postcopy_ram_supported_by_host(mis)) {
1219             /* postcopy_ram_supported_by_host will have emitted a more
1220              * detailed message
1221              */
1222             error_setg(errp, "Postcopy is not supported");
1223             return false;
1224         }
1225
1226         if (cap_list[MIGRATION_CAPABILITY_X_IGNORE_SHARED]) {
1227             error_setg(errp, "Postcopy is not compatible with ignore-shared");
1228             return false;
1229         }
1230     }
1231
1232     if (cap_list[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT]) {
1233         WriteTrackingSupport wt_support;
1234         int idx;
1235         /*
1236          * Check if 'background-snapshot' capability is supported by
1237          * host kernel and compatible with guest memory configuration.
1238          */
1239         wt_support = migrate_query_write_tracking();
1240         if (wt_support < WT_SUPPORT_AVAILABLE) {
1241             error_setg(errp, "Background-snapshot is not supported by host kernel");
1242             return false;
1243         }
1244         if (wt_support < WT_SUPPORT_COMPATIBLE) {
1245             error_setg(errp, "Background-snapshot is not compatible "
1246                     "with guest memory configuration");
1247             return false;
1248         }
1249
1250         /*
1251          * Check if there are any migration capabilities
1252          * incompatible with 'background-snapshot'.
1253          */
1254         for (idx = 0; idx < check_caps_background_snapshot.size; idx++) {
1255             int incomp_cap = check_caps_background_snapshot.caps[idx];
1256             if (cap_list[incomp_cap]) {
1257                 error_setg(errp,
1258                         "Background-snapshot is not compatible with %s",
1259                         MigrationCapability_str(incomp_cap));
1260                 return false;
1261             }
1262         }
1263     }
1264
1265     return true;
1266 }
1267
1268 static void fill_destination_migration_info(MigrationInfo *info)
1269 {
1270     MigrationIncomingState *mis = migration_incoming_get_current();
1271
1272     if (mis->socket_address_list) {
1273         info->has_socket_address = true;
1274         info->socket_address =
1275             QAPI_CLONE(SocketAddressList, mis->socket_address_list);
1276     }
1277
1278     switch (mis->state) {
1279     case MIGRATION_STATUS_NONE:
1280         return;
1281     case MIGRATION_STATUS_SETUP:
1282     case MIGRATION_STATUS_CANCELLING:
1283     case MIGRATION_STATUS_CANCELLED:
1284     case MIGRATION_STATUS_ACTIVE:
1285     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1286     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1287     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1288     case MIGRATION_STATUS_FAILED:
1289     case MIGRATION_STATUS_COLO:
1290         info->has_status = true;
1291         break;
1292     case MIGRATION_STATUS_COMPLETED:
1293         info->has_status = true;
1294         fill_destination_postcopy_migration_info(info);
1295         break;
1296     }
1297     info->status = mis->state;
1298 }
1299
1300 MigrationInfo *qmp_query_migrate(Error **errp)
1301 {
1302     MigrationInfo *info = g_malloc0(sizeof(*info));
1303
1304     fill_destination_migration_info(info);
1305     fill_source_migration_info(info);
1306
1307     return info;
1308 }
1309
1310 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
1311                                   Error **errp)
1312 {
1313     MigrationState *s = migrate_get_current();
1314     MigrationCapabilityStatusList *cap;
1315     bool cap_list[MIGRATION_CAPABILITY__MAX];
1316
1317     if (migration_is_running(s->state)) {
1318         error_setg(errp, QERR_MIGRATION_ACTIVE);
1319         return;
1320     }
1321
1322     memcpy(cap_list, s->enabled_capabilities, sizeof(cap_list));
1323     if (!migrate_caps_check(cap_list, params, errp)) {
1324         return;
1325     }
1326
1327     for (cap = params; cap; cap = cap->next) {
1328         s->enabled_capabilities[cap->value->capability] = cap->value->state;
1329     }
1330 }
1331
1332 /*
1333  * Check whether the parameters are valid. Error will be put into errp
1334  * (if provided). Return true if valid, otherwise false.
1335  */
1336 static bool migrate_params_check(MigrationParameters *params, Error **errp)
1337 {
1338     if (params->has_compress_level &&
1339         (params->compress_level > 9)) {
1340         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level",
1341                    "a value between 0 and 9");
1342         return false;
1343     }
1344
1345     if (params->has_compress_threads && (params->compress_threads < 1)) {
1346         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1347                    "compress_threads",
1348                    "a value between 1 and 255");
1349         return false;
1350     }
1351
1352     if (params->has_decompress_threads && (params->decompress_threads < 1)) {
1353         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1354                    "decompress_threads",
1355                    "a value between 1 and 255");
1356         return false;
1357     }
1358
1359     if (params->has_throttle_trigger_threshold &&
1360         (params->throttle_trigger_threshold < 1 ||
1361          params->throttle_trigger_threshold > 100)) {
1362         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1363                    "throttle_trigger_threshold",
1364                    "an integer in the range of 1 to 100");
1365         return false;
1366     }
1367
1368     if (params->has_cpu_throttle_initial &&
1369         (params->cpu_throttle_initial < 1 ||
1370          params->cpu_throttle_initial > 99)) {
1371         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1372                    "cpu_throttle_initial",
1373                    "an integer in the range of 1 to 99");
1374         return false;
1375     }
1376
1377     if (params->has_cpu_throttle_increment &&
1378         (params->cpu_throttle_increment < 1 ||
1379          params->cpu_throttle_increment > 99)) {
1380         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1381                    "cpu_throttle_increment",
1382                    "an integer in the range of 1 to 99");
1383         return false;
1384     }
1385
1386     if (params->has_max_bandwidth && (params->max_bandwidth > SIZE_MAX)) {
1387         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1388                    "max_bandwidth",
1389                    "an integer in the range of 0 to "stringify(SIZE_MAX)
1390                    " bytes/second");
1391         return false;
1392     }
1393
1394     if (params->has_downtime_limit &&
1395         (params->downtime_limit > MAX_MIGRATE_DOWNTIME)) {
1396         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1397                    "downtime_limit",
1398                    "an integer in the range of 0 to "
1399                     stringify(MAX_MIGRATE_DOWNTIME)" ms");
1400         return false;
1401     }
1402
1403     /* x_checkpoint_delay is now always positive */
1404
1405     if (params->has_multifd_channels && (params->multifd_channels < 1)) {
1406         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1407                    "multifd_channels",
1408                    "a value between 1 and 255");
1409         return false;
1410     }
1411
1412     if (params->has_multifd_zlib_level &&
1413         (params->multifd_zlib_level > 9)) {
1414         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zlib_level",
1415                    "a value between 0 and 9");
1416         return false;
1417     }
1418
1419     if (params->has_multifd_zstd_level &&
1420         (params->multifd_zstd_level > 20)) {
1421         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zstd_level",
1422                    "a value between 0 and 20");
1423         return false;
1424     }
1425
1426     if (params->has_xbzrle_cache_size &&
1427         (params->xbzrle_cache_size < qemu_target_page_size() ||
1428          !is_power_of_2(params->xbzrle_cache_size))) {
1429         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1430                    "xbzrle_cache_size",
1431                    "a power of two no less than the target page size");
1432         return false;
1433     }
1434
1435     if (params->has_max_cpu_throttle &&
1436         (params->max_cpu_throttle < params->cpu_throttle_initial ||
1437          params->max_cpu_throttle > 99)) {
1438         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1439                    "max_cpu_throttle",
1440                    "an integer in the range of cpu_throttle_initial to 99");
1441         return false;
1442     }
1443
1444     if (params->has_announce_initial &&
1445         params->announce_initial > 100000) {
1446         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1447                    "announce_initial",
1448                    "a value between 0 and 100000");
1449         return false;
1450     }
1451     if (params->has_announce_max &&
1452         params->announce_max > 100000) {
1453         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1454                    "announce_max",
1455                    "a value between 0 and 100000");
1456        return false;
1457     }
1458     if (params->has_announce_rounds &&
1459         params->announce_rounds > 1000) {
1460         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1461                    "announce_rounds",
1462                    "a value between 0 and 1000");
1463        return false;
1464     }
1465     if (params->has_announce_step &&
1466         (params->announce_step < 1 ||
1467         params->announce_step > 10000)) {
1468         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1469                    "announce_step",
1470                    "a value between 0 and 10000");
1471        return false;
1472     }
1473
1474     if (params->has_block_bitmap_mapping &&
1475         !check_dirty_bitmap_mig_alias_map(params->block_bitmap_mapping, errp)) {
1476         error_prepend(errp, "Invalid mapping given for block-bitmap-mapping: ");
1477         return false;
1478     }
1479
1480     return true;
1481 }
1482
1483 static void migrate_params_test_apply(MigrateSetParameters *params,
1484                                       MigrationParameters *dest)
1485 {
1486     *dest = migrate_get_current()->parameters;
1487
1488     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1489
1490     if (params->has_compress_level) {
1491         dest->compress_level = params->compress_level;
1492     }
1493
1494     if (params->has_compress_threads) {
1495         dest->compress_threads = params->compress_threads;
1496     }
1497
1498     if (params->has_compress_wait_thread) {
1499         dest->compress_wait_thread = params->compress_wait_thread;
1500     }
1501
1502     if (params->has_decompress_threads) {
1503         dest->decompress_threads = params->decompress_threads;
1504     }
1505
1506     if (params->has_throttle_trigger_threshold) {
1507         dest->throttle_trigger_threshold = params->throttle_trigger_threshold;
1508     }
1509
1510     if (params->has_cpu_throttle_initial) {
1511         dest->cpu_throttle_initial = params->cpu_throttle_initial;
1512     }
1513
1514     if (params->has_cpu_throttle_increment) {
1515         dest->cpu_throttle_increment = params->cpu_throttle_increment;
1516     }
1517
1518     if (params->has_cpu_throttle_tailslow) {
1519         dest->cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1520     }
1521
1522     if (params->has_tls_creds) {
1523         assert(params->tls_creds->type == QTYPE_QSTRING);
1524         dest->tls_creds = params->tls_creds->u.s;
1525     }
1526
1527     if (params->has_tls_hostname) {
1528         assert(params->tls_hostname->type == QTYPE_QSTRING);
1529         dest->tls_hostname = params->tls_hostname->u.s;
1530     }
1531
1532     if (params->has_max_bandwidth) {
1533         dest->max_bandwidth = params->max_bandwidth;
1534     }
1535
1536     if (params->has_downtime_limit) {
1537         dest->downtime_limit = params->downtime_limit;
1538     }
1539
1540     if (params->has_x_checkpoint_delay) {
1541         dest->x_checkpoint_delay = params->x_checkpoint_delay;
1542     }
1543
1544     if (params->has_block_incremental) {
1545         dest->block_incremental = params->block_incremental;
1546     }
1547     if (params->has_multifd_channels) {
1548         dest->multifd_channels = params->multifd_channels;
1549     }
1550     if (params->has_multifd_compression) {
1551         dest->multifd_compression = params->multifd_compression;
1552     }
1553     if (params->has_xbzrle_cache_size) {
1554         dest->xbzrle_cache_size = params->xbzrle_cache_size;
1555     }
1556     if (params->has_max_postcopy_bandwidth) {
1557         dest->max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1558     }
1559     if (params->has_max_cpu_throttle) {
1560         dest->max_cpu_throttle = params->max_cpu_throttle;
1561     }
1562     if (params->has_announce_initial) {
1563         dest->announce_initial = params->announce_initial;
1564     }
1565     if (params->has_announce_max) {
1566         dest->announce_max = params->announce_max;
1567     }
1568     if (params->has_announce_rounds) {
1569         dest->announce_rounds = params->announce_rounds;
1570     }
1571     if (params->has_announce_step) {
1572         dest->announce_step = params->announce_step;
1573     }
1574
1575     if (params->has_block_bitmap_mapping) {
1576         dest->has_block_bitmap_mapping = true;
1577         dest->block_bitmap_mapping = params->block_bitmap_mapping;
1578     }
1579 }
1580
1581 static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
1582 {
1583     MigrationState *s = migrate_get_current();
1584
1585     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1586
1587     if (params->has_compress_level) {
1588         s->parameters.compress_level = params->compress_level;
1589     }
1590
1591     if (params->has_compress_threads) {
1592         s->parameters.compress_threads = params->compress_threads;
1593     }
1594
1595     if (params->has_compress_wait_thread) {
1596         s->parameters.compress_wait_thread = params->compress_wait_thread;
1597     }
1598
1599     if (params->has_decompress_threads) {
1600         s->parameters.decompress_threads = params->decompress_threads;
1601     }
1602
1603     if (params->has_throttle_trigger_threshold) {
1604         s->parameters.throttle_trigger_threshold = params->throttle_trigger_threshold;
1605     }
1606
1607     if (params->has_cpu_throttle_initial) {
1608         s->parameters.cpu_throttle_initial = params->cpu_throttle_initial;
1609     }
1610
1611     if (params->has_cpu_throttle_increment) {
1612         s->parameters.cpu_throttle_increment = params->cpu_throttle_increment;
1613     }
1614
1615     if (params->has_cpu_throttle_tailslow) {
1616         s->parameters.cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1617     }
1618
1619     if (params->has_tls_creds) {
1620         g_free(s->parameters.tls_creds);
1621         assert(params->tls_creds->type == QTYPE_QSTRING);
1622         s->parameters.tls_creds = g_strdup(params->tls_creds->u.s);
1623     }
1624
1625     if (params->has_tls_hostname) {
1626         g_free(s->parameters.tls_hostname);
1627         assert(params->tls_hostname->type == QTYPE_QSTRING);
1628         s->parameters.tls_hostname = g_strdup(params->tls_hostname->u.s);
1629     }
1630
1631     if (params->has_tls_authz) {
1632         g_free(s->parameters.tls_authz);
1633         assert(params->tls_authz->type == QTYPE_QSTRING);
1634         s->parameters.tls_authz = g_strdup(params->tls_authz->u.s);
1635     }
1636
1637     if (params->has_max_bandwidth) {
1638         s->parameters.max_bandwidth = params->max_bandwidth;
1639         if (s->to_dst_file && !migration_in_postcopy()) {
1640             qemu_file_set_rate_limit(s->to_dst_file,
1641                                 s->parameters.max_bandwidth / XFER_LIMIT_RATIO);
1642         }
1643     }
1644
1645     if (params->has_downtime_limit) {
1646         s->parameters.downtime_limit = params->downtime_limit;
1647     }
1648
1649     if (params->has_x_checkpoint_delay) {
1650         s->parameters.x_checkpoint_delay = params->x_checkpoint_delay;
1651         if (migration_in_colo_state()) {
1652             colo_checkpoint_notify(s);
1653         }
1654     }
1655
1656     if (params->has_block_incremental) {
1657         s->parameters.block_incremental = params->block_incremental;
1658     }
1659     if (params->has_multifd_channels) {
1660         s->parameters.multifd_channels = params->multifd_channels;
1661     }
1662     if (params->has_multifd_compression) {
1663         s->parameters.multifd_compression = params->multifd_compression;
1664     }
1665     if (params->has_xbzrle_cache_size) {
1666         s->parameters.xbzrle_cache_size = params->xbzrle_cache_size;
1667         xbzrle_cache_resize(params->xbzrle_cache_size, errp);
1668     }
1669     if (params->has_max_postcopy_bandwidth) {
1670         s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1671         if (s->to_dst_file && migration_in_postcopy()) {
1672             qemu_file_set_rate_limit(s->to_dst_file,
1673                     s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO);
1674         }
1675     }
1676     if (params->has_max_cpu_throttle) {
1677         s->parameters.max_cpu_throttle = params->max_cpu_throttle;
1678     }
1679     if (params->has_announce_initial) {
1680         s->parameters.announce_initial = params->announce_initial;
1681     }
1682     if (params->has_announce_max) {
1683         s->parameters.announce_max = params->announce_max;
1684     }
1685     if (params->has_announce_rounds) {
1686         s->parameters.announce_rounds = params->announce_rounds;
1687     }
1688     if (params->has_announce_step) {
1689         s->parameters.announce_step = params->announce_step;
1690     }
1691
1692     if (params->has_block_bitmap_mapping) {
1693         qapi_free_BitmapMigrationNodeAliasList(
1694             s->parameters.block_bitmap_mapping);
1695
1696         s->parameters.has_block_bitmap_mapping = true;
1697         s->parameters.block_bitmap_mapping =
1698             QAPI_CLONE(BitmapMigrationNodeAliasList,
1699                        params->block_bitmap_mapping);
1700     }
1701 }
1702
1703 void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
1704 {
1705     MigrationParameters tmp;
1706
1707     /* TODO Rewrite "" to null instead */
1708     if (params->has_tls_creds
1709         && params->tls_creds->type == QTYPE_QNULL) {
1710         qobject_unref(params->tls_creds->u.n);
1711         params->tls_creds->type = QTYPE_QSTRING;
1712         params->tls_creds->u.s = strdup("");
1713     }
1714     /* TODO Rewrite "" to null instead */
1715     if (params->has_tls_hostname
1716         && params->tls_hostname->type == QTYPE_QNULL) {
1717         qobject_unref(params->tls_hostname->u.n);
1718         params->tls_hostname->type = QTYPE_QSTRING;
1719         params->tls_hostname->u.s = strdup("");
1720     }
1721
1722     migrate_params_test_apply(params, &tmp);
1723
1724     if (!migrate_params_check(&tmp, errp)) {
1725         /* Invalid parameter */
1726         return;
1727     }
1728
1729     migrate_params_apply(params, errp);
1730 }
1731
1732
1733 void qmp_migrate_start_postcopy(Error **errp)
1734 {
1735     MigrationState *s = migrate_get_current();
1736
1737     if (!migrate_postcopy()) {
1738         error_setg(errp, "Enable postcopy with migrate_set_capability before"
1739                          " the start of migration");
1740         return;
1741     }
1742
1743     if (s->state == MIGRATION_STATUS_NONE) {
1744         error_setg(errp, "Postcopy must be started after migration has been"
1745                          " started");
1746         return;
1747     }
1748     /*
1749      * we don't error if migration has finished since that would be racy
1750      * with issuing this command.
1751      */
1752     qatomic_set(&s->start_postcopy, true);
1753 }
1754
1755 /* shared migration helpers */
1756
1757 void migrate_set_state(int *state, int old_state, int new_state)
1758 {
1759     assert(new_state < MIGRATION_STATUS__MAX);
1760     if (qatomic_cmpxchg(state, old_state, new_state) == old_state) {
1761         trace_migrate_set_state(MigrationStatus_str(new_state));
1762         migrate_generate_event(new_state);
1763     }
1764 }
1765
1766 static MigrationCapabilityStatus *migrate_cap_add(MigrationCapability index,
1767                                                   bool state)
1768 {
1769     MigrationCapabilityStatus *cap;
1770
1771     cap = g_new0(MigrationCapabilityStatus, 1);
1772     cap->capability = index;
1773     cap->state = state;
1774
1775     return cap;
1776 }
1777
1778 void migrate_set_block_enabled(bool value, Error **errp)
1779 {
1780     MigrationCapabilityStatusList *cap = NULL;
1781
1782     QAPI_LIST_PREPEND(cap, migrate_cap_add(MIGRATION_CAPABILITY_BLOCK, value));
1783     qmp_migrate_set_capabilities(cap, errp);
1784     qapi_free_MigrationCapabilityStatusList(cap);
1785 }
1786
1787 static void migrate_set_block_incremental(MigrationState *s, bool value)
1788 {
1789     s->parameters.block_incremental = value;
1790 }
1791
1792 static void block_cleanup_parameters(MigrationState *s)
1793 {
1794     if (s->must_remove_block_options) {
1795         /* setting to false can never fail */
1796         migrate_set_block_enabled(false, &error_abort);
1797         migrate_set_block_incremental(s, false);
1798         s->must_remove_block_options = false;
1799     }
1800 }
1801
1802 static void migrate_fd_cleanup(MigrationState *s)
1803 {
1804     qemu_bh_delete(s->cleanup_bh);
1805     s->cleanup_bh = NULL;
1806
1807     qemu_savevm_state_cleanup();
1808
1809     if (s->to_dst_file) {
1810         QEMUFile *tmp;
1811
1812         trace_migrate_fd_cleanup();
1813         qemu_mutex_unlock_iothread();
1814         if (s->migration_thread_running) {
1815             qemu_thread_join(&s->thread);
1816             s->migration_thread_running = false;
1817         }
1818         qemu_mutex_lock_iothread();
1819
1820         multifd_save_cleanup();
1821         qemu_mutex_lock(&s->qemu_file_lock);
1822         tmp = s->to_dst_file;
1823         s->to_dst_file = NULL;
1824         qemu_mutex_unlock(&s->qemu_file_lock);
1825         /*
1826          * Close the file handle without the lock to make sure the
1827          * critical section won't block for long.
1828          */
1829         qemu_fclose(tmp);
1830     }
1831
1832     assert(!migration_is_active(s));
1833
1834     if (s->state == MIGRATION_STATUS_CANCELLING) {
1835         migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING,
1836                           MIGRATION_STATUS_CANCELLED);
1837     }
1838
1839     if (s->error) {
1840         /* It is used on info migrate.  We can't free it */
1841         error_report_err(error_copy(s->error));
1842     }
1843     notifier_list_notify(&migration_state_notifiers, s);
1844     block_cleanup_parameters(s);
1845     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
1846 }
1847
1848 static void migrate_fd_cleanup_schedule(MigrationState *s)
1849 {
1850     /*
1851      * Ref the state for bh, because it may be called when
1852      * there're already no other refs
1853      */
1854     object_ref(OBJECT(s));
1855     qemu_bh_schedule(s->cleanup_bh);
1856 }
1857
1858 static void migrate_fd_cleanup_bh(void *opaque)
1859 {
1860     MigrationState *s = opaque;
1861     migrate_fd_cleanup(s);
1862     object_unref(OBJECT(s));
1863 }
1864
1865 void migrate_set_error(MigrationState *s, const Error *error)
1866 {
1867     QEMU_LOCK_GUARD(&s->error_mutex);
1868     if (!s->error) {
1869         s->error = error_copy(error);
1870     }
1871 }
1872
1873 void migrate_fd_error(MigrationState *s, const Error *error)
1874 {
1875     trace_migrate_fd_error(error_get_pretty(error));
1876     assert(s->to_dst_file == NULL);
1877     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
1878                       MIGRATION_STATUS_FAILED);
1879     migrate_set_error(s, error);
1880 }
1881
1882 static void migrate_fd_cancel(MigrationState *s)
1883 {
1884     int old_state ;
1885     QEMUFile *f = migrate_get_current()->to_dst_file;
1886     trace_migrate_fd_cancel();
1887
1888     if (s->rp_state.from_dst_file) {
1889         /* shutdown the rp socket, so causing the rp thread to shutdown */
1890         qemu_file_shutdown(s->rp_state.from_dst_file);
1891     }
1892
1893     do {
1894         old_state = s->state;
1895         if (!migration_is_running(old_state)) {
1896             break;
1897         }
1898         /* If the migration is paused, kick it out of the pause */
1899         if (old_state == MIGRATION_STATUS_PRE_SWITCHOVER) {
1900             qemu_sem_post(&s->pause_sem);
1901         }
1902         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING);
1903     } while (s->state != MIGRATION_STATUS_CANCELLING);
1904
1905     /*
1906      * If we're unlucky the migration code might be stuck somewhere in a
1907      * send/write while the network has failed and is waiting to timeout;
1908      * if we've got shutdown(2) available then we can force it to quit.
1909      * The outgoing qemu file gets closed in migrate_fd_cleanup that is
1910      * called in a bh, so there is no race against this cancel.
1911      */
1912     if (s->state == MIGRATION_STATUS_CANCELLING && f) {
1913         qemu_file_shutdown(f);
1914     }
1915     if (s->state == MIGRATION_STATUS_CANCELLING && s->block_inactive) {
1916         Error *local_err = NULL;
1917
1918         bdrv_invalidate_cache_all(&local_err);
1919         if (local_err) {
1920             error_report_err(local_err);
1921         } else {
1922             s->block_inactive = false;
1923         }
1924     }
1925 }
1926
1927 void add_migration_state_change_notifier(Notifier *notify)
1928 {
1929     notifier_list_add(&migration_state_notifiers, notify);
1930 }
1931
1932 void remove_migration_state_change_notifier(Notifier *notify)
1933 {
1934     notifier_remove(notify);
1935 }
1936
1937 bool migration_in_setup(MigrationState *s)
1938 {
1939     return s->state == MIGRATION_STATUS_SETUP;
1940 }
1941
1942 bool migration_has_finished(MigrationState *s)
1943 {
1944     return s->state == MIGRATION_STATUS_COMPLETED;
1945 }
1946
1947 bool migration_has_failed(MigrationState *s)
1948 {
1949     return (s->state == MIGRATION_STATUS_CANCELLED ||
1950             s->state == MIGRATION_STATUS_FAILED);
1951 }
1952
1953 bool migration_in_postcopy(void)
1954 {
1955     MigrationState *s = migrate_get_current();
1956
1957     switch (s->state) {
1958     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1959     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1960     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1961         return true;
1962     default:
1963         return false;
1964     }
1965 }
1966
1967 bool migration_in_postcopy_after_devices(MigrationState *s)
1968 {
1969     return migration_in_postcopy() && s->postcopy_after_devices;
1970 }
1971
1972 bool migration_in_incoming_postcopy(void)
1973 {
1974     PostcopyState ps = postcopy_state_get();
1975
1976     return ps >= POSTCOPY_INCOMING_DISCARD && ps < POSTCOPY_INCOMING_END;
1977 }
1978
1979 bool migration_is_idle(void)
1980 {
1981     MigrationState *s = current_migration;
1982
1983     if (!s) {
1984         return true;
1985     }
1986
1987     switch (s->state) {
1988     case MIGRATION_STATUS_NONE:
1989     case MIGRATION_STATUS_CANCELLED:
1990     case MIGRATION_STATUS_COMPLETED:
1991     case MIGRATION_STATUS_FAILED:
1992         return true;
1993     case MIGRATION_STATUS_SETUP:
1994     case MIGRATION_STATUS_CANCELLING:
1995     case MIGRATION_STATUS_ACTIVE:
1996     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1997     case MIGRATION_STATUS_COLO:
1998     case MIGRATION_STATUS_PRE_SWITCHOVER:
1999     case MIGRATION_STATUS_DEVICE:
2000     case MIGRATION_STATUS_WAIT_UNPLUG:
2001         return false;
2002     case MIGRATION_STATUS__MAX:
2003         g_assert_not_reached();
2004     }
2005
2006     return false;
2007 }
2008
2009 bool migration_is_active(MigrationState *s)
2010 {
2011     return (s->state == MIGRATION_STATUS_ACTIVE ||
2012             s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
2013 }
2014
2015 void migrate_init(MigrationState *s)
2016 {
2017     /*
2018      * Reinitialise all migration state, except
2019      * parameters/capabilities that the user set, and
2020      * locks.
2021      */
2022     s->cleanup_bh = 0;
2023     s->vm_start_bh = 0;
2024     s->to_dst_file = NULL;
2025     s->state = MIGRATION_STATUS_NONE;
2026     s->rp_state.from_dst_file = NULL;
2027     s->rp_state.error = false;
2028     s->mbps = 0.0;
2029     s->pages_per_second = 0.0;
2030     s->downtime = 0;
2031     s->expected_downtime = 0;
2032     s->setup_time = 0;
2033     s->start_postcopy = false;
2034     s->postcopy_after_devices = false;
2035     s->migration_thread_running = false;
2036     error_free(s->error);
2037     s->error = NULL;
2038     s->hostname = NULL;
2039
2040     migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
2041
2042     s->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2043     s->total_time = 0;
2044     s->vm_was_running = false;
2045     s->iteration_initial_bytes = 0;
2046     s->threshold_size = 0;
2047 }
2048
2049 int migrate_add_blocker(Error *reason, Error **errp)
2050 {
2051     if (only_migratable) {
2052         error_propagate_prepend(errp, error_copy(reason),
2053                                 "disallowing migration blocker "
2054                                 "(--only-migratable) for: ");
2055         return -EACCES;
2056     }
2057
2058     if (migration_is_idle()) {
2059         migration_blockers = g_slist_prepend(migration_blockers, reason);
2060         return 0;
2061     }
2062
2063     error_propagate_prepend(errp, error_copy(reason),
2064                             "disallowing migration blocker "
2065                             "(migration in progress) for: ");
2066     return -EBUSY;
2067 }
2068
2069 void migrate_del_blocker(Error *reason)
2070 {
2071     migration_blockers = g_slist_remove(migration_blockers, reason);
2072 }
2073
2074 void qmp_migrate_incoming(const char *uri, Error **errp)
2075 {
2076     Error *local_err = NULL;
2077     static bool once = true;
2078
2079     if (!once) {
2080         error_setg(errp, "The incoming migration has already been started");
2081         return;
2082     }
2083     if (!runstate_check(RUN_STATE_INMIGRATE)) {
2084         error_setg(errp, "'-incoming' was not specified on the command line");
2085         return;
2086     }
2087
2088     qemu_start_incoming_migration(uri, &local_err);
2089
2090     if (local_err) {
2091         error_propagate(errp, local_err);
2092         return;
2093     }
2094
2095     once = false;
2096 }
2097
2098 void qmp_migrate_recover(const char *uri, Error **errp)
2099 {
2100     MigrationIncomingState *mis = migration_incoming_get_current();
2101
2102     if (mis->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2103         error_setg(errp, "Migrate recover can only be run "
2104                    "when postcopy is paused.");
2105         return;
2106     }
2107
2108     if (qatomic_cmpxchg(&mis->postcopy_recover_triggered,
2109                        false, true) == true) {
2110         error_setg(errp, "Migrate recovery is triggered already");
2111         return;
2112     }
2113
2114     /*
2115      * Note that this call will never start a real migration; it will
2116      * only re-setup the migration stream and poke existing migration
2117      * to continue using that newly established channel.
2118      */
2119     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2120     qemu_start_incoming_migration(uri, errp);
2121 }
2122
2123 void qmp_migrate_pause(Error **errp)
2124 {
2125     MigrationState *ms = migrate_get_current();
2126     MigrationIncomingState *mis = migration_incoming_get_current();
2127     int ret;
2128
2129     if (ms->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2130         /* Source side, during postcopy */
2131         qemu_mutex_lock(&ms->qemu_file_lock);
2132         ret = qemu_file_shutdown(ms->to_dst_file);
2133         qemu_mutex_unlock(&ms->qemu_file_lock);
2134         if (ret) {
2135             error_setg(errp, "Failed to pause source migration");
2136         }
2137         return;
2138     }
2139
2140     if (mis->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2141         ret = qemu_file_shutdown(mis->from_src_file);
2142         if (ret) {
2143             error_setg(errp, "Failed to pause destination migration");
2144         }
2145         return;
2146     }
2147
2148     error_setg(errp, "migrate-pause is currently only supported "
2149                "during postcopy-active state");
2150 }
2151
2152 bool migration_is_blocked(Error **errp)
2153 {
2154     if (qemu_savevm_state_blocked(errp)) {
2155         return true;
2156     }
2157
2158     if (migration_blockers) {
2159         error_propagate(errp, error_copy(migration_blockers->data));
2160         return true;
2161     }
2162
2163     return false;
2164 }
2165
2166 /* Returns true if continue to migrate, or false if error detected */
2167 static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
2168                             bool resume, Error **errp)
2169 {
2170     Error *local_err = NULL;
2171
2172     if (resume) {
2173         if (s->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2174             error_setg(errp, "Cannot resume if there is no "
2175                        "paused migration");
2176             return false;
2177         }
2178
2179         /*
2180          * Postcopy recovery won't work well with release-ram
2181          * capability since release-ram will drop the page buffer as
2182          * long as the page is put into the send buffer.  So if there
2183          * is a network failure happened, any page buffers that have
2184          * not yet reached the destination VM but have already been
2185          * sent from the source VM will be lost forever.  Let's refuse
2186          * the client from resuming such a postcopy migration.
2187          * Luckily release-ram was designed to only be used when src
2188          * and destination VMs are on the same host, so it should be
2189          * fine.
2190          */
2191         if (migrate_release_ram()) {
2192             error_setg(errp, "Postcopy recovery cannot work "
2193                        "when release-ram capability is set");
2194             return false;
2195         }
2196
2197         /* This is a resume, skip init status */
2198         return true;
2199     }
2200
2201     if (migration_is_running(s->state)) {
2202         error_setg(errp, QERR_MIGRATION_ACTIVE);
2203         return false;
2204     }
2205
2206     if (runstate_check(RUN_STATE_INMIGRATE)) {
2207         error_setg(errp, "Guest is waiting for an incoming migration");
2208         return false;
2209     }
2210
2211     if (runstate_check(RUN_STATE_POSTMIGRATE)) {
2212         error_setg(errp, "Can't migrate the vm that was paused due to "
2213                    "previous migration");
2214         return false;
2215     }
2216
2217     if (migration_is_blocked(errp)) {
2218         return false;
2219     }
2220
2221     if (blk || blk_inc) {
2222         if (migrate_use_block() || migrate_use_block_incremental()) {
2223             error_setg(errp, "Command options are incompatible with "
2224                        "current migration capabilities");
2225             return false;
2226         }
2227         migrate_set_block_enabled(true, &local_err);
2228         if (local_err) {
2229             error_propagate(errp, local_err);
2230             return false;
2231         }
2232         s->must_remove_block_options = true;
2233     }
2234
2235     if (blk_inc) {
2236         migrate_set_block_incremental(s, true);
2237     }
2238
2239     migrate_init(s);
2240     /*
2241      * set ram_counters memory to zero for a
2242      * new migration
2243      */
2244     memset(&ram_counters, 0, sizeof(ram_counters));
2245
2246     return true;
2247 }
2248
2249 void qmp_migrate(const char *uri, bool has_blk, bool blk,
2250                  bool has_inc, bool inc, bool has_detach, bool detach,
2251                  bool has_resume, bool resume, Error **errp)
2252 {
2253     Error *local_err = NULL;
2254     MigrationState *s = migrate_get_current();
2255     const char *p = NULL;
2256
2257     if (!migrate_prepare(s, has_blk && blk, has_inc && inc,
2258                          has_resume && resume, errp)) {
2259         /* Error detected, put into errp */
2260         return;
2261     }
2262
2263     if (!(has_resume && resume)) {
2264         if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2265             return;
2266         }
2267     }
2268
2269     if (strstart(uri, "tcp:", &p) ||
2270         strstart(uri, "unix:", NULL) ||
2271         strstart(uri, "vsock:", NULL)) {
2272         socket_start_outgoing_migration(s, p ? p : uri, &local_err);
2273 #ifdef CONFIG_RDMA
2274     } else if (strstart(uri, "rdma:", &p)) {
2275         rdma_start_outgoing_migration(s, p, &local_err);
2276 #endif
2277     } else if (strstart(uri, "exec:", &p)) {
2278         exec_start_outgoing_migration(s, p, &local_err);
2279     } else if (strstart(uri, "fd:", &p)) {
2280         fd_start_outgoing_migration(s, p, &local_err);
2281     } else {
2282         if (!(has_resume && resume)) {
2283             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2284         }
2285         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri",
2286                    "a valid migration protocol");
2287         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
2288                           MIGRATION_STATUS_FAILED);
2289         block_cleanup_parameters(s);
2290         return;
2291     }
2292
2293     if (local_err) {
2294         if (!(has_resume && resume)) {
2295             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2296         }
2297         migrate_fd_error(s, local_err);
2298         error_propagate(errp, local_err);
2299         return;
2300     }
2301 }
2302
2303 void qmp_migrate_cancel(Error **errp)
2304 {
2305     migrate_fd_cancel(migrate_get_current());
2306 }
2307
2308 void qmp_migrate_continue(MigrationStatus state, Error **errp)
2309 {
2310     MigrationState *s = migrate_get_current();
2311     if (s->state != state) {
2312         error_setg(errp,  "Migration not in expected state: %s",
2313                    MigrationStatus_str(s->state));
2314         return;
2315     }
2316     qemu_sem_post(&s->pause_sem);
2317 }
2318
2319 void qmp_migrate_set_cache_size(int64_t value, Error **errp)
2320 {
2321     MigrateSetParameters p = {
2322         .has_xbzrle_cache_size = true,
2323         .xbzrle_cache_size = value,
2324     };
2325
2326     qmp_migrate_set_parameters(&p, errp);
2327 }
2328
2329 uint64_t qmp_query_migrate_cache_size(Error **errp)
2330 {
2331     return migrate_xbzrle_cache_size();
2332 }
2333
2334 void qmp_migrate_set_speed(int64_t value, Error **errp)
2335 {
2336     MigrateSetParameters p = {
2337         .has_max_bandwidth = true,
2338         .max_bandwidth = value,
2339     };
2340
2341     qmp_migrate_set_parameters(&p, errp);
2342 }
2343
2344 void qmp_migrate_set_downtime(double value, Error **errp)
2345 {
2346     if (value < 0 || value > MAX_MIGRATE_DOWNTIME_SECONDS) {
2347         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
2348                    "downtime_limit",
2349                    "an integer in the range of 0 to "
2350                     stringify(MAX_MIGRATE_DOWNTIME_SECONDS)" seconds");
2351         return;
2352     }
2353
2354     value *= 1000; /* Convert to milliseconds */
2355
2356     MigrateSetParameters p = {
2357         .has_downtime_limit = true,
2358         .downtime_limit = (int64_t)value,
2359     };
2360
2361     qmp_migrate_set_parameters(&p, errp);
2362 }
2363
2364 bool migrate_release_ram(void)
2365 {
2366     MigrationState *s;
2367
2368     s = migrate_get_current();
2369
2370     return s->enabled_capabilities[MIGRATION_CAPABILITY_RELEASE_RAM];
2371 }
2372
2373 bool migrate_postcopy_ram(void)
2374 {
2375     MigrationState *s;
2376
2377     s = migrate_get_current();
2378
2379     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM];
2380 }
2381
2382 bool migrate_postcopy(void)
2383 {
2384     return migrate_postcopy_ram() || migrate_dirty_bitmaps();
2385 }
2386
2387 bool migrate_auto_converge(void)
2388 {
2389     MigrationState *s;
2390
2391     s = migrate_get_current();
2392
2393     return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
2394 }
2395
2396 bool migrate_zero_blocks(void)
2397 {
2398     MigrationState *s;
2399
2400     s = migrate_get_current();
2401
2402     return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
2403 }
2404
2405 bool migrate_postcopy_blocktime(void)
2406 {
2407     MigrationState *s;
2408
2409     s = migrate_get_current();
2410
2411     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME];
2412 }
2413
2414 bool migrate_use_compression(void)
2415 {
2416     MigrationState *s;
2417
2418     s = migrate_get_current();
2419
2420     return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS];
2421 }
2422
2423 int migrate_compress_level(void)
2424 {
2425     MigrationState *s;
2426
2427     s = migrate_get_current();
2428
2429     return s->parameters.compress_level;
2430 }
2431
2432 int migrate_compress_threads(void)
2433 {
2434     MigrationState *s;
2435
2436     s = migrate_get_current();
2437
2438     return s->parameters.compress_threads;
2439 }
2440
2441 int migrate_compress_wait_thread(void)
2442 {
2443     MigrationState *s;
2444
2445     s = migrate_get_current();
2446
2447     return s->parameters.compress_wait_thread;
2448 }
2449
2450 int migrate_decompress_threads(void)
2451 {
2452     MigrationState *s;
2453
2454     s = migrate_get_current();
2455
2456     return s->parameters.decompress_threads;
2457 }
2458
2459 bool migrate_dirty_bitmaps(void)
2460 {
2461     MigrationState *s;
2462
2463     s = migrate_get_current();
2464
2465     return s->enabled_capabilities[MIGRATION_CAPABILITY_DIRTY_BITMAPS];
2466 }
2467
2468 bool migrate_ignore_shared(void)
2469 {
2470     MigrationState *s;
2471
2472     s = migrate_get_current();
2473
2474     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_IGNORE_SHARED];
2475 }
2476
2477 bool migrate_validate_uuid(void)
2478 {
2479     MigrationState *s;
2480
2481     s = migrate_get_current();
2482
2483     return s->enabled_capabilities[MIGRATION_CAPABILITY_VALIDATE_UUID];
2484 }
2485
2486 bool migrate_use_events(void)
2487 {
2488     MigrationState *s;
2489
2490     s = migrate_get_current();
2491
2492     return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS];
2493 }
2494
2495 bool migrate_use_multifd(void)
2496 {
2497     MigrationState *s;
2498
2499     s = migrate_get_current();
2500
2501     return s->enabled_capabilities[MIGRATION_CAPABILITY_MULTIFD];
2502 }
2503
2504 bool migrate_pause_before_switchover(void)
2505 {
2506     MigrationState *s;
2507
2508     s = migrate_get_current();
2509
2510     return s->enabled_capabilities[
2511         MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER];
2512 }
2513
2514 int migrate_multifd_channels(void)
2515 {
2516     MigrationState *s;
2517
2518     s = migrate_get_current();
2519
2520     return s->parameters.multifd_channels;
2521 }
2522
2523 MultiFDCompression migrate_multifd_compression(void)
2524 {
2525     MigrationState *s;
2526
2527     s = migrate_get_current();
2528
2529     return s->parameters.multifd_compression;
2530 }
2531
2532 int migrate_multifd_zlib_level(void)
2533 {
2534     MigrationState *s;
2535
2536     s = migrate_get_current();
2537
2538     return s->parameters.multifd_zlib_level;
2539 }
2540
2541 int migrate_multifd_zstd_level(void)
2542 {
2543     MigrationState *s;
2544
2545     s = migrate_get_current();
2546
2547     return s->parameters.multifd_zstd_level;
2548 }
2549
2550 int migrate_use_xbzrle(void)
2551 {
2552     MigrationState *s;
2553
2554     s = migrate_get_current();
2555
2556     return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
2557 }
2558
2559 uint64_t migrate_xbzrle_cache_size(void)
2560 {
2561     MigrationState *s;
2562
2563     s = migrate_get_current();
2564
2565     return s->parameters.xbzrle_cache_size;
2566 }
2567
2568 static int64_t migrate_max_postcopy_bandwidth(void)
2569 {
2570     MigrationState *s;
2571
2572     s = migrate_get_current();
2573
2574     return s->parameters.max_postcopy_bandwidth;
2575 }
2576
2577 bool migrate_use_block(void)
2578 {
2579     MigrationState *s;
2580
2581     s = migrate_get_current();
2582
2583     return s->enabled_capabilities[MIGRATION_CAPABILITY_BLOCK];
2584 }
2585
2586 bool migrate_use_return_path(void)
2587 {
2588     MigrationState *s;
2589
2590     s = migrate_get_current();
2591
2592     return s->enabled_capabilities[MIGRATION_CAPABILITY_RETURN_PATH];
2593 }
2594
2595 bool migrate_use_block_incremental(void)
2596 {
2597     MigrationState *s;
2598
2599     s = migrate_get_current();
2600
2601     return s->parameters.block_incremental;
2602 }
2603
2604 bool migrate_background_snapshot(void)
2605 {
2606     MigrationState *s;
2607
2608     s = migrate_get_current();
2609
2610     return s->enabled_capabilities[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT];
2611 }
2612
2613 /* migration thread support */
2614 /*
2615  * Something bad happened to the RP stream, mark an error
2616  * The caller shall print or trace something to indicate why
2617  */
2618 static void mark_source_rp_bad(MigrationState *s)
2619 {
2620     s->rp_state.error = true;
2621 }
2622
2623 static struct rp_cmd_args {
2624     ssize_t     len; /* -1 = variable */
2625     const char *name;
2626 } rp_cmd_args[] = {
2627     [MIG_RP_MSG_INVALID]        = { .len = -1, .name = "INVALID" },
2628     [MIG_RP_MSG_SHUT]           = { .len =  4, .name = "SHUT" },
2629     [MIG_RP_MSG_PONG]           = { .len =  4, .name = "PONG" },
2630     [MIG_RP_MSG_REQ_PAGES]      = { .len = 12, .name = "REQ_PAGES" },
2631     [MIG_RP_MSG_REQ_PAGES_ID]   = { .len = -1, .name = "REQ_PAGES_ID" },
2632     [MIG_RP_MSG_RECV_BITMAP]    = { .len = -1, .name = "RECV_BITMAP" },
2633     [MIG_RP_MSG_RESUME_ACK]     = { .len =  4, .name = "RESUME_ACK" },
2634     [MIG_RP_MSG_MAX]            = { .len = -1, .name = "MAX" },
2635 };
2636
2637 /*
2638  * Process a request for pages received on the return path,
2639  * We're allowed to send more than requested (e.g. to round to our page size)
2640  * and we don't need to send pages that have already been sent.
2641  */
2642 static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
2643                                        ram_addr_t start, size_t len)
2644 {
2645     long our_host_ps = qemu_real_host_page_size;
2646
2647     trace_migrate_handle_rp_req_pages(rbname, start, len);
2648
2649     /*
2650      * Since we currently insist on matching page sizes, just sanity check
2651      * we're being asked for whole host pages.
2652      */
2653     if (start & (our_host_ps - 1) ||
2654        (len & (our_host_ps - 1))) {
2655         error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
2656                      " len: %zd", __func__, start, len);
2657         mark_source_rp_bad(ms);
2658         return;
2659     }
2660
2661     if (ram_save_queue_pages(rbname, start, len)) {
2662         mark_source_rp_bad(ms);
2663     }
2664 }
2665
2666 /* Return true to retry, false to quit */
2667 static bool postcopy_pause_return_path_thread(MigrationState *s)
2668 {
2669     trace_postcopy_pause_return_path();
2670
2671     qemu_sem_wait(&s->postcopy_pause_rp_sem);
2672
2673     trace_postcopy_pause_return_path_continued();
2674
2675     return true;
2676 }
2677
2678 static int migrate_handle_rp_recv_bitmap(MigrationState *s, char *block_name)
2679 {
2680     RAMBlock *block = qemu_ram_block_by_name(block_name);
2681
2682     if (!block) {
2683         error_report("%s: invalid block name '%s'", __func__, block_name);
2684         return -EINVAL;
2685     }
2686
2687     /* Fetch the received bitmap and refresh the dirty bitmap */
2688     return ram_dirty_bitmap_reload(s, block);
2689 }
2690
2691 static int migrate_handle_rp_resume_ack(MigrationState *s, uint32_t value)
2692 {
2693     trace_source_return_path_thread_resume_ack(value);
2694
2695     if (value != MIGRATION_RESUME_ACK_VALUE) {
2696         error_report("%s: illegal resume_ack value %"PRIu32,
2697                      __func__, value);
2698         return -1;
2699     }
2700
2701     /* Now both sides are active. */
2702     migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_RECOVER,
2703                       MIGRATION_STATUS_POSTCOPY_ACTIVE);
2704
2705     /* Notify send thread that time to continue send pages */
2706     qemu_sem_post(&s->rp_state.rp_sem);
2707
2708     return 0;
2709 }
2710
2711 /*
2712  * Handles messages sent on the return path towards the source VM
2713  *
2714  */
2715 static void *source_return_path_thread(void *opaque)
2716 {
2717     MigrationState *ms = opaque;
2718     QEMUFile *rp = ms->rp_state.from_dst_file;
2719     uint16_t header_len, header_type;
2720     uint8_t buf[512];
2721     uint32_t tmp32, sibling_error;
2722     ram_addr_t start = 0; /* =0 to silence warning */
2723     size_t  len = 0, expected_len;
2724     int res;
2725
2726     trace_source_return_path_thread_entry();
2727     rcu_register_thread();
2728
2729 retry:
2730     while (!ms->rp_state.error && !qemu_file_get_error(rp) &&
2731            migration_is_setup_or_active(ms->state)) {
2732         trace_source_return_path_thread_loop_top();
2733         header_type = qemu_get_be16(rp);
2734         header_len = qemu_get_be16(rp);
2735
2736         if (qemu_file_get_error(rp)) {
2737             mark_source_rp_bad(ms);
2738             goto out;
2739         }
2740
2741         if (header_type >= MIG_RP_MSG_MAX ||
2742             header_type == MIG_RP_MSG_INVALID) {
2743             error_report("RP: Received invalid message 0x%04x length 0x%04x",
2744                          header_type, header_len);
2745             mark_source_rp_bad(ms);
2746             goto out;
2747         }
2748
2749         if ((rp_cmd_args[header_type].len != -1 &&
2750             header_len != rp_cmd_args[header_type].len) ||
2751             header_len > sizeof(buf)) {
2752             error_report("RP: Received '%s' message (0x%04x) with"
2753                          "incorrect length %d expecting %zu",
2754                          rp_cmd_args[header_type].name, header_type, header_len,
2755                          (size_t)rp_cmd_args[header_type].len);
2756             mark_source_rp_bad(ms);
2757             goto out;
2758         }
2759
2760         /* We know we've got a valid header by this point */
2761         res = qemu_get_buffer(rp, buf, header_len);
2762         if (res != header_len) {
2763             error_report("RP: Failed reading data for message 0x%04x"
2764                          " read %d expected %d",
2765                          header_type, res, header_len);
2766             mark_source_rp_bad(ms);
2767             goto out;
2768         }
2769
2770         /* OK, we have the message and the data */
2771         switch (header_type) {
2772         case MIG_RP_MSG_SHUT:
2773             sibling_error = ldl_be_p(buf);
2774             trace_source_return_path_thread_shut(sibling_error);
2775             if (sibling_error) {
2776                 error_report("RP: Sibling indicated error %d", sibling_error);
2777                 mark_source_rp_bad(ms);
2778             }
2779             /*
2780              * We'll let the main thread deal with closing the RP
2781              * we could do a shutdown(2) on it, but we're the only user
2782              * anyway, so there's nothing gained.
2783              */
2784             goto out;
2785
2786         case MIG_RP_MSG_PONG:
2787             tmp32 = ldl_be_p(buf);
2788             trace_source_return_path_thread_pong(tmp32);
2789             break;
2790
2791         case MIG_RP_MSG_REQ_PAGES:
2792             start = ldq_be_p(buf);
2793             len = ldl_be_p(buf + 8);
2794             migrate_handle_rp_req_pages(ms, NULL, start, len);
2795             break;
2796
2797         case MIG_RP_MSG_REQ_PAGES_ID:
2798             expected_len = 12 + 1; /* header + termination */
2799
2800             if (header_len >= expected_len) {
2801                 start = ldq_be_p(buf);
2802                 len = ldl_be_p(buf + 8);
2803                 /* Now we expect an idstr */
2804                 tmp32 = buf[12]; /* Length of the following idstr */
2805                 buf[13 + tmp32] = '\0';
2806                 expected_len += tmp32;
2807             }
2808             if (header_len != expected_len) {
2809                 error_report("RP: Req_Page_id with length %d expecting %zd",
2810                              header_len, expected_len);
2811                 mark_source_rp_bad(ms);
2812                 goto out;
2813             }
2814             migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len);
2815             break;
2816
2817         case MIG_RP_MSG_RECV_BITMAP:
2818             if (header_len < 1) {
2819                 error_report("%s: missing block name", __func__);
2820                 mark_source_rp_bad(ms);
2821                 goto out;
2822             }
2823             /* Format: len (1B) + idstr (<255B). This ends the idstr. */
2824             buf[buf[0] + 1] = '\0';
2825             if (migrate_handle_rp_recv_bitmap(ms, (char *)(buf + 1))) {
2826                 mark_source_rp_bad(ms);
2827                 goto out;
2828             }
2829             break;
2830
2831         case MIG_RP_MSG_RESUME_ACK:
2832             tmp32 = ldl_be_p(buf);
2833             if (migrate_handle_rp_resume_ack(ms, tmp32)) {
2834                 mark_source_rp_bad(ms);
2835                 goto out;
2836             }
2837             break;
2838
2839         default:
2840             break;
2841         }
2842     }
2843
2844 out:
2845     res = qemu_file_get_error(rp);
2846     if (res) {
2847         if (res == -EIO && migration_in_postcopy()) {
2848             /*
2849              * Maybe there is something we can do: it looks like a
2850              * network down issue, and we pause for a recovery.
2851              */
2852             if (postcopy_pause_return_path_thread(ms)) {
2853                 /* Reload rp, reset the rest */
2854                 if (rp != ms->rp_state.from_dst_file) {
2855                     qemu_fclose(rp);
2856                     rp = ms->rp_state.from_dst_file;
2857                 }
2858                 ms->rp_state.error = false;
2859                 goto retry;
2860             }
2861         }
2862
2863         trace_source_return_path_thread_bad_end();
2864         mark_source_rp_bad(ms);
2865     }
2866
2867     trace_source_return_path_thread_end();
2868     ms->rp_state.from_dst_file = NULL;
2869     qemu_fclose(rp);
2870     rcu_unregister_thread();
2871     return NULL;
2872 }
2873
2874 static int open_return_path_on_source(MigrationState *ms,
2875                                       bool create_thread)
2876 {
2877
2878     ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file);
2879     if (!ms->rp_state.from_dst_file) {
2880         return -1;
2881     }
2882
2883     trace_open_return_path_on_source();
2884
2885     if (!create_thread) {
2886         /* We're done */
2887         return 0;
2888     }
2889
2890     qemu_thread_create(&ms->rp_state.rp_thread, "return path",
2891                        source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
2892
2893     trace_open_return_path_on_source_continue();
2894
2895     return 0;
2896 }
2897
2898 /* Returns 0 if the RP was ok, otherwise there was an error on the RP */
2899 static int await_return_path_close_on_source(MigrationState *ms)
2900 {
2901     /*
2902      * If this is a normal exit then the destination will send a SHUT and the
2903      * rp_thread will exit, however if there's an error we need to cause
2904      * it to exit.
2905      */
2906     if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) {
2907         /*
2908          * shutdown(2), if we have it, will cause it to unblock if it's stuck
2909          * waiting for the destination.
2910          */
2911         qemu_file_shutdown(ms->rp_state.from_dst_file);
2912         mark_source_rp_bad(ms);
2913     }
2914     trace_await_return_path_close_on_source_joining();
2915     qemu_thread_join(&ms->rp_state.rp_thread);
2916     trace_await_return_path_close_on_source_close();
2917     return ms->rp_state.error;
2918 }
2919
2920 /*
2921  * Switch from normal iteration to postcopy
2922  * Returns non-0 on error
2923  */
2924 static int postcopy_start(MigrationState *ms)
2925 {
2926     int ret;
2927     QIOChannelBuffer *bioc;
2928     QEMUFile *fb;
2929     int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2930     int64_t bandwidth = migrate_max_postcopy_bandwidth();
2931     bool restart_block = false;
2932     int cur_state = MIGRATION_STATUS_ACTIVE;
2933     if (!migrate_pause_before_switchover()) {
2934         migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
2935                           MIGRATION_STATUS_POSTCOPY_ACTIVE);
2936     }
2937
2938     trace_postcopy_start();
2939     qemu_mutex_lock_iothread();
2940     trace_postcopy_start_set_run();
2941
2942     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
2943     global_state_store();
2944     ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
2945     if (ret < 0) {
2946         goto fail;
2947     }
2948
2949     ret = migration_maybe_pause(ms, &cur_state,
2950                                 MIGRATION_STATUS_POSTCOPY_ACTIVE);
2951     if (ret < 0) {
2952         goto fail;
2953     }
2954
2955     ret = bdrv_inactivate_all();
2956     if (ret < 0) {
2957         goto fail;
2958     }
2959     restart_block = true;
2960
2961     /*
2962      * Cause any non-postcopiable, but iterative devices to
2963      * send out their final data.
2964      */
2965     qemu_savevm_state_complete_precopy(ms->to_dst_file, true, false);
2966
2967     /*
2968      * in Finish migrate and with the io-lock held everything should
2969      * be quiet, but we've potentially still got dirty pages and we
2970      * need to tell the destination to throw any pages it's already received
2971      * that are dirty
2972      */
2973     if (migrate_postcopy_ram()) {
2974         if (ram_postcopy_send_discard_bitmap(ms)) {
2975             error_report("postcopy send discard bitmap failed");
2976             goto fail;
2977         }
2978     }
2979
2980     /*
2981      * send rest of state - note things that are doing postcopy
2982      * will notice we're in POSTCOPY_ACTIVE and not actually
2983      * wrap their state up here
2984      */
2985     /* 0 max-postcopy-bandwidth means unlimited */
2986     if (!bandwidth) {
2987         qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX);
2988     } else {
2989         qemu_file_set_rate_limit(ms->to_dst_file, bandwidth / XFER_LIMIT_RATIO);
2990     }
2991     if (migrate_postcopy_ram()) {
2992         /* Ping just for debugging, helps line traces up */
2993         qemu_savevm_send_ping(ms->to_dst_file, 2);
2994     }
2995
2996     /*
2997      * While loading the device state we may trigger page transfer
2998      * requests and the fd must be free to process those, and thus
2999      * the destination must read the whole device state off the fd before
3000      * it starts processing it.  Unfortunately the ad-hoc migration format
3001      * doesn't allow the destination to know the size to read without fully
3002      * parsing it through each devices load-state code (especially the open
3003      * coded devices that use get/put).
3004      * So we wrap the device state up in a package with a length at the start;
3005      * to do this we use a qemu_buf to hold the whole of the device state.
3006      */
3007     bioc = qio_channel_buffer_new(4096);
3008     qio_channel_set_name(QIO_CHANNEL(bioc), "migration-postcopy-buffer");
3009     fb = qemu_fopen_channel_output(QIO_CHANNEL(bioc));
3010     object_unref(OBJECT(bioc));
3011
3012     /*
3013      * Make sure the receiver can get incoming pages before we send the rest
3014      * of the state
3015      */
3016     qemu_savevm_send_postcopy_listen(fb);
3017
3018     qemu_savevm_state_complete_precopy(fb, false, false);
3019     if (migrate_postcopy_ram()) {
3020         qemu_savevm_send_ping(fb, 3);
3021     }
3022
3023     qemu_savevm_send_postcopy_run(fb);
3024
3025     /* <><> end of stuff going into the package */
3026
3027     /* Last point of recovery; as soon as we send the package the destination
3028      * can open devices and potentially start running.
3029      * Lets just check again we've not got any errors.
3030      */
3031     ret = qemu_file_get_error(ms->to_dst_file);
3032     if (ret) {
3033         error_report("postcopy_start: Migration stream errored (pre package)");
3034         goto fail_closefb;
3035     }
3036
3037     restart_block = false;
3038
3039     /* Now send that blob */
3040     if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
3041         goto fail_closefb;
3042     }
3043     qemu_fclose(fb);
3044
3045     /* Send a notify to give a chance for anything that needs to happen
3046      * at the transition to postcopy and after the device state; in particular
3047      * spice needs to trigger a transition now
3048      */
3049     ms->postcopy_after_devices = true;
3050     notifier_list_notify(&migration_state_notifiers, ms);
3051
3052     ms->downtime =  qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop;
3053
3054     qemu_mutex_unlock_iothread();
3055
3056     if (migrate_postcopy_ram()) {
3057         /*
3058          * Although this ping is just for debug, it could potentially be
3059          * used for getting a better measurement of downtime at the source.
3060          */
3061         qemu_savevm_send_ping(ms->to_dst_file, 4);
3062     }
3063
3064     if (migrate_release_ram()) {
3065         ram_postcopy_migrated_memory_release(ms);
3066     }
3067
3068     ret = qemu_file_get_error(ms->to_dst_file);
3069     if (ret) {
3070         error_report("postcopy_start: Migration stream errored");
3071         migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3072                               MIGRATION_STATUS_FAILED);
3073     }
3074
3075     return ret;
3076
3077 fail_closefb:
3078     qemu_fclose(fb);
3079 fail:
3080     migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3081                           MIGRATION_STATUS_FAILED);
3082     if (restart_block) {
3083         /* A failure happened early enough that we know the destination hasn't
3084          * accessed block devices, so we're safe to recover.
3085          */
3086         Error *local_err = NULL;
3087
3088         bdrv_invalidate_cache_all(&local_err);
3089         if (local_err) {
3090             error_report_err(local_err);
3091         }
3092     }
3093     qemu_mutex_unlock_iothread();
3094     return -1;
3095 }
3096
3097 /**
3098  * migration_maybe_pause: Pause if required to by
3099  * migrate_pause_before_switchover called with the iothread locked
3100  * Returns: 0 on success
3101  */
3102 static int migration_maybe_pause(MigrationState *s,
3103                                  int *current_active_state,
3104                                  int new_state)
3105 {
3106     if (!migrate_pause_before_switchover()) {
3107         return 0;
3108     }
3109
3110     /* Since leaving this state is not atomic with posting the semaphore
3111      * it's possible that someone could have issued multiple migrate_continue
3112      * and the semaphore is incorrectly positive at this point;
3113      * the docs say it's undefined to reinit a semaphore that's already
3114      * init'd, so use timedwait to eat up any existing posts.
3115      */
3116     while (qemu_sem_timedwait(&s->pause_sem, 1) == 0) {
3117         /* This block intentionally left blank */
3118     }
3119
3120     /*
3121      * If the migration is cancelled when it is in the completion phase,
3122      * the migration state is set to MIGRATION_STATUS_CANCELLING.
3123      * So we don't need to wait a semaphore, otherwise we would always
3124      * wait for the 'pause_sem' semaphore.
3125      */
3126     if (s->state != MIGRATION_STATUS_CANCELLING) {
3127         qemu_mutex_unlock_iothread();
3128         migrate_set_state(&s->state, *current_active_state,
3129                           MIGRATION_STATUS_PRE_SWITCHOVER);
3130         qemu_sem_wait(&s->pause_sem);
3131         migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER,
3132                           new_state);
3133         *current_active_state = new_state;
3134         qemu_mutex_lock_iothread();
3135     }
3136
3137     return s->state == new_state ? 0 : -EINVAL;
3138 }
3139
3140 /**
3141  * migration_completion: Used by migration_thread when there's not much left.
3142  *   The caller 'breaks' the loop when this returns.
3143  *
3144  * @s: Current migration state
3145  */
3146 static void migration_completion(MigrationState *s)
3147 {
3148     int ret;
3149     int current_active_state = s->state;
3150
3151     if (s->state == MIGRATION_STATUS_ACTIVE) {
3152         qemu_mutex_lock_iothread();
3153         s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3154         qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3155         s->vm_was_running = runstate_is_running();
3156         ret = global_state_store();
3157
3158         if (!ret) {
3159             bool inactivate = !migrate_colo_enabled();
3160             ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
3161             if (ret >= 0) {
3162                 ret = migration_maybe_pause(s, &current_active_state,
3163                                             MIGRATION_STATUS_DEVICE);
3164             }
3165             if (ret >= 0) {
3166                 qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3167                 ret = qemu_savevm_state_complete_precopy(s->to_dst_file, false,
3168                                                          inactivate);
3169             }
3170             if (inactivate && ret >= 0) {
3171                 s->block_inactive = true;
3172             }
3173         }
3174         qemu_mutex_unlock_iothread();
3175
3176         if (ret < 0) {
3177             goto fail;
3178         }
3179     } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3180         trace_migration_completion_postcopy_end();
3181
3182         qemu_savevm_state_complete_postcopy(s->to_dst_file);
3183         trace_migration_completion_postcopy_end_after_complete();
3184     } else if (s->state == MIGRATION_STATUS_CANCELLING) {
3185         goto fail;
3186     }
3187
3188     /*
3189      * If rp was opened we must clean up the thread before
3190      * cleaning everything else up (since if there are no failures
3191      * it will wait for the destination to send it's status in
3192      * a SHUT command).
3193      */
3194     if (s->rp_state.from_dst_file) {
3195         int rp_error;
3196         trace_migration_return_path_end_before();
3197         rp_error = await_return_path_close_on_source(s);
3198         trace_migration_return_path_end_after(rp_error);
3199         if (rp_error) {
3200             goto fail_invalidate;
3201         }
3202     }
3203
3204     if (qemu_file_get_error(s->to_dst_file)) {
3205         trace_migration_completion_file_err();
3206         goto fail_invalidate;
3207     }
3208
3209     if (!migrate_colo_enabled()) {
3210         migrate_set_state(&s->state, current_active_state,
3211                           MIGRATION_STATUS_COMPLETED);
3212     }
3213
3214     return;
3215
3216 fail_invalidate:
3217     /* If not doing postcopy, vm_start() will be called: let's regain
3218      * control on images.
3219      */
3220     if (s->state == MIGRATION_STATUS_ACTIVE ||
3221         s->state == MIGRATION_STATUS_DEVICE) {
3222         Error *local_err = NULL;
3223
3224         qemu_mutex_lock_iothread();
3225         bdrv_invalidate_cache_all(&local_err);
3226         if (local_err) {
3227             error_report_err(local_err);
3228         } else {
3229             s->block_inactive = false;
3230         }
3231         qemu_mutex_unlock_iothread();
3232     }
3233
3234 fail:
3235     migrate_set_state(&s->state, current_active_state,
3236                       MIGRATION_STATUS_FAILED);
3237 }
3238
3239 /**
3240  * bg_migration_completion: Used by bg_migration_thread when after all the
3241  *   RAM has been saved. The caller 'breaks' the loop when this returns.
3242  *
3243  * @s: Current migration state
3244  */
3245 static void bg_migration_completion(MigrationState *s)
3246 {
3247     int current_active_state = s->state;
3248
3249     /*
3250      * Stop tracking RAM writes - un-protect memory, un-register UFFD
3251      * memory ranges, flush kernel wait queues and wake up threads
3252      * waiting for write fault to be resolved.
3253      */
3254     ram_write_tracking_stop();
3255
3256     if (s->state == MIGRATION_STATUS_ACTIVE) {
3257         /*
3258          * By this moment we have RAM content saved into the migration stream.
3259          * The next step is to flush the non-RAM content (device state)
3260          * right after the ram content. The device state has been stored into
3261          * the temporary buffer before RAM saving started.
3262          */
3263         qemu_put_buffer(s->to_dst_file, s->bioc->data, s->bioc->usage);
3264         qemu_fflush(s->to_dst_file);
3265     } else if (s->state == MIGRATION_STATUS_CANCELLING) {
3266         goto fail;
3267     }
3268
3269     if (qemu_file_get_error(s->to_dst_file)) {
3270         trace_migration_completion_file_err();
3271         goto fail;
3272     }
3273
3274     migrate_set_state(&s->state, current_active_state,
3275                       MIGRATION_STATUS_COMPLETED);
3276     return;
3277
3278 fail:
3279     migrate_set_state(&s->state, current_active_state,
3280                       MIGRATION_STATUS_FAILED);
3281 }
3282
3283 bool migrate_colo_enabled(void)
3284 {
3285     MigrationState *s = migrate_get_current();
3286     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_COLO];
3287 }
3288
3289 typedef enum MigThrError {
3290     /* No error detected */
3291     MIG_THR_ERR_NONE = 0,
3292     /* Detected error, but resumed successfully */
3293     MIG_THR_ERR_RECOVERED = 1,
3294     /* Detected fatal error, need to exit */
3295     MIG_THR_ERR_FATAL = 2,
3296 } MigThrError;
3297
3298 static int postcopy_resume_handshake(MigrationState *s)
3299 {
3300     qemu_savevm_send_postcopy_resume(s->to_dst_file);
3301
3302     while (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3303         qemu_sem_wait(&s->rp_state.rp_sem);
3304     }
3305
3306     if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3307         return 0;
3308     }
3309
3310     return -1;
3311 }
3312
3313 /* Return zero if success, or <0 for error */
3314 static int postcopy_do_resume(MigrationState *s)
3315 {
3316     int ret;
3317
3318     /*
3319      * Call all the resume_prepare() hooks, so that modules can be
3320      * ready for the migration resume.
3321      */
3322     ret = qemu_savevm_state_resume_prepare(s);
3323     if (ret) {
3324         error_report("%s: resume_prepare() failure detected: %d",
3325                      __func__, ret);
3326         return ret;
3327     }
3328
3329     /*
3330      * Last handshake with destination on the resume (destination will
3331      * switch to postcopy-active afterwards)
3332      */
3333     ret = postcopy_resume_handshake(s);
3334     if (ret) {
3335         error_report("%s: handshake failed: %d", __func__, ret);
3336         return ret;
3337     }
3338
3339     return 0;
3340 }
3341
3342 /*
3343  * We don't return until we are in a safe state to continue current
3344  * postcopy migration.  Returns MIG_THR_ERR_RECOVERED if recovered, or
3345  * MIG_THR_ERR_FATAL if unrecovery failure happened.
3346  */
3347 static MigThrError postcopy_pause(MigrationState *s)
3348 {
3349     assert(s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
3350
3351     while (true) {
3352         QEMUFile *file;
3353
3354         /* Current channel is possibly broken. Release it. */
3355         assert(s->to_dst_file);
3356         qemu_mutex_lock(&s->qemu_file_lock);
3357         file = s->to_dst_file;
3358         s->to_dst_file = NULL;
3359         qemu_mutex_unlock(&s->qemu_file_lock);
3360
3361         qemu_file_shutdown(file);
3362         qemu_fclose(file);
3363
3364         migrate_set_state(&s->state, s->state,
3365                           MIGRATION_STATUS_POSTCOPY_PAUSED);
3366
3367         error_report("Detected IO failure for postcopy. "
3368                      "Migration paused.");
3369
3370         /*
3371          * We wait until things fixed up. Then someone will setup the
3372          * status back for us.
3373          */
3374         while (s->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
3375             qemu_sem_wait(&s->postcopy_pause_sem);
3376         }
3377
3378         if (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3379             /* Woken up by a recover procedure. Give it a shot */
3380
3381             /*
3382              * Firstly, let's wake up the return path now, with a new
3383              * return path channel.
3384              */
3385             qemu_sem_post(&s->postcopy_pause_rp_sem);
3386
3387             /* Do the resume logic */
3388             if (postcopy_do_resume(s) == 0) {
3389                 /* Let's continue! */
3390                 trace_postcopy_pause_continued();
3391                 return MIG_THR_ERR_RECOVERED;
3392             } else {
3393                 /*
3394                  * Something wrong happened during the recovery, let's
3395                  * pause again. Pause is always better than throwing
3396                  * data away.
3397                  */
3398                 continue;
3399             }
3400         } else {
3401             /* This is not right... Time to quit. */
3402             return MIG_THR_ERR_FATAL;
3403         }
3404     }
3405 }
3406
3407 static MigThrError migration_detect_error(MigrationState *s)
3408 {
3409     int ret;
3410     int state = s->state;
3411     Error *local_error = NULL;
3412
3413     if (state == MIGRATION_STATUS_CANCELLING ||
3414         state == MIGRATION_STATUS_CANCELLED) {
3415         /* End the migration, but don't set the state to failed */
3416         return MIG_THR_ERR_FATAL;
3417     }
3418
3419     /* Try to detect any file errors */
3420     ret = qemu_file_get_error_obj(s->to_dst_file, &local_error);
3421     if (!ret) {
3422         /* Everything is fine */
3423         assert(!local_error);
3424         return MIG_THR_ERR_NONE;
3425     }
3426
3427     if (local_error) {
3428         migrate_set_error(s, local_error);
3429         error_free(local_error);
3430     }
3431
3432     if (state == MIGRATION_STATUS_POSTCOPY_ACTIVE && ret == -EIO) {
3433         /*
3434          * For postcopy, we allow the network to be down for a
3435          * while. After that, it can be continued by a
3436          * recovery phase.
3437          */
3438         return postcopy_pause(s);
3439     } else {
3440         /*
3441          * For precopy (or postcopy with error outside IO), we fail
3442          * with no time.
3443          */
3444         migrate_set_state(&s->state, state, MIGRATION_STATUS_FAILED);
3445         trace_migration_thread_file_err();
3446
3447         /* Time to stop the migration, now. */
3448         return MIG_THR_ERR_FATAL;
3449     }
3450 }
3451
3452 /* How many bytes have we transferred since the beginning of the migration */
3453 static uint64_t migration_total_bytes(MigrationState *s)
3454 {
3455     return qemu_ftell(s->to_dst_file) + ram_counters.multifd_bytes;
3456 }
3457
3458 static void migration_calculate_complete(MigrationState *s)
3459 {
3460     uint64_t bytes = migration_total_bytes(s);
3461     int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3462     int64_t transfer_time;
3463
3464     s->total_time = end_time - s->start_time;
3465     if (!s->downtime) {
3466         /*
3467          * It's still not set, so we are precopy migration.  For
3468          * postcopy, downtime is calculated during postcopy_start().
3469          */
3470         s->downtime = end_time - s->downtime_start;
3471     }
3472
3473     transfer_time = s->total_time - s->setup_time;
3474     if (transfer_time) {
3475         s->mbps = ((double) bytes * 8.0) / transfer_time / 1000;
3476     }
3477 }
3478
3479 static void update_iteration_initial_status(MigrationState *s)
3480 {
3481     /*
3482      * Update these three fields at the same time to avoid mismatch info lead
3483      * wrong speed calculation.
3484      */
3485     s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3486     s->iteration_initial_bytes = migration_total_bytes(s);
3487     s->iteration_initial_pages = ram_get_total_transferred_pages();
3488 }
3489
3490 static void migration_update_counters(MigrationState *s,
3491                                       int64_t current_time)
3492 {
3493     uint64_t transferred, transferred_pages, time_spent;
3494     uint64_t current_bytes; /* bytes transferred since the beginning */
3495     double bandwidth;
3496
3497     if (current_time < s->iteration_start_time + BUFFER_DELAY) {
3498         return;
3499     }
3500
3501     current_bytes = migration_total_bytes(s);
3502     transferred = current_bytes - s->iteration_initial_bytes;
3503     time_spent = current_time - s->iteration_start_time;
3504     bandwidth = (double)transferred / time_spent;
3505     s->threshold_size = bandwidth * s->parameters.downtime_limit;
3506
3507     s->mbps = (((double) transferred * 8.0) /
3508                ((double) time_spent / 1000.0)) / 1000.0 / 1000.0;
3509
3510     transferred_pages = ram_get_total_transferred_pages() -
3511                             s->iteration_initial_pages;
3512     s->pages_per_second = (double) transferred_pages /
3513                              (((double) time_spent / 1000.0));
3514
3515     /*
3516      * if we haven't sent anything, we don't want to
3517      * recalculate. 10000 is a small enough number for our purposes
3518      */
3519     if (ram_counters.dirty_pages_rate && transferred > 10000) {
3520         s->expected_downtime = ram_counters.remaining / bandwidth;
3521     }
3522
3523     qemu_file_reset_rate_limit(s->to_dst_file);
3524
3525     update_iteration_initial_status(s);
3526
3527     trace_migrate_transferred(transferred, time_spent,
3528                               bandwidth, s->threshold_size);
3529 }
3530
3531 /* Migration thread iteration status */
3532 typedef enum {
3533     MIG_ITERATE_RESUME,         /* Resume current iteration */
3534     MIG_ITERATE_SKIP,           /* Skip current iteration */
3535     MIG_ITERATE_BREAK,          /* Break the loop */
3536 } MigIterateState;
3537
3538 /*
3539  * Return true if continue to the next iteration directly, false
3540  * otherwise.
3541  */
3542 static MigIterateState migration_iteration_run(MigrationState *s)
3543 {
3544     uint64_t pending_size, pend_pre, pend_compat, pend_post;
3545     bool in_postcopy = s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE;
3546
3547     qemu_savevm_state_pending(s->to_dst_file, s->threshold_size, &pend_pre,
3548                               &pend_compat, &pend_post);
3549     pending_size = pend_pre + pend_compat + pend_post;
3550
3551     trace_migrate_pending(pending_size, s->threshold_size,
3552                           pend_pre, pend_compat, pend_post);
3553
3554     if (pending_size && pending_size >= s->threshold_size) {
3555         /* Still a significant amount to transfer */
3556         if (!in_postcopy && pend_pre <= s->threshold_size &&
3557             qatomic_read(&s->start_postcopy)) {
3558             if (postcopy_start(s)) {
3559                 error_report("%s: postcopy failed to start", __func__);
3560             }
3561             return MIG_ITERATE_SKIP;
3562         }
3563         /* Just another iteration step */
3564         qemu_savevm_state_iterate(s->to_dst_file, in_postcopy);
3565     } else {
3566         trace_migration_thread_low_pending(pending_size);
3567         migration_completion(s);
3568         return MIG_ITERATE_BREAK;
3569     }
3570
3571     return MIG_ITERATE_RESUME;
3572 }
3573
3574 static void migration_iteration_finish(MigrationState *s)
3575 {
3576     /* If we enabled cpu throttling for auto-converge, turn it off. */
3577     cpu_throttle_stop();
3578
3579     qemu_mutex_lock_iothread();
3580     switch (s->state) {
3581     case MIGRATION_STATUS_COMPLETED:
3582         migration_calculate_complete(s);
3583         runstate_set(RUN_STATE_POSTMIGRATE);
3584         break;
3585
3586     case MIGRATION_STATUS_ACTIVE:
3587         /*
3588          * We should really assert here, but since it's during
3589          * migration, let's try to reduce the usage of assertions.
3590          */
3591         if (!migrate_colo_enabled()) {
3592             error_report("%s: critical error: calling COLO code without "
3593                          "COLO enabled", __func__);
3594         }
3595         migrate_start_colo_process(s);
3596         /*
3597          * Fixme: we will run VM in COLO no matter its old running state.
3598          * After exited COLO, we will keep running.
3599          */
3600         s->vm_was_running = true;
3601         /* Fallthrough */
3602     case MIGRATION_STATUS_FAILED:
3603     case MIGRATION_STATUS_CANCELLED:
3604     case MIGRATION_STATUS_CANCELLING:
3605         if (s->vm_was_running) {
3606             vm_start();
3607         } else {
3608             if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
3609                 runstate_set(RUN_STATE_POSTMIGRATE);
3610             }
3611         }
3612         break;
3613
3614     default:
3615         /* Should not reach here, but if so, forgive the VM. */
3616         error_report("%s: Unknown ending state %d", __func__, s->state);
3617         break;
3618     }
3619     migrate_fd_cleanup_schedule(s);
3620     qemu_mutex_unlock_iothread();
3621 }
3622
3623 static void bg_migration_iteration_finish(MigrationState *s)
3624 {
3625     qemu_mutex_lock_iothread();
3626     switch (s->state) {
3627     case MIGRATION_STATUS_COMPLETED:
3628         migration_calculate_complete(s);
3629         break;
3630
3631     case MIGRATION_STATUS_ACTIVE:
3632     case MIGRATION_STATUS_FAILED:
3633     case MIGRATION_STATUS_CANCELLED:
3634     case MIGRATION_STATUS_CANCELLING:
3635         break;
3636
3637     default:
3638         /* Should not reach here, but if so, forgive the VM. */
3639         error_report("%s: Unknown ending state %d", __func__, s->state);
3640         break;
3641     }
3642
3643     migrate_fd_cleanup_schedule(s);
3644     qemu_mutex_unlock_iothread();
3645 }
3646
3647 /*
3648  * Return true if continue to the next iteration directly, false
3649  * otherwise.
3650  */
3651 static MigIterateState bg_migration_iteration_run(MigrationState *s)
3652 {
3653     int res;
3654
3655     res = qemu_savevm_state_iterate(s->to_dst_file, false);
3656     if (res > 0) {
3657         bg_migration_completion(s);
3658         return MIG_ITERATE_BREAK;
3659     }
3660
3661     return MIG_ITERATE_RESUME;
3662 }
3663
3664 void migration_make_urgent_request(void)
3665 {
3666     qemu_sem_post(&migrate_get_current()->rate_limit_sem);
3667 }
3668
3669 void migration_consume_urgent_request(void)
3670 {
3671     qemu_sem_wait(&migrate_get_current()->rate_limit_sem);
3672 }
3673
3674 /* Returns true if the rate limiting was broken by an urgent request */
3675 bool migration_rate_limit(void)
3676 {
3677     int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3678     MigrationState *s = migrate_get_current();
3679
3680     bool urgent = false;
3681     migration_update_counters(s, now);
3682     if (qemu_file_rate_limit(s->to_dst_file)) {
3683
3684         if (qemu_file_get_error(s->to_dst_file)) {
3685             return false;
3686         }
3687         /*
3688          * Wait for a delay to do rate limiting OR
3689          * something urgent to post the semaphore.
3690          */
3691         int ms = s->iteration_start_time + BUFFER_DELAY - now;
3692         trace_migration_rate_limit_pre(ms);
3693         if (qemu_sem_timedwait(&s->rate_limit_sem, ms) == 0) {
3694             /*
3695              * We were woken by one or more urgent things but
3696              * the timedwait will have consumed one of them.
3697              * The service routine for the urgent wake will dec
3698              * the semaphore itself for each item it consumes,
3699              * so add this one we just eat back.
3700              */
3701             qemu_sem_post(&s->rate_limit_sem);
3702             urgent = true;
3703         }
3704         trace_migration_rate_limit_post(urgent);
3705     }
3706     return urgent;
3707 }
3708
3709 /*
3710  * Master migration thread on the source VM.
3711  * It drives the migration and pumps the data down the outgoing channel.
3712  */
3713 static void *migration_thread(void *opaque)
3714 {
3715     MigrationState *s = opaque;
3716     int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3717     MigThrError thr_error;
3718     bool urgent = false;
3719
3720     rcu_register_thread();
3721
3722     object_ref(OBJECT(s));
3723     update_iteration_initial_status(s);
3724
3725     qemu_savevm_state_header(s->to_dst_file);
3726
3727     /*
3728      * If we opened the return path, we need to make sure dst has it
3729      * opened as well.
3730      */
3731     if (s->rp_state.from_dst_file) {
3732         /* Now tell the dest that it should open its end so it can reply */
3733         qemu_savevm_send_open_return_path(s->to_dst_file);
3734
3735         /* And do a ping that will make stuff easier to debug */
3736         qemu_savevm_send_ping(s->to_dst_file, 1);
3737     }
3738
3739     if (migrate_postcopy()) {
3740         /*
3741          * Tell the destination that we *might* want to do postcopy later;
3742          * if the other end can't do postcopy it should fail now, nice and
3743          * early.
3744          */
3745         qemu_savevm_send_postcopy_advise(s->to_dst_file);
3746     }
3747
3748     if (migrate_colo_enabled()) {
3749         /* Notify migration destination that we enable COLO */
3750         qemu_savevm_send_colo_enable(s->to_dst_file);
3751     }
3752
3753     qemu_savevm_state_setup(s->to_dst_file);
3754
3755     if (qemu_savevm_state_guest_unplug_pending()) {
3756         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3757                           MIGRATION_STATUS_WAIT_UNPLUG);
3758
3759         while (s->state == MIGRATION_STATUS_WAIT_UNPLUG &&
3760                qemu_savevm_state_guest_unplug_pending()) {
3761             qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3762         }
3763
3764         migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG,
3765                 MIGRATION_STATUS_ACTIVE);
3766     }
3767
3768     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3769     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3770                       MIGRATION_STATUS_ACTIVE);
3771
3772     trace_migration_thread_setup_complete();
3773
3774     while (migration_is_active(s)) {
3775         if (urgent || !qemu_file_rate_limit(s->to_dst_file)) {
3776             MigIterateState iter_state = migration_iteration_run(s);
3777             if (iter_state == MIG_ITERATE_SKIP) {
3778                 continue;
3779             } else if (iter_state == MIG_ITERATE_BREAK) {
3780                 break;
3781             }
3782         }
3783
3784         /*
3785          * Try to detect any kind of failures, and see whether we
3786          * should stop the migration now.
3787          */
3788         thr_error = migration_detect_error(s);
3789         if (thr_error == MIG_THR_ERR_FATAL) {
3790             /* Stop migration */
3791             break;
3792         } else if (thr_error == MIG_THR_ERR_RECOVERED) {
3793             /*
3794              * Just recovered from a e.g. network failure, reset all
3795              * the local variables. This is important to avoid
3796              * breaking transferred_bytes and bandwidth calculation
3797              */
3798             update_iteration_initial_status(s);
3799         }
3800
3801         urgent = migration_rate_limit();
3802     }
3803
3804     trace_migration_thread_after_loop();
3805     migration_iteration_finish(s);
3806     object_unref(OBJECT(s));
3807     rcu_unregister_thread();
3808     return NULL;
3809 }
3810
3811 static void bg_migration_vm_start_bh(void *opaque)
3812 {
3813     MigrationState *s = opaque;
3814
3815     qemu_bh_delete(s->vm_start_bh);
3816     s->vm_start_bh = NULL;
3817
3818     vm_start();
3819     s->downtime = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - s->downtime_start;
3820 }
3821
3822 /**
3823  * Background snapshot thread, based on live migration code.
3824  * This is an alternative implementation of live migration mechanism
3825  * introduced specifically to support background snapshots.
3826  *
3827  * It takes advantage of userfault_fd write protection mechanism introduced
3828  * in v5.7 kernel. Compared to existing dirty page logging migration much
3829  * lesser stream traffic is produced resulting in smaller snapshot images,
3830  * simply cause of no page duplicates can get into the stream.
3831  *
3832  * Another key point is that generated vmstate stream reflects machine state
3833  * 'frozen' at the beginning of snapshot creation compared to dirty page logging
3834  * mechanism, which effectively results in that saved snapshot is the state of VM
3835  * at the end of the process.
3836  */
3837 static void *bg_migration_thread(void *opaque)
3838 {
3839     MigrationState *s = opaque;
3840     int64_t setup_start;
3841     MigThrError thr_error;
3842     QEMUFile *fb;
3843     bool early_fail = true;
3844
3845     rcu_register_thread();
3846     object_ref(OBJECT(s));
3847
3848     qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3849
3850     setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3851     /*
3852      * We want to save vmstate for the moment when migration has been
3853      * initiated but also we want to save RAM content while VM is running.
3854      * The RAM content should appear first in the vmstate. So, we first
3855      * stash the non-RAM part of the vmstate to the temporary buffer,
3856      * then write RAM part of the vmstate to the migration stream
3857      * with vCPUs running and, finally, write stashed non-RAM part of
3858      * the vmstate from the buffer to the migration stream.
3859      */
3860     s->bioc = qio_channel_buffer_new(128 * 1024);
3861     qio_channel_set_name(QIO_CHANNEL(s->bioc), "vmstate-buffer");
3862     fb = qemu_fopen_channel_output(QIO_CHANNEL(s->bioc));
3863     object_unref(OBJECT(s->bioc));
3864
3865     update_iteration_initial_status(s);
3866
3867     qemu_savevm_state_header(s->to_dst_file);
3868     qemu_savevm_state_setup(s->to_dst_file);
3869
3870     if (qemu_savevm_state_guest_unplug_pending()) {
3871         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3872                           MIGRATION_STATUS_WAIT_UNPLUG);
3873
3874         while (s->state == MIGRATION_STATUS_WAIT_UNPLUG &&
3875                qemu_savevm_state_guest_unplug_pending()) {
3876             qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3877         }
3878
3879         migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG,
3880                           MIGRATION_STATUS_ACTIVE);
3881     } else {
3882         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3883                 MIGRATION_STATUS_ACTIVE);
3884     }
3885     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3886
3887     trace_migration_thread_setup_complete();
3888     s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3889
3890     qemu_mutex_lock_iothread();
3891
3892     /*
3893      * If VM is currently in suspended state, then, to make a valid runstate
3894      * transition in vm_stop_force_state() we need to wakeup it up.
3895      */
3896     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3897     s->vm_was_running = runstate_is_running();
3898
3899     if (global_state_store()) {
3900         goto fail;
3901     }
3902     /* Forcibly stop VM before saving state of vCPUs and devices */
3903     if (vm_stop_force_state(RUN_STATE_PAUSED)) {
3904         goto fail;
3905     }
3906     /*
3907      * Put vCPUs in sync with shadow context structures, then
3908      * save their state to channel-buffer along with devices.
3909      */
3910     cpu_synchronize_all_states();
3911     if (qemu_savevm_state_complete_precopy_non_iterable(fb, false, false)) {
3912         goto fail;
3913     }
3914     /* Now initialize UFFD context and start tracking RAM writes */
3915     if (ram_write_tracking_start()) {
3916         goto fail;
3917     }
3918     early_fail = false;
3919
3920     /*
3921      * Start VM from BH handler to avoid write-fault lock here.
3922      * UFFD-WP protection for the whole RAM is already enabled so
3923      * calling VM state change notifiers from vm_start() would initiate
3924      * writes to virtio VQs memory which is in write-protected region.
3925      */
3926     s->vm_start_bh = qemu_bh_new(bg_migration_vm_start_bh, s);
3927     qemu_bh_schedule(s->vm_start_bh);
3928
3929     qemu_mutex_unlock_iothread();
3930
3931     while (migration_is_active(s)) {
3932         MigIterateState iter_state = bg_migration_iteration_run(s);
3933         if (iter_state == MIG_ITERATE_SKIP) {
3934             continue;
3935         } else if (iter_state == MIG_ITERATE_BREAK) {
3936             break;
3937         }
3938
3939         /*
3940          * Try to detect any kind of failures, and see whether we
3941          * should stop the migration now.
3942          */
3943         thr_error = migration_detect_error(s);
3944         if (thr_error == MIG_THR_ERR_FATAL) {
3945             /* Stop migration */
3946             break;
3947         }
3948
3949         migration_update_counters(s, qemu_clock_get_ms(QEMU_CLOCK_REALTIME));
3950     }
3951
3952     trace_migration_thread_after_loop();
3953
3954 fail:
3955     if (early_fail) {
3956         migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE,
3957                 MIGRATION_STATUS_FAILED);
3958         qemu_mutex_unlock_iothread();
3959     }
3960
3961     bg_migration_iteration_finish(s);
3962
3963     qemu_fclose(fb);
3964     object_unref(OBJECT(s));
3965     rcu_unregister_thread();
3966
3967     return NULL;
3968 }
3969
3970 void migrate_fd_connect(MigrationState *s, Error *error_in)
3971 {
3972     Error *local_err = NULL;
3973     int64_t rate_limit;
3974     bool resume = s->state == MIGRATION_STATUS_POSTCOPY_PAUSED;
3975
3976     s->expected_downtime = s->parameters.downtime_limit;
3977     if (resume) {
3978         assert(s->cleanup_bh);
3979     } else {
3980         assert(!s->cleanup_bh);
3981         s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup_bh, s);
3982     }
3983     if (error_in) {
3984         migrate_fd_error(s, error_in);
3985         migrate_fd_cleanup(s);
3986         return;
3987     }
3988
3989     if (resume) {
3990         /* This is a resumed migration */
3991         rate_limit = s->parameters.max_postcopy_bandwidth /
3992             XFER_LIMIT_RATIO;
3993     } else {
3994         /* This is a fresh new migration */
3995         rate_limit = s->parameters.max_bandwidth / XFER_LIMIT_RATIO;
3996
3997         /* Notify before starting migration thread */
3998         notifier_list_notify(&migration_state_notifiers, s);
3999     }
4000
4001     qemu_file_set_rate_limit(s->to_dst_file, rate_limit);
4002     qemu_file_set_blocking(s->to_dst_file, true);
4003
4004     /*
4005      * Open the return path. For postcopy, it is used exclusively. For
4006      * precopy, only if user specified "return-path" capability would
4007      * QEMU uses the return path.
4008      */
4009     if (migrate_postcopy_ram() || migrate_use_return_path()) {
4010         if (open_return_path_on_source(s, !resume)) {
4011             error_report("Unable to open return-path for postcopy");
4012             migrate_set_state(&s->state, s->state, MIGRATION_STATUS_FAILED);
4013             migrate_fd_cleanup(s);
4014             return;
4015         }
4016     }
4017
4018     if (resume) {
4019         /* Wakeup the main migration thread to do the recovery */
4020         migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
4021                           MIGRATION_STATUS_POSTCOPY_RECOVER);
4022         qemu_sem_post(&s->postcopy_pause_sem);
4023         return;
4024     }
4025
4026     if (multifd_save_setup(&local_err) != 0) {
4027         error_report_err(local_err);
4028         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
4029                           MIGRATION_STATUS_FAILED);
4030         migrate_fd_cleanup(s);
4031         return;
4032     }
4033
4034     if (migrate_background_snapshot()) {
4035         qemu_thread_create(&s->thread, "bg_snapshot",
4036                 bg_migration_thread, s, QEMU_THREAD_JOINABLE);
4037     } else {
4038         qemu_thread_create(&s->thread, "live_migration",
4039                 migration_thread, s, QEMU_THREAD_JOINABLE);
4040     }
4041     s->migration_thread_running = true;
4042 }
4043
4044 void migration_global_dump(Monitor *mon)
4045 {
4046     MigrationState *ms = migrate_get_current();
4047
4048     monitor_printf(mon, "globals:\n");
4049     monitor_printf(mon, "store-global-state: %s\n",
4050                    ms->store_global_state ? "on" : "off");
4051     monitor_printf(mon, "only-migratable: %s\n",
4052                    only_migratable ? "on" : "off");
4053     monitor_printf(mon, "send-configuration: %s\n",
4054                    ms->send_configuration ? "on" : "off");
4055     monitor_printf(mon, "send-section-footer: %s\n",
4056                    ms->send_section_footer ? "on" : "off");
4057     monitor_printf(mon, "decompress-error-check: %s\n",
4058                    ms->decompress_error_check ? "on" : "off");
4059     monitor_printf(mon, "clear-bitmap-shift: %u\n",
4060                    ms->clear_bitmap_shift);
4061 }
4062
4063 #define DEFINE_PROP_MIG_CAP(name, x)             \
4064     DEFINE_PROP_BOOL(name, MigrationState, enabled_capabilities[x], false)
4065
4066 static Property migration_properties[] = {
4067     DEFINE_PROP_BOOL("store-global-state", MigrationState,
4068                      store_global_state, true),
4069     DEFINE_PROP_BOOL("send-configuration", MigrationState,
4070                      send_configuration, true),
4071     DEFINE_PROP_BOOL("send-section-footer", MigrationState,
4072                      send_section_footer, true),
4073     DEFINE_PROP_BOOL("decompress-error-check", MigrationState,
4074                       decompress_error_check, true),
4075     DEFINE_PROP_UINT8("x-clear-bitmap-shift", MigrationState,
4076                       clear_bitmap_shift, CLEAR_BITMAP_SHIFT_DEFAULT),
4077
4078     /* Migration parameters */
4079     DEFINE_PROP_UINT8("x-compress-level", MigrationState,
4080                       parameters.compress_level,
4081                       DEFAULT_MIGRATE_COMPRESS_LEVEL),
4082     DEFINE_PROP_UINT8("x-compress-threads", MigrationState,
4083                       parameters.compress_threads,
4084                       DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT),
4085     DEFINE_PROP_BOOL("x-compress-wait-thread", MigrationState,
4086                       parameters.compress_wait_thread, true),
4087     DEFINE_PROP_UINT8("x-decompress-threads", MigrationState,
4088                       parameters.decompress_threads,
4089                       DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT),
4090     DEFINE_PROP_UINT8("x-throttle-trigger-threshold", MigrationState,
4091                       parameters.throttle_trigger_threshold,
4092                       DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD),
4093     DEFINE_PROP_UINT8("x-cpu-throttle-initial", MigrationState,
4094                       parameters.cpu_throttle_initial,
4095                       DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL),
4096     DEFINE_PROP_UINT8("x-cpu-throttle-increment", MigrationState,
4097                       parameters.cpu_throttle_increment,
4098                       DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT),
4099     DEFINE_PROP_BOOL("x-cpu-throttle-tailslow", MigrationState,
4100                       parameters.cpu_throttle_tailslow, false),
4101     DEFINE_PROP_SIZE("x-max-bandwidth", MigrationState,
4102                       parameters.max_bandwidth, MAX_THROTTLE),
4103     DEFINE_PROP_UINT64("x-downtime-limit", MigrationState,
4104                       parameters.downtime_limit,
4105                       DEFAULT_MIGRATE_SET_DOWNTIME),
4106     DEFINE_PROP_UINT32("x-checkpoint-delay", MigrationState,
4107                       parameters.x_checkpoint_delay,
4108                       DEFAULT_MIGRATE_X_CHECKPOINT_DELAY),
4109     DEFINE_PROP_UINT8("multifd-channels", MigrationState,
4110                       parameters.multifd_channels,
4111                       DEFAULT_MIGRATE_MULTIFD_CHANNELS),
4112     DEFINE_PROP_MULTIFD_COMPRESSION("multifd-compression", MigrationState,
4113                       parameters.multifd_compression,
4114                       DEFAULT_MIGRATE_MULTIFD_COMPRESSION),
4115     DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState,
4116                       parameters.multifd_zlib_level,
4117                       DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL),
4118     DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState,
4119                       parameters.multifd_zstd_level,
4120                       DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL),
4121     DEFINE_PROP_SIZE("xbzrle-cache-size", MigrationState,
4122                       parameters.xbzrle_cache_size,
4123                       DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE),
4124     DEFINE_PROP_SIZE("max-postcopy-bandwidth", MigrationState,
4125                       parameters.max_postcopy_bandwidth,
4126                       DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH),
4127     DEFINE_PROP_UINT8("max-cpu-throttle", MigrationState,
4128                       parameters.max_cpu_throttle,
4129                       DEFAULT_MIGRATE_MAX_CPU_THROTTLE),
4130     DEFINE_PROP_SIZE("announce-initial", MigrationState,
4131                       parameters.announce_initial,
4132                       DEFAULT_MIGRATE_ANNOUNCE_INITIAL),
4133     DEFINE_PROP_SIZE("announce-max", MigrationState,
4134                       parameters.announce_max,
4135                       DEFAULT_MIGRATE_ANNOUNCE_MAX),
4136     DEFINE_PROP_SIZE("announce-rounds", MigrationState,
4137                       parameters.announce_rounds,
4138                       DEFAULT_MIGRATE_ANNOUNCE_ROUNDS),
4139     DEFINE_PROP_SIZE("announce-step", MigrationState,
4140                       parameters.announce_step,
4141                       DEFAULT_MIGRATE_ANNOUNCE_STEP),
4142
4143     /* Migration capabilities */
4144     DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
4145     DEFINE_PROP_MIG_CAP("x-rdma-pin-all", MIGRATION_CAPABILITY_RDMA_PIN_ALL),
4146     DEFINE_PROP_MIG_CAP("x-auto-converge", MIGRATION_CAPABILITY_AUTO_CONVERGE),
4147     DEFINE_PROP_MIG_CAP("x-zero-blocks", MIGRATION_CAPABILITY_ZERO_BLOCKS),
4148     DEFINE_PROP_MIG_CAP("x-compress", MIGRATION_CAPABILITY_COMPRESS),
4149     DEFINE_PROP_MIG_CAP("x-events", MIGRATION_CAPABILITY_EVENTS),
4150     DEFINE_PROP_MIG_CAP("x-postcopy-ram", MIGRATION_CAPABILITY_POSTCOPY_RAM),
4151     DEFINE_PROP_MIG_CAP("x-colo", MIGRATION_CAPABILITY_X_COLO),
4152     DEFINE_PROP_MIG_CAP("x-release-ram", MIGRATION_CAPABILITY_RELEASE_RAM),
4153     DEFINE_PROP_MIG_CAP("x-block", MIGRATION_CAPABILITY_BLOCK),
4154     DEFINE_PROP_MIG_CAP("x-return-path", MIGRATION_CAPABILITY_RETURN_PATH),
4155     DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD),
4156     DEFINE_PROP_MIG_CAP("x-background-snapshot",
4157             MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT),
4158
4159     DEFINE_PROP_END_OF_LIST(),
4160 };
4161
4162 static void migration_class_init(ObjectClass *klass, void *data)
4163 {
4164     DeviceClass *dc = DEVICE_CLASS(klass);
4165
4166     dc->user_creatable = false;
4167     device_class_set_props(dc, migration_properties);
4168 }
4169
4170 static void migration_instance_finalize(Object *obj)
4171 {
4172     MigrationState *ms = MIGRATION_OBJ(obj);
4173     MigrationParameters *params = &ms->parameters;
4174
4175     qemu_mutex_destroy(&ms->error_mutex);
4176     qemu_mutex_destroy(&ms->qemu_file_lock);
4177     g_free(params->tls_hostname);
4178     g_free(params->tls_creds);
4179     qemu_sem_destroy(&ms->wait_unplug_sem);
4180     qemu_sem_destroy(&ms->rate_limit_sem);
4181     qemu_sem_destroy(&ms->pause_sem);
4182     qemu_sem_destroy(&ms->postcopy_pause_sem);
4183     qemu_sem_destroy(&ms->postcopy_pause_rp_sem);
4184     qemu_sem_destroy(&ms->rp_state.rp_sem);
4185     error_free(ms->error);
4186 }
4187
4188 static void migration_instance_init(Object *obj)
4189 {
4190     MigrationState *ms = MIGRATION_OBJ(obj);
4191     MigrationParameters *params = &ms->parameters;
4192
4193     ms->state = MIGRATION_STATUS_NONE;
4194     ms->mbps = -1;
4195     ms->pages_per_second = -1;
4196     qemu_sem_init(&ms->pause_sem, 0);
4197     qemu_mutex_init(&ms->error_mutex);
4198
4199     params->tls_hostname = g_strdup("");
4200     params->tls_creds = g_strdup("");
4201
4202     /* Set has_* up only for parameter checks */
4203     params->has_compress_level = true;
4204     params->has_compress_threads = true;
4205     params->has_decompress_threads = true;
4206     params->has_throttle_trigger_threshold = true;
4207     params->has_cpu_throttle_initial = true;
4208     params->has_cpu_throttle_increment = true;
4209     params->has_cpu_throttle_tailslow = true;
4210     params->has_max_bandwidth = true;
4211     params->has_downtime_limit = true;
4212     params->has_x_checkpoint_delay = true;
4213     params->has_block_incremental = true;
4214     params->has_multifd_channels = true;
4215     params->has_multifd_compression = true;
4216     params->has_multifd_zlib_level = true;
4217     params->has_multifd_zstd_level = true;
4218     params->has_xbzrle_cache_size = true;
4219     params->has_max_postcopy_bandwidth = true;
4220     params->has_max_cpu_throttle = true;
4221     params->has_announce_initial = true;
4222     params->has_announce_max = true;
4223     params->has_announce_rounds = true;
4224     params->has_announce_step = true;
4225
4226     qemu_sem_init(&ms->postcopy_pause_sem, 0);
4227     qemu_sem_init(&ms->postcopy_pause_rp_sem, 0);
4228     qemu_sem_init(&ms->rp_state.rp_sem, 0);
4229     qemu_sem_init(&ms->rate_limit_sem, 0);
4230     qemu_sem_init(&ms->wait_unplug_sem, 0);
4231     qemu_mutex_init(&ms->qemu_file_lock);
4232 }
4233
4234 /*
4235  * Return true if check pass, false otherwise. Error will be put
4236  * inside errp if provided.
4237  */
4238 static bool migration_object_check(MigrationState *ms, Error **errp)
4239 {
4240     MigrationCapabilityStatusList *head = NULL;
4241     /* Assuming all off */
4242     bool cap_list[MIGRATION_CAPABILITY__MAX] = { 0 }, ret;
4243     int i;
4244
4245     if (!migrate_params_check(&ms->parameters, errp)) {
4246         return false;
4247     }
4248
4249     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
4250         if (ms->enabled_capabilities[i]) {
4251             QAPI_LIST_PREPEND(head, migrate_cap_add(i, true));
4252         }
4253     }
4254
4255     ret = migrate_caps_check(cap_list, head, errp);
4256
4257     /* It works with head == NULL */
4258     qapi_free_MigrationCapabilityStatusList(head);
4259
4260     return ret;
4261 }
4262
4263 static const TypeInfo migration_type = {
4264     .name = TYPE_MIGRATION,
4265     /*
4266      * NOTE: TYPE_MIGRATION is not really a device, as the object is
4267      * not created using qdev_new(), it is not attached to the qdev
4268      * device tree, and it is never realized.
4269      *
4270      * TODO: Make this TYPE_OBJECT once QOM provides something like
4271      * TYPE_DEVICE's "-global" properties.
4272      */
4273     .parent = TYPE_DEVICE,
4274     .class_init = migration_class_init,
4275     .class_size = sizeof(MigrationClass),
4276     .instance_size = sizeof(MigrationState),
4277     .instance_init = migration_instance_init,
4278     .instance_finalize = migration_instance_finalize,
4279 };
4280
4281 static void register_migration_types(void)
4282 {
4283     type_register_static(&migration_type);
4284 }
4285
4286 type_init(register_migration_types);