migration/migration.c

   1 /*
   2  * QEMU live migration
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *
   6  * Authors:
   7  *  Anthony Liguori   <aliguori@us.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include "qemu/cutils.h"
  18 #include "qemu/error-report.h"
  19 #include "qemu/main-loop.h"
  20 #include "migration/blocker.h"
  21 #include "exec.h"
  22 #include "fd.h"
  23 #include "socket.h"
  24 #include "sysemu/runstate.h"
  25 #include "sysemu/sysemu.h"
  26 #include "sysemu/cpu-throttle.h"
  27 #include "rdma.h"
  28 #include "ram.h"
  29 #include "migration/global_state.h"
  30 #include "migration/misc.h"
  31 #include "migration.h"
  32 #include "savevm.h"
  33 #include "qemu-file-channel.h"
  34 #include "qemu-file.h"
  35 #include "migration/vmstate.h"
  36 #include "block/block.h"
  37 #include "qapi/error.h"
  38 #include "qapi/clone-visitor.h"
  39 #include "qapi/qapi-visit-migration.h"
  40 #include "qapi/qapi-visit-sockets.h"
  41 #include "qapi/qapi-commands-migration.h"
  42 #include "qapi/qapi-events-migration.h"
  43 #include "qapi/qmp/qerror.h"
  44 #include "qapi/qmp/qnull.h"
  45 #include "qemu/rcu.h"
  46 #include "block.h"
  47 #include "postcopy-ram.h"
  48 #include "qemu/thread.h"
  49 #include "trace.h"
  50 #include "exec/target_page.h"
  51 #include "io/channel-buffer.h"
  52 #include "migration/colo.h"
  53 #include "hw/boards.h"
  54 #include "hw/qdev-properties.h"
  55 #include "hw/qdev-properties-system.h"
  56 #include "monitor/monitor.h"
  57 #include "net/announce.h"
  58 #include "qemu/queue.h"
  59 #include "multifd.h"
  60 #include "qemu/yank.h"
  61 #include "sysemu/cpus.h"
  62 #include "yank_functions.h"
  63 #include "sysemu/qtest.h"
  64
  65 #define MAX_THROTTLE  (128 << 20)      /* Migration transfer speed throttling */
  66
  67 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
  68  * data. */
  69 #define BUFFER_DELAY     100
  70 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
  71
  72 /* Time in milliseconds we are allowed to stop the source,
  73  * for sending the last part */
  74 #define DEFAULT_MIGRATE_SET_DOWNTIME 300
  75
  76 /* Maximum migrate downtime set to 2000 seconds */
  77 #define MAX_MIGRATE_DOWNTIME_SECONDS 2000
  78 #define MAX_MIGRATE_DOWNTIME (MAX_MIGRATE_DOWNTIME_SECONDS * 1000)
  79
  80 /* Default compression thread count */
  81 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8
  82 /* Default decompression thread count, usually decompression is at
  83  * least 4 times as fast as compression.*/
  84 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2
  85 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */
  86 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1
  87 /* Define default autoconverge cpu throttle migration parameters */
  88 #define DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD 50
  89 #define DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL 20
  90 #define DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT 10
  91 #define DEFAULT_MIGRATE_MAX_CPU_THROTTLE 99
  92
  93 /* Migration XBZRLE default cache size */
  94 #define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024)
  95
  96 /* The delay time (in ms) between two COLO checkpoints */
  97 #define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100)
  98 #define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2
  99 #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE
 100 /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */
 101 #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1
 102 /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */
 103 #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1
 104
 105 /* Background transfer rate for postcopy, 0 means unlimited, note
 106  * that page requests can still exceed this limit.
 107  */
 108 #define DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH 0
 109
 110 /*
 111  * Parameters for self_announce_delay giving a stream of RARP/ARP
 112  * packets after migration.
 113  */
 114 #define DEFAULT_MIGRATE_ANNOUNCE_INITIAL  50
 115 #define DEFAULT_MIGRATE_ANNOUNCE_MAX     550
 116 #define DEFAULT_MIGRATE_ANNOUNCE_ROUNDS    5
 117 #define DEFAULT_MIGRATE_ANNOUNCE_STEP    100
 118
 119 static NotifierList migration_state_notifiers =
 120     NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
 121
 122 /* Messages sent on the return path from destination to source */
 123 enum mig_rp_message_type {
 124     MIG_RP_MSG_INVALID = 0,  /* Must be 0 */
 125     MIG_RP_MSG_SHUT,         /* sibling will not send any more RP messages */
 126     MIG_RP_MSG_PONG,         /* Response to a PING; data (seq: be32 ) */
 127
 128     MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
 129     MIG_RP_MSG_REQ_PAGES,    /* data (start: be64, len: be32) */
 130     MIG_RP_MSG_RECV_BITMAP,  /* send recved_bitmap back to source */
 131     MIG_RP_MSG_RESUME_ACK,   /* tell source that we are ready to resume */
 132
 133     MIG_RP_MSG_MAX
 134 };
 135
 136 /* Migration capabilities set */
 137 struct MigrateCapsSet {
 138     int size;                       /* Capability set size */
 139     MigrationCapability caps[];     /* Variadic array of capabilities */
 140 };
 141 typedef struct MigrateCapsSet MigrateCapsSet;
 142
 143 /* Define and initialize MigrateCapsSet */
 144 #define INITIALIZE_MIGRATE_CAPS_SET(_name, ...)   \
 145     MigrateCapsSet _name = {    \
 146         .size = sizeof((int []) { __VA_ARGS__ }) / sizeof(int), \
 147         .caps = { __VA_ARGS__ } \
 148     }
 149
 150 /* Background-snapshot compatibility check list */
 151 static const
 152 INITIALIZE_MIGRATE_CAPS_SET(check_caps_background_snapshot,
 153     MIGRATION_CAPABILITY_POSTCOPY_RAM,
 154     MIGRATION_CAPABILITY_DIRTY_BITMAPS,
 155     MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME,
 156     MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE,
 157     MIGRATION_CAPABILITY_RETURN_PATH,
 158     MIGRATION_CAPABILITY_MULTIFD,
 159     MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER,
 160     MIGRATION_CAPABILITY_AUTO_CONVERGE,
 161     MIGRATION_CAPABILITY_RELEASE_RAM,
 162     MIGRATION_CAPABILITY_RDMA_PIN_ALL,
 163     MIGRATION_CAPABILITY_COMPRESS,
 164     MIGRATION_CAPABILITY_XBZRLE,
 165     MIGRATION_CAPABILITY_X_COLO,
 166     MIGRATION_CAPABILITY_VALIDATE_UUID);
 167
 168 /* When we add fault tolerance, we could have several
 169    migrations at once.  For now we don't need to add
 170    dynamic creation of migration */
 171
 172 static MigrationState *current_migration;
 173 static MigrationIncomingState *current_incoming;
 174
 175 static GSList *migration_blockers;
 176
 177 static bool migration_object_check(MigrationState *ms, Error **errp);
 178 static int migration_maybe_pause(MigrationState *s,
 179                                  int *current_active_state,
 180                                  int new_state);
 181 static void migrate_fd_cancel(MigrationState *s);
 182
 183 static bool migrate_allow_multi_channels = true;
 184
 185 void migrate_protocol_allow_multi_channels(bool allow)
 186 {
 187     migrate_allow_multi_channels = allow;
 188 }
 189
 190 bool migrate_multi_channels_is_allowed(void)
 191 {
 192     return migrate_allow_multi_channels;
 193 }
 194
 195 static gint page_request_addr_cmp(gconstpointer ap, gconstpointer bp)
 196 {
 197     uintptr_t a = (uintptr_t) ap, b = (uintptr_t) bp;
 198
 199     return (a > b) - (a < b);
 200 }
 201
 202 void migration_object_init(void)
 203 {
 204     /* This can only be called once. */
 205     assert(!current_migration);
 206     current_migration = MIGRATION_OBJ(object_new(TYPE_MIGRATION));
 207
 208     /*
 209      * Init the migrate incoming object as well no matter whether
 210      * we'll use it or not.
 211      */
 212     assert(!current_incoming);
 213     current_incoming = g_new0(MigrationIncomingState, 1);
 214     current_incoming->state = MIGRATION_STATUS_NONE;
 215     current_incoming->postcopy_remote_fds =
 216         g_array_new(FALSE, TRUE, sizeof(struct PostCopyFD));
 217     qemu_mutex_init(&current_incoming->rp_mutex);
 218     qemu_event_init(&current_incoming->main_thread_load_event, false);
 219     qemu_sem_init(&current_incoming->postcopy_pause_sem_dst, 0);
 220     qemu_sem_init(&current_incoming->postcopy_pause_sem_fault, 0);
 221     qemu_mutex_init(&current_incoming->page_request_mutex);
 222     current_incoming->page_requested = g_tree_new(page_request_addr_cmp);
 223
 224     migration_object_check(current_migration, &error_fatal);
 225
 226     blk_mig_init();
 227     ram_mig_init();
 228     dirty_bitmap_mig_init();
 229 }
 230
 231 void migration_cancel(const Error *error)
 232 {
 233     if (error) {
 234         migrate_set_error(current_migration, error);
 235     }
 236     migrate_fd_cancel(current_migration);
 237 }
 238
 239 void migration_shutdown(void)
 240 {
 241     /*
 242      * When the QEMU main thread exit, the COLO thread
 243      * may wait a semaphore. So, we should wakeup the
 244      * COLO thread before migration shutdown.
 245      */
 246     colo_shutdown();
 247     /*
 248      * Cancel the current migration - that will (eventually)
 249      * stop the migration using this structure
 250      */
 251     migration_cancel(NULL);
 252     object_unref(OBJECT(current_migration));
 253
 254     /*
 255      * Cancel outgoing migration of dirty bitmaps. It should
 256      * at least unref used block nodes.
 257      */
 258     dirty_bitmap_mig_cancel_outgoing();
 259
 260     /*
 261      * Cancel incoming migration of dirty bitmaps. Dirty bitmaps
 262      * are non-critical data, and their loss never considered as
 263      * something serious.
 264      */
 265     dirty_bitmap_mig_cancel_incoming();
 266 }
 267
 268 /* For outgoing */
 269 MigrationState *migrate_get_current(void)
 270 {
 271     /* This can only be called after the object created. */
 272     assert(current_migration);
 273     return current_migration;
 274 }
 275
 276 MigrationIncomingState *migration_incoming_get_current(void)
 277 {
 278     assert(current_incoming);
 279     return current_incoming;
 280 }
 281
 282 void migration_incoming_transport_cleanup(MigrationIncomingState *mis)
 283 {
 284     if (mis->socket_address_list) {
 285         qapi_free_SocketAddressList(mis->socket_address_list);
 286         mis->socket_address_list = NULL;
 287     }
 288
 289     if (mis->transport_cleanup) {
 290         mis->transport_cleanup(mis->transport_data);
 291         mis->transport_data = mis->transport_cleanup = NULL;
 292     }
 293 }
 294
 295 void migration_incoming_state_destroy(void)
 296 {
 297     struct MigrationIncomingState *mis = migration_incoming_get_current();
 298
 299     if (mis->to_src_file) {
 300         /* Tell source that we are done */
 301         migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
 302         qemu_fclose(mis->to_src_file);
 303         mis->to_src_file = NULL;
 304     }
 305
 306     if (mis->from_src_file) {
 307         migration_ioc_unregister_yank_from_file(mis->from_src_file);
 308         qemu_fclose(mis->from_src_file);
 309         mis->from_src_file = NULL;
 310     }
 311     if (mis->postcopy_remote_fds) {
 312         g_array_free(mis->postcopy_remote_fds, TRUE);
 313         mis->postcopy_remote_fds = NULL;
 314     }
 315
 316     migration_incoming_transport_cleanup(mis);
 317     qemu_event_reset(&mis->main_thread_load_event);
 318
 319     if (mis->page_requested) {
 320         g_tree_destroy(mis->page_requested);
 321         mis->page_requested = NULL;
 322     }
 323
 324     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
 325 }
 326
 327 static void migrate_generate_event(int new_state)
 328 {
 329     if (migrate_use_events()) {
 330         qapi_event_send_migration(new_state);
 331     }
 332 }
 333
 334 static bool migrate_late_block_activate(void)
 335 {
 336     MigrationState *s;
 337
 338     s = migrate_get_current();
 339
 340     return s->enabled_capabilities[
 341         MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE];
 342 }
 343
 344 /*
 345  * Send a message on the return channel back to the source
 346  * of the migration.
 347  */
 348 static int migrate_send_rp_message(MigrationIncomingState *mis,
 349                                    enum mig_rp_message_type message_type,
 350                                    uint16_t len, void *data)
 351 {
 352     int ret = 0;
 353
 354     trace_migrate_send_rp_message((int)message_type, len);
 355     QEMU_LOCK_GUARD(&mis->rp_mutex);
 356
 357     /*
 358      * It's possible that the file handle got lost due to network
 359      * failures.
 360      */
 361     if (!mis->to_src_file) {
 362         ret = -EIO;
 363         return ret;
 364     }
 365
 366     qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
 367     qemu_put_be16(mis->to_src_file, len);
 368     qemu_put_buffer(mis->to_src_file, data, len);
 369     qemu_fflush(mis->to_src_file);
 370
 371     /* It's possible that qemu file got error during sending */
 372     ret = qemu_file_get_error(mis->to_src_file);
 373
 374     return ret;
 375 }
 376
 377 /* Request one page from the source VM at the given start address.
 378  *   rb: the RAMBlock to request the page in
 379  *   Start: Address offset within the RB
 380  *   Len: Length in bytes required - must be a multiple of pagesize
 381  */
 382 int migrate_send_rp_message_req_pages(MigrationIncomingState *mis,
 383                                       RAMBlock *rb, ram_addr_t start)
 384 {
 385     uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */
 386     size_t msglen = 12; /* start + len */
 387     size_t len = qemu_ram_pagesize(rb);
 388     enum mig_rp_message_type msg_type;
 389     const char *rbname;
 390     int rbname_len;
 391
 392     *(uint64_t *)bufc = cpu_to_be64((uint64_t)start);
 393     *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len);
 394
 395     /*
 396      * We maintain the last ramblock that we requested for page.  Note that we
 397      * don't need locking because this function will only be called within the
 398      * postcopy ram fault thread.
 399      */
 400     if (rb != mis->last_rb) {
 401         mis->last_rb = rb;
 402
 403         rbname = qemu_ram_get_idstr(rb);
 404         rbname_len = strlen(rbname);
 405
 406         assert(rbname_len < 256);
 407
 408         bufc[msglen++] = rbname_len;
 409         memcpy(bufc + msglen, rbname, rbname_len);
 410         msglen += rbname_len;
 411         msg_type = MIG_RP_MSG_REQ_PAGES_ID;
 412     } else {
 413         msg_type = MIG_RP_MSG_REQ_PAGES;
 414     }
 415
 416     return migrate_send_rp_message(mis, msg_type, msglen, bufc);
 417 }
 418
 419 int migrate_send_rp_req_pages(MigrationIncomingState *mis,
 420                               RAMBlock *rb, ram_addr_t start, uint64_t haddr)
 421 {
 422     void *aligned = (void *)(uintptr_t)ROUND_DOWN(haddr, qemu_ram_pagesize(rb));
 423     bool received = false;
 424
 425     WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) {
 426         received = ramblock_recv_bitmap_test_byte_offset(rb, start);
 427         if (!received && !g_tree_lookup(mis->page_requested, aligned)) {
 428             /*
 429              * The page has not been received, and it's not yet in the page
 430              * request list.  Queue it.  Set the value of element to 1, so that
 431              * things like g_tree_lookup() will return TRUE (1) when found.
 432              */
 433             g_tree_insert(mis->page_requested, aligned, (gpointer)1);
 434             mis->page_requested_count++;
 435             trace_postcopy_page_req_add(aligned, mis->page_requested_count);
 436         }
 437     }
 438
 439     /*
 440      * If the page is there, skip sending the message.  We don't even need the
 441      * lock because as long as the page arrived, it'll be there forever.
 442      */
 443     if (received) {
 444         return 0;
 445     }
 446
 447     return migrate_send_rp_message_req_pages(mis, rb, start);
 448 }
 449
 450 static bool migration_colo_enabled;
 451 bool migration_incoming_colo_enabled(void)
 452 {
 453     return migration_colo_enabled;
 454 }
 455
 456 void migration_incoming_disable_colo(void)
 457 {
 458     ram_block_discard_disable(false);
 459     migration_colo_enabled = false;
 460 }
 461
 462 int migration_incoming_enable_colo(void)
 463 {
 464     if (ram_block_discard_disable(true)) {
 465         error_report("COLO: cannot disable RAM discard");
 466         return -EBUSY;
 467     }
 468     migration_colo_enabled = true;
 469     return 0;
 470 }
 471
 472 void migrate_add_address(SocketAddress *address)
 473 {
 474     MigrationIncomingState *mis = migration_incoming_get_current();
 475
 476     QAPI_LIST_PREPEND(mis->socket_address_list,
 477                       QAPI_CLONE(SocketAddress, address));
 478 }
 479
 480 static void qemu_start_incoming_migration(const char *uri, Error **errp)
 481 {
 482     const char *p = NULL;
 483
 484     migrate_protocol_allow_multi_channels(false); /* reset it anyway */
 485     qapi_event_send_migration(MIGRATION_STATUS_SETUP);
 486     if (strstart(uri, "tcp:", &p) ||
 487         strstart(uri, "unix:", NULL) ||
 488         strstart(uri, "vsock:", NULL)) {
 489         migrate_protocol_allow_multi_channels(true);
 490         socket_start_incoming_migration(p ? p : uri, errp);
 491 #ifdef CONFIG_RDMA
 492     } else if (strstart(uri, "rdma:", &p)) {
 493         rdma_start_incoming_migration(p, errp);
 494 #endif
 495     } else if (strstart(uri, "exec:", &p)) {
 496         exec_start_incoming_migration(p, errp);
 497     } else if (strstart(uri, "fd:", &p)) {
 498         fd_start_incoming_migration(p, errp);
 499     } else {
 500         error_setg(errp, "unknown migration protocol: %s", uri);
 501     }
 502 }
 503
 504 static void process_incoming_migration_bh(void *opaque)
 505 {
 506     Error *local_err = NULL;
 507     MigrationIncomingState *mis = opaque;
 508
 509     /* If capability late_block_activate is set:
 510      * Only fire up the block code now if we're going to restart the
 511      * VM, else 'cont' will do it.
 512      * This causes file locking to happen; so we don't want it to happen
 513      * unless we really are starting the VM.
 514      */
 515     if (!migrate_late_block_activate() ||
 516          (autostart && (!global_state_received() ||
 517             global_state_get_runstate() == RUN_STATE_RUNNING))) {
 518         /* Make sure all file formats throw away their mutable metadata.
 519          * If we get an error here, just don't restart the VM yet. */
 520         bdrv_activate_all(&local_err);
 521         if (local_err) {
 522             error_report_err(local_err);
 523             local_err = NULL;
 524             autostart = false;
 525         }
 526     }
 527
 528     /*
 529      * This must happen after all error conditions are dealt with and
 530      * we're sure the VM is going to be running on this host.
 531      */
 532     qemu_announce_self(&mis->announce_timer, migrate_announce_params());
 533
 534     if (multifd_load_cleanup(&local_err) != 0) {
 535         error_report_err(local_err);
 536         autostart = false;
 537     }
 538     /* If global state section was not received or we are in running
 539        state, we need to obey autostart. Any other state is set with
 540        runstate_set. */
 541
 542     dirty_bitmap_mig_before_vm_start();
 543
 544     if (!global_state_received() ||
 545         global_state_get_runstate() == RUN_STATE_RUNNING) {
 546         if (autostart) {
 547             vm_start();
 548         } else {
 549             runstate_set(RUN_STATE_PAUSED);
 550         }
 551     } else if (migration_incoming_colo_enabled()) {
 552         migration_incoming_disable_colo();
 553         vm_start();
 554     } else {
 555         runstate_set(global_state_get_runstate());
 556     }
 557     /*
 558      * This must happen after any state changes since as soon as an external
 559      * observer sees this event they might start to prod at the VM assuming
 560      * it's ready to use.
 561      */
 562     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 563                       MIGRATION_STATUS_COMPLETED);
 564     qemu_bh_delete(mis->bh);
 565     migration_incoming_state_destroy();
 566 }
 567
 568 static void process_incoming_migration_co(void *opaque)
 569 {
 570     MigrationIncomingState *mis = migration_incoming_get_current();
 571     PostcopyState ps;
 572     int ret;
 573     Error *local_err = NULL;
 574
 575     assert(mis->from_src_file);
 576     mis->migration_incoming_co = qemu_coroutine_self();
 577     mis->largest_page_size = qemu_ram_pagesize_largest();
 578     postcopy_state_set(POSTCOPY_INCOMING_NONE);
 579     migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
 580                       MIGRATION_STATUS_ACTIVE);
 581     ret = qemu_loadvm_state(mis->from_src_file);
 582
 583     ps = postcopy_state_get();
 584     trace_process_incoming_migration_co_end(ret, ps);
 585     if (ps != POSTCOPY_INCOMING_NONE) {
 586         if (ps == POSTCOPY_INCOMING_ADVISE) {
 587             /*
 588              * Where a migration had postcopy enabled (and thus went to advise)
 589              * but managed to complete within the precopy period, we can use
 590              * the normal exit.
 591              */
 592             postcopy_ram_incoming_cleanup(mis);
 593         } else if (ret >= 0) {
 594             /*
 595              * Postcopy was started, cleanup should happen at the end of the
 596              * postcopy thread.
 597              */
 598             trace_process_incoming_migration_co_postcopy_end_main();
 599             return;
 600         }
 601         /* Else if something went wrong then just fall out of the normal exit */
 602     }
 603
 604     /* we get COLO info, and know if we are in COLO mode */
 605     if (!ret && migration_incoming_colo_enabled()) {
 606         /* Make sure all file formats throw away their mutable metadata */
 607         bdrv_activate_all(&local_err);
 608         if (local_err) {
 609             error_report_err(local_err);
 610             goto fail;
 611         }
 612
 613         qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
 614              colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
 615         mis->have_colo_incoming_thread = true;
 616         qemu_coroutine_yield();
 617
 618         qemu_mutex_unlock_iothread();
 619         /* Wait checkpoint incoming thread exit before free resource */
 620         qemu_thread_join(&mis->colo_incoming_thread);
 621         qemu_mutex_lock_iothread();
 622         /* We hold the global iothread lock, so it is safe here */
 623         colo_release_ram_cache();
 624     }
 625
 626     if (ret < 0) {
 627         error_report("load of migration failed: %s", strerror(-ret));
 628         goto fail;
 629     }
 630     mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
 631     qemu_bh_schedule(mis->bh);
 632     mis->migration_incoming_co = NULL;
 633     return;
 634 fail:
 635     local_err = NULL;
 636     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 637                       MIGRATION_STATUS_FAILED);
 638     qemu_fclose(mis->from_src_file);
 639     if (multifd_load_cleanup(&local_err) != 0) {
 640         error_report_err(local_err);
 641     }
 642     exit(EXIT_FAILURE);
 643 }
 644
 645 /**
 646  * migration_incoming_setup: Setup incoming migration
 647  * @f: file for main migration channel
 648  * @errp: where to put errors
 649  *
 650  * Returns: %true on success, %false on error.
 651  */
 652 static bool migration_incoming_setup(QEMUFile *f, Error **errp)
 653 {
 654     MigrationIncomingState *mis = migration_incoming_get_current();
 655
 656     if (multifd_load_setup(errp) != 0) {
 657         return false;
 658     }
 659
 660     if (!mis->from_src_file) {
 661         mis->from_src_file = f;
 662     }
 663     qemu_file_set_blocking(f, false);
 664     return true;
 665 }
 666
 667 void migration_incoming_process(void)
 668 {
 669     Coroutine *co = qemu_coroutine_create(process_incoming_migration_co, NULL);
 670     qemu_coroutine_enter(co);
 671 }
 672
 673 /* Returns true if recovered from a paused migration, otherwise false */
 674 static bool postcopy_try_recover(void)
 675 {
 676     MigrationIncomingState *mis = migration_incoming_get_current();
 677
 678     if (mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
 679         /* Resumed from a paused postcopy migration */
 680
 681         /* This should be set already in migration_incoming_setup() */
 682         assert(mis->from_src_file);
 683         /* Postcopy has standalone thread to do vm load */
 684         qemu_file_set_blocking(mis->from_src_file, true);
 685
 686         /* Re-configure the return path */
 687         mis->to_src_file = qemu_file_get_return_path(mis->from_src_file);
 688
 689         migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
 690                           MIGRATION_STATUS_POSTCOPY_RECOVER);
 691
 692         /*
 693          * Here, we only wake up the main loading thread (while the
 694          * fault thread will still be waiting), so that we can receive
 695          * commands from source now, and answer it if needed. The
 696          * fault thread will be woken up afterwards until we are sure
 697          * that source is ready to reply to page requests.
 698          */
 699         qemu_sem_post(&mis->postcopy_pause_sem_dst);
 700         return true;
 701     }
 702
 703     return false;
 704 }
 705
 706 void migration_fd_process_incoming(QEMUFile *f, Error **errp)
 707 {
 708     if (!migration_incoming_setup(f, errp)) {
 709         return;
 710     }
 711     if (postcopy_try_recover()) {
 712         return;
 713     }
 714     migration_incoming_process();
 715 }
 716
 717 void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp)
 718 {
 719     MigrationIncomingState *mis = migration_incoming_get_current();
 720     Error *local_err = NULL;
 721     bool start_migration;
 722
 723     if (!mis->from_src_file) {
 724         /* The first connection (multifd may have multiple) */
 725         QEMUFile *f = qemu_fopen_channel_input(ioc);
 726
 727         if (!migration_incoming_setup(f, errp)) {
 728             return;
 729         }
 730
 731         /*
 732          * Common migration only needs one channel, so we can start
 733          * right now.  Multifd needs more than one channel, we wait.
 734          */
 735         start_migration = !migrate_use_multifd();
 736     } else {
 737         /* Multiple connections */
 738         assert(migrate_use_multifd());
 739         start_migration = multifd_recv_new_channel(ioc, &local_err);
 740         if (local_err) {
 741             error_propagate(errp, local_err);
 742             return;
 743         }
 744     }
 745
 746     if (start_migration) {
 747         /* If it's a recovery, we're done */
 748         if (postcopy_try_recover()) {
 749             return;
 750         }
 751         migration_incoming_process();
 752     }
 753 }
 754
 755 /**
 756  * @migration_has_all_channels: We have received all channels that we need
 757  *
 758  * Returns true when we have got connections to all the channels that
 759  * we need for migration.
 760  */
 761 bool migration_has_all_channels(void)
 762 {
 763     MigrationIncomingState *mis = migration_incoming_get_current();
 764     bool all_channels;
 765
 766     all_channels = multifd_recv_all_channels_created();
 767
 768     return all_channels && mis->from_src_file != NULL;
 769 }
 770
 771 /*
 772  * Send a 'SHUT' message on the return channel with the given value
 773  * to indicate that we've finished with the RP.  Non-0 value indicates
 774  * error.
 775  */
 776 void migrate_send_rp_shut(MigrationIncomingState *mis,
 777                           uint32_t value)
 778 {
 779     uint32_t buf;
 780
 781     buf = cpu_to_be32(value);
 782     migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
 783 }
 784
 785 /*
 786  * Send a 'PONG' message on the return channel with the given value
 787  * (normally in response to a 'PING')
 788  */
 789 void migrate_send_rp_pong(MigrationIncomingState *mis,
 790                           uint32_t value)
 791 {
 792     uint32_t buf;
 793
 794     buf = cpu_to_be32(value);
 795     migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
 796 }
 797
 798 void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
 799                                  char *block_name)
 800 {
 801     char buf[512];
 802     int len;
 803     int64_t res;
 804
 805     /*
 806      * First, we send the header part. It contains only the len of
 807      * idstr, and the idstr itself.
 808      */
 809     len = strlen(block_name);
 810     buf[0] = len;
 811     memcpy(buf + 1, block_name, len);
 812
 813     if (mis->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
 814         error_report("%s: MSG_RP_RECV_BITMAP only used for recovery",
 815                      __func__);
 816         return;
 817     }
 818
 819     migrate_send_rp_message(mis, MIG_RP_MSG_RECV_BITMAP, len + 1, buf);
 820
 821     /*
 822      * Next, we dump the received bitmap to the stream.
 823      *
 824      * TODO: currently we are safe since we are the only one that is
 825      * using the to_src_file handle (fault thread is still paused),
 826      * and it's ok even not taking the mutex. However the best way is
 827      * to take the lock before sending the message header, and release
 828      * the lock after sending the bitmap.
 829      */
 830     qemu_mutex_lock(&mis->rp_mutex);
 831     res = ramblock_recv_bitmap_send(mis->to_src_file, block_name);
 832     qemu_mutex_unlock(&mis->rp_mutex);
 833
 834     trace_migrate_send_rp_recv_bitmap(block_name, res);
 835 }
 836
 837 void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value)
 838 {
 839     uint32_t buf;
 840
 841     buf = cpu_to_be32(value);
 842     migrate_send_rp_message(mis, MIG_RP_MSG_RESUME_ACK, sizeof(buf), &buf);
 843 }
 844
 845 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
 846 {
 847     MigrationCapabilityStatusList *head = NULL, **tail = &head;
 848     MigrationCapabilityStatus *caps;
 849     MigrationState *s = migrate_get_current();
 850     int i;
 851
 852     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
 853 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
 854         if (i == MIGRATION_CAPABILITY_BLOCK) {
 855             continue;
 856         }
 857 #endif
 858         caps = g_malloc0(sizeof(*caps));
 859         caps->capability = i;
 860         caps->state = s->enabled_capabilities[i];
 861         QAPI_LIST_APPEND(tail, caps);
 862     }
 863
 864     return head;
 865 }
 866
 867 MigrationParameters *qmp_query_migrate_parameters(Error **errp)
 868 {
 869     MigrationParameters *params;
 870     MigrationState *s = migrate_get_current();
 871
 872     /* TODO use QAPI_CLONE() instead of duplicating it inline */
 873     params = g_malloc0(sizeof(*params));
 874     params->has_compress_level = true;
 875     params->compress_level = s->parameters.compress_level;
 876     params->has_compress_threads = true;
 877     params->compress_threads = s->parameters.compress_threads;
 878     params->has_compress_wait_thread = true;
 879     params->compress_wait_thread = s->parameters.compress_wait_thread;
 880     params->has_decompress_threads = true;
 881     params->decompress_threads = s->parameters.decompress_threads;
 882     params->has_throttle_trigger_threshold = true;
 883     params->throttle_trigger_threshold = s->parameters.throttle_trigger_threshold;
 884     params->has_cpu_throttle_initial = true;
 885     params->cpu_throttle_initial = s->parameters.cpu_throttle_initial;
 886     params->has_cpu_throttle_increment = true;
 887     params->cpu_throttle_increment = s->parameters.cpu_throttle_increment;
 888     params->has_cpu_throttle_tailslow = true;
 889     params->cpu_throttle_tailslow = s->parameters.cpu_throttle_tailslow;
 890     params->has_tls_creds = true;
 891     params->tls_creds = g_strdup(s->parameters.tls_creds);
 892     params->has_tls_hostname = true;
 893     params->tls_hostname = g_strdup(s->parameters.tls_hostname);
 894     params->has_tls_authz = true;
 895     params->tls_authz = g_strdup(s->parameters.tls_authz ?
 896                                  s->parameters.tls_authz : "");
 897     params->has_max_bandwidth = true;
 898     params->max_bandwidth = s->parameters.max_bandwidth;
 899     params->has_downtime_limit = true;
 900     params->downtime_limit = s->parameters.downtime_limit;
 901     params->has_x_checkpoint_delay = true;
 902     params->x_checkpoint_delay = s->parameters.x_checkpoint_delay;
 903     params->has_block_incremental = true;
 904     params->block_incremental = s->parameters.block_incremental;
 905     params->has_multifd_channels = true;
 906     params->multifd_channels = s->parameters.multifd_channels;
 907     params->has_multifd_compression = true;
 908     params->multifd_compression = s->parameters.multifd_compression;
 909     params->has_multifd_zlib_level = true;
 910     params->multifd_zlib_level = s->parameters.multifd_zlib_level;
 911     params->has_multifd_zstd_level = true;
 912     params->multifd_zstd_level = s->parameters.multifd_zstd_level;
 913 #ifdef CONFIG_LINUX
 914     params->has_zero_copy_send = true;
 915     params->zero_copy_send = s->parameters.zero_copy_send;
 916 #endif
 917     params->has_xbzrle_cache_size = true;
 918     params->xbzrle_cache_size = s->parameters.xbzrle_cache_size;
 919     params->has_max_postcopy_bandwidth = true;
 920     params->max_postcopy_bandwidth = s->parameters.max_postcopy_bandwidth;
 921     params->has_max_cpu_throttle = true;
 922     params->max_cpu_throttle = s->parameters.max_cpu_throttle;
 923     params->has_announce_initial = true;
 924     params->announce_initial = s->parameters.announce_initial;
 925     params->has_announce_max = true;
 926     params->announce_max = s->parameters.announce_max;
 927     params->has_announce_rounds = true;
 928     params->announce_rounds = s->parameters.announce_rounds;
 929     params->has_announce_step = true;
 930     params->announce_step = s->parameters.announce_step;
 931
 932     if (s->parameters.has_block_bitmap_mapping) {
 933         params->has_block_bitmap_mapping = true;
 934         params->block_bitmap_mapping =
 935             QAPI_CLONE(BitmapMigrationNodeAliasList,
 936                        s->parameters.block_bitmap_mapping);
 937     }
 938
 939     return params;
 940 }
 941
 942 AnnounceParameters *migrate_announce_params(void)
 943 {
 944     static AnnounceParameters ap;
 945
 946     MigrationState *s = migrate_get_current();
 947
 948     ap.initial = s->parameters.announce_initial;
 949     ap.max = s->parameters.announce_max;
 950     ap.rounds = s->parameters.announce_rounds;
 951     ap.step = s->parameters.announce_step;
 952
 953     return &ap;
 954 }
 955
 956 /*
 957  * Return true if we're already in the middle of a migration
 958  * (i.e. any of the active or setup states)
 959  */
 960 bool migration_is_setup_or_active(int state)
 961 {
 962     switch (state) {
 963     case MIGRATION_STATUS_ACTIVE:
 964     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 965     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 966     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 967     case MIGRATION_STATUS_SETUP:
 968     case MIGRATION_STATUS_PRE_SWITCHOVER:
 969     case MIGRATION_STATUS_DEVICE:
 970     case MIGRATION_STATUS_WAIT_UNPLUG:
 971     case MIGRATION_STATUS_COLO:
 972         return true;
 973
 974     default:
 975         return false;
 976
 977     }
 978 }
 979
 980 bool migration_is_running(int state)
 981 {
 982     switch (state) {
 983     case MIGRATION_STATUS_ACTIVE:
 984     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 985     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 986     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 987     case MIGRATION_STATUS_SETUP:
 988     case MIGRATION_STATUS_PRE_SWITCHOVER:
 989     case MIGRATION_STATUS_DEVICE:
 990     case MIGRATION_STATUS_WAIT_UNPLUG:
 991     case MIGRATION_STATUS_CANCELLING:
 992         return true;
 993
 994     default:
 995         return false;
 996
 997     }
 998 }
 999
1000 static void populate_time_info(MigrationInfo *info, MigrationState *s)
1001 {
1002     info->has_status = true;
1003     info->has_setup_time = true;
1004     info->setup_time = s->setup_time;
1005     if (s->state == MIGRATION_STATUS_COMPLETED) {
1006         info->has_total_time = true;
1007         info->total_time = s->total_time;
1008         info->has_downtime = true;
1009         info->downtime = s->downtime;
1010     } else {
1011         info->has_total_time = true;
1012         info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) -
1013                            s->start_time;
1014         info->has_expected_downtime = true;
1015         info->expected_downtime = s->expected_downtime;
1016     }
1017 }
1018
1019 static void populate_ram_info(MigrationInfo *info, MigrationState *s)
1020 {
1021     size_t page_size = qemu_target_page_size();
1022
1023     info->has_ram = true;
1024     info->ram = g_malloc0(sizeof(*info->ram));
1025     info->ram->transferred = ram_counters.transferred;
1026     info->ram->total = ram_bytes_total();
1027     info->ram->duplicate = ram_counters.duplicate;
1028     /* legacy value.  It is not used anymore */
1029     info->ram->skipped = 0;
1030     info->ram->normal = ram_counters.normal;
1031     info->ram->normal_bytes = ram_counters.normal * page_size;
1032     info->ram->mbps = s->mbps;
1033     info->ram->dirty_sync_count = ram_counters.dirty_sync_count;
1034     info->ram->postcopy_requests = ram_counters.postcopy_requests;
1035     info->ram->page_size = page_size;
1036     info->ram->multifd_bytes = ram_counters.multifd_bytes;
1037     info->ram->pages_per_second = s->pages_per_second;
1038     info->ram->precopy_bytes = ram_counters.precopy_bytes;
1039     info->ram->downtime_bytes = ram_counters.downtime_bytes;
1040     info->ram->postcopy_bytes = ram_counters.postcopy_bytes;
1041
1042     if (migrate_use_xbzrle()) {
1043         info->has_xbzrle_cache = true;
1044         info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
1045         info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
1046         info->xbzrle_cache->bytes = xbzrle_counters.bytes;
1047         info->xbzrle_cache->pages = xbzrle_counters.pages;
1048         info->xbzrle_cache->cache_miss = xbzrle_counters.cache_miss;
1049         info->xbzrle_cache->cache_miss_rate = xbzrle_counters.cache_miss_rate;
1050         info->xbzrle_cache->encoding_rate = xbzrle_counters.encoding_rate;
1051         info->xbzrle_cache->overflow = xbzrle_counters.overflow;
1052     }
1053
1054     if (migrate_use_compression()) {
1055         info->has_compression = true;
1056         info->compression = g_malloc0(sizeof(*info->compression));
1057         info->compression->pages = compression_counters.pages;
1058         info->compression->busy = compression_counters.busy;
1059         info->compression->busy_rate = compression_counters.busy_rate;
1060         info->compression->compressed_size =
1061                                     compression_counters.compressed_size;
1062         info->compression->compression_rate =
1063                                     compression_counters.compression_rate;
1064     }
1065
1066     if (cpu_throttle_active()) {
1067         info->has_cpu_throttle_percentage = true;
1068         info->cpu_throttle_percentage = cpu_throttle_get_percentage();
1069     }
1070
1071     if (s->state != MIGRATION_STATUS_COMPLETED) {
1072         info->ram->remaining = ram_bytes_remaining();
1073         info->ram->dirty_pages_rate = ram_counters.dirty_pages_rate;
1074     }
1075 }
1076
1077 static void populate_disk_info(MigrationInfo *info)
1078 {
1079     if (blk_mig_active()) {
1080         info->has_disk = true;
1081         info->disk = g_malloc0(sizeof(*info->disk));
1082         info->disk->transferred = blk_mig_bytes_transferred();
1083         info->disk->remaining = blk_mig_bytes_remaining();
1084         info->disk->total = blk_mig_bytes_total();
1085     }
1086 }
1087
1088 static void fill_source_migration_info(MigrationInfo *info)
1089 {
1090     MigrationState *s = migrate_get_current();
1091     int state = qatomic_read(&s->state);
1092     GSList *cur_blocker = migration_blockers;
1093
1094     info->blocked_reasons = NULL;
1095
1096     /*
1097      * There are two types of reasons a migration might be blocked;
1098      * a) devices marked in VMState as non-migratable, and
1099      * b) Explicit migration blockers
1100      * We need to add both of them here.
1101      */
1102     qemu_savevm_non_migratable_list(&info->blocked_reasons);
1103
1104     while (cur_blocker) {
1105         QAPI_LIST_PREPEND(info->blocked_reasons,
1106                           g_strdup(error_get_pretty(cur_blocker->data)));
1107         cur_blocker = g_slist_next(cur_blocker);
1108     }
1109     info->has_blocked_reasons = info->blocked_reasons != NULL;
1110
1111     switch (state) {
1112     case MIGRATION_STATUS_NONE:
1113         /* no migration has happened ever */
1114         /* do not overwrite destination migration status */
1115         return;
1116     case MIGRATION_STATUS_SETUP:
1117         info->has_status = true;
1118         info->has_total_time = false;
1119         break;
1120     case MIGRATION_STATUS_ACTIVE:
1121     case MIGRATION_STATUS_CANCELLING:
1122     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1123     case MIGRATION_STATUS_PRE_SWITCHOVER:
1124     case MIGRATION_STATUS_DEVICE:
1125     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1126     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1127         /* TODO add some postcopy stats */
1128         populate_time_info(info, s);
1129         populate_ram_info(info, s);
1130         populate_disk_info(info);
1131         populate_vfio_info(info);
1132         break;
1133     case MIGRATION_STATUS_COLO:
1134         info->has_status = true;
1135         /* TODO: display COLO specific information (checkpoint info etc.) */
1136         break;
1137     case MIGRATION_STATUS_COMPLETED:
1138         populate_time_info(info, s);
1139         populate_ram_info(info, s);
1140         populate_vfio_info(info);
1141         break;
1142     case MIGRATION_STATUS_FAILED:
1143         info->has_status = true;
1144         if (s->error) {
1145             info->has_error_desc = true;
1146             info->error_desc = g_strdup(error_get_pretty(s->error));
1147         }
1148         break;
1149     case MIGRATION_STATUS_CANCELLED:
1150         info->has_status = true;
1151         break;
1152     case MIGRATION_STATUS_WAIT_UNPLUG:
1153         info->has_status = true;
1154         break;
1155     }
1156     info->status = state;
1157 }
1158
1159 typedef enum WriteTrackingSupport {
1160     WT_SUPPORT_UNKNOWN = 0,
1161     WT_SUPPORT_ABSENT,
1162     WT_SUPPORT_AVAILABLE,
1163     WT_SUPPORT_COMPATIBLE
1164 } WriteTrackingSupport;
1165
1166 static
1167 WriteTrackingSupport migrate_query_write_tracking(void)
1168 {
1169     /* Check if kernel supports required UFFD features */
1170     if (!ram_write_tracking_available()) {
1171         return WT_SUPPORT_ABSENT;
1172     }
1173     /*
1174      * Check if current memory configuration is
1175      * compatible with required UFFD features.
1176      */
1177     if (!ram_write_tracking_compatible()) {
1178         return WT_SUPPORT_AVAILABLE;
1179     }
1180
1181     return WT_SUPPORT_COMPATIBLE;
1182 }
1183
1184 /**
1185  * @migration_caps_check - check capability validity
1186  *
1187  * @cap_list: old capability list, array of bool
1188  * @params: new capabilities to be applied soon
1189  * @errp: set *errp if the check failed, with reason
1190  *
1191  * Returns true if check passed, otherwise false.
1192  */
1193 static bool migrate_caps_check(bool *cap_list,
1194                                MigrationCapabilityStatusList *params,
1195                                Error **errp)
1196 {
1197     MigrationCapabilityStatusList *cap;
1198     bool old_postcopy_cap;
1199     MigrationIncomingState *mis = migration_incoming_get_current();
1200
1201     old_postcopy_cap = cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM];
1202
1203     for (cap = params; cap; cap = cap->next) {
1204         cap_list[cap->value->capability] = cap->value->state;
1205     }
1206
1207 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
1208     if (cap_list[MIGRATION_CAPABILITY_BLOCK]) {
1209         error_setg(errp, "QEMU compiled without old-style (blk/-b, inc/-i) "
1210                    "block migration");
1211         error_append_hint(errp, "Use drive_mirror+NBD instead.\n");
1212         return false;
1213     }
1214 #endif
1215
1216 #ifndef CONFIG_REPLICATION
1217     if (cap_list[MIGRATION_CAPABILITY_X_COLO]) {
1218         error_setg(errp, "QEMU compiled without replication module"
1219                    " can't enable COLO");
1220         error_append_hint(errp, "Please enable replication before COLO.\n");
1221         return false;
1222     }
1223 #endif
1224
1225     if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
1226         /* This check is reasonably expensive, so only when it's being
1227          * set the first time, also it's only the destination that needs
1228          * special support.
1229          */
1230         if (!old_postcopy_cap && runstate_check(RUN_STATE_INMIGRATE) &&
1231             !postcopy_ram_supported_by_host(mis)) {
1232             /* postcopy_ram_supported_by_host will have emitted a more
1233              * detailed message
1234              */
1235             error_setg(errp, "Postcopy is not supported");
1236             return false;
1237         }
1238
1239         if (cap_list[MIGRATION_CAPABILITY_X_IGNORE_SHARED]) {
1240             error_setg(errp, "Postcopy is not compatible with ignore-shared");
1241             return false;
1242         }
1243     }
1244
1245     if (cap_list[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT]) {
1246         WriteTrackingSupport wt_support;
1247         int idx;
1248         /*
1249          * Check if 'background-snapshot' capability is supported by
1250          * host kernel and compatible with guest memory configuration.
1251          */
1252         wt_support = migrate_query_write_tracking();
1253         if (wt_support < WT_SUPPORT_AVAILABLE) {
1254             error_setg(errp, "Background-snapshot is not supported by host kernel");
1255             return false;
1256         }
1257         if (wt_support < WT_SUPPORT_COMPATIBLE) {
1258             error_setg(errp, "Background-snapshot is not compatible "
1259                     "with guest memory configuration");
1260             return false;
1261         }
1262
1263         /*
1264          * Check if there are any migration capabilities
1265          * incompatible with 'background-snapshot'.
1266          */
1267         for (idx = 0; idx < check_caps_background_snapshot.size; idx++) {
1268             int incomp_cap = check_caps_background_snapshot.caps[idx];
1269             if (cap_list[incomp_cap]) {
1270                 error_setg(errp,
1271                         "Background-snapshot is not compatible with %s",
1272                         MigrationCapability_str(incomp_cap));
1273                 return false;
1274             }
1275         }
1276     }
1277
1278     /* incoming side only */
1279     if (runstate_check(RUN_STATE_INMIGRATE) &&
1280         !migrate_multi_channels_is_allowed() &&
1281         cap_list[MIGRATION_CAPABILITY_MULTIFD]) {
1282         error_setg(errp, "multifd is not supported by current protocol");
1283         return false;
1284     }
1285
1286     return true;
1287 }
1288
1289 static void fill_destination_migration_info(MigrationInfo *info)
1290 {
1291     MigrationIncomingState *mis = migration_incoming_get_current();
1292
1293     if (mis->socket_address_list) {
1294         info->has_socket_address = true;
1295         info->socket_address =
1296             QAPI_CLONE(SocketAddressList, mis->socket_address_list);
1297     }
1298
1299     switch (mis->state) {
1300     case MIGRATION_STATUS_NONE:
1301         return;
1302     case MIGRATION_STATUS_SETUP:
1303     case MIGRATION_STATUS_CANCELLING:
1304     case MIGRATION_STATUS_CANCELLED:
1305     case MIGRATION_STATUS_ACTIVE:
1306     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1307     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1308     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1309     case MIGRATION_STATUS_FAILED:
1310     case MIGRATION_STATUS_COLO:
1311         info->has_status = true;
1312         break;
1313     case MIGRATION_STATUS_COMPLETED:
1314         info->has_status = true;
1315         fill_destination_postcopy_migration_info(info);
1316         break;
1317     }
1318     info->status = mis->state;
1319 }
1320
1321 MigrationInfo *qmp_query_migrate(Error **errp)
1322 {
1323     MigrationInfo *info = g_malloc0(sizeof(*info));
1324
1325     fill_destination_migration_info(info);
1326     fill_source_migration_info(info);
1327
1328     return info;
1329 }
1330
1331 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
1332                                   Error **errp)
1333 {
1334     MigrationState *s = migrate_get_current();
1335     MigrationCapabilityStatusList *cap;
1336     bool cap_list[MIGRATION_CAPABILITY__MAX];
1337
1338     if (migration_is_running(s->state)) {
1339         error_setg(errp, QERR_MIGRATION_ACTIVE);
1340         return;
1341     }
1342
1343     memcpy(cap_list, s->enabled_capabilities, sizeof(cap_list));
1344     if (!migrate_caps_check(cap_list, params, errp)) {
1345         return;
1346     }
1347
1348     for (cap = params; cap; cap = cap->next) {
1349         s->enabled_capabilities[cap->value->capability] = cap->value->state;
1350     }
1351 }
1352
1353 /*
1354  * Check whether the parameters are valid. Error will be put into errp
1355  * (if provided). Return true if valid, otherwise false.
1356  */
1357 static bool migrate_params_check(MigrationParameters *params, Error **errp)
1358 {
1359     if (params->has_compress_level &&
1360         (params->compress_level > 9)) {
1361         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level",
1362                    "a value between 0 and 9");
1363         return false;
1364     }
1365
1366     if (params->has_compress_threads && (params->compress_threads < 1)) {
1367         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1368                    "compress_threads",
1369                    "a value between 1 and 255");
1370         return false;
1371     }
1372
1373     if (params->has_decompress_threads && (params->decompress_threads < 1)) {
1374         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1375                    "decompress_threads",
1376                    "a value between 1 and 255");
1377         return false;
1378     }
1379
1380     if (params->has_throttle_trigger_threshold &&
1381         (params->throttle_trigger_threshold < 1 ||
1382          params->throttle_trigger_threshold > 100)) {
1383         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1384                    "throttle_trigger_threshold",
1385                    "an integer in the range of 1 to 100");
1386         return false;
1387     }
1388
1389     if (params->has_cpu_throttle_initial &&
1390         (params->cpu_throttle_initial < 1 ||
1391          params->cpu_throttle_initial > 99)) {
1392         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1393                    "cpu_throttle_initial",
1394                    "an integer in the range of 1 to 99");
1395         return false;
1396     }
1397
1398     if (params->has_cpu_throttle_increment &&
1399         (params->cpu_throttle_increment < 1 ||
1400          params->cpu_throttle_increment > 99)) {
1401         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1402                    "cpu_throttle_increment",
1403                    "an integer in the range of 1 to 99");
1404         return false;
1405     }
1406
1407     if (params->has_max_bandwidth && (params->max_bandwidth > SIZE_MAX)) {
1408         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1409                    "max_bandwidth",
1410                    "an integer in the range of 0 to "stringify(SIZE_MAX)
1411                    " bytes/second");
1412         return false;
1413     }
1414
1415     if (params->has_downtime_limit &&
1416         (params->downtime_limit > MAX_MIGRATE_DOWNTIME)) {
1417         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1418                    "downtime_limit",
1419                    "an integer in the range of 0 to "
1420                     stringify(MAX_MIGRATE_DOWNTIME)" ms");
1421         return false;
1422     }
1423
1424     /* x_checkpoint_delay is now always positive */
1425
1426     if (params->has_multifd_channels && (params->multifd_channels < 1)) {
1427         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1428                    "multifd_channels",
1429                    "a value between 1 and 255");
1430         return false;
1431     }
1432
1433     if (params->has_multifd_zlib_level &&
1434         (params->multifd_zlib_level > 9)) {
1435         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zlib_level",
1436                    "a value between 0 and 9");
1437         return false;
1438     }
1439
1440     if (params->has_multifd_zstd_level &&
1441         (params->multifd_zstd_level > 20)) {
1442         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zstd_level",
1443                    "a value between 0 and 20");
1444         return false;
1445     }
1446
1447     if (params->has_xbzrle_cache_size &&
1448         (params->xbzrle_cache_size < qemu_target_page_size() ||
1449          !is_power_of_2(params->xbzrle_cache_size))) {
1450         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1451                    "xbzrle_cache_size",
1452                    "a power of two no less than the target page size");
1453         return false;
1454     }
1455
1456     if (params->has_max_cpu_throttle &&
1457         (params->max_cpu_throttle < params->cpu_throttle_initial ||
1458          params->max_cpu_throttle > 99)) {
1459         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1460                    "max_cpu_throttle",
1461                    "an integer in the range of cpu_throttle_initial to 99");
1462         return false;
1463     }
1464
1465     if (params->has_announce_initial &&
1466         params->announce_initial > 100000) {
1467         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1468                    "announce_initial",
1469                    "a value between 0 and 100000");
1470         return false;
1471     }
1472     if (params->has_announce_max &&
1473         params->announce_max > 100000) {
1474         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1475                    "announce_max",
1476                    "a value between 0 and 100000");
1477        return false;
1478     }
1479     if (params->has_announce_rounds &&
1480         params->announce_rounds > 1000) {
1481         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1482                    "announce_rounds",
1483                    "a value between 0 and 1000");
1484        return false;
1485     }
1486     if (params->has_announce_step &&
1487         (params->announce_step < 1 ||
1488         params->announce_step > 10000)) {
1489         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1490                    "announce_step",
1491                    "a value between 0 and 10000");
1492        return false;
1493     }
1494
1495     if (params->has_block_bitmap_mapping &&
1496         !check_dirty_bitmap_mig_alias_map(params->block_bitmap_mapping, errp)) {
1497         error_prepend(errp, "Invalid mapping given for block-bitmap-mapping: ");
1498         return false;
1499     }
1500 #ifdef CONFIG_LINUX
1501     if (params->zero_copy_send &&
1502         (!migrate_use_multifd() ||
1503          params->multifd_compression != MULTIFD_COMPRESSION_NONE ||
1504          (params->tls_creds && *params->tls_creds))) {
1505         error_setg(errp,
1506                    "Zero copy only available for non-compressed non-TLS multifd migration");
1507         return false;
1508     }
1509 #endif
1510     return true;
1511 }
1512
1513 static void migrate_params_test_apply(MigrateSetParameters *params,
1514                                       MigrationParameters *dest)
1515 {
1516     *dest = migrate_get_current()->parameters;
1517
1518     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1519
1520     if (params->has_compress_level) {
1521         dest->compress_level = params->compress_level;
1522     }
1523
1524     if (params->has_compress_threads) {
1525         dest->compress_threads = params->compress_threads;
1526     }
1527
1528     if (params->has_compress_wait_thread) {
1529         dest->compress_wait_thread = params->compress_wait_thread;
1530     }
1531
1532     if (params->has_decompress_threads) {
1533         dest->decompress_threads = params->decompress_threads;
1534     }
1535
1536     if (params->has_throttle_trigger_threshold) {
1537         dest->throttle_trigger_threshold = params->throttle_trigger_threshold;
1538     }
1539
1540     if (params->has_cpu_throttle_initial) {
1541         dest->cpu_throttle_initial = params->cpu_throttle_initial;
1542     }
1543
1544     if (params->has_cpu_throttle_increment) {
1545         dest->cpu_throttle_increment = params->cpu_throttle_increment;
1546     }
1547
1548     if (params->has_cpu_throttle_tailslow) {
1549         dest->cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1550     }
1551
1552     if (params->has_tls_creds) {
1553         assert(params->tls_creds->type == QTYPE_QSTRING);
1554         dest->tls_creds = params->tls_creds->u.s;
1555     }
1556
1557     if (params->has_tls_hostname) {
1558         assert(params->tls_hostname->type == QTYPE_QSTRING);
1559         dest->tls_hostname = params->tls_hostname->u.s;
1560     }
1561
1562     if (params->has_max_bandwidth) {
1563         dest->max_bandwidth = params->max_bandwidth;
1564     }
1565
1566     if (params->has_downtime_limit) {
1567         dest->downtime_limit = params->downtime_limit;
1568     }
1569
1570     if (params->has_x_checkpoint_delay) {
1571         dest->x_checkpoint_delay = params->x_checkpoint_delay;
1572     }
1573
1574     if (params->has_block_incremental) {
1575         dest->block_incremental = params->block_incremental;
1576     }
1577     if (params->has_multifd_channels) {
1578         dest->multifd_channels = params->multifd_channels;
1579     }
1580     if (params->has_multifd_compression) {
1581         dest->multifd_compression = params->multifd_compression;
1582     }
1583 #ifdef CONFIG_LINUX
1584     if (params->has_zero_copy_send) {
1585         dest->zero_copy_send = params->zero_copy_send;
1586     }
1587 #endif
1588     if (params->has_xbzrle_cache_size) {
1589         dest->xbzrle_cache_size = params->xbzrle_cache_size;
1590     }
1591     if (params->has_max_postcopy_bandwidth) {
1592         dest->max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1593     }
1594     if (params->has_max_cpu_throttle) {
1595         dest->max_cpu_throttle = params->max_cpu_throttle;
1596     }
1597     if (params->has_announce_initial) {
1598         dest->announce_initial = params->announce_initial;
1599     }
1600     if (params->has_announce_max) {
1601         dest->announce_max = params->announce_max;
1602     }
1603     if (params->has_announce_rounds) {
1604         dest->announce_rounds = params->announce_rounds;
1605     }
1606     if (params->has_announce_step) {
1607         dest->announce_step = params->announce_step;
1608     }
1609
1610     if (params->has_block_bitmap_mapping) {
1611         dest->has_block_bitmap_mapping = true;
1612         dest->block_bitmap_mapping = params->block_bitmap_mapping;
1613     }
1614 }
1615
1616 static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
1617 {
1618     MigrationState *s = migrate_get_current();
1619
1620     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1621
1622     if (params->has_compress_level) {
1623         s->parameters.compress_level = params->compress_level;
1624     }
1625
1626     if (params->has_compress_threads) {
1627         s->parameters.compress_threads = params->compress_threads;
1628     }
1629
1630     if (params->has_compress_wait_thread) {
1631         s->parameters.compress_wait_thread = params->compress_wait_thread;
1632     }
1633
1634     if (params->has_decompress_threads) {
1635         s->parameters.decompress_threads = params->decompress_threads;
1636     }
1637
1638     if (params->has_throttle_trigger_threshold) {
1639         s->parameters.throttle_trigger_threshold = params->throttle_trigger_threshold;
1640     }
1641
1642     if (params->has_cpu_throttle_initial) {
1643         s->parameters.cpu_throttle_initial = params->cpu_throttle_initial;
1644     }
1645
1646     if (params->has_cpu_throttle_increment) {
1647         s->parameters.cpu_throttle_increment = params->cpu_throttle_increment;
1648     }
1649
1650     if (params->has_cpu_throttle_tailslow) {
1651         s->parameters.cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1652     }
1653
1654     if (params->has_tls_creds) {
1655         g_free(s->parameters.tls_creds);
1656         assert(params->tls_creds->type == QTYPE_QSTRING);
1657         s->parameters.tls_creds = g_strdup(params->tls_creds->u.s);
1658     }
1659
1660     if (params->has_tls_hostname) {
1661         g_free(s->parameters.tls_hostname);
1662         assert(params->tls_hostname->type == QTYPE_QSTRING);
1663         s->parameters.tls_hostname = g_strdup(params->tls_hostname->u.s);
1664     }
1665
1666     if (params->has_tls_authz) {
1667         g_free(s->parameters.tls_authz);
1668         assert(params->tls_authz->type == QTYPE_QSTRING);
1669         s->parameters.tls_authz = g_strdup(params->tls_authz->u.s);
1670     }
1671
1672     if (params->has_max_bandwidth) {
1673         s->parameters.max_bandwidth = params->max_bandwidth;
1674         if (s->to_dst_file && !migration_in_postcopy()) {
1675             qemu_file_set_rate_limit(s->to_dst_file,
1676                                 s->parameters.max_bandwidth / XFER_LIMIT_RATIO);
1677         }
1678     }
1679
1680     if (params->has_downtime_limit) {
1681         s->parameters.downtime_limit = params->downtime_limit;
1682     }
1683
1684     if (params->has_x_checkpoint_delay) {
1685         s->parameters.x_checkpoint_delay = params->x_checkpoint_delay;
1686         if (migration_in_colo_state()) {
1687             colo_checkpoint_notify(s);
1688         }
1689     }
1690
1691     if (params->has_block_incremental) {
1692         s->parameters.block_incremental = params->block_incremental;
1693     }
1694     if (params->has_multifd_channels) {
1695         s->parameters.multifd_channels = params->multifd_channels;
1696     }
1697     if (params->has_multifd_compression) {
1698         s->parameters.multifd_compression = params->multifd_compression;
1699     }
1700 #ifdef CONFIG_LINUX
1701     if (params->has_zero_copy_send) {
1702         s->parameters.zero_copy_send = params->zero_copy_send;
1703     }
1704 #endif
1705     if (params->has_xbzrle_cache_size) {
1706         s->parameters.xbzrle_cache_size = params->xbzrle_cache_size;
1707         xbzrle_cache_resize(params->xbzrle_cache_size, errp);
1708     }
1709     if (params->has_max_postcopy_bandwidth) {
1710         s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1711         if (s->to_dst_file && migration_in_postcopy()) {
1712             qemu_file_set_rate_limit(s->to_dst_file,
1713                     s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO);
1714         }
1715     }
1716     if (params->has_max_cpu_throttle) {
1717         s->parameters.max_cpu_throttle = params->max_cpu_throttle;
1718     }
1719     if (params->has_announce_initial) {
1720         s->parameters.announce_initial = params->announce_initial;
1721     }
1722     if (params->has_announce_max) {
1723         s->parameters.announce_max = params->announce_max;
1724     }
1725     if (params->has_announce_rounds) {
1726         s->parameters.announce_rounds = params->announce_rounds;
1727     }
1728     if (params->has_announce_step) {
1729         s->parameters.announce_step = params->announce_step;
1730     }
1731
1732     if (params->has_block_bitmap_mapping) {
1733         qapi_free_BitmapMigrationNodeAliasList(
1734             s->parameters.block_bitmap_mapping);
1735
1736         s->parameters.has_block_bitmap_mapping = true;
1737         s->parameters.block_bitmap_mapping =
1738             QAPI_CLONE(BitmapMigrationNodeAliasList,
1739                        params->block_bitmap_mapping);
1740     }
1741 }
1742
1743 void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
1744 {
1745     MigrationParameters tmp;
1746
1747     /* TODO Rewrite "" to null instead */
1748     if (params->has_tls_creds
1749         && params->tls_creds->type == QTYPE_QNULL) {
1750         qobject_unref(params->tls_creds->u.n);
1751         params->tls_creds->type = QTYPE_QSTRING;
1752         params->tls_creds->u.s = strdup("");
1753     }
1754     /* TODO Rewrite "" to null instead */
1755     if (params->has_tls_hostname
1756         && params->tls_hostname->type == QTYPE_QNULL) {
1757         qobject_unref(params->tls_hostname->u.n);
1758         params->tls_hostname->type = QTYPE_QSTRING;
1759         params->tls_hostname->u.s = strdup("");
1760     }
1761
1762     migrate_params_test_apply(params, &tmp);
1763
1764     if (!migrate_params_check(&tmp, errp)) {
1765         /* Invalid parameter */
1766         return;
1767     }
1768
1769     migrate_params_apply(params, errp);
1770 }
1771
1772
1773 void qmp_migrate_start_postcopy(Error **errp)
1774 {
1775     MigrationState *s = migrate_get_current();
1776
1777     if (!migrate_postcopy()) {
1778         error_setg(errp, "Enable postcopy with migrate_set_capability before"
1779                          " the start of migration");
1780         return;
1781     }
1782
1783     if (s->state == MIGRATION_STATUS_NONE) {
1784         error_setg(errp, "Postcopy must be started after migration has been"
1785                          " started");
1786         return;
1787     }
1788     /*
1789      * we don't error if migration has finished since that would be racy
1790      * with issuing this command.
1791      */
1792     qatomic_set(&s->start_postcopy, true);
1793 }
1794
1795 /* shared migration helpers */
1796
1797 void migrate_set_state(int *state, int old_state, int new_state)
1798 {
1799     assert(new_state < MIGRATION_STATUS__MAX);
1800     if (qatomic_cmpxchg(state, old_state, new_state) == old_state) {
1801         trace_migrate_set_state(MigrationStatus_str(new_state));
1802         migrate_generate_event(new_state);
1803     }
1804 }
1805
1806 static MigrationCapabilityStatus *migrate_cap_add(MigrationCapability index,
1807                                                   bool state)
1808 {
1809     MigrationCapabilityStatus *cap;
1810
1811     cap = g_new0(MigrationCapabilityStatus, 1);
1812     cap->capability = index;
1813     cap->state = state;
1814
1815     return cap;
1816 }
1817
1818 void migrate_set_block_enabled(bool value, Error **errp)
1819 {
1820     MigrationCapabilityStatusList *cap = NULL;
1821
1822     QAPI_LIST_PREPEND(cap, migrate_cap_add(MIGRATION_CAPABILITY_BLOCK, value));
1823     qmp_migrate_set_capabilities(cap, errp);
1824     qapi_free_MigrationCapabilityStatusList(cap);
1825 }
1826
1827 static void migrate_set_block_incremental(MigrationState *s, bool value)
1828 {
1829     s->parameters.block_incremental = value;
1830 }
1831
1832 static void block_cleanup_parameters(MigrationState *s)
1833 {
1834     if (s->must_remove_block_options) {
1835         /* setting to false can never fail */
1836         migrate_set_block_enabled(false, &error_abort);
1837         migrate_set_block_incremental(s, false);
1838         s->must_remove_block_options = false;
1839     }
1840 }
1841
1842 static void migrate_fd_cleanup(MigrationState *s)
1843 {
1844     qemu_bh_delete(s->cleanup_bh);
1845     s->cleanup_bh = NULL;
1846
1847     g_free(s->hostname);
1848     s->hostname = NULL;
1849
1850     qemu_savevm_state_cleanup();
1851
1852     if (s->to_dst_file) {
1853         QEMUFile *tmp;
1854
1855         trace_migrate_fd_cleanup();
1856         qemu_mutex_unlock_iothread();
1857         if (s->migration_thread_running) {
1858             qemu_thread_join(&s->thread);
1859             s->migration_thread_running = false;
1860         }
1861         qemu_mutex_lock_iothread();
1862
1863         multifd_save_cleanup();
1864         qemu_mutex_lock(&s->qemu_file_lock);
1865         tmp = s->to_dst_file;
1866         s->to_dst_file = NULL;
1867         qemu_mutex_unlock(&s->qemu_file_lock);
1868         /*
1869          * Close the file handle without the lock to make sure the
1870          * critical section won't block for long.
1871          */
1872         migration_ioc_unregister_yank_from_file(tmp);
1873         qemu_fclose(tmp);
1874     }
1875
1876     assert(!migration_is_active(s));
1877
1878     if (s->state == MIGRATION_STATUS_CANCELLING) {
1879         migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING,
1880                           MIGRATION_STATUS_CANCELLED);
1881     }
1882
1883     if (s->error) {
1884         /* It is used on info migrate.  We can't free it */
1885         error_report_err(error_copy(s->error));
1886     }
1887     notifier_list_notify(&migration_state_notifiers, s);
1888     block_cleanup_parameters(s);
1889     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
1890 }
1891
1892 static void migrate_fd_cleanup_schedule(MigrationState *s)
1893 {
1894     /*
1895      * Ref the state for bh, because it may be called when
1896      * there're already no other refs
1897      */
1898     object_ref(OBJECT(s));
1899     qemu_bh_schedule(s->cleanup_bh);
1900 }
1901
1902 static void migrate_fd_cleanup_bh(void *opaque)
1903 {
1904     MigrationState *s = opaque;
1905     migrate_fd_cleanup(s);
1906     object_unref(OBJECT(s));
1907 }
1908
1909 void migrate_set_error(MigrationState *s, const Error *error)
1910 {
1911     QEMU_LOCK_GUARD(&s->error_mutex);
1912     if (!s->error) {
1913         s->error = error_copy(error);
1914     }
1915 }
1916
1917 static void migrate_error_free(MigrationState *s)
1918 {
1919     QEMU_LOCK_GUARD(&s->error_mutex);
1920     if (s->error) {
1921         error_free(s->error);
1922         s->error = NULL;
1923     }
1924 }
1925
1926 void migrate_fd_error(MigrationState *s, const Error *error)
1927 {
1928     trace_migrate_fd_error(error_get_pretty(error));
1929     assert(s->to_dst_file == NULL);
1930     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
1931                       MIGRATION_STATUS_FAILED);
1932     migrate_set_error(s, error);
1933 }
1934
1935 static void migrate_fd_cancel(MigrationState *s)
1936 {
1937     int old_state ;
1938     QEMUFile *f = migrate_get_current()->to_dst_file;
1939     trace_migrate_fd_cancel();
1940
1941     WITH_QEMU_LOCK_GUARD(&s->qemu_file_lock) {
1942         if (s->rp_state.from_dst_file) {
1943             /* shutdown the rp socket, so causing the rp thread to shutdown */
1944             qemu_file_shutdown(s->rp_state.from_dst_file);
1945         }
1946     }
1947
1948     do {
1949         old_state = s->state;
1950         if (!migration_is_running(old_state)) {
1951             break;
1952         }
1953         /* If the migration is paused, kick it out of the pause */
1954         if (old_state == MIGRATION_STATUS_PRE_SWITCHOVER) {
1955             qemu_sem_post(&s->pause_sem);
1956         }
1957         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING);
1958     } while (s->state != MIGRATION_STATUS_CANCELLING);
1959
1960     /*
1961      * If we're unlucky the migration code might be stuck somewhere in a
1962      * send/write while the network has failed and is waiting to timeout;
1963      * if we've got shutdown(2) available then we can force it to quit.
1964      * The outgoing qemu file gets closed in migrate_fd_cleanup that is
1965      * called in a bh, so there is no race against this cancel.
1966      */
1967     if (s->state == MIGRATION_STATUS_CANCELLING && f) {
1968         qemu_file_shutdown(f);
1969     }
1970     if (s->state == MIGRATION_STATUS_CANCELLING && s->block_inactive) {
1971         Error *local_err = NULL;
1972
1973         bdrv_activate_all(&local_err);
1974         if (local_err) {
1975             error_report_err(local_err);
1976         } else {
1977             s->block_inactive = false;
1978         }
1979     }
1980 }
1981
1982 void add_migration_state_change_notifier(Notifier *notify)
1983 {
1984     notifier_list_add(&migration_state_notifiers, notify);
1985 }
1986
1987 void remove_migration_state_change_notifier(Notifier *notify)
1988 {
1989     notifier_remove(notify);
1990 }
1991
1992 bool migration_in_setup(MigrationState *s)
1993 {
1994     return s->state == MIGRATION_STATUS_SETUP;
1995 }
1996
1997 bool migration_has_finished(MigrationState *s)
1998 {
1999     return s->state == MIGRATION_STATUS_COMPLETED;
2000 }
2001
2002 bool migration_has_failed(MigrationState *s)
2003 {
2004     return (s->state == MIGRATION_STATUS_CANCELLED ||
2005             s->state == MIGRATION_STATUS_FAILED);
2006 }
2007
2008 bool migration_in_postcopy(void)
2009 {
2010     MigrationState *s = migrate_get_current();
2011
2012     switch (s->state) {
2013     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
2014     case MIGRATION_STATUS_POSTCOPY_PAUSED:
2015     case MIGRATION_STATUS_POSTCOPY_RECOVER:
2016         return true;
2017     default:
2018         return false;
2019     }
2020 }
2021
2022 bool migration_in_postcopy_after_devices(MigrationState *s)
2023 {
2024     return migration_in_postcopy() && s->postcopy_after_devices;
2025 }
2026
2027 bool migration_in_incoming_postcopy(void)
2028 {
2029     PostcopyState ps = postcopy_state_get();
2030
2031     return ps >= POSTCOPY_INCOMING_DISCARD && ps < POSTCOPY_INCOMING_END;
2032 }
2033
2034 bool migration_in_bg_snapshot(void)
2035 {
2036     MigrationState *s = migrate_get_current();
2037
2038     return migrate_background_snapshot() &&
2039             migration_is_setup_or_active(s->state);
2040 }
2041
2042 bool migration_is_idle(void)
2043 {
2044     MigrationState *s = current_migration;
2045
2046     if (!s) {
2047         return true;
2048     }
2049
2050     switch (s->state) {
2051     case MIGRATION_STATUS_NONE:
2052     case MIGRATION_STATUS_CANCELLED:
2053     case MIGRATION_STATUS_COMPLETED:
2054     case MIGRATION_STATUS_FAILED:
2055         return true;
2056     case MIGRATION_STATUS_SETUP:
2057     case MIGRATION_STATUS_CANCELLING:
2058     case MIGRATION_STATUS_ACTIVE:
2059     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
2060     case MIGRATION_STATUS_COLO:
2061     case MIGRATION_STATUS_PRE_SWITCHOVER:
2062     case MIGRATION_STATUS_DEVICE:
2063     case MIGRATION_STATUS_WAIT_UNPLUG:
2064         return false;
2065     case MIGRATION_STATUS__MAX:
2066         g_assert_not_reached();
2067     }
2068
2069     return false;
2070 }
2071
2072 bool migration_is_active(MigrationState *s)
2073 {
2074     return (s->state == MIGRATION_STATUS_ACTIVE ||
2075             s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
2076 }
2077
2078 void migrate_init(MigrationState *s)
2079 {
2080     /*
2081      * Reinitialise all migration state, except
2082      * parameters/capabilities that the user set, and
2083      * locks.
2084      */
2085     s->cleanup_bh = 0;
2086     s->vm_start_bh = 0;
2087     s->to_dst_file = NULL;
2088     s->state = MIGRATION_STATUS_NONE;
2089     s->rp_state.from_dst_file = NULL;
2090     s->rp_state.error = false;
2091     s->mbps = 0.0;
2092     s->pages_per_second = 0.0;
2093     s->downtime = 0;
2094     s->expected_downtime = 0;
2095     s->setup_time = 0;
2096     s->start_postcopy = false;
2097     s->postcopy_after_devices = false;
2098     s->migration_thread_running = false;
2099     error_free(s->error);
2100     s->error = NULL;
2101     s->hostname = NULL;
2102
2103     migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
2104
2105     s->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2106     s->total_time = 0;
2107     s->vm_was_running = false;
2108     s->iteration_initial_bytes = 0;
2109     s->threshold_size = 0;
2110 }
2111
2112 int migrate_add_blocker_internal(Error *reason, Error **errp)
2113 {
2114     /* Snapshots are similar to migrations, so check RUN_STATE_SAVE_VM too. */
2115     if (runstate_check(RUN_STATE_SAVE_VM) || !migration_is_idle()) {
2116         error_propagate_prepend(errp, error_copy(reason),
2117                                 "disallowing migration blocker "
2118                                 "(migration/snapshot in progress) for: ");
2119         return -EBUSY;
2120     }
2121
2122     migration_blockers = g_slist_prepend(migration_blockers, reason);
2123     return 0;
2124 }
2125
2126 int migrate_add_blocker(Error *reason, Error **errp)
2127 {
2128     if (only_migratable) {
2129         error_propagate_prepend(errp, error_copy(reason),
2130                                 "disallowing migration blocker "
2131                                 "(--only-migratable) for: ");
2132         return -EACCES;
2133     }
2134
2135     return migrate_add_blocker_internal(reason, errp);
2136 }
2137
2138 void migrate_del_blocker(Error *reason)
2139 {
2140     migration_blockers = g_slist_remove(migration_blockers, reason);
2141 }
2142
2143 void qmp_migrate_incoming(const char *uri, Error **errp)
2144 {
2145     Error *local_err = NULL;
2146     static bool once = true;
2147
2148     if (!once) {
2149         error_setg(errp, "The incoming migration has already been started");
2150         return;
2151     }
2152     if (!runstate_check(RUN_STATE_INMIGRATE)) {
2153         error_setg(errp, "'-incoming' was not specified on the command line");
2154         return;
2155     }
2156
2157     if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2158         return;
2159     }
2160
2161     qemu_start_incoming_migration(uri, &local_err);
2162
2163     if (local_err) {
2164         yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2165         error_propagate(errp, local_err);
2166         return;
2167     }
2168
2169     once = false;
2170 }
2171
2172 void qmp_migrate_recover(const char *uri, Error **errp)
2173 {
2174     MigrationIncomingState *mis = migration_incoming_get_current();
2175
2176     /*
2177      * Don't even bother to use ERRP_GUARD() as it _must_ always be set by
2178      * callers (no one should ignore a recover failure); if there is, it's a
2179      * programming error.
2180      */
2181     assert(errp);
2182
2183     if (mis->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2184         error_setg(errp, "Migrate recover can only be run "
2185                    "when postcopy is paused.");
2186         return;
2187     }
2188
2189     /* If there's an existing transport, release it */
2190     migration_incoming_transport_cleanup(mis);
2191
2192     /*
2193      * Note that this call will never start a real migration; it will
2194      * only re-setup the migration stream and poke existing migration
2195      * to continue using that newly established channel.
2196      */
2197     qemu_start_incoming_migration(uri, errp);
2198 }
2199
2200 void qmp_migrate_pause(Error **errp)
2201 {
2202     MigrationState *ms = migrate_get_current();
2203     MigrationIncomingState *mis = migration_incoming_get_current();
2204     int ret;
2205
2206     if (ms->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2207         /* Source side, during postcopy */
2208         qemu_mutex_lock(&ms->qemu_file_lock);
2209         ret = qemu_file_shutdown(ms->to_dst_file);
2210         qemu_mutex_unlock(&ms->qemu_file_lock);
2211         if (ret) {
2212             error_setg(errp, "Failed to pause source migration");
2213         }
2214         return;
2215     }
2216
2217     if (mis->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2218         ret = qemu_file_shutdown(mis->from_src_file);
2219         if (ret) {
2220             error_setg(errp, "Failed to pause destination migration");
2221         }
2222         return;
2223     }
2224
2225     error_setg(errp, "migrate-pause is currently only supported "
2226                "during postcopy-active state");
2227 }
2228
2229 bool migration_is_blocked(Error **errp)
2230 {
2231     if (qemu_savevm_state_blocked(errp)) {
2232         return true;
2233     }
2234
2235     if (migration_blockers) {
2236         error_propagate(errp, error_copy(migration_blockers->data));
2237         return true;
2238     }
2239
2240     return false;
2241 }
2242
2243 /* Returns true if continue to migrate, or false if error detected */
2244 static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
2245                             bool resume, Error **errp)
2246 {
2247     Error *local_err = NULL;
2248
2249     if (resume) {
2250         if (s->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2251             error_setg(errp, "Cannot resume if there is no "
2252                        "paused migration");
2253             return false;
2254         }
2255
2256         /*
2257          * Postcopy recovery won't work well with release-ram
2258          * capability since release-ram will drop the page buffer as
2259          * long as the page is put into the send buffer.  So if there
2260          * is a network failure happened, any page buffers that have
2261          * not yet reached the destination VM but have already been
2262          * sent from the source VM will be lost forever.  Let's refuse
2263          * the client from resuming such a postcopy migration.
2264          * Luckily release-ram was designed to only be used when src
2265          * and destination VMs are on the same host, so it should be
2266          * fine.
2267          */
2268         if (migrate_release_ram()) {
2269             error_setg(errp, "Postcopy recovery cannot work "
2270                        "when release-ram capability is set");
2271             return false;
2272         }
2273
2274         /* This is a resume, skip init status */
2275         return true;
2276     }
2277
2278     if (migration_is_running(s->state)) {
2279         error_setg(errp, QERR_MIGRATION_ACTIVE);
2280         return false;
2281     }
2282
2283     if (runstate_check(RUN_STATE_INMIGRATE)) {
2284         error_setg(errp, "Guest is waiting for an incoming migration");
2285         return false;
2286     }
2287
2288     if (runstate_check(RUN_STATE_POSTMIGRATE)) {
2289         error_setg(errp, "Can't migrate the vm that was paused due to "
2290                    "previous migration");
2291         return false;
2292     }
2293
2294     if (migration_is_blocked(errp)) {
2295         return false;
2296     }
2297
2298     if (blk || blk_inc) {
2299         if (migrate_colo_enabled()) {
2300             error_setg(errp, "No disk migration is required in COLO mode");
2301             return false;
2302         }
2303         if (migrate_use_block() || migrate_use_block_incremental()) {
2304             error_setg(errp, "Command options are incompatible with "
2305                        "current migration capabilities");
2306             return false;
2307         }
2308         migrate_set_block_enabled(true, &local_err);
2309         if (local_err) {
2310             error_propagate(errp, local_err);
2311             return false;
2312         }
2313         s->must_remove_block_options = true;
2314     }
2315
2316     if (blk_inc) {
2317         migrate_set_block_incremental(s, true);
2318     }
2319
2320     migrate_init(s);
2321     /*
2322      * set ram_counters compression_counters memory to zero for a
2323      * new migration
2324      */
2325     memset(&ram_counters, 0, sizeof(ram_counters));
2326     memset(&compression_counters, 0, sizeof(compression_counters));
2327
2328     return true;
2329 }
2330
2331 void qmp_migrate(const char *uri, bool has_blk, bool blk,
2332                  bool has_inc, bool inc, bool has_detach, bool detach,
2333                  bool has_resume, bool resume, Error **errp)
2334 {
2335     Error *local_err = NULL;
2336     MigrationState *s = migrate_get_current();
2337     const char *p = NULL;
2338
2339     if (!migrate_prepare(s, has_blk && blk, has_inc && inc,
2340                          has_resume && resume, errp)) {
2341         /* Error detected, put into errp */
2342         return;
2343     }
2344
2345     if (!(has_resume && resume)) {
2346         if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2347             return;
2348         }
2349     }
2350
2351     migrate_protocol_allow_multi_channels(false);
2352     if (strstart(uri, "tcp:", &p) ||
2353         strstart(uri, "unix:", NULL) ||
2354         strstart(uri, "vsock:", NULL)) {
2355         migrate_protocol_allow_multi_channels(true);
2356         socket_start_outgoing_migration(s, p ? p : uri, &local_err);
2357 #ifdef CONFIG_RDMA
2358     } else if (strstart(uri, "rdma:", &p)) {
2359         rdma_start_outgoing_migration(s, p, &local_err);
2360 #endif
2361     } else if (strstart(uri, "exec:", &p)) {
2362         exec_start_outgoing_migration(s, p, &local_err);
2363     } else if (strstart(uri, "fd:", &p)) {
2364         fd_start_outgoing_migration(s, p, &local_err);
2365     } else {
2366         if (!(has_resume && resume)) {
2367             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2368         }
2369         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri",
2370                    "a valid migration protocol");
2371         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
2372                           MIGRATION_STATUS_FAILED);
2373         block_cleanup_parameters(s);
2374         return;
2375     }
2376
2377     if (local_err) {
2378         if (!(has_resume && resume)) {
2379             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2380         }
2381         migrate_fd_error(s, local_err);
2382         error_propagate(errp, local_err);
2383         return;
2384     }
2385 }
2386
2387 void qmp_migrate_cancel(Error **errp)
2388 {
2389     migration_cancel(NULL);
2390 }
2391
2392 void qmp_migrate_continue(MigrationStatus state, Error **errp)
2393 {
2394     MigrationState *s = migrate_get_current();
2395     if (s->state != state) {
2396         error_setg(errp,  "Migration not in expected state: %s",
2397                    MigrationStatus_str(s->state));
2398         return;
2399     }
2400     qemu_sem_post(&s->pause_sem);
2401 }
2402
2403 bool migrate_release_ram(void)
2404 {
2405     MigrationState *s;
2406
2407     s = migrate_get_current();
2408
2409     return s->enabled_capabilities[MIGRATION_CAPABILITY_RELEASE_RAM];
2410 }
2411
2412 bool migrate_postcopy_ram(void)
2413 {
2414     MigrationState *s;
2415
2416     s = migrate_get_current();
2417
2418     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM];
2419 }
2420
2421 bool migrate_postcopy(void)
2422 {
2423     return migrate_postcopy_ram() || migrate_dirty_bitmaps();
2424 }
2425
2426 bool migrate_auto_converge(void)
2427 {
2428     MigrationState *s;
2429
2430     s = migrate_get_current();
2431
2432     return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
2433 }
2434
2435 bool migrate_zero_blocks(void)
2436 {
2437     MigrationState *s;
2438
2439     s = migrate_get_current();
2440
2441     return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
2442 }
2443
2444 bool migrate_postcopy_blocktime(void)
2445 {
2446     MigrationState *s;
2447
2448     s = migrate_get_current();
2449
2450     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME];
2451 }
2452
2453 bool migrate_use_compression(void)
2454 {
2455     MigrationState *s;
2456
2457     s = migrate_get_current();
2458
2459     return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS];
2460 }
2461
2462 int migrate_compress_level(void)
2463 {
2464     MigrationState *s;
2465
2466     s = migrate_get_current();
2467
2468     return s->parameters.compress_level;
2469 }
2470
2471 int migrate_compress_threads(void)
2472 {
2473     MigrationState *s;
2474
2475     s = migrate_get_current();
2476
2477     return s->parameters.compress_threads;
2478 }
2479
2480 int migrate_compress_wait_thread(void)
2481 {
2482     MigrationState *s;
2483
2484     s = migrate_get_current();
2485
2486     return s->parameters.compress_wait_thread;
2487 }
2488
2489 int migrate_decompress_threads(void)
2490 {
2491     MigrationState *s;
2492
2493     s = migrate_get_current();
2494
2495     return s->parameters.decompress_threads;
2496 }
2497
2498 bool migrate_dirty_bitmaps(void)
2499 {
2500     MigrationState *s;
2501
2502     s = migrate_get_current();
2503
2504     return s->enabled_capabilities[MIGRATION_CAPABILITY_DIRTY_BITMAPS];
2505 }
2506
2507 bool migrate_ignore_shared(void)
2508 {
2509     MigrationState *s;
2510
2511     s = migrate_get_current();
2512
2513     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_IGNORE_SHARED];
2514 }
2515
2516 bool migrate_validate_uuid(void)
2517 {
2518     MigrationState *s;
2519
2520     s = migrate_get_current();
2521
2522     return s->enabled_capabilities[MIGRATION_CAPABILITY_VALIDATE_UUID];
2523 }
2524
2525 bool migrate_use_events(void)
2526 {
2527     MigrationState *s;
2528
2529     s = migrate_get_current();
2530
2531     return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS];
2532 }
2533
2534 bool migrate_use_multifd(void)
2535 {
2536     MigrationState *s;
2537
2538     s = migrate_get_current();
2539
2540     return s->enabled_capabilities[MIGRATION_CAPABILITY_MULTIFD];
2541 }
2542
2543 bool migrate_pause_before_switchover(void)
2544 {
2545     MigrationState *s;
2546
2547     s = migrate_get_current();
2548
2549     return s->enabled_capabilities[
2550         MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER];
2551 }
2552
2553 int migrate_multifd_channels(void)
2554 {
2555     MigrationState *s;
2556
2557     s = migrate_get_current();
2558
2559     return s->parameters.multifd_channels;
2560 }
2561
2562 MultiFDCompression migrate_multifd_compression(void)
2563 {
2564     MigrationState *s;
2565
2566     s = migrate_get_current();
2567
2568     return s->parameters.multifd_compression;
2569 }
2570
2571 int migrate_multifd_zlib_level(void)
2572 {
2573     MigrationState *s;
2574
2575     s = migrate_get_current();
2576
2577     return s->parameters.multifd_zlib_level;
2578 }
2579
2580 int migrate_multifd_zstd_level(void)
2581 {
2582     MigrationState *s;
2583
2584     s = migrate_get_current();
2585
2586     return s->parameters.multifd_zstd_level;
2587 }
2588
2589 #ifdef CONFIG_LINUX
2590 bool migrate_use_zero_copy_send(void)
2591 {
2592     MigrationState *s;
2593
2594     s = migrate_get_current();
2595
2596     return s->parameters.zero_copy_send;
2597 }
2598 #endif
2599
2600 int migrate_use_tls(void)
2601 {
2602     MigrationState *s;
2603
2604     s = migrate_get_current();
2605
2606     return s->parameters.tls_creds && *s->parameters.tls_creds;
2607 }
2608
2609 int migrate_use_xbzrle(void)
2610 {
2611     MigrationState *s;
2612
2613     s = migrate_get_current();
2614
2615     return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
2616 }
2617
2618 uint64_t migrate_xbzrle_cache_size(void)
2619 {
2620     MigrationState *s;
2621
2622     s = migrate_get_current();
2623
2624     return s->parameters.xbzrle_cache_size;
2625 }
2626
2627 static int64_t migrate_max_postcopy_bandwidth(void)
2628 {
2629     MigrationState *s;
2630
2631     s = migrate_get_current();
2632
2633     return s->parameters.max_postcopy_bandwidth;
2634 }
2635
2636 bool migrate_use_block(void)
2637 {
2638     MigrationState *s;
2639
2640     s = migrate_get_current();
2641
2642     return s->enabled_capabilities[MIGRATION_CAPABILITY_BLOCK];
2643 }
2644
2645 bool migrate_use_return_path(void)
2646 {
2647     MigrationState *s;
2648
2649     s = migrate_get_current();
2650
2651     return s->enabled_capabilities[MIGRATION_CAPABILITY_RETURN_PATH];
2652 }
2653
2654 bool migrate_use_block_incremental(void)
2655 {
2656     MigrationState *s;
2657
2658     s = migrate_get_current();
2659
2660     return s->parameters.block_incremental;
2661 }
2662
2663 bool migrate_background_snapshot(void)
2664 {
2665     MigrationState *s;
2666
2667     s = migrate_get_current();
2668
2669     return s->enabled_capabilities[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT];
2670 }
2671
2672 /* migration thread support */
2673 /*
2674  * Something bad happened to the RP stream, mark an error
2675  * The caller shall print or trace something to indicate why
2676  */
2677 static void mark_source_rp_bad(MigrationState *s)
2678 {
2679     s->rp_state.error = true;
2680 }
2681
2682 static struct rp_cmd_args {
2683     ssize_t     len; /* -1 = variable */
2684     const char *name;
2685 } rp_cmd_args[] = {
2686     [MIG_RP_MSG_INVALID]        = { .len = -1, .name = "INVALID" },
2687     [MIG_RP_MSG_SHUT]           = { .len =  4, .name = "SHUT" },
2688     [MIG_RP_MSG_PONG]           = { .len =  4, .name = "PONG" },
2689     [MIG_RP_MSG_REQ_PAGES]      = { .len = 12, .name = "REQ_PAGES" },
2690     [MIG_RP_MSG_REQ_PAGES_ID]   = { .len = -1, .name = "REQ_PAGES_ID" },
2691     [MIG_RP_MSG_RECV_BITMAP]    = { .len = -1, .name = "RECV_BITMAP" },
2692     [MIG_RP_MSG_RESUME_ACK]     = { .len =  4, .name = "RESUME_ACK" },
2693     [MIG_RP_MSG_MAX]            = { .len = -1, .name = "MAX" },
2694 };
2695
2696 /*
2697  * Process a request for pages received on the return path,
2698  * We're allowed to send more than requested (e.g. to round to our page size)
2699  * and we don't need to send pages that have already been sent.
2700  */
2701 static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
2702                                        ram_addr_t start, size_t len)
2703 {
2704     long our_host_ps = qemu_real_host_page_size();
2705
2706     trace_migrate_handle_rp_req_pages(rbname, start, len);
2707
2708     /*
2709      * Since we currently insist on matching page sizes, just sanity check
2710      * we're being asked for whole host pages.
2711      */
2712     if (!QEMU_IS_ALIGNED(start, our_host_ps) ||
2713         !QEMU_IS_ALIGNED(len, our_host_ps)) {
2714         error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
2715                      " len: %zd", __func__, start, len);
2716         mark_source_rp_bad(ms);
2717         return;
2718     }
2719
2720     if (ram_save_queue_pages(rbname, start, len)) {
2721         mark_source_rp_bad(ms);
2722     }
2723 }
2724
2725 /* Return true to retry, false to quit */
2726 static bool postcopy_pause_return_path_thread(MigrationState *s)
2727 {
2728     trace_postcopy_pause_return_path();
2729
2730     qemu_sem_wait(&s->postcopy_pause_rp_sem);
2731
2732     trace_postcopy_pause_return_path_continued();
2733
2734     return true;
2735 }
2736
2737 static int migrate_handle_rp_recv_bitmap(MigrationState *s, char *block_name)
2738 {
2739     RAMBlock *block = qemu_ram_block_by_name(block_name);
2740
2741     if (!block) {
2742         error_report("%s: invalid block name '%s'", __func__, block_name);
2743         return -EINVAL;
2744     }
2745
2746     /* Fetch the received bitmap and refresh the dirty bitmap */
2747     return ram_dirty_bitmap_reload(s, block);
2748 }
2749
2750 static int migrate_handle_rp_resume_ack(MigrationState *s, uint32_t value)
2751 {
2752     trace_source_return_path_thread_resume_ack(value);
2753
2754     if (value != MIGRATION_RESUME_ACK_VALUE) {
2755         error_report("%s: illegal resume_ack value %"PRIu32,
2756                      __func__, value);
2757         return -1;
2758     }
2759
2760     /* Now both sides are active. */
2761     migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_RECOVER,
2762                       MIGRATION_STATUS_POSTCOPY_ACTIVE);
2763
2764     /* Notify send thread that time to continue send pages */
2765     qemu_sem_post(&s->rp_state.rp_sem);
2766
2767     return 0;
2768 }
2769
2770 /* Release ms->rp_state.from_dst_file in a safe way */
2771 static void migration_release_from_dst_file(MigrationState *ms)
2772 {
2773     QEMUFile *file;
2774
2775     WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) {
2776         /*
2777          * Reset the from_dst_file pointer first before releasing it, as we
2778          * can't block within lock section
2779          */
2780         file = ms->rp_state.from_dst_file;
2781         ms->rp_state.from_dst_file = NULL;
2782     }
2783
2784     qemu_fclose(file);
2785 }
2786
2787 /*
2788  * Handles messages sent on the return path towards the source VM
2789  *
2790  */
2791 static void *source_return_path_thread(void *opaque)
2792 {
2793     MigrationState *ms = opaque;
2794     QEMUFile *rp = ms->rp_state.from_dst_file;
2795     uint16_t header_len, header_type;
2796     uint8_t buf[512];
2797     uint32_t tmp32, sibling_error;
2798     ram_addr_t start = 0; /* =0 to silence warning */
2799     size_t  len = 0, expected_len;
2800     int res;
2801
2802     trace_source_return_path_thread_entry();
2803     rcu_register_thread();
2804
2805 retry:
2806     while (!ms->rp_state.error && !qemu_file_get_error(rp) &&
2807            migration_is_setup_or_active(ms->state)) {
2808         trace_source_return_path_thread_loop_top();
2809         header_type = qemu_get_be16(rp);
2810         header_len = qemu_get_be16(rp);
2811
2812         if (qemu_file_get_error(rp)) {
2813             mark_source_rp_bad(ms);
2814             goto out;
2815         }
2816
2817         if (header_type >= MIG_RP_MSG_MAX ||
2818             header_type == MIG_RP_MSG_INVALID) {
2819             error_report("RP: Received invalid message 0x%04x length 0x%04x",
2820                          header_type, header_len);
2821             mark_source_rp_bad(ms);
2822             goto out;
2823         }
2824
2825         if ((rp_cmd_args[header_type].len != -1 &&
2826             header_len != rp_cmd_args[header_type].len) ||
2827             header_len > sizeof(buf)) {
2828             error_report("RP: Received '%s' message (0x%04x) with"
2829                          "incorrect length %d expecting %zu",
2830                          rp_cmd_args[header_type].name, header_type, header_len,
2831                          (size_t)rp_cmd_args[header_type].len);
2832             mark_source_rp_bad(ms);
2833             goto out;
2834         }
2835
2836         /* We know we've got a valid header by this point */
2837         res = qemu_get_buffer(rp, buf, header_len);
2838         if (res != header_len) {
2839             error_report("RP: Failed reading data for message 0x%04x"
2840                          " read %d expected %d",
2841                          header_type, res, header_len);
2842             mark_source_rp_bad(ms);
2843             goto out;
2844         }
2845
2846         /* OK, we have the message and the data */
2847         switch (header_type) {
2848         case MIG_RP_MSG_SHUT:
2849             sibling_error = ldl_be_p(buf);
2850             trace_source_return_path_thread_shut(sibling_error);
2851             if (sibling_error) {
2852                 error_report("RP: Sibling indicated error %d", sibling_error);
2853                 mark_source_rp_bad(ms);
2854             }
2855             /*
2856              * We'll let the main thread deal with closing the RP
2857              * we could do a shutdown(2) on it, but we're the only user
2858              * anyway, so there's nothing gained.
2859              */
2860             goto out;
2861
2862         case MIG_RP_MSG_PONG:
2863             tmp32 = ldl_be_p(buf);
2864             trace_source_return_path_thread_pong(tmp32);
2865             break;
2866
2867         case MIG_RP_MSG_REQ_PAGES:
2868             start = ldq_be_p(buf);
2869             len = ldl_be_p(buf + 8);
2870             migrate_handle_rp_req_pages(ms, NULL, start, len);
2871             break;
2872
2873         case MIG_RP_MSG_REQ_PAGES_ID:
2874             expected_len = 12 + 1; /* header + termination */
2875
2876             if (header_len >= expected_len) {
2877                 start = ldq_be_p(buf);
2878                 len = ldl_be_p(buf + 8);
2879                 /* Now we expect an idstr */
2880                 tmp32 = buf[12]; /* Length of the following idstr */
2881                 buf[13 + tmp32] = '\0';
2882                 expected_len += tmp32;
2883             }
2884             if (header_len != expected_len) {
2885                 error_report("RP: Req_Page_id with length %d expecting %zd",
2886                              header_len, expected_len);
2887                 mark_source_rp_bad(ms);
2888                 goto out;
2889             }
2890             migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len);
2891             break;
2892
2893         case MIG_RP_MSG_RECV_BITMAP:
2894             if (header_len < 1) {
2895                 error_report("%s: missing block name", __func__);
2896                 mark_source_rp_bad(ms);
2897                 goto out;
2898             }
2899             /* Format: len (1B) + idstr (<255B). This ends the idstr. */
2900             buf[buf[0] + 1] = '\0';
2901             if (migrate_handle_rp_recv_bitmap(ms, (char *)(buf + 1))) {
2902                 mark_source_rp_bad(ms);
2903                 goto out;
2904             }
2905             break;
2906
2907         case MIG_RP_MSG_RESUME_ACK:
2908             tmp32 = ldl_be_p(buf);
2909             if (migrate_handle_rp_resume_ack(ms, tmp32)) {
2910                 mark_source_rp_bad(ms);
2911                 goto out;
2912             }
2913             break;
2914
2915         default:
2916             break;
2917         }
2918     }
2919
2920 out:
2921     res = qemu_file_get_error(rp);
2922     if (res) {
2923         if (res && migration_in_postcopy()) {
2924             /*
2925              * Maybe there is something we can do: it looks like a
2926              * network down issue, and we pause for a recovery.
2927              */
2928             migration_release_from_dst_file(ms);
2929             rp = NULL;
2930             if (postcopy_pause_return_path_thread(ms)) {
2931                 /*
2932                  * Reload rp, reset the rest.  Referencing it is safe since
2933                  * it's reset only by us above, or when migration completes
2934                  */
2935                 rp = ms->rp_state.from_dst_file;
2936                 ms->rp_state.error = false;
2937                 goto retry;
2938             }
2939         }
2940
2941         trace_source_return_path_thread_bad_end();
2942         mark_source_rp_bad(ms);
2943     }
2944
2945     trace_source_return_path_thread_end();
2946     migration_release_from_dst_file(ms);
2947     rcu_unregister_thread();
2948     return NULL;
2949 }
2950
2951 static int open_return_path_on_source(MigrationState *ms,
2952                                       bool create_thread)
2953 {
2954     ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file);
2955     if (!ms->rp_state.from_dst_file) {
2956         return -1;
2957     }
2958
2959     trace_open_return_path_on_source();
2960
2961     if (!create_thread) {
2962         /* We're done */
2963         return 0;
2964     }
2965
2966     qemu_thread_create(&ms->rp_state.rp_thread, "return path",
2967                        source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
2968     ms->rp_state.rp_thread_created = true;
2969
2970     trace_open_return_path_on_source_continue();
2971
2972     return 0;
2973 }
2974
2975 /* Returns 0 if the RP was ok, otherwise there was an error on the RP */
2976 static int await_return_path_close_on_source(MigrationState *ms)
2977 {
2978     /*
2979      * If this is a normal exit then the destination will send a SHUT and the
2980      * rp_thread will exit, however if there's an error we need to cause
2981      * it to exit.
2982      */
2983     if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) {
2984         /*
2985          * shutdown(2), if we have it, will cause it to unblock if it's stuck
2986          * waiting for the destination.
2987          */
2988         qemu_file_shutdown(ms->rp_state.from_dst_file);
2989         mark_source_rp_bad(ms);
2990     }
2991     trace_await_return_path_close_on_source_joining();
2992     qemu_thread_join(&ms->rp_state.rp_thread);
2993     ms->rp_state.rp_thread_created = false;
2994     trace_await_return_path_close_on_source_close();
2995     return ms->rp_state.error;
2996 }
2997
2998 /*
2999  * Switch from normal iteration to postcopy
3000  * Returns non-0 on error
3001  */
3002 static int postcopy_start(MigrationState *ms)
3003 {
3004     int ret;
3005     QIOChannelBuffer *bioc;
3006     QEMUFile *fb;
3007     int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3008     int64_t bandwidth = migrate_max_postcopy_bandwidth();
3009     bool restart_block = false;
3010     int cur_state = MIGRATION_STATUS_ACTIVE;
3011     if (!migrate_pause_before_switchover()) {
3012         migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
3013                           MIGRATION_STATUS_POSTCOPY_ACTIVE);
3014     }
3015
3016     trace_postcopy_start();
3017     qemu_mutex_lock_iothread();
3018     trace_postcopy_start_set_run();
3019
3020     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3021     global_state_store();
3022     ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
3023     if (ret < 0) {
3024         goto fail;
3025     }
3026
3027     ret = migration_maybe_pause(ms, &cur_state,
3028                                 MIGRATION_STATUS_POSTCOPY_ACTIVE);
3029     if (ret < 0) {
3030         goto fail;
3031     }
3032
3033     ret = bdrv_inactivate_all();
3034     if (ret < 0) {
3035         goto fail;
3036     }
3037     restart_block = true;
3038
3039     /*
3040      * Cause any non-postcopiable, but iterative devices to
3041      * send out their final data.
3042      */
3043     qemu_savevm_state_complete_precopy(ms->to_dst_file, true, false);
3044
3045     /*
3046      * in Finish migrate and with the io-lock held everything should
3047      * be quiet, but we've potentially still got dirty pages and we
3048      * need to tell the destination to throw any pages it's already received
3049      * that are dirty
3050      */
3051     if (migrate_postcopy_ram()) {
3052         ram_postcopy_send_discard_bitmap(ms);
3053     }
3054
3055     /*
3056      * send rest of state - note things that are doing postcopy
3057      * will notice we're in POSTCOPY_ACTIVE and not actually
3058      * wrap their state up here
3059      */
3060     /* 0 max-postcopy-bandwidth means unlimited */
3061     if (!bandwidth) {
3062         qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX);
3063     } else {
3064         qemu_file_set_rate_limit(ms->to_dst_file, bandwidth / XFER_LIMIT_RATIO);
3065     }
3066     if (migrate_postcopy_ram()) {
3067         /* Ping just for debugging, helps line traces up */
3068         qemu_savevm_send_ping(ms->to_dst_file, 2);
3069     }
3070
3071     /*
3072      * While loading the device state we may trigger page transfer
3073      * requests and the fd must be free to process those, and thus
3074      * the destination must read the whole device state off the fd before
3075      * it starts processing it.  Unfortunately the ad-hoc migration format
3076      * doesn't allow the destination to know the size to read without fully
3077      * parsing it through each devices load-state code (especially the open
3078      * coded devices that use get/put).
3079      * So we wrap the device state up in a package with a length at the start;
3080      * to do this we use a qemu_buf to hold the whole of the device state.
3081      */
3082     bioc = qio_channel_buffer_new(4096);
3083     qio_channel_set_name(QIO_CHANNEL(bioc), "migration-postcopy-buffer");
3084     fb = qemu_fopen_channel_output(QIO_CHANNEL(bioc));
3085     object_unref(OBJECT(bioc));
3086
3087     /*
3088      * Make sure the receiver can get incoming pages before we send the rest
3089      * of the state
3090      */
3091     qemu_savevm_send_postcopy_listen(fb);
3092
3093     qemu_savevm_state_complete_precopy(fb, false, false);
3094     if (migrate_postcopy_ram()) {
3095         qemu_savevm_send_ping(fb, 3);
3096     }
3097
3098     qemu_savevm_send_postcopy_run(fb);
3099
3100     /* <><> end of stuff going into the package */
3101
3102     /* Last point of recovery; as soon as we send the package the destination
3103      * can open devices and potentially start running.
3104      * Lets just check again we've not got any errors.
3105      */
3106     ret = qemu_file_get_error(ms->to_dst_file);
3107     if (ret) {
3108         error_report("postcopy_start: Migration stream errored (pre package)");
3109         goto fail_closefb;
3110     }
3111
3112     restart_block = false;
3113
3114     /* Now send that blob */
3115     if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
3116         goto fail_closefb;
3117     }
3118     qemu_fclose(fb);
3119
3120     /* Send a notify to give a chance for anything that needs to happen
3121      * at the transition to postcopy and after the device state; in particular
3122      * spice needs to trigger a transition now
3123      */
3124     ms->postcopy_after_devices = true;
3125     notifier_list_notify(&migration_state_notifiers, ms);
3126
3127     ms->downtime =  qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop;
3128
3129     qemu_mutex_unlock_iothread();
3130
3131     if (migrate_postcopy_ram()) {
3132         /*
3133          * Although this ping is just for debug, it could potentially be
3134          * used for getting a better measurement of downtime at the source.
3135          */
3136         qemu_savevm_send_ping(ms->to_dst_file, 4);
3137     }
3138
3139     if (migrate_release_ram()) {
3140         ram_postcopy_migrated_memory_release(ms);
3141     }
3142
3143     ret = qemu_file_get_error(ms->to_dst_file);
3144     if (ret) {
3145         error_report("postcopy_start: Migration stream errored");
3146         migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3147                               MIGRATION_STATUS_FAILED);
3148     }
3149
3150     return ret;
3151
3152 fail_closefb:
3153     qemu_fclose(fb);
3154 fail:
3155     migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3156                           MIGRATION_STATUS_FAILED);
3157     if (restart_block) {
3158         /* A failure happened early enough that we know the destination hasn't
3159          * accessed block devices, so we're safe to recover.
3160          */
3161         Error *local_err = NULL;
3162
3163         bdrv_activate_all(&local_err);
3164         if (local_err) {
3165             error_report_err(local_err);
3166         }
3167     }
3168     qemu_mutex_unlock_iothread();
3169     return -1;
3170 }
3171
3172 /**
3173  * migration_maybe_pause: Pause if required to by
3174  * migrate_pause_before_switchover called with the iothread locked
3175  * Returns: 0 on success
3176  */
3177 static int migration_maybe_pause(MigrationState *s,
3178                                  int *current_active_state,
3179                                  int new_state)
3180 {
3181     if (!migrate_pause_before_switchover()) {
3182         return 0;
3183     }
3184
3185     /* Since leaving this state is not atomic with posting the semaphore
3186      * it's possible that someone could have issued multiple migrate_continue
3187      * and the semaphore is incorrectly positive at this point;
3188      * the docs say it's undefined to reinit a semaphore that's already
3189      * init'd, so use timedwait to eat up any existing posts.
3190      */
3191     while (qemu_sem_timedwait(&s->pause_sem, 1) == 0) {
3192         /* This block intentionally left blank */
3193     }
3194
3195     /*
3196      * If the migration is cancelled when it is in the completion phase,
3197      * the migration state is set to MIGRATION_STATUS_CANCELLING.
3198      * So we don't need to wait a semaphore, otherwise we would always
3199      * wait for the 'pause_sem' semaphore.
3200      */
3201     if (s->state != MIGRATION_STATUS_CANCELLING) {
3202         qemu_mutex_unlock_iothread();
3203         migrate_set_state(&s->state, *current_active_state,
3204                           MIGRATION_STATUS_PRE_SWITCHOVER);
3205         qemu_sem_wait(&s->pause_sem);
3206         migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER,
3207                           new_state);
3208         *current_active_state = new_state;
3209         qemu_mutex_lock_iothread();
3210     }
3211
3212     return s->state == new_state ? 0 : -EINVAL;
3213 }
3214
3215 /**
3216  * migration_completion: Used by migration_thread when there's not much left.
3217  *   The caller 'breaks' the loop when this returns.
3218  *
3219  * @s: Current migration state
3220  */
3221 static void migration_completion(MigrationState *s)
3222 {
3223     int ret;
3224     int current_active_state = s->state;
3225
3226     if (s->state == MIGRATION_STATUS_ACTIVE) {
3227         qemu_mutex_lock_iothread();
3228         s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3229         qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3230         s->vm_was_running = runstate_is_running();
3231         ret = global_state_store();
3232
3233         if (!ret) {
3234             bool inactivate = !migrate_colo_enabled();
3235             ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
3236             trace_migration_completion_vm_stop(ret);
3237             if (ret >= 0) {
3238                 ret = migration_maybe_pause(s, &current_active_state,
3239                                             MIGRATION_STATUS_DEVICE);
3240             }
3241             if (ret >= 0) {
3242                 qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3243                 ret = qemu_savevm_state_complete_precopy(s->to_dst_file, false,
3244                                                          inactivate);
3245             }
3246             if (inactivate && ret >= 0) {
3247                 s->block_inactive = true;
3248             }
3249         }
3250         qemu_mutex_unlock_iothread();
3251
3252         if (ret < 0) {
3253             goto fail;
3254         }
3255     } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3256         trace_migration_completion_postcopy_end();
3257
3258         qemu_mutex_lock_iothread();
3259         qemu_savevm_state_complete_postcopy(s->to_dst_file);
3260         qemu_mutex_unlock_iothread();
3261
3262         trace_migration_completion_postcopy_end_after_complete();
3263     } else {
3264         goto fail;
3265     }
3266
3267     /*
3268      * If rp was opened we must clean up the thread before
3269      * cleaning everything else up (since if there are no failures
3270      * it will wait for the destination to send it's status in
3271      * a SHUT command).
3272      */
3273     if (s->rp_state.rp_thread_created) {
3274         int rp_error;
3275         trace_migration_return_path_end_before();
3276         rp_error = await_return_path_close_on_source(s);
3277         trace_migration_return_path_end_after(rp_error);
3278         if (rp_error) {
3279             goto fail_invalidate;
3280         }
3281     }
3282
3283     if (qemu_file_get_error(s->to_dst_file)) {
3284         trace_migration_completion_file_err();
3285         goto fail_invalidate;
3286     }
3287
3288     if (migrate_colo_enabled() && s->state == MIGRATION_STATUS_ACTIVE) {
3289         /* COLO does not support postcopy */
3290         migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE,
3291                           MIGRATION_STATUS_COLO);
3292     } else {
3293         migrate_set_state(&s->state, current_active_state,
3294                           MIGRATION_STATUS_COMPLETED);
3295     }
3296
3297     return;
3298
3299 fail_invalidate:
3300     /* If not doing postcopy, vm_start() will be called: let's regain
3301      * control on images.
3302      */
3303     if (s->state == MIGRATION_STATUS_ACTIVE ||
3304         s->state == MIGRATION_STATUS_DEVICE) {
3305         Error *local_err = NULL;
3306
3307         qemu_mutex_lock_iothread();
3308         bdrv_activate_all(&local_err);
3309         if (local_err) {
3310             error_report_err(local_err);
3311         } else {
3312             s->block_inactive = false;
3313         }
3314         qemu_mutex_unlock_iothread();
3315     }
3316
3317 fail:
3318     migrate_set_state(&s->state, current_active_state,
3319                       MIGRATION_STATUS_FAILED);
3320 }
3321
3322 /**
3323  * bg_migration_completion: Used by bg_migration_thread when after all the
3324  *   RAM has been saved. The caller 'breaks' the loop when this returns.
3325  *
3326  * @s: Current migration state
3327  */
3328 static void bg_migration_completion(MigrationState *s)
3329 {
3330     int current_active_state = s->state;
3331
3332     /*
3333      * Stop tracking RAM writes - un-protect memory, un-register UFFD
3334      * memory ranges, flush kernel wait queues and wake up threads
3335      * waiting for write fault to be resolved.
3336      */
3337     ram_write_tracking_stop();
3338
3339     if (s->state == MIGRATION_STATUS_ACTIVE) {
3340         /*
3341          * By this moment we have RAM content saved into the migration stream.
3342          * The next step is to flush the non-RAM content (device state)
3343          * right after the ram content. The device state has been stored into
3344          * the temporary buffer before RAM saving started.
3345          */
3346         qemu_put_buffer(s->to_dst_file, s->bioc->data, s->bioc->usage);
3347         qemu_fflush(s->to_dst_file);
3348     } else if (s->state == MIGRATION_STATUS_CANCELLING) {
3349         goto fail;
3350     }
3351
3352     if (qemu_file_get_error(s->to_dst_file)) {
3353         trace_migration_completion_file_err();
3354         goto fail;
3355     }
3356
3357     migrate_set_state(&s->state, current_active_state,
3358                       MIGRATION_STATUS_COMPLETED);
3359     return;
3360
3361 fail:
3362     migrate_set_state(&s->state, current_active_state,
3363                       MIGRATION_STATUS_FAILED);
3364 }
3365
3366 bool migrate_colo_enabled(void)
3367 {
3368     MigrationState *s = migrate_get_current();
3369     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_COLO];
3370 }
3371
3372 typedef enum MigThrError {
3373     /* No error detected */
3374     MIG_THR_ERR_NONE = 0,
3375     /* Detected error, but resumed successfully */
3376     MIG_THR_ERR_RECOVERED = 1,
3377     /* Detected fatal error, need to exit */
3378     MIG_THR_ERR_FATAL = 2,
3379 } MigThrError;
3380
3381 static int postcopy_resume_handshake(MigrationState *s)
3382 {
3383     qemu_savevm_send_postcopy_resume(s->to_dst_file);
3384
3385     while (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3386         qemu_sem_wait(&s->rp_state.rp_sem);
3387     }
3388
3389     if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3390         return 0;
3391     }
3392
3393     return -1;
3394 }
3395
3396 /* Return zero if success, or <0 for error */
3397 static int postcopy_do_resume(MigrationState *s)
3398 {
3399     int ret;
3400
3401     /*
3402      * Call all the resume_prepare() hooks, so that modules can be
3403      * ready for the migration resume.
3404      */
3405     ret = qemu_savevm_state_resume_prepare(s);
3406     if (ret) {
3407         error_report("%s: resume_prepare() failure detected: %d",
3408                      __func__, ret);
3409         return ret;
3410     }
3411
3412     /*
3413      * Last handshake with destination on the resume (destination will
3414      * switch to postcopy-active afterwards)
3415      */
3416     ret = postcopy_resume_handshake(s);
3417     if (ret) {
3418         error_report("%s: handshake failed: %d", __func__, ret);
3419         return ret;
3420     }
3421
3422     return 0;
3423 }
3424
3425 /*
3426  * We don't return until we are in a safe state to continue current
3427  * postcopy migration.  Returns MIG_THR_ERR_RECOVERED if recovered, or
3428  * MIG_THR_ERR_FATAL if unrecovery failure happened.
3429  */
3430 static MigThrError postcopy_pause(MigrationState *s)
3431 {
3432     assert(s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
3433
3434     while (true) {
3435         QEMUFile *file;
3436
3437         /*
3438          * Current channel is possibly broken. Release it.  Note that this is
3439          * guaranteed even without lock because to_dst_file should only be
3440          * modified by the migration thread.  That also guarantees that the
3441          * unregister of yank is safe too without the lock.  It should be safe
3442          * even to be within the qemu_file_lock, but we didn't do that to avoid
3443          * taking more mutex (yank_lock) within qemu_file_lock.  TL;DR: we make
3444          * the qemu_file_lock critical section as small as possible.
3445          */
3446         assert(s->to_dst_file);
3447         migration_ioc_unregister_yank_from_file(s->to_dst_file);
3448         qemu_mutex_lock(&s->qemu_file_lock);
3449         file = s->to_dst_file;
3450         s->to_dst_file = NULL;
3451         qemu_mutex_unlock(&s->qemu_file_lock);
3452
3453         qemu_file_shutdown(file);
3454         qemu_fclose(file);
3455
3456         migrate_set_state(&s->state, s->state,
3457                           MIGRATION_STATUS_POSTCOPY_PAUSED);
3458
3459         error_report("Detected IO failure for postcopy. "
3460                      "Migration paused.");
3461
3462         /*
3463          * We wait until things fixed up. Then someone will setup the
3464          * status back for us.
3465          */
3466         while (s->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
3467             qemu_sem_wait(&s->postcopy_pause_sem);
3468         }
3469
3470         if (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3471             /* Woken up by a recover procedure. Give it a shot */
3472
3473             /*
3474              * Firstly, let's wake up the return path now, with a new
3475              * return path channel.
3476              */
3477             qemu_sem_post(&s->postcopy_pause_rp_sem);
3478
3479             /* Do the resume logic */
3480             if (postcopy_do_resume(s) == 0) {
3481                 /* Let's continue! */
3482                 trace_postcopy_pause_continued();
3483                 return MIG_THR_ERR_RECOVERED;
3484             } else {
3485                 /*
3486                  * Something wrong happened during the recovery, let's
3487                  * pause again. Pause is always better than throwing
3488                  * data away.
3489                  */
3490                 continue;
3491             }
3492         } else {
3493             /* This is not right... Time to quit. */
3494             return MIG_THR_ERR_FATAL;
3495         }
3496     }
3497 }
3498
3499 static MigThrError migration_detect_error(MigrationState *s)
3500 {
3501     int ret;
3502     int state = s->state;
3503     Error *local_error = NULL;
3504
3505     if (state == MIGRATION_STATUS_CANCELLING ||
3506         state == MIGRATION_STATUS_CANCELLED) {
3507         /* End the migration, but don't set the state to failed */
3508         return MIG_THR_ERR_FATAL;
3509     }
3510
3511     /* Try to detect any file errors */
3512     ret = qemu_file_get_error_obj(s->to_dst_file, &local_error);
3513     if (!ret) {
3514         /* Everything is fine */
3515         assert(!local_error);
3516         return MIG_THR_ERR_NONE;
3517     }
3518
3519     if (local_error) {
3520         migrate_set_error(s, local_error);
3521         error_free(local_error);
3522     }
3523
3524     if (state == MIGRATION_STATUS_POSTCOPY_ACTIVE && ret) {
3525         /*
3526          * For postcopy, we allow the network to be down for a
3527          * while. After that, it can be continued by a
3528          * recovery phase.
3529          */
3530         return postcopy_pause(s);
3531     } else {
3532         /*
3533          * For precopy (or postcopy with error outside IO), we fail
3534          * with no time.
3535          */
3536         migrate_set_state(&s->state, state, MIGRATION_STATUS_FAILED);
3537         trace_migration_thread_file_err();
3538
3539         /* Time to stop the migration, now. */
3540         return MIG_THR_ERR_FATAL;
3541     }
3542 }
3543
3544 /* How many bytes have we transferred since the beginning of the migration */
3545 static uint64_t migration_total_bytes(MigrationState *s)
3546 {
3547     return qemu_ftell(s->to_dst_file) + ram_counters.multifd_bytes;
3548 }
3549
3550 static void migration_calculate_complete(MigrationState *s)
3551 {
3552     uint64_t bytes = migration_total_bytes(s);
3553     int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3554     int64_t transfer_time;
3555
3556     s->total_time = end_time - s->start_time;
3557     if (!s->downtime) {
3558         /*
3559          * It's still not set, so we are precopy migration.  For
3560          * postcopy, downtime is calculated during postcopy_start().
3561          */
3562         s->downtime = end_time - s->downtime_start;
3563     }
3564
3565     transfer_time = s->total_time - s->setup_time;
3566     if (transfer_time) {
3567         s->mbps = ((double) bytes * 8.0) / transfer_time / 1000;
3568     }
3569 }
3570
3571 static void update_iteration_initial_status(MigrationState *s)
3572 {
3573     /*
3574      * Update these three fields at the same time to avoid mismatch info lead
3575      * wrong speed calculation.
3576      */
3577     s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3578     s->iteration_initial_bytes = migration_total_bytes(s);
3579     s->iteration_initial_pages = ram_get_total_transferred_pages();
3580 }
3581
3582 static void migration_update_counters(MigrationState *s,
3583                                       int64_t current_time)
3584 {
3585     uint64_t transferred, transferred_pages, time_spent;
3586     uint64_t current_bytes; /* bytes transferred since the beginning */
3587     double bandwidth;
3588
3589     if (current_time < s->iteration_start_time + BUFFER_DELAY) {
3590         return;
3591     }
3592
3593     current_bytes = migration_total_bytes(s);
3594     transferred = current_bytes - s->iteration_initial_bytes;
3595     time_spent = current_time - s->iteration_start_time;
3596     bandwidth = (double)transferred / time_spent;
3597     s->threshold_size = bandwidth * s->parameters.downtime_limit;
3598
3599     s->mbps = (((double) transferred * 8.0) /
3600                ((double) time_spent / 1000.0)) / 1000.0 / 1000.0;
3601
3602     transferred_pages = ram_get_total_transferred_pages() -
3603                             s->iteration_initial_pages;
3604     s->pages_per_second = (double) transferred_pages /
3605                              (((double) time_spent / 1000.0));
3606
3607     /*
3608      * if we haven't sent anything, we don't want to
3609      * recalculate. 10000 is a small enough number for our purposes
3610      */
3611     if (ram_counters.dirty_pages_rate && transferred > 10000) {
3612         s->expected_downtime = ram_counters.remaining / bandwidth;
3613     }
3614
3615     qemu_file_reset_rate_limit(s->to_dst_file);
3616
3617     update_iteration_initial_status(s);
3618
3619     trace_migrate_transferred(transferred, time_spent,
3620                               bandwidth, s->threshold_size);
3621 }
3622
3623 /* Migration thread iteration status */
3624 typedef enum {
3625     MIG_ITERATE_RESUME,         /* Resume current iteration */
3626     MIG_ITERATE_SKIP,           /* Skip current iteration */
3627     MIG_ITERATE_BREAK,          /* Break the loop */
3628 } MigIterateState;
3629
3630 /*
3631  * Return true if continue to the next iteration directly, false
3632  * otherwise.
3633  */
3634 static MigIterateState migration_iteration_run(MigrationState *s)
3635 {
3636     uint64_t pending_size, pend_pre, pend_compat, pend_post;
3637     bool in_postcopy = s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE;
3638
3639     qemu_savevm_state_pending(s->to_dst_file, s->threshold_size, &pend_pre,
3640                               &pend_compat, &pend_post);
3641     pending_size = pend_pre + pend_compat + pend_post;
3642
3643     trace_migrate_pending(pending_size, s->threshold_size,
3644                           pend_pre, pend_compat, pend_post);
3645
3646     if (pending_size && pending_size >= s->threshold_size) {
3647         /* Still a significant amount to transfer */
3648         if (!in_postcopy && pend_pre <= s->threshold_size &&
3649             qatomic_read(&s->start_postcopy)) {
3650             if (postcopy_start(s)) {
3651                 error_report("%s: postcopy failed to start", __func__);
3652             }
3653             return MIG_ITERATE_SKIP;
3654         }
3655         /* Just another iteration step */
3656         qemu_savevm_state_iterate(s->to_dst_file, in_postcopy);
3657     } else {
3658         trace_migration_thread_low_pending(pending_size);
3659         migration_completion(s);
3660         return MIG_ITERATE_BREAK;
3661     }
3662
3663     return MIG_ITERATE_RESUME;
3664 }
3665
3666 static void migration_iteration_finish(MigrationState *s)
3667 {
3668     /* If we enabled cpu throttling for auto-converge, turn it off. */
3669     cpu_throttle_stop();
3670
3671     qemu_mutex_lock_iothread();
3672     switch (s->state) {
3673     case MIGRATION_STATUS_COMPLETED:
3674         migration_calculate_complete(s);
3675         runstate_set(RUN_STATE_POSTMIGRATE);
3676         break;
3677     case MIGRATION_STATUS_COLO:
3678         if (!migrate_colo_enabled()) {
3679             error_report("%s: critical error: calling COLO code without "
3680                          "COLO enabled", __func__);
3681         }
3682         migrate_start_colo_process(s);
3683         s->vm_was_running = true;
3684         /* Fallthrough */
3685     case MIGRATION_STATUS_FAILED:
3686     case MIGRATION_STATUS_CANCELLED:
3687     case MIGRATION_STATUS_CANCELLING:
3688         if (s->vm_was_running) {
3689             if (!runstate_check(RUN_STATE_SHUTDOWN)) {
3690                 vm_start();
3691             }
3692         } else {
3693             if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
3694                 runstate_set(RUN_STATE_POSTMIGRATE);
3695             }
3696         }
3697         break;
3698
3699     default:
3700         /* Should not reach here, but if so, forgive the VM. */
3701         error_report("%s: Unknown ending state %d", __func__, s->state);
3702         break;
3703     }
3704     migrate_fd_cleanup_schedule(s);
3705     qemu_mutex_unlock_iothread();
3706 }
3707
3708 static void bg_migration_iteration_finish(MigrationState *s)
3709 {
3710     qemu_mutex_lock_iothread();
3711     switch (s->state) {
3712     case MIGRATION_STATUS_COMPLETED:
3713         migration_calculate_complete(s);
3714         break;
3715
3716     case MIGRATION_STATUS_ACTIVE:
3717     case MIGRATION_STATUS_FAILED:
3718     case MIGRATION_STATUS_CANCELLED:
3719     case MIGRATION_STATUS_CANCELLING:
3720         break;
3721
3722     default:
3723         /* Should not reach here, but if so, forgive the VM. */
3724         error_report("%s: Unknown ending state %d", __func__, s->state);
3725         break;
3726     }
3727
3728     migrate_fd_cleanup_schedule(s);
3729     qemu_mutex_unlock_iothread();
3730 }
3731
3732 /*
3733  * Return true if continue to the next iteration directly, false
3734  * otherwise.
3735  */
3736 static MigIterateState bg_migration_iteration_run(MigrationState *s)
3737 {
3738     int res;
3739
3740     res = qemu_savevm_state_iterate(s->to_dst_file, false);
3741     if (res > 0) {
3742         bg_migration_completion(s);
3743         return MIG_ITERATE_BREAK;
3744     }
3745
3746     return MIG_ITERATE_RESUME;
3747 }
3748
3749 void migration_make_urgent_request(void)
3750 {
3751     qemu_sem_post(&migrate_get_current()->rate_limit_sem);
3752 }
3753
3754 void migration_consume_urgent_request(void)
3755 {
3756     qemu_sem_wait(&migrate_get_current()->rate_limit_sem);
3757 }
3758
3759 /* Returns true if the rate limiting was broken by an urgent request */
3760 bool migration_rate_limit(void)
3761 {
3762     int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3763     MigrationState *s = migrate_get_current();
3764
3765     bool urgent = false;
3766     migration_update_counters(s, now);
3767     if (qemu_file_rate_limit(s->to_dst_file)) {
3768
3769         if (qemu_file_get_error(s->to_dst_file)) {
3770             return false;
3771         }
3772         /*
3773          * Wait for a delay to do rate limiting OR
3774          * something urgent to post the semaphore.
3775          */
3776         int ms = s->iteration_start_time + BUFFER_DELAY - now;
3777         trace_migration_rate_limit_pre(ms);
3778         if (qemu_sem_timedwait(&s->rate_limit_sem, ms) == 0) {
3779             /*
3780              * We were woken by one or more urgent things but
3781              * the timedwait will have consumed one of them.
3782              * The service routine for the urgent wake will dec
3783              * the semaphore itself for each item it consumes,
3784              * so add this one we just eat back.
3785              */
3786             qemu_sem_post(&s->rate_limit_sem);
3787             urgent = true;
3788         }
3789         trace_migration_rate_limit_post(urgent);
3790     }
3791     return urgent;
3792 }
3793
3794 /*
3795  * if failover devices are present, wait they are completely
3796  * unplugged
3797  */
3798
3799 static void qemu_savevm_wait_unplug(MigrationState *s, int old_state,
3800                                     int new_state)
3801 {
3802     if (qemu_savevm_state_guest_unplug_pending()) {
3803         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_WAIT_UNPLUG);
3804
3805         while (s->state == MIGRATION_STATUS_WAIT_UNPLUG &&
3806                qemu_savevm_state_guest_unplug_pending()) {
3807             qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3808         }
3809         if (s->state != MIGRATION_STATUS_WAIT_UNPLUG) {
3810             int timeout = 120; /* 30 seconds */
3811             /*
3812              * migration has been canceled
3813              * but as we have started an unplug we must wait the end
3814              * to be able to plug back the card
3815              */
3816             while (timeout-- && qemu_savevm_state_guest_unplug_pending()) {
3817                 qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3818             }
3819             if (qemu_savevm_state_guest_unplug_pending() &&
3820                 !qtest_enabled()) {
3821                 warn_report("migration: partially unplugged device on "
3822                             "failure");
3823             }
3824         }
3825
3826         migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG, new_state);
3827     } else {
3828         migrate_set_state(&s->state, old_state, new_state);
3829     }
3830 }
3831
3832 /*
3833  * Master migration thread on the source VM.
3834  * It drives the migration and pumps the data down the outgoing channel.
3835  */
3836 static void *migration_thread(void *opaque)
3837 {
3838     MigrationState *s = opaque;
3839     int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3840     MigThrError thr_error;
3841     bool urgent = false;
3842
3843     rcu_register_thread();
3844
3845     object_ref(OBJECT(s));
3846     update_iteration_initial_status(s);
3847
3848     qemu_savevm_state_header(s->to_dst_file);
3849
3850     /*
3851      * If we opened the return path, we need to make sure dst has it
3852      * opened as well.
3853      */
3854     if (s->rp_state.rp_thread_created) {
3855         /* Now tell the dest that it should open its end so it can reply */
3856         qemu_savevm_send_open_return_path(s->to_dst_file);
3857
3858         /* And do a ping that will make stuff easier to debug */
3859         qemu_savevm_send_ping(s->to_dst_file, 1);
3860     }
3861
3862     if (migrate_postcopy()) {
3863         /*
3864          * Tell the destination that we *might* want to do postcopy later;
3865          * if the other end can't do postcopy it should fail now, nice and
3866          * early.
3867          */
3868         qemu_savevm_send_postcopy_advise(s->to_dst_file);
3869     }
3870
3871     if (migrate_colo_enabled()) {
3872         /* Notify migration destination that we enable COLO */
3873         qemu_savevm_send_colo_enable(s->to_dst_file);
3874     }
3875
3876     qemu_savevm_state_setup(s->to_dst_file);
3877
3878     qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
3879                                MIGRATION_STATUS_ACTIVE);
3880
3881     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3882
3883     trace_migration_thread_setup_complete();
3884
3885     while (migration_is_active(s)) {
3886         if (urgent || !qemu_file_rate_limit(s->to_dst_file)) {
3887             MigIterateState iter_state = migration_iteration_run(s);
3888             if (iter_state == MIG_ITERATE_SKIP) {
3889                 continue;
3890             } else if (iter_state == MIG_ITERATE_BREAK) {
3891                 break;
3892             }
3893         }
3894
3895         /*
3896          * Try to detect any kind of failures, and see whether we
3897          * should stop the migration now.
3898          */
3899         thr_error = migration_detect_error(s);
3900         if (thr_error == MIG_THR_ERR_FATAL) {
3901             /* Stop migration */
3902             break;
3903         } else if (thr_error == MIG_THR_ERR_RECOVERED) {
3904             /*
3905              * Just recovered from a e.g. network failure, reset all
3906              * the local variables. This is important to avoid
3907              * breaking transferred_bytes and bandwidth calculation
3908              */
3909             update_iteration_initial_status(s);
3910         }
3911
3912         urgent = migration_rate_limit();
3913     }
3914
3915     trace_migration_thread_after_loop();
3916     migration_iteration_finish(s);
3917     object_unref(OBJECT(s));
3918     rcu_unregister_thread();
3919     return NULL;
3920 }
3921
3922 static void bg_migration_vm_start_bh(void *opaque)
3923 {
3924     MigrationState *s = opaque;
3925
3926     qemu_bh_delete(s->vm_start_bh);
3927     s->vm_start_bh = NULL;
3928
3929     vm_start();
3930     s->downtime = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - s->downtime_start;
3931 }
3932
3933 /**
3934  * Background snapshot thread, based on live migration code.
3935  * This is an alternative implementation of live migration mechanism
3936  * introduced specifically to support background snapshots.
3937  *
3938  * It takes advantage of userfault_fd write protection mechanism introduced
3939  * in v5.7 kernel. Compared to existing dirty page logging migration much
3940  * lesser stream traffic is produced resulting in smaller snapshot images,
3941  * simply cause of no page duplicates can get into the stream.
3942  *
3943  * Another key point is that generated vmstate stream reflects machine state
3944  * 'frozen' at the beginning of snapshot creation compared to dirty page logging
3945  * mechanism, which effectively results in that saved snapshot is the state of VM
3946  * at the end of the process.
3947  */
3948 static void *bg_migration_thread(void *opaque)
3949 {
3950     MigrationState *s = opaque;
3951     int64_t setup_start;
3952     MigThrError thr_error;
3953     QEMUFile *fb;
3954     bool early_fail = true;
3955
3956     rcu_register_thread();
3957     object_ref(OBJECT(s));
3958
3959     qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3960
3961     setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3962     /*
3963      * We want to save vmstate for the moment when migration has been
3964      * initiated but also we want to save RAM content while VM is running.
3965      * The RAM content should appear first in the vmstate. So, we first
3966      * stash the non-RAM part of the vmstate to the temporary buffer,
3967      * then write RAM part of the vmstate to the migration stream
3968      * with vCPUs running and, finally, write stashed non-RAM part of
3969      * the vmstate from the buffer to the migration stream.
3970      */
3971     s->bioc = qio_channel_buffer_new(512 * 1024);
3972     qio_channel_set_name(QIO_CHANNEL(s->bioc), "vmstate-buffer");
3973     fb = qemu_fopen_channel_output(QIO_CHANNEL(s->bioc));
3974     object_unref(OBJECT(s->bioc));
3975
3976     update_iteration_initial_status(s);
3977
3978     /*
3979      * Prepare for tracking memory writes with UFFD-WP - populate
3980      * RAM pages before protecting.
3981      */
3982 #ifdef __linux__
3983     ram_write_tracking_prepare();
3984 #endif
3985
3986     qemu_savevm_state_header(s->to_dst_file);
3987     qemu_savevm_state_setup(s->to_dst_file);
3988
3989     qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
3990                                MIGRATION_STATUS_ACTIVE);
3991
3992     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3993
3994     trace_migration_thread_setup_complete();
3995     s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3996
3997     qemu_mutex_lock_iothread();
3998
3999     /*
4000      * If VM is currently in suspended state, then, to make a valid runstate
4001      * transition in vm_stop_force_state() we need to wakeup it up.
4002      */
4003     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
4004     s->vm_was_running = runstate_is_running();
4005
4006     if (global_state_store()) {
4007         goto fail;
4008     }
4009     /* Forcibly stop VM before saving state of vCPUs and devices */
4010     if (vm_stop_force_state(RUN_STATE_PAUSED)) {
4011         goto fail;
4012     }
4013     /*
4014      * Put vCPUs in sync with shadow context structures, then
4015      * save their state to channel-buffer along with devices.
4016      */
4017     cpu_synchronize_all_states();
4018     if (qemu_savevm_state_complete_precopy_non_iterable(fb, false, false)) {
4019         goto fail;
4020     }
4021     /*
4022      * Since we are going to get non-iterable state data directly
4023      * from s->bioc->data, explicit flush is needed here.
4024      */
4025     qemu_fflush(fb);
4026
4027     /* Now initialize UFFD context and start tracking RAM writes */
4028     if (ram_write_tracking_start()) {
4029         goto fail;
4030     }
4031     early_fail = false;
4032
4033     /*
4034      * Start VM from BH handler to avoid write-fault lock here.
4035      * UFFD-WP protection for the whole RAM is already enabled so
4036      * calling VM state change notifiers from vm_start() would initiate
4037      * writes to virtio VQs memory which is in write-protected region.
4038      */
4039     s->vm_start_bh = qemu_bh_new(bg_migration_vm_start_bh, s);
4040     qemu_bh_schedule(s->vm_start_bh);
4041
4042     qemu_mutex_unlock_iothread();
4043
4044     while (migration_is_active(s)) {
4045         MigIterateState iter_state = bg_migration_iteration_run(s);
4046         if (iter_state == MIG_ITERATE_SKIP) {
4047             continue;
4048         } else if (iter_state == MIG_ITERATE_BREAK) {
4049             break;
4050         }
4051
4052         /*
4053          * Try to detect any kind of failures, and see whether we
4054          * should stop the migration now.
4055          */
4056         thr_error = migration_detect_error(s);
4057         if (thr_error == MIG_THR_ERR_FATAL) {
4058             /* Stop migration */
4059             break;
4060         }
4061
4062         migration_update_counters(s, qemu_clock_get_ms(QEMU_CLOCK_REALTIME));
4063     }
4064
4065     trace_migration_thread_after_loop();
4066
4067 fail:
4068     if (early_fail) {
4069         migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE,
4070                 MIGRATION_STATUS_FAILED);
4071         qemu_mutex_unlock_iothread();
4072     }
4073
4074     bg_migration_iteration_finish(s);
4075
4076     qemu_fclose(fb);
4077     object_unref(OBJECT(s));
4078     rcu_unregister_thread();
4079
4080     return NULL;
4081 }
4082
4083 void migrate_fd_connect(MigrationState *s, Error *error_in)
4084 {
4085     Error *local_err = NULL;
4086     int64_t rate_limit;
4087     bool resume = s->state == MIGRATION_STATUS_POSTCOPY_PAUSED;
4088
4089     /*
4090      * If there's a previous error, free it and prepare for another one.
4091      * Meanwhile if migration completes successfully, there won't have an error
4092      * dumped when calling migrate_fd_cleanup().
4093      */
4094     migrate_error_free(s);
4095
4096     s->expected_downtime = s->parameters.downtime_limit;
4097     if (resume) {
4098         assert(s->cleanup_bh);
4099     } else {
4100         assert(!s->cleanup_bh);
4101         s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup_bh, s);
4102     }
4103     if (error_in) {
4104         migrate_fd_error(s, error_in);
4105         if (resume) {
4106             /*
4107              * Don't do cleanup for resume if channel is invalid, but only dump
4108              * the error.  We wait for another channel connect from the user.
4109              * The error_report still gives HMP user a hint on what failed.
4110              * It's normally done in migrate_fd_cleanup(), but call it here
4111              * explicitly.
4112              */
4113             error_report_err(error_copy(s->error));
4114         } else {
4115             migrate_fd_cleanup(s);
4116         }
4117         return;
4118     }
4119
4120     if (resume) {
4121         /* This is a resumed migration */
4122         rate_limit = s->parameters.max_postcopy_bandwidth /
4123             XFER_LIMIT_RATIO;
4124     } else {
4125         /* This is a fresh new migration */
4126         rate_limit = s->parameters.max_bandwidth / XFER_LIMIT_RATIO;
4127
4128         /* Notify before starting migration thread */
4129         notifier_list_notify(&migration_state_notifiers, s);
4130     }
4131
4132     qemu_file_set_rate_limit(s->to_dst_file, rate_limit);
4133     qemu_file_set_blocking(s->to_dst_file, true);
4134
4135     /*
4136      * Open the return path. For postcopy, it is used exclusively. For
4137      * precopy, only if user specified "return-path" capability would
4138      * QEMU uses the return path.
4139      */
4140     if (migrate_postcopy_ram() || migrate_use_return_path()) {
4141         if (open_return_path_on_source(s, !resume)) {
4142             error_report("Unable to open return-path for postcopy");
4143             migrate_set_state(&s->state, s->state, MIGRATION_STATUS_FAILED);
4144             migrate_fd_cleanup(s);
4145             return;
4146         }
4147     }
4148
4149     if (resume) {
4150         /* Wakeup the main migration thread to do the recovery */
4151         migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
4152                           MIGRATION_STATUS_POSTCOPY_RECOVER);
4153         qemu_sem_post(&s->postcopy_pause_sem);
4154         return;
4155     }
4156
4157     if (multifd_save_setup(&local_err) != 0) {
4158         error_report_err(local_err);
4159         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
4160                           MIGRATION_STATUS_FAILED);
4161         migrate_fd_cleanup(s);
4162         return;
4163     }
4164
4165     if (migrate_background_snapshot()) {
4166         qemu_thread_create(&s->thread, "bg_snapshot",
4167                 bg_migration_thread, s, QEMU_THREAD_JOINABLE);
4168     } else {
4169         qemu_thread_create(&s->thread, "live_migration",
4170                 migration_thread, s, QEMU_THREAD_JOINABLE);
4171     }
4172     s->migration_thread_running = true;
4173 }
4174
4175 void migration_global_dump(Monitor *mon)
4176 {
4177     MigrationState *ms = migrate_get_current();
4178
4179     monitor_printf(mon, "globals:\n");
4180     monitor_printf(mon, "store-global-state: %s\n",
4181                    ms->store_global_state ? "on" : "off");
4182     monitor_printf(mon, "only-migratable: %s\n",
4183                    only_migratable ? "on" : "off");
4184     monitor_printf(mon, "send-configuration: %s\n",
4185                    ms->send_configuration ? "on" : "off");
4186     monitor_printf(mon, "send-section-footer: %s\n",
4187                    ms->send_section_footer ? "on" : "off");
4188     monitor_printf(mon, "decompress-error-check: %s\n",
4189                    ms->decompress_error_check ? "on" : "off");
4190     monitor_printf(mon, "clear-bitmap-shift: %u\n",
4191                    ms->clear_bitmap_shift);
4192 }
4193
4194 #define DEFINE_PROP_MIG_CAP(name, x)             \
4195     DEFINE_PROP_BOOL(name, MigrationState, enabled_capabilities[x], false)
4196
4197 static Property migration_properties[] = {
4198     DEFINE_PROP_BOOL("store-global-state", MigrationState,
4199                      store_global_state, true),
4200     DEFINE_PROP_BOOL("send-configuration", MigrationState,
4201                      send_configuration, true),
4202     DEFINE_PROP_BOOL("send-section-footer", MigrationState,
4203                      send_section_footer, true),
4204     DEFINE_PROP_BOOL("decompress-error-check", MigrationState,
4205                       decompress_error_check, true),
4206     DEFINE_PROP_UINT8("x-clear-bitmap-shift", MigrationState,
4207                       clear_bitmap_shift, CLEAR_BITMAP_SHIFT_DEFAULT),
4208
4209     /* Migration parameters */
4210     DEFINE_PROP_UINT8("x-compress-level", MigrationState,
4211                       parameters.compress_level,
4212                       DEFAULT_MIGRATE_COMPRESS_LEVEL),
4213     DEFINE_PROP_UINT8("x-compress-threads", MigrationState,
4214                       parameters.compress_threads,
4215                       DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT),
4216     DEFINE_PROP_BOOL("x-compress-wait-thread", MigrationState,
4217                       parameters.compress_wait_thread, true),
4218     DEFINE_PROP_UINT8("x-decompress-threads", MigrationState,
4219                       parameters.decompress_threads,
4220                       DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT),
4221     DEFINE_PROP_UINT8("x-throttle-trigger-threshold", MigrationState,
4222                       parameters.throttle_trigger_threshold,
4223                       DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD),
4224     DEFINE_PROP_UINT8("x-cpu-throttle-initial", MigrationState,
4225                       parameters.cpu_throttle_initial,
4226                       DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL),
4227     DEFINE_PROP_UINT8("x-cpu-throttle-increment", MigrationState,
4228                       parameters.cpu_throttle_increment,
4229                       DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT),
4230     DEFINE_PROP_BOOL("x-cpu-throttle-tailslow", MigrationState,
4231                       parameters.cpu_throttle_tailslow, false),
4232     DEFINE_PROP_SIZE("x-max-bandwidth", MigrationState,
4233                       parameters.max_bandwidth, MAX_THROTTLE),
4234     DEFINE_PROP_UINT64("x-downtime-limit", MigrationState,
4235                       parameters.downtime_limit,
4236                       DEFAULT_MIGRATE_SET_DOWNTIME),
4237     DEFINE_PROP_UINT32("x-checkpoint-delay", MigrationState,
4238                       parameters.x_checkpoint_delay,
4239                       DEFAULT_MIGRATE_X_CHECKPOINT_DELAY),
4240     DEFINE_PROP_UINT8("multifd-channels", MigrationState,
4241                       parameters.multifd_channels,
4242                       DEFAULT_MIGRATE_MULTIFD_CHANNELS),
4243     DEFINE_PROP_MULTIFD_COMPRESSION("multifd-compression", MigrationState,
4244                       parameters.multifd_compression,
4245                       DEFAULT_MIGRATE_MULTIFD_COMPRESSION),
4246     DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState,
4247                       parameters.multifd_zlib_level,
4248                       DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL),
4249     DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState,
4250                       parameters.multifd_zstd_level,
4251                       DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL),
4252 #ifdef CONFIG_LINUX
4253     DEFINE_PROP_BOOL("zero_copy_send", MigrationState,
4254                       parameters.zero_copy_send, false),
4255 #endif
4256     DEFINE_PROP_SIZE("xbzrle-cache-size", MigrationState,
4257                       parameters.xbzrle_cache_size,
4258                       DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE),
4259     DEFINE_PROP_SIZE("max-postcopy-bandwidth", MigrationState,
4260                       parameters.max_postcopy_bandwidth,
4261                       DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH),
4262     DEFINE_PROP_UINT8("max-cpu-throttle", MigrationState,
4263                       parameters.max_cpu_throttle,
4264                       DEFAULT_MIGRATE_MAX_CPU_THROTTLE),
4265     DEFINE_PROP_SIZE("announce-initial", MigrationState,
4266                       parameters.announce_initial,
4267                       DEFAULT_MIGRATE_ANNOUNCE_INITIAL),
4268     DEFINE_PROP_SIZE("announce-max", MigrationState,
4269                       parameters.announce_max,
4270                       DEFAULT_MIGRATE_ANNOUNCE_MAX),
4271     DEFINE_PROP_SIZE("announce-rounds", MigrationState,
4272                       parameters.announce_rounds,
4273                       DEFAULT_MIGRATE_ANNOUNCE_ROUNDS),
4274     DEFINE_PROP_SIZE("announce-step", MigrationState,
4275                       parameters.announce_step,
4276                       DEFAULT_MIGRATE_ANNOUNCE_STEP),
4277
4278     /* Migration capabilities */
4279     DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
4280     DEFINE_PROP_MIG_CAP("x-rdma-pin-all", MIGRATION_CAPABILITY_RDMA_PIN_ALL),
4281     DEFINE_PROP_MIG_CAP("x-auto-converge", MIGRATION_CAPABILITY_AUTO_CONVERGE),
4282     DEFINE_PROP_MIG_CAP("x-zero-blocks", MIGRATION_CAPABILITY_ZERO_BLOCKS),
4283     DEFINE_PROP_MIG_CAP("x-compress", MIGRATION_CAPABILITY_COMPRESS),
4284     DEFINE_PROP_MIG_CAP("x-events", MIGRATION_CAPABILITY_EVENTS),
4285     DEFINE_PROP_MIG_CAP("x-postcopy-ram", MIGRATION_CAPABILITY_POSTCOPY_RAM),
4286     DEFINE_PROP_MIG_CAP("x-colo", MIGRATION_CAPABILITY_X_COLO),
4287     DEFINE_PROP_MIG_CAP("x-release-ram", MIGRATION_CAPABILITY_RELEASE_RAM),
4288     DEFINE_PROP_MIG_CAP("x-block", MIGRATION_CAPABILITY_BLOCK),
4289     DEFINE_PROP_MIG_CAP("x-return-path", MIGRATION_CAPABILITY_RETURN_PATH),
4290     DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD),
4291     DEFINE_PROP_MIG_CAP("x-background-snapshot",
4292             MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT),
4293
4294     DEFINE_PROP_END_OF_LIST(),
4295 };
4296
4297 static void migration_class_init(ObjectClass *klass, void *data)
4298 {
4299     DeviceClass *dc = DEVICE_CLASS(klass);
4300
4301     dc->user_creatable = false;
4302     device_class_set_props(dc, migration_properties);
4303 }
4304
4305 static void migration_instance_finalize(Object *obj)
4306 {
4307     MigrationState *ms = MIGRATION_OBJ(obj);
4308     MigrationParameters *params = &ms->parameters;
4309
4310     qemu_mutex_destroy(&ms->error_mutex);
4311     qemu_mutex_destroy(&ms->qemu_file_lock);
4312     g_free(params->tls_hostname);
4313     g_free(params->tls_creds);
4314     qemu_sem_destroy(&ms->wait_unplug_sem);
4315     qemu_sem_destroy(&ms->rate_limit_sem);
4316     qemu_sem_destroy(&ms->pause_sem);
4317     qemu_sem_destroy(&ms->postcopy_pause_sem);
4318     qemu_sem_destroy(&ms->postcopy_pause_rp_sem);
4319     qemu_sem_destroy(&ms->rp_state.rp_sem);
4320     error_free(ms->error);
4321 }
4322
4323 static void migration_instance_init(Object *obj)
4324 {
4325     MigrationState *ms = MIGRATION_OBJ(obj);
4326     MigrationParameters *params = &ms->parameters;
4327
4328     ms->state = MIGRATION_STATUS_NONE;
4329     ms->mbps = -1;
4330     ms->pages_per_second = -1;
4331     qemu_sem_init(&ms->pause_sem, 0);
4332     qemu_mutex_init(&ms->error_mutex);
4333
4334     params->tls_hostname = g_strdup("");
4335     params->tls_creds = g_strdup("");
4336
4337     /* Set has_* up only for parameter checks */
4338     params->has_compress_level = true;
4339     params->has_compress_threads = true;
4340     params->has_decompress_threads = true;
4341     params->has_throttle_trigger_threshold = true;
4342     params->has_cpu_throttle_initial = true;
4343     params->has_cpu_throttle_increment = true;
4344     params->has_cpu_throttle_tailslow = true;
4345     params->has_max_bandwidth = true;
4346     params->has_downtime_limit = true;
4347     params->has_x_checkpoint_delay = true;
4348     params->has_block_incremental = true;
4349     params->has_multifd_channels = true;
4350     params->has_multifd_compression = true;
4351     params->has_multifd_zlib_level = true;
4352     params->has_multifd_zstd_level = true;
4353 #ifdef CONFIG_LINUX
4354     params->has_zero_copy_send = true;
4355 #endif
4356     params->has_xbzrle_cache_size = true;
4357     params->has_max_postcopy_bandwidth = true;
4358     params->has_max_cpu_throttle = true;
4359     params->has_announce_initial = true;
4360     params->has_announce_max = true;
4361     params->has_announce_rounds = true;
4362     params->has_announce_step = true;
4363
4364     qemu_sem_init(&ms->postcopy_pause_sem, 0);
4365     qemu_sem_init(&ms->postcopy_pause_rp_sem, 0);
4366     qemu_sem_init(&ms->rp_state.rp_sem, 0);
4367     qemu_sem_init(&ms->rate_limit_sem, 0);
4368     qemu_sem_init(&ms->wait_unplug_sem, 0);
4369     qemu_mutex_init(&ms->qemu_file_lock);
4370 }
4371
4372 /*
4373  * Return true if check pass, false otherwise. Error will be put
4374  * inside errp if provided.
4375  */
4376 static bool migration_object_check(MigrationState *ms, Error **errp)
4377 {
4378     MigrationCapabilityStatusList *head = NULL;
4379     /* Assuming all off */
4380     bool cap_list[MIGRATION_CAPABILITY__MAX] = { 0 }, ret;
4381     int i;
4382
4383     if (!migrate_params_check(&ms->parameters, errp)) {
4384         return false;
4385     }
4386
4387     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
4388         if (ms->enabled_capabilities[i]) {
4389             QAPI_LIST_PREPEND(head, migrate_cap_add(i, true));
4390         }
4391     }
4392
4393     ret = migrate_caps_check(cap_list, head, errp);
4394
4395     /* It works with head == NULL */
4396     qapi_free_MigrationCapabilityStatusList(head);
4397
4398     return ret;
4399 }
4400
4401 static const TypeInfo migration_type = {
4402     .name = TYPE_MIGRATION,
4403     /*
4404      * NOTE: TYPE_MIGRATION is not really a device, as the object is
4405      * not created using qdev_new(), it is not attached to the qdev
4406      * device tree, and it is never realized.
4407      *
4408      * TODO: Make this TYPE_OBJECT once QOM provides something like
4409      * TYPE_DEVICE's "-global" properties.
4410      */
4411     .parent = TYPE_DEVICE,
4412     .class_init = migration_class_init,
4413     .class_size = sizeof(MigrationClass),
4414     .instance_size = sizeof(MigrationState),
4415     .instance_init = migration_instance_init,
4416     .instance_finalize = migration_instance_finalize,
4417 };
4418
4419 static void register_migration_types(void)
4420 {
4421     type_register_static(&migration_type);
4422 }
4423
4424 type_init(register_migration_types);