migration/migration.c

   1 /*
   2  * QEMU live migration
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *
   6  * Authors:
   7  *  Anthony Liguori   <aliguori@us.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include "qemu/cutils.h"
  18 #include "qemu/error-report.h"
  19 #include "qemu/main-loop.h"
  20 #include "migration/blocker.h"
  21 #include "exec.h"
  22 #include "fd.h"
  23 #include "socket.h"
  24 #include "sysemu/runstate.h"
  25 #include "sysemu/sysemu.h"
  26 #include "sysemu/cpu-throttle.h"
  27 #include "rdma.h"
  28 #include "ram.h"
  29 #include "migration/global_state.h"
  30 #include "migration/misc.h"
  31 #include "migration.h"
  32 #include "savevm.h"
  33 #include "qemu-file.h"
  34 #include "migration/vmstate.h"
  35 #include "block/block.h"
  36 #include "qapi/error.h"
  37 #include "qapi/clone-visitor.h"
  38 #include "qapi/qapi-visit-migration.h"
  39 #include "qapi/qapi-visit-sockets.h"
  40 #include "qapi/qapi-commands-migration.h"
  41 #include "qapi/qapi-events-migration.h"
  42 #include "qapi/qmp/qerror.h"
  43 #include "qapi/qmp/qnull.h"
  44 #include "qemu/rcu.h"
  45 #include "block.h"
  46 #include "postcopy-ram.h"
  47 #include "qemu/thread.h"
  48 #include "trace.h"
  49 #include "exec/target_page.h"
  50 #include "io/channel-buffer.h"
  51 #include "migration/colo.h"
  52 #include "hw/boards.h"
  53 #include "hw/qdev-properties.h"
  54 #include "hw/qdev-properties-system.h"
  55 #include "monitor/monitor.h"
  56 #include "net/announce.h"
  57 #include "qemu/queue.h"
  58 #include "multifd.h"
  59 #include "qemu/yank.h"
  60 #include "sysemu/cpus.h"
  61 #include "yank_functions.h"
  62 #include "sysemu/qtest.h"
  63
  64 #define MAX_THROTTLE  (128 << 20)      /* Migration transfer speed throttling */
  65
  66 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
  67  * data. */
  68 #define BUFFER_DELAY     100
  69 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
  70
  71 /* Time in milliseconds we are allowed to stop the source,
  72  * for sending the last part */
  73 #define DEFAULT_MIGRATE_SET_DOWNTIME 300
  74
  75 /* Maximum migrate downtime set to 2000 seconds */
  76 #define MAX_MIGRATE_DOWNTIME_SECONDS 2000
  77 #define MAX_MIGRATE_DOWNTIME (MAX_MIGRATE_DOWNTIME_SECONDS * 1000)
  78
  79 /* Default compression thread count */
  80 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8
  81 /* Default decompression thread count, usually decompression is at
  82  * least 4 times as fast as compression.*/
  83 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2
  84 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */
  85 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1
  86 /* Define default autoconverge cpu throttle migration parameters */
  87 #define DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD 50
  88 #define DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL 20
  89 #define DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT 10
  90 #define DEFAULT_MIGRATE_MAX_CPU_THROTTLE 99
  91
  92 /* Migration XBZRLE default cache size */
  93 #define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024)
  94
  95 /* The delay time (in ms) between two COLO checkpoints */
  96 #define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100)
  97 #define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2
  98 #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE
  99 /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */
 100 #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1
 101 /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */
 102 #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1
 103
 104 /* Background transfer rate for postcopy, 0 means unlimited, note
 105  * that page requests can still exceed this limit.
 106  */
 107 #define DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH 0
 108
 109 /*
 110  * Parameters for self_announce_delay giving a stream of RARP/ARP
 111  * packets after migration.
 112  */
 113 #define DEFAULT_MIGRATE_ANNOUNCE_INITIAL  50
 114 #define DEFAULT_MIGRATE_ANNOUNCE_MAX     550
 115 #define DEFAULT_MIGRATE_ANNOUNCE_ROUNDS    5
 116 #define DEFAULT_MIGRATE_ANNOUNCE_STEP    100
 117
 118 static NotifierList migration_state_notifiers =
 119     NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
 120
 121 /* Messages sent on the return path from destination to source */
 122 enum mig_rp_message_type {
 123     MIG_RP_MSG_INVALID = 0,  /* Must be 0 */
 124     MIG_RP_MSG_SHUT,         /* sibling will not send any more RP messages */
 125     MIG_RP_MSG_PONG,         /* Response to a PING; data (seq: be32 ) */
 126
 127     MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
 128     MIG_RP_MSG_REQ_PAGES,    /* data (start: be64, len: be32) */
 129     MIG_RP_MSG_RECV_BITMAP,  /* send recved_bitmap back to source */
 130     MIG_RP_MSG_RESUME_ACK,   /* tell source that we are ready to resume */
 131
 132     MIG_RP_MSG_MAX
 133 };
 134
 135 /* Migration capabilities set */
 136 struct MigrateCapsSet {
 137     int size;                       /* Capability set size */
 138     MigrationCapability caps[];     /* Variadic array of capabilities */
 139 };
 140 typedef struct MigrateCapsSet MigrateCapsSet;
 141
 142 /* Define and initialize MigrateCapsSet */
 143 #define INITIALIZE_MIGRATE_CAPS_SET(_name, ...)   \
 144     MigrateCapsSet _name = {    \
 145         .size = sizeof((int []) { __VA_ARGS__ }) / sizeof(int), \
 146         .caps = { __VA_ARGS__ } \
 147     }
 148
 149 /* Background-snapshot compatibility check list */
 150 static const
 151 INITIALIZE_MIGRATE_CAPS_SET(check_caps_background_snapshot,
 152     MIGRATION_CAPABILITY_POSTCOPY_RAM,
 153     MIGRATION_CAPABILITY_DIRTY_BITMAPS,
 154     MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME,
 155     MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE,
 156     MIGRATION_CAPABILITY_RETURN_PATH,
 157     MIGRATION_CAPABILITY_MULTIFD,
 158     MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER,
 159     MIGRATION_CAPABILITY_AUTO_CONVERGE,
 160     MIGRATION_CAPABILITY_RELEASE_RAM,
 161     MIGRATION_CAPABILITY_RDMA_PIN_ALL,
 162     MIGRATION_CAPABILITY_COMPRESS,
 163     MIGRATION_CAPABILITY_XBZRLE,
 164     MIGRATION_CAPABILITY_X_COLO,
 165     MIGRATION_CAPABILITY_VALIDATE_UUID,
 166     MIGRATION_CAPABILITY_ZERO_COPY_SEND);
 167
 168 /* When we add fault tolerance, we could have several
 169    migrations at once.  For now we don't need to add
 170    dynamic creation of migration */
 171
 172 static MigrationState *current_migration;
 173 static MigrationIncomingState *current_incoming;
 174
 175 static GSList *migration_blockers;
 176
 177 static bool migration_object_check(MigrationState *ms, Error **errp);
 178 static int migration_maybe_pause(MigrationState *s,
 179                                  int *current_active_state,
 180                                  int new_state);
 181 static void migrate_fd_cancel(MigrationState *s);
 182
 183 static bool migrate_allow_multi_channels = true;
 184
 185 void migrate_protocol_allow_multi_channels(bool allow)
 186 {
 187     migrate_allow_multi_channels = allow;
 188 }
 189
 190 bool migrate_multi_channels_is_allowed(void)
 191 {
 192     return migrate_allow_multi_channels;
 193 }
 194
 195 static gint page_request_addr_cmp(gconstpointer ap, gconstpointer bp)
 196 {
 197     uintptr_t a = (uintptr_t) ap, b = (uintptr_t) bp;
 198
 199     return (a > b) - (a < b);
 200 }
 201
 202 void migration_object_init(void)
 203 {
 204     /* This can only be called once. */
 205     assert(!current_migration);
 206     current_migration = MIGRATION_OBJ(object_new(TYPE_MIGRATION));
 207
 208     /*
 209      * Init the migrate incoming object as well no matter whether
 210      * we'll use it or not.
 211      */
 212     assert(!current_incoming);
 213     current_incoming = g_new0(MigrationIncomingState, 1);
 214     current_incoming->state = MIGRATION_STATUS_NONE;
 215     current_incoming->postcopy_remote_fds =
 216         g_array_new(FALSE, TRUE, sizeof(struct PostCopyFD));
 217     qemu_mutex_init(&current_incoming->rp_mutex);
 218     qemu_mutex_init(&current_incoming->postcopy_prio_thread_mutex);
 219     qemu_event_init(&current_incoming->main_thread_load_event, false);
 220     qemu_sem_init(&current_incoming->postcopy_pause_sem_dst, 0);
 221     qemu_sem_init(&current_incoming->postcopy_pause_sem_fault, 0);
 222     qemu_sem_init(&current_incoming->postcopy_pause_sem_fast_load, 0);
 223     qemu_mutex_init(&current_incoming->page_request_mutex);
 224     current_incoming->page_requested = g_tree_new(page_request_addr_cmp);
 225
 226     migration_object_check(current_migration, &error_fatal);
 227
 228     blk_mig_init();
 229     ram_mig_init();
 230     dirty_bitmap_mig_init();
 231 }
 232
 233 void migration_cancel(const Error *error)
 234 {
 235     if (error) {
 236         migrate_set_error(current_migration, error);
 237     }
 238     migrate_fd_cancel(current_migration);
 239 }
 240
 241 void migration_shutdown(void)
 242 {
 243     /*
 244      * When the QEMU main thread exit, the COLO thread
 245      * may wait a semaphore. So, we should wakeup the
 246      * COLO thread before migration shutdown.
 247      */
 248     colo_shutdown();
 249     /*
 250      * Cancel the current migration - that will (eventually)
 251      * stop the migration using this structure
 252      */
 253     migration_cancel(NULL);
 254     object_unref(OBJECT(current_migration));
 255
 256     /*
 257      * Cancel outgoing migration of dirty bitmaps. It should
 258      * at least unref used block nodes.
 259      */
 260     dirty_bitmap_mig_cancel_outgoing();
 261
 262     /*
 263      * Cancel incoming migration of dirty bitmaps. Dirty bitmaps
 264      * are non-critical data, and their loss never considered as
 265      * something serious.
 266      */
 267     dirty_bitmap_mig_cancel_incoming();
 268 }
 269
 270 /* For outgoing */
 271 MigrationState *migrate_get_current(void)
 272 {
 273     /* This can only be called after the object created. */
 274     assert(current_migration);
 275     return current_migration;
 276 }
 277
 278 MigrationIncomingState *migration_incoming_get_current(void)
 279 {
 280     assert(current_incoming);
 281     return current_incoming;
 282 }
 283
 284 void migration_incoming_transport_cleanup(MigrationIncomingState *mis)
 285 {
 286     if (mis->socket_address_list) {
 287         qapi_free_SocketAddressList(mis->socket_address_list);
 288         mis->socket_address_list = NULL;
 289     }
 290
 291     if (mis->transport_cleanup) {
 292         mis->transport_cleanup(mis->transport_data);
 293         mis->transport_data = mis->transport_cleanup = NULL;
 294     }
 295 }
 296
 297 void migration_incoming_state_destroy(void)
 298 {
 299     struct MigrationIncomingState *mis = migration_incoming_get_current();
 300
 301     if (mis->to_src_file) {
 302         /* Tell source that we are done */
 303         migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
 304         qemu_fclose(mis->to_src_file);
 305         mis->to_src_file = NULL;
 306     }
 307
 308     if (mis->from_src_file) {
 309         migration_ioc_unregister_yank_from_file(mis->from_src_file);
 310         qemu_fclose(mis->from_src_file);
 311         mis->from_src_file = NULL;
 312     }
 313     if (mis->postcopy_remote_fds) {
 314         g_array_free(mis->postcopy_remote_fds, TRUE);
 315         mis->postcopy_remote_fds = NULL;
 316     }
 317
 318     migration_incoming_transport_cleanup(mis);
 319     qemu_event_reset(&mis->main_thread_load_event);
 320
 321     if (mis->page_requested) {
 322         g_tree_destroy(mis->page_requested);
 323         mis->page_requested = NULL;
 324     }
 325
 326     if (mis->postcopy_qemufile_dst) {
 327         migration_ioc_unregister_yank_from_file(mis->postcopy_qemufile_dst);
 328         qemu_fclose(mis->postcopy_qemufile_dst);
 329         mis->postcopy_qemufile_dst = NULL;
 330     }
 331
 332     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
 333 }
 334
 335 static void migrate_generate_event(int new_state)
 336 {
 337     if (migrate_use_events()) {
 338         qapi_event_send_migration(new_state);
 339     }
 340 }
 341
 342 static bool migrate_late_block_activate(void)
 343 {
 344     MigrationState *s;
 345
 346     s = migrate_get_current();
 347
 348     return s->enabled_capabilities[
 349         MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE];
 350 }
 351
 352 /*
 353  * Send a message on the return channel back to the source
 354  * of the migration.
 355  */
 356 static int migrate_send_rp_message(MigrationIncomingState *mis,
 357                                    enum mig_rp_message_type message_type,
 358                                    uint16_t len, void *data)
 359 {
 360     int ret = 0;
 361
 362     trace_migrate_send_rp_message((int)message_type, len);
 363     QEMU_LOCK_GUARD(&mis->rp_mutex);
 364
 365     /*
 366      * It's possible that the file handle got lost due to network
 367      * failures.
 368      */
 369     if (!mis->to_src_file) {
 370         ret = -EIO;
 371         return ret;
 372     }
 373
 374     qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
 375     qemu_put_be16(mis->to_src_file, len);
 376     qemu_put_buffer(mis->to_src_file, data, len);
 377     qemu_fflush(mis->to_src_file);
 378
 379     /* It's possible that qemu file got error during sending */
 380     ret = qemu_file_get_error(mis->to_src_file);
 381
 382     return ret;
 383 }
 384
 385 /* Request one page from the source VM at the given start address.
 386  *   rb: the RAMBlock to request the page in
 387  *   Start: Address offset within the RB
 388  *   Len: Length in bytes required - must be a multiple of pagesize
 389  */
 390 int migrate_send_rp_message_req_pages(MigrationIncomingState *mis,
 391                                       RAMBlock *rb, ram_addr_t start)
 392 {
 393     uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */
 394     size_t msglen = 12; /* start + len */
 395     size_t len = qemu_ram_pagesize(rb);
 396     enum mig_rp_message_type msg_type;
 397     const char *rbname;
 398     int rbname_len;
 399
 400     *(uint64_t *)bufc = cpu_to_be64((uint64_t)start);
 401     *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len);
 402
 403     /*
 404      * We maintain the last ramblock that we requested for page.  Note that we
 405      * don't need locking because this function will only be called within the
 406      * postcopy ram fault thread.
 407      */
 408     if (rb != mis->last_rb) {
 409         mis->last_rb = rb;
 410
 411         rbname = qemu_ram_get_idstr(rb);
 412         rbname_len = strlen(rbname);
 413
 414         assert(rbname_len < 256);
 415
 416         bufc[msglen++] = rbname_len;
 417         memcpy(bufc + msglen, rbname, rbname_len);
 418         msglen += rbname_len;
 419         msg_type = MIG_RP_MSG_REQ_PAGES_ID;
 420     } else {
 421         msg_type = MIG_RP_MSG_REQ_PAGES;
 422     }
 423
 424     return migrate_send_rp_message(mis, msg_type, msglen, bufc);
 425 }
 426
 427 int migrate_send_rp_req_pages(MigrationIncomingState *mis,
 428                               RAMBlock *rb, ram_addr_t start, uint64_t haddr)
 429 {
 430     void *aligned = (void *)(uintptr_t)ROUND_DOWN(haddr, qemu_ram_pagesize(rb));
 431     bool received = false;
 432
 433     WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) {
 434         received = ramblock_recv_bitmap_test_byte_offset(rb, start);
 435         if (!received && !g_tree_lookup(mis->page_requested, aligned)) {
 436             /*
 437              * The page has not been received, and it's not yet in the page
 438              * request list.  Queue it.  Set the value of element to 1, so that
 439              * things like g_tree_lookup() will return TRUE (1) when found.
 440              */
 441             g_tree_insert(mis->page_requested, aligned, (gpointer)1);
 442             mis->page_requested_count++;
 443             trace_postcopy_page_req_add(aligned, mis->page_requested_count);
 444         }
 445     }
 446
 447     /*
 448      * If the page is there, skip sending the message.  We don't even need the
 449      * lock because as long as the page arrived, it'll be there forever.
 450      */
 451     if (received) {
 452         return 0;
 453     }
 454
 455     return migrate_send_rp_message_req_pages(mis, rb, start);
 456 }
 457
 458 static bool migration_colo_enabled;
 459 bool migration_incoming_colo_enabled(void)
 460 {
 461     return migration_colo_enabled;
 462 }
 463
 464 void migration_incoming_disable_colo(void)
 465 {
 466     ram_block_discard_disable(false);
 467     migration_colo_enabled = false;
 468 }
 469
 470 int migration_incoming_enable_colo(void)
 471 {
 472     if (ram_block_discard_disable(true)) {
 473         error_report("COLO: cannot disable RAM discard");
 474         return -EBUSY;
 475     }
 476     migration_colo_enabled = true;
 477     return 0;
 478 }
 479
 480 void migrate_add_address(SocketAddress *address)
 481 {
 482     MigrationIncomingState *mis = migration_incoming_get_current();
 483
 484     QAPI_LIST_PREPEND(mis->socket_address_list,
 485                       QAPI_CLONE(SocketAddress, address));
 486 }
 487
 488 static void qemu_start_incoming_migration(const char *uri, Error **errp)
 489 {
 490     const char *p = NULL;
 491
 492     migrate_protocol_allow_multi_channels(false); /* reset it anyway */
 493     qapi_event_send_migration(MIGRATION_STATUS_SETUP);
 494     if (strstart(uri, "tcp:", &p) ||
 495         strstart(uri, "unix:", NULL) ||
 496         strstart(uri, "vsock:", NULL)) {
 497         migrate_protocol_allow_multi_channels(true);
 498         socket_start_incoming_migration(p ? p : uri, errp);
 499 #ifdef CONFIG_RDMA
 500     } else if (strstart(uri, "rdma:", &p)) {
 501         rdma_start_incoming_migration(p, errp);
 502 #endif
 503     } else if (strstart(uri, "exec:", &p)) {
 504         exec_start_incoming_migration(p, errp);
 505     } else if (strstart(uri, "fd:", &p)) {
 506         fd_start_incoming_migration(p, errp);
 507     } else {
 508         error_setg(errp, "unknown migration protocol: %s", uri);
 509     }
 510 }
 511
 512 static void process_incoming_migration_bh(void *opaque)
 513 {
 514     Error *local_err = NULL;
 515     MigrationIncomingState *mis = opaque;
 516
 517     /* If capability late_block_activate is set:
 518      * Only fire up the block code now if we're going to restart the
 519      * VM, else 'cont' will do it.
 520      * This causes file locking to happen; so we don't want it to happen
 521      * unless we really are starting the VM.
 522      */
 523     if (!migrate_late_block_activate() ||
 524          (autostart && (!global_state_received() ||
 525             global_state_get_runstate() == RUN_STATE_RUNNING))) {
 526         /* Make sure all file formats throw away their mutable metadata.
 527          * If we get an error here, just don't restart the VM yet. */
 528         bdrv_activate_all(&local_err);
 529         if (local_err) {
 530             error_report_err(local_err);
 531             local_err = NULL;
 532             autostart = false;
 533         }
 534     }
 535
 536     /*
 537      * This must happen after all error conditions are dealt with and
 538      * we're sure the VM is going to be running on this host.
 539      */
 540     qemu_announce_self(&mis->announce_timer, migrate_announce_params());
 541
 542     if (multifd_load_cleanup(&local_err) != 0) {
 543         error_report_err(local_err);
 544         autostart = false;
 545     }
 546     /* If global state section was not received or we are in running
 547        state, we need to obey autostart. Any other state is set with
 548        runstate_set. */
 549
 550     dirty_bitmap_mig_before_vm_start();
 551
 552     if (!global_state_received() ||
 553         global_state_get_runstate() == RUN_STATE_RUNNING) {
 554         if (autostart) {
 555             vm_start();
 556         } else {
 557             runstate_set(RUN_STATE_PAUSED);
 558         }
 559     } else if (migration_incoming_colo_enabled()) {
 560         migration_incoming_disable_colo();
 561         vm_start();
 562     } else {
 563         runstate_set(global_state_get_runstate());
 564     }
 565     /*
 566      * This must happen after any state changes since as soon as an external
 567      * observer sees this event they might start to prod at the VM assuming
 568      * it's ready to use.
 569      */
 570     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 571                       MIGRATION_STATUS_COMPLETED);
 572     qemu_bh_delete(mis->bh);
 573     migration_incoming_state_destroy();
 574 }
 575
 576 static void process_incoming_migration_co(void *opaque)
 577 {
 578     MigrationIncomingState *mis = migration_incoming_get_current();
 579     PostcopyState ps;
 580     int ret;
 581     Error *local_err = NULL;
 582
 583     assert(mis->from_src_file);
 584     mis->migration_incoming_co = qemu_coroutine_self();
 585     mis->largest_page_size = qemu_ram_pagesize_largest();
 586     postcopy_state_set(POSTCOPY_INCOMING_NONE);
 587     migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
 588                       MIGRATION_STATUS_ACTIVE);
 589     ret = qemu_loadvm_state(mis->from_src_file);
 590
 591     ps = postcopy_state_get();
 592     trace_process_incoming_migration_co_end(ret, ps);
 593     if (ps != POSTCOPY_INCOMING_NONE) {
 594         if (ps == POSTCOPY_INCOMING_ADVISE) {
 595             /*
 596              * Where a migration had postcopy enabled (and thus went to advise)
 597              * but managed to complete within the precopy period, we can use
 598              * the normal exit.
 599              */
 600             postcopy_ram_incoming_cleanup(mis);
 601         } else if (ret >= 0) {
 602             /*
 603              * Postcopy was started, cleanup should happen at the end of the
 604              * postcopy thread.
 605              */
 606             trace_process_incoming_migration_co_postcopy_end_main();
 607             return;
 608         }
 609         /* Else if something went wrong then just fall out of the normal exit */
 610     }
 611
 612     /* we get COLO info, and know if we are in COLO mode */
 613     if (!ret && migration_incoming_colo_enabled()) {
 614         /* Make sure all file formats throw away their mutable metadata */
 615         bdrv_activate_all(&local_err);
 616         if (local_err) {
 617             error_report_err(local_err);
 618             goto fail;
 619         }
 620
 621         qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
 622              colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
 623         mis->have_colo_incoming_thread = true;
 624         qemu_coroutine_yield();
 625
 626         qemu_mutex_unlock_iothread();
 627         /* Wait checkpoint incoming thread exit before free resource */
 628         qemu_thread_join(&mis->colo_incoming_thread);
 629         qemu_mutex_lock_iothread();
 630         /* We hold the global iothread lock, so it is safe here */
 631         colo_release_ram_cache();
 632     }
 633
 634     if (ret < 0) {
 635         error_report("load of migration failed: %s", strerror(-ret));
 636         goto fail;
 637     }
 638     mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
 639     qemu_bh_schedule(mis->bh);
 640     mis->migration_incoming_co = NULL;
 641     return;
 642 fail:
 643     local_err = NULL;
 644     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 645                       MIGRATION_STATUS_FAILED);
 646     qemu_fclose(mis->from_src_file);
 647     if (multifd_load_cleanup(&local_err) != 0) {
 648         error_report_err(local_err);
 649     }
 650     exit(EXIT_FAILURE);
 651 }
 652
 653 /**
 654  * migration_incoming_setup: Setup incoming migration
 655  * @f: file for main migration channel
 656  * @errp: where to put errors
 657  *
 658  * Returns: %true on success, %false on error.
 659  */
 660 static bool migration_incoming_setup(QEMUFile *f, Error **errp)
 661 {
 662     MigrationIncomingState *mis = migration_incoming_get_current();
 663
 664     if (multifd_load_setup(errp) != 0) {
 665         return false;
 666     }
 667
 668     if (!mis->from_src_file) {
 669         mis->from_src_file = f;
 670     }
 671     qemu_file_set_blocking(f, false);
 672     return true;
 673 }
 674
 675 void migration_incoming_process(void)
 676 {
 677     Coroutine *co = qemu_coroutine_create(process_incoming_migration_co, NULL);
 678     qemu_coroutine_enter(co);
 679 }
 680
 681 /* Returns true if recovered from a paused migration, otherwise false */
 682 static bool postcopy_try_recover(void)
 683 {
 684     MigrationIncomingState *mis = migration_incoming_get_current();
 685
 686     if (mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
 687         /* Resumed from a paused postcopy migration */
 688
 689         /* This should be set already in migration_incoming_setup() */
 690         assert(mis->from_src_file);
 691         /* Postcopy has standalone thread to do vm load */
 692         qemu_file_set_blocking(mis->from_src_file, true);
 693
 694         /* Re-configure the return path */
 695         mis->to_src_file = qemu_file_get_return_path(mis->from_src_file);
 696
 697         migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
 698                           MIGRATION_STATUS_POSTCOPY_RECOVER);
 699
 700         /*
 701          * Here, we only wake up the main loading thread (while the
 702          * rest threads will still be waiting), so that we can receive
 703          * commands from source now, and answer it if needed. The
 704          * rest threads will be woken up afterwards until we are sure
 705          * that source is ready to reply to page requests.
 706          */
 707         qemu_sem_post(&mis->postcopy_pause_sem_dst);
 708         return true;
 709     }
 710
 711     return false;
 712 }
 713
 714 void migration_fd_process_incoming(QEMUFile *f, Error **errp)
 715 {
 716     if (!migration_incoming_setup(f, errp)) {
 717         return;
 718     }
 719     if (postcopy_try_recover()) {
 720         return;
 721     }
 722     migration_incoming_process();
 723 }
 724
 725 static bool migration_needs_multiple_sockets(void)
 726 {
 727     return migrate_use_multifd() || migrate_postcopy_preempt();
 728 }
 729
 730 void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp)
 731 {
 732     MigrationIncomingState *mis = migration_incoming_get_current();
 733     Error *local_err = NULL;
 734     bool start_migration;
 735     QEMUFile *f;
 736
 737     if (!mis->from_src_file) {
 738         /* The first connection (multifd may have multiple) */
 739         f = qemu_file_new_input(ioc);
 740
 741         if (!migration_incoming_setup(f, errp)) {
 742             return;
 743         }
 744
 745         /*
 746          * Common migration only needs one channel, so we can start
 747          * right now.  Some features need more than one channel, we wait.
 748          */
 749         start_migration = !migration_needs_multiple_sockets();
 750     } else {
 751         /* Multiple connections */
 752         assert(migration_needs_multiple_sockets());
 753         if (migrate_use_multifd()) {
 754             start_migration = multifd_recv_new_channel(ioc, &local_err);
 755         } else {
 756             assert(migrate_postcopy_preempt());
 757             f = qemu_file_new_input(ioc);
 758             start_migration = postcopy_preempt_new_channel(mis, f);
 759         }
 760         if (local_err) {
 761             error_propagate(errp, local_err);
 762             return;
 763         }
 764     }
 765
 766     if (start_migration) {
 767         /* If it's a recovery, we're done */
 768         if (postcopy_try_recover()) {
 769             return;
 770         }
 771         migration_incoming_process();
 772     }
 773 }
 774
 775 /**
 776  * @migration_has_all_channels: We have received all channels that we need
 777  *
 778  * Returns true when we have got connections to all the channels that
 779  * we need for migration.
 780  */
 781 bool migration_has_all_channels(void)
 782 {
 783     MigrationIncomingState *mis = migration_incoming_get_current();
 784
 785     if (!mis->from_src_file) {
 786         return false;
 787     }
 788
 789     if (migrate_use_multifd()) {
 790         return multifd_recv_all_channels_created();
 791     }
 792
 793     if (migrate_postcopy_preempt()) {
 794         return mis->postcopy_qemufile_dst != NULL;
 795     }
 796
 797     return true;
 798 }
 799
 800 /*
 801  * Send a 'SHUT' message on the return channel with the given value
 802  * to indicate that we've finished with the RP.  Non-0 value indicates
 803  * error.
 804  */
 805 void migrate_send_rp_shut(MigrationIncomingState *mis,
 806                           uint32_t value)
 807 {
 808     uint32_t buf;
 809
 810     buf = cpu_to_be32(value);
 811     migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
 812 }
 813
 814 /*
 815  * Send a 'PONG' message on the return channel with the given value
 816  * (normally in response to a 'PING')
 817  */
 818 void migrate_send_rp_pong(MigrationIncomingState *mis,
 819                           uint32_t value)
 820 {
 821     uint32_t buf;
 822
 823     buf = cpu_to_be32(value);
 824     migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
 825 }
 826
 827 void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
 828                                  char *block_name)
 829 {
 830     char buf[512];
 831     int len;
 832     int64_t res;
 833
 834     /*
 835      * First, we send the header part. It contains only the len of
 836      * idstr, and the idstr itself.
 837      */
 838     len = strlen(block_name);
 839     buf[0] = len;
 840     memcpy(buf + 1, block_name, len);
 841
 842     if (mis->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
 843         error_report("%s: MSG_RP_RECV_BITMAP only used for recovery",
 844                      __func__);
 845         return;
 846     }
 847
 848     migrate_send_rp_message(mis, MIG_RP_MSG_RECV_BITMAP, len + 1, buf);
 849
 850     /*
 851      * Next, we dump the received bitmap to the stream.
 852      *
 853      * TODO: currently we are safe since we are the only one that is
 854      * using the to_src_file handle (fault thread is still paused),
 855      * and it's ok even not taking the mutex. However the best way is
 856      * to take the lock before sending the message header, and release
 857      * the lock after sending the bitmap.
 858      */
 859     qemu_mutex_lock(&mis->rp_mutex);
 860     res = ramblock_recv_bitmap_send(mis->to_src_file, block_name);
 861     qemu_mutex_unlock(&mis->rp_mutex);
 862
 863     trace_migrate_send_rp_recv_bitmap(block_name, res);
 864 }
 865
 866 void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value)
 867 {
 868     uint32_t buf;
 869
 870     buf = cpu_to_be32(value);
 871     migrate_send_rp_message(mis, MIG_RP_MSG_RESUME_ACK, sizeof(buf), &buf);
 872 }
 873
 874 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
 875 {
 876     MigrationCapabilityStatusList *head = NULL, **tail = &head;
 877     MigrationCapabilityStatus *caps;
 878     MigrationState *s = migrate_get_current();
 879     int i;
 880
 881     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
 882 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
 883         if (i == MIGRATION_CAPABILITY_BLOCK) {
 884             continue;
 885         }
 886 #endif
 887         caps = g_malloc0(sizeof(*caps));
 888         caps->capability = i;
 889         caps->state = s->enabled_capabilities[i];
 890         QAPI_LIST_APPEND(tail, caps);
 891     }
 892
 893     return head;
 894 }
 895
 896 MigrationParameters *qmp_query_migrate_parameters(Error **errp)
 897 {
 898     MigrationParameters *params;
 899     MigrationState *s = migrate_get_current();
 900
 901     /* TODO use QAPI_CLONE() instead of duplicating it inline */
 902     params = g_malloc0(sizeof(*params));
 903     params->has_compress_level = true;
 904     params->compress_level = s->parameters.compress_level;
 905     params->has_compress_threads = true;
 906     params->compress_threads = s->parameters.compress_threads;
 907     params->has_compress_wait_thread = true;
 908     params->compress_wait_thread = s->parameters.compress_wait_thread;
 909     params->has_decompress_threads = true;
 910     params->decompress_threads = s->parameters.decompress_threads;
 911     params->has_throttle_trigger_threshold = true;
 912     params->throttle_trigger_threshold = s->parameters.throttle_trigger_threshold;
 913     params->has_cpu_throttle_initial = true;
 914     params->cpu_throttle_initial = s->parameters.cpu_throttle_initial;
 915     params->has_cpu_throttle_increment = true;
 916     params->cpu_throttle_increment = s->parameters.cpu_throttle_increment;
 917     params->has_cpu_throttle_tailslow = true;
 918     params->cpu_throttle_tailslow = s->parameters.cpu_throttle_tailslow;
 919     params->has_tls_creds = true;
 920     params->tls_creds = g_strdup(s->parameters.tls_creds);
 921     params->has_tls_hostname = true;
 922     params->tls_hostname = g_strdup(s->parameters.tls_hostname);
 923     params->has_tls_authz = true;
 924     params->tls_authz = g_strdup(s->parameters.tls_authz ?
 925                                  s->parameters.tls_authz : "");
 926     params->has_max_bandwidth = true;
 927     params->max_bandwidth = s->parameters.max_bandwidth;
 928     params->has_downtime_limit = true;
 929     params->downtime_limit = s->parameters.downtime_limit;
 930     params->has_x_checkpoint_delay = true;
 931     params->x_checkpoint_delay = s->parameters.x_checkpoint_delay;
 932     params->has_block_incremental = true;
 933     params->block_incremental = s->parameters.block_incremental;
 934     params->has_multifd_channels = true;
 935     params->multifd_channels = s->parameters.multifd_channels;
 936     params->has_multifd_compression = true;
 937     params->multifd_compression = s->parameters.multifd_compression;
 938     params->has_multifd_zlib_level = true;
 939     params->multifd_zlib_level = s->parameters.multifd_zlib_level;
 940     params->has_multifd_zstd_level = true;
 941     params->multifd_zstd_level = s->parameters.multifd_zstd_level;
 942     params->has_xbzrle_cache_size = true;
 943     params->xbzrle_cache_size = s->parameters.xbzrle_cache_size;
 944     params->has_max_postcopy_bandwidth = true;
 945     params->max_postcopy_bandwidth = s->parameters.max_postcopy_bandwidth;
 946     params->has_max_cpu_throttle = true;
 947     params->max_cpu_throttle = s->parameters.max_cpu_throttle;
 948     params->has_announce_initial = true;
 949     params->announce_initial = s->parameters.announce_initial;
 950     params->has_announce_max = true;
 951     params->announce_max = s->parameters.announce_max;
 952     params->has_announce_rounds = true;
 953     params->announce_rounds = s->parameters.announce_rounds;
 954     params->has_announce_step = true;
 955     params->announce_step = s->parameters.announce_step;
 956
 957     if (s->parameters.has_block_bitmap_mapping) {
 958         params->has_block_bitmap_mapping = true;
 959         params->block_bitmap_mapping =
 960             QAPI_CLONE(BitmapMigrationNodeAliasList,
 961                        s->parameters.block_bitmap_mapping);
 962     }
 963
 964     return params;
 965 }
 966
 967 AnnounceParameters *migrate_announce_params(void)
 968 {
 969     static AnnounceParameters ap;
 970
 971     MigrationState *s = migrate_get_current();
 972
 973     ap.initial = s->parameters.announce_initial;
 974     ap.max = s->parameters.announce_max;
 975     ap.rounds = s->parameters.announce_rounds;
 976     ap.step = s->parameters.announce_step;
 977
 978     return &ap;
 979 }
 980
 981 /*
 982  * Return true if we're already in the middle of a migration
 983  * (i.e. any of the active or setup states)
 984  */
 985 bool migration_is_setup_or_active(int state)
 986 {
 987     switch (state) {
 988     case MIGRATION_STATUS_ACTIVE:
 989     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 990     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 991     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 992     case MIGRATION_STATUS_SETUP:
 993     case MIGRATION_STATUS_PRE_SWITCHOVER:
 994     case MIGRATION_STATUS_DEVICE:
 995     case MIGRATION_STATUS_WAIT_UNPLUG:
 996     case MIGRATION_STATUS_COLO:
 997         return true;
 998
 999     default:
1000         return false;
1001
1002     }
1003 }
1004
1005 bool migration_is_running(int state)
1006 {
1007     switch (state) {
1008     case MIGRATION_STATUS_ACTIVE:
1009     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1010     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1011     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1012     case MIGRATION_STATUS_SETUP:
1013     case MIGRATION_STATUS_PRE_SWITCHOVER:
1014     case MIGRATION_STATUS_DEVICE:
1015     case MIGRATION_STATUS_WAIT_UNPLUG:
1016     case MIGRATION_STATUS_CANCELLING:
1017         return true;
1018
1019     default:
1020         return false;
1021
1022     }
1023 }
1024
1025 static void populate_time_info(MigrationInfo *info, MigrationState *s)
1026 {
1027     info->has_status = true;
1028     info->has_setup_time = true;
1029     info->setup_time = s->setup_time;
1030     if (s->state == MIGRATION_STATUS_COMPLETED) {
1031         info->has_total_time = true;
1032         info->total_time = s->total_time;
1033         info->has_downtime = true;
1034         info->downtime = s->downtime;
1035     } else {
1036         info->has_total_time = true;
1037         info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) -
1038                            s->start_time;
1039         info->has_expected_downtime = true;
1040         info->expected_downtime = s->expected_downtime;
1041     }
1042 }
1043
1044 static void populate_ram_info(MigrationInfo *info, MigrationState *s)
1045 {
1046     size_t page_size = qemu_target_page_size();
1047
1048     info->has_ram = true;
1049     info->ram = g_malloc0(sizeof(*info->ram));
1050     info->ram->transferred = ram_counters.transferred;
1051     info->ram->total = ram_bytes_total();
1052     info->ram->duplicate = ram_counters.duplicate;
1053     /* legacy value.  It is not used anymore */
1054     info->ram->skipped = 0;
1055     info->ram->normal = ram_counters.normal;
1056     info->ram->normal_bytes = ram_counters.normal * page_size;
1057     info->ram->mbps = s->mbps;
1058     info->ram->dirty_sync_count = ram_counters.dirty_sync_count;
1059     info->ram->postcopy_requests = ram_counters.postcopy_requests;
1060     info->ram->page_size = page_size;
1061     info->ram->multifd_bytes = ram_counters.multifd_bytes;
1062     info->ram->pages_per_second = s->pages_per_second;
1063     info->ram->precopy_bytes = ram_counters.precopy_bytes;
1064     info->ram->downtime_bytes = ram_counters.downtime_bytes;
1065     info->ram->postcopy_bytes = ram_counters.postcopy_bytes;
1066
1067     if (migrate_use_xbzrle()) {
1068         info->has_xbzrle_cache = true;
1069         info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
1070         info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
1071         info->xbzrle_cache->bytes = xbzrle_counters.bytes;
1072         info->xbzrle_cache->pages = xbzrle_counters.pages;
1073         info->xbzrle_cache->cache_miss = xbzrle_counters.cache_miss;
1074         info->xbzrle_cache->cache_miss_rate = xbzrle_counters.cache_miss_rate;
1075         info->xbzrle_cache->encoding_rate = xbzrle_counters.encoding_rate;
1076         info->xbzrle_cache->overflow = xbzrle_counters.overflow;
1077     }
1078
1079     if (migrate_use_compression()) {
1080         info->has_compression = true;
1081         info->compression = g_malloc0(sizeof(*info->compression));
1082         info->compression->pages = compression_counters.pages;
1083         info->compression->busy = compression_counters.busy;
1084         info->compression->busy_rate = compression_counters.busy_rate;
1085         info->compression->compressed_size =
1086                                     compression_counters.compressed_size;
1087         info->compression->compression_rate =
1088                                     compression_counters.compression_rate;
1089     }
1090
1091     if (cpu_throttle_active()) {
1092         info->has_cpu_throttle_percentage = true;
1093         info->cpu_throttle_percentage = cpu_throttle_get_percentage();
1094     }
1095
1096     if (s->state != MIGRATION_STATUS_COMPLETED) {
1097         info->ram->remaining = ram_bytes_remaining();
1098         info->ram->dirty_pages_rate = ram_counters.dirty_pages_rate;
1099     }
1100 }
1101
1102 static void populate_disk_info(MigrationInfo *info)
1103 {
1104     if (blk_mig_active()) {
1105         info->has_disk = true;
1106         info->disk = g_malloc0(sizeof(*info->disk));
1107         info->disk->transferred = blk_mig_bytes_transferred();
1108         info->disk->remaining = blk_mig_bytes_remaining();
1109         info->disk->total = blk_mig_bytes_total();
1110     }
1111 }
1112
1113 static void fill_source_migration_info(MigrationInfo *info)
1114 {
1115     MigrationState *s = migrate_get_current();
1116     int state = qatomic_read(&s->state);
1117     GSList *cur_blocker = migration_blockers;
1118
1119     info->blocked_reasons = NULL;
1120
1121     /*
1122      * There are two types of reasons a migration might be blocked;
1123      * a) devices marked in VMState as non-migratable, and
1124      * b) Explicit migration blockers
1125      * We need to add both of them here.
1126      */
1127     qemu_savevm_non_migratable_list(&info->blocked_reasons);
1128
1129     while (cur_blocker) {
1130         QAPI_LIST_PREPEND(info->blocked_reasons,
1131                           g_strdup(error_get_pretty(cur_blocker->data)));
1132         cur_blocker = g_slist_next(cur_blocker);
1133     }
1134     info->has_blocked_reasons = info->blocked_reasons != NULL;
1135
1136     switch (state) {
1137     case MIGRATION_STATUS_NONE:
1138         /* no migration has happened ever */
1139         /* do not overwrite destination migration status */
1140         return;
1141     case MIGRATION_STATUS_SETUP:
1142         info->has_status = true;
1143         info->has_total_time = false;
1144         break;
1145     case MIGRATION_STATUS_ACTIVE:
1146     case MIGRATION_STATUS_CANCELLING:
1147     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1148     case MIGRATION_STATUS_PRE_SWITCHOVER:
1149     case MIGRATION_STATUS_DEVICE:
1150     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1151     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1152         /* TODO add some postcopy stats */
1153         populate_time_info(info, s);
1154         populate_ram_info(info, s);
1155         populate_disk_info(info);
1156         populate_vfio_info(info);
1157         break;
1158     case MIGRATION_STATUS_COLO:
1159         info->has_status = true;
1160         /* TODO: display COLO specific information (checkpoint info etc.) */
1161         break;
1162     case MIGRATION_STATUS_COMPLETED:
1163         populate_time_info(info, s);
1164         populate_ram_info(info, s);
1165         populate_vfio_info(info);
1166         break;
1167     case MIGRATION_STATUS_FAILED:
1168         info->has_status = true;
1169         if (s->error) {
1170             info->has_error_desc = true;
1171             info->error_desc = g_strdup(error_get_pretty(s->error));
1172         }
1173         break;
1174     case MIGRATION_STATUS_CANCELLED:
1175         info->has_status = true;
1176         break;
1177     case MIGRATION_STATUS_WAIT_UNPLUG:
1178         info->has_status = true;
1179         break;
1180     }
1181     info->status = state;
1182 }
1183
1184 typedef enum WriteTrackingSupport {
1185     WT_SUPPORT_UNKNOWN = 0,
1186     WT_SUPPORT_ABSENT,
1187     WT_SUPPORT_AVAILABLE,
1188     WT_SUPPORT_COMPATIBLE
1189 } WriteTrackingSupport;
1190
1191 static
1192 WriteTrackingSupport migrate_query_write_tracking(void)
1193 {
1194     /* Check if kernel supports required UFFD features */
1195     if (!ram_write_tracking_available()) {
1196         return WT_SUPPORT_ABSENT;
1197     }
1198     /*
1199      * Check if current memory configuration is
1200      * compatible with required UFFD features.
1201      */
1202     if (!ram_write_tracking_compatible()) {
1203         return WT_SUPPORT_AVAILABLE;
1204     }
1205
1206     return WT_SUPPORT_COMPATIBLE;
1207 }
1208
1209 /**
1210  * @migration_caps_check - check capability validity
1211  *
1212  * @cap_list: old capability list, array of bool
1213  * @params: new capabilities to be applied soon
1214  * @errp: set *errp if the check failed, with reason
1215  *
1216  * Returns true if check passed, otherwise false.
1217  */
1218 static bool migrate_caps_check(bool *cap_list,
1219                                MigrationCapabilityStatusList *params,
1220                                Error **errp)
1221 {
1222     MigrationCapabilityStatusList *cap;
1223     bool old_postcopy_cap;
1224     MigrationIncomingState *mis = migration_incoming_get_current();
1225
1226     old_postcopy_cap = cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM];
1227
1228     for (cap = params; cap; cap = cap->next) {
1229         cap_list[cap->value->capability] = cap->value->state;
1230     }
1231
1232 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
1233     if (cap_list[MIGRATION_CAPABILITY_BLOCK]) {
1234         error_setg(errp, "QEMU compiled without old-style (blk/-b, inc/-i) "
1235                    "block migration");
1236         error_append_hint(errp, "Use drive_mirror+NBD instead.\n");
1237         return false;
1238     }
1239 #endif
1240
1241 #ifndef CONFIG_REPLICATION
1242     if (cap_list[MIGRATION_CAPABILITY_X_COLO]) {
1243         error_setg(errp, "QEMU compiled without replication module"
1244                    " can't enable COLO");
1245         error_append_hint(errp, "Please enable replication before COLO.\n");
1246         return false;
1247     }
1248 #endif
1249
1250     if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
1251         /* This check is reasonably expensive, so only when it's being
1252          * set the first time, also it's only the destination that needs
1253          * special support.
1254          */
1255         if (!old_postcopy_cap && runstate_check(RUN_STATE_INMIGRATE) &&
1256             !postcopy_ram_supported_by_host(mis)) {
1257             /* postcopy_ram_supported_by_host will have emitted a more
1258              * detailed message
1259              */
1260             error_setg(errp, "Postcopy is not supported");
1261             return false;
1262         }
1263
1264         if (cap_list[MIGRATION_CAPABILITY_X_IGNORE_SHARED]) {
1265             error_setg(errp, "Postcopy is not compatible with ignore-shared");
1266             return false;
1267         }
1268     }
1269
1270     if (cap_list[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT]) {
1271         WriteTrackingSupport wt_support;
1272         int idx;
1273         /*
1274          * Check if 'background-snapshot' capability is supported by
1275          * host kernel and compatible with guest memory configuration.
1276          */
1277         wt_support = migrate_query_write_tracking();
1278         if (wt_support < WT_SUPPORT_AVAILABLE) {
1279             error_setg(errp, "Background-snapshot is not supported by host kernel");
1280             return false;
1281         }
1282         if (wt_support < WT_SUPPORT_COMPATIBLE) {
1283             error_setg(errp, "Background-snapshot is not compatible "
1284                     "with guest memory configuration");
1285             return false;
1286         }
1287
1288         /*
1289          * Check if there are any migration capabilities
1290          * incompatible with 'background-snapshot'.
1291          */
1292         for (idx = 0; idx < check_caps_background_snapshot.size; idx++) {
1293             int incomp_cap = check_caps_background_snapshot.caps[idx];
1294             if (cap_list[incomp_cap]) {
1295                 error_setg(errp,
1296                         "Background-snapshot is not compatible with %s",
1297                         MigrationCapability_str(incomp_cap));
1298                 return false;
1299             }
1300         }
1301     }
1302
1303 #ifdef CONFIG_LINUX
1304     if (cap_list[MIGRATION_CAPABILITY_ZERO_COPY_SEND] &&
1305         (!cap_list[MIGRATION_CAPABILITY_MULTIFD] ||
1306          migrate_use_compression() ||
1307          migrate_use_tls())) {
1308         error_setg(errp,
1309                    "Zero copy only available for non-compressed non-TLS multifd migration");
1310         return false;
1311     }
1312 #else
1313     if (cap_list[MIGRATION_CAPABILITY_ZERO_COPY_SEND]) {
1314         error_setg(errp,
1315                    "Zero copy currently only available on Linux");
1316         return false;
1317     }
1318 #endif
1319
1320
1321     /* incoming side only */
1322     if (runstate_check(RUN_STATE_INMIGRATE) &&
1323         !migrate_multi_channels_is_allowed() &&
1324         cap_list[MIGRATION_CAPABILITY_MULTIFD]) {
1325         error_setg(errp, "multifd is not supported by current protocol");
1326         return false;
1327     }
1328
1329     if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_PREEMPT]) {
1330         if (!cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
1331             error_setg(errp, "Postcopy preempt requires postcopy-ram");
1332             return false;
1333         }
1334     }
1335
1336     return true;
1337 }
1338
1339 static void fill_destination_migration_info(MigrationInfo *info)
1340 {
1341     MigrationIncomingState *mis = migration_incoming_get_current();
1342
1343     if (mis->socket_address_list) {
1344         info->has_socket_address = true;
1345         info->socket_address =
1346             QAPI_CLONE(SocketAddressList, mis->socket_address_list);
1347     }
1348
1349     switch (mis->state) {
1350     case MIGRATION_STATUS_NONE:
1351         return;
1352     case MIGRATION_STATUS_SETUP:
1353     case MIGRATION_STATUS_CANCELLING:
1354     case MIGRATION_STATUS_CANCELLED:
1355     case MIGRATION_STATUS_ACTIVE:
1356     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1357     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1358     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1359     case MIGRATION_STATUS_FAILED:
1360     case MIGRATION_STATUS_COLO:
1361         info->has_status = true;
1362         break;
1363     case MIGRATION_STATUS_COMPLETED:
1364         info->has_status = true;
1365         fill_destination_postcopy_migration_info(info);
1366         break;
1367     }
1368     info->status = mis->state;
1369 }
1370
1371 MigrationInfo *qmp_query_migrate(Error **errp)
1372 {
1373     MigrationInfo *info = g_malloc0(sizeof(*info));
1374
1375     fill_destination_migration_info(info);
1376     fill_source_migration_info(info);
1377
1378     return info;
1379 }
1380
1381 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
1382                                   Error **errp)
1383 {
1384     MigrationState *s = migrate_get_current();
1385     MigrationCapabilityStatusList *cap;
1386     bool cap_list[MIGRATION_CAPABILITY__MAX];
1387
1388     if (migration_is_running(s->state)) {
1389         error_setg(errp, QERR_MIGRATION_ACTIVE);
1390         return;
1391     }
1392
1393     memcpy(cap_list, s->enabled_capabilities, sizeof(cap_list));
1394     if (!migrate_caps_check(cap_list, params, errp)) {
1395         return;
1396     }
1397
1398     for (cap = params; cap; cap = cap->next) {
1399         s->enabled_capabilities[cap->value->capability] = cap->value->state;
1400     }
1401 }
1402
1403 /*
1404  * Check whether the parameters are valid. Error will be put into errp
1405  * (if provided). Return true if valid, otherwise false.
1406  */
1407 static bool migrate_params_check(MigrationParameters *params, Error **errp)
1408 {
1409     if (params->has_compress_level &&
1410         (params->compress_level > 9)) {
1411         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level",
1412                    "a value between 0 and 9");
1413         return false;
1414     }
1415
1416     if (params->has_compress_threads && (params->compress_threads < 1)) {
1417         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1418                    "compress_threads",
1419                    "a value between 1 and 255");
1420         return false;
1421     }
1422
1423     if (params->has_decompress_threads && (params->decompress_threads < 1)) {
1424         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1425                    "decompress_threads",
1426                    "a value between 1 and 255");
1427         return false;
1428     }
1429
1430     if (params->has_throttle_trigger_threshold &&
1431         (params->throttle_trigger_threshold < 1 ||
1432          params->throttle_trigger_threshold > 100)) {
1433         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1434                    "throttle_trigger_threshold",
1435                    "an integer in the range of 1 to 100");
1436         return false;
1437     }
1438
1439     if (params->has_cpu_throttle_initial &&
1440         (params->cpu_throttle_initial < 1 ||
1441          params->cpu_throttle_initial > 99)) {
1442         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1443                    "cpu_throttle_initial",
1444                    "an integer in the range of 1 to 99");
1445         return false;
1446     }
1447
1448     if (params->has_cpu_throttle_increment &&
1449         (params->cpu_throttle_increment < 1 ||
1450          params->cpu_throttle_increment > 99)) {
1451         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1452                    "cpu_throttle_increment",
1453                    "an integer in the range of 1 to 99");
1454         return false;
1455     }
1456
1457     if (params->has_max_bandwidth && (params->max_bandwidth > SIZE_MAX)) {
1458         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1459                    "max_bandwidth",
1460                    "an integer in the range of 0 to "stringify(SIZE_MAX)
1461                    " bytes/second");
1462         return false;
1463     }
1464
1465     if (params->has_downtime_limit &&
1466         (params->downtime_limit > MAX_MIGRATE_DOWNTIME)) {
1467         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1468                    "downtime_limit",
1469                    "an integer in the range of 0 to "
1470                     stringify(MAX_MIGRATE_DOWNTIME)" ms");
1471         return false;
1472     }
1473
1474     /* x_checkpoint_delay is now always positive */
1475
1476     if (params->has_multifd_channels && (params->multifd_channels < 1)) {
1477         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1478                    "multifd_channels",
1479                    "a value between 1 and 255");
1480         return false;
1481     }
1482
1483     if (params->has_multifd_zlib_level &&
1484         (params->multifd_zlib_level > 9)) {
1485         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zlib_level",
1486                    "a value between 0 and 9");
1487         return false;
1488     }
1489
1490     if (params->has_multifd_zstd_level &&
1491         (params->multifd_zstd_level > 20)) {
1492         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zstd_level",
1493                    "a value between 0 and 20");
1494         return false;
1495     }
1496
1497     if (params->has_xbzrle_cache_size &&
1498         (params->xbzrle_cache_size < qemu_target_page_size() ||
1499          !is_power_of_2(params->xbzrle_cache_size))) {
1500         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1501                    "xbzrle_cache_size",
1502                    "a power of two no less than the target page size");
1503         return false;
1504     }
1505
1506     if (params->has_max_cpu_throttle &&
1507         (params->max_cpu_throttle < params->cpu_throttle_initial ||
1508          params->max_cpu_throttle > 99)) {
1509         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1510                    "max_cpu_throttle",
1511                    "an integer in the range of cpu_throttle_initial to 99");
1512         return false;
1513     }
1514
1515     if (params->has_announce_initial &&
1516         params->announce_initial > 100000) {
1517         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1518                    "announce_initial",
1519                    "a value between 0 and 100000");
1520         return false;
1521     }
1522     if (params->has_announce_max &&
1523         params->announce_max > 100000) {
1524         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1525                    "announce_max",
1526                    "a value between 0 and 100000");
1527        return false;
1528     }
1529     if (params->has_announce_rounds &&
1530         params->announce_rounds > 1000) {
1531         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1532                    "announce_rounds",
1533                    "a value between 0 and 1000");
1534        return false;
1535     }
1536     if (params->has_announce_step &&
1537         (params->announce_step < 1 ||
1538         params->announce_step > 10000)) {
1539         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1540                    "announce_step",
1541                    "a value between 0 and 10000");
1542        return false;
1543     }
1544
1545     if (params->has_block_bitmap_mapping &&
1546         !check_dirty_bitmap_mig_alias_map(params->block_bitmap_mapping, errp)) {
1547         error_prepend(errp, "Invalid mapping given for block-bitmap-mapping: ");
1548         return false;
1549     }
1550     return true;
1551 }
1552
1553 static void migrate_params_test_apply(MigrateSetParameters *params,
1554                                       MigrationParameters *dest)
1555 {
1556     *dest = migrate_get_current()->parameters;
1557
1558     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1559
1560     if (params->has_compress_level) {
1561         dest->compress_level = params->compress_level;
1562     }
1563
1564     if (params->has_compress_threads) {
1565         dest->compress_threads = params->compress_threads;
1566     }
1567
1568     if (params->has_compress_wait_thread) {
1569         dest->compress_wait_thread = params->compress_wait_thread;
1570     }
1571
1572     if (params->has_decompress_threads) {
1573         dest->decompress_threads = params->decompress_threads;
1574     }
1575
1576     if (params->has_throttle_trigger_threshold) {
1577         dest->throttle_trigger_threshold = params->throttle_trigger_threshold;
1578     }
1579
1580     if (params->has_cpu_throttle_initial) {
1581         dest->cpu_throttle_initial = params->cpu_throttle_initial;
1582     }
1583
1584     if (params->has_cpu_throttle_increment) {
1585         dest->cpu_throttle_increment = params->cpu_throttle_increment;
1586     }
1587
1588     if (params->has_cpu_throttle_tailslow) {
1589         dest->cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1590     }
1591
1592     if (params->has_tls_creds) {
1593         assert(params->tls_creds->type == QTYPE_QSTRING);
1594         dest->tls_creds = params->tls_creds->u.s;
1595     }
1596
1597     if (params->has_tls_hostname) {
1598         assert(params->tls_hostname->type == QTYPE_QSTRING);
1599         dest->tls_hostname = params->tls_hostname->u.s;
1600     }
1601
1602     if (params->has_max_bandwidth) {
1603         dest->max_bandwidth = params->max_bandwidth;
1604     }
1605
1606     if (params->has_downtime_limit) {
1607         dest->downtime_limit = params->downtime_limit;
1608     }
1609
1610     if (params->has_x_checkpoint_delay) {
1611         dest->x_checkpoint_delay = params->x_checkpoint_delay;
1612     }
1613
1614     if (params->has_block_incremental) {
1615         dest->block_incremental = params->block_incremental;
1616     }
1617     if (params->has_multifd_channels) {
1618         dest->multifd_channels = params->multifd_channels;
1619     }
1620     if (params->has_multifd_compression) {
1621         dest->multifd_compression = params->multifd_compression;
1622     }
1623     if (params->has_xbzrle_cache_size) {
1624         dest->xbzrle_cache_size = params->xbzrle_cache_size;
1625     }
1626     if (params->has_max_postcopy_bandwidth) {
1627         dest->max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1628     }
1629     if (params->has_max_cpu_throttle) {
1630         dest->max_cpu_throttle = params->max_cpu_throttle;
1631     }
1632     if (params->has_announce_initial) {
1633         dest->announce_initial = params->announce_initial;
1634     }
1635     if (params->has_announce_max) {
1636         dest->announce_max = params->announce_max;
1637     }
1638     if (params->has_announce_rounds) {
1639         dest->announce_rounds = params->announce_rounds;
1640     }
1641     if (params->has_announce_step) {
1642         dest->announce_step = params->announce_step;
1643     }
1644
1645     if (params->has_block_bitmap_mapping) {
1646         dest->has_block_bitmap_mapping = true;
1647         dest->block_bitmap_mapping = params->block_bitmap_mapping;
1648     }
1649 }
1650
1651 static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
1652 {
1653     MigrationState *s = migrate_get_current();
1654
1655     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1656
1657     if (params->has_compress_level) {
1658         s->parameters.compress_level = params->compress_level;
1659     }
1660
1661     if (params->has_compress_threads) {
1662         s->parameters.compress_threads = params->compress_threads;
1663     }
1664
1665     if (params->has_compress_wait_thread) {
1666         s->parameters.compress_wait_thread = params->compress_wait_thread;
1667     }
1668
1669     if (params->has_decompress_threads) {
1670         s->parameters.decompress_threads = params->decompress_threads;
1671     }
1672
1673     if (params->has_throttle_trigger_threshold) {
1674         s->parameters.throttle_trigger_threshold = params->throttle_trigger_threshold;
1675     }
1676
1677     if (params->has_cpu_throttle_initial) {
1678         s->parameters.cpu_throttle_initial = params->cpu_throttle_initial;
1679     }
1680
1681     if (params->has_cpu_throttle_increment) {
1682         s->parameters.cpu_throttle_increment = params->cpu_throttle_increment;
1683     }
1684
1685     if (params->has_cpu_throttle_tailslow) {
1686         s->parameters.cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1687     }
1688
1689     if (params->has_tls_creds) {
1690         g_free(s->parameters.tls_creds);
1691         assert(params->tls_creds->type == QTYPE_QSTRING);
1692         s->parameters.tls_creds = g_strdup(params->tls_creds->u.s);
1693     }
1694
1695     if (params->has_tls_hostname) {
1696         g_free(s->parameters.tls_hostname);
1697         assert(params->tls_hostname->type == QTYPE_QSTRING);
1698         s->parameters.tls_hostname = g_strdup(params->tls_hostname->u.s);
1699     }
1700
1701     if (params->has_tls_authz) {
1702         g_free(s->parameters.tls_authz);
1703         assert(params->tls_authz->type == QTYPE_QSTRING);
1704         s->parameters.tls_authz = g_strdup(params->tls_authz->u.s);
1705     }
1706
1707     if (params->has_max_bandwidth) {
1708         s->parameters.max_bandwidth = params->max_bandwidth;
1709         if (s->to_dst_file && !migration_in_postcopy()) {
1710             qemu_file_set_rate_limit(s->to_dst_file,
1711                                 s->parameters.max_bandwidth / XFER_LIMIT_RATIO);
1712         }
1713     }
1714
1715     if (params->has_downtime_limit) {
1716         s->parameters.downtime_limit = params->downtime_limit;
1717     }
1718
1719     if (params->has_x_checkpoint_delay) {
1720         s->parameters.x_checkpoint_delay = params->x_checkpoint_delay;
1721         if (migration_in_colo_state()) {
1722             colo_checkpoint_notify(s);
1723         }
1724     }
1725
1726     if (params->has_block_incremental) {
1727         s->parameters.block_incremental = params->block_incremental;
1728     }
1729     if (params->has_multifd_channels) {
1730         s->parameters.multifd_channels = params->multifd_channels;
1731     }
1732     if (params->has_multifd_compression) {
1733         s->parameters.multifd_compression = params->multifd_compression;
1734     }
1735     if (params->has_xbzrle_cache_size) {
1736         s->parameters.xbzrle_cache_size = params->xbzrle_cache_size;
1737         xbzrle_cache_resize(params->xbzrle_cache_size, errp);
1738     }
1739     if (params->has_max_postcopy_bandwidth) {
1740         s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1741         if (s->to_dst_file && migration_in_postcopy()) {
1742             qemu_file_set_rate_limit(s->to_dst_file,
1743                     s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO);
1744         }
1745     }
1746     if (params->has_max_cpu_throttle) {
1747         s->parameters.max_cpu_throttle = params->max_cpu_throttle;
1748     }
1749     if (params->has_announce_initial) {
1750         s->parameters.announce_initial = params->announce_initial;
1751     }
1752     if (params->has_announce_max) {
1753         s->parameters.announce_max = params->announce_max;
1754     }
1755     if (params->has_announce_rounds) {
1756         s->parameters.announce_rounds = params->announce_rounds;
1757     }
1758     if (params->has_announce_step) {
1759         s->parameters.announce_step = params->announce_step;
1760     }
1761
1762     if (params->has_block_bitmap_mapping) {
1763         qapi_free_BitmapMigrationNodeAliasList(
1764             s->parameters.block_bitmap_mapping);
1765
1766         s->parameters.has_block_bitmap_mapping = true;
1767         s->parameters.block_bitmap_mapping =
1768             QAPI_CLONE(BitmapMigrationNodeAliasList,
1769                        params->block_bitmap_mapping);
1770     }
1771 }
1772
1773 void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
1774 {
1775     MigrationParameters tmp;
1776
1777     /* TODO Rewrite "" to null instead */
1778     if (params->has_tls_creds
1779         && params->tls_creds->type == QTYPE_QNULL) {
1780         qobject_unref(params->tls_creds->u.n);
1781         params->tls_creds->type = QTYPE_QSTRING;
1782         params->tls_creds->u.s = strdup("");
1783     }
1784     /* TODO Rewrite "" to null instead */
1785     if (params->has_tls_hostname
1786         && params->tls_hostname->type == QTYPE_QNULL) {
1787         qobject_unref(params->tls_hostname->u.n);
1788         params->tls_hostname->type = QTYPE_QSTRING;
1789         params->tls_hostname->u.s = strdup("");
1790     }
1791
1792     migrate_params_test_apply(params, &tmp);
1793
1794     if (!migrate_params_check(&tmp, errp)) {
1795         /* Invalid parameter */
1796         return;
1797     }
1798
1799     migrate_params_apply(params, errp);
1800 }
1801
1802
1803 void qmp_migrate_start_postcopy(Error **errp)
1804 {
1805     MigrationState *s = migrate_get_current();
1806
1807     if (!migrate_postcopy()) {
1808         error_setg(errp, "Enable postcopy with migrate_set_capability before"
1809                          " the start of migration");
1810         return;
1811     }
1812
1813     if (s->state == MIGRATION_STATUS_NONE) {
1814         error_setg(errp, "Postcopy must be started after migration has been"
1815                          " started");
1816         return;
1817     }
1818     /*
1819      * we don't error if migration has finished since that would be racy
1820      * with issuing this command.
1821      */
1822     qatomic_set(&s->start_postcopy, true);
1823 }
1824
1825 /* shared migration helpers */
1826
1827 void migrate_set_state(int *state, int old_state, int new_state)
1828 {
1829     assert(new_state < MIGRATION_STATUS__MAX);
1830     if (qatomic_cmpxchg(state, old_state, new_state) == old_state) {
1831         trace_migrate_set_state(MigrationStatus_str(new_state));
1832         migrate_generate_event(new_state);
1833     }
1834 }
1835
1836 static MigrationCapabilityStatus *migrate_cap_add(MigrationCapability index,
1837                                                   bool state)
1838 {
1839     MigrationCapabilityStatus *cap;
1840
1841     cap = g_new0(MigrationCapabilityStatus, 1);
1842     cap->capability = index;
1843     cap->state = state;
1844
1845     return cap;
1846 }
1847
1848 void migrate_set_block_enabled(bool value, Error **errp)
1849 {
1850     MigrationCapabilityStatusList *cap = NULL;
1851
1852     QAPI_LIST_PREPEND(cap, migrate_cap_add(MIGRATION_CAPABILITY_BLOCK, value));
1853     qmp_migrate_set_capabilities(cap, errp);
1854     qapi_free_MigrationCapabilityStatusList(cap);
1855 }
1856
1857 static void migrate_set_block_incremental(MigrationState *s, bool value)
1858 {
1859     s->parameters.block_incremental = value;
1860 }
1861
1862 static void block_cleanup_parameters(MigrationState *s)
1863 {
1864     if (s->must_remove_block_options) {
1865         /* setting to false can never fail */
1866         migrate_set_block_enabled(false, &error_abort);
1867         migrate_set_block_incremental(s, false);
1868         s->must_remove_block_options = false;
1869     }
1870 }
1871
1872 static void migrate_fd_cleanup(MigrationState *s)
1873 {
1874     qemu_bh_delete(s->cleanup_bh);
1875     s->cleanup_bh = NULL;
1876
1877     g_free(s->hostname);
1878     s->hostname = NULL;
1879
1880     qemu_savevm_state_cleanup();
1881
1882     if (s->to_dst_file) {
1883         QEMUFile *tmp;
1884
1885         trace_migrate_fd_cleanup();
1886         qemu_mutex_unlock_iothread();
1887         if (s->migration_thread_running) {
1888             qemu_thread_join(&s->thread);
1889             s->migration_thread_running = false;
1890         }
1891         qemu_mutex_lock_iothread();
1892
1893         multifd_save_cleanup();
1894         qemu_mutex_lock(&s->qemu_file_lock);
1895         tmp = s->to_dst_file;
1896         s->to_dst_file = NULL;
1897         qemu_mutex_unlock(&s->qemu_file_lock);
1898         /*
1899          * Close the file handle without the lock to make sure the
1900          * critical section won't block for long.
1901          */
1902         migration_ioc_unregister_yank_from_file(tmp);
1903         qemu_fclose(tmp);
1904     }
1905
1906     if (s->postcopy_qemufile_src) {
1907         migration_ioc_unregister_yank_from_file(s->postcopy_qemufile_src);
1908         qemu_fclose(s->postcopy_qemufile_src);
1909         s->postcopy_qemufile_src = NULL;
1910     }
1911
1912     assert(!migration_is_active(s));
1913
1914     if (s->state == MIGRATION_STATUS_CANCELLING) {
1915         migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING,
1916                           MIGRATION_STATUS_CANCELLED);
1917     }
1918
1919     if (s->error) {
1920         /* It is used on info migrate.  We can't free it */
1921         error_report_err(error_copy(s->error));
1922     }
1923     notifier_list_notify(&migration_state_notifiers, s);
1924     block_cleanup_parameters(s);
1925     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
1926 }
1927
1928 static void migrate_fd_cleanup_schedule(MigrationState *s)
1929 {
1930     /*
1931      * Ref the state for bh, because it may be called when
1932      * there're already no other refs
1933      */
1934     object_ref(OBJECT(s));
1935     qemu_bh_schedule(s->cleanup_bh);
1936 }
1937
1938 static void migrate_fd_cleanup_bh(void *opaque)
1939 {
1940     MigrationState *s = opaque;
1941     migrate_fd_cleanup(s);
1942     object_unref(OBJECT(s));
1943 }
1944
1945 void migrate_set_error(MigrationState *s, const Error *error)
1946 {
1947     QEMU_LOCK_GUARD(&s->error_mutex);
1948     if (!s->error) {
1949         s->error = error_copy(error);
1950     }
1951 }
1952
1953 static void migrate_error_free(MigrationState *s)
1954 {
1955     QEMU_LOCK_GUARD(&s->error_mutex);
1956     if (s->error) {
1957         error_free(s->error);
1958         s->error = NULL;
1959     }
1960 }
1961
1962 void migrate_fd_error(MigrationState *s, const Error *error)
1963 {
1964     trace_migrate_fd_error(error_get_pretty(error));
1965     assert(s->to_dst_file == NULL);
1966     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
1967                       MIGRATION_STATUS_FAILED);
1968     migrate_set_error(s, error);
1969 }
1970
1971 static void migrate_fd_cancel(MigrationState *s)
1972 {
1973     int old_state ;
1974     QEMUFile *f = migrate_get_current()->to_dst_file;
1975     trace_migrate_fd_cancel();
1976
1977     WITH_QEMU_LOCK_GUARD(&s->qemu_file_lock) {
1978         if (s->rp_state.from_dst_file) {
1979             /* shutdown the rp socket, so causing the rp thread to shutdown */
1980             qemu_file_shutdown(s->rp_state.from_dst_file);
1981         }
1982     }
1983
1984     do {
1985         old_state = s->state;
1986         if (!migration_is_running(old_state)) {
1987             break;
1988         }
1989         /* If the migration is paused, kick it out of the pause */
1990         if (old_state == MIGRATION_STATUS_PRE_SWITCHOVER) {
1991             qemu_sem_post(&s->pause_sem);
1992         }
1993         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING);
1994     } while (s->state != MIGRATION_STATUS_CANCELLING);
1995
1996     /*
1997      * If we're unlucky the migration code might be stuck somewhere in a
1998      * send/write while the network has failed and is waiting to timeout;
1999      * if we've got shutdown(2) available then we can force it to quit.
2000      * The outgoing qemu file gets closed in migrate_fd_cleanup that is
2001      * called in a bh, so there is no race against this cancel.
2002      */
2003     if (s->state == MIGRATION_STATUS_CANCELLING && f) {
2004         qemu_file_shutdown(f);
2005     }
2006     if (s->state == MIGRATION_STATUS_CANCELLING && s->block_inactive) {
2007         Error *local_err = NULL;
2008
2009         bdrv_activate_all(&local_err);
2010         if (local_err) {
2011             error_report_err(local_err);
2012         } else {
2013             s->block_inactive = false;
2014         }
2015     }
2016 }
2017
2018 void add_migration_state_change_notifier(Notifier *notify)
2019 {
2020     notifier_list_add(&migration_state_notifiers, notify);
2021 }
2022
2023 void remove_migration_state_change_notifier(Notifier *notify)
2024 {
2025     notifier_remove(notify);
2026 }
2027
2028 bool migration_in_setup(MigrationState *s)
2029 {
2030     return s->state == MIGRATION_STATUS_SETUP;
2031 }
2032
2033 bool migration_has_finished(MigrationState *s)
2034 {
2035     return s->state == MIGRATION_STATUS_COMPLETED;
2036 }
2037
2038 bool migration_has_failed(MigrationState *s)
2039 {
2040     return (s->state == MIGRATION_STATUS_CANCELLED ||
2041             s->state == MIGRATION_STATUS_FAILED);
2042 }
2043
2044 bool migration_in_postcopy(void)
2045 {
2046     MigrationState *s = migrate_get_current();
2047
2048     switch (s->state) {
2049     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
2050     case MIGRATION_STATUS_POSTCOPY_PAUSED:
2051     case MIGRATION_STATUS_POSTCOPY_RECOVER:
2052         return true;
2053     default:
2054         return false;
2055     }
2056 }
2057
2058 bool migration_in_postcopy_after_devices(MigrationState *s)
2059 {
2060     return migration_in_postcopy() && s->postcopy_after_devices;
2061 }
2062
2063 bool migration_in_incoming_postcopy(void)
2064 {
2065     PostcopyState ps = postcopy_state_get();
2066
2067     return ps >= POSTCOPY_INCOMING_DISCARD && ps < POSTCOPY_INCOMING_END;
2068 }
2069
2070 bool migration_in_bg_snapshot(void)
2071 {
2072     MigrationState *s = migrate_get_current();
2073
2074     return migrate_background_snapshot() &&
2075             migration_is_setup_or_active(s->state);
2076 }
2077
2078 bool migration_is_idle(void)
2079 {
2080     MigrationState *s = current_migration;
2081
2082     if (!s) {
2083         return true;
2084     }
2085
2086     switch (s->state) {
2087     case MIGRATION_STATUS_NONE:
2088     case MIGRATION_STATUS_CANCELLED:
2089     case MIGRATION_STATUS_COMPLETED:
2090     case MIGRATION_STATUS_FAILED:
2091         return true;
2092     case MIGRATION_STATUS_SETUP:
2093     case MIGRATION_STATUS_CANCELLING:
2094     case MIGRATION_STATUS_ACTIVE:
2095     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
2096     case MIGRATION_STATUS_COLO:
2097     case MIGRATION_STATUS_PRE_SWITCHOVER:
2098     case MIGRATION_STATUS_DEVICE:
2099     case MIGRATION_STATUS_WAIT_UNPLUG:
2100         return false;
2101     case MIGRATION_STATUS__MAX:
2102         g_assert_not_reached();
2103     }
2104
2105     return false;
2106 }
2107
2108 bool migration_is_active(MigrationState *s)
2109 {
2110     return (s->state == MIGRATION_STATUS_ACTIVE ||
2111             s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
2112 }
2113
2114 void migrate_init(MigrationState *s)
2115 {
2116     /*
2117      * Reinitialise all migration state, except
2118      * parameters/capabilities that the user set, and
2119      * locks.
2120      */
2121     s->cleanup_bh = 0;
2122     s->vm_start_bh = 0;
2123     s->to_dst_file = NULL;
2124     s->state = MIGRATION_STATUS_NONE;
2125     s->rp_state.from_dst_file = NULL;
2126     s->rp_state.error = false;
2127     s->mbps = 0.0;
2128     s->pages_per_second = 0.0;
2129     s->downtime = 0;
2130     s->expected_downtime = 0;
2131     s->setup_time = 0;
2132     s->start_postcopy = false;
2133     s->postcopy_after_devices = false;
2134     s->migration_thread_running = false;
2135     error_free(s->error);
2136     s->error = NULL;
2137     s->hostname = NULL;
2138
2139     migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
2140
2141     s->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2142     s->total_time = 0;
2143     s->vm_was_running = false;
2144     s->iteration_initial_bytes = 0;
2145     s->threshold_size = 0;
2146 }
2147
2148 int migrate_add_blocker_internal(Error *reason, Error **errp)
2149 {
2150     /* Snapshots are similar to migrations, so check RUN_STATE_SAVE_VM too. */
2151     if (runstate_check(RUN_STATE_SAVE_VM) || !migration_is_idle()) {
2152         error_propagate_prepend(errp, error_copy(reason),
2153                                 "disallowing migration blocker "
2154                                 "(migration/snapshot in progress) for: ");
2155         return -EBUSY;
2156     }
2157
2158     migration_blockers = g_slist_prepend(migration_blockers, reason);
2159     return 0;
2160 }
2161
2162 int migrate_add_blocker(Error *reason, Error **errp)
2163 {
2164     if (only_migratable) {
2165         error_propagate_prepend(errp, error_copy(reason),
2166                                 "disallowing migration blocker "
2167                                 "(--only-migratable) for: ");
2168         return -EACCES;
2169     }
2170
2171     return migrate_add_blocker_internal(reason, errp);
2172 }
2173
2174 void migrate_del_blocker(Error *reason)
2175 {
2176     migration_blockers = g_slist_remove(migration_blockers, reason);
2177 }
2178
2179 void qmp_migrate_incoming(const char *uri, Error **errp)
2180 {
2181     Error *local_err = NULL;
2182     static bool once = true;
2183
2184     if (!once) {
2185         error_setg(errp, "The incoming migration has already been started");
2186         return;
2187     }
2188     if (!runstate_check(RUN_STATE_INMIGRATE)) {
2189         error_setg(errp, "'-incoming' was not specified on the command line");
2190         return;
2191     }
2192
2193     if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2194         return;
2195     }
2196
2197     qemu_start_incoming_migration(uri, &local_err);
2198
2199     if (local_err) {
2200         yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2201         error_propagate(errp, local_err);
2202         return;
2203     }
2204
2205     once = false;
2206 }
2207
2208 void qmp_migrate_recover(const char *uri, Error **errp)
2209 {
2210     MigrationIncomingState *mis = migration_incoming_get_current();
2211
2212     /*
2213      * Don't even bother to use ERRP_GUARD() as it _must_ always be set by
2214      * callers (no one should ignore a recover failure); if there is, it's a
2215      * programming error.
2216      */
2217     assert(errp);
2218
2219     if (mis->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2220         error_setg(errp, "Migrate recover can only be run "
2221                    "when postcopy is paused.");
2222         return;
2223     }
2224
2225     /* If there's an existing transport, release it */
2226     migration_incoming_transport_cleanup(mis);
2227
2228     /*
2229      * Note that this call will never start a real migration; it will
2230      * only re-setup the migration stream and poke existing migration
2231      * to continue using that newly established channel.
2232      */
2233     qemu_start_incoming_migration(uri, errp);
2234 }
2235
2236 void qmp_migrate_pause(Error **errp)
2237 {
2238     MigrationState *ms = migrate_get_current();
2239     MigrationIncomingState *mis = migration_incoming_get_current();
2240     int ret;
2241
2242     if (ms->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2243         /* Source side, during postcopy */
2244         qemu_mutex_lock(&ms->qemu_file_lock);
2245         ret = qemu_file_shutdown(ms->to_dst_file);
2246         qemu_mutex_unlock(&ms->qemu_file_lock);
2247         if (ret) {
2248             error_setg(errp, "Failed to pause source migration");
2249         }
2250         return;
2251     }
2252
2253     if (mis->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2254         ret = qemu_file_shutdown(mis->from_src_file);
2255         if (ret) {
2256             error_setg(errp, "Failed to pause destination migration");
2257         }
2258         return;
2259     }
2260
2261     error_setg(errp, "migrate-pause is currently only supported "
2262                "during postcopy-active state");
2263 }
2264
2265 bool migration_is_blocked(Error **errp)
2266 {
2267     if (qemu_savevm_state_blocked(errp)) {
2268         return true;
2269     }
2270
2271     if (migration_blockers) {
2272         error_propagate(errp, error_copy(migration_blockers->data));
2273         return true;
2274     }
2275
2276     return false;
2277 }
2278
2279 /* Returns true if continue to migrate, or false if error detected */
2280 static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
2281                             bool resume, Error **errp)
2282 {
2283     Error *local_err = NULL;
2284
2285     if (resume) {
2286         if (s->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2287             error_setg(errp, "Cannot resume if there is no "
2288                        "paused migration");
2289             return false;
2290         }
2291
2292         /*
2293          * Postcopy recovery won't work well with release-ram
2294          * capability since release-ram will drop the page buffer as
2295          * long as the page is put into the send buffer.  So if there
2296          * is a network failure happened, any page buffers that have
2297          * not yet reached the destination VM but have already been
2298          * sent from the source VM will be lost forever.  Let's refuse
2299          * the client from resuming such a postcopy migration.
2300          * Luckily release-ram was designed to only be used when src
2301          * and destination VMs are on the same host, so it should be
2302          * fine.
2303          */
2304         if (migrate_release_ram()) {
2305             error_setg(errp, "Postcopy recovery cannot work "
2306                        "when release-ram capability is set");
2307             return false;
2308         }
2309
2310         /* This is a resume, skip init status */
2311         return true;
2312     }
2313
2314     if (migration_is_running(s->state)) {
2315         error_setg(errp, QERR_MIGRATION_ACTIVE);
2316         return false;
2317     }
2318
2319     if (runstate_check(RUN_STATE_INMIGRATE)) {
2320         error_setg(errp, "Guest is waiting for an incoming migration");
2321         return false;
2322     }
2323
2324     if (runstate_check(RUN_STATE_POSTMIGRATE)) {
2325         error_setg(errp, "Can't migrate the vm that was paused due to "
2326                    "previous migration");
2327         return false;
2328     }
2329
2330     if (migration_is_blocked(errp)) {
2331         return false;
2332     }
2333
2334     if (blk || blk_inc) {
2335         if (migrate_colo_enabled()) {
2336             error_setg(errp, "No disk migration is required in COLO mode");
2337             return false;
2338         }
2339         if (migrate_use_block() || migrate_use_block_incremental()) {
2340             error_setg(errp, "Command options are incompatible with "
2341                        "current migration capabilities");
2342             return false;
2343         }
2344         migrate_set_block_enabled(true, &local_err);
2345         if (local_err) {
2346             error_propagate(errp, local_err);
2347             return false;
2348         }
2349         s->must_remove_block_options = true;
2350     }
2351
2352     if (blk_inc) {
2353         migrate_set_block_incremental(s, true);
2354     }
2355
2356     migrate_init(s);
2357     /*
2358      * set ram_counters compression_counters memory to zero for a
2359      * new migration
2360      */
2361     memset(&ram_counters, 0, sizeof(ram_counters));
2362     memset(&compression_counters, 0, sizeof(compression_counters));
2363
2364     return true;
2365 }
2366
2367 void qmp_migrate(const char *uri, bool has_blk, bool blk,
2368                  bool has_inc, bool inc, bool has_detach, bool detach,
2369                  bool has_resume, bool resume, Error **errp)
2370 {
2371     Error *local_err = NULL;
2372     MigrationState *s = migrate_get_current();
2373     const char *p = NULL;
2374
2375     if (!migrate_prepare(s, has_blk && blk, has_inc && inc,
2376                          has_resume && resume, errp)) {
2377         /* Error detected, put into errp */
2378         return;
2379     }
2380
2381     if (!(has_resume && resume)) {
2382         if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2383             return;
2384         }
2385     }
2386
2387     migrate_protocol_allow_multi_channels(false);
2388     if (strstart(uri, "tcp:", &p) ||
2389         strstart(uri, "unix:", NULL) ||
2390         strstart(uri, "vsock:", NULL)) {
2391         migrate_protocol_allow_multi_channels(true);
2392         socket_start_outgoing_migration(s, p ? p : uri, &local_err);
2393 #ifdef CONFIG_RDMA
2394     } else if (strstart(uri, "rdma:", &p)) {
2395         rdma_start_outgoing_migration(s, p, &local_err);
2396 #endif
2397     } else if (strstart(uri, "exec:", &p)) {
2398         exec_start_outgoing_migration(s, p, &local_err);
2399     } else if (strstart(uri, "fd:", &p)) {
2400         fd_start_outgoing_migration(s, p, &local_err);
2401     } else {
2402         if (!(has_resume && resume)) {
2403             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2404         }
2405         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri",
2406                    "a valid migration protocol");
2407         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
2408                           MIGRATION_STATUS_FAILED);
2409         block_cleanup_parameters(s);
2410         return;
2411     }
2412
2413     if (local_err) {
2414         if (!(has_resume && resume)) {
2415             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2416         }
2417         migrate_fd_error(s, local_err);
2418         error_propagate(errp, local_err);
2419         return;
2420     }
2421 }
2422
2423 void qmp_migrate_cancel(Error **errp)
2424 {
2425     migration_cancel(NULL);
2426 }
2427
2428 void qmp_migrate_continue(MigrationStatus state, Error **errp)
2429 {
2430     MigrationState *s = migrate_get_current();
2431     if (s->state != state) {
2432         error_setg(errp,  "Migration not in expected state: %s",
2433                    MigrationStatus_str(s->state));
2434         return;
2435     }
2436     qemu_sem_post(&s->pause_sem);
2437 }
2438
2439 bool migrate_release_ram(void)
2440 {
2441     MigrationState *s;
2442
2443     s = migrate_get_current();
2444
2445     return s->enabled_capabilities[MIGRATION_CAPABILITY_RELEASE_RAM];
2446 }
2447
2448 bool migrate_postcopy_ram(void)
2449 {
2450     MigrationState *s;
2451
2452     s = migrate_get_current();
2453
2454     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM];
2455 }
2456
2457 bool migrate_postcopy(void)
2458 {
2459     return migrate_postcopy_ram() || migrate_dirty_bitmaps();
2460 }
2461
2462 bool migrate_auto_converge(void)
2463 {
2464     MigrationState *s;
2465
2466     s = migrate_get_current();
2467
2468     return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
2469 }
2470
2471 bool migrate_zero_blocks(void)
2472 {
2473     MigrationState *s;
2474
2475     s = migrate_get_current();
2476
2477     return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
2478 }
2479
2480 bool migrate_postcopy_blocktime(void)
2481 {
2482     MigrationState *s;
2483
2484     s = migrate_get_current();
2485
2486     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME];
2487 }
2488
2489 bool migrate_use_compression(void)
2490 {
2491     MigrationState *s;
2492
2493     s = migrate_get_current();
2494
2495     return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS];
2496 }
2497
2498 int migrate_compress_level(void)
2499 {
2500     MigrationState *s;
2501
2502     s = migrate_get_current();
2503
2504     return s->parameters.compress_level;
2505 }
2506
2507 int migrate_compress_threads(void)
2508 {
2509     MigrationState *s;
2510
2511     s = migrate_get_current();
2512
2513     return s->parameters.compress_threads;
2514 }
2515
2516 int migrate_compress_wait_thread(void)
2517 {
2518     MigrationState *s;
2519
2520     s = migrate_get_current();
2521
2522     return s->parameters.compress_wait_thread;
2523 }
2524
2525 int migrate_decompress_threads(void)
2526 {
2527     MigrationState *s;
2528
2529     s = migrate_get_current();
2530
2531     return s->parameters.decompress_threads;
2532 }
2533
2534 bool migrate_dirty_bitmaps(void)
2535 {
2536     MigrationState *s;
2537
2538     s = migrate_get_current();
2539
2540     return s->enabled_capabilities[MIGRATION_CAPABILITY_DIRTY_BITMAPS];
2541 }
2542
2543 bool migrate_ignore_shared(void)
2544 {
2545     MigrationState *s;
2546
2547     s = migrate_get_current();
2548
2549     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_IGNORE_SHARED];
2550 }
2551
2552 bool migrate_validate_uuid(void)
2553 {
2554     MigrationState *s;
2555
2556     s = migrate_get_current();
2557
2558     return s->enabled_capabilities[MIGRATION_CAPABILITY_VALIDATE_UUID];
2559 }
2560
2561 bool migrate_use_events(void)
2562 {
2563     MigrationState *s;
2564
2565     s = migrate_get_current();
2566
2567     return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS];
2568 }
2569
2570 bool migrate_use_multifd(void)
2571 {
2572     MigrationState *s;
2573
2574     s = migrate_get_current();
2575
2576     return s->enabled_capabilities[MIGRATION_CAPABILITY_MULTIFD];
2577 }
2578
2579 bool migrate_pause_before_switchover(void)
2580 {
2581     MigrationState *s;
2582
2583     s = migrate_get_current();
2584
2585     return s->enabled_capabilities[
2586         MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER];
2587 }
2588
2589 int migrate_multifd_channels(void)
2590 {
2591     MigrationState *s;
2592
2593     s = migrate_get_current();
2594
2595     return s->parameters.multifd_channels;
2596 }
2597
2598 MultiFDCompression migrate_multifd_compression(void)
2599 {
2600     MigrationState *s;
2601
2602     s = migrate_get_current();
2603
2604     return s->parameters.multifd_compression;
2605 }
2606
2607 int migrate_multifd_zlib_level(void)
2608 {
2609     MigrationState *s;
2610
2611     s = migrate_get_current();
2612
2613     return s->parameters.multifd_zlib_level;
2614 }
2615
2616 int migrate_multifd_zstd_level(void)
2617 {
2618     MigrationState *s;
2619
2620     s = migrate_get_current();
2621
2622     return s->parameters.multifd_zstd_level;
2623 }
2624
2625 #ifdef CONFIG_LINUX
2626 bool migrate_use_zero_copy_send(void)
2627 {
2628     MigrationState *s;
2629
2630     s = migrate_get_current();
2631
2632     return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_COPY_SEND];
2633 }
2634 #endif
2635
2636 int migrate_use_tls(void)
2637 {
2638     MigrationState *s;
2639
2640     s = migrate_get_current();
2641
2642     return s->parameters.tls_creds && *s->parameters.tls_creds;
2643 }
2644
2645 int migrate_use_xbzrle(void)
2646 {
2647     MigrationState *s;
2648
2649     s = migrate_get_current();
2650
2651     return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
2652 }
2653
2654 uint64_t migrate_xbzrle_cache_size(void)
2655 {
2656     MigrationState *s;
2657
2658     s = migrate_get_current();
2659
2660     return s->parameters.xbzrle_cache_size;
2661 }
2662
2663 static int64_t migrate_max_postcopy_bandwidth(void)
2664 {
2665     MigrationState *s;
2666
2667     s = migrate_get_current();
2668
2669     return s->parameters.max_postcopy_bandwidth;
2670 }
2671
2672 bool migrate_use_block(void)
2673 {
2674     MigrationState *s;
2675
2676     s = migrate_get_current();
2677
2678     return s->enabled_capabilities[MIGRATION_CAPABILITY_BLOCK];
2679 }
2680
2681 bool migrate_use_return_path(void)
2682 {
2683     MigrationState *s;
2684
2685     s = migrate_get_current();
2686
2687     return s->enabled_capabilities[MIGRATION_CAPABILITY_RETURN_PATH];
2688 }
2689
2690 bool migrate_use_block_incremental(void)
2691 {
2692     MigrationState *s;
2693
2694     s = migrate_get_current();
2695
2696     return s->parameters.block_incremental;
2697 }
2698
2699 bool migrate_background_snapshot(void)
2700 {
2701     MigrationState *s;
2702
2703     s = migrate_get_current();
2704
2705     return s->enabled_capabilities[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT];
2706 }
2707
2708 bool migrate_postcopy_preempt(void)
2709 {
2710     MigrationState *s;
2711
2712     s = migrate_get_current();
2713
2714     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_PREEMPT];
2715 }
2716
2717 /* migration thread support */
2718 /*
2719  * Something bad happened to the RP stream, mark an error
2720  * The caller shall print or trace something to indicate why
2721  */
2722 static void mark_source_rp_bad(MigrationState *s)
2723 {
2724     s->rp_state.error = true;
2725 }
2726
2727 static struct rp_cmd_args {
2728     ssize_t     len; /* -1 = variable */
2729     const char *name;
2730 } rp_cmd_args[] = {
2731     [MIG_RP_MSG_INVALID]        = { .len = -1, .name = "INVALID" },
2732     [MIG_RP_MSG_SHUT]           = { .len =  4, .name = "SHUT" },
2733     [MIG_RP_MSG_PONG]           = { .len =  4, .name = "PONG" },
2734     [MIG_RP_MSG_REQ_PAGES]      = { .len = 12, .name = "REQ_PAGES" },
2735     [MIG_RP_MSG_REQ_PAGES_ID]   = { .len = -1, .name = "REQ_PAGES_ID" },
2736     [MIG_RP_MSG_RECV_BITMAP]    = { .len = -1, .name = "RECV_BITMAP" },
2737     [MIG_RP_MSG_RESUME_ACK]     = { .len =  4, .name = "RESUME_ACK" },
2738     [MIG_RP_MSG_MAX]            = { .len = -1, .name = "MAX" },
2739 };
2740
2741 /*
2742  * Process a request for pages received on the return path,
2743  * We're allowed to send more than requested (e.g. to round to our page size)
2744  * and we don't need to send pages that have already been sent.
2745  */
2746 static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
2747                                        ram_addr_t start, size_t len)
2748 {
2749     long our_host_ps = qemu_real_host_page_size();
2750
2751     trace_migrate_handle_rp_req_pages(rbname, start, len);
2752
2753     /*
2754      * Since we currently insist on matching page sizes, just sanity check
2755      * we're being asked for whole host pages.
2756      */
2757     if (!QEMU_IS_ALIGNED(start, our_host_ps) ||
2758         !QEMU_IS_ALIGNED(len, our_host_ps)) {
2759         error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
2760                      " len: %zd", __func__, start, len);
2761         mark_source_rp_bad(ms);
2762         return;
2763     }
2764
2765     if (ram_save_queue_pages(rbname, start, len)) {
2766         mark_source_rp_bad(ms);
2767     }
2768 }
2769
2770 /* Return true to retry, false to quit */
2771 static bool postcopy_pause_return_path_thread(MigrationState *s)
2772 {
2773     trace_postcopy_pause_return_path();
2774
2775     qemu_sem_wait(&s->postcopy_pause_rp_sem);
2776
2777     trace_postcopy_pause_return_path_continued();
2778
2779     return true;
2780 }
2781
2782 static int migrate_handle_rp_recv_bitmap(MigrationState *s, char *block_name)
2783 {
2784     RAMBlock *block = qemu_ram_block_by_name(block_name);
2785
2786     if (!block) {
2787         error_report("%s: invalid block name '%s'", __func__, block_name);
2788         return -EINVAL;
2789     }
2790
2791     /* Fetch the received bitmap and refresh the dirty bitmap */
2792     return ram_dirty_bitmap_reload(s, block);
2793 }
2794
2795 static int migrate_handle_rp_resume_ack(MigrationState *s, uint32_t value)
2796 {
2797     trace_source_return_path_thread_resume_ack(value);
2798
2799     if (value != MIGRATION_RESUME_ACK_VALUE) {
2800         error_report("%s: illegal resume_ack value %"PRIu32,
2801                      __func__, value);
2802         return -1;
2803     }
2804
2805     /* Now both sides are active. */
2806     migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_RECOVER,
2807                       MIGRATION_STATUS_POSTCOPY_ACTIVE);
2808
2809     /* Notify send thread that time to continue send pages */
2810     qemu_sem_post(&s->rp_state.rp_sem);
2811
2812     return 0;
2813 }
2814
2815 /* Release ms->rp_state.from_dst_file in a safe way */
2816 static void migration_release_from_dst_file(MigrationState *ms)
2817 {
2818     QEMUFile *file;
2819
2820     WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) {
2821         /*
2822          * Reset the from_dst_file pointer first before releasing it, as we
2823          * can't block within lock section
2824          */
2825         file = ms->rp_state.from_dst_file;
2826         ms->rp_state.from_dst_file = NULL;
2827     }
2828
2829     qemu_fclose(file);
2830 }
2831
2832 /*
2833  * Handles messages sent on the return path towards the source VM
2834  *
2835  */
2836 static void *source_return_path_thread(void *opaque)
2837 {
2838     MigrationState *ms = opaque;
2839     QEMUFile *rp = ms->rp_state.from_dst_file;
2840     uint16_t header_len, header_type;
2841     uint8_t buf[512];
2842     uint32_t tmp32, sibling_error;
2843     ram_addr_t start = 0; /* =0 to silence warning */
2844     size_t  len = 0, expected_len;
2845     int res;
2846
2847     trace_source_return_path_thread_entry();
2848     rcu_register_thread();
2849
2850 retry:
2851     while (!ms->rp_state.error && !qemu_file_get_error(rp) &&
2852            migration_is_setup_or_active(ms->state)) {
2853         trace_source_return_path_thread_loop_top();
2854         header_type = qemu_get_be16(rp);
2855         header_len = qemu_get_be16(rp);
2856
2857         if (qemu_file_get_error(rp)) {
2858             mark_source_rp_bad(ms);
2859             goto out;
2860         }
2861
2862         if (header_type >= MIG_RP_MSG_MAX ||
2863             header_type == MIG_RP_MSG_INVALID) {
2864             error_report("RP: Received invalid message 0x%04x length 0x%04x",
2865                          header_type, header_len);
2866             mark_source_rp_bad(ms);
2867             goto out;
2868         }
2869
2870         if ((rp_cmd_args[header_type].len != -1 &&
2871             header_len != rp_cmd_args[header_type].len) ||
2872             header_len > sizeof(buf)) {
2873             error_report("RP: Received '%s' message (0x%04x) with"
2874                          "incorrect length %d expecting %zu",
2875                          rp_cmd_args[header_type].name, header_type, header_len,
2876                          (size_t)rp_cmd_args[header_type].len);
2877             mark_source_rp_bad(ms);
2878             goto out;
2879         }
2880
2881         /* We know we've got a valid header by this point */
2882         res = qemu_get_buffer(rp, buf, header_len);
2883         if (res != header_len) {
2884             error_report("RP: Failed reading data for message 0x%04x"
2885                          " read %d expected %d",
2886                          header_type, res, header_len);
2887             mark_source_rp_bad(ms);
2888             goto out;
2889         }
2890
2891         /* OK, we have the message and the data */
2892         switch (header_type) {
2893         case MIG_RP_MSG_SHUT:
2894             sibling_error = ldl_be_p(buf);
2895             trace_source_return_path_thread_shut(sibling_error);
2896             if (sibling_error) {
2897                 error_report("RP: Sibling indicated error %d", sibling_error);
2898                 mark_source_rp_bad(ms);
2899             }
2900             /*
2901              * We'll let the main thread deal with closing the RP
2902              * we could do a shutdown(2) on it, but we're the only user
2903              * anyway, so there's nothing gained.
2904              */
2905             goto out;
2906
2907         case MIG_RP_MSG_PONG:
2908             tmp32 = ldl_be_p(buf);
2909             trace_source_return_path_thread_pong(tmp32);
2910             break;
2911
2912         case MIG_RP_MSG_REQ_PAGES:
2913             start = ldq_be_p(buf);
2914             len = ldl_be_p(buf + 8);
2915             migrate_handle_rp_req_pages(ms, NULL, start, len);
2916             break;
2917
2918         case MIG_RP_MSG_REQ_PAGES_ID:
2919             expected_len = 12 + 1; /* header + termination */
2920
2921             if (header_len >= expected_len) {
2922                 start = ldq_be_p(buf);
2923                 len = ldl_be_p(buf + 8);
2924                 /* Now we expect an idstr */
2925                 tmp32 = buf[12]; /* Length of the following idstr */
2926                 buf[13 + tmp32] = '\0';
2927                 expected_len += tmp32;
2928             }
2929             if (header_len != expected_len) {
2930                 error_report("RP: Req_Page_id with length %d expecting %zd",
2931                              header_len, expected_len);
2932                 mark_source_rp_bad(ms);
2933                 goto out;
2934             }
2935             migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len);
2936             break;
2937
2938         case MIG_RP_MSG_RECV_BITMAP:
2939             if (header_len < 1) {
2940                 error_report("%s: missing block name", __func__);
2941                 mark_source_rp_bad(ms);
2942                 goto out;
2943             }
2944             /* Format: len (1B) + idstr (<255B). This ends the idstr. */
2945             buf[buf[0] + 1] = '\0';
2946             if (migrate_handle_rp_recv_bitmap(ms, (char *)(buf + 1))) {
2947                 mark_source_rp_bad(ms);
2948                 goto out;
2949             }
2950             break;
2951
2952         case MIG_RP_MSG_RESUME_ACK:
2953             tmp32 = ldl_be_p(buf);
2954             if (migrate_handle_rp_resume_ack(ms, tmp32)) {
2955                 mark_source_rp_bad(ms);
2956                 goto out;
2957             }
2958             break;
2959
2960         default:
2961             break;
2962         }
2963     }
2964
2965 out:
2966     res = qemu_file_get_error(rp);
2967     if (res) {
2968         if (res && migration_in_postcopy()) {
2969             /*
2970              * Maybe there is something we can do: it looks like a
2971              * network down issue, and we pause for a recovery.
2972              */
2973             migration_release_from_dst_file(ms);
2974             rp = NULL;
2975             if (postcopy_pause_return_path_thread(ms)) {
2976                 /*
2977                  * Reload rp, reset the rest.  Referencing it is safe since
2978                  * it's reset only by us above, or when migration completes
2979                  */
2980                 rp = ms->rp_state.from_dst_file;
2981                 ms->rp_state.error = false;
2982                 goto retry;
2983             }
2984         }
2985
2986         trace_source_return_path_thread_bad_end();
2987         mark_source_rp_bad(ms);
2988     }
2989
2990     trace_source_return_path_thread_end();
2991     migration_release_from_dst_file(ms);
2992     rcu_unregister_thread();
2993     return NULL;
2994 }
2995
2996 static int open_return_path_on_source(MigrationState *ms,
2997                                       bool create_thread)
2998 {
2999     ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file);
3000     if (!ms->rp_state.from_dst_file) {
3001         return -1;
3002     }
3003
3004     trace_open_return_path_on_source();
3005
3006     if (!create_thread) {
3007         /* We're done */
3008         return 0;
3009     }
3010
3011     qemu_thread_create(&ms->rp_state.rp_thread, "return path",
3012                        source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
3013     ms->rp_state.rp_thread_created = true;
3014
3015     trace_open_return_path_on_source_continue();
3016
3017     return 0;
3018 }
3019
3020 /* Returns 0 if the RP was ok, otherwise there was an error on the RP */
3021 static int await_return_path_close_on_source(MigrationState *ms)
3022 {
3023     /*
3024      * If this is a normal exit then the destination will send a SHUT and the
3025      * rp_thread will exit, however if there's an error we need to cause
3026      * it to exit.
3027      */
3028     if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) {
3029         /*
3030          * shutdown(2), if we have it, will cause it to unblock if it's stuck
3031          * waiting for the destination.
3032          */
3033         qemu_file_shutdown(ms->rp_state.from_dst_file);
3034         mark_source_rp_bad(ms);
3035     }
3036     trace_await_return_path_close_on_source_joining();
3037     qemu_thread_join(&ms->rp_state.rp_thread);
3038     ms->rp_state.rp_thread_created = false;
3039     trace_await_return_path_close_on_source_close();
3040     return ms->rp_state.error;
3041 }
3042
3043 /*
3044  * Switch from normal iteration to postcopy
3045  * Returns non-0 on error
3046  */
3047 static int postcopy_start(MigrationState *ms)
3048 {
3049     int ret;
3050     QIOChannelBuffer *bioc;
3051     QEMUFile *fb;
3052     int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3053     int64_t bandwidth = migrate_max_postcopy_bandwidth();
3054     bool restart_block = false;
3055     int cur_state = MIGRATION_STATUS_ACTIVE;
3056
3057     if (postcopy_preempt_wait_channel(ms)) {
3058         migrate_set_state(&ms->state, ms->state, MIGRATION_STATUS_FAILED);
3059         return -1;
3060     }
3061
3062     if (!migrate_pause_before_switchover()) {
3063         migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
3064                           MIGRATION_STATUS_POSTCOPY_ACTIVE);
3065     }
3066
3067     trace_postcopy_start();
3068     qemu_mutex_lock_iothread();
3069     trace_postcopy_start_set_run();
3070
3071     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3072     global_state_store();
3073     ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
3074     if (ret < 0) {
3075         goto fail;
3076     }
3077
3078     ret = migration_maybe_pause(ms, &cur_state,
3079                                 MIGRATION_STATUS_POSTCOPY_ACTIVE);
3080     if (ret < 0) {
3081         goto fail;
3082     }
3083
3084     ret = bdrv_inactivate_all();
3085     if (ret < 0) {
3086         goto fail;
3087     }
3088     restart_block = true;
3089
3090     /*
3091      * Cause any non-postcopiable, but iterative devices to
3092      * send out their final data.
3093      */
3094     qemu_savevm_state_complete_precopy(ms->to_dst_file, true, false);
3095
3096     /*
3097      * in Finish migrate and with the io-lock held everything should
3098      * be quiet, but we've potentially still got dirty pages and we
3099      * need to tell the destination to throw any pages it's already received
3100      * that are dirty
3101      */
3102     if (migrate_postcopy_ram()) {
3103         ram_postcopy_send_discard_bitmap(ms);
3104     }
3105
3106     /*
3107      * send rest of state - note things that are doing postcopy
3108      * will notice we're in POSTCOPY_ACTIVE and not actually
3109      * wrap their state up here
3110      */
3111     /* 0 max-postcopy-bandwidth means unlimited */
3112     if (!bandwidth) {
3113         qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX);
3114     } else {
3115         qemu_file_set_rate_limit(ms->to_dst_file, bandwidth / XFER_LIMIT_RATIO);
3116     }
3117     if (migrate_postcopy_ram()) {
3118         /* Ping just for debugging, helps line traces up */
3119         qemu_savevm_send_ping(ms->to_dst_file, 2);
3120     }
3121
3122     /*
3123      * While loading the device state we may trigger page transfer
3124      * requests and the fd must be free to process those, and thus
3125      * the destination must read the whole device state off the fd before
3126      * it starts processing it.  Unfortunately the ad-hoc migration format
3127      * doesn't allow the destination to know the size to read without fully
3128      * parsing it through each devices load-state code (especially the open
3129      * coded devices that use get/put).
3130      * So we wrap the device state up in a package with a length at the start;
3131      * to do this we use a qemu_buf to hold the whole of the device state.
3132      */
3133     bioc = qio_channel_buffer_new(4096);
3134     qio_channel_set_name(QIO_CHANNEL(bioc), "migration-postcopy-buffer");
3135     fb = qemu_file_new_output(QIO_CHANNEL(bioc));
3136     object_unref(OBJECT(bioc));
3137
3138     /*
3139      * Make sure the receiver can get incoming pages before we send the rest
3140      * of the state
3141      */
3142     qemu_savevm_send_postcopy_listen(fb);
3143
3144     qemu_savevm_state_complete_precopy(fb, false, false);
3145     if (migrate_postcopy_ram()) {
3146         qemu_savevm_send_ping(fb, 3);
3147     }
3148
3149     qemu_savevm_send_postcopy_run(fb);
3150
3151     /* <><> end of stuff going into the package */
3152
3153     /* Last point of recovery; as soon as we send the package the destination
3154      * can open devices and potentially start running.
3155      * Lets just check again we've not got any errors.
3156      */
3157     ret = qemu_file_get_error(ms->to_dst_file);
3158     if (ret) {
3159         error_report("postcopy_start: Migration stream errored (pre package)");
3160         goto fail_closefb;
3161     }
3162
3163     restart_block = false;
3164
3165     /* Now send that blob */
3166     if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
3167         goto fail_closefb;
3168     }
3169     qemu_fclose(fb);
3170
3171     /* Send a notify to give a chance for anything that needs to happen
3172      * at the transition to postcopy and after the device state; in particular
3173      * spice needs to trigger a transition now
3174      */
3175     ms->postcopy_after_devices = true;
3176     notifier_list_notify(&migration_state_notifiers, ms);
3177
3178     ms->downtime =  qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop;
3179
3180     qemu_mutex_unlock_iothread();
3181
3182     if (migrate_postcopy_ram()) {
3183         /*
3184          * Although this ping is just for debug, it could potentially be
3185          * used for getting a better measurement of downtime at the source.
3186          */
3187         qemu_savevm_send_ping(ms->to_dst_file, 4);
3188     }
3189
3190     if (migrate_release_ram()) {
3191         ram_postcopy_migrated_memory_release(ms);
3192     }
3193
3194     ret = qemu_file_get_error(ms->to_dst_file);
3195     if (ret) {
3196         error_report("postcopy_start: Migration stream errored");
3197         migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3198                               MIGRATION_STATUS_FAILED);
3199     }
3200
3201     trace_postcopy_preempt_enabled(migrate_postcopy_preempt());
3202
3203     return ret;
3204
3205 fail_closefb:
3206     qemu_fclose(fb);
3207 fail:
3208     migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3209                           MIGRATION_STATUS_FAILED);
3210     if (restart_block) {
3211         /* A failure happened early enough that we know the destination hasn't
3212          * accessed block devices, so we're safe to recover.
3213          */
3214         Error *local_err = NULL;
3215
3216         bdrv_activate_all(&local_err);
3217         if (local_err) {
3218             error_report_err(local_err);
3219         }
3220     }
3221     qemu_mutex_unlock_iothread();
3222     return -1;
3223 }
3224
3225 /**
3226  * migration_maybe_pause: Pause if required to by
3227  * migrate_pause_before_switchover called with the iothread locked
3228  * Returns: 0 on success
3229  */
3230 static int migration_maybe_pause(MigrationState *s,
3231                                  int *current_active_state,
3232                                  int new_state)
3233 {
3234     if (!migrate_pause_before_switchover()) {
3235         return 0;
3236     }
3237
3238     /* Since leaving this state is not atomic with posting the semaphore
3239      * it's possible that someone could have issued multiple migrate_continue
3240      * and the semaphore is incorrectly positive at this point;
3241      * the docs say it's undefined to reinit a semaphore that's already
3242      * init'd, so use timedwait to eat up any existing posts.
3243      */
3244     while (qemu_sem_timedwait(&s->pause_sem, 1) == 0) {
3245         /* This block intentionally left blank */
3246     }
3247
3248     /*
3249      * If the migration is cancelled when it is in the completion phase,
3250      * the migration state is set to MIGRATION_STATUS_CANCELLING.
3251      * So we don't need to wait a semaphore, otherwise we would always
3252      * wait for the 'pause_sem' semaphore.
3253      */
3254     if (s->state != MIGRATION_STATUS_CANCELLING) {
3255         qemu_mutex_unlock_iothread();
3256         migrate_set_state(&s->state, *current_active_state,
3257                           MIGRATION_STATUS_PRE_SWITCHOVER);
3258         qemu_sem_wait(&s->pause_sem);
3259         migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER,
3260                           new_state);
3261         *current_active_state = new_state;
3262         qemu_mutex_lock_iothread();
3263     }
3264
3265     return s->state == new_state ? 0 : -EINVAL;
3266 }
3267
3268 /**
3269  * migration_completion: Used by migration_thread when there's not much left.
3270  *   The caller 'breaks' the loop when this returns.
3271  *
3272  * @s: Current migration state
3273  */
3274 static void migration_completion(MigrationState *s)
3275 {
3276     int ret;
3277     int current_active_state = s->state;
3278
3279     if (s->state == MIGRATION_STATUS_ACTIVE) {
3280         qemu_mutex_lock_iothread();
3281         s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3282         qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3283         s->vm_was_running = runstate_is_running();
3284         ret = global_state_store();
3285
3286         if (!ret) {
3287             bool inactivate = !migrate_colo_enabled();
3288             ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
3289             trace_migration_completion_vm_stop(ret);
3290             if (ret >= 0) {
3291                 ret = migration_maybe_pause(s, &current_active_state,
3292                                             MIGRATION_STATUS_DEVICE);
3293             }
3294             if (ret >= 0) {
3295                 qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3296                 ret = qemu_savevm_state_complete_precopy(s->to_dst_file, false,
3297                                                          inactivate);
3298             }
3299             if (inactivate && ret >= 0) {
3300                 s->block_inactive = true;
3301             }
3302         }
3303         qemu_mutex_unlock_iothread();
3304
3305         if (ret < 0) {
3306             goto fail;
3307         }
3308     } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3309         trace_migration_completion_postcopy_end();
3310
3311         qemu_mutex_lock_iothread();
3312         qemu_savevm_state_complete_postcopy(s->to_dst_file);
3313         qemu_mutex_unlock_iothread();
3314
3315         /* Shutdown the postcopy fast path thread */
3316         if (migrate_postcopy_preempt()) {
3317             postcopy_preempt_shutdown_file(s);
3318         }
3319
3320         trace_migration_completion_postcopy_end_after_complete();
3321     } else {
3322         goto fail;
3323     }
3324
3325     /*
3326      * If rp was opened we must clean up the thread before
3327      * cleaning everything else up (since if there are no failures
3328      * it will wait for the destination to send it's status in
3329      * a SHUT command).
3330      */
3331     if (s->rp_state.rp_thread_created) {
3332         int rp_error;
3333         trace_migration_return_path_end_before();
3334         rp_error = await_return_path_close_on_source(s);
3335         trace_migration_return_path_end_after(rp_error);
3336         if (rp_error) {
3337             goto fail_invalidate;
3338         }
3339     }
3340
3341     if (qemu_file_get_error(s->to_dst_file)) {
3342         trace_migration_completion_file_err();
3343         goto fail_invalidate;
3344     }
3345
3346     if (migrate_colo_enabled() && s->state == MIGRATION_STATUS_ACTIVE) {
3347         /* COLO does not support postcopy */
3348         migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE,
3349                           MIGRATION_STATUS_COLO);
3350     } else {
3351         migrate_set_state(&s->state, current_active_state,
3352                           MIGRATION_STATUS_COMPLETED);
3353     }
3354
3355     return;
3356
3357 fail_invalidate:
3358     /* If not doing postcopy, vm_start() will be called: let's regain
3359      * control on images.
3360      */
3361     if (s->state == MIGRATION_STATUS_ACTIVE ||
3362         s->state == MIGRATION_STATUS_DEVICE) {
3363         Error *local_err = NULL;
3364
3365         qemu_mutex_lock_iothread();
3366         bdrv_activate_all(&local_err);
3367         if (local_err) {
3368             error_report_err(local_err);
3369         } else {
3370             s->block_inactive = false;
3371         }
3372         qemu_mutex_unlock_iothread();
3373     }
3374
3375 fail:
3376     migrate_set_state(&s->state, current_active_state,
3377                       MIGRATION_STATUS_FAILED);
3378 }
3379
3380 /**
3381  * bg_migration_completion: Used by bg_migration_thread when after all the
3382  *   RAM has been saved. The caller 'breaks' the loop when this returns.
3383  *
3384  * @s: Current migration state
3385  */
3386 static void bg_migration_completion(MigrationState *s)
3387 {
3388     int current_active_state = s->state;
3389
3390     /*
3391      * Stop tracking RAM writes - un-protect memory, un-register UFFD
3392      * memory ranges, flush kernel wait queues and wake up threads
3393      * waiting for write fault to be resolved.
3394      */
3395     ram_write_tracking_stop();
3396
3397     if (s->state == MIGRATION_STATUS_ACTIVE) {
3398         /*
3399          * By this moment we have RAM content saved into the migration stream.
3400          * The next step is to flush the non-RAM content (device state)
3401          * right after the ram content. The device state has been stored into
3402          * the temporary buffer before RAM saving started.
3403          */
3404         qemu_put_buffer(s->to_dst_file, s->bioc->data, s->bioc->usage);
3405         qemu_fflush(s->to_dst_file);
3406     } else if (s->state == MIGRATION_STATUS_CANCELLING) {
3407         goto fail;
3408     }
3409
3410     if (qemu_file_get_error(s->to_dst_file)) {
3411         trace_migration_completion_file_err();
3412         goto fail;
3413     }
3414
3415     migrate_set_state(&s->state, current_active_state,
3416                       MIGRATION_STATUS_COMPLETED);
3417     return;
3418
3419 fail:
3420     migrate_set_state(&s->state, current_active_state,
3421                       MIGRATION_STATUS_FAILED);
3422 }
3423
3424 bool migrate_colo_enabled(void)
3425 {
3426     MigrationState *s = migrate_get_current();
3427     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_COLO];
3428 }
3429
3430 typedef enum MigThrError {
3431     /* No error detected */
3432     MIG_THR_ERR_NONE = 0,
3433     /* Detected error, but resumed successfully */
3434     MIG_THR_ERR_RECOVERED = 1,
3435     /* Detected fatal error, need to exit */
3436     MIG_THR_ERR_FATAL = 2,
3437 } MigThrError;
3438
3439 static int postcopy_resume_handshake(MigrationState *s)
3440 {
3441     qemu_savevm_send_postcopy_resume(s->to_dst_file);
3442
3443     while (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3444         qemu_sem_wait(&s->rp_state.rp_sem);
3445     }
3446
3447     if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3448         return 0;
3449     }
3450
3451     return -1;
3452 }
3453
3454 /* Return zero if success, or <0 for error */
3455 static int postcopy_do_resume(MigrationState *s)
3456 {
3457     int ret;
3458
3459     /*
3460      * Call all the resume_prepare() hooks, so that modules can be
3461      * ready for the migration resume.
3462      */
3463     ret = qemu_savevm_state_resume_prepare(s);
3464     if (ret) {
3465         error_report("%s: resume_prepare() failure detected: %d",
3466                      __func__, ret);
3467         return ret;
3468     }
3469
3470     /*
3471      * Last handshake with destination on the resume (destination will
3472      * switch to postcopy-active afterwards)
3473      */
3474     ret = postcopy_resume_handshake(s);
3475     if (ret) {
3476         error_report("%s: handshake failed: %d", __func__, ret);
3477         return ret;
3478     }
3479
3480     return 0;
3481 }
3482
3483 /*
3484  * We don't return until we are in a safe state to continue current
3485  * postcopy migration.  Returns MIG_THR_ERR_RECOVERED if recovered, or
3486  * MIG_THR_ERR_FATAL if unrecovery failure happened.
3487  */
3488 static MigThrError postcopy_pause(MigrationState *s)
3489 {
3490     assert(s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
3491
3492     while (true) {
3493         QEMUFile *file;
3494
3495         /*
3496          * Current channel is possibly broken. Release it.  Note that this is
3497          * guaranteed even without lock because to_dst_file should only be
3498          * modified by the migration thread.  That also guarantees that the
3499          * unregister of yank is safe too without the lock.  It should be safe
3500          * even to be within the qemu_file_lock, but we didn't do that to avoid
3501          * taking more mutex (yank_lock) within qemu_file_lock.  TL;DR: we make
3502          * the qemu_file_lock critical section as small as possible.
3503          */
3504         assert(s->to_dst_file);
3505         migration_ioc_unregister_yank_from_file(s->to_dst_file);
3506         qemu_mutex_lock(&s->qemu_file_lock);
3507         file = s->to_dst_file;
3508         s->to_dst_file = NULL;
3509         qemu_mutex_unlock(&s->qemu_file_lock);
3510
3511         qemu_file_shutdown(file);
3512         qemu_fclose(file);
3513
3514         /*
3515          * Do the same to postcopy fast path socket too if there is.  No
3516          * locking needed because no racer as long as we do this before setting
3517          * status to paused.
3518          */
3519         if (s->postcopy_qemufile_src) {
3520             migration_ioc_unregister_yank_from_file(s->postcopy_qemufile_src);
3521             qemu_file_shutdown(s->postcopy_qemufile_src);
3522             qemu_fclose(s->postcopy_qemufile_src);
3523             s->postcopy_qemufile_src = NULL;
3524         }
3525
3526         migrate_set_state(&s->state, s->state,
3527                           MIGRATION_STATUS_POSTCOPY_PAUSED);
3528
3529         error_report("Detected IO failure for postcopy. "
3530                      "Migration paused.");
3531
3532         /*
3533          * We wait until things fixed up. Then someone will setup the
3534          * status back for us.
3535          */
3536         while (s->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
3537             qemu_sem_wait(&s->postcopy_pause_sem);
3538         }
3539
3540         if (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3541             /* Woken up by a recover procedure. Give it a shot */
3542
3543             if (postcopy_preempt_wait_channel(s)) {
3544                 /*
3545                  * Preempt enabled, and new channel create failed; loop
3546                  * back to wait for another recovery.
3547                  */
3548                 continue;
3549             }
3550
3551             /*
3552              * Firstly, let's wake up the return path now, with a new
3553              * return path channel.
3554              */
3555             qemu_sem_post(&s->postcopy_pause_rp_sem);
3556
3557             /* Do the resume logic */
3558             if (postcopy_do_resume(s) == 0) {
3559                 /* Let's continue! */
3560                 trace_postcopy_pause_continued();
3561                 return MIG_THR_ERR_RECOVERED;
3562             } else {
3563                 /*
3564                  * Something wrong happened during the recovery, let's
3565                  * pause again. Pause is always better than throwing
3566                  * data away.
3567                  */
3568                 continue;
3569             }
3570         } else {
3571             /* This is not right... Time to quit. */
3572             return MIG_THR_ERR_FATAL;
3573         }
3574     }
3575 }
3576
3577 static MigThrError migration_detect_error(MigrationState *s)
3578 {
3579     int ret;
3580     int state = s->state;
3581     Error *local_error = NULL;
3582
3583     if (state == MIGRATION_STATUS_CANCELLING ||
3584         state == MIGRATION_STATUS_CANCELLED) {
3585         /* End the migration, but don't set the state to failed */
3586         return MIG_THR_ERR_FATAL;
3587     }
3588
3589     /*
3590      * Try to detect any file errors.  Note that postcopy_qemufile_src will
3591      * be NULL when postcopy preempt is not enabled.
3592      */
3593     ret = qemu_file_get_error_obj_any(s->to_dst_file,
3594                                       s->postcopy_qemufile_src,
3595                                       &local_error);
3596     if (!ret) {
3597         /* Everything is fine */
3598         assert(!local_error);
3599         return MIG_THR_ERR_NONE;
3600     }
3601
3602     if (local_error) {
3603         migrate_set_error(s, local_error);
3604         error_free(local_error);
3605     }
3606
3607     if (state == MIGRATION_STATUS_POSTCOPY_ACTIVE && ret) {
3608         /*
3609          * For postcopy, we allow the network to be down for a
3610          * while. After that, it can be continued by a
3611          * recovery phase.
3612          */
3613         return postcopy_pause(s);
3614     } else {
3615         /*
3616          * For precopy (or postcopy with error outside IO), we fail
3617          * with no time.
3618          */
3619         migrate_set_state(&s->state, state, MIGRATION_STATUS_FAILED);
3620         trace_migration_thread_file_err();
3621
3622         /* Time to stop the migration, now. */
3623         return MIG_THR_ERR_FATAL;
3624     }
3625 }
3626
3627 /* How many bytes have we transferred since the beginning of the migration */
3628 static uint64_t migration_total_bytes(MigrationState *s)
3629 {
3630     return qemu_file_total_transferred(s->to_dst_file) +
3631         ram_counters.multifd_bytes;
3632 }
3633
3634 static void migration_calculate_complete(MigrationState *s)
3635 {
3636     uint64_t bytes = migration_total_bytes(s);
3637     int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3638     int64_t transfer_time;
3639
3640     s->total_time = end_time - s->start_time;
3641     if (!s->downtime) {
3642         /*
3643          * It's still not set, so we are precopy migration.  For
3644          * postcopy, downtime is calculated during postcopy_start().
3645          */
3646         s->downtime = end_time - s->downtime_start;
3647     }
3648
3649     transfer_time = s->total_time - s->setup_time;
3650     if (transfer_time) {
3651         s->mbps = ((double) bytes * 8.0) / transfer_time / 1000;
3652     }
3653 }
3654
3655 static void update_iteration_initial_status(MigrationState *s)
3656 {
3657     /*
3658      * Update these three fields at the same time to avoid mismatch info lead
3659      * wrong speed calculation.
3660      */
3661     s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3662     s->iteration_initial_bytes = migration_total_bytes(s);
3663     s->iteration_initial_pages = ram_get_total_transferred_pages();
3664 }
3665
3666 static void migration_update_counters(MigrationState *s,
3667                                       int64_t current_time)
3668 {
3669     uint64_t transferred, transferred_pages, time_spent;
3670     uint64_t current_bytes; /* bytes transferred since the beginning */
3671     double bandwidth;
3672
3673     if (current_time < s->iteration_start_time + BUFFER_DELAY) {
3674         return;
3675     }
3676
3677     current_bytes = migration_total_bytes(s);
3678     transferred = current_bytes - s->iteration_initial_bytes;
3679     time_spent = current_time - s->iteration_start_time;
3680     bandwidth = (double)transferred / time_spent;
3681     s->threshold_size = bandwidth * s->parameters.downtime_limit;
3682
3683     s->mbps = (((double) transferred * 8.0) /
3684                ((double) time_spent / 1000.0)) / 1000.0 / 1000.0;
3685
3686     transferred_pages = ram_get_total_transferred_pages() -
3687                             s->iteration_initial_pages;
3688     s->pages_per_second = (double) transferred_pages /
3689                              (((double) time_spent / 1000.0));
3690
3691     /*
3692      * if we haven't sent anything, we don't want to
3693      * recalculate. 10000 is a small enough number for our purposes
3694      */
3695     if (ram_counters.dirty_pages_rate && transferred > 10000) {
3696         s->expected_downtime = ram_counters.remaining / bandwidth;
3697     }
3698
3699     qemu_file_reset_rate_limit(s->to_dst_file);
3700
3701     update_iteration_initial_status(s);
3702
3703     trace_migrate_transferred(transferred, time_spent,
3704                               bandwidth, s->threshold_size);
3705 }
3706
3707 /* Migration thread iteration status */
3708 typedef enum {
3709     MIG_ITERATE_RESUME,         /* Resume current iteration */
3710     MIG_ITERATE_SKIP,           /* Skip current iteration */
3711     MIG_ITERATE_BREAK,          /* Break the loop */
3712 } MigIterateState;
3713
3714 /*
3715  * Return true if continue to the next iteration directly, false
3716  * otherwise.
3717  */
3718 static MigIterateState migration_iteration_run(MigrationState *s)
3719 {
3720     uint64_t pending_size, pend_pre, pend_compat, pend_post;
3721     bool in_postcopy = s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE;
3722
3723     qemu_savevm_state_pending(s->to_dst_file, s->threshold_size, &pend_pre,
3724                               &pend_compat, &pend_post);
3725     pending_size = pend_pre + pend_compat + pend_post;
3726
3727     trace_migrate_pending(pending_size, s->threshold_size,
3728                           pend_pre, pend_compat, pend_post);
3729
3730     if (pending_size && pending_size >= s->threshold_size) {
3731         /* Still a significant amount to transfer */
3732         if (!in_postcopy && pend_pre <= s->threshold_size &&
3733             qatomic_read(&s->start_postcopy)) {
3734             if (postcopy_start(s)) {
3735                 error_report("%s: postcopy failed to start", __func__);
3736             }
3737             return MIG_ITERATE_SKIP;
3738         }
3739         /* Just another iteration step */
3740         qemu_savevm_state_iterate(s->to_dst_file, in_postcopy);
3741     } else {
3742         trace_migration_thread_low_pending(pending_size);
3743         migration_completion(s);
3744         return MIG_ITERATE_BREAK;
3745     }
3746
3747     return MIG_ITERATE_RESUME;
3748 }
3749
3750 static void migration_iteration_finish(MigrationState *s)
3751 {
3752     /* If we enabled cpu throttling for auto-converge, turn it off. */
3753     cpu_throttle_stop();
3754
3755     qemu_mutex_lock_iothread();
3756     switch (s->state) {
3757     case MIGRATION_STATUS_COMPLETED:
3758         migration_calculate_complete(s);
3759         runstate_set(RUN_STATE_POSTMIGRATE);
3760         break;
3761     case MIGRATION_STATUS_COLO:
3762         if (!migrate_colo_enabled()) {
3763             error_report("%s: critical error: calling COLO code without "
3764                          "COLO enabled", __func__);
3765         }
3766         migrate_start_colo_process(s);
3767         s->vm_was_running = true;
3768         /* Fallthrough */
3769     case MIGRATION_STATUS_FAILED:
3770     case MIGRATION_STATUS_CANCELLED:
3771     case MIGRATION_STATUS_CANCELLING:
3772         if (s->vm_was_running) {
3773             if (!runstate_check(RUN_STATE_SHUTDOWN)) {
3774                 vm_start();
3775             }
3776         } else {
3777             if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
3778                 runstate_set(RUN_STATE_POSTMIGRATE);
3779             }
3780         }
3781         break;
3782
3783     default:
3784         /* Should not reach here, but if so, forgive the VM. */
3785         error_report("%s: Unknown ending state %d", __func__, s->state);
3786         break;
3787     }
3788     migrate_fd_cleanup_schedule(s);
3789     qemu_mutex_unlock_iothread();
3790 }
3791
3792 static void bg_migration_iteration_finish(MigrationState *s)
3793 {
3794     qemu_mutex_lock_iothread();
3795     switch (s->state) {
3796     case MIGRATION_STATUS_COMPLETED:
3797         migration_calculate_complete(s);
3798         break;
3799
3800     case MIGRATION_STATUS_ACTIVE:
3801     case MIGRATION_STATUS_FAILED:
3802     case MIGRATION_STATUS_CANCELLED:
3803     case MIGRATION_STATUS_CANCELLING:
3804         break;
3805
3806     default:
3807         /* Should not reach here, but if so, forgive the VM. */
3808         error_report("%s: Unknown ending state %d", __func__, s->state);
3809         break;
3810     }
3811
3812     migrate_fd_cleanup_schedule(s);
3813     qemu_mutex_unlock_iothread();
3814 }
3815
3816 /*
3817  * Return true if continue to the next iteration directly, false
3818  * otherwise.
3819  */
3820 static MigIterateState bg_migration_iteration_run(MigrationState *s)
3821 {
3822     int res;
3823
3824     res = qemu_savevm_state_iterate(s->to_dst_file, false);
3825     if (res > 0) {
3826         bg_migration_completion(s);
3827         return MIG_ITERATE_BREAK;
3828     }
3829
3830     return MIG_ITERATE_RESUME;
3831 }
3832
3833 void migration_make_urgent_request(void)
3834 {
3835     qemu_sem_post(&migrate_get_current()->rate_limit_sem);
3836 }
3837
3838 void migration_consume_urgent_request(void)
3839 {
3840     qemu_sem_wait(&migrate_get_current()->rate_limit_sem);
3841 }
3842
3843 /* Returns true if the rate limiting was broken by an urgent request */
3844 bool migration_rate_limit(void)
3845 {
3846     int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3847     MigrationState *s = migrate_get_current();
3848
3849     bool urgent = false;
3850     migration_update_counters(s, now);
3851     if (qemu_file_rate_limit(s->to_dst_file)) {
3852
3853         if (qemu_file_get_error(s->to_dst_file)) {
3854             return false;
3855         }
3856         /*
3857          * Wait for a delay to do rate limiting OR
3858          * something urgent to post the semaphore.
3859          */
3860         int ms = s->iteration_start_time + BUFFER_DELAY - now;
3861         trace_migration_rate_limit_pre(ms);
3862         if (qemu_sem_timedwait(&s->rate_limit_sem, ms) == 0) {
3863             /*
3864              * We were woken by one or more urgent things but
3865              * the timedwait will have consumed one of them.
3866              * The service routine for the urgent wake will dec
3867              * the semaphore itself for each item it consumes,
3868              * so add this one we just eat back.
3869              */
3870             qemu_sem_post(&s->rate_limit_sem);
3871             urgent = true;
3872         }
3873         trace_migration_rate_limit_post(urgent);
3874     }
3875     return urgent;
3876 }
3877
3878 /*
3879  * if failover devices are present, wait they are completely
3880  * unplugged
3881  */
3882
3883 static void qemu_savevm_wait_unplug(MigrationState *s, int old_state,
3884                                     int new_state)
3885 {
3886     if (qemu_savevm_state_guest_unplug_pending()) {
3887         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_WAIT_UNPLUG);
3888
3889         while (s->state == MIGRATION_STATUS_WAIT_UNPLUG &&
3890                qemu_savevm_state_guest_unplug_pending()) {
3891             qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3892         }
3893         if (s->state != MIGRATION_STATUS_WAIT_UNPLUG) {
3894             int timeout = 120; /* 30 seconds */
3895             /*
3896              * migration has been canceled
3897              * but as we have started an unplug we must wait the end
3898              * to be able to plug back the card
3899              */
3900             while (timeout-- && qemu_savevm_state_guest_unplug_pending()) {
3901                 qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3902             }
3903             if (qemu_savevm_state_guest_unplug_pending() &&
3904                 !qtest_enabled()) {
3905                 warn_report("migration: partially unplugged device on "
3906                             "failure");
3907             }
3908         }
3909
3910         migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG, new_state);
3911     } else {
3912         migrate_set_state(&s->state, old_state, new_state);
3913     }
3914 }
3915
3916 /*
3917  * Master migration thread on the source VM.
3918  * It drives the migration and pumps the data down the outgoing channel.
3919  */
3920 static void *migration_thread(void *opaque)
3921 {
3922     MigrationState *s = opaque;
3923     int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3924     MigThrError thr_error;
3925     bool urgent = false;
3926
3927     rcu_register_thread();
3928
3929     object_ref(OBJECT(s));
3930     update_iteration_initial_status(s);
3931
3932     qemu_savevm_state_header(s->to_dst_file);
3933
3934     /*
3935      * If we opened the return path, we need to make sure dst has it
3936      * opened as well.
3937      */
3938     if (s->rp_state.rp_thread_created) {
3939         /* Now tell the dest that it should open its end so it can reply */
3940         qemu_savevm_send_open_return_path(s->to_dst_file);
3941
3942         /* And do a ping that will make stuff easier to debug */
3943         qemu_savevm_send_ping(s->to_dst_file, 1);
3944     }
3945
3946     if (migrate_postcopy()) {
3947         /*
3948          * Tell the destination that we *might* want to do postcopy later;
3949          * if the other end can't do postcopy it should fail now, nice and
3950          * early.
3951          */
3952         qemu_savevm_send_postcopy_advise(s->to_dst_file);
3953     }
3954
3955     if (migrate_colo_enabled()) {
3956         /* Notify migration destination that we enable COLO */
3957         qemu_savevm_send_colo_enable(s->to_dst_file);
3958     }
3959
3960     qemu_savevm_state_setup(s->to_dst_file);
3961
3962     qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
3963                                MIGRATION_STATUS_ACTIVE);
3964
3965     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3966
3967     trace_migration_thread_setup_complete();
3968
3969     while (migration_is_active(s)) {
3970         if (urgent || !qemu_file_rate_limit(s->to_dst_file)) {
3971             MigIterateState iter_state = migration_iteration_run(s);
3972             if (iter_state == MIG_ITERATE_SKIP) {
3973                 continue;
3974             } else if (iter_state == MIG_ITERATE_BREAK) {
3975                 break;
3976             }
3977         }
3978
3979         /*
3980          * Try to detect any kind of failures, and see whether we
3981          * should stop the migration now.
3982          */
3983         thr_error = migration_detect_error(s);
3984         if (thr_error == MIG_THR_ERR_FATAL) {
3985             /* Stop migration */
3986             break;
3987         } else if (thr_error == MIG_THR_ERR_RECOVERED) {
3988             /*
3989              * Just recovered from a e.g. network failure, reset all
3990              * the local variables. This is important to avoid
3991              * breaking transferred_bytes and bandwidth calculation
3992              */
3993             update_iteration_initial_status(s);
3994         }
3995
3996         urgent = migration_rate_limit();
3997     }
3998
3999     trace_migration_thread_after_loop();
4000     migration_iteration_finish(s);
4001     object_unref(OBJECT(s));
4002     rcu_unregister_thread();
4003     return NULL;
4004 }
4005
4006 static void bg_migration_vm_start_bh(void *opaque)
4007 {
4008     MigrationState *s = opaque;
4009
4010     qemu_bh_delete(s->vm_start_bh);
4011     s->vm_start_bh = NULL;
4012
4013     vm_start();
4014     s->downtime = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - s->downtime_start;
4015 }
4016
4017 /**
4018  * Background snapshot thread, based on live migration code.
4019  * This is an alternative implementation of live migration mechanism
4020  * introduced specifically to support background snapshots.
4021  *
4022  * It takes advantage of userfault_fd write protection mechanism introduced
4023  * in v5.7 kernel. Compared to existing dirty page logging migration much
4024  * lesser stream traffic is produced resulting in smaller snapshot images,
4025  * simply cause of no page duplicates can get into the stream.
4026  *
4027  * Another key point is that generated vmstate stream reflects machine state
4028  * 'frozen' at the beginning of snapshot creation compared to dirty page logging
4029  * mechanism, which effectively results in that saved snapshot is the state of VM
4030  * at the end of the process.
4031  */
4032 static void *bg_migration_thread(void *opaque)
4033 {
4034     MigrationState *s = opaque;
4035     int64_t setup_start;
4036     MigThrError thr_error;
4037     QEMUFile *fb;
4038     bool early_fail = true;
4039
4040     rcu_register_thread();
4041     object_ref(OBJECT(s));
4042
4043     qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
4044
4045     setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
4046     /*
4047      * We want to save vmstate for the moment when migration has been
4048      * initiated but also we want to save RAM content while VM is running.
4049      * The RAM content should appear first in the vmstate. So, we first
4050      * stash the non-RAM part of the vmstate to the temporary buffer,
4051      * then write RAM part of the vmstate to the migration stream
4052      * with vCPUs running and, finally, write stashed non-RAM part of
4053      * the vmstate from the buffer to the migration stream.
4054      */
4055     s->bioc = qio_channel_buffer_new(512 * 1024);
4056     qio_channel_set_name(QIO_CHANNEL(s->bioc), "vmstate-buffer");
4057     fb = qemu_file_new_output(QIO_CHANNEL(s->bioc));
4058     object_unref(OBJECT(s->bioc));
4059
4060     update_iteration_initial_status(s);
4061
4062     /*
4063      * Prepare for tracking memory writes with UFFD-WP - populate
4064      * RAM pages before protecting.
4065      */
4066 #ifdef __linux__
4067     ram_write_tracking_prepare();
4068 #endif
4069
4070     qemu_savevm_state_header(s->to_dst_file);
4071     qemu_savevm_state_setup(s->to_dst_file);
4072
4073     qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
4074                                MIGRATION_STATUS_ACTIVE);
4075
4076     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
4077
4078     trace_migration_thread_setup_complete();
4079     s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
4080
4081     qemu_mutex_lock_iothread();
4082
4083     /*
4084      * If VM is currently in suspended state, then, to make a valid runstate
4085      * transition in vm_stop_force_state() we need to wakeup it up.
4086      */
4087     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
4088     s->vm_was_running = runstate_is_running();
4089
4090     if (global_state_store()) {
4091         goto fail;
4092     }
4093     /* Forcibly stop VM before saving state of vCPUs and devices */
4094     if (vm_stop_force_state(RUN_STATE_PAUSED)) {
4095         goto fail;
4096     }
4097     /*
4098      * Put vCPUs in sync with shadow context structures, then
4099      * save their state to channel-buffer along with devices.
4100      */
4101     cpu_synchronize_all_states();
4102     if (qemu_savevm_state_complete_precopy_non_iterable(fb, false, false)) {
4103         goto fail;
4104     }
4105     /*
4106      * Since we are going to get non-iterable state data directly
4107      * from s->bioc->data, explicit flush is needed here.
4108      */
4109     qemu_fflush(fb);
4110
4111     /* Now initialize UFFD context and start tracking RAM writes */
4112     if (ram_write_tracking_start()) {
4113         goto fail;
4114     }
4115     early_fail = false;
4116
4117     /*
4118      * Start VM from BH handler to avoid write-fault lock here.
4119      * UFFD-WP protection for the whole RAM is already enabled so
4120      * calling VM state change notifiers from vm_start() would initiate
4121      * writes to virtio VQs memory which is in write-protected region.
4122      */
4123     s->vm_start_bh = qemu_bh_new(bg_migration_vm_start_bh, s);
4124     qemu_bh_schedule(s->vm_start_bh);
4125
4126     qemu_mutex_unlock_iothread();
4127
4128     while (migration_is_active(s)) {
4129         MigIterateState iter_state = bg_migration_iteration_run(s);
4130         if (iter_state == MIG_ITERATE_SKIP) {
4131             continue;
4132         } else if (iter_state == MIG_ITERATE_BREAK) {
4133             break;
4134         }
4135
4136         /*
4137          * Try to detect any kind of failures, and see whether we
4138          * should stop the migration now.
4139          */
4140         thr_error = migration_detect_error(s);
4141         if (thr_error == MIG_THR_ERR_FATAL) {
4142             /* Stop migration */
4143             break;
4144         }
4145
4146         migration_update_counters(s, qemu_clock_get_ms(QEMU_CLOCK_REALTIME));
4147     }
4148
4149     trace_migration_thread_after_loop();
4150
4151 fail:
4152     if (early_fail) {
4153         migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE,
4154                 MIGRATION_STATUS_FAILED);
4155         qemu_mutex_unlock_iothread();
4156     }
4157
4158     bg_migration_iteration_finish(s);
4159
4160     qemu_fclose(fb);
4161     object_unref(OBJECT(s));
4162     rcu_unregister_thread();
4163
4164     return NULL;
4165 }
4166
4167 void migrate_fd_connect(MigrationState *s, Error *error_in)
4168 {
4169     Error *local_err = NULL;
4170     int64_t rate_limit;
4171     bool resume = s->state == MIGRATION_STATUS_POSTCOPY_PAUSED;
4172
4173     /*
4174      * If there's a previous error, free it and prepare for another one.
4175      * Meanwhile if migration completes successfully, there won't have an error
4176      * dumped when calling migrate_fd_cleanup().
4177      */
4178     migrate_error_free(s);
4179
4180     s->expected_downtime = s->parameters.downtime_limit;
4181     if (resume) {
4182         assert(s->cleanup_bh);
4183     } else {
4184         assert(!s->cleanup_bh);
4185         s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup_bh, s);
4186     }
4187     if (error_in) {
4188         migrate_fd_error(s, error_in);
4189         if (resume) {
4190             /*
4191              * Don't do cleanup for resume if channel is invalid, but only dump
4192              * the error.  We wait for another channel connect from the user.
4193              * The error_report still gives HMP user a hint on what failed.
4194              * It's normally done in migrate_fd_cleanup(), but call it here
4195              * explicitly.
4196              */
4197             error_report_err(error_copy(s->error));
4198         } else {
4199             migrate_fd_cleanup(s);
4200         }
4201         return;
4202     }
4203
4204     if (resume) {
4205         /* This is a resumed migration */
4206         rate_limit = s->parameters.max_postcopy_bandwidth /
4207             XFER_LIMIT_RATIO;
4208     } else {
4209         /* This is a fresh new migration */
4210         rate_limit = s->parameters.max_bandwidth / XFER_LIMIT_RATIO;
4211
4212         /* Notify before starting migration thread */
4213         notifier_list_notify(&migration_state_notifiers, s);
4214     }
4215
4216     qemu_file_set_rate_limit(s->to_dst_file, rate_limit);
4217     qemu_file_set_blocking(s->to_dst_file, true);
4218
4219     /*
4220      * Open the return path. For postcopy, it is used exclusively. For
4221      * precopy, only if user specified "return-path" capability would
4222      * QEMU uses the return path.
4223      */
4224     if (migrate_postcopy_ram() || migrate_use_return_path()) {
4225         if (open_return_path_on_source(s, !resume)) {
4226             error_report("Unable to open return-path for postcopy");
4227             migrate_set_state(&s->state, s->state, MIGRATION_STATUS_FAILED);
4228             migrate_fd_cleanup(s);
4229             return;
4230         }
4231     }
4232
4233     /* This needs to be done before resuming a postcopy */
4234     if (postcopy_preempt_setup(s, &local_err)) {
4235         error_report_err(local_err);
4236         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
4237                           MIGRATION_STATUS_FAILED);
4238         migrate_fd_cleanup(s);
4239         return;
4240     }
4241
4242     if (resume) {
4243         /* Wakeup the main migration thread to do the recovery */
4244         migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
4245                           MIGRATION_STATUS_POSTCOPY_RECOVER);
4246         qemu_sem_post(&s->postcopy_pause_sem);
4247         return;
4248     }
4249
4250     if (multifd_save_setup(&local_err) != 0) {
4251         error_report_err(local_err);
4252         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
4253                           MIGRATION_STATUS_FAILED);
4254         migrate_fd_cleanup(s);
4255         return;
4256     }
4257
4258     if (migrate_background_snapshot()) {
4259         qemu_thread_create(&s->thread, "bg_snapshot",
4260                 bg_migration_thread, s, QEMU_THREAD_JOINABLE);
4261     } else {
4262         qemu_thread_create(&s->thread, "live_migration",
4263                 migration_thread, s, QEMU_THREAD_JOINABLE);
4264     }
4265     s->migration_thread_running = true;
4266 }
4267
4268 void migration_global_dump(Monitor *mon)
4269 {
4270     MigrationState *ms = migrate_get_current();
4271
4272     monitor_printf(mon, "globals:\n");
4273     monitor_printf(mon, "store-global-state: %s\n",
4274                    ms->store_global_state ? "on" : "off");
4275     monitor_printf(mon, "only-migratable: %s\n",
4276                    only_migratable ? "on" : "off");
4277     monitor_printf(mon, "send-configuration: %s\n",
4278                    ms->send_configuration ? "on" : "off");
4279     monitor_printf(mon, "send-section-footer: %s\n",
4280                    ms->send_section_footer ? "on" : "off");
4281     monitor_printf(mon, "decompress-error-check: %s\n",
4282                    ms->decompress_error_check ? "on" : "off");
4283     monitor_printf(mon, "clear-bitmap-shift: %u\n",
4284                    ms->clear_bitmap_shift);
4285 }
4286
4287 #define DEFINE_PROP_MIG_CAP(name, x)             \
4288     DEFINE_PROP_BOOL(name, MigrationState, enabled_capabilities[x], false)
4289
4290 static Property migration_properties[] = {
4291     DEFINE_PROP_BOOL("store-global-state", MigrationState,
4292                      store_global_state, true),
4293     DEFINE_PROP_BOOL("send-configuration", MigrationState,
4294                      send_configuration, true),
4295     DEFINE_PROP_BOOL("send-section-footer", MigrationState,
4296                      send_section_footer, true),
4297     DEFINE_PROP_BOOL("decompress-error-check", MigrationState,
4298                       decompress_error_check, true),
4299     DEFINE_PROP_UINT8("x-clear-bitmap-shift", MigrationState,
4300                       clear_bitmap_shift, CLEAR_BITMAP_SHIFT_DEFAULT),
4301
4302     /* Migration parameters */
4303     DEFINE_PROP_UINT8("x-compress-level", MigrationState,
4304                       parameters.compress_level,
4305                       DEFAULT_MIGRATE_COMPRESS_LEVEL),
4306     DEFINE_PROP_UINT8("x-compress-threads", MigrationState,
4307                       parameters.compress_threads,
4308                       DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT),
4309     DEFINE_PROP_BOOL("x-compress-wait-thread", MigrationState,
4310                       parameters.compress_wait_thread, true),
4311     DEFINE_PROP_UINT8("x-decompress-threads", MigrationState,
4312                       parameters.decompress_threads,
4313                       DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT),
4314     DEFINE_PROP_UINT8("x-throttle-trigger-threshold", MigrationState,
4315                       parameters.throttle_trigger_threshold,
4316                       DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD),
4317     DEFINE_PROP_UINT8("x-cpu-throttle-initial", MigrationState,
4318                       parameters.cpu_throttle_initial,
4319                       DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL),
4320     DEFINE_PROP_UINT8("x-cpu-throttle-increment", MigrationState,
4321                       parameters.cpu_throttle_increment,
4322                       DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT),
4323     DEFINE_PROP_BOOL("x-cpu-throttle-tailslow", MigrationState,
4324                       parameters.cpu_throttle_tailslow, false),
4325     DEFINE_PROP_SIZE("x-max-bandwidth", MigrationState,
4326                       parameters.max_bandwidth, MAX_THROTTLE),
4327     DEFINE_PROP_UINT64("x-downtime-limit", MigrationState,
4328                       parameters.downtime_limit,
4329                       DEFAULT_MIGRATE_SET_DOWNTIME),
4330     DEFINE_PROP_UINT32("x-checkpoint-delay", MigrationState,
4331                       parameters.x_checkpoint_delay,
4332                       DEFAULT_MIGRATE_X_CHECKPOINT_DELAY),
4333     DEFINE_PROP_UINT8("multifd-channels", MigrationState,
4334                       parameters.multifd_channels,
4335                       DEFAULT_MIGRATE_MULTIFD_CHANNELS),
4336     DEFINE_PROP_MULTIFD_COMPRESSION("multifd-compression", MigrationState,
4337                       parameters.multifd_compression,
4338                       DEFAULT_MIGRATE_MULTIFD_COMPRESSION),
4339     DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState,
4340                       parameters.multifd_zlib_level,
4341                       DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL),
4342     DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState,
4343                       parameters.multifd_zstd_level,
4344                       DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL),
4345     DEFINE_PROP_SIZE("xbzrle-cache-size", MigrationState,
4346                       parameters.xbzrle_cache_size,
4347                       DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE),
4348     DEFINE_PROP_SIZE("max-postcopy-bandwidth", MigrationState,
4349                       parameters.max_postcopy_bandwidth,
4350                       DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH),
4351     DEFINE_PROP_UINT8("max-cpu-throttle", MigrationState,
4352                       parameters.max_cpu_throttle,
4353                       DEFAULT_MIGRATE_MAX_CPU_THROTTLE),
4354     DEFINE_PROP_SIZE("announce-initial", MigrationState,
4355                       parameters.announce_initial,
4356                       DEFAULT_MIGRATE_ANNOUNCE_INITIAL),
4357     DEFINE_PROP_SIZE("announce-max", MigrationState,
4358                       parameters.announce_max,
4359                       DEFAULT_MIGRATE_ANNOUNCE_MAX),
4360     DEFINE_PROP_SIZE("announce-rounds", MigrationState,
4361                       parameters.announce_rounds,
4362                       DEFAULT_MIGRATE_ANNOUNCE_ROUNDS),
4363     DEFINE_PROP_SIZE("announce-step", MigrationState,
4364                       parameters.announce_step,
4365                       DEFAULT_MIGRATE_ANNOUNCE_STEP),
4366
4367     /* Migration capabilities */
4368     DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
4369     DEFINE_PROP_MIG_CAP("x-rdma-pin-all", MIGRATION_CAPABILITY_RDMA_PIN_ALL),
4370     DEFINE_PROP_MIG_CAP("x-auto-converge", MIGRATION_CAPABILITY_AUTO_CONVERGE),
4371     DEFINE_PROP_MIG_CAP("x-zero-blocks", MIGRATION_CAPABILITY_ZERO_BLOCKS),
4372     DEFINE_PROP_MIG_CAP("x-compress", MIGRATION_CAPABILITY_COMPRESS),
4373     DEFINE_PROP_MIG_CAP("x-events", MIGRATION_CAPABILITY_EVENTS),
4374     DEFINE_PROP_MIG_CAP("x-postcopy-ram", MIGRATION_CAPABILITY_POSTCOPY_RAM),
4375     DEFINE_PROP_MIG_CAP("x-postcopy-preempt",
4376                         MIGRATION_CAPABILITY_POSTCOPY_PREEMPT),
4377     DEFINE_PROP_MIG_CAP("x-colo", MIGRATION_CAPABILITY_X_COLO),
4378     DEFINE_PROP_MIG_CAP("x-release-ram", MIGRATION_CAPABILITY_RELEASE_RAM),
4379     DEFINE_PROP_MIG_CAP("x-block", MIGRATION_CAPABILITY_BLOCK),
4380     DEFINE_PROP_MIG_CAP("x-return-path", MIGRATION_CAPABILITY_RETURN_PATH),
4381     DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD),
4382     DEFINE_PROP_MIG_CAP("x-background-snapshot",
4383             MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT),
4384 #ifdef CONFIG_LINUX
4385     DEFINE_PROP_MIG_CAP("x-zero-copy-send",
4386             MIGRATION_CAPABILITY_ZERO_COPY_SEND),
4387 #endif
4388
4389     DEFINE_PROP_END_OF_LIST(),
4390 };
4391
4392 static void migration_class_init(ObjectClass *klass, void *data)
4393 {
4394     DeviceClass *dc = DEVICE_CLASS(klass);
4395
4396     dc->user_creatable = false;
4397     device_class_set_props(dc, migration_properties);
4398 }
4399
4400 static void migration_instance_finalize(Object *obj)
4401 {
4402     MigrationState *ms = MIGRATION_OBJ(obj);
4403     MigrationParameters *params = &ms->parameters;
4404
4405     qemu_mutex_destroy(&ms->error_mutex);
4406     qemu_mutex_destroy(&ms->qemu_file_lock);
4407     g_free(params->tls_hostname);
4408     g_free(params->tls_creds);
4409     qemu_sem_destroy(&ms->wait_unplug_sem);
4410     qemu_sem_destroy(&ms->rate_limit_sem);
4411     qemu_sem_destroy(&ms->pause_sem);
4412     qemu_sem_destroy(&ms->postcopy_pause_sem);
4413     qemu_sem_destroy(&ms->postcopy_pause_rp_sem);
4414     qemu_sem_destroy(&ms->rp_state.rp_sem);
4415     qemu_sem_destroy(&ms->postcopy_qemufile_src_sem);
4416     error_free(ms->error);
4417 }
4418
4419 static void migration_instance_init(Object *obj)
4420 {
4421     MigrationState *ms = MIGRATION_OBJ(obj);
4422     MigrationParameters *params = &ms->parameters;
4423
4424     ms->state = MIGRATION_STATUS_NONE;
4425     ms->mbps = -1;
4426     ms->pages_per_second = -1;
4427     qemu_sem_init(&ms->pause_sem, 0);
4428     qemu_mutex_init(&ms->error_mutex);
4429
4430     params->tls_hostname = g_strdup("");
4431     params->tls_creds = g_strdup("");
4432
4433     /* Set has_* up only for parameter checks */
4434     params->has_compress_level = true;
4435     params->has_compress_threads = true;
4436     params->has_decompress_threads = true;
4437     params->has_throttle_trigger_threshold = true;
4438     params->has_cpu_throttle_initial = true;
4439     params->has_cpu_throttle_increment = true;
4440     params->has_cpu_throttle_tailslow = true;
4441     params->has_max_bandwidth = true;
4442     params->has_downtime_limit = true;
4443     params->has_x_checkpoint_delay = true;
4444     params->has_block_incremental = true;
4445     params->has_multifd_channels = true;
4446     params->has_multifd_compression = true;
4447     params->has_multifd_zlib_level = true;
4448     params->has_multifd_zstd_level = true;
4449     params->has_xbzrle_cache_size = true;
4450     params->has_max_postcopy_bandwidth = true;
4451     params->has_max_cpu_throttle = true;
4452     params->has_announce_initial = true;
4453     params->has_announce_max = true;
4454     params->has_announce_rounds = true;
4455     params->has_announce_step = true;
4456
4457     qemu_sem_init(&ms->postcopy_pause_sem, 0);
4458     qemu_sem_init(&ms->postcopy_pause_rp_sem, 0);
4459     qemu_sem_init(&ms->rp_state.rp_sem, 0);
4460     qemu_sem_init(&ms->rate_limit_sem, 0);
4461     qemu_sem_init(&ms->wait_unplug_sem, 0);
4462     qemu_sem_init(&ms->postcopy_qemufile_src_sem, 0);
4463     qemu_mutex_init(&ms->qemu_file_lock);
4464 }
4465
4466 /*
4467  * Return true if check pass, false otherwise. Error will be put
4468  * inside errp if provided.
4469  */
4470 static bool migration_object_check(MigrationState *ms, Error **errp)
4471 {
4472     MigrationCapabilityStatusList *head = NULL;
4473     /* Assuming all off */
4474     bool cap_list[MIGRATION_CAPABILITY__MAX] = { 0 }, ret;
4475     int i;
4476
4477     if (!migrate_params_check(&ms->parameters, errp)) {
4478         return false;
4479     }
4480
4481     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
4482         if (ms->enabled_capabilities[i]) {
4483             QAPI_LIST_PREPEND(head, migrate_cap_add(i, true));
4484         }
4485     }
4486
4487     ret = migrate_caps_check(cap_list, head, errp);
4488
4489     /* It works with head == NULL */
4490     qapi_free_MigrationCapabilityStatusList(head);
4491
4492     return ret;
4493 }
4494
4495 static const TypeInfo migration_type = {
4496     .name = TYPE_MIGRATION,
4497     /*
4498      * NOTE: TYPE_MIGRATION is not really a device, as the object is
4499      * not created using qdev_new(), it is not attached to the qdev
4500      * device tree, and it is never realized.
4501      *
4502      * TODO: Make this TYPE_OBJECT once QOM provides something like
4503      * TYPE_DEVICE's "-global" properties.
4504      */
4505     .parent = TYPE_DEVICE,
4506     .class_init = migration_class_init,
4507     .class_size = sizeof(MigrationClass),
4508     .instance_size = sizeof(MigrationState),
4509     .instance_init = migration_instance_init,
4510     .instance_finalize = migration_instance_finalize,
4511 };
4512
4513 static void register_migration_types(void)
4514 {
4515     type_register_static(&migration_type);
4516 }
4517
4518 type_init(register_migration_types);