migration/migration.c

   1 /*
   2  * QEMU live migration
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *
   6  * Authors:
   7  *  Anthony Liguori   <aliguori@us.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include "qemu/cutils.h"
  18 #include "qemu/error-report.h"
  19 #include "qemu/main-loop.h"
  20 #include "migration/blocker.h"
  21 #include "exec.h"
  22 #include "fd.h"
  23 #include "socket.h"
  24 #include "sysemu/runstate.h"
  25 #include "sysemu/sysemu.h"
  26 #include "sysemu/cpu-throttle.h"
  27 #include "rdma.h"
  28 #include "ram.h"
  29 #include "migration/global_state.h"
  30 #include "migration/misc.h"
  31 #include "migration.h"
  32 #include "savevm.h"
  33 #include "qemu-file-channel.h"
  34 #include "qemu-file.h"
  35 #include "migration/vmstate.h"
  36 #include "block/block.h"
  37 #include "qapi/error.h"
  38 #include "qapi/clone-visitor.h"
  39 #include "qapi/qapi-visit-migration.h"
  40 #include "qapi/qapi-visit-sockets.h"
  41 #include "qapi/qapi-commands-migration.h"
  42 #include "qapi/qapi-events-migration.h"
  43 #include "qapi/qmp/qerror.h"
  44 #include "qapi/qmp/qnull.h"
  45 #include "qemu/rcu.h"
  46 #include "block.h"
  47 #include "postcopy-ram.h"
  48 #include "qemu/thread.h"
  49 #include "trace.h"
  50 #include "exec/target_page.h"
  51 #include "io/channel-buffer.h"
  52 #include "migration/colo.h"
  53 #include "hw/boards.h"
  54 #include "hw/qdev-properties.h"
  55 #include "hw/qdev-properties-system.h"
  56 #include "monitor/monitor.h"
  57 #include "net/announce.h"
  58 #include "qemu/queue.h"
  59 #include "multifd.h"
  60 #include "qemu/yank.h"
  61 #include "sysemu/cpus.h"
  62 #include "yank_functions.h"
  63
  64 #define MAX_THROTTLE  (128 << 20)      /* Migration transfer speed throttling */
  65
  66 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
  67  * data. */
  68 #define BUFFER_DELAY     100
  69 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
  70
  71 /* Time in milliseconds we are allowed to stop the source,
  72  * for sending the last part */
  73 #define DEFAULT_MIGRATE_SET_DOWNTIME 300
  74
  75 /* Maximum migrate downtime set to 2000 seconds */
  76 #define MAX_MIGRATE_DOWNTIME_SECONDS 2000
  77 #define MAX_MIGRATE_DOWNTIME (MAX_MIGRATE_DOWNTIME_SECONDS * 1000)
  78
  79 /* Default compression thread count */
  80 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8
  81 /* Default decompression thread count, usually decompression is at
  82  * least 4 times as fast as compression.*/
  83 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2
  84 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */
  85 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1
  86 /* Define default autoconverge cpu throttle migration parameters */
  87 #define DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD 50
  88 #define DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL 20
  89 #define DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT 10
  90 #define DEFAULT_MIGRATE_MAX_CPU_THROTTLE 99
  91
  92 /* Migration XBZRLE default cache size */
  93 #define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024)
  94
  95 /* The delay time (in ms) between two COLO checkpoints */
  96 #define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100)
  97 #define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2
  98 #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE
  99 /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */
 100 #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1
 101 /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */
 102 #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1
 103
 104 /* Background transfer rate for postcopy, 0 means unlimited, note
 105  * that page requests can still exceed this limit.
 106  */
 107 #define DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH 0
 108
 109 /*
 110  * Parameters for self_announce_delay giving a stream of RARP/ARP
 111  * packets after migration.
 112  */
 113 #define DEFAULT_MIGRATE_ANNOUNCE_INITIAL  50
 114 #define DEFAULT_MIGRATE_ANNOUNCE_MAX     550
 115 #define DEFAULT_MIGRATE_ANNOUNCE_ROUNDS    5
 116 #define DEFAULT_MIGRATE_ANNOUNCE_STEP    100
 117
 118 static NotifierList migration_state_notifiers =
 119     NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
 120
 121 /* Messages sent on the return path from destination to source */
 122 enum mig_rp_message_type {
 123     MIG_RP_MSG_INVALID = 0,  /* Must be 0 */
 124     MIG_RP_MSG_SHUT,         /* sibling will not send any more RP messages */
 125     MIG_RP_MSG_PONG,         /* Response to a PING; data (seq: be32 ) */
 126
 127     MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
 128     MIG_RP_MSG_REQ_PAGES,    /* data (start: be64, len: be32) */
 129     MIG_RP_MSG_RECV_BITMAP,  /* send recved_bitmap back to source */
 130     MIG_RP_MSG_RESUME_ACK,   /* tell source that we are ready to resume */
 131
 132     MIG_RP_MSG_MAX
 133 };
 134
 135 /* Migration capabilities set */
 136 struct MigrateCapsSet {
 137     int size;                       /* Capability set size */
 138     MigrationCapability caps[];     /* Variadic array of capabilities */
 139 };
 140 typedef struct MigrateCapsSet MigrateCapsSet;
 141
 142 /* Define and initialize MigrateCapsSet */
 143 #define INITIALIZE_MIGRATE_CAPS_SET(_name, ...)   \
 144     MigrateCapsSet _name = {    \
 145         .size = sizeof((int []) { __VA_ARGS__ }) / sizeof(int), \
 146         .caps = { __VA_ARGS__ } \
 147     }
 148
 149 /* Background-snapshot compatibility check list */
 150 static const
 151 INITIALIZE_MIGRATE_CAPS_SET(check_caps_background_snapshot,
 152     MIGRATION_CAPABILITY_POSTCOPY_RAM,
 153     MIGRATION_CAPABILITY_DIRTY_BITMAPS,
 154     MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME,
 155     MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE,
 156     MIGRATION_CAPABILITY_RETURN_PATH,
 157     MIGRATION_CAPABILITY_MULTIFD,
 158     MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER,
 159     MIGRATION_CAPABILITY_AUTO_CONVERGE,
 160     MIGRATION_CAPABILITY_RELEASE_RAM,
 161     MIGRATION_CAPABILITY_RDMA_PIN_ALL,
 162     MIGRATION_CAPABILITY_COMPRESS,
 163     MIGRATION_CAPABILITY_XBZRLE,
 164     MIGRATION_CAPABILITY_X_COLO,
 165     MIGRATION_CAPABILITY_VALIDATE_UUID);
 166
 167 /* When we add fault tolerance, we could have several
 168    migrations at once.  For now we don't need to add
 169    dynamic creation of migration */
 170
 171 static MigrationState *current_migration;
 172 static MigrationIncomingState *current_incoming;
 173
 174 static GSList *migration_blockers;
 175
 176 static bool migration_object_check(MigrationState *ms, Error **errp);
 177 static int migration_maybe_pause(MigrationState *s,
 178                                  int *current_active_state,
 179                                  int new_state);
 180 static void migrate_fd_cancel(MigrationState *s);
 181
 182 static gint page_request_addr_cmp(gconstpointer ap, gconstpointer bp)
 183 {
 184     uintptr_t a = (uintptr_t) ap, b = (uintptr_t) bp;
 185
 186     return (a > b) - (a < b);
 187 }
 188
 189 void migration_object_init(void)
 190 {
 191     /* This can only be called once. */
 192     assert(!current_migration);
 193     current_migration = MIGRATION_OBJ(object_new(TYPE_MIGRATION));
 194
 195     /*
 196      * Init the migrate incoming object as well no matter whether
 197      * we'll use it or not.
 198      */
 199     assert(!current_incoming);
 200     current_incoming = g_new0(MigrationIncomingState, 1);
 201     current_incoming->state = MIGRATION_STATUS_NONE;
 202     current_incoming->postcopy_remote_fds =
 203         g_array_new(FALSE, TRUE, sizeof(struct PostCopyFD));
 204     qemu_mutex_init(&current_incoming->rp_mutex);
 205     qemu_event_init(&current_incoming->main_thread_load_event, false);
 206     qemu_sem_init(&current_incoming->postcopy_pause_sem_dst, 0);
 207     qemu_sem_init(&current_incoming->postcopy_pause_sem_fault, 0);
 208     qemu_mutex_init(&current_incoming->page_request_mutex);
 209     current_incoming->page_requested = g_tree_new(page_request_addr_cmp);
 210
 211     migration_object_check(current_migration, &error_fatal);
 212
 213     blk_mig_init();
 214     ram_mig_init();
 215     dirty_bitmap_mig_init();
 216 }
 217
 218 void migration_cancel(const Error *error)
 219 {
 220     if (error) {
 221         migrate_set_error(current_migration, error);
 222     }
 223     migrate_fd_cancel(current_migration);
 224 }
 225
 226 void migration_shutdown(void)
 227 {
 228     /*
 229      * Cancel the current migration - that will (eventually)
 230      * stop the migration using this structure
 231      */
 232     migration_cancel(NULL);
 233     object_unref(OBJECT(current_migration));
 234
 235     /*
 236      * Cancel outgoing migration of dirty bitmaps. It should
 237      * at least unref used block nodes.
 238      */
 239     dirty_bitmap_mig_cancel_outgoing();
 240
 241     /*
 242      * Cancel incoming migration of dirty bitmaps. Dirty bitmaps
 243      * are non-critical data, and their loss never considered as
 244      * something serious.
 245      */
 246     dirty_bitmap_mig_cancel_incoming();
 247 }
 248
 249 /* For outgoing */
 250 MigrationState *migrate_get_current(void)
 251 {
 252     /* This can only be called after the object created. */
 253     assert(current_migration);
 254     return current_migration;
 255 }
 256
 257 MigrationIncomingState *migration_incoming_get_current(void)
 258 {
 259     assert(current_incoming);
 260     return current_incoming;
 261 }
 262
 263 void migration_incoming_state_destroy(void)
 264 {
 265     struct MigrationIncomingState *mis = migration_incoming_get_current();
 266
 267     if (mis->to_src_file) {
 268         /* Tell source that we are done */
 269         migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
 270         qemu_fclose(mis->to_src_file);
 271         mis->to_src_file = NULL;
 272     }
 273
 274     if (mis->from_src_file) {
 275         migration_ioc_unregister_yank_from_file(mis->from_src_file);
 276         qemu_fclose(mis->from_src_file);
 277         mis->from_src_file = NULL;
 278     }
 279     if (mis->postcopy_remote_fds) {
 280         g_array_free(mis->postcopy_remote_fds, TRUE);
 281         mis->postcopy_remote_fds = NULL;
 282     }
 283     if (mis->transport_cleanup) {
 284         mis->transport_cleanup(mis->transport_data);
 285     }
 286
 287     qemu_event_reset(&mis->main_thread_load_event);
 288
 289     if (mis->page_requested) {
 290         g_tree_destroy(mis->page_requested);
 291         mis->page_requested = NULL;
 292     }
 293
 294     if (mis->socket_address_list) {
 295         qapi_free_SocketAddressList(mis->socket_address_list);
 296         mis->socket_address_list = NULL;
 297     }
 298
 299     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
 300 }
 301
 302 static void migrate_generate_event(int new_state)
 303 {
 304     if (migrate_use_events()) {
 305         qapi_event_send_migration(new_state);
 306     }
 307 }
 308
 309 static bool migrate_late_block_activate(void)
 310 {
 311     MigrationState *s;
 312
 313     s = migrate_get_current();
 314
 315     return s->enabled_capabilities[
 316         MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE];
 317 }
 318
 319 /*
 320  * Send a message on the return channel back to the source
 321  * of the migration.
 322  */
 323 static int migrate_send_rp_message(MigrationIncomingState *mis,
 324                                    enum mig_rp_message_type message_type,
 325                                    uint16_t len, void *data)
 326 {
 327     int ret = 0;
 328
 329     trace_migrate_send_rp_message((int)message_type, len);
 330     QEMU_LOCK_GUARD(&mis->rp_mutex);
 331
 332     /*
 333      * It's possible that the file handle got lost due to network
 334      * failures.
 335      */
 336     if (!mis->to_src_file) {
 337         ret = -EIO;
 338         return ret;
 339     }
 340
 341     qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
 342     qemu_put_be16(mis->to_src_file, len);
 343     qemu_put_buffer(mis->to_src_file, data, len);
 344     qemu_fflush(mis->to_src_file);
 345
 346     /* It's possible that qemu file got error during sending */
 347     ret = qemu_file_get_error(mis->to_src_file);
 348
 349     return ret;
 350 }
 351
 352 /* Request one page from the source VM at the given start address.
 353  *   rb: the RAMBlock to request the page in
 354  *   Start: Address offset within the RB
 355  *   Len: Length in bytes required - must be a multiple of pagesize
 356  */
 357 int migrate_send_rp_message_req_pages(MigrationIncomingState *mis,
 358                                       RAMBlock *rb, ram_addr_t start)
 359 {
 360     uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */
 361     size_t msglen = 12; /* start + len */
 362     size_t len = qemu_ram_pagesize(rb);
 363     enum mig_rp_message_type msg_type;
 364     const char *rbname;
 365     int rbname_len;
 366
 367     *(uint64_t *)bufc = cpu_to_be64((uint64_t)start);
 368     *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len);
 369
 370     /*
 371      * We maintain the last ramblock that we requested for page.  Note that we
 372      * don't need locking because this function will only be called within the
 373      * postcopy ram fault thread.
 374      */
 375     if (rb != mis->last_rb) {
 376         mis->last_rb = rb;
 377
 378         rbname = qemu_ram_get_idstr(rb);
 379         rbname_len = strlen(rbname);
 380
 381         assert(rbname_len < 256);
 382
 383         bufc[msglen++] = rbname_len;
 384         memcpy(bufc + msglen, rbname, rbname_len);
 385         msglen += rbname_len;
 386         msg_type = MIG_RP_MSG_REQ_PAGES_ID;
 387     } else {
 388         msg_type = MIG_RP_MSG_REQ_PAGES;
 389     }
 390
 391     return migrate_send_rp_message(mis, msg_type, msglen, bufc);
 392 }
 393
 394 int migrate_send_rp_req_pages(MigrationIncomingState *mis,
 395                               RAMBlock *rb, ram_addr_t start, uint64_t haddr)
 396 {
 397     void *aligned = (void *)(uintptr_t)ROUND_DOWN(haddr, qemu_ram_pagesize(rb));
 398     bool received = false;
 399
 400     WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) {
 401         received = ramblock_recv_bitmap_test_byte_offset(rb, start);
 402         if (!received && !g_tree_lookup(mis->page_requested, aligned)) {
 403             /*
 404              * The page has not been received, and it's not yet in the page
 405              * request list.  Queue it.  Set the value of element to 1, so that
 406              * things like g_tree_lookup() will return TRUE (1) when found.
 407              */
 408             g_tree_insert(mis->page_requested, aligned, (gpointer)1);
 409             mis->page_requested_count++;
 410             trace_postcopy_page_req_add(aligned, mis->page_requested_count);
 411         }
 412     }
 413
 414     /*
 415      * If the page is there, skip sending the message.  We don't even need the
 416      * lock because as long as the page arrived, it'll be there forever.
 417      */
 418     if (received) {
 419         return 0;
 420     }
 421
 422     return migrate_send_rp_message_req_pages(mis, rb, start);
 423 }
 424
 425 static bool migration_colo_enabled;
 426 bool migration_incoming_colo_enabled(void)
 427 {
 428     return migration_colo_enabled;
 429 }
 430
 431 void migration_incoming_disable_colo(void)
 432 {
 433     ram_block_discard_disable(false);
 434     migration_colo_enabled = false;
 435 }
 436
 437 int migration_incoming_enable_colo(void)
 438 {
 439     if (ram_block_discard_disable(true)) {
 440         error_report("COLO: cannot disable RAM discard");
 441         return -EBUSY;
 442     }
 443     migration_colo_enabled = true;
 444     return 0;
 445 }
 446
 447 void migrate_add_address(SocketAddress *address)
 448 {
 449     MigrationIncomingState *mis = migration_incoming_get_current();
 450
 451     QAPI_LIST_PREPEND(mis->socket_address_list,
 452                       QAPI_CLONE(SocketAddress, address));
 453 }
 454
 455 static void qemu_start_incoming_migration(const char *uri, Error **errp)
 456 {
 457     const char *p = NULL;
 458
 459     migrate_protocol_allow_multifd(false); /* reset it anyway */
 460     qapi_event_send_migration(MIGRATION_STATUS_SETUP);
 461     if (strstart(uri, "tcp:", &p) ||
 462         strstart(uri, "unix:", NULL) ||
 463         strstart(uri, "vsock:", NULL)) {
 464         migrate_protocol_allow_multifd(true);
 465         socket_start_incoming_migration(p ? p : uri, errp);
 466 #ifdef CONFIG_RDMA
 467     } else if (strstart(uri, "rdma:", &p)) {
 468         rdma_start_incoming_migration(p, errp);
 469 #endif
 470     } else if (strstart(uri, "exec:", &p)) {
 471         exec_start_incoming_migration(p, errp);
 472     } else if (strstart(uri, "fd:", &p)) {
 473         fd_start_incoming_migration(p, errp);
 474     } else {
 475         error_setg(errp, "unknown migration protocol: %s", uri);
 476     }
 477 }
 478
 479 static void process_incoming_migration_bh(void *opaque)
 480 {
 481     Error *local_err = NULL;
 482     MigrationIncomingState *mis = opaque;
 483
 484     /* If capability late_block_activate is set:
 485      * Only fire up the block code now if we're going to restart the
 486      * VM, else 'cont' will do it.
 487      * This causes file locking to happen; so we don't want it to happen
 488      * unless we really are starting the VM.
 489      */
 490     if (!migrate_late_block_activate() ||
 491          (autostart && (!global_state_received() ||
 492             global_state_get_runstate() == RUN_STATE_RUNNING))) {
 493         /* Make sure all file formats flush their mutable metadata.
 494          * If we get an error here, just don't restart the VM yet. */
 495         bdrv_invalidate_cache_all(&local_err);
 496         if (local_err) {
 497             error_report_err(local_err);
 498             local_err = NULL;
 499             autostart = false;
 500         }
 501     }
 502
 503     /*
 504      * This must happen after all error conditions are dealt with and
 505      * we're sure the VM is going to be running on this host.
 506      */
 507     qemu_announce_self(&mis->announce_timer, migrate_announce_params());
 508
 509     if (multifd_load_cleanup(&local_err) != 0) {
 510         error_report_err(local_err);
 511         autostart = false;
 512     }
 513     /* If global state section was not received or we are in running
 514        state, we need to obey autostart. Any other state is set with
 515        runstate_set. */
 516
 517     dirty_bitmap_mig_before_vm_start();
 518
 519     if (!global_state_received() ||
 520         global_state_get_runstate() == RUN_STATE_RUNNING) {
 521         if (autostart) {
 522             vm_start();
 523         } else {
 524             runstate_set(RUN_STATE_PAUSED);
 525         }
 526     } else if (migration_incoming_colo_enabled()) {
 527         migration_incoming_disable_colo();
 528         vm_start();
 529     } else {
 530         runstate_set(global_state_get_runstate());
 531     }
 532     /*
 533      * This must happen after any state changes since as soon as an external
 534      * observer sees this event they might start to prod at the VM assuming
 535      * it's ready to use.
 536      */
 537     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 538                       MIGRATION_STATUS_COMPLETED);
 539     qemu_bh_delete(mis->bh);
 540     migration_incoming_state_destroy();
 541 }
 542
 543 static void process_incoming_migration_co(void *opaque)
 544 {
 545     MigrationIncomingState *mis = migration_incoming_get_current();
 546     PostcopyState ps;
 547     int ret;
 548     Error *local_err = NULL;
 549
 550     assert(mis->from_src_file);
 551     mis->migration_incoming_co = qemu_coroutine_self();
 552     mis->largest_page_size = qemu_ram_pagesize_largest();
 553     postcopy_state_set(POSTCOPY_INCOMING_NONE);
 554     migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
 555                       MIGRATION_STATUS_ACTIVE);
 556     ret = qemu_loadvm_state(mis->from_src_file);
 557
 558     ps = postcopy_state_get();
 559     trace_process_incoming_migration_co_end(ret, ps);
 560     if (ps != POSTCOPY_INCOMING_NONE) {
 561         if (ps == POSTCOPY_INCOMING_ADVISE) {
 562             /*
 563              * Where a migration had postcopy enabled (and thus went to advise)
 564              * but managed to complete within the precopy period, we can use
 565              * the normal exit.
 566              */
 567             postcopy_ram_incoming_cleanup(mis);
 568         } else if (ret >= 0) {
 569             /*
 570              * Postcopy was started, cleanup should happen at the end of the
 571              * postcopy thread.
 572              */
 573             trace_process_incoming_migration_co_postcopy_end_main();
 574             return;
 575         }
 576         /* Else if something went wrong then just fall out of the normal exit */
 577     }
 578
 579     /* we get COLO info, and know if we are in COLO mode */
 580     if (!ret && migration_incoming_colo_enabled()) {
 581         /* Make sure all file formats flush their mutable metadata */
 582         bdrv_invalidate_cache_all(&local_err);
 583         if (local_err) {
 584             error_report_err(local_err);
 585             goto fail;
 586         }
 587
 588         qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
 589              colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
 590         mis->have_colo_incoming_thread = true;
 591         qemu_coroutine_yield();
 592
 593         qemu_mutex_unlock_iothread();
 594         /* Wait checkpoint incoming thread exit before free resource */
 595         qemu_thread_join(&mis->colo_incoming_thread);
 596         qemu_mutex_lock_iothread();
 597         /* We hold the global iothread lock, so it is safe here */
 598         colo_release_ram_cache();
 599     }
 600
 601     if (ret < 0) {
 602         error_report("load of migration failed: %s", strerror(-ret));
 603         goto fail;
 604     }
 605     mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
 606     qemu_bh_schedule(mis->bh);
 607     mis->migration_incoming_co = NULL;
 608     return;
 609 fail:
 610     local_err = NULL;
 611     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 612                       MIGRATION_STATUS_FAILED);
 613     qemu_fclose(mis->from_src_file);
 614     if (multifd_load_cleanup(&local_err) != 0) {
 615         error_report_err(local_err);
 616     }
 617     exit(EXIT_FAILURE);
 618 }
 619
 620 /**
 621  * migration_incoming_setup: Setup incoming migration
 622  * @f: file for main migration channel
 623  * @errp: where to put errors
 624  *
 625  * Returns: %true on success, %false on error.
 626  */
 627 static bool migration_incoming_setup(QEMUFile *f, Error **errp)
 628 {
 629     MigrationIncomingState *mis = migration_incoming_get_current();
 630
 631     if (multifd_load_setup(errp) != 0) {
 632         return false;
 633     }
 634
 635     if (!mis->from_src_file) {
 636         mis->from_src_file = f;
 637     }
 638     qemu_file_set_blocking(f, false);
 639     return true;
 640 }
 641
 642 void migration_incoming_process(void)
 643 {
 644     Coroutine *co = qemu_coroutine_create(process_incoming_migration_co, NULL);
 645     qemu_coroutine_enter(co);
 646 }
 647
 648 /* Returns true if recovered from a paused migration, otherwise false */
 649 static bool postcopy_try_recover(QEMUFile *f)
 650 {
 651     MigrationIncomingState *mis = migration_incoming_get_current();
 652
 653     if (mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
 654         /* Resumed from a paused postcopy migration */
 655
 656         mis->from_src_file = f;
 657         /* Postcopy has standalone thread to do vm load */
 658         qemu_file_set_blocking(f, true);
 659
 660         /* Re-configure the return path */
 661         mis->to_src_file = qemu_file_get_return_path(f);
 662
 663         migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
 664                           MIGRATION_STATUS_POSTCOPY_RECOVER);
 665
 666         /*
 667          * Here, we only wake up the main loading thread (while the
 668          * fault thread will still be waiting), so that we can receive
 669          * commands from source now, and answer it if needed. The
 670          * fault thread will be woken up afterwards until we are sure
 671          * that source is ready to reply to page requests.
 672          */
 673         qemu_sem_post(&mis->postcopy_pause_sem_dst);
 674         return true;
 675     }
 676
 677     return false;
 678 }
 679
 680 void migration_fd_process_incoming(QEMUFile *f, Error **errp)
 681 {
 682     if (postcopy_try_recover(f)) {
 683         return;
 684     }
 685
 686     if (!migration_incoming_setup(f, errp)) {
 687         return;
 688     }
 689     migration_incoming_process();
 690 }
 691
 692 void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp)
 693 {
 694     MigrationIncomingState *mis = migration_incoming_get_current();
 695     Error *local_err = NULL;
 696     bool start_migration;
 697
 698     if (!mis->from_src_file) {
 699         /* The first connection (multifd may have multiple) */
 700         QEMUFile *f = qemu_fopen_channel_input(ioc);
 701
 702         /* If it's a recovery, we're done */
 703         if (postcopy_try_recover(f)) {
 704             return;
 705         }
 706
 707         if (!migration_incoming_setup(f, errp)) {
 708             return;
 709         }
 710
 711         /*
 712          * Common migration only needs one channel, so we can start
 713          * right now.  Multifd needs more than one channel, we wait.
 714          */
 715         start_migration = !migrate_use_multifd();
 716     } else {
 717         /* Multiple connections */
 718         assert(migrate_use_multifd());
 719         start_migration = multifd_recv_new_channel(ioc, &local_err);
 720         if (local_err) {
 721             error_propagate(errp, local_err);
 722             return;
 723         }
 724     }
 725
 726     if (start_migration) {
 727         migration_incoming_process();
 728     }
 729 }
 730
 731 /**
 732  * @migration_has_all_channels: We have received all channels that we need
 733  *
 734  * Returns true when we have got connections to all the channels that
 735  * we need for migration.
 736  */
 737 bool migration_has_all_channels(void)
 738 {
 739     MigrationIncomingState *mis = migration_incoming_get_current();
 740     bool all_channels;
 741
 742     all_channels = multifd_recv_all_channels_created();
 743
 744     return all_channels && mis->from_src_file != NULL;
 745 }
 746
 747 /*
 748  * Send a 'SHUT' message on the return channel with the given value
 749  * to indicate that we've finished with the RP.  Non-0 value indicates
 750  * error.
 751  */
 752 void migrate_send_rp_shut(MigrationIncomingState *mis,
 753                           uint32_t value)
 754 {
 755     uint32_t buf;
 756
 757     buf = cpu_to_be32(value);
 758     migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
 759 }
 760
 761 /*
 762  * Send a 'PONG' message on the return channel with the given value
 763  * (normally in response to a 'PING')
 764  */
 765 void migrate_send_rp_pong(MigrationIncomingState *mis,
 766                           uint32_t value)
 767 {
 768     uint32_t buf;
 769
 770     buf = cpu_to_be32(value);
 771     migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
 772 }
 773
 774 void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
 775                                  char *block_name)
 776 {
 777     char buf[512];
 778     int len;
 779     int64_t res;
 780
 781     /*
 782      * First, we send the header part. It contains only the len of
 783      * idstr, and the idstr itself.
 784      */
 785     len = strlen(block_name);
 786     buf[0] = len;
 787     memcpy(buf + 1, block_name, len);
 788
 789     if (mis->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
 790         error_report("%s: MSG_RP_RECV_BITMAP only used for recovery",
 791                      __func__);
 792         return;
 793     }
 794
 795     migrate_send_rp_message(mis, MIG_RP_MSG_RECV_BITMAP, len + 1, buf);
 796
 797     /*
 798      * Next, we dump the received bitmap to the stream.
 799      *
 800      * TODO: currently we are safe since we are the only one that is
 801      * using the to_src_file handle (fault thread is still paused),
 802      * and it's ok even not taking the mutex. However the best way is
 803      * to take the lock before sending the message header, and release
 804      * the lock after sending the bitmap.
 805      */
 806     qemu_mutex_lock(&mis->rp_mutex);
 807     res = ramblock_recv_bitmap_send(mis->to_src_file, block_name);
 808     qemu_mutex_unlock(&mis->rp_mutex);
 809
 810     trace_migrate_send_rp_recv_bitmap(block_name, res);
 811 }
 812
 813 void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value)
 814 {
 815     uint32_t buf;
 816
 817     buf = cpu_to_be32(value);
 818     migrate_send_rp_message(mis, MIG_RP_MSG_RESUME_ACK, sizeof(buf), &buf);
 819 }
 820
 821 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
 822 {
 823     MigrationCapabilityStatusList *head = NULL, **tail = &head;
 824     MigrationCapabilityStatus *caps;
 825     MigrationState *s = migrate_get_current();
 826     int i;
 827
 828     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
 829 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
 830         if (i == MIGRATION_CAPABILITY_BLOCK) {
 831             continue;
 832         }
 833 #endif
 834         caps = g_malloc0(sizeof(*caps));
 835         caps->capability = i;
 836         caps->state = s->enabled_capabilities[i];
 837         QAPI_LIST_APPEND(tail, caps);
 838     }
 839
 840     return head;
 841 }
 842
 843 MigrationParameters *qmp_query_migrate_parameters(Error **errp)
 844 {
 845     MigrationParameters *params;
 846     MigrationState *s = migrate_get_current();
 847
 848     /* TODO use QAPI_CLONE() instead of duplicating it inline */
 849     params = g_malloc0(sizeof(*params));
 850     params->has_compress_level = true;
 851     params->compress_level = s->parameters.compress_level;
 852     params->has_compress_threads = true;
 853     params->compress_threads = s->parameters.compress_threads;
 854     params->has_compress_wait_thread = true;
 855     params->compress_wait_thread = s->parameters.compress_wait_thread;
 856     params->has_decompress_threads = true;
 857     params->decompress_threads = s->parameters.decompress_threads;
 858     params->has_throttle_trigger_threshold = true;
 859     params->throttle_trigger_threshold = s->parameters.throttle_trigger_threshold;
 860     params->has_cpu_throttle_initial = true;
 861     params->cpu_throttle_initial = s->parameters.cpu_throttle_initial;
 862     params->has_cpu_throttle_increment = true;
 863     params->cpu_throttle_increment = s->parameters.cpu_throttle_increment;
 864     params->has_cpu_throttle_tailslow = true;
 865     params->cpu_throttle_tailslow = s->parameters.cpu_throttle_tailslow;
 866     params->has_tls_creds = true;
 867     params->tls_creds = g_strdup(s->parameters.tls_creds);
 868     params->has_tls_hostname = true;
 869     params->tls_hostname = g_strdup(s->parameters.tls_hostname);
 870     params->has_tls_authz = true;
 871     params->tls_authz = g_strdup(s->parameters.tls_authz ?
 872                                  s->parameters.tls_authz : "");
 873     params->has_max_bandwidth = true;
 874     params->max_bandwidth = s->parameters.max_bandwidth;
 875     params->has_downtime_limit = true;
 876     params->downtime_limit = s->parameters.downtime_limit;
 877     params->has_x_checkpoint_delay = true;
 878     params->x_checkpoint_delay = s->parameters.x_checkpoint_delay;
 879     params->has_block_incremental = true;
 880     params->block_incremental = s->parameters.block_incremental;
 881     params->has_multifd_channels = true;
 882     params->multifd_channels = s->parameters.multifd_channels;
 883     params->has_multifd_compression = true;
 884     params->multifd_compression = s->parameters.multifd_compression;
 885     params->has_multifd_zlib_level = true;
 886     params->multifd_zlib_level = s->parameters.multifd_zlib_level;
 887     params->has_multifd_zstd_level = true;
 888     params->multifd_zstd_level = s->parameters.multifd_zstd_level;
 889     params->has_xbzrle_cache_size = true;
 890     params->xbzrle_cache_size = s->parameters.xbzrle_cache_size;
 891     params->has_max_postcopy_bandwidth = true;
 892     params->max_postcopy_bandwidth = s->parameters.max_postcopy_bandwidth;
 893     params->has_max_cpu_throttle = true;
 894     params->max_cpu_throttle = s->parameters.max_cpu_throttle;
 895     params->has_announce_initial = true;
 896     params->announce_initial = s->parameters.announce_initial;
 897     params->has_announce_max = true;
 898     params->announce_max = s->parameters.announce_max;
 899     params->has_announce_rounds = true;
 900     params->announce_rounds = s->parameters.announce_rounds;
 901     params->has_announce_step = true;
 902     params->announce_step = s->parameters.announce_step;
 903
 904     if (s->parameters.has_block_bitmap_mapping) {
 905         params->has_block_bitmap_mapping = true;
 906         params->block_bitmap_mapping =
 907             QAPI_CLONE(BitmapMigrationNodeAliasList,
 908                        s->parameters.block_bitmap_mapping);
 909     }
 910
 911     return params;
 912 }
 913
 914 AnnounceParameters *migrate_announce_params(void)
 915 {
 916     static AnnounceParameters ap;
 917
 918     MigrationState *s = migrate_get_current();
 919
 920     ap.initial = s->parameters.announce_initial;
 921     ap.max = s->parameters.announce_max;
 922     ap.rounds = s->parameters.announce_rounds;
 923     ap.step = s->parameters.announce_step;
 924
 925     return &ap;
 926 }
 927
 928 /*
 929  * Return true if we're already in the middle of a migration
 930  * (i.e. any of the active or setup states)
 931  */
 932 bool migration_is_setup_or_active(int state)
 933 {
 934     switch (state) {
 935     case MIGRATION_STATUS_ACTIVE:
 936     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 937     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 938     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 939     case MIGRATION_STATUS_SETUP:
 940     case MIGRATION_STATUS_PRE_SWITCHOVER:
 941     case MIGRATION_STATUS_DEVICE:
 942     case MIGRATION_STATUS_WAIT_UNPLUG:
 943     case MIGRATION_STATUS_COLO:
 944         return true;
 945
 946     default:
 947         return false;
 948
 949     }
 950 }
 951
 952 bool migration_is_running(int state)
 953 {
 954     switch (state) {
 955     case MIGRATION_STATUS_ACTIVE:
 956     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 957     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 958     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 959     case MIGRATION_STATUS_SETUP:
 960     case MIGRATION_STATUS_PRE_SWITCHOVER:
 961     case MIGRATION_STATUS_DEVICE:
 962     case MIGRATION_STATUS_WAIT_UNPLUG:
 963     case MIGRATION_STATUS_CANCELLING:
 964         return true;
 965
 966     default:
 967         return false;
 968
 969     }
 970 }
 971
 972 static void populate_time_info(MigrationInfo *info, MigrationState *s)
 973 {
 974     info->has_status = true;
 975     info->has_setup_time = true;
 976     info->setup_time = s->setup_time;
 977     if (s->state == MIGRATION_STATUS_COMPLETED) {
 978         info->has_total_time = true;
 979         info->total_time = s->total_time;
 980         info->has_downtime = true;
 981         info->downtime = s->downtime;
 982     } else {
 983         info->has_total_time = true;
 984         info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) -
 985                            s->start_time;
 986         info->has_expected_downtime = true;
 987         info->expected_downtime = s->expected_downtime;
 988     }
 989 }
 990
 991 static void populate_ram_info(MigrationInfo *info, MigrationState *s)
 992 {
 993     info->has_ram = true;
 994     info->ram = g_malloc0(sizeof(*info->ram));
 995     info->ram->transferred = ram_counters.transferred;
 996     info->ram->total = ram_bytes_total();
 997     info->ram->duplicate = ram_counters.duplicate;
 998     /* legacy value.  It is not used anymore */
 999     info->ram->skipped = 0;
1000     info->ram->normal = ram_counters.normal;
1001     info->ram->normal_bytes = ram_counters.normal *
1002         qemu_target_page_size();
1003     info->ram->mbps = s->mbps;
1004     info->ram->dirty_sync_count = ram_counters.dirty_sync_count;
1005     info->ram->postcopy_requests = ram_counters.postcopy_requests;
1006     info->ram->page_size = qemu_target_page_size();
1007     info->ram->multifd_bytes = ram_counters.multifd_bytes;
1008     info->ram->pages_per_second = s->pages_per_second;
1009
1010     if (migrate_use_xbzrle()) {
1011         info->has_xbzrle_cache = true;
1012         info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
1013         info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
1014         info->xbzrle_cache->bytes = xbzrle_counters.bytes;
1015         info->xbzrle_cache->pages = xbzrle_counters.pages;
1016         info->xbzrle_cache->cache_miss = xbzrle_counters.cache_miss;
1017         info->xbzrle_cache->cache_miss_rate = xbzrle_counters.cache_miss_rate;
1018         info->xbzrle_cache->encoding_rate = xbzrle_counters.encoding_rate;
1019         info->xbzrle_cache->overflow = xbzrle_counters.overflow;
1020     }
1021
1022     if (migrate_use_compression()) {
1023         info->has_compression = true;
1024         info->compression = g_malloc0(sizeof(*info->compression));
1025         info->compression->pages = compression_counters.pages;
1026         info->compression->busy = compression_counters.busy;
1027         info->compression->busy_rate = compression_counters.busy_rate;
1028         info->compression->compressed_size =
1029                                     compression_counters.compressed_size;
1030         info->compression->compression_rate =
1031                                     compression_counters.compression_rate;
1032     }
1033
1034     if (cpu_throttle_active()) {
1035         info->has_cpu_throttle_percentage = true;
1036         info->cpu_throttle_percentage = cpu_throttle_get_percentage();
1037     }
1038
1039     if (s->state != MIGRATION_STATUS_COMPLETED) {
1040         info->ram->remaining = ram_bytes_remaining();
1041         info->ram->dirty_pages_rate = ram_counters.dirty_pages_rate;
1042     }
1043 }
1044
1045 static void populate_disk_info(MigrationInfo *info)
1046 {
1047     if (blk_mig_active()) {
1048         info->has_disk = true;
1049         info->disk = g_malloc0(sizeof(*info->disk));
1050         info->disk->transferred = blk_mig_bytes_transferred();
1051         info->disk->remaining = blk_mig_bytes_remaining();
1052         info->disk->total = blk_mig_bytes_total();
1053     }
1054 }
1055
1056 static void fill_source_migration_info(MigrationInfo *info)
1057 {
1058     MigrationState *s = migrate_get_current();
1059     GSList *cur_blocker = migration_blockers;
1060
1061     info->blocked_reasons = NULL;
1062
1063     /*
1064      * There are two types of reasons a migration might be blocked;
1065      * a) devices marked in VMState as non-migratable, and
1066      * b) Explicit migration blockers
1067      * We need to add both of them here.
1068      */
1069     qemu_savevm_non_migratable_list(&info->blocked_reasons);
1070
1071     while (cur_blocker) {
1072         QAPI_LIST_PREPEND(info->blocked_reasons,
1073                           g_strdup(error_get_pretty(cur_blocker->data)));
1074         cur_blocker = g_slist_next(cur_blocker);
1075     }
1076     info->has_blocked_reasons = info->blocked_reasons != NULL;
1077
1078     switch (s->state) {
1079     case MIGRATION_STATUS_NONE:
1080         /* no migration has happened ever */
1081         /* do not overwrite destination migration status */
1082         return;
1083     case MIGRATION_STATUS_SETUP:
1084         info->has_status = true;
1085         info->has_total_time = false;
1086         break;
1087     case MIGRATION_STATUS_ACTIVE:
1088     case MIGRATION_STATUS_CANCELLING:
1089     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1090     case MIGRATION_STATUS_PRE_SWITCHOVER:
1091     case MIGRATION_STATUS_DEVICE:
1092     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1093     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1094         /* TODO add some postcopy stats */
1095         populate_time_info(info, s);
1096         populate_ram_info(info, s);
1097         populate_disk_info(info);
1098         populate_vfio_info(info);
1099         break;
1100     case MIGRATION_STATUS_COLO:
1101         info->has_status = true;
1102         /* TODO: display COLO specific information (checkpoint info etc.) */
1103         break;
1104     case MIGRATION_STATUS_COMPLETED:
1105         populate_time_info(info, s);
1106         populate_ram_info(info, s);
1107         populate_vfio_info(info);
1108         break;
1109     case MIGRATION_STATUS_FAILED:
1110         info->has_status = true;
1111         if (s->error) {
1112             info->has_error_desc = true;
1113             info->error_desc = g_strdup(error_get_pretty(s->error));
1114         }
1115         break;
1116     case MIGRATION_STATUS_CANCELLED:
1117         info->has_status = true;
1118         break;
1119     case MIGRATION_STATUS_WAIT_UNPLUG:
1120         info->has_status = true;
1121         break;
1122     }
1123     info->status = s->state;
1124 }
1125
1126 typedef enum WriteTrackingSupport {
1127     WT_SUPPORT_UNKNOWN = 0,
1128     WT_SUPPORT_ABSENT,
1129     WT_SUPPORT_AVAILABLE,
1130     WT_SUPPORT_COMPATIBLE
1131 } WriteTrackingSupport;
1132
1133 static
1134 WriteTrackingSupport migrate_query_write_tracking(void)
1135 {
1136     /* Check if kernel supports required UFFD features */
1137     if (!ram_write_tracking_available()) {
1138         return WT_SUPPORT_ABSENT;
1139     }
1140     /*
1141      * Check if current memory configuration is
1142      * compatible with required UFFD features.
1143      */
1144     if (!ram_write_tracking_compatible()) {
1145         return WT_SUPPORT_AVAILABLE;
1146     }
1147
1148     return WT_SUPPORT_COMPATIBLE;
1149 }
1150
1151 /**
1152  * @migration_caps_check - check capability validity
1153  *
1154  * @cap_list: old capability list, array of bool
1155  * @params: new capabilities to be applied soon
1156  * @errp: set *errp if the check failed, with reason
1157  *
1158  * Returns true if check passed, otherwise false.
1159  */
1160 static bool migrate_caps_check(bool *cap_list,
1161                                MigrationCapabilityStatusList *params,
1162                                Error **errp)
1163 {
1164     MigrationCapabilityStatusList *cap;
1165     bool old_postcopy_cap;
1166     MigrationIncomingState *mis = migration_incoming_get_current();
1167
1168     old_postcopy_cap = cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM];
1169
1170     for (cap = params; cap; cap = cap->next) {
1171         cap_list[cap->value->capability] = cap->value->state;
1172     }
1173
1174 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
1175     if (cap_list[MIGRATION_CAPABILITY_BLOCK]) {
1176         error_setg(errp, "QEMU compiled without old-style (blk/-b, inc/-i) "
1177                    "block migration");
1178         error_append_hint(errp, "Use drive_mirror+NBD instead.\n");
1179         return false;
1180     }
1181 #endif
1182
1183 #ifndef CONFIG_REPLICATION
1184     if (cap_list[MIGRATION_CAPABILITY_X_COLO]) {
1185         error_setg(errp, "QEMU compiled without replication module"
1186                    " can't enable COLO");
1187         error_append_hint(errp, "Please enable replication before COLO.\n");
1188         return false;
1189     }
1190 #endif
1191
1192     if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
1193         /* This check is reasonably expensive, so only when it's being
1194          * set the first time, also it's only the destination that needs
1195          * special support.
1196          */
1197         if (!old_postcopy_cap && runstate_check(RUN_STATE_INMIGRATE) &&
1198             !postcopy_ram_supported_by_host(mis)) {
1199             /* postcopy_ram_supported_by_host will have emitted a more
1200              * detailed message
1201              */
1202             error_setg(errp, "Postcopy is not supported");
1203             return false;
1204         }
1205
1206         if (cap_list[MIGRATION_CAPABILITY_X_IGNORE_SHARED]) {
1207             error_setg(errp, "Postcopy is not compatible with ignore-shared");
1208             return false;
1209         }
1210     }
1211
1212     if (cap_list[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT]) {
1213         WriteTrackingSupport wt_support;
1214         int idx;
1215         /*
1216          * Check if 'background-snapshot' capability is supported by
1217          * host kernel and compatible with guest memory configuration.
1218          */
1219         wt_support = migrate_query_write_tracking();
1220         if (wt_support < WT_SUPPORT_AVAILABLE) {
1221             error_setg(errp, "Background-snapshot is not supported by host kernel");
1222             return false;
1223         }
1224         if (wt_support < WT_SUPPORT_COMPATIBLE) {
1225             error_setg(errp, "Background-snapshot is not compatible "
1226                     "with guest memory configuration");
1227             return false;
1228         }
1229
1230         /*
1231          * Check if there are any migration capabilities
1232          * incompatible with 'background-snapshot'.
1233          */
1234         for (idx = 0; idx < check_caps_background_snapshot.size; idx++) {
1235             int incomp_cap = check_caps_background_snapshot.caps[idx];
1236             if (cap_list[incomp_cap]) {
1237                 error_setg(errp,
1238                         "Background-snapshot is not compatible with %s",
1239                         MigrationCapability_str(incomp_cap));
1240                 return false;
1241             }
1242         }
1243     }
1244
1245     /* incoming side only */
1246     if (runstate_check(RUN_STATE_INMIGRATE) &&
1247         !migrate_multifd_is_allowed() &&
1248         cap_list[MIGRATION_CAPABILITY_MULTIFD]) {
1249         error_setg(errp, "multifd is not supported by current protocol");
1250         return false;
1251     }
1252
1253     return true;
1254 }
1255
1256 static void fill_destination_migration_info(MigrationInfo *info)
1257 {
1258     MigrationIncomingState *mis = migration_incoming_get_current();
1259
1260     if (mis->socket_address_list) {
1261         info->has_socket_address = true;
1262         info->socket_address =
1263             QAPI_CLONE(SocketAddressList, mis->socket_address_list);
1264     }
1265
1266     switch (mis->state) {
1267     case MIGRATION_STATUS_NONE:
1268         return;
1269     case MIGRATION_STATUS_SETUP:
1270     case MIGRATION_STATUS_CANCELLING:
1271     case MIGRATION_STATUS_CANCELLED:
1272     case MIGRATION_STATUS_ACTIVE:
1273     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1274     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1275     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1276     case MIGRATION_STATUS_FAILED:
1277     case MIGRATION_STATUS_COLO:
1278         info->has_status = true;
1279         break;
1280     case MIGRATION_STATUS_COMPLETED:
1281         info->has_status = true;
1282         fill_destination_postcopy_migration_info(info);
1283         break;
1284     }
1285     info->status = mis->state;
1286 }
1287
1288 MigrationInfo *qmp_query_migrate(Error **errp)
1289 {
1290     MigrationInfo *info = g_malloc0(sizeof(*info));
1291
1292     fill_destination_migration_info(info);
1293     fill_source_migration_info(info);
1294
1295     return info;
1296 }
1297
1298 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
1299                                   Error **errp)
1300 {
1301     MigrationState *s = migrate_get_current();
1302     MigrationCapabilityStatusList *cap;
1303     bool cap_list[MIGRATION_CAPABILITY__MAX];
1304
1305     if (migration_is_running(s->state)) {
1306         error_setg(errp, QERR_MIGRATION_ACTIVE);
1307         return;
1308     }
1309
1310     memcpy(cap_list, s->enabled_capabilities, sizeof(cap_list));
1311     if (!migrate_caps_check(cap_list, params, errp)) {
1312         return;
1313     }
1314
1315     for (cap = params; cap; cap = cap->next) {
1316         s->enabled_capabilities[cap->value->capability] = cap->value->state;
1317     }
1318 }
1319
1320 /*
1321  * Check whether the parameters are valid. Error will be put into errp
1322  * (if provided). Return true if valid, otherwise false.
1323  */
1324 static bool migrate_params_check(MigrationParameters *params, Error **errp)
1325 {
1326     if (params->has_compress_level &&
1327         (params->compress_level > 9)) {
1328         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level",
1329                    "a value between 0 and 9");
1330         return false;
1331     }
1332
1333     if (params->has_compress_threads && (params->compress_threads < 1)) {
1334         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1335                    "compress_threads",
1336                    "a value between 1 and 255");
1337         return false;
1338     }
1339
1340     if (params->has_decompress_threads && (params->decompress_threads < 1)) {
1341         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1342                    "decompress_threads",
1343                    "a value between 1 and 255");
1344         return false;
1345     }
1346
1347     if (params->has_throttle_trigger_threshold &&
1348         (params->throttle_trigger_threshold < 1 ||
1349          params->throttle_trigger_threshold > 100)) {
1350         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1351                    "throttle_trigger_threshold",
1352                    "an integer in the range of 1 to 100");
1353         return false;
1354     }
1355
1356     if (params->has_cpu_throttle_initial &&
1357         (params->cpu_throttle_initial < 1 ||
1358          params->cpu_throttle_initial > 99)) {
1359         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1360                    "cpu_throttle_initial",
1361                    "an integer in the range of 1 to 99");
1362         return false;
1363     }
1364
1365     if (params->has_cpu_throttle_increment &&
1366         (params->cpu_throttle_increment < 1 ||
1367          params->cpu_throttle_increment > 99)) {
1368         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1369                    "cpu_throttle_increment",
1370                    "an integer in the range of 1 to 99");
1371         return false;
1372     }
1373
1374     if (params->has_max_bandwidth && (params->max_bandwidth > SIZE_MAX)) {
1375         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1376                    "max_bandwidth",
1377                    "an integer in the range of 0 to "stringify(SIZE_MAX)
1378                    " bytes/second");
1379         return false;
1380     }
1381
1382     if (params->has_downtime_limit &&
1383         (params->downtime_limit > MAX_MIGRATE_DOWNTIME)) {
1384         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1385                    "downtime_limit",
1386                    "an integer in the range of 0 to "
1387                     stringify(MAX_MIGRATE_DOWNTIME)" ms");
1388         return false;
1389     }
1390
1391     /* x_checkpoint_delay is now always positive */
1392
1393     if (params->has_multifd_channels && (params->multifd_channels < 1)) {
1394         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1395                    "multifd_channels",
1396                    "a value between 1 and 255");
1397         return false;
1398     }
1399
1400     if (params->has_multifd_zlib_level &&
1401         (params->multifd_zlib_level > 9)) {
1402         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zlib_level",
1403                    "a value between 0 and 9");
1404         return false;
1405     }
1406
1407     if (params->has_multifd_zstd_level &&
1408         (params->multifd_zstd_level > 20)) {
1409         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zstd_level",
1410                    "a value between 0 and 20");
1411         return false;
1412     }
1413
1414     if (params->has_xbzrle_cache_size &&
1415         (params->xbzrle_cache_size < qemu_target_page_size() ||
1416          !is_power_of_2(params->xbzrle_cache_size))) {
1417         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1418                    "xbzrle_cache_size",
1419                    "a power of two no less than the target page size");
1420         return false;
1421     }
1422
1423     if (params->has_max_cpu_throttle &&
1424         (params->max_cpu_throttle < params->cpu_throttle_initial ||
1425          params->max_cpu_throttle > 99)) {
1426         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1427                    "max_cpu_throttle",
1428                    "an integer in the range of cpu_throttle_initial to 99");
1429         return false;
1430     }
1431
1432     if (params->has_announce_initial &&
1433         params->announce_initial > 100000) {
1434         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1435                    "announce_initial",
1436                    "a value between 0 and 100000");
1437         return false;
1438     }
1439     if (params->has_announce_max &&
1440         params->announce_max > 100000) {
1441         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1442                    "announce_max",
1443                    "a value between 0 and 100000");
1444        return false;
1445     }
1446     if (params->has_announce_rounds &&
1447         params->announce_rounds > 1000) {
1448         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1449                    "announce_rounds",
1450                    "a value between 0 and 1000");
1451        return false;
1452     }
1453     if (params->has_announce_step &&
1454         (params->announce_step < 1 ||
1455         params->announce_step > 10000)) {
1456         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1457                    "announce_step",
1458                    "a value between 0 and 10000");
1459        return false;
1460     }
1461
1462     if (params->has_block_bitmap_mapping &&
1463         !check_dirty_bitmap_mig_alias_map(params->block_bitmap_mapping, errp)) {
1464         error_prepend(errp, "Invalid mapping given for block-bitmap-mapping: ");
1465         return false;
1466     }
1467
1468     return true;
1469 }
1470
1471 static void migrate_params_test_apply(MigrateSetParameters *params,
1472                                       MigrationParameters *dest)
1473 {
1474     *dest = migrate_get_current()->parameters;
1475
1476     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1477
1478     if (params->has_compress_level) {
1479         dest->compress_level = params->compress_level;
1480     }
1481
1482     if (params->has_compress_threads) {
1483         dest->compress_threads = params->compress_threads;
1484     }
1485
1486     if (params->has_compress_wait_thread) {
1487         dest->compress_wait_thread = params->compress_wait_thread;
1488     }
1489
1490     if (params->has_decompress_threads) {
1491         dest->decompress_threads = params->decompress_threads;
1492     }
1493
1494     if (params->has_throttle_trigger_threshold) {
1495         dest->throttle_trigger_threshold = params->throttle_trigger_threshold;
1496     }
1497
1498     if (params->has_cpu_throttle_initial) {
1499         dest->cpu_throttle_initial = params->cpu_throttle_initial;
1500     }
1501
1502     if (params->has_cpu_throttle_increment) {
1503         dest->cpu_throttle_increment = params->cpu_throttle_increment;
1504     }
1505
1506     if (params->has_cpu_throttle_tailslow) {
1507         dest->cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1508     }
1509
1510     if (params->has_tls_creds) {
1511         assert(params->tls_creds->type == QTYPE_QSTRING);
1512         dest->tls_creds = params->tls_creds->u.s;
1513     }
1514
1515     if (params->has_tls_hostname) {
1516         assert(params->tls_hostname->type == QTYPE_QSTRING);
1517         dest->tls_hostname = params->tls_hostname->u.s;
1518     }
1519
1520     if (params->has_max_bandwidth) {
1521         dest->max_bandwidth = params->max_bandwidth;
1522     }
1523
1524     if (params->has_downtime_limit) {
1525         dest->downtime_limit = params->downtime_limit;
1526     }
1527
1528     if (params->has_x_checkpoint_delay) {
1529         dest->x_checkpoint_delay = params->x_checkpoint_delay;
1530     }
1531
1532     if (params->has_block_incremental) {
1533         dest->block_incremental = params->block_incremental;
1534     }
1535     if (params->has_multifd_channels) {
1536         dest->multifd_channels = params->multifd_channels;
1537     }
1538     if (params->has_multifd_compression) {
1539         dest->multifd_compression = params->multifd_compression;
1540     }
1541     if (params->has_xbzrle_cache_size) {
1542         dest->xbzrle_cache_size = params->xbzrle_cache_size;
1543     }
1544     if (params->has_max_postcopy_bandwidth) {
1545         dest->max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1546     }
1547     if (params->has_max_cpu_throttle) {
1548         dest->max_cpu_throttle = params->max_cpu_throttle;
1549     }
1550     if (params->has_announce_initial) {
1551         dest->announce_initial = params->announce_initial;
1552     }
1553     if (params->has_announce_max) {
1554         dest->announce_max = params->announce_max;
1555     }
1556     if (params->has_announce_rounds) {
1557         dest->announce_rounds = params->announce_rounds;
1558     }
1559     if (params->has_announce_step) {
1560         dest->announce_step = params->announce_step;
1561     }
1562
1563     if (params->has_block_bitmap_mapping) {
1564         dest->has_block_bitmap_mapping = true;
1565         dest->block_bitmap_mapping = params->block_bitmap_mapping;
1566     }
1567 }
1568
1569 static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
1570 {
1571     MigrationState *s = migrate_get_current();
1572
1573     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1574
1575     if (params->has_compress_level) {
1576         s->parameters.compress_level = params->compress_level;
1577     }
1578
1579     if (params->has_compress_threads) {
1580         s->parameters.compress_threads = params->compress_threads;
1581     }
1582
1583     if (params->has_compress_wait_thread) {
1584         s->parameters.compress_wait_thread = params->compress_wait_thread;
1585     }
1586
1587     if (params->has_decompress_threads) {
1588         s->parameters.decompress_threads = params->decompress_threads;
1589     }
1590
1591     if (params->has_throttle_trigger_threshold) {
1592         s->parameters.throttle_trigger_threshold = params->throttle_trigger_threshold;
1593     }
1594
1595     if (params->has_cpu_throttle_initial) {
1596         s->parameters.cpu_throttle_initial = params->cpu_throttle_initial;
1597     }
1598
1599     if (params->has_cpu_throttle_increment) {
1600         s->parameters.cpu_throttle_increment = params->cpu_throttle_increment;
1601     }
1602
1603     if (params->has_cpu_throttle_tailslow) {
1604         s->parameters.cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1605     }
1606
1607     if (params->has_tls_creds) {
1608         g_free(s->parameters.tls_creds);
1609         assert(params->tls_creds->type == QTYPE_QSTRING);
1610         s->parameters.tls_creds = g_strdup(params->tls_creds->u.s);
1611     }
1612
1613     if (params->has_tls_hostname) {
1614         g_free(s->parameters.tls_hostname);
1615         assert(params->tls_hostname->type == QTYPE_QSTRING);
1616         s->parameters.tls_hostname = g_strdup(params->tls_hostname->u.s);
1617     }
1618
1619     if (params->has_tls_authz) {
1620         g_free(s->parameters.tls_authz);
1621         assert(params->tls_authz->type == QTYPE_QSTRING);
1622         s->parameters.tls_authz = g_strdup(params->tls_authz->u.s);
1623     }
1624
1625     if (params->has_max_bandwidth) {
1626         s->parameters.max_bandwidth = params->max_bandwidth;
1627         if (s->to_dst_file && !migration_in_postcopy()) {
1628             qemu_file_set_rate_limit(s->to_dst_file,
1629                                 s->parameters.max_bandwidth / XFER_LIMIT_RATIO);
1630         }
1631     }
1632
1633     if (params->has_downtime_limit) {
1634         s->parameters.downtime_limit = params->downtime_limit;
1635     }
1636
1637     if (params->has_x_checkpoint_delay) {
1638         s->parameters.x_checkpoint_delay = params->x_checkpoint_delay;
1639         if (migration_in_colo_state()) {
1640             colo_checkpoint_notify(s);
1641         }
1642     }
1643
1644     if (params->has_block_incremental) {
1645         s->parameters.block_incremental = params->block_incremental;
1646     }
1647     if (params->has_multifd_channels) {
1648         s->parameters.multifd_channels = params->multifd_channels;
1649     }
1650     if (params->has_multifd_compression) {
1651         s->parameters.multifd_compression = params->multifd_compression;
1652     }
1653     if (params->has_xbzrle_cache_size) {
1654         s->parameters.xbzrle_cache_size = params->xbzrle_cache_size;
1655         xbzrle_cache_resize(params->xbzrle_cache_size, errp);
1656     }
1657     if (params->has_max_postcopy_bandwidth) {
1658         s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1659         if (s->to_dst_file && migration_in_postcopy()) {
1660             qemu_file_set_rate_limit(s->to_dst_file,
1661                     s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO);
1662         }
1663     }
1664     if (params->has_max_cpu_throttle) {
1665         s->parameters.max_cpu_throttle = params->max_cpu_throttle;
1666     }
1667     if (params->has_announce_initial) {
1668         s->parameters.announce_initial = params->announce_initial;
1669     }
1670     if (params->has_announce_max) {
1671         s->parameters.announce_max = params->announce_max;
1672     }
1673     if (params->has_announce_rounds) {
1674         s->parameters.announce_rounds = params->announce_rounds;
1675     }
1676     if (params->has_announce_step) {
1677         s->parameters.announce_step = params->announce_step;
1678     }
1679
1680     if (params->has_block_bitmap_mapping) {
1681         qapi_free_BitmapMigrationNodeAliasList(
1682             s->parameters.block_bitmap_mapping);
1683
1684         s->parameters.has_block_bitmap_mapping = true;
1685         s->parameters.block_bitmap_mapping =
1686             QAPI_CLONE(BitmapMigrationNodeAliasList,
1687                        params->block_bitmap_mapping);
1688     }
1689 }
1690
1691 void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
1692 {
1693     MigrationParameters tmp;
1694
1695     /* TODO Rewrite "" to null instead */
1696     if (params->has_tls_creds
1697         && params->tls_creds->type == QTYPE_QNULL) {
1698         qobject_unref(params->tls_creds->u.n);
1699         params->tls_creds->type = QTYPE_QSTRING;
1700         params->tls_creds->u.s = strdup("");
1701     }
1702     /* TODO Rewrite "" to null instead */
1703     if (params->has_tls_hostname
1704         && params->tls_hostname->type == QTYPE_QNULL) {
1705         qobject_unref(params->tls_hostname->u.n);
1706         params->tls_hostname->type = QTYPE_QSTRING;
1707         params->tls_hostname->u.s = strdup("");
1708     }
1709
1710     migrate_params_test_apply(params, &tmp);
1711
1712     if (!migrate_params_check(&tmp, errp)) {
1713         /* Invalid parameter */
1714         return;
1715     }
1716
1717     migrate_params_apply(params, errp);
1718 }
1719
1720
1721 void qmp_migrate_start_postcopy(Error **errp)
1722 {
1723     MigrationState *s = migrate_get_current();
1724
1725     if (!migrate_postcopy()) {
1726         error_setg(errp, "Enable postcopy with migrate_set_capability before"
1727                          " the start of migration");
1728         return;
1729     }
1730
1731     if (s->state == MIGRATION_STATUS_NONE) {
1732         error_setg(errp, "Postcopy must be started after migration has been"
1733                          " started");
1734         return;
1735     }
1736     /*
1737      * we don't error if migration has finished since that would be racy
1738      * with issuing this command.
1739      */
1740     qatomic_set(&s->start_postcopy, true);
1741 }
1742
1743 /* shared migration helpers */
1744
1745 void migrate_set_state(int *state, int old_state, int new_state)
1746 {
1747     assert(new_state < MIGRATION_STATUS__MAX);
1748     if (qatomic_cmpxchg(state, old_state, new_state) == old_state) {
1749         trace_migrate_set_state(MigrationStatus_str(new_state));
1750         migrate_generate_event(new_state);
1751     }
1752 }
1753
1754 static MigrationCapabilityStatus *migrate_cap_add(MigrationCapability index,
1755                                                   bool state)
1756 {
1757     MigrationCapabilityStatus *cap;
1758
1759     cap = g_new0(MigrationCapabilityStatus, 1);
1760     cap->capability = index;
1761     cap->state = state;
1762
1763     return cap;
1764 }
1765
1766 void migrate_set_block_enabled(bool value, Error **errp)
1767 {
1768     MigrationCapabilityStatusList *cap = NULL;
1769
1770     QAPI_LIST_PREPEND(cap, migrate_cap_add(MIGRATION_CAPABILITY_BLOCK, value));
1771     qmp_migrate_set_capabilities(cap, errp);
1772     qapi_free_MigrationCapabilityStatusList(cap);
1773 }
1774
1775 static void migrate_set_block_incremental(MigrationState *s, bool value)
1776 {
1777     s->parameters.block_incremental = value;
1778 }
1779
1780 static void block_cleanup_parameters(MigrationState *s)
1781 {
1782     if (s->must_remove_block_options) {
1783         /* setting to false can never fail */
1784         migrate_set_block_enabled(false, &error_abort);
1785         migrate_set_block_incremental(s, false);
1786         s->must_remove_block_options = false;
1787     }
1788 }
1789
1790 static void migrate_fd_cleanup(MigrationState *s)
1791 {
1792     qemu_bh_delete(s->cleanup_bh);
1793     s->cleanup_bh = NULL;
1794
1795     qemu_savevm_state_cleanup();
1796
1797     if (s->to_dst_file) {
1798         QEMUFile *tmp;
1799
1800         trace_migrate_fd_cleanup();
1801         qemu_mutex_unlock_iothread();
1802         if (s->migration_thread_running) {
1803             qemu_thread_join(&s->thread);
1804             s->migration_thread_running = false;
1805         }
1806         qemu_mutex_lock_iothread();
1807
1808         multifd_save_cleanup();
1809         qemu_mutex_lock(&s->qemu_file_lock);
1810         tmp = s->to_dst_file;
1811         s->to_dst_file = NULL;
1812         qemu_mutex_unlock(&s->qemu_file_lock);
1813         /*
1814          * Close the file handle without the lock to make sure the
1815          * critical section won't block for long.
1816          */
1817         migration_ioc_unregister_yank_from_file(tmp);
1818         qemu_fclose(tmp);
1819     }
1820
1821     assert(!migration_is_active(s));
1822
1823     if (s->state == MIGRATION_STATUS_CANCELLING) {
1824         migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING,
1825                           MIGRATION_STATUS_CANCELLED);
1826     }
1827
1828     if (s->error) {
1829         /* It is used on info migrate.  We can't free it */
1830         error_report_err(error_copy(s->error));
1831     }
1832     notifier_list_notify(&migration_state_notifiers, s);
1833     block_cleanup_parameters(s);
1834     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
1835 }
1836
1837 static void migrate_fd_cleanup_schedule(MigrationState *s)
1838 {
1839     /*
1840      * Ref the state for bh, because it may be called when
1841      * there're already no other refs
1842      */
1843     object_ref(OBJECT(s));
1844     qemu_bh_schedule(s->cleanup_bh);
1845 }
1846
1847 static void migrate_fd_cleanup_bh(void *opaque)
1848 {
1849     MigrationState *s = opaque;
1850     migrate_fd_cleanup(s);
1851     object_unref(OBJECT(s));
1852 }
1853
1854 void migrate_set_error(MigrationState *s, const Error *error)
1855 {
1856     QEMU_LOCK_GUARD(&s->error_mutex);
1857     if (!s->error) {
1858         s->error = error_copy(error);
1859     }
1860 }
1861
1862 static void migrate_error_free(MigrationState *s)
1863 {
1864     QEMU_LOCK_GUARD(&s->error_mutex);
1865     if (s->error) {
1866         error_free(s->error);
1867         s->error = NULL;
1868     }
1869 }
1870
1871 void migrate_fd_error(MigrationState *s, const Error *error)
1872 {
1873     trace_migrate_fd_error(error_get_pretty(error));
1874     assert(s->to_dst_file == NULL);
1875     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
1876                       MIGRATION_STATUS_FAILED);
1877     migrate_set_error(s, error);
1878 }
1879
1880 static void migrate_fd_cancel(MigrationState *s)
1881 {
1882     int old_state ;
1883     QEMUFile *f = migrate_get_current()->to_dst_file;
1884     trace_migrate_fd_cancel();
1885
1886     WITH_QEMU_LOCK_GUARD(&s->qemu_file_lock) {
1887         if (s->rp_state.from_dst_file) {
1888             /* shutdown the rp socket, so causing the rp thread to shutdown */
1889             qemu_file_shutdown(s->rp_state.from_dst_file);
1890         }
1891     }
1892
1893     do {
1894         old_state = s->state;
1895         if (!migration_is_running(old_state)) {
1896             break;
1897         }
1898         /* If the migration is paused, kick it out of the pause */
1899         if (old_state == MIGRATION_STATUS_PRE_SWITCHOVER) {
1900             qemu_sem_post(&s->pause_sem);
1901         }
1902         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING);
1903     } while (s->state != MIGRATION_STATUS_CANCELLING);
1904
1905     /*
1906      * If we're unlucky the migration code might be stuck somewhere in a
1907      * send/write while the network has failed and is waiting to timeout;
1908      * if we've got shutdown(2) available then we can force it to quit.
1909      * The outgoing qemu file gets closed in migrate_fd_cleanup that is
1910      * called in a bh, so there is no race against this cancel.
1911      */
1912     if (s->state == MIGRATION_STATUS_CANCELLING && f) {
1913         qemu_file_shutdown(f);
1914     }
1915     if (s->state == MIGRATION_STATUS_CANCELLING && s->block_inactive) {
1916         Error *local_err = NULL;
1917
1918         bdrv_invalidate_cache_all(&local_err);
1919         if (local_err) {
1920             error_report_err(local_err);
1921         } else {
1922             s->block_inactive = false;
1923         }
1924     }
1925 }
1926
1927 void add_migration_state_change_notifier(Notifier *notify)
1928 {
1929     notifier_list_add(&migration_state_notifiers, notify);
1930 }
1931
1932 void remove_migration_state_change_notifier(Notifier *notify)
1933 {
1934     notifier_remove(notify);
1935 }
1936
1937 bool migration_in_setup(MigrationState *s)
1938 {
1939     return s->state == MIGRATION_STATUS_SETUP;
1940 }
1941
1942 bool migration_has_finished(MigrationState *s)
1943 {
1944     return s->state == MIGRATION_STATUS_COMPLETED;
1945 }
1946
1947 bool migration_has_failed(MigrationState *s)
1948 {
1949     return (s->state == MIGRATION_STATUS_CANCELLED ||
1950             s->state == MIGRATION_STATUS_FAILED);
1951 }
1952
1953 bool migration_in_postcopy(void)
1954 {
1955     MigrationState *s = migrate_get_current();
1956
1957     switch (s->state) {
1958     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1959     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1960     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1961         return true;
1962     default:
1963         return false;
1964     }
1965 }
1966
1967 bool migration_in_postcopy_after_devices(MigrationState *s)
1968 {
1969     return migration_in_postcopy() && s->postcopy_after_devices;
1970 }
1971
1972 bool migration_in_incoming_postcopy(void)
1973 {
1974     PostcopyState ps = postcopy_state_get();
1975
1976     return ps >= POSTCOPY_INCOMING_DISCARD && ps < POSTCOPY_INCOMING_END;
1977 }
1978
1979 bool migration_in_bg_snapshot(void)
1980 {
1981     MigrationState *s = migrate_get_current();
1982
1983     return migrate_background_snapshot() &&
1984             migration_is_setup_or_active(s->state);
1985 }
1986
1987 bool migration_is_idle(void)
1988 {
1989     MigrationState *s = current_migration;
1990
1991     if (!s) {
1992         return true;
1993     }
1994
1995     switch (s->state) {
1996     case MIGRATION_STATUS_NONE:
1997     case MIGRATION_STATUS_CANCELLED:
1998     case MIGRATION_STATUS_COMPLETED:
1999     case MIGRATION_STATUS_FAILED:
2000         return true;
2001     case MIGRATION_STATUS_SETUP:
2002     case MIGRATION_STATUS_CANCELLING:
2003     case MIGRATION_STATUS_ACTIVE:
2004     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
2005     case MIGRATION_STATUS_COLO:
2006     case MIGRATION_STATUS_PRE_SWITCHOVER:
2007     case MIGRATION_STATUS_DEVICE:
2008     case MIGRATION_STATUS_WAIT_UNPLUG:
2009         return false;
2010     case MIGRATION_STATUS__MAX:
2011         g_assert_not_reached();
2012     }
2013
2014     return false;
2015 }
2016
2017 bool migration_is_active(MigrationState *s)
2018 {
2019     return (s->state == MIGRATION_STATUS_ACTIVE ||
2020             s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
2021 }
2022
2023 void migrate_init(MigrationState *s)
2024 {
2025     /*
2026      * Reinitialise all migration state, except
2027      * parameters/capabilities that the user set, and
2028      * locks.
2029      */
2030     s->cleanup_bh = 0;
2031     s->vm_start_bh = 0;
2032     s->to_dst_file = NULL;
2033     s->state = MIGRATION_STATUS_NONE;
2034     s->rp_state.from_dst_file = NULL;
2035     s->rp_state.error = false;
2036     s->mbps = 0.0;
2037     s->pages_per_second = 0.0;
2038     s->downtime = 0;
2039     s->expected_downtime = 0;
2040     s->setup_time = 0;
2041     s->start_postcopy = false;
2042     s->postcopy_after_devices = false;
2043     s->migration_thread_running = false;
2044     error_free(s->error);
2045     s->error = NULL;
2046     s->hostname = NULL;
2047
2048     migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
2049
2050     s->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2051     s->total_time = 0;
2052     s->vm_was_running = false;
2053     s->iteration_initial_bytes = 0;
2054     s->threshold_size = 0;
2055 }
2056
2057 int migrate_add_blocker_internal(Error *reason, Error **errp)
2058 {
2059     /* Snapshots are similar to migrations, so check RUN_STATE_SAVE_VM too. */
2060     if (runstate_check(RUN_STATE_SAVE_VM) || !migration_is_idle()) {
2061         error_propagate_prepend(errp, error_copy(reason),
2062                                 "disallowing migration blocker "
2063                                 "(migration/snapshot in progress) for: ");
2064         return -EBUSY;
2065     }
2066
2067     migration_blockers = g_slist_prepend(migration_blockers, reason);
2068     return 0;
2069 }
2070
2071 int migrate_add_blocker(Error *reason, Error **errp)
2072 {
2073     if (only_migratable) {
2074         error_propagate_prepend(errp, error_copy(reason),
2075                                 "disallowing migration blocker "
2076                                 "(--only-migratable) for: ");
2077         return -EACCES;
2078     }
2079
2080     return migrate_add_blocker_internal(reason, errp);
2081 }
2082
2083 void migrate_del_blocker(Error *reason)
2084 {
2085     migration_blockers = g_slist_remove(migration_blockers, reason);
2086 }
2087
2088 void qmp_migrate_incoming(const char *uri, Error **errp)
2089 {
2090     Error *local_err = NULL;
2091     static bool once = true;
2092
2093     if (!once) {
2094         error_setg(errp, "The incoming migration has already been started");
2095         return;
2096     }
2097     if (!runstate_check(RUN_STATE_INMIGRATE)) {
2098         error_setg(errp, "'-incoming' was not specified on the command line");
2099         return;
2100     }
2101
2102     if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2103         return;
2104     }
2105
2106     qemu_start_incoming_migration(uri, &local_err);
2107
2108     if (local_err) {
2109         yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2110         error_propagate(errp, local_err);
2111         return;
2112     }
2113
2114     once = false;
2115 }
2116
2117 void qmp_migrate_recover(const char *uri, Error **errp)
2118 {
2119     MigrationIncomingState *mis = migration_incoming_get_current();
2120
2121     /*
2122      * Don't even bother to use ERRP_GUARD() as it _must_ always be set by
2123      * callers (no one should ignore a recover failure); if there is, it's a
2124      * programming error.
2125      */
2126     assert(errp);
2127
2128     if (mis->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2129         error_setg(errp, "Migrate recover can only be run "
2130                    "when postcopy is paused.");
2131         return;
2132     }
2133
2134     if (qatomic_cmpxchg(&mis->postcopy_recover_triggered,
2135                        false, true) == true) {
2136         error_setg(errp, "Migrate recovery is triggered already");
2137         return;
2138     }
2139
2140     /*
2141      * Note that this call will never start a real migration; it will
2142      * only re-setup the migration stream and poke existing migration
2143      * to continue using that newly established channel.
2144      */
2145     qemu_start_incoming_migration(uri, errp);
2146
2147     /* Safe to dereference with the assert above */
2148     if (*errp) {
2149         /* Reset the flag so user could still retry */
2150         qatomic_set(&mis->postcopy_recover_triggered, false);
2151     }
2152 }
2153
2154 void qmp_migrate_pause(Error **errp)
2155 {
2156     MigrationState *ms = migrate_get_current();
2157     MigrationIncomingState *mis = migration_incoming_get_current();
2158     int ret;
2159
2160     if (ms->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2161         /* Source side, during postcopy */
2162         qemu_mutex_lock(&ms->qemu_file_lock);
2163         ret = qemu_file_shutdown(ms->to_dst_file);
2164         qemu_mutex_unlock(&ms->qemu_file_lock);
2165         if (ret) {
2166             error_setg(errp, "Failed to pause source migration");
2167         }
2168         return;
2169     }
2170
2171     if (mis->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2172         ret = qemu_file_shutdown(mis->from_src_file);
2173         if (ret) {
2174             error_setg(errp, "Failed to pause destination migration");
2175         }
2176         return;
2177     }
2178
2179     error_setg(errp, "migrate-pause is currently only supported "
2180                "during postcopy-active state");
2181 }
2182
2183 bool migration_is_blocked(Error **errp)
2184 {
2185     if (qemu_savevm_state_blocked(errp)) {
2186         return true;
2187     }
2188
2189     if (migration_blockers) {
2190         error_propagate(errp, error_copy(migration_blockers->data));
2191         return true;
2192     }
2193
2194     return false;
2195 }
2196
2197 /* Returns true if continue to migrate, or false if error detected */
2198 static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
2199                             bool resume, Error **errp)
2200 {
2201     Error *local_err = NULL;
2202
2203     if (resume) {
2204         if (s->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2205             error_setg(errp, "Cannot resume if there is no "
2206                        "paused migration");
2207             return false;
2208         }
2209
2210         /*
2211          * Postcopy recovery won't work well with release-ram
2212          * capability since release-ram will drop the page buffer as
2213          * long as the page is put into the send buffer.  So if there
2214          * is a network failure happened, any page buffers that have
2215          * not yet reached the destination VM but have already been
2216          * sent from the source VM will be lost forever.  Let's refuse
2217          * the client from resuming such a postcopy migration.
2218          * Luckily release-ram was designed to only be used when src
2219          * and destination VMs are on the same host, so it should be
2220          * fine.
2221          */
2222         if (migrate_release_ram()) {
2223             error_setg(errp, "Postcopy recovery cannot work "
2224                        "when release-ram capability is set");
2225             return false;
2226         }
2227
2228         /* This is a resume, skip init status */
2229         return true;
2230     }
2231
2232     if (migration_is_running(s->state)) {
2233         error_setg(errp, QERR_MIGRATION_ACTIVE);
2234         return false;
2235     }
2236
2237     if (runstate_check(RUN_STATE_INMIGRATE)) {
2238         error_setg(errp, "Guest is waiting for an incoming migration");
2239         return false;
2240     }
2241
2242     if (runstate_check(RUN_STATE_POSTMIGRATE)) {
2243         error_setg(errp, "Can't migrate the vm that was paused due to "
2244                    "previous migration");
2245         return false;
2246     }
2247
2248     if (migration_is_blocked(errp)) {
2249         return false;
2250     }
2251
2252     if (blk || blk_inc) {
2253         if (migrate_colo_enabled()) {
2254             error_setg(errp, "No disk migration is required in COLO mode");
2255             return false;
2256         }
2257         if (migrate_use_block() || migrate_use_block_incremental()) {
2258             error_setg(errp, "Command options are incompatible with "
2259                        "current migration capabilities");
2260             return false;
2261         }
2262         migrate_set_block_enabled(true, &local_err);
2263         if (local_err) {
2264             error_propagate(errp, local_err);
2265             return false;
2266         }
2267         s->must_remove_block_options = true;
2268     }
2269
2270     if (blk_inc) {
2271         migrate_set_block_incremental(s, true);
2272     }
2273
2274     migrate_init(s);
2275     /*
2276      * set ram_counters compression_counters memory to zero for a
2277      * new migration
2278      */
2279     memset(&ram_counters, 0, sizeof(ram_counters));
2280     memset(&compression_counters, 0, sizeof(compression_counters));
2281
2282     return true;
2283 }
2284
2285 void qmp_migrate(const char *uri, bool has_blk, bool blk,
2286                  bool has_inc, bool inc, bool has_detach, bool detach,
2287                  bool has_resume, bool resume, Error **errp)
2288 {
2289     Error *local_err = NULL;
2290     MigrationState *s = migrate_get_current();
2291     const char *p = NULL;
2292
2293     if (!migrate_prepare(s, has_blk && blk, has_inc && inc,
2294                          has_resume && resume, errp)) {
2295         /* Error detected, put into errp */
2296         return;
2297     }
2298
2299     if (!(has_resume && resume)) {
2300         if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2301             return;
2302         }
2303     }
2304
2305     migrate_protocol_allow_multifd(false);
2306     if (strstart(uri, "tcp:", &p) ||
2307         strstart(uri, "unix:", NULL) ||
2308         strstart(uri, "vsock:", NULL)) {
2309         migrate_protocol_allow_multifd(true);
2310         socket_start_outgoing_migration(s, p ? p : uri, &local_err);
2311 #ifdef CONFIG_RDMA
2312     } else if (strstart(uri, "rdma:", &p)) {
2313         rdma_start_outgoing_migration(s, p, &local_err);
2314 #endif
2315     } else if (strstart(uri, "exec:", &p)) {
2316         exec_start_outgoing_migration(s, p, &local_err);
2317     } else if (strstart(uri, "fd:", &p)) {
2318         fd_start_outgoing_migration(s, p, &local_err);
2319     } else {
2320         if (!(has_resume && resume)) {
2321             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2322         }
2323         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri",
2324                    "a valid migration protocol");
2325         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
2326                           MIGRATION_STATUS_FAILED);
2327         block_cleanup_parameters(s);
2328         return;
2329     }
2330
2331     if (local_err) {
2332         if (!(has_resume && resume)) {
2333             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2334         }
2335         migrate_fd_error(s, local_err);
2336         error_propagate(errp, local_err);
2337         return;
2338     }
2339 }
2340
2341 void qmp_migrate_cancel(Error **errp)
2342 {
2343     migration_cancel(NULL);
2344 }
2345
2346 void qmp_migrate_continue(MigrationStatus state, Error **errp)
2347 {
2348     MigrationState *s = migrate_get_current();
2349     if (s->state != state) {
2350         error_setg(errp,  "Migration not in expected state: %s",
2351                    MigrationStatus_str(s->state));
2352         return;
2353     }
2354     qemu_sem_post(&s->pause_sem);
2355 }
2356
2357 bool migrate_release_ram(void)
2358 {
2359     MigrationState *s;
2360
2361     s = migrate_get_current();
2362
2363     return s->enabled_capabilities[MIGRATION_CAPABILITY_RELEASE_RAM];
2364 }
2365
2366 bool migrate_postcopy_ram(void)
2367 {
2368     MigrationState *s;
2369
2370     s = migrate_get_current();
2371
2372     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM];
2373 }
2374
2375 bool migrate_postcopy(void)
2376 {
2377     return migrate_postcopy_ram() || migrate_dirty_bitmaps();
2378 }
2379
2380 bool migrate_auto_converge(void)
2381 {
2382     MigrationState *s;
2383
2384     s = migrate_get_current();
2385
2386     return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
2387 }
2388
2389 bool migrate_zero_blocks(void)
2390 {
2391     MigrationState *s;
2392
2393     s = migrate_get_current();
2394
2395     return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
2396 }
2397
2398 bool migrate_postcopy_blocktime(void)
2399 {
2400     MigrationState *s;
2401
2402     s = migrate_get_current();
2403
2404     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME];
2405 }
2406
2407 bool migrate_use_compression(void)
2408 {
2409     MigrationState *s;
2410
2411     s = migrate_get_current();
2412
2413     return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS];
2414 }
2415
2416 int migrate_compress_level(void)
2417 {
2418     MigrationState *s;
2419
2420     s = migrate_get_current();
2421
2422     return s->parameters.compress_level;
2423 }
2424
2425 int migrate_compress_threads(void)
2426 {
2427     MigrationState *s;
2428
2429     s = migrate_get_current();
2430
2431     return s->parameters.compress_threads;
2432 }
2433
2434 int migrate_compress_wait_thread(void)
2435 {
2436     MigrationState *s;
2437
2438     s = migrate_get_current();
2439
2440     return s->parameters.compress_wait_thread;
2441 }
2442
2443 int migrate_decompress_threads(void)
2444 {
2445     MigrationState *s;
2446
2447     s = migrate_get_current();
2448
2449     return s->parameters.decompress_threads;
2450 }
2451
2452 bool migrate_dirty_bitmaps(void)
2453 {
2454     MigrationState *s;
2455
2456     s = migrate_get_current();
2457
2458     return s->enabled_capabilities[MIGRATION_CAPABILITY_DIRTY_BITMAPS];
2459 }
2460
2461 bool migrate_ignore_shared(void)
2462 {
2463     MigrationState *s;
2464
2465     s = migrate_get_current();
2466
2467     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_IGNORE_SHARED];
2468 }
2469
2470 bool migrate_validate_uuid(void)
2471 {
2472     MigrationState *s;
2473
2474     s = migrate_get_current();
2475
2476     return s->enabled_capabilities[MIGRATION_CAPABILITY_VALIDATE_UUID];
2477 }
2478
2479 bool migrate_use_events(void)
2480 {
2481     MigrationState *s;
2482
2483     s = migrate_get_current();
2484
2485     return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS];
2486 }
2487
2488 bool migrate_use_multifd(void)
2489 {
2490     MigrationState *s;
2491
2492     s = migrate_get_current();
2493
2494     return s->enabled_capabilities[MIGRATION_CAPABILITY_MULTIFD];
2495 }
2496
2497 bool migrate_pause_before_switchover(void)
2498 {
2499     MigrationState *s;
2500
2501     s = migrate_get_current();
2502
2503     return s->enabled_capabilities[
2504         MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER];
2505 }
2506
2507 int migrate_multifd_channels(void)
2508 {
2509     MigrationState *s;
2510
2511     s = migrate_get_current();
2512
2513     return s->parameters.multifd_channels;
2514 }
2515
2516 MultiFDCompression migrate_multifd_compression(void)
2517 {
2518     MigrationState *s;
2519
2520     s = migrate_get_current();
2521
2522     return s->parameters.multifd_compression;
2523 }
2524
2525 int migrate_multifd_zlib_level(void)
2526 {
2527     MigrationState *s;
2528
2529     s = migrate_get_current();
2530
2531     return s->parameters.multifd_zlib_level;
2532 }
2533
2534 int migrate_multifd_zstd_level(void)
2535 {
2536     MigrationState *s;
2537
2538     s = migrate_get_current();
2539
2540     return s->parameters.multifd_zstd_level;
2541 }
2542
2543 int migrate_use_xbzrle(void)
2544 {
2545     MigrationState *s;
2546
2547     s = migrate_get_current();
2548
2549     return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
2550 }
2551
2552 uint64_t migrate_xbzrle_cache_size(void)
2553 {
2554     MigrationState *s;
2555
2556     s = migrate_get_current();
2557
2558     return s->parameters.xbzrle_cache_size;
2559 }
2560
2561 static int64_t migrate_max_postcopy_bandwidth(void)
2562 {
2563     MigrationState *s;
2564
2565     s = migrate_get_current();
2566
2567     return s->parameters.max_postcopy_bandwidth;
2568 }
2569
2570 bool migrate_use_block(void)
2571 {
2572     MigrationState *s;
2573
2574     s = migrate_get_current();
2575
2576     return s->enabled_capabilities[MIGRATION_CAPABILITY_BLOCK];
2577 }
2578
2579 bool migrate_use_return_path(void)
2580 {
2581     MigrationState *s;
2582
2583     s = migrate_get_current();
2584
2585     return s->enabled_capabilities[MIGRATION_CAPABILITY_RETURN_PATH];
2586 }
2587
2588 bool migrate_use_block_incremental(void)
2589 {
2590     MigrationState *s;
2591
2592     s = migrate_get_current();
2593
2594     return s->parameters.block_incremental;
2595 }
2596
2597 bool migrate_background_snapshot(void)
2598 {
2599     MigrationState *s;
2600
2601     s = migrate_get_current();
2602
2603     return s->enabled_capabilities[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT];
2604 }
2605
2606 /* migration thread support */
2607 /*
2608  * Something bad happened to the RP stream, mark an error
2609  * The caller shall print or trace something to indicate why
2610  */
2611 static void mark_source_rp_bad(MigrationState *s)
2612 {
2613     s->rp_state.error = true;
2614 }
2615
2616 static struct rp_cmd_args {
2617     ssize_t     len; /* -1 = variable */
2618     const char *name;
2619 } rp_cmd_args[] = {
2620     [MIG_RP_MSG_INVALID]        = { .len = -1, .name = "INVALID" },
2621     [MIG_RP_MSG_SHUT]           = { .len =  4, .name = "SHUT" },
2622     [MIG_RP_MSG_PONG]           = { .len =  4, .name = "PONG" },
2623     [MIG_RP_MSG_REQ_PAGES]      = { .len = 12, .name = "REQ_PAGES" },
2624     [MIG_RP_MSG_REQ_PAGES_ID]   = { .len = -1, .name = "REQ_PAGES_ID" },
2625     [MIG_RP_MSG_RECV_BITMAP]    = { .len = -1, .name = "RECV_BITMAP" },
2626     [MIG_RP_MSG_RESUME_ACK]     = { .len =  4, .name = "RESUME_ACK" },
2627     [MIG_RP_MSG_MAX]            = { .len = -1, .name = "MAX" },
2628 };
2629
2630 /*
2631  * Process a request for pages received on the return path,
2632  * We're allowed to send more than requested (e.g. to round to our page size)
2633  * and we don't need to send pages that have already been sent.
2634  */
2635 static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
2636                                        ram_addr_t start, size_t len)
2637 {
2638     long our_host_ps = qemu_real_host_page_size;
2639
2640     trace_migrate_handle_rp_req_pages(rbname, start, len);
2641
2642     /*
2643      * Since we currently insist on matching page sizes, just sanity check
2644      * we're being asked for whole host pages.
2645      */
2646     if (!QEMU_IS_ALIGNED(start, our_host_ps) ||
2647         !QEMU_IS_ALIGNED(len, our_host_ps)) {
2648         error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
2649                      " len: %zd", __func__, start, len);
2650         mark_source_rp_bad(ms);
2651         return;
2652     }
2653
2654     if (ram_save_queue_pages(rbname, start, len)) {
2655         mark_source_rp_bad(ms);
2656     }
2657 }
2658
2659 /* Return true to retry, false to quit */
2660 static bool postcopy_pause_return_path_thread(MigrationState *s)
2661 {
2662     trace_postcopy_pause_return_path();
2663
2664     qemu_sem_wait(&s->postcopy_pause_rp_sem);
2665
2666     trace_postcopy_pause_return_path_continued();
2667
2668     return true;
2669 }
2670
2671 static int migrate_handle_rp_recv_bitmap(MigrationState *s, char *block_name)
2672 {
2673     RAMBlock *block = qemu_ram_block_by_name(block_name);
2674
2675     if (!block) {
2676         error_report("%s: invalid block name '%s'", __func__, block_name);
2677         return -EINVAL;
2678     }
2679
2680     /* Fetch the received bitmap and refresh the dirty bitmap */
2681     return ram_dirty_bitmap_reload(s, block);
2682 }
2683
2684 static int migrate_handle_rp_resume_ack(MigrationState *s, uint32_t value)
2685 {
2686     trace_source_return_path_thread_resume_ack(value);
2687
2688     if (value != MIGRATION_RESUME_ACK_VALUE) {
2689         error_report("%s: illegal resume_ack value %"PRIu32,
2690                      __func__, value);
2691         return -1;
2692     }
2693
2694     /* Now both sides are active. */
2695     migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_RECOVER,
2696                       MIGRATION_STATUS_POSTCOPY_ACTIVE);
2697
2698     /* Notify send thread that time to continue send pages */
2699     qemu_sem_post(&s->rp_state.rp_sem);
2700
2701     return 0;
2702 }
2703
2704 /* Release ms->rp_state.from_dst_file in a safe way */
2705 static void migration_release_from_dst_file(MigrationState *ms)
2706 {
2707     QEMUFile *file;
2708
2709     WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) {
2710         /*
2711          * Reset the from_dst_file pointer first before releasing it, as we
2712          * can't block within lock section
2713          */
2714         file = ms->rp_state.from_dst_file;
2715         ms->rp_state.from_dst_file = NULL;
2716     }
2717
2718     qemu_fclose(file);
2719 }
2720
2721 /*
2722  * Handles messages sent on the return path towards the source VM
2723  *
2724  */
2725 static void *source_return_path_thread(void *opaque)
2726 {
2727     MigrationState *ms = opaque;
2728     QEMUFile *rp = ms->rp_state.from_dst_file;
2729     uint16_t header_len, header_type;
2730     uint8_t buf[512];
2731     uint32_t tmp32, sibling_error;
2732     ram_addr_t start = 0; /* =0 to silence warning */
2733     size_t  len = 0, expected_len;
2734     int res;
2735
2736     trace_source_return_path_thread_entry();
2737     rcu_register_thread();
2738
2739 retry:
2740     while (!ms->rp_state.error && !qemu_file_get_error(rp) &&
2741            migration_is_setup_or_active(ms->state)) {
2742         trace_source_return_path_thread_loop_top();
2743         header_type = qemu_get_be16(rp);
2744         header_len = qemu_get_be16(rp);
2745
2746         if (qemu_file_get_error(rp)) {
2747             mark_source_rp_bad(ms);
2748             goto out;
2749         }
2750
2751         if (header_type >= MIG_RP_MSG_MAX ||
2752             header_type == MIG_RP_MSG_INVALID) {
2753             error_report("RP: Received invalid message 0x%04x length 0x%04x",
2754                          header_type, header_len);
2755             mark_source_rp_bad(ms);
2756             goto out;
2757         }
2758
2759         if ((rp_cmd_args[header_type].len != -1 &&
2760             header_len != rp_cmd_args[header_type].len) ||
2761             header_len > sizeof(buf)) {
2762             error_report("RP: Received '%s' message (0x%04x) with"
2763                          "incorrect length %d expecting %zu",
2764                          rp_cmd_args[header_type].name, header_type, header_len,
2765                          (size_t)rp_cmd_args[header_type].len);
2766             mark_source_rp_bad(ms);
2767             goto out;
2768         }
2769
2770         /* We know we've got a valid header by this point */
2771         res = qemu_get_buffer(rp, buf, header_len);
2772         if (res != header_len) {
2773             error_report("RP: Failed reading data for message 0x%04x"
2774                          " read %d expected %d",
2775                          header_type, res, header_len);
2776             mark_source_rp_bad(ms);
2777             goto out;
2778         }
2779
2780         /* OK, we have the message and the data */
2781         switch (header_type) {
2782         case MIG_RP_MSG_SHUT:
2783             sibling_error = ldl_be_p(buf);
2784             trace_source_return_path_thread_shut(sibling_error);
2785             if (sibling_error) {
2786                 error_report("RP: Sibling indicated error %d", sibling_error);
2787                 mark_source_rp_bad(ms);
2788             }
2789             /*
2790              * We'll let the main thread deal with closing the RP
2791              * we could do a shutdown(2) on it, but we're the only user
2792              * anyway, so there's nothing gained.
2793              */
2794             goto out;
2795
2796         case MIG_RP_MSG_PONG:
2797             tmp32 = ldl_be_p(buf);
2798             trace_source_return_path_thread_pong(tmp32);
2799             break;
2800
2801         case MIG_RP_MSG_REQ_PAGES:
2802             start = ldq_be_p(buf);
2803             len = ldl_be_p(buf + 8);
2804             migrate_handle_rp_req_pages(ms, NULL, start, len);
2805             break;
2806
2807         case MIG_RP_MSG_REQ_PAGES_ID:
2808             expected_len = 12 + 1; /* header + termination */
2809
2810             if (header_len >= expected_len) {
2811                 start = ldq_be_p(buf);
2812                 len = ldl_be_p(buf + 8);
2813                 /* Now we expect an idstr */
2814                 tmp32 = buf[12]; /* Length of the following idstr */
2815                 buf[13 + tmp32] = '\0';
2816                 expected_len += tmp32;
2817             }
2818             if (header_len != expected_len) {
2819                 error_report("RP: Req_Page_id with length %d expecting %zd",
2820                              header_len, expected_len);
2821                 mark_source_rp_bad(ms);
2822                 goto out;
2823             }
2824             migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len);
2825             break;
2826
2827         case MIG_RP_MSG_RECV_BITMAP:
2828             if (header_len < 1) {
2829                 error_report("%s: missing block name", __func__);
2830                 mark_source_rp_bad(ms);
2831                 goto out;
2832             }
2833             /* Format: len (1B) + idstr (<255B). This ends the idstr. */
2834             buf[buf[0] + 1] = '\0';
2835             if (migrate_handle_rp_recv_bitmap(ms, (char *)(buf + 1))) {
2836                 mark_source_rp_bad(ms);
2837                 goto out;
2838             }
2839             break;
2840
2841         case MIG_RP_MSG_RESUME_ACK:
2842             tmp32 = ldl_be_p(buf);
2843             if (migrate_handle_rp_resume_ack(ms, tmp32)) {
2844                 mark_source_rp_bad(ms);
2845                 goto out;
2846             }
2847             break;
2848
2849         default:
2850             break;
2851         }
2852     }
2853
2854 out:
2855     res = qemu_file_get_error(rp);
2856     if (res) {
2857         if (res == -EIO && migration_in_postcopy()) {
2858             /*
2859              * Maybe there is something we can do: it looks like a
2860              * network down issue, and we pause for a recovery.
2861              */
2862             migration_release_from_dst_file(ms);
2863             rp = NULL;
2864             if (postcopy_pause_return_path_thread(ms)) {
2865                 /*
2866                  * Reload rp, reset the rest.  Referencing it is safe since
2867                  * it's reset only by us above, or when migration completes
2868                  */
2869                 rp = ms->rp_state.from_dst_file;
2870                 ms->rp_state.error = false;
2871                 goto retry;
2872             }
2873         }
2874
2875         trace_source_return_path_thread_bad_end();
2876         mark_source_rp_bad(ms);
2877     }
2878
2879     trace_source_return_path_thread_end();
2880     migration_release_from_dst_file(ms);
2881     rcu_unregister_thread();
2882     return NULL;
2883 }
2884
2885 static int open_return_path_on_source(MigrationState *ms,
2886                                       bool create_thread)
2887 {
2888     ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file);
2889     if (!ms->rp_state.from_dst_file) {
2890         return -1;
2891     }
2892
2893     trace_open_return_path_on_source();
2894
2895     if (!create_thread) {
2896         /* We're done */
2897         return 0;
2898     }
2899
2900     qemu_thread_create(&ms->rp_state.rp_thread, "return path",
2901                        source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
2902     ms->rp_state.rp_thread_created = true;
2903
2904     trace_open_return_path_on_source_continue();
2905
2906     return 0;
2907 }
2908
2909 /* Returns 0 if the RP was ok, otherwise there was an error on the RP */
2910 static int await_return_path_close_on_source(MigrationState *ms)
2911 {
2912     /*
2913      * If this is a normal exit then the destination will send a SHUT and the
2914      * rp_thread will exit, however if there's an error we need to cause
2915      * it to exit.
2916      */
2917     if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) {
2918         /*
2919          * shutdown(2), if we have it, will cause it to unblock if it's stuck
2920          * waiting for the destination.
2921          */
2922         qemu_file_shutdown(ms->rp_state.from_dst_file);
2923         mark_source_rp_bad(ms);
2924     }
2925     trace_await_return_path_close_on_source_joining();
2926     qemu_thread_join(&ms->rp_state.rp_thread);
2927     ms->rp_state.rp_thread_created = false;
2928     trace_await_return_path_close_on_source_close();
2929     return ms->rp_state.error;
2930 }
2931
2932 /*
2933  * Switch from normal iteration to postcopy
2934  * Returns non-0 on error
2935  */
2936 static int postcopy_start(MigrationState *ms)
2937 {
2938     int ret;
2939     QIOChannelBuffer *bioc;
2940     QEMUFile *fb;
2941     int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2942     int64_t bandwidth = migrate_max_postcopy_bandwidth();
2943     bool restart_block = false;
2944     int cur_state = MIGRATION_STATUS_ACTIVE;
2945     if (!migrate_pause_before_switchover()) {
2946         migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
2947                           MIGRATION_STATUS_POSTCOPY_ACTIVE);
2948     }
2949
2950     trace_postcopy_start();
2951     qemu_mutex_lock_iothread();
2952     trace_postcopy_start_set_run();
2953
2954     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
2955     global_state_store();
2956     ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
2957     if (ret < 0) {
2958         goto fail;
2959     }
2960
2961     ret = migration_maybe_pause(ms, &cur_state,
2962                                 MIGRATION_STATUS_POSTCOPY_ACTIVE);
2963     if (ret < 0) {
2964         goto fail;
2965     }
2966
2967     ret = bdrv_inactivate_all();
2968     if (ret < 0) {
2969         goto fail;
2970     }
2971     restart_block = true;
2972
2973     /*
2974      * Cause any non-postcopiable, but iterative devices to
2975      * send out their final data.
2976      */
2977     qemu_savevm_state_complete_precopy(ms->to_dst_file, true, false);
2978
2979     /*
2980      * in Finish migrate and with the io-lock held everything should
2981      * be quiet, but we've potentially still got dirty pages and we
2982      * need to tell the destination to throw any pages it's already received
2983      * that are dirty
2984      */
2985     if (migrate_postcopy_ram()) {
2986         if (ram_postcopy_send_discard_bitmap(ms)) {
2987             error_report("postcopy send discard bitmap failed");
2988             goto fail;
2989         }
2990     }
2991
2992     /*
2993      * send rest of state - note things that are doing postcopy
2994      * will notice we're in POSTCOPY_ACTIVE and not actually
2995      * wrap their state up here
2996      */
2997     /* 0 max-postcopy-bandwidth means unlimited */
2998     if (!bandwidth) {
2999         qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX);
3000     } else {
3001         qemu_file_set_rate_limit(ms->to_dst_file, bandwidth / XFER_LIMIT_RATIO);
3002     }
3003     if (migrate_postcopy_ram()) {
3004         /* Ping just for debugging, helps line traces up */
3005         qemu_savevm_send_ping(ms->to_dst_file, 2);
3006     }
3007
3008     /*
3009      * While loading the device state we may trigger page transfer
3010      * requests and the fd must be free to process those, and thus
3011      * the destination must read the whole device state off the fd before
3012      * it starts processing it.  Unfortunately the ad-hoc migration format
3013      * doesn't allow the destination to know the size to read without fully
3014      * parsing it through each devices load-state code (especially the open
3015      * coded devices that use get/put).
3016      * So we wrap the device state up in a package with a length at the start;
3017      * to do this we use a qemu_buf to hold the whole of the device state.
3018      */
3019     bioc = qio_channel_buffer_new(4096);
3020     qio_channel_set_name(QIO_CHANNEL(bioc), "migration-postcopy-buffer");
3021     fb = qemu_fopen_channel_output(QIO_CHANNEL(bioc));
3022     object_unref(OBJECT(bioc));
3023
3024     /*
3025      * Make sure the receiver can get incoming pages before we send the rest
3026      * of the state
3027      */
3028     qemu_savevm_send_postcopy_listen(fb);
3029
3030     qemu_savevm_state_complete_precopy(fb, false, false);
3031     if (migrate_postcopy_ram()) {
3032         qemu_savevm_send_ping(fb, 3);
3033     }
3034
3035     qemu_savevm_send_postcopy_run(fb);
3036
3037     /* <><> end of stuff going into the package */
3038
3039     /* Last point of recovery; as soon as we send the package the destination
3040      * can open devices and potentially start running.
3041      * Lets just check again we've not got any errors.
3042      */
3043     ret = qemu_file_get_error(ms->to_dst_file);
3044     if (ret) {
3045         error_report("postcopy_start: Migration stream errored (pre package)");
3046         goto fail_closefb;
3047     }
3048
3049     restart_block = false;
3050
3051     /* Now send that blob */
3052     if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
3053         goto fail_closefb;
3054     }
3055     qemu_fclose(fb);
3056
3057     /* Send a notify to give a chance for anything that needs to happen
3058      * at the transition to postcopy and after the device state; in particular
3059      * spice needs to trigger a transition now
3060      */
3061     ms->postcopy_after_devices = true;
3062     notifier_list_notify(&migration_state_notifiers, ms);
3063
3064     ms->downtime =  qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop;
3065
3066     qemu_mutex_unlock_iothread();
3067
3068     if (migrate_postcopy_ram()) {
3069         /*
3070          * Although this ping is just for debug, it could potentially be
3071          * used for getting a better measurement of downtime at the source.
3072          */
3073         qemu_savevm_send_ping(ms->to_dst_file, 4);
3074     }
3075
3076     if (migrate_release_ram()) {
3077         ram_postcopy_migrated_memory_release(ms);
3078     }
3079
3080     ret = qemu_file_get_error(ms->to_dst_file);
3081     if (ret) {
3082         error_report("postcopy_start: Migration stream errored");
3083         migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3084                               MIGRATION_STATUS_FAILED);
3085     }
3086
3087     return ret;
3088
3089 fail_closefb:
3090     qemu_fclose(fb);
3091 fail:
3092     migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3093                           MIGRATION_STATUS_FAILED);
3094     if (restart_block) {
3095         /* A failure happened early enough that we know the destination hasn't
3096          * accessed block devices, so we're safe to recover.
3097          */
3098         Error *local_err = NULL;
3099
3100         bdrv_invalidate_cache_all(&local_err);
3101         if (local_err) {
3102             error_report_err(local_err);
3103         }
3104     }
3105     qemu_mutex_unlock_iothread();
3106     return -1;
3107 }
3108
3109 /**
3110  * migration_maybe_pause: Pause if required to by
3111  * migrate_pause_before_switchover called with the iothread locked
3112  * Returns: 0 on success
3113  */
3114 static int migration_maybe_pause(MigrationState *s,
3115                                  int *current_active_state,
3116                                  int new_state)
3117 {
3118     if (!migrate_pause_before_switchover()) {
3119         return 0;
3120     }
3121
3122     /* Since leaving this state is not atomic with posting the semaphore
3123      * it's possible that someone could have issued multiple migrate_continue
3124      * and the semaphore is incorrectly positive at this point;
3125      * the docs say it's undefined to reinit a semaphore that's already
3126      * init'd, so use timedwait to eat up any existing posts.
3127      */
3128     while (qemu_sem_timedwait(&s->pause_sem, 1) == 0) {
3129         /* This block intentionally left blank */
3130     }
3131
3132     /*
3133      * If the migration is cancelled when it is in the completion phase,
3134      * the migration state is set to MIGRATION_STATUS_CANCELLING.
3135      * So we don't need to wait a semaphore, otherwise we would always
3136      * wait for the 'pause_sem' semaphore.
3137      */
3138     if (s->state != MIGRATION_STATUS_CANCELLING) {
3139         qemu_mutex_unlock_iothread();
3140         migrate_set_state(&s->state, *current_active_state,
3141                           MIGRATION_STATUS_PRE_SWITCHOVER);
3142         qemu_sem_wait(&s->pause_sem);
3143         migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER,
3144                           new_state);
3145         *current_active_state = new_state;
3146         qemu_mutex_lock_iothread();
3147     }
3148
3149     return s->state == new_state ? 0 : -EINVAL;
3150 }
3151
3152 /**
3153  * migration_completion: Used by migration_thread when there's not much left.
3154  *   The caller 'breaks' the loop when this returns.
3155  *
3156  * @s: Current migration state
3157  */
3158 static void migration_completion(MigrationState *s)
3159 {
3160     int ret;
3161     int current_active_state = s->state;
3162
3163     if (s->state == MIGRATION_STATUS_ACTIVE) {
3164         qemu_mutex_lock_iothread();
3165         s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3166         qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3167         s->vm_was_running = runstate_is_running();
3168         ret = global_state_store();
3169
3170         if (!ret) {
3171             bool inactivate = !migrate_colo_enabled();
3172             ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
3173             trace_migration_completion_vm_stop(ret);
3174             if (ret >= 0) {
3175                 ret = migration_maybe_pause(s, &current_active_state,
3176                                             MIGRATION_STATUS_DEVICE);
3177             }
3178             if (ret >= 0) {
3179                 qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3180                 ret = qemu_savevm_state_complete_precopy(s->to_dst_file, false,
3181                                                          inactivate);
3182             }
3183             if (inactivate && ret >= 0) {
3184                 s->block_inactive = true;
3185             }
3186         }
3187         qemu_mutex_unlock_iothread();
3188
3189         if (ret < 0) {
3190             goto fail;
3191         }
3192     } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3193         trace_migration_completion_postcopy_end();
3194
3195         qemu_mutex_lock_iothread();
3196         qemu_savevm_state_complete_postcopy(s->to_dst_file);
3197         qemu_mutex_unlock_iothread();
3198
3199         trace_migration_completion_postcopy_end_after_complete();
3200     } else if (s->state == MIGRATION_STATUS_CANCELLING) {
3201         goto fail;
3202     }
3203
3204     /*
3205      * If rp was opened we must clean up the thread before
3206      * cleaning everything else up (since if there are no failures
3207      * it will wait for the destination to send it's status in
3208      * a SHUT command).
3209      */
3210     if (s->rp_state.rp_thread_created) {
3211         int rp_error;
3212         trace_migration_return_path_end_before();
3213         rp_error = await_return_path_close_on_source(s);
3214         trace_migration_return_path_end_after(rp_error);
3215         if (rp_error) {
3216             goto fail_invalidate;
3217         }
3218     }
3219
3220     if (qemu_file_get_error(s->to_dst_file)) {
3221         trace_migration_completion_file_err();
3222         goto fail_invalidate;
3223     }
3224
3225     if (!migrate_colo_enabled()) {
3226         migrate_set_state(&s->state, current_active_state,
3227                           MIGRATION_STATUS_COMPLETED);
3228     }
3229
3230     return;
3231
3232 fail_invalidate:
3233     /* If not doing postcopy, vm_start() will be called: let's regain
3234      * control on images.
3235      */
3236     if (s->state == MIGRATION_STATUS_ACTIVE ||
3237         s->state == MIGRATION_STATUS_DEVICE) {
3238         Error *local_err = NULL;
3239
3240         qemu_mutex_lock_iothread();
3241         bdrv_invalidate_cache_all(&local_err);
3242         if (local_err) {
3243             error_report_err(local_err);
3244         } else {
3245             s->block_inactive = false;
3246         }
3247         qemu_mutex_unlock_iothread();
3248     }
3249
3250 fail:
3251     migrate_set_state(&s->state, current_active_state,
3252                       MIGRATION_STATUS_FAILED);
3253 }
3254
3255 /**
3256  * bg_migration_completion: Used by bg_migration_thread when after all the
3257  *   RAM has been saved. The caller 'breaks' the loop when this returns.
3258  *
3259  * @s: Current migration state
3260  */
3261 static void bg_migration_completion(MigrationState *s)
3262 {
3263     int current_active_state = s->state;
3264
3265     /*
3266      * Stop tracking RAM writes - un-protect memory, un-register UFFD
3267      * memory ranges, flush kernel wait queues and wake up threads
3268      * waiting for write fault to be resolved.
3269      */
3270     ram_write_tracking_stop();
3271
3272     if (s->state == MIGRATION_STATUS_ACTIVE) {
3273         /*
3274          * By this moment we have RAM content saved into the migration stream.
3275          * The next step is to flush the non-RAM content (device state)
3276          * right after the ram content. The device state has been stored into
3277          * the temporary buffer before RAM saving started.
3278          */
3279         qemu_put_buffer(s->to_dst_file, s->bioc->data, s->bioc->usage);
3280         qemu_fflush(s->to_dst_file);
3281     } else if (s->state == MIGRATION_STATUS_CANCELLING) {
3282         goto fail;
3283     }
3284
3285     if (qemu_file_get_error(s->to_dst_file)) {
3286         trace_migration_completion_file_err();
3287         goto fail;
3288     }
3289
3290     migrate_set_state(&s->state, current_active_state,
3291                       MIGRATION_STATUS_COMPLETED);
3292     return;
3293
3294 fail:
3295     migrate_set_state(&s->state, current_active_state,
3296                       MIGRATION_STATUS_FAILED);
3297 }
3298
3299 bool migrate_colo_enabled(void)
3300 {
3301     MigrationState *s = migrate_get_current();
3302     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_COLO];
3303 }
3304
3305 typedef enum MigThrError {
3306     /* No error detected */
3307     MIG_THR_ERR_NONE = 0,
3308     /* Detected error, but resumed successfully */
3309     MIG_THR_ERR_RECOVERED = 1,
3310     /* Detected fatal error, need to exit */
3311     MIG_THR_ERR_FATAL = 2,
3312 } MigThrError;
3313
3314 static int postcopy_resume_handshake(MigrationState *s)
3315 {
3316     qemu_savevm_send_postcopy_resume(s->to_dst_file);
3317
3318     while (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3319         qemu_sem_wait(&s->rp_state.rp_sem);
3320     }
3321
3322     if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3323         return 0;
3324     }
3325
3326     return -1;
3327 }
3328
3329 /* Return zero if success, or <0 for error */
3330 static int postcopy_do_resume(MigrationState *s)
3331 {
3332     int ret;
3333
3334     /*
3335      * Call all the resume_prepare() hooks, so that modules can be
3336      * ready for the migration resume.
3337      */
3338     ret = qemu_savevm_state_resume_prepare(s);
3339     if (ret) {
3340         error_report("%s: resume_prepare() failure detected: %d",
3341                      __func__, ret);
3342         return ret;
3343     }
3344
3345     /*
3346      * Last handshake with destination on the resume (destination will
3347      * switch to postcopy-active afterwards)
3348      */
3349     ret = postcopy_resume_handshake(s);
3350     if (ret) {
3351         error_report("%s: handshake failed: %d", __func__, ret);
3352         return ret;
3353     }
3354
3355     return 0;
3356 }
3357
3358 /*
3359  * We don't return until we are in a safe state to continue current
3360  * postcopy migration.  Returns MIG_THR_ERR_RECOVERED if recovered, or
3361  * MIG_THR_ERR_FATAL if unrecovery failure happened.
3362  */
3363 static MigThrError postcopy_pause(MigrationState *s)
3364 {
3365     assert(s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
3366
3367     while (true) {
3368         QEMUFile *file;
3369
3370         /*
3371          * Current channel is possibly broken. Release it.  Note that this is
3372          * guaranteed even without lock because to_dst_file should only be
3373          * modified by the migration thread.  That also guarantees that the
3374          * unregister of yank is safe too without the lock.  It should be safe
3375          * even to be within the qemu_file_lock, but we didn't do that to avoid
3376          * taking more mutex (yank_lock) within qemu_file_lock.  TL;DR: we make
3377          * the qemu_file_lock critical section as small as possible.
3378          */
3379         assert(s->to_dst_file);
3380         migration_ioc_unregister_yank_from_file(s->to_dst_file);
3381         qemu_mutex_lock(&s->qemu_file_lock);
3382         file = s->to_dst_file;
3383         s->to_dst_file = NULL;
3384         qemu_mutex_unlock(&s->qemu_file_lock);
3385
3386         qemu_file_shutdown(file);
3387         qemu_fclose(file);
3388
3389         migrate_set_state(&s->state, s->state,
3390                           MIGRATION_STATUS_POSTCOPY_PAUSED);
3391
3392         error_report("Detected IO failure for postcopy. "
3393                      "Migration paused.");
3394
3395         /*
3396          * We wait until things fixed up. Then someone will setup the
3397          * status back for us.
3398          */
3399         while (s->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
3400             qemu_sem_wait(&s->postcopy_pause_sem);
3401         }
3402
3403         if (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3404             /* Woken up by a recover procedure. Give it a shot */
3405
3406             /*
3407              * Firstly, let's wake up the return path now, with a new
3408              * return path channel.
3409              */
3410             qemu_sem_post(&s->postcopy_pause_rp_sem);
3411
3412             /* Do the resume logic */
3413             if (postcopy_do_resume(s) == 0) {
3414                 /* Let's continue! */
3415                 trace_postcopy_pause_continued();
3416                 return MIG_THR_ERR_RECOVERED;
3417             } else {
3418                 /*
3419                  * Something wrong happened during the recovery, let's
3420                  * pause again. Pause is always better than throwing
3421                  * data away.
3422                  */
3423                 continue;
3424             }
3425         } else {
3426             /* This is not right... Time to quit. */
3427             return MIG_THR_ERR_FATAL;
3428         }
3429     }
3430 }
3431
3432 static MigThrError migration_detect_error(MigrationState *s)
3433 {
3434     int ret;
3435     int state = s->state;
3436     Error *local_error = NULL;
3437
3438     if (state == MIGRATION_STATUS_CANCELLING ||
3439         state == MIGRATION_STATUS_CANCELLED) {
3440         /* End the migration, but don't set the state to failed */
3441         return MIG_THR_ERR_FATAL;
3442     }
3443
3444     /* Try to detect any file errors */
3445     ret = qemu_file_get_error_obj(s->to_dst_file, &local_error);
3446     if (!ret) {
3447         /* Everything is fine */
3448         assert(!local_error);
3449         return MIG_THR_ERR_NONE;
3450     }
3451
3452     if (local_error) {
3453         migrate_set_error(s, local_error);
3454         error_free(local_error);
3455     }
3456
3457     if (state == MIGRATION_STATUS_POSTCOPY_ACTIVE && ret == -EIO) {
3458         /*
3459          * For postcopy, we allow the network to be down for a
3460          * while. After that, it can be continued by a
3461          * recovery phase.
3462          */
3463         return postcopy_pause(s);
3464     } else {
3465         /*
3466          * For precopy (or postcopy with error outside IO), we fail
3467          * with no time.
3468          */
3469         migrate_set_state(&s->state, state, MIGRATION_STATUS_FAILED);
3470         trace_migration_thread_file_err();
3471
3472         /* Time to stop the migration, now. */
3473         return MIG_THR_ERR_FATAL;
3474     }
3475 }
3476
3477 /* How many bytes have we transferred since the beginning of the migration */
3478 static uint64_t migration_total_bytes(MigrationState *s)
3479 {
3480     return qemu_ftell(s->to_dst_file) + ram_counters.multifd_bytes;
3481 }
3482
3483 static void migration_calculate_complete(MigrationState *s)
3484 {
3485     uint64_t bytes = migration_total_bytes(s);
3486     int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3487     int64_t transfer_time;
3488
3489     s->total_time = end_time - s->start_time;
3490     if (!s->downtime) {
3491         /*
3492          * It's still not set, so we are precopy migration.  For
3493          * postcopy, downtime is calculated during postcopy_start().
3494          */
3495         s->downtime = end_time - s->downtime_start;
3496     }
3497
3498     transfer_time = s->total_time - s->setup_time;
3499     if (transfer_time) {
3500         s->mbps = ((double) bytes * 8.0) / transfer_time / 1000;
3501     }
3502 }
3503
3504 static void update_iteration_initial_status(MigrationState *s)
3505 {
3506     /*
3507      * Update these three fields at the same time to avoid mismatch info lead
3508      * wrong speed calculation.
3509      */
3510     s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3511     s->iteration_initial_bytes = migration_total_bytes(s);
3512     s->iteration_initial_pages = ram_get_total_transferred_pages();
3513 }
3514
3515 static void migration_update_counters(MigrationState *s,
3516                                       int64_t current_time)
3517 {
3518     uint64_t transferred, transferred_pages, time_spent;
3519     uint64_t current_bytes; /* bytes transferred since the beginning */
3520     double bandwidth;
3521
3522     if (current_time < s->iteration_start_time + BUFFER_DELAY) {
3523         return;
3524     }
3525
3526     current_bytes = migration_total_bytes(s);
3527     transferred = current_bytes - s->iteration_initial_bytes;
3528     time_spent = current_time - s->iteration_start_time;
3529     bandwidth = (double)transferred / time_spent;
3530     s->threshold_size = bandwidth * s->parameters.downtime_limit;
3531
3532     s->mbps = (((double) transferred * 8.0) /
3533                ((double) time_spent / 1000.0)) / 1000.0 / 1000.0;
3534
3535     transferred_pages = ram_get_total_transferred_pages() -
3536                             s->iteration_initial_pages;
3537     s->pages_per_second = (double) transferred_pages /
3538                              (((double) time_spent / 1000.0));
3539
3540     /*
3541      * if we haven't sent anything, we don't want to
3542      * recalculate. 10000 is a small enough number for our purposes
3543      */
3544     if (ram_counters.dirty_pages_rate && transferred > 10000) {
3545         s->expected_downtime = ram_counters.remaining / bandwidth;
3546     }
3547
3548     qemu_file_reset_rate_limit(s->to_dst_file);
3549
3550     update_iteration_initial_status(s);
3551
3552     trace_migrate_transferred(transferred, time_spent,
3553                               bandwidth, s->threshold_size);
3554 }
3555
3556 /* Migration thread iteration status */
3557 typedef enum {
3558     MIG_ITERATE_RESUME,         /* Resume current iteration */
3559     MIG_ITERATE_SKIP,           /* Skip current iteration */
3560     MIG_ITERATE_BREAK,          /* Break the loop */
3561 } MigIterateState;
3562
3563 /*
3564  * Return true if continue to the next iteration directly, false
3565  * otherwise.
3566  */
3567 static MigIterateState migration_iteration_run(MigrationState *s)
3568 {
3569     uint64_t pending_size, pend_pre, pend_compat, pend_post;
3570     bool in_postcopy = s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE;
3571
3572     qemu_savevm_state_pending(s->to_dst_file, s->threshold_size, &pend_pre,
3573                               &pend_compat, &pend_post);
3574     pending_size = pend_pre + pend_compat + pend_post;
3575
3576     trace_migrate_pending(pending_size, s->threshold_size,
3577                           pend_pre, pend_compat, pend_post);
3578
3579     if (pending_size && pending_size >= s->threshold_size) {
3580         /* Still a significant amount to transfer */
3581         if (!in_postcopy && pend_pre <= s->threshold_size &&
3582             qatomic_read(&s->start_postcopy)) {
3583             if (postcopy_start(s)) {
3584                 error_report("%s: postcopy failed to start", __func__);
3585             }
3586             return MIG_ITERATE_SKIP;
3587         }
3588         /* Just another iteration step */
3589         qemu_savevm_state_iterate(s->to_dst_file, in_postcopy);
3590     } else {
3591         trace_migration_thread_low_pending(pending_size);
3592         migration_completion(s);
3593         return MIG_ITERATE_BREAK;
3594     }
3595
3596     return MIG_ITERATE_RESUME;
3597 }
3598
3599 static void migration_iteration_finish(MigrationState *s)
3600 {
3601     /* If we enabled cpu throttling for auto-converge, turn it off. */
3602     cpu_throttle_stop();
3603
3604     qemu_mutex_lock_iothread();
3605     switch (s->state) {
3606     case MIGRATION_STATUS_COMPLETED:
3607         migration_calculate_complete(s);
3608         runstate_set(RUN_STATE_POSTMIGRATE);
3609         break;
3610
3611     case MIGRATION_STATUS_ACTIVE:
3612         /*
3613          * We should really assert here, but since it's during
3614          * migration, let's try to reduce the usage of assertions.
3615          */
3616         if (!migrate_colo_enabled()) {
3617             error_report("%s: critical error: calling COLO code without "
3618                          "COLO enabled", __func__);
3619         }
3620         migrate_start_colo_process(s);
3621         /*
3622          * Fixme: we will run VM in COLO no matter its old running state.
3623          * After exited COLO, we will keep running.
3624          */
3625         s->vm_was_running = true;
3626         /* Fallthrough */
3627     case MIGRATION_STATUS_FAILED:
3628     case MIGRATION_STATUS_CANCELLED:
3629     case MIGRATION_STATUS_CANCELLING:
3630         if (s->vm_was_running) {
3631             if (!runstate_check(RUN_STATE_SHUTDOWN)) {
3632                 vm_start();
3633             }
3634         } else {
3635             if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
3636                 runstate_set(RUN_STATE_POSTMIGRATE);
3637             }
3638         }
3639         break;
3640
3641     default:
3642         /* Should not reach here, but if so, forgive the VM. */
3643         error_report("%s: Unknown ending state %d", __func__, s->state);
3644         break;
3645     }
3646     migrate_fd_cleanup_schedule(s);
3647     qemu_mutex_unlock_iothread();
3648 }
3649
3650 static void bg_migration_iteration_finish(MigrationState *s)
3651 {
3652     qemu_mutex_lock_iothread();
3653     switch (s->state) {
3654     case MIGRATION_STATUS_COMPLETED:
3655         migration_calculate_complete(s);
3656         break;
3657
3658     case MIGRATION_STATUS_ACTIVE:
3659     case MIGRATION_STATUS_FAILED:
3660     case MIGRATION_STATUS_CANCELLED:
3661     case MIGRATION_STATUS_CANCELLING:
3662         break;
3663
3664     default:
3665         /* Should not reach here, but if so, forgive the VM. */
3666         error_report("%s: Unknown ending state %d", __func__, s->state);
3667         break;
3668     }
3669
3670     migrate_fd_cleanup_schedule(s);
3671     qemu_mutex_unlock_iothread();
3672 }
3673
3674 /*
3675  * Return true if continue to the next iteration directly, false
3676  * otherwise.
3677  */
3678 static MigIterateState bg_migration_iteration_run(MigrationState *s)
3679 {
3680     int res;
3681
3682     res = qemu_savevm_state_iterate(s->to_dst_file, false);
3683     if (res > 0) {
3684         bg_migration_completion(s);
3685         return MIG_ITERATE_BREAK;
3686     }
3687
3688     return MIG_ITERATE_RESUME;
3689 }
3690
3691 void migration_make_urgent_request(void)
3692 {
3693     qemu_sem_post(&migrate_get_current()->rate_limit_sem);
3694 }
3695
3696 void migration_consume_urgent_request(void)
3697 {
3698     qemu_sem_wait(&migrate_get_current()->rate_limit_sem);
3699 }
3700
3701 /* Returns true if the rate limiting was broken by an urgent request */
3702 bool migration_rate_limit(void)
3703 {
3704     int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3705     MigrationState *s = migrate_get_current();
3706
3707     bool urgent = false;
3708     migration_update_counters(s, now);
3709     if (qemu_file_rate_limit(s->to_dst_file)) {
3710
3711         if (qemu_file_get_error(s->to_dst_file)) {
3712             return false;
3713         }
3714         /*
3715          * Wait for a delay to do rate limiting OR
3716          * something urgent to post the semaphore.
3717          */
3718         int ms = s->iteration_start_time + BUFFER_DELAY - now;
3719         trace_migration_rate_limit_pre(ms);
3720         if (qemu_sem_timedwait(&s->rate_limit_sem, ms) == 0) {
3721             /*
3722              * We were woken by one or more urgent things but
3723              * the timedwait will have consumed one of them.
3724              * The service routine for the urgent wake will dec
3725              * the semaphore itself for each item it consumes,
3726              * so add this one we just eat back.
3727              */
3728             qemu_sem_post(&s->rate_limit_sem);
3729             urgent = true;
3730         }
3731         trace_migration_rate_limit_post(urgent);
3732     }
3733     return urgent;
3734 }
3735
3736 /*
3737  * if failover devices are present, wait they are completely
3738  * unplugged
3739  */
3740
3741 static void qemu_savevm_wait_unplug(MigrationState *s, int old_state,
3742                                     int new_state)
3743 {
3744     if (qemu_savevm_state_guest_unplug_pending()) {
3745         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_WAIT_UNPLUG);
3746
3747         while (s->state == MIGRATION_STATUS_WAIT_UNPLUG &&
3748                qemu_savevm_state_guest_unplug_pending()) {
3749             qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3750         }
3751         if (s->state != MIGRATION_STATUS_WAIT_UNPLUG) {
3752             int timeout = 120; /* 30 seconds */
3753             /*
3754              * migration has been canceled
3755              * but as we have started an unplug we must wait the end
3756              * to be able to plug back the card
3757              */
3758             while (timeout-- && qemu_savevm_state_guest_unplug_pending()) {
3759                 qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3760             }
3761             if (qemu_savevm_state_guest_unplug_pending()) {
3762                 warn_report("migration: partially unplugged device on "
3763                             "failure");
3764             }
3765         }
3766
3767         migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG, new_state);
3768     } else {
3769         migrate_set_state(&s->state, old_state, new_state);
3770     }
3771 }
3772
3773 /*
3774  * Master migration thread on the source VM.
3775  * It drives the migration and pumps the data down the outgoing channel.
3776  */
3777 static void *migration_thread(void *opaque)
3778 {
3779     MigrationState *s = opaque;
3780     int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3781     MigThrError thr_error;
3782     bool urgent = false;
3783
3784     rcu_register_thread();
3785
3786     object_ref(OBJECT(s));
3787     update_iteration_initial_status(s);
3788
3789     qemu_savevm_state_header(s->to_dst_file);
3790
3791     /*
3792      * If we opened the return path, we need to make sure dst has it
3793      * opened as well.
3794      */
3795     if (s->rp_state.rp_thread_created) {
3796         /* Now tell the dest that it should open its end so it can reply */
3797         qemu_savevm_send_open_return_path(s->to_dst_file);
3798
3799         /* And do a ping that will make stuff easier to debug */
3800         qemu_savevm_send_ping(s->to_dst_file, 1);
3801     }
3802
3803     if (migrate_postcopy()) {
3804         /*
3805          * Tell the destination that we *might* want to do postcopy later;
3806          * if the other end can't do postcopy it should fail now, nice and
3807          * early.
3808          */
3809         qemu_savevm_send_postcopy_advise(s->to_dst_file);
3810     }
3811
3812     if (migrate_colo_enabled()) {
3813         /* Notify migration destination that we enable COLO */
3814         qemu_savevm_send_colo_enable(s->to_dst_file);
3815     }
3816
3817     qemu_savevm_state_setup(s->to_dst_file);
3818
3819     qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
3820                                MIGRATION_STATUS_ACTIVE);
3821
3822     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3823
3824     trace_migration_thread_setup_complete();
3825
3826     while (migration_is_active(s)) {
3827         if (urgent || !qemu_file_rate_limit(s->to_dst_file)) {
3828             MigIterateState iter_state = migration_iteration_run(s);
3829             if (iter_state == MIG_ITERATE_SKIP) {
3830                 continue;
3831             } else if (iter_state == MIG_ITERATE_BREAK) {
3832                 break;
3833             }
3834         }
3835
3836         /*
3837          * Try to detect any kind of failures, and see whether we
3838          * should stop the migration now.
3839          */
3840         thr_error = migration_detect_error(s);
3841         if (thr_error == MIG_THR_ERR_FATAL) {
3842             /* Stop migration */
3843             break;
3844         } else if (thr_error == MIG_THR_ERR_RECOVERED) {
3845             /*
3846              * Just recovered from a e.g. network failure, reset all
3847              * the local variables. This is important to avoid
3848              * breaking transferred_bytes and bandwidth calculation
3849              */
3850             update_iteration_initial_status(s);
3851         }
3852
3853         urgent = migration_rate_limit();
3854     }
3855
3856     trace_migration_thread_after_loop();
3857     migration_iteration_finish(s);
3858     object_unref(OBJECT(s));
3859     rcu_unregister_thread();
3860     return NULL;
3861 }
3862
3863 static void bg_migration_vm_start_bh(void *opaque)
3864 {
3865     MigrationState *s = opaque;
3866
3867     qemu_bh_delete(s->vm_start_bh);
3868     s->vm_start_bh = NULL;
3869
3870     vm_start();
3871     s->downtime = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - s->downtime_start;
3872 }
3873
3874 /**
3875  * Background snapshot thread, based on live migration code.
3876  * This is an alternative implementation of live migration mechanism
3877  * introduced specifically to support background snapshots.
3878  *
3879  * It takes advantage of userfault_fd write protection mechanism introduced
3880  * in v5.7 kernel. Compared to existing dirty page logging migration much
3881  * lesser stream traffic is produced resulting in smaller snapshot images,
3882  * simply cause of no page duplicates can get into the stream.
3883  *
3884  * Another key point is that generated vmstate stream reflects machine state
3885  * 'frozen' at the beginning of snapshot creation compared to dirty page logging
3886  * mechanism, which effectively results in that saved snapshot is the state of VM
3887  * at the end of the process.
3888  */
3889 static void *bg_migration_thread(void *opaque)
3890 {
3891     MigrationState *s = opaque;
3892     int64_t setup_start;
3893     MigThrError thr_error;
3894     QEMUFile *fb;
3895     bool early_fail = true;
3896
3897     rcu_register_thread();
3898     object_ref(OBJECT(s));
3899
3900     qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3901
3902     setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3903     /*
3904      * We want to save vmstate for the moment when migration has been
3905      * initiated but also we want to save RAM content while VM is running.
3906      * The RAM content should appear first in the vmstate. So, we first
3907      * stash the non-RAM part of the vmstate to the temporary buffer,
3908      * then write RAM part of the vmstate to the migration stream
3909      * with vCPUs running and, finally, write stashed non-RAM part of
3910      * the vmstate from the buffer to the migration stream.
3911      */
3912     s->bioc = qio_channel_buffer_new(512 * 1024);
3913     qio_channel_set_name(QIO_CHANNEL(s->bioc), "vmstate-buffer");
3914     fb = qemu_fopen_channel_output(QIO_CHANNEL(s->bioc));
3915     object_unref(OBJECT(s->bioc));
3916
3917     update_iteration_initial_status(s);
3918
3919     /*
3920      * Prepare for tracking memory writes with UFFD-WP - populate
3921      * RAM pages before protecting.
3922      */
3923 #ifdef __linux__
3924     ram_write_tracking_prepare();
3925 #endif
3926
3927     qemu_savevm_state_header(s->to_dst_file);
3928     qemu_savevm_state_setup(s->to_dst_file);
3929
3930     qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
3931                                MIGRATION_STATUS_ACTIVE);
3932
3933     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3934
3935     trace_migration_thread_setup_complete();
3936     s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3937
3938     qemu_mutex_lock_iothread();
3939
3940     /*
3941      * If VM is currently in suspended state, then, to make a valid runstate
3942      * transition in vm_stop_force_state() we need to wakeup it up.
3943      */
3944     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3945     s->vm_was_running = runstate_is_running();
3946
3947     if (global_state_store()) {
3948         goto fail;
3949     }
3950     /* Forcibly stop VM before saving state of vCPUs and devices */
3951     if (vm_stop_force_state(RUN_STATE_PAUSED)) {
3952         goto fail;
3953     }
3954     /*
3955      * Put vCPUs in sync with shadow context structures, then
3956      * save their state to channel-buffer along with devices.
3957      */
3958     cpu_synchronize_all_states();
3959     if (qemu_savevm_state_complete_precopy_non_iterable(fb, false, false)) {
3960         goto fail;
3961     }
3962     /*
3963      * Since we are going to get non-iterable state data directly
3964      * from s->bioc->data, explicit flush is needed here.
3965      */
3966     qemu_fflush(fb);
3967
3968     /* Now initialize UFFD context and start tracking RAM writes */
3969     if (ram_write_tracking_start()) {
3970         goto fail;
3971     }
3972     early_fail = false;
3973
3974     /*
3975      * Start VM from BH handler to avoid write-fault lock here.
3976      * UFFD-WP protection for the whole RAM is already enabled so
3977      * calling VM state change notifiers from vm_start() would initiate
3978      * writes to virtio VQs memory which is in write-protected region.
3979      */
3980     s->vm_start_bh = qemu_bh_new(bg_migration_vm_start_bh, s);
3981     qemu_bh_schedule(s->vm_start_bh);
3982
3983     qemu_mutex_unlock_iothread();
3984
3985     while (migration_is_active(s)) {
3986         MigIterateState iter_state = bg_migration_iteration_run(s);
3987         if (iter_state == MIG_ITERATE_SKIP) {
3988             continue;
3989         } else if (iter_state == MIG_ITERATE_BREAK) {
3990             break;
3991         }
3992
3993         /*
3994          * Try to detect any kind of failures, and see whether we
3995          * should stop the migration now.
3996          */
3997         thr_error = migration_detect_error(s);
3998         if (thr_error == MIG_THR_ERR_FATAL) {
3999             /* Stop migration */
4000             break;
4001         }
4002
4003         migration_update_counters(s, qemu_clock_get_ms(QEMU_CLOCK_REALTIME));
4004     }
4005
4006     trace_migration_thread_after_loop();
4007
4008 fail:
4009     if (early_fail) {
4010         migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE,
4011                 MIGRATION_STATUS_FAILED);
4012         qemu_mutex_unlock_iothread();
4013     }
4014
4015     bg_migration_iteration_finish(s);
4016
4017     qemu_fclose(fb);
4018     object_unref(OBJECT(s));
4019     rcu_unregister_thread();
4020
4021     return NULL;
4022 }
4023
4024 void migrate_fd_connect(MigrationState *s, Error *error_in)
4025 {
4026     Error *local_err = NULL;
4027     int64_t rate_limit;
4028     bool resume = s->state == MIGRATION_STATUS_POSTCOPY_PAUSED;
4029
4030     /*
4031      * If there's a previous error, free it and prepare for another one.
4032      * Meanwhile if migration completes successfully, there won't have an error
4033      * dumped when calling migrate_fd_cleanup().
4034      */
4035     migrate_error_free(s);
4036
4037     s->expected_downtime = s->parameters.downtime_limit;
4038     if (resume) {
4039         assert(s->cleanup_bh);
4040     } else {
4041         assert(!s->cleanup_bh);
4042         s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup_bh, s);
4043     }
4044     if (error_in) {
4045         migrate_fd_error(s, error_in);
4046         if (resume) {
4047             /*
4048              * Don't do cleanup for resume if channel is invalid, but only dump
4049              * the error.  We wait for another channel connect from the user.
4050              * The error_report still gives HMP user a hint on what failed.
4051              * It's normally done in migrate_fd_cleanup(), but call it here
4052              * explicitly.
4053              */
4054             error_report_err(error_copy(s->error));
4055         } else {
4056             migrate_fd_cleanup(s);
4057         }
4058         return;
4059     }
4060
4061     if (resume) {
4062         /* This is a resumed migration */
4063         rate_limit = s->parameters.max_postcopy_bandwidth /
4064             XFER_LIMIT_RATIO;
4065     } else {
4066         /* This is a fresh new migration */
4067         rate_limit = s->parameters.max_bandwidth / XFER_LIMIT_RATIO;
4068
4069         /* Notify before starting migration thread */
4070         notifier_list_notify(&migration_state_notifiers, s);
4071     }
4072
4073     qemu_file_set_rate_limit(s->to_dst_file, rate_limit);
4074     qemu_file_set_blocking(s->to_dst_file, true);
4075
4076     /*
4077      * Open the return path. For postcopy, it is used exclusively. For
4078      * precopy, only if user specified "return-path" capability would
4079      * QEMU uses the return path.
4080      */
4081     if (migrate_postcopy_ram() || migrate_use_return_path()) {
4082         if (open_return_path_on_source(s, !resume)) {
4083             error_report("Unable to open return-path for postcopy");
4084             migrate_set_state(&s->state, s->state, MIGRATION_STATUS_FAILED);
4085             migrate_fd_cleanup(s);
4086             return;
4087         }
4088     }
4089
4090     if (resume) {
4091         /* Wakeup the main migration thread to do the recovery */
4092         migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
4093                           MIGRATION_STATUS_POSTCOPY_RECOVER);
4094         qemu_sem_post(&s->postcopy_pause_sem);
4095         return;
4096     }
4097
4098     if (multifd_save_setup(&local_err) != 0) {
4099         error_report_err(local_err);
4100         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
4101                           MIGRATION_STATUS_FAILED);
4102         migrate_fd_cleanup(s);
4103         return;
4104     }
4105
4106     if (migrate_background_snapshot()) {
4107         qemu_thread_create(&s->thread, "bg_snapshot",
4108                 bg_migration_thread, s, QEMU_THREAD_JOINABLE);
4109     } else {
4110         qemu_thread_create(&s->thread, "live_migration",
4111                 migration_thread, s, QEMU_THREAD_JOINABLE);
4112     }
4113     s->migration_thread_running = true;
4114 }
4115
4116 void migration_global_dump(Monitor *mon)
4117 {
4118     MigrationState *ms = migrate_get_current();
4119
4120     monitor_printf(mon, "globals:\n");
4121     monitor_printf(mon, "store-global-state: %s\n",
4122                    ms->store_global_state ? "on" : "off");
4123     monitor_printf(mon, "only-migratable: %s\n",
4124                    only_migratable ? "on" : "off");
4125     monitor_printf(mon, "send-configuration: %s\n",
4126                    ms->send_configuration ? "on" : "off");
4127     monitor_printf(mon, "send-section-footer: %s\n",
4128                    ms->send_section_footer ? "on" : "off");
4129     monitor_printf(mon, "decompress-error-check: %s\n",
4130                    ms->decompress_error_check ? "on" : "off");
4131     monitor_printf(mon, "clear-bitmap-shift: %u\n",
4132                    ms->clear_bitmap_shift);
4133 }
4134
4135 #define DEFINE_PROP_MIG_CAP(name, x)             \
4136     DEFINE_PROP_BOOL(name, MigrationState, enabled_capabilities[x], false)
4137
4138 static Property migration_properties[] = {
4139     DEFINE_PROP_BOOL("store-global-state", MigrationState,
4140                      store_global_state, true),
4141     DEFINE_PROP_BOOL("send-configuration", MigrationState,
4142                      send_configuration, true),
4143     DEFINE_PROP_BOOL("send-section-footer", MigrationState,
4144                      send_section_footer, true),
4145     DEFINE_PROP_BOOL("decompress-error-check", MigrationState,
4146                       decompress_error_check, true),
4147     DEFINE_PROP_UINT8("x-clear-bitmap-shift", MigrationState,
4148                       clear_bitmap_shift, CLEAR_BITMAP_SHIFT_DEFAULT),
4149
4150     /* Migration parameters */
4151     DEFINE_PROP_UINT8("x-compress-level", MigrationState,
4152                       parameters.compress_level,
4153                       DEFAULT_MIGRATE_COMPRESS_LEVEL),
4154     DEFINE_PROP_UINT8("x-compress-threads", MigrationState,
4155                       parameters.compress_threads,
4156                       DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT),
4157     DEFINE_PROP_BOOL("x-compress-wait-thread", MigrationState,
4158                       parameters.compress_wait_thread, true),
4159     DEFINE_PROP_UINT8("x-decompress-threads", MigrationState,
4160                       parameters.decompress_threads,
4161                       DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT),
4162     DEFINE_PROP_UINT8("x-throttle-trigger-threshold", MigrationState,
4163                       parameters.throttle_trigger_threshold,
4164                       DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD),
4165     DEFINE_PROP_UINT8("x-cpu-throttle-initial", MigrationState,
4166                       parameters.cpu_throttle_initial,
4167                       DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL),
4168     DEFINE_PROP_UINT8("x-cpu-throttle-increment", MigrationState,
4169                       parameters.cpu_throttle_increment,
4170                       DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT),
4171     DEFINE_PROP_BOOL("x-cpu-throttle-tailslow", MigrationState,
4172                       parameters.cpu_throttle_tailslow, false),
4173     DEFINE_PROP_SIZE("x-max-bandwidth", MigrationState,
4174                       parameters.max_bandwidth, MAX_THROTTLE),
4175     DEFINE_PROP_UINT64("x-downtime-limit", MigrationState,
4176                       parameters.downtime_limit,
4177                       DEFAULT_MIGRATE_SET_DOWNTIME),
4178     DEFINE_PROP_UINT32("x-checkpoint-delay", MigrationState,
4179                       parameters.x_checkpoint_delay,
4180                       DEFAULT_MIGRATE_X_CHECKPOINT_DELAY),
4181     DEFINE_PROP_UINT8("multifd-channels", MigrationState,
4182                       parameters.multifd_channels,
4183                       DEFAULT_MIGRATE_MULTIFD_CHANNELS),
4184     DEFINE_PROP_MULTIFD_COMPRESSION("multifd-compression", MigrationState,
4185                       parameters.multifd_compression,
4186                       DEFAULT_MIGRATE_MULTIFD_COMPRESSION),
4187     DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState,
4188                       parameters.multifd_zlib_level,
4189                       DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL),
4190     DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState,
4191                       parameters.multifd_zstd_level,
4192                       DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL),
4193     DEFINE_PROP_SIZE("xbzrle-cache-size", MigrationState,
4194                       parameters.xbzrle_cache_size,
4195                       DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE),
4196     DEFINE_PROP_SIZE("max-postcopy-bandwidth", MigrationState,
4197                       parameters.max_postcopy_bandwidth,
4198                       DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH),
4199     DEFINE_PROP_UINT8("max-cpu-throttle", MigrationState,
4200                       parameters.max_cpu_throttle,
4201                       DEFAULT_MIGRATE_MAX_CPU_THROTTLE),
4202     DEFINE_PROP_SIZE("announce-initial", MigrationState,
4203                       parameters.announce_initial,
4204                       DEFAULT_MIGRATE_ANNOUNCE_INITIAL),
4205     DEFINE_PROP_SIZE("announce-max", MigrationState,
4206                       parameters.announce_max,
4207                       DEFAULT_MIGRATE_ANNOUNCE_MAX),
4208     DEFINE_PROP_SIZE("announce-rounds", MigrationState,
4209                       parameters.announce_rounds,
4210                       DEFAULT_MIGRATE_ANNOUNCE_ROUNDS),
4211     DEFINE_PROP_SIZE("announce-step", MigrationState,
4212                       parameters.announce_step,
4213                       DEFAULT_MIGRATE_ANNOUNCE_STEP),
4214
4215     /* Migration capabilities */
4216     DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
4217     DEFINE_PROP_MIG_CAP("x-rdma-pin-all", MIGRATION_CAPABILITY_RDMA_PIN_ALL),
4218     DEFINE_PROP_MIG_CAP("x-auto-converge", MIGRATION_CAPABILITY_AUTO_CONVERGE),
4219     DEFINE_PROP_MIG_CAP("x-zero-blocks", MIGRATION_CAPABILITY_ZERO_BLOCKS),
4220     DEFINE_PROP_MIG_CAP("x-compress", MIGRATION_CAPABILITY_COMPRESS),
4221     DEFINE_PROP_MIG_CAP("x-events", MIGRATION_CAPABILITY_EVENTS),
4222     DEFINE_PROP_MIG_CAP("x-postcopy-ram", MIGRATION_CAPABILITY_POSTCOPY_RAM),
4223     DEFINE_PROP_MIG_CAP("x-colo", MIGRATION_CAPABILITY_X_COLO),
4224     DEFINE_PROP_MIG_CAP("x-release-ram", MIGRATION_CAPABILITY_RELEASE_RAM),
4225     DEFINE_PROP_MIG_CAP("x-block", MIGRATION_CAPABILITY_BLOCK),
4226     DEFINE_PROP_MIG_CAP("x-return-path", MIGRATION_CAPABILITY_RETURN_PATH),
4227     DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD),
4228     DEFINE_PROP_MIG_CAP("x-background-snapshot",
4229             MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT),
4230
4231     DEFINE_PROP_END_OF_LIST(),
4232 };
4233
4234 static void migration_class_init(ObjectClass *klass, void *data)
4235 {
4236     DeviceClass *dc = DEVICE_CLASS(klass);
4237
4238     dc->user_creatable = false;
4239     device_class_set_props(dc, migration_properties);
4240 }
4241
4242 static void migration_instance_finalize(Object *obj)
4243 {
4244     MigrationState *ms = MIGRATION_OBJ(obj);
4245     MigrationParameters *params = &ms->parameters;
4246
4247     qemu_mutex_destroy(&ms->error_mutex);
4248     qemu_mutex_destroy(&ms->qemu_file_lock);
4249     g_free(params->tls_hostname);
4250     g_free(params->tls_creds);
4251     qemu_sem_destroy(&ms->wait_unplug_sem);
4252     qemu_sem_destroy(&ms->rate_limit_sem);
4253     qemu_sem_destroy(&ms->pause_sem);
4254     qemu_sem_destroy(&ms->postcopy_pause_sem);
4255     qemu_sem_destroy(&ms->postcopy_pause_rp_sem);
4256     qemu_sem_destroy(&ms->rp_state.rp_sem);
4257     error_free(ms->error);
4258 }
4259
4260 static void migration_instance_init(Object *obj)
4261 {
4262     MigrationState *ms = MIGRATION_OBJ(obj);
4263     MigrationParameters *params = &ms->parameters;
4264
4265     ms->state = MIGRATION_STATUS_NONE;
4266     ms->mbps = -1;
4267     ms->pages_per_second = -1;
4268     qemu_sem_init(&ms->pause_sem, 0);
4269     qemu_mutex_init(&ms->error_mutex);
4270
4271     params->tls_hostname = g_strdup("");
4272     params->tls_creds = g_strdup("");
4273
4274     /* Set has_* up only for parameter checks */
4275     params->has_compress_level = true;
4276     params->has_compress_threads = true;
4277     params->has_decompress_threads = true;
4278     params->has_throttle_trigger_threshold = true;
4279     params->has_cpu_throttle_initial = true;
4280     params->has_cpu_throttle_increment = true;
4281     params->has_cpu_throttle_tailslow = true;
4282     params->has_max_bandwidth = true;
4283     params->has_downtime_limit = true;
4284     params->has_x_checkpoint_delay = true;
4285     params->has_block_incremental = true;
4286     params->has_multifd_channels = true;
4287     params->has_multifd_compression = true;
4288     params->has_multifd_zlib_level = true;
4289     params->has_multifd_zstd_level = true;
4290     params->has_xbzrle_cache_size = true;
4291     params->has_max_postcopy_bandwidth = true;
4292     params->has_max_cpu_throttle = true;
4293     params->has_announce_initial = true;
4294     params->has_announce_max = true;
4295     params->has_announce_rounds = true;
4296     params->has_announce_step = true;
4297
4298     qemu_sem_init(&ms->postcopy_pause_sem, 0);
4299     qemu_sem_init(&ms->postcopy_pause_rp_sem, 0);
4300     qemu_sem_init(&ms->rp_state.rp_sem, 0);
4301     qemu_sem_init(&ms->rate_limit_sem, 0);
4302     qemu_sem_init(&ms->wait_unplug_sem, 0);
4303     qemu_mutex_init(&ms->qemu_file_lock);
4304 }
4305
4306 /*
4307  * Return true if check pass, false otherwise. Error will be put
4308  * inside errp if provided.
4309  */
4310 static bool migration_object_check(MigrationState *ms, Error **errp)
4311 {
4312     MigrationCapabilityStatusList *head = NULL;
4313     /* Assuming all off */
4314     bool cap_list[MIGRATION_CAPABILITY__MAX] = { 0 }, ret;
4315     int i;
4316
4317     if (!migrate_params_check(&ms->parameters, errp)) {
4318         return false;
4319     }
4320
4321     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
4322         if (ms->enabled_capabilities[i]) {
4323             QAPI_LIST_PREPEND(head, migrate_cap_add(i, true));
4324         }
4325     }
4326
4327     ret = migrate_caps_check(cap_list, head, errp);
4328
4329     /* It works with head == NULL */
4330     qapi_free_MigrationCapabilityStatusList(head);
4331
4332     return ret;
4333 }
4334
4335 static const TypeInfo migration_type = {
4336     .name = TYPE_MIGRATION,
4337     /*
4338      * NOTE: TYPE_MIGRATION is not really a device, as the object is
4339      * not created using qdev_new(), it is not attached to the qdev
4340      * device tree, and it is never realized.
4341      *
4342      * TODO: Make this TYPE_OBJECT once QOM provides something like
4343      * TYPE_DEVICE's "-global" properties.
4344      */
4345     .parent = TYPE_DEVICE,
4346     .class_init = migration_class_init,
4347     .class_size = sizeof(MigrationClass),
4348     .instance_size = sizeof(MigrationState),
4349     .instance_init = migration_instance_init,
4350     .instance_finalize = migration_instance_finalize,
4351 };
4352
4353 static void register_migration_types(void)
4354 {
4355     type_register_static(&migration_type);
4356 }
4357
4358 type_init(register_migration_types);