migration/migration.c

   1 /*
   2  * QEMU live migration
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *
   6  * Authors:
   7  *  Anthony Liguori   <aliguori@us.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include "qemu/cutils.h"
  18 #include "qemu/error-report.h"
  19 #include "qemu/main-loop.h"
  20 #include "migration/blocker.h"
  21 #include "exec.h"
  22 #include "fd.h"
  23 #include "socket.h"
  24 #include "sysemu/runstate.h"
  25 #include "sysemu/sysemu.h"
  26 #include "sysemu/cpu-throttle.h"
  27 #include "rdma.h"
  28 #include "ram.h"
  29 #include "migration/global_state.h"
  30 #include "migration/misc.h"
  31 #include "migration.h"
  32 #include "savevm.h"
  33 #include "qemu-file-channel.h"
  34 #include "qemu-file.h"
  35 #include "migration/vmstate.h"
  36 #include "block/block.h"
  37 #include "qapi/error.h"
  38 #include "qapi/clone-visitor.h"
  39 #include "qapi/qapi-visit-migration.h"
  40 #include "qapi/qapi-visit-sockets.h"
  41 #include "qapi/qapi-commands-migration.h"
  42 #include "qapi/qapi-events-migration.h"
  43 #include "qapi/qmp/qerror.h"
  44 #include "qapi/qmp/qnull.h"
  45 #include "qemu/rcu.h"
  46 #include "block.h"
  47 #include "postcopy-ram.h"
  48 #include "qemu/thread.h"
  49 #include "trace.h"
  50 #include "exec/target_page.h"
  51 #include "io/channel-buffer.h"
  52 #include "migration/colo.h"
  53 #include "hw/boards.h"
  54 #include "hw/qdev-properties.h"
  55 #include "hw/qdev-properties-system.h"
  56 #include "monitor/monitor.h"
  57 #include "net/announce.h"
  58 #include "qemu/queue.h"
  59 #include "multifd.h"
  60 #include "qemu/yank.h"
  61 #include "sysemu/cpus.h"
  62
  63 #ifdef CONFIG_VFIO
  64 #include "hw/vfio/vfio-common.h"
  65 #endif
  66
  67 #define MAX_THROTTLE  (128 << 20)      /* Migration transfer speed throttling */
  68
  69 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
  70  * data. */
  71 #define BUFFER_DELAY     100
  72 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
  73
  74 /* Time in milliseconds we are allowed to stop the source,
  75  * for sending the last part */
  76 #define DEFAULT_MIGRATE_SET_DOWNTIME 300
  77
  78 /* Maximum migrate downtime set to 2000 seconds */
  79 #define MAX_MIGRATE_DOWNTIME_SECONDS 2000
  80 #define MAX_MIGRATE_DOWNTIME (MAX_MIGRATE_DOWNTIME_SECONDS * 1000)
  81
  82 /* Default compression thread count */
  83 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8
  84 /* Default decompression thread count, usually decompression is at
  85  * least 4 times as fast as compression.*/
  86 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2
  87 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */
  88 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1
  89 /* Define default autoconverge cpu throttle migration parameters */
  90 #define DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD 50
  91 #define DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL 20
  92 #define DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT 10
  93 #define DEFAULT_MIGRATE_MAX_CPU_THROTTLE 99
  94
  95 /* Migration XBZRLE default cache size */
  96 #define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024)
  97
  98 /* The delay time (in ms) between two COLO checkpoints */
  99 #define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100)
 100 #define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2
 101 #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE
 102 /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */
 103 #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1
 104 /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */
 105 #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1
 106
 107 /* Background transfer rate for postcopy, 0 means unlimited, note
 108  * that page requests can still exceed this limit.
 109  */
 110 #define DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH 0
 111
 112 /*
 113  * Parameters for self_announce_delay giving a stream of RARP/ARP
 114  * packets after migration.
 115  */
 116 #define DEFAULT_MIGRATE_ANNOUNCE_INITIAL  50
 117 #define DEFAULT_MIGRATE_ANNOUNCE_MAX     550
 118 #define DEFAULT_MIGRATE_ANNOUNCE_ROUNDS    5
 119 #define DEFAULT_MIGRATE_ANNOUNCE_STEP    100
 120
 121 static NotifierList migration_state_notifiers =
 122     NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
 123
 124 /* Messages sent on the return path from destination to source */
 125 enum mig_rp_message_type {
 126     MIG_RP_MSG_INVALID = 0,  /* Must be 0 */
 127     MIG_RP_MSG_SHUT,         /* sibling will not send any more RP messages */
 128     MIG_RP_MSG_PONG,         /* Response to a PING; data (seq: be32 ) */
 129
 130     MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
 131     MIG_RP_MSG_REQ_PAGES,    /* data (start: be64, len: be32) */
 132     MIG_RP_MSG_RECV_BITMAP,  /* send recved_bitmap back to source */
 133     MIG_RP_MSG_RESUME_ACK,   /* tell source that we are ready to resume */
 134
 135     MIG_RP_MSG_MAX
 136 };
 137
 138 /* Migration capabilities set */
 139 struct MigrateCapsSet {
 140     int size;                       /* Capability set size */
 141     MigrationCapability caps[];     /* Variadic array of capabilities */
 142 };
 143 typedef struct MigrateCapsSet MigrateCapsSet;
 144
 145 /* Define and initialize MigrateCapsSet */
 146 #define INITIALIZE_MIGRATE_CAPS_SET(_name, ...)   \
 147     MigrateCapsSet _name = {    \
 148         .size = sizeof((int []) { __VA_ARGS__ }) / sizeof(int), \
 149         .caps = { __VA_ARGS__ } \
 150     }
 151
 152 /* Background-snapshot compatibility check list */
 153 static const
 154 INITIALIZE_MIGRATE_CAPS_SET(check_caps_background_snapshot,
 155     MIGRATION_CAPABILITY_POSTCOPY_RAM,
 156     MIGRATION_CAPABILITY_DIRTY_BITMAPS,
 157     MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME,
 158     MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE,
 159     MIGRATION_CAPABILITY_RETURN_PATH,
 160     MIGRATION_CAPABILITY_MULTIFD,
 161     MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER,
 162     MIGRATION_CAPABILITY_AUTO_CONVERGE,
 163     MIGRATION_CAPABILITY_RELEASE_RAM,
 164     MIGRATION_CAPABILITY_RDMA_PIN_ALL,
 165     MIGRATION_CAPABILITY_COMPRESS,
 166     MIGRATION_CAPABILITY_XBZRLE,
 167     MIGRATION_CAPABILITY_X_COLO,
 168     MIGRATION_CAPABILITY_VALIDATE_UUID);
 169
 170 /* When we add fault tolerance, we could have several
 171    migrations at once.  For now we don't need to add
 172    dynamic creation of migration */
 173
 174 static MigrationState *current_migration;
 175 static MigrationIncomingState *current_incoming;
 176
 177 static GSList *migration_blockers;
 178
 179 static bool migration_object_check(MigrationState *ms, Error **errp);
 180 static int migration_maybe_pause(MigrationState *s,
 181                                  int *current_active_state,
 182                                  int new_state);
 183 static void migrate_fd_cancel(MigrationState *s);
 184
 185 static gint page_request_addr_cmp(gconstpointer ap, gconstpointer bp)
 186 {
 187     uintptr_t a = (uintptr_t) ap, b = (uintptr_t) bp;
 188
 189     return (a > b) - (a < b);
 190 }
 191
 192 void migration_object_init(void)
 193 {
 194     Error *err = NULL;
 195
 196     /* This can only be called once. */
 197     assert(!current_migration);
 198     current_migration = MIGRATION_OBJ(object_new(TYPE_MIGRATION));
 199
 200     /*
 201      * Init the migrate incoming object as well no matter whether
 202      * we'll use it or not.
 203      */
 204     assert(!current_incoming);
 205     current_incoming = g_new0(MigrationIncomingState, 1);
 206     current_incoming->state = MIGRATION_STATUS_NONE;
 207     current_incoming->postcopy_remote_fds =
 208         g_array_new(FALSE, TRUE, sizeof(struct PostCopyFD));
 209     qemu_mutex_init(&current_incoming->rp_mutex);
 210     qemu_event_init(&current_incoming->main_thread_load_event, false);
 211     qemu_sem_init(&current_incoming->postcopy_pause_sem_dst, 0);
 212     qemu_sem_init(&current_incoming->postcopy_pause_sem_fault, 0);
 213     qemu_mutex_init(&current_incoming->page_request_mutex);
 214     current_incoming->page_requested = g_tree_new(page_request_addr_cmp);
 215
 216     if (!migration_object_check(current_migration, &err)) {
 217         error_report_err(err);
 218         exit(1);
 219     }
 220
 221     blk_mig_init();
 222     ram_mig_init();
 223     dirty_bitmap_mig_init();
 224 }
 225
 226 void migration_shutdown(void)
 227 {
 228     /*
 229      * Cancel the current migration - that will (eventually)
 230      * stop the migration using this structure
 231      */
 232     migrate_fd_cancel(current_migration);
 233     object_unref(OBJECT(current_migration));
 234
 235     /*
 236      * Cancel outgoing migration of dirty bitmaps. It should
 237      * at least unref used block nodes.
 238      */
 239     dirty_bitmap_mig_cancel_outgoing();
 240
 241     /*
 242      * Cancel incoming migration of dirty bitmaps. Dirty bitmaps
 243      * are non-critical data, and their loss never considered as
 244      * something serious.
 245      */
 246     dirty_bitmap_mig_cancel_incoming();
 247 }
 248
 249 /* For outgoing */
 250 MigrationState *migrate_get_current(void)
 251 {
 252     /* This can only be called after the object created. */
 253     assert(current_migration);
 254     return current_migration;
 255 }
 256
 257 MigrationIncomingState *migration_incoming_get_current(void)
 258 {
 259     assert(current_incoming);
 260     return current_incoming;
 261 }
 262
 263 void migration_incoming_state_destroy(void)
 264 {
 265     struct MigrationIncomingState *mis = migration_incoming_get_current();
 266
 267     if (mis->to_src_file) {
 268         /* Tell source that we are done */
 269         migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
 270         qemu_fclose(mis->to_src_file);
 271         mis->to_src_file = NULL;
 272     }
 273
 274     if (mis->from_src_file) {
 275         qemu_fclose(mis->from_src_file);
 276         mis->from_src_file = NULL;
 277     }
 278     if (mis->postcopy_remote_fds) {
 279         g_array_free(mis->postcopy_remote_fds, TRUE);
 280         mis->postcopy_remote_fds = NULL;
 281     }
 282
 283     qemu_event_reset(&mis->main_thread_load_event);
 284
 285     if (mis->page_requested) {
 286         g_tree_destroy(mis->page_requested);
 287         mis->page_requested = NULL;
 288     }
 289
 290     if (mis->socket_address_list) {
 291         qapi_free_SocketAddressList(mis->socket_address_list);
 292         mis->socket_address_list = NULL;
 293     }
 294
 295     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
 296 }
 297
 298 static void migrate_generate_event(int new_state)
 299 {
 300     if (migrate_use_events()) {
 301         qapi_event_send_migration(new_state);
 302     }
 303 }
 304
 305 static bool migrate_late_block_activate(void)
 306 {
 307     MigrationState *s;
 308
 309     s = migrate_get_current();
 310
 311     return s->enabled_capabilities[
 312         MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE];
 313 }
 314
 315 /*
 316  * Send a message on the return channel back to the source
 317  * of the migration.
 318  */
 319 static int migrate_send_rp_message(MigrationIncomingState *mis,
 320                                    enum mig_rp_message_type message_type,
 321                                    uint16_t len, void *data)
 322 {
 323     int ret = 0;
 324
 325     trace_migrate_send_rp_message((int)message_type, len);
 326     qemu_mutex_lock(&mis->rp_mutex);
 327
 328     /*
 329      * It's possible that the file handle got lost due to network
 330      * failures.
 331      */
 332     if (!mis->to_src_file) {
 333         ret = -EIO;
 334         goto error;
 335     }
 336
 337     qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
 338     qemu_put_be16(mis->to_src_file, len);
 339     qemu_put_buffer(mis->to_src_file, data, len);
 340     qemu_fflush(mis->to_src_file);
 341
 342     /* It's possible that qemu file got error during sending */
 343     ret = qemu_file_get_error(mis->to_src_file);
 344
 345 error:
 346     qemu_mutex_unlock(&mis->rp_mutex);
 347     return ret;
 348 }
 349
 350 /* Request one page from the source VM at the given start address.
 351  *   rb: the RAMBlock to request the page in
 352  *   Start: Address offset within the RB
 353  *   Len: Length in bytes required - must be a multiple of pagesize
 354  */
 355 int migrate_send_rp_message_req_pages(MigrationIncomingState *mis,
 356                                       RAMBlock *rb, ram_addr_t start)
 357 {
 358     uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */
 359     size_t msglen = 12; /* start + len */
 360     size_t len = qemu_ram_pagesize(rb);
 361     enum mig_rp_message_type msg_type;
 362     const char *rbname;
 363     int rbname_len;
 364
 365     *(uint64_t *)bufc = cpu_to_be64((uint64_t)start);
 366     *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len);
 367
 368     /*
 369      * We maintain the last ramblock that we requested for page.  Note that we
 370      * don't need locking because this function will only be called within the
 371      * postcopy ram fault thread.
 372      */
 373     if (rb != mis->last_rb) {
 374         mis->last_rb = rb;
 375
 376         rbname = qemu_ram_get_idstr(rb);
 377         rbname_len = strlen(rbname);
 378
 379         assert(rbname_len < 256);
 380
 381         bufc[msglen++] = rbname_len;
 382         memcpy(bufc + msglen, rbname, rbname_len);
 383         msglen += rbname_len;
 384         msg_type = MIG_RP_MSG_REQ_PAGES_ID;
 385     } else {
 386         msg_type = MIG_RP_MSG_REQ_PAGES;
 387     }
 388
 389     return migrate_send_rp_message(mis, msg_type, msglen, bufc);
 390 }
 391
 392 int migrate_send_rp_req_pages(MigrationIncomingState *mis,
 393                               RAMBlock *rb, ram_addr_t start, uint64_t haddr)
 394 {
 395     void *aligned = (void *)(uintptr_t)(haddr & (-qemu_ram_pagesize(rb)));
 396     bool received = false;
 397
 398     WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) {
 399         received = ramblock_recv_bitmap_test_byte_offset(rb, start);
 400         if (!received && !g_tree_lookup(mis->page_requested, aligned)) {
 401             /*
 402              * The page has not been received, and it's not yet in the page
 403              * request list.  Queue it.  Set the value of element to 1, so that
 404              * things like g_tree_lookup() will return TRUE (1) when found.
 405              */
 406             g_tree_insert(mis->page_requested, aligned, (gpointer)1);
 407             mis->page_requested_count++;
 408             trace_postcopy_page_req_add(aligned, mis->page_requested_count);
 409         }
 410     }
 411
 412     /*
 413      * If the page is there, skip sending the message.  We don't even need the
 414      * lock because as long as the page arrived, it'll be there forever.
 415      */
 416     if (received) {
 417         return 0;
 418     }
 419
 420     return migrate_send_rp_message_req_pages(mis, rb, start);
 421 }
 422
 423 static bool migration_colo_enabled;
 424 bool migration_incoming_colo_enabled(void)
 425 {
 426     return migration_colo_enabled;
 427 }
 428
 429 void migration_incoming_disable_colo(void)
 430 {
 431     ram_block_discard_disable(false);
 432     migration_colo_enabled = false;
 433 }
 434
 435 int migration_incoming_enable_colo(void)
 436 {
 437     if (ram_block_discard_disable(true)) {
 438         error_report("COLO: cannot disable RAM discard");
 439         return -EBUSY;
 440     }
 441     migration_colo_enabled = true;
 442     return 0;
 443 }
 444
 445 void migrate_add_address(SocketAddress *address)
 446 {
 447     MigrationIncomingState *mis = migration_incoming_get_current();
 448
 449     QAPI_LIST_PREPEND(mis->socket_address_list,
 450                       QAPI_CLONE(SocketAddress, address));
 451 }
 452
 453 static void qemu_start_incoming_migration(const char *uri, Error **errp)
 454 {
 455     const char *p = NULL;
 456
 457     if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
 458         return;
 459     }
 460
 461     qapi_event_send_migration(MIGRATION_STATUS_SETUP);
 462     if (strstart(uri, "tcp:", &p) ||
 463         strstart(uri, "unix:", NULL) ||
 464         strstart(uri, "vsock:", NULL)) {
 465         socket_start_incoming_migration(p ? p : uri, errp);
 466 #ifdef CONFIG_RDMA
 467     } else if (strstart(uri, "rdma:", &p)) {
 468         rdma_start_incoming_migration(p, errp);
 469 #endif
 470     } else if (strstart(uri, "exec:", &p)) {
 471         exec_start_incoming_migration(p, errp);
 472     } else if (strstart(uri, "fd:", &p)) {
 473         fd_start_incoming_migration(p, errp);
 474     } else {
 475         yank_unregister_instance(MIGRATION_YANK_INSTANCE);
 476         error_setg(errp, "unknown migration protocol: %s", uri);
 477     }
 478 }
 479
 480 static void process_incoming_migration_bh(void *opaque)
 481 {
 482     Error *local_err = NULL;
 483     MigrationIncomingState *mis = opaque;
 484
 485     /* If capability late_block_activate is set:
 486      * Only fire up the block code now if we're going to restart the
 487      * VM, else 'cont' will do it.
 488      * This causes file locking to happen; so we don't want it to happen
 489      * unless we really are starting the VM.
 490      */
 491     if (!migrate_late_block_activate() ||
 492          (autostart && (!global_state_received() ||
 493             global_state_get_runstate() == RUN_STATE_RUNNING))) {
 494         /* Make sure all file formats flush their mutable metadata.
 495          * If we get an error here, just don't restart the VM yet. */
 496         bdrv_invalidate_cache_all(&local_err);
 497         if (local_err) {
 498             error_report_err(local_err);
 499             local_err = NULL;
 500             autostart = false;
 501         }
 502     }
 503
 504     /*
 505      * This must happen after all error conditions are dealt with and
 506      * we're sure the VM is going to be running on this host.
 507      */
 508     qemu_announce_self(&mis->announce_timer, migrate_announce_params());
 509
 510     if (multifd_load_cleanup(&local_err) != 0) {
 511         error_report_err(local_err);
 512         autostart = false;
 513     }
 514     /* If global state section was not received or we are in running
 515        state, we need to obey autostart. Any other state is set with
 516        runstate_set. */
 517
 518     dirty_bitmap_mig_before_vm_start();
 519
 520     if (!global_state_received() ||
 521         global_state_get_runstate() == RUN_STATE_RUNNING) {
 522         if (autostart) {
 523             vm_start();
 524         } else {
 525             runstate_set(RUN_STATE_PAUSED);
 526         }
 527     } else if (migration_incoming_colo_enabled()) {
 528         migration_incoming_disable_colo();
 529         vm_start();
 530     } else {
 531         runstate_set(global_state_get_runstate());
 532     }
 533     /*
 534      * This must happen after any state changes since as soon as an external
 535      * observer sees this event they might start to prod at the VM assuming
 536      * it's ready to use.
 537      */
 538     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 539                       MIGRATION_STATUS_COMPLETED);
 540     qemu_bh_delete(mis->bh);
 541     migration_incoming_state_destroy();
 542 }
 543
 544 static void process_incoming_migration_co(void *opaque)
 545 {
 546     MigrationIncomingState *mis = migration_incoming_get_current();
 547     PostcopyState ps;
 548     int ret;
 549     Error *local_err = NULL;
 550
 551     assert(mis->from_src_file);
 552     mis->migration_incoming_co = qemu_coroutine_self();
 553     mis->largest_page_size = qemu_ram_pagesize_largest();
 554     postcopy_state_set(POSTCOPY_INCOMING_NONE);
 555     migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
 556                       MIGRATION_STATUS_ACTIVE);
 557     ret = qemu_loadvm_state(mis->from_src_file);
 558
 559     ps = postcopy_state_get();
 560     trace_process_incoming_migration_co_end(ret, ps);
 561     if (ps != POSTCOPY_INCOMING_NONE) {
 562         if (ps == POSTCOPY_INCOMING_ADVISE) {
 563             /*
 564              * Where a migration had postcopy enabled (and thus went to advise)
 565              * but managed to complete within the precopy period, we can use
 566              * the normal exit.
 567              */
 568             postcopy_ram_incoming_cleanup(mis);
 569         } else if (ret >= 0) {
 570             /*
 571              * Postcopy was started, cleanup should happen at the end of the
 572              * postcopy thread.
 573              */
 574             trace_process_incoming_migration_co_postcopy_end_main();
 575             return;
 576         }
 577         /* Else if something went wrong then just fall out of the normal exit */
 578     }
 579
 580     /* we get COLO info, and know if we are in COLO mode */
 581     if (!ret && migration_incoming_colo_enabled()) {
 582         /* Make sure all file formats flush their mutable metadata */
 583         bdrv_invalidate_cache_all(&local_err);
 584         if (local_err) {
 585             error_report_err(local_err);
 586             goto fail;
 587         }
 588
 589         qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
 590              colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
 591         mis->have_colo_incoming_thread = true;
 592         qemu_coroutine_yield();
 593
 594         /* Wait checkpoint incoming thread exit before free resource */
 595         qemu_thread_join(&mis->colo_incoming_thread);
 596         /* We hold the global iothread lock, so it is safe here */
 597         colo_release_ram_cache();
 598     }
 599
 600     if (ret < 0) {
 601         error_report("load of migration failed: %s", strerror(-ret));
 602         goto fail;
 603     }
 604     mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
 605     qemu_bh_schedule(mis->bh);
 606     mis->migration_incoming_co = NULL;
 607     return;
 608 fail:
 609     local_err = NULL;
 610     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 611                       MIGRATION_STATUS_FAILED);
 612     qemu_fclose(mis->from_src_file);
 613     if (multifd_load_cleanup(&local_err) != 0) {
 614         error_report_err(local_err);
 615     }
 616     exit(EXIT_FAILURE);
 617 }
 618
 619 /**
 620  * @migration_incoming_setup: Setup incoming migration
 621  *
 622  * Returns 0 for no error or 1 for error
 623  *
 624  * @f: file for main migration channel
 625  * @errp: where to put errors
 626  */
 627 static int migration_incoming_setup(QEMUFile *f, Error **errp)
 628 {
 629     MigrationIncomingState *mis = migration_incoming_get_current();
 630     Error *local_err = NULL;
 631
 632     if (multifd_load_setup(&local_err) != 0) {
 633         /* We haven't been able to create multifd threads
 634            nothing better to do */
 635         error_report_err(local_err);
 636         exit(EXIT_FAILURE);
 637     }
 638
 639     if (!mis->from_src_file) {
 640         mis->from_src_file = f;
 641     }
 642     qemu_file_set_blocking(f, false);
 643     return 0;
 644 }
 645
 646 void migration_incoming_process(void)
 647 {
 648     Coroutine *co = qemu_coroutine_create(process_incoming_migration_co, NULL);
 649     qemu_coroutine_enter(co);
 650 }
 651
 652 /* Returns true if recovered from a paused migration, otherwise false */
 653 static bool postcopy_try_recover(QEMUFile *f)
 654 {
 655     MigrationIncomingState *mis = migration_incoming_get_current();
 656
 657     if (mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
 658         /* Resumed from a paused postcopy migration */
 659
 660         mis->from_src_file = f;
 661         /* Postcopy has standalone thread to do vm load */
 662         qemu_file_set_blocking(f, true);
 663
 664         /* Re-configure the return path */
 665         mis->to_src_file = qemu_file_get_return_path(f);
 666
 667         migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
 668                           MIGRATION_STATUS_POSTCOPY_RECOVER);
 669
 670         /*
 671          * Here, we only wake up the main loading thread (while the
 672          * fault thread will still be waiting), so that we can receive
 673          * commands from source now, and answer it if needed. The
 674          * fault thread will be woken up afterwards until we are sure
 675          * that source is ready to reply to page requests.
 676          */
 677         qemu_sem_post(&mis->postcopy_pause_sem_dst);
 678         return true;
 679     }
 680
 681     return false;
 682 }
 683
 684 void migration_fd_process_incoming(QEMUFile *f, Error **errp)
 685 {
 686     Error *local_err = NULL;
 687
 688     if (postcopy_try_recover(f)) {
 689         return;
 690     }
 691
 692     if (migration_incoming_setup(f, &local_err)) {
 693         error_propagate(errp, local_err);
 694         return;
 695     }
 696     migration_incoming_process();
 697 }
 698
 699 void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp)
 700 {
 701     MigrationIncomingState *mis = migration_incoming_get_current();
 702     Error *local_err = NULL;
 703     bool start_migration;
 704
 705     if (!mis->from_src_file) {
 706         /* The first connection (multifd may have multiple) */
 707         QEMUFile *f = qemu_fopen_channel_input(ioc);
 708
 709         /* If it's a recovery, we're done */
 710         if (postcopy_try_recover(f)) {
 711             return;
 712         }
 713
 714         if (migration_incoming_setup(f, &local_err)) {
 715             error_propagate(errp, local_err);
 716             return;
 717         }
 718
 719         /*
 720          * Common migration only needs one channel, so we can start
 721          * right now.  Multifd needs more than one channel, we wait.
 722          */
 723         start_migration = !migrate_use_multifd();
 724     } else {
 725         /* Multiple connections */
 726         assert(migrate_use_multifd());
 727         start_migration = multifd_recv_new_channel(ioc, &local_err);
 728         if (local_err) {
 729             error_propagate(errp, local_err);
 730             return;
 731         }
 732     }
 733
 734     if (start_migration) {
 735         migration_incoming_process();
 736     }
 737 }
 738
 739 /**
 740  * @migration_has_all_channels: We have received all channels that we need
 741  *
 742  * Returns true when we have got connections to all the channels that
 743  * we need for migration.
 744  */
 745 bool migration_has_all_channels(void)
 746 {
 747     MigrationIncomingState *mis = migration_incoming_get_current();
 748     bool all_channels;
 749
 750     all_channels = multifd_recv_all_channels_created();
 751
 752     return all_channels && mis->from_src_file != NULL;
 753 }
 754
 755 /*
 756  * Send a 'SHUT' message on the return channel with the given value
 757  * to indicate that we've finished with the RP.  Non-0 value indicates
 758  * error.
 759  */
 760 void migrate_send_rp_shut(MigrationIncomingState *mis,
 761                           uint32_t value)
 762 {
 763     uint32_t buf;
 764
 765     buf = cpu_to_be32(value);
 766     migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
 767 }
 768
 769 /*
 770  * Send a 'PONG' message on the return channel with the given value
 771  * (normally in response to a 'PING')
 772  */
 773 void migrate_send_rp_pong(MigrationIncomingState *mis,
 774                           uint32_t value)
 775 {
 776     uint32_t buf;
 777
 778     buf = cpu_to_be32(value);
 779     migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
 780 }
 781
 782 void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
 783                                  char *block_name)
 784 {
 785     char buf[512];
 786     int len;
 787     int64_t res;
 788
 789     /*
 790      * First, we send the header part. It contains only the len of
 791      * idstr, and the idstr itself.
 792      */
 793     len = strlen(block_name);
 794     buf[0] = len;
 795     memcpy(buf + 1, block_name, len);
 796
 797     if (mis->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
 798         error_report("%s: MSG_RP_RECV_BITMAP only used for recovery",
 799                      __func__);
 800         return;
 801     }
 802
 803     migrate_send_rp_message(mis, MIG_RP_MSG_RECV_BITMAP, len + 1, buf);
 804
 805     /*
 806      * Next, we dump the received bitmap to the stream.
 807      *
 808      * TODO: currently we are safe since we are the only one that is
 809      * using the to_src_file handle (fault thread is still paused),
 810      * and it's ok even not taking the mutex. However the best way is
 811      * to take the lock before sending the message header, and release
 812      * the lock after sending the bitmap.
 813      */
 814     qemu_mutex_lock(&mis->rp_mutex);
 815     res = ramblock_recv_bitmap_send(mis->to_src_file, block_name);
 816     qemu_mutex_unlock(&mis->rp_mutex);
 817
 818     trace_migrate_send_rp_recv_bitmap(block_name, res);
 819 }
 820
 821 void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value)
 822 {
 823     uint32_t buf;
 824
 825     buf = cpu_to_be32(value);
 826     migrate_send_rp_message(mis, MIG_RP_MSG_RESUME_ACK, sizeof(buf), &buf);
 827 }
 828
 829 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
 830 {
 831     MigrationCapabilityStatusList *head = NULL, **tail = &head;
 832     MigrationCapabilityStatus *caps;
 833     MigrationState *s = migrate_get_current();
 834     int i;
 835
 836     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
 837 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
 838         if (i == MIGRATION_CAPABILITY_BLOCK) {
 839             continue;
 840         }
 841 #endif
 842         caps = g_malloc0(sizeof(*caps));
 843         caps->capability = i;
 844         caps->state = s->enabled_capabilities[i];
 845         QAPI_LIST_APPEND(tail, caps);
 846     }
 847
 848     return head;
 849 }
 850
 851 MigrationParameters *qmp_query_migrate_parameters(Error **errp)
 852 {
 853     MigrationParameters *params;
 854     MigrationState *s = migrate_get_current();
 855
 856     /* TODO use QAPI_CLONE() instead of duplicating it inline */
 857     params = g_malloc0(sizeof(*params));
 858     params->has_compress_level = true;
 859     params->compress_level = s->parameters.compress_level;
 860     params->has_compress_threads = true;
 861     params->compress_threads = s->parameters.compress_threads;
 862     params->has_compress_wait_thread = true;
 863     params->compress_wait_thread = s->parameters.compress_wait_thread;
 864     params->has_decompress_threads = true;
 865     params->decompress_threads = s->parameters.decompress_threads;
 866     params->has_throttle_trigger_threshold = true;
 867     params->throttle_trigger_threshold = s->parameters.throttle_trigger_threshold;
 868     params->has_cpu_throttle_initial = true;
 869     params->cpu_throttle_initial = s->parameters.cpu_throttle_initial;
 870     params->has_cpu_throttle_increment = true;
 871     params->cpu_throttle_increment = s->parameters.cpu_throttle_increment;
 872     params->has_cpu_throttle_tailslow = true;
 873     params->cpu_throttle_tailslow = s->parameters.cpu_throttle_tailslow;
 874     params->has_tls_creds = true;
 875     params->tls_creds = g_strdup(s->parameters.tls_creds);
 876     params->has_tls_hostname = true;
 877     params->tls_hostname = g_strdup(s->parameters.tls_hostname);
 878     params->has_tls_authz = true;
 879     params->tls_authz = g_strdup(s->parameters.tls_authz ?
 880                                  s->parameters.tls_authz : "");
 881     params->has_max_bandwidth = true;
 882     params->max_bandwidth = s->parameters.max_bandwidth;
 883     params->has_downtime_limit = true;
 884     params->downtime_limit = s->parameters.downtime_limit;
 885     params->has_x_checkpoint_delay = true;
 886     params->x_checkpoint_delay = s->parameters.x_checkpoint_delay;
 887     params->has_block_incremental = true;
 888     params->block_incremental = s->parameters.block_incremental;
 889     params->has_multifd_channels = true;
 890     params->multifd_channels = s->parameters.multifd_channels;
 891     params->has_multifd_compression = true;
 892     params->multifd_compression = s->parameters.multifd_compression;
 893     params->has_multifd_zlib_level = true;
 894     params->multifd_zlib_level = s->parameters.multifd_zlib_level;
 895     params->has_multifd_zstd_level = true;
 896     params->multifd_zstd_level = s->parameters.multifd_zstd_level;
 897     params->has_xbzrle_cache_size = true;
 898     params->xbzrle_cache_size = s->parameters.xbzrle_cache_size;
 899     params->has_max_postcopy_bandwidth = true;
 900     params->max_postcopy_bandwidth = s->parameters.max_postcopy_bandwidth;
 901     params->has_max_cpu_throttle = true;
 902     params->max_cpu_throttle = s->parameters.max_cpu_throttle;
 903     params->has_announce_initial = true;
 904     params->announce_initial = s->parameters.announce_initial;
 905     params->has_announce_max = true;
 906     params->announce_max = s->parameters.announce_max;
 907     params->has_announce_rounds = true;
 908     params->announce_rounds = s->parameters.announce_rounds;
 909     params->has_announce_step = true;
 910     params->announce_step = s->parameters.announce_step;
 911
 912     if (s->parameters.has_block_bitmap_mapping) {
 913         params->has_block_bitmap_mapping = true;
 914         params->block_bitmap_mapping =
 915             QAPI_CLONE(BitmapMigrationNodeAliasList,
 916                        s->parameters.block_bitmap_mapping);
 917     }
 918
 919     return params;
 920 }
 921
 922 AnnounceParameters *migrate_announce_params(void)
 923 {
 924     static AnnounceParameters ap;
 925
 926     MigrationState *s = migrate_get_current();
 927
 928     ap.initial = s->parameters.announce_initial;
 929     ap.max = s->parameters.announce_max;
 930     ap.rounds = s->parameters.announce_rounds;
 931     ap.step = s->parameters.announce_step;
 932
 933     return &ap;
 934 }
 935
 936 /*
 937  * Return true if we're already in the middle of a migration
 938  * (i.e. any of the active or setup states)
 939  */
 940 bool migration_is_setup_or_active(int state)
 941 {
 942     switch (state) {
 943     case MIGRATION_STATUS_ACTIVE:
 944     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 945     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 946     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 947     case MIGRATION_STATUS_SETUP:
 948     case MIGRATION_STATUS_PRE_SWITCHOVER:
 949     case MIGRATION_STATUS_DEVICE:
 950     case MIGRATION_STATUS_WAIT_UNPLUG:
 951     case MIGRATION_STATUS_COLO:
 952         return true;
 953
 954     default:
 955         return false;
 956
 957     }
 958 }
 959
 960 bool migration_is_running(int state)
 961 {
 962     switch (state) {
 963     case MIGRATION_STATUS_ACTIVE:
 964     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 965     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 966     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 967     case MIGRATION_STATUS_SETUP:
 968     case MIGRATION_STATUS_PRE_SWITCHOVER:
 969     case MIGRATION_STATUS_DEVICE:
 970     case MIGRATION_STATUS_WAIT_UNPLUG:
 971     case MIGRATION_STATUS_CANCELLING:
 972         return true;
 973
 974     default:
 975         return false;
 976
 977     }
 978 }
 979
 980 static void populate_time_info(MigrationInfo *info, MigrationState *s)
 981 {
 982     info->has_status = true;
 983     info->has_setup_time = true;
 984     info->setup_time = s->setup_time;
 985     if (s->state == MIGRATION_STATUS_COMPLETED) {
 986         info->has_total_time = true;
 987         info->total_time = s->total_time;
 988         info->has_downtime = true;
 989         info->downtime = s->downtime;
 990     } else {
 991         info->has_total_time = true;
 992         info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) -
 993                            s->start_time;
 994         info->has_expected_downtime = true;
 995         info->expected_downtime = s->expected_downtime;
 996     }
 997 }
 998
 999 static void populate_ram_info(MigrationInfo *info, MigrationState *s)
1000 {
1001     info->has_ram = true;
1002     info->ram = g_malloc0(sizeof(*info->ram));
1003     info->ram->transferred = ram_counters.transferred;
1004     info->ram->total = ram_bytes_total();
1005     info->ram->duplicate = ram_counters.duplicate;
1006     /* legacy value.  It is not used anymore */
1007     info->ram->skipped = 0;
1008     info->ram->normal = ram_counters.normal;
1009     info->ram->normal_bytes = ram_counters.normal *
1010         qemu_target_page_size();
1011     info->ram->mbps = s->mbps;
1012     info->ram->dirty_sync_count = ram_counters.dirty_sync_count;
1013     info->ram->postcopy_requests = ram_counters.postcopy_requests;
1014     info->ram->page_size = qemu_target_page_size();
1015     info->ram->multifd_bytes = ram_counters.multifd_bytes;
1016     info->ram->pages_per_second = s->pages_per_second;
1017
1018     if (migrate_use_xbzrle()) {
1019         info->has_xbzrle_cache = true;
1020         info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
1021         info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
1022         info->xbzrle_cache->bytes = xbzrle_counters.bytes;
1023         info->xbzrle_cache->pages = xbzrle_counters.pages;
1024         info->xbzrle_cache->cache_miss = xbzrle_counters.cache_miss;
1025         info->xbzrle_cache->cache_miss_rate = xbzrle_counters.cache_miss_rate;
1026         info->xbzrle_cache->encoding_rate = xbzrle_counters.encoding_rate;
1027         info->xbzrle_cache->overflow = xbzrle_counters.overflow;
1028     }
1029
1030     if (migrate_use_compression()) {
1031         info->has_compression = true;
1032         info->compression = g_malloc0(sizeof(*info->compression));
1033         info->compression->pages = compression_counters.pages;
1034         info->compression->busy = compression_counters.busy;
1035         info->compression->busy_rate = compression_counters.busy_rate;
1036         info->compression->compressed_size =
1037                                     compression_counters.compressed_size;
1038         info->compression->compression_rate =
1039                                     compression_counters.compression_rate;
1040     }
1041
1042     if (cpu_throttle_active()) {
1043         info->has_cpu_throttle_percentage = true;
1044         info->cpu_throttle_percentage = cpu_throttle_get_percentage();
1045     }
1046
1047     if (s->state != MIGRATION_STATUS_COMPLETED) {
1048         info->ram->remaining = ram_bytes_remaining();
1049         info->ram->dirty_pages_rate = ram_counters.dirty_pages_rate;
1050     }
1051 }
1052
1053 static void populate_disk_info(MigrationInfo *info)
1054 {
1055     if (blk_mig_active()) {
1056         info->has_disk = true;
1057         info->disk = g_malloc0(sizeof(*info->disk));
1058         info->disk->transferred = blk_mig_bytes_transferred();
1059         info->disk->remaining = blk_mig_bytes_remaining();
1060         info->disk->total = blk_mig_bytes_total();
1061     }
1062 }
1063
1064 static void populate_vfio_info(MigrationInfo *info)
1065 {
1066 #ifdef CONFIG_VFIO
1067     if (vfio_mig_active()) {
1068         info->has_vfio = true;
1069         info->vfio = g_malloc0(sizeof(*info->vfio));
1070         info->vfio->transferred = vfio_mig_bytes_transferred();
1071     }
1072 #endif
1073 }
1074
1075 static void fill_source_migration_info(MigrationInfo *info)
1076 {
1077     MigrationState *s = migrate_get_current();
1078
1079     info->blocked = migration_is_blocked(NULL);
1080     info->has_blocked_reasons = info->blocked;
1081     info->blocked_reasons = NULL;
1082     if (info->blocked) {
1083         GSList *cur_blocker = migration_blockers;
1084
1085         /*
1086          * There are two types of reasons a migration might be blocked;
1087          * a) devices marked in VMState as non-migratable, and
1088          * b) Explicit migration blockers
1089          * We need to add both of them here.
1090          */
1091         qemu_savevm_non_migratable_list(&info->blocked_reasons);
1092
1093         while (cur_blocker) {
1094             QAPI_LIST_PREPEND(info->blocked_reasons,
1095                               g_strdup(error_get_pretty(cur_blocker->data)));
1096             cur_blocker = g_slist_next(cur_blocker);
1097         }
1098     }
1099
1100     switch (s->state) {
1101     case MIGRATION_STATUS_NONE:
1102         /* no migration has happened ever */
1103         /* do not overwrite destination migration status */
1104         return;
1105     case MIGRATION_STATUS_SETUP:
1106         info->has_status = true;
1107         info->has_total_time = false;
1108         break;
1109     case MIGRATION_STATUS_ACTIVE:
1110     case MIGRATION_STATUS_CANCELLING:
1111     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1112     case MIGRATION_STATUS_PRE_SWITCHOVER:
1113     case MIGRATION_STATUS_DEVICE:
1114     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1115     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1116         /* TODO add some postcopy stats */
1117         populate_time_info(info, s);
1118         populate_ram_info(info, s);
1119         populate_disk_info(info);
1120         populate_vfio_info(info);
1121         break;
1122     case MIGRATION_STATUS_COLO:
1123         info->has_status = true;
1124         /* TODO: display COLO specific information (checkpoint info etc.) */
1125         break;
1126     case MIGRATION_STATUS_COMPLETED:
1127         populate_time_info(info, s);
1128         populate_ram_info(info, s);
1129         populate_vfio_info(info);
1130         break;
1131     case MIGRATION_STATUS_FAILED:
1132         info->has_status = true;
1133         if (s->error) {
1134             info->has_error_desc = true;
1135             info->error_desc = g_strdup(error_get_pretty(s->error));
1136         }
1137         break;
1138     case MIGRATION_STATUS_CANCELLED:
1139         info->has_status = true;
1140         break;
1141     case MIGRATION_STATUS_WAIT_UNPLUG:
1142         info->has_status = true;
1143         break;
1144     }
1145     info->status = s->state;
1146 }
1147
1148 typedef enum WriteTrackingSupport {
1149     WT_SUPPORT_UNKNOWN = 0,
1150     WT_SUPPORT_ABSENT,
1151     WT_SUPPORT_AVAILABLE,
1152     WT_SUPPORT_COMPATIBLE
1153 } WriteTrackingSupport;
1154
1155 static
1156 WriteTrackingSupport migrate_query_write_tracking(void)
1157 {
1158     /* Check if kernel supports required UFFD features */
1159     if (!ram_write_tracking_available()) {
1160         return WT_SUPPORT_ABSENT;
1161     }
1162     /*
1163      * Check if current memory configuration is
1164      * compatible with required UFFD features.
1165      */
1166     if (!ram_write_tracking_compatible()) {
1167         return WT_SUPPORT_AVAILABLE;
1168     }
1169
1170     return WT_SUPPORT_COMPATIBLE;
1171 }
1172
1173 /**
1174  * @migration_caps_check - check capability validity
1175  *
1176  * @cap_list: old capability list, array of bool
1177  * @params: new capabilities to be applied soon
1178  * @errp: set *errp if the check failed, with reason
1179  *
1180  * Returns true if check passed, otherwise false.
1181  */
1182 static bool migrate_caps_check(bool *cap_list,
1183                                MigrationCapabilityStatusList *params,
1184                                Error **errp)
1185 {
1186     MigrationCapabilityStatusList *cap;
1187     bool old_postcopy_cap;
1188     MigrationIncomingState *mis = migration_incoming_get_current();
1189
1190     old_postcopy_cap = cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM];
1191
1192     for (cap = params; cap; cap = cap->next) {
1193         cap_list[cap->value->capability] = cap->value->state;
1194     }
1195
1196 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
1197     if (cap_list[MIGRATION_CAPABILITY_BLOCK]) {
1198         error_setg(errp, "QEMU compiled without old-style (blk/-b, inc/-i) "
1199                    "block migration");
1200         error_append_hint(errp, "Use drive_mirror+NBD instead.\n");
1201         return false;
1202     }
1203 #endif
1204
1205 #ifndef CONFIG_REPLICATION
1206     if (cap_list[MIGRATION_CAPABILITY_X_COLO]) {
1207         error_setg(errp, "QEMU compiled without replication module"
1208                    " can't enable COLO");
1209         error_append_hint(errp, "Please enable replication before COLO.\n");
1210         return false;
1211     }
1212 #endif
1213
1214     if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
1215         /* This check is reasonably expensive, so only when it's being
1216          * set the first time, also it's only the destination that needs
1217          * special support.
1218          */
1219         if (!old_postcopy_cap && runstate_check(RUN_STATE_INMIGRATE) &&
1220             !postcopy_ram_supported_by_host(mis)) {
1221             /* postcopy_ram_supported_by_host will have emitted a more
1222              * detailed message
1223              */
1224             error_setg(errp, "Postcopy is not supported");
1225             return false;
1226         }
1227
1228         if (cap_list[MIGRATION_CAPABILITY_X_IGNORE_SHARED]) {
1229             error_setg(errp, "Postcopy is not compatible with ignore-shared");
1230             return false;
1231         }
1232     }
1233
1234     if (cap_list[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT]) {
1235         WriteTrackingSupport wt_support;
1236         int idx;
1237         /*
1238          * Check if 'background-snapshot' capability is supported by
1239          * host kernel and compatible with guest memory configuration.
1240          */
1241         wt_support = migrate_query_write_tracking();
1242         if (wt_support < WT_SUPPORT_AVAILABLE) {
1243             error_setg(errp, "Background-snapshot is not supported by host kernel");
1244             return false;
1245         }
1246         if (wt_support < WT_SUPPORT_COMPATIBLE) {
1247             error_setg(errp, "Background-snapshot is not compatible "
1248                     "with guest memory configuration");
1249             return false;
1250         }
1251
1252         /*
1253          * Check if there are any migration capabilities
1254          * incompatible with 'background-snapshot'.
1255          */
1256         for (idx = 0; idx < check_caps_background_snapshot.size; idx++) {
1257             int incomp_cap = check_caps_background_snapshot.caps[idx];
1258             if (cap_list[incomp_cap]) {
1259                 error_setg(errp,
1260                         "Background-snapshot is not compatible with %s",
1261                         MigrationCapability_str(incomp_cap));
1262                 return false;
1263             }
1264         }
1265     }
1266
1267     return true;
1268 }
1269
1270 static void fill_destination_migration_info(MigrationInfo *info)
1271 {
1272     MigrationIncomingState *mis = migration_incoming_get_current();
1273
1274     if (mis->socket_address_list) {
1275         info->has_socket_address = true;
1276         info->socket_address =
1277             QAPI_CLONE(SocketAddressList, mis->socket_address_list);
1278     }
1279
1280     switch (mis->state) {
1281     case MIGRATION_STATUS_NONE:
1282         return;
1283     case MIGRATION_STATUS_SETUP:
1284     case MIGRATION_STATUS_CANCELLING:
1285     case MIGRATION_STATUS_CANCELLED:
1286     case MIGRATION_STATUS_ACTIVE:
1287     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1288     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1289     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1290     case MIGRATION_STATUS_FAILED:
1291     case MIGRATION_STATUS_COLO:
1292         info->has_status = true;
1293         break;
1294     case MIGRATION_STATUS_COMPLETED:
1295         info->has_status = true;
1296         fill_destination_postcopy_migration_info(info);
1297         break;
1298     }
1299     info->status = mis->state;
1300 }
1301
1302 MigrationInfo *qmp_query_migrate(Error **errp)
1303 {
1304     MigrationInfo *info = g_malloc0(sizeof(*info));
1305
1306     fill_destination_migration_info(info);
1307     fill_source_migration_info(info);
1308
1309     return info;
1310 }
1311
1312 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
1313                                   Error **errp)
1314 {
1315     MigrationState *s = migrate_get_current();
1316     MigrationCapabilityStatusList *cap;
1317     bool cap_list[MIGRATION_CAPABILITY__MAX];
1318
1319     if (migration_is_running(s->state)) {
1320         error_setg(errp, QERR_MIGRATION_ACTIVE);
1321         return;
1322     }
1323
1324     memcpy(cap_list, s->enabled_capabilities, sizeof(cap_list));
1325     if (!migrate_caps_check(cap_list, params, errp)) {
1326         return;
1327     }
1328
1329     for (cap = params; cap; cap = cap->next) {
1330         s->enabled_capabilities[cap->value->capability] = cap->value->state;
1331     }
1332 }
1333
1334 /*
1335  * Check whether the parameters are valid. Error will be put into errp
1336  * (if provided). Return true if valid, otherwise false.
1337  */
1338 static bool migrate_params_check(MigrationParameters *params, Error **errp)
1339 {
1340     if (params->has_compress_level &&
1341         (params->compress_level > 9)) {
1342         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level",
1343                    "a value between 0 and 9");
1344         return false;
1345     }
1346
1347     if (params->has_compress_threads && (params->compress_threads < 1)) {
1348         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1349                    "compress_threads",
1350                    "a value between 1 and 255");
1351         return false;
1352     }
1353
1354     if (params->has_decompress_threads && (params->decompress_threads < 1)) {
1355         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1356                    "decompress_threads",
1357                    "a value between 1 and 255");
1358         return false;
1359     }
1360
1361     if (params->has_throttle_trigger_threshold &&
1362         (params->throttle_trigger_threshold < 1 ||
1363          params->throttle_trigger_threshold > 100)) {
1364         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1365                    "throttle_trigger_threshold",
1366                    "an integer in the range of 1 to 100");
1367         return false;
1368     }
1369
1370     if (params->has_cpu_throttle_initial &&
1371         (params->cpu_throttle_initial < 1 ||
1372          params->cpu_throttle_initial > 99)) {
1373         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1374                    "cpu_throttle_initial",
1375                    "an integer in the range of 1 to 99");
1376         return false;
1377     }
1378
1379     if (params->has_cpu_throttle_increment &&
1380         (params->cpu_throttle_increment < 1 ||
1381          params->cpu_throttle_increment > 99)) {
1382         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1383                    "cpu_throttle_increment",
1384                    "an integer in the range of 1 to 99");
1385         return false;
1386     }
1387
1388     if (params->has_max_bandwidth && (params->max_bandwidth > SIZE_MAX)) {
1389         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1390                    "max_bandwidth",
1391                    "an integer in the range of 0 to "stringify(SIZE_MAX)
1392                    " bytes/second");
1393         return false;
1394     }
1395
1396     if (params->has_downtime_limit &&
1397         (params->downtime_limit > MAX_MIGRATE_DOWNTIME)) {
1398         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1399                    "downtime_limit",
1400                    "an integer in the range of 0 to "
1401                     stringify(MAX_MIGRATE_DOWNTIME)" ms");
1402         return false;
1403     }
1404
1405     /* x_checkpoint_delay is now always positive */
1406
1407     if (params->has_multifd_channels && (params->multifd_channels < 1)) {
1408         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1409                    "multifd_channels",
1410                    "a value between 1 and 255");
1411         return false;
1412     }
1413
1414     if (params->has_multifd_zlib_level &&
1415         (params->multifd_zlib_level > 9)) {
1416         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zlib_level",
1417                    "a value between 0 and 9");
1418         return false;
1419     }
1420
1421     if (params->has_multifd_zstd_level &&
1422         (params->multifd_zstd_level > 20)) {
1423         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zstd_level",
1424                    "a value between 0 and 20");
1425         return false;
1426     }
1427
1428     if (params->has_xbzrle_cache_size &&
1429         (params->xbzrle_cache_size < qemu_target_page_size() ||
1430          !is_power_of_2(params->xbzrle_cache_size))) {
1431         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1432                    "xbzrle_cache_size",
1433                    "a power of two no less than the target page size");
1434         return false;
1435     }
1436
1437     if (params->has_max_cpu_throttle &&
1438         (params->max_cpu_throttle < params->cpu_throttle_initial ||
1439          params->max_cpu_throttle > 99)) {
1440         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1441                    "max_cpu_throttle",
1442                    "an integer in the range of cpu_throttle_initial to 99");
1443         return false;
1444     }
1445
1446     if (params->has_announce_initial &&
1447         params->announce_initial > 100000) {
1448         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1449                    "announce_initial",
1450                    "a value between 0 and 100000");
1451         return false;
1452     }
1453     if (params->has_announce_max &&
1454         params->announce_max > 100000) {
1455         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1456                    "announce_max",
1457                    "a value between 0 and 100000");
1458        return false;
1459     }
1460     if (params->has_announce_rounds &&
1461         params->announce_rounds > 1000) {
1462         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1463                    "announce_rounds",
1464                    "a value between 0 and 1000");
1465        return false;
1466     }
1467     if (params->has_announce_step &&
1468         (params->announce_step < 1 ||
1469         params->announce_step > 10000)) {
1470         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1471                    "announce_step",
1472                    "a value between 0 and 10000");
1473        return false;
1474     }
1475
1476     if (params->has_block_bitmap_mapping &&
1477         !check_dirty_bitmap_mig_alias_map(params->block_bitmap_mapping, errp)) {
1478         error_prepend(errp, "Invalid mapping given for block-bitmap-mapping: ");
1479         return false;
1480     }
1481
1482     return true;
1483 }
1484
1485 static void migrate_params_test_apply(MigrateSetParameters *params,
1486                                       MigrationParameters *dest)
1487 {
1488     *dest = migrate_get_current()->parameters;
1489
1490     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1491
1492     if (params->has_compress_level) {
1493         dest->compress_level = params->compress_level;
1494     }
1495
1496     if (params->has_compress_threads) {
1497         dest->compress_threads = params->compress_threads;
1498     }
1499
1500     if (params->has_compress_wait_thread) {
1501         dest->compress_wait_thread = params->compress_wait_thread;
1502     }
1503
1504     if (params->has_decompress_threads) {
1505         dest->decompress_threads = params->decompress_threads;
1506     }
1507
1508     if (params->has_throttle_trigger_threshold) {
1509         dest->throttle_trigger_threshold = params->throttle_trigger_threshold;
1510     }
1511
1512     if (params->has_cpu_throttle_initial) {
1513         dest->cpu_throttle_initial = params->cpu_throttle_initial;
1514     }
1515
1516     if (params->has_cpu_throttle_increment) {
1517         dest->cpu_throttle_increment = params->cpu_throttle_increment;
1518     }
1519
1520     if (params->has_cpu_throttle_tailslow) {
1521         dest->cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1522     }
1523
1524     if (params->has_tls_creds) {
1525         assert(params->tls_creds->type == QTYPE_QSTRING);
1526         dest->tls_creds = params->tls_creds->u.s;
1527     }
1528
1529     if (params->has_tls_hostname) {
1530         assert(params->tls_hostname->type == QTYPE_QSTRING);
1531         dest->tls_hostname = params->tls_hostname->u.s;
1532     }
1533
1534     if (params->has_max_bandwidth) {
1535         dest->max_bandwidth = params->max_bandwidth;
1536     }
1537
1538     if (params->has_downtime_limit) {
1539         dest->downtime_limit = params->downtime_limit;
1540     }
1541
1542     if (params->has_x_checkpoint_delay) {
1543         dest->x_checkpoint_delay = params->x_checkpoint_delay;
1544     }
1545
1546     if (params->has_block_incremental) {
1547         dest->block_incremental = params->block_incremental;
1548     }
1549     if (params->has_multifd_channels) {
1550         dest->multifd_channels = params->multifd_channels;
1551     }
1552     if (params->has_multifd_compression) {
1553         dest->multifd_compression = params->multifd_compression;
1554     }
1555     if (params->has_xbzrle_cache_size) {
1556         dest->xbzrle_cache_size = params->xbzrle_cache_size;
1557     }
1558     if (params->has_max_postcopy_bandwidth) {
1559         dest->max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1560     }
1561     if (params->has_max_cpu_throttle) {
1562         dest->max_cpu_throttle = params->max_cpu_throttle;
1563     }
1564     if (params->has_announce_initial) {
1565         dest->announce_initial = params->announce_initial;
1566     }
1567     if (params->has_announce_max) {
1568         dest->announce_max = params->announce_max;
1569     }
1570     if (params->has_announce_rounds) {
1571         dest->announce_rounds = params->announce_rounds;
1572     }
1573     if (params->has_announce_step) {
1574         dest->announce_step = params->announce_step;
1575     }
1576
1577     if (params->has_block_bitmap_mapping) {
1578         dest->has_block_bitmap_mapping = true;
1579         dest->block_bitmap_mapping = params->block_bitmap_mapping;
1580     }
1581 }
1582
1583 static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
1584 {
1585     MigrationState *s = migrate_get_current();
1586
1587     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1588
1589     if (params->has_compress_level) {
1590         s->parameters.compress_level = params->compress_level;
1591     }
1592
1593     if (params->has_compress_threads) {
1594         s->parameters.compress_threads = params->compress_threads;
1595     }
1596
1597     if (params->has_compress_wait_thread) {
1598         s->parameters.compress_wait_thread = params->compress_wait_thread;
1599     }
1600
1601     if (params->has_decompress_threads) {
1602         s->parameters.decompress_threads = params->decompress_threads;
1603     }
1604
1605     if (params->has_throttle_trigger_threshold) {
1606         s->parameters.throttle_trigger_threshold = params->throttle_trigger_threshold;
1607     }
1608
1609     if (params->has_cpu_throttle_initial) {
1610         s->parameters.cpu_throttle_initial = params->cpu_throttle_initial;
1611     }
1612
1613     if (params->has_cpu_throttle_increment) {
1614         s->parameters.cpu_throttle_increment = params->cpu_throttle_increment;
1615     }
1616
1617     if (params->has_cpu_throttle_tailslow) {
1618         s->parameters.cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1619     }
1620
1621     if (params->has_tls_creds) {
1622         g_free(s->parameters.tls_creds);
1623         assert(params->tls_creds->type == QTYPE_QSTRING);
1624         s->parameters.tls_creds = g_strdup(params->tls_creds->u.s);
1625     }
1626
1627     if (params->has_tls_hostname) {
1628         g_free(s->parameters.tls_hostname);
1629         assert(params->tls_hostname->type == QTYPE_QSTRING);
1630         s->parameters.tls_hostname = g_strdup(params->tls_hostname->u.s);
1631     }
1632
1633     if (params->has_tls_authz) {
1634         g_free(s->parameters.tls_authz);
1635         assert(params->tls_authz->type == QTYPE_QSTRING);
1636         s->parameters.tls_authz = g_strdup(params->tls_authz->u.s);
1637     }
1638
1639     if (params->has_max_bandwidth) {
1640         s->parameters.max_bandwidth = params->max_bandwidth;
1641         if (s->to_dst_file && !migration_in_postcopy()) {
1642             qemu_file_set_rate_limit(s->to_dst_file,
1643                                 s->parameters.max_bandwidth / XFER_LIMIT_RATIO);
1644         }
1645     }
1646
1647     if (params->has_downtime_limit) {
1648         s->parameters.downtime_limit = params->downtime_limit;
1649     }
1650
1651     if (params->has_x_checkpoint_delay) {
1652         s->parameters.x_checkpoint_delay = params->x_checkpoint_delay;
1653         if (migration_in_colo_state()) {
1654             colo_checkpoint_notify(s);
1655         }
1656     }
1657
1658     if (params->has_block_incremental) {
1659         s->parameters.block_incremental = params->block_incremental;
1660     }
1661     if (params->has_multifd_channels) {
1662         s->parameters.multifd_channels = params->multifd_channels;
1663     }
1664     if (params->has_multifd_compression) {
1665         s->parameters.multifd_compression = params->multifd_compression;
1666     }
1667     if (params->has_xbzrle_cache_size) {
1668         s->parameters.xbzrle_cache_size = params->xbzrle_cache_size;
1669         xbzrle_cache_resize(params->xbzrle_cache_size, errp);
1670     }
1671     if (params->has_max_postcopy_bandwidth) {
1672         s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1673         if (s->to_dst_file && migration_in_postcopy()) {
1674             qemu_file_set_rate_limit(s->to_dst_file,
1675                     s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO);
1676         }
1677     }
1678     if (params->has_max_cpu_throttle) {
1679         s->parameters.max_cpu_throttle = params->max_cpu_throttle;
1680     }
1681     if (params->has_announce_initial) {
1682         s->parameters.announce_initial = params->announce_initial;
1683     }
1684     if (params->has_announce_max) {
1685         s->parameters.announce_max = params->announce_max;
1686     }
1687     if (params->has_announce_rounds) {
1688         s->parameters.announce_rounds = params->announce_rounds;
1689     }
1690     if (params->has_announce_step) {
1691         s->parameters.announce_step = params->announce_step;
1692     }
1693
1694     if (params->has_block_bitmap_mapping) {
1695         qapi_free_BitmapMigrationNodeAliasList(
1696             s->parameters.block_bitmap_mapping);
1697
1698         s->parameters.has_block_bitmap_mapping = true;
1699         s->parameters.block_bitmap_mapping =
1700             QAPI_CLONE(BitmapMigrationNodeAliasList,
1701                        params->block_bitmap_mapping);
1702     }
1703 }
1704
1705 void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
1706 {
1707     MigrationParameters tmp;
1708
1709     /* TODO Rewrite "" to null instead */
1710     if (params->has_tls_creds
1711         && params->tls_creds->type == QTYPE_QNULL) {
1712         qobject_unref(params->tls_creds->u.n);
1713         params->tls_creds->type = QTYPE_QSTRING;
1714         params->tls_creds->u.s = strdup("");
1715     }
1716     /* TODO Rewrite "" to null instead */
1717     if (params->has_tls_hostname
1718         && params->tls_hostname->type == QTYPE_QNULL) {
1719         qobject_unref(params->tls_hostname->u.n);
1720         params->tls_hostname->type = QTYPE_QSTRING;
1721         params->tls_hostname->u.s = strdup("");
1722     }
1723
1724     migrate_params_test_apply(params, &tmp);
1725
1726     if (!migrate_params_check(&tmp, errp)) {
1727         /* Invalid parameter */
1728         return;
1729     }
1730
1731     migrate_params_apply(params, errp);
1732 }
1733
1734
1735 void qmp_migrate_start_postcopy(Error **errp)
1736 {
1737     MigrationState *s = migrate_get_current();
1738
1739     if (!migrate_postcopy()) {
1740         error_setg(errp, "Enable postcopy with migrate_set_capability before"
1741                          " the start of migration");
1742         return;
1743     }
1744
1745     if (s->state == MIGRATION_STATUS_NONE) {
1746         error_setg(errp, "Postcopy must be started after migration has been"
1747                          " started");
1748         return;
1749     }
1750     /*
1751      * we don't error if migration has finished since that would be racy
1752      * with issuing this command.
1753      */
1754     qatomic_set(&s->start_postcopy, true);
1755 }
1756
1757 /* shared migration helpers */
1758
1759 void migrate_set_state(int *state, int old_state, int new_state)
1760 {
1761     assert(new_state < MIGRATION_STATUS__MAX);
1762     if (qatomic_cmpxchg(state, old_state, new_state) == old_state) {
1763         trace_migrate_set_state(MigrationStatus_str(new_state));
1764         migrate_generate_event(new_state);
1765     }
1766 }
1767
1768 static MigrationCapabilityStatus *migrate_cap_add(MigrationCapability index,
1769                                                   bool state)
1770 {
1771     MigrationCapabilityStatus *cap;
1772
1773     cap = g_new0(MigrationCapabilityStatus, 1);
1774     cap->capability = index;
1775     cap->state = state;
1776
1777     return cap;
1778 }
1779
1780 void migrate_set_block_enabled(bool value, Error **errp)
1781 {
1782     MigrationCapabilityStatusList *cap = NULL;
1783
1784     QAPI_LIST_PREPEND(cap, migrate_cap_add(MIGRATION_CAPABILITY_BLOCK, value));
1785     qmp_migrate_set_capabilities(cap, errp);
1786     qapi_free_MigrationCapabilityStatusList(cap);
1787 }
1788
1789 static void migrate_set_block_incremental(MigrationState *s, bool value)
1790 {
1791     s->parameters.block_incremental = value;
1792 }
1793
1794 static void block_cleanup_parameters(MigrationState *s)
1795 {
1796     if (s->must_remove_block_options) {
1797         /* setting to false can never fail */
1798         migrate_set_block_enabled(false, &error_abort);
1799         migrate_set_block_incremental(s, false);
1800         s->must_remove_block_options = false;
1801     }
1802 }
1803
1804 static void migrate_fd_cleanup(MigrationState *s)
1805 {
1806     qemu_bh_delete(s->cleanup_bh);
1807     s->cleanup_bh = NULL;
1808
1809     qemu_savevm_state_cleanup();
1810
1811     if (s->to_dst_file) {
1812         QEMUFile *tmp;
1813
1814         trace_migrate_fd_cleanup();
1815         qemu_mutex_unlock_iothread();
1816         if (s->migration_thread_running) {
1817             qemu_thread_join(&s->thread);
1818             s->migration_thread_running = false;
1819         }
1820         qemu_mutex_lock_iothread();
1821
1822         multifd_save_cleanup();
1823         qemu_mutex_lock(&s->qemu_file_lock);
1824         tmp = s->to_dst_file;
1825         s->to_dst_file = NULL;
1826         qemu_mutex_unlock(&s->qemu_file_lock);
1827         /*
1828          * Close the file handle without the lock to make sure the
1829          * critical section won't block for long.
1830          */
1831         qemu_fclose(tmp);
1832     }
1833
1834     assert(!migration_is_active(s));
1835
1836     if (s->state == MIGRATION_STATUS_CANCELLING) {
1837         migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING,
1838                           MIGRATION_STATUS_CANCELLED);
1839     }
1840
1841     if (s->error) {
1842         /* It is used on info migrate.  We can't free it */
1843         error_report_err(error_copy(s->error));
1844     }
1845     notifier_list_notify(&migration_state_notifiers, s);
1846     block_cleanup_parameters(s);
1847     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
1848 }
1849
1850 static void migrate_fd_cleanup_schedule(MigrationState *s)
1851 {
1852     /*
1853      * Ref the state for bh, because it may be called when
1854      * there're already no other refs
1855      */
1856     object_ref(OBJECT(s));
1857     qemu_bh_schedule(s->cleanup_bh);
1858 }
1859
1860 static void migrate_fd_cleanup_bh(void *opaque)
1861 {
1862     MigrationState *s = opaque;
1863     migrate_fd_cleanup(s);
1864     object_unref(OBJECT(s));
1865 }
1866
1867 void migrate_set_error(MigrationState *s, const Error *error)
1868 {
1869     QEMU_LOCK_GUARD(&s->error_mutex);
1870     if (!s->error) {
1871         s->error = error_copy(error);
1872     }
1873 }
1874
1875 void migrate_fd_error(MigrationState *s, const Error *error)
1876 {
1877     trace_migrate_fd_error(error_get_pretty(error));
1878     assert(s->to_dst_file == NULL);
1879     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
1880                       MIGRATION_STATUS_FAILED);
1881     migrate_set_error(s, error);
1882 }
1883
1884 static void migrate_fd_cancel(MigrationState *s)
1885 {
1886     int old_state ;
1887     QEMUFile *f = migrate_get_current()->to_dst_file;
1888     trace_migrate_fd_cancel();
1889
1890     if (s->rp_state.from_dst_file) {
1891         /* shutdown the rp socket, so causing the rp thread to shutdown */
1892         qemu_file_shutdown(s->rp_state.from_dst_file);
1893     }
1894
1895     do {
1896         old_state = s->state;
1897         if (!migration_is_running(old_state)) {
1898             break;
1899         }
1900         /* If the migration is paused, kick it out of the pause */
1901         if (old_state == MIGRATION_STATUS_PRE_SWITCHOVER) {
1902             qemu_sem_post(&s->pause_sem);
1903         }
1904         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING);
1905     } while (s->state != MIGRATION_STATUS_CANCELLING);
1906
1907     /*
1908      * If we're unlucky the migration code might be stuck somewhere in a
1909      * send/write while the network has failed and is waiting to timeout;
1910      * if we've got shutdown(2) available then we can force it to quit.
1911      * The outgoing qemu file gets closed in migrate_fd_cleanup that is
1912      * called in a bh, so there is no race against this cancel.
1913      */
1914     if (s->state == MIGRATION_STATUS_CANCELLING && f) {
1915         qemu_file_shutdown(f);
1916     }
1917     if (s->state == MIGRATION_STATUS_CANCELLING && s->block_inactive) {
1918         Error *local_err = NULL;
1919
1920         bdrv_invalidate_cache_all(&local_err);
1921         if (local_err) {
1922             error_report_err(local_err);
1923         } else {
1924             s->block_inactive = false;
1925         }
1926     }
1927 }
1928
1929 void add_migration_state_change_notifier(Notifier *notify)
1930 {
1931     notifier_list_add(&migration_state_notifiers, notify);
1932 }
1933
1934 void remove_migration_state_change_notifier(Notifier *notify)
1935 {
1936     notifier_remove(notify);
1937 }
1938
1939 bool migration_in_setup(MigrationState *s)
1940 {
1941     return s->state == MIGRATION_STATUS_SETUP;
1942 }
1943
1944 bool migration_has_finished(MigrationState *s)
1945 {
1946     return s->state == MIGRATION_STATUS_COMPLETED;
1947 }
1948
1949 bool migration_has_failed(MigrationState *s)
1950 {
1951     return (s->state == MIGRATION_STATUS_CANCELLED ||
1952             s->state == MIGRATION_STATUS_FAILED);
1953 }
1954
1955 bool migration_in_postcopy(void)
1956 {
1957     MigrationState *s = migrate_get_current();
1958
1959     switch (s->state) {
1960     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1961     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1962     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1963         return true;
1964     default:
1965         return false;
1966     }
1967 }
1968
1969 bool migration_in_postcopy_after_devices(MigrationState *s)
1970 {
1971     return migration_in_postcopy() && s->postcopy_after_devices;
1972 }
1973
1974 bool migration_in_incoming_postcopy(void)
1975 {
1976     PostcopyState ps = postcopy_state_get();
1977
1978     return ps >= POSTCOPY_INCOMING_DISCARD && ps < POSTCOPY_INCOMING_END;
1979 }
1980
1981 bool migration_is_idle(void)
1982 {
1983     MigrationState *s = current_migration;
1984
1985     if (!s) {
1986         return true;
1987     }
1988
1989     switch (s->state) {
1990     case MIGRATION_STATUS_NONE:
1991     case MIGRATION_STATUS_CANCELLED:
1992     case MIGRATION_STATUS_COMPLETED:
1993     case MIGRATION_STATUS_FAILED:
1994         return true;
1995     case MIGRATION_STATUS_SETUP:
1996     case MIGRATION_STATUS_CANCELLING:
1997     case MIGRATION_STATUS_ACTIVE:
1998     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1999     case MIGRATION_STATUS_COLO:
2000     case MIGRATION_STATUS_PRE_SWITCHOVER:
2001     case MIGRATION_STATUS_DEVICE:
2002     case MIGRATION_STATUS_WAIT_UNPLUG:
2003         return false;
2004     case MIGRATION_STATUS__MAX:
2005         g_assert_not_reached();
2006     }
2007
2008     return false;
2009 }
2010
2011 bool migration_is_active(MigrationState *s)
2012 {
2013     return (s->state == MIGRATION_STATUS_ACTIVE ||
2014             s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
2015 }
2016
2017 void migrate_init(MigrationState *s)
2018 {
2019     /*
2020      * Reinitialise all migration state, except
2021      * parameters/capabilities that the user set, and
2022      * locks.
2023      */
2024     s->cleanup_bh = 0;
2025     s->vm_start_bh = 0;
2026     s->to_dst_file = NULL;
2027     s->state = MIGRATION_STATUS_NONE;
2028     s->rp_state.from_dst_file = NULL;
2029     s->rp_state.error = false;
2030     s->mbps = 0.0;
2031     s->pages_per_second = 0.0;
2032     s->downtime = 0;
2033     s->expected_downtime = 0;
2034     s->setup_time = 0;
2035     s->start_postcopy = false;
2036     s->postcopy_after_devices = false;
2037     s->migration_thread_running = false;
2038     error_free(s->error);
2039     s->error = NULL;
2040     s->hostname = NULL;
2041
2042     migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
2043
2044     s->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2045     s->total_time = 0;
2046     s->vm_was_running = false;
2047     s->iteration_initial_bytes = 0;
2048     s->threshold_size = 0;
2049 }
2050
2051 int migrate_add_blocker(Error *reason, Error **errp)
2052 {
2053     if (only_migratable) {
2054         error_propagate_prepend(errp, error_copy(reason),
2055                                 "disallowing migration blocker "
2056                                 "(--only-migratable) for: ");
2057         return -EACCES;
2058     }
2059
2060     if (migration_is_idle()) {
2061         migration_blockers = g_slist_prepend(migration_blockers, reason);
2062         return 0;
2063     }
2064
2065     error_propagate_prepend(errp, error_copy(reason),
2066                             "disallowing migration blocker "
2067                             "(migration in progress) for: ");
2068     return -EBUSY;
2069 }
2070
2071 void migrate_del_blocker(Error *reason)
2072 {
2073     migration_blockers = g_slist_remove(migration_blockers, reason);
2074 }
2075
2076 void qmp_migrate_incoming(const char *uri, Error **errp)
2077 {
2078     Error *local_err = NULL;
2079     static bool once = true;
2080
2081     if (!once) {
2082         error_setg(errp, "The incoming migration has already been started");
2083         return;
2084     }
2085     if (!runstate_check(RUN_STATE_INMIGRATE)) {
2086         error_setg(errp, "'-incoming' was not specified on the command line");
2087         return;
2088     }
2089
2090     qemu_start_incoming_migration(uri, &local_err);
2091
2092     if (local_err) {
2093         error_propagate(errp, local_err);
2094         return;
2095     }
2096
2097     once = false;
2098 }
2099
2100 void qmp_migrate_recover(const char *uri, Error **errp)
2101 {
2102     MigrationIncomingState *mis = migration_incoming_get_current();
2103
2104     if (mis->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2105         error_setg(errp, "Migrate recover can only be run "
2106                    "when postcopy is paused.");
2107         return;
2108     }
2109
2110     if (qatomic_cmpxchg(&mis->postcopy_recover_triggered,
2111                        false, true) == true) {
2112         error_setg(errp, "Migrate recovery is triggered already");
2113         return;
2114     }
2115
2116     /*
2117      * Note that this call will never start a real migration; it will
2118      * only re-setup the migration stream and poke existing migration
2119      * to continue using that newly established channel.
2120      */
2121     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2122     qemu_start_incoming_migration(uri, errp);
2123 }
2124
2125 void qmp_migrate_pause(Error **errp)
2126 {
2127     MigrationState *ms = migrate_get_current();
2128     MigrationIncomingState *mis = migration_incoming_get_current();
2129     int ret;
2130
2131     if (ms->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2132         /* Source side, during postcopy */
2133         qemu_mutex_lock(&ms->qemu_file_lock);
2134         ret = qemu_file_shutdown(ms->to_dst_file);
2135         qemu_mutex_unlock(&ms->qemu_file_lock);
2136         if (ret) {
2137             error_setg(errp, "Failed to pause source migration");
2138         }
2139         return;
2140     }
2141
2142     if (mis->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2143         ret = qemu_file_shutdown(mis->from_src_file);
2144         if (ret) {
2145             error_setg(errp, "Failed to pause destination migration");
2146         }
2147         return;
2148     }
2149
2150     error_setg(errp, "migrate-pause is currently only supported "
2151                "during postcopy-active state");
2152 }
2153
2154 bool migration_is_blocked(Error **errp)
2155 {
2156     if (qemu_savevm_state_blocked(errp)) {
2157         return true;
2158     }
2159
2160     if (migration_blockers) {
2161         error_propagate(errp, error_copy(migration_blockers->data));
2162         return true;
2163     }
2164
2165     return false;
2166 }
2167
2168 /* Returns true if continue to migrate, or false if error detected */
2169 static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
2170                             bool resume, Error **errp)
2171 {
2172     Error *local_err = NULL;
2173
2174     if (resume) {
2175         if (s->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2176             error_setg(errp, "Cannot resume if there is no "
2177                        "paused migration");
2178             return false;
2179         }
2180
2181         /*
2182          * Postcopy recovery won't work well with release-ram
2183          * capability since release-ram will drop the page buffer as
2184          * long as the page is put into the send buffer.  So if there
2185          * is a network failure happened, any page buffers that have
2186          * not yet reached the destination VM but have already been
2187          * sent from the source VM will be lost forever.  Let's refuse
2188          * the client from resuming such a postcopy migration.
2189          * Luckily release-ram was designed to only be used when src
2190          * and destination VMs are on the same host, so it should be
2191          * fine.
2192          */
2193         if (migrate_release_ram()) {
2194             error_setg(errp, "Postcopy recovery cannot work "
2195                        "when release-ram capability is set");
2196             return false;
2197         }
2198
2199         /* This is a resume, skip init status */
2200         return true;
2201     }
2202
2203     if (migration_is_running(s->state)) {
2204         error_setg(errp, QERR_MIGRATION_ACTIVE);
2205         return false;
2206     }
2207
2208     if (runstate_check(RUN_STATE_INMIGRATE)) {
2209         error_setg(errp, "Guest is waiting for an incoming migration");
2210         return false;
2211     }
2212
2213     if (runstate_check(RUN_STATE_POSTMIGRATE)) {
2214         error_setg(errp, "Can't migrate the vm that was paused due to "
2215                    "previous migration");
2216         return false;
2217     }
2218
2219     if (migration_is_blocked(errp)) {
2220         return false;
2221     }
2222
2223     if (blk || blk_inc) {
2224         if (migrate_use_block() || migrate_use_block_incremental()) {
2225             error_setg(errp, "Command options are incompatible with "
2226                        "current migration capabilities");
2227             return false;
2228         }
2229         migrate_set_block_enabled(true, &local_err);
2230         if (local_err) {
2231             error_propagate(errp, local_err);
2232             return false;
2233         }
2234         s->must_remove_block_options = true;
2235     }
2236
2237     if (blk_inc) {
2238         migrate_set_block_incremental(s, true);
2239     }
2240
2241     migrate_init(s);
2242     /*
2243      * set ram_counters memory to zero for a
2244      * new migration
2245      */
2246     memset(&ram_counters, 0, sizeof(ram_counters));
2247
2248     return true;
2249 }
2250
2251 void qmp_migrate(const char *uri, bool has_blk, bool blk,
2252                  bool has_inc, bool inc, bool has_detach, bool detach,
2253                  bool has_resume, bool resume, Error **errp)
2254 {
2255     Error *local_err = NULL;
2256     MigrationState *s = migrate_get_current();
2257     const char *p = NULL;
2258
2259     if (!migrate_prepare(s, has_blk && blk, has_inc && inc,
2260                          has_resume && resume, errp)) {
2261         /* Error detected, put into errp */
2262         return;
2263     }
2264
2265     if (!(has_resume && resume)) {
2266         if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2267             return;
2268         }
2269     }
2270
2271     if (strstart(uri, "tcp:", &p) ||
2272         strstart(uri, "unix:", NULL) ||
2273         strstart(uri, "vsock:", NULL)) {
2274         socket_start_outgoing_migration(s, p ? p : uri, &local_err);
2275 #ifdef CONFIG_RDMA
2276     } else if (strstart(uri, "rdma:", &p)) {
2277         rdma_start_outgoing_migration(s, p, &local_err);
2278 #endif
2279     } else if (strstart(uri, "exec:", &p)) {
2280         exec_start_outgoing_migration(s, p, &local_err);
2281     } else if (strstart(uri, "fd:", &p)) {
2282         fd_start_outgoing_migration(s, p, &local_err);
2283     } else {
2284         if (!(has_resume && resume)) {
2285             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2286         }
2287         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri",
2288                    "a valid migration protocol");
2289         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
2290                           MIGRATION_STATUS_FAILED);
2291         block_cleanup_parameters(s);
2292         return;
2293     }
2294
2295     if (local_err) {
2296         if (!(has_resume && resume)) {
2297             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2298         }
2299         migrate_fd_error(s, local_err);
2300         error_propagate(errp, local_err);
2301         return;
2302     }
2303 }
2304
2305 void qmp_migrate_cancel(Error **errp)
2306 {
2307     migrate_fd_cancel(migrate_get_current());
2308 }
2309
2310 void qmp_migrate_continue(MigrationStatus state, Error **errp)
2311 {
2312     MigrationState *s = migrate_get_current();
2313     if (s->state != state) {
2314         error_setg(errp,  "Migration not in expected state: %s",
2315                    MigrationStatus_str(s->state));
2316         return;
2317     }
2318     qemu_sem_post(&s->pause_sem);
2319 }
2320
2321 void qmp_migrate_set_cache_size(int64_t value, Error **errp)
2322 {
2323     MigrateSetParameters p = {
2324         .has_xbzrle_cache_size = true,
2325         .xbzrle_cache_size = value,
2326     };
2327
2328     qmp_migrate_set_parameters(&p, errp);
2329 }
2330
2331 uint64_t qmp_query_migrate_cache_size(Error **errp)
2332 {
2333     return migrate_xbzrle_cache_size();
2334 }
2335
2336 void qmp_migrate_set_speed(int64_t value, Error **errp)
2337 {
2338     MigrateSetParameters p = {
2339         .has_max_bandwidth = true,
2340         .max_bandwidth = value,
2341     };
2342
2343     qmp_migrate_set_parameters(&p, errp);
2344 }
2345
2346 void qmp_migrate_set_downtime(double value, Error **errp)
2347 {
2348     if (value < 0 || value > MAX_MIGRATE_DOWNTIME_SECONDS) {
2349         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
2350                    "downtime_limit",
2351                    "an integer in the range of 0 to "
2352                     stringify(MAX_MIGRATE_DOWNTIME_SECONDS)" seconds");
2353         return;
2354     }
2355
2356     value *= 1000; /* Convert to milliseconds */
2357
2358     MigrateSetParameters p = {
2359         .has_downtime_limit = true,
2360         .downtime_limit = (int64_t)value,
2361     };
2362
2363     qmp_migrate_set_parameters(&p, errp);
2364 }
2365
2366 bool migrate_release_ram(void)
2367 {
2368     MigrationState *s;
2369
2370     s = migrate_get_current();
2371
2372     return s->enabled_capabilities[MIGRATION_CAPABILITY_RELEASE_RAM];
2373 }
2374
2375 bool migrate_postcopy_ram(void)
2376 {
2377     MigrationState *s;
2378
2379     s = migrate_get_current();
2380
2381     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM];
2382 }
2383
2384 bool migrate_postcopy(void)
2385 {
2386     return migrate_postcopy_ram() || migrate_dirty_bitmaps();
2387 }
2388
2389 bool migrate_auto_converge(void)
2390 {
2391     MigrationState *s;
2392
2393     s = migrate_get_current();
2394
2395     return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
2396 }
2397
2398 bool migrate_zero_blocks(void)
2399 {
2400     MigrationState *s;
2401
2402     s = migrate_get_current();
2403
2404     return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
2405 }
2406
2407 bool migrate_postcopy_blocktime(void)
2408 {
2409     MigrationState *s;
2410
2411     s = migrate_get_current();
2412
2413     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME];
2414 }
2415
2416 bool migrate_use_compression(void)
2417 {
2418     MigrationState *s;
2419
2420     s = migrate_get_current();
2421
2422     return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS];
2423 }
2424
2425 int migrate_compress_level(void)
2426 {
2427     MigrationState *s;
2428
2429     s = migrate_get_current();
2430
2431     return s->parameters.compress_level;
2432 }
2433
2434 int migrate_compress_threads(void)
2435 {
2436     MigrationState *s;
2437
2438     s = migrate_get_current();
2439
2440     return s->parameters.compress_threads;
2441 }
2442
2443 int migrate_compress_wait_thread(void)
2444 {
2445     MigrationState *s;
2446
2447     s = migrate_get_current();
2448
2449     return s->parameters.compress_wait_thread;
2450 }
2451
2452 int migrate_decompress_threads(void)
2453 {
2454     MigrationState *s;
2455
2456     s = migrate_get_current();
2457
2458     return s->parameters.decompress_threads;
2459 }
2460
2461 bool migrate_dirty_bitmaps(void)
2462 {
2463     MigrationState *s;
2464
2465     s = migrate_get_current();
2466
2467     return s->enabled_capabilities[MIGRATION_CAPABILITY_DIRTY_BITMAPS];
2468 }
2469
2470 bool migrate_ignore_shared(void)
2471 {
2472     MigrationState *s;
2473
2474     s = migrate_get_current();
2475
2476     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_IGNORE_SHARED];
2477 }
2478
2479 bool migrate_validate_uuid(void)
2480 {
2481     MigrationState *s;
2482
2483     s = migrate_get_current();
2484
2485     return s->enabled_capabilities[MIGRATION_CAPABILITY_VALIDATE_UUID];
2486 }
2487
2488 bool migrate_use_events(void)
2489 {
2490     MigrationState *s;
2491
2492     s = migrate_get_current();
2493
2494     return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS];
2495 }
2496
2497 bool migrate_use_multifd(void)
2498 {
2499     MigrationState *s;
2500
2501     s = migrate_get_current();
2502
2503     return s->enabled_capabilities[MIGRATION_CAPABILITY_MULTIFD];
2504 }
2505
2506 bool migrate_pause_before_switchover(void)
2507 {
2508     MigrationState *s;
2509
2510     s = migrate_get_current();
2511
2512     return s->enabled_capabilities[
2513         MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER];
2514 }
2515
2516 int migrate_multifd_channels(void)
2517 {
2518     MigrationState *s;
2519
2520     s = migrate_get_current();
2521
2522     return s->parameters.multifd_channels;
2523 }
2524
2525 MultiFDCompression migrate_multifd_compression(void)
2526 {
2527     MigrationState *s;
2528
2529     s = migrate_get_current();
2530
2531     return s->parameters.multifd_compression;
2532 }
2533
2534 int migrate_multifd_zlib_level(void)
2535 {
2536     MigrationState *s;
2537
2538     s = migrate_get_current();
2539
2540     return s->parameters.multifd_zlib_level;
2541 }
2542
2543 int migrate_multifd_zstd_level(void)
2544 {
2545     MigrationState *s;
2546
2547     s = migrate_get_current();
2548
2549     return s->parameters.multifd_zstd_level;
2550 }
2551
2552 int migrate_use_xbzrle(void)
2553 {
2554     MigrationState *s;
2555
2556     s = migrate_get_current();
2557
2558     return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
2559 }
2560
2561 uint64_t migrate_xbzrle_cache_size(void)
2562 {
2563     MigrationState *s;
2564
2565     s = migrate_get_current();
2566
2567     return s->parameters.xbzrle_cache_size;
2568 }
2569
2570 static int64_t migrate_max_postcopy_bandwidth(void)
2571 {
2572     MigrationState *s;
2573
2574     s = migrate_get_current();
2575
2576     return s->parameters.max_postcopy_bandwidth;
2577 }
2578
2579 bool migrate_use_block(void)
2580 {
2581     MigrationState *s;
2582
2583     s = migrate_get_current();
2584
2585     return s->enabled_capabilities[MIGRATION_CAPABILITY_BLOCK];
2586 }
2587
2588 bool migrate_use_return_path(void)
2589 {
2590     MigrationState *s;
2591
2592     s = migrate_get_current();
2593
2594     return s->enabled_capabilities[MIGRATION_CAPABILITY_RETURN_PATH];
2595 }
2596
2597 bool migrate_use_block_incremental(void)
2598 {
2599     MigrationState *s;
2600
2601     s = migrate_get_current();
2602
2603     return s->parameters.block_incremental;
2604 }
2605
2606 bool migrate_background_snapshot(void)
2607 {
2608     MigrationState *s;
2609
2610     s = migrate_get_current();
2611
2612     return s->enabled_capabilities[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT];
2613 }
2614
2615 /* migration thread support */
2616 /*
2617  * Something bad happened to the RP stream, mark an error
2618  * The caller shall print or trace something to indicate why
2619  */
2620 static void mark_source_rp_bad(MigrationState *s)
2621 {
2622     s->rp_state.error = true;
2623 }
2624
2625 static struct rp_cmd_args {
2626     ssize_t     len; /* -1 = variable */
2627     const char *name;
2628 } rp_cmd_args[] = {
2629     [MIG_RP_MSG_INVALID]        = { .len = -1, .name = "INVALID" },
2630     [MIG_RP_MSG_SHUT]           = { .len =  4, .name = "SHUT" },
2631     [MIG_RP_MSG_PONG]           = { .len =  4, .name = "PONG" },
2632     [MIG_RP_MSG_REQ_PAGES]      = { .len = 12, .name = "REQ_PAGES" },
2633     [MIG_RP_MSG_REQ_PAGES_ID]   = { .len = -1, .name = "REQ_PAGES_ID" },
2634     [MIG_RP_MSG_RECV_BITMAP]    = { .len = -1, .name = "RECV_BITMAP" },
2635     [MIG_RP_MSG_RESUME_ACK]     = { .len =  4, .name = "RESUME_ACK" },
2636     [MIG_RP_MSG_MAX]            = { .len = -1, .name = "MAX" },
2637 };
2638
2639 /*
2640  * Process a request for pages received on the return path,
2641  * We're allowed to send more than requested (e.g. to round to our page size)
2642  * and we don't need to send pages that have already been sent.
2643  */
2644 static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
2645                                        ram_addr_t start, size_t len)
2646 {
2647     long our_host_ps = qemu_real_host_page_size;
2648
2649     trace_migrate_handle_rp_req_pages(rbname, start, len);
2650
2651     /*
2652      * Since we currently insist on matching page sizes, just sanity check
2653      * we're being asked for whole host pages.
2654      */
2655     if (start & (our_host_ps - 1) ||
2656        (len & (our_host_ps - 1))) {
2657         error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
2658                      " len: %zd", __func__, start, len);
2659         mark_source_rp_bad(ms);
2660         return;
2661     }
2662
2663     if (ram_save_queue_pages(rbname, start, len)) {
2664         mark_source_rp_bad(ms);
2665     }
2666 }
2667
2668 /* Return true to retry, false to quit */
2669 static bool postcopy_pause_return_path_thread(MigrationState *s)
2670 {
2671     trace_postcopy_pause_return_path();
2672
2673     qemu_sem_wait(&s->postcopy_pause_rp_sem);
2674
2675     trace_postcopy_pause_return_path_continued();
2676
2677     return true;
2678 }
2679
2680 static int migrate_handle_rp_recv_bitmap(MigrationState *s, char *block_name)
2681 {
2682     RAMBlock *block = qemu_ram_block_by_name(block_name);
2683
2684     if (!block) {
2685         error_report("%s: invalid block name '%s'", __func__, block_name);
2686         return -EINVAL;
2687     }
2688
2689     /* Fetch the received bitmap and refresh the dirty bitmap */
2690     return ram_dirty_bitmap_reload(s, block);
2691 }
2692
2693 static int migrate_handle_rp_resume_ack(MigrationState *s, uint32_t value)
2694 {
2695     trace_source_return_path_thread_resume_ack(value);
2696
2697     if (value != MIGRATION_RESUME_ACK_VALUE) {
2698         error_report("%s: illegal resume_ack value %"PRIu32,
2699                      __func__, value);
2700         return -1;
2701     }
2702
2703     /* Now both sides are active. */
2704     migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_RECOVER,
2705                       MIGRATION_STATUS_POSTCOPY_ACTIVE);
2706
2707     /* Notify send thread that time to continue send pages */
2708     qemu_sem_post(&s->rp_state.rp_sem);
2709
2710     return 0;
2711 }
2712
2713 /*
2714  * Handles messages sent on the return path towards the source VM
2715  *
2716  */
2717 static void *source_return_path_thread(void *opaque)
2718 {
2719     MigrationState *ms = opaque;
2720     QEMUFile *rp = ms->rp_state.from_dst_file;
2721     uint16_t header_len, header_type;
2722     uint8_t buf[512];
2723     uint32_t tmp32, sibling_error;
2724     ram_addr_t start = 0; /* =0 to silence warning */
2725     size_t  len = 0, expected_len;
2726     int res;
2727
2728     trace_source_return_path_thread_entry();
2729     rcu_register_thread();
2730
2731 retry:
2732     while (!ms->rp_state.error && !qemu_file_get_error(rp) &&
2733            migration_is_setup_or_active(ms->state)) {
2734         trace_source_return_path_thread_loop_top();
2735         header_type = qemu_get_be16(rp);
2736         header_len = qemu_get_be16(rp);
2737
2738         if (qemu_file_get_error(rp)) {
2739             mark_source_rp_bad(ms);
2740             goto out;
2741         }
2742
2743         if (header_type >= MIG_RP_MSG_MAX ||
2744             header_type == MIG_RP_MSG_INVALID) {
2745             error_report("RP: Received invalid message 0x%04x length 0x%04x",
2746                          header_type, header_len);
2747             mark_source_rp_bad(ms);
2748             goto out;
2749         }
2750
2751         if ((rp_cmd_args[header_type].len != -1 &&
2752             header_len != rp_cmd_args[header_type].len) ||
2753             header_len > sizeof(buf)) {
2754             error_report("RP: Received '%s' message (0x%04x) with"
2755                          "incorrect length %d expecting %zu",
2756                          rp_cmd_args[header_type].name, header_type, header_len,
2757                          (size_t)rp_cmd_args[header_type].len);
2758             mark_source_rp_bad(ms);
2759             goto out;
2760         }
2761
2762         /* We know we've got a valid header by this point */
2763         res = qemu_get_buffer(rp, buf, header_len);
2764         if (res != header_len) {
2765             error_report("RP: Failed reading data for message 0x%04x"
2766                          " read %d expected %d",
2767                          header_type, res, header_len);
2768             mark_source_rp_bad(ms);
2769             goto out;
2770         }
2771
2772         /* OK, we have the message and the data */
2773         switch (header_type) {
2774         case MIG_RP_MSG_SHUT:
2775             sibling_error = ldl_be_p(buf);
2776             trace_source_return_path_thread_shut(sibling_error);
2777             if (sibling_error) {
2778                 error_report("RP: Sibling indicated error %d", sibling_error);
2779                 mark_source_rp_bad(ms);
2780             }
2781             /*
2782              * We'll let the main thread deal with closing the RP
2783              * we could do a shutdown(2) on it, but we're the only user
2784              * anyway, so there's nothing gained.
2785              */
2786             goto out;
2787
2788         case MIG_RP_MSG_PONG:
2789             tmp32 = ldl_be_p(buf);
2790             trace_source_return_path_thread_pong(tmp32);
2791             break;
2792
2793         case MIG_RP_MSG_REQ_PAGES:
2794             start = ldq_be_p(buf);
2795             len = ldl_be_p(buf + 8);
2796             migrate_handle_rp_req_pages(ms, NULL, start, len);
2797             break;
2798
2799         case MIG_RP_MSG_REQ_PAGES_ID:
2800             expected_len = 12 + 1; /* header + termination */
2801
2802             if (header_len >= expected_len) {
2803                 start = ldq_be_p(buf);
2804                 len = ldl_be_p(buf + 8);
2805                 /* Now we expect an idstr */
2806                 tmp32 = buf[12]; /* Length of the following idstr */
2807                 buf[13 + tmp32] = '\0';
2808                 expected_len += tmp32;
2809             }
2810             if (header_len != expected_len) {
2811                 error_report("RP: Req_Page_id with length %d expecting %zd",
2812                              header_len, expected_len);
2813                 mark_source_rp_bad(ms);
2814                 goto out;
2815             }
2816             migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len);
2817             break;
2818
2819         case MIG_RP_MSG_RECV_BITMAP:
2820             if (header_len < 1) {
2821                 error_report("%s: missing block name", __func__);
2822                 mark_source_rp_bad(ms);
2823                 goto out;
2824             }
2825             /* Format: len (1B) + idstr (<255B). This ends the idstr. */
2826             buf[buf[0] + 1] = '\0';
2827             if (migrate_handle_rp_recv_bitmap(ms, (char *)(buf + 1))) {
2828                 mark_source_rp_bad(ms);
2829                 goto out;
2830             }
2831             break;
2832
2833         case MIG_RP_MSG_RESUME_ACK:
2834             tmp32 = ldl_be_p(buf);
2835             if (migrate_handle_rp_resume_ack(ms, tmp32)) {
2836                 mark_source_rp_bad(ms);
2837                 goto out;
2838             }
2839             break;
2840
2841         default:
2842             break;
2843         }
2844     }
2845
2846 out:
2847     res = qemu_file_get_error(rp);
2848     if (res) {
2849         if (res == -EIO && migration_in_postcopy()) {
2850             /*
2851              * Maybe there is something we can do: it looks like a
2852              * network down issue, and we pause for a recovery.
2853              */
2854             if (postcopy_pause_return_path_thread(ms)) {
2855                 /* Reload rp, reset the rest */
2856                 if (rp != ms->rp_state.from_dst_file) {
2857                     qemu_fclose(rp);
2858                     rp = ms->rp_state.from_dst_file;
2859                 }
2860                 ms->rp_state.error = false;
2861                 goto retry;
2862             }
2863         }
2864
2865         trace_source_return_path_thread_bad_end();
2866         mark_source_rp_bad(ms);
2867     }
2868
2869     trace_source_return_path_thread_end();
2870     ms->rp_state.from_dst_file = NULL;
2871     qemu_fclose(rp);
2872     rcu_unregister_thread();
2873     return NULL;
2874 }
2875
2876 static int open_return_path_on_source(MigrationState *ms,
2877                                       bool create_thread)
2878 {
2879
2880     ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file);
2881     if (!ms->rp_state.from_dst_file) {
2882         return -1;
2883     }
2884
2885     trace_open_return_path_on_source();
2886
2887     if (!create_thread) {
2888         /* We're done */
2889         return 0;
2890     }
2891
2892     qemu_thread_create(&ms->rp_state.rp_thread, "return path",
2893                        source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
2894
2895     trace_open_return_path_on_source_continue();
2896
2897     return 0;
2898 }
2899
2900 /* Returns 0 if the RP was ok, otherwise there was an error on the RP */
2901 static int await_return_path_close_on_source(MigrationState *ms)
2902 {
2903     /*
2904      * If this is a normal exit then the destination will send a SHUT and the
2905      * rp_thread will exit, however if there's an error we need to cause
2906      * it to exit.
2907      */
2908     if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) {
2909         /*
2910          * shutdown(2), if we have it, will cause it to unblock if it's stuck
2911          * waiting for the destination.
2912          */
2913         qemu_file_shutdown(ms->rp_state.from_dst_file);
2914         mark_source_rp_bad(ms);
2915     }
2916     trace_await_return_path_close_on_source_joining();
2917     qemu_thread_join(&ms->rp_state.rp_thread);
2918     trace_await_return_path_close_on_source_close();
2919     return ms->rp_state.error;
2920 }
2921
2922 /*
2923  * Switch from normal iteration to postcopy
2924  * Returns non-0 on error
2925  */
2926 static int postcopy_start(MigrationState *ms)
2927 {
2928     int ret;
2929     QIOChannelBuffer *bioc;
2930     QEMUFile *fb;
2931     int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2932     int64_t bandwidth = migrate_max_postcopy_bandwidth();
2933     bool restart_block = false;
2934     int cur_state = MIGRATION_STATUS_ACTIVE;
2935     if (!migrate_pause_before_switchover()) {
2936         migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
2937                           MIGRATION_STATUS_POSTCOPY_ACTIVE);
2938     }
2939
2940     trace_postcopy_start();
2941     qemu_mutex_lock_iothread();
2942     trace_postcopy_start_set_run();
2943
2944     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
2945     global_state_store();
2946     ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
2947     if (ret < 0) {
2948         goto fail;
2949     }
2950
2951     ret = migration_maybe_pause(ms, &cur_state,
2952                                 MIGRATION_STATUS_POSTCOPY_ACTIVE);
2953     if (ret < 0) {
2954         goto fail;
2955     }
2956
2957     ret = bdrv_inactivate_all();
2958     if (ret < 0) {
2959         goto fail;
2960     }
2961     restart_block = true;
2962
2963     /*
2964      * Cause any non-postcopiable, but iterative devices to
2965      * send out their final data.
2966      */
2967     qemu_savevm_state_complete_precopy(ms->to_dst_file, true, false);
2968
2969     /*
2970      * in Finish migrate and with the io-lock held everything should
2971      * be quiet, but we've potentially still got dirty pages and we
2972      * need to tell the destination to throw any pages it's already received
2973      * that are dirty
2974      */
2975     if (migrate_postcopy_ram()) {
2976         if (ram_postcopy_send_discard_bitmap(ms)) {
2977             error_report("postcopy send discard bitmap failed");
2978             goto fail;
2979         }
2980     }
2981
2982     /*
2983      * send rest of state - note things that are doing postcopy
2984      * will notice we're in POSTCOPY_ACTIVE and not actually
2985      * wrap their state up here
2986      */
2987     /* 0 max-postcopy-bandwidth means unlimited */
2988     if (!bandwidth) {
2989         qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX);
2990     } else {
2991         qemu_file_set_rate_limit(ms->to_dst_file, bandwidth / XFER_LIMIT_RATIO);
2992     }
2993     if (migrate_postcopy_ram()) {
2994         /* Ping just for debugging, helps line traces up */
2995         qemu_savevm_send_ping(ms->to_dst_file, 2);
2996     }
2997
2998     /*
2999      * While loading the device state we may trigger page transfer
3000      * requests and the fd must be free to process those, and thus
3001      * the destination must read the whole device state off the fd before
3002      * it starts processing it.  Unfortunately the ad-hoc migration format
3003      * doesn't allow the destination to know the size to read without fully
3004      * parsing it through each devices load-state code (especially the open
3005      * coded devices that use get/put).
3006      * So we wrap the device state up in a package with a length at the start;
3007      * to do this we use a qemu_buf to hold the whole of the device state.
3008      */
3009     bioc = qio_channel_buffer_new(4096);
3010     qio_channel_set_name(QIO_CHANNEL(bioc), "migration-postcopy-buffer");
3011     fb = qemu_fopen_channel_output(QIO_CHANNEL(bioc));
3012     object_unref(OBJECT(bioc));
3013
3014     /*
3015      * Make sure the receiver can get incoming pages before we send the rest
3016      * of the state
3017      */
3018     qemu_savevm_send_postcopy_listen(fb);
3019
3020     qemu_savevm_state_complete_precopy(fb, false, false);
3021     if (migrate_postcopy_ram()) {
3022         qemu_savevm_send_ping(fb, 3);
3023     }
3024
3025     qemu_savevm_send_postcopy_run(fb);
3026
3027     /* <><> end of stuff going into the package */
3028
3029     /* Last point of recovery; as soon as we send the package the destination
3030      * can open devices and potentially start running.
3031      * Lets just check again we've not got any errors.
3032      */
3033     ret = qemu_file_get_error(ms->to_dst_file);
3034     if (ret) {
3035         error_report("postcopy_start: Migration stream errored (pre package)");
3036         goto fail_closefb;
3037     }
3038
3039     restart_block = false;
3040
3041     /* Now send that blob */
3042     if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
3043         goto fail_closefb;
3044     }
3045     qemu_fclose(fb);
3046
3047     /* Send a notify to give a chance for anything that needs to happen
3048      * at the transition to postcopy and after the device state; in particular
3049      * spice needs to trigger a transition now
3050      */
3051     ms->postcopy_after_devices = true;
3052     notifier_list_notify(&migration_state_notifiers, ms);
3053
3054     ms->downtime =  qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop;
3055
3056     qemu_mutex_unlock_iothread();
3057
3058     if (migrate_postcopy_ram()) {
3059         /*
3060          * Although this ping is just for debug, it could potentially be
3061          * used for getting a better measurement of downtime at the source.
3062          */
3063         qemu_savevm_send_ping(ms->to_dst_file, 4);
3064     }
3065
3066     if (migrate_release_ram()) {
3067         ram_postcopy_migrated_memory_release(ms);
3068     }
3069
3070     ret = qemu_file_get_error(ms->to_dst_file);
3071     if (ret) {
3072         error_report("postcopy_start: Migration stream errored");
3073         migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3074                               MIGRATION_STATUS_FAILED);
3075     }
3076
3077     return ret;
3078
3079 fail_closefb:
3080     qemu_fclose(fb);
3081 fail:
3082     migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3083                           MIGRATION_STATUS_FAILED);
3084     if (restart_block) {
3085         /* A failure happened early enough that we know the destination hasn't
3086          * accessed block devices, so we're safe to recover.
3087          */
3088         Error *local_err = NULL;
3089
3090         bdrv_invalidate_cache_all(&local_err);
3091         if (local_err) {
3092             error_report_err(local_err);
3093         }
3094     }
3095     qemu_mutex_unlock_iothread();
3096     return -1;
3097 }
3098
3099 /**
3100  * migration_maybe_pause: Pause if required to by
3101  * migrate_pause_before_switchover called with the iothread locked
3102  * Returns: 0 on success
3103  */
3104 static int migration_maybe_pause(MigrationState *s,
3105                                  int *current_active_state,
3106                                  int new_state)
3107 {
3108     if (!migrate_pause_before_switchover()) {
3109         return 0;
3110     }
3111
3112     /* Since leaving this state is not atomic with posting the semaphore
3113      * it's possible that someone could have issued multiple migrate_continue
3114      * and the semaphore is incorrectly positive at this point;
3115      * the docs say it's undefined to reinit a semaphore that's already
3116      * init'd, so use timedwait to eat up any existing posts.
3117      */
3118     while (qemu_sem_timedwait(&s->pause_sem, 1) == 0) {
3119         /* This block intentionally left blank */
3120     }
3121
3122     /*
3123      * If the migration is cancelled when it is in the completion phase,
3124      * the migration state is set to MIGRATION_STATUS_CANCELLING.
3125      * So we don't need to wait a semaphore, otherwise we would always
3126      * wait for the 'pause_sem' semaphore.
3127      */
3128     if (s->state != MIGRATION_STATUS_CANCELLING) {
3129         qemu_mutex_unlock_iothread();
3130         migrate_set_state(&s->state, *current_active_state,
3131                           MIGRATION_STATUS_PRE_SWITCHOVER);
3132         qemu_sem_wait(&s->pause_sem);
3133         migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER,
3134                           new_state);
3135         *current_active_state = new_state;
3136         qemu_mutex_lock_iothread();
3137     }
3138
3139     return s->state == new_state ? 0 : -EINVAL;
3140 }
3141
3142 /**
3143  * migration_completion: Used by migration_thread when there's not much left.
3144  *   The caller 'breaks' the loop when this returns.
3145  *
3146  * @s: Current migration state
3147  */
3148 static void migration_completion(MigrationState *s)
3149 {
3150     int ret;
3151     int current_active_state = s->state;
3152
3153     if (s->state == MIGRATION_STATUS_ACTIVE) {
3154         qemu_mutex_lock_iothread();
3155         s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3156         qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3157         s->vm_was_running = runstate_is_running();
3158         ret = global_state_store();
3159
3160         if (!ret) {
3161             bool inactivate = !migrate_colo_enabled();
3162             ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
3163             if (ret >= 0) {
3164                 ret = migration_maybe_pause(s, &current_active_state,
3165                                             MIGRATION_STATUS_DEVICE);
3166             }
3167             if (ret >= 0) {
3168                 qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3169                 ret = qemu_savevm_state_complete_precopy(s->to_dst_file, false,
3170                                                          inactivate);
3171             }
3172             if (inactivate && ret >= 0) {
3173                 s->block_inactive = true;
3174             }
3175         }
3176         qemu_mutex_unlock_iothread();
3177
3178         if (ret < 0) {
3179             goto fail;
3180         }
3181     } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3182         trace_migration_completion_postcopy_end();
3183
3184         qemu_savevm_state_complete_postcopy(s->to_dst_file);
3185         trace_migration_completion_postcopy_end_after_complete();
3186     } else if (s->state == MIGRATION_STATUS_CANCELLING) {
3187         goto fail;
3188     }
3189
3190     /*
3191      * If rp was opened we must clean up the thread before
3192      * cleaning everything else up (since if there are no failures
3193      * it will wait for the destination to send it's status in
3194      * a SHUT command).
3195      */
3196     if (s->rp_state.from_dst_file) {
3197         int rp_error;
3198         trace_migration_return_path_end_before();
3199         rp_error = await_return_path_close_on_source(s);
3200         trace_migration_return_path_end_after(rp_error);
3201         if (rp_error) {
3202             goto fail_invalidate;
3203         }
3204     }
3205
3206     if (qemu_file_get_error(s->to_dst_file)) {
3207         trace_migration_completion_file_err();
3208         goto fail_invalidate;
3209     }
3210
3211     if (!migrate_colo_enabled()) {
3212         migrate_set_state(&s->state, current_active_state,
3213                           MIGRATION_STATUS_COMPLETED);
3214     }
3215
3216     return;
3217
3218 fail_invalidate:
3219     /* If not doing postcopy, vm_start() will be called: let's regain
3220      * control on images.
3221      */
3222     if (s->state == MIGRATION_STATUS_ACTIVE ||
3223         s->state == MIGRATION_STATUS_DEVICE) {
3224         Error *local_err = NULL;
3225
3226         qemu_mutex_lock_iothread();
3227         bdrv_invalidate_cache_all(&local_err);
3228         if (local_err) {
3229             error_report_err(local_err);
3230         } else {
3231             s->block_inactive = false;
3232         }
3233         qemu_mutex_unlock_iothread();
3234     }
3235
3236 fail:
3237     migrate_set_state(&s->state, current_active_state,
3238                       MIGRATION_STATUS_FAILED);
3239 }
3240
3241 /**
3242  * bg_migration_completion: Used by bg_migration_thread when after all the
3243  *   RAM has been saved. The caller 'breaks' the loop when this returns.
3244  *
3245  * @s: Current migration state
3246  */
3247 static void bg_migration_completion(MigrationState *s)
3248 {
3249     int current_active_state = s->state;
3250
3251     /*
3252      * Stop tracking RAM writes - un-protect memory, un-register UFFD
3253      * memory ranges, flush kernel wait queues and wake up threads
3254      * waiting for write fault to be resolved.
3255      */
3256     ram_write_tracking_stop();
3257
3258     if (s->state == MIGRATION_STATUS_ACTIVE) {
3259         /*
3260          * By this moment we have RAM content saved into the migration stream.
3261          * The next step is to flush the non-RAM content (device state)
3262          * right after the ram content. The device state has been stored into
3263          * the temporary buffer before RAM saving started.
3264          */
3265         qemu_put_buffer(s->to_dst_file, s->bioc->data, s->bioc->usage);
3266         qemu_fflush(s->to_dst_file);
3267     } else if (s->state == MIGRATION_STATUS_CANCELLING) {
3268         goto fail;
3269     }
3270
3271     if (qemu_file_get_error(s->to_dst_file)) {
3272         trace_migration_completion_file_err();
3273         goto fail;
3274     }
3275
3276     migrate_set_state(&s->state, current_active_state,
3277                       MIGRATION_STATUS_COMPLETED);
3278     return;
3279
3280 fail:
3281     migrate_set_state(&s->state, current_active_state,
3282                       MIGRATION_STATUS_FAILED);
3283 }
3284
3285 bool migrate_colo_enabled(void)
3286 {
3287     MigrationState *s = migrate_get_current();
3288     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_COLO];
3289 }
3290
3291 typedef enum MigThrError {
3292     /* No error detected */
3293     MIG_THR_ERR_NONE = 0,
3294     /* Detected error, but resumed successfully */
3295     MIG_THR_ERR_RECOVERED = 1,
3296     /* Detected fatal error, need to exit */
3297     MIG_THR_ERR_FATAL = 2,
3298 } MigThrError;
3299
3300 static int postcopy_resume_handshake(MigrationState *s)
3301 {
3302     qemu_savevm_send_postcopy_resume(s->to_dst_file);
3303
3304     while (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3305         qemu_sem_wait(&s->rp_state.rp_sem);
3306     }
3307
3308     if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3309         return 0;
3310     }
3311
3312     return -1;
3313 }
3314
3315 /* Return zero if success, or <0 for error */
3316 static int postcopy_do_resume(MigrationState *s)
3317 {
3318     int ret;
3319
3320     /*
3321      * Call all the resume_prepare() hooks, so that modules can be
3322      * ready for the migration resume.
3323      */
3324     ret = qemu_savevm_state_resume_prepare(s);
3325     if (ret) {
3326         error_report("%s: resume_prepare() failure detected: %d",
3327                      __func__, ret);
3328         return ret;
3329     }
3330
3331     /*
3332      * Last handshake with destination on the resume (destination will
3333      * switch to postcopy-active afterwards)
3334      */
3335     ret = postcopy_resume_handshake(s);
3336     if (ret) {
3337         error_report("%s: handshake failed: %d", __func__, ret);
3338         return ret;
3339     }
3340
3341     return 0;
3342 }
3343
3344 /*
3345  * We don't return until we are in a safe state to continue current
3346  * postcopy migration.  Returns MIG_THR_ERR_RECOVERED if recovered, or
3347  * MIG_THR_ERR_FATAL if unrecovery failure happened.
3348  */
3349 static MigThrError postcopy_pause(MigrationState *s)
3350 {
3351     assert(s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
3352
3353     while (true) {
3354         QEMUFile *file;
3355
3356         /* Current channel is possibly broken. Release it. */
3357         assert(s->to_dst_file);
3358         qemu_mutex_lock(&s->qemu_file_lock);
3359         file = s->to_dst_file;
3360         s->to_dst_file = NULL;
3361         qemu_mutex_unlock(&s->qemu_file_lock);
3362
3363         qemu_file_shutdown(file);
3364         qemu_fclose(file);
3365
3366         migrate_set_state(&s->state, s->state,
3367                           MIGRATION_STATUS_POSTCOPY_PAUSED);
3368
3369         error_report("Detected IO failure for postcopy. "
3370                      "Migration paused.");
3371
3372         /*
3373          * We wait until things fixed up. Then someone will setup the
3374          * status back for us.
3375          */
3376         while (s->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
3377             qemu_sem_wait(&s->postcopy_pause_sem);
3378         }
3379
3380         if (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3381             /* Woken up by a recover procedure. Give it a shot */
3382
3383             /*
3384              * Firstly, let's wake up the return path now, with a new
3385              * return path channel.
3386              */
3387             qemu_sem_post(&s->postcopy_pause_rp_sem);
3388
3389             /* Do the resume logic */
3390             if (postcopy_do_resume(s) == 0) {
3391                 /* Let's continue! */
3392                 trace_postcopy_pause_continued();
3393                 return MIG_THR_ERR_RECOVERED;
3394             } else {
3395                 /*
3396                  * Something wrong happened during the recovery, let's
3397                  * pause again. Pause is always better than throwing
3398                  * data away.
3399                  */
3400                 continue;
3401             }
3402         } else {
3403             /* This is not right... Time to quit. */
3404             return MIG_THR_ERR_FATAL;
3405         }
3406     }
3407 }
3408
3409 static MigThrError migration_detect_error(MigrationState *s)
3410 {
3411     int ret;
3412     int state = s->state;
3413     Error *local_error = NULL;
3414
3415     if (state == MIGRATION_STATUS_CANCELLING ||
3416         state == MIGRATION_STATUS_CANCELLED) {
3417         /* End the migration, but don't set the state to failed */
3418         return MIG_THR_ERR_FATAL;
3419     }
3420
3421     /* Try to detect any file errors */
3422     ret = qemu_file_get_error_obj(s->to_dst_file, &local_error);
3423     if (!ret) {
3424         /* Everything is fine */
3425         assert(!local_error);
3426         return MIG_THR_ERR_NONE;
3427     }
3428
3429     if (local_error) {
3430         migrate_set_error(s, local_error);
3431         error_free(local_error);
3432     }
3433
3434     if (state == MIGRATION_STATUS_POSTCOPY_ACTIVE && ret == -EIO) {
3435         /*
3436          * For postcopy, we allow the network to be down for a
3437          * while. After that, it can be continued by a
3438          * recovery phase.
3439          */
3440         return postcopy_pause(s);
3441     } else {
3442         /*
3443          * For precopy (or postcopy with error outside IO), we fail
3444          * with no time.
3445          */
3446         migrate_set_state(&s->state, state, MIGRATION_STATUS_FAILED);
3447         trace_migration_thread_file_err();
3448
3449         /* Time to stop the migration, now. */
3450         return MIG_THR_ERR_FATAL;
3451     }
3452 }
3453
3454 /* How many bytes have we transferred since the beginning of the migration */
3455 static uint64_t migration_total_bytes(MigrationState *s)
3456 {
3457     return qemu_ftell(s->to_dst_file) + ram_counters.multifd_bytes;
3458 }
3459
3460 static void migration_calculate_complete(MigrationState *s)
3461 {
3462     uint64_t bytes = migration_total_bytes(s);
3463     int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3464     int64_t transfer_time;
3465
3466     s->total_time = end_time - s->start_time;
3467     if (!s->downtime) {
3468         /*
3469          * It's still not set, so we are precopy migration.  For
3470          * postcopy, downtime is calculated during postcopy_start().
3471          */
3472         s->downtime = end_time - s->downtime_start;
3473     }
3474
3475     transfer_time = s->total_time - s->setup_time;
3476     if (transfer_time) {
3477         s->mbps = ((double) bytes * 8.0) / transfer_time / 1000;
3478     }
3479 }
3480
3481 static void update_iteration_initial_status(MigrationState *s)
3482 {
3483     /*
3484      * Update these three fields at the same time to avoid mismatch info lead
3485      * wrong speed calculation.
3486      */
3487     s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3488     s->iteration_initial_bytes = migration_total_bytes(s);
3489     s->iteration_initial_pages = ram_get_total_transferred_pages();
3490 }
3491
3492 static void migration_update_counters(MigrationState *s,
3493                                       int64_t current_time)
3494 {
3495     uint64_t transferred, transferred_pages, time_spent;
3496     uint64_t current_bytes; /* bytes transferred since the beginning */
3497     double bandwidth;
3498
3499     if (current_time < s->iteration_start_time + BUFFER_DELAY) {
3500         return;
3501     }
3502
3503     current_bytes = migration_total_bytes(s);
3504     transferred = current_bytes - s->iteration_initial_bytes;
3505     time_spent = current_time - s->iteration_start_time;
3506     bandwidth = (double)transferred / time_spent;
3507     s->threshold_size = bandwidth * s->parameters.downtime_limit;
3508
3509     s->mbps = (((double) transferred * 8.0) /
3510                ((double) time_spent / 1000.0)) / 1000.0 / 1000.0;
3511
3512     transferred_pages = ram_get_total_transferred_pages() -
3513                             s->iteration_initial_pages;
3514     s->pages_per_second = (double) transferred_pages /
3515                              (((double) time_spent / 1000.0));
3516
3517     /*
3518      * if we haven't sent anything, we don't want to
3519      * recalculate. 10000 is a small enough number for our purposes
3520      */
3521     if (ram_counters.dirty_pages_rate && transferred > 10000) {
3522         s->expected_downtime = ram_counters.remaining / bandwidth;
3523     }
3524
3525     qemu_file_reset_rate_limit(s->to_dst_file);
3526
3527     update_iteration_initial_status(s);
3528
3529     trace_migrate_transferred(transferred, time_spent,
3530                               bandwidth, s->threshold_size);
3531 }
3532
3533 /* Migration thread iteration status */
3534 typedef enum {
3535     MIG_ITERATE_RESUME,         /* Resume current iteration */
3536     MIG_ITERATE_SKIP,           /* Skip current iteration */
3537     MIG_ITERATE_BREAK,          /* Break the loop */
3538 } MigIterateState;
3539
3540 /*
3541  * Return true if continue to the next iteration directly, false
3542  * otherwise.
3543  */
3544 static MigIterateState migration_iteration_run(MigrationState *s)
3545 {
3546     uint64_t pending_size, pend_pre, pend_compat, pend_post;
3547     bool in_postcopy = s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE;
3548
3549     qemu_savevm_state_pending(s->to_dst_file, s->threshold_size, &pend_pre,
3550                               &pend_compat, &pend_post);
3551     pending_size = pend_pre + pend_compat + pend_post;
3552
3553     trace_migrate_pending(pending_size, s->threshold_size,
3554                           pend_pre, pend_compat, pend_post);
3555
3556     if (pending_size && pending_size >= s->threshold_size) {
3557         /* Still a significant amount to transfer */
3558         if (!in_postcopy && pend_pre <= s->threshold_size &&
3559             qatomic_read(&s->start_postcopy)) {
3560             if (postcopy_start(s)) {
3561                 error_report("%s: postcopy failed to start", __func__);
3562             }
3563             return MIG_ITERATE_SKIP;
3564         }
3565         /* Just another iteration step */
3566         qemu_savevm_state_iterate(s->to_dst_file, in_postcopy);
3567     } else {
3568         trace_migration_thread_low_pending(pending_size);
3569         migration_completion(s);
3570         return MIG_ITERATE_BREAK;
3571     }
3572
3573     return MIG_ITERATE_RESUME;
3574 }
3575
3576 static void migration_iteration_finish(MigrationState *s)
3577 {
3578     /* If we enabled cpu throttling for auto-converge, turn it off. */
3579     cpu_throttle_stop();
3580
3581     qemu_mutex_lock_iothread();
3582     switch (s->state) {
3583     case MIGRATION_STATUS_COMPLETED:
3584         migration_calculate_complete(s);
3585         runstate_set(RUN_STATE_POSTMIGRATE);
3586         break;
3587
3588     case MIGRATION_STATUS_ACTIVE:
3589         /*
3590          * We should really assert here, but since it's during
3591          * migration, let's try to reduce the usage of assertions.
3592          */
3593         if (!migrate_colo_enabled()) {
3594             error_report("%s: critical error: calling COLO code without "
3595                          "COLO enabled", __func__);
3596         }
3597         migrate_start_colo_process(s);
3598         /*
3599          * Fixme: we will run VM in COLO no matter its old running state.
3600          * After exited COLO, we will keep running.
3601          */
3602         s->vm_was_running = true;
3603         /* Fallthrough */
3604     case MIGRATION_STATUS_FAILED:
3605     case MIGRATION_STATUS_CANCELLED:
3606     case MIGRATION_STATUS_CANCELLING:
3607         if (s->vm_was_running) {
3608             vm_start();
3609         } else {
3610             if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
3611                 runstate_set(RUN_STATE_POSTMIGRATE);
3612             }
3613         }
3614         break;
3615
3616     default:
3617         /* Should not reach here, but if so, forgive the VM. */
3618         error_report("%s: Unknown ending state %d", __func__, s->state);
3619         break;
3620     }
3621     migrate_fd_cleanup_schedule(s);
3622     qemu_mutex_unlock_iothread();
3623 }
3624
3625 static void bg_migration_iteration_finish(MigrationState *s)
3626 {
3627     qemu_mutex_lock_iothread();
3628     switch (s->state) {
3629     case MIGRATION_STATUS_COMPLETED:
3630         migration_calculate_complete(s);
3631         break;
3632
3633     case MIGRATION_STATUS_ACTIVE:
3634     case MIGRATION_STATUS_FAILED:
3635     case MIGRATION_STATUS_CANCELLED:
3636     case MIGRATION_STATUS_CANCELLING:
3637         break;
3638
3639     default:
3640         /* Should not reach here, but if so, forgive the VM. */
3641         error_report("%s: Unknown ending state %d", __func__, s->state);
3642         break;
3643     }
3644
3645     migrate_fd_cleanup_schedule(s);
3646     qemu_mutex_unlock_iothread();
3647 }
3648
3649 /*
3650  * Return true if continue to the next iteration directly, false
3651  * otherwise.
3652  */
3653 static MigIterateState bg_migration_iteration_run(MigrationState *s)
3654 {
3655     int res;
3656
3657     res = qemu_savevm_state_iterate(s->to_dst_file, false);
3658     if (res > 0) {
3659         bg_migration_completion(s);
3660         return MIG_ITERATE_BREAK;
3661     }
3662
3663     return MIG_ITERATE_RESUME;
3664 }
3665
3666 void migration_make_urgent_request(void)
3667 {
3668     qemu_sem_post(&migrate_get_current()->rate_limit_sem);
3669 }
3670
3671 void migration_consume_urgent_request(void)
3672 {
3673     qemu_sem_wait(&migrate_get_current()->rate_limit_sem);
3674 }
3675
3676 /* Returns true if the rate limiting was broken by an urgent request */
3677 bool migration_rate_limit(void)
3678 {
3679     int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3680     MigrationState *s = migrate_get_current();
3681
3682     bool urgent = false;
3683     migration_update_counters(s, now);
3684     if (qemu_file_rate_limit(s->to_dst_file)) {
3685
3686         if (qemu_file_get_error(s->to_dst_file)) {
3687             return false;
3688         }
3689         /*
3690          * Wait for a delay to do rate limiting OR
3691          * something urgent to post the semaphore.
3692          */
3693         int ms = s->iteration_start_time + BUFFER_DELAY - now;
3694         trace_migration_rate_limit_pre(ms);
3695         if (qemu_sem_timedwait(&s->rate_limit_sem, ms) == 0) {
3696             /*
3697              * We were woken by one or more urgent things but
3698              * the timedwait will have consumed one of them.
3699              * The service routine for the urgent wake will dec
3700              * the semaphore itself for each item it consumes,
3701              * so add this one we just eat back.
3702              */
3703             qemu_sem_post(&s->rate_limit_sem);
3704             urgent = true;
3705         }
3706         trace_migration_rate_limit_post(urgent);
3707     }
3708     return urgent;
3709 }
3710
3711 /*
3712  * Master migration thread on the source VM.
3713  * It drives the migration and pumps the data down the outgoing channel.
3714  */
3715 static void *migration_thread(void *opaque)
3716 {
3717     MigrationState *s = opaque;
3718     int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3719     MigThrError thr_error;
3720     bool urgent = false;
3721
3722     rcu_register_thread();
3723
3724     object_ref(OBJECT(s));
3725     update_iteration_initial_status(s);
3726
3727     qemu_savevm_state_header(s->to_dst_file);
3728
3729     /*
3730      * If we opened the return path, we need to make sure dst has it
3731      * opened as well.
3732      */
3733     if (s->rp_state.from_dst_file) {
3734         /* Now tell the dest that it should open its end so it can reply */
3735         qemu_savevm_send_open_return_path(s->to_dst_file);
3736
3737         /* And do a ping that will make stuff easier to debug */
3738         qemu_savevm_send_ping(s->to_dst_file, 1);
3739     }
3740
3741     if (migrate_postcopy()) {
3742         /*
3743          * Tell the destination that we *might* want to do postcopy later;
3744          * if the other end can't do postcopy it should fail now, nice and
3745          * early.
3746          */
3747         qemu_savevm_send_postcopy_advise(s->to_dst_file);
3748     }
3749
3750     if (migrate_colo_enabled()) {
3751         /* Notify migration destination that we enable COLO */
3752         qemu_savevm_send_colo_enable(s->to_dst_file);
3753     }
3754
3755     qemu_savevm_state_setup(s->to_dst_file);
3756
3757     if (qemu_savevm_state_guest_unplug_pending()) {
3758         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3759                           MIGRATION_STATUS_WAIT_UNPLUG);
3760
3761         while (s->state == MIGRATION_STATUS_WAIT_UNPLUG &&
3762                qemu_savevm_state_guest_unplug_pending()) {
3763             qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3764         }
3765
3766         migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG,
3767                 MIGRATION_STATUS_ACTIVE);
3768     }
3769
3770     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3771     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3772                       MIGRATION_STATUS_ACTIVE);
3773
3774     trace_migration_thread_setup_complete();
3775
3776     while (migration_is_active(s)) {
3777         if (urgent || !qemu_file_rate_limit(s->to_dst_file)) {
3778             MigIterateState iter_state = migration_iteration_run(s);
3779             if (iter_state == MIG_ITERATE_SKIP) {
3780                 continue;
3781             } else if (iter_state == MIG_ITERATE_BREAK) {
3782                 break;
3783             }
3784         }
3785
3786         /*
3787          * Try to detect any kind of failures, and see whether we
3788          * should stop the migration now.
3789          */
3790         thr_error = migration_detect_error(s);
3791         if (thr_error == MIG_THR_ERR_FATAL) {
3792             /* Stop migration */
3793             break;
3794         } else if (thr_error == MIG_THR_ERR_RECOVERED) {
3795             /*
3796              * Just recovered from a e.g. network failure, reset all
3797              * the local variables. This is important to avoid
3798              * breaking transferred_bytes and bandwidth calculation
3799              */
3800             update_iteration_initial_status(s);
3801         }
3802
3803         urgent = migration_rate_limit();
3804     }
3805
3806     trace_migration_thread_after_loop();
3807     migration_iteration_finish(s);
3808     object_unref(OBJECT(s));
3809     rcu_unregister_thread();
3810     return NULL;
3811 }
3812
3813 static void bg_migration_vm_start_bh(void *opaque)
3814 {
3815     MigrationState *s = opaque;
3816
3817     qemu_bh_delete(s->vm_start_bh);
3818     s->vm_start_bh = NULL;
3819
3820     vm_start();
3821     s->downtime = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - s->downtime_start;
3822 }
3823
3824 /**
3825  * Background snapshot thread, based on live migration code.
3826  * This is an alternative implementation of live migration mechanism
3827  * introduced specifically to support background snapshots.
3828  *
3829  * It takes advantage of userfault_fd write protection mechanism introduced
3830  * in v5.7 kernel. Compared to existing dirty page logging migration much
3831  * lesser stream traffic is produced resulting in smaller snapshot images,
3832  * simply cause of no page duplicates can get into the stream.
3833  *
3834  * Another key point is that generated vmstate stream reflects machine state
3835  * 'frozen' at the beginning of snapshot creation compared to dirty page logging
3836  * mechanism, which effectively results in that saved snapshot is the state of VM
3837  * at the end of the process.
3838  */
3839 static void *bg_migration_thread(void *opaque)
3840 {
3841     MigrationState *s = opaque;
3842     int64_t setup_start;
3843     MigThrError thr_error;
3844     QEMUFile *fb;
3845     bool early_fail = true;
3846
3847     rcu_register_thread();
3848     object_ref(OBJECT(s));
3849
3850     qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3851
3852     setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3853     /*
3854      * We want to save vmstate for the moment when migration has been
3855      * initiated but also we want to save RAM content while VM is running.
3856      * The RAM content should appear first in the vmstate. So, we first
3857      * stash the non-RAM part of the vmstate to the temporary buffer,
3858      * then write RAM part of the vmstate to the migration stream
3859      * with vCPUs running and, finally, write stashed non-RAM part of
3860      * the vmstate from the buffer to the migration stream.
3861      */
3862     s->bioc = qio_channel_buffer_new(128 * 1024);
3863     qio_channel_set_name(QIO_CHANNEL(s->bioc), "vmstate-buffer");
3864     fb = qemu_fopen_channel_output(QIO_CHANNEL(s->bioc));
3865     object_unref(OBJECT(s->bioc));
3866
3867     update_iteration_initial_status(s);
3868
3869     qemu_savevm_state_header(s->to_dst_file);
3870     qemu_savevm_state_setup(s->to_dst_file);
3871
3872     if (qemu_savevm_state_guest_unplug_pending()) {
3873         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3874                           MIGRATION_STATUS_WAIT_UNPLUG);
3875
3876         while (s->state == MIGRATION_STATUS_WAIT_UNPLUG &&
3877                qemu_savevm_state_guest_unplug_pending()) {
3878             qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3879         }
3880
3881         migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG,
3882                           MIGRATION_STATUS_ACTIVE);
3883     } else {
3884         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3885                 MIGRATION_STATUS_ACTIVE);
3886     }
3887     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3888
3889     trace_migration_thread_setup_complete();
3890     s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3891
3892     qemu_mutex_lock_iothread();
3893
3894     /*
3895      * If VM is currently in suspended state, then, to make a valid runstate
3896      * transition in vm_stop_force_state() we need to wakeup it up.
3897      */
3898     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3899     s->vm_was_running = runstate_is_running();
3900
3901     if (global_state_store()) {
3902         goto fail;
3903     }
3904     /* Forcibly stop VM before saving state of vCPUs and devices */
3905     if (vm_stop_force_state(RUN_STATE_PAUSED)) {
3906         goto fail;
3907     }
3908     /*
3909      * Put vCPUs in sync with shadow context structures, then
3910      * save their state to channel-buffer along with devices.
3911      */
3912     cpu_synchronize_all_states();
3913     if (qemu_savevm_state_complete_precopy_non_iterable(fb, false, false)) {
3914         goto fail;
3915     }
3916     /* Now initialize UFFD context and start tracking RAM writes */
3917     if (ram_write_tracking_start()) {
3918         goto fail;
3919     }
3920     early_fail = false;
3921
3922     /*
3923      * Start VM from BH handler to avoid write-fault lock here.
3924      * UFFD-WP protection for the whole RAM is already enabled so
3925      * calling VM state change notifiers from vm_start() would initiate
3926      * writes to virtio VQs memory which is in write-protected region.
3927      */
3928     s->vm_start_bh = qemu_bh_new(bg_migration_vm_start_bh, s);
3929     qemu_bh_schedule(s->vm_start_bh);
3930
3931     qemu_mutex_unlock_iothread();
3932
3933     while (migration_is_active(s)) {
3934         MigIterateState iter_state = bg_migration_iteration_run(s);
3935         if (iter_state == MIG_ITERATE_SKIP) {
3936             continue;
3937         } else if (iter_state == MIG_ITERATE_BREAK) {
3938             break;
3939         }
3940
3941         /*
3942          * Try to detect any kind of failures, and see whether we
3943          * should stop the migration now.
3944          */
3945         thr_error = migration_detect_error(s);
3946         if (thr_error == MIG_THR_ERR_FATAL) {
3947             /* Stop migration */
3948             break;
3949         }
3950
3951         migration_update_counters(s, qemu_clock_get_ms(QEMU_CLOCK_REALTIME));
3952     }
3953
3954     trace_migration_thread_after_loop();
3955
3956 fail:
3957     if (early_fail) {
3958         migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE,
3959                 MIGRATION_STATUS_FAILED);
3960         qemu_mutex_unlock_iothread();
3961     }
3962
3963     bg_migration_iteration_finish(s);
3964
3965     qemu_fclose(fb);
3966     object_unref(OBJECT(s));
3967     rcu_unregister_thread();
3968
3969     return NULL;
3970 }
3971
3972 void migrate_fd_connect(MigrationState *s, Error *error_in)
3973 {
3974     Error *local_err = NULL;
3975     int64_t rate_limit;
3976     bool resume = s->state == MIGRATION_STATUS_POSTCOPY_PAUSED;
3977
3978     s->expected_downtime = s->parameters.downtime_limit;
3979     if (resume) {
3980         assert(s->cleanup_bh);
3981     } else {
3982         assert(!s->cleanup_bh);
3983         s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup_bh, s);
3984     }
3985     if (error_in) {
3986         migrate_fd_error(s, error_in);
3987         migrate_fd_cleanup(s);
3988         return;
3989     }
3990
3991     if (resume) {
3992         /* This is a resumed migration */
3993         rate_limit = s->parameters.max_postcopy_bandwidth /
3994             XFER_LIMIT_RATIO;
3995     } else {
3996         /* This is a fresh new migration */
3997         rate_limit = s->parameters.max_bandwidth / XFER_LIMIT_RATIO;
3998
3999         /* Notify before starting migration thread */
4000         notifier_list_notify(&migration_state_notifiers, s);
4001     }
4002
4003     qemu_file_set_rate_limit(s->to_dst_file, rate_limit);
4004     qemu_file_set_blocking(s->to_dst_file, true);
4005
4006     /*
4007      * Open the return path. For postcopy, it is used exclusively. For
4008      * precopy, only if user specified "return-path" capability would
4009      * QEMU uses the return path.
4010      */
4011     if (migrate_postcopy_ram() || migrate_use_return_path()) {
4012         if (open_return_path_on_source(s, !resume)) {
4013             error_report("Unable to open return-path for postcopy");
4014             migrate_set_state(&s->state, s->state, MIGRATION_STATUS_FAILED);
4015             migrate_fd_cleanup(s);
4016             return;
4017         }
4018     }
4019
4020     if (resume) {
4021         /* Wakeup the main migration thread to do the recovery */
4022         migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
4023                           MIGRATION_STATUS_POSTCOPY_RECOVER);
4024         qemu_sem_post(&s->postcopy_pause_sem);
4025         return;
4026     }
4027
4028     if (multifd_save_setup(&local_err) != 0) {
4029         error_report_err(local_err);
4030         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
4031                           MIGRATION_STATUS_FAILED);
4032         migrate_fd_cleanup(s);
4033         return;
4034     }
4035
4036     if (migrate_background_snapshot()) {
4037         qemu_thread_create(&s->thread, "bg_snapshot",
4038                 bg_migration_thread, s, QEMU_THREAD_JOINABLE);
4039     } else {
4040         qemu_thread_create(&s->thread, "live_migration",
4041                 migration_thread, s, QEMU_THREAD_JOINABLE);
4042     }
4043     s->migration_thread_running = true;
4044 }
4045
4046 void migration_global_dump(Monitor *mon)
4047 {
4048     MigrationState *ms = migrate_get_current();
4049
4050     monitor_printf(mon, "globals:\n");
4051     monitor_printf(mon, "store-global-state: %s\n",
4052                    ms->store_global_state ? "on" : "off");
4053     monitor_printf(mon, "only-migratable: %s\n",
4054                    only_migratable ? "on" : "off");
4055     monitor_printf(mon, "send-configuration: %s\n",
4056                    ms->send_configuration ? "on" : "off");
4057     monitor_printf(mon, "send-section-footer: %s\n",
4058                    ms->send_section_footer ? "on" : "off");
4059     monitor_printf(mon, "decompress-error-check: %s\n",
4060                    ms->decompress_error_check ? "on" : "off");
4061     monitor_printf(mon, "clear-bitmap-shift: %u\n",
4062                    ms->clear_bitmap_shift);
4063 }
4064
4065 #define DEFINE_PROP_MIG_CAP(name, x)             \
4066     DEFINE_PROP_BOOL(name, MigrationState, enabled_capabilities[x], false)
4067
4068 static Property migration_properties[] = {
4069     DEFINE_PROP_BOOL("store-global-state", MigrationState,
4070                      store_global_state, true),
4071     DEFINE_PROP_BOOL("send-configuration", MigrationState,
4072                      send_configuration, true),
4073     DEFINE_PROP_BOOL("send-section-footer", MigrationState,
4074                      send_section_footer, true),
4075     DEFINE_PROP_BOOL("decompress-error-check", MigrationState,
4076                       decompress_error_check, true),
4077     DEFINE_PROP_UINT8("x-clear-bitmap-shift", MigrationState,
4078                       clear_bitmap_shift, CLEAR_BITMAP_SHIFT_DEFAULT),
4079
4080     /* Migration parameters */
4081     DEFINE_PROP_UINT8("x-compress-level", MigrationState,
4082                       parameters.compress_level,
4083                       DEFAULT_MIGRATE_COMPRESS_LEVEL),
4084     DEFINE_PROP_UINT8("x-compress-threads", MigrationState,
4085                       parameters.compress_threads,
4086                       DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT),
4087     DEFINE_PROP_BOOL("x-compress-wait-thread", MigrationState,
4088                       parameters.compress_wait_thread, true),
4089     DEFINE_PROP_UINT8("x-decompress-threads", MigrationState,
4090                       parameters.decompress_threads,
4091                       DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT),
4092     DEFINE_PROP_UINT8("x-throttle-trigger-threshold", MigrationState,
4093                       parameters.throttle_trigger_threshold,
4094                       DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD),
4095     DEFINE_PROP_UINT8("x-cpu-throttle-initial", MigrationState,
4096                       parameters.cpu_throttle_initial,
4097                       DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL),
4098     DEFINE_PROP_UINT8("x-cpu-throttle-increment", MigrationState,
4099                       parameters.cpu_throttle_increment,
4100                       DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT),
4101     DEFINE_PROP_BOOL("x-cpu-throttle-tailslow", MigrationState,
4102                       parameters.cpu_throttle_tailslow, false),
4103     DEFINE_PROP_SIZE("x-max-bandwidth", MigrationState,
4104                       parameters.max_bandwidth, MAX_THROTTLE),
4105     DEFINE_PROP_UINT64("x-downtime-limit", MigrationState,
4106                       parameters.downtime_limit,
4107                       DEFAULT_MIGRATE_SET_DOWNTIME),
4108     DEFINE_PROP_UINT32("x-checkpoint-delay", MigrationState,
4109                       parameters.x_checkpoint_delay,
4110                       DEFAULT_MIGRATE_X_CHECKPOINT_DELAY),
4111     DEFINE_PROP_UINT8("multifd-channels", MigrationState,
4112                       parameters.multifd_channels,
4113                       DEFAULT_MIGRATE_MULTIFD_CHANNELS),
4114     DEFINE_PROP_MULTIFD_COMPRESSION("multifd-compression", MigrationState,
4115                       parameters.multifd_compression,
4116                       DEFAULT_MIGRATE_MULTIFD_COMPRESSION),
4117     DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState,
4118                       parameters.multifd_zlib_level,
4119                       DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL),
4120     DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState,
4121                       parameters.multifd_zstd_level,
4122                       DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL),
4123     DEFINE_PROP_SIZE("xbzrle-cache-size", MigrationState,
4124                       parameters.xbzrle_cache_size,
4125                       DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE),
4126     DEFINE_PROP_SIZE("max-postcopy-bandwidth", MigrationState,
4127                       parameters.max_postcopy_bandwidth,
4128                       DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH),
4129     DEFINE_PROP_UINT8("max-cpu-throttle", MigrationState,
4130                       parameters.max_cpu_throttle,
4131                       DEFAULT_MIGRATE_MAX_CPU_THROTTLE),
4132     DEFINE_PROP_SIZE("announce-initial", MigrationState,
4133                       parameters.announce_initial,
4134                       DEFAULT_MIGRATE_ANNOUNCE_INITIAL),
4135     DEFINE_PROP_SIZE("announce-max", MigrationState,
4136                       parameters.announce_max,
4137                       DEFAULT_MIGRATE_ANNOUNCE_MAX),
4138     DEFINE_PROP_SIZE("announce-rounds", MigrationState,
4139                       parameters.announce_rounds,
4140                       DEFAULT_MIGRATE_ANNOUNCE_ROUNDS),
4141     DEFINE_PROP_SIZE("announce-step", MigrationState,
4142                       parameters.announce_step,
4143                       DEFAULT_MIGRATE_ANNOUNCE_STEP),
4144
4145     /* Migration capabilities */
4146     DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
4147     DEFINE_PROP_MIG_CAP("x-rdma-pin-all", MIGRATION_CAPABILITY_RDMA_PIN_ALL),
4148     DEFINE_PROP_MIG_CAP("x-auto-converge", MIGRATION_CAPABILITY_AUTO_CONVERGE),
4149     DEFINE_PROP_MIG_CAP("x-zero-blocks", MIGRATION_CAPABILITY_ZERO_BLOCKS),
4150     DEFINE_PROP_MIG_CAP("x-compress", MIGRATION_CAPABILITY_COMPRESS),
4151     DEFINE_PROP_MIG_CAP("x-events", MIGRATION_CAPABILITY_EVENTS),
4152     DEFINE_PROP_MIG_CAP("x-postcopy-ram", MIGRATION_CAPABILITY_POSTCOPY_RAM),
4153     DEFINE_PROP_MIG_CAP("x-colo", MIGRATION_CAPABILITY_X_COLO),
4154     DEFINE_PROP_MIG_CAP("x-release-ram", MIGRATION_CAPABILITY_RELEASE_RAM),
4155     DEFINE_PROP_MIG_CAP("x-block", MIGRATION_CAPABILITY_BLOCK),
4156     DEFINE_PROP_MIG_CAP("x-return-path", MIGRATION_CAPABILITY_RETURN_PATH),
4157     DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD),
4158     DEFINE_PROP_MIG_CAP("x-background-snapshot",
4159             MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT),
4160
4161     DEFINE_PROP_END_OF_LIST(),
4162 };
4163
4164 static void migration_class_init(ObjectClass *klass, void *data)
4165 {
4166     DeviceClass *dc = DEVICE_CLASS(klass);
4167
4168     dc->user_creatable = false;
4169     device_class_set_props(dc, migration_properties);
4170 }
4171
4172 static void migration_instance_finalize(Object *obj)
4173 {
4174     MigrationState *ms = MIGRATION_OBJ(obj);
4175     MigrationParameters *params = &ms->parameters;
4176
4177     qemu_mutex_destroy(&ms->error_mutex);
4178     qemu_mutex_destroy(&ms->qemu_file_lock);
4179     g_free(params->tls_hostname);
4180     g_free(params->tls_creds);
4181     qemu_sem_destroy(&ms->wait_unplug_sem);
4182     qemu_sem_destroy(&ms->rate_limit_sem);
4183     qemu_sem_destroy(&ms->pause_sem);
4184     qemu_sem_destroy(&ms->postcopy_pause_sem);
4185     qemu_sem_destroy(&ms->postcopy_pause_rp_sem);
4186     qemu_sem_destroy(&ms->rp_state.rp_sem);
4187     error_free(ms->error);
4188 }
4189
4190 static void migration_instance_init(Object *obj)
4191 {
4192     MigrationState *ms = MIGRATION_OBJ(obj);
4193     MigrationParameters *params = &ms->parameters;
4194
4195     ms->state = MIGRATION_STATUS_NONE;
4196     ms->mbps = -1;
4197     ms->pages_per_second = -1;
4198     qemu_sem_init(&ms->pause_sem, 0);
4199     qemu_mutex_init(&ms->error_mutex);
4200
4201     params->tls_hostname = g_strdup("");
4202     params->tls_creds = g_strdup("");
4203
4204     /* Set has_* up only for parameter checks */
4205     params->has_compress_level = true;
4206     params->has_compress_threads = true;
4207     params->has_decompress_threads = true;
4208     params->has_throttle_trigger_threshold = true;
4209     params->has_cpu_throttle_initial = true;
4210     params->has_cpu_throttle_increment = true;
4211     params->has_cpu_throttle_tailslow = true;
4212     params->has_max_bandwidth = true;
4213     params->has_downtime_limit = true;
4214     params->has_x_checkpoint_delay = true;
4215     params->has_block_incremental = true;
4216     params->has_multifd_channels = true;
4217     params->has_multifd_compression = true;
4218     params->has_multifd_zlib_level = true;
4219     params->has_multifd_zstd_level = true;
4220     params->has_xbzrle_cache_size = true;
4221     params->has_max_postcopy_bandwidth = true;
4222     params->has_max_cpu_throttle = true;
4223     params->has_announce_initial = true;
4224     params->has_announce_max = true;
4225     params->has_announce_rounds = true;
4226     params->has_announce_step = true;
4227
4228     qemu_sem_init(&ms->postcopy_pause_sem, 0);
4229     qemu_sem_init(&ms->postcopy_pause_rp_sem, 0);
4230     qemu_sem_init(&ms->rp_state.rp_sem, 0);
4231     qemu_sem_init(&ms->rate_limit_sem, 0);
4232     qemu_sem_init(&ms->wait_unplug_sem, 0);
4233     qemu_mutex_init(&ms->qemu_file_lock);
4234 }
4235
4236 /*
4237  * Return true if check pass, false otherwise. Error will be put
4238  * inside errp if provided.
4239  */
4240 static bool migration_object_check(MigrationState *ms, Error **errp)
4241 {
4242     MigrationCapabilityStatusList *head = NULL;
4243     /* Assuming all off */
4244     bool cap_list[MIGRATION_CAPABILITY__MAX] = { 0 }, ret;
4245     int i;
4246
4247     if (!migrate_params_check(&ms->parameters, errp)) {
4248         return false;
4249     }
4250
4251     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
4252         if (ms->enabled_capabilities[i]) {
4253             QAPI_LIST_PREPEND(head, migrate_cap_add(i, true));
4254         }
4255     }
4256
4257     ret = migrate_caps_check(cap_list, head, errp);
4258
4259     /* It works with head == NULL */
4260     qapi_free_MigrationCapabilityStatusList(head);
4261
4262     return ret;
4263 }
4264
4265 static const TypeInfo migration_type = {
4266     .name = TYPE_MIGRATION,
4267     /*
4268      * NOTE: TYPE_MIGRATION is not really a device, as the object is
4269      * not created using qdev_new(), it is not attached to the qdev
4270      * device tree, and it is never realized.
4271      *
4272      * TODO: Make this TYPE_OBJECT once QOM provides something like
4273      * TYPE_DEVICE's "-global" properties.
4274      */
4275     .parent = TYPE_DEVICE,
4276     .class_init = migration_class_init,
4277     .class_size = sizeof(MigrationClass),
4278     .instance_size = sizeof(MigrationState),
4279     .instance_init = migration_instance_init,
4280     .instance_finalize = migration_instance_finalize,
4281 };
4282
4283 static void register_migration_types(void)
4284 {
4285     type_register_static(&migration_type);
4286 }
4287
4288 type_init(register_migration_types);