migration/migration.c

   1 /*
   2  * QEMU live migration
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *
   6  * Authors:
   7  *  Anthony Liguori   <aliguori@us.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include "qemu/cutils.h"
  18 #include "qemu/error-report.h"
  19 #include "qemu/main-loop.h"
  20 #include "migration/blocker.h"
  21 #include "exec.h"
  22 #include "fd.h"
  23 #include "socket.h"
  24 #include "sysemu/runstate.h"
  25 #include "sysemu/sysemu.h"
  26 #include "sysemu/cpu-throttle.h"
  27 #include "rdma.h"
  28 #include "ram.h"
  29 #include "migration/global_state.h"
  30 #include "migration/misc.h"
  31 #include "migration.h"
  32 #include "savevm.h"
  33 #include "qemu-file-channel.h"
  34 #include "qemu-file.h"
  35 #include "migration/vmstate.h"
  36 #include "block/block.h"
  37 #include "qapi/error.h"
  38 #include "qapi/clone-visitor.h"
  39 #include "qapi/qapi-visit-migration.h"
  40 #include "qapi/qapi-visit-sockets.h"
  41 #include "qapi/qapi-commands-migration.h"
  42 #include "qapi/qapi-events-migration.h"
  43 #include "qapi/qmp/qerror.h"
  44 #include "qapi/qmp/qnull.h"
  45 #include "qemu/rcu.h"
  46 #include "block.h"
  47 #include "postcopy-ram.h"
  48 #include "qemu/thread.h"
  49 #include "trace.h"
  50 #include "exec/target_page.h"
  51 #include "io/channel-buffer.h"
  52 #include "migration/colo.h"
  53 #include "hw/boards.h"
  54 #include "hw/qdev-properties.h"
  55 #include "hw/qdev-properties-system.h"
  56 #include "monitor/monitor.h"
  57 #include "net/announce.h"
  58 #include "qemu/queue.h"
  59 #include "multifd.h"
  60 #include "qemu/yank.h"
  61 #include "sysemu/cpus.h"
  62
  63 #define MAX_THROTTLE  (128 << 20)      /* Migration transfer speed throttling */
  64
  65 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
  66  * data. */
  67 #define BUFFER_DELAY     100
  68 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
  69
  70 /* Time in milliseconds we are allowed to stop the source,
  71  * for sending the last part */
  72 #define DEFAULT_MIGRATE_SET_DOWNTIME 300
  73
  74 /* Maximum migrate downtime set to 2000 seconds */
  75 #define MAX_MIGRATE_DOWNTIME_SECONDS 2000
  76 #define MAX_MIGRATE_DOWNTIME (MAX_MIGRATE_DOWNTIME_SECONDS * 1000)
  77
  78 /* Default compression thread count */
  79 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8
  80 /* Default decompression thread count, usually decompression is at
  81  * least 4 times as fast as compression.*/
  82 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2
  83 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */
  84 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1
  85 /* Define default autoconverge cpu throttle migration parameters */
  86 #define DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD 50
  87 #define DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL 20
  88 #define DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT 10
  89 #define DEFAULT_MIGRATE_MAX_CPU_THROTTLE 99
  90
  91 /* Migration XBZRLE default cache size */
  92 #define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024)
  93
  94 /* The delay time (in ms) between two COLO checkpoints */
  95 #define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100)
  96 #define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2
  97 #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE
  98 /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */
  99 #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1
 100 /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */
 101 #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1
 102
 103 /* Background transfer rate for postcopy, 0 means unlimited, note
 104  * that page requests can still exceed this limit.
 105  */
 106 #define DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH 0
 107
 108 /*
 109  * Parameters for self_announce_delay giving a stream of RARP/ARP
 110  * packets after migration.
 111  */
 112 #define DEFAULT_MIGRATE_ANNOUNCE_INITIAL  50
 113 #define DEFAULT_MIGRATE_ANNOUNCE_MAX     550
 114 #define DEFAULT_MIGRATE_ANNOUNCE_ROUNDS    5
 115 #define DEFAULT_MIGRATE_ANNOUNCE_STEP    100
 116
 117 static NotifierList migration_state_notifiers =
 118     NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
 119
 120 /* Messages sent on the return path from destination to source */
 121 enum mig_rp_message_type {
 122     MIG_RP_MSG_INVALID = 0,  /* Must be 0 */
 123     MIG_RP_MSG_SHUT,         /* sibling will not send any more RP messages */
 124     MIG_RP_MSG_PONG,         /* Response to a PING; data (seq: be32 ) */
 125
 126     MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
 127     MIG_RP_MSG_REQ_PAGES,    /* data (start: be64, len: be32) */
 128     MIG_RP_MSG_RECV_BITMAP,  /* send recved_bitmap back to source */
 129     MIG_RP_MSG_RESUME_ACK,   /* tell source that we are ready to resume */
 130
 131     MIG_RP_MSG_MAX
 132 };
 133
 134 /* Migration capabilities set */
 135 struct MigrateCapsSet {
 136     int size;                       /* Capability set size */
 137     MigrationCapability caps[];     /* Variadic array of capabilities */
 138 };
 139 typedef struct MigrateCapsSet MigrateCapsSet;
 140
 141 /* Define and initialize MigrateCapsSet */
 142 #define INITIALIZE_MIGRATE_CAPS_SET(_name, ...)   \
 143     MigrateCapsSet _name = {    \
 144         .size = sizeof((int []) { __VA_ARGS__ }) / sizeof(int), \
 145         .caps = { __VA_ARGS__ } \
 146     }
 147
 148 /* Background-snapshot compatibility check list */
 149 static const
 150 INITIALIZE_MIGRATE_CAPS_SET(check_caps_background_snapshot,
 151     MIGRATION_CAPABILITY_POSTCOPY_RAM,
 152     MIGRATION_CAPABILITY_DIRTY_BITMAPS,
 153     MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME,
 154     MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE,
 155     MIGRATION_CAPABILITY_RETURN_PATH,
 156     MIGRATION_CAPABILITY_MULTIFD,
 157     MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER,
 158     MIGRATION_CAPABILITY_AUTO_CONVERGE,
 159     MIGRATION_CAPABILITY_RELEASE_RAM,
 160     MIGRATION_CAPABILITY_RDMA_PIN_ALL,
 161     MIGRATION_CAPABILITY_COMPRESS,
 162     MIGRATION_CAPABILITY_XBZRLE,
 163     MIGRATION_CAPABILITY_X_COLO,
 164     MIGRATION_CAPABILITY_VALIDATE_UUID);
 165
 166 /* When we add fault tolerance, we could have several
 167    migrations at once.  For now we don't need to add
 168    dynamic creation of migration */
 169
 170 static MigrationState *current_migration;
 171 static MigrationIncomingState *current_incoming;
 172
 173 static GSList *migration_blockers;
 174
 175 static bool migration_object_check(MigrationState *ms, Error **errp);
 176 static int migration_maybe_pause(MigrationState *s,
 177                                  int *current_active_state,
 178                                  int new_state);
 179 static void migrate_fd_cancel(MigrationState *s);
 180
 181 static gint page_request_addr_cmp(gconstpointer ap, gconstpointer bp)
 182 {
 183     uintptr_t a = (uintptr_t) ap, b = (uintptr_t) bp;
 184
 185     return (a > b) - (a < b);
 186 }
 187
 188 void migration_object_init(void)
 189 {
 190     Error *err = NULL;
 191
 192     /* This can only be called once. */
 193     assert(!current_migration);
 194     current_migration = MIGRATION_OBJ(object_new(TYPE_MIGRATION));
 195
 196     /*
 197      * Init the migrate incoming object as well no matter whether
 198      * we'll use it or not.
 199      */
 200     assert(!current_incoming);
 201     current_incoming = g_new0(MigrationIncomingState, 1);
 202     current_incoming->state = MIGRATION_STATUS_NONE;
 203     current_incoming->postcopy_remote_fds =
 204         g_array_new(FALSE, TRUE, sizeof(struct PostCopyFD));
 205     qemu_mutex_init(&current_incoming->rp_mutex);
 206     qemu_event_init(&current_incoming->main_thread_load_event, false);
 207     qemu_sem_init(&current_incoming->postcopy_pause_sem_dst, 0);
 208     qemu_sem_init(&current_incoming->postcopy_pause_sem_fault, 0);
 209     qemu_mutex_init(&current_incoming->page_request_mutex);
 210     current_incoming->page_requested = g_tree_new(page_request_addr_cmp);
 211
 212     if (!migration_object_check(current_migration, &err)) {
 213         error_report_err(err);
 214         exit(1);
 215     }
 216
 217     blk_mig_init();
 218     ram_mig_init();
 219     dirty_bitmap_mig_init();
 220 }
 221
 222 void migration_cancel(void)
 223 {
 224     migrate_fd_cancel(current_migration);
 225 }
 226
 227 void migration_shutdown(void)
 228 {
 229     /*
 230      * Cancel the current migration - that will (eventually)
 231      * stop the migration using this structure
 232      */
 233     migration_cancel();
 234     object_unref(OBJECT(current_migration));
 235
 236     /*
 237      * Cancel outgoing migration of dirty bitmaps. It should
 238      * at least unref used block nodes.
 239      */
 240     dirty_bitmap_mig_cancel_outgoing();
 241
 242     /*
 243      * Cancel incoming migration of dirty bitmaps. Dirty bitmaps
 244      * are non-critical data, and their loss never considered as
 245      * something serious.
 246      */
 247     dirty_bitmap_mig_cancel_incoming();
 248 }
 249
 250 /* For outgoing */
 251 MigrationState *migrate_get_current(void)
 252 {
 253     /* This can only be called after the object created. */
 254     assert(current_migration);
 255     return current_migration;
 256 }
 257
 258 MigrationIncomingState *migration_incoming_get_current(void)
 259 {
 260     assert(current_incoming);
 261     return current_incoming;
 262 }
 263
 264 void migration_incoming_state_destroy(void)
 265 {
 266     struct MigrationIncomingState *mis = migration_incoming_get_current();
 267
 268     if (mis->to_src_file) {
 269         /* Tell source that we are done */
 270         migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
 271         qemu_fclose(mis->to_src_file);
 272         mis->to_src_file = NULL;
 273     }
 274
 275     if (mis->from_src_file) {
 276         qemu_fclose(mis->from_src_file);
 277         mis->from_src_file = NULL;
 278     }
 279     if (mis->postcopy_remote_fds) {
 280         g_array_free(mis->postcopy_remote_fds, TRUE);
 281         mis->postcopy_remote_fds = NULL;
 282     }
 283     if (mis->transport_cleanup) {
 284         mis->transport_cleanup(mis->transport_data);
 285     }
 286
 287     qemu_event_reset(&mis->main_thread_load_event);
 288
 289     if (mis->page_requested) {
 290         g_tree_destroy(mis->page_requested);
 291         mis->page_requested = NULL;
 292     }
 293
 294     if (mis->socket_address_list) {
 295         qapi_free_SocketAddressList(mis->socket_address_list);
 296         mis->socket_address_list = NULL;
 297     }
 298
 299     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
 300 }
 301
 302 static void migrate_generate_event(int new_state)
 303 {
 304     if (migrate_use_events()) {
 305         qapi_event_send_migration(new_state);
 306     }
 307 }
 308
 309 static bool migrate_late_block_activate(void)
 310 {
 311     MigrationState *s;
 312
 313     s = migrate_get_current();
 314
 315     return s->enabled_capabilities[
 316         MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE];
 317 }
 318
 319 /*
 320  * Send a message on the return channel back to the source
 321  * of the migration.
 322  */
 323 static int migrate_send_rp_message(MigrationIncomingState *mis,
 324                                    enum mig_rp_message_type message_type,
 325                                    uint16_t len, void *data)
 326 {
 327     int ret = 0;
 328
 329     trace_migrate_send_rp_message((int)message_type, len);
 330     QEMU_LOCK_GUARD(&mis->rp_mutex);
 331
 332     /*
 333      * It's possible that the file handle got lost due to network
 334      * failures.
 335      */
 336     if (!mis->to_src_file) {
 337         ret = -EIO;
 338         return ret;
 339     }
 340
 341     qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
 342     qemu_put_be16(mis->to_src_file, len);
 343     qemu_put_buffer(mis->to_src_file, data, len);
 344     qemu_fflush(mis->to_src_file);
 345
 346     /* It's possible that qemu file got error during sending */
 347     ret = qemu_file_get_error(mis->to_src_file);
 348
 349     return ret;
 350 }
 351
 352 /* Request one page from the source VM at the given start address.
 353  *   rb: the RAMBlock to request the page in
 354  *   Start: Address offset within the RB
 355  *   Len: Length in bytes required - must be a multiple of pagesize
 356  */
 357 int migrate_send_rp_message_req_pages(MigrationIncomingState *mis,
 358                                       RAMBlock *rb, ram_addr_t start)
 359 {
 360     uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */
 361     size_t msglen = 12; /* start + len */
 362     size_t len = qemu_ram_pagesize(rb);
 363     enum mig_rp_message_type msg_type;
 364     const char *rbname;
 365     int rbname_len;
 366
 367     *(uint64_t *)bufc = cpu_to_be64((uint64_t)start);
 368     *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len);
 369
 370     /*
 371      * We maintain the last ramblock that we requested for page.  Note that we
 372      * don't need locking because this function will only be called within the
 373      * postcopy ram fault thread.
 374      */
 375     if (rb != mis->last_rb) {
 376         mis->last_rb = rb;
 377
 378         rbname = qemu_ram_get_idstr(rb);
 379         rbname_len = strlen(rbname);
 380
 381         assert(rbname_len < 256);
 382
 383         bufc[msglen++] = rbname_len;
 384         memcpy(bufc + msglen, rbname, rbname_len);
 385         msglen += rbname_len;
 386         msg_type = MIG_RP_MSG_REQ_PAGES_ID;
 387     } else {
 388         msg_type = MIG_RP_MSG_REQ_PAGES;
 389     }
 390
 391     return migrate_send_rp_message(mis, msg_type, msglen, bufc);
 392 }
 393
 394 int migrate_send_rp_req_pages(MigrationIncomingState *mis,
 395                               RAMBlock *rb, ram_addr_t start, uint64_t haddr)
 396 {
 397     void *aligned = (void *)(uintptr_t)(haddr & (-qemu_ram_pagesize(rb)));
 398     bool received = false;
 399
 400     WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) {
 401         received = ramblock_recv_bitmap_test_byte_offset(rb, start);
 402         if (!received && !g_tree_lookup(mis->page_requested, aligned)) {
 403             /*
 404              * The page has not been received, and it's not yet in the page
 405              * request list.  Queue it.  Set the value of element to 1, so that
 406              * things like g_tree_lookup() will return TRUE (1) when found.
 407              */
 408             g_tree_insert(mis->page_requested, aligned, (gpointer)1);
 409             mis->page_requested_count++;
 410             trace_postcopy_page_req_add(aligned, mis->page_requested_count);
 411         }
 412     }
 413
 414     /*
 415      * If the page is there, skip sending the message.  We don't even need the
 416      * lock because as long as the page arrived, it'll be there forever.
 417      */
 418     if (received) {
 419         return 0;
 420     }
 421
 422     return migrate_send_rp_message_req_pages(mis, rb, start);
 423 }
 424
 425 static bool migration_colo_enabled;
 426 bool migration_incoming_colo_enabled(void)
 427 {
 428     return migration_colo_enabled;
 429 }
 430
 431 void migration_incoming_disable_colo(void)
 432 {
 433     ram_block_discard_disable(false);
 434     migration_colo_enabled = false;
 435 }
 436
 437 int migration_incoming_enable_colo(void)
 438 {
 439     if (ram_block_discard_disable(true)) {
 440         error_report("COLO: cannot disable RAM discard");
 441         return -EBUSY;
 442     }
 443     migration_colo_enabled = true;
 444     return 0;
 445 }
 446
 447 void migrate_add_address(SocketAddress *address)
 448 {
 449     MigrationIncomingState *mis = migration_incoming_get_current();
 450
 451     QAPI_LIST_PREPEND(mis->socket_address_list,
 452                       QAPI_CLONE(SocketAddress, address));
 453 }
 454
 455 static void qemu_start_incoming_migration(const char *uri, Error **errp)
 456 {
 457     const char *p = NULL;
 458
 459     qapi_event_send_migration(MIGRATION_STATUS_SETUP);
 460     if (strstart(uri, "tcp:", &p) ||
 461         strstart(uri, "unix:", NULL) ||
 462         strstart(uri, "vsock:", NULL)) {
 463         socket_start_incoming_migration(p ? p : uri, errp);
 464 #ifdef CONFIG_RDMA
 465     } else if (strstart(uri, "rdma:", &p)) {
 466         rdma_start_incoming_migration(p, errp);
 467 #endif
 468     } else if (strstart(uri, "exec:", &p)) {
 469         exec_start_incoming_migration(p, errp);
 470     } else if (strstart(uri, "fd:", &p)) {
 471         fd_start_incoming_migration(p, errp);
 472     } else {
 473         error_setg(errp, "unknown migration protocol: %s", uri);
 474     }
 475 }
 476
 477 static void process_incoming_migration_bh(void *opaque)
 478 {
 479     Error *local_err = NULL;
 480     MigrationIncomingState *mis = opaque;
 481
 482     /* If capability late_block_activate is set:
 483      * Only fire up the block code now if we're going to restart the
 484      * VM, else 'cont' will do it.
 485      * This causes file locking to happen; so we don't want it to happen
 486      * unless we really are starting the VM.
 487      */
 488     if (!migrate_late_block_activate() ||
 489          (autostart && (!global_state_received() ||
 490             global_state_get_runstate() == RUN_STATE_RUNNING))) {
 491         /* Make sure all file formats flush their mutable metadata.
 492          * If we get an error here, just don't restart the VM yet. */
 493         bdrv_invalidate_cache_all(&local_err);
 494         if (local_err) {
 495             error_report_err(local_err);
 496             local_err = NULL;
 497             autostart = false;
 498         }
 499     }
 500
 501     /*
 502      * This must happen after all error conditions are dealt with and
 503      * we're sure the VM is going to be running on this host.
 504      */
 505     qemu_announce_self(&mis->announce_timer, migrate_announce_params());
 506
 507     if (multifd_load_cleanup(&local_err) != 0) {
 508         error_report_err(local_err);
 509         autostart = false;
 510     }
 511     /* If global state section was not received or we are in running
 512        state, we need to obey autostart. Any other state is set with
 513        runstate_set. */
 514
 515     dirty_bitmap_mig_before_vm_start();
 516
 517     if (!global_state_received() ||
 518         global_state_get_runstate() == RUN_STATE_RUNNING) {
 519         if (autostart) {
 520             vm_start();
 521         } else {
 522             runstate_set(RUN_STATE_PAUSED);
 523         }
 524     } else if (migration_incoming_colo_enabled()) {
 525         migration_incoming_disable_colo();
 526         vm_start();
 527     } else {
 528         runstate_set(global_state_get_runstate());
 529     }
 530     /*
 531      * This must happen after any state changes since as soon as an external
 532      * observer sees this event they might start to prod at the VM assuming
 533      * it's ready to use.
 534      */
 535     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 536                       MIGRATION_STATUS_COMPLETED);
 537     qemu_bh_delete(mis->bh);
 538     migration_incoming_state_destroy();
 539 }
 540
 541 static void process_incoming_migration_co(void *opaque)
 542 {
 543     MigrationIncomingState *mis = migration_incoming_get_current();
 544     PostcopyState ps;
 545     int ret;
 546     Error *local_err = NULL;
 547
 548     assert(mis->from_src_file);
 549     mis->migration_incoming_co = qemu_coroutine_self();
 550     mis->largest_page_size = qemu_ram_pagesize_largest();
 551     postcopy_state_set(POSTCOPY_INCOMING_NONE);
 552     migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
 553                       MIGRATION_STATUS_ACTIVE);
 554     ret = qemu_loadvm_state(mis->from_src_file);
 555
 556     ps = postcopy_state_get();
 557     trace_process_incoming_migration_co_end(ret, ps);
 558     if (ps != POSTCOPY_INCOMING_NONE) {
 559         if (ps == POSTCOPY_INCOMING_ADVISE) {
 560             /*
 561              * Where a migration had postcopy enabled (and thus went to advise)
 562              * but managed to complete within the precopy period, we can use
 563              * the normal exit.
 564              */
 565             postcopy_ram_incoming_cleanup(mis);
 566         } else if (ret >= 0) {
 567             /*
 568              * Postcopy was started, cleanup should happen at the end of the
 569              * postcopy thread.
 570              */
 571             trace_process_incoming_migration_co_postcopy_end_main();
 572             return;
 573         }
 574         /* Else if something went wrong then just fall out of the normal exit */
 575     }
 576
 577     /* we get COLO info, and know if we are in COLO mode */
 578     if (!ret && migration_incoming_colo_enabled()) {
 579         /* Make sure all file formats flush their mutable metadata */
 580         bdrv_invalidate_cache_all(&local_err);
 581         if (local_err) {
 582             error_report_err(local_err);
 583             goto fail;
 584         }
 585
 586         qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
 587              colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
 588         mis->have_colo_incoming_thread = true;
 589         qemu_coroutine_yield();
 590
 591         /* Wait checkpoint incoming thread exit before free resource */
 592         qemu_thread_join(&mis->colo_incoming_thread);
 593         /* We hold the global iothread lock, so it is safe here */
 594         colo_release_ram_cache();
 595     }
 596
 597     if (ret < 0) {
 598         error_report("load of migration failed: %s", strerror(-ret));
 599         goto fail;
 600     }
 601     mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
 602     qemu_bh_schedule(mis->bh);
 603     mis->migration_incoming_co = NULL;
 604     return;
 605 fail:
 606     local_err = NULL;
 607     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 608                       MIGRATION_STATUS_FAILED);
 609     qemu_fclose(mis->from_src_file);
 610     if (multifd_load_cleanup(&local_err) != 0) {
 611         error_report_err(local_err);
 612     }
 613     exit(EXIT_FAILURE);
 614 }
 615
 616 /**
 617  * @migration_incoming_setup: Setup incoming migration
 618  *
 619  * Returns 0 for no error or 1 for error
 620  *
 621  * @f: file for main migration channel
 622  * @errp: where to put errors
 623  */
 624 static int migration_incoming_setup(QEMUFile *f, Error **errp)
 625 {
 626     MigrationIncomingState *mis = migration_incoming_get_current();
 627     Error *local_err = NULL;
 628
 629     if (multifd_load_setup(&local_err) != 0) {
 630         /* We haven't been able to create multifd threads
 631            nothing better to do */
 632         error_report_err(local_err);
 633         exit(EXIT_FAILURE);
 634     }
 635
 636     if (!mis->from_src_file) {
 637         mis->from_src_file = f;
 638     }
 639     qemu_file_set_blocking(f, false);
 640     return 0;
 641 }
 642
 643 void migration_incoming_process(void)
 644 {
 645     Coroutine *co = qemu_coroutine_create(process_incoming_migration_co, NULL);
 646     qemu_coroutine_enter(co);
 647 }
 648
 649 /* Returns true if recovered from a paused migration, otherwise false */
 650 static bool postcopy_try_recover(QEMUFile *f)
 651 {
 652     MigrationIncomingState *mis = migration_incoming_get_current();
 653
 654     if (mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
 655         /* Resumed from a paused postcopy migration */
 656
 657         mis->from_src_file = f;
 658         /* Postcopy has standalone thread to do vm load */
 659         qemu_file_set_blocking(f, true);
 660
 661         /* Re-configure the return path */
 662         mis->to_src_file = qemu_file_get_return_path(f);
 663
 664         migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
 665                           MIGRATION_STATUS_POSTCOPY_RECOVER);
 666
 667         /*
 668          * Here, we only wake up the main loading thread (while the
 669          * fault thread will still be waiting), so that we can receive
 670          * commands from source now, and answer it if needed. The
 671          * fault thread will be woken up afterwards until we are sure
 672          * that source is ready to reply to page requests.
 673          */
 674         qemu_sem_post(&mis->postcopy_pause_sem_dst);
 675         return true;
 676     }
 677
 678     return false;
 679 }
 680
 681 void migration_fd_process_incoming(QEMUFile *f, Error **errp)
 682 {
 683     Error *local_err = NULL;
 684
 685     if (postcopy_try_recover(f)) {
 686         return;
 687     }
 688
 689     if (migration_incoming_setup(f, &local_err)) {
 690         error_propagate(errp, local_err);
 691         return;
 692     }
 693     migration_incoming_process();
 694 }
 695
 696 void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp)
 697 {
 698     MigrationIncomingState *mis = migration_incoming_get_current();
 699     Error *local_err = NULL;
 700     bool start_migration;
 701
 702     if (!mis->from_src_file) {
 703         /* The first connection (multifd may have multiple) */
 704         QEMUFile *f = qemu_fopen_channel_input(ioc);
 705
 706         /* If it's a recovery, we're done */
 707         if (postcopy_try_recover(f)) {
 708             return;
 709         }
 710
 711         if (migration_incoming_setup(f, &local_err)) {
 712             error_propagate(errp, local_err);
 713             return;
 714         }
 715
 716         /*
 717          * Common migration only needs one channel, so we can start
 718          * right now.  Multifd needs more than one channel, we wait.
 719          */
 720         start_migration = !migrate_use_multifd();
 721     } else {
 722         /* Multiple connections */
 723         assert(migrate_use_multifd());
 724         start_migration = multifd_recv_new_channel(ioc, &local_err);
 725         if (local_err) {
 726             error_propagate(errp, local_err);
 727             return;
 728         }
 729     }
 730
 731     if (start_migration) {
 732         migration_incoming_process();
 733     }
 734 }
 735
 736 /**
 737  * @migration_has_all_channels: We have received all channels that we need
 738  *
 739  * Returns true when we have got connections to all the channels that
 740  * we need for migration.
 741  */
 742 bool migration_has_all_channels(void)
 743 {
 744     MigrationIncomingState *mis = migration_incoming_get_current();
 745     bool all_channels;
 746
 747     all_channels = multifd_recv_all_channels_created();
 748
 749     return all_channels && mis->from_src_file != NULL;
 750 }
 751
 752 /*
 753  * Send a 'SHUT' message on the return channel with the given value
 754  * to indicate that we've finished with the RP.  Non-0 value indicates
 755  * error.
 756  */
 757 void migrate_send_rp_shut(MigrationIncomingState *mis,
 758                           uint32_t value)
 759 {
 760     uint32_t buf;
 761
 762     buf = cpu_to_be32(value);
 763     migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
 764 }
 765
 766 /*
 767  * Send a 'PONG' message on the return channel with the given value
 768  * (normally in response to a 'PING')
 769  */
 770 void migrate_send_rp_pong(MigrationIncomingState *mis,
 771                           uint32_t value)
 772 {
 773     uint32_t buf;
 774
 775     buf = cpu_to_be32(value);
 776     migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
 777 }
 778
 779 void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
 780                                  char *block_name)
 781 {
 782     char buf[512];
 783     int len;
 784     int64_t res;
 785
 786     /*
 787      * First, we send the header part. It contains only the len of
 788      * idstr, and the idstr itself.
 789      */
 790     len = strlen(block_name);
 791     buf[0] = len;
 792     memcpy(buf + 1, block_name, len);
 793
 794     if (mis->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
 795         error_report("%s: MSG_RP_RECV_BITMAP only used for recovery",
 796                      __func__);
 797         return;
 798     }
 799
 800     migrate_send_rp_message(mis, MIG_RP_MSG_RECV_BITMAP, len + 1, buf);
 801
 802     /*
 803      * Next, we dump the received bitmap to the stream.
 804      *
 805      * TODO: currently we are safe since we are the only one that is
 806      * using the to_src_file handle (fault thread is still paused),
 807      * and it's ok even not taking the mutex. However the best way is
 808      * to take the lock before sending the message header, and release
 809      * the lock after sending the bitmap.
 810      */
 811     qemu_mutex_lock(&mis->rp_mutex);
 812     res = ramblock_recv_bitmap_send(mis->to_src_file, block_name);
 813     qemu_mutex_unlock(&mis->rp_mutex);
 814
 815     trace_migrate_send_rp_recv_bitmap(block_name, res);
 816 }
 817
 818 void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value)
 819 {
 820     uint32_t buf;
 821
 822     buf = cpu_to_be32(value);
 823     migrate_send_rp_message(mis, MIG_RP_MSG_RESUME_ACK, sizeof(buf), &buf);
 824 }
 825
 826 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
 827 {
 828     MigrationCapabilityStatusList *head = NULL, **tail = &head;
 829     MigrationCapabilityStatus *caps;
 830     MigrationState *s = migrate_get_current();
 831     int i;
 832
 833     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
 834 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
 835         if (i == MIGRATION_CAPABILITY_BLOCK) {
 836             continue;
 837         }
 838 #endif
 839         caps = g_malloc0(sizeof(*caps));
 840         caps->capability = i;
 841         caps->state = s->enabled_capabilities[i];
 842         QAPI_LIST_APPEND(tail, caps);
 843     }
 844
 845     return head;
 846 }
 847
 848 MigrationParameters *qmp_query_migrate_parameters(Error **errp)
 849 {
 850     MigrationParameters *params;
 851     MigrationState *s = migrate_get_current();
 852
 853     /* TODO use QAPI_CLONE() instead of duplicating it inline */
 854     params = g_malloc0(sizeof(*params));
 855     params->has_compress_level = true;
 856     params->compress_level = s->parameters.compress_level;
 857     params->has_compress_threads = true;
 858     params->compress_threads = s->parameters.compress_threads;
 859     params->has_compress_wait_thread = true;
 860     params->compress_wait_thread = s->parameters.compress_wait_thread;
 861     params->has_decompress_threads = true;
 862     params->decompress_threads = s->parameters.decompress_threads;
 863     params->has_throttle_trigger_threshold = true;
 864     params->throttle_trigger_threshold = s->parameters.throttle_trigger_threshold;
 865     params->has_cpu_throttle_initial = true;
 866     params->cpu_throttle_initial = s->parameters.cpu_throttle_initial;
 867     params->has_cpu_throttle_increment = true;
 868     params->cpu_throttle_increment = s->parameters.cpu_throttle_increment;
 869     params->has_cpu_throttle_tailslow = true;
 870     params->cpu_throttle_tailslow = s->parameters.cpu_throttle_tailslow;
 871     params->has_tls_creds = true;
 872     params->tls_creds = g_strdup(s->parameters.tls_creds);
 873     params->has_tls_hostname = true;
 874     params->tls_hostname = g_strdup(s->parameters.tls_hostname);
 875     params->has_tls_authz = true;
 876     params->tls_authz = g_strdup(s->parameters.tls_authz ?
 877                                  s->parameters.tls_authz : "");
 878     params->has_max_bandwidth = true;
 879     params->max_bandwidth = s->parameters.max_bandwidth;
 880     params->has_downtime_limit = true;
 881     params->downtime_limit = s->parameters.downtime_limit;
 882     params->has_x_checkpoint_delay = true;
 883     params->x_checkpoint_delay = s->parameters.x_checkpoint_delay;
 884     params->has_block_incremental = true;
 885     params->block_incremental = s->parameters.block_incremental;
 886     params->has_multifd_channels = true;
 887     params->multifd_channels = s->parameters.multifd_channels;
 888     params->has_multifd_compression = true;
 889     params->multifd_compression = s->parameters.multifd_compression;
 890     params->has_multifd_zlib_level = true;
 891     params->multifd_zlib_level = s->parameters.multifd_zlib_level;
 892     params->has_multifd_zstd_level = true;
 893     params->multifd_zstd_level = s->parameters.multifd_zstd_level;
 894     params->has_xbzrle_cache_size = true;
 895     params->xbzrle_cache_size = s->parameters.xbzrle_cache_size;
 896     params->has_max_postcopy_bandwidth = true;
 897     params->max_postcopy_bandwidth = s->parameters.max_postcopy_bandwidth;
 898     params->has_max_cpu_throttle = true;
 899     params->max_cpu_throttle = s->parameters.max_cpu_throttle;
 900     params->has_announce_initial = true;
 901     params->announce_initial = s->parameters.announce_initial;
 902     params->has_announce_max = true;
 903     params->announce_max = s->parameters.announce_max;
 904     params->has_announce_rounds = true;
 905     params->announce_rounds = s->parameters.announce_rounds;
 906     params->has_announce_step = true;
 907     params->announce_step = s->parameters.announce_step;
 908
 909     if (s->parameters.has_block_bitmap_mapping) {
 910         params->has_block_bitmap_mapping = true;
 911         params->block_bitmap_mapping =
 912             QAPI_CLONE(BitmapMigrationNodeAliasList,
 913                        s->parameters.block_bitmap_mapping);
 914     }
 915
 916     return params;
 917 }
 918
 919 AnnounceParameters *migrate_announce_params(void)
 920 {
 921     static AnnounceParameters ap;
 922
 923     MigrationState *s = migrate_get_current();
 924
 925     ap.initial = s->parameters.announce_initial;
 926     ap.max = s->parameters.announce_max;
 927     ap.rounds = s->parameters.announce_rounds;
 928     ap.step = s->parameters.announce_step;
 929
 930     return &ap;
 931 }
 932
 933 /*
 934  * Return true if we're already in the middle of a migration
 935  * (i.e. any of the active or setup states)
 936  */
 937 bool migration_is_setup_or_active(int state)
 938 {
 939     switch (state) {
 940     case MIGRATION_STATUS_ACTIVE:
 941     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 942     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 943     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 944     case MIGRATION_STATUS_SETUP:
 945     case MIGRATION_STATUS_PRE_SWITCHOVER:
 946     case MIGRATION_STATUS_DEVICE:
 947     case MIGRATION_STATUS_WAIT_UNPLUG:
 948     case MIGRATION_STATUS_COLO:
 949         return true;
 950
 951     default:
 952         return false;
 953
 954     }
 955 }
 956
 957 bool migration_is_running(int state)
 958 {
 959     switch (state) {
 960     case MIGRATION_STATUS_ACTIVE:
 961     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 962     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 963     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 964     case MIGRATION_STATUS_SETUP:
 965     case MIGRATION_STATUS_PRE_SWITCHOVER:
 966     case MIGRATION_STATUS_DEVICE:
 967     case MIGRATION_STATUS_WAIT_UNPLUG:
 968     case MIGRATION_STATUS_CANCELLING:
 969         return true;
 970
 971     default:
 972         return false;
 973
 974     }
 975 }
 976
 977 static void populate_time_info(MigrationInfo *info, MigrationState *s)
 978 {
 979     info->has_status = true;
 980     info->has_setup_time = true;
 981     info->setup_time = s->setup_time;
 982     if (s->state == MIGRATION_STATUS_COMPLETED) {
 983         info->has_total_time = true;
 984         info->total_time = s->total_time;
 985         info->has_downtime = true;
 986         info->downtime = s->downtime;
 987     } else {
 988         info->has_total_time = true;
 989         info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) -
 990                            s->start_time;
 991         info->has_expected_downtime = true;
 992         info->expected_downtime = s->expected_downtime;
 993     }
 994 }
 995
 996 static void populate_ram_info(MigrationInfo *info, MigrationState *s)
 997 {
 998     info->has_ram = true;
 999     info->ram = g_malloc0(sizeof(*info->ram));
1000     info->ram->transferred = ram_counters.transferred;
1001     info->ram->total = ram_bytes_total();
1002     info->ram->duplicate = ram_counters.duplicate;
1003     /* legacy value.  It is not used anymore */
1004     info->ram->skipped = 0;
1005     info->ram->normal = ram_counters.normal;
1006     info->ram->normal_bytes = ram_counters.normal *
1007         qemu_target_page_size();
1008     info->ram->mbps = s->mbps;
1009     info->ram->dirty_sync_count = ram_counters.dirty_sync_count;
1010     info->ram->postcopy_requests = ram_counters.postcopy_requests;
1011     info->ram->page_size = qemu_target_page_size();
1012     info->ram->multifd_bytes = ram_counters.multifd_bytes;
1013     info->ram->pages_per_second = s->pages_per_second;
1014
1015     if (migrate_use_xbzrle()) {
1016         info->has_xbzrle_cache = true;
1017         info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
1018         info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
1019         info->xbzrle_cache->bytes = xbzrle_counters.bytes;
1020         info->xbzrle_cache->pages = xbzrle_counters.pages;
1021         info->xbzrle_cache->cache_miss = xbzrle_counters.cache_miss;
1022         info->xbzrle_cache->cache_miss_rate = xbzrle_counters.cache_miss_rate;
1023         info->xbzrle_cache->encoding_rate = xbzrle_counters.encoding_rate;
1024         info->xbzrle_cache->overflow = xbzrle_counters.overflow;
1025     }
1026
1027     if (migrate_use_compression()) {
1028         info->has_compression = true;
1029         info->compression = g_malloc0(sizeof(*info->compression));
1030         info->compression->pages = compression_counters.pages;
1031         info->compression->busy = compression_counters.busy;
1032         info->compression->busy_rate = compression_counters.busy_rate;
1033         info->compression->compressed_size =
1034                                     compression_counters.compressed_size;
1035         info->compression->compression_rate =
1036                                     compression_counters.compression_rate;
1037     }
1038
1039     if (cpu_throttle_active()) {
1040         info->has_cpu_throttle_percentage = true;
1041         info->cpu_throttle_percentage = cpu_throttle_get_percentage();
1042     }
1043
1044     if (s->state != MIGRATION_STATUS_COMPLETED) {
1045         info->ram->remaining = ram_bytes_remaining();
1046         info->ram->dirty_pages_rate = ram_counters.dirty_pages_rate;
1047     }
1048 }
1049
1050 static void populate_disk_info(MigrationInfo *info)
1051 {
1052     if (blk_mig_active()) {
1053         info->has_disk = true;
1054         info->disk = g_malloc0(sizeof(*info->disk));
1055         info->disk->transferred = blk_mig_bytes_transferred();
1056         info->disk->remaining = blk_mig_bytes_remaining();
1057         info->disk->total = blk_mig_bytes_total();
1058     }
1059 }
1060
1061 static void fill_source_migration_info(MigrationInfo *info)
1062 {
1063     MigrationState *s = migrate_get_current();
1064     GSList *cur_blocker = migration_blockers;
1065
1066     info->blocked_reasons = NULL;
1067
1068     /*
1069      * There are two types of reasons a migration might be blocked;
1070      * a) devices marked in VMState as non-migratable, and
1071      * b) Explicit migration blockers
1072      * We need to add both of them here.
1073      */
1074     qemu_savevm_non_migratable_list(&info->blocked_reasons);
1075
1076     while (cur_blocker) {
1077         QAPI_LIST_PREPEND(info->blocked_reasons,
1078                           g_strdup(error_get_pretty(cur_blocker->data)));
1079         cur_blocker = g_slist_next(cur_blocker);
1080     }
1081     info->has_blocked_reasons = info->blocked_reasons != NULL;
1082
1083     switch (s->state) {
1084     case MIGRATION_STATUS_NONE:
1085         /* no migration has happened ever */
1086         /* do not overwrite destination migration status */
1087         return;
1088     case MIGRATION_STATUS_SETUP:
1089         info->has_status = true;
1090         info->has_total_time = false;
1091         break;
1092     case MIGRATION_STATUS_ACTIVE:
1093     case MIGRATION_STATUS_CANCELLING:
1094     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1095     case MIGRATION_STATUS_PRE_SWITCHOVER:
1096     case MIGRATION_STATUS_DEVICE:
1097     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1098     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1099         /* TODO add some postcopy stats */
1100         populate_time_info(info, s);
1101         populate_ram_info(info, s);
1102         populate_disk_info(info);
1103         populate_vfio_info(info);
1104         break;
1105     case MIGRATION_STATUS_COLO:
1106         info->has_status = true;
1107         /* TODO: display COLO specific information (checkpoint info etc.) */
1108         break;
1109     case MIGRATION_STATUS_COMPLETED:
1110         populate_time_info(info, s);
1111         populate_ram_info(info, s);
1112         populate_vfio_info(info);
1113         break;
1114     case MIGRATION_STATUS_FAILED:
1115         info->has_status = true;
1116         if (s->error) {
1117             info->has_error_desc = true;
1118             info->error_desc = g_strdup(error_get_pretty(s->error));
1119         }
1120         break;
1121     case MIGRATION_STATUS_CANCELLED:
1122         info->has_status = true;
1123         break;
1124     case MIGRATION_STATUS_WAIT_UNPLUG:
1125         info->has_status = true;
1126         break;
1127     }
1128     info->status = s->state;
1129 }
1130
1131 typedef enum WriteTrackingSupport {
1132     WT_SUPPORT_UNKNOWN = 0,
1133     WT_SUPPORT_ABSENT,
1134     WT_SUPPORT_AVAILABLE,
1135     WT_SUPPORT_COMPATIBLE
1136 } WriteTrackingSupport;
1137
1138 static
1139 WriteTrackingSupport migrate_query_write_tracking(void)
1140 {
1141     /* Check if kernel supports required UFFD features */
1142     if (!ram_write_tracking_available()) {
1143         return WT_SUPPORT_ABSENT;
1144     }
1145     /*
1146      * Check if current memory configuration is
1147      * compatible with required UFFD features.
1148      */
1149     if (!ram_write_tracking_compatible()) {
1150         return WT_SUPPORT_AVAILABLE;
1151     }
1152
1153     return WT_SUPPORT_COMPATIBLE;
1154 }
1155
1156 /**
1157  * @migration_caps_check - check capability validity
1158  *
1159  * @cap_list: old capability list, array of bool
1160  * @params: new capabilities to be applied soon
1161  * @errp: set *errp if the check failed, with reason
1162  *
1163  * Returns true if check passed, otherwise false.
1164  */
1165 static bool migrate_caps_check(bool *cap_list,
1166                                MigrationCapabilityStatusList *params,
1167                                Error **errp)
1168 {
1169     MigrationCapabilityStatusList *cap;
1170     bool old_postcopy_cap;
1171     MigrationIncomingState *mis = migration_incoming_get_current();
1172
1173     old_postcopy_cap = cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM];
1174
1175     for (cap = params; cap; cap = cap->next) {
1176         cap_list[cap->value->capability] = cap->value->state;
1177     }
1178
1179 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
1180     if (cap_list[MIGRATION_CAPABILITY_BLOCK]) {
1181         error_setg(errp, "QEMU compiled without old-style (blk/-b, inc/-i) "
1182                    "block migration");
1183         error_append_hint(errp, "Use drive_mirror+NBD instead.\n");
1184         return false;
1185     }
1186 #endif
1187
1188 #ifndef CONFIG_REPLICATION
1189     if (cap_list[MIGRATION_CAPABILITY_X_COLO]) {
1190         error_setg(errp, "QEMU compiled without replication module"
1191                    " can't enable COLO");
1192         error_append_hint(errp, "Please enable replication before COLO.\n");
1193         return false;
1194     }
1195 #endif
1196
1197     if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
1198         /* This check is reasonably expensive, so only when it's being
1199          * set the first time, also it's only the destination that needs
1200          * special support.
1201          */
1202         if (!old_postcopy_cap && runstate_check(RUN_STATE_INMIGRATE) &&
1203             !postcopy_ram_supported_by_host(mis)) {
1204             /* postcopy_ram_supported_by_host will have emitted a more
1205              * detailed message
1206              */
1207             error_setg(errp, "Postcopy is not supported");
1208             return false;
1209         }
1210
1211         if (cap_list[MIGRATION_CAPABILITY_X_IGNORE_SHARED]) {
1212             error_setg(errp, "Postcopy is not compatible with ignore-shared");
1213             return false;
1214         }
1215     }
1216
1217     if (cap_list[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT]) {
1218         WriteTrackingSupport wt_support;
1219         int idx;
1220         /*
1221          * Check if 'background-snapshot' capability is supported by
1222          * host kernel and compatible with guest memory configuration.
1223          */
1224         wt_support = migrate_query_write_tracking();
1225         if (wt_support < WT_SUPPORT_AVAILABLE) {
1226             error_setg(errp, "Background-snapshot is not supported by host kernel");
1227             return false;
1228         }
1229         if (wt_support < WT_SUPPORT_COMPATIBLE) {
1230             error_setg(errp, "Background-snapshot is not compatible "
1231                     "with guest memory configuration");
1232             return false;
1233         }
1234
1235         /*
1236          * Check if there are any migration capabilities
1237          * incompatible with 'background-snapshot'.
1238          */
1239         for (idx = 0; idx < check_caps_background_snapshot.size; idx++) {
1240             int incomp_cap = check_caps_background_snapshot.caps[idx];
1241             if (cap_list[incomp_cap]) {
1242                 error_setg(errp,
1243                         "Background-snapshot is not compatible with %s",
1244                         MigrationCapability_str(incomp_cap));
1245                 return false;
1246             }
1247         }
1248     }
1249
1250     return true;
1251 }
1252
1253 static void fill_destination_migration_info(MigrationInfo *info)
1254 {
1255     MigrationIncomingState *mis = migration_incoming_get_current();
1256
1257     if (mis->socket_address_list) {
1258         info->has_socket_address = true;
1259         info->socket_address =
1260             QAPI_CLONE(SocketAddressList, mis->socket_address_list);
1261     }
1262
1263     switch (mis->state) {
1264     case MIGRATION_STATUS_NONE:
1265         return;
1266     case MIGRATION_STATUS_SETUP:
1267     case MIGRATION_STATUS_CANCELLING:
1268     case MIGRATION_STATUS_CANCELLED:
1269     case MIGRATION_STATUS_ACTIVE:
1270     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1271     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1272     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1273     case MIGRATION_STATUS_FAILED:
1274     case MIGRATION_STATUS_COLO:
1275         info->has_status = true;
1276         break;
1277     case MIGRATION_STATUS_COMPLETED:
1278         info->has_status = true;
1279         fill_destination_postcopy_migration_info(info);
1280         break;
1281     }
1282     info->status = mis->state;
1283 }
1284
1285 MigrationInfo *qmp_query_migrate(Error **errp)
1286 {
1287     MigrationInfo *info = g_malloc0(sizeof(*info));
1288
1289     fill_destination_migration_info(info);
1290     fill_source_migration_info(info);
1291
1292     return info;
1293 }
1294
1295 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
1296                                   Error **errp)
1297 {
1298     MigrationState *s = migrate_get_current();
1299     MigrationCapabilityStatusList *cap;
1300     bool cap_list[MIGRATION_CAPABILITY__MAX];
1301
1302     if (migration_is_running(s->state)) {
1303         error_setg(errp, QERR_MIGRATION_ACTIVE);
1304         return;
1305     }
1306
1307     memcpy(cap_list, s->enabled_capabilities, sizeof(cap_list));
1308     if (!migrate_caps_check(cap_list, params, errp)) {
1309         return;
1310     }
1311
1312     for (cap = params; cap; cap = cap->next) {
1313         s->enabled_capabilities[cap->value->capability] = cap->value->state;
1314     }
1315 }
1316
1317 /*
1318  * Check whether the parameters are valid. Error will be put into errp
1319  * (if provided). Return true if valid, otherwise false.
1320  */
1321 static bool migrate_params_check(MigrationParameters *params, Error **errp)
1322 {
1323     if (params->has_compress_level &&
1324         (params->compress_level > 9)) {
1325         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level",
1326                    "a value between 0 and 9");
1327         return false;
1328     }
1329
1330     if (params->has_compress_threads && (params->compress_threads < 1)) {
1331         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1332                    "compress_threads",
1333                    "a value between 1 and 255");
1334         return false;
1335     }
1336
1337     if (params->has_decompress_threads && (params->decompress_threads < 1)) {
1338         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1339                    "decompress_threads",
1340                    "a value between 1 and 255");
1341         return false;
1342     }
1343
1344     if (params->has_throttle_trigger_threshold &&
1345         (params->throttle_trigger_threshold < 1 ||
1346          params->throttle_trigger_threshold > 100)) {
1347         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1348                    "throttle_trigger_threshold",
1349                    "an integer in the range of 1 to 100");
1350         return false;
1351     }
1352
1353     if (params->has_cpu_throttle_initial &&
1354         (params->cpu_throttle_initial < 1 ||
1355          params->cpu_throttle_initial > 99)) {
1356         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1357                    "cpu_throttle_initial",
1358                    "an integer in the range of 1 to 99");
1359         return false;
1360     }
1361
1362     if (params->has_cpu_throttle_increment &&
1363         (params->cpu_throttle_increment < 1 ||
1364          params->cpu_throttle_increment > 99)) {
1365         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1366                    "cpu_throttle_increment",
1367                    "an integer in the range of 1 to 99");
1368         return false;
1369     }
1370
1371     if (params->has_max_bandwidth && (params->max_bandwidth > SIZE_MAX)) {
1372         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1373                    "max_bandwidth",
1374                    "an integer in the range of 0 to "stringify(SIZE_MAX)
1375                    " bytes/second");
1376         return false;
1377     }
1378
1379     if (params->has_downtime_limit &&
1380         (params->downtime_limit > MAX_MIGRATE_DOWNTIME)) {
1381         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1382                    "downtime_limit",
1383                    "an integer in the range of 0 to "
1384                     stringify(MAX_MIGRATE_DOWNTIME)" ms");
1385         return false;
1386     }
1387
1388     /* x_checkpoint_delay is now always positive */
1389
1390     if (params->has_multifd_channels && (params->multifd_channels < 1)) {
1391         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1392                    "multifd_channels",
1393                    "a value between 1 and 255");
1394         return false;
1395     }
1396
1397     if (params->has_multifd_zlib_level &&
1398         (params->multifd_zlib_level > 9)) {
1399         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zlib_level",
1400                    "a value between 0 and 9");
1401         return false;
1402     }
1403
1404     if (params->has_multifd_zstd_level &&
1405         (params->multifd_zstd_level > 20)) {
1406         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zstd_level",
1407                    "a value between 0 and 20");
1408         return false;
1409     }
1410
1411     if (params->has_xbzrle_cache_size &&
1412         (params->xbzrle_cache_size < qemu_target_page_size() ||
1413          !is_power_of_2(params->xbzrle_cache_size))) {
1414         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1415                    "xbzrle_cache_size",
1416                    "a power of two no less than the target page size");
1417         return false;
1418     }
1419
1420     if (params->has_max_cpu_throttle &&
1421         (params->max_cpu_throttle < params->cpu_throttle_initial ||
1422          params->max_cpu_throttle > 99)) {
1423         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1424                    "max_cpu_throttle",
1425                    "an integer in the range of cpu_throttle_initial to 99");
1426         return false;
1427     }
1428
1429     if (params->has_announce_initial &&
1430         params->announce_initial > 100000) {
1431         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1432                    "announce_initial",
1433                    "a value between 0 and 100000");
1434         return false;
1435     }
1436     if (params->has_announce_max &&
1437         params->announce_max > 100000) {
1438         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1439                    "announce_max",
1440                    "a value between 0 and 100000");
1441        return false;
1442     }
1443     if (params->has_announce_rounds &&
1444         params->announce_rounds > 1000) {
1445         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1446                    "announce_rounds",
1447                    "a value between 0 and 1000");
1448        return false;
1449     }
1450     if (params->has_announce_step &&
1451         (params->announce_step < 1 ||
1452         params->announce_step > 10000)) {
1453         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1454                    "announce_step",
1455                    "a value between 0 and 10000");
1456        return false;
1457     }
1458
1459     if (params->has_block_bitmap_mapping &&
1460         !check_dirty_bitmap_mig_alias_map(params->block_bitmap_mapping, errp)) {
1461         error_prepend(errp, "Invalid mapping given for block-bitmap-mapping: ");
1462         return false;
1463     }
1464
1465     return true;
1466 }
1467
1468 static void migrate_params_test_apply(MigrateSetParameters *params,
1469                                       MigrationParameters *dest)
1470 {
1471     *dest = migrate_get_current()->parameters;
1472
1473     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1474
1475     if (params->has_compress_level) {
1476         dest->compress_level = params->compress_level;
1477     }
1478
1479     if (params->has_compress_threads) {
1480         dest->compress_threads = params->compress_threads;
1481     }
1482
1483     if (params->has_compress_wait_thread) {
1484         dest->compress_wait_thread = params->compress_wait_thread;
1485     }
1486
1487     if (params->has_decompress_threads) {
1488         dest->decompress_threads = params->decompress_threads;
1489     }
1490
1491     if (params->has_throttle_trigger_threshold) {
1492         dest->throttle_trigger_threshold = params->throttle_trigger_threshold;
1493     }
1494
1495     if (params->has_cpu_throttle_initial) {
1496         dest->cpu_throttle_initial = params->cpu_throttle_initial;
1497     }
1498
1499     if (params->has_cpu_throttle_increment) {
1500         dest->cpu_throttle_increment = params->cpu_throttle_increment;
1501     }
1502
1503     if (params->has_cpu_throttle_tailslow) {
1504         dest->cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1505     }
1506
1507     if (params->has_tls_creds) {
1508         assert(params->tls_creds->type == QTYPE_QSTRING);
1509         dest->tls_creds = params->tls_creds->u.s;
1510     }
1511
1512     if (params->has_tls_hostname) {
1513         assert(params->tls_hostname->type == QTYPE_QSTRING);
1514         dest->tls_hostname = params->tls_hostname->u.s;
1515     }
1516
1517     if (params->has_max_bandwidth) {
1518         dest->max_bandwidth = params->max_bandwidth;
1519     }
1520
1521     if (params->has_downtime_limit) {
1522         dest->downtime_limit = params->downtime_limit;
1523     }
1524
1525     if (params->has_x_checkpoint_delay) {
1526         dest->x_checkpoint_delay = params->x_checkpoint_delay;
1527     }
1528
1529     if (params->has_block_incremental) {
1530         dest->block_incremental = params->block_incremental;
1531     }
1532     if (params->has_multifd_channels) {
1533         dest->multifd_channels = params->multifd_channels;
1534     }
1535     if (params->has_multifd_compression) {
1536         dest->multifd_compression = params->multifd_compression;
1537     }
1538     if (params->has_xbzrle_cache_size) {
1539         dest->xbzrle_cache_size = params->xbzrle_cache_size;
1540     }
1541     if (params->has_max_postcopy_bandwidth) {
1542         dest->max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1543     }
1544     if (params->has_max_cpu_throttle) {
1545         dest->max_cpu_throttle = params->max_cpu_throttle;
1546     }
1547     if (params->has_announce_initial) {
1548         dest->announce_initial = params->announce_initial;
1549     }
1550     if (params->has_announce_max) {
1551         dest->announce_max = params->announce_max;
1552     }
1553     if (params->has_announce_rounds) {
1554         dest->announce_rounds = params->announce_rounds;
1555     }
1556     if (params->has_announce_step) {
1557         dest->announce_step = params->announce_step;
1558     }
1559
1560     if (params->has_block_bitmap_mapping) {
1561         dest->has_block_bitmap_mapping = true;
1562         dest->block_bitmap_mapping = params->block_bitmap_mapping;
1563     }
1564 }
1565
1566 static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
1567 {
1568     MigrationState *s = migrate_get_current();
1569
1570     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1571
1572     if (params->has_compress_level) {
1573         s->parameters.compress_level = params->compress_level;
1574     }
1575
1576     if (params->has_compress_threads) {
1577         s->parameters.compress_threads = params->compress_threads;
1578     }
1579
1580     if (params->has_compress_wait_thread) {
1581         s->parameters.compress_wait_thread = params->compress_wait_thread;
1582     }
1583
1584     if (params->has_decompress_threads) {
1585         s->parameters.decompress_threads = params->decompress_threads;
1586     }
1587
1588     if (params->has_throttle_trigger_threshold) {
1589         s->parameters.throttle_trigger_threshold = params->throttle_trigger_threshold;
1590     }
1591
1592     if (params->has_cpu_throttle_initial) {
1593         s->parameters.cpu_throttle_initial = params->cpu_throttle_initial;
1594     }
1595
1596     if (params->has_cpu_throttle_increment) {
1597         s->parameters.cpu_throttle_increment = params->cpu_throttle_increment;
1598     }
1599
1600     if (params->has_cpu_throttle_tailslow) {
1601         s->parameters.cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1602     }
1603
1604     if (params->has_tls_creds) {
1605         g_free(s->parameters.tls_creds);
1606         assert(params->tls_creds->type == QTYPE_QSTRING);
1607         s->parameters.tls_creds = g_strdup(params->tls_creds->u.s);
1608     }
1609
1610     if (params->has_tls_hostname) {
1611         g_free(s->parameters.tls_hostname);
1612         assert(params->tls_hostname->type == QTYPE_QSTRING);
1613         s->parameters.tls_hostname = g_strdup(params->tls_hostname->u.s);
1614     }
1615
1616     if (params->has_tls_authz) {
1617         g_free(s->parameters.tls_authz);
1618         assert(params->tls_authz->type == QTYPE_QSTRING);
1619         s->parameters.tls_authz = g_strdup(params->tls_authz->u.s);
1620     }
1621
1622     if (params->has_max_bandwidth) {
1623         s->parameters.max_bandwidth = params->max_bandwidth;
1624         if (s->to_dst_file && !migration_in_postcopy()) {
1625             qemu_file_set_rate_limit(s->to_dst_file,
1626                                 s->parameters.max_bandwidth / XFER_LIMIT_RATIO);
1627         }
1628     }
1629
1630     if (params->has_downtime_limit) {
1631         s->parameters.downtime_limit = params->downtime_limit;
1632     }
1633
1634     if (params->has_x_checkpoint_delay) {
1635         s->parameters.x_checkpoint_delay = params->x_checkpoint_delay;
1636         if (migration_in_colo_state()) {
1637             colo_checkpoint_notify(s);
1638         }
1639     }
1640
1641     if (params->has_block_incremental) {
1642         s->parameters.block_incremental = params->block_incremental;
1643     }
1644     if (params->has_multifd_channels) {
1645         s->parameters.multifd_channels = params->multifd_channels;
1646     }
1647     if (params->has_multifd_compression) {
1648         s->parameters.multifd_compression = params->multifd_compression;
1649     }
1650     if (params->has_xbzrle_cache_size) {
1651         s->parameters.xbzrle_cache_size = params->xbzrle_cache_size;
1652         xbzrle_cache_resize(params->xbzrle_cache_size, errp);
1653     }
1654     if (params->has_max_postcopy_bandwidth) {
1655         s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1656         if (s->to_dst_file && migration_in_postcopy()) {
1657             qemu_file_set_rate_limit(s->to_dst_file,
1658                     s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO);
1659         }
1660     }
1661     if (params->has_max_cpu_throttle) {
1662         s->parameters.max_cpu_throttle = params->max_cpu_throttle;
1663     }
1664     if (params->has_announce_initial) {
1665         s->parameters.announce_initial = params->announce_initial;
1666     }
1667     if (params->has_announce_max) {
1668         s->parameters.announce_max = params->announce_max;
1669     }
1670     if (params->has_announce_rounds) {
1671         s->parameters.announce_rounds = params->announce_rounds;
1672     }
1673     if (params->has_announce_step) {
1674         s->parameters.announce_step = params->announce_step;
1675     }
1676
1677     if (params->has_block_bitmap_mapping) {
1678         qapi_free_BitmapMigrationNodeAliasList(
1679             s->parameters.block_bitmap_mapping);
1680
1681         s->parameters.has_block_bitmap_mapping = true;
1682         s->parameters.block_bitmap_mapping =
1683             QAPI_CLONE(BitmapMigrationNodeAliasList,
1684                        params->block_bitmap_mapping);
1685     }
1686 }
1687
1688 void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
1689 {
1690     MigrationParameters tmp;
1691
1692     /* TODO Rewrite "" to null instead */
1693     if (params->has_tls_creds
1694         && params->tls_creds->type == QTYPE_QNULL) {
1695         qobject_unref(params->tls_creds->u.n);
1696         params->tls_creds->type = QTYPE_QSTRING;
1697         params->tls_creds->u.s = strdup("");
1698     }
1699     /* TODO Rewrite "" to null instead */
1700     if (params->has_tls_hostname
1701         && params->tls_hostname->type == QTYPE_QNULL) {
1702         qobject_unref(params->tls_hostname->u.n);
1703         params->tls_hostname->type = QTYPE_QSTRING;
1704         params->tls_hostname->u.s = strdup("");
1705     }
1706
1707     migrate_params_test_apply(params, &tmp);
1708
1709     if (!migrate_params_check(&tmp, errp)) {
1710         /* Invalid parameter */
1711         return;
1712     }
1713
1714     migrate_params_apply(params, errp);
1715 }
1716
1717
1718 void qmp_migrate_start_postcopy(Error **errp)
1719 {
1720     MigrationState *s = migrate_get_current();
1721
1722     if (!migrate_postcopy()) {
1723         error_setg(errp, "Enable postcopy with migrate_set_capability before"
1724                          " the start of migration");
1725         return;
1726     }
1727
1728     if (s->state == MIGRATION_STATUS_NONE) {
1729         error_setg(errp, "Postcopy must be started after migration has been"
1730                          " started");
1731         return;
1732     }
1733     /*
1734      * we don't error if migration has finished since that would be racy
1735      * with issuing this command.
1736      */
1737     qatomic_set(&s->start_postcopy, true);
1738 }
1739
1740 /* shared migration helpers */
1741
1742 void migrate_set_state(int *state, int old_state, int new_state)
1743 {
1744     assert(new_state < MIGRATION_STATUS__MAX);
1745     if (qatomic_cmpxchg(state, old_state, new_state) == old_state) {
1746         trace_migrate_set_state(MigrationStatus_str(new_state));
1747         migrate_generate_event(new_state);
1748     }
1749 }
1750
1751 static MigrationCapabilityStatus *migrate_cap_add(MigrationCapability index,
1752                                                   bool state)
1753 {
1754     MigrationCapabilityStatus *cap;
1755
1756     cap = g_new0(MigrationCapabilityStatus, 1);
1757     cap->capability = index;
1758     cap->state = state;
1759
1760     return cap;
1761 }
1762
1763 void migrate_set_block_enabled(bool value, Error **errp)
1764 {
1765     MigrationCapabilityStatusList *cap = NULL;
1766
1767     QAPI_LIST_PREPEND(cap, migrate_cap_add(MIGRATION_CAPABILITY_BLOCK, value));
1768     qmp_migrate_set_capabilities(cap, errp);
1769     qapi_free_MigrationCapabilityStatusList(cap);
1770 }
1771
1772 static void migrate_set_block_incremental(MigrationState *s, bool value)
1773 {
1774     s->parameters.block_incremental = value;
1775 }
1776
1777 static void block_cleanup_parameters(MigrationState *s)
1778 {
1779     if (s->must_remove_block_options) {
1780         /* setting to false can never fail */
1781         migrate_set_block_enabled(false, &error_abort);
1782         migrate_set_block_incremental(s, false);
1783         s->must_remove_block_options = false;
1784     }
1785 }
1786
1787 static void migrate_fd_cleanup(MigrationState *s)
1788 {
1789     qemu_bh_delete(s->cleanup_bh);
1790     s->cleanup_bh = NULL;
1791
1792     qemu_savevm_state_cleanup();
1793
1794     if (s->to_dst_file) {
1795         QEMUFile *tmp;
1796
1797         trace_migrate_fd_cleanup();
1798         qemu_mutex_unlock_iothread();
1799         if (s->migration_thread_running) {
1800             qemu_thread_join(&s->thread);
1801             s->migration_thread_running = false;
1802         }
1803         qemu_mutex_lock_iothread();
1804
1805         multifd_save_cleanup();
1806         qemu_mutex_lock(&s->qemu_file_lock);
1807         tmp = s->to_dst_file;
1808         s->to_dst_file = NULL;
1809         qemu_mutex_unlock(&s->qemu_file_lock);
1810         /*
1811          * Close the file handle without the lock to make sure the
1812          * critical section won't block for long.
1813          */
1814         qemu_fclose(tmp);
1815     }
1816
1817     assert(!migration_is_active(s));
1818
1819     if (s->state == MIGRATION_STATUS_CANCELLING) {
1820         migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING,
1821                           MIGRATION_STATUS_CANCELLED);
1822     }
1823
1824     if (s->error) {
1825         /* It is used on info migrate.  We can't free it */
1826         error_report_err(error_copy(s->error));
1827     }
1828     notifier_list_notify(&migration_state_notifiers, s);
1829     block_cleanup_parameters(s);
1830     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
1831 }
1832
1833 static void migrate_fd_cleanup_schedule(MigrationState *s)
1834 {
1835     /*
1836      * Ref the state for bh, because it may be called when
1837      * there're already no other refs
1838      */
1839     object_ref(OBJECT(s));
1840     qemu_bh_schedule(s->cleanup_bh);
1841 }
1842
1843 static void migrate_fd_cleanup_bh(void *opaque)
1844 {
1845     MigrationState *s = opaque;
1846     migrate_fd_cleanup(s);
1847     object_unref(OBJECT(s));
1848 }
1849
1850 void migrate_set_error(MigrationState *s, const Error *error)
1851 {
1852     QEMU_LOCK_GUARD(&s->error_mutex);
1853     if (!s->error) {
1854         s->error = error_copy(error);
1855     }
1856 }
1857
1858 static void migrate_error_free(MigrationState *s)
1859 {
1860     QEMU_LOCK_GUARD(&s->error_mutex);
1861     if (s->error) {
1862         error_free(s->error);
1863         s->error = NULL;
1864     }
1865 }
1866
1867 void migrate_fd_error(MigrationState *s, const Error *error)
1868 {
1869     trace_migrate_fd_error(error_get_pretty(error));
1870     assert(s->to_dst_file == NULL);
1871     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
1872                       MIGRATION_STATUS_FAILED);
1873     migrate_set_error(s, error);
1874 }
1875
1876 static void migrate_fd_cancel(MigrationState *s)
1877 {
1878     int old_state ;
1879     QEMUFile *f = migrate_get_current()->to_dst_file;
1880     trace_migrate_fd_cancel();
1881
1882     WITH_QEMU_LOCK_GUARD(&s->qemu_file_lock) {
1883         if (s->rp_state.from_dst_file) {
1884             /* shutdown the rp socket, so causing the rp thread to shutdown */
1885             qemu_file_shutdown(s->rp_state.from_dst_file);
1886         }
1887     }
1888
1889     do {
1890         old_state = s->state;
1891         if (!migration_is_running(old_state)) {
1892             break;
1893         }
1894         /* If the migration is paused, kick it out of the pause */
1895         if (old_state == MIGRATION_STATUS_PRE_SWITCHOVER) {
1896             qemu_sem_post(&s->pause_sem);
1897         }
1898         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING);
1899     } while (s->state != MIGRATION_STATUS_CANCELLING);
1900
1901     /*
1902      * If we're unlucky the migration code might be stuck somewhere in a
1903      * send/write while the network has failed and is waiting to timeout;
1904      * if we've got shutdown(2) available then we can force it to quit.
1905      * The outgoing qemu file gets closed in migrate_fd_cleanup that is
1906      * called in a bh, so there is no race against this cancel.
1907      */
1908     if (s->state == MIGRATION_STATUS_CANCELLING && f) {
1909         qemu_file_shutdown(f);
1910     }
1911     if (s->state == MIGRATION_STATUS_CANCELLING && s->block_inactive) {
1912         Error *local_err = NULL;
1913
1914         bdrv_invalidate_cache_all(&local_err);
1915         if (local_err) {
1916             error_report_err(local_err);
1917         } else {
1918             s->block_inactive = false;
1919         }
1920     }
1921 }
1922
1923 void add_migration_state_change_notifier(Notifier *notify)
1924 {
1925     notifier_list_add(&migration_state_notifiers, notify);
1926 }
1927
1928 void remove_migration_state_change_notifier(Notifier *notify)
1929 {
1930     notifier_remove(notify);
1931 }
1932
1933 bool migration_in_setup(MigrationState *s)
1934 {
1935     return s->state == MIGRATION_STATUS_SETUP;
1936 }
1937
1938 bool migration_has_finished(MigrationState *s)
1939 {
1940     return s->state == MIGRATION_STATUS_COMPLETED;
1941 }
1942
1943 bool migration_has_failed(MigrationState *s)
1944 {
1945     return (s->state == MIGRATION_STATUS_CANCELLED ||
1946             s->state == MIGRATION_STATUS_FAILED);
1947 }
1948
1949 bool migration_in_postcopy(void)
1950 {
1951     MigrationState *s = migrate_get_current();
1952
1953     switch (s->state) {
1954     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1955     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1956     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1957         return true;
1958     default:
1959         return false;
1960     }
1961 }
1962
1963 bool migration_in_postcopy_after_devices(MigrationState *s)
1964 {
1965     return migration_in_postcopy() && s->postcopy_after_devices;
1966 }
1967
1968 bool migration_in_incoming_postcopy(void)
1969 {
1970     PostcopyState ps = postcopy_state_get();
1971
1972     return ps >= POSTCOPY_INCOMING_DISCARD && ps < POSTCOPY_INCOMING_END;
1973 }
1974
1975 bool migration_in_bg_snapshot(void)
1976 {
1977     MigrationState *s = migrate_get_current();
1978
1979     return migrate_background_snapshot() &&
1980             migration_is_setup_or_active(s->state);
1981 }
1982
1983 bool migration_is_idle(void)
1984 {
1985     MigrationState *s = current_migration;
1986
1987     if (!s) {
1988         return true;
1989     }
1990
1991     switch (s->state) {
1992     case MIGRATION_STATUS_NONE:
1993     case MIGRATION_STATUS_CANCELLED:
1994     case MIGRATION_STATUS_COMPLETED:
1995     case MIGRATION_STATUS_FAILED:
1996         return true;
1997     case MIGRATION_STATUS_SETUP:
1998     case MIGRATION_STATUS_CANCELLING:
1999     case MIGRATION_STATUS_ACTIVE:
2000     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
2001     case MIGRATION_STATUS_COLO:
2002     case MIGRATION_STATUS_PRE_SWITCHOVER:
2003     case MIGRATION_STATUS_DEVICE:
2004     case MIGRATION_STATUS_WAIT_UNPLUG:
2005         return false;
2006     case MIGRATION_STATUS__MAX:
2007         g_assert_not_reached();
2008     }
2009
2010     return false;
2011 }
2012
2013 bool migration_is_active(MigrationState *s)
2014 {
2015     return (s->state == MIGRATION_STATUS_ACTIVE ||
2016             s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
2017 }
2018
2019 void migrate_init(MigrationState *s)
2020 {
2021     /*
2022      * Reinitialise all migration state, except
2023      * parameters/capabilities that the user set, and
2024      * locks.
2025      */
2026     s->cleanup_bh = 0;
2027     s->vm_start_bh = 0;
2028     s->to_dst_file = NULL;
2029     s->state = MIGRATION_STATUS_NONE;
2030     s->rp_state.from_dst_file = NULL;
2031     s->rp_state.error = false;
2032     s->mbps = 0.0;
2033     s->pages_per_second = 0.0;
2034     s->downtime = 0;
2035     s->expected_downtime = 0;
2036     s->setup_time = 0;
2037     s->start_postcopy = false;
2038     s->postcopy_after_devices = false;
2039     s->migration_thread_running = false;
2040     error_free(s->error);
2041     s->error = NULL;
2042     s->hostname = NULL;
2043
2044     migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
2045
2046     s->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2047     s->total_time = 0;
2048     s->vm_was_running = false;
2049     s->iteration_initial_bytes = 0;
2050     s->threshold_size = 0;
2051 }
2052
2053 int migrate_add_blocker(Error *reason, Error **errp)
2054 {
2055     if (only_migratable) {
2056         error_propagate_prepend(errp, error_copy(reason),
2057                                 "disallowing migration blocker "
2058                                 "(--only-migratable) for: ");
2059         return -EACCES;
2060     }
2061
2062     if (migration_is_idle()) {
2063         migration_blockers = g_slist_prepend(migration_blockers, reason);
2064         return 0;
2065     }
2066
2067     error_propagate_prepend(errp, error_copy(reason),
2068                             "disallowing migration blocker "
2069                             "(migration in progress) for: ");
2070     return -EBUSY;
2071 }
2072
2073 void migrate_del_blocker(Error *reason)
2074 {
2075     migration_blockers = g_slist_remove(migration_blockers, reason);
2076 }
2077
2078 void qmp_migrate_incoming(const char *uri, Error **errp)
2079 {
2080     Error *local_err = NULL;
2081     static bool once = true;
2082
2083     if (!once) {
2084         error_setg(errp, "The incoming migration has already been started");
2085         return;
2086     }
2087     if (!runstate_check(RUN_STATE_INMIGRATE)) {
2088         error_setg(errp, "'-incoming' was not specified on the command line");
2089         return;
2090     }
2091
2092     if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2093         return;
2094     }
2095
2096     qemu_start_incoming_migration(uri, &local_err);
2097
2098     if (local_err) {
2099         yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2100         error_propagate(errp, local_err);
2101         return;
2102     }
2103
2104     once = false;
2105 }
2106
2107 void qmp_migrate_recover(const char *uri, Error **errp)
2108 {
2109     MigrationIncomingState *mis = migration_incoming_get_current();
2110
2111     /*
2112      * Don't even bother to use ERRP_GUARD() as it _must_ always be set by
2113      * callers (no one should ignore a recover failure); if there is, it's a
2114      * programming error.
2115      */
2116     assert(errp);
2117
2118     if (mis->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2119         error_setg(errp, "Migrate recover can only be run "
2120                    "when postcopy is paused.");
2121         return;
2122     }
2123
2124     if (qatomic_cmpxchg(&mis->postcopy_recover_triggered,
2125                        false, true) == true) {
2126         error_setg(errp, "Migrate recovery is triggered already");
2127         return;
2128     }
2129
2130     /*
2131      * Note that this call will never start a real migration; it will
2132      * only re-setup the migration stream and poke existing migration
2133      * to continue using that newly established channel.
2134      */
2135     qemu_start_incoming_migration(uri, errp);
2136
2137     /* Safe to dereference with the assert above */
2138     if (*errp) {
2139         /* Reset the flag so user could still retry */
2140         qatomic_set(&mis->postcopy_recover_triggered, false);
2141     }
2142 }
2143
2144 void qmp_migrate_pause(Error **errp)
2145 {
2146     MigrationState *ms = migrate_get_current();
2147     MigrationIncomingState *mis = migration_incoming_get_current();
2148     int ret;
2149
2150     if (ms->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2151         /* Source side, during postcopy */
2152         qemu_mutex_lock(&ms->qemu_file_lock);
2153         ret = qemu_file_shutdown(ms->to_dst_file);
2154         qemu_mutex_unlock(&ms->qemu_file_lock);
2155         if (ret) {
2156             error_setg(errp, "Failed to pause source migration");
2157         }
2158         return;
2159     }
2160
2161     if (mis->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2162         ret = qemu_file_shutdown(mis->from_src_file);
2163         if (ret) {
2164             error_setg(errp, "Failed to pause destination migration");
2165         }
2166         return;
2167     }
2168
2169     error_setg(errp, "migrate-pause is currently only supported "
2170                "during postcopy-active state");
2171 }
2172
2173 bool migration_is_blocked(Error **errp)
2174 {
2175     if (qemu_savevm_state_blocked(errp)) {
2176         return true;
2177     }
2178
2179     if (migration_blockers) {
2180         error_propagate(errp, error_copy(migration_blockers->data));
2181         return true;
2182     }
2183
2184     return false;
2185 }
2186
2187 /* Returns true if continue to migrate, or false if error detected */
2188 static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
2189                             bool resume, Error **errp)
2190 {
2191     Error *local_err = NULL;
2192
2193     if (resume) {
2194         if (s->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2195             error_setg(errp, "Cannot resume if there is no "
2196                        "paused migration");
2197             return false;
2198         }
2199
2200         /*
2201          * Postcopy recovery won't work well with release-ram
2202          * capability since release-ram will drop the page buffer as
2203          * long as the page is put into the send buffer.  So if there
2204          * is a network failure happened, any page buffers that have
2205          * not yet reached the destination VM but have already been
2206          * sent from the source VM will be lost forever.  Let's refuse
2207          * the client from resuming such a postcopy migration.
2208          * Luckily release-ram was designed to only be used when src
2209          * and destination VMs are on the same host, so it should be
2210          * fine.
2211          */
2212         if (migrate_release_ram()) {
2213             error_setg(errp, "Postcopy recovery cannot work "
2214                        "when release-ram capability is set");
2215             return false;
2216         }
2217
2218         /* This is a resume, skip init status */
2219         return true;
2220     }
2221
2222     if (migration_is_running(s->state)) {
2223         error_setg(errp, QERR_MIGRATION_ACTIVE);
2224         return false;
2225     }
2226
2227     if (runstate_check(RUN_STATE_INMIGRATE)) {
2228         error_setg(errp, "Guest is waiting for an incoming migration");
2229         return false;
2230     }
2231
2232     if (runstate_check(RUN_STATE_POSTMIGRATE)) {
2233         error_setg(errp, "Can't migrate the vm that was paused due to "
2234                    "previous migration");
2235         return false;
2236     }
2237
2238     if (migration_is_blocked(errp)) {
2239         return false;
2240     }
2241
2242     if (blk || blk_inc) {
2243         if (migrate_colo_enabled()) {
2244             error_setg(errp, "No disk migration is required in COLO mode");
2245             return false;
2246         }
2247         if (migrate_use_block() || migrate_use_block_incremental()) {
2248             error_setg(errp, "Command options are incompatible with "
2249                        "current migration capabilities");
2250             return false;
2251         }
2252         migrate_set_block_enabled(true, &local_err);
2253         if (local_err) {
2254             error_propagate(errp, local_err);
2255             return false;
2256         }
2257         s->must_remove_block_options = true;
2258     }
2259
2260     if (blk_inc) {
2261         migrate_set_block_incremental(s, true);
2262     }
2263
2264     migrate_init(s);
2265     /*
2266      * set ram_counters memory to zero for a
2267      * new migration
2268      */
2269     memset(&ram_counters, 0, sizeof(ram_counters));
2270
2271     return true;
2272 }
2273
2274 void qmp_migrate(const char *uri, bool has_blk, bool blk,
2275                  bool has_inc, bool inc, bool has_detach, bool detach,
2276                  bool has_resume, bool resume, Error **errp)
2277 {
2278     Error *local_err = NULL;
2279     MigrationState *s = migrate_get_current();
2280     const char *p = NULL;
2281
2282     if (!migrate_prepare(s, has_blk && blk, has_inc && inc,
2283                          has_resume && resume, errp)) {
2284         /* Error detected, put into errp */
2285         return;
2286     }
2287
2288     if (!(has_resume && resume)) {
2289         if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2290             return;
2291         }
2292     }
2293
2294     if (strstart(uri, "tcp:", &p) ||
2295         strstart(uri, "unix:", NULL) ||
2296         strstart(uri, "vsock:", NULL)) {
2297         socket_start_outgoing_migration(s, p ? p : uri, &local_err);
2298 #ifdef CONFIG_RDMA
2299     } else if (strstart(uri, "rdma:", &p)) {
2300         rdma_start_outgoing_migration(s, p, &local_err);
2301 #endif
2302     } else if (strstart(uri, "exec:", &p)) {
2303         exec_start_outgoing_migration(s, p, &local_err);
2304     } else if (strstart(uri, "fd:", &p)) {
2305         fd_start_outgoing_migration(s, p, &local_err);
2306     } else {
2307         if (!(has_resume && resume)) {
2308             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2309         }
2310         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri",
2311                    "a valid migration protocol");
2312         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
2313                           MIGRATION_STATUS_FAILED);
2314         block_cleanup_parameters(s);
2315         return;
2316     }
2317
2318     if (local_err) {
2319         if (!(has_resume && resume)) {
2320             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2321         }
2322         migrate_fd_error(s, local_err);
2323         error_propagate(errp, local_err);
2324         return;
2325     }
2326 }
2327
2328 void qmp_migrate_cancel(Error **errp)
2329 {
2330     migration_cancel();
2331 }
2332
2333 void qmp_migrate_continue(MigrationStatus state, Error **errp)
2334 {
2335     MigrationState *s = migrate_get_current();
2336     if (s->state != state) {
2337         error_setg(errp,  "Migration not in expected state: %s",
2338                    MigrationStatus_str(s->state));
2339         return;
2340     }
2341     qemu_sem_post(&s->pause_sem);
2342 }
2343
2344 bool migrate_release_ram(void)
2345 {
2346     MigrationState *s;
2347
2348     s = migrate_get_current();
2349
2350     return s->enabled_capabilities[MIGRATION_CAPABILITY_RELEASE_RAM];
2351 }
2352
2353 bool migrate_postcopy_ram(void)
2354 {
2355     MigrationState *s;
2356
2357     s = migrate_get_current();
2358
2359     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM];
2360 }
2361
2362 bool migrate_postcopy(void)
2363 {
2364     return migrate_postcopy_ram() || migrate_dirty_bitmaps();
2365 }
2366
2367 bool migrate_auto_converge(void)
2368 {
2369     MigrationState *s;
2370
2371     s = migrate_get_current();
2372
2373     return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
2374 }
2375
2376 bool migrate_zero_blocks(void)
2377 {
2378     MigrationState *s;
2379
2380     s = migrate_get_current();
2381
2382     return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
2383 }
2384
2385 bool migrate_postcopy_blocktime(void)
2386 {
2387     MigrationState *s;
2388
2389     s = migrate_get_current();
2390
2391     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME];
2392 }
2393
2394 bool migrate_use_compression(void)
2395 {
2396     MigrationState *s;
2397
2398     s = migrate_get_current();
2399
2400     return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS];
2401 }
2402
2403 int migrate_compress_level(void)
2404 {
2405     MigrationState *s;
2406
2407     s = migrate_get_current();
2408
2409     return s->parameters.compress_level;
2410 }
2411
2412 int migrate_compress_threads(void)
2413 {
2414     MigrationState *s;
2415
2416     s = migrate_get_current();
2417
2418     return s->parameters.compress_threads;
2419 }
2420
2421 int migrate_compress_wait_thread(void)
2422 {
2423     MigrationState *s;
2424
2425     s = migrate_get_current();
2426
2427     return s->parameters.compress_wait_thread;
2428 }
2429
2430 int migrate_decompress_threads(void)
2431 {
2432     MigrationState *s;
2433
2434     s = migrate_get_current();
2435
2436     return s->parameters.decompress_threads;
2437 }
2438
2439 bool migrate_dirty_bitmaps(void)
2440 {
2441     MigrationState *s;
2442
2443     s = migrate_get_current();
2444
2445     return s->enabled_capabilities[MIGRATION_CAPABILITY_DIRTY_BITMAPS];
2446 }
2447
2448 bool migrate_ignore_shared(void)
2449 {
2450     MigrationState *s;
2451
2452     s = migrate_get_current();
2453
2454     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_IGNORE_SHARED];
2455 }
2456
2457 bool migrate_validate_uuid(void)
2458 {
2459     MigrationState *s;
2460
2461     s = migrate_get_current();
2462
2463     return s->enabled_capabilities[MIGRATION_CAPABILITY_VALIDATE_UUID];
2464 }
2465
2466 bool migrate_use_events(void)
2467 {
2468     MigrationState *s;
2469
2470     s = migrate_get_current();
2471
2472     return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS];
2473 }
2474
2475 bool migrate_use_multifd(void)
2476 {
2477     MigrationState *s;
2478
2479     s = migrate_get_current();
2480
2481     return s->enabled_capabilities[MIGRATION_CAPABILITY_MULTIFD];
2482 }
2483
2484 bool migrate_pause_before_switchover(void)
2485 {
2486     MigrationState *s;
2487
2488     s = migrate_get_current();
2489
2490     return s->enabled_capabilities[
2491         MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER];
2492 }
2493
2494 int migrate_multifd_channels(void)
2495 {
2496     MigrationState *s;
2497
2498     s = migrate_get_current();
2499
2500     return s->parameters.multifd_channels;
2501 }
2502
2503 MultiFDCompression migrate_multifd_compression(void)
2504 {
2505     MigrationState *s;
2506
2507     s = migrate_get_current();
2508
2509     return s->parameters.multifd_compression;
2510 }
2511
2512 int migrate_multifd_zlib_level(void)
2513 {
2514     MigrationState *s;
2515
2516     s = migrate_get_current();
2517
2518     return s->parameters.multifd_zlib_level;
2519 }
2520
2521 int migrate_multifd_zstd_level(void)
2522 {
2523     MigrationState *s;
2524
2525     s = migrate_get_current();
2526
2527     return s->parameters.multifd_zstd_level;
2528 }
2529
2530 int migrate_use_xbzrle(void)
2531 {
2532     MigrationState *s;
2533
2534     s = migrate_get_current();
2535
2536     return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
2537 }
2538
2539 uint64_t migrate_xbzrle_cache_size(void)
2540 {
2541     MigrationState *s;
2542
2543     s = migrate_get_current();
2544
2545     return s->parameters.xbzrle_cache_size;
2546 }
2547
2548 static int64_t migrate_max_postcopy_bandwidth(void)
2549 {
2550     MigrationState *s;
2551
2552     s = migrate_get_current();
2553
2554     return s->parameters.max_postcopy_bandwidth;
2555 }
2556
2557 bool migrate_use_block(void)
2558 {
2559     MigrationState *s;
2560
2561     s = migrate_get_current();
2562
2563     return s->enabled_capabilities[MIGRATION_CAPABILITY_BLOCK];
2564 }
2565
2566 bool migrate_use_return_path(void)
2567 {
2568     MigrationState *s;
2569
2570     s = migrate_get_current();
2571
2572     return s->enabled_capabilities[MIGRATION_CAPABILITY_RETURN_PATH];
2573 }
2574
2575 bool migrate_use_block_incremental(void)
2576 {
2577     MigrationState *s;
2578
2579     s = migrate_get_current();
2580
2581     return s->parameters.block_incremental;
2582 }
2583
2584 bool migrate_background_snapshot(void)
2585 {
2586     MigrationState *s;
2587
2588     s = migrate_get_current();
2589
2590     return s->enabled_capabilities[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT];
2591 }
2592
2593 /* migration thread support */
2594 /*
2595  * Something bad happened to the RP stream, mark an error
2596  * The caller shall print or trace something to indicate why
2597  */
2598 static void mark_source_rp_bad(MigrationState *s)
2599 {
2600     s->rp_state.error = true;
2601 }
2602
2603 static struct rp_cmd_args {
2604     ssize_t     len; /* -1 = variable */
2605     const char *name;
2606 } rp_cmd_args[] = {
2607     [MIG_RP_MSG_INVALID]        = { .len = -1, .name = "INVALID" },
2608     [MIG_RP_MSG_SHUT]           = { .len =  4, .name = "SHUT" },
2609     [MIG_RP_MSG_PONG]           = { .len =  4, .name = "PONG" },
2610     [MIG_RP_MSG_REQ_PAGES]      = { .len = 12, .name = "REQ_PAGES" },
2611     [MIG_RP_MSG_REQ_PAGES_ID]   = { .len = -1, .name = "REQ_PAGES_ID" },
2612     [MIG_RP_MSG_RECV_BITMAP]    = { .len = -1, .name = "RECV_BITMAP" },
2613     [MIG_RP_MSG_RESUME_ACK]     = { .len =  4, .name = "RESUME_ACK" },
2614     [MIG_RP_MSG_MAX]            = { .len = -1, .name = "MAX" },
2615 };
2616
2617 /*
2618  * Process a request for pages received on the return path,
2619  * We're allowed to send more than requested (e.g. to round to our page size)
2620  * and we don't need to send pages that have already been sent.
2621  */
2622 static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
2623                                        ram_addr_t start, size_t len)
2624 {
2625     long our_host_ps = qemu_real_host_page_size;
2626
2627     trace_migrate_handle_rp_req_pages(rbname, start, len);
2628
2629     /*
2630      * Since we currently insist on matching page sizes, just sanity check
2631      * we're being asked for whole host pages.
2632      */
2633     if (start & (our_host_ps - 1) ||
2634        (len & (our_host_ps - 1))) {
2635         error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
2636                      " len: %zd", __func__, start, len);
2637         mark_source_rp_bad(ms);
2638         return;
2639     }
2640
2641     if (ram_save_queue_pages(rbname, start, len)) {
2642         mark_source_rp_bad(ms);
2643     }
2644 }
2645
2646 /* Return true to retry, false to quit */
2647 static bool postcopy_pause_return_path_thread(MigrationState *s)
2648 {
2649     trace_postcopy_pause_return_path();
2650
2651     qemu_sem_wait(&s->postcopy_pause_rp_sem);
2652
2653     trace_postcopy_pause_return_path_continued();
2654
2655     return true;
2656 }
2657
2658 static int migrate_handle_rp_recv_bitmap(MigrationState *s, char *block_name)
2659 {
2660     RAMBlock *block = qemu_ram_block_by_name(block_name);
2661
2662     if (!block) {
2663         error_report("%s: invalid block name '%s'", __func__, block_name);
2664         return -EINVAL;
2665     }
2666
2667     /* Fetch the received bitmap and refresh the dirty bitmap */
2668     return ram_dirty_bitmap_reload(s, block);
2669 }
2670
2671 static int migrate_handle_rp_resume_ack(MigrationState *s, uint32_t value)
2672 {
2673     trace_source_return_path_thread_resume_ack(value);
2674
2675     if (value != MIGRATION_RESUME_ACK_VALUE) {
2676         error_report("%s: illegal resume_ack value %"PRIu32,
2677                      __func__, value);
2678         return -1;
2679     }
2680
2681     /* Now both sides are active. */
2682     migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_RECOVER,
2683                       MIGRATION_STATUS_POSTCOPY_ACTIVE);
2684
2685     /* Notify send thread that time to continue send pages */
2686     qemu_sem_post(&s->rp_state.rp_sem);
2687
2688     return 0;
2689 }
2690
2691 /* Release ms->rp_state.from_dst_file in a safe way */
2692 static void migration_release_from_dst_file(MigrationState *ms)
2693 {
2694     QEMUFile *file;
2695
2696     WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) {
2697         /*
2698          * Reset the from_dst_file pointer first before releasing it, as we
2699          * can't block within lock section
2700          */
2701         file = ms->rp_state.from_dst_file;
2702         ms->rp_state.from_dst_file = NULL;
2703     }
2704
2705     qemu_fclose(file);
2706 }
2707
2708 /*
2709  * Handles messages sent on the return path towards the source VM
2710  *
2711  */
2712 static void *source_return_path_thread(void *opaque)
2713 {
2714     MigrationState *ms = opaque;
2715     QEMUFile *rp = ms->rp_state.from_dst_file;
2716     uint16_t header_len, header_type;
2717     uint8_t buf[512];
2718     uint32_t tmp32, sibling_error;
2719     ram_addr_t start = 0; /* =0 to silence warning */
2720     size_t  len = 0, expected_len;
2721     int res;
2722
2723     trace_source_return_path_thread_entry();
2724     rcu_register_thread();
2725
2726 retry:
2727     while (!ms->rp_state.error && !qemu_file_get_error(rp) &&
2728            migration_is_setup_or_active(ms->state)) {
2729         trace_source_return_path_thread_loop_top();
2730         header_type = qemu_get_be16(rp);
2731         header_len = qemu_get_be16(rp);
2732
2733         if (qemu_file_get_error(rp)) {
2734             mark_source_rp_bad(ms);
2735             goto out;
2736         }
2737
2738         if (header_type >= MIG_RP_MSG_MAX ||
2739             header_type == MIG_RP_MSG_INVALID) {
2740             error_report("RP: Received invalid message 0x%04x length 0x%04x",
2741                          header_type, header_len);
2742             mark_source_rp_bad(ms);
2743             goto out;
2744         }
2745
2746         if ((rp_cmd_args[header_type].len != -1 &&
2747             header_len != rp_cmd_args[header_type].len) ||
2748             header_len > sizeof(buf)) {
2749             error_report("RP: Received '%s' message (0x%04x) with"
2750                          "incorrect length %d expecting %zu",
2751                          rp_cmd_args[header_type].name, header_type, header_len,
2752                          (size_t)rp_cmd_args[header_type].len);
2753             mark_source_rp_bad(ms);
2754             goto out;
2755         }
2756
2757         /* We know we've got a valid header by this point */
2758         res = qemu_get_buffer(rp, buf, header_len);
2759         if (res != header_len) {
2760             error_report("RP: Failed reading data for message 0x%04x"
2761                          " read %d expected %d",
2762                          header_type, res, header_len);
2763             mark_source_rp_bad(ms);
2764             goto out;
2765         }
2766
2767         /* OK, we have the message and the data */
2768         switch (header_type) {
2769         case MIG_RP_MSG_SHUT:
2770             sibling_error = ldl_be_p(buf);
2771             trace_source_return_path_thread_shut(sibling_error);
2772             if (sibling_error) {
2773                 error_report("RP: Sibling indicated error %d", sibling_error);
2774                 mark_source_rp_bad(ms);
2775             }
2776             /*
2777              * We'll let the main thread deal with closing the RP
2778              * we could do a shutdown(2) on it, but we're the only user
2779              * anyway, so there's nothing gained.
2780              */
2781             goto out;
2782
2783         case MIG_RP_MSG_PONG:
2784             tmp32 = ldl_be_p(buf);
2785             trace_source_return_path_thread_pong(tmp32);
2786             break;
2787
2788         case MIG_RP_MSG_REQ_PAGES:
2789             start = ldq_be_p(buf);
2790             len = ldl_be_p(buf + 8);
2791             migrate_handle_rp_req_pages(ms, NULL, start, len);
2792             break;
2793
2794         case MIG_RP_MSG_REQ_PAGES_ID:
2795             expected_len = 12 + 1; /* header + termination */
2796
2797             if (header_len >= expected_len) {
2798                 start = ldq_be_p(buf);
2799                 len = ldl_be_p(buf + 8);
2800                 /* Now we expect an idstr */
2801                 tmp32 = buf[12]; /* Length of the following idstr */
2802                 buf[13 + tmp32] = '\0';
2803                 expected_len += tmp32;
2804             }
2805             if (header_len != expected_len) {
2806                 error_report("RP: Req_Page_id with length %d expecting %zd",
2807                              header_len, expected_len);
2808                 mark_source_rp_bad(ms);
2809                 goto out;
2810             }
2811             migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len);
2812             break;
2813
2814         case MIG_RP_MSG_RECV_BITMAP:
2815             if (header_len < 1) {
2816                 error_report("%s: missing block name", __func__);
2817                 mark_source_rp_bad(ms);
2818                 goto out;
2819             }
2820             /* Format: len (1B) + idstr (<255B). This ends the idstr. */
2821             buf[buf[0] + 1] = '\0';
2822             if (migrate_handle_rp_recv_bitmap(ms, (char *)(buf + 1))) {
2823                 mark_source_rp_bad(ms);
2824                 goto out;
2825             }
2826             break;
2827
2828         case MIG_RP_MSG_RESUME_ACK:
2829             tmp32 = ldl_be_p(buf);
2830             if (migrate_handle_rp_resume_ack(ms, tmp32)) {
2831                 mark_source_rp_bad(ms);
2832                 goto out;
2833             }
2834             break;
2835
2836         default:
2837             break;
2838         }
2839     }
2840
2841 out:
2842     res = qemu_file_get_error(rp);
2843     if (res) {
2844         if (res == -EIO && migration_in_postcopy()) {
2845             /*
2846              * Maybe there is something we can do: it looks like a
2847              * network down issue, and we pause for a recovery.
2848              */
2849             migration_release_from_dst_file(ms);
2850             rp = NULL;
2851             if (postcopy_pause_return_path_thread(ms)) {
2852                 /*
2853                  * Reload rp, reset the rest.  Referencing it is safe since
2854                  * it's reset only by us above, or when migration completes
2855                  */
2856                 rp = ms->rp_state.from_dst_file;
2857                 ms->rp_state.error = false;
2858                 goto retry;
2859             }
2860         }
2861
2862         trace_source_return_path_thread_bad_end();
2863         mark_source_rp_bad(ms);
2864     }
2865
2866     trace_source_return_path_thread_end();
2867     migration_release_from_dst_file(ms);
2868     rcu_unregister_thread();
2869     return NULL;
2870 }
2871
2872 static int open_return_path_on_source(MigrationState *ms,
2873                                       bool create_thread)
2874 {
2875     ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file);
2876     if (!ms->rp_state.from_dst_file) {
2877         return -1;
2878     }
2879
2880     trace_open_return_path_on_source();
2881
2882     if (!create_thread) {
2883         /* We're done */
2884         return 0;
2885     }
2886
2887     qemu_thread_create(&ms->rp_state.rp_thread, "return path",
2888                        source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
2889     ms->rp_state.rp_thread_created = true;
2890
2891     trace_open_return_path_on_source_continue();
2892
2893     return 0;
2894 }
2895
2896 /* Returns 0 if the RP was ok, otherwise there was an error on the RP */
2897 static int await_return_path_close_on_source(MigrationState *ms)
2898 {
2899     /*
2900      * If this is a normal exit then the destination will send a SHUT and the
2901      * rp_thread will exit, however if there's an error we need to cause
2902      * it to exit.
2903      */
2904     if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) {
2905         /*
2906          * shutdown(2), if we have it, will cause it to unblock if it's stuck
2907          * waiting for the destination.
2908          */
2909         qemu_file_shutdown(ms->rp_state.from_dst_file);
2910         mark_source_rp_bad(ms);
2911     }
2912     trace_await_return_path_close_on_source_joining();
2913     qemu_thread_join(&ms->rp_state.rp_thread);
2914     ms->rp_state.rp_thread_created = false;
2915     trace_await_return_path_close_on_source_close();
2916     return ms->rp_state.error;
2917 }
2918
2919 /*
2920  * Switch from normal iteration to postcopy
2921  * Returns non-0 on error
2922  */
2923 static int postcopy_start(MigrationState *ms)
2924 {
2925     int ret;
2926     QIOChannelBuffer *bioc;
2927     QEMUFile *fb;
2928     int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2929     int64_t bandwidth = migrate_max_postcopy_bandwidth();
2930     bool restart_block = false;
2931     int cur_state = MIGRATION_STATUS_ACTIVE;
2932     if (!migrate_pause_before_switchover()) {
2933         migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
2934                           MIGRATION_STATUS_POSTCOPY_ACTIVE);
2935     }
2936
2937     trace_postcopy_start();
2938     qemu_mutex_lock_iothread();
2939     trace_postcopy_start_set_run();
2940
2941     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
2942     global_state_store();
2943     ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
2944     if (ret < 0) {
2945         goto fail;
2946     }
2947
2948     ret = migration_maybe_pause(ms, &cur_state,
2949                                 MIGRATION_STATUS_POSTCOPY_ACTIVE);
2950     if (ret < 0) {
2951         goto fail;
2952     }
2953
2954     ret = bdrv_inactivate_all();
2955     if (ret < 0) {
2956         goto fail;
2957     }
2958     restart_block = true;
2959
2960     /*
2961      * Cause any non-postcopiable, but iterative devices to
2962      * send out their final data.
2963      */
2964     qemu_savevm_state_complete_precopy(ms->to_dst_file, true, false);
2965
2966     /*
2967      * in Finish migrate and with the io-lock held everything should
2968      * be quiet, but we've potentially still got dirty pages and we
2969      * need to tell the destination to throw any pages it's already received
2970      * that are dirty
2971      */
2972     if (migrate_postcopy_ram()) {
2973         if (ram_postcopy_send_discard_bitmap(ms)) {
2974             error_report("postcopy send discard bitmap failed");
2975             goto fail;
2976         }
2977     }
2978
2979     /*
2980      * send rest of state - note things that are doing postcopy
2981      * will notice we're in POSTCOPY_ACTIVE and not actually
2982      * wrap their state up here
2983      */
2984     /* 0 max-postcopy-bandwidth means unlimited */
2985     if (!bandwidth) {
2986         qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX);
2987     } else {
2988         qemu_file_set_rate_limit(ms->to_dst_file, bandwidth / XFER_LIMIT_RATIO);
2989     }
2990     if (migrate_postcopy_ram()) {
2991         /* Ping just for debugging, helps line traces up */
2992         qemu_savevm_send_ping(ms->to_dst_file, 2);
2993     }
2994
2995     /*
2996      * While loading the device state we may trigger page transfer
2997      * requests and the fd must be free to process those, and thus
2998      * the destination must read the whole device state off the fd before
2999      * it starts processing it.  Unfortunately the ad-hoc migration format
3000      * doesn't allow the destination to know the size to read without fully
3001      * parsing it through each devices load-state code (especially the open
3002      * coded devices that use get/put).
3003      * So we wrap the device state up in a package with a length at the start;
3004      * to do this we use a qemu_buf to hold the whole of the device state.
3005      */
3006     bioc = qio_channel_buffer_new(4096);
3007     qio_channel_set_name(QIO_CHANNEL(bioc), "migration-postcopy-buffer");
3008     fb = qemu_fopen_channel_output(QIO_CHANNEL(bioc));
3009     object_unref(OBJECT(bioc));
3010
3011     /*
3012      * Make sure the receiver can get incoming pages before we send the rest
3013      * of the state
3014      */
3015     qemu_savevm_send_postcopy_listen(fb);
3016
3017     qemu_savevm_state_complete_precopy(fb, false, false);
3018     if (migrate_postcopy_ram()) {
3019         qemu_savevm_send_ping(fb, 3);
3020     }
3021
3022     qemu_savevm_send_postcopy_run(fb);
3023
3024     /* <><> end of stuff going into the package */
3025
3026     /* Last point of recovery; as soon as we send the package the destination
3027      * can open devices and potentially start running.
3028      * Lets just check again we've not got any errors.
3029      */
3030     ret = qemu_file_get_error(ms->to_dst_file);
3031     if (ret) {
3032         error_report("postcopy_start: Migration stream errored (pre package)");
3033         goto fail_closefb;
3034     }
3035
3036     restart_block = false;
3037
3038     /* Now send that blob */
3039     if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
3040         goto fail_closefb;
3041     }
3042     qemu_fclose(fb);
3043
3044     /* Send a notify to give a chance for anything that needs to happen
3045      * at the transition to postcopy and after the device state; in particular
3046      * spice needs to trigger a transition now
3047      */
3048     ms->postcopy_after_devices = true;
3049     notifier_list_notify(&migration_state_notifiers, ms);
3050
3051     ms->downtime =  qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop;
3052
3053     qemu_mutex_unlock_iothread();
3054
3055     if (migrate_postcopy_ram()) {
3056         /*
3057          * Although this ping is just for debug, it could potentially be
3058          * used for getting a better measurement of downtime at the source.
3059          */
3060         qemu_savevm_send_ping(ms->to_dst_file, 4);
3061     }
3062
3063     if (migrate_release_ram()) {
3064         ram_postcopy_migrated_memory_release(ms);
3065     }
3066
3067     ret = qemu_file_get_error(ms->to_dst_file);
3068     if (ret) {
3069         error_report("postcopy_start: Migration stream errored");
3070         migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3071                               MIGRATION_STATUS_FAILED);
3072     }
3073
3074     return ret;
3075
3076 fail_closefb:
3077     qemu_fclose(fb);
3078 fail:
3079     migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3080                           MIGRATION_STATUS_FAILED);
3081     if (restart_block) {
3082         /* A failure happened early enough that we know the destination hasn't
3083          * accessed block devices, so we're safe to recover.
3084          */
3085         Error *local_err = NULL;
3086
3087         bdrv_invalidate_cache_all(&local_err);
3088         if (local_err) {
3089             error_report_err(local_err);
3090         }
3091     }
3092     qemu_mutex_unlock_iothread();
3093     return -1;
3094 }
3095
3096 /**
3097  * migration_maybe_pause: Pause if required to by
3098  * migrate_pause_before_switchover called with the iothread locked
3099  * Returns: 0 on success
3100  */
3101 static int migration_maybe_pause(MigrationState *s,
3102                                  int *current_active_state,
3103                                  int new_state)
3104 {
3105     if (!migrate_pause_before_switchover()) {
3106         return 0;
3107     }
3108
3109     /* Since leaving this state is not atomic with posting the semaphore
3110      * it's possible that someone could have issued multiple migrate_continue
3111      * and the semaphore is incorrectly positive at this point;
3112      * the docs say it's undefined to reinit a semaphore that's already
3113      * init'd, so use timedwait to eat up any existing posts.
3114      */
3115     while (qemu_sem_timedwait(&s->pause_sem, 1) == 0) {
3116         /* This block intentionally left blank */
3117     }
3118
3119     /*
3120      * If the migration is cancelled when it is in the completion phase,
3121      * the migration state is set to MIGRATION_STATUS_CANCELLING.
3122      * So we don't need to wait a semaphore, otherwise we would always
3123      * wait for the 'pause_sem' semaphore.
3124      */
3125     if (s->state != MIGRATION_STATUS_CANCELLING) {
3126         qemu_mutex_unlock_iothread();
3127         migrate_set_state(&s->state, *current_active_state,
3128                           MIGRATION_STATUS_PRE_SWITCHOVER);
3129         qemu_sem_wait(&s->pause_sem);
3130         migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER,
3131                           new_state);
3132         *current_active_state = new_state;
3133         qemu_mutex_lock_iothread();
3134     }
3135
3136     return s->state == new_state ? 0 : -EINVAL;
3137 }
3138
3139 /**
3140  * migration_completion: Used by migration_thread when there's not much left.
3141  *   The caller 'breaks' the loop when this returns.
3142  *
3143  * @s: Current migration state
3144  */
3145 static void migration_completion(MigrationState *s)
3146 {
3147     int ret;
3148     int current_active_state = s->state;
3149
3150     if (s->state == MIGRATION_STATUS_ACTIVE) {
3151         qemu_mutex_lock_iothread();
3152         s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3153         qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3154         s->vm_was_running = runstate_is_running();
3155         ret = global_state_store();
3156
3157         if (!ret) {
3158             bool inactivate = !migrate_colo_enabled();
3159             ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
3160             trace_migration_completion_vm_stop(ret);
3161             if (ret >= 0) {
3162                 ret = migration_maybe_pause(s, &current_active_state,
3163                                             MIGRATION_STATUS_DEVICE);
3164             }
3165             if (ret >= 0) {
3166                 qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3167                 ret = qemu_savevm_state_complete_precopy(s->to_dst_file, false,
3168                                                          inactivate);
3169             }
3170             if (inactivate && ret >= 0) {
3171                 s->block_inactive = true;
3172             }
3173         }
3174         qemu_mutex_unlock_iothread();
3175
3176         if (ret < 0) {
3177             goto fail;
3178         }
3179     } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3180         trace_migration_completion_postcopy_end();
3181
3182         qemu_savevm_state_complete_postcopy(s->to_dst_file);
3183         trace_migration_completion_postcopy_end_after_complete();
3184     } else if (s->state == MIGRATION_STATUS_CANCELLING) {
3185         goto fail;
3186     }
3187
3188     /*
3189      * If rp was opened we must clean up the thread before
3190      * cleaning everything else up (since if there are no failures
3191      * it will wait for the destination to send it's status in
3192      * a SHUT command).
3193      */
3194     if (s->rp_state.rp_thread_created) {
3195         int rp_error;
3196         trace_migration_return_path_end_before();
3197         rp_error = await_return_path_close_on_source(s);
3198         trace_migration_return_path_end_after(rp_error);
3199         if (rp_error) {
3200             goto fail_invalidate;
3201         }
3202     }
3203
3204     if (qemu_file_get_error(s->to_dst_file)) {
3205         trace_migration_completion_file_err();
3206         goto fail_invalidate;
3207     }
3208
3209     if (!migrate_colo_enabled()) {
3210         migrate_set_state(&s->state, current_active_state,
3211                           MIGRATION_STATUS_COMPLETED);
3212     }
3213
3214     return;
3215
3216 fail_invalidate:
3217     /* If not doing postcopy, vm_start() will be called: let's regain
3218      * control on images.
3219      */
3220     if (s->state == MIGRATION_STATUS_ACTIVE ||
3221         s->state == MIGRATION_STATUS_DEVICE) {
3222         Error *local_err = NULL;
3223
3224         qemu_mutex_lock_iothread();
3225         bdrv_invalidate_cache_all(&local_err);
3226         if (local_err) {
3227             error_report_err(local_err);
3228         } else {
3229             s->block_inactive = false;
3230         }
3231         qemu_mutex_unlock_iothread();
3232     }
3233
3234 fail:
3235     migrate_set_state(&s->state, current_active_state,
3236                       MIGRATION_STATUS_FAILED);
3237 }
3238
3239 /**
3240  * bg_migration_completion: Used by bg_migration_thread when after all the
3241  *   RAM has been saved. The caller 'breaks' the loop when this returns.
3242  *
3243  * @s: Current migration state
3244  */
3245 static void bg_migration_completion(MigrationState *s)
3246 {
3247     int current_active_state = s->state;
3248
3249     /*
3250      * Stop tracking RAM writes - un-protect memory, un-register UFFD
3251      * memory ranges, flush kernel wait queues and wake up threads
3252      * waiting for write fault to be resolved.
3253      */
3254     ram_write_tracking_stop();
3255
3256     if (s->state == MIGRATION_STATUS_ACTIVE) {
3257         /*
3258          * By this moment we have RAM content saved into the migration stream.
3259          * The next step is to flush the non-RAM content (device state)
3260          * right after the ram content. The device state has been stored into
3261          * the temporary buffer before RAM saving started.
3262          */
3263         qemu_put_buffer(s->to_dst_file, s->bioc->data, s->bioc->usage);
3264         qemu_fflush(s->to_dst_file);
3265     } else if (s->state == MIGRATION_STATUS_CANCELLING) {
3266         goto fail;
3267     }
3268
3269     if (qemu_file_get_error(s->to_dst_file)) {
3270         trace_migration_completion_file_err();
3271         goto fail;
3272     }
3273
3274     migrate_set_state(&s->state, current_active_state,
3275                       MIGRATION_STATUS_COMPLETED);
3276     return;
3277
3278 fail:
3279     migrate_set_state(&s->state, current_active_state,
3280                       MIGRATION_STATUS_FAILED);
3281 }
3282
3283 bool migrate_colo_enabled(void)
3284 {
3285     MigrationState *s = migrate_get_current();
3286     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_COLO];
3287 }
3288
3289 typedef enum MigThrError {
3290     /* No error detected */
3291     MIG_THR_ERR_NONE = 0,
3292     /* Detected error, but resumed successfully */
3293     MIG_THR_ERR_RECOVERED = 1,
3294     /* Detected fatal error, need to exit */
3295     MIG_THR_ERR_FATAL = 2,
3296 } MigThrError;
3297
3298 static int postcopy_resume_handshake(MigrationState *s)
3299 {
3300     qemu_savevm_send_postcopy_resume(s->to_dst_file);
3301
3302     while (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3303         qemu_sem_wait(&s->rp_state.rp_sem);
3304     }
3305
3306     if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3307         return 0;
3308     }
3309
3310     return -1;
3311 }
3312
3313 /* Return zero if success, or <0 for error */
3314 static int postcopy_do_resume(MigrationState *s)
3315 {
3316     int ret;
3317
3318     /*
3319      * Call all the resume_prepare() hooks, so that modules can be
3320      * ready for the migration resume.
3321      */
3322     ret = qemu_savevm_state_resume_prepare(s);
3323     if (ret) {
3324         error_report("%s: resume_prepare() failure detected: %d",
3325                      __func__, ret);
3326         return ret;
3327     }
3328
3329     /*
3330      * Last handshake with destination on the resume (destination will
3331      * switch to postcopy-active afterwards)
3332      */
3333     ret = postcopy_resume_handshake(s);
3334     if (ret) {
3335         error_report("%s: handshake failed: %d", __func__, ret);
3336         return ret;
3337     }
3338
3339     return 0;
3340 }
3341
3342 /*
3343  * We don't return until we are in a safe state to continue current
3344  * postcopy migration.  Returns MIG_THR_ERR_RECOVERED if recovered, or
3345  * MIG_THR_ERR_FATAL if unrecovery failure happened.
3346  */
3347 static MigThrError postcopy_pause(MigrationState *s)
3348 {
3349     assert(s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
3350
3351     while (true) {
3352         QEMUFile *file;
3353
3354         /* Current channel is possibly broken. Release it. */
3355         assert(s->to_dst_file);
3356         qemu_mutex_lock(&s->qemu_file_lock);
3357         file = s->to_dst_file;
3358         s->to_dst_file = NULL;
3359         qemu_mutex_unlock(&s->qemu_file_lock);
3360
3361         qemu_file_shutdown(file);
3362         qemu_fclose(file);
3363
3364         migrate_set_state(&s->state, s->state,
3365                           MIGRATION_STATUS_POSTCOPY_PAUSED);
3366
3367         error_report("Detected IO failure for postcopy. "
3368                      "Migration paused.");
3369
3370         /*
3371          * We wait until things fixed up. Then someone will setup the
3372          * status back for us.
3373          */
3374         while (s->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
3375             qemu_sem_wait(&s->postcopy_pause_sem);
3376         }
3377
3378         if (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3379             /* Woken up by a recover procedure. Give it a shot */
3380
3381             /*
3382              * Firstly, let's wake up the return path now, with a new
3383              * return path channel.
3384              */
3385             qemu_sem_post(&s->postcopy_pause_rp_sem);
3386
3387             /* Do the resume logic */
3388             if (postcopy_do_resume(s) == 0) {
3389                 /* Let's continue! */
3390                 trace_postcopy_pause_continued();
3391                 return MIG_THR_ERR_RECOVERED;
3392             } else {
3393                 /*
3394                  * Something wrong happened during the recovery, let's
3395                  * pause again. Pause is always better than throwing
3396                  * data away.
3397                  */
3398                 continue;
3399             }
3400         } else {
3401             /* This is not right... Time to quit. */
3402             return MIG_THR_ERR_FATAL;
3403         }
3404     }
3405 }
3406
3407 static MigThrError migration_detect_error(MigrationState *s)
3408 {
3409     int ret;
3410     int state = s->state;
3411     Error *local_error = NULL;
3412
3413     if (state == MIGRATION_STATUS_CANCELLING ||
3414         state == MIGRATION_STATUS_CANCELLED) {
3415         /* End the migration, but don't set the state to failed */
3416         return MIG_THR_ERR_FATAL;
3417     }
3418
3419     /* Try to detect any file errors */
3420     ret = qemu_file_get_error_obj(s->to_dst_file, &local_error);
3421     if (!ret) {
3422         /* Everything is fine */
3423         assert(!local_error);
3424         return MIG_THR_ERR_NONE;
3425     }
3426
3427     if (local_error) {
3428         migrate_set_error(s, local_error);
3429         error_free(local_error);
3430     }
3431
3432     if (state == MIGRATION_STATUS_POSTCOPY_ACTIVE && ret == -EIO) {
3433         /*
3434          * For postcopy, we allow the network to be down for a
3435          * while. After that, it can be continued by a
3436          * recovery phase.
3437          */
3438         return postcopy_pause(s);
3439     } else {
3440         /*
3441          * For precopy (or postcopy with error outside IO), we fail
3442          * with no time.
3443          */
3444         migrate_set_state(&s->state, state, MIGRATION_STATUS_FAILED);
3445         trace_migration_thread_file_err();
3446
3447         /* Time to stop the migration, now. */
3448         return MIG_THR_ERR_FATAL;
3449     }
3450 }
3451
3452 /* How many bytes have we transferred since the beginning of the migration */
3453 static uint64_t migration_total_bytes(MigrationState *s)
3454 {
3455     return qemu_ftell(s->to_dst_file) + ram_counters.multifd_bytes;
3456 }
3457
3458 static void migration_calculate_complete(MigrationState *s)
3459 {
3460     uint64_t bytes = migration_total_bytes(s);
3461     int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3462     int64_t transfer_time;
3463
3464     s->total_time = end_time - s->start_time;
3465     if (!s->downtime) {
3466         /*
3467          * It's still not set, so we are precopy migration.  For
3468          * postcopy, downtime is calculated during postcopy_start().
3469          */
3470         s->downtime = end_time - s->downtime_start;
3471     }
3472
3473     transfer_time = s->total_time - s->setup_time;
3474     if (transfer_time) {
3475         s->mbps = ((double) bytes * 8.0) / transfer_time / 1000;
3476     }
3477 }
3478
3479 static void update_iteration_initial_status(MigrationState *s)
3480 {
3481     /*
3482      * Update these three fields at the same time to avoid mismatch info lead
3483      * wrong speed calculation.
3484      */
3485     s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3486     s->iteration_initial_bytes = migration_total_bytes(s);
3487     s->iteration_initial_pages = ram_get_total_transferred_pages();
3488 }
3489
3490 static void migration_update_counters(MigrationState *s,
3491                                       int64_t current_time)
3492 {
3493     uint64_t transferred, transferred_pages, time_spent;
3494     uint64_t current_bytes; /* bytes transferred since the beginning */
3495     double bandwidth;
3496
3497     if (current_time < s->iteration_start_time + BUFFER_DELAY) {
3498         return;
3499     }
3500
3501     current_bytes = migration_total_bytes(s);
3502     transferred = current_bytes - s->iteration_initial_bytes;
3503     time_spent = current_time - s->iteration_start_time;
3504     bandwidth = (double)transferred / time_spent;
3505     s->threshold_size = bandwidth * s->parameters.downtime_limit;
3506
3507     s->mbps = (((double) transferred * 8.0) /
3508                ((double) time_spent / 1000.0)) / 1000.0 / 1000.0;
3509
3510     transferred_pages = ram_get_total_transferred_pages() -
3511                             s->iteration_initial_pages;
3512     s->pages_per_second = (double) transferred_pages /
3513                              (((double) time_spent / 1000.0));
3514
3515     /*
3516      * if we haven't sent anything, we don't want to
3517      * recalculate. 10000 is a small enough number for our purposes
3518      */
3519     if (ram_counters.dirty_pages_rate && transferred > 10000) {
3520         s->expected_downtime = ram_counters.remaining / bandwidth;
3521     }
3522
3523     qemu_file_reset_rate_limit(s->to_dst_file);
3524
3525     update_iteration_initial_status(s);
3526
3527     trace_migrate_transferred(transferred, time_spent,
3528                               bandwidth, s->threshold_size);
3529 }
3530
3531 /* Migration thread iteration status */
3532 typedef enum {
3533     MIG_ITERATE_RESUME,         /* Resume current iteration */
3534     MIG_ITERATE_SKIP,           /* Skip current iteration */
3535     MIG_ITERATE_BREAK,          /* Break the loop */
3536 } MigIterateState;
3537
3538 /*
3539  * Return true if continue to the next iteration directly, false
3540  * otherwise.
3541  */
3542 static MigIterateState migration_iteration_run(MigrationState *s)
3543 {
3544     uint64_t pending_size, pend_pre, pend_compat, pend_post;
3545     bool in_postcopy = s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE;
3546
3547     qemu_savevm_state_pending(s->to_dst_file, s->threshold_size, &pend_pre,
3548                               &pend_compat, &pend_post);
3549     pending_size = pend_pre + pend_compat + pend_post;
3550
3551     trace_migrate_pending(pending_size, s->threshold_size,
3552                           pend_pre, pend_compat, pend_post);
3553
3554     if (pending_size && pending_size >= s->threshold_size) {
3555         /* Still a significant amount to transfer */
3556         if (!in_postcopy && pend_pre <= s->threshold_size &&
3557             qatomic_read(&s->start_postcopy)) {
3558             if (postcopy_start(s)) {
3559                 error_report("%s: postcopy failed to start", __func__);
3560             }
3561             return MIG_ITERATE_SKIP;
3562         }
3563         /* Just another iteration step */
3564         qemu_savevm_state_iterate(s->to_dst_file, in_postcopy);
3565     } else {
3566         trace_migration_thread_low_pending(pending_size);
3567         migration_completion(s);
3568         return MIG_ITERATE_BREAK;
3569     }
3570
3571     return MIG_ITERATE_RESUME;
3572 }
3573
3574 static void migration_iteration_finish(MigrationState *s)
3575 {
3576     /* If we enabled cpu throttling for auto-converge, turn it off. */
3577     cpu_throttle_stop();
3578
3579     qemu_mutex_lock_iothread();
3580     switch (s->state) {
3581     case MIGRATION_STATUS_COMPLETED:
3582         migration_calculate_complete(s);
3583         runstate_set(RUN_STATE_POSTMIGRATE);
3584         break;
3585
3586     case MIGRATION_STATUS_ACTIVE:
3587         /*
3588          * We should really assert here, but since it's during
3589          * migration, let's try to reduce the usage of assertions.
3590          */
3591         if (!migrate_colo_enabled()) {
3592             error_report("%s: critical error: calling COLO code without "
3593                          "COLO enabled", __func__);
3594         }
3595         migrate_start_colo_process(s);
3596         /*
3597          * Fixme: we will run VM in COLO no matter its old running state.
3598          * After exited COLO, we will keep running.
3599          */
3600         s->vm_was_running = true;
3601         /* Fallthrough */
3602     case MIGRATION_STATUS_FAILED:
3603     case MIGRATION_STATUS_CANCELLED:
3604     case MIGRATION_STATUS_CANCELLING:
3605         if (s->vm_was_running) {
3606             vm_start();
3607         } else {
3608             if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
3609                 runstate_set(RUN_STATE_POSTMIGRATE);
3610             }
3611         }
3612         break;
3613
3614     default:
3615         /* Should not reach here, but if so, forgive the VM. */
3616         error_report("%s: Unknown ending state %d", __func__, s->state);
3617         break;
3618     }
3619     migrate_fd_cleanup_schedule(s);
3620     qemu_mutex_unlock_iothread();
3621 }
3622
3623 static void bg_migration_iteration_finish(MigrationState *s)
3624 {
3625     qemu_mutex_lock_iothread();
3626     switch (s->state) {
3627     case MIGRATION_STATUS_COMPLETED:
3628         migration_calculate_complete(s);
3629         break;
3630
3631     case MIGRATION_STATUS_ACTIVE:
3632     case MIGRATION_STATUS_FAILED:
3633     case MIGRATION_STATUS_CANCELLED:
3634     case MIGRATION_STATUS_CANCELLING:
3635         break;
3636
3637     default:
3638         /* Should not reach here, but if so, forgive the VM. */
3639         error_report("%s: Unknown ending state %d", __func__, s->state);
3640         break;
3641     }
3642
3643     migrate_fd_cleanup_schedule(s);
3644     qemu_mutex_unlock_iothread();
3645 }
3646
3647 /*
3648  * Return true if continue to the next iteration directly, false
3649  * otherwise.
3650  */
3651 static MigIterateState bg_migration_iteration_run(MigrationState *s)
3652 {
3653     int res;
3654
3655     res = qemu_savevm_state_iterate(s->to_dst_file, false);
3656     if (res > 0) {
3657         bg_migration_completion(s);
3658         return MIG_ITERATE_BREAK;
3659     }
3660
3661     return MIG_ITERATE_RESUME;
3662 }
3663
3664 void migration_make_urgent_request(void)
3665 {
3666     qemu_sem_post(&migrate_get_current()->rate_limit_sem);
3667 }
3668
3669 void migration_consume_urgent_request(void)
3670 {
3671     qemu_sem_wait(&migrate_get_current()->rate_limit_sem);
3672 }
3673
3674 /* Returns true if the rate limiting was broken by an urgent request */
3675 bool migration_rate_limit(void)
3676 {
3677     int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3678     MigrationState *s = migrate_get_current();
3679
3680     bool urgent = false;
3681     migration_update_counters(s, now);
3682     if (qemu_file_rate_limit(s->to_dst_file)) {
3683
3684         if (qemu_file_get_error(s->to_dst_file)) {
3685             return false;
3686         }
3687         /*
3688          * Wait for a delay to do rate limiting OR
3689          * something urgent to post the semaphore.
3690          */
3691         int ms = s->iteration_start_time + BUFFER_DELAY - now;
3692         trace_migration_rate_limit_pre(ms);
3693         if (qemu_sem_timedwait(&s->rate_limit_sem, ms) == 0) {
3694             /*
3695              * We were woken by one or more urgent things but
3696              * the timedwait will have consumed one of them.
3697              * The service routine for the urgent wake will dec
3698              * the semaphore itself for each item it consumes,
3699              * so add this one we just eat back.
3700              */
3701             qemu_sem_post(&s->rate_limit_sem);
3702             urgent = true;
3703         }
3704         trace_migration_rate_limit_post(urgent);
3705     }
3706     return urgent;
3707 }
3708
3709 /*
3710  * if failover devices are present, wait they are completely
3711  * unplugged
3712  */
3713
3714 static void qemu_savevm_wait_unplug(MigrationState *s, int old_state,
3715                                     int new_state)
3716 {
3717     if (qemu_savevm_state_guest_unplug_pending()) {
3718         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_WAIT_UNPLUG);
3719
3720         while (s->state == MIGRATION_STATUS_WAIT_UNPLUG &&
3721                qemu_savevm_state_guest_unplug_pending()) {
3722             qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3723         }
3724         if (s->state != MIGRATION_STATUS_WAIT_UNPLUG) {
3725             int timeout = 120; /* 30 seconds */
3726             /*
3727              * migration has been canceled
3728              * but as we have started an unplug we must wait the end
3729              * to be able to plug back the card
3730              */
3731             while (timeout-- && qemu_savevm_state_guest_unplug_pending()) {
3732                 qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3733             }
3734             if (qemu_savevm_state_guest_unplug_pending()) {
3735                 warn_report("migration: partially unplugged device on "
3736                             "failure");
3737             }
3738         }
3739
3740         migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG, new_state);
3741     } else {
3742         migrate_set_state(&s->state, old_state, new_state);
3743     }
3744 }
3745
3746 /*
3747  * Master migration thread on the source VM.
3748  * It drives the migration and pumps the data down the outgoing channel.
3749  */
3750 static void *migration_thread(void *opaque)
3751 {
3752     MigrationState *s = opaque;
3753     int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3754     MigThrError thr_error;
3755     bool urgent = false;
3756
3757     rcu_register_thread();
3758
3759     object_ref(OBJECT(s));
3760     update_iteration_initial_status(s);
3761
3762     qemu_savevm_state_header(s->to_dst_file);
3763
3764     /*
3765      * If we opened the return path, we need to make sure dst has it
3766      * opened as well.
3767      */
3768     if (s->rp_state.rp_thread_created) {
3769         /* Now tell the dest that it should open its end so it can reply */
3770         qemu_savevm_send_open_return_path(s->to_dst_file);
3771
3772         /* And do a ping that will make stuff easier to debug */
3773         qemu_savevm_send_ping(s->to_dst_file, 1);
3774     }
3775
3776     if (migrate_postcopy()) {
3777         /*
3778          * Tell the destination that we *might* want to do postcopy later;
3779          * if the other end can't do postcopy it should fail now, nice and
3780          * early.
3781          */
3782         qemu_savevm_send_postcopy_advise(s->to_dst_file);
3783     }
3784
3785     if (migrate_colo_enabled()) {
3786         /* Notify migration destination that we enable COLO */
3787         qemu_savevm_send_colo_enable(s->to_dst_file);
3788     }
3789
3790     qemu_savevm_state_setup(s->to_dst_file);
3791
3792     qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
3793                                MIGRATION_STATUS_ACTIVE);
3794
3795     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3796
3797     trace_migration_thread_setup_complete();
3798
3799     while (migration_is_active(s)) {
3800         if (urgent || !qemu_file_rate_limit(s->to_dst_file)) {
3801             MigIterateState iter_state = migration_iteration_run(s);
3802             if (iter_state == MIG_ITERATE_SKIP) {
3803                 continue;
3804             } else if (iter_state == MIG_ITERATE_BREAK) {
3805                 break;
3806             }
3807         }
3808
3809         /*
3810          * Try to detect any kind of failures, and see whether we
3811          * should stop the migration now.
3812          */
3813         thr_error = migration_detect_error(s);
3814         if (thr_error == MIG_THR_ERR_FATAL) {
3815             /* Stop migration */
3816             break;
3817         } else if (thr_error == MIG_THR_ERR_RECOVERED) {
3818             /*
3819              * Just recovered from a e.g. network failure, reset all
3820              * the local variables. This is important to avoid
3821              * breaking transferred_bytes and bandwidth calculation
3822              */
3823             update_iteration_initial_status(s);
3824         }
3825
3826         urgent = migration_rate_limit();
3827     }
3828
3829     trace_migration_thread_after_loop();
3830     migration_iteration_finish(s);
3831     object_unref(OBJECT(s));
3832     rcu_unregister_thread();
3833     return NULL;
3834 }
3835
3836 static void bg_migration_vm_start_bh(void *opaque)
3837 {
3838     MigrationState *s = opaque;
3839
3840     qemu_bh_delete(s->vm_start_bh);
3841     s->vm_start_bh = NULL;
3842
3843     vm_start();
3844     s->downtime = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - s->downtime_start;
3845 }
3846
3847 /**
3848  * Background snapshot thread, based on live migration code.
3849  * This is an alternative implementation of live migration mechanism
3850  * introduced specifically to support background snapshots.
3851  *
3852  * It takes advantage of userfault_fd write protection mechanism introduced
3853  * in v5.7 kernel. Compared to existing dirty page logging migration much
3854  * lesser stream traffic is produced resulting in smaller snapshot images,
3855  * simply cause of no page duplicates can get into the stream.
3856  *
3857  * Another key point is that generated vmstate stream reflects machine state
3858  * 'frozen' at the beginning of snapshot creation compared to dirty page logging
3859  * mechanism, which effectively results in that saved snapshot is the state of VM
3860  * at the end of the process.
3861  */
3862 static void *bg_migration_thread(void *opaque)
3863 {
3864     MigrationState *s = opaque;
3865     int64_t setup_start;
3866     MigThrError thr_error;
3867     QEMUFile *fb;
3868     bool early_fail = true;
3869
3870     rcu_register_thread();
3871     object_ref(OBJECT(s));
3872
3873     qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3874
3875     setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3876     /*
3877      * We want to save vmstate for the moment when migration has been
3878      * initiated but also we want to save RAM content while VM is running.
3879      * The RAM content should appear first in the vmstate. So, we first
3880      * stash the non-RAM part of the vmstate to the temporary buffer,
3881      * then write RAM part of the vmstate to the migration stream
3882      * with vCPUs running and, finally, write stashed non-RAM part of
3883      * the vmstate from the buffer to the migration stream.
3884      */
3885     s->bioc = qio_channel_buffer_new(512 * 1024);
3886     qio_channel_set_name(QIO_CHANNEL(s->bioc), "vmstate-buffer");
3887     fb = qemu_fopen_channel_output(QIO_CHANNEL(s->bioc));
3888     object_unref(OBJECT(s->bioc));
3889
3890     update_iteration_initial_status(s);
3891
3892     /*
3893      * Prepare for tracking memory writes with UFFD-WP - populate
3894      * RAM pages before protecting.
3895      */
3896 #ifdef __linux__
3897     ram_write_tracking_prepare();
3898 #endif
3899
3900     qemu_savevm_state_header(s->to_dst_file);
3901     qemu_savevm_state_setup(s->to_dst_file);
3902
3903     qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
3904                                MIGRATION_STATUS_ACTIVE);
3905
3906     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3907
3908     trace_migration_thread_setup_complete();
3909     s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3910
3911     qemu_mutex_lock_iothread();
3912
3913     /*
3914      * If VM is currently in suspended state, then, to make a valid runstate
3915      * transition in vm_stop_force_state() we need to wakeup it up.
3916      */
3917     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3918     s->vm_was_running = runstate_is_running();
3919
3920     if (global_state_store()) {
3921         goto fail;
3922     }
3923     /* Forcibly stop VM before saving state of vCPUs and devices */
3924     if (vm_stop_force_state(RUN_STATE_PAUSED)) {
3925         goto fail;
3926     }
3927     /*
3928      * Put vCPUs in sync with shadow context structures, then
3929      * save their state to channel-buffer along with devices.
3930      */
3931     cpu_synchronize_all_states();
3932     if (qemu_savevm_state_complete_precopy_non_iterable(fb, false, false)) {
3933         goto fail;
3934     }
3935     /*
3936      * Since we are going to get non-iterable state data directly
3937      * from s->bioc->data, explicit flush is needed here.
3938      */
3939     qemu_fflush(fb);
3940
3941     /* Now initialize UFFD context and start tracking RAM writes */
3942     if (ram_write_tracking_start()) {
3943         goto fail;
3944     }
3945     early_fail = false;
3946
3947     /*
3948      * Start VM from BH handler to avoid write-fault lock here.
3949      * UFFD-WP protection for the whole RAM is already enabled so
3950      * calling VM state change notifiers from vm_start() would initiate
3951      * writes to virtio VQs memory which is in write-protected region.
3952      */
3953     s->vm_start_bh = qemu_bh_new(bg_migration_vm_start_bh, s);
3954     qemu_bh_schedule(s->vm_start_bh);
3955
3956     qemu_mutex_unlock_iothread();
3957
3958     while (migration_is_active(s)) {
3959         MigIterateState iter_state = bg_migration_iteration_run(s);
3960         if (iter_state == MIG_ITERATE_SKIP) {
3961             continue;
3962         } else if (iter_state == MIG_ITERATE_BREAK) {
3963             break;
3964         }
3965
3966         /*
3967          * Try to detect any kind of failures, and see whether we
3968          * should stop the migration now.
3969          */
3970         thr_error = migration_detect_error(s);
3971         if (thr_error == MIG_THR_ERR_FATAL) {
3972             /* Stop migration */
3973             break;
3974         }
3975
3976         migration_update_counters(s, qemu_clock_get_ms(QEMU_CLOCK_REALTIME));
3977     }
3978
3979     trace_migration_thread_after_loop();
3980
3981 fail:
3982     if (early_fail) {
3983         migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE,
3984                 MIGRATION_STATUS_FAILED);
3985         qemu_mutex_unlock_iothread();
3986     }
3987
3988     bg_migration_iteration_finish(s);
3989
3990     qemu_fclose(fb);
3991     object_unref(OBJECT(s));
3992     rcu_unregister_thread();
3993
3994     return NULL;
3995 }
3996
3997 void migrate_fd_connect(MigrationState *s, Error *error_in)
3998 {
3999     Error *local_err = NULL;
4000     int64_t rate_limit;
4001     bool resume = s->state == MIGRATION_STATUS_POSTCOPY_PAUSED;
4002
4003     /*
4004      * If there's a previous error, free it and prepare for another one.
4005      * Meanwhile if migration completes successfully, there won't have an error
4006      * dumped when calling migrate_fd_cleanup().
4007      */
4008     migrate_error_free(s);
4009
4010     s->expected_downtime = s->parameters.downtime_limit;
4011     if (resume) {
4012         assert(s->cleanup_bh);
4013     } else {
4014         assert(!s->cleanup_bh);
4015         s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup_bh, s);
4016     }
4017     if (error_in) {
4018         migrate_fd_error(s, error_in);
4019         if (resume) {
4020             /*
4021              * Don't do cleanup for resume if channel is invalid, but only dump
4022              * the error.  We wait for another channel connect from the user.
4023              * The error_report still gives HMP user a hint on what failed.
4024              * It's normally done in migrate_fd_cleanup(), but call it here
4025              * explicitly.
4026              */
4027             error_report_err(error_copy(s->error));
4028         } else {
4029             migrate_fd_cleanup(s);
4030         }
4031         return;
4032     }
4033
4034     if (resume) {
4035         /* This is a resumed migration */
4036         rate_limit = s->parameters.max_postcopy_bandwidth /
4037             XFER_LIMIT_RATIO;
4038     } else {
4039         /* This is a fresh new migration */
4040         rate_limit = s->parameters.max_bandwidth / XFER_LIMIT_RATIO;
4041
4042         /* Notify before starting migration thread */
4043         notifier_list_notify(&migration_state_notifiers, s);
4044     }
4045
4046     qemu_file_set_rate_limit(s->to_dst_file, rate_limit);
4047     qemu_file_set_blocking(s->to_dst_file, true);
4048
4049     /*
4050      * Open the return path. For postcopy, it is used exclusively. For
4051      * precopy, only if user specified "return-path" capability would
4052      * QEMU uses the return path.
4053      */
4054     if (migrate_postcopy_ram() || migrate_use_return_path()) {
4055         if (open_return_path_on_source(s, !resume)) {
4056             error_report("Unable to open return-path for postcopy");
4057             migrate_set_state(&s->state, s->state, MIGRATION_STATUS_FAILED);
4058             migrate_fd_cleanup(s);
4059             return;
4060         }
4061     }
4062
4063     if (resume) {
4064         /* Wakeup the main migration thread to do the recovery */
4065         migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
4066                           MIGRATION_STATUS_POSTCOPY_RECOVER);
4067         qemu_sem_post(&s->postcopy_pause_sem);
4068         return;
4069     }
4070
4071     if (multifd_save_setup(&local_err) != 0) {
4072         error_report_err(local_err);
4073         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
4074                           MIGRATION_STATUS_FAILED);
4075         migrate_fd_cleanup(s);
4076         return;
4077     }
4078
4079     if (migrate_background_snapshot()) {
4080         qemu_thread_create(&s->thread, "bg_snapshot",
4081                 bg_migration_thread, s, QEMU_THREAD_JOINABLE);
4082     } else {
4083         qemu_thread_create(&s->thread, "live_migration",
4084                 migration_thread, s, QEMU_THREAD_JOINABLE);
4085     }
4086     s->migration_thread_running = true;
4087 }
4088
4089 void migration_global_dump(Monitor *mon)
4090 {
4091     MigrationState *ms = migrate_get_current();
4092
4093     monitor_printf(mon, "globals:\n");
4094     monitor_printf(mon, "store-global-state: %s\n",
4095                    ms->store_global_state ? "on" : "off");
4096     monitor_printf(mon, "only-migratable: %s\n",
4097                    only_migratable ? "on" : "off");
4098     monitor_printf(mon, "send-configuration: %s\n",
4099                    ms->send_configuration ? "on" : "off");
4100     monitor_printf(mon, "send-section-footer: %s\n",
4101                    ms->send_section_footer ? "on" : "off");
4102     monitor_printf(mon, "decompress-error-check: %s\n",
4103                    ms->decompress_error_check ? "on" : "off");
4104     monitor_printf(mon, "clear-bitmap-shift: %u\n",
4105                    ms->clear_bitmap_shift);
4106 }
4107
4108 #define DEFINE_PROP_MIG_CAP(name, x)             \
4109     DEFINE_PROP_BOOL(name, MigrationState, enabled_capabilities[x], false)
4110
4111 static Property migration_properties[] = {
4112     DEFINE_PROP_BOOL("store-global-state", MigrationState,
4113                      store_global_state, true),
4114     DEFINE_PROP_BOOL("send-configuration", MigrationState,
4115                      send_configuration, true),
4116     DEFINE_PROP_BOOL("send-section-footer", MigrationState,
4117                      send_section_footer, true),
4118     DEFINE_PROP_BOOL("decompress-error-check", MigrationState,
4119                       decompress_error_check, true),
4120     DEFINE_PROP_UINT8("x-clear-bitmap-shift", MigrationState,
4121                       clear_bitmap_shift, CLEAR_BITMAP_SHIFT_DEFAULT),
4122
4123     /* Migration parameters */
4124     DEFINE_PROP_UINT8("x-compress-level", MigrationState,
4125                       parameters.compress_level,
4126                       DEFAULT_MIGRATE_COMPRESS_LEVEL),
4127     DEFINE_PROP_UINT8("x-compress-threads", MigrationState,
4128                       parameters.compress_threads,
4129                       DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT),
4130     DEFINE_PROP_BOOL("x-compress-wait-thread", MigrationState,
4131                       parameters.compress_wait_thread, true),
4132     DEFINE_PROP_UINT8("x-decompress-threads", MigrationState,
4133                       parameters.decompress_threads,
4134                       DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT),
4135     DEFINE_PROP_UINT8("x-throttle-trigger-threshold", MigrationState,
4136                       parameters.throttle_trigger_threshold,
4137                       DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD),
4138     DEFINE_PROP_UINT8("x-cpu-throttle-initial", MigrationState,
4139                       parameters.cpu_throttle_initial,
4140                       DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL),
4141     DEFINE_PROP_UINT8("x-cpu-throttle-increment", MigrationState,
4142                       parameters.cpu_throttle_increment,
4143                       DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT),
4144     DEFINE_PROP_BOOL("x-cpu-throttle-tailslow", MigrationState,
4145                       parameters.cpu_throttle_tailslow, false),
4146     DEFINE_PROP_SIZE("x-max-bandwidth", MigrationState,
4147                       parameters.max_bandwidth, MAX_THROTTLE),
4148     DEFINE_PROP_UINT64("x-downtime-limit", MigrationState,
4149                       parameters.downtime_limit,
4150                       DEFAULT_MIGRATE_SET_DOWNTIME),
4151     DEFINE_PROP_UINT32("x-checkpoint-delay", MigrationState,
4152                       parameters.x_checkpoint_delay,
4153                       DEFAULT_MIGRATE_X_CHECKPOINT_DELAY),
4154     DEFINE_PROP_UINT8("multifd-channels", MigrationState,
4155                       parameters.multifd_channels,
4156                       DEFAULT_MIGRATE_MULTIFD_CHANNELS),
4157     DEFINE_PROP_MULTIFD_COMPRESSION("multifd-compression", MigrationState,
4158                       parameters.multifd_compression,
4159                       DEFAULT_MIGRATE_MULTIFD_COMPRESSION),
4160     DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState,
4161                       parameters.multifd_zlib_level,
4162                       DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL),
4163     DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState,
4164                       parameters.multifd_zstd_level,
4165                       DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL),
4166     DEFINE_PROP_SIZE("xbzrle-cache-size", MigrationState,
4167                       parameters.xbzrle_cache_size,
4168                       DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE),
4169     DEFINE_PROP_SIZE("max-postcopy-bandwidth", MigrationState,
4170                       parameters.max_postcopy_bandwidth,
4171                       DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH),
4172     DEFINE_PROP_UINT8("max-cpu-throttle", MigrationState,
4173                       parameters.max_cpu_throttle,
4174                       DEFAULT_MIGRATE_MAX_CPU_THROTTLE),
4175     DEFINE_PROP_SIZE("announce-initial", MigrationState,
4176                       parameters.announce_initial,
4177                       DEFAULT_MIGRATE_ANNOUNCE_INITIAL),
4178     DEFINE_PROP_SIZE("announce-max", MigrationState,
4179                       parameters.announce_max,
4180                       DEFAULT_MIGRATE_ANNOUNCE_MAX),
4181     DEFINE_PROP_SIZE("announce-rounds", MigrationState,
4182                       parameters.announce_rounds,
4183                       DEFAULT_MIGRATE_ANNOUNCE_ROUNDS),
4184     DEFINE_PROP_SIZE("announce-step", MigrationState,
4185                       parameters.announce_step,
4186                       DEFAULT_MIGRATE_ANNOUNCE_STEP),
4187
4188     /* Migration capabilities */
4189     DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
4190     DEFINE_PROP_MIG_CAP("x-rdma-pin-all", MIGRATION_CAPABILITY_RDMA_PIN_ALL),
4191     DEFINE_PROP_MIG_CAP("x-auto-converge", MIGRATION_CAPABILITY_AUTO_CONVERGE),
4192     DEFINE_PROP_MIG_CAP("x-zero-blocks", MIGRATION_CAPABILITY_ZERO_BLOCKS),
4193     DEFINE_PROP_MIG_CAP("x-compress", MIGRATION_CAPABILITY_COMPRESS),
4194     DEFINE_PROP_MIG_CAP("x-events", MIGRATION_CAPABILITY_EVENTS),
4195     DEFINE_PROP_MIG_CAP("x-postcopy-ram", MIGRATION_CAPABILITY_POSTCOPY_RAM),
4196     DEFINE_PROP_MIG_CAP("x-colo", MIGRATION_CAPABILITY_X_COLO),
4197     DEFINE_PROP_MIG_CAP("x-release-ram", MIGRATION_CAPABILITY_RELEASE_RAM),
4198     DEFINE_PROP_MIG_CAP("x-block", MIGRATION_CAPABILITY_BLOCK),
4199     DEFINE_PROP_MIG_CAP("x-return-path", MIGRATION_CAPABILITY_RETURN_PATH),
4200     DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD),
4201     DEFINE_PROP_MIG_CAP("x-background-snapshot",
4202             MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT),
4203
4204     DEFINE_PROP_END_OF_LIST(),
4205 };
4206
4207 static void migration_class_init(ObjectClass *klass, void *data)
4208 {
4209     DeviceClass *dc = DEVICE_CLASS(klass);
4210
4211     dc->user_creatable = false;
4212     device_class_set_props(dc, migration_properties);
4213 }
4214
4215 static void migration_instance_finalize(Object *obj)
4216 {
4217     MigrationState *ms = MIGRATION_OBJ(obj);
4218     MigrationParameters *params = &ms->parameters;
4219
4220     qemu_mutex_destroy(&ms->error_mutex);
4221     qemu_mutex_destroy(&ms->qemu_file_lock);
4222     g_free(params->tls_hostname);
4223     g_free(params->tls_creds);
4224     qemu_sem_destroy(&ms->wait_unplug_sem);
4225     qemu_sem_destroy(&ms->rate_limit_sem);
4226     qemu_sem_destroy(&ms->pause_sem);
4227     qemu_sem_destroy(&ms->postcopy_pause_sem);
4228     qemu_sem_destroy(&ms->postcopy_pause_rp_sem);
4229     qemu_sem_destroy(&ms->rp_state.rp_sem);
4230     error_free(ms->error);
4231 }
4232
4233 static void migration_instance_init(Object *obj)
4234 {
4235     MigrationState *ms = MIGRATION_OBJ(obj);
4236     MigrationParameters *params = &ms->parameters;
4237
4238     ms->state = MIGRATION_STATUS_NONE;
4239     ms->mbps = -1;
4240     ms->pages_per_second = -1;
4241     qemu_sem_init(&ms->pause_sem, 0);
4242     qemu_mutex_init(&ms->error_mutex);
4243
4244     params->tls_hostname = g_strdup("");
4245     params->tls_creds = g_strdup("");
4246
4247     /* Set has_* up only for parameter checks */
4248     params->has_compress_level = true;
4249     params->has_compress_threads = true;
4250     params->has_decompress_threads = true;
4251     params->has_throttle_trigger_threshold = true;
4252     params->has_cpu_throttle_initial = true;
4253     params->has_cpu_throttle_increment = true;
4254     params->has_cpu_throttle_tailslow = true;
4255     params->has_max_bandwidth = true;
4256     params->has_downtime_limit = true;
4257     params->has_x_checkpoint_delay = true;
4258     params->has_block_incremental = true;
4259     params->has_multifd_channels = true;
4260     params->has_multifd_compression = true;
4261     params->has_multifd_zlib_level = true;
4262     params->has_multifd_zstd_level = true;
4263     params->has_xbzrle_cache_size = true;
4264     params->has_max_postcopy_bandwidth = true;
4265     params->has_max_cpu_throttle = true;
4266     params->has_announce_initial = true;
4267     params->has_announce_max = true;
4268     params->has_announce_rounds = true;
4269     params->has_announce_step = true;
4270
4271     qemu_sem_init(&ms->postcopy_pause_sem, 0);
4272     qemu_sem_init(&ms->postcopy_pause_rp_sem, 0);
4273     qemu_sem_init(&ms->rp_state.rp_sem, 0);
4274     qemu_sem_init(&ms->rate_limit_sem, 0);
4275     qemu_sem_init(&ms->wait_unplug_sem, 0);
4276     qemu_mutex_init(&ms->qemu_file_lock);
4277 }
4278
4279 /*
4280  * Return true if check pass, false otherwise. Error will be put
4281  * inside errp if provided.
4282  */
4283 static bool migration_object_check(MigrationState *ms, Error **errp)
4284 {
4285     MigrationCapabilityStatusList *head = NULL;
4286     /* Assuming all off */
4287     bool cap_list[MIGRATION_CAPABILITY__MAX] = { 0 }, ret;
4288     int i;
4289
4290     if (!migrate_params_check(&ms->parameters, errp)) {
4291         return false;
4292     }
4293
4294     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
4295         if (ms->enabled_capabilities[i]) {
4296             QAPI_LIST_PREPEND(head, migrate_cap_add(i, true));
4297         }
4298     }
4299
4300     ret = migrate_caps_check(cap_list, head, errp);
4301
4302     /* It works with head == NULL */
4303     qapi_free_MigrationCapabilityStatusList(head);
4304
4305     return ret;
4306 }
4307
4308 static const TypeInfo migration_type = {
4309     .name = TYPE_MIGRATION,
4310     /*
4311      * NOTE: TYPE_MIGRATION is not really a device, as the object is
4312      * not created using qdev_new(), it is not attached to the qdev
4313      * device tree, and it is never realized.
4314      *
4315      * TODO: Make this TYPE_OBJECT once QOM provides something like
4316      * TYPE_DEVICE's "-global" properties.
4317      */
4318     .parent = TYPE_DEVICE,
4319     .class_init = migration_class_init,
4320     .class_size = sizeof(MigrationClass),
4321     .instance_size = sizeof(MigrationState),
4322     .instance_init = migration_instance_init,
4323     .instance_finalize = migration_instance_finalize,
4324 };
4325
4326 static void register_migration_types(void)
4327 {
4328     type_register_static(&migration_type);
4329 }
4330
4331 type_init(register_migration_types);