migration/migration.c

   1 /*
   2  * QEMU live migration
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *
   6  * Authors:
   7  *  Anthony Liguori   <aliguori@us.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include "qemu/cutils.h"
  18 #include "qemu/error-report.h"
  19 #include "qemu/main-loop.h"
  20 #include "migration/blocker.h"
  21 #include "exec.h"
  22 #include "fd.h"
  23 #include "socket.h"
  24 #include "sysemu/runstate.h"
  25 #include "sysemu/sysemu.h"
  26 #include "sysemu/cpu-throttle.h"
  27 #include "rdma.h"
  28 #include "ram.h"
  29 #include "migration/global_state.h"
  30 #include "migration/misc.h"
  31 #include "migration.h"
  32 #include "savevm.h"
  33 #include "qemu-file-channel.h"
  34 #include "qemu-file.h"
  35 #include "migration/vmstate.h"
  36 #include "block/block.h"
  37 #include "qapi/error.h"
  38 #include "qapi/clone-visitor.h"
  39 #include "qapi/qapi-visit-migration.h"
  40 #include "qapi/qapi-visit-sockets.h"
  41 #include "qapi/qapi-commands-migration.h"
  42 #include "qapi/qapi-events-migration.h"
  43 #include "qapi/qmp/qerror.h"
  44 #include "qapi/qmp/qnull.h"
  45 #include "qemu/rcu.h"
  46 #include "block.h"
  47 #include "postcopy-ram.h"
  48 #include "qemu/thread.h"
  49 #include "trace.h"
  50 #include "exec/target_page.h"
  51 #include "io/channel-buffer.h"
  52 #include "migration/colo.h"
  53 #include "hw/boards.h"
  54 #include "hw/qdev-properties.h"
  55 #include "hw/qdev-properties-system.h"
  56 #include "monitor/monitor.h"
  57 #include "net/announce.h"
  58 #include "qemu/queue.h"
  59 #include "multifd.h"
  60
  61 #ifdef CONFIG_VFIO
  62 #include "hw/vfio/vfio-common.h"
  63 #endif
  64
  65 #define MAX_THROTTLE  (128 << 20)      /* Migration transfer speed throttling */
  66
  67 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
  68  * data. */
  69 #define BUFFER_DELAY     100
  70 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
  71
  72 /* Time in milliseconds we are allowed to stop the source,
  73  * for sending the last part */
  74 #define DEFAULT_MIGRATE_SET_DOWNTIME 300
  75
  76 /* Maximum migrate downtime set to 2000 seconds */
  77 #define MAX_MIGRATE_DOWNTIME_SECONDS 2000
  78 #define MAX_MIGRATE_DOWNTIME (MAX_MIGRATE_DOWNTIME_SECONDS * 1000)
  79
  80 /* Default compression thread count */
  81 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8
  82 /* Default decompression thread count, usually decompression is at
  83  * least 4 times as fast as compression.*/
  84 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2
  85 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */
  86 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1
  87 /* Define default autoconverge cpu throttle migration parameters */
  88 #define DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD 50
  89 #define DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL 20
  90 #define DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT 10
  91 #define DEFAULT_MIGRATE_MAX_CPU_THROTTLE 99
  92
  93 /* Migration XBZRLE default cache size */
  94 #define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024)
  95
  96 /* The delay time (in ms) between two COLO checkpoints */
  97 #define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100)
  98 #define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2
  99 #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE
 100 /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */
 101 #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1
 102 /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */
 103 #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1
 104
 105 /* Background transfer rate for postcopy, 0 means unlimited, note
 106  * that page requests can still exceed this limit.
 107  */
 108 #define DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH 0
 109
 110 /*
 111  * Parameters for self_announce_delay giving a stream of RARP/ARP
 112  * packets after migration.
 113  */
 114 #define DEFAULT_MIGRATE_ANNOUNCE_INITIAL  50
 115 #define DEFAULT_MIGRATE_ANNOUNCE_MAX     550
 116 #define DEFAULT_MIGRATE_ANNOUNCE_ROUNDS    5
 117 #define DEFAULT_MIGRATE_ANNOUNCE_STEP    100
 118
 119 static NotifierList migration_state_notifiers =
 120     NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
 121
 122 /* Messages sent on the return path from destination to source */
 123 enum mig_rp_message_type {
 124     MIG_RP_MSG_INVALID = 0,  /* Must be 0 */
 125     MIG_RP_MSG_SHUT,         /* sibling will not send any more RP messages */
 126     MIG_RP_MSG_PONG,         /* Response to a PING; data (seq: be32 ) */
 127
 128     MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
 129     MIG_RP_MSG_REQ_PAGES,    /* data (start: be64, len: be32) */
 130     MIG_RP_MSG_RECV_BITMAP,  /* send recved_bitmap back to source */
 131     MIG_RP_MSG_RESUME_ACK,   /* tell source that we are ready to resume */
 132
 133     MIG_RP_MSG_MAX
 134 };
 135
 136 /* When we add fault tolerance, we could have several
 137    migrations at once.  For now we don't need to add
 138    dynamic creation of migration */
 139
 140 static MigrationState *current_migration;
 141 static MigrationIncomingState *current_incoming;
 142
 143 static bool migration_object_check(MigrationState *ms, Error **errp);
 144 static int migration_maybe_pause(MigrationState *s,
 145                                  int *current_active_state,
 146                                  int new_state);
 147 static void migrate_fd_cancel(MigrationState *s);
 148
 149 static gint page_request_addr_cmp(gconstpointer ap, gconstpointer bp)
 150 {
 151     uintptr_t a = (uintptr_t) ap, b = (uintptr_t) bp;
 152
 153     return (a > b) - (a < b);
 154 }
 155
 156 void migration_object_init(void)
 157 {
 158     Error *err = NULL;
 159
 160     /* This can only be called once. */
 161     assert(!current_migration);
 162     current_migration = MIGRATION_OBJ(object_new(TYPE_MIGRATION));
 163
 164     /*
 165      * Init the migrate incoming object as well no matter whether
 166      * we'll use it or not.
 167      */
 168     assert(!current_incoming);
 169     current_incoming = g_new0(MigrationIncomingState, 1);
 170     current_incoming->state = MIGRATION_STATUS_NONE;
 171     current_incoming->postcopy_remote_fds =
 172         g_array_new(FALSE, TRUE, sizeof(struct PostCopyFD));
 173     qemu_mutex_init(&current_incoming->rp_mutex);
 174     qemu_event_init(&current_incoming->main_thread_load_event, false);
 175     qemu_sem_init(&current_incoming->postcopy_pause_sem_dst, 0);
 176     qemu_sem_init(&current_incoming->postcopy_pause_sem_fault, 0);
 177     qemu_mutex_init(&current_incoming->page_request_mutex);
 178     current_incoming->page_requested = g_tree_new(page_request_addr_cmp);
 179
 180     if (!migration_object_check(current_migration, &err)) {
 181         error_report_err(err);
 182         exit(1);
 183     }
 184
 185     blk_mig_init();
 186     ram_mig_init();
 187     dirty_bitmap_mig_init();
 188 }
 189
 190 void migration_shutdown(void)
 191 {
 192     /*
 193      * Cancel the current migration - that will (eventually)
 194      * stop the migration using this structure
 195      */
 196     migrate_fd_cancel(current_migration);
 197     object_unref(OBJECT(current_migration));
 198
 199     /*
 200      * Cancel outgoing migration of dirty bitmaps. It should
 201      * at least unref used block nodes.
 202      */
 203     dirty_bitmap_mig_cancel_outgoing();
 204
 205     /*
 206      * Cancel incoming migration of dirty bitmaps. Dirty bitmaps
 207      * are non-critical data, and their loss never considered as
 208      * something serious.
 209      */
 210     dirty_bitmap_mig_cancel_incoming();
 211 }
 212
 213 /* For outgoing */
 214 MigrationState *migrate_get_current(void)
 215 {
 216     /* This can only be called after the object created. */
 217     assert(current_migration);
 218     return current_migration;
 219 }
 220
 221 MigrationIncomingState *migration_incoming_get_current(void)
 222 {
 223     assert(current_incoming);
 224     return current_incoming;
 225 }
 226
 227 void migration_incoming_state_destroy(void)
 228 {
 229     struct MigrationIncomingState *mis = migration_incoming_get_current();
 230
 231     if (mis->to_src_file) {
 232         /* Tell source that we are done */
 233         migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
 234         qemu_fclose(mis->to_src_file);
 235         mis->to_src_file = NULL;
 236     }
 237
 238     if (mis->from_src_file) {
 239         qemu_fclose(mis->from_src_file);
 240         mis->from_src_file = NULL;
 241     }
 242     if (mis->postcopy_remote_fds) {
 243         g_array_free(mis->postcopy_remote_fds, TRUE);
 244         mis->postcopy_remote_fds = NULL;
 245     }
 246
 247     qemu_event_reset(&mis->main_thread_load_event);
 248
 249     if (mis->page_requested) {
 250         g_tree_destroy(mis->page_requested);
 251         mis->page_requested = NULL;
 252     }
 253
 254     if (mis->socket_address_list) {
 255         qapi_free_SocketAddressList(mis->socket_address_list);
 256         mis->socket_address_list = NULL;
 257     }
 258 }
 259
 260 static void migrate_generate_event(int new_state)
 261 {
 262     if (migrate_use_events()) {
 263         qapi_event_send_migration(new_state);
 264     }
 265 }
 266
 267 static bool migrate_late_block_activate(void)
 268 {
 269     MigrationState *s;
 270
 271     s = migrate_get_current();
 272
 273     return s->enabled_capabilities[
 274         MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE];
 275 }
 276
 277 /*
 278  * Send a message on the return channel back to the source
 279  * of the migration.
 280  */
 281 static int migrate_send_rp_message(MigrationIncomingState *mis,
 282                                    enum mig_rp_message_type message_type,
 283                                    uint16_t len, void *data)
 284 {
 285     int ret = 0;
 286
 287     trace_migrate_send_rp_message((int)message_type, len);
 288     qemu_mutex_lock(&mis->rp_mutex);
 289
 290     /*
 291      * It's possible that the file handle got lost due to network
 292      * failures.
 293      */
 294     if (!mis->to_src_file) {
 295         ret = -EIO;
 296         goto error;
 297     }
 298
 299     qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
 300     qemu_put_be16(mis->to_src_file, len);
 301     qemu_put_buffer(mis->to_src_file, data, len);
 302     qemu_fflush(mis->to_src_file);
 303
 304     /* It's possible that qemu file got error during sending */
 305     ret = qemu_file_get_error(mis->to_src_file);
 306
 307 error:
 308     qemu_mutex_unlock(&mis->rp_mutex);
 309     return ret;
 310 }
 311
 312 /* Request one page from the source VM at the given start address.
 313  *   rb: the RAMBlock to request the page in
 314  *   Start: Address offset within the RB
 315  *   Len: Length in bytes required - must be a multiple of pagesize
 316  */
 317 int migrate_send_rp_message_req_pages(MigrationIncomingState *mis,
 318                                       RAMBlock *rb, ram_addr_t start)
 319 {
 320     uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */
 321     size_t msglen = 12; /* start + len */
 322     size_t len = qemu_ram_pagesize(rb);
 323     enum mig_rp_message_type msg_type;
 324     const char *rbname;
 325     int rbname_len;
 326
 327     *(uint64_t *)bufc = cpu_to_be64((uint64_t)start);
 328     *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len);
 329
 330     /*
 331      * We maintain the last ramblock that we requested for page.  Note that we
 332      * don't need locking because this function will only be called within the
 333      * postcopy ram fault thread.
 334      */
 335     if (rb != mis->last_rb) {
 336         mis->last_rb = rb;
 337
 338         rbname = qemu_ram_get_idstr(rb);
 339         rbname_len = strlen(rbname);
 340
 341         assert(rbname_len < 256);
 342
 343         bufc[msglen++] = rbname_len;
 344         memcpy(bufc + msglen, rbname, rbname_len);
 345         msglen += rbname_len;
 346         msg_type = MIG_RP_MSG_REQ_PAGES_ID;
 347     } else {
 348         msg_type = MIG_RP_MSG_REQ_PAGES;
 349     }
 350
 351     return migrate_send_rp_message(mis, msg_type, msglen, bufc);
 352 }
 353
 354 int migrate_send_rp_req_pages(MigrationIncomingState *mis,
 355                               RAMBlock *rb, ram_addr_t start, uint64_t haddr)
 356 {
 357     void *aligned = (void *)(uintptr_t)(haddr & (-qemu_ram_pagesize(rb)));
 358     bool received = false;
 359
 360     WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) {
 361         received = ramblock_recv_bitmap_test_byte_offset(rb, start);
 362         if (!received && !g_tree_lookup(mis->page_requested, aligned)) {
 363             /*
 364              * The page has not been received, and it's not yet in the page
 365              * request list.  Queue it.  Set the value of element to 1, so that
 366              * things like g_tree_lookup() will return TRUE (1) when found.
 367              */
 368             g_tree_insert(mis->page_requested, aligned, (gpointer)1);
 369             mis->page_requested_count++;
 370             trace_postcopy_page_req_add(aligned, mis->page_requested_count);
 371         }
 372     }
 373
 374     /*
 375      * If the page is there, skip sending the message.  We don't even need the
 376      * lock because as long as the page arrived, it'll be there forever.
 377      */
 378     if (received) {
 379         return 0;
 380     }
 381
 382     return migrate_send_rp_message_req_pages(mis, rb, start);
 383 }
 384
 385 static bool migration_colo_enabled;
 386 bool migration_incoming_colo_enabled(void)
 387 {
 388     return migration_colo_enabled;
 389 }
 390
 391 void migration_incoming_disable_colo(void)
 392 {
 393     ram_block_discard_disable(false);
 394     migration_colo_enabled = false;
 395 }
 396
 397 int migration_incoming_enable_colo(void)
 398 {
 399     if (ram_block_discard_disable(true)) {
 400         error_report("COLO: cannot disable RAM discard");
 401         return -EBUSY;
 402     }
 403     migration_colo_enabled = true;
 404     return 0;
 405 }
 406
 407 void migrate_add_address(SocketAddress *address)
 408 {
 409     MigrationIncomingState *mis = migration_incoming_get_current();
 410
 411     QAPI_LIST_PREPEND(mis->socket_address_list,
 412                       QAPI_CLONE(SocketAddress, address));
 413 }
 414
 415 static void qemu_start_incoming_migration(const char *uri, Error **errp)
 416 {
 417     const char *p = NULL;
 418
 419     qapi_event_send_migration(MIGRATION_STATUS_SETUP);
 420     if (strstart(uri, "tcp:", &p) ||
 421         strstart(uri, "unix:", NULL) ||
 422         strstart(uri, "vsock:", NULL)) {
 423         socket_start_incoming_migration(p ? p : uri, errp);
 424 #ifdef CONFIG_RDMA
 425     } else if (strstart(uri, "rdma:", &p)) {
 426         rdma_start_incoming_migration(p, errp);
 427 #endif
 428     } else if (strstart(uri, "exec:", &p)) {
 429         exec_start_incoming_migration(p, errp);
 430     } else if (strstart(uri, "fd:", &p)) {
 431         fd_start_incoming_migration(p, errp);
 432     } else {
 433         error_setg(errp, "unknown migration protocol: %s", uri);
 434     }
 435 }
 436
 437 static void process_incoming_migration_bh(void *opaque)
 438 {
 439     Error *local_err = NULL;
 440     MigrationIncomingState *mis = opaque;
 441
 442     /* If capability late_block_activate is set:
 443      * Only fire up the block code now if we're going to restart the
 444      * VM, else 'cont' will do it.
 445      * This causes file locking to happen; so we don't want it to happen
 446      * unless we really are starting the VM.
 447      */
 448     if (!migrate_late_block_activate() ||
 449          (autostart && (!global_state_received() ||
 450             global_state_get_runstate() == RUN_STATE_RUNNING))) {
 451         /* Make sure all file formats flush their mutable metadata.
 452          * If we get an error here, just don't restart the VM yet. */
 453         bdrv_invalidate_cache_all(&local_err);
 454         if (local_err) {
 455             error_report_err(local_err);
 456             local_err = NULL;
 457             autostart = false;
 458         }
 459     }
 460
 461     /*
 462      * This must happen after all error conditions are dealt with and
 463      * we're sure the VM is going to be running on this host.
 464      */
 465     qemu_announce_self(&mis->announce_timer, migrate_announce_params());
 466
 467     if (multifd_load_cleanup(&local_err) != 0) {
 468         error_report_err(local_err);
 469         autostart = false;
 470     }
 471     /* If global state section was not received or we are in running
 472        state, we need to obey autostart. Any other state is set with
 473        runstate_set. */
 474
 475     dirty_bitmap_mig_before_vm_start();
 476
 477     if (!global_state_received() ||
 478         global_state_get_runstate() == RUN_STATE_RUNNING) {
 479         if (autostart) {
 480             vm_start();
 481         } else {
 482             runstate_set(RUN_STATE_PAUSED);
 483         }
 484     } else if (migration_incoming_colo_enabled()) {
 485         migration_incoming_disable_colo();
 486         vm_start();
 487     } else {
 488         runstate_set(global_state_get_runstate());
 489     }
 490     /*
 491      * This must happen after any state changes since as soon as an external
 492      * observer sees this event they might start to prod at the VM assuming
 493      * it's ready to use.
 494      */
 495     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 496                       MIGRATION_STATUS_COMPLETED);
 497     qemu_bh_delete(mis->bh);
 498     migration_incoming_state_destroy();
 499 }
 500
 501 static void process_incoming_migration_co(void *opaque)
 502 {
 503     MigrationIncomingState *mis = migration_incoming_get_current();
 504     PostcopyState ps;
 505     int ret;
 506     Error *local_err = NULL;
 507
 508     assert(mis->from_src_file);
 509     mis->migration_incoming_co = qemu_coroutine_self();
 510     mis->largest_page_size = qemu_ram_pagesize_largest();
 511     postcopy_state_set(POSTCOPY_INCOMING_NONE);
 512     migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
 513                       MIGRATION_STATUS_ACTIVE);
 514     ret = qemu_loadvm_state(mis->from_src_file);
 515
 516     ps = postcopy_state_get();
 517     trace_process_incoming_migration_co_end(ret, ps);
 518     if (ps != POSTCOPY_INCOMING_NONE) {
 519         if (ps == POSTCOPY_INCOMING_ADVISE) {
 520             /*
 521              * Where a migration had postcopy enabled (and thus went to advise)
 522              * but managed to complete within the precopy period, we can use
 523              * the normal exit.
 524              */
 525             postcopy_ram_incoming_cleanup(mis);
 526         } else if (ret >= 0) {
 527             /*
 528              * Postcopy was started, cleanup should happen at the end of the
 529              * postcopy thread.
 530              */
 531             trace_process_incoming_migration_co_postcopy_end_main();
 532             return;
 533         }
 534         /* Else if something went wrong then just fall out of the normal exit */
 535     }
 536
 537     /* we get COLO info, and know if we are in COLO mode */
 538     if (!ret && migration_incoming_colo_enabled()) {
 539         /* Make sure all file formats flush their mutable metadata */
 540         bdrv_invalidate_cache_all(&local_err);
 541         if (local_err) {
 542             error_report_err(local_err);
 543             goto fail;
 544         }
 545
 546         qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
 547              colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
 548         mis->have_colo_incoming_thread = true;
 549         qemu_coroutine_yield();
 550
 551         /* Wait checkpoint incoming thread exit before free resource */
 552         qemu_thread_join(&mis->colo_incoming_thread);
 553         /* We hold the global iothread lock, so it is safe here */
 554         colo_release_ram_cache();
 555     }
 556
 557     if (ret < 0) {
 558         error_report("load of migration failed: %s", strerror(-ret));
 559         goto fail;
 560     }
 561     mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
 562     qemu_bh_schedule(mis->bh);
 563     mis->migration_incoming_co = NULL;
 564     return;
 565 fail:
 566     local_err = NULL;
 567     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 568                       MIGRATION_STATUS_FAILED);
 569     qemu_fclose(mis->from_src_file);
 570     if (multifd_load_cleanup(&local_err) != 0) {
 571         error_report_err(local_err);
 572     }
 573     exit(EXIT_FAILURE);
 574 }
 575
 576 /**
 577  * @migration_incoming_setup: Setup incoming migration
 578  *
 579  * Returns 0 for no error or 1 for error
 580  *
 581  * @f: file for main migration channel
 582  * @errp: where to put errors
 583  */
 584 static int migration_incoming_setup(QEMUFile *f, Error **errp)
 585 {
 586     MigrationIncomingState *mis = migration_incoming_get_current();
 587     Error *local_err = NULL;
 588
 589     if (multifd_load_setup(&local_err) != 0) {
 590         /* We haven't been able to create multifd threads
 591            nothing better to do */
 592         error_report_err(local_err);
 593         exit(EXIT_FAILURE);
 594     }
 595
 596     if (!mis->from_src_file) {
 597         mis->from_src_file = f;
 598     }
 599     qemu_file_set_blocking(f, false);
 600     return 0;
 601 }
 602
 603 void migration_incoming_process(void)
 604 {
 605     Coroutine *co = qemu_coroutine_create(process_incoming_migration_co, NULL);
 606     qemu_coroutine_enter(co);
 607 }
 608
 609 /* Returns true if recovered from a paused migration, otherwise false */
 610 static bool postcopy_try_recover(QEMUFile *f)
 611 {
 612     MigrationIncomingState *mis = migration_incoming_get_current();
 613
 614     if (mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
 615         /* Resumed from a paused postcopy migration */
 616
 617         mis->from_src_file = f;
 618         /* Postcopy has standalone thread to do vm load */
 619         qemu_file_set_blocking(f, true);
 620
 621         /* Re-configure the return path */
 622         mis->to_src_file = qemu_file_get_return_path(f);
 623
 624         migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
 625                           MIGRATION_STATUS_POSTCOPY_RECOVER);
 626
 627         /*
 628          * Here, we only wake up the main loading thread (while the
 629          * fault thread will still be waiting), so that we can receive
 630          * commands from source now, and answer it if needed. The
 631          * fault thread will be woken up afterwards until we are sure
 632          * that source is ready to reply to page requests.
 633          */
 634         qemu_sem_post(&mis->postcopy_pause_sem_dst);
 635         return true;
 636     }
 637
 638     return false;
 639 }
 640
 641 void migration_fd_process_incoming(QEMUFile *f, Error **errp)
 642 {
 643     Error *local_err = NULL;
 644
 645     if (postcopy_try_recover(f)) {
 646         return;
 647     }
 648
 649     if (migration_incoming_setup(f, &local_err)) {
 650         error_propagate(errp, local_err);
 651         return;
 652     }
 653     migration_incoming_process();
 654 }
 655
 656 void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp)
 657 {
 658     MigrationIncomingState *mis = migration_incoming_get_current();
 659     Error *local_err = NULL;
 660     bool start_migration;
 661
 662     if (!mis->from_src_file) {
 663         /* The first connection (multifd may have multiple) */
 664         QEMUFile *f = qemu_fopen_channel_input(ioc);
 665
 666         /* If it's a recovery, we're done */
 667         if (postcopy_try_recover(f)) {
 668             return;
 669         }
 670
 671         if (migration_incoming_setup(f, &local_err)) {
 672             error_propagate(errp, local_err);
 673             return;
 674         }
 675
 676         /*
 677          * Common migration only needs one channel, so we can start
 678          * right now.  Multifd needs more than one channel, we wait.
 679          */
 680         start_migration = !migrate_use_multifd();
 681     } else {
 682         /* Multiple connections */
 683         assert(migrate_use_multifd());
 684         start_migration = multifd_recv_new_channel(ioc, &local_err);
 685         if (local_err) {
 686             error_propagate(errp, local_err);
 687             return;
 688         }
 689     }
 690
 691     if (start_migration) {
 692         migration_incoming_process();
 693     }
 694 }
 695
 696 /**
 697  * @migration_has_all_channels: We have received all channels that we need
 698  *
 699  * Returns true when we have got connections to all the channels that
 700  * we need for migration.
 701  */
 702 bool migration_has_all_channels(void)
 703 {
 704     MigrationIncomingState *mis = migration_incoming_get_current();
 705     bool all_channels;
 706
 707     all_channels = multifd_recv_all_channels_created();
 708
 709     return all_channels && mis->from_src_file != NULL;
 710 }
 711
 712 /*
 713  * Send a 'SHUT' message on the return channel with the given value
 714  * to indicate that we've finished with the RP.  Non-0 value indicates
 715  * error.
 716  */
 717 void migrate_send_rp_shut(MigrationIncomingState *mis,
 718                           uint32_t value)
 719 {
 720     uint32_t buf;
 721
 722     buf = cpu_to_be32(value);
 723     migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
 724 }
 725
 726 /*
 727  * Send a 'PONG' message on the return channel with the given value
 728  * (normally in response to a 'PING')
 729  */
 730 void migrate_send_rp_pong(MigrationIncomingState *mis,
 731                           uint32_t value)
 732 {
 733     uint32_t buf;
 734
 735     buf = cpu_to_be32(value);
 736     migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
 737 }
 738
 739 void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
 740                                  char *block_name)
 741 {
 742     char buf[512];
 743     int len;
 744     int64_t res;
 745
 746     /*
 747      * First, we send the header part. It contains only the len of
 748      * idstr, and the idstr itself.
 749      */
 750     len = strlen(block_name);
 751     buf[0] = len;
 752     memcpy(buf + 1, block_name, len);
 753
 754     if (mis->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
 755         error_report("%s: MSG_RP_RECV_BITMAP only used for recovery",
 756                      __func__);
 757         return;
 758     }
 759
 760     migrate_send_rp_message(mis, MIG_RP_MSG_RECV_BITMAP, len + 1, buf);
 761
 762     /*
 763      * Next, we dump the received bitmap to the stream.
 764      *
 765      * TODO: currently we are safe since we are the only one that is
 766      * using the to_src_file handle (fault thread is still paused),
 767      * and it's ok even not taking the mutex. However the best way is
 768      * to take the lock before sending the message header, and release
 769      * the lock after sending the bitmap.
 770      */
 771     qemu_mutex_lock(&mis->rp_mutex);
 772     res = ramblock_recv_bitmap_send(mis->to_src_file, block_name);
 773     qemu_mutex_unlock(&mis->rp_mutex);
 774
 775     trace_migrate_send_rp_recv_bitmap(block_name, res);
 776 }
 777
 778 void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value)
 779 {
 780     uint32_t buf;
 781
 782     buf = cpu_to_be32(value);
 783     migrate_send_rp_message(mis, MIG_RP_MSG_RESUME_ACK, sizeof(buf), &buf);
 784 }
 785
 786 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
 787 {
 788     MigrationCapabilityStatusList *head = NULL;
 789     MigrationCapabilityStatusList *caps;
 790     MigrationState *s = migrate_get_current();
 791     int i;
 792
 793     caps = NULL; /* silence compiler warning */
 794     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
 795 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
 796         if (i == MIGRATION_CAPABILITY_BLOCK) {
 797             continue;
 798         }
 799 #endif
 800         if (head == NULL) {
 801             head = g_malloc0(sizeof(*caps));
 802             caps = head;
 803         } else {
 804             caps->next = g_malloc0(sizeof(*caps));
 805             caps = caps->next;
 806         }
 807         caps->value =
 808             g_malloc(sizeof(*caps->value));
 809         caps->value->capability = i;
 810         caps->value->state = s->enabled_capabilities[i];
 811     }
 812
 813     return head;
 814 }
 815
 816 MigrationParameters *qmp_query_migrate_parameters(Error **errp)
 817 {
 818     MigrationParameters *params;
 819     MigrationState *s = migrate_get_current();
 820
 821     /* TODO use QAPI_CLONE() instead of duplicating it inline */
 822     params = g_malloc0(sizeof(*params));
 823     params->has_compress_level = true;
 824     params->compress_level = s->parameters.compress_level;
 825     params->has_compress_threads = true;
 826     params->compress_threads = s->parameters.compress_threads;
 827     params->has_compress_wait_thread = true;
 828     params->compress_wait_thread = s->parameters.compress_wait_thread;
 829     params->has_decompress_threads = true;
 830     params->decompress_threads = s->parameters.decompress_threads;
 831     params->has_throttle_trigger_threshold = true;
 832     params->throttle_trigger_threshold = s->parameters.throttle_trigger_threshold;
 833     params->has_cpu_throttle_initial = true;
 834     params->cpu_throttle_initial = s->parameters.cpu_throttle_initial;
 835     params->has_cpu_throttle_increment = true;
 836     params->cpu_throttle_increment = s->parameters.cpu_throttle_increment;
 837     params->has_cpu_throttle_tailslow = true;
 838     params->cpu_throttle_tailslow = s->parameters.cpu_throttle_tailslow;
 839     params->has_tls_creds = true;
 840     params->tls_creds = g_strdup(s->parameters.tls_creds);
 841     params->has_tls_hostname = true;
 842     params->tls_hostname = g_strdup(s->parameters.tls_hostname);
 843     params->has_tls_authz = true;
 844     params->tls_authz = g_strdup(s->parameters.tls_authz ?
 845                                  s->parameters.tls_authz : "");
 846     params->has_max_bandwidth = true;
 847     params->max_bandwidth = s->parameters.max_bandwidth;
 848     params->has_downtime_limit = true;
 849     params->downtime_limit = s->parameters.downtime_limit;
 850     params->has_x_checkpoint_delay = true;
 851     params->x_checkpoint_delay = s->parameters.x_checkpoint_delay;
 852     params->has_block_incremental = true;
 853     params->block_incremental = s->parameters.block_incremental;
 854     params->has_multifd_channels = true;
 855     params->multifd_channels = s->parameters.multifd_channels;
 856     params->has_multifd_compression = true;
 857     params->multifd_compression = s->parameters.multifd_compression;
 858     params->has_multifd_zlib_level = true;
 859     params->multifd_zlib_level = s->parameters.multifd_zlib_level;
 860     params->has_multifd_zstd_level = true;
 861     params->multifd_zstd_level = s->parameters.multifd_zstd_level;
 862     params->has_xbzrle_cache_size = true;
 863     params->xbzrle_cache_size = s->parameters.xbzrle_cache_size;
 864     params->has_max_postcopy_bandwidth = true;
 865     params->max_postcopy_bandwidth = s->parameters.max_postcopy_bandwidth;
 866     params->has_max_cpu_throttle = true;
 867     params->max_cpu_throttle = s->parameters.max_cpu_throttle;
 868     params->has_announce_initial = true;
 869     params->announce_initial = s->parameters.announce_initial;
 870     params->has_announce_max = true;
 871     params->announce_max = s->parameters.announce_max;
 872     params->has_announce_rounds = true;
 873     params->announce_rounds = s->parameters.announce_rounds;
 874     params->has_announce_step = true;
 875     params->announce_step = s->parameters.announce_step;
 876
 877     if (s->parameters.has_block_bitmap_mapping) {
 878         params->has_block_bitmap_mapping = true;
 879         params->block_bitmap_mapping =
 880             QAPI_CLONE(BitmapMigrationNodeAliasList,
 881                        s->parameters.block_bitmap_mapping);
 882     }
 883
 884     return params;
 885 }
 886
 887 AnnounceParameters *migrate_announce_params(void)
 888 {
 889     static AnnounceParameters ap;
 890
 891     MigrationState *s = migrate_get_current();
 892
 893     ap.initial = s->parameters.announce_initial;
 894     ap.max = s->parameters.announce_max;
 895     ap.rounds = s->parameters.announce_rounds;
 896     ap.step = s->parameters.announce_step;
 897
 898     return &ap;
 899 }
 900
 901 /*
 902  * Return true if we're already in the middle of a migration
 903  * (i.e. any of the active or setup states)
 904  */
 905 bool migration_is_setup_or_active(int state)
 906 {
 907     switch (state) {
 908     case MIGRATION_STATUS_ACTIVE:
 909     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 910     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 911     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 912     case MIGRATION_STATUS_SETUP:
 913     case MIGRATION_STATUS_PRE_SWITCHOVER:
 914     case MIGRATION_STATUS_DEVICE:
 915     case MIGRATION_STATUS_WAIT_UNPLUG:
 916     case MIGRATION_STATUS_COLO:
 917         return true;
 918
 919     default:
 920         return false;
 921
 922     }
 923 }
 924
 925 bool migration_is_running(int state)
 926 {
 927     switch (state) {
 928     case MIGRATION_STATUS_ACTIVE:
 929     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 930     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 931     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 932     case MIGRATION_STATUS_SETUP:
 933     case MIGRATION_STATUS_PRE_SWITCHOVER:
 934     case MIGRATION_STATUS_DEVICE:
 935     case MIGRATION_STATUS_WAIT_UNPLUG:
 936     case MIGRATION_STATUS_CANCELLING:
 937         return true;
 938
 939     default:
 940         return false;
 941
 942     }
 943 }
 944
 945 static void populate_time_info(MigrationInfo *info, MigrationState *s)
 946 {
 947     info->has_status = true;
 948     info->has_setup_time = true;
 949     info->setup_time = s->setup_time;
 950     if (s->state == MIGRATION_STATUS_COMPLETED) {
 951         info->has_total_time = true;
 952         info->total_time = s->total_time;
 953         info->has_downtime = true;
 954         info->downtime = s->downtime;
 955     } else {
 956         info->has_total_time = true;
 957         info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) -
 958                            s->start_time;
 959         info->has_expected_downtime = true;
 960         info->expected_downtime = s->expected_downtime;
 961     }
 962 }
 963
 964 static void populate_ram_info(MigrationInfo *info, MigrationState *s)
 965 {
 966     info->has_ram = true;
 967     info->ram = g_malloc0(sizeof(*info->ram));
 968     info->ram->transferred = ram_counters.transferred;
 969     info->ram->total = ram_bytes_total();
 970     info->ram->duplicate = ram_counters.duplicate;
 971     /* legacy value.  It is not used anymore */
 972     info->ram->skipped = 0;
 973     info->ram->normal = ram_counters.normal;
 974     info->ram->normal_bytes = ram_counters.normal *
 975         qemu_target_page_size();
 976     info->ram->mbps = s->mbps;
 977     info->ram->dirty_sync_count = ram_counters.dirty_sync_count;
 978     info->ram->postcopy_requests = ram_counters.postcopy_requests;
 979     info->ram->page_size = qemu_target_page_size();
 980     info->ram->multifd_bytes = ram_counters.multifd_bytes;
 981     info->ram->pages_per_second = s->pages_per_second;
 982
 983     if (migrate_use_xbzrle()) {
 984         info->has_xbzrle_cache = true;
 985         info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
 986         info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
 987         info->xbzrle_cache->bytes = xbzrle_counters.bytes;
 988         info->xbzrle_cache->pages = xbzrle_counters.pages;
 989         info->xbzrle_cache->cache_miss = xbzrle_counters.cache_miss;
 990         info->xbzrle_cache->cache_miss_rate = xbzrle_counters.cache_miss_rate;
 991         info->xbzrle_cache->encoding_rate = xbzrle_counters.encoding_rate;
 992         info->xbzrle_cache->overflow = xbzrle_counters.overflow;
 993     }
 994
 995     if (migrate_use_compression()) {
 996         info->has_compression = true;
 997         info->compression = g_malloc0(sizeof(*info->compression));
 998         info->compression->pages = compression_counters.pages;
 999         info->compression->busy = compression_counters.busy;
1000         info->compression->busy_rate = compression_counters.busy_rate;
1001         info->compression->compressed_size =
1002                                     compression_counters.compressed_size;
1003         info->compression->compression_rate =
1004                                     compression_counters.compression_rate;
1005     }
1006
1007     if (cpu_throttle_active()) {
1008         info->has_cpu_throttle_percentage = true;
1009         info->cpu_throttle_percentage = cpu_throttle_get_percentage();
1010     }
1011
1012     if (s->state != MIGRATION_STATUS_COMPLETED) {
1013         info->ram->remaining = ram_bytes_remaining();
1014         info->ram->dirty_pages_rate = ram_counters.dirty_pages_rate;
1015     }
1016 }
1017
1018 static void populate_disk_info(MigrationInfo *info)
1019 {
1020     if (blk_mig_active()) {
1021         info->has_disk = true;
1022         info->disk = g_malloc0(sizeof(*info->disk));
1023         info->disk->transferred = blk_mig_bytes_transferred();
1024         info->disk->remaining = blk_mig_bytes_remaining();
1025         info->disk->total = blk_mig_bytes_total();
1026     }
1027 }
1028
1029 static void populate_vfio_info(MigrationInfo *info)
1030 {
1031 #ifdef CONFIG_VFIO
1032     if (vfio_mig_active()) {
1033         info->has_vfio = true;
1034         info->vfio = g_malloc0(sizeof(*info->vfio));
1035         info->vfio->transferred = vfio_mig_bytes_transferred();
1036     }
1037 #endif
1038 }
1039
1040 static void fill_source_migration_info(MigrationInfo *info)
1041 {
1042     MigrationState *s = migrate_get_current();
1043
1044     switch (s->state) {
1045     case MIGRATION_STATUS_NONE:
1046         /* no migration has happened ever */
1047         /* do not overwrite destination migration status */
1048         return;
1049     case MIGRATION_STATUS_SETUP:
1050         info->has_status = true;
1051         info->has_total_time = false;
1052         break;
1053     case MIGRATION_STATUS_ACTIVE:
1054     case MIGRATION_STATUS_CANCELLING:
1055     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1056     case MIGRATION_STATUS_PRE_SWITCHOVER:
1057     case MIGRATION_STATUS_DEVICE:
1058     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1059     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1060         /* TODO add some postcopy stats */
1061         populate_time_info(info, s);
1062         populate_ram_info(info, s);
1063         populate_disk_info(info);
1064         populate_vfio_info(info);
1065         break;
1066     case MIGRATION_STATUS_COLO:
1067         info->has_status = true;
1068         /* TODO: display COLO specific information (checkpoint info etc.) */
1069         break;
1070     case MIGRATION_STATUS_COMPLETED:
1071         populate_time_info(info, s);
1072         populate_ram_info(info, s);
1073         populate_vfio_info(info);
1074         break;
1075     case MIGRATION_STATUS_FAILED:
1076         info->has_status = true;
1077         if (s->error) {
1078             info->has_error_desc = true;
1079             info->error_desc = g_strdup(error_get_pretty(s->error));
1080         }
1081         break;
1082     case MIGRATION_STATUS_CANCELLED:
1083         info->has_status = true;
1084         break;
1085     case MIGRATION_STATUS_WAIT_UNPLUG:
1086         info->has_status = true;
1087         break;
1088     }
1089     info->status = s->state;
1090 }
1091
1092 /**
1093  * @migration_caps_check - check capability validity
1094  *
1095  * @cap_list: old capability list, array of bool
1096  * @params: new capabilities to be applied soon
1097  * @errp: set *errp if the check failed, with reason
1098  *
1099  * Returns true if check passed, otherwise false.
1100  */
1101 static bool migrate_caps_check(bool *cap_list,
1102                                MigrationCapabilityStatusList *params,
1103                                Error **errp)
1104 {
1105     MigrationCapabilityStatusList *cap;
1106     bool old_postcopy_cap;
1107     MigrationIncomingState *mis = migration_incoming_get_current();
1108
1109     old_postcopy_cap = cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM];
1110
1111     for (cap = params; cap; cap = cap->next) {
1112         cap_list[cap->value->capability] = cap->value->state;
1113     }
1114
1115 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
1116     if (cap_list[MIGRATION_CAPABILITY_BLOCK]) {
1117         error_setg(errp, "QEMU compiled without old-style (blk/-b, inc/-i) "
1118                    "block migration");
1119         error_append_hint(errp, "Use drive_mirror+NBD instead.\n");
1120         return false;
1121     }
1122 #endif
1123
1124 #ifndef CONFIG_REPLICATION
1125     if (cap_list[MIGRATION_CAPABILITY_X_COLO]) {
1126         error_setg(errp, "QEMU compiled without replication module"
1127                    " can't enable COLO");
1128         error_append_hint(errp, "Please enable replication before COLO.\n");
1129         return false;
1130     }
1131 #endif
1132
1133     if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
1134         /* This check is reasonably expensive, so only when it's being
1135          * set the first time, also it's only the destination that needs
1136          * special support.
1137          */
1138         if (!old_postcopy_cap && runstate_check(RUN_STATE_INMIGRATE) &&
1139             !postcopy_ram_supported_by_host(mis)) {
1140             /* postcopy_ram_supported_by_host will have emitted a more
1141              * detailed message
1142              */
1143             error_setg(errp, "Postcopy is not supported");
1144             return false;
1145         }
1146
1147         if (cap_list[MIGRATION_CAPABILITY_X_IGNORE_SHARED]) {
1148             error_setg(errp, "Postcopy is not compatible with ignore-shared");
1149             return false;
1150         }
1151     }
1152
1153     return true;
1154 }
1155
1156 static void fill_destination_migration_info(MigrationInfo *info)
1157 {
1158     MigrationIncomingState *mis = migration_incoming_get_current();
1159
1160     if (mis->socket_address_list) {
1161         info->has_socket_address = true;
1162         info->socket_address =
1163             QAPI_CLONE(SocketAddressList, mis->socket_address_list);
1164     }
1165
1166     switch (mis->state) {
1167     case MIGRATION_STATUS_NONE:
1168         return;
1169     case MIGRATION_STATUS_SETUP:
1170     case MIGRATION_STATUS_CANCELLING:
1171     case MIGRATION_STATUS_CANCELLED:
1172     case MIGRATION_STATUS_ACTIVE:
1173     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1174     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1175     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1176     case MIGRATION_STATUS_FAILED:
1177     case MIGRATION_STATUS_COLO:
1178         info->has_status = true;
1179         break;
1180     case MIGRATION_STATUS_COMPLETED:
1181         info->has_status = true;
1182         fill_destination_postcopy_migration_info(info);
1183         break;
1184     }
1185     info->status = mis->state;
1186 }
1187
1188 MigrationInfo *qmp_query_migrate(Error **errp)
1189 {
1190     MigrationInfo *info = g_malloc0(sizeof(*info));
1191
1192     fill_destination_migration_info(info);
1193     fill_source_migration_info(info);
1194
1195     return info;
1196 }
1197
1198 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
1199                                   Error **errp)
1200 {
1201     MigrationState *s = migrate_get_current();
1202     MigrationCapabilityStatusList *cap;
1203     bool cap_list[MIGRATION_CAPABILITY__MAX];
1204
1205     if (migration_is_running(s->state)) {
1206         error_setg(errp, QERR_MIGRATION_ACTIVE);
1207         return;
1208     }
1209
1210     memcpy(cap_list, s->enabled_capabilities, sizeof(cap_list));
1211     if (!migrate_caps_check(cap_list, params, errp)) {
1212         return;
1213     }
1214
1215     for (cap = params; cap; cap = cap->next) {
1216         s->enabled_capabilities[cap->value->capability] = cap->value->state;
1217     }
1218 }
1219
1220 /*
1221  * Check whether the parameters are valid. Error will be put into errp
1222  * (if provided). Return true if valid, otherwise false.
1223  */
1224 static bool migrate_params_check(MigrationParameters *params, Error **errp)
1225 {
1226     if (params->has_compress_level &&
1227         (params->compress_level > 9)) {
1228         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level",
1229                    "is invalid, it should be in the range of 0 to 9");
1230         return false;
1231     }
1232
1233     if (params->has_compress_threads && (params->compress_threads < 1)) {
1234         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1235                    "compress_threads",
1236                    "is invalid, it should be in the range of 1 to 255");
1237         return false;
1238     }
1239
1240     if (params->has_decompress_threads && (params->decompress_threads < 1)) {
1241         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1242                    "decompress_threads",
1243                    "is invalid, it should be in the range of 1 to 255");
1244         return false;
1245     }
1246
1247     if (params->has_throttle_trigger_threshold &&
1248         (params->throttle_trigger_threshold < 1 ||
1249          params->throttle_trigger_threshold > 100)) {
1250         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1251                    "throttle_trigger_threshold",
1252                    "an integer in the range of 1 to 100");
1253         return false;
1254     }
1255
1256     if (params->has_cpu_throttle_initial &&
1257         (params->cpu_throttle_initial < 1 ||
1258          params->cpu_throttle_initial > 99)) {
1259         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1260                    "cpu_throttle_initial",
1261                    "an integer in the range of 1 to 99");
1262         return false;
1263     }
1264
1265     if (params->has_cpu_throttle_increment &&
1266         (params->cpu_throttle_increment < 1 ||
1267          params->cpu_throttle_increment > 99)) {
1268         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1269                    "cpu_throttle_increment",
1270                    "an integer in the range of 1 to 99");
1271         return false;
1272     }
1273
1274     if (params->has_max_bandwidth && (params->max_bandwidth > SIZE_MAX)) {
1275         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1276                    "max_bandwidth",
1277                    "an integer in the range of 0 to "stringify(SIZE_MAX)
1278                    " bytes/second");
1279         return false;
1280     }
1281
1282     if (params->has_downtime_limit &&
1283         (params->downtime_limit > MAX_MIGRATE_DOWNTIME)) {
1284         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1285                    "downtime_limit",
1286                    "an integer in the range of 0 to "
1287                     stringify(MAX_MIGRATE_DOWNTIME)" ms");
1288         return false;
1289     }
1290
1291     /* x_checkpoint_delay is now always positive */
1292
1293     if (params->has_multifd_channels && (params->multifd_channels < 1)) {
1294         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1295                    "multifd_channels",
1296                    "is invalid, it should be in the range of 1 to 255");
1297         return false;
1298     }
1299
1300     if (params->has_multifd_zlib_level &&
1301         (params->multifd_zlib_level > 9)) {
1302         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zlib_level",
1303                    "is invalid, it should be in the range of 0 to 9");
1304         return false;
1305     }
1306
1307     if (params->has_multifd_zstd_level &&
1308         (params->multifd_zstd_level > 20)) {
1309         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zstd_level",
1310                    "is invalid, it should be in the range of 0 to 20");
1311         return false;
1312     }
1313
1314     if (params->has_xbzrle_cache_size &&
1315         (params->xbzrle_cache_size < qemu_target_page_size() ||
1316          !is_power_of_2(params->xbzrle_cache_size))) {
1317         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1318                    "xbzrle_cache_size",
1319                    "is invalid, it should be bigger than target page size"
1320                    " and a power of 2");
1321         return false;
1322     }
1323
1324     if (params->has_max_cpu_throttle &&
1325         (params->max_cpu_throttle < params->cpu_throttle_initial ||
1326          params->max_cpu_throttle > 99)) {
1327         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1328                    "max_cpu_throttle",
1329                    "an integer in the range of cpu_throttle_initial to 99");
1330         return false;
1331     }
1332
1333     if (params->has_announce_initial &&
1334         params->announce_initial > 100000) {
1335         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1336                    "announce_initial",
1337                    "is invalid, it must be less than 100000 ms");
1338         return false;
1339     }
1340     if (params->has_announce_max &&
1341         params->announce_max > 100000) {
1342         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1343                    "announce_max",
1344                    "is invalid, it must be less than 100000 ms");
1345        return false;
1346     }
1347     if (params->has_announce_rounds &&
1348         params->announce_rounds > 1000) {
1349         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1350                    "announce_rounds",
1351                    "is invalid, it must be in the range of 0 to 1000");
1352        return false;
1353     }
1354     if (params->has_announce_step &&
1355         (params->announce_step < 1 ||
1356         params->announce_step > 10000)) {
1357         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1358                    "announce_step",
1359                    "is invalid, it must be in the range of 1 to 10000 ms");
1360        return false;
1361     }
1362
1363     if (params->has_block_bitmap_mapping &&
1364         !check_dirty_bitmap_mig_alias_map(params->block_bitmap_mapping, errp)) {
1365         error_prepend(errp, "Invalid mapping given for block-bitmap-mapping: ");
1366         return false;
1367     }
1368
1369     return true;
1370 }
1371
1372 static void migrate_params_test_apply(MigrateSetParameters *params,
1373                                       MigrationParameters *dest)
1374 {
1375     *dest = migrate_get_current()->parameters;
1376
1377     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1378
1379     if (params->has_compress_level) {
1380         dest->compress_level = params->compress_level;
1381     }
1382
1383     if (params->has_compress_threads) {
1384         dest->compress_threads = params->compress_threads;
1385     }
1386
1387     if (params->has_compress_wait_thread) {
1388         dest->compress_wait_thread = params->compress_wait_thread;
1389     }
1390
1391     if (params->has_decompress_threads) {
1392         dest->decompress_threads = params->decompress_threads;
1393     }
1394
1395     if (params->has_throttle_trigger_threshold) {
1396         dest->throttle_trigger_threshold = params->throttle_trigger_threshold;
1397     }
1398
1399     if (params->has_cpu_throttle_initial) {
1400         dest->cpu_throttle_initial = params->cpu_throttle_initial;
1401     }
1402
1403     if (params->has_cpu_throttle_increment) {
1404         dest->cpu_throttle_increment = params->cpu_throttle_increment;
1405     }
1406
1407     if (params->has_cpu_throttle_tailslow) {
1408         dest->cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1409     }
1410
1411     if (params->has_tls_creds) {
1412         assert(params->tls_creds->type == QTYPE_QSTRING);
1413         dest->tls_creds = params->tls_creds->u.s;
1414     }
1415
1416     if (params->has_tls_hostname) {
1417         assert(params->tls_hostname->type == QTYPE_QSTRING);
1418         dest->tls_hostname = params->tls_hostname->u.s;
1419     }
1420
1421     if (params->has_max_bandwidth) {
1422         dest->max_bandwidth = params->max_bandwidth;
1423     }
1424
1425     if (params->has_downtime_limit) {
1426         dest->downtime_limit = params->downtime_limit;
1427     }
1428
1429     if (params->has_x_checkpoint_delay) {
1430         dest->x_checkpoint_delay = params->x_checkpoint_delay;
1431     }
1432
1433     if (params->has_block_incremental) {
1434         dest->block_incremental = params->block_incremental;
1435     }
1436     if (params->has_multifd_channels) {
1437         dest->multifd_channels = params->multifd_channels;
1438     }
1439     if (params->has_multifd_compression) {
1440         dest->multifd_compression = params->multifd_compression;
1441     }
1442     if (params->has_xbzrle_cache_size) {
1443         dest->xbzrle_cache_size = params->xbzrle_cache_size;
1444     }
1445     if (params->has_max_postcopy_bandwidth) {
1446         dest->max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1447     }
1448     if (params->has_max_cpu_throttle) {
1449         dest->max_cpu_throttle = params->max_cpu_throttle;
1450     }
1451     if (params->has_announce_initial) {
1452         dest->announce_initial = params->announce_initial;
1453     }
1454     if (params->has_announce_max) {
1455         dest->announce_max = params->announce_max;
1456     }
1457     if (params->has_announce_rounds) {
1458         dest->announce_rounds = params->announce_rounds;
1459     }
1460     if (params->has_announce_step) {
1461         dest->announce_step = params->announce_step;
1462     }
1463
1464     if (params->has_block_bitmap_mapping) {
1465         dest->has_block_bitmap_mapping = true;
1466         dest->block_bitmap_mapping = params->block_bitmap_mapping;
1467     }
1468 }
1469
1470 static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
1471 {
1472     MigrationState *s = migrate_get_current();
1473
1474     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1475
1476     if (params->has_compress_level) {
1477         s->parameters.compress_level = params->compress_level;
1478     }
1479
1480     if (params->has_compress_threads) {
1481         s->parameters.compress_threads = params->compress_threads;
1482     }
1483
1484     if (params->has_compress_wait_thread) {
1485         s->parameters.compress_wait_thread = params->compress_wait_thread;
1486     }
1487
1488     if (params->has_decompress_threads) {
1489         s->parameters.decompress_threads = params->decompress_threads;
1490     }
1491
1492     if (params->has_throttle_trigger_threshold) {
1493         s->parameters.throttle_trigger_threshold = params->throttle_trigger_threshold;
1494     }
1495
1496     if (params->has_cpu_throttle_initial) {
1497         s->parameters.cpu_throttle_initial = params->cpu_throttle_initial;
1498     }
1499
1500     if (params->has_cpu_throttle_increment) {
1501         s->parameters.cpu_throttle_increment = params->cpu_throttle_increment;
1502     }
1503
1504     if (params->has_cpu_throttle_tailslow) {
1505         s->parameters.cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1506     }
1507
1508     if (params->has_tls_creds) {
1509         g_free(s->parameters.tls_creds);
1510         assert(params->tls_creds->type == QTYPE_QSTRING);
1511         s->parameters.tls_creds = g_strdup(params->tls_creds->u.s);
1512     }
1513
1514     if (params->has_tls_hostname) {
1515         g_free(s->parameters.tls_hostname);
1516         assert(params->tls_hostname->type == QTYPE_QSTRING);
1517         s->parameters.tls_hostname = g_strdup(params->tls_hostname->u.s);
1518     }
1519
1520     if (params->has_tls_authz) {
1521         g_free(s->parameters.tls_authz);
1522         assert(params->tls_authz->type == QTYPE_QSTRING);
1523         s->parameters.tls_authz = g_strdup(params->tls_authz->u.s);
1524     }
1525
1526     if (params->has_max_bandwidth) {
1527         s->parameters.max_bandwidth = params->max_bandwidth;
1528         if (s->to_dst_file && !migration_in_postcopy()) {
1529             qemu_file_set_rate_limit(s->to_dst_file,
1530                                 s->parameters.max_bandwidth / XFER_LIMIT_RATIO);
1531         }
1532     }
1533
1534     if (params->has_downtime_limit) {
1535         s->parameters.downtime_limit = params->downtime_limit;
1536     }
1537
1538     if (params->has_x_checkpoint_delay) {
1539         s->parameters.x_checkpoint_delay = params->x_checkpoint_delay;
1540         if (migration_in_colo_state()) {
1541             colo_checkpoint_notify(s);
1542         }
1543     }
1544
1545     if (params->has_block_incremental) {
1546         s->parameters.block_incremental = params->block_incremental;
1547     }
1548     if (params->has_multifd_channels) {
1549         s->parameters.multifd_channels = params->multifd_channels;
1550     }
1551     if (params->has_multifd_compression) {
1552         s->parameters.multifd_compression = params->multifd_compression;
1553     }
1554     if (params->has_xbzrle_cache_size) {
1555         s->parameters.xbzrle_cache_size = params->xbzrle_cache_size;
1556         xbzrle_cache_resize(params->xbzrle_cache_size, errp);
1557     }
1558     if (params->has_max_postcopy_bandwidth) {
1559         s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1560         if (s->to_dst_file && migration_in_postcopy()) {
1561             qemu_file_set_rate_limit(s->to_dst_file,
1562                     s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO);
1563         }
1564     }
1565     if (params->has_max_cpu_throttle) {
1566         s->parameters.max_cpu_throttle = params->max_cpu_throttle;
1567     }
1568     if (params->has_announce_initial) {
1569         s->parameters.announce_initial = params->announce_initial;
1570     }
1571     if (params->has_announce_max) {
1572         s->parameters.announce_max = params->announce_max;
1573     }
1574     if (params->has_announce_rounds) {
1575         s->parameters.announce_rounds = params->announce_rounds;
1576     }
1577     if (params->has_announce_step) {
1578         s->parameters.announce_step = params->announce_step;
1579     }
1580
1581     if (params->has_block_bitmap_mapping) {
1582         qapi_free_BitmapMigrationNodeAliasList(
1583             s->parameters.block_bitmap_mapping);
1584
1585         s->parameters.has_block_bitmap_mapping = true;
1586         s->parameters.block_bitmap_mapping =
1587             QAPI_CLONE(BitmapMigrationNodeAliasList,
1588                        params->block_bitmap_mapping);
1589     }
1590 }
1591
1592 void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
1593 {
1594     MigrationParameters tmp;
1595
1596     /* TODO Rewrite "" to null instead */
1597     if (params->has_tls_creds
1598         && params->tls_creds->type == QTYPE_QNULL) {
1599         qobject_unref(params->tls_creds->u.n);
1600         params->tls_creds->type = QTYPE_QSTRING;
1601         params->tls_creds->u.s = strdup("");
1602     }
1603     /* TODO Rewrite "" to null instead */
1604     if (params->has_tls_hostname
1605         && params->tls_hostname->type == QTYPE_QNULL) {
1606         qobject_unref(params->tls_hostname->u.n);
1607         params->tls_hostname->type = QTYPE_QSTRING;
1608         params->tls_hostname->u.s = strdup("");
1609     }
1610
1611     migrate_params_test_apply(params, &tmp);
1612
1613     if (!migrate_params_check(&tmp, errp)) {
1614         /* Invalid parameter */
1615         return;
1616     }
1617
1618     migrate_params_apply(params, errp);
1619 }
1620
1621
1622 void qmp_migrate_start_postcopy(Error **errp)
1623 {
1624     MigrationState *s = migrate_get_current();
1625
1626     if (!migrate_postcopy()) {
1627         error_setg(errp, "Enable postcopy with migrate_set_capability before"
1628                          " the start of migration");
1629         return;
1630     }
1631
1632     if (s->state == MIGRATION_STATUS_NONE) {
1633         error_setg(errp, "Postcopy must be started after migration has been"
1634                          " started");
1635         return;
1636     }
1637     /*
1638      * we don't error if migration has finished since that would be racy
1639      * with issuing this command.
1640      */
1641     qatomic_set(&s->start_postcopy, true);
1642 }
1643
1644 /* shared migration helpers */
1645
1646 void migrate_set_state(int *state, int old_state, int new_state)
1647 {
1648     assert(new_state < MIGRATION_STATUS__MAX);
1649     if (qatomic_cmpxchg(state, old_state, new_state) == old_state) {
1650         trace_migrate_set_state(MigrationStatus_str(new_state));
1651         migrate_generate_event(new_state);
1652     }
1653 }
1654
1655 static MigrationCapabilityStatus *migrate_cap_add(MigrationCapability index,
1656                                                   bool state)
1657 {
1658     MigrationCapabilityStatus *cap;
1659
1660     cap = g_new0(MigrationCapabilityStatus, 1);
1661     cap->capability = index;
1662     cap->state = state;
1663
1664     return cap;
1665 }
1666
1667 void migrate_set_block_enabled(bool value, Error **errp)
1668 {
1669     MigrationCapabilityStatusList *cap = NULL;
1670
1671     QAPI_LIST_PREPEND(cap, migrate_cap_add(MIGRATION_CAPABILITY_BLOCK, value));
1672     qmp_migrate_set_capabilities(cap, errp);
1673     qapi_free_MigrationCapabilityStatusList(cap);
1674 }
1675
1676 static void migrate_set_block_incremental(MigrationState *s, bool value)
1677 {
1678     s->parameters.block_incremental = value;
1679 }
1680
1681 static void block_cleanup_parameters(MigrationState *s)
1682 {
1683     if (s->must_remove_block_options) {
1684         /* setting to false can never fail */
1685         migrate_set_block_enabled(false, &error_abort);
1686         migrate_set_block_incremental(s, false);
1687         s->must_remove_block_options = false;
1688     }
1689 }
1690
1691 static void migrate_fd_cleanup(MigrationState *s)
1692 {
1693     qemu_bh_delete(s->cleanup_bh);
1694     s->cleanup_bh = NULL;
1695
1696     qemu_savevm_state_cleanup();
1697
1698     if (s->to_dst_file) {
1699         QEMUFile *tmp;
1700
1701         trace_migrate_fd_cleanup();
1702         qemu_mutex_unlock_iothread();
1703         if (s->migration_thread_running) {
1704             qemu_thread_join(&s->thread);
1705             s->migration_thread_running = false;
1706         }
1707         qemu_mutex_lock_iothread();
1708
1709         multifd_save_cleanup();
1710         qemu_mutex_lock(&s->qemu_file_lock);
1711         tmp = s->to_dst_file;
1712         s->to_dst_file = NULL;
1713         qemu_mutex_unlock(&s->qemu_file_lock);
1714         /*
1715          * Close the file handle without the lock to make sure the
1716          * critical section won't block for long.
1717          */
1718         qemu_fclose(tmp);
1719     }
1720
1721     assert(!migration_is_active(s));
1722
1723     if (s->state == MIGRATION_STATUS_CANCELLING) {
1724         migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING,
1725                           MIGRATION_STATUS_CANCELLED);
1726     }
1727
1728     if (s->error) {
1729         /* It is used on info migrate.  We can't free it */
1730         error_report_err(error_copy(s->error));
1731     }
1732     notifier_list_notify(&migration_state_notifiers, s);
1733     block_cleanup_parameters(s);
1734 }
1735
1736 static void migrate_fd_cleanup_schedule(MigrationState *s)
1737 {
1738     /*
1739      * Ref the state for bh, because it may be called when
1740      * there're already no other refs
1741      */
1742     object_ref(OBJECT(s));
1743     qemu_bh_schedule(s->cleanup_bh);
1744 }
1745
1746 static void migrate_fd_cleanup_bh(void *opaque)
1747 {
1748     MigrationState *s = opaque;
1749     migrate_fd_cleanup(s);
1750     object_unref(OBJECT(s));
1751 }
1752
1753 void migrate_set_error(MigrationState *s, const Error *error)
1754 {
1755     QEMU_LOCK_GUARD(&s->error_mutex);
1756     if (!s->error) {
1757         s->error = error_copy(error);
1758     }
1759 }
1760
1761 void migrate_fd_error(MigrationState *s, const Error *error)
1762 {
1763     trace_migrate_fd_error(error_get_pretty(error));
1764     assert(s->to_dst_file == NULL);
1765     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
1766                       MIGRATION_STATUS_FAILED);
1767     migrate_set_error(s, error);
1768 }
1769
1770 static void migrate_fd_cancel(MigrationState *s)
1771 {
1772     int old_state ;
1773     QEMUFile *f = migrate_get_current()->to_dst_file;
1774     trace_migrate_fd_cancel();
1775
1776     if (s->rp_state.from_dst_file) {
1777         /* shutdown the rp socket, so causing the rp thread to shutdown */
1778         qemu_file_shutdown(s->rp_state.from_dst_file);
1779     }
1780
1781     do {
1782         old_state = s->state;
1783         if (!migration_is_running(old_state)) {
1784             break;
1785         }
1786         /* If the migration is paused, kick it out of the pause */
1787         if (old_state == MIGRATION_STATUS_PRE_SWITCHOVER) {
1788             qemu_sem_post(&s->pause_sem);
1789         }
1790         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING);
1791     } while (s->state != MIGRATION_STATUS_CANCELLING);
1792
1793     /*
1794      * If we're unlucky the migration code might be stuck somewhere in a
1795      * send/write while the network has failed and is waiting to timeout;
1796      * if we've got shutdown(2) available then we can force it to quit.
1797      * The outgoing qemu file gets closed in migrate_fd_cleanup that is
1798      * called in a bh, so there is no race against this cancel.
1799      */
1800     if (s->state == MIGRATION_STATUS_CANCELLING && f) {
1801         qemu_file_shutdown(f);
1802     }
1803     if (s->state == MIGRATION_STATUS_CANCELLING && s->block_inactive) {
1804         Error *local_err = NULL;
1805
1806         bdrv_invalidate_cache_all(&local_err);
1807         if (local_err) {
1808             error_report_err(local_err);
1809         } else {
1810             s->block_inactive = false;
1811         }
1812     }
1813 }
1814
1815 void add_migration_state_change_notifier(Notifier *notify)
1816 {
1817     notifier_list_add(&migration_state_notifiers, notify);
1818 }
1819
1820 void remove_migration_state_change_notifier(Notifier *notify)
1821 {
1822     notifier_remove(notify);
1823 }
1824
1825 bool migration_in_setup(MigrationState *s)
1826 {
1827     return s->state == MIGRATION_STATUS_SETUP;
1828 }
1829
1830 bool migration_has_finished(MigrationState *s)
1831 {
1832     return s->state == MIGRATION_STATUS_COMPLETED;
1833 }
1834
1835 bool migration_has_failed(MigrationState *s)
1836 {
1837     return (s->state == MIGRATION_STATUS_CANCELLED ||
1838             s->state == MIGRATION_STATUS_FAILED);
1839 }
1840
1841 bool migration_in_postcopy(void)
1842 {
1843     MigrationState *s = migrate_get_current();
1844
1845     switch (s->state) {
1846     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1847     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1848     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1849         return true;
1850     default:
1851         return false;
1852     }
1853 }
1854
1855 bool migration_in_postcopy_after_devices(MigrationState *s)
1856 {
1857     return migration_in_postcopy() && s->postcopy_after_devices;
1858 }
1859
1860 bool migration_in_incoming_postcopy(void)
1861 {
1862     PostcopyState ps = postcopy_state_get();
1863
1864     return ps >= POSTCOPY_INCOMING_DISCARD && ps < POSTCOPY_INCOMING_END;
1865 }
1866
1867 bool migration_is_idle(void)
1868 {
1869     MigrationState *s = current_migration;
1870
1871     if (!s) {
1872         return true;
1873     }
1874
1875     switch (s->state) {
1876     case MIGRATION_STATUS_NONE:
1877     case MIGRATION_STATUS_CANCELLED:
1878     case MIGRATION_STATUS_COMPLETED:
1879     case MIGRATION_STATUS_FAILED:
1880         return true;
1881     case MIGRATION_STATUS_SETUP:
1882     case MIGRATION_STATUS_CANCELLING:
1883     case MIGRATION_STATUS_ACTIVE:
1884     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1885     case MIGRATION_STATUS_COLO:
1886     case MIGRATION_STATUS_PRE_SWITCHOVER:
1887     case MIGRATION_STATUS_DEVICE:
1888     case MIGRATION_STATUS_WAIT_UNPLUG:
1889         return false;
1890     case MIGRATION_STATUS__MAX:
1891         g_assert_not_reached();
1892     }
1893
1894     return false;
1895 }
1896
1897 bool migration_is_active(MigrationState *s)
1898 {
1899     return (s->state == MIGRATION_STATUS_ACTIVE ||
1900             s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
1901 }
1902
1903 void migrate_init(MigrationState *s)
1904 {
1905     /*
1906      * Reinitialise all migration state, except
1907      * parameters/capabilities that the user set, and
1908      * locks.
1909      */
1910     s->cleanup_bh = 0;
1911     s->to_dst_file = NULL;
1912     s->state = MIGRATION_STATUS_NONE;
1913     s->rp_state.from_dst_file = NULL;
1914     s->rp_state.error = false;
1915     s->mbps = 0.0;
1916     s->pages_per_second = 0.0;
1917     s->downtime = 0;
1918     s->expected_downtime = 0;
1919     s->setup_time = 0;
1920     s->start_postcopy = false;
1921     s->postcopy_after_devices = false;
1922     s->migration_thread_running = false;
1923     error_free(s->error);
1924     s->error = NULL;
1925     s->hostname = NULL;
1926
1927     migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
1928
1929     s->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1930     s->total_time = 0;
1931     s->vm_was_running = false;
1932     s->iteration_initial_bytes = 0;
1933     s->threshold_size = 0;
1934 }
1935
1936 static GSList *migration_blockers;
1937
1938 int migrate_add_blocker(Error *reason, Error **errp)
1939 {
1940     if (only_migratable) {
1941         error_propagate_prepend(errp, error_copy(reason),
1942                                 "disallowing migration blocker "
1943                                 "(--only-migratable) for: ");
1944         return -EACCES;
1945     }
1946
1947     if (migration_is_idle()) {
1948         migration_blockers = g_slist_prepend(migration_blockers, reason);
1949         return 0;
1950     }
1951
1952     error_propagate_prepend(errp, error_copy(reason),
1953                             "disallowing migration blocker "
1954                             "(migration in progress) for: ");
1955     return -EBUSY;
1956 }
1957
1958 void migrate_del_blocker(Error *reason)
1959 {
1960     migration_blockers = g_slist_remove(migration_blockers, reason);
1961 }
1962
1963 void qmp_migrate_incoming(const char *uri, Error **errp)
1964 {
1965     Error *local_err = NULL;
1966     static bool once = true;
1967
1968     if (!once) {
1969         error_setg(errp, "The incoming migration has already been started");
1970         return;
1971     }
1972     if (!runstate_check(RUN_STATE_INMIGRATE)) {
1973         error_setg(errp, "'-incoming' was not specified on the command line");
1974         return;
1975     }
1976
1977     qemu_start_incoming_migration(uri, &local_err);
1978
1979     if (local_err) {
1980         error_propagate(errp, local_err);
1981         return;
1982     }
1983
1984     once = false;
1985 }
1986
1987 void qmp_migrate_recover(const char *uri, Error **errp)
1988 {
1989     MigrationIncomingState *mis = migration_incoming_get_current();
1990
1991     if (mis->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
1992         error_setg(errp, "Migrate recover can only be run "
1993                    "when postcopy is paused.");
1994         return;
1995     }
1996
1997     if (qatomic_cmpxchg(&mis->postcopy_recover_triggered,
1998                        false, true) == true) {
1999         error_setg(errp, "Migrate recovery is triggered already");
2000         return;
2001     }
2002
2003     /*
2004      * Note that this call will never start a real migration; it will
2005      * only re-setup the migration stream and poke existing migration
2006      * to continue using that newly established channel.
2007      */
2008     qemu_start_incoming_migration(uri, errp);
2009 }
2010
2011 void qmp_migrate_pause(Error **errp)
2012 {
2013     MigrationState *ms = migrate_get_current();
2014     MigrationIncomingState *mis = migration_incoming_get_current();
2015     int ret;
2016
2017     if (ms->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2018         /* Source side, during postcopy */
2019         qemu_mutex_lock(&ms->qemu_file_lock);
2020         ret = qemu_file_shutdown(ms->to_dst_file);
2021         qemu_mutex_unlock(&ms->qemu_file_lock);
2022         if (ret) {
2023             error_setg(errp, "Failed to pause source migration");
2024         }
2025         return;
2026     }
2027
2028     if (mis->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2029         ret = qemu_file_shutdown(mis->from_src_file);
2030         if (ret) {
2031             error_setg(errp, "Failed to pause destination migration");
2032         }
2033         return;
2034     }
2035
2036     error_setg(errp, "migrate-pause is currently only supported "
2037                "during postcopy-active state");
2038 }
2039
2040 bool migration_is_blocked(Error **errp)
2041 {
2042     if (qemu_savevm_state_blocked(errp)) {
2043         return true;
2044     }
2045
2046     if (migration_blockers) {
2047         error_propagate(errp, error_copy(migration_blockers->data));
2048         return true;
2049     }
2050
2051     return false;
2052 }
2053
2054 /* Returns true if continue to migrate, or false if error detected */
2055 static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
2056                             bool resume, Error **errp)
2057 {
2058     Error *local_err = NULL;
2059
2060     if (resume) {
2061         if (s->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2062             error_setg(errp, "Cannot resume if there is no "
2063                        "paused migration");
2064             return false;
2065         }
2066
2067         /*
2068          * Postcopy recovery won't work well with release-ram
2069          * capability since release-ram will drop the page buffer as
2070          * long as the page is put into the send buffer.  So if there
2071          * is a network failure happened, any page buffers that have
2072          * not yet reached the destination VM but have already been
2073          * sent from the source VM will be lost forever.  Let's refuse
2074          * the client from resuming such a postcopy migration.
2075          * Luckily release-ram was designed to only be used when src
2076          * and destination VMs are on the same host, so it should be
2077          * fine.
2078          */
2079         if (migrate_release_ram()) {
2080             error_setg(errp, "Postcopy recovery cannot work "
2081                        "when release-ram capability is set");
2082             return false;
2083         }
2084
2085         /* This is a resume, skip init status */
2086         return true;
2087     }
2088
2089     if (migration_is_running(s->state)) {
2090         error_setg(errp, QERR_MIGRATION_ACTIVE);
2091         return false;
2092     }
2093
2094     if (runstate_check(RUN_STATE_INMIGRATE)) {
2095         error_setg(errp, "Guest is waiting for an incoming migration");
2096         return false;
2097     }
2098
2099     if (runstate_check(RUN_STATE_POSTMIGRATE)) {
2100         error_setg(errp, "Can't migrate the vm that was paused due to "
2101                    "previous migration");
2102         return false;
2103     }
2104
2105     if (migration_is_blocked(errp)) {
2106         return false;
2107     }
2108
2109     if (blk || blk_inc) {
2110         if (migrate_use_block() || migrate_use_block_incremental()) {
2111             error_setg(errp, "Command options are incompatible with "
2112                        "current migration capabilities");
2113             return false;
2114         }
2115         migrate_set_block_enabled(true, &local_err);
2116         if (local_err) {
2117             error_propagate(errp, local_err);
2118             return false;
2119         }
2120         s->must_remove_block_options = true;
2121     }
2122
2123     if (blk_inc) {
2124         migrate_set_block_incremental(s, true);
2125     }
2126
2127     migrate_init(s);
2128     /*
2129      * set ram_counters memory to zero for a
2130      * new migration
2131      */
2132     memset(&ram_counters, 0, sizeof(ram_counters));
2133
2134     return true;
2135 }
2136
2137 void qmp_migrate(const char *uri, bool has_blk, bool blk,
2138                  bool has_inc, bool inc, bool has_detach, bool detach,
2139                  bool has_resume, bool resume, Error **errp)
2140 {
2141     Error *local_err = NULL;
2142     MigrationState *s = migrate_get_current();
2143     const char *p = NULL;
2144
2145     if (!migrate_prepare(s, has_blk && blk, has_inc && inc,
2146                          has_resume && resume, errp)) {
2147         /* Error detected, put into errp */
2148         return;
2149     }
2150
2151     if (strstart(uri, "tcp:", &p) ||
2152         strstart(uri, "unix:", NULL) ||
2153         strstart(uri, "vsock:", NULL)) {
2154         socket_start_outgoing_migration(s, p ? p : uri, &local_err);
2155 #ifdef CONFIG_RDMA
2156     } else if (strstart(uri, "rdma:", &p)) {
2157         rdma_start_outgoing_migration(s, p, &local_err);
2158 #endif
2159     } else if (strstart(uri, "exec:", &p)) {
2160         exec_start_outgoing_migration(s, p, &local_err);
2161     } else if (strstart(uri, "fd:", &p)) {
2162         fd_start_outgoing_migration(s, p, &local_err);
2163     } else {
2164         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri",
2165                    "a valid migration protocol");
2166         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
2167                           MIGRATION_STATUS_FAILED);
2168         block_cleanup_parameters(s);
2169         return;
2170     }
2171
2172     if (local_err) {
2173         migrate_fd_error(s, local_err);
2174         error_propagate(errp, local_err);
2175         return;
2176     }
2177 }
2178
2179 void qmp_migrate_cancel(Error **errp)
2180 {
2181     migrate_fd_cancel(migrate_get_current());
2182 }
2183
2184 void qmp_migrate_continue(MigrationStatus state, Error **errp)
2185 {
2186     MigrationState *s = migrate_get_current();
2187     if (s->state != state) {
2188         error_setg(errp,  "Migration not in expected state: %s",
2189                    MigrationStatus_str(s->state));
2190         return;
2191     }
2192     qemu_sem_post(&s->pause_sem);
2193 }
2194
2195 void qmp_migrate_set_cache_size(int64_t value, Error **errp)
2196 {
2197     MigrateSetParameters p = {
2198         .has_xbzrle_cache_size = true,
2199         .xbzrle_cache_size = value,
2200     };
2201
2202     qmp_migrate_set_parameters(&p, errp);
2203 }
2204
2205 int64_t qmp_query_migrate_cache_size(Error **errp)
2206 {
2207     return migrate_xbzrle_cache_size();
2208 }
2209
2210 void qmp_migrate_set_speed(int64_t value, Error **errp)
2211 {
2212     MigrateSetParameters p = {
2213         .has_max_bandwidth = true,
2214         .max_bandwidth = value,
2215     };
2216
2217     qmp_migrate_set_parameters(&p, errp);
2218 }
2219
2220 void qmp_migrate_set_downtime(double value, Error **errp)
2221 {
2222     if (value < 0 || value > MAX_MIGRATE_DOWNTIME_SECONDS) {
2223         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
2224                    "downtime_limit",
2225                    "an integer in the range of 0 to "
2226                     stringify(MAX_MIGRATE_DOWNTIME_SECONDS)" seconds");
2227         return;
2228     }
2229
2230     value *= 1000; /* Convert to milliseconds */
2231
2232     MigrateSetParameters p = {
2233         .has_downtime_limit = true,
2234         .downtime_limit = (int64_t)value,
2235     };
2236
2237     qmp_migrate_set_parameters(&p, errp);
2238 }
2239
2240 bool migrate_release_ram(void)
2241 {
2242     MigrationState *s;
2243
2244     s = migrate_get_current();
2245
2246     return s->enabled_capabilities[MIGRATION_CAPABILITY_RELEASE_RAM];
2247 }
2248
2249 bool migrate_postcopy_ram(void)
2250 {
2251     MigrationState *s;
2252
2253     s = migrate_get_current();
2254
2255     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM];
2256 }
2257
2258 bool migrate_postcopy(void)
2259 {
2260     return migrate_postcopy_ram() || migrate_dirty_bitmaps();
2261 }
2262
2263 bool migrate_auto_converge(void)
2264 {
2265     MigrationState *s;
2266
2267     s = migrate_get_current();
2268
2269     return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
2270 }
2271
2272 bool migrate_zero_blocks(void)
2273 {
2274     MigrationState *s;
2275
2276     s = migrate_get_current();
2277
2278     return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
2279 }
2280
2281 bool migrate_postcopy_blocktime(void)
2282 {
2283     MigrationState *s;
2284
2285     s = migrate_get_current();
2286
2287     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME];
2288 }
2289
2290 bool migrate_use_compression(void)
2291 {
2292     MigrationState *s;
2293
2294     s = migrate_get_current();
2295
2296     return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS];
2297 }
2298
2299 int migrate_compress_level(void)
2300 {
2301     MigrationState *s;
2302
2303     s = migrate_get_current();
2304
2305     return s->parameters.compress_level;
2306 }
2307
2308 int migrate_compress_threads(void)
2309 {
2310     MigrationState *s;
2311
2312     s = migrate_get_current();
2313
2314     return s->parameters.compress_threads;
2315 }
2316
2317 int migrate_compress_wait_thread(void)
2318 {
2319     MigrationState *s;
2320
2321     s = migrate_get_current();
2322
2323     return s->parameters.compress_wait_thread;
2324 }
2325
2326 int migrate_decompress_threads(void)
2327 {
2328     MigrationState *s;
2329
2330     s = migrate_get_current();
2331
2332     return s->parameters.decompress_threads;
2333 }
2334
2335 bool migrate_dirty_bitmaps(void)
2336 {
2337     MigrationState *s;
2338
2339     s = migrate_get_current();
2340
2341     return s->enabled_capabilities[MIGRATION_CAPABILITY_DIRTY_BITMAPS];
2342 }
2343
2344 bool migrate_ignore_shared(void)
2345 {
2346     MigrationState *s;
2347
2348     s = migrate_get_current();
2349
2350     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_IGNORE_SHARED];
2351 }
2352
2353 bool migrate_validate_uuid(void)
2354 {
2355     MigrationState *s;
2356
2357     s = migrate_get_current();
2358
2359     return s->enabled_capabilities[MIGRATION_CAPABILITY_VALIDATE_UUID];
2360 }
2361
2362 bool migrate_use_events(void)
2363 {
2364     MigrationState *s;
2365
2366     s = migrate_get_current();
2367
2368     return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS];
2369 }
2370
2371 bool migrate_use_multifd(void)
2372 {
2373     MigrationState *s;
2374
2375     s = migrate_get_current();
2376
2377     return s->enabled_capabilities[MIGRATION_CAPABILITY_MULTIFD];
2378 }
2379
2380 bool migrate_pause_before_switchover(void)
2381 {
2382     MigrationState *s;
2383
2384     s = migrate_get_current();
2385
2386     return s->enabled_capabilities[
2387         MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER];
2388 }
2389
2390 int migrate_multifd_channels(void)
2391 {
2392     MigrationState *s;
2393
2394     s = migrate_get_current();
2395
2396     return s->parameters.multifd_channels;
2397 }
2398
2399 MultiFDCompression migrate_multifd_compression(void)
2400 {
2401     MigrationState *s;
2402
2403     s = migrate_get_current();
2404
2405     return s->parameters.multifd_compression;
2406 }
2407
2408 int migrate_multifd_zlib_level(void)
2409 {
2410     MigrationState *s;
2411
2412     s = migrate_get_current();
2413
2414     return s->parameters.multifd_zlib_level;
2415 }
2416
2417 int migrate_multifd_zstd_level(void)
2418 {
2419     MigrationState *s;
2420
2421     s = migrate_get_current();
2422
2423     return s->parameters.multifd_zstd_level;
2424 }
2425
2426 int migrate_use_xbzrle(void)
2427 {
2428     MigrationState *s;
2429
2430     s = migrate_get_current();
2431
2432     return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
2433 }
2434
2435 int64_t migrate_xbzrle_cache_size(void)
2436 {
2437     MigrationState *s;
2438
2439     s = migrate_get_current();
2440
2441     return s->parameters.xbzrle_cache_size;
2442 }
2443
2444 static int64_t migrate_max_postcopy_bandwidth(void)
2445 {
2446     MigrationState *s;
2447
2448     s = migrate_get_current();
2449
2450     return s->parameters.max_postcopy_bandwidth;
2451 }
2452
2453 bool migrate_use_block(void)
2454 {
2455     MigrationState *s;
2456
2457     s = migrate_get_current();
2458
2459     return s->enabled_capabilities[MIGRATION_CAPABILITY_BLOCK];
2460 }
2461
2462 bool migrate_use_return_path(void)
2463 {
2464     MigrationState *s;
2465
2466     s = migrate_get_current();
2467
2468     return s->enabled_capabilities[MIGRATION_CAPABILITY_RETURN_PATH];
2469 }
2470
2471 bool migrate_use_block_incremental(void)
2472 {
2473     MigrationState *s;
2474
2475     s = migrate_get_current();
2476
2477     return s->parameters.block_incremental;
2478 }
2479
2480 /* migration thread support */
2481 /*
2482  * Something bad happened to the RP stream, mark an error
2483  * The caller shall print or trace something to indicate why
2484  */
2485 static void mark_source_rp_bad(MigrationState *s)
2486 {
2487     s->rp_state.error = true;
2488 }
2489
2490 static struct rp_cmd_args {
2491     ssize_t     len; /* -1 = variable */
2492     const char *name;
2493 } rp_cmd_args[] = {
2494     [MIG_RP_MSG_INVALID]        = { .len = -1, .name = "INVALID" },
2495     [MIG_RP_MSG_SHUT]           = { .len =  4, .name = "SHUT" },
2496     [MIG_RP_MSG_PONG]           = { .len =  4, .name = "PONG" },
2497     [MIG_RP_MSG_REQ_PAGES]      = { .len = 12, .name = "REQ_PAGES" },
2498     [MIG_RP_MSG_REQ_PAGES_ID]   = { .len = -1, .name = "REQ_PAGES_ID" },
2499     [MIG_RP_MSG_RECV_BITMAP]    = { .len = -1, .name = "RECV_BITMAP" },
2500     [MIG_RP_MSG_RESUME_ACK]     = { .len =  4, .name = "RESUME_ACK" },
2501     [MIG_RP_MSG_MAX]            = { .len = -1, .name = "MAX" },
2502 };
2503
2504 /*
2505  * Process a request for pages received on the return path,
2506  * We're allowed to send more than requested (e.g. to round to our page size)
2507  * and we don't need to send pages that have already been sent.
2508  */
2509 static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
2510                                        ram_addr_t start, size_t len)
2511 {
2512     long our_host_ps = qemu_real_host_page_size;
2513
2514     trace_migrate_handle_rp_req_pages(rbname, start, len);
2515
2516     /*
2517      * Since we currently insist on matching page sizes, just sanity check
2518      * we're being asked for whole host pages.
2519      */
2520     if (start & (our_host_ps - 1) ||
2521        (len & (our_host_ps - 1))) {
2522         error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
2523                      " len: %zd", __func__, start, len);
2524         mark_source_rp_bad(ms);
2525         return;
2526     }
2527
2528     if (ram_save_queue_pages(rbname, start, len)) {
2529         mark_source_rp_bad(ms);
2530     }
2531 }
2532
2533 /* Return true to retry, false to quit */
2534 static bool postcopy_pause_return_path_thread(MigrationState *s)
2535 {
2536     trace_postcopy_pause_return_path();
2537
2538     qemu_sem_wait(&s->postcopy_pause_rp_sem);
2539
2540     trace_postcopy_pause_return_path_continued();
2541
2542     return true;
2543 }
2544
2545 static int migrate_handle_rp_recv_bitmap(MigrationState *s, char *block_name)
2546 {
2547     RAMBlock *block = qemu_ram_block_by_name(block_name);
2548
2549     if (!block) {
2550         error_report("%s: invalid block name '%s'", __func__, block_name);
2551         return -EINVAL;
2552     }
2553
2554     /* Fetch the received bitmap and refresh the dirty bitmap */
2555     return ram_dirty_bitmap_reload(s, block);
2556 }
2557
2558 static int migrate_handle_rp_resume_ack(MigrationState *s, uint32_t value)
2559 {
2560     trace_source_return_path_thread_resume_ack(value);
2561
2562     if (value != MIGRATION_RESUME_ACK_VALUE) {
2563         error_report("%s: illegal resume_ack value %"PRIu32,
2564                      __func__, value);
2565         return -1;
2566     }
2567
2568     /* Now both sides are active. */
2569     migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_RECOVER,
2570                       MIGRATION_STATUS_POSTCOPY_ACTIVE);
2571
2572     /* Notify send thread that time to continue send pages */
2573     qemu_sem_post(&s->rp_state.rp_sem);
2574
2575     return 0;
2576 }
2577
2578 /*
2579  * Handles messages sent on the return path towards the source VM
2580  *
2581  */
2582 static void *source_return_path_thread(void *opaque)
2583 {
2584     MigrationState *ms = opaque;
2585     QEMUFile *rp = ms->rp_state.from_dst_file;
2586     uint16_t header_len, header_type;
2587     uint8_t buf[512];
2588     uint32_t tmp32, sibling_error;
2589     ram_addr_t start = 0; /* =0 to silence warning */
2590     size_t  len = 0, expected_len;
2591     int res;
2592
2593     trace_source_return_path_thread_entry();
2594     rcu_register_thread();
2595
2596 retry:
2597     while (!ms->rp_state.error && !qemu_file_get_error(rp) &&
2598            migration_is_setup_or_active(ms->state)) {
2599         trace_source_return_path_thread_loop_top();
2600         header_type = qemu_get_be16(rp);
2601         header_len = qemu_get_be16(rp);
2602
2603         if (qemu_file_get_error(rp)) {
2604             mark_source_rp_bad(ms);
2605             goto out;
2606         }
2607
2608         if (header_type >= MIG_RP_MSG_MAX ||
2609             header_type == MIG_RP_MSG_INVALID) {
2610             error_report("RP: Received invalid message 0x%04x length 0x%04x",
2611                          header_type, header_len);
2612             mark_source_rp_bad(ms);
2613             goto out;
2614         }
2615
2616         if ((rp_cmd_args[header_type].len != -1 &&
2617             header_len != rp_cmd_args[header_type].len) ||
2618             header_len > sizeof(buf)) {
2619             error_report("RP: Received '%s' message (0x%04x) with"
2620                          "incorrect length %d expecting %zu",
2621                          rp_cmd_args[header_type].name, header_type, header_len,
2622                          (size_t)rp_cmd_args[header_type].len);
2623             mark_source_rp_bad(ms);
2624             goto out;
2625         }
2626
2627         /* We know we've got a valid header by this point */
2628         res = qemu_get_buffer(rp, buf, header_len);
2629         if (res != header_len) {
2630             error_report("RP: Failed reading data for message 0x%04x"
2631                          " read %d expected %d",
2632                          header_type, res, header_len);
2633             mark_source_rp_bad(ms);
2634             goto out;
2635         }
2636
2637         /* OK, we have the message and the data */
2638         switch (header_type) {
2639         case MIG_RP_MSG_SHUT:
2640             sibling_error = ldl_be_p(buf);
2641             trace_source_return_path_thread_shut(sibling_error);
2642             if (sibling_error) {
2643                 error_report("RP: Sibling indicated error %d", sibling_error);
2644                 mark_source_rp_bad(ms);
2645             }
2646             /*
2647              * We'll let the main thread deal with closing the RP
2648              * we could do a shutdown(2) on it, but we're the only user
2649              * anyway, so there's nothing gained.
2650              */
2651             goto out;
2652
2653         case MIG_RP_MSG_PONG:
2654             tmp32 = ldl_be_p(buf);
2655             trace_source_return_path_thread_pong(tmp32);
2656             break;
2657
2658         case MIG_RP_MSG_REQ_PAGES:
2659             start = ldq_be_p(buf);
2660             len = ldl_be_p(buf + 8);
2661             migrate_handle_rp_req_pages(ms, NULL, start, len);
2662             break;
2663
2664         case MIG_RP_MSG_REQ_PAGES_ID:
2665             expected_len = 12 + 1; /* header + termination */
2666
2667             if (header_len >= expected_len) {
2668                 start = ldq_be_p(buf);
2669                 len = ldl_be_p(buf + 8);
2670                 /* Now we expect an idstr */
2671                 tmp32 = buf[12]; /* Length of the following idstr */
2672                 buf[13 + tmp32] = '\0';
2673                 expected_len += tmp32;
2674             }
2675             if (header_len != expected_len) {
2676                 error_report("RP: Req_Page_id with length %d expecting %zd",
2677                              header_len, expected_len);
2678                 mark_source_rp_bad(ms);
2679                 goto out;
2680             }
2681             migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len);
2682             break;
2683
2684         case MIG_RP_MSG_RECV_BITMAP:
2685             if (header_len < 1) {
2686                 error_report("%s: missing block name", __func__);
2687                 mark_source_rp_bad(ms);
2688                 goto out;
2689             }
2690             /* Format: len (1B) + idstr (<255B). This ends the idstr. */
2691             buf[buf[0] + 1] = '\0';
2692             if (migrate_handle_rp_recv_bitmap(ms, (char *)(buf + 1))) {
2693                 mark_source_rp_bad(ms);
2694                 goto out;
2695             }
2696             break;
2697
2698         case MIG_RP_MSG_RESUME_ACK:
2699             tmp32 = ldl_be_p(buf);
2700             if (migrate_handle_rp_resume_ack(ms, tmp32)) {
2701                 mark_source_rp_bad(ms);
2702                 goto out;
2703             }
2704             break;
2705
2706         default:
2707             break;
2708         }
2709     }
2710
2711 out:
2712     res = qemu_file_get_error(rp);
2713     if (res) {
2714         if (res == -EIO && migration_in_postcopy()) {
2715             /*
2716              * Maybe there is something we can do: it looks like a
2717              * network down issue, and we pause for a recovery.
2718              */
2719             if (postcopy_pause_return_path_thread(ms)) {
2720                 /* Reload rp, reset the rest */
2721                 if (rp != ms->rp_state.from_dst_file) {
2722                     qemu_fclose(rp);
2723                     rp = ms->rp_state.from_dst_file;
2724                 }
2725                 ms->rp_state.error = false;
2726                 goto retry;
2727             }
2728         }
2729
2730         trace_source_return_path_thread_bad_end();
2731         mark_source_rp_bad(ms);
2732     }
2733
2734     trace_source_return_path_thread_end();
2735     ms->rp_state.from_dst_file = NULL;
2736     qemu_fclose(rp);
2737     rcu_unregister_thread();
2738     return NULL;
2739 }
2740
2741 static int open_return_path_on_source(MigrationState *ms,
2742                                       bool create_thread)
2743 {
2744
2745     ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file);
2746     if (!ms->rp_state.from_dst_file) {
2747         return -1;
2748     }
2749
2750     trace_open_return_path_on_source();
2751
2752     if (!create_thread) {
2753         /* We're done */
2754         return 0;
2755     }
2756
2757     qemu_thread_create(&ms->rp_state.rp_thread, "return path",
2758                        source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
2759
2760     trace_open_return_path_on_source_continue();
2761
2762     return 0;
2763 }
2764
2765 /* Returns 0 if the RP was ok, otherwise there was an error on the RP */
2766 static int await_return_path_close_on_source(MigrationState *ms)
2767 {
2768     /*
2769      * If this is a normal exit then the destination will send a SHUT and the
2770      * rp_thread will exit, however if there's an error we need to cause
2771      * it to exit.
2772      */
2773     if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) {
2774         /*
2775          * shutdown(2), if we have it, will cause it to unblock if it's stuck
2776          * waiting for the destination.
2777          */
2778         qemu_file_shutdown(ms->rp_state.from_dst_file);
2779         mark_source_rp_bad(ms);
2780     }
2781     trace_await_return_path_close_on_source_joining();
2782     qemu_thread_join(&ms->rp_state.rp_thread);
2783     trace_await_return_path_close_on_source_close();
2784     return ms->rp_state.error;
2785 }
2786
2787 /*
2788  * Switch from normal iteration to postcopy
2789  * Returns non-0 on error
2790  */
2791 static int postcopy_start(MigrationState *ms)
2792 {
2793     int ret;
2794     QIOChannelBuffer *bioc;
2795     QEMUFile *fb;
2796     int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2797     int64_t bandwidth = migrate_max_postcopy_bandwidth();
2798     bool restart_block = false;
2799     int cur_state = MIGRATION_STATUS_ACTIVE;
2800     if (!migrate_pause_before_switchover()) {
2801         migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
2802                           MIGRATION_STATUS_POSTCOPY_ACTIVE);
2803     }
2804
2805     trace_postcopy_start();
2806     qemu_mutex_lock_iothread();
2807     trace_postcopy_start_set_run();
2808
2809     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
2810     global_state_store();
2811     ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
2812     if (ret < 0) {
2813         goto fail;
2814     }
2815
2816     ret = migration_maybe_pause(ms, &cur_state,
2817                                 MIGRATION_STATUS_POSTCOPY_ACTIVE);
2818     if (ret < 0) {
2819         goto fail;
2820     }
2821
2822     ret = bdrv_inactivate_all();
2823     if (ret < 0) {
2824         goto fail;
2825     }
2826     restart_block = true;
2827
2828     /*
2829      * Cause any non-postcopiable, but iterative devices to
2830      * send out their final data.
2831      */
2832     qemu_savevm_state_complete_precopy(ms->to_dst_file, true, false);
2833
2834     /*
2835      * in Finish migrate and with the io-lock held everything should
2836      * be quiet, but we've potentially still got dirty pages and we
2837      * need to tell the destination to throw any pages it's already received
2838      * that are dirty
2839      */
2840     if (migrate_postcopy_ram()) {
2841         if (ram_postcopy_send_discard_bitmap(ms)) {
2842             error_report("postcopy send discard bitmap failed");
2843             goto fail;
2844         }
2845     }
2846
2847     /*
2848      * send rest of state - note things that are doing postcopy
2849      * will notice we're in POSTCOPY_ACTIVE and not actually
2850      * wrap their state up here
2851      */
2852     /* 0 max-postcopy-bandwidth means unlimited */
2853     if (!bandwidth) {
2854         qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX);
2855     } else {
2856         qemu_file_set_rate_limit(ms->to_dst_file, bandwidth / XFER_LIMIT_RATIO);
2857     }
2858     if (migrate_postcopy_ram()) {
2859         /* Ping just for debugging, helps line traces up */
2860         qemu_savevm_send_ping(ms->to_dst_file, 2);
2861     }
2862
2863     /*
2864      * While loading the device state we may trigger page transfer
2865      * requests and the fd must be free to process those, and thus
2866      * the destination must read the whole device state off the fd before
2867      * it starts processing it.  Unfortunately the ad-hoc migration format
2868      * doesn't allow the destination to know the size to read without fully
2869      * parsing it through each devices load-state code (especially the open
2870      * coded devices that use get/put).
2871      * So we wrap the device state up in a package with a length at the start;
2872      * to do this we use a qemu_buf to hold the whole of the device state.
2873      */
2874     bioc = qio_channel_buffer_new(4096);
2875     qio_channel_set_name(QIO_CHANNEL(bioc), "migration-postcopy-buffer");
2876     fb = qemu_fopen_channel_output(QIO_CHANNEL(bioc));
2877     object_unref(OBJECT(bioc));
2878
2879     /*
2880      * Make sure the receiver can get incoming pages before we send the rest
2881      * of the state
2882      */
2883     qemu_savevm_send_postcopy_listen(fb);
2884
2885     qemu_savevm_state_complete_precopy(fb, false, false);
2886     if (migrate_postcopy_ram()) {
2887         qemu_savevm_send_ping(fb, 3);
2888     }
2889
2890     qemu_savevm_send_postcopy_run(fb);
2891
2892     /* <><> end of stuff going into the package */
2893
2894     /* Last point of recovery; as soon as we send the package the destination
2895      * can open devices and potentially start running.
2896      * Lets just check again we've not got any errors.
2897      */
2898     ret = qemu_file_get_error(ms->to_dst_file);
2899     if (ret) {
2900         error_report("postcopy_start: Migration stream errored (pre package)");
2901         goto fail_closefb;
2902     }
2903
2904     restart_block = false;
2905
2906     /* Now send that blob */
2907     if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
2908         goto fail_closefb;
2909     }
2910     qemu_fclose(fb);
2911
2912     /* Send a notify to give a chance for anything that needs to happen
2913      * at the transition to postcopy and after the device state; in particular
2914      * spice needs to trigger a transition now
2915      */
2916     ms->postcopy_after_devices = true;
2917     notifier_list_notify(&migration_state_notifiers, ms);
2918
2919     ms->downtime =  qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop;
2920
2921     qemu_mutex_unlock_iothread();
2922
2923     if (migrate_postcopy_ram()) {
2924         /*
2925          * Although this ping is just for debug, it could potentially be
2926          * used for getting a better measurement of downtime at the source.
2927          */
2928         qemu_savevm_send_ping(ms->to_dst_file, 4);
2929     }
2930
2931     if (migrate_release_ram()) {
2932         ram_postcopy_migrated_memory_release(ms);
2933     }
2934
2935     ret = qemu_file_get_error(ms->to_dst_file);
2936     if (ret) {
2937         error_report("postcopy_start: Migration stream errored");
2938         migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
2939                               MIGRATION_STATUS_FAILED);
2940     }
2941
2942     return ret;
2943
2944 fail_closefb:
2945     qemu_fclose(fb);
2946 fail:
2947     migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
2948                           MIGRATION_STATUS_FAILED);
2949     if (restart_block) {
2950         /* A failure happened early enough that we know the destination hasn't
2951          * accessed block devices, so we're safe to recover.
2952          */
2953         Error *local_err = NULL;
2954
2955         bdrv_invalidate_cache_all(&local_err);
2956         if (local_err) {
2957             error_report_err(local_err);
2958         }
2959     }
2960     qemu_mutex_unlock_iothread();
2961     return -1;
2962 }
2963
2964 /**
2965  * migration_maybe_pause: Pause if required to by
2966  * migrate_pause_before_switchover called with the iothread locked
2967  * Returns: 0 on success
2968  */
2969 static int migration_maybe_pause(MigrationState *s,
2970                                  int *current_active_state,
2971                                  int new_state)
2972 {
2973     if (!migrate_pause_before_switchover()) {
2974         return 0;
2975     }
2976
2977     /* Since leaving this state is not atomic with posting the semaphore
2978      * it's possible that someone could have issued multiple migrate_continue
2979      * and the semaphore is incorrectly positive at this point;
2980      * the docs say it's undefined to reinit a semaphore that's already
2981      * init'd, so use timedwait to eat up any existing posts.
2982      */
2983     while (qemu_sem_timedwait(&s->pause_sem, 1) == 0) {
2984         /* This block intentionally left blank */
2985     }
2986
2987     /*
2988      * If the migration is cancelled when it is in the completion phase,
2989      * the migration state is set to MIGRATION_STATUS_CANCELLING.
2990      * So we don't need to wait a semaphore, otherwise we would always
2991      * wait for the 'pause_sem' semaphore.
2992      */
2993     if (s->state != MIGRATION_STATUS_CANCELLING) {
2994         qemu_mutex_unlock_iothread();
2995         migrate_set_state(&s->state, *current_active_state,
2996                           MIGRATION_STATUS_PRE_SWITCHOVER);
2997         qemu_sem_wait(&s->pause_sem);
2998         migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER,
2999                           new_state);
3000         *current_active_state = new_state;
3001         qemu_mutex_lock_iothread();
3002     }
3003
3004     return s->state == new_state ? 0 : -EINVAL;
3005 }
3006
3007 /**
3008  * migration_completion: Used by migration_thread when there's not much left.
3009  *   The caller 'breaks' the loop when this returns.
3010  *
3011  * @s: Current migration state
3012  */
3013 static void migration_completion(MigrationState *s)
3014 {
3015     int ret;
3016     int current_active_state = s->state;
3017
3018     if (s->state == MIGRATION_STATUS_ACTIVE) {
3019         qemu_mutex_lock_iothread();
3020         s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3021         qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3022         s->vm_was_running = runstate_is_running();
3023         ret = global_state_store();
3024
3025         if (!ret) {
3026             bool inactivate = !migrate_colo_enabled();
3027             ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
3028             if (ret >= 0) {
3029                 ret = migration_maybe_pause(s, &current_active_state,
3030                                             MIGRATION_STATUS_DEVICE);
3031             }
3032             if (ret >= 0) {
3033                 qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3034                 ret = qemu_savevm_state_complete_precopy(s->to_dst_file, false,
3035                                                          inactivate);
3036             }
3037             if (inactivate && ret >= 0) {
3038                 s->block_inactive = true;
3039             }
3040         }
3041         qemu_mutex_unlock_iothread();
3042
3043         if (ret < 0) {
3044             goto fail;
3045         }
3046     } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3047         trace_migration_completion_postcopy_end();
3048
3049         qemu_savevm_state_complete_postcopy(s->to_dst_file);
3050         trace_migration_completion_postcopy_end_after_complete();
3051     } else if (s->state == MIGRATION_STATUS_CANCELLING) {
3052         goto fail;
3053     }
3054
3055     /*
3056      * If rp was opened we must clean up the thread before
3057      * cleaning everything else up (since if there are no failures
3058      * it will wait for the destination to send it's status in
3059      * a SHUT command).
3060      */
3061     if (s->rp_state.from_dst_file) {
3062         int rp_error;
3063         trace_migration_return_path_end_before();
3064         rp_error = await_return_path_close_on_source(s);
3065         trace_migration_return_path_end_after(rp_error);
3066         if (rp_error) {
3067             goto fail_invalidate;
3068         }
3069     }
3070
3071     if (qemu_file_get_error(s->to_dst_file)) {
3072         trace_migration_completion_file_err();
3073         goto fail_invalidate;
3074     }
3075
3076     if (!migrate_colo_enabled()) {
3077         migrate_set_state(&s->state, current_active_state,
3078                           MIGRATION_STATUS_COMPLETED);
3079     }
3080
3081     return;
3082
3083 fail_invalidate:
3084     /* If not doing postcopy, vm_start() will be called: let's regain
3085      * control on images.
3086      */
3087     if (s->state == MIGRATION_STATUS_ACTIVE ||
3088         s->state == MIGRATION_STATUS_DEVICE) {
3089         Error *local_err = NULL;
3090
3091         qemu_mutex_lock_iothread();
3092         bdrv_invalidate_cache_all(&local_err);
3093         if (local_err) {
3094             error_report_err(local_err);
3095         } else {
3096             s->block_inactive = false;
3097         }
3098         qemu_mutex_unlock_iothread();
3099     }
3100
3101 fail:
3102     migrate_set_state(&s->state, current_active_state,
3103                       MIGRATION_STATUS_FAILED);
3104 }
3105
3106 bool migrate_colo_enabled(void)
3107 {
3108     MigrationState *s = migrate_get_current();
3109     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_COLO];
3110 }
3111
3112 typedef enum MigThrError {
3113     /* No error detected */
3114     MIG_THR_ERR_NONE = 0,
3115     /* Detected error, but resumed successfully */
3116     MIG_THR_ERR_RECOVERED = 1,
3117     /* Detected fatal error, need to exit */
3118     MIG_THR_ERR_FATAL = 2,
3119 } MigThrError;
3120
3121 static int postcopy_resume_handshake(MigrationState *s)
3122 {
3123     qemu_savevm_send_postcopy_resume(s->to_dst_file);
3124
3125     while (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3126         qemu_sem_wait(&s->rp_state.rp_sem);
3127     }
3128
3129     if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3130         return 0;
3131     }
3132
3133     return -1;
3134 }
3135
3136 /* Return zero if success, or <0 for error */
3137 static int postcopy_do_resume(MigrationState *s)
3138 {
3139     int ret;
3140
3141     /*
3142      * Call all the resume_prepare() hooks, so that modules can be
3143      * ready for the migration resume.
3144      */
3145     ret = qemu_savevm_state_resume_prepare(s);
3146     if (ret) {
3147         error_report("%s: resume_prepare() failure detected: %d",
3148                      __func__, ret);
3149         return ret;
3150     }
3151
3152     /*
3153      * Last handshake with destination on the resume (destination will
3154      * switch to postcopy-active afterwards)
3155      */
3156     ret = postcopy_resume_handshake(s);
3157     if (ret) {
3158         error_report("%s: handshake failed: %d", __func__, ret);
3159         return ret;
3160     }
3161
3162     return 0;
3163 }
3164
3165 /*
3166  * We don't return until we are in a safe state to continue current
3167  * postcopy migration.  Returns MIG_THR_ERR_RECOVERED if recovered, or
3168  * MIG_THR_ERR_FATAL if unrecovery failure happened.
3169  */
3170 static MigThrError postcopy_pause(MigrationState *s)
3171 {
3172     assert(s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
3173
3174     while (true) {
3175         QEMUFile *file;
3176
3177         /* Current channel is possibly broken. Release it. */
3178         assert(s->to_dst_file);
3179         qemu_mutex_lock(&s->qemu_file_lock);
3180         file = s->to_dst_file;
3181         s->to_dst_file = NULL;
3182         qemu_mutex_unlock(&s->qemu_file_lock);
3183
3184         qemu_file_shutdown(file);
3185         qemu_fclose(file);
3186
3187         migrate_set_state(&s->state, s->state,
3188                           MIGRATION_STATUS_POSTCOPY_PAUSED);
3189
3190         error_report("Detected IO failure for postcopy. "
3191                      "Migration paused.");
3192
3193         /*
3194          * We wait until things fixed up. Then someone will setup the
3195          * status back for us.
3196          */
3197         while (s->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
3198             qemu_sem_wait(&s->postcopy_pause_sem);
3199         }
3200
3201         if (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3202             /* Woken up by a recover procedure. Give it a shot */
3203
3204             /*
3205              * Firstly, let's wake up the return path now, with a new
3206              * return path channel.
3207              */
3208             qemu_sem_post(&s->postcopy_pause_rp_sem);
3209
3210             /* Do the resume logic */
3211             if (postcopy_do_resume(s) == 0) {
3212                 /* Let's continue! */
3213                 trace_postcopy_pause_continued();
3214                 return MIG_THR_ERR_RECOVERED;
3215             } else {
3216                 /*
3217                  * Something wrong happened during the recovery, let's
3218                  * pause again. Pause is always better than throwing
3219                  * data away.
3220                  */
3221                 continue;
3222             }
3223         } else {
3224             /* This is not right... Time to quit. */
3225             return MIG_THR_ERR_FATAL;
3226         }
3227     }
3228 }
3229
3230 static MigThrError migration_detect_error(MigrationState *s)
3231 {
3232     int ret;
3233     int state = s->state;
3234     Error *local_error = NULL;
3235
3236     if (state == MIGRATION_STATUS_CANCELLING ||
3237         state == MIGRATION_STATUS_CANCELLED) {
3238         /* End the migration, but don't set the state to failed */
3239         return MIG_THR_ERR_FATAL;
3240     }
3241
3242     /* Try to detect any file errors */
3243     ret = qemu_file_get_error_obj(s->to_dst_file, &local_error);
3244     if (!ret) {
3245         /* Everything is fine */
3246         assert(!local_error);
3247         return MIG_THR_ERR_NONE;
3248     }
3249
3250     if (local_error) {
3251         migrate_set_error(s, local_error);
3252         error_free(local_error);
3253     }
3254
3255     if (state == MIGRATION_STATUS_POSTCOPY_ACTIVE && ret == -EIO) {
3256         /*
3257          * For postcopy, we allow the network to be down for a
3258          * while. After that, it can be continued by a
3259          * recovery phase.
3260          */
3261         return postcopy_pause(s);
3262     } else {
3263         /*
3264          * For precopy (or postcopy with error outside IO), we fail
3265          * with no time.
3266          */
3267         migrate_set_state(&s->state, state, MIGRATION_STATUS_FAILED);
3268         trace_migration_thread_file_err();
3269
3270         /* Time to stop the migration, now. */
3271         return MIG_THR_ERR_FATAL;
3272     }
3273 }
3274
3275 /* How many bytes have we transferred since the beginning of the migration */
3276 static uint64_t migration_total_bytes(MigrationState *s)
3277 {
3278     return qemu_ftell(s->to_dst_file) + ram_counters.multifd_bytes;
3279 }
3280
3281 static void migration_calculate_complete(MigrationState *s)
3282 {
3283     uint64_t bytes = migration_total_bytes(s);
3284     int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3285     int64_t transfer_time;
3286
3287     s->total_time = end_time - s->start_time;
3288     if (!s->downtime) {
3289         /*
3290          * It's still not set, so we are precopy migration.  For
3291          * postcopy, downtime is calculated during postcopy_start().
3292          */
3293         s->downtime = end_time - s->downtime_start;
3294     }
3295
3296     transfer_time = s->total_time - s->setup_time;
3297     if (transfer_time) {
3298         s->mbps = ((double) bytes * 8.0) / transfer_time / 1000;
3299     }
3300 }
3301
3302 static void update_iteration_initial_status(MigrationState *s)
3303 {
3304     /*
3305      * Update these three fields at the same time to avoid mismatch info lead
3306      * wrong speed calculation.
3307      */
3308     s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3309     s->iteration_initial_bytes = migration_total_bytes(s);
3310     s->iteration_initial_pages = ram_get_total_transferred_pages();
3311 }
3312
3313 static void migration_update_counters(MigrationState *s,
3314                                       int64_t current_time)
3315 {
3316     uint64_t transferred, transferred_pages, time_spent;
3317     uint64_t current_bytes; /* bytes transferred since the beginning */
3318     double bandwidth;
3319
3320     if (current_time < s->iteration_start_time + BUFFER_DELAY) {
3321         return;
3322     }
3323
3324     current_bytes = migration_total_bytes(s);
3325     transferred = current_bytes - s->iteration_initial_bytes;
3326     time_spent = current_time - s->iteration_start_time;
3327     bandwidth = (double)transferred / time_spent;
3328     s->threshold_size = bandwidth * s->parameters.downtime_limit;
3329
3330     s->mbps = (((double) transferred * 8.0) /
3331                ((double) time_spent / 1000.0)) / 1000.0 / 1000.0;
3332
3333     transferred_pages = ram_get_total_transferred_pages() -
3334                             s->iteration_initial_pages;
3335     s->pages_per_second = (double) transferred_pages /
3336                              (((double) time_spent / 1000.0));
3337
3338     /*
3339      * if we haven't sent anything, we don't want to
3340      * recalculate. 10000 is a small enough number for our purposes
3341      */
3342     if (ram_counters.dirty_pages_rate && transferred > 10000) {
3343         s->expected_downtime = ram_counters.remaining / bandwidth;
3344     }
3345
3346     qemu_file_reset_rate_limit(s->to_dst_file);
3347
3348     update_iteration_initial_status(s);
3349
3350     trace_migrate_transferred(transferred, time_spent,
3351                               bandwidth, s->threshold_size);
3352 }
3353
3354 /* Migration thread iteration status */
3355 typedef enum {
3356     MIG_ITERATE_RESUME,         /* Resume current iteration */
3357     MIG_ITERATE_SKIP,           /* Skip current iteration */
3358     MIG_ITERATE_BREAK,          /* Break the loop */
3359 } MigIterateState;
3360
3361 /*
3362  * Return true if continue to the next iteration directly, false
3363  * otherwise.
3364  */
3365 static MigIterateState migration_iteration_run(MigrationState *s)
3366 {
3367     uint64_t pending_size, pend_pre, pend_compat, pend_post;
3368     bool in_postcopy = s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE;
3369
3370     qemu_savevm_state_pending(s->to_dst_file, s->threshold_size, &pend_pre,
3371                               &pend_compat, &pend_post);
3372     pending_size = pend_pre + pend_compat + pend_post;
3373
3374     trace_migrate_pending(pending_size, s->threshold_size,
3375                           pend_pre, pend_compat, pend_post);
3376
3377     if (pending_size && pending_size >= s->threshold_size) {
3378         /* Still a significant amount to transfer */
3379         if (!in_postcopy && pend_pre <= s->threshold_size &&
3380             qatomic_read(&s->start_postcopy)) {
3381             if (postcopy_start(s)) {
3382                 error_report("%s: postcopy failed to start", __func__);
3383             }
3384             return MIG_ITERATE_SKIP;
3385         }
3386         /* Just another iteration step */
3387         qemu_savevm_state_iterate(s->to_dst_file, in_postcopy);
3388     } else {
3389         trace_migration_thread_low_pending(pending_size);
3390         migration_completion(s);
3391         return MIG_ITERATE_BREAK;
3392     }
3393
3394     return MIG_ITERATE_RESUME;
3395 }
3396
3397 static void migration_iteration_finish(MigrationState *s)
3398 {
3399     /* If we enabled cpu throttling for auto-converge, turn it off. */
3400     cpu_throttle_stop();
3401
3402     qemu_mutex_lock_iothread();
3403     switch (s->state) {
3404     case MIGRATION_STATUS_COMPLETED:
3405         migration_calculate_complete(s);
3406         runstate_set(RUN_STATE_POSTMIGRATE);
3407         break;
3408
3409     case MIGRATION_STATUS_ACTIVE:
3410         /*
3411          * We should really assert here, but since it's during
3412          * migration, let's try to reduce the usage of assertions.
3413          */
3414         if (!migrate_colo_enabled()) {
3415             error_report("%s: critical error: calling COLO code without "
3416                          "COLO enabled", __func__);
3417         }
3418         migrate_start_colo_process(s);
3419         /*
3420          * Fixme: we will run VM in COLO no matter its old running state.
3421          * After exited COLO, we will keep running.
3422          */
3423         s->vm_was_running = true;
3424         /* Fallthrough */
3425     case MIGRATION_STATUS_FAILED:
3426     case MIGRATION_STATUS_CANCELLED:
3427     case MIGRATION_STATUS_CANCELLING:
3428         if (s->vm_was_running) {
3429             vm_start();
3430         } else {
3431             if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
3432                 runstate_set(RUN_STATE_POSTMIGRATE);
3433             }
3434         }
3435         break;
3436
3437     default:
3438         /* Should not reach here, but if so, forgive the VM. */
3439         error_report("%s: Unknown ending state %d", __func__, s->state);
3440         break;
3441     }
3442     migrate_fd_cleanup_schedule(s);
3443     qemu_mutex_unlock_iothread();
3444 }
3445
3446 void migration_make_urgent_request(void)
3447 {
3448     qemu_sem_post(&migrate_get_current()->rate_limit_sem);
3449 }
3450
3451 void migration_consume_urgent_request(void)
3452 {
3453     qemu_sem_wait(&migrate_get_current()->rate_limit_sem);
3454 }
3455
3456 /* Returns true if the rate limiting was broken by an urgent request */
3457 bool migration_rate_limit(void)
3458 {
3459     int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3460     MigrationState *s = migrate_get_current();
3461
3462     bool urgent = false;
3463     migration_update_counters(s, now);
3464     if (qemu_file_rate_limit(s->to_dst_file)) {
3465
3466         if (qemu_file_get_error(s->to_dst_file)) {
3467             return false;
3468         }
3469         /*
3470          * Wait for a delay to do rate limiting OR
3471          * something urgent to post the semaphore.
3472          */
3473         int ms = s->iteration_start_time + BUFFER_DELAY - now;
3474         trace_migration_rate_limit_pre(ms);
3475         if (qemu_sem_timedwait(&s->rate_limit_sem, ms) == 0) {
3476             /*
3477              * We were woken by one or more urgent things but
3478              * the timedwait will have consumed one of them.
3479              * The service routine for the urgent wake will dec
3480              * the semaphore itself for each item it consumes,
3481              * so add this one we just eat back.
3482              */
3483             qemu_sem_post(&s->rate_limit_sem);
3484             urgent = true;
3485         }
3486         trace_migration_rate_limit_post(urgent);
3487     }
3488     return urgent;
3489 }
3490
3491 /*
3492  * Master migration thread on the source VM.
3493  * It drives the migration and pumps the data down the outgoing channel.
3494  */
3495 static void *migration_thread(void *opaque)
3496 {
3497     MigrationState *s = opaque;
3498     int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3499     MigThrError thr_error;
3500     bool urgent = false;
3501
3502     rcu_register_thread();
3503
3504     object_ref(OBJECT(s));
3505     update_iteration_initial_status(s);
3506
3507     qemu_savevm_state_header(s->to_dst_file);
3508
3509     /*
3510      * If we opened the return path, we need to make sure dst has it
3511      * opened as well.
3512      */
3513     if (s->rp_state.from_dst_file) {
3514         /* Now tell the dest that it should open its end so it can reply */
3515         qemu_savevm_send_open_return_path(s->to_dst_file);
3516
3517         /* And do a ping that will make stuff easier to debug */
3518         qemu_savevm_send_ping(s->to_dst_file, 1);
3519     }
3520
3521     if (migrate_postcopy()) {
3522         /*
3523          * Tell the destination that we *might* want to do postcopy later;
3524          * if the other end can't do postcopy it should fail now, nice and
3525          * early.
3526          */
3527         qemu_savevm_send_postcopy_advise(s->to_dst_file);
3528     }
3529
3530     if (migrate_colo_enabled()) {
3531         /* Notify migration destination that we enable COLO */
3532         qemu_savevm_send_colo_enable(s->to_dst_file);
3533     }
3534
3535     qemu_savevm_state_setup(s->to_dst_file);
3536
3537     if (qemu_savevm_state_guest_unplug_pending()) {
3538         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3539                           MIGRATION_STATUS_WAIT_UNPLUG);
3540
3541         while (s->state == MIGRATION_STATUS_WAIT_UNPLUG &&
3542                qemu_savevm_state_guest_unplug_pending()) {
3543             qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3544         }
3545
3546         migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG,
3547                 MIGRATION_STATUS_ACTIVE);
3548     }
3549
3550     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3551     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3552                       MIGRATION_STATUS_ACTIVE);
3553
3554     trace_migration_thread_setup_complete();
3555
3556     while (migration_is_active(s)) {
3557         if (urgent || !qemu_file_rate_limit(s->to_dst_file)) {
3558             MigIterateState iter_state = migration_iteration_run(s);
3559             if (iter_state == MIG_ITERATE_SKIP) {
3560                 continue;
3561             } else if (iter_state == MIG_ITERATE_BREAK) {
3562                 break;
3563             }
3564         }
3565
3566         /*
3567          * Try to detect any kind of failures, and see whether we
3568          * should stop the migration now.
3569          */
3570         thr_error = migration_detect_error(s);
3571         if (thr_error == MIG_THR_ERR_FATAL) {
3572             /* Stop migration */
3573             break;
3574         } else if (thr_error == MIG_THR_ERR_RECOVERED) {
3575             /*
3576              * Just recovered from a e.g. network failure, reset all
3577              * the local variables. This is important to avoid
3578              * breaking transferred_bytes and bandwidth calculation
3579              */
3580             update_iteration_initial_status(s);
3581         }
3582
3583         urgent = migration_rate_limit();
3584     }
3585
3586     trace_migration_thread_after_loop();
3587     migration_iteration_finish(s);
3588     object_unref(OBJECT(s));
3589     rcu_unregister_thread();
3590     return NULL;
3591 }
3592
3593 void migrate_fd_connect(MigrationState *s, Error *error_in)
3594 {
3595     Error *local_err = NULL;
3596     int64_t rate_limit;
3597     bool resume = s->state == MIGRATION_STATUS_POSTCOPY_PAUSED;
3598
3599     s->expected_downtime = s->parameters.downtime_limit;
3600     if (resume) {
3601         assert(s->cleanup_bh);
3602     } else {
3603         assert(!s->cleanup_bh);
3604         s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup_bh, s);
3605     }
3606     if (error_in) {
3607         migrate_fd_error(s, error_in);
3608         migrate_fd_cleanup(s);
3609         return;
3610     }
3611
3612     if (resume) {
3613         /* This is a resumed migration */
3614         rate_limit = s->parameters.max_postcopy_bandwidth /
3615             XFER_LIMIT_RATIO;
3616     } else {
3617         /* This is a fresh new migration */
3618         rate_limit = s->parameters.max_bandwidth / XFER_LIMIT_RATIO;
3619
3620         /* Notify before starting migration thread */
3621         notifier_list_notify(&migration_state_notifiers, s);
3622     }
3623
3624     qemu_file_set_rate_limit(s->to_dst_file, rate_limit);
3625     qemu_file_set_blocking(s->to_dst_file, true);
3626
3627     /*
3628      * Open the return path. For postcopy, it is used exclusively. For
3629      * precopy, only if user specified "return-path" capability would
3630      * QEMU uses the return path.
3631      */
3632     if (migrate_postcopy_ram() || migrate_use_return_path()) {
3633         if (open_return_path_on_source(s, !resume)) {
3634             error_report("Unable to open return-path for postcopy");
3635             migrate_set_state(&s->state, s->state, MIGRATION_STATUS_FAILED);
3636             migrate_fd_cleanup(s);
3637             return;
3638         }
3639     }
3640
3641     if (resume) {
3642         /* Wakeup the main migration thread to do the recovery */
3643         migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
3644                           MIGRATION_STATUS_POSTCOPY_RECOVER);
3645         qemu_sem_post(&s->postcopy_pause_sem);
3646         return;
3647     }
3648
3649     if (multifd_save_setup(&local_err) != 0) {
3650         error_report_err(local_err);
3651         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3652                           MIGRATION_STATUS_FAILED);
3653         migrate_fd_cleanup(s);
3654         return;
3655     }
3656     qemu_thread_create(&s->thread, "live_migration", migration_thread, s,
3657                        QEMU_THREAD_JOINABLE);
3658     s->migration_thread_running = true;
3659 }
3660
3661 void migration_global_dump(Monitor *mon)
3662 {
3663     MigrationState *ms = migrate_get_current();
3664
3665     monitor_printf(mon, "globals:\n");
3666     monitor_printf(mon, "store-global-state: %s\n",
3667                    ms->store_global_state ? "on" : "off");
3668     monitor_printf(mon, "only-migratable: %s\n",
3669                    only_migratable ? "on" : "off");
3670     monitor_printf(mon, "send-configuration: %s\n",
3671                    ms->send_configuration ? "on" : "off");
3672     monitor_printf(mon, "send-section-footer: %s\n",
3673                    ms->send_section_footer ? "on" : "off");
3674     monitor_printf(mon, "decompress-error-check: %s\n",
3675                    ms->decompress_error_check ? "on" : "off");
3676     monitor_printf(mon, "clear-bitmap-shift: %u\n",
3677                    ms->clear_bitmap_shift);
3678 }
3679
3680 #define DEFINE_PROP_MIG_CAP(name, x)             \
3681     DEFINE_PROP_BOOL(name, MigrationState, enabled_capabilities[x], false)
3682
3683 static Property migration_properties[] = {
3684     DEFINE_PROP_BOOL("store-global-state", MigrationState,
3685                      store_global_state, true),
3686     DEFINE_PROP_BOOL("send-configuration", MigrationState,
3687                      send_configuration, true),
3688     DEFINE_PROP_BOOL("send-section-footer", MigrationState,
3689                      send_section_footer, true),
3690     DEFINE_PROP_BOOL("decompress-error-check", MigrationState,
3691                       decompress_error_check, true),
3692     DEFINE_PROP_UINT8("x-clear-bitmap-shift", MigrationState,
3693                       clear_bitmap_shift, CLEAR_BITMAP_SHIFT_DEFAULT),
3694
3695     /* Migration parameters */
3696     DEFINE_PROP_UINT8("x-compress-level", MigrationState,
3697                       parameters.compress_level,
3698                       DEFAULT_MIGRATE_COMPRESS_LEVEL),
3699     DEFINE_PROP_UINT8("x-compress-threads", MigrationState,
3700                       parameters.compress_threads,
3701                       DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT),
3702     DEFINE_PROP_BOOL("x-compress-wait-thread", MigrationState,
3703                       parameters.compress_wait_thread, true),
3704     DEFINE_PROP_UINT8("x-decompress-threads", MigrationState,
3705                       parameters.decompress_threads,
3706                       DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT),
3707     DEFINE_PROP_UINT8("x-throttle-trigger-threshold", MigrationState,
3708                       parameters.throttle_trigger_threshold,
3709                       DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD),
3710     DEFINE_PROP_UINT8("x-cpu-throttle-initial", MigrationState,
3711                       parameters.cpu_throttle_initial,
3712                       DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL),
3713     DEFINE_PROP_UINT8("x-cpu-throttle-increment", MigrationState,
3714                       parameters.cpu_throttle_increment,
3715                       DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT),
3716     DEFINE_PROP_BOOL("x-cpu-throttle-tailslow", MigrationState,
3717                       parameters.cpu_throttle_tailslow, false),
3718     DEFINE_PROP_SIZE("x-max-bandwidth", MigrationState,
3719                       parameters.max_bandwidth, MAX_THROTTLE),
3720     DEFINE_PROP_UINT64("x-downtime-limit", MigrationState,
3721                       parameters.downtime_limit,
3722                       DEFAULT_MIGRATE_SET_DOWNTIME),
3723     DEFINE_PROP_UINT32("x-checkpoint-delay", MigrationState,
3724                       parameters.x_checkpoint_delay,
3725                       DEFAULT_MIGRATE_X_CHECKPOINT_DELAY),
3726     DEFINE_PROP_UINT8("multifd-channels", MigrationState,
3727                       parameters.multifd_channels,
3728                       DEFAULT_MIGRATE_MULTIFD_CHANNELS),
3729     DEFINE_PROP_MULTIFD_COMPRESSION("multifd-compression", MigrationState,
3730                       parameters.multifd_compression,
3731                       DEFAULT_MIGRATE_MULTIFD_COMPRESSION),
3732     DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState,
3733                       parameters.multifd_zlib_level,
3734                       DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL),
3735     DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState,
3736                       parameters.multifd_zstd_level,
3737                       DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL),
3738     DEFINE_PROP_SIZE("xbzrle-cache-size", MigrationState,
3739                       parameters.xbzrle_cache_size,
3740                       DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE),
3741     DEFINE_PROP_SIZE("max-postcopy-bandwidth", MigrationState,
3742                       parameters.max_postcopy_bandwidth,
3743                       DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH),
3744     DEFINE_PROP_UINT8("max-cpu-throttle", MigrationState,
3745                       parameters.max_cpu_throttle,
3746                       DEFAULT_MIGRATE_MAX_CPU_THROTTLE),
3747     DEFINE_PROP_SIZE("announce-initial", MigrationState,
3748                       parameters.announce_initial,
3749                       DEFAULT_MIGRATE_ANNOUNCE_INITIAL),
3750     DEFINE_PROP_SIZE("announce-max", MigrationState,
3751                       parameters.announce_max,
3752                       DEFAULT_MIGRATE_ANNOUNCE_MAX),
3753     DEFINE_PROP_SIZE("announce-rounds", MigrationState,
3754                       parameters.announce_rounds,
3755                       DEFAULT_MIGRATE_ANNOUNCE_ROUNDS),
3756     DEFINE_PROP_SIZE("announce-step", MigrationState,
3757                       parameters.announce_step,
3758                       DEFAULT_MIGRATE_ANNOUNCE_STEP),
3759
3760     /* Migration capabilities */
3761     DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
3762     DEFINE_PROP_MIG_CAP("x-rdma-pin-all", MIGRATION_CAPABILITY_RDMA_PIN_ALL),
3763     DEFINE_PROP_MIG_CAP("x-auto-converge", MIGRATION_CAPABILITY_AUTO_CONVERGE),
3764     DEFINE_PROP_MIG_CAP("x-zero-blocks", MIGRATION_CAPABILITY_ZERO_BLOCKS),
3765     DEFINE_PROP_MIG_CAP("x-compress", MIGRATION_CAPABILITY_COMPRESS),
3766     DEFINE_PROP_MIG_CAP("x-events", MIGRATION_CAPABILITY_EVENTS),
3767     DEFINE_PROP_MIG_CAP("x-postcopy-ram", MIGRATION_CAPABILITY_POSTCOPY_RAM),
3768     DEFINE_PROP_MIG_CAP("x-colo", MIGRATION_CAPABILITY_X_COLO),
3769     DEFINE_PROP_MIG_CAP("x-release-ram", MIGRATION_CAPABILITY_RELEASE_RAM),
3770     DEFINE_PROP_MIG_CAP("x-block", MIGRATION_CAPABILITY_BLOCK),
3771     DEFINE_PROP_MIG_CAP("x-return-path", MIGRATION_CAPABILITY_RETURN_PATH),
3772     DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD),
3773
3774     DEFINE_PROP_END_OF_LIST(),
3775 };
3776
3777 static void migration_class_init(ObjectClass *klass, void *data)
3778 {
3779     DeviceClass *dc = DEVICE_CLASS(klass);
3780
3781     dc->user_creatable = false;
3782     device_class_set_props(dc, migration_properties);
3783 }
3784
3785 static void migration_instance_finalize(Object *obj)
3786 {
3787     MigrationState *ms = MIGRATION_OBJ(obj);
3788     MigrationParameters *params = &ms->parameters;
3789
3790     qemu_mutex_destroy(&ms->error_mutex);
3791     qemu_mutex_destroy(&ms->qemu_file_lock);
3792     g_free(params->tls_hostname);
3793     g_free(params->tls_creds);
3794     qemu_sem_destroy(&ms->wait_unplug_sem);
3795     qemu_sem_destroy(&ms->rate_limit_sem);
3796     qemu_sem_destroy(&ms->pause_sem);
3797     qemu_sem_destroy(&ms->postcopy_pause_sem);
3798     qemu_sem_destroy(&ms->postcopy_pause_rp_sem);
3799     qemu_sem_destroy(&ms->rp_state.rp_sem);
3800     error_free(ms->error);
3801 }
3802
3803 static void migration_instance_init(Object *obj)
3804 {
3805     MigrationState *ms = MIGRATION_OBJ(obj);
3806     MigrationParameters *params = &ms->parameters;
3807
3808     ms->state = MIGRATION_STATUS_NONE;
3809     ms->mbps = -1;
3810     ms->pages_per_second = -1;
3811     qemu_sem_init(&ms->pause_sem, 0);
3812     qemu_mutex_init(&ms->error_mutex);
3813
3814     params->tls_hostname = g_strdup("");
3815     params->tls_creds = g_strdup("");
3816
3817     /* Set has_* up only for parameter checks */
3818     params->has_compress_level = true;
3819     params->has_compress_threads = true;
3820     params->has_decompress_threads = true;
3821     params->has_throttle_trigger_threshold = true;
3822     params->has_cpu_throttle_initial = true;
3823     params->has_cpu_throttle_increment = true;
3824     params->has_cpu_throttle_tailslow = true;
3825     params->has_max_bandwidth = true;
3826     params->has_downtime_limit = true;
3827     params->has_x_checkpoint_delay = true;
3828     params->has_block_incremental = true;
3829     params->has_multifd_channels = true;
3830     params->has_multifd_compression = true;
3831     params->has_multifd_zlib_level = true;
3832     params->has_multifd_zstd_level = true;
3833     params->has_xbzrle_cache_size = true;
3834     params->has_max_postcopy_bandwidth = true;
3835     params->has_max_cpu_throttle = true;
3836     params->has_announce_initial = true;
3837     params->has_announce_max = true;
3838     params->has_announce_rounds = true;
3839     params->has_announce_step = true;
3840
3841     qemu_sem_init(&ms->postcopy_pause_sem, 0);
3842     qemu_sem_init(&ms->postcopy_pause_rp_sem, 0);
3843     qemu_sem_init(&ms->rp_state.rp_sem, 0);
3844     qemu_sem_init(&ms->rate_limit_sem, 0);
3845     qemu_sem_init(&ms->wait_unplug_sem, 0);
3846     qemu_mutex_init(&ms->qemu_file_lock);
3847 }
3848
3849 /*
3850  * Return true if check pass, false otherwise. Error will be put
3851  * inside errp if provided.
3852  */
3853 static bool migration_object_check(MigrationState *ms, Error **errp)
3854 {
3855     MigrationCapabilityStatusList *head = NULL;
3856     /* Assuming all off */
3857     bool cap_list[MIGRATION_CAPABILITY__MAX] = { 0 }, ret;
3858     int i;
3859
3860     if (!migrate_params_check(&ms->parameters, errp)) {
3861         return false;
3862     }
3863
3864     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
3865         if (ms->enabled_capabilities[i]) {
3866             QAPI_LIST_PREPEND(head, migrate_cap_add(i, true));
3867         }
3868     }
3869
3870     ret = migrate_caps_check(cap_list, head, errp);
3871
3872     /* It works with head == NULL */
3873     qapi_free_MigrationCapabilityStatusList(head);
3874
3875     return ret;
3876 }
3877
3878 static const TypeInfo migration_type = {
3879     .name = TYPE_MIGRATION,
3880     /*
3881      * NOTE: TYPE_MIGRATION is not really a device, as the object is
3882      * not created using qdev_new(), it is not attached to the qdev
3883      * device tree, and it is never realized.
3884      *
3885      * TODO: Make this TYPE_OBJECT once QOM provides something like
3886      * TYPE_DEVICE's "-global" properties.
3887      */
3888     .parent = TYPE_DEVICE,
3889     .class_init = migration_class_init,
3890     .class_size = sizeof(MigrationClass),
3891     .instance_size = sizeof(MigrationState),
3892     .instance_init = migration_instance_init,
3893     .instance_finalize = migration_instance_finalize,
3894 };
3895
3896 static void register_migration_types(void)
3897 {
3898     type_register_static(&migration_type);
3899 }
3900
3901 type_init(register_migration_types);