migration/migration.c

   1 /*
   2  * QEMU live migration
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *
   6  * Authors:
   7  *  Anthony Liguori   <aliguori@us.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include "qemu/cutils.h"
  18 #include "qemu/error-report.h"
  19 #include "qemu/main-loop.h"
  20 #include "migration/blocker.h"
  21 #include "exec.h"
  22 #include "fd.h"
  23 #include "socket.h"
  24 #include "sysemu/runstate.h"
  25 #include "sysemu/sysemu.h"
  26 #include "sysemu/cpu-throttle.h"
  27 #include "rdma.h"
  28 #include "ram.h"
  29 #include "migration/global_state.h"
  30 #include "migration/misc.h"
  31 #include "migration.h"
  32 #include "savevm.h"
  33 #include "qemu-file-channel.h"
  34 #include "qemu-file.h"
  35 #include "migration/vmstate.h"
  36 #include "block/block.h"
  37 #include "qapi/error.h"
  38 #include "qapi/clone-visitor.h"
  39 #include "qapi/qapi-visit-sockets.h"
  40 #include "qapi/qapi-commands-migration.h"
  41 #include "qapi/qapi-events-migration.h"
  42 #include "qapi/qmp/qerror.h"
  43 #include "qapi/qmp/qnull.h"
  44 #include "qemu/rcu.h"
  45 #include "block.h"
  46 #include "postcopy-ram.h"
  47 #include "qemu/thread.h"
  48 #include "trace.h"
  49 #include "exec/target_page.h"
  50 #include "io/channel-buffer.h"
  51 #include "migration/colo.h"
  52 #include "hw/boards.h"
  53 #include "hw/qdev-properties.h"
  54 #include "monitor/monitor.h"
  55 #include "net/announce.h"
  56 #include "qemu/queue.h"
  57 #include "multifd.h"
  58
  59 #define MAX_THROTTLE  (32 << 20)      /* Migration transfer speed throttling */
  60
  61 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
  62  * data. */
  63 #define BUFFER_DELAY     100
  64 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
  65
  66 /* Time in milliseconds we are allowed to stop the source,
  67  * for sending the last part */
  68 #define DEFAULT_MIGRATE_SET_DOWNTIME 300
  69
  70 /* Maximum migrate downtime set to 2000 seconds */
  71 #define MAX_MIGRATE_DOWNTIME_SECONDS 2000
  72 #define MAX_MIGRATE_DOWNTIME (MAX_MIGRATE_DOWNTIME_SECONDS * 1000)
  73
  74 /* Default compression thread count */
  75 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8
  76 /* Default decompression thread count, usually decompression is at
  77  * least 4 times as fast as compression.*/
  78 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2
  79 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */
  80 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1
  81 /* Define default autoconverge cpu throttle migration parameters */
  82 #define DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD 50
  83 #define DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL 20
  84 #define DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT 10
  85 #define DEFAULT_MIGRATE_MAX_CPU_THROTTLE 99
  86
  87 /* Migration XBZRLE default cache size */
  88 #define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024)
  89
  90 /* The delay time (in ms) between two COLO checkpoints */
  91 #define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100)
  92 #define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2
  93 #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE
  94 /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */
  95 #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1
  96 /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */
  97 #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1
  98
  99 /* Background transfer rate for postcopy, 0 means unlimited, note
 100  * that page requests can still exceed this limit.
 101  */
 102 #define DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH 0
 103
 104 /*
 105  * Parameters for self_announce_delay giving a stream of RARP/ARP
 106  * packets after migration.
 107  */
 108 #define DEFAULT_MIGRATE_ANNOUNCE_INITIAL  50
 109 #define DEFAULT_MIGRATE_ANNOUNCE_MAX     550
 110 #define DEFAULT_MIGRATE_ANNOUNCE_ROUNDS    5
 111 #define DEFAULT_MIGRATE_ANNOUNCE_STEP    100
 112
 113 static NotifierList migration_state_notifiers =
 114     NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
 115
 116 static bool deferred_incoming;
 117
 118 /* Messages sent on the return path from destination to source */
 119 enum mig_rp_message_type {
 120     MIG_RP_MSG_INVALID = 0,  /* Must be 0 */
 121     MIG_RP_MSG_SHUT,         /* sibling will not send any more RP messages */
 122     MIG_RP_MSG_PONG,         /* Response to a PING; data (seq: be32 ) */
 123
 124     MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
 125     MIG_RP_MSG_REQ_PAGES,    /* data (start: be64, len: be32) */
 126     MIG_RP_MSG_RECV_BITMAP,  /* send recved_bitmap back to source */
 127     MIG_RP_MSG_RESUME_ACK,   /* tell source that we are ready to resume */
 128
 129     MIG_RP_MSG_MAX
 130 };
 131
 132 /* When we add fault tolerance, we could have several
 133    migrations at once.  For now we don't need to add
 134    dynamic creation of migration */
 135
 136 static MigrationState *current_migration;
 137 static MigrationIncomingState *current_incoming;
 138
 139 static bool migration_object_check(MigrationState *ms, Error **errp);
 140 static int migration_maybe_pause(MigrationState *s,
 141                                  int *current_active_state,
 142                                  int new_state);
 143 static void migrate_fd_cancel(MigrationState *s);
 144
 145 void migration_object_init(void)
 146 {
 147     MachineState *ms = MACHINE(qdev_get_machine());
 148     Error *err = NULL;
 149
 150     /* This can only be called once. */
 151     assert(!current_migration);
 152     current_migration = MIGRATION_OBJ(object_new(TYPE_MIGRATION));
 153
 154     /*
 155      * Init the migrate incoming object as well no matter whether
 156      * we'll use it or not.
 157      */
 158     assert(!current_incoming);
 159     current_incoming = g_new0(MigrationIncomingState, 1);
 160     current_incoming->state = MIGRATION_STATUS_NONE;
 161     current_incoming->postcopy_remote_fds =
 162         g_array_new(FALSE, TRUE, sizeof(struct PostCopyFD));
 163     qemu_mutex_init(&current_incoming->rp_mutex);
 164     qemu_event_init(&current_incoming->main_thread_load_event, false);
 165     qemu_sem_init(&current_incoming->postcopy_pause_sem_dst, 0);
 166     qemu_sem_init(&current_incoming->postcopy_pause_sem_fault, 0);
 167
 168     if (!migration_object_check(current_migration, &err)) {
 169         error_report_err(err);
 170         exit(1);
 171     }
 172
 173     /*
 174      * We cannot really do this in migration_instance_init() since at
 175      * that time global properties are not yet applied, then this
 176      * value will be definitely replaced by something else.
 177      */
 178     if (ms->enforce_config_section) {
 179         current_migration->send_configuration = true;
 180     }
 181 }
 182
 183 void migration_shutdown(void)
 184 {
 185     /*
 186      * Cancel the current migration - that will (eventually)
 187      * stop the migration using this structure
 188      */
 189     migrate_fd_cancel(current_migration);
 190     object_unref(OBJECT(current_migration));
 191 }
 192
 193 /* For outgoing */
 194 MigrationState *migrate_get_current(void)
 195 {
 196     /* This can only be called after the object created. */
 197     assert(current_migration);
 198     return current_migration;
 199 }
 200
 201 MigrationIncomingState *migration_incoming_get_current(void)
 202 {
 203     assert(current_incoming);
 204     return current_incoming;
 205 }
 206
 207 void migration_incoming_state_destroy(void)
 208 {
 209     struct MigrationIncomingState *mis = migration_incoming_get_current();
 210
 211     if (mis->to_src_file) {
 212         /* Tell source that we are done */
 213         migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
 214         qemu_fclose(mis->to_src_file);
 215         mis->to_src_file = NULL;
 216     }
 217
 218     if (mis->from_src_file) {
 219         qemu_fclose(mis->from_src_file);
 220         mis->from_src_file = NULL;
 221     }
 222     if (mis->postcopy_remote_fds) {
 223         g_array_free(mis->postcopy_remote_fds, TRUE);
 224         mis->postcopy_remote_fds = NULL;
 225     }
 226
 227     qemu_event_reset(&mis->main_thread_load_event);
 228
 229     if (mis->socket_address_list) {
 230         qapi_free_SocketAddressList(mis->socket_address_list);
 231         mis->socket_address_list = NULL;
 232     }
 233 }
 234
 235 static void migrate_generate_event(int new_state)
 236 {
 237     if (migrate_use_events()) {
 238         qapi_event_send_migration(new_state);
 239     }
 240 }
 241
 242 static bool migrate_late_block_activate(void)
 243 {
 244     MigrationState *s;
 245
 246     s = migrate_get_current();
 247
 248     return s->enabled_capabilities[
 249         MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE];
 250 }
 251
 252 /*
 253  * Called on -incoming with a defer: uri.
 254  * The migration can be started later after any parameters have been
 255  * changed.
 256  */
 257 static void deferred_incoming_migration(Error **errp)
 258 {
 259     if (deferred_incoming) {
 260         error_setg(errp, "Incoming migration already deferred");
 261     }
 262     deferred_incoming = true;
 263 }
 264
 265 /*
 266  * Send a message on the return channel back to the source
 267  * of the migration.
 268  */
 269 static int migrate_send_rp_message(MigrationIncomingState *mis,
 270                                    enum mig_rp_message_type message_type,
 271                                    uint16_t len, void *data)
 272 {
 273     int ret = 0;
 274
 275     trace_migrate_send_rp_message((int)message_type, len);
 276     qemu_mutex_lock(&mis->rp_mutex);
 277
 278     /*
 279      * It's possible that the file handle got lost due to network
 280      * failures.
 281      */
 282     if (!mis->to_src_file) {
 283         ret = -EIO;
 284         goto error;
 285     }
 286
 287     qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
 288     qemu_put_be16(mis->to_src_file, len);
 289     qemu_put_buffer(mis->to_src_file, data, len);
 290     qemu_fflush(mis->to_src_file);
 291
 292     /* It's possible that qemu file got error during sending */
 293     ret = qemu_file_get_error(mis->to_src_file);
 294
 295 error:
 296     qemu_mutex_unlock(&mis->rp_mutex);
 297     return ret;
 298 }
 299
 300 /* Request a range of pages from the source VM at the given
 301  * start address.
 302  *   rbname: Name of the RAMBlock to request the page in, if NULL it's the same
 303  *           as the last request (a name must have been given previously)
 304  *   Start: Address offset within the RB
 305  *   Len: Length in bytes required - must be a multiple of pagesize
 306  */
 307 int migrate_send_rp_req_pages(MigrationIncomingState *mis, const char *rbname,
 308                               ram_addr_t start, size_t len)
 309 {
 310     uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */
 311     size_t msglen = 12; /* start + len */
 312     enum mig_rp_message_type msg_type;
 313
 314     *(uint64_t *)bufc = cpu_to_be64((uint64_t)start);
 315     *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len);
 316
 317     if (rbname) {
 318         int rbname_len = strlen(rbname);
 319         assert(rbname_len < 256);
 320
 321         bufc[msglen++] = rbname_len;
 322         memcpy(bufc + msglen, rbname, rbname_len);
 323         msglen += rbname_len;
 324         msg_type = MIG_RP_MSG_REQ_PAGES_ID;
 325     } else {
 326         msg_type = MIG_RP_MSG_REQ_PAGES;
 327     }
 328
 329     return migrate_send_rp_message(mis, msg_type, msglen, bufc);
 330 }
 331
 332 static bool migration_colo_enabled;
 333 bool migration_incoming_colo_enabled(void)
 334 {
 335     return migration_colo_enabled;
 336 }
 337
 338 void migration_incoming_disable_colo(void)
 339 {
 340     ram_block_discard_disable(false);
 341     migration_colo_enabled = false;
 342 }
 343
 344 int migration_incoming_enable_colo(void)
 345 {
 346     if (ram_block_discard_disable(true)) {
 347         error_report("COLO: cannot disable RAM discard");
 348         return -EBUSY;
 349     }
 350     migration_colo_enabled = true;
 351     return 0;
 352 }
 353
 354 void migrate_add_address(SocketAddress *address)
 355 {
 356     MigrationIncomingState *mis = migration_incoming_get_current();
 357     SocketAddressList *addrs;
 358
 359     addrs = g_new0(SocketAddressList, 1);
 360     addrs->next = mis->socket_address_list;
 361     mis->socket_address_list = addrs;
 362     addrs->value = QAPI_CLONE(SocketAddress, address);
 363 }
 364
 365 void qemu_start_incoming_migration(const char *uri, Error **errp)
 366 {
 367     const char *p;
 368
 369     qapi_event_send_migration(MIGRATION_STATUS_SETUP);
 370     if (!strcmp(uri, "defer")) {
 371         deferred_incoming_migration(errp);
 372     } else if (strstart(uri, "tcp:", &p)) {
 373         tcp_start_incoming_migration(p, errp);
 374 #ifdef CONFIG_RDMA
 375     } else if (strstart(uri, "rdma:", &p)) {
 376         rdma_start_incoming_migration(p, errp);
 377 #endif
 378     } else if (strstart(uri, "exec:", &p)) {
 379         exec_start_incoming_migration(p, errp);
 380     } else if (strstart(uri, "unix:", &p)) {
 381         unix_start_incoming_migration(p, errp);
 382     } else if (strstart(uri, "fd:", &p)) {
 383         fd_start_incoming_migration(p, errp);
 384     } else {
 385         error_setg(errp, "unknown migration protocol: %s", uri);
 386     }
 387 }
 388
 389 static void process_incoming_migration_bh(void *opaque)
 390 {
 391     Error *local_err = NULL;
 392     MigrationIncomingState *mis = opaque;
 393
 394     /* If capability late_block_activate is set:
 395      * Only fire up the block code now if we're going to restart the
 396      * VM, else 'cont' will do it.
 397      * This causes file locking to happen; so we don't want it to happen
 398      * unless we really are starting the VM.
 399      */
 400     if (!migrate_late_block_activate() ||
 401          (autostart && (!global_state_received() ||
 402             global_state_get_runstate() == RUN_STATE_RUNNING))) {
 403         /* Make sure all file formats flush their mutable metadata.
 404          * If we get an error here, just don't restart the VM yet. */
 405         bdrv_invalidate_cache_all(&local_err);
 406         if (local_err) {
 407             error_report_err(local_err);
 408             local_err = NULL;
 409             autostart = false;
 410         }
 411     }
 412
 413     /*
 414      * This must happen after all error conditions are dealt with and
 415      * we're sure the VM is going to be running on this host.
 416      */
 417     qemu_announce_self(&mis->announce_timer, migrate_announce_params());
 418
 419     if (multifd_load_cleanup(&local_err) != 0) {
 420         error_report_err(local_err);
 421         autostart = false;
 422     }
 423     /* If global state section was not received or we are in running
 424        state, we need to obey autostart. Any other state is set with
 425        runstate_set. */
 426
 427     dirty_bitmap_mig_before_vm_start();
 428
 429     if (!global_state_received() ||
 430         global_state_get_runstate() == RUN_STATE_RUNNING) {
 431         if (autostart) {
 432             vm_start();
 433         } else {
 434             runstate_set(RUN_STATE_PAUSED);
 435         }
 436     } else if (migration_incoming_colo_enabled()) {
 437         migration_incoming_disable_colo();
 438         vm_start();
 439     } else {
 440         runstate_set(global_state_get_runstate());
 441     }
 442     /*
 443      * This must happen after any state changes since as soon as an external
 444      * observer sees this event they might start to prod at the VM assuming
 445      * it's ready to use.
 446      */
 447     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 448                       MIGRATION_STATUS_COMPLETED);
 449     qemu_bh_delete(mis->bh);
 450     migration_incoming_state_destroy();
 451 }
 452
 453 static void process_incoming_migration_co(void *opaque)
 454 {
 455     MigrationIncomingState *mis = migration_incoming_get_current();
 456     PostcopyState ps;
 457     int ret;
 458     Error *local_err = NULL;
 459
 460     assert(mis->from_src_file);
 461     mis->migration_incoming_co = qemu_coroutine_self();
 462     mis->largest_page_size = qemu_ram_pagesize_largest();
 463     postcopy_state_set(POSTCOPY_INCOMING_NONE);
 464     migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
 465                       MIGRATION_STATUS_ACTIVE);
 466     ret = qemu_loadvm_state(mis->from_src_file);
 467
 468     ps = postcopy_state_get();
 469     trace_process_incoming_migration_co_end(ret, ps);
 470     if (ps != POSTCOPY_INCOMING_NONE) {
 471         if (ps == POSTCOPY_INCOMING_ADVISE) {
 472             /*
 473              * Where a migration had postcopy enabled (and thus went to advise)
 474              * but managed to complete within the precopy period, we can use
 475              * the normal exit.
 476              */
 477             postcopy_ram_incoming_cleanup(mis);
 478         } else if (ret >= 0) {
 479             /*
 480              * Postcopy was started, cleanup should happen at the end of the
 481              * postcopy thread.
 482              */
 483             trace_process_incoming_migration_co_postcopy_end_main();
 484             return;
 485         }
 486         /* Else if something went wrong then just fall out of the normal exit */
 487     }
 488
 489     /* we get COLO info, and know if we are in COLO mode */
 490     if (!ret && migration_incoming_colo_enabled()) {
 491         /* Make sure all file formats flush their mutable metadata */
 492         bdrv_invalidate_cache_all(&local_err);
 493         if (local_err) {
 494             error_report_err(local_err);
 495             goto fail;
 496         }
 497
 498         qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
 499              colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
 500         mis->have_colo_incoming_thread = true;
 501         qemu_coroutine_yield();
 502
 503         /* Wait checkpoint incoming thread exit before free resource */
 504         qemu_thread_join(&mis->colo_incoming_thread);
 505         /* We hold the global iothread lock, so it is safe here */
 506         colo_release_ram_cache();
 507     }
 508
 509     if (ret < 0) {
 510         error_report("load of migration failed: %s", strerror(-ret));
 511         goto fail;
 512     }
 513     mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
 514     qemu_bh_schedule(mis->bh);
 515     mis->migration_incoming_co = NULL;
 516     return;
 517 fail:
 518     local_err = NULL;
 519     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 520                       MIGRATION_STATUS_FAILED);
 521     qemu_fclose(mis->from_src_file);
 522     if (multifd_load_cleanup(&local_err) != 0) {
 523         error_report_err(local_err);
 524     }
 525     exit(EXIT_FAILURE);
 526 }
 527
 528 /**
 529  * @migration_incoming_setup: Setup incoming migration
 530  *
 531  * Returns 0 for no error or 1 for error
 532  *
 533  * @f: file for main migration channel
 534  * @errp: where to put errors
 535  */
 536 static int migration_incoming_setup(QEMUFile *f, Error **errp)
 537 {
 538     MigrationIncomingState *mis = migration_incoming_get_current();
 539     Error *local_err = NULL;
 540
 541     if (multifd_load_setup(&local_err) != 0) {
 542         /* We haven't been able to create multifd threads
 543            nothing better to do */
 544         error_report_err(local_err);
 545         exit(EXIT_FAILURE);
 546     }
 547
 548     if (!mis->from_src_file) {
 549         mis->from_src_file = f;
 550     }
 551     qemu_file_set_blocking(f, false);
 552     return 0;
 553 }
 554
 555 void migration_incoming_process(void)
 556 {
 557     Coroutine *co = qemu_coroutine_create(process_incoming_migration_co, NULL);
 558     qemu_coroutine_enter(co);
 559 }
 560
 561 /* Returns true if recovered from a paused migration, otherwise false */
 562 static bool postcopy_try_recover(QEMUFile *f)
 563 {
 564     MigrationIncomingState *mis = migration_incoming_get_current();
 565
 566     if (mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
 567         /* Resumed from a paused postcopy migration */
 568
 569         mis->from_src_file = f;
 570         /* Postcopy has standalone thread to do vm load */
 571         qemu_file_set_blocking(f, true);
 572
 573         /* Re-configure the return path */
 574         mis->to_src_file = qemu_file_get_return_path(f);
 575
 576         migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
 577                           MIGRATION_STATUS_POSTCOPY_RECOVER);
 578
 579         /*
 580          * Here, we only wake up the main loading thread (while the
 581          * fault thread will still be waiting), so that we can receive
 582          * commands from source now, and answer it if needed. The
 583          * fault thread will be woken up afterwards until we are sure
 584          * that source is ready to reply to page requests.
 585          */
 586         qemu_sem_post(&mis->postcopy_pause_sem_dst);
 587         return true;
 588     }
 589
 590     return false;
 591 }
 592
 593 void migration_fd_process_incoming(QEMUFile *f, Error **errp)
 594 {
 595     Error *local_err = NULL;
 596
 597     if (postcopy_try_recover(f)) {
 598         return;
 599     }
 600
 601     if (migration_incoming_setup(f, &local_err)) {
 602         if (local_err) {
 603             error_propagate(errp, local_err);
 604         }
 605         return;
 606     }
 607     migration_incoming_process();
 608 }
 609
 610 void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp)
 611 {
 612     MigrationIncomingState *mis = migration_incoming_get_current();
 613     Error *local_err = NULL;
 614     bool start_migration;
 615
 616     if (!mis->from_src_file) {
 617         /* The first connection (multifd may have multiple) */
 618         QEMUFile *f = qemu_fopen_channel_input(ioc);
 619
 620         /* If it's a recovery, we're done */
 621         if (postcopy_try_recover(f)) {
 622             return;
 623         }
 624
 625         if (migration_incoming_setup(f, &local_err)) {
 626             if (local_err) {
 627                 error_propagate(errp, local_err);
 628             }
 629             return;
 630         }
 631
 632         /*
 633          * Common migration only needs one channel, so we can start
 634          * right now.  Multifd needs more than one channel, we wait.
 635          */
 636         start_migration = !migrate_use_multifd();
 637     } else {
 638         /* Multiple connections */
 639         assert(migrate_use_multifd());
 640         start_migration = multifd_recv_new_channel(ioc, &local_err);
 641         if (local_err) {
 642             error_propagate(errp, local_err);
 643             return;
 644         }
 645     }
 646
 647     if (start_migration) {
 648         migration_incoming_process();
 649     }
 650 }
 651
 652 /**
 653  * @migration_has_all_channels: We have received all channels that we need
 654  *
 655  * Returns true when we have got connections to all the channels that
 656  * we need for migration.
 657  */
 658 bool migration_has_all_channels(void)
 659 {
 660     MigrationIncomingState *mis = migration_incoming_get_current();
 661     bool all_channels;
 662
 663     all_channels = multifd_recv_all_channels_created();
 664
 665     return all_channels && mis->from_src_file != NULL;
 666 }
 667
 668 /*
 669  * Send a 'SHUT' message on the return channel with the given value
 670  * to indicate that we've finished with the RP.  Non-0 value indicates
 671  * error.
 672  */
 673 void migrate_send_rp_shut(MigrationIncomingState *mis,
 674                           uint32_t value)
 675 {
 676     uint32_t buf;
 677
 678     buf = cpu_to_be32(value);
 679     migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
 680 }
 681
 682 /*
 683  * Send a 'PONG' message on the return channel with the given value
 684  * (normally in response to a 'PING')
 685  */
 686 void migrate_send_rp_pong(MigrationIncomingState *mis,
 687                           uint32_t value)
 688 {
 689     uint32_t buf;
 690
 691     buf = cpu_to_be32(value);
 692     migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
 693 }
 694
 695 void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
 696                                  char *block_name)
 697 {
 698     char buf[512];
 699     int len;
 700     int64_t res;
 701
 702     /*
 703      * First, we send the header part. It contains only the len of
 704      * idstr, and the idstr itself.
 705      */
 706     len = strlen(block_name);
 707     buf[0] = len;
 708     memcpy(buf + 1, block_name, len);
 709
 710     if (mis->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
 711         error_report("%s: MSG_RP_RECV_BITMAP only used for recovery",
 712                      __func__);
 713         return;
 714     }
 715
 716     migrate_send_rp_message(mis, MIG_RP_MSG_RECV_BITMAP, len + 1, buf);
 717
 718     /*
 719      * Next, we dump the received bitmap to the stream.
 720      *
 721      * TODO: currently we are safe since we are the only one that is
 722      * using the to_src_file handle (fault thread is still paused),
 723      * and it's ok even not taking the mutex. However the best way is
 724      * to take the lock before sending the message header, and release
 725      * the lock after sending the bitmap.
 726      */
 727     qemu_mutex_lock(&mis->rp_mutex);
 728     res = ramblock_recv_bitmap_send(mis->to_src_file, block_name);
 729     qemu_mutex_unlock(&mis->rp_mutex);
 730
 731     trace_migrate_send_rp_recv_bitmap(block_name, res);
 732 }
 733
 734 void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value)
 735 {
 736     uint32_t buf;
 737
 738     buf = cpu_to_be32(value);
 739     migrate_send_rp_message(mis, MIG_RP_MSG_RESUME_ACK, sizeof(buf), &buf);
 740 }
 741
 742 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
 743 {
 744     MigrationCapabilityStatusList *head = NULL;
 745     MigrationCapabilityStatusList *caps;
 746     MigrationState *s = migrate_get_current();
 747     int i;
 748
 749     caps = NULL; /* silence compiler warning */
 750     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
 751 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
 752         if (i == MIGRATION_CAPABILITY_BLOCK) {
 753             continue;
 754         }
 755 #endif
 756         if (head == NULL) {
 757             head = g_malloc0(sizeof(*caps));
 758             caps = head;
 759         } else {
 760             caps->next = g_malloc0(sizeof(*caps));
 761             caps = caps->next;
 762         }
 763         caps->value =
 764             g_malloc(sizeof(*caps->value));
 765         caps->value->capability = i;
 766         caps->value->state = s->enabled_capabilities[i];
 767     }
 768
 769     return head;
 770 }
 771
 772 MigrationParameters *qmp_query_migrate_parameters(Error **errp)
 773 {
 774     MigrationParameters *params;
 775     MigrationState *s = migrate_get_current();
 776
 777     /* TODO use QAPI_CLONE() instead of duplicating it inline */
 778     params = g_malloc0(sizeof(*params));
 779     params->has_compress_level = true;
 780     params->compress_level = s->parameters.compress_level;
 781     params->has_compress_threads = true;
 782     params->compress_threads = s->parameters.compress_threads;
 783     params->has_compress_wait_thread = true;
 784     params->compress_wait_thread = s->parameters.compress_wait_thread;
 785     params->has_decompress_threads = true;
 786     params->decompress_threads = s->parameters.decompress_threads;
 787     params->has_throttle_trigger_threshold = true;
 788     params->throttle_trigger_threshold = s->parameters.throttle_trigger_threshold;
 789     params->has_cpu_throttle_initial = true;
 790     params->cpu_throttle_initial = s->parameters.cpu_throttle_initial;
 791     params->has_cpu_throttle_increment = true;
 792     params->cpu_throttle_increment = s->parameters.cpu_throttle_increment;
 793     params->has_cpu_throttle_tailslow = true;
 794     params->cpu_throttle_tailslow = s->parameters.cpu_throttle_tailslow;
 795     params->has_tls_creds = true;
 796     params->tls_creds = g_strdup(s->parameters.tls_creds);
 797     params->has_tls_hostname = true;
 798     params->tls_hostname = g_strdup(s->parameters.tls_hostname);
 799     params->has_tls_authz = true;
 800     params->tls_authz = g_strdup(s->parameters.tls_authz ?
 801                                  s->parameters.tls_authz : "");
 802     params->has_max_bandwidth = true;
 803     params->max_bandwidth = s->parameters.max_bandwidth;
 804     params->has_downtime_limit = true;
 805     params->downtime_limit = s->parameters.downtime_limit;
 806     params->has_x_checkpoint_delay = true;
 807     params->x_checkpoint_delay = s->parameters.x_checkpoint_delay;
 808     params->has_block_incremental = true;
 809     params->block_incremental = s->parameters.block_incremental;
 810     params->has_multifd_channels = true;
 811     params->multifd_channels = s->parameters.multifd_channels;
 812     params->has_multifd_compression = true;
 813     params->multifd_compression = s->parameters.multifd_compression;
 814     params->has_multifd_zlib_level = true;
 815     params->multifd_zlib_level = s->parameters.multifd_zlib_level;
 816     params->has_multifd_zstd_level = true;
 817     params->multifd_zstd_level = s->parameters.multifd_zstd_level;
 818     params->has_xbzrle_cache_size = true;
 819     params->xbzrle_cache_size = s->parameters.xbzrle_cache_size;
 820     params->has_max_postcopy_bandwidth = true;
 821     params->max_postcopy_bandwidth = s->parameters.max_postcopy_bandwidth;
 822     params->has_max_cpu_throttle = true;
 823     params->max_cpu_throttle = s->parameters.max_cpu_throttle;
 824     params->has_announce_initial = true;
 825     params->announce_initial = s->parameters.announce_initial;
 826     params->has_announce_max = true;
 827     params->announce_max = s->parameters.announce_max;
 828     params->has_announce_rounds = true;
 829     params->announce_rounds = s->parameters.announce_rounds;
 830     params->has_announce_step = true;
 831     params->announce_step = s->parameters.announce_step;
 832
 833     return params;
 834 }
 835
 836 AnnounceParameters *migrate_announce_params(void)
 837 {
 838     static AnnounceParameters ap;
 839
 840     MigrationState *s = migrate_get_current();
 841
 842     ap.initial = s->parameters.announce_initial;
 843     ap.max = s->parameters.announce_max;
 844     ap.rounds = s->parameters.announce_rounds;
 845     ap.step = s->parameters.announce_step;
 846
 847     return &ap;
 848 }
 849
 850 /*
 851  * Return true if we're already in the middle of a migration
 852  * (i.e. any of the active or setup states)
 853  */
 854 bool migration_is_setup_or_active(int state)
 855 {
 856     switch (state) {
 857     case MIGRATION_STATUS_ACTIVE:
 858     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 859     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 860     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 861     case MIGRATION_STATUS_SETUP:
 862     case MIGRATION_STATUS_PRE_SWITCHOVER:
 863     case MIGRATION_STATUS_DEVICE:
 864     case MIGRATION_STATUS_WAIT_UNPLUG:
 865     case MIGRATION_STATUS_COLO:
 866         return true;
 867
 868     default:
 869         return false;
 870
 871     }
 872 }
 873
 874 bool migration_is_running(int state)
 875 {
 876     switch (state) {
 877     case MIGRATION_STATUS_ACTIVE:
 878     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 879     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 880     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 881     case MIGRATION_STATUS_SETUP:
 882     case MIGRATION_STATUS_PRE_SWITCHOVER:
 883     case MIGRATION_STATUS_DEVICE:
 884     case MIGRATION_STATUS_WAIT_UNPLUG:
 885     case MIGRATION_STATUS_CANCELLING:
 886         return true;
 887
 888     default:
 889         return false;
 890
 891     }
 892 }
 893
 894 static void populate_time_info(MigrationInfo *info, MigrationState *s)
 895 {
 896     info->has_status = true;
 897     info->has_setup_time = true;
 898     info->setup_time = s->setup_time;
 899     if (s->state == MIGRATION_STATUS_COMPLETED) {
 900         info->has_total_time = true;
 901         info->total_time = s->total_time;
 902         info->has_downtime = true;
 903         info->downtime = s->downtime;
 904     } else {
 905         info->has_total_time = true;
 906         info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) -
 907                            s->start_time;
 908         info->has_expected_downtime = true;
 909         info->expected_downtime = s->expected_downtime;
 910     }
 911 }
 912
 913 static void populate_ram_info(MigrationInfo *info, MigrationState *s)
 914 {
 915     info->has_ram = true;
 916     info->ram = g_malloc0(sizeof(*info->ram));
 917     info->ram->transferred = ram_counters.transferred;
 918     info->ram->total = ram_bytes_total();
 919     info->ram->duplicate = ram_counters.duplicate;
 920     /* legacy value.  It is not used anymore */
 921     info->ram->skipped = 0;
 922     info->ram->normal = ram_counters.normal;
 923     info->ram->normal_bytes = ram_counters.normal *
 924         qemu_target_page_size();
 925     info->ram->mbps = s->mbps;
 926     info->ram->dirty_sync_count = ram_counters.dirty_sync_count;
 927     info->ram->postcopy_requests = ram_counters.postcopy_requests;
 928     info->ram->page_size = qemu_target_page_size();
 929     info->ram->multifd_bytes = ram_counters.multifd_bytes;
 930     info->ram->pages_per_second = s->pages_per_second;
 931
 932     if (migrate_use_xbzrle()) {
 933         info->has_xbzrle_cache = true;
 934         info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
 935         info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
 936         info->xbzrle_cache->bytes = xbzrle_counters.bytes;
 937         info->xbzrle_cache->pages = xbzrle_counters.pages;
 938         info->xbzrle_cache->cache_miss = xbzrle_counters.cache_miss;
 939         info->xbzrle_cache->cache_miss_rate = xbzrle_counters.cache_miss_rate;
 940         info->xbzrle_cache->encoding_rate = xbzrle_counters.encoding_rate;
 941         info->xbzrle_cache->overflow = xbzrle_counters.overflow;
 942     }
 943
 944     if (migrate_use_compression()) {
 945         info->has_compression = true;
 946         info->compression = g_malloc0(sizeof(*info->compression));
 947         info->compression->pages = compression_counters.pages;
 948         info->compression->busy = compression_counters.busy;
 949         info->compression->busy_rate = compression_counters.busy_rate;
 950         info->compression->compressed_size =
 951                                     compression_counters.compressed_size;
 952         info->compression->compression_rate =
 953                                     compression_counters.compression_rate;
 954     }
 955
 956     if (cpu_throttle_active()) {
 957         info->has_cpu_throttle_percentage = true;
 958         info->cpu_throttle_percentage = cpu_throttle_get_percentage();
 959     }
 960
 961     if (s->state != MIGRATION_STATUS_COMPLETED) {
 962         info->ram->remaining = ram_bytes_remaining();
 963         info->ram->dirty_pages_rate = ram_counters.dirty_pages_rate;
 964     }
 965 }
 966
 967 static void populate_disk_info(MigrationInfo *info)
 968 {
 969     if (blk_mig_active()) {
 970         info->has_disk = true;
 971         info->disk = g_malloc0(sizeof(*info->disk));
 972         info->disk->transferred = blk_mig_bytes_transferred();
 973         info->disk->remaining = blk_mig_bytes_remaining();
 974         info->disk->total = blk_mig_bytes_total();
 975     }
 976 }
 977
 978 static void fill_source_migration_info(MigrationInfo *info)
 979 {
 980     MigrationState *s = migrate_get_current();
 981
 982     switch (s->state) {
 983     case MIGRATION_STATUS_NONE:
 984         /* no migration has happened ever */
 985         /* do not overwrite destination migration status */
 986         return;
 987     case MIGRATION_STATUS_SETUP:
 988         info->has_status = true;
 989         info->has_total_time = false;
 990         break;
 991     case MIGRATION_STATUS_ACTIVE:
 992     case MIGRATION_STATUS_CANCELLING:
 993     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 994     case MIGRATION_STATUS_PRE_SWITCHOVER:
 995     case MIGRATION_STATUS_DEVICE:
 996     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 997     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 998         /* TODO add some postcopy stats */
 999         populate_time_info(info, s);
1000         populate_ram_info(info, s);
1001         populate_disk_info(info);
1002         break;
1003     case MIGRATION_STATUS_COLO:
1004         info->has_status = true;
1005         /* TODO: display COLO specific information (checkpoint info etc.) */
1006         break;
1007     case MIGRATION_STATUS_COMPLETED:
1008         populate_time_info(info, s);
1009         populate_ram_info(info, s);
1010         break;
1011     case MIGRATION_STATUS_FAILED:
1012         info->has_status = true;
1013         if (s->error) {
1014             info->has_error_desc = true;
1015             info->error_desc = g_strdup(error_get_pretty(s->error));
1016         }
1017         break;
1018     case MIGRATION_STATUS_CANCELLED:
1019         info->has_status = true;
1020         break;
1021     case MIGRATION_STATUS_WAIT_UNPLUG:
1022         info->has_status = true;
1023         break;
1024     }
1025     info->status = s->state;
1026 }
1027
1028 /**
1029  * @migration_caps_check - check capability validity
1030  *
1031  * @cap_list: old capability list, array of bool
1032  * @params: new capabilities to be applied soon
1033  * @errp: set *errp if the check failed, with reason
1034  *
1035  * Returns true if check passed, otherwise false.
1036  */
1037 static bool migrate_caps_check(bool *cap_list,
1038                                MigrationCapabilityStatusList *params,
1039                                Error **errp)
1040 {
1041     MigrationCapabilityStatusList *cap;
1042     bool old_postcopy_cap;
1043     MigrationIncomingState *mis = migration_incoming_get_current();
1044
1045     old_postcopy_cap = cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM];
1046
1047     for (cap = params; cap; cap = cap->next) {
1048         cap_list[cap->value->capability] = cap->value->state;
1049     }
1050
1051 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
1052     if (cap_list[MIGRATION_CAPABILITY_BLOCK]) {
1053         error_setg(errp, "QEMU compiled without old-style (blk/-b, inc/-i) "
1054                    "block migration");
1055         error_append_hint(errp, "Use drive_mirror+NBD instead.\n");
1056         return false;
1057     }
1058 #endif
1059
1060 #ifndef CONFIG_REPLICATION
1061     if (cap_list[MIGRATION_CAPABILITY_X_COLO]) {
1062         error_setg(errp, "QEMU compiled without replication module"
1063                    " can't enable COLO");
1064         error_append_hint(errp, "Please enable replication before COLO.\n");
1065         return false;
1066     }
1067 #endif
1068
1069     if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
1070         /* This check is reasonably expensive, so only when it's being
1071          * set the first time, also it's only the destination that needs
1072          * special support.
1073          */
1074         if (!old_postcopy_cap && runstate_check(RUN_STATE_INMIGRATE) &&
1075             !postcopy_ram_supported_by_host(mis)) {
1076             /* postcopy_ram_supported_by_host will have emitted a more
1077              * detailed message
1078              */
1079             error_setg(errp, "Postcopy is not supported");
1080             return false;
1081         }
1082
1083         if (cap_list[MIGRATION_CAPABILITY_X_IGNORE_SHARED]) {
1084             error_setg(errp, "Postcopy is not compatible with ignore-shared");
1085             return false;
1086         }
1087     }
1088
1089     return true;
1090 }
1091
1092 static void fill_destination_migration_info(MigrationInfo *info)
1093 {
1094     MigrationIncomingState *mis = migration_incoming_get_current();
1095
1096     if (mis->socket_address_list) {
1097         info->has_socket_address = true;
1098         info->socket_address =
1099             QAPI_CLONE(SocketAddressList, mis->socket_address_list);
1100     }
1101
1102     switch (mis->state) {
1103     case MIGRATION_STATUS_NONE:
1104         return;
1105     case MIGRATION_STATUS_SETUP:
1106     case MIGRATION_STATUS_CANCELLING:
1107     case MIGRATION_STATUS_CANCELLED:
1108     case MIGRATION_STATUS_ACTIVE:
1109     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1110     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1111     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1112     case MIGRATION_STATUS_FAILED:
1113     case MIGRATION_STATUS_COLO:
1114         info->has_status = true;
1115         break;
1116     case MIGRATION_STATUS_COMPLETED:
1117         info->has_status = true;
1118         fill_destination_postcopy_migration_info(info);
1119         break;
1120     }
1121     info->status = mis->state;
1122 }
1123
1124 MigrationInfo *qmp_query_migrate(Error **errp)
1125 {
1126     MigrationInfo *info = g_malloc0(sizeof(*info));
1127
1128     fill_destination_migration_info(info);
1129     fill_source_migration_info(info);
1130
1131     return info;
1132 }
1133
1134 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
1135                                   Error **errp)
1136 {
1137     MigrationState *s = migrate_get_current();
1138     MigrationCapabilityStatusList *cap;
1139     bool cap_list[MIGRATION_CAPABILITY__MAX];
1140
1141     if (migration_is_running(s->state)) {
1142         error_setg(errp, QERR_MIGRATION_ACTIVE);
1143         return;
1144     }
1145
1146     memcpy(cap_list, s->enabled_capabilities, sizeof(cap_list));
1147     if (!migrate_caps_check(cap_list, params, errp)) {
1148         return;
1149     }
1150
1151     for (cap = params; cap; cap = cap->next) {
1152         s->enabled_capabilities[cap->value->capability] = cap->value->state;
1153     }
1154 }
1155
1156 /*
1157  * Check whether the parameters are valid. Error will be put into errp
1158  * (if provided). Return true if valid, otherwise false.
1159  */
1160 static bool migrate_params_check(MigrationParameters *params, Error **errp)
1161 {
1162     if (params->has_compress_level &&
1163         (params->compress_level > 9)) {
1164         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level",
1165                    "is invalid, it should be in the range of 0 to 9");
1166         return false;
1167     }
1168
1169     if (params->has_compress_threads && (params->compress_threads < 1)) {
1170         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1171                    "compress_threads",
1172                    "is invalid, it should be in the range of 1 to 255");
1173         return false;
1174     }
1175
1176     if (params->has_decompress_threads && (params->decompress_threads < 1)) {
1177         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1178                    "decompress_threads",
1179                    "is invalid, it should be in the range of 1 to 255");
1180         return false;
1181     }
1182
1183     if (params->has_throttle_trigger_threshold &&
1184         (params->throttle_trigger_threshold < 1 ||
1185          params->throttle_trigger_threshold > 100)) {
1186         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1187                    "throttle_trigger_threshold",
1188                    "an integer in the range of 1 to 100");
1189         return false;
1190     }
1191
1192     if (params->has_cpu_throttle_initial &&
1193         (params->cpu_throttle_initial < 1 ||
1194          params->cpu_throttle_initial > 99)) {
1195         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1196                    "cpu_throttle_initial",
1197                    "an integer in the range of 1 to 99");
1198         return false;
1199     }
1200
1201     if (params->has_cpu_throttle_increment &&
1202         (params->cpu_throttle_increment < 1 ||
1203          params->cpu_throttle_increment > 99)) {
1204         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1205                    "cpu_throttle_increment",
1206                    "an integer in the range of 1 to 99");
1207         return false;
1208     }
1209
1210     if (params->has_max_bandwidth && (params->max_bandwidth > SIZE_MAX)) {
1211         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1212                    "max_bandwidth",
1213                    "an integer in the range of 0 to "stringify(SIZE_MAX)
1214                    " bytes/second");
1215         return false;
1216     }
1217
1218     if (params->has_downtime_limit &&
1219         (params->downtime_limit > MAX_MIGRATE_DOWNTIME)) {
1220         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1221                    "downtime_limit",
1222                    "an integer in the range of 0 to "
1223                     stringify(MAX_MIGRATE_DOWNTIME)" ms");
1224         return false;
1225     }
1226
1227     /* x_checkpoint_delay is now always positive */
1228
1229     if (params->has_multifd_channels && (params->multifd_channels < 1)) {
1230         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1231                    "multifd_channels",
1232                    "is invalid, it should be in the range of 1 to 255");
1233         return false;
1234     }
1235
1236     if (params->has_multifd_zlib_level &&
1237         (params->multifd_zlib_level > 9)) {
1238         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zlib_level",
1239                    "is invalid, it should be in the range of 0 to 9");
1240         return false;
1241     }
1242
1243     if (params->has_multifd_zstd_level &&
1244         (params->multifd_zstd_level > 20)) {
1245         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zstd_level",
1246                    "is invalid, it should be in the range of 0 to 20");
1247         return false;
1248     }
1249
1250     if (params->has_xbzrle_cache_size &&
1251         (params->xbzrle_cache_size < qemu_target_page_size() ||
1252          !is_power_of_2(params->xbzrle_cache_size))) {
1253         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1254                    "xbzrle_cache_size",
1255                    "is invalid, it should be bigger than target page size"
1256                    " and a power of 2");
1257         return false;
1258     }
1259
1260     if (params->has_max_cpu_throttle &&
1261         (params->max_cpu_throttle < params->cpu_throttle_initial ||
1262          params->max_cpu_throttle > 99)) {
1263         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1264                    "max_cpu_throttle",
1265                    "an integer in the range of cpu_throttle_initial to 99");
1266         return false;
1267     }
1268
1269     if (params->has_announce_initial &&
1270         params->announce_initial > 100000) {
1271         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1272                    "announce_initial",
1273                    "is invalid, it must be less than 100000 ms");
1274         return false;
1275     }
1276     if (params->has_announce_max &&
1277         params->announce_max > 100000) {
1278         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1279                    "announce_max",
1280                    "is invalid, it must be less than 100000 ms");
1281        return false;
1282     }
1283     if (params->has_announce_rounds &&
1284         params->announce_rounds > 1000) {
1285         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1286                    "announce_rounds",
1287                    "is invalid, it must be in the range of 0 to 1000");
1288        return false;
1289     }
1290     if (params->has_announce_step &&
1291         (params->announce_step < 1 ||
1292         params->announce_step > 10000)) {
1293         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1294                    "announce_step",
1295                    "is invalid, it must be in the range of 1 to 10000 ms");
1296        return false;
1297     }
1298     return true;
1299 }
1300
1301 static void migrate_params_test_apply(MigrateSetParameters *params,
1302                                       MigrationParameters *dest)
1303 {
1304     *dest = migrate_get_current()->parameters;
1305
1306     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1307
1308     if (params->has_compress_level) {
1309         dest->compress_level = params->compress_level;
1310     }
1311
1312     if (params->has_compress_threads) {
1313         dest->compress_threads = params->compress_threads;
1314     }
1315
1316     if (params->has_compress_wait_thread) {
1317         dest->compress_wait_thread = params->compress_wait_thread;
1318     }
1319
1320     if (params->has_decompress_threads) {
1321         dest->decompress_threads = params->decompress_threads;
1322     }
1323
1324     if (params->has_throttle_trigger_threshold) {
1325         dest->throttle_trigger_threshold = params->throttle_trigger_threshold;
1326     }
1327
1328     if (params->has_cpu_throttle_initial) {
1329         dest->cpu_throttle_initial = params->cpu_throttle_initial;
1330     }
1331
1332     if (params->has_cpu_throttle_increment) {
1333         dest->cpu_throttle_increment = params->cpu_throttle_increment;
1334     }
1335
1336     if (params->has_cpu_throttle_tailslow) {
1337         dest->cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1338     }
1339
1340     if (params->has_tls_creds) {
1341         assert(params->tls_creds->type == QTYPE_QSTRING);
1342         dest->tls_creds = params->tls_creds->u.s;
1343     }
1344
1345     if (params->has_tls_hostname) {
1346         assert(params->tls_hostname->type == QTYPE_QSTRING);
1347         dest->tls_hostname = params->tls_hostname->u.s;
1348     }
1349
1350     if (params->has_max_bandwidth) {
1351         dest->max_bandwidth = params->max_bandwidth;
1352     }
1353
1354     if (params->has_downtime_limit) {
1355         dest->downtime_limit = params->downtime_limit;
1356     }
1357
1358     if (params->has_x_checkpoint_delay) {
1359         dest->x_checkpoint_delay = params->x_checkpoint_delay;
1360     }
1361
1362     if (params->has_block_incremental) {
1363         dest->block_incremental = params->block_incremental;
1364     }
1365     if (params->has_multifd_channels) {
1366         dest->multifd_channels = params->multifd_channels;
1367     }
1368     if (params->has_multifd_compression) {
1369         dest->multifd_compression = params->multifd_compression;
1370     }
1371     if (params->has_xbzrle_cache_size) {
1372         dest->xbzrle_cache_size = params->xbzrle_cache_size;
1373     }
1374     if (params->has_max_postcopy_bandwidth) {
1375         dest->max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1376     }
1377     if (params->has_max_cpu_throttle) {
1378         dest->max_cpu_throttle = params->max_cpu_throttle;
1379     }
1380     if (params->has_announce_initial) {
1381         dest->announce_initial = params->announce_initial;
1382     }
1383     if (params->has_announce_max) {
1384         dest->announce_max = params->announce_max;
1385     }
1386     if (params->has_announce_rounds) {
1387         dest->announce_rounds = params->announce_rounds;
1388     }
1389     if (params->has_announce_step) {
1390         dest->announce_step = params->announce_step;
1391     }
1392 }
1393
1394 static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
1395 {
1396     MigrationState *s = migrate_get_current();
1397
1398     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1399
1400     if (params->has_compress_level) {
1401         s->parameters.compress_level = params->compress_level;
1402     }
1403
1404     if (params->has_compress_threads) {
1405         s->parameters.compress_threads = params->compress_threads;
1406     }
1407
1408     if (params->has_compress_wait_thread) {
1409         s->parameters.compress_wait_thread = params->compress_wait_thread;
1410     }
1411
1412     if (params->has_decompress_threads) {
1413         s->parameters.decompress_threads = params->decompress_threads;
1414     }
1415
1416     if (params->has_throttle_trigger_threshold) {
1417         s->parameters.throttle_trigger_threshold = params->throttle_trigger_threshold;
1418     }
1419
1420     if (params->has_cpu_throttle_initial) {
1421         s->parameters.cpu_throttle_initial = params->cpu_throttle_initial;
1422     }
1423
1424     if (params->has_cpu_throttle_increment) {
1425         s->parameters.cpu_throttle_increment = params->cpu_throttle_increment;
1426     }
1427
1428     if (params->has_cpu_throttle_tailslow) {
1429         s->parameters.cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1430     }
1431
1432     if (params->has_tls_creds) {
1433         g_free(s->parameters.tls_creds);
1434         assert(params->tls_creds->type == QTYPE_QSTRING);
1435         s->parameters.tls_creds = g_strdup(params->tls_creds->u.s);
1436     }
1437
1438     if (params->has_tls_hostname) {
1439         g_free(s->parameters.tls_hostname);
1440         assert(params->tls_hostname->type == QTYPE_QSTRING);
1441         s->parameters.tls_hostname = g_strdup(params->tls_hostname->u.s);
1442     }
1443
1444     if (params->has_tls_authz) {
1445         g_free(s->parameters.tls_authz);
1446         assert(params->tls_authz->type == QTYPE_QSTRING);
1447         s->parameters.tls_authz = g_strdup(params->tls_authz->u.s);
1448     }
1449
1450     if (params->has_max_bandwidth) {
1451         s->parameters.max_bandwidth = params->max_bandwidth;
1452         if (s->to_dst_file && !migration_in_postcopy()) {
1453             qemu_file_set_rate_limit(s->to_dst_file,
1454                                 s->parameters.max_bandwidth / XFER_LIMIT_RATIO);
1455         }
1456     }
1457
1458     if (params->has_downtime_limit) {
1459         s->parameters.downtime_limit = params->downtime_limit;
1460     }
1461
1462     if (params->has_x_checkpoint_delay) {
1463         s->parameters.x_checkpoint_delay = params->x_checkpoint_delay;
1464         if (migration_in_colo_state()) {
1465             colo_checkpoint_notify(s);
1466         }
1467     }
1468
1469     if (params->has_block_incremental) {
1470         s->parameters.block_incremental = params->block_incremental;
1471     }
1472     if (params->has_multifd_channels) {
1473         s->parameters.multifd_channels = params->multifd_channels;
1474     }
1475     if (params->has_multifd_compression) {
1476         s->parameters.multifd_compression = params->multifd_compression;
1477     }
1478     if (params->has_xbzrle_cache_size) {
1479         s->parameters.xbzrle_cache_size = params->xbzrle_cache_size;
1480         xbzrle_cache_resize(params->xbzrle_cache_size, errp);
1481     }
1482     if (params->has_max_postcopy_bandwidth) {
1483         s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1484         if (s->to_dst_file && migration_in_postcopy()) {
1485             qemu_file_set_rate_limit(s->to_dst_file,
1486                     s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO);
1487         }
1488     }
1489     if (params->has_max_cpu_throttle) {
1490         s->parameters.max_cpu_throttle = params->max_cpu_throttle;
1491     }
1492     if (params->has_announce_initial) {
1493         s->parameters.announce_initial = params->announce_initial;
1494     }
1495     if (params->has_announce_max) {
1496         s->parameters.announce_max = params->announce_max;
1497     }
1498     if (params->has_announce_rounds) {
1499         s->parameters.announce_rounds = params->announce_rounds;
1500     }
1501     if (params->has_announce_step) {
1502         s->parameters.announce_step = params->announce_step;
1503     }
1504 }
1505
1506 void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
1507 {
1508     MigrationParameters tmp;
1509
1510     /* TODO Rewrite "" to null instead */
1511     if (params->has_tls_creds
1512         && params->tls_creds->type == QTYPE_QNULL) {
1513         qobject_unref(params->tls_creds->u.n);
1514         params->tls_creds->type = QTYPE_QSTRING;
1515         params->tls_creds->u.s = strdup("");
1516     }
1517     /* TODO Rewrite "" to null instead */
1518     if (params->has_tls_hostname
1519         && params->tls_hostname->type == QTYPE_QNULL) {
1520         qobject_unref(params->tls_hostname->u.n);
1521         params->tls_hostname->type = QTYPE_QSTRING;
1522         params->tls_hostname->u.s = strdup("");
1523     }
1524
1525     migrate_params_test_apply(params, &tmp);
1526
1527     if (!migrate_params_check(&tmp, errp)) {
1528         /* Invalid parameter */
1529         return;
1530     }
1531
1532     migrate_params_apply(params, errp);
1533 }
1534
1535
1536 void qmp_migrate_start_postcopy(Error **errp)
1537 {
1538     MigrationState *s = migrate_get_current();
1539
1540     if (!migrate_postcopy()) {
1541         error_setg(errp, "Enable postcopy with migrate_set_capability before"
1542                          " the start of migration");
1543         return;
1544     }
1545
1546     if (s->state == MIGRATION_STATUS_NONE) {
1547         error_setg(errp, "Postcopy must be started after migration has been"
1548                          " started");
1549         return;
1550     }
1551     /*
1552      * we don't error if migration has finished since that would be racy
1553      * with issuing this command.
1554      */
1555     atomic_set(&s->start_postcopy, true);
1556 }
1557
1558 /* shared migration helpers */
1559
1560 void migrate_set_state(int *state, int old_state, int new_state)
1561 {
1562     assert(new_state < MIGRATION_STATUS__MAX);
1563     if (atomic_cmpxchg(state, old_state, new_state) == old_state) {
1564         trace_migrate_set_state(MigrationStatus_str(new_state));
1565         migrate_generate_event(new_state);
1566     }
1567 }
1568
1569 static MigrationCapabilityStatusList *migrate_cap_add(
1570     MigrationCapabilityStatusList *list,
1571     MigrationCapability index,
1572     bool state)
1573 {
1574     MigrationCapabilityStatusList *cap;
1575
1576     cap = g_new0(MigrationCapabilityStatusList, 1);
1577     cap->value = g_new0(MigrationCapabilityStatus, 1);
1578     cap->value->capability = index;
1579     cap->value->state = state;
1580     cap->next = list;
1581
1582     return cap;
1583 }
1584
1585 void migrate_set_block_enabled(bool value, Error **errp)
1586 {
1587     MigrationCapabilityStatusList *cap;
1588
1589     cap = migrate_cap_add(NULL, MIGRATION_CAPABILITY_BLOCK, value);
1590     qmp_migrate_set_capabilities(cap, errp);
1591     qapi_free_MigrationCapabilityStatusList(cap);
1592 }
1593
1594 static void migrate_set_block_incremental(MigrationState *s, bool value)
1595 {
1596     s->parameters.block_incremental = value;
1597 }
1598
1599 static void block_cleanup_parameters(MigrationState *s)
1600 {
1601     if (s->must_remove_block_options) {
1602         /* setting to false can never fail */
1603         migrate_set_block_enabled(false, &error_abort);
1604         migrate_set_block_incremental(s, false);
1605         s->must_remove_block_options = false;
1606     }
1607 }
1608
1609 static void migrate_fd_cleanup(MigrationState *s)
1610 {
1611     qemu_bh_delete(s->cleanup_bh);
1612     s->cleanup_bh = NULL;
1613
1614     qemu_savevm_state_cleanup();
1615
1616     if (s->to_dst_file) {
1617         QEMUFile *tmp;
1618
1619         trace_migrate_fd_cleanup();
1620         qemu_mutex_unlock_iothread();
1621         if (s->migration_thread_running) {
1622             qemu_thread_join(&s->thread);
1623             s->migration_thread_running = false;
1624         }
1625         qemu_mutex_lock_iothread();
1626
1627         multifd_save_cleanup();
1628         qemu_mutex_lock(&s->qemu_file_lock);
1629         tmp = s->to_dst_file;
1630         s->to_dst_file = NULL;
1631         qemu_mutex_unlock(&s->qemu_file_lock);
1632         /*
1633          * Close the file handle without the lock to make sure the
1634          * critical section won't block for long.
1635          */
1636         qemu_fclose(tmp);
1637     }
1638
1639     assert(!migration_is_active(s));
1640
1641     if (s->state == MIGRATION_STATUS_CANCELLING) {
1642         migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING,
1643                           MIGRATION_STATUS_CANCELLED);
1644     }
1645
1646     if (s->error) {
1647         /* It is used on info migrate.  We can't free it */
1648         error_report_err(error_copy(s->error));
1649     }
1650     notifier_list_notify(&migration_state_notifiers, s);
1651     block_cleanup_parameters(s);
1652 }
1653
1654 static void migrate_fd_cleanup_schedule(MigrationState *s)
1655 {
1656     /*
1657      * Ref the state for bh, because it may be called when
1658      * there're already no other refs
1659      */
1660     object_ref(OBJECT(s));
1661     qemu_bh_schedule(s->cleanup_bh);
1662 }
1663
1664 static void migrate_fd_cleanup_bh(void *opaque)
1665 {
1666     MigrationState *s = opaque;
1667     migrate_fd_cleanup(s);
1668     object_unref(OBJECT(s));
1669 }
1670
1671 void migrate_set_error(MigrationState *s, const Error *error)
1672 {
1673     QEMU_LOCK_GUARD(&s->error_mutex);
1674     if (!s->error) {
1675         s->error = error_copy(error);
1676     }
1677 }
1678
1679 void migrate_fd_error(MigrationState *s, const Error *error)
1680 {
1681     trace_migrate_fd_error(error_get_pretty(error));
1682     assert(s->to_dst_file == NULL);
1683     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
1684                       MIGRATION_STATUS_FAILED);
1685     migrate_set_error(s, error);
1686 }
1687
1688 static void migrate_fd_cancel(MigrationState *s)
1689 {
1690     int old_state ;
1691     QEMUFile *f = migrate_get_current()->to_dst_file;
1692     trace_migrate_fd_cancel();
1693
1694     if (s->rp_state.from_dst_file) {
1695         /* shutdown the rp socket, so causing the rp thread to shutdown */
1696         qemu_file_shutdown(s->rp_state.from_dst_file);
1697     }
1698
1699     do {
1700         old_state = s->state;
1701         if (!migration_is_running(old_state)) {
1702             break;
1703         }
1704         /* If the migration is paused, kick it out of the pause */
1705         if (old_state == MIGRATION_STATUS_PRE_SWITCHOVER) {
1706             qemu_sem_post(&s->pause_sem);
1707         }
1708         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING);
1709     } while (s->state != MIGRATION_STATUS_CANCELLING);
1710
1711     /*
1712      * If we're unlucky the migration code might be stuck somewhere in a
1713      * send/write while the network has failed and is waiting to timeout;
1714      * if we've got shutdown(2) available then we can force it to quit.
1715      * The outgoing qemu file gets closed in migrate_fd_cleanup that is
1716      * called in a bh, so there is no race against this cancel.
1717      */
1718     if (s->state == MIGRATION_STATUS_CANCELLING && f) {
1719         qemu_file_shutdown(f);
1720     }
1721     if (s->state == MIGRATION_STATUS_CANCELLING && s->block_inactive) {
1722         Error *local_err = NULL;
1723
1724         bdrv_invalidate_cache_all(&local_err);
1725         if (local_err) {
1726             error_report_err(local_err);
1727         } else {
1728             s->block_inactive = false;
1729         }
1730     }
1731 }
1732
1733 void add_migration_state_change_notifier(Notifier *notify)
1734 {
1735     notifier_list_add(&migration_state_notifiers, notify);
1736 }
1737
1738 void remove_migration_state_change_notifier(Notifier *notify)
1739 {
1740     notifier_remove(notify);
1741 }
1742
1743 bool migration_in_setup(MigrationState *s)
1744 {
1745     return s->state == MIGRATION_STATUS_SETUP;
1746 }
1747
1748 bool migration_has_finished(MigrationState *s)
1749 {
1750     return s->state == MIGRATION_STATUS_COMPLETED;
1751 }
1752
1753 bool migration_has_failed(MigrationState *s)
1754 {
1755     return (s->state == MIGRATION_STATUS_CANCELLED ||
1756             s->state == MIGRATION_STATUS_FAILED);
1757 }
1758
1759 bool migration_in_postcopy(void)
1760 {
1761     MigrationState *s = migrate_get_current();
1762
1763     switch (s->state) {
1764     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1765     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1766     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1767         return true;
1768     default:
1769         return false;
1770     }
1771 }
1772
1773 bool migration_in_postcopy_after_devices(MigrationState *s)
1774 {
1775     return migration_in_postcopy() && s->postcopy_after_devices;
1776 }
1777
1778 bool migration_in_incoming_postcopy(void)
1779 {
1780     PostcopyState ps = postcopy_state_get();
1781
1782     return ps >= POSTCOPY_INCOMING_DISCARD && ps < POSTCOPY_INCOMING_END;
1783 }
1784
1785 bool migration_is_idle(void)
1786 {
1787     MigrationState *s = current_migration;
1788
1789     if (!s) {
1790         return true;
1791     }
1792
1793     switch (s->state) {
1794     case MIGRATION_STATUS_NONE:
1795     case MIGRATION_STATUS_CANCELLED:
1796     case MIGRATION_STATUS_COMPLETED:
1797     case MIGRATION_STATUS_FAILED:
1798         return true;
1799     case MIGRATION_STATUS_SETUP:
1800     case MIGRATION_STATUS_CANCELLING:
1801     case MIGRATION_STATUS_ACTIVE:
1802     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1803     case MIGRATION_STATUS_COLO:
1804     case MIGRATION_STATUS_PRE_SWITCHOVER:
1805     case MIGRATION_STATUS_DEVICE:
1806     case MIGRATION_STATUS_WAIT_UNPLUG:
1807         return false;
1808     case MIGRATION_STATUS__MAX:
1809         g_assert_not_reached();
1810     }
1811
1812     return false;
1813 }
1814
1815 bool migration_is_active(MigrationState *s)
1816 {
1817     return (s->state == MIGRATION_STATUS_ACTIVE ||
1818             s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
1819 }
1820
1821 void migrate_init(MigrationState *s)
1822 {
1823     /*
1824      * Reinitialise all migration state, except
1825      * parameters/capabilities that the user set, and
1826      * locks.
1827      */
1828     s->cleanup_bh = 0;
1829     s->to_dst_file = NULL;
1830     s->state = MIGRATION_STATUS_NONE;
1831     s->rp_state.from_dst_file = NULL;
1832     s->rp_state.error = false;
1833     s->mbps = 0.0;
1834     s->pages_per_second = 0.0;
1835     s->downtime = 0;
1836     s->expected_downtime = 0;
1837     s->setup_time = 0;
1838     s->start_postcopy = false;
1839     s->postcopy_after_devices = false;
1840     s->migration_thread_running = false;
1841     error_free(s->error);
1842     s->error = NULL;
1843
1844     migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
1845
1846     s->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1847     s->total_time = 0;
1848     s->vm_was_running = false;
1849     s->iteration_initial_bytes = 0;
1850     s->threshold_size = 0;
1851 }
1852
1853 static GSList *migration_blockers;
1854
1855 int migrate_add_blocker(Error *reason, Error **errp)
1856 {
1857     if (only_migratable) {
1858         error_propagate_prepend(errp, error_copy(reason),
1859                                 "disallowing migration blocker "
1860                                 "(--only-migratable) for: ");
1861         return -EACCES;
1862     }
1863
1864     if (migration_is_idle()) {
1865         migration_blockers = g_slist_prepend(migration_blockers, reason);
1866         return 0;
1867     }
1868
1869     error_propagate_prepend(errp, error_copy(reason),
1870                             "disallowing migration blocker "
1871                             "(migration in progress) for: ");
1872     return -EBUSY;
1873 }
1874
1875 void migrate_del_blocker(Error *reason)
1876 {
1877     migration_blockers = g_slist_remove(migration_blockers, reason);
1878 }
1879
1880 void qmp_migrate_incoming(const char *uri, Error **errp)
1881 {
1882     Error *local_err = NULL;
1883     static bool once = true;
1884
1885     if (!deferred_incoming) {
1886         error_setg(errp, "For use with '-incoming defer'");
1887         return;
1888     }
1889     if (!once) {
1890         error_setg(errp, "The incoming migration has already been started");
1891         return;
1892     }
1893
1894     qemu_start_incoming_migration(uri, &local_err);
1895
1896     if (local_err) {
1897         error_propagate(errp, local_err);
1898         return;
1899     }
1900
1901     once = false;
1902 }
1903
1904 void qmp_migrate_recover(const char *uri, Error **errp)
1905 {
1906     MigrationIncomingState *mis = migration_incoming_get_current();
1907
1908     if (mis->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
1909         error_setg(errp, "Migrate recover can only be run "
1910                    "when postcopy is paused.");
1911         return;
1912     }
1913
1914     if (atomic_cmpxchg(&mis->postcopy_recover_triggered,
1915                        false, true) == true) {
1916         error_setg(errp, "Migrate recovery is triggered already");
1917         return;
1918     }
1919
1920     /*
1921      * Note that this call will never start a real migration; it will
1922      * only re-setup the migration stream and poke existing migration
1923      * to continue using that newly established channel.
1924      */
1925     qemu_start_incoming_migration(uri, errp);
1926 }
1927
1928 void qmp_migrate_pause(Error **errp)
1929 {
1930     MigrationState *ms = migrate_get_current();
1931     MigrationIncomingState *mis = migration_incoming_get_current();
1932     int ret;
1933
1934     if (ms->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
1935         /* Source side, during postcopy */
1936         qemu_mutex_lock(&ms->qemu_file_lock);
1937         ret = qemu_file_shutdown(ms->to_dst_file);
1938         qemu_mutex_unlock(&ms->qemu_file_lock);
1939         if (ret) {
1940             error_setg(errp, "Failed to pause source migration");
1941         }
1942         return;
1943     }
1944
1945     if (mis->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
1946         ret = qemu_file_shutdown(mis->from_src_file);
1947         if (ret) {
1948             error_setg(errp, "Failed to pause destination migration");
1949         }
1950         return;
1951     }
1952
1953     error_setg(errp, "migrate-pause is currently only supported "
1954                "during postcopy-active state");
1955 }
1956
1957 bool migration_is_blocked(Error **errp)
1958 {
1959     if (qemu_savevm_state_blocked(errp)) {
1960         return true;
1961     }
1962
1963     if (migration_blockers) {
1964         error_propagate(errp, error_copy(migration_blockers->data));
1965         return true;
1966     }
1967
1968     return false;
1969 }
1970
1971 /* Returns true if continue to migrate, or false if error detected */
1972 static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
1973                             bool resume, Error **errp)
1974 {
1975     Error *local_err = NULL;
1976
1977     if (resume) {
1978         if (s->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
1979             error_setg(errp, "Cannot resume if there is no "
1980                        "paused migration");
1981             return false;
1982         }
1983
1984         /*
1985          * Postcopy recovery won't work well with release-ram
1986          * capability since release-ram will drop the page buffer as
1987          * long as the page is put into the send buffer.  So if there
1988          * is a network failure happened, any page buffers that have
1989          * not yet reached the destination VM but have already been
1990          * sent from the source VM will be lost forever.  Let's refuse
1991          * the client from resuming such a postcopy migration.
1992          * Luckily release-ram was designed to only be used when src
1993          * and destination VMs are on the same host, so it should be
1994          * fine.
1995          */
1996         if (migrate_release_ram()) {
1997             error_setg(errp, "Postcopy recovery cannot work "
1998                        "when release-ram capability is set");
1999             return false;
2000         }
2001
2002         /* This is a resume, skip init status */
2003         return true;
2004     }
2005
2006     if (migration_is_running(s->state)) {
2007         error_setg(errp, QERR_MIGRATION_ACTIVE);
2008         return false;
2009     }
2010
2011     if (runstate_check(RUN_STATE_INMIGRATE)) {
2012         error_setg(errp, "Guest is waiting for an incoming migration");
2013         return false;
2014     }
2015
2016     if (migration_is_blocked(errp)) {
2017         return false;
2018     }
2019
2020     if (blk || blk_inc) {
2021         if (migrate_use_block() || migrate_use_block_incremental()) {
2022             error_setg(errp, "Command options are incompatible with "
2023                        "current migration capabilities");
2024             return false;
2025         }
2026         migrate_set_block_enabled(true, &local_err);
2027         if (local_err) {
2028             error_propagate(errp, local_err);
2029             return false;
2030         }
2031         s->must_remove_block_options = true;
2032     }
2033
2034     if (blk_inc) {
2035         migrate_set_block_incremental(s, true);
2036     }
2037
2038     migrate_init(s);
2039     /*
2040      * set ram_counters memory to zero for a
2041      * new migration
2042      */
2043     memset(&ram_counters, 0, sizeof(ram_counters));
2044
2045     return true;
2046 }
2047
2048 void qmp_migrate(const char *uri, bool has_blk, bool blk,
2049                  bool has_inc, bool inc, bool has_detach, bool detach,
2050                  bool has_resume, bool resume, Error **errp)
2051 {
2052     Error *local_err = NULL;
2053     MigrationState *s = migrate_get_current();
2054     const char *p;
2055
2056     if (!migrate_prepare(s, has_blk && blk, has_inc && inc,
2057                          has_resume && resume, errp)) {
2058         /* Error detected, put into errp */
2059         return;
2060     }
2061
2062     if (strstart(uri, "tcp:", &p)) {
2063         tcp_start_outgoing_migration(s, p, &local_err);
2064 #ifdef CONFIG_RDMA
2065     } else if (strstart(uri, "rdma:", &p)) {
2066         rdma_start_outgoing_migration(s, p, &local_err);
2067 #endif
2068     } else if (strstart(uri, "exec:", &p)) {
2069         exec_start_outgoing_migration(s, p, &local_err);
2070     } else if (strstart(uri, "unix:", &p)) {
2071         unix_start_outgoing_migration(s, p, &local_err);
2072     } else if (strstart(uri, "fd:", &p)) {
2073         fd_start_outgoing_migration(s, p, &local_err);
2074     } else {
2075         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri",
2076                    "a valid migration protocol");
2077         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
2078                           MIGRATION_STATUS_FAILED);
2079         block_cleanup_parameters(s);
2080         return;
2081     }
2082
2083     if (local_err) {
2084         migrate_fd_error(s, local_err);
2085         error_propagate(errp, local_err);
2086         return;
2087     }
2088 }
2089
2090 void qmp_migrate_cancel(Error **errp)
2091 {
2092     migrate_fd_cancel(migrate_get_current());
2093 }
2094
2095 void qmp_migrate_continue(MigrationStatus state, Error **errp)
2096 {
2097     MigrationState *s = migrate_get_current();
2098     if (s->state != state) {
2099         error_setg(errp,  "Migration not in expected state: %s",
2100                    MigrationStatus_str(s->state));
2101         return;
2102     }
2103     qemu_sem_post(&s->pause_sem);
2104 }
2105
2106 void qmp_migrate_set_cache_size(int64_t value, Error **errp)
2107 {
2108     MigrateSetParameters p = {
2109         .has_xbzrle_cache_size = true,
2110         .xbzrle_cache_size = value,
2111     };
2112
2113     qmp_migrate_set_parameters(&p, errp);
2114 }
2115
2116 int64_t qmp_query_migrate_cache_size(Error **errp)
2117 {
2118     return migrate_xbzrle_cache_size();
2119 }
2120
2121 void qmp_migrate_set_speed(int64_t value, Error **errp)
2122 {
2123     MigrateSetParameters p = {
2124         .has_max_bandwidth = true,
2125         .max_bandwidth = value,
2126     };
2127
2128     qmp_migrate_set_parameters(&p, errp);
2129 }
2130
2131 void qmp_migrate_set_downtime(double value, Error **errp)
2132 {
2133     if (value < 0 || value > MAX_MIGRATE_DOWNTIME_SECONDS) {
2134         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
2135                    "downtime_limit",
2136                    "an integer in the range of 0 to "
2137                     stringify(MAX_MIGRATE_DOWNTIME_SECONDS)" seconds");
2138         return;
2139     }
2140
2141     value *= 1000; /* Convert to milliseconds */
2142
2143     MigrateSetParameters p = {
2144         .has_downtime_limit = true,
2145         .downtime_limit = (int64_t)value,
2146     };
2147
2148     qmp_migrate_set_parameters(&p, errp);
2149 }
2150
2151 bool migrate_release_ram(void)
2152 {
2153     MigrationState *s;
2154
2155     s = migrate_get_current();
2156
2157     return s->enabled_capabilities[MIGRATION_CAPABILITY_RELEASE_RAM];
2158 }
2159
2160 bool migrate_postcopy_ram(void)
2161 {
2162     MigrationState *s;
2163
2164     s = migrate_get_current();
2165
2166     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM];
2167 }
2168
2169 bool migrate_postcopy(void)
2170 {
2171     return migrate_postcopy_ram() || migrate_dirty_bitmaps();
2172 }
2173
2174 bool migrate_auto_converge(void)
2175 {
2176     MigrationState *s;
2177
2178     s = migrate_get_current();
2179
2180     return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
2181 }
2182
2183 bool migrate_zero_blocks(void)
2184 {
2185     MigrationState *s;
2186
2187     s = migrate_get_current();
2188
2189     return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
2190 }
2191
2192 bool migrate_postcopy_blocktime(void)
2193 {
2194     MigrationState *s;
2195
2196     s = migrate_get_current();
2197
2198     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME];
2199 }
2200
2201 bool migrate_use_compression(void)
2202 {
2203     MigrationState *s;
2204
2205     s = migrate_get_current();
2206
2207     return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS];
2208 }
2209
2210 int migrate_compress_level(void)
2211 {
2212     MigrationState *s;
2213
2214     s = migrate_get_current();
2215
2216     return s->parameters.compress_level;
2217 }
2218
2219 int migrate_compress_threads(void)
2220 {
2221     MigrationState *s;
2222
2223     s = migrate_get_current();
2224
2225     return s->parameters.compress_threads;
2226 }
2227
2228 int migrate_compress_wait_thread(void)
2229 {
2230     MigrationState *s;
2231
2232     s = migrate_get_current();
2233
2234     return s->parameters.compress_wait_thread;
2235 }
2236
2237 int migrate_decompress_threads(void)
2238 {
2239     MigrationState *s;
2240
2241     s = migrate_get_current();
2242
2243     return s->parameters.decompress_threads;
2244 }
2245
2246 bool migrate_dirty_bitmaps(void)
2247 {
2248     MigrationState *s;
2249
2250     s = migrate_get_current();
2251
2252     return s->enabled_capabilities[MIGRATION_CAPABILITY_DIRTY_BITMAPS];
2253 }
2254
2255 bool migrate_ignore_shared(void)
2256 {
2257     MigrationState *s;
2258
2259     s = migrate_get_current();
2260
2261     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_IGNORE_SHARED];
2262 }
2263
2264 bool migrate_validate_uuid(void)
2265 {
2266     MigrationState *s;
2267
2268     s = migrate_get_current();
2269
2270     return s->enabled_capabilities[MIGRATION_CAPABILITY_VALIDATE_UUID];
2271 }
2272
2273 bool migrate_use_events(void)
2274 {
2275     MigrationState *s;
2276
2277     s = migrate_get_current();
2278
2279     return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS];
2280 }
2281
2282 bool migrate_use_multifd(void)
2283 {
2284     MigrationState *s;
2285
2286     s = migrate_get_current();
2287
2288     return s->enabled_capabilities[MIGRATION_CAPABILITY_MULTIFD];
2289 }
2290
2291 bool migrate_pause_before_switchover(void)
2292 {
2293     MigrationState *s;
2294
2295     s = migrate_get_current();
2296
2297     return s->enabled_capabilities[
2298         MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER];
2299 }
2300
2301 int migrate_multifd_channels(void)
2302 {
2303     MigrationState *s;
2304
2305     s = migrate_get_current();
2306
2307     return s->parameters.multifd_channels;
2308 }
2309
2310 MultiFDCompression migrate_multifd_compression(void)
2311 {
2312     MigrationState *s;
2313
2314     s = migrate_get_current();
2315
2316     return s->parameters.multifd_compression;
2317 }
2318
2319 int migrate_multifd_zlib_level(void)
2320 {
2321     MigrationState *s;
2322
2323     s = migrate_get_current();
2324
2325     return s->parameters.multifd_zlib_level;
2326 }
2327
2328 int migrate_multifd_zstd_level(void)
2329 {
2330     MigrationState *s;
2331
2332     s = migrate_get_current();
2333
2334     return s->parameters.multifd_zstd_level;
2335 }
2336
2337 int migrate_use_xbzrle(void)
2338 {
2339     MigrationState *s;
2340
2341     s = migrate_get_current();
2342
2343     return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
2344 }
2345
2346 int64_t migrate_xbzrle_cache_size(void)
2347 {
2348     MigrationState *s;
2349
2350     s = migrate_get_current();
2351
2352     return s->parameters.xbzrle_cache_size;
2353 }
2354
2355 static int64_t migrate_max_postcopy_bandwidth(void)
2356 {
2357     MigrationState *s;
2358
2359     s = migrate_get_current();
2360
2361     return s->parameters.max_postcopy_bandwidth;
2362 }
2363
2364 bool migrate_use_block(void)
2365 {
2366     MigrationState *s;
2367
2368     s = migrate_get_current();
2369
2370     return s->enabled_capabilities[MIGRATION_CAPABILITY_BLOCK];
2371 }
2372
2373 bool migrate_use_return_path(void)
2374 {
2375     MigrationState *s;
2376
2377     s = migrate_get_current();
2378
2379     return s->enabled_capabilities[MIGRATION_CAPABILITY_RETURN_PATH];
2380 }
2381
2382 bool migrate_use_block_incremental(void)
2383 {
2384     MigrationState *s;
2385
2386     s = migrate_get_current();
2387
2388     return s->parameters.block_incremental;
2389 }
2390
2391 /* migration thread support */
2392 /*
2393  * Something bad happened to the RP stream, mark an error
2394  * The caller shall print or trace something to indicate why
2395  */
2396 static void mark_source_rp_bad(MigrationState *s)
2397 {
2398     s->rp_state.error = true;
2399 }
2400
2401 static struct rp_cmd_args {
2402     ssize_t     len; /* -1 = variable */
2403     const char *name;
2404 } rp_cmd_args[] = {
2405     [MIG_RP_MSG_INVALID]        = { .len = -1, .name = "INVALID" },
2406     [MIG_RP_MSG_SHUT]           = { .len =  4, .name = "SHUT" },
2407     [MIG_RP_MSG_PONG]           = { .len =  4, .name = "PONG" },
2408     [MIG_RP_MSG_REQ_PAGES]      = { .len = 12, .name = "REQ_PAGES" },
2409     [MIG_RP_MSG_REQ_PAGES_ID]   = { .len = -1, .name = "REQ_PAGES_ID" },
2410     [MIG_RP_MSG_RECV_BITMAP]    = { .len = -1, .name = "RECV_BITMAP" },
2411     [MIG_RP_MSG_RESUME_ACK]     = { .len =  4, .name = "RESUME_ACK" },
2412     [MIG_RP_MSG_MAX]            = { .len = -1, .name = "MAX" },
2413 };
2414
2415 /*
2416  * Process a request for pages received on the return path,
2417  * We're allowed to send more than requested (e.g. to round to our page size)
2418  * and we don't need to send pages that have already been sent.
2419  */
2420 static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
2421                                        ram_addr_t start, size_t len)
2422 {
2423     long our_host_ps = qemu_real_host_page_size;
2424
2425     trace_migrate_handle_rp_req_pages(rbname, start, len);
2426
2427     /*
2428      * Since we currently insist on matching page sizes, just sanity check
2429      * we're being asked for whole host pages.
2430      */
2431     if (start & (our_host_ps-1) ||
2432        (len & (our_host_ps-1))) {
2433         error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
2434                      " len: %zd", __func__, start, len);
2435         mark_source_rp_bad(ms);
2436         return;
2437     }
2438
2439     if (ram_save_queue_pages(rbname, start, len)) {
2440         mark_source_rp_bad(ms);
2441     }
2442 }
2443
2444 /* Return true to retry, false to quit */
2445 static bool postcopy_pause_return_path_thread(MigrationState *s)
2446 {
2447     trace_postcopy_pause_return_path();
2448
2449     qemu_sem_wait(&s->postcopy_pause_rp_sem);
2450
2451     trace_postcopy_pause_return_path_continued();
2452
2453     return true;
2454 }
2455
2456 static int migrate_handle_rp_recv_bitmap(MigrationState *s, char *block_name)
2457 {
2458     RAMBlock *block = qemu_ram_block_by_name(block_name);
2459
2460     if (!block) {
2461         error_report("%s: invalid block name '%s'", __func__, block_name);
2462         return -EINVAL;
2463     }
2464
2465     /* Fetch the received bitmap and refresh the dirty bitmap */
2466     return ram_dirty_bitmap_reload(s, block);
2467 }
2468
2469 static int migrate_handle_rp_resume_ack(MigrationState *s, uint32_t value)
2470 {
2471     trace_source_return_path_thread_resume_ack(value);
2472
2473     if (value != MIGRATION_RESUME_ACK_VALUE) {
2474         error_report("%s: illegal resume_ack value %"PRIu32,
2475                      __func__, value);
2476         return -1;
2477     }
2478
2479     /* Now both sides are active. */
2480     migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_RECOVER,
2481                       MIGRATION_STATUS_POSTCOPY_ACTIVE);
2482
2483     /* Notify send thread that time to continue send pages */
2484     qemu_sem_post(&s->rp_state.rp_sem);
2485
2486     return 0;
2487 }
2488
2489 /*
2490  * Handles messages sent on the return path towards the source VM
2491  *
2492  */
2493 static void *source_return_path_thread(void *opaque)
2494 {
2495     MigrationState *ms = opaque;
2496     QEMUFile *rp = ms->rp_state.from_dst_file;
2497     uint16_t header_len, header_type;
2498     uint8_t buf[512];
2499     uint32_t tmp32, sibling_error;
2500     ram_addr_t start = 0; /* =0 to silence warning */
2501     size_t  len = 0, expected_len;
2502     int res;
2503
2504     trace_source_return_path_thread_entry();
2505     rcu_register_thread();
2506
2507 retry:
2508     while (!ms->rp_state.error && !qemu_file_get_error(rp) &&
2509            migration_is_setup_or_active(ms->state)) {
2510         trace_source_return_path_thread_loop_top();
2511         header_type = qemu_get_be16(rp);
2512         header_len = qemu_get_be16(rp);
2513
2514         if (qemu_file_get_error(rp)) {
2515             mark_source_rp_bad(ms);
2516             goto out;
2517         }
2518
2519         if (header_type >= MIG_RP_MSG_MAX ||
2520             header_type == MIG_RP_MSG_INVALID) {
2521             error_report("RP: Received invalid message 0x%04x length 0x%04x",
2522                          header_type, header_len);
2523             mark_source_rp_bad(ms);
2524             goto out;
2525         }
2526
2527         if ((rp_cmd_args[header_type].len != -1 &&
2528             header_len != rp_cmd_args[header_type].len) ||
2529             header_len > sizeof(buf)) {
2530             error_report("RP: Received '%s' message (0x%04x) with"
2531                          "incorrect length %d expecting %zu",
2532                          rp_cmd_args[header_type].name, header_type, header_len,
2533                          (size_t)rp_cmd_args[header_type].len);
2534             mark_source_rp_bad(ms);
2535             goto out;
2536         }
2537
2538         /* We know we've got a valid header by this point */
2539         res = qemu_get_buffer(rp, buf, header_len);
2540         if (res != header_len) {
2541             error_report("RP: Failed reading data for message 0x%04x"
2542                          " read %d expected %d",
2543                          header_type, res, header_len);
2544             mark_source_rp_bad(ms);
2545             goto out;
2546         }
2547
2548         /* OK, we have the message and the data */
2549         switch (header_type) {
2550         case MIG_RP_MSG_SHUT:
2551             sibling_error = ldl_be_p(buf);
2552             trace_source_return_path_thread_shut(sibling_error);
2553             if (sibling_error) {
2554                 error_report("RP: Sibling indicated error %d", sibling_error);
2555                 mark_source_rp_bad(ms);
2556             }
2557             /*
2558              * We'll let the main thread deal with closing the RP
2559              * we could do a shutdown(2) on it, but we're the only user
2560              * anyway, so there's nothing gained.
2561              */
2562             goto out;
2563
2564         case MIG_RP_MSG_PONG:
2565             tmp32 = ldl_be_p(buf);
2566             trace_source_return_path_thread_pong(tmp32);
2567             break;
2568
2569         case MIG_RP_MSG_REQ_PAGES:
2570             start = ldq_be_p(buf);
2571             len = ldl_be_p(buf + 8);
2572             migrate_handle_rp_req_pages(ms, NULL, start, len);
2573             break;
2574
2575         case MIG_RP_MSG_REQ_PAGES_ID:
2576             expected_len = 12 + 1; /* header + termination */
2577
2578             if (header_len >= expected_len) {
2579                 start = ldq_be_p(buf);
2580                 len = ldl_be_p(buf + 8);
2581                 /* Now we expect an idstr */
2582                 tmp32 = buf[12]; /* Length of the following idstr */
2583                 buf[13 + tmp32] = '\0';
2584                 expected_len += tmp32;
2585             }
2586             if (header_len != expected_len) {
2587                 error_report("RP: Req_Page_id with length %d expecting %zd",
2588                              header_len, expected_len);
2589                 mark_source_rp_bad(ms);
2590                 goto out;
2591             }
2592             migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len);
2593             break;
2594
2595         case MIG_RP_MSG_RECV_BITMAP:
2596             if (header_len < 1) {
2597                 error_report("%s: missing block name", __func__);
2598                 mark_source_rp_bad(ms);
2599                 goto out;
2600             }
2601             /* Format: len (1B) + idstr (<255B). This ends the idstr. */
2602             buf[buf[0] + 1] = '\0';
2603             if (migrate_handle_rp_recv_bitmap(ms, (char *)(buf + 1))) {
2604                 mark_source_rp_bad(ms);
2605                 goto out;
2606             }
2607             break;
2608
2609         case MIG_RP_MSG_RESUME_ACK:
2610             tmp32 = ldl_be_p(buf);
2611             if (migrate_handle_rp_resume_ack(ms, tmp32)) {
2612                 mark_source_rp_bad(ms);
2613                 goto out;
2614             }
2615             break;
2616
2617         default:
2618             break;
2619         }
2620     }
2621
2622 out:
2623     res = qemu_file_get_error(rp);
2624     if (res) {
2625         if (res == -EIO && migration_in_postcopy()) {
2626             /*
2627              * Maybe there is something we can do: it looks like a
2628              * network down issue, and we pause for a recovery.
2629              */
2630             if (postcopy_pause_return_path_thread(ms)) {
2631                 /* Reload rp, reset the rest */
2632                 if (rp != ms->rp_state.from_dst_file) {
2633                     qemu_fclose(rp);
2634                     rp = ms->rp_state.from_dst_file;
2635                 }
2636                 ms->rp_state.error = false;
2637                 goto retry;
2638             }
2639         }
2640
2641         trace_source_return_path_thread_bad_end();
2642         mark_source_rp_bad(ms);
2643     }
2644
2645     trace_source_return_path_thread_end();
2646     ms->rp_state.from_dst_file = NULL;
2647     qemu_fclose(rp);
2648     rcu_unregister_thread();
2649     return NULL;
2650 }
2651
2652 static int open_return_path_on_source(MigrationState *ms,
2653                                       bool create_thread)
2654 {
2655
2656     ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file);
2657     if (!ms->rp_state.from_dst_file) {
2658         return -1;
2659     }
2660
2661     trace_open_return_path_on_source();
2662
2663     if (!create_thread) {
2664         /* We're done */
2665         return 0;
2666     }
2667
2668     qemu_thread_create(&ms->rp_state.rp_thread, "return path",
2669                        source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
2670
2671     trace_open_return_path_on_source_continue();
2672
2673     return 0;
2674 }
2675
2676 /* Returns 0 if the RP was ok, otherwise there was an error on the RP */
2677 static int await_return_path_close_on_source(MigrationState *ms)
2678 {
2679     /*
2680      * If this is a normal exit then the destination will send a SHUT and the
2681      * rp_thread will exit, however if there's an error we need to cause
2682      * it to exit.
2683      */
2684     if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) {
2685         /*
2686          * shutdown(2), if we have it, will cause it to unblock if it's stuck
2687          * waiting for the destination.
2688          */
2689         qemu_file_shutdown(ms->rp_state.from_dst_file);
2690         mark_source_rp_bad(ms);
2691     }
2692     trace_await_return_path_close_on_source_joining();
2693     qemu_thread_join(&ms->rp_state.rp_thread);
2694     trace_await_return_path_close_on_source_close();
2695     return ms->rp_state.error;
2696 }
2697
2698 /*
2699  * Switch from normal iteration to postcopy
2700  * Returns non-0 on error
2701  */
2702 static int postcopy_start(MigrationState *ms)
2703 {
2704     int ret;
2705     QIOChannelBuffer *bioc;
2706     QEMUFile *fb;
2707     int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2708     int64_t bandwidth = migrate_max_postcopy_bandwidth();
2709     bool restart_block = false;
2710     int cur_state = MIGRATION_STATUS_ACTIVE;
2711     if (!migrate_pause_before_switchover()) {
2712         migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
2713                           MIGRATION_STATUS_POSTCOPY_ACTIVE);
2714     }
2715
2716     trace_postcopy_start();
2717     qemu_mutex_lock_iothread();
2718     trace_postcopy_start_set_run();
2719
2720     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
2721     global_state_store();
2722     ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
2723     if (ret < 0) {
2724         goto fail;
2725     }
2726
2727     ret = migration_maybe_pause(ms, &cur_state,
2728                                 MIGRATION_STATUS_POSTCOPY_ACTIVE);
2729     if (ret < 0) {
2730         goto fail;
2731     }
2732
2733     ret = bdrv_inactivate_all();
2734     if (ret < 0) {
2735         goto fail;
2736     }
2737     restart_block = true;
2738
2739     /*
2740      * Cause any non-postcopiable, but iterative devices to
2741      * send out their final data.
2742      */
2743     qemu_savevm_state_complete_precopy(ms->to_dst_file, true, false);
2744
2745     /*
2746      * in Finish migrate and with the io-lock held everything should
2747      * be quiet, but we've potentially still got dirty pages and we
2748      * need to tell the destination to throw any pages it's already received
2749      * that are dirty
2750      */
2751     if (migrate_postcopy_ram()) {
2752         if (ram_postcopy_send_discard_bitmap(ms)) {
2753             error_report("postcopy send discard bitmap failed");
2754             goto fail;
2755         }
2756     }
2757
2758     /*
2759      * send rest of state - note things that are doing postcopy
2760      * will notice we're in POSTCOPY_ACTIVE and not actually
2761      * wrap their state up here
2762      */
2763     /* 0 max-postcopy-bandwidth means unlimited */
2764     if (!bandwidth) {
2765         qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX);
2766     } else {
2767         qemu_file_set_rate_limit(ms->to_dst_file, bandwidth / XFER_LIMIT_RATIO);
2768     }
2769     if (migrate_postcopy_ram()) {
2770         /* Ping just for debugging, helps line traces up */
2771         qemu_savevm_send_ping(ms->to_dst_file, 2);
2772     }
2773
2774     /*
2775      * While loading the device state we may trigger page transfer
2776      * requests and the fd must be free to process those, and thus
2777      * the destination must read the whole device state off the fd before
2778      * it starts processing it.  Unfortunately the ad-hoc migration format
2779      * doesn't allow the destination to know the size to read without fully
2780      * parsing it through each devices load-state code (especially the open
2781      * coded devices that use get/put).
2782      * So we wrap the device state up in a package with a length at the start;
2783      * to do this we use a qemu_buf to hold the whole of the device state.
2784      */
2785     bioc = qio_channel_buffer_new(4096);
2786     qio_channel_set_name(QIO_CHANNEL(bioc), "migration-postcopy-buffer");
2787     fb = qemu_fopen_channel_output(QIO_CHANNEL(bioc));
2788     object_unref(OBJECT(bioc));
2789
2790     /*
2791      * Make sure the receiver can get incoming pages before we send the rest
2792      * of the state
2793      */
2794     qemu_savevm_send_postcopy_listen(fb);
2795
2796     qemu_savevm_state_complete_precopy(fb, false, false);
2797     if (migrate_postcopy_ram()) {
2798         qemu_savevm_send_ping(fb, 3);
2799     }
2800
2801     qemu_savevm_send_postcopy_run(fb);
2802
2803     /* <><> end of stuff going into the package */
2804
2805     /* Last point of recovery; as soon as we send the package the destination
2806      * can open devices and potentially start running.
2807      * Lets just check again we've not got any errors.
2808      */
2809     ret = qemu_file_get_error(ms->to_dst_file);
2810     if (ret) {
2811         error_report("postcopy_start: Migration stream errored (pre package)");
2812         goto fail_closefb;
2813     }
2814
2815     restart_block = false;
2816
2817     /* Now send that blob */
2818     if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
2819         goto fail_closefb;
2820     }
2821     qemu_fclose(fb);
2822
2823     /* Send a notify to give a chance for anything that needs to happen
2824      * at the transition to postcopy and after the device state; in particular
2825      * spice needs to trigger a transition now
2826      */
2827     ms->postcopy_after_devices = true;
2828     notifier_list_notify(&migration_state_notifiers, ms);
2829
2830     ms->downtime =  qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop;
2831
2832     qemu_mutex_unlock_iothread();
2833
2834     if (migrate_postcopy_ram()) {
2835         /*
2836          * Although this ping is just for debug, it could potentially be
2837          * used for getting a better measurement of downtime at the source.
2838          */
2839         qemu_savevm_send_ping(ms->to_dst_file, 4);
2840     }
2841
2842     if (migrate_release_ram()) {
2843         ram_postcopy_migrated_memory_release(ms);
2844     }
2845
2846     ret = qemu_file_get_error(ms->to_dst_file);
2847     if (ret) {
2848         error_report("postcopy_start: Migration stream errored");
2849         migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
2850                               MIGRATION_STATUS_FAILED);
2851     }
2852
2853     return ret;
2854
2855 fail_closefb:
2856     qemu_fclose(fb);
2857 fail:
2858     migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
2859                           MIGRATION_STATUS_FAILED);
2860     if (restart_block) {
2861         /* A failure happened early enough that we know the destination hasn't
2862          * accessed block devices, so we're safe to recover.
2863          */
2864         Error *local_err = NULL;
2865
2866         bdrv_invalidate_cache_all(&local_err);
2867         if (local_err) {
2868             error_report_err(local_err);
2869         }
2870     }
2871     qemu_mutex_unlock_iothread();
2872     return -1;
2873 }
2874
2875 /**
2876  * migration_maybe_pause: Pause if required to by
2877  * migrate_pause_before_switchover called with the iothread locked
2878  * Returns: 0 on success
2879  */
2880 static int migration_maybe_pause(MigrationState *s,
2881                                  int *current_active_state,
2882                                  int new_state)
2883 {
2884     if (!migrate_pause_before_switchover()) {
2885         return 0;
2886     }
2887
2888     /* Since leaving this state is not atomic with posting the semaphore
2889      * it's possible that someone could have issued multiple migrate_continue
2890      * and the semaphore is incorrectly positive at this point;
2891      * the docs say it's undefined to reinit a semaphore that's already
2892      * init'd, so use timedwait to eat up any existing posts.
2893      */
2894     while (qemu_sem_timedwait(&s->pause_sem, 1) == 0) {
2895         /* This block intentionally left blank */
2896     }
2897
2898     /*
2899      * If the migration is cancelled when it is in the completion phase,
2900      * the migration state is set to MIGRATION_STATUS_CANCELLING.
2901      * So we don't need to wait a semaphore, otherwise we would always
2902      * wait for the 'pause_sem' semaphore.
2903      */
2904     if (s->state != MIGRATION_STATUS_CANCELLING) {
2905         qemu_mutex_unlock_iothread();
2906         migrate_set_state(&s->state, *current_active_state,
2907                           MIGRATION_STATUS_PRE_SWITCHOVER);
2908         qemu_sem_wait(&s->pause_sem);
2909         migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER,
2910                           new_state);
2911         *current_active_state = new_state;
2912         qemu_mutex_lock_iothread();
2913     }
2914
2915     return s->state == new_state ? 0 : -EINVAL;
2916 }
2917
2918 /**
2919  * migration_completion: Used by migration_thread when there's not much left.
2920  *   The caller 'breaks' the loop when this returns.
2921  *
2922  * @s: Current migration state
2923  */
2924 static void migration_completion(MigrationState *s)
2925 {
2926     int ret;
2927     int current_active_state = s->state;
2928
2929     if (s->state == MIGRATION_STATUS_ACTIVE) {
2930         qemu_mutex_lock_iothread();
2931         s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2932         qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
2933         s->vm_was_running = runstate_is_running();
2934         ret = global_state_store();
2935
2936         if (!ret) {
2937             bool inactivate = !migrate_colo_enabled();
2938             ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
2939             if (ret >= 0) {
2940                 ret = migration_maybe_pause(s, &current_active_state,
2941                                             MIGRATION_STATUS_DEVICE);
2942             }
2943             if (ret >= 0) {
2944                 qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
2945                 ret = qemu_savevm_state_complete_precopy(s->to_dst_file, false,
2946                                                          inactivate);
2947             }
2948             if (inactivate && ret >= 0) {
2949                 s->block_inactive = true;
2950             }
2951         }
2952         qemu_mutex_unlock_iothread();
2953
2954         if (ret < 0) {
2955             goto fail;
2956         }
2957     } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2958         trace_migration_completion_postcopy_end();
2959
2960         qemu_savevm_state_complete_postcopy(s->to_dst_file);
2961         trace_migration_completion_postcopy_end_after_complete();
2962     }
2963
2964     /*
2965      * If rp was opened we must clean up the thread before
2966      * cleaning everything else up (since if there are no failures
2967      * it will wait for the destination to send it's status in
2968      * a SHUT command).
2969      */
2970     if (s->rp_state.from_dst_file) {
2971         int rp_error;
2972         trace_migration_return_path_end_before();
2973         rp_error = await_return_path_close_on_source(s);
2974         trace_migration_return_path_end_after(rp_error);
2975         if (rp_error) {
2976             goto fail_invalidate;
2977         }
2978     }
2979
2980     if (qemu_file_get_error(s->to_dst_file)) {
2981         trace_migration_completion_file_err();
2982         goto fail_invalidate;
2983     }
2984
2985     if (!migrate_colo_enabled()) {
2986         migrate_set_state(&s->state, current_active_state,
2987                           MIGRATION_STATUS_COMPLETED);
2988     }
2989
2990     return;
2991
2992 fail_invalidate:
2993     /* If not doing postcopy, vm_start() will be called: let's regain
2994      * control on images.
2995      */
2996     if (s->state == MIGRATION_STATUS_ACTIVE ||
2997         s->state == MIGRATION_STATUS_DEVICE) {
2998         Error *local_err = NULL;
2999
3000         qemu_mutex_lock_iothread();
3001         bdrv_invalidate_cache_all(&local_err);
3002         if (local_err) {
3003             error_report_err(local_err);
3004         } else {
3005             s->block_inactive = false;
3006         }
3007         qemu_mutex_unlock_iothread();
3008     }
3009
3010 fail:
3011     migrate_set_state(&s->state, current_active_state,
3012                       MIGRATION_STATUS_FAILED);
3013 }
3014
3015 bool migrate_colo_enabled(void)
3016 {
3017     MigrationState *s = migrate_get_current();
3018     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_COLO];
3019 }
3020
3021 typedef enum MigThrError {
3022     /* No error detected */
3023     MIG_THR_ERR_NONE = 0,
3024     /* Detected error, but resumed successfully */
3025     MIG_THR_ERR_RECOVERED = 1,
3026     /* Detected fatal error, need to exit */
3027     MIG_THR_ERR_FATAL = 2,
3028 } MigThrError;
3029
3030 static int postcopy_resume_handshake(MigrationState *s)
3031 {
3032     qemu_savevm_send_postcopy_resume(s->to_dst_file);
3033
3034     while (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3035         qemu_sem_wait(&s->rp_state.rp_sem);
3036     }
3037
3038     if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3039         return 0;
3040     }
3041
3042     return -1;
3043 }
3044
3045 /* Return zero if success, or <0 for error */
3046 static int postcopy_do_resume(MigrationState *s)
3047 {
3048     int ret;
3049
3050     /*
3051      * Call all the resume_prepare() hooks, so that modules can be
3052      * ready for the migration resume.
3053      */
3054     ret = qemu_savevm_state_resume_prepare(s);
3055     if (ret) {
3056         error_report("%s: resume_prepare() failure detected: %d",
3057                      __func__, ret);
3058         return ret;
3059     }
3060
3061     /*
3062      * Last handshake with destination on the resume (destination will
3063      * switch to postcopy-active afterwards)
3064      */
3065     ret = postcopy_resume_handshake(s);
3066     if (ret) {
3067         error_report("%s: handshake failed: %d", __func__, ret);
3068         return ret;
3069     }
3070
3071     return 0;
3072 }
3073
3074 /*
3075  * We don't return until we are in a safe state to continue current
3076  * postcopy migration.  Returns MIG_THR_ERR_RECOVERED if recovered, or
3077  * MIG_THR_ERR_FATAL if unrecovery failure happened.
3078  */
3079 static MigThrError postcopy_pause(MigrationState *s)
3080 {
3081     assert(s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
3082
3083     while (true) {
3084         QEMUFile *file;
3085
3086         migrate_set_state(&s->state, s->state,
3087                           MIGRATION_STATUS_POSTCOPY_PAUSED);
3088
3089         /* Current channel is possibly broken. Release it. */
3090         assert(s->to_dst_file);
3091         qemu_mutex_lock(&s->qemu_file_lock);
3092         file = s->to_dst_file;
3093         s->to_dst_file = NULL;
3094         qemu_mutex_unlock(&s->qemu_file_lock);
3095
3096         qemu_file_shutdown(file);
3097         qemu_fclose(file);
3098
3099         error_report("Detected IO failure for postcopy. "
3100                      "Migration paused.");
3101
3102         /*
3103          * We wait until things fixed up. Then someone will setup the
3104          * status back for us.
3105          */
3106         while (s->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
3107             qemu_sem_wait(&s->postcopy_pause_sem);
3108         }
3109
3110         if (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3111             /* Woken up by a recover procedure. Give it a shot */
3112
3113             /*
3114              * Firstly, let's wake up the return path now, with a new
3115              * return path channel.
3116              */
3117             qemu_sem_post(&s->postcopy_pause_rp_sem);
3118
3119             /* Do the resume logic */
3120             if (postcopy_do_resume(s) == 0) {
3121                 /* Let's continue! */
3122                 trace_postcopy_pause_continued();
3123                 return MIG_THR_ERR_RECOVERED;
3124             } else {
3125                 /*
3126                  * Something wrong happened during the recovery, let's
3127                  * pause again. Pause is always better than throwing
3128                  * data away.
3129                  */
3130                 continue;
3131             }
3132         } else {
3133             /* This is not right... Time to quit. */
3134             return MIG_THR_ERR_FATAL;
3135         }
3136     }
3137 }
3138
3139 static MigThrError migration_detect_error(MigrationState *s)
3140 {
3141     int ret;
3142     int state = s->state;
3143     Error *local_error = NULL;
3144
3145     if (state == MIGRATION_STATUS_CANCELLING ||
3146         state == MIGRATION_STATUS_CANCELLED) {
3147         /* End the migration, but don't set the state to failed */
3148         return MIG_THR_ERR_FATAL;
3149     }
3150
3151     /* Try to detect any file errors */
3152     ret = qemu_file_get_error_obj(s->to_dst_file, &local_error);
3153     if (!ret) {
3154         /* Everything is fine */
3155         assert(!local_error);
3156         return MIG_THR_ERR_NONE;
3157     }
3158
3159     if (local_error) {
3160         migrate_set_error(s, local_error);
3161         error_free(local_error);
3162     }
3163
3164     if (state == MIGRATION_STATUS_POSTCOPY_ACTIVE && ret == -EIO) {
3165         /*
3166          * For postcopy, we allow the network to be down for a
3167          * while. After that, it can be continued by a
3168          * recovery phase.
3169          */
3170         return postcopy_pause(s);
3171     } else {
3172         /*
3173          * For precopy (or postcopy with error outside IO), we fail
3174          * with no time.
3175          */
3176         migrate_set_state(&s->state, state, MIGRATION_STATUS_FAILED);
3177         trace_migration_thread_file_err();
3178
3179         /* Time to stop the migration, now. */
3180         return MIG_THR_ERR_FATAL;
3181     }
3182 }
3183
3184 /* How many bytes have we transferred since the beginning of the migration */
3185 static uint64_t migration_total_bytes(MigrationState *s)
3186 {
3187     return qemu_ftell(s->to_dst_file) + ram_counters.multifd_bytes;
3188 }
3189
3190 static void migration_calculate_complete(MigrationState *s)
3191 {
3192     uint64_t bytes = migration_total_bytes(s);
3193     int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3194     int64_t transfer_time;
3195
3196     s->total_time = end_time - s->start_time;
3197     if (!s->downtime) {
3198         /*
3199          * It's still not set, so we are precopy migration.  For
3200          * postcopy, downtime is calculated during postcopy_start().
3201          */
3202         s->downtime = end_time - s->downtime_start;
3203     }
3204
3205     transfer_time = s->total_time - s->setup_time;
3206     if (transfer_time) {
3207         s->mbps = ((double) bytes * 8.0) / transfer_time / 1000;
3208     }
3209 }
3210
3211 static void update_iteration_initial_status(MigrationState *s)
3212 {
3213     /*
3214      * Update these three fields at the same time to avoid mismatch info lead
3215      * wrong speed calculation.
3216      */
3217     s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3218     s->iteration_initial_bytes = migration_total_bytes(s);
3219     s->iteration_initial_pages = ram_get_total_transferred_pages();
3220 }
3221
3222 static void migration_update_counters(MigrationState *s,
3223                                       int64_t current_time)
3224 {
3225     uint64_t transferred, transferred_pages, time_spent;
3226     uint64_t current_bytes; /* bytes transferred since the beginning */
3227     double bandwidth;
3228
3229     if (current_time < s->iteration_start_time + BUFFER_DELAY) {
3230         return;
3231     }
3232
3233     current_bytes = migration_total_bytes(s);
3234     transferred = current_bytes - s->iteration_initial_bytes;
3235     time_spent = current_time - s->iteration_start_time;
3236     bandwidth = (double)transferred / time_spent;
3237     s->threshold_size = bandwidth * s->parameters.downtime_limit;
3238
3239     s->mbps = (((double) transferred * 8.0) /
3240                ((double) time_spent / 1000.0)) / 1000.0 / 1000.0;
3241
3242     transferred_pages = ram_get_total_transferred_pages() -
3243                             s->iteration_initial_pages;
3244     s->pages_per_second = (double) transferred_pages /
3245                              (((double) time_spent / 1000.0));
3246
3247     /*
3248      * if we haven't sent anything, we don't want to
3249      * recalculate. 10000 is a small enough number for our purposes
3250      */
3251     if (ram_counters.dirty_pages_rate && transferred > 10000) {
3252         s->expected_downtime = ram_counters.remaining / bandwidth;
3253     }
3254
3255     qemu_file_reset_rate_limit(s->to_dst_file);
3256
3257     update_iteration_initial_status(s);
3258
3259     trace_migrate_transferred(transferred, time_spent,
3260                               bandwidth, s->threshold_size);
3261 }
3262
3263 /* Migration thread iteration status */
3264 typedef enum {
3265     MIG_ITERATE_RESUME,         /* Resume current iteration */
3266     MIG_ITERATE_SKIP,           /* Skip current iteration */
3267     MIG_ITERATE_BREAK,          /* Break the loop */
3268 } MigIterateState;
3269
3270 /*
3271  * Return true if continue to the next iteration directly, false
3272  * otherwise.
3273  */
3274 static MigIterateState migration_iteration_run(MigrationState *s)
3275 {
3276     uint64_t pending_size, pend_pre, pend_compat, pend_post;
3277     bool in_postcopy = s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE;
3278
3279     qemu_savevm_state_pending(s->to_dst_file, s->threshold_size, &pend_pre,
3280                               &pend_compat, &pend_post);
3281     pending_size = pend_pre + pend_compat + pend_post;
3282
3283     trace_migrate_pending(pending_size, s->threshold_size,
3284                           pend_pre, pend_compat, pend_post);
3285
3286     if (pending_size && pending_size >= s->threshold_size) {
3287         /* Still a significant amount to transfer */
3288         if (!in_postcopy && pend_pre <= s->threshold_size &&
3289             atomic_read(&s->start_postcopy)) {
3290             if (postcopy_start(s)) {
3291                 error_report("%s: postcopy failed to start", __func__);
3292             }
3293             return MIG_ITERATE_SKIP;
3294         }
3295         /* Just another iteration step */
3296         qemu_savevm_state_iterate(s->to_dst_file, in_postcopy);
3297     } else {
3298         trace_migration_thread_low_pending(pending_size);
3299         migration_completion(s);
3300         return MIG_ITERATE_BREAK;
3301     }
3302
3303     return MIG_ITERATE_RESUME;
3304 }
3305
3306 static void migration_iteration_finish(MigrationState *s)
3307 {
3308     /* If we enabled cpu throttling for auto-converge, turn it off. */
3309     cpu_throttle_stop();
3310
3311     qemu_mutex_lock_iothread();
3312     switch (s->state) {
3313     case MIGRATION_STATUS_COMPLETED:
3314         migration_calculate_complete(s);
3315         runstate_set(RUN_STATE_POSTMIGRATE);
3316         break;
3317
3318     case MIGRATION_STATUS_ACTIVE:
3319         /*
3320          * We should really assert here, but since it's during
3321          * migration, let's try to reduce the usage of assertions.
3322          */
3323         if (!migrate_colo_enabled()) {
3324             error_report("%s: critical error: calling COLO code without "
3325                          "COLO enabled", __func__);
3326         }
3327         migrate_start_colo_process(s);
3328         /*
3329          * Fixme: we will run VM in COLO no matter its old running state.
3330          * After exited COLO, we will keep running.
3331          */
3332         s->vm_was_running = true;
3333         /* Fallthrough */
3334     case MIGRATION_STATUS_FAILED:
3335     case MIGRATION_STATUS_CANCELLED:
3336     case MIGRATION_STATUS_CANCELLING:
3337         if (s->vm_was_running) {
3338             vm_start();
3339         } else {
3340             if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
3341                 runstate_set(RUN_STATE_POSTMIGRATE);
3342             }
3343         }
3344         break;
3345
3346     default:
3347         /* Should not reach here, but if so, forgive the VM. */
3348         error_report("%s: Unknown ending state %d", __func__, s->state);
3349         break;
3350     }
3351     migrate_fd_cleanup_schedule(s);
3352     qemu_mutex_unlock_iothread();
3353 }
3354
3355 void migration_make_urgent_request(void)
3356 {
3357     qemu_sem_post(&migrate_get_current()->rate_limit_sem);
3358 }
3359
3360 void migration_consume_urgent_request(void)
3361 {
3362     qemu_sem_wait(&migrate_get_current()->rate_limit_sem);
3363 }
3364
3365 /* Returns true if the rate limiting was broken by an urgent request */
3366 bool migration_rate_limit(void)
3367 {
3368     int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3369     MigrationState *s = migrate_get_current();
3370
3371     bool urgent = false;
3372     migration_update_counters(s, now);
3373     if (qemu_file_rate_limit(s->to_dst_file)) {
3374
3375         if (qemu_file_get_error(s->to_dst_file)) {
3376             return false;
3377         }
3378         /*
3379          * Wait for a delay to do rate limiting OR
3380          * something urgent to post the semaphore.
3381          */
3382         int ms = s->iteration_start_time + BUFFER_DELAY - now;
3383         trace_migration_rate_limit_pre(ms);
3384         if (qemu_sem_timedwait(&s->rate_limit_sem, ms) == 0) {
3385             /*
3386              * We were woken by one or more urgent things but
3387              * the timedwait will have consumed one of them.
3388              * The service routine for the urgent wake will dec
3389              * the semaphore itself for each item it consumes,
3390              * so add this one we just eat back.
3391              */
3392             qemu_sem_post(&s->rate_limit_sem);
3393             urgent = true;
3394         }
3395         trace_migration_rate_limit_post(urgent);
3396     }
3397     return urgent;
3398 }
3399
3400 /*
3401  * Master migration thread on the source VM.
3402  * It drives the migration and pumps the data down the outgoing channel.
3403  */
3404 static void *migration_thread(void *opaque)
3405 {
3406     MigrationState *s = opaque;
3407     int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3408     MigThrError thr_error;
3409     bool urgent = false;
3410
3411     rcu_register_thread();
3412
3413     object_ref(OBJECT(s));
3414     update_iteration_initial_status(s);
3415
3416     qemu_savevm_state_header(s->to_dst_file);
3417
3418     /*
3419      * If we opened the return path, we need to make sure dst has it
3420      * opened as well.
3421      */
3422     if (s->rp_state.from_dst_file) {
3423         /* Now tell the dest that it should open its end so it can reply */
3424         qemu_savevm_send_open_return_path(s->to_dst_file);
3425
3426         /* And do a ping that will make stuff easier to debug */
3427         qemu_savevm_send_ping(s->to_dst_file, 1);
3428     }
3429
3430     if (migrate_postcopy()) {
3431         /*
3432          * Tell the destination that we *might* want to do postcopy later;
3433          * if the other end can't do postcopy it should fail now, nice and
3434          * early.
3435          */
3436         qemu_savevm_send_postcopy_advise(s->to_dst_file);
3437     }
3438
3439     if (migrate_colo_enabled()) {
3440         /* Notify migration destination that we enable COLO */
3441         qemu_savevm_send_colo_enable(s->to_dst_file);
3442     }
3443
3444     qemu_savevm_state_setup(s->to_dst_file);
3445
3446     if (qemu_savevm_state_guest_unplug_pending()) {
3447         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3448                           MIGRATION_STATUS_WAIT_UNPLUG);
3449
3450         while (s->state == MIGRATION_STATUS_WAIT_UNPLUG &&
3451                qemu_savevm_state_guest_unplug_pending()) {
3452             qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3453         }
3454
3455         migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG,
3456                 MIGRATION_STATUS_ACTIVE);
3457     }
3458
3459     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3460     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3461                       MIGRATION_STATUS_ACTIVE);
3462
3463     trace_migration_thread_setup_complete();
3464
3465     while (migration_is_active(s)) {
3466         if (urgent || !qemu_file_rate_limit(s->to_dst_file)) {
3467             MigIterateState iter_state = migration_iteration_run(s);
3468             if (iter_state == MIG_ITERATE_SKIP) {
3469                 continue;
3470             } else if (iter_state == MIG_ITERATE_BREAK) {
3471                 break;
3472             }
3473         }
3474
3475         /*
3476          * Try to detect any kind of failures, and see whether we
3477          * should stop the migration now.
3478          */
3479         thr_error = migration_detect_error(s);
3480         if (thr_error == MIG_THR_ERR_FATAL) {
3481             /* Stop migration */
3482             break;
3483         } else if (thr_error == MIG_THR_ERR_RECOVERED) {
3484             /*
3485              * Just recovered from a e.g. network failure, reset all
3486              * the local variables. This is important to avoid
3487              * breaking transferred_bytes and bandwidth calculation
3488              */
3489             update_iteration_initial_status(s);
3490         }
3491
3492         urgent = migration_rate_limit();
3493     }
3494
3495     trace_migration_thread_after_loop();
3496     migration_iteration_finish(s);
3497     object_unref(OBJECT(s));
3498     rcu_unregister_thread();
3499     return NULL;
3500 }
3501
3502 void migrate_fd_connect(MigrationState *s, Error *error_in)
3503 {
3504     Error *local_err = NULL;
3505     int64_t rate_limit;
3506     bool resume = s->state == MIGRATION_STATUS_POSTCOPY_PAUSED;
3507
3508     s->expected_downtime = s->parameters.downtime_limit;
3509     if (resume) {
3510         assert(s->cleanup_bh);
3511     } else {
3512         assert(!s->cleanup_bh);
3513         s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup_bh, s);
3514     }
3515     if (error_in) {
3516         migrate_fd_error(s, error_in);
3517         migrate_fd_cleanup(s);
3518         return;
3519     }
3520
3521     if (resume) {
3522         /* This is a resumed migration */
3523         rate_limit = s->parameters.max_postcopy_bandwidth /
3524             XFER_LIMIT_RATIO;
3525     } else {
3526         /* This is a fresh new migration */
3527         rate_limit = s->parameters.max_bandwidth / XFER_LIMIT_RATIO;
3528
3529         /* Notify before starting migration thread */
3530         notifier_list_notify(&migration_state_notifiers, s);
3531     }
3532
3533     qemu_file_set_rate_limit(s->to_dst_file, rate_limit);
3534     qemu_file_set_blocking(s->to_dst_file, true);
3535
3536     /*
3537      * Open the return path. For postcopy, it is used exclusively. For
3538      * precopy, only if user specified "return-path" capability would
3539      * QEMU uses the return path.
3540      */
3541     if (migrate_postcopy_ram() || migrate_use_return_path()) {
3542         if (open_return_path_on_source(s, !resume)) {
3543             error_report("Unable to open return-path for postcopy");
3544             migrate_set_state(&s->state, s->state, MIGRATION_STATUS_FAILED);
3545             migrate_fd_cleanup(s);
3546             return;
3547         }
3548     }
3549
3550     if (resume) {
3551         /* Wakeup the main migration thread to do the recovery */
3552         migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
3553                           MIGRATION_STATUS_POSTCOPY_RECOVER);
3554         qemu_sem_post(&s->postcopy_pause_sem);
3555         return;
3556     }
3557
3558     if (multifd_save_setup(&local_err) != 0) {
3559         error_report_err(local_err);
3560         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3561                           MIGRATION_STATUS_FAILED);
3562         migrate_fd_cleanup(s);
3563         return;
3564     }
3565     qemu_thread_create(&s->thread, "live_migration", migration_thread, s,
3566                        QEMU_THREAD_JOINABLE);
3567     s->migration_thread_running = true;
3568 }
3569
3570 void migration_global_dump(Monitor *mon)
3571 {
3572     MigrationState *ms = migrate_get_current();
3573
3574     monitor_printf(mon, "globals:\n");
3575     monitor_printf(mon, "store-global-state: %s\n",
3576                    ms->store_global_state ? "on" : "off");
3577     monitor_printf(mon, "only-migratable: %s\n",
3578                    only_migratable ? "on" : "off");
3579     monitor_printf(mon, "send-configuration: %s\n",
3580                    ms->send_configuration ? "on" : "off");
3581     monitor_printf(mon, "send-section-footer: %s\n",
3582                    ms->send_section_footer ? "on" : "off");
3583     monitor_printf(mon, "decompress-error-check: %s\n",
3584                    ms->decompress_error_check ? "on" : "off");
3585     monitor_printf(mon, "clear-bitmap-shift: %u\n",
3586                    ms->clear_bitmap_shift);
3587 }
3588
3589 #define DEFINE_PROP_MIG_CAP(name, x)             \
3590     DEFINE_PROP_BOOL(name, MigrationState, enabled_capabilities[x], false)
3591
3592 static Property migration_properties[] = {
3593     DEFINE_PROP_BOOL("store-global-state", MigrationState,
3594                      store_global_state, true),
3595     DEFINE_PROP_BOOL("send-configuration", MigrationState,
3596                      send_configuration, true),
3597     DEFINE_PROP_BOOL("send-section-footer", MigrationState,
3598                      send_section_footer, true),
3599     DEFINE_PROP_BOOL("decompress-error-check", MigrationState,
3600                       decompress_error_check, true),
3601     DEFINE_PROP_UINT8("x-clear-bitmap-shift", MigrationState,
3602                       clear_bitmap_shift, CLEAR_BITMAP_SHIFT_DEFAULT),
3603
3604     /* Migration parameters */
3605     DEFINE_PROP_UINT8("x-compress-level", MigrationState,
3606                       parameters.compress_level,
3607                       DEFAULT_MIGRATE_COMPRESS_LEVEL),
3608     DEFINE_PROP_UINT8("x-compress-threads", MigrationState,
3609                       parameters.compress_threads,
3610                       DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT),
3611     DEFINE_PROP_BOOL("x-compress-wait-thread", MigrationState,
3612                       parameters.compress_wait_thread, true),
3613     DEFINE_PROP_UINT8("x-decompress-threads", MigrationState,
3614                       parameters.decompress_threads,
3615                       DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT),
3616     DEFINE_PROP_UINT8("x-throttle-trigger-threshold", MigrationState,
3617                       parameters.throttle_trigger_threshold,
3618                       DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD),
3619     DEFINE_PROP_UINT8("x-cpu-throttle-initial", MigrationState,
3620                       parameters.cpu_throttle_initial,
3621                       DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL),
3622     DEFINE_PROP_UINT8("x-cpu-throttle-increment", MigrationState,
3623                       parameters.cpu_throttle_increment,
3624                       DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT),
3625     DEFINE_PROP_BOOL("x-cpu-throttle-tailslow", MigrationState,
3626                       parameters.cpu_throttle_tailslow, false),
3627     DEFINE_PROP_SIZE("x-max-bandwidth", MigrationState,
3628                       parameters.max_bandwidth, MAX_THROTTLE),
3629     DEFINE_PROP_UINT64("x-downtime-limit", MigrationState,
3630                       parameters.downtime_limit,
3631                       DEFAULT_MIGRATE_SET_DOWNTIME),
3632     DEFINE_PROP_UINT32("x-checkpoint-delay", MigrationState,
3633                       parameters.x_checkpoint_delay,
3634                       DEFAULT_MIGRATE_X_CHECKPOINT_DELAY),
3635     DEFINE_PROP_UINT8("multifd-channels", MigrationState,
3636                       parameters.multifd_channels,
3637                       DEFAULT_MIGRATE_MULTIFD_CHANNELS),
3638     DEFINE_PROP_MULTIFD_COMPRESSION("multifd-compression", MigrationState,
3639                       parameters.multifd_compression,
3640                       DEFAULT_MIGRATE_MULTIFD_COMPRESSION),
3641     DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState,
3642                       parameters.multifd_zlib_level,
3643                       DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL),
3644     DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState,
3645                       parameters.multifd_zstd_level,
3646                       DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL),
3647     DEFINE_PROP_SIZE("xbzrle-cache-size", MigrationState,
3648                       parameters.xbzrle_cache_size,
3649                       DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE),
3650     DEFINE_PROP_SIZE("max-postcopy-bandwidth", MigrationState,
3651                       parameters.max_postcopy_bandwidth,
3652                       DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH),
3653     DEFINE_PROP_UINT8("max-cpu-throttle", MigrationState,
3654                       parameters.max_cpu_throttle,
3655                       DEFAULT_MIGRATE_MAX_CPU_THROTTLE),
3656     DEFINE_PROP_SIZE("announce-initial", MigrationState,
3657                       parameters.announce_initial,
3658                       DEFAULT_MIGRATE_ANNOUNCE_INITIAL),
3659     DEFINE_PROP_SIZE("announce-max", MigrationState,
3660                       parameters.announce_max,
3661                       DEFAULT_MIGRATE_ANNOUNCE_MAX),
3662     DEFINE_PROP_SIZE("announce-rounds", MigrationState,
3663                       parameters.announce_rounds,
3664                       DEFAULT_MIGRATE_ANNOUNCE_ROUNDS),
3665     DEFINE_PROP_SIZE("announce-step", MigrationState,
3666                       parameters.announce_step,
3667                       DEFAULT_MIGRATE_ANNOUNCE_STEP),
3668
3669     /* Migration capabilities */
3670     DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
3671     DEFINE_PROP_MIG_CAP("x-rdma-pin-all", MIGRATION_CAPABILITY_RDMA_PIN_ALL),
3672     DEFINE_PROP_MIG_CAP("x-auto-converge", MIGRATION_CAPABILITY_AUTO_CONVERGE),
3673     DEFINE_PROP_MIG_CAP("x-zero-blocks", MIGRATION_CAPABILITY_ZERO_BLOCKS),
3674     DEFINE_PROP_MIG_CAP("x-compress", MIGRATION_CAPABILITY_COMPRESS),
3675     DEFINE_PROP_MIG_CAP("x-events", MIGRATION_CAPABILITY_EVENTS),
3676     DEFINE_PROP_MIG_CAP("x-postcopy-ram", MIGRATION_CAPABILITY_POSTCOPY_RAM),
3677     DEFINE_PROP_MIG_CAP("x-colo", MIGRATION_CAPABILITY_X_COLO),
3678     DEFINE_PROP_MIG_CAP("x-release-ram", MIGRATION_CAPABILITY_RELEASE_RAM),
3679     DEFINE_PROP_MIG_CAP("x-block", MIGRATION_CAPABILITY_BLOCK),
3680     DEFINE_PROP_MIG_CAP("x-return-path", MIGRATION_CAPABILITY_RETURN_PATH),
3681     DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD),
3682
3683     DEFINE_PROP_END_OF_LIST(),
3684 };
3685
3686 static void migration_class_init(ObjectClass *klass, void *data)
3687 {
3688     DeviceClass *dc = DEVICE_CLASS(klass);
3689
3690     dc->user_creatable = false;
3691     device_class_set_props(dc, migration_properties);
3692 }
3693
3694 static void migration_instance_finalize(Object *obj)
3695 {
3696     MigrationState *ms = MIGRATION_OBJ(obj);
3697     MigrationParameters *params = &ms->parameters;
3698
3699     qemu_mutex_destroy(&ms->error_mutex);
3700     qemu_mutex_destroy(&ms->qemu_file_lock);
3701     g_free(params->tls_hostname);
3702     g_free(params->tls_creds);
3703     qemu_sem_destroy(&ms->wait_unplug_sem);
3704     qemu_sem_destroy(&ms->rate_limit_sem);
3705     qemu_sem_destroy(&ms->pause_sem);
3706     qemu_sem_destroy(&ms->postcopy_pause_sem);
3707     qemu_sem_destroy(&ms->postcopy_pause_rp_sem);
3708     qemu_sem_destroy(&ms->rp_state.rp_sem);
3709     error_free(ms->error);
3710 }
3711
3712 static void migration_instance_init(Object *obj)
3713 {
3714     MigrationState *ms = MIGRATION_OBJ(obj);
3715     MigrationParameters *params = &ms->parameters;
3716
3717     ms->state = MIGRATION_STATUS_NONE;
3718     ms->mbps = -1;
3719     ms->pages_per_second = -1;
3720     qemu_sem_init(&ms->pause_sem, 0);
3721     qemu_mutex_init(&ms->error_mutex);
3722
3723     params->tls_hostname = g_strdup("");
3724     params->tls_creds = g_strdup("");
3725
3726     /* Set has_* up only for parameter checks */
3727     params->has_compress_level = true;
3728     params->has_compress_threads = true;
3729     params->has_decompress_threads = true;
3730     params->has_throttle_trigger_threshold = true;
3731     params->has_cpu_throttle_initial = true;
3732     params->has_cpu_throttle_increment = true;
3733     params->has_cpu_throttle_tailslow = true;
3734     params->has_max_bandwidth = true;
3735     params->has_downtime_limit = true;
3736     params->has_x_checkpoint_delay = true;
3737     params->has_block_incremental = true;
3738     params->has_multifd_channels = true;
3739     params->has_multifd_compression = true;
3740     params->has_multifd_zlib_level = true;
3741     params->has_multifd_zstd_level = true;
3742     params->has_xbzrle_cache_size = true;
3743     params->has_max_postcopy_bandwidth = true;
3744     params->has_max_cpu_throttle = true;
3745     params->has_announce_initial = true;
3746     params->has_announce_max = true;
3747     params->has_announce_rounds = true;
3748     params->has_announce_step = true;
3749
3750     qemu_sem_init(&ms->postcopy_pause_sem, 0);
3751     qemu_sem_init(&ms->postcopy_pause_rp_sem, 0);
3752     qemu_sem_init(&ms->rp_state.rp_sem, 0);
3753     qemu_sem_init(&ms->rate_limit_sem, 0);
3754     qemu_sem_init(&ms->wait_unplug_sem, 0);
3755     qemu_mutex_init(&ms->qemu_file_lock);
3756 }
3757
3758 /*
3759  * Return true if check pass, false otherwise. Error will be put
3760  * inside errp if provided.
3761  */
3762 static bool migration_object_check(MigrationState *ms, Error **errp)
3763 {
3764     MigrationCapabilityStatusList *head = NULL;
3765     /* Assuming all off */
3766     bool cap_list[MIGRATION_CAPABILITY__MAX] = { 0 }, ret;
3767     int i;
3768
3769     if (!migrate_params_check(&ms->parameters, errp)) {
3770         return false;
3771     }
3772
3773     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
3774         if (ms->enabled_capabilities[i]) {
3775             head = migrate_cap_add(head, i, true);
3776         }
3777     }
3778
3779     ret = migrate_caps_check(cap_list, head, errp);
3780
3781     /* It works with head == NULL */
3782     qapi_free_MigrationCapabilityStatusList(head);
3783
3784     return ret;
3785 }
3786
3787 static const TypeInfo migration_type = {
3788     .name = TYPE_MIGRATION,
3789     /*
3790      * NOTE: TYPE_MIGRATION is not really a device, as the object is
3791      * not created using qdev_new(), it is not attached to the qdev
3792      * device tree, and it is never realized.
3793      *
3794      * TODO: Make this TYPE_OBJECT once QOM provides something like
3795      * TYPE_DEVICE's "-global" properties.
3796      */
3797     .parent = TYPE_DEVICE,
3798     .class_init = migration_class_init,
3799     .class_size = sizeof(MigrationClass),
3800     .instance_size = sizeof(MigrationState),
3801     .instance_init = migration_instance_init,
3802     .instance_finalize = migration_instance_finalize,
3803 };
3804
3805 static void register_migration_types(void)
3806 {
3807     type_register_static(&migration_type);
3808 }
3809
3810 type_init(register_migration_types);