migration/migration.h

   1 /*
   2  * QEMU live migration
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *
   6  * Authors:
   7  *  Anthony Liguori   <aliguori@us.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  */
  13
  14 #ifndef QEMU_MIGRATION_H
  15 #define QEMU_MIGRATION_H
  16
  17 #include "exec/cpu-common.h"
  18 #include "hw/qdev-core.h"
  19 #include "qapi/qapi-types-migration.h"
  20 #include "qemu/thread.h"
  21 #include "qemu/coroutine_int.h"
  22 #include "io/channel.h"
  23 #include "net/announce.h"
  24 #include "qom/object.h"
  25
  26 struct PostcopyBlocktimeContext;
  27
  28 #define  MIGRATION_RESUME_ACK_VALUE  (1)
  29
  30 /*
  31  * 1<<6=64 pages -> 256K chunk when page size is 4K.  This gives us
  32  * the benefit that all the chunks are 64 pages aligned then the
  33  * bitmaps are always aligned to LONG.
  34  */
  35 #define CLEAR_BITMAP_SHIFT_MIN             6
  36 /*
  37  * 1<<18=256K pages -> 1G chunk when page size is 4K.  This is the
  38  * default value to use if no one specified.
  39  */
  40 #define CLEAR_BITMAP_SHIFT_DEFAULT        18
  41 /*
  42  * 1<<31=2G pages -> 8T chunk when page size is 4K.  This should be
  43  * big enough and make sure we won't overflow easily.
  44  */
  45 #define CLEAR_BITMAP_SHIFT_MAX            31
  46
  47 /* State for the incoming migration */
  48 struct MigrationIncomingState {
  49     QEMUFile *from_src_file;
  50
  51     /*
  52      * Free at the start of the main state load, set as the main thread finishes
  53      * loading state.
  54      */
  55     QemuEvent main_thread_load_event;
  56
  57     /* For network announces */
  58     AnnounceTimer  announce_timer;
  59
  60     size_t         largest_page_size;
  61     bool           have_fault_thread;
  62     QemuThread     fault_thread;
  63     QemuSemaphore  fault_thread_sem;
  64     /* Set this when we want the fault thread to quit */
  65     bool           fault_thread_quit;
  66
  67     bool           have_listen_thread;
  68     QemuThread     listen_thread;
  69     QemuSemaphore  listen_thread_sem;
  70
  71     /* For the kernel to send us notifications */
  72     int       userfault_fd;
  73     /* To notify the fault_thread to wake, e.g., when need to quit */
  74     int       userfault_event_fd;
  75     QEMUFile *to_src_file;
  76     QemuMutex rp_mutex;    /* We send replies from multiple threads */
  77     /* RAMBlock of last request sent to source */
  78     RAMBlock *last_rb;
  79     void     *postcopy_tmp_page;
  80     void     *postcopy_tmp_zero_page;
  81     /* PostCopyFD's for external userfaultfds & handlers of shared memory */
  82     GArray   *postcopy_remote_fds;
  83
  84     QEMUBH *bh;
  85
  86     int state;
  87
  88     bool have_colo_incoming_thread;
  89     QemuThread colo_incoming_thread;
  90     /* The coroutine we should enter (back) after failover */
  91     Coroutine *migration_incoming_co;
  92     QemuSemaphore colo_incoming_sem;
  93
  94     /*
  95      * PostcopyBlocktimeContext to keep information for postcopy
  96      * live migration, to calculate vCPU block time
  97      * */
  98     struct PostcopyBlocktimeContext *blocktime_ctx;
  99
 100     /* notify PAUSED postcopy incoming migrations to try to continue */
 101     bool postcopy_recover_triggered;
 102     QemuSemaphore postcopy_pause_sem_dst;
 103     QemuSemaphore postcopy_pause_sem_fault;
 104
 105     /* List of listening socket addresses  */
 106     SocketAddressList *socket_address_list;
 107
 108     /* A tree of pages that we requested to the source VM */
 109     GTree *page_requested;
 110     /* For debugging purpose only, but would be nice to keep */
 111     int page_requested_count;
 112     /*
 113      * The mutex helps to maintain the requested pages that we sent to the
 114      * source, IOW, to guarantee coherent between the page_requests tree and
 115      * the per-ramblock receivedmap.  Note! This does not guarantee consistency
 116      * of the real page copy procedures (using UFFDIO_[ZERO]COPY).  E.g., even
 117      * if one bit in receivedmap is cleared, UFFDIO_COPY could have happened
 118      * for that page already.  This is intended so that the mutex won't
 119      * serialize and blocked by slow operations like UFFDIO_* ioctls.  However
 120      * this should be enough to make sure the page_requested tree always
 121      * contains valid information.
 122      */
 123     QemuMutex page_request_mutex;
 124 };
 125
 126 MigrationIncomingState *migration_incoming_get_current(void);
 127 void migration_incoming_state_destroy(void);
 128 /*
 129  * Functions to work with blocktime context
 130  */
 131 void fill_destination_postcopy_migration_info(MigrationInfo *info);
 132
 133 #define TYPE_MIGRATION "migration"
 134
 135 typedef struct MigrationClass MigrationClass;
 136 DECLARE_OBJ_CHECKERS(MigrationState, MigrationClass,
 137                      MIGRATION_OBJ, TYPE_MIGRATION)
 138
 139 struct MigrationClass {
 140     /*< private >*/
 141     DeviceClass parent_class;
 142 };
 143
 144 struct MigrationState {
 145     /*< private >*/
 146     DeviceState parent_obj;
 147
 148     /*< public >*/
 149     QemuThread thread;
 150     QEMUBH *cleanup_bh;
 151     QEMUFile *to_dst_file;
 152     /*
 153      * Protects to_dst_file pointer.  We need to make sure we won't
 154      * yield or hang during the critical section, since this lock will
 155      * be used in OOB command handler.
 156      */
 157     QemuMutex qemu_file_lock;
 158
 159     /*
 160      * Used to allow urgent requests to override rate limiting.
 161      */
 162     QemuSemaphore rate_limit_sem;
 163
 164     /* pages already send at the beginning of current iteration */
 165     uint64_t iteration_initial_pages;
 166
 167     /* pages transferred per second */
 168     double pages_per_second;
 169
 170     /* bytes already send at the beginning of current iteration */
 171     uint64_t iteration_initial_bytes;
 172     /* time at the start of current iteration */
 173     int64_t iteration_start_time;
 174     /*
 175      * The final stage happens when the remaining data is smaller than
 176      * this threshold; it's calculated from the requested downtime and
 177      * measured bandwidth
 178      */
 179     int64_t threshold_size;
 180
 181     /* params from 'migrate-set-parameters' */
 182     MigrationParameters parameters;
 183
 184     int state;
 185
 186     /* State related to return path */
 187     struct {
 188         QEMUFile     *from_dst_file;
 189         QemuThread    rp_thread;
 190         bool          error;
 191         QemuSemaphore rp_sem;
 192     } rp_state;
 193
 194     double mbps;
 195     /* Timestamp when recent migration starts (ms) */
 196     int64_t start_time;
 197     /* Total time used by latest migration (ms) */
 198     int64_t total_time;
 199     /* Timestamp when VM is down (ms) to migrate the last stuff */
 200     int64_t downtime_start;
 201     int64_t downtime;
 202     int64_t expected_downtime;
 203     bool enabled_capabilities[MIGRATION_CAPABILITY__MAX];
 204     int64_t setup_time;
 205     /*
 206      * Whether guest was running when we enter the completion stage.
 207      * If migration is interrupted by any reason, we need to continue
 208      * running the guest on source.
 209      */
 210     bool vm_was_running;
 211
 212     /* Flag set once the migration has been asked to enter postcopy */
 213     bool start_postcopy;
 214     /* Flag set after postcopy has sent the device state */
 215     bool postcopy_after_devices;
 216
 217     /* Flag set once the migration thread is running (and needs joining) */
 218     bool migration_thread_running;
 219
 220     /* Flag set once the migration thread called bdrv_inactivate_all */
 221     bool block_inactive;
 222
 223     /* Migration is waiting for guest to unplug device */
 224     QemuSemaphore wait_unplug_sem;
 225
 226     /* Migration is paused due to pause-before-switchover */
 227     QemuSemaphore pause_sem;
 228
 229     /* The semaphore is used to notify COLO thread that failover is finished */
 230     QemuSemaphore colo_exit_sem;
 231
 232     /* The event is used to notify COLO thread to do checkpoint */
 233     QemuEvent colo_checkpoint_event;
 234     int64_t colo_checkpoint_time;
 235     QEMUTimer *colo_delay_timer;
 236
 237     /* The first error that has occurred.
 238        We used the mutex to be able to return the 1st error message */
 239     Error *error;
 240     /* mutex to protect errp */
 241     QemuMutex error_mutex;
 242
 243     /* Do we have to clean up -b/-i from old migrate parameters */
 244     /* This feature is deprecated and will be removed */
 245     bool must_remove_block_options;
 246
 247     /*
 248      * Global switch on whether we need to store the global state
 249      * during migration.
 250      */
 251     bool store_global_state;
 252
 253     /* Whether we send QEMU_VM_CONFIGURATION during migration */
 254     bool send_configuration;
 255     /* Whether we send section footer during migration */
 256     bool send_section_footer;
 257
 258     /* Needed by postcopy-pause state */
 259     QemuSemaphore postcopy_pause_sem;
 260     QemuSemaphore postcopy_pause_rp_sem;
 261     /*
 262      * Whether we abort the migration if decompression errors are
 263      * detected at the destination. It is left at false for qemu
 264      * older than 3.0, since only newer qemu sends streams that
 265      * do not trigger spurious decompression errors.
 266      */
 267     bool decompress_error_check;
 268
 269     /*
 270      * This decides the size of guest memory chunk that will be used
 271      * to track dirty bitmap clearing.  The size of memory chunk will
 272      * be GUEST_PAGE_SIZE << N.  Say, N=0 means we will clear dirty
 273      * bitmap for each page to send (1<<0=1); N=10 means we will clear
 274      * dirty bitmap only once for 1<<10=1K continuous guest pages
 275      * (which is in 4M chunk).
 276      */
 277     uint8_t clear_bitmap_shift;
 278
 279     /*
 280      * This save hostname when out-going migration starts
 281      */
 282     char *hostname;
 283 };
 284
 285 void migrate_set_state(int *state, int old_state, int new_state);
 286
 287 void migration_fd_process_incoming(QEMUFile *f, Error **errp);
 288 void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp);
 289 void migration_incoming_process(void);
 290
 291 bool  migration_has_all_channels(void);
 292
 293 uint64_t migrate_max_downtime(void);
 294
 295 void migrate_set_error(MigrationState *s, const Error *error);
 296 void migrate_fd_error(MigrationState *s, const Error *error);
 297
 298 void migrate_fd_connect(MigrationState *s, Error *error_in);
 299
 300 bool migration_is_setup_or_active(int state);
 301 bool migration_is_running(int state);
 302
 303 void migrate_init(MigrationState *s);
 304 bool migration_is_blocked(Error **errp);
 305 /* True if outgoing migration has entered postcopy phase */
 306 bool migration_in_postcopy(void);
 307 MigrationState *migrate_get_current(void);
 308
 309 bool migrate_postcopy(void);
 310
 311 bool migrate_release_ram(void);
 312 bool migrate_postcopy_ram(void);
 313 bool migrate_zero_blocks(void);
 314 bool migrate_dirty_bitmaps(void);
 315 bool migrate_ignore_shared(void);
 316 bool migrate_validate_uuid(void);
 317
 318 bool migrate_auto_converge(void);
 319 bool migrate_use_multifd(void);
 320 bool migrate_pause_before_switchover(void);
 321 int migrate_multifd_channels(void);
 322 MultiFDCompression migrate_multifd_compression(void);
 323 int migrate_multifd_zlib_level(void);
 324 int migrate_multifd_zstd_level(void);
 325
 326 int migrate_use_xbzrle(void);
 327 int64_t migrate_xbzrle_cache_size(void);
 328 bool migrate_colo_enabled(void);
 329
 330 bool migrate_use_block(void);
 331 bool migrate_use_block_incremental(void);
 332 int migrate_max_cpu_throttle(void);
 333 bool migrate_use_return_path(void);
 334
 335 uint64_t ram_get_total_transferred_pages(void);
 336
 337 bool migrate_use_compression(void);
 338 int migrate_compress_level(void);
 339 int migrate_compress_threads(void);
 340 int migrate_compress_wait_thread(void);
 341 int migrate_decompress_threads(void);
 342 bool migrate_use_events(void);
 343 bool migrate_postcopy_blocktime(void);
 344
 345 /* Sending on the return path - generic and then for each message type */
 346 void migrate_send_rp_shut(MigrationIncomingState *mis,
 347                           uint32_t value);
 348 void migrate_send_rp_pong(MigrationIncomingState *mis,
 349                           uint32_t value);
 350 int migrate_send_rp_req_pages(MigrationIncomingState *mis, RAMBlock *rb,
 351                               ram_addr_t start, uint64_t haddr);
 352 int migrate_send_rp_message_req_pages(MigrationIncomingState *mis,
 353                                       RAMBlock *rb, ram_addr_t start);
 354 void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
 355                                  char *block_name);
 356 void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value);
 357
 358 void dirty_bitmap_mig_before_vm_start(void);
 359 void dirty_bitmap_mig_cancel_outgoing(void);
 360 void dirty_bitmap_mig_cancel_incoming(void);
 361 bool check_dirty_bitmap_mig_alias_map(const BitmapMigrationNodeAliasList *bbm,
 362                                       Error **errp);
 363
 364 void migrate_add_address(SocketAddress *address);
 365
 366 int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque);
 367
 368 #define qemu_ram_foreach_block \
 369   #warning "Use foreach_not_ignored_block in migration code"
 370
 371 void migration_make_urgent_request(void);
 372 void migration_consume_urgent_request(void);
 373 bool migration_rate_limit(void);
 374
 375 #endif