hw/vfio/migration.c

   1 /*
   2  * Migration support for VFIO devices
   3  *
   4  * Copyright NVIDIA, Inc. 2020
   5  *
   6  * This work is licensed under the terms of the GNU GPL, version 2. See
   7  * the COPYING file in the top-level directory.
   8  */
   9
  10 #include "qemu/osdep.h"
  11 #include "qemu/main-loop.h"
  12 #include "qemu/cutils.h"
  13 #include "qemu/units.h"
  14 #include "qemu/error-report.h"
  15 #include <linux/vfio.h>
  16 #include <sys/ioctl.h>
  17
  18 #include "sysemu/runstate.h"
  19 #include "hw/vfio/vfio-common.h"
  20 #include "migration/misc.h"
  21 #include "migration/savevm.h"
  22 #include "migration/vmstate.h"
  23 #include "migration/qemu-file.h"
  24 #include "migration/register.h"
  25 #include "migration/blocker.h"
  26 #include "qapi/error.h"
  27 #include "qapi/qapi-events-vfio.h"
  28 #include "exec/ramlist.h"
  29 #include "exec/ram_addr.h"
  30 #include "pci.h"
  31 #include "trace.h"
  32 #include "hw/hw.h"
  33
  34 /*
  35  * Flags to be used as unique delimiters for VFIO devices in the migration
  36  * stream. These flags are composed as:
  37  * 0xffffffff => MSB 32-bit all 1s
  38  * 0xef10     => Magic ID, represents emulated (virtual) function IO
  39  * 0x0000     => 16-bits reserved for flags
  40  *
  41  * The beginning of state information is marked by _DEV_CONFIG_STATE,
  42  * _DEV_SETUP_STATE, or _DEV_DATA_STATE, respectively. The end of a
  43  * certain state information is marked by _END_OF_STATE.
  44  */
  45 #define VFIO_MIG_FLAG_END_OF_STATE      (0xffffffffef100001ULL)
  46 #define VFIO_MIG_FLAG_DEV_CONFIG_STATE  (0xffffffffef100002ULL)
  47 #define VFIO_MIG_FLAG_DEV_SETUP_STATE   (0xffffffffef100003ULL)
  48 #define VFIO_MIG_FLAG_DEV_DATA_STATE    (0xffffffffef100004ULL)
  49 #define VFIO_MIG_FLAG_DEV_INIT_DATA_SENT (0xffffffffef100005ULL)
  50
  51 /*
  52  * This is an arbitrary size based on migration of mlx5 devices, where typically
  53  * total device migration size is on the order of 100s of MB. Testing with
  54  * larger values, e.g. 128MB and 1GB, did not show a performance improvement.
  55  */
  56 #define VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE (1 * MiB)
  57
  58 static int64_t bytes_transferred;
  59
  60 static const char *mig_state_to_str(enum vfio_device_mig_state state)
  61 {
  62     switch (state) {
  63     case VFIO_DEVICE_STATE_ERROR:
  64         return "ERROR";
  65     case VFIO_DEVICE_STATE_STOP:
  66         return "STOP";
  67     case VFIO_DEVICE_STATE_RUNNING:
  68         return "RUNNING";
  69     case VFIO_DEVICE_STATE_STOP_COPY:
  70         return "STOP_COPY";
  71     case VFIO_DEVICE_STATE_RESUMING:
  72         return "RESUMING";
  73     case VFIO_DEVICE_STATE_RUNNING_P2P:
  74         return "RUNNING_P2P";
  75     case VFIO_DEVICE_STATE_PRE_COPY:
  76         return "PRE_COPY";
  77     case VFIO_DEVICE_STATE_PRE_COPY_P2P:
  78         return "PRE_COPY_P2P";
  79     default:
  80         return "UNKNOWN STATE";
  81     }
  82 }
  83
  84 static VfioMigrationState
  85 mig_state_to_qapi_state(enum vfio_device_mig_state state)
  86 {
  87     switch (state) {
  88     case VFIO_DEVICE_STATE_STOP:
  89         return QAPI_VFIO_MIGRATION_STATE_STOP;
  90     case VFIO_DEVICE_STATE_RUNNING:
  91         return QAPI_VFIO_MIGRATION_STATE_RUNNING;
  92     case VFIO_DEVICE_STATE_STOP_COPY:
  93         return QAPI_VFIO_MIGRATION_STATE_STOP_COPY;
  94     case VFIO_DEVICE_STATE_RESUMING:
  95         return QAPI_VFIO_MIGRATION_STATE_RESUMING;
  96     case VFIO_DEVICE_STATE_RUNNING_P2P:
  97         return QAPI_VFIO_MIGRATION_STATE_RUNNING_P2P;
  98     case VFIO_DEVICE_STATE_PRE_COPY:
  99         return QAPI_VFIO_MIGRATION_STATE_PRE_COPY;
 100     case VFIO_DEVICE_STATE_PRE_COPY_P2P:
 101         return QAPI_VFIO_MIGRATION_STATE_PRE_COPY_P2P;
 102     default:
 103         g_assert_not_reached();
 104     }
 105 }
 106
 107 static void vfio_migration_send_event(VFIODevice *vbasedev)
 108 {
 109     VFIOMigration *migration = vbasedev->migration;
 110     DeviceState *dev = vbasedev->dev;
 111     g_autofree char *qom_path = NULL;
 112     Object *obj;
 113
 114     if (!vbasedev->migration_events) {
 115         return;
 116     }
 117
 118     g_assert(vbasedev->ops->vfio_get_object);
 119     obj = vbasedev->ops->vfio_get_object(vbasedev);
 120     g_assert(obj);
 121     qom_path = object_get_canonical_path(obj);
 122
 123     qapi_event_send_vfio_migration(
 124         dev->id, qom_path, mig_state_to_qapi_state(migration->device_state));
 125 }
 126
 127 static void vfio_migration_set_device_state(VFIODevice *vbasedev,
 128                                             enum vfio_device_mig_state state)
 129 {
 130     VFIOMigration *migration = vbasedev->migration;
 131
 132     trace_vfio_migration_set_device_state(vbasedev->name,
 133                                           mig_state_to_str(state));
 134
 135     migration->device_state = state;
 136     vfio_migration_send_event(vbasedev);
 137 }
 138
 139 static int vfio_migration_set_state(VFIODevice *vbasedev,
 140                                     enum vfio_device_mig_state new_state,
 141                                     enum vfio_device_mig_state recover_state,
 142                                     Error **errp)
 143 {
 144     VFIOMigration *migration = vbasedev->migration;
 145     uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) +
 146                               sizeof(struct vfio_device_feature_mig_state),
 147                               sizeof(uint64_t))] = {};
 148     struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
 149     struct vfio_device_feature_mig_state *mig_state =
 150         (struct vfio_device_feature_mig_state *)feature->data;
 151     int ret;
 152     g_autofree char *error_prefix =
 153         g_strdup_printf("%s: Failed setting device state to %s.",
 154                         vbasedev->name, mig_state_to_str(new_state));
 155
 156     trace_vfio_migration_set_state(vbasedev->name, mig_state_to_str(new_state),
 157                                    mig_state_to_str(recover_state));
 158
 159     if (new_state == migration->device_state) {
 160         return 0;
 161     }
 162
 163     feature->argsz = sizeof(buf);
 164     feature->flags =
 165         VFIO_DEVICE_FEATURE_SET | VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE;
 166     mig_state->device_state = new_state;
 167     if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) {
 168         /* Try to set the device in some good state */
 169         ret = -errno;
 170
 171         if (recover_state == VFIO_DEVICE_STATE_ERROR) {
 172             error_setg_errno(errp, errno,
 173                              "%s Recover state is ERROR. Resetting device",
 174                              error_prefix);
 175
 176             goto reset_device;
 177         }
 178
 179         error_setg_errno(errp, errno,
 180                          "%s Setting device in recover state %s",
 181                          error_prefix, mig_state_to_str(recover_state));
 182
 183         mig_state->device_state = recover_state;
 184         if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) {
 185             ret = -errno;
 186             /*
 187              * If setting the device in recover state fails, report
 188              * the error here and propagate the first error.
 189              */
 190             error_report(
 191                 "%s: Failed setting device in recover state, err: %s. Resetting device",
 192                          vbasedev->name, strerror(errno));
 193
 194             goto reset_device;
 195         }
 196
 197         vfio_migration_set_device_state(vbasedev, recover_state);
 198
 199         return ret;
 200     }
 201
 202     vfio_migration_set_device_state(vbasedev, new_state);
 203     if (mig_state->data_fd != -1) {
 204         if (migration->data_fd != -1) {
 205             /*
 206              * This can happen if the device is asynchronously reset and
 207              * terminates a data transfer.
 208              */
 209             error_setg(errp, "%s: data_fd out of sync", vbasedev->name);
 210             close(mig_state->data_fd);
 211
 212             return -EBADF;
 213         }
 214
 215         migration->data_fd = mig_state->data_fd;
 216     }
 217
 218     return 0;
 219
 220 reset_device:
 221     if (ioctl(vbasedev->fd, VFIO_DEVICE_RESET)) {
 222         hw_error("%s: Failed resetting device, err: %s", vbasedev->name,
 223                  strerror(errno));
 224     }
 225
 226     vfio_migration_set_device_state(vbasedev, VFIO_DEVICE_STATE_RUNNING);
 227
 228     return ret;
 229 }
 230
 231 /*
 232  * Some device state transitions require resetting the device if they fail.
 233  * This function sets the device in new_state and resets the device if that
 234  * fails. Reset is done by using ERROR as the recover state.
 235  */
 236 static int
 237 vfio_migration_set_state_or_reset(VFIODevice *vbasedev,
 238                                   enum vfio_device_mig_state new_state,
 239                                   Error **errp)
 240 {
 241     return vfio_migration_set_state(vbasedev, new_state,
 242                                     VFIO_DEVICE_STATE_ERROR, errp);
 243 }
 244
 245 static int vfio_load_buffer(QEMUFile *f, VFIODevice *vbasedev,
 246                             uint64_t data_size)
 247 {
 248     VFIOMigration *migration = vbasedev->migration;
 249     int ret;
 250
 251     ret = qemu_file_get_to_fd(f, migration->data_fd, data_size);
 252     trace_vfio_load_state_device_data(vbasedev->name, data_size, ret);
 253
 254     return ret;
 255 }
 256
 257 static int vfio_save_device_config_state(QEMUFile *f, void *opaque,
 258                                          Error **errp)
 259 {
 260     VFIODevice *vbasedev = opaque;
 261     int ret;
 262
 263     qemu_put_be64(f, VFIO_MIG_FLAG_DEV_CONFIG_STATE);
 264
 265     if (vbasedev->ops && vbasedev->ops->vfio_save_config) {
 266         ret = vbasedev->ops->vfio_save_config(vbasedev, f, errp);
 267         if (ret) {
 268             return ret;
 269         }
 270     }
 271
 272     qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
 273
 274     trace_vfio_save_device_config_state(vbasedev->name);
 275
 276     ret = qemu_file_get_error(f);
 277     if (ret < 0) {
 278         error_setg_errno(errp, -ret, "Failed to save state");
 279     }
 280     return ret;
 281 }
 282
 283 static int vfio_load_device_config_state(QEMUFile *f, void *opaque)
 284 {
 285     VFIODevice *vbasedev = opaque;
 286     uint64_t data;
 287
 288     if (vbasedev->ops && vbasedev->ops->vfio_load_config) {
 289         int ret;
 290
 291         ret = vbasedev->ops->vfio_load_config(vbasedev, f);
 292         if (ret) {
 293             error_report("%s: Failed to load device config space",
 294                          vbasedev->name);
 295             return ret;
 296         }
 297     }
 298
 299     data = qemu_get_be64(f);
 300     if (data != VFIO_MIG_FLAG_END_OF_STATE) {
 301         error_report("%s: Failed loading device config space, "
 302                      "end flag incorrect 0x%"PRIx64, vbasedev->name, data);
 303         return -EINVAL;
 304     }
 305
 306     trace_vfio_load_device_config_state(vbasedev->name);
 307     return qemu_file_get_error(f);
 308 }
 309
 310 static void vfio_migration_cleanup(VFIODevice *vbasedev)
 311 {
 312     VFIOMigration *migration = vbasedev->migration;
 313
 314     close(migration->data_fd);
 315     migration->data_fd = -1;
 316 }
 317
 318 static int vfio_query_stop_copy_size(VFIODevice *vbasedev,
 319                                      uint64_t *stop_copy_size)
 320 {
 321     uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) +
 322                               sizeof(struct vfio_device_feature_mig_data_size),
 323                               sizeof(uint64_t))] = {};
 324     struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
 325     struct vfio_device_feature_mig_data_size *mig_data_size =
 326         (struct vfio_device_feature_mig_data_size *)feature->data;
 327
 328     feature->argsz = sizeof(buf);
 329     feature->flags =
 330         VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_MIG_DATA_SIZE;
 331
 332     if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) {
 333         return -errno;
 334     }
 335
 336     *stop_copy_size = mig_data_size->stop_copy_length;
 337
 338     return 0;
 339 }
 340
 341 static int vfio_query_precopy_size(VFIOMigration *migration)
 342 {
 343     struct vfio_precopy_info precopy = {
 344         .argsz = sizeof(precopy),
 345     };
 346
 347     migration->precopy_init_size = 0;
 348     migration->precopy_dirty_size = 0;
 349
 350     if (ioctl(migration->data_fd, VFIO_MIG_GET_PRECOPY_INFO, &precopy)) {
 351         return -errno;
 352     }
 353
 354     migration->precopy_init_size = precopy.initial_bytes;
 355     migration->precopy_dirty_size = precopy.dirty_bytes;
 356
 357     return 0;
 358 }
 359
 360 /* Returns the size of saved data on success and -errno on error */
 361 static ssize_t vfio_save_block(QEMUFile *f, VFIOMigration *migration)
 362 {
 363     ssize_t data_size;
 364
 365     data_size = read(migration->data_fd, migration->data_buffer,
 366                      migration->data_buffer_size);
 367     if (data_size < 0) {
 368         /*
 369          * Pre-copy emptied all the device state for now. For more information,
 370          * please refer to the Linux kernel VFIO uAPI.
 371          */
 372         if (errno == ENOMSG) {
 373             return 0;
 374         }
 375
 376         return -errno;
 377     }
 378     if (data_size == 0) {
 379         return 0;
 380     }
 381
 382     qemu_put_be64(f, VFIO_MIG_FLAG_DEV_DATA_STATE);
 383     qemu_put_be64(f, data_size);
 384     qemu_put_buffer(f, migration->data_buffer, data_size);
 385     bytes_transferred += data_size;
 386
 387     trace_vfio_save_block(migration->vbasedev->name, data_size);
 388
 389     return qemu_file_get_error(f) ?: data_size;
 390 }
 391
 392 static void vfio_update_estimated_pending_data(VFIOMigration *migration,
 393                                                uint64_t data_size)
 394 {
 395     if (!data_size) {
 396         /*
 397          * Pre-copy emptied all the device state for now, update estimated sizes
 398          * accordingly.
 399          */
 400         migration->precopy_init_size = 0;
 401         migration->precopy_dirty_size = 0;
 402
 403         return;
 404     }
 405
 406     if (migration->precopy_init_size) {
 407         uint64_t init_size = MIN(migration->precopy_init_size, data_size);
 408
 409         migration->precopy_init_size -= init_size;
 410         data_size -= init_size;
 411     }
 412
 413     migration->precopy_dirty_size -= MIN(migration->precopy_dirty_size,
 414                                          data_size);
 415 }
 416
 417 static bool vfio_precopy_supported(VFIODevice *vbasedev)
 418 {
 419     VFIOMigration *migration = vbasedev->migration;
 420
 421     return migration->mig_flags & VFIO_MIGRATION_PRE_COPY;
 422 }
 423
 424 /* ---------------------------------------------------------------------- */
 425
 426 static int vfio_save_prepare(void *opaque, Error **errp)
 427 {
 428     VFIODevice *vbasedev = opaque;
 429
 430     /*
 431      * Snapshot doesn't use postcopy nor background snapshot, so allow snapshot
 432      * even if they are on.
 433      */
 434     if (runstate_check(RUN_STATE_SAVE_VM)) {
 435         return 0;
 436     }
 437
 438     if (migrate_postcopy_ram()) {
 439         error_setg(
 440             errp, "%s: VFIO migration is not supported with postcopy migration",
 441             vbasedev->name);
 442         return -EOPNOTSUPP;
 443     }
 444
 445     if (migrate_background_snapshot()) {
 446         error_setg(
 447             errp,
 448             "%s: VFIO migration is not supported with background snapshot",
 449             vbasedev->name);
 450         return -EOPNOTSUPP;
 451     }
 452
 453     return 0;
 454 }
 455
 456 static int vfio_save_setup(QEMUFile *f, void *opaque, Error **errp)
 457 {
 458     VFIODevice *vbasedev = opaque;
 459     VFIOMigration *migration = vbasedev->migration;
 460     uint64_t stop_copy_size = VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE;
 461     int ret;
 462
 463     qemu_put_be64(f, VFIO_MIG_FLAG_DEV_SETUP_STATE);
 464
 465     vfio_query_stop_copy_size(vbasedev, &stop_copy_size);
 466     migration->data_buffer_size = MIN(VFIO_MIG_DEFAULT_DATA_BUFFER_SIZE,
 467                                       stop_copy_size);
 468     migration->data_buffer = g_try_malloc0(migration->data_buffer_size);
 469     if (!migration->data_buffer) {
 470         error_setg(errp, "%s: Failed to allocate migration data buffer",
 471                    vbasedev->name);
 472         return -ENOMEM;
 473     }
 474
 475     if (vfio_precopy_supported(vbasedev)) {
 476         switch (migration->device_state) {
 477         case VFIO_DEVICE_STATE_RUNNING:
 478             ret = vfio_migration_set_state(vbasedev, VFIO_DEVICE_STATE_PRE_COPY,
 479                                            VFIO_DEVICE_STATE_RUNNING, errp);
 480             if (ret) {
 481                 return ret;
 482             }
 483
 484             vfio_query_precopy_size(migration);
 485
 486             break;
 487         case VFIO_DEVICE_STATE_STOP:
 488             /* vfio_save_complete_precopy() will go to STOP_COPY */
 489             break;
 490         default:
 491             error_setg(errp, "%s: Invalid device state %d", vbasedev->name,
 492                        migration->device_state);
 493             return -EINVAL;
 494         }
 495     }
 496
 497     trace_vfio_save_setup(vbasedev->name, migration->data_buffer_size);
 498
 499     qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
 500
 501     ret = qemu_file_get_error(f);
 502     if (ret < 0) {
 503         error_setg_errno(errp, -ret, "%s: save setup failed", vbasedev->name);
 504     }
 505
 506     return ret;
 507 }
 508
 509 static void vfio_save_cleanup(void *opaque)
 510 {
 511     VFIODevice *vbasedev = opaque;
 512     VFIOMigration *migration = vbasedev->migration;
 513     Error *local_err = NULL;
 514     int ret;
 515
 516     /*
 517      * Changing device state from STOP_COPY to STOP can take time. Do it here,
 518      * after migration has completed, so it won't increase downtime.
 519      */
 520     if (migration->device_state == VFIO_DEVICE_STATE_STOP_COPY) {
 521         ret = vfio_migration_set_state_or_reset(vbasedev,
 522                                                 VFIO_DEVICE_STATE_STOP,
 523                                                 &local_err);
 524         if (ret) {
 525             error_report_err(local_err);
 526         }
 527     }
 528
 529     g_free(migration->data_buffer);
 530     migration->data_buffer = NULL;
 531     migration->precopy_init_size = 0;
 532     migration->precopy_dirty_size = 0;
 533     migration->initial_data_sent = false;
 534     vfio_migration_cleanup(vbasedev);
 535     trace_vfio_save_cleanup(vbasedev->name);
 536 }
 537
 538 static void vfio_state_pending_estimate(void *opaque, uint64_t *must_precopy,
 539                                         uint64_t *can_postcopy)
 540 {
 541     VFIODevice *vbasedev = opaque;
 542     VFIOMigration *migration = vbasedev->migration;
 543
 544     if (!vfio_device_state_is_precopy(vbasedev)) {
 545         return;
 546     }
 547
 548     *must_precopy +=
 549         migration->precopy_init_size + migration->precopy_dirty_size;
 550
 551     trace_vfio_state_pending_estimate(vbasedev->name, *must_precopy,
 552                                       *can_postcopy,
 553                                       migration->precopy_init_size,
 554                                       migration->precopy_dirty_size);
 555 }
 556
 557 /*
 558  * Migration size of VFIO devices can be as little as a few KBs or as big as
 559  * many GBs. This value should be big enough to cover the worst case.
 560  */
 561 #define VFIO_MIG_STOP_COPY_SIZE (100 * GiB)
 562
 563 static void vfio_state_pending_exact(void *opaque, uint64_t *must_precopy,
 564                                      uint64_t *can_postcopy)
 565 {
 566     VFIODevice *vbasedev = opaque;
 567     VFIOMigration *migration = vbasedev->migration;
 568     uint64_t stop_copy_size = VFIO_MIG_STOP_COPY_SIZE;
 569
 570     /*
 571      * If getting pending migration size fails, VFIO_MIG_STOP_COPY_SIZE is
 572      * reported so downtime limit won't be violated.
 573      */
 574     vfio_query_stop_copy_size(vbasedev, &stop_copy_size);
 575     *must_precopy += stop_copy_size;
 576
 577     if (vfio_device_state_is_precopy(vbasedev)) {
 578         vfio_query_precopy_size(migration);
 579
 580         *must_precopy +=
 581             migration->precopy_init_size + migration->precopy_dirty_size;
 582     }
 583
 584     trace_vfio_state_pending_exact(vbasedev->name, *must_precopy, *can_postcopy,
 585                                    stop_copy_size, migration->precopy_init_size,
 586                                    migration->precopy_dirty_size);
 587 }
 588
 589 static bool vfio_is_active_iterate(void *opaque)
 590 {
 591     VFIODevice *vbasedev = opaque;
 592
 593     return vfio_device_state_is_precopy(vbasedev);
 594 }
 595
 596 /*
 597  * Note about migration rate limiting: VFIO migration buffer size is currently
 598  * limited to 1MB, so there is no need to check if migration rate exceeded (as
 599  * in the worst case it will exceed by 1MB). However, if the buffer size is
 600  * later changed to a bigger value, migration rate should be enforced here.
 601  */
 602 static int vfio_save_iterate(QEMUFile *f, void *opaque)
 603 {
 604     VFIODevice *vbasedev = opaque;
 605     VFIOMigration *migration = vbasedev->migration;
 606     ssize_t data_size;
 607
 608     data_size = vfio_save_block(f, migration);
 609     if (data_size < 0) {
 610         return data_size;
 611     }
 612
 613     vfio_update_estimated_pending_data(migration, data_size);
 614
 615     if (migrate_switchover_ack() && !migration->precopy_init_size &&
 616         !migration->initial_data_sent) {
 617         qemu_put_be64(f, VFIO_MIG_FLAG_DEV_INIT_DATA_SENT);
 618         migration->initial_data_sent = true;
 619     } else {
 620         qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
 621     }
 622
 623     trace_vfio_save_iterate(vbasedev->name, migration->precopy_init_size,
 624                             migration->precopy_dirty_size);
 625
 626     return !migration->precopy_init_size && !migration->precopy_dirty_size;
 627 }
 628
 629 static int vfio_save_complete_precopy(QEMUFile *f, void *opaque)
 630 {
 631     VFIODevice *vbasedev = opaque;
 632     ssize_t data_size;
 633     int ret;
 634     Error *local_err = NULL;
 635
 636     /* We reach here with device state STOP or STOP_COPY only */
 637     ret = vfio_migration_set_state(vbasedev, VFIO_DEVICE_STATE_STOP_COPY,
 638                                    VFIO_DEVICE_STATE_STOP, &local_err);
 639     if (ret) {
 640         error_report_err(local_err);
 641         return ret;
 642     }
 643
 644     do {
 645         data_size = vfio_save_block(f, vbasedev->migration);
 646         if (data_size < 0) {
 647             return data_size;
 648         }
 649     } while (data_size);
 650
 651     qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
 652     ret = qemu_file_get_error(f);
 653
 654     trace_vfio_save_complete_precopy(vbasedev->name, ret);
 655
 656     return ret;
 657 }
 658
 659 static void vfio_save_state(QEMUFile *f, void *opaque)
 660 {
 661     VFIODevice *vbasedev = opaque;
 662     Error *local_err = NULL;
 663     int ret;
 664
 665     ret = vfio_save_device_config_state(f, opaque, &local_err);
 666     if (ret) {
 667         error_prepend(&local_err,
 668                       "vfio: Failed to save device config space of %s - ",
 669                       vbasedev->name);
 670         qemu_file_set_error_obj(f, ret, local_err);
 671     }
 672 }
 673
 674 static int vfio_load_setup(QEMUFile *f, void *opaque, Error **errp)
 675 {
 676     VFIODevice *vbasedev = opaque;
 677
 678     return vfio_migration_set_state(vbasedev, VFIO_DEVICE_STATE_RESUMING,
 679                                     vbasedev->migration->device_state, errp);
 680 }
 681
 682 static int vfio_load_cleanup(void *opaque)
 683 {
 684     VFIODevice *vbasedev = opaque;
 685
 686     vfio_migration_cleanup(vbasedev);
 687     trace_vfio_load_cleanup(vbasedev->name);
 688
 689     return 0;
 690 }
 691
 692 static int vfio_load_state(QEMUFile *f, void *opaque, int version_id)
 693 {
 694     VFIODevice *vbasedev = opaque;
 695     int ret = 0;
 696     uint64_t data;
 697
 698     data = qemu_get_be64(f);
 699     while (data != VFIO_MIG_FLAG_END_OF_STATE) {
 700
 701         trace_vfio_load_state(vbasedev->name, data);
 702
 703         switch (data) {
 704         case VFIO_MIG_FLAG_DEV_CONFIG_STATE:
 705         {
 706             return vfio_load_device_config_state(f, opaque);
 707         }
 708         case VFIO_MIG_FLAG_DEV_SETUP_STATE:
 709         {
 710             data = qemu_get_be64(f);
 711             if (data == VFIO_MIG_FLAG_END_OF_STATE) {
 712                 return ret;
 713             } else {
 714                 error_report("%s: SETUP STATE: EOS not found 0x%"PRIx64,
 715                              vbasedev->name, data);
 716                 return -EINVAL;
 717             }
 718             break;
 719         }
 720         case VFIO_MIG_FLAG_DEV_DATA_STATE:
 721         {
 722             uint64_t data_size = qemu_get_be64(f);
 723
 724             if (data_size) {
 725                 ret = vfio_load_buffer(f, vbasedev, data_size);
 726                 if (ret < 0) {
 727                     return ret;
 728                 }
 729             }
 730             break;
 731         }
 732         case VFIO_MIG_FLAG_DEV_INIT_DATA_SENT:
 733         {
 734             if (!vfio_precopy_supported(vbasedev) ||
 735                 !migrate_switchover_ack()) {
 736                 error_report("%s: Received INIT_DATA_SENT but switchover ack "
 737                              "is not used", vbasedev->name);
 738                 return -EINVAL;
 739             }
 740
 741             ret = qemu_loadvm_approve_switchover();
 742             if (ret) {
 743                 error_report(
 744                     "%s: qemu_loadvm_approve_switchover failed, err=%d (%s)",
 745                     vbasedev->name, ret, strerror(-ret));
 746             }
 747
 748             return ret;
 749         }
 750         default:
 751             error_report("%s: Unknown tag 0x%"PRIx64, vbasedev->name, data);
 752             return -EINVAL;
 753         }
 754
 755         data = qemu_get_be64(f);
 756         ret = qemu_file_get_error(f);
 757         if (ret) {
 758             return ret;
 759         }
 760     }
 761     return ret;
 762 }
 763
 764 static bool vfio_switchover_ack_needed(void *opaque)
 765 {
 766     VFIODevice *vbasedev = opaque;
 767
 768     return vfio_precopy_supported(vbasedev);
 769 }
 770
 771 static const SaveVMHandlers savevm_vfio_handlers = {
 772     .save_prepare = vfio_save_prepare,
 773     .save_setup = vfio_save_setup,
 774     .save_cleanup = vfio_save_cleanup,
 775     .state_pending_estimate = vfio_state_pending_estimate,
 776     .state_pending_exact = vfio_state_pending_exact,
 777     .is_active_iterate = vfio_is_active_iterate,
 778     .save_live_iterate = vfio_save_iterate,
 779     .save_live_complete_precopy = vfio_save_complete_precopy,
 780     .save_state = vfio_save_state,
 781     .load_setup = vfio_load_setup,
 782     .load_cleanup = vfio_load_cleanup,
 783     .load_state = vfio_load_state,
 784     .switchover_ack_needed = vfio_switchover_ack_needed,
 785 };
 786
 787 /* ---------------------------------------------------------------------- */
 788
 789 static void vfio_vmstate_change_prepare(void *opaque, bool running,
 790                                         RunState state)
 791 {
 792     VFIODevice *vbasedev = opaque;
 793     VFIOMigration *migration = vbasedev->migration;
 794     enum vfio_device_mig_state new_state;
 795     Error *local_err = NULL;
 796     int ret;
 797
 798     new_state = migration->device_state == VFIO_DEVICE_STATE_PRE_COPY ?
 799                     VFIO_DEVICE_STATE_PRE_COPY_P2P :
 800                     VFIO_DEVICE_STATE_RUNNING_P2P;
 801
 802     ret = vfio_migration_set_state_or_reset(vbasedev, new_state, &local_err);
 803     if (ret) {
 804         /*
 805          * Migration should be aborted in this case, but vm_state_notify()
 806          * currently does not support reporting failures.
 807          */
 808         migration_file_set_error(ret, local_err);
 809     }
 810
 811     trace_vfio_vmstate_change_prepare(vbasedev->name, running,
 812                                       RunState_str(state),
 813                                       mig_state_to_str(new_state));
 814 }
 815
 816 static void vfio_vmstate_change(void *opaque, bool running, RunState state)
 817 {
 818     VFIODevice *vbasedev = opaque;
 819     enum vfio_device_mig_state new_state;
 820     Error *local_err = NULL;
 821     int ret;
 822
 823     if (running) {
 824         new_state = VFIO_DEVICE_STATE_RUNNING;
 825     } else {
 826         new_state =
 827             (vfio_device_state_is_precopy(vbasedev) &&
 828              (state == RUN_STATE_FINISH_MIGRATE || state == RUN_STATE_PAUSED)) ?
 829                 VFIO_DEVICE_STATE_STOP_COPY :
 830                 VFIO_DEVICE_STATE_STOP;
 831     }
 832
 833     ret = vfio_migration_set_state_or_reset(vbasedev, new_state, &local_err);
 834     if (ret) {
 835         /*
 836          * Migration should be aborted in this case, but vm_state_notify()
 837          * currently does not support reporting failures.
 838          */
 839         migration_file_set_error(ret, local_err);
 840     }
 841
 842     trace_vfio_vmstate_change(vbasedev->name, running, RunState_str(state),
 843                               mig_state_to_str(new_state));
 844 }
 845
 846 static int vfio_migration_state_notifier(NotifierWithReturn *notifier,
 847                                          MigrationEvent *e, Error **errp)
 848 {
 849     VFIOMigration *migration = container_of(notifier, VFIOMigration,
 850                                             migration_state);
 851     VFIODevice *vbasedev = migration->vbasedev;
 852     Error *local_err = NULL;
 853     int ret;
 854
 855     trace_vfio_migration_state_notifier(vbasedev->name, e->type);
 856
 857     if (e->type == MIG_EVENT_PRECOPY_FAILED) {
 858         /*
 859          * MigrationNotifyFunc may not return an error code and an Error
 860          * object for MIG_EVENT_PRECOPY_FAILED. Hence, report the error
 861          * locally and ignore the errp argument.
 862          */
 863         ret = vfio_migration_set_state_or_reset(vbasedev,
 864                                                 VFIO_DEVICE_STATE_RUNNING,
 865                                                 &local_err);
 866         if (ret) {
 867             error_report_err(local_err);
 868         }
 869     }
 870     return 0;
 871 }
 872
 873 static void vfio_migration_free(VFIODevice *vbasedev)
 874 {
 875     g_free(vbasedev->migration);
 876     vbasedev->migration = NULL;
 877 }
 878
 879 static int vfio_migration_query_flags(VFIODevice *vbasedev, uint64_t *mig_flags)
 880 {
 881     uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) +
 882                                   sizeof(struct vfio_device_feature_migration),
 883                               sizeof(uint64_t))] = {};
 884     struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
 885     struct vfio_device_feature_migration *mig =
 886         (struct vfio_device_feature_migration *)feature->data;
 887
 888     feature->argsz = sizeof(buf);
 889     feature->flags = VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_MIGRATION;
 890     if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) {
 891         return -errno;
 892     }
 893
 894     *mig_flags = mig->flags;
 895
 896     return 0;
 897 }
 898
 899 static bool vfio_dma_logging_supported(VFIODevice *vbasedev)
 900 {
 901     uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature),
 902                               sizeof(uint64_t))] = {};
 903     struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
 904
 905     feature->argsz = sizeof(buf);
 906     feature->flags = VFIO_DEVICE_FEATURE_PROBE |
 907                      VFIO_DEVICE_FEATURE_DMA_LOGGING_START;
 908
 909     return !ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature);
 910 }
 911
 912 static int vfio_migration_init(VFIODevice *vbasedev)
 913 {
 914     int ret;
 915     Object *obj;
 916     VFIOMigration *migration;
 917     char id[256] = "";
 918     g_autofree char *path = NULL, *oid = NULL;
 919     uint64_t mig_flags = 0;
 920     VMChangeStateHandler *prepare_cb;
 921
 922     if (!vbasedev->ops->vfio_get_object) {
 923         return -EINVAL;
 924     }
 925
 926     obj = vbasedev->ops->vfio_get_object(vbasedev);
 927     if (!obj) {
 928         return -EINVAL;
 929     }
 930
 931     ret = vfio_migration_query_flags(vbasedev, &mig_flags);
 932     if (ret) {
 933         return ret;
 934     }
 935
 936     /* Basic migration functionality must be supported */
 937     if (!(mig_flags & VFIO_MIGRATION_STOP_COPY)) {
 938         return -EOPNOTSUPP;
 939     }
 940
 941     vbasedev->migration = g_new0(VFIOMigration, 1);
 942     migration = vbasedev->migration;
 943     migration->vbasedev = vbasedev;
 944     migration->device_state = VFIO_DEVICE_STATE_RUNNING;
 945     migration->data_fd = -1;
 946     migration->mig_flags = mig_flags;
 947
 948     vbasedev->dirty_pages_supported = vfio_dma_logging_supported(vbasedev);
 949
 950     oid = vmstate_if_get_id(VMSTATE_IF(DEVICE(obj)));
 951     if (oid) {
 952         path = g_strdup_printf("%s/vfio", oid);
 953     } else {
 954         path = g_strdup("vfio");
 955     }
 956     strpadcpy(id, sizeof(id), path, '\0');
 957
 958     register_savevm_live(id, VMSTATE_INSTANCE_ID_ANY, 1, &savevm_vfio_handlers,
 959                          vbasedev);
 960
 961     prepare_cb = migration->mig_flags & VFIO_MIGRATION_P2P ?
 962                      vfio_vmstate_change_prepare :
 963                      NULL;
 964     migration->vm_state = qdev_add_vm_change_state_handler_full(
 965         vbasedev->dev, vfio_vmstate_change, prepare_cb, vbasedev);
 966     migration_add_notifier(&migration->migration_state,
 967                            vfio_migration_state_notifier);
 968
 969     return 0;
 970 }
 971
 972 static void vfio_migration_deinit(VFIODevice *vbasedev)
 973 {
 974     VFIOMigration *migration = vbasedev->migration;
 975
 976     migration_remove_notifier(&migration->migration_state);
 977     qemu_del_vm_change_state_handler(migration->vm_state);
 978     unregister_savevm(VMSTATE_IF(vbasedev->dev), "vfio", vbasedev);
 979     vfio_migration_free(vbasedev);
 980     vfio_unblock_multiple_devices_migration();
 981 }
 982
 983 static int vfio_block_migration(VFIODevice *vbasedev, Error *err, Error **errp)
 984 {
 985     if (vbasedev->enable_migration == ON_OFF_AUTO_ON) {
 986         error_propagate(errp, err);
 987         return -EINVAL;
 988     }
 989
 990     vbasedev->migration_blocker = error_copy(err);
 991     error_free(err);
 992
 993     return migrate_add_blocker_normal(&vbasedev->migration_blocker, errp);
 994 }
 995
 996 /* ---------------------------------------------------------------------- */
 997
 998 int64_t vfio_mig_bytes_transferred(void)
 999 {
1000     return bytes_transferred;
1001 }
1002
1003 void vfio_reset_bytes_transferred(void)
1004 {
1005     bytes_transferred = 0;
1006 }
1007
1008 /*
1009  * Return true when either migration initialized or blocker registered.
1010  * Currently only return false when adding blocker fails which will
1011  * de-register vfio device.
1012  */
1013 bool vfio_migration_realize(VFIODevice *vbasedev, Error **errp)
1014 {
1015     Error *err = NULL;
1016     int ret;
1017
1018     if (vbasedev->enable_migration == ON_OFF_AUTO_OFF) {
1019         error_setg(&err, "%s: Migration is disabled for VFIO device",
1020                    vbasedev->name);
1021         return !vfio_block_migration(vbasedev, err, errp);
1022     }
1023
1024     ret = vfio_migration_init(vbasedev);
1025     if (ret) {
1026         if (ret == -ENOTTY) {
1027             error_setg(&err, "%s: VFIO migration is not supported in kernel",
1028                        vbasedev->name);
1029         } else {
1030             error_setg(&err,
1031                        "%s: Migration couldn't be initialized for VFIO device, "
1032                        "err: %d (%s)",
1033                        vbasedev->name, ret, strerror(-ret));
1034         }
1035
1036         return !vfio_block_migration(vbasedev, err, errp);
1037     }
1038
1039     if ((!vbasedev->dirty_pages_supported ||
1040          vbasedev->device_dirty_page_tracking == ON_OFF_AUTO_OFF) &&
1041         !vbasedev->iommu_dirty_tracking) {
1042         if (vbasedev->enable_migration == ON_OFF_AUTO_AUTO) {
1043             error_setg(&err,
1044                        "%s: VFIO device doesn't support device and "
1045                        "IOMMU dirty tracking", vbasedev->name);
1046             goto add_blocker;
1047         }
1048
1049         warn_report("%s: VFIO device doesn't support device and "
1050                     "IOMMU dirty tracking", vbasedev->name);
1051     }
1052
1053     ret = vfio_block_multiple_devices_migration(vbasedev, errp);
1054     if (ret) {
1055         goto out_deinit;
1056     }
1057
1058     if (vfio_viommu_preset(vbasedev)) {
1059         error_setg(&err, "%s: Migration is currently not supported "
1060                    "with vIOMMU enabled", vbasedev->name);
1061         goto add_blocker;
1062     }
1063
1064     trace_vfio_migration_realize(vbasedev->name);
1065     return true;
1066
1067 add_blocker:
1068     ret = vfio_block_migration(vbasedev, err, errp);
1069 out_deinit:
1070     if (ret) {
1071         vfio_migration_deinit(vbasedev);
1072     }
1073     return !ret;
1074 }
1075
1076 void vfio_migration_exit(VFIODevice *vbasedev)
1077 {
1078     if (vbasedev->migration) {
1079         vfio_migration_deinit(vbasedev);
1080     }
1081
1082     migrate_del_blocker(&vbasedev->migration_blocker);
1083 }