hw/vfio/container.c

   1 /*
   2  * generic functions used by VFIO devices
   3  *
   4  * Copyright Red Hat, Inc. 2012
   5  *
   6  * Authors:
   7  *  Alex Williamson <alex.williamson@redhat.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Based on qemu-kvm device-assignment:
  13  *  Adapted for KVM by Qumranet.
  14  *  Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
  15  *  Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
  16  *  Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
  17  *  Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
  18  *  Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
  19  */
  20
  21 #include "qemu/osdep.h"
  22 #include <sys/ioctl.h>
  23 #include <linux/vfio.h>
  24
  25 #include "hw/vfio/vfio-common.h"
  26 #include "exec/address-spaces.h"
  27 #include "exec/memory.h"
  28 #include "exec/ram_addr.h"
  29 #include "hw/hw.h"
  30 #include "qemu/error-report.h"
  31 #include "qemu/range.h"
  32 #include "sysemu/reset.h"
  33 #include "trace.h"
  34 #include "qapi/error.h"
  35 #include "migration/migration.h"
  36 #include "pci.h"
  37
  38 VFIOGroupList vfio_group_list =
  39     QLIST_HEAD_INITIALIZER(vfio_group_list);
  40
  41 static int vfio_ram_block_discard_disable(VFIOContainer *container, bool state)
  42 {
  43     switch (container->iommu_type) {
  44     case VFIO_TYPE1v2_IOMMU:
  45     case VFIO_TYPE1_IOMMU:
  46         /*
  47          * We support coordinated discarding of RAM via the RamDiscardManager.
  48          */
  49         return ram_block_uncoordinated_discard_disable(state);
  50     default:
  51         /*
  52          * VFIO_SPAPR_TCE_IOMMU most probably works just fine with
  53          * RamDiscardManager, however, it is completely untested.
  54          *
  55          * VFIO_SPAPR_TCE_v2_IOMMU with "DMA memory preregistering" does
  56          * completely the opposite of managing mapping/pinning dynamically as
  57          * required by RamDiscardManager. We would have to special-case sections
  58          * with a RamDiscardManager.
  59          */
  60         return ram_block_discard_disable(state);
  61     }
  62 }
  63
  64 static int vfio_dma_unmap_bitmap(const VFIOContainer *container,
  65                                  hwaddr iova, ram_addr_t size,
  66                                  IOMMUTLBEntry *iotlb)
  67 {
  68     const VFIOContainerBase *bcontainer = &container->bcontainer;
  69     struct vfio_iommu_type1_dma_unmap *unmap;
  70     struct vfio_bitmap *bitmap;
  71     VFIOBitmap vbmap;
  72     int ret;
  73
  74     ret = vfio_bitmap_alloc(&vbmap, size);
  75     if (ret) {
  76         return ret;
  77     }
  78
  79     unmap = g_malloc0(sizeof(*unmap) + sizeof(*bitmap));
  80
  81     unmap->argsz = sizeof(*unmap) + sizeof(*bitmap);
  82     unmap->iova = iova;
  83     unmap->size = size;
  84     unmap->flags |= VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP;
  85     bitmap = (struct vfio_bitmap *)&unmap->data;
  86
  87     /*
  88      * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of
  89      * qemu_real_host_page_size to mark those dirty. Hence set bitmap_pgsize
  90      * to qemu_real_host_page_size.
  91      */
  92     bitmap->pgsize = qemu_real_host_page_size();
  93     bitmap->size = vbmap.size;
  94     bitmap->data = (__u64 *)vbmap.bitmap;
  95
  96     if (vbmap.size > bcontainer->max_dirty_bitmap_size) {
  97         error_report("UNMAP: Size of bitmap too big 0x%"PRIx64, vbmap.size);
  98         ret = -E2BIG;
  99         goto unmap_exit;
 100     }
 101
 102     ret = ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, unmap);
 103     if (!ret) {
 104         cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap,
 105                 iotlb->translated_addr, vbmap.pages);
 106     } else {
 107         error_report("VFIO_UNMAP_DMA with DIRTY_BITMAP : %m");
 108     }
 109
 110 unmap_exit:
 111     g_free(unmap);
 112     g_free(vbmap.bitmap);
 113
 114     return ret;
 115 }
 116
 117 /*
 118  * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86
 119  */
 120 static int vfio_legacy_dma_unmap(const VFIOContainerBase *bcontainer,
 121                                  hwaddr iova, ram_addr_t size,
 122                                  IOMMUTLBEntry *iotlb)
 123 {
 124     const VFIOContainer *container = container_of(bcontainer, VFIOContainer,
 125                                                   bcontainer);
 126     struct vfio_iommu_type1_dma_unmap unmap = {
 127         .argsz = sizeof(unmap),
 128         .flags = 0,
 129         .iova = iova,
 130         .size = size,
 131     };
 132     bool need_dirty_sync = false;
 133     int ret;
 134
 135     if (iotlb && vfio_devices_all_running_and_mig_active(bcontainer)) {
 136         if (!vfio_devices_all_device_dirty_tracking(bcontainer) &&
 137             bcontainer->dirty_pages_supported) {
 138             return vfio_dma_unmap_bitmap(container, iova, size, iotlb);
 139         }
 140
 141         need_dirty_sync = true;
 142     }
 143
 144     while (ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, &unmap)) {
 145         /*
 146          * The type1 backend has an off-by-one bug in the kernel (71a7d3d78e3c
 147          * v4.15) where an overflow in its wrap-around check prevents us from
 148          * unmapping the last page of the address space.  Test for the error
 149          * condition and re-try the unmap excluding the last page.  The
 150          * expectation is that we've never mapped the last page anyway and this
 151          * unmap request comes via vIOMMU support which also makes it unlikely
 152          * that this page is used.  This bug was introduced well after type1 v2
 153          * support was introduced, so we shouldn't need to test for v1.  A fix
 154          * is queued for kernel v5.0 so this workaround can be removed once
 155          * affected kernels are sufficiently deprecated.
 156          */
 157         if (errno == EINVAL && unmap.size && !(unmap.iova + unmap.size) &&
 158             container->iommu_type == VFIO_TYPE1v2_IOMMU) {
 159             trace_vfio_legacy_dma_unmap_overflow_workaround();
 160             unmap.size -= 1ULL << ctz64(bcontainer->pgsizes);
 161             continue;
 162         }
 163         error_report("VFIO_UNMAP_DMA failed: %s", strerror(errno));
 164         return -errno;
 165     }
 166
 167     if (need_dirty_sync) {
 168         ret = vfio_get_dirty_bitmap(bcontainer, iova, size,
 169                                     iotlb->translated_addr);
 170         if (ret) {
 171             return ret;
 172         }
 173     }
 174
 175     return 0;
 176 }
 177
 178 static int vfio_legacy_dma_map(const VFIOContainerBase *bcontainer, hwaddr iova,
 179                                ram_addr_t size, void *vaddr, bool readonly)
 180 {
 181     const VFIOContainer *container = container_of(bcontainer, VFIOContainer,
 182                                                   bcontainer);
 183     struct vfio_iommu_type1_dma_map map = {
 184         .argsz = sizeof(map),
 185         .flags = VFIO_DMA_MAP_FLAG_READ,
 186         .vaddr = (__u64)(uintptr_t)vaddr,
 187         .iova = iova,
 188         .size = size,
 189     };
 190
 191     if (!readonly) {
 192         map.flags |= VFIO_DMA_MAP_FLAG_WRITE;
 193     }
 194
 195     /*
 196      * Try the mapping, if it fails with EBUSY, unmap the region and try
 197      * again.  This shouldn't be necessary, but we sometimes see it in
 198      * the VGA ROM space.
 199      */
 200     if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 ||
 201         (errno == EBUSY &&
 202          vfio_legacy_dma_unmap(bcontainer, iova, size, NULL) == 0 &&
 203          ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0)) {
 204         return 0;
 205     }
 206
 207     error_report("VFIO_MAP_DMA failed: %s", strerror(errno));
 208     return -errno;
 209 }
 210
 211 static int
 212 vfio_legacy_set_dirty_page_tracking(const VFIOContainerBase *bcontainer,
 213                                     bool start)
 214 {
 215     const VFIOContainer *container = container_of(bcontainer, VFIOContainer,
 216                                                   bcontainer);
 217     int ret;
 218     struct vfio_iommu_type1_dirty_bitmap dirty = {
 219         .argsz = sizeof(dirty),
 220     };
 221
 222     if (start) {
 223         dirty.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_START;
 224     } else {
 225         dirty.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP;
 226     }
 227
 228     ret = ioctl(container->fd, VFIO_IOMMU_DIRTY_PAGES, &dirty);
 229     if (ret) {
 230         ret = -errno;
 231         error_report("Failed to set dirty tracking flag 0x%x errno: %d",
 232                      dirty.flags, errno);
 233     }
 234
 235     return ret;
 236 }
 237
 238 static int vfio_legacy_query_dirty_bitmap(const VFIOContainerBase *bcontainer,
 239                                           VFIOBitmap *vbmap,
 240                                           hwaddr iova, hwaddr size)
 241 {
 242     const VFIOContainer *container = container_of(bcontainer, VFIOContainer,
 243                                                   bcontainer);
 244     struct vfio_iommu_type1_dirty_bitmap *dbitmap;
 245     struct vfio_iommu_type1_dirty_bitmap_get *range;
 246     int ret;
 247
 248     dbitmap = g_malloc0(sizeof(*dbitmap) + sizeof(*range));
 249
 250     dbitmap->argsz = sizeof(*dbitmap) + sizeof(*range);
 251     dbitmap->flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP;
 252     range = (struct vfio_iommu_type1_dirty_bitmap_get *)&dbitmap->data;
 253     range->iova = iova;
 254     range->size = size;
 255
 256     /*
 257      * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of
 258      * qemu_real_host_page_size to mark those dirty. Hence set bitmap's pgsize
 259      * to qemu_real_host_page_size.
 260      */
 261     range->bitmap.pgsize = qemu_real_host_page_size();
 262     range->bitmap.size = vbmap->size;
 263     range->bitmap.data = (__u64 *)vbmap->bitmap;
 264
 265     ret = ioctl(container->fd, VFIO_IOMMU_DIRTY_PAGES, dbitmap);
 266     if (ret) {
 267         ret = -errno;
 268         error_report("Failed to get dirty bitmap for iova: 0x%"PRIx64
 269                 " size: 0x%"PRIx64" err: %d", (uint64_t)range->iova,
 270                 (uint64_t)range->size, errno);
 271     }
 272
 273     g_free(dbitmap);
 274
 275     return ret;
 276 }
 277
 278 static struct vfio_info_cap_header *
 279 vfio_get_iommu_type1_info_cap(struct vfio_iommu_type1_info *info, uint16_t id)
 280 {
 281     if (!(info->flags & VFIO_IOMMU_INFO_CAPS)) {
 282         return NULL;
 283     }
 284
 285     return vfio_get_cap((void *)info, info->cap_offset, id);
 286 }
 287
 288 bool vfio_get_info_dma_avail(struct vfio_iommu_type1_info *info,
 289                              unsigned int *avail)
 290 {
 291     struct vfio_info_cap_header *hdr;
 292     struct vfio_iommu_type1_info_dma_avail *cap;
 293
 294     /* If the capability cannot be found, assume no DMA limiting */
 295     hdr = vfio_get_iommu_type1_info_cap(info,
 296                                         VFIO_IOMMU_TYPE1_INFO_DMA_AVAIL);
 297     if (!hdr) {
 298         return false;
 299     }
 300
 301     if (avail != NULL) {
 302         cap = (void *) hdr;
 303         *avail = cap->avail;
 304     }
 305
 306     return true;
 307 }
 308
 309 static bool vfio_get_info_iova_range(struct vfio_iommu_type1_info *info,
 310                                      VFIOContainerBase *bcontainer)
 311 {
 312     struct vfio_info_cap_header *hdr;
 313     struct vfio_iommu_type1_info_cap_iova_range *cap;
 314
 315     hdr = vfio_get_iommu_type1_info_cap(info,
 316                                         VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE);
 317     if (!hdr) {
 318         return false;
 319     }
 320
 321     cap = (void *)hdr;
 322
 323     for (int i = 0; i < cap->nr_iovas; i++) {
 324         Range *range = g_new(Range, 1);
 325
 326         range_set_bounds(range, cap->iova_ranges[i].start,
 327                          cap->iova_ranges[i].end);
 328         bcontainer->iova_ranges =
 329             range_list_insert(bcontainer->iova_ranges, range);
 330     }
 331
 332     return true;
 333 }
 334
 335 static void vfio_kvm_device_add_group(VFIOGroup *group)
 336 {
 337     Error *err = NULL;
 338
 339     if (vfio_kvm_device_add_fd(group->fd, &err)) {
 340         error_reportf_err(err, "group ID %d: ", group->groupid);
 341     }
 342 }
 343
 344 static void vfio_kvm_device_del_group(VFIOGroup *group)
 345 {
 346     Error *err = NULL;
 347
 348     if (vfio_kvm_device_del_fd(group->fd, &err)) {
 349         error_reportf_err(err, "group ID %d: ", group->groupid);
 350     }
 351 }
 352
 353 /*
 354  * vfio_get_iommu_type - selects the richest iommu_type (v2 first)
 355  */
 356 static int vfio_get_iommu_type(VFIOContainer *container,
 357                                Error **errp)
 358 {
 359     int iommu_types[] = { VFIO_TYPE1v2_IOMMU, VFIO_TYPE1_IOMMU,
 360                           VFIO_SPAPR_TCE_v2_IOMMU, VFIO_SPAPR_TCE_IOMMU };
 361     int i;
 362
 363     for (i = 0; i < ARRAY_SIZE(iommu_types); i++) {
 364         if (ioctl(container->fd, VFIO_CHECK_EXTENSION, iommu_types[i])) {
 365             return iommu_types[i];
 366         }
 367     }
 368     error_setg(errp, "No available IOMMU models");
 369     return -EINVAL;
 370 }
 371
 372 static int vfio_init_container(VFIOContainer *container, int group_fd,
 373                                Error **errp)
 374 {
 375     int iommu_type, ret;
 376
 377     iommu_type = vfio_get_iommu_type(container, errp);
 378     if (iommu_type < 0) {
 379         return iommu_type;
 380     }
 381
 382     ret = ioctl(group_fd, VFIO_GROUP_SET_CONTAINER, &container->fd);
 383     if (ret) {
 384         error_setg_errno(errp, errno, "Failed to set group container");
 385         return -errno;
 386     }
 387
 388     while (ioctl(container->fd, VFIO_SET_IOMMU, iommu_type)) {
 389         if (iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) {
 390             /*
 391              * On sPAPR, despite the IOMMU subdriver always advertises v1 and
 392              * v2, the running platform may not support v2 and there is no
 393              * way to guess it until an IOMMU group gets added to the container.
 394              * So in case it fails with v2, try v1 as a fallback.
 395              */
 396             iommu_type = VFIO_SPAPR_TCE_IOMMU;
 397             continue;
 398         }
 399         error_setg_errno(errp, errno, "Failed to set iommu for container");
 400         return -errno;
 401     }
 402
 403     container->iommu_type = iommu_type;
 404     return 0;
 405 }
 406
 407 static int vfio_get_iommu_info(VFIOContainer *container,
 408                                struct vfio_iommu_type1_info **info)
 409 {
 410
 411     size_t argsz = sizeof(struct vfio_iommu_type1_info);
 412
 413     *info = g_new0(struct vfio_iommu_type1_info, 1);
 414 again:
 415     (*info)->argsz = argsz;
 416
 417     if (ioctl(container->fd, VFIO_IOMMU_GET_INFO, *info)) {
 418         g_free(*info);
 419         *info = NULL;
 420         return -errno;
 421     }
 422
 423     if (((*info)->argsz > argsz)) {
 424         argsz = (*info)->argsz;
 425         *info = g_realloc(*info, argsz);
 426         goto again;
 427     }
 428
 429     return 0;
 430 }
 431
 432 static struct vfio_info_cap_header *
 433 vfio_get_iommu_info_cap(struct vfio_iommu_type1_info *info, uint16_t id)
 434 {
 435     struct vfio_info_cap_header *hdr;
 436     void *ptr = info;
 437
 438     if (!(info->flags & VFIO_IOMMU_INFO_CAPS)) {
 439         return NULL;
 440     }
 441
 442     for (hdr = ptr + info->cap_offset; hdr != ptr; hdr = ptr + hdr->next) {
 443         if (hdr->id == id) {
 444             return hdr;
 445         }
 446     }
 447
 448     return NULL;
 449 }
 450
 451 static void vfio_get_iommu_info_migration(VFIOContainer *container,
 452                                           struct vfio_iommu_type1_info *info)
 453 {
 454     struct vfio_info_cap_header *hdr;
 455     struct vfio_iommu_type1_info_cap_migration *cap_mig;
 456     VFIOContainerBase *bcontainer = &container->bcontainer;
 457
 458     hdr = vfio_get_iommu_info_cap(info, VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION);
 459     if (!hdr) {
 460         return;
 461     }
 462
 463     cap_mig = container_of(hdr, struct vfio_iommu_type1_info_cap_migration,
 464                             header);
 465
 466     /*
 467      * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of
 468      * qemu_real_host_page_size to mark those dirty.
 469      */
 470     if (cap_mig->pgsize_bitmap & qemu_real_host_page_size()) {
 471         bcontainer->dirty_pages_supported = true;
 472         bcontainer->max_dirty_bitmap_size = cap_mig->max_dirty_bitmap_size;
 473         bcontainer->dirty_pgsizes = cap_mig->pgsize_bitmap;
 474     }
 475 }
 476
 477 static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
 478                                   Error **errp)
 479 {
 480     VFIOContainer *container;
 481     VFIOContainerBase *bcontainer;
 482     int ret, fd;
 483     VFIOAddressSpace *space;
 484
 485     space = vfio_get_address_space(as);
 486
 487     /*
 488      * VFIO is currently incompatible with discarding of RAM insofar as the
 489      * madvise to purge (zap) the page from QEMU's address space does not
 490      * interact with the memory API and therefore leaves stale virtual to
 491      * physical mappings in the IOMMU if the page was previously pinned.  We
 492      * therefore set discarding broken for each group added to a container,
 493      * whether the container is used individually or shared.  This provides
 494      * us with options to allow devices within a group to opt-in and allow
 495      * discarding, so long as it is done consistently for a group (for instance
 496      * if the device is an mdev device where it is known that the host vendor
 497      * driver will never pin pages outside of the working set of the guest
 498      * driver, which would thus not be discarding candidates).
 499      *
 500      * The first opportunity to induce pinning occurs here where we attempt to
 501      * attach the group to existing containers within the AddressSpace.  If any
 502      * pages are already zapped from the virtual address space, such as from
 503      * previous discards, new pinning will cause valid mappings to be
 504      * re-established.  Likewise, when the overall MemoryListener for a new
 505      * container is registered, a replay of mappings within the AddressSpace
 506      * will occur, re-establishing any previously zapped pages as well.
 507      *
 508      * Especially virtio-balloon is currently only prevented from discarding
 509      * new memory, it will not yet set ram_block_discard_set_required() and
 510      * therefore, neither stops us here or deals with the sudden memory
 511      * consumption of inflated memory.
 512      *
 513      * We do support discarding of memory coordinated via the RamDiscardManager
 514      * with some IOMMU types. vfio_ram_block_discard_disable() handles the
 515      * details once we know which type of IOMMU we are using.
 516      */
 517
 518     QLIST_FOREACH(bcontainer, &space->containers, next) {
 519         container = container_of(bcontainer, VFIOContainer, bcontainer);
 520         if (!ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) {
 521             ret = vfio_ram_block_discard_disable(container, true);
 522             if (ret) {
 523                 error_setg_errno(errp, -ret,
 524                                  "Cannot set discarding of RAM broken");
 525                 if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER,
 526                           &container->fd)) {
 527                     error_report("vfio: error disconnecting group %d from"
 528                                  " container", group->groupid);
 529                 }
 530                 return ret;
 531             }
 532             group->container = container;
 533             QLIST_INSERT_HEAD(&container->group_list, group, container_next);
 534             vfio_kvm_device_add_group(group);
 535             return 0;
 536         }
 537     }
 538
 539     fd = qemu_open_old("/dev/vfio/vfio", O_RDWR);
 540     if (fd < 0) {
 541         error_setg_errno(errp, errno, "failed to open /dev/vfio/vfio");
 542         ret = -errno;
 543         goto put_space_exit;
 544     }
 545
 546     ret = ioctl(fd, VFIO_GET_API_VERSION);
 547     if (ret != VFIO_API_VERSION) {
 548         error_setg(errp, "supported vfio version: %d, "
 549                    "reported version: %d", VFIO_API_VERSION, ret);
 550         ret = -EINVAL;
 551         goto close_fd_exit;
 552     }
 553
 554     container = g_malloc0(sizeof(*container));
 555     container->fd = fd;
 556     bcontainer = &container->bcontainer;
 557     vfio_container_init(bcontainer, space, &vfio_legacy_ops);
 558
 559     ret = vfio_init_container(container, group->fd, errp);
 560     if (ret) {
 561         goto free_container_exit;
 562     }
 563
 564     ret = vfio_ram_block_discard_disable(container, true);
 565     if (ret) {
 566         error_setg_errno(errp, -ret, "Cannot set discarding of RAM broken");
 567         goto free_container_exit;
 568     }
 569
 570     switch (container->iommu_type) {
 571     case VFIO_TYPE1v2_IOMMU:
 572     case VFIO_TYPE1_IOMMU:
 573     {
 574         struct vfio_iommu_type1_info *info;
 575
 576         ret = vfio_get_iommu_info(container, &info);
 577         if (ret) {
 578             error_setg_errno(errp, -ret, "Failed to get VFIO IOMMU info");
 579             goto enable_discards_exit;
 580         }
 581
 582         if (info->flags & VFIO_IOMMU_INFO_PGSIZES) {
 583             bcontainer->pgsizes = info->iova_pgsizes;
 584         } else {
 585             bcontainer->pgsizes = qemu_real_host_page_size();
 586         }
 587
 588         if (!vfio_get_info_dma_avail(info, &bcontainer->dma_max_mappings)) {
 589             bcontainer->dma_max_mappings = 65535;
 590         }
 591
 592         vfio_get_info_iova_range(info, bcontainer);
 593
 594         vfio_get_iommu_info_migration(container, info);
 595         g_free(info);
 596         break;
 597     }
 598     case VFIO_SPAPR_TCE_v2_IOMMU:
 599     case VFIO_SPAPR_TCE_IOMMU:
 600     {
 601         ret = vfio_spapr_container_init(container, errp);
 602         if (ret) {
 603             goto enable_discards_exit;
 604         }
 605         break;
 606     }
 607     }
 608
 609     vfio_kvm_device_add_group(group);
 610
 611     QLIST_INIT(&container->group_list);
 612     QLIST_INSERT_HEAD(&space->containers, bcontainer, next);
 613
 614     group->container = container;
 615     QLIST_INSERT_HEAD(&container->group_list, group, container_next);
 616
 617     bcontainer->listener = vfio_memory_listener;
 618     memory_listener_register(&bcontainer->listener, bcontainer->space->as);
 619
 620     if (bcontainer->error) {
 621         ret = -1;
 622         error_propagate_prepend(errp, bcontainer->error,
 623             "memory listener initialization failed: ");
 624         goto listener_release_exit;
 625     }
 626
 627     bcontainer->initialized = true;
 628
 629     return 0;
 630 listener_release_exit:
 631     QLIST_REMOVE(group, container_next);
 632     QLIST_REMOVE(bcontainer, next);
 633     vfio_kvm_device_del_group(group);
 634     memory_listener_unregister(&bcontainer->listener);
 635     if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU ||
 636         container->iommu_type == VFIO_SPAPR_TCE_IOMMU) {
 637         vfio_spapr_container_deinit(container);
 638     }
 639
 640 enable_discards_exit:
 641     vfio_ram_block_discard_disable(container, false);
 642
 643 free_container_exit:
 644     g_free(container);
 645
 646 close_fd_exit:
 647     close(fd);
 648
 649 put_space_exit:
 650     vfio_put_address_space(space);
 651
 652     return ret;
 653 }
 654
 655 static void vfio_disconnect_container(VFIOGroup *group)
 656 {
 657     VFIOContainer *container = group->container;
 658     VFIOContainerBase *bcontainer = &container->bcontainer;
 659
 660     QLIST_REMOVE(group, container_next);
 661     group->container = NULL;
 662
 663     /*
 664      * Explicitly release the listener first before unset container,
 665      * since unset may destroy the backend container if it's the last
 666      * group.
 667      */
 668     if (QLIST_EMPTY(&container->group_list)) {
 669         memory_listener_unregister(&bcontainer->listener);
 670         if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU ||
 671             container->iommu_type == VFIO_SPAPR_TCE_IOMMU) {
 672             vfio_spapr_container_deinit(container);
 673         }
 674     }
 675
 676     if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, &container->fd)) {
 677         error_report("vfio: error disconnecting group %d from container",
 678                      group->groupid);
 679     }
 680
 681     if (QLIST_EMPTY(&container->group_list)) {
 682         VFIOAddressSpace *space = bcontainer->space;
 683
 684         vfio_container_destroy(bcontainer);
 685
 686         trace_vfio_disconnect_container(container->fd);
 687         close(container->fd);
 688         g_free(container);
 689
 690         vfio_put_address_space(space);
 691     }
 692 }
 693
 694 static VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp)
 695 {
 696     VFIOGroup *group;
 697     char path[32];
 698     struct vfio_group_status status = { .argsz = sizeof(status) };
 699
 700     QLIST_FOREACH(group, &vfio_group_list, next) {
 701         if (group->groupid == groupid) {
 702             /* Found it.  Now is it already in the right context? */
 703             if (group->container->bcontainer.space->as == as) {
 704                 return group;
 705             } else {
 706                 error_setg(errp, "group %d used in multiple address spaces",
 707                            group->groupid);
 708                 return NULL;
 709             }
 710         }
 711     }
 712
 713     group = g_malloc0(sizeof(*group));
 714
 715     snprintf(path, sizeof(path), "/dev/vfio/%d", groupid);
 716     group->fd = qemu_open_old(path, O_RDWR);
 717     if (group->fd < 0) {
 718         error_setg_errno(errp, errno, "failed to open %s", path);
 719         goto free_group_exit;
 720     }
 721
 722     if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &status)) {
 723         error_setg_errno(errp, errno, "failed to get group %d status", groupid);
 724         goto close_fd_exit;
 725     }
 726
 727     if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
 728         error_setg(errp, "group %d is not viable", groupid);
 729         error_append_hint(errp,
 730                           "Please ensure all devices within the iommu_group "
 731                           "are bound to their vfio bus driver.\n");
 732         goto close_fd_exit;
 733     }
 734
 735     group->groupid = groupid;
 736     QLIST_INIT(&group->device_list);
 737
 738     if (vfio_connect_container(group, as, errp)) {
 739         error_prepend(errp, "failed to setup container for group %d: ",
 740                       groupid);
 741         goto close_fd_exit;
 742     }
 743
 744     QLIST_INSERT_HEAD(&vfio_group_list, group, next);
 745
 746     return group;
 747
 748 close_fd_exit:
 749     close(group->fd);
 750
 751 free_group_exit:
 752     g_free(group);
 753
 754     return NULL;
 755 }
 756
 757 static void vfio_put_group(VFIOGroup *group)
 758 {
 759     if (!group || !QLIST_EMPTY(&group->device_list)) {
 760         return;
 761     }
 762
 763     if (!group->ram_block_discard_allowed) {
 764         vfio_ram_block_discard_disable(group->container, false);
 765     }
 766     vfio_kvm_device_del_group(group);
 767     vfio_disconnect_container(group);
 768     QLIST_REMOVE(group, next);
 769     trace_vfio_put_group(group->fd);
 770     close(group->fd);
 771     g_free(group);
 772 }
 773
 774 static int vfio_get_device(VFIOGroup *group, const char *name,
 775                            VFIODevice *vbasedev, Error **errp)
 776 {
 777     g_autofree struct vfio_device_info *info = NULL;
 778     int fd;
 779
 780     fd = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, name);
 781     if (fd < 0) {
 782         error_setg_errno(errp, errno, "error getting device from group %d",
 783                          group->groupid);
 784         error_append_hint(errp,
 785                       "Verify all devices in group %d are bound to vfio-<bus> "
 786                       "or pci-stub and not already in use\n", group->groupid);
 787         return fd;
 788     }
 789
 790     info = vfio_get_device_info(fd);
 791     if (!info) {
 792         error_setg_errno(errp, errno, "error getting device info");
 793         close(fd);
 794         return -1;
 795     }
 796
 797     /*
 798      * Set discarding of RAM as not broken for this group if the driver knows
 799      * the device operates compatibly with discarding.  Setting must be
 800      * consistent per group, but since compatibility is really only possible
 801      * with mdev currently, we expect singleton groups.
 802      */
 803     if (vbasedev->ram_block_discard_allowed !=
 804         group->ram_block_discard_allowed) {
 805         if (!QLIST_EMPTY(&group->device_list)) {
 806             error_setg(errp, "Inconsistent setting of support for discarding "
 807                        "RAM (e.g., balloon) within group");
 808             close(fd);
 809             return -1;
 810         }
 811
 812         if (!group->ram_block_discard_allowed) {
 813             group->ram_block_discard_allowed = true;
 814             vfio_ram_block_discard_disable(group->container, false);
 815         }
 816     }
 817
 818     vbasedev->fd = fd;
 819     vbasedev->group = group;
 820     QLIST_INSERT_HEAD(&group->device_list, vbasedev, next);
 821
 822     vbasedev->num_irqs = info->num_irqs;
 823     vbasedev->num_regions = info->num_regions;
 824     vbasedev->flags = info->flags;
 825
 826     trace_vfio_get_device(name, info->flags, info->num_regions, info->num_irqs);
 827
 828     vbasedev->reset_works = !!(info->flags & VFIO_DEVICE_FLAGS_RESET);
 829
 830     return 0;
 831 }
 832
 833 static void vfio_put_base_device(VFIODevice *vbasedev)
 834 {
 835     if (!vbasedev->group) {
 836         return;
 837     }
 838     QLIST_REMOVE(vbasedev, next);
 839     vbasedev->group = NULL;
 840     trace_vfio_put_base_device(vbasedev->fd);
 841     close(vbasedev->fd);
 842 }
 843
 844 static int vfio_device_groupid(VFIODevice *vbasedev, Error **errp)
 845 {
 846     char *tmp, group_path[PATH_MAX], *group_name;
 847     int ret, groupid;
 848     ssize_t len;
 849
 850     tmp = g_strdup_printf("%s/iommu_group", vbasedev->sysfsdev);
 851     len = readlink(tmp, group_path, sizeof(group_path));
 852     g_free(tmp);
 853
 854     if (len <= 0 || len >= sizeof(group_path)) {
 855         ret = len < 0 ? -errno : -ENAMETOOLONG;
 856         error_setg_errno(errp, -ret, "no iommu_group found");
 857         return ret;
 858     }
 859
 860     group_path[len] = 0;
 861
 862     group_name = basename(group_path);
 863     if (sscanf(group_name, "%d", &groupid) != 1) {
 864         error_setg_errno(errp, errno, "failed to read %s", group_path);
 865         return -errno;
 866     }
 867     return groupid;
 868 }
 869
 870 /*
 871  * vfio_attach_device: attach a device to a security context
 872  * @name and @vbasedev->name are likely to be different depending
 873  * on the type of the device, hence the need for passing @name
 874  */
 875 static int vfio_legacy_attach_device(const char *name, VFIODevice *vbasedev,
 876                                      AddressSpace *as, Error **errp)
 877 {
 878     int groupid = vfio_device_groupid(vbasedev, errp);
 879     VFIODevice *vbasedev_iter;
 880     VFIOGroup *group;
 881     VFIOContainerBase *bcontainer;
 882     int ret;
 883
 884     if (groupid < 0) {
 885         return groupid;
 886     }
 887
 888     trace_vfio_attach_device(vbasedev->name, groupid);
 889
 890     group = vfio_get_group(groupid, as, errp);
 891     if (!group) {
 892         return -ENOENT;
 893     }
 894
 895     QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
 896         if (strcmp(vbasedev_iter->name, vbasedev->name) == 0) {
 897             error_setg(errp, "device is already attached");
 898             vfio_put_group(group);
 899             return -EBUSY;
 900         }
 901     }
 902     ret = vfio_get_device(group, name, vbasedev, errp);
 903     if (ret) {
 904         vfio_put_group(group);
 905         return ret;
 906     }
 907
 908     bcontainer = &group->container->bcontainer;
 909     vbasedev->bcontainer = bcontainer;
 910     QLIST_INSERT_HEAD(&bcontainer->device_list, vbasedev, container_next);
 911     QLIST_INSERT_HEAD(&vfio_device_list, vbasedev, global_next);
 912
 913     return ret;
 914 }
 915
 916 static void vfio_legacy_detach_device(VFIODevice *vbasedev)
 917 {
 918     VFIOGroup *group = vbasedev->group;
 919
 920     QLIST_REMOVE(vbasedev, global_next);
 921     QLIST_REMOVE(vbasedev, container_next);
 922     vbasedev->bcontainer = NULL;
 923     trace_vfio_detach_device(vbasedev->name, group->groupid);
 924     vfio_put_base_device(vbasedev);
 925     vfio_put_group(group);
 926 }
 927
 928 static int vfio_legacy_pci_hot_reset(VFIODevice *vbasedev, bool single)
 929 {
 930     VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
 931     VFIOGroup *group;
 932     struct vfio_pci_hot_reset_info *info = NULL;
 933     struct vfio_pci_dependent_device *devices;
 934     struct vfio_pci_hot_reset *reset;
 935     int32_t *fds;
 936     int ret, i, count;
 937     bool multi = false;
 938
 939     trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
 940
 941     if (!single) {
 942         vfio_pci_pre_reset(vdev);
 943     }
 944     vdev->vbasedev.needs_reset = false;
 945
 946     ret = vfio_pci_get_pci_hot_reset_info(vdev, &info);
 947
 948     if (ret) {
 949         goto out_single;
 950     }
 951     devices = &info->devices[0];
 952
 953     trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name);
 954
 955     /* Verify that we have all the groups required */
 956     for (i = 0; i < info->count; i++) {
 957         PCIHostDeviceAddress host;
 958         VFIOPCIDevice *tmp;
 959         VFIODevice *vbasedev_iter;
 960
 961         host.domain = devices[i].segment;
 962         host.bus = devices[i].bus;
 963         host.slot = PCI_SLOT(devices[i].devfn);
 964         host.function = PCI_FUNC(devices[i].devfn);
 965
 966         trace_vfio_pci_hot_reset_dep_devices(host.domain,
 967                 host.bus, host.slot, host.function, devices[i].group_id);
 968
 969         if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
 970             continue;
 971         }
 972
 973         QLIST_FOREACH(group, &vfio_group_list, next) {
 974             if (group->groupid == devices[i].group_id) {
 975                 break;
 976             }
 977         }
 978
 979         if (!group) {
 980             if (!vdev->has_pm_reset) {
 981                 error_report("vfio: Cannot reset device %s, "
 982                              "depends on group %d which is not owned.",
 983                              vdev->vbasedev.name, devices[i].group_id);
 984             }
 985             ret = -EPERM;
 986             goto out;
 987         }
 988
 989         /* Prep dependent devices for reset and clear our marker. */
 990         QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
 991             if (!vbasedev_iter->dev->realized ||
 992                 vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
 993                 continue;
 994             }
 995             tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
 996             if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
 997                 if (single) {
 998                     ret = -EINVAL;
 999                     goto out_single;
1000                 }
1001                 vfio_pci_pre_reset(tmp);
1002                 tmp->vbasedev.needs_reset = false;
1003                 multi = true;
1004                 break;
1005             }
1006         }
1007     }
1008
1009     if (!single && !multi) {
1010         ret = -EINVAL;
1011         goto out_single;
1012     }
1013
1014     /* Determine how many group fds need to be passed */
1015     count = 0;
1016     QLIST_FOREACH(group, &vfio_group_list, next) {
1017         for (i = 0; i < info->count; i++) {
1018             if (group->groupid == devices[i].group_id) {
1019                 count++;
1020                 break;
1021             }
1022         }
1023     }
1024
1025     reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds)));
1026     reset->argsz = sizeof(*reset) + (count * sizeof(*fds));
1027     fds = &reset->group_fds[0];
1028
1029     /* Fill in group fds */
1030     QLIST_FOREACH(group, &vfio_group_list, next) {
1031         for (i = 0; i < info->count; i++) {
1032             if (group->groupid == devices[i].group_id) {
1033                 fds[reset->count++] = group->fd;
1034                 break;
1035             }
1036         }
1037     }
1038
1039     /* Bus reset! */
1040     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
1041     g_free(reset);
1042     if (ret) {
1043         ret = -errno;
1044     }
1045
1046     trace_vfio_pci_hot_reset_result(vdev->vbasedev.name,
1047                                     ret ? strerror(errno) : "Success");
1048
1049 out:
1050     /* Re-enable INTx on affected devices */
1051     for (i = 0; i < info->count; i++) {
1052         PCIHostDeviceAddress host;
1053         VFIOPCIDevice *tmp;
1054         VFIODevice *vbasedev_iter;
1055
1056         host.domain = devices[i].segment;
1057         host.bus = devices[i].bus;
1058         host.slot = PCI_SLOT(devices[i].devfn);
1059         host.function = PCI_FUNC(devices[i].devfn);
1060
1061         if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
1062             continue;
1063         }
1064
1065         QLIST_FOREACH(group, &vfio_group_list, next) {
1066             if (group->groupid == devices[i].group_id) {
1067                 break;
1068             }
1069         }
1070
1071         if (!group) {
1072             break;
1073         }
1074
1075         QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
1076             if (!vbasedev_iter->dev->realized ||
1077                 vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
1078                 continue;
1079             }
1080             tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
1081             if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
1082                 vfio_pci_post_reset(tmp);
1083                 break;
1084             }
1085         }
1086     }
1087 out_single:
1088     if (!single) {
1089         vfio_pci_post_reset(vdev);
1090     }
1091     g_free(info);
1092
1093     return ret;
1094 }
1095
1096 const VFIOIOMMUOps vfio_legacy_ops = {
1097     .dma_map = vfio_legacy_dma_map,
1098     .dma_unmap = vfio_legacy_dma_unmap,
1099     .attach_device = vfio_legacy_attach_device,
1100     .detach_device = vfio_legacy_detach_device,
1101     .set_dirty_page_tracking = vfio_legacy_set_dirty_page_tracking,
1102     .query_dirty_bitmap = vfio_legacy_query_dirty_bitmap,
1103     .pci_hot_reset = vfio_legacy_pci_hot_reset,
1104 };