hw/vfio/container.c

   1 /*
   2  * generic functions used by VFIO devices
   3  *
   4  * Copyright Red Hat, Inc. 2012
   5  *
   6  * Authors:
   7  *  Alex Williamson <alex.williamson@redhat.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Based on qemu-kvm device-assignment:
  13  *  Adapted for KVM by Qumranet.
  14  *  Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
  15  *  Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
  16  *  Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
  17  *  Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
  18  *  Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
  19  */
  20
  21 #include "qemu/osdep.h"
  22 #include <sys/ioctl.h>
  23 #include <linux/vfio.h>
  24
  25 #include "hw/vfio/vfio-common.h"
  26 #include "exec/address-spaces.h"
  27 #include "exec/memory.h"
  28 #include "exec/ram_addr.h"
  29 #include "hw/hw.h"
  30 #include "qemu/error-report.h"
  31 #include "qemu/range.h"
  32 #include "sysemu/reset.h"
  33 #include "trace.h"
  34 #include "qapi/error.h"
  35 #include "migration/migration.h"
  36
  37 VFIOGroupList vfio_group_list =
  38     QLIST_HEAD_INITIALIZER(vfio_group_list);
  39
  40 static int vfio_ram_block_discard_disable(VFIOContainer *container, bool state)
  41 {
  42     switch (container->iommu_type) {
  43     case VFIO_TYPE1v2_IOMMU:
  44     case VFIO_TYPE1_IOMMU:
  45         /*
  46          * We support coordinated discarding of RAM via the RamDiscardManager.
  47          */
  48         return ram_block_uncoordinated_discard_disable(state);
  49     default:
  50         /*
  51          * VFIO_SPAPR_TCE_IOMMU most probably works just fine with
  52          * RamDiscardManager, however, it is completely untested.
  53          *
  54          * VFIO_SPAPR_TCE_v2_IOMMU with "DMA memory preregistering" does
  55          * completely the opposite of managing mapping/pinning dynamically as
  56          * required by RamDiscardManager. We would have to special-case sections
  57          * with a RamDiscardManager.
  58          */
  59         return ram_block_discard_disable(state);
  60     }
  61 }
  62
  63 static int vfio_dma_unmap_bitmap(VFIOContainer *container,
  64                                  hwaddr iova, ram_addr_t size,
  65                                  IOMMUTLBEntry *iotlb)
  66 {
  67     struct vfio_iommu_type1_dma_unmap *unmap;
  68     struct vfio_bitmap *bitmap;
  69     VFIOBitmap vbmap;
  70     int ret;
  71
  72     ret = vfio_bitmap_alloc(&vbmap, size);
  73     if (ret) {
  74         return ret;
  75     }
  76
  77     unmap = g_malloc0(sizeof(*unmap) + sizeof(*bitmap));
  78
  79     unmap->argsz = sizeof(*unmap) + sizeof(*bitmap);
  80     unmap->iova = iova;
  81     unmap->size = size;
  82     unmap->flags |= VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP;
  83     bitmap = (struct vfio_bitmap *)&unmap->data;
  84
  85     /*
  86      * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of
  87      * qemu_real_host_page_size to mark those dirty. Hence set bitmap_pgsize
  88      * to qemu_real_host_page_size.
  89      */
  90     bitmap->pgsize = qemu_real_host_page_size();
  91     bitmap->size = vbmap.size;
  92     bitmap->data = (__u64 *)vbmap.bitmap;
  93
  94     if (vbmap.size > container->max_dirty_bitmap_size) {
  95         error_report("UNMAP: Size of bitmap too big 0x%"PRIx64, vbmap.size);
  96         ret = -E2BIG;
  97         goto unmap_exit;
  98     }
  99
 100     ret = ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, unmap);
 101     if (!ret) {
 102         cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap,
 103                 iotlb->translated_addr, vbmap.pages);
 104     } else {
 105         error_report("VFIO_UNMAP_DMA with DIRTY_BITMAP : %m");
 106     }
 107
 108 unmap_exit:
 109     g_free(unmap);
 110     g_free(vbmap.bitmap);
 111
 112     return ret;
 113 }
 114
 115 /*
 116  * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86
 117  */
 118 static int vfio_legacy_dma_unmap(VFIOContainerBase *bcontainer, hwaddr iova,
 119                                  ram_addr_t size, IOMMUTLBEntry *iotlb)
 120 {
 121     VFIOContainer *container = container_of(bcontainer, VFIOContainer,
 122                                             bcontainer);
 123     struct vfio_iommu_type1_dma_unmap unmap = {
 124         .argsz = sizeof(unmap),
 125         .flags = 0,
 126         .iova = iova,
 127         .size = size,
 128     };
 129     bool need_dirty_sync = false;
 130     int ret;
 131
 132     if (iotlb && vfio_devices_all_running_and_mig_active(bcontainer)) {
 133         if (!vfio_devices_all_device_dirty_tracking(bcontainer) &&
 134             container->bcontainer.dirty_pages_supported) {
 135             return vfio_dma_unmap_bitmap(container, iova, size, iotlb);
 136         }
 137
 138         need_dirty_sync = true;
 139     }
 140
 141     while (ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, &unmap)) {
 142         /*
 143          * The type1 backend has an off-by-one bug in the kernel (71a7d3d78e3c
 144          * v4.15) where an overflow in its wrap-around check prevents us from
 145          * unmapping the last page of the address space.  Test for the error
 146          * condition and re-try the unmap excluding the last page.  The
 147          * expectation is that we've never mapped the last page anyway and this
 148          * unmap request comes via vIOMMU support which also makes it unlikely
 149          * that this page is used.  This bug was introduced well after type1 v2
 150          * support was introduced, so we shouldn't need to test for v1.  A fix
 151          * is queued for kernel v5.0 so this workaround can be removed once
 152          * affected kernels are sufficiently deprecated.
 153          */
 154         if (errno == EINVAL && unmap.size && !(unmap.iova + unmap.size) &&
 155             container->iommu_type == VFIO_TYPE1v2_IOMMU) {
 156             trace_vfio_legacy_dma_unmap_overflow_workaround();
 157             unmap.size -= 1ULL << ctz64(bcontainer->pgsizes);
 158             continue;
 159         }
 160         error_report("VFIO_UNMAP_DMA failed: %s", strerror(errno));
 161         return -errno;
 162     }
 163
 164     if (need_dirty_sync) {
 165         ret = vfio_get_dirty_bitmap(bcontainer, iova, size,
 166                                     iotlb->translated_addr);
 167         if (ret) {
 168             return ret;
 169         }
 170     }
 171
 172     return 0;
 173 }
 174
 175 static int vfio_legacy_dma_map(VFIOContainerBase *bcontainer, hwaddr iova,
 176                                ram_addr_t size, void *vaddr, bool readonly)
 177 {
 178     VFIOContainer *container = container_of(bcontainer, VFIOContainer,
 179                                             bcontainer);
 180     struct vfio_iommu_type1_dma_map map = {
 181         .argsz = sizeof(map),
 182         .flags = VFIO_DMA_MAP_FLAG_READ,
 183         .vaddr = (__u64)(uintptr_t)vaddr,
 184         .iova = iova,
 185         .size = size,
 186     };
 187
 188     if (!readonly) {
 189         map.flags |= VFIO_DMA_MAP_FLAG_WRITE;
 190     }
 191
 192     /*
 193      * Try the mapping, if it fails with EBUSY, unmap the region and try
 194      * again.  This shouldn't be necessary, but we sometimes see it in
 195      * the VGA ROM space.
 196      */
 197     if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 ||
 198         (errno == EBUSY &&
 199          vfio_legacy_dma_unmap(bcontainer, iova, size, NULL) == 0 &&
 200          ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0)) {
 201         return 0;
 202     }
 203
 204     error_report("VFIO_MAP_DMA failed: %s", strerror(errno));
 205     return -errno;
 206 }
 207
 208 static int vfio_legacy_set_dirty_page_tracking(VFIOContainerBase *bcontainer,
 209                                                bool start)
 210 {
 211     VFIOContainer *container = container_of(bcontainer, VFIOContainer,
 212                                             bcontainer);
 213     int ret;
 214     struct vfio_iommu_type1_dirty_bitmap dirty = {
 215         .argsz = sizeof(dirty),
 216     };
 217
 218     if (!bcontainer->dirty_pages_supported) {
 219         return 0;
 220     }
 221
 222     if (start) {
 223         dirty.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_START;
 224     } else {
 225         dirty.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP;
 226     }
 227
 228     ret = ioctl(container->fd, VFIO_IOMMU_DIRTY_PAGES, &dirty);
 229     if (ret) {
 230         ret = -errno;
 231         error_report("Failed to set dirty tracking flag 0x%x errno: %d",
 232                      dirty.flags, errno);
 233     }
 234
 235     return ret;
 236 }
 237
 238 static int vfio_legacy_query_dirty_bitmap(VFIOContainerBase *bcontainer,
 239                                           VFIOBitmap *vbmap,
 240                                           hwaddr iova, hwaddr size)
 241 {
 242     VFIOContainer *container = container_of(bcontainer, VFIOContainer,
 243                                             bcontainer);
 244     struct vfio_iommu_type1_dirty_bitmap *dbitmap;
 245     struct vfio_iommu_type1_dirty_bitmap_get *range;
 246     int ret;
 247
 248     dbitmap = g_malloc0(sizeof(*dbitmap) + sizeof(*range));
 249
 250     dbitmap->argsz = sizeof(*dbitmap) + sizeof(*range);
 251     dbitmap->flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP;
 252     range = (struct vfio_iommu_type1_dirty_bitmap_get *)&dbitmap->data;
 253     range->iova = iova;
 254     range->size = size;
 255
 256     /*
 257      * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of
 258      * qemu_real_host_page_size to mark those dirty. Hence set bitmap's pgsize
 259      * to qemu_real_host_page_size.
 260      */
 261     range->bitmap.pgsize = qemu_real_host_page_size();
 262     range->bitmap.size = vbmap->size;
 263     range->bitmap.data = (__u64 *)vbmap->bitmap;
 264
 265     ret = ioctl(container->fd, VFIO_IOMMU_DIRTY_PAGES, dbitmap);
 266     if (ret) {
 267         ret = -errno;
 268         error_report("Failed to get dirty bitmap for iova: 0x%"PRIx64
 269                 " size: 0x%"PRIx64" err: %d", (uint64_t)range->iova,
 270                 (uint64_t)range->size, errno);
 271     }
 272
 273     g_free(dbitmap);
 274
 275     return ret;
 276 }
 277
 278 static struct vfio_info_cap_header *
 279 vfio_get_iommu_type1_info_cap(struct vfio_iommu_type1_info *info, uint16_t id)
 280 {
 281     if (!(info->flags & VFIO_IOMMU_INFO_CAPS)) {
 282         return NULL;
 283     }
 284
 285     return vfio_get_cap((void *)info, info->cap_offset, id);
 286 }
 287
 288 bool vfio_get_info_dma_avail(struct vfio_iommu_type1_info *info,
 289                              unsigned int *avail)
 290 {
 291     struct vfio_info_cap_header *hdr;
 292     struct vfio_iommu_type1_info_dma_avail *cap;
 293
 294     /* If the capability cannot be found, assume no DMA limiting */
 295     hdr = vfio_get_iommu_type1_info_cap(info,
 296                                         VFIO_IOMMU_TYPE1_INFO_DMA_AVAIL);
 297     if (!hdr) {
 298         return false;
 299     }
 300
 301     if (avail != NULL) {
 302         cap = (void *) hdr;
 303         *avail = cap->avail;
 304     }
 305
 306     return true;
 307 }
 308
 309 static bool vfio_get_info_iova_range(struct vfio_iommu_type1_info *info,
 310                                      VFIOContainer *container)
 311 {
 312     struct vfio_info_cap_header *hdr;
 313     struct vfio_iommu_type1_info_cap_iova_range *cap;
 314
 315     hdr = vfio_get_iommu_type1_info_cap(info,
 316                                         VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE);
 317     if (!hdr) {
 318         return false;
 319     }
 320
 321     cap = (void *)hdr;
 322
 323     for (int i = 0; i < cap->nr_iovas; i++) {
 324         Range *range = g_new(Range, 1);
 325
 326         range_set_bounds(range, cap->iova_ranges[i].start,
 327                          cap->iova_ranges[i].end);
 328         container->iova_ranges =
 329             range_list_insert(container->iova_ranges, range);
 330     }
 331
 332     return true;
 333 }
 334
 335 static void vfio_kvm_device_add_group(VFIOGroup *group)
 336 {
 337     Error *err = NULL;
 338
 339     if (vfio_kvm_device_add_fd(group->fd, &err)) {
 340         error_reportf_err(err, "group ID %d: ", group->groupid);
 341     }
 342 }
 343
 344 static void vfio_kvm_device_del_group(VFIOGroup *group)
 345 {
 346     Error *err = NULL;
 347
 348     if (vfio_kvm_device_del_fd(group->fd, &err)) {
 349         error_reportf_err(err, "group ID %d: ", group->groupid);
 350     }
 351 }
 352
 353 /*
 354  * vfio_get_iommu_type - selects the richest iommu_type (v2 first)
 355  */
 356 static int vfio_get_iommu_type(VFIOContainer *container,
 357                                Error **errp)
 358 {
 359     int iommu_types[] = { VFIO_TYPE1v2_IOMMU, VFIO_TYPE1_IOMMU,
 360                           VFIO_SPAPR_TCE_v2_IOMMU, VFIO_SPAPR_TCE_IOMMU };
 361     int i;
 362
 363     for (i = 0; i < ARRAY_SIZE(iommu_types); i++) {
 364         if (ioctl(container->fd, VFIO_CHECK_EXTENSION, iommu_types[i])) {
 365             return iommu_types[i];
 366         }
 367     }
 368     error_setg(errp, "No available IOMMU models");
 369     return -EINVAL;
 370 }
 371
 372 static int vfio_init_container(VFIOContainer *container, int group_fd,
 373                                Error **errp)
 374 {
 375     int iommu_type, ret;
 376
 377     iommu_type = vfio_get_iommu_type(container, errp);
 378     if (iommu_type < 0) {
 379         return iommu_type;
 380     }
 381
 382     ret = ioctl(group_fd, VFIO_GROUP_SET_CONTAINER, &container->fd);
 383     if (ret) {
 384         error_setg_errno(errp, errno, "Failed to set group container");
 385         return -errno;
 386     }
 387
 388     while (ioctl(container->fd, VFIO_SET_IOMMU, iommu_type)) {
 389         if (iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) {
 390             /*
 391              * On sPAPR, despite the IOMMU subdriver always advertises v1 and
 392              * v2, the running platform may not support v2 and there is no
 393              * way to guess it until an IOMMU group gets added to the container.
 394              * So in case it fails with v2, try v1 as a fallback.
 395              */
 396             iommu_type = VFIO_SPAPR_TCE_IOMMU;
 397             continue;
 398         }
 399         error_setg_errno(errp, errno, "Failed to set iommu for container");
 400         return -errno;
 401     }
 402
 403     container->iommu_type = iommu_type;
 404     return 0;
 405 }
 406
 407 static int vfio_get_iommu_info(VFIOContainer *container,
 408                                struct vfio_iommu_type1_info **info)
 409 {
 410
 411     size_t argsz = sizeof(struct vfio_iommu_type1_info);
 412
 413     *info = g_new0(struct vfio_iommu_type1_info, 1);
 414 again:
 415     (*info)->argsz = argsz;
 416
 417     if (ioctl(container->fd, VFIO_IOMMU_GET_INFO, *info)) {
 418         g_free(*info);
 419         *info = NULL;
 420         return -errno;
 421     }
 422
 423     if (((*info)->argsz > argsz)) {
 424         argsz = (*info)->argsz;
 425         *info = g_realloc(*info, argsz);
 426         goto again;
 427     }
 428
 429     return 0;
 430 }
 431
 432 static struct vfio_info_cap_header *
 433 vfio_get_iommu_info_cap(struct vfio_iommu_type1_info *info, uint16_t id)
 434 {
 435     struct vfio_info_cap_header *hdr;
 436     void *ptr = info;
 437
 438     if (!(info->flags & VFIO_IOMMU_INFO_CAPS)) {
 439         return NULL;
 440     }
 441
 442     for (hdr = ptr + info->cap_offset; hdr != ptr; hdr = ptr + hdr->next) {
 443         if (hdr->id == id) {
 444             return hdr;
 445         }
 446     }
 447
 448     return NULL;
 449 }
 450
 451 static void vfio_get_iommu_info_migration(VFIOContainer *container,
 452                                           struct vfio_iommu_type1_info *info)
 453 {
 454     struct vfio_info_cap_header *hdr;
 455     struct vfio_iommu_type1_info_cap_migration *cap_mig;
 456     VFIOContainerBase *bcontainer = &container->bcontainer;
 457
 458     hdr = vfio_get_iommu_info_cap(info, VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION);
 459     if (!hdr) {
 460         return;
 461     }
 462
 463     cap_mig = container_of(hdr, struct vfio_iommu_type1_info_cap_migration,
 464                             header);
 465
 466     /*
 467      * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of
 468      * qemu_real_host_page_size to mark those dirty.
 469      */
 470     if (cap_mig->pgsize_bitmap & qemu_real_host_page_size()) {
 471         bcontainer->dirty_pages_supported = true;
 472         container->max_dirty_bitmap_size = cap_mig->max_dirty_bitmap_size;
 473         container->dirty_pgsizes = cap_mig->pgsize_bitmap;
 474     }
 475 }
 476
 477 static void vfio_free_container(VFIOContainer *container)
 478 {
 479     g_list_free_full(container->iova_ranges, g_free);
 480     g_free(container);
 481 }
 482
 483 static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
 484                                   Error **errp)
 485 {
 486     VFIOContainer *container;
 487     VFIOContainerBase *bcontainer;
 488     int ret, fd;
 489     VFIOAddressSpace *space;
 490
 491     space = vfio_get_address_space(as);
 492
 493     /*
 494      * VFIO is currently incompatible with discarding of RAM insofar as the
 495      * madvise to purge (zap) the page from QEMU's address space does not
 496      * interact with the memory API and therefore leaves stale virtual to
 497      * physical mappings in the IOMMU if the page was previously pinned.  We
 498      * therefore set discarding broken for each group added to a container,
 499      * whether the container is used individually or shared.  This provides
 500      * us with options to allow devices within a group to opt-in and allow
 501      * discarding, so long as it is done consistently for a group (for instance
 502      * if the device is an mdev device where it is known that the host vendor
 503      * driver will never pin pages outside of the working set of the guest
 504      * driver, which would thus not be discarding candidates).
 505      *
 506      * The first opportunity to induce pinning occurs here where we attempt to
 507      * attach the group to existing containers within the AddressSpace.  If any
 508      * pages are already zapped from the virtual address space, such as from
 509      * previous discards, new pinning will cause valid mappings to be
 510      * re-established.  Likewise, when the overall MemoryListener for a new
 511      * container is registered, a replay of mappings within the AddressSpace
 512      * will occur, re-establishing any previously zapped pages as well.
 513      *
 514      * Especially virtio-balloon is currently only prevented from discarding
 515      * new memory, it will not yet set ram_block_discard_set_required() and
 516      * therefore, neither stops us here or deals with the sudden memory
 517      * consumption of inflated memory.
 518      *
 519      * We do support discarding of memory coordinated via the RamDiscardManager
 520      * with some IOMMU types. vfio_ram_block_discard_disable() handles the
 521      * details once we know which type of IOMMU we are using.
 522      */
 523
 524     QLIST_FOREACH(bcontainer, &space->containers, next) {
 525         container = container_of(bcontainer, VFIOContainer, bcontainer);
 526         if (!ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) {
 527             ret = vfio_ram_block_discard_disable(container, true);
 528             if (ret) {
 529                 error_setg_errno(errp, -ret,
 530                                  "Cannot set discarding of RAM broken");
 531                 if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER,
 532                           &container->fd)) {
 533                     error_report("vfio: error disconnecting group %d from"
 534                                  " container", group->groupid);
 535                 }
 536                 return ret;
 537             }
 538             group->container = container;
 539             QLIST_INSERT_HEAD(&container->group_list, group, container_next);
 540             vfio_kvm_device_add_group(group);
 541             return 0;
 542         }
 543     }
 544
 545     fd = qemu_open_old("/dev/vfio/vfio", O_RDWR);
 546     if (fd < 0) {
 547         error_setg_errno(errp, errno, "failed to open /dev/vfio/vfio");
 548         ret = -errno;
 549         goto put_space_exit;
 550     }
 551
 552     ret = ioctl(fd, VFIO_GET_API_VERSION);
 553     if (ret != VFIO_API_VERSION) {
 554         error_setg(errp, "supported vfio version: %d, "
 555                    "reported version: %d", VFIO_API_VERSION, ret);
 556         ret = -EINVAL;
 557         goto close_fd_exit;
 558     }
 559
 560     container = g_malloc0(sizeof(*container));
 561     container->fd = fd;
 562     container->iova_ranges = NULL;
 563     bcontainer = &container->bcontainer;
 564     vfio_container_init(bcontainer, space, &vfio_legacy_ops);
 565
 566     ret = vfio_init_container(container, group->fd, errp);
 567     if (ret) {
 568         goto free_container_exit;
 569     }
 570
 571     ret = vfio_ram_block_discard_disable(container, true);
 572     if (ret) {
 573         error_setg_errno(errp, -ret, "Cannot set discarding of RAM broken");
 574         goto free_container_exit;
 575     }
 576
 577     switch (container->iommu_type) {
 578     case VFIO_TYPE1v2_IOMMU:
 579     case VFIO_TYPE1_IOMMU:
 580     {
 581         struct vfio_iommu_type1_info *info;
 582
 583         ret = vfio_get_iommu_info(container, &info);
 584         if (ret) {
 585             error_setg_errno(errp, -ret, "Failed to get VFIO IOMMU info");
 586             goto enable_discards_exit;
 587         }
 588
 589         if (info->flags & VFIO_IOMMU_INFO_PGSIZES) {
 590             bcontainer->pgsizes = info->iova_pgsizes;
 591         } else {
 592             bcontainer->pgsizes = qemu_real_host_page_size();
 593         }
 594
 595         if (!vfio_get_info_dma_avail(info, &bcontainer->dma_max_mappings)) {
 596             bcontainer->dma_max_mappings = 65535;
 597         }
 598
 599         vfio_get_info_iova_range(info, container);
 600
 601         vfio_get_iommu_info_migration(container, info);
 602         g_free(info);
 603         break;
 604     }
 605     case VFIO_SPAPR_TCE_v2_IOMMU:
 606     case VFIO_SPAPR_TCE_IOMMU:
 607     {
 608         ret = vfio_spapr_container_init(container, errp);
 609         if (ret) {
 610             goto enable_discards_exit;
 611         }
 612         break;
 613     }
 614     }
 615
 616     vfio_kvm_device_add_group(group);
 617
 618     QLIST_INIT(&container->group_list);
 619     QLIST_INSERT_HEAD(&space->containers, bcontainer, next);
 620
 621     group->container = container;
 622     QLIST_INSERT_HEAD(&container->group_list, group, container_next);
 623
 624     bcontainer->listener = vfio_memory_listener;
 625     memory_listener_register(&bcontainer->listener, bcontainer->space->as);
 626
 627     if (bcontainer->error) {
 628         ret = -1;
 629         error_propagate_prepend(errp, bcontainer->error,
 630             "memory listener initialization failed: ");
 631         goto listener_release_exit;
 632     }
 633
 634     bcontainer->initialized = true;
 635
 636     return 0;
 637 listener_release_exit:
 638     QLIST_REMOVE(group, container_next);
 639     QLIST_REMOVE(bcontainer, next);
 640     vfio_kvm_device_del_group(group);
 641     memory_listener_unregister(&bcontainer->listener);
 642     if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU ||
 643         container->iommu_type == VFIO_SPAPR_TCE_IOMMU) {
 644         vfio_spapr_container_deinit(container);
 645     }
 646
 647 enable_discards_exit:
 648     vfio_ram_block_discard_disable(container, false);
 649
 650 free_container_exit:
 651     vfio_free_container(container);
 652
 653 close_fd_exit:
 654     close(fd);
 655
 656 put_space_exit:
 657     vfio_put_address_space(space);
 658
 659     return ret;
 660 }
 661
 662 static void vfio_disconnect_container(VFIOGroup *group)
 663 {
 664     VFIOContainer *container = group->container;
 665     VFIOContainerBase *bcontainer = &container->bcontainer;
 666
 667     QLIST_REMOVE(group, container_next);
 668     group->container = NULL;
 669
 670     /*
 671      * Explicitly release the listener first before unset container,
 672      * since unset may destroy the backend container if it's the last
 673      * group.
 674      */
 675     if (QLIST_EMPTY(&container->group_list)) {
 676         memory_listener_unregister(&bcontainer->listener);
 677         if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU ||
 678             container->iommu_type == VFIO_SPAPR_TCE_IOMMU) {
 679             vfio_spapr_container_deinit(container);
 680         }
 681     }
 682
 683     if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, &container->fd)) {
 684         error_report("vfio: error disconnecting group %d from container",
 685                      group->groupid);
 686     }
 687
 688     if (QLIST_EMPTY(&container->group_list)) {
 689         VFIOAddressSpace *space = bcontainer->space;
 690
 691         vfio_container_destroy(bcontainer);
 692
 693         trace_vfio_disconnect_container(container->fd);
 694         close(container->fd);
 695         vfio_free_container(container);
 696
 697         vfio_put_address_space(space);
 698     }
 699 }
 700
 701 static VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp)
 702 {
 703     VFIOGroup *group;
 704     char path[32];
 705     struct vfio_group_status status = { .argsz = sizeof(status) };
 706
 707     QLIST_FOREACH(group, &vfio_group_list, next) {
 708         if (group->groupid == groupid) {
 709             /* Found it.  Now is it already in the right context? */
 710             if (group->container->bcontainer.space->as == as) {
 711                 return group;
 712             } else {
 713                 error_setg(errp, "group %d used in multiple address spaces",
 714                            group->groupid);
 715                 return NULL;
 716             }
 717         }
 718     }
 719
 720     group = g_malloc0(sizeof(*group));
 721
 722     snprintf(path, sizeof(path), "/dev/vfio/%d", groupid);
 723     group->fd = qemu_open_old(path, O_RDWR);
 724     if (group->fd < 0) {
 725         error_setg_errno(errp, errno, "failed to open %s", path);
 726         goto free_group_exit;
 727     }
 728
 729     if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &status)) {
 730         error_setg_errno(errp, errno, "failed to get group %d status", groupid);
 731         goto close_fd_exit;
 732     }
 733
 734     if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
 735         error_setg(errp, "group %d is not viable", groupid);
 736         error_append_hint(errp,
 737                           "Please ensure all devices within the iommu_group "
 738                           "are bound to their vfio bus driver.\n");
 739         goto close_fd_exit;
 740     }
 741
 742     group->groupid = groupid;
 743     QLIST_INIT(&group->device_list);
 744
 745     if (vfio_connect_container(group, as, errp)) {
 746         error_prepend(errp, "failed to setup container for group %d: ",
 747                       groupid);
 748         goto close_fd_exit;
 749     }
 750
 751     QLIST_INSERT_HEAD(&vfio_group_list, group, next);
 752
 753     return group;
 754
 755 close_fd_exit:
 756     close(group->fd);
 757
 758 free_group_exit:
 759     g_free(group);
 760
 761     return NULL;
 762 }
 763
 764 static void vfio_put_group(VFIOGroup *group)
 765 {
 766     if (!group || !QLIST_EMPTY(&group->device_list)) {
 767         return;
 768     }
 769
 770     if (!group->ram_block_discard_allowed) {
 771         vfio_ram_block_discard_disable(group->container, false);
 772     }
 773     vfio_kvm_device_del_group(group);
 774     vfio_disconnect_container(group);
 775     QLIST_REMOVE(group, next);
 776     trace_vfio_put_group(group->fd);
 777     close(group->fd);
 778     g_free(group);
 779 }
 780
 781 static int vfio_get_device(VFIOGroup *group, const char *name,
 782                            VFIODevice *vbasedev, Error **errp)
 783 {
 784     g_autofree struct vfio_device_info *info = NULL;
 785     int fd;
 786
 787     fd = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, name);
 788     if (fd < 0) {
 789         error_setg_errno(errp, errno, "error getting device from group %d",
 790                          group->groupid);
 791         error_append_hint(errp,
 792                       "Verify all devices in group %d are bound to vfio-<bus> "
 793                       "or pci-stub and not already in use\n", group->groupid);
 794         return fd;
 795     }
 796
 797     info = vfio_get_device_info(fd);
 798     if (!info) {
 799         error_setg_errno(errp, errno, "error getting device info");
 800         close(fd);
 801         return -1;
 802     }
 803
 804     /*
 805      * Set discarding of RAM as not broken for this group if the driver knows
 806      * the device operates compatibly with discarding.  Setting must be
 807      * consistent per group, but since compatibility is really only possible
 808      * with mdev currently, we expect singleton groups.
 809      */
 810     if (vbasedev->ram_block_discard_allowed !=
 811         group->ram_block_discard_allowed) {
 812         if (!QLIST_EMPTY(&group->device_list)) {
 813             error_setg(errp, "Inconsistent setting of support for discarding "
 814                        "RAM (e.g., balloon) within group");
 815             close(fd);
 816             return -1;
 817         }
 818
 819         if (!group->ram_block_discard_allowed) {
 820             group->ram_block_discard_allowed = true;
 821             vfio_ram_block_discard_disable(group->container, false);
 822         }
 823     }
 824
 825     vbasedev->fd = fd;
 826     vbasedev->group = group;
 827     QLIST_INSERT_HEAD(&group->device_list, vbasedev, next);
 828
 829     vbasedev->num_irqs = info->num_irqs;
 830     vbasedev->num_regions = info->num_regions;
 831     vbasedev->flags = info->flags;
 832
 833     trace_vfio_get_device(name, info->flags, info->num_regions, info->num_irqs);
 834
 835     vbasedev->reset_works = !!(info->flags & VFIO_DEVICE_FLAGS_RESET);
 836
 837     return 0;
 838 }
 839
 840 static void vfio_put_base_device(VFIODevice *vbasedev)
 841 {
 842     if (!vbasedev->group) {
 843         return;
 844     }
 845     QLIST_REMOVE(vbasedev, next);
 846     vbasedev->group = NULL;
 847     trace_vfio_put_base_device(vbasedev->fd);
 848     close(vbasedev->fd);
 849 }
 850
 851 static int vfio_device_groupid(VFIODevice *vbasedev, Error **errp)
 852 {
 853     char *tmp, group_path[PATH_MAX], *group_name;
 854     int ret, groupid;
 855     ssize_t len;
 856
 857     tmp = g_strdup_printf("%s/iommu_group", vbasedev->sysfsdev);
 858     len = readlink(tmp, group_path, sizeof(group_path));
 859     g_free(tmp);
 860
 861     if (len <= 0 || len >= sizeof(group_path)) {
 862         ret = len < 0 ? -errno : -ENAMETOOLONG;
 863         error_setg_errno(errp, -ret, "no iommu_group found");
 864         return ret;
 865     }
 866
 867     group_path[len] = 0;
 868
 869     group_name = basename(group_path);
 870     if (sscanf(group_name, "%d", &groupid) != 1) {
 871         error_setg_errno(errp, errno, "failed to read %s", group_path);
 872         return -errno;
 873     }
 874     return groupid;
 875 }
 876
 877 /*
 878  * vfio_attach_device: attach a device to a security context
 879  * @name and @vbasedev->name are likely to be different depending
 880  * on the type of the device, hence the need for passing @name
 881  */
 882 int vfio_attach_device(char *name, VFIODevice *vbasedev,
 883                        AddressSpace *as, Error **errp)
 884 {
 885     int groupid = vfio_device_groupid(vbasedev, errp);
 886     VFIODevice *vbasedev_iter;
 887     VFIOGroup *group;
 888     VFIOContainerBase *bcontainer;
 889     int ret;
 890
 891     if (groupid < 0) {
 892         return groupid;
 893     }
 894
 895     trace_vfio_attach_device(vbasedev->name, groupid);
 896
 897     group = vfio_get_group(groupid, as, errp);
 898     if (!group) {
 899         return -ENOENT;
 900     }
 901
 902     QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
 903         if (strcmp(vbasedev_iter->name, vbasedev->name) == 0) {
 904             error_setg(errp, "device is already attached");
 905             vfio_put_group(group);
 906             return -EBUSY;
 907         }
 908     }
 909     ret = vfio_get_device(group, name, vbasedev, errp);
 910     if (ret) {
 911         vfio_put_group(group);
 912         return ret;
 913     }
 914
 915     bcontainer = &group->container->bcontainer;
 916     vbasedev->bcontainer = bcontainer;
 917     QLIST_INSERT_HEAD(&bcontainer->device_list, vbasedev, container_next);
 918     QLIST_INSERT_HEAD(&vfio_device_list, vbasedev, global_next);
 919
 920     return ret;
 921 }
 922
 923 void vfio_detach_device(VFIODevice *vbasedev)
 924 {
 925     VFIOGroup *group = vbasedev->group;
 926
 927     if (!vbasedev->bcontainer) {
 928         return;
 929     }
 930
 931     QLIST_REMOVE(vbasedev, global_next);
 932     QLIST_REMOVE(vbasedev, container_next);
 933     vbasedev->bcontainer = NULL;
 934     trace_vfio_detach_device(vbasedev->name, group->groupid);
 935     vfio_put_base_device(vbasedev);
 936     vfio_put_group(group);
 937 }
 938
 939 const VFIOIOMMUOps vfio_legacy_ops = {
 940     .dma_map = vfio_legacy_dma_map,
 941     .dma_unmap = vfio_legacy_dma_unmap,
 942     .set_dirty_page_tracking = vfio_legacy_set_dirty_page_tracking,
 943     .query_dirty_bitmap = vfio_legacy_query_dirty_bitmap,
 944 };