hw/vfio/spapr.c

   1 /*
   2  * DMA memory preregistration
   3  *
   4  * Authors:
   5  *  Alexey Kardashevskiy <aik@ozlabs.ru>
   6  *
   7  * This work is licensed under the terms of the GNU GPL, version 2.  See
   8  * the COPYING file in the top-level directory.
   9  */
  10
  11 #include "qemu/osdep.h"
  12 #include <sys/ioctl.h>
  13 #include <linux/vfio.h>
  14 #ifdef CONFIG_KVM
  15 #include <linux/kvm.h>
  16 #endif
  17 #include "sysemu/kvm.h"
  18 #include "exec/address-spaces.h"
  19
  20 #include "hw/vfio/vfio-common.h"
  21 #include "hw/hw.h"
  22 #include "exec/ram_addr.h"
  23 #include "qemu/error-report.h"
  24 #include "qapi/error.h"
  25 #include "trace.h"
  26
  27 typedef struct VFIOSpaprContainer {
  28     VFIOContainer container;
  29 } VFIOSpaprContainer;
  30
  31 static bool vfio_prereg_listener_skipped_section(MemoryRegionSection *section)
  32 {
  33     if (memory_region_is_iommu(section->mr)) {
  34         hw_error("Cannot possibly preregister IOMMU memory");
  35     }
  36
  37     return !memory_region_is_ram(section->mr) ||
  38             memory_region_is_ram_device(section->mr);
  39 }
  40
  41 static void *vfio_prereg_gpa_to_vaddr(MemoryRegionSection *section, hwaddr gpa)
  42 {
  43     return memory_region_get_ram_ptr(section->mr) +
  44         section->offset_within_region +
  45         (gpa - section->offset_within_address_space);
  46 }
  47
  48 static void vfio_prereg_listener_region_add(MemoryListener *listener,
  49                                             MemoryRegionSection *section)
  50 {
  51     VFIOContainer *container = container_of(listener, VFIOContainer,
  52                                             prereg_listener);
  53     VFIOContainerBase *bcontainer = &container->bcontainer;
  54     const hwaddr gpa = section->offset_within_address_space;
  55     hwaddr end;
  56     int ret;
  57     hwaddr page_mask = qemu_real_host_page_mask();
  58     struct vfio_iommu_spapr_register_memory reg = {
  59         .argsz = sizeof(reg),
  60         .flags = 0,
  61     };
  62
  63     if (vfio_prereg_listener_skipped_section(section)) {
  64         trace_vfio_prereg_listener_region_add_skip(
  65                 section->offset_within_address_space,
  66                 section->offset_within_address_space +
  67                 int128_get64(int128_sub(section->size, int128_one())));
  68         return;
  69     }
  70
  71     if (unlikely((section->offset_within_address_space & ~page_mask) ||
  72                  (section->offset_within_region & ~page_mask) ||
  73                  (int128_get64(section->size) & ~page_mask))) {
  74         error_report("%s received unaligned region", __func__);
  75         return;
  76     }
  77
  78     end = section->offset_within_address_space + int128_get64(section->size);
  79     if (gpa >= end) {
  80         return;
  81     }
  82
  83     memory_region_ref(section->mr);
  84
  85     reg.vaddr = (uintptr_t) vfio_prereg_gpa_to_vaddr(section, gpa);
  86     reg.size = end - gpa;
  87
  88     ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_REGISTER_MEMORY, &reg);
  89     trace_vfio_prereg_register(reg.vaddr, reg.size, ret ? -errno : 0);
  90     if (ret) {
  91         /*
  92          * On the initfn path, store the first error in the container so we
  93          * can gracefully fail.  Runtime, there's not much we can do other
  94          * than throw a hardware error.
  95          */
  96         if (!bcontainer->initialized) {
  97             if (!bcontainer->error) {
  98                 error_setg_errno(&bcontainer->error, -ret,
  99                                  "Memory registering failed");
 100             }
 101         } else {
 102             hw_error("vfio: Memory registering failed, unable to continue");
 103         }
 104     }
 105 }
 106
 107 static void vfio_prereg_listener_region_del(MemoryListener *listener,
 108                                             MemoryRegionSection *section)
 109 {
 110     VFIOContainer *container = container_of(listener, VFIOContainer,
 111                                             prereg_listener);
 112     const hwaddr gpa = section->offset_within_address_space;
 113     hwaddr end;
 114     int ret;
 115     hwaddr page_mask = qemu_real_host_page_mask();
 116     struct vfio_iommu_spapr_register_memory reg = {
 117         .argsz = sizeof(reg),
 118         .flags = 0,
 119     };
 120
 121     if (vfio_prereg_listener_skipped_section(section)) {
 122         trace_vfio_prereg_listener_region_del_skip(
 123                 section->offset_within_address_space,
 124                 section->offset_within_address_space +
 125                 int128_get64(int128_sub(section->size, int128_one())));
 126         return;
 127     }
 128
 129     if (unlikely((section->offset_within_address_space & ~page_mask) ||
 130                  (section->offset_within_region & ~page_mask) ||
 131                  (int128_get64(section->size) & ~page_mask))) {
 132         error_report("%s received unaligned region", __func__);
 133         return;
 134     }
 135
 136     end = section->offset_within_address_space + int128_get64(section->size);
 137     if (gpa >= end) {
 138         return;
 139     }
 140
 141     reg.vaddr = (uintptr_t) vfio_prereg_gpa_to_vaddr(section, gpa);
 142     reg.size = end - gpa;
 143
 144     ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY, &reg);
 145     trace_vfio_prereg_unregister(reg.vaddr, reg.size, ret ? -errno : 0);
 146 }
 147
 148 static const MemoryListener vfio_prereg_listener = {
 149     .name = "vfio-pre-reg",
 150     .region_add = vfio_prereg_listener_region_add,
 151     .region_del = vfio_prereg_listener_region_del,
 152 };
 153
 154 static void vfio_host_win_add(VFIOContainer *container, hwaddr min_iova,
 155                               hwaddr max_iova, uint64_t iova_pgsizes)
 156 {
 157     VFIOHostDMAWindow *hostwin;
 158
 159     QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) {
 160         if (ranges_overlap(hostwin->min_iova,
 161                            hostwin->max_iova - hostwin->min_iova + 1,
 162                            min_iova,
 163                            max_iova - min_iova + 1)) {
 164             hw_error("%s: Overlapped IOMMU are not enabled", __func__);
 165         }
 166     }
 167
 168     hostwin = g_malloc0(sizeof(*hostwin));
 169
 170     hostwin->min_iova = min_iova;
 171     hostwin->max_iova = max_iova;
 172     hostwin->iova_pgsizes = iova_pgsizes;
 173     QLIST_INSERT_HEAD(&container->hostwin_list, hostwin, hostwin_next);
 174 }
 175
 176 static int vfio_host_win_del(VFIOContainer *container,
 177                              hwaddr min_iova, hwaddr max_iova)
 178 {
 179     VFIOHostDMAWindow *hostwin;
 180
 181     QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) {
 182         if (hostwin->min_iova == min_iova && hostwin->max_iova == max_iova) {
 183             QLIST_REMOVE(hostwin, hostwin_next);
 184             g_free(hostwin);
 185             return 0;
 186         }
 187     }
 188
 189     return -1;
 190 }
 191
 192 static VFIOHostDMAWindow *vfio_find_hostwin(VFIOContainer *container,
 193                                             hwaddr iova, hwaddr end)
 194 {
 195     VFIOHostDMAWindow *hostwin;
 196     bool hostwin_found = false;
 197
 198     QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) {
 199         if (hostwin->min_iova <= iova && end <= hostwin->max_iova) {
 200             hostwin_found = true;
 201             break;
 202         }
 203     }
 204
 205     return hostwin_found ? hostwin : NULL;
 206 }
 207
 208 static int vfio_spapr_remove_window(VFIOContainer *container,
 209                                     hwaddr offset_within_address_space)
 210 {
 211     struct vfio_iommu_spapr_tce_remove remove = {
 212         .argsz = sizeof(remove),
 213         .start_addr = offset_within_address_space,
 214     };
 215     int ret;
 216
 217     ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_TCE_REMOVE, &remove);
 218     if (ret) {
 219         error_report("Failed to remove window at %"PRIx64,
 220                      (uint64_t)remove.start_addr);
 221         return -errno;
 222     }
 223
 224     trace_vfio_spapr_remove_window(offset_within_address_space);
 225
 226     return 0;
 227 }
 228
 229 static int vfio_spapr_create_window(VFIOContainer *container,
 230                                     MemoryRegionSection *section,
 231                                     hwaddr *pgsize)
 232 {
 233     int ret = 0;
 234     VFIOContainerBase *bcontainer = &container->bcontainer;
 235     IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr);
 236     uint64_t pagesize = memory_region_iommu_get_min_page_size(iommu_mr), pgmask;
 237     unsigned entries, bits_total, bits_per_level, max_levels;
 238     struct vfio_iommu_spapr_tce_create create = { .argsz = sizeof(create) };
 239     long rampagesize = qemu_minrampagesize();
 240
 241     /*
 242      * The host might not support the guest supported IOMMU page size,
 243      * so we will use smaller physical IOMMU pages to back them.
 244      */
 245     if (pagesize > rampagesize) {
 246         pagesize = rampagesize;
 247     }
 248     pgmask = bcontainer->pgsizes & (pagesize | (pagesize - 1));
 249     pagesize = pgmask ? (1ULL << (63 - clz64(pgmask))) : 0;
 250     if (!pagesize) {
 251         error_report("Host doesn't support page size 0x%"PRIx64
 252                      ", the supported mask is 0x%lx",
 253                      memory_region_iommu_get_min_page_size(iommu_mr),
 254                      bcontainer->pgsizes);
 255         return -EINVAL;
 256     }
 257
 258     /*
 259      * FIXME: For VFIO iommu types which have KVM acceleration to
 260      * avoid bouncing all map/unmaps through qemu this way, this
 261      * would be the right place to wire that up (tell the KVM
 262      * device emulation the VFIO iommu handles to use).
 263      */
 264     create.window_size = int128_get64(section->size);
 265     create.page_shift = ctz64(pagesize);
 266     /*
 267      * SPAPR host supports multilevel TCE tables. We try to guess optimal
 268      * levels number and if this fails (for example due to the host memory
 269      * fragmentation), we increase levels. The DMA address structure is:
 270      * rrrrrrrr rxxxxxxx xxxxxxxx xxxxxxxx  xxxxxxxx xxxxxxxx xxxxxxxx iiiiiiii
 271      * where:
 272      *   r = reserved (bits >= 55 are reserved in the existing hardware)
 273      *   i = IOMMU page offset (64K in this example)
 274      *   x = bits to index a TCE which can be split to equal chunks to index
 275      *      within the level.
 276      * The aim is to split "x" to smaller possible number of levels.
 277      */
 278     entries = create.window_size >> create.page_shift;
 279     /* bits_total is number of "x" needed */
 280     bits_total = ctz64(entries * sizeof(uint64_t));
 281     /*
 282      * bits_per_level is a safe guess of how much we can allocate per level:
 283      * 8 is the current minimum for CONFIG_FORCE_MAX_ZONEORDER and MAX_ORDER
 284      * is usually bigger than that.
 285      * Below we look at qemu_real_host_page_size as TCEs are allocated from
 286      * system pages.
 287      */
 288     bits_per_level = ctz64(qemu_real_host_page_size()) + 8;
 289     create.levels = bits_total / bits_per_level;
 290     if (bits_total % bits_per_level) {
 291         ++create.levels;
 292     }
 293     max_levels = (64 - create.page_shift) / ctz64(qemu_real_host_page_size());
 294     for ( ; create.levels <= max_levels; ++create.levels) {
 295         ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_TCE_CREATE, &create);
 296         if (!ret) {
 297             break;
 298         }
 299     }
 300     if (ret) {
 301         error_report("Failed to create a window, ret = %d (%m)", ret);
 302         return -errno;
 303     }
 304
 305     if (create.start_addr != section->offset_within_address_space) {
 306         vfio_spapr_remove_window(container, create.start_addr);
 307
 308         error_report("Host doesn't support DMA window at %"HWADDR_PRIx", must be %"PRIx64,
 309                      section->offset_within_address_space,
 310                      (uint64_t)create.start_addr);
 311         return -EINVAL;
 312     }
 313     trace_vfio_spapr_create_window(create.page_shift,
 314                                    create.levels,
 315                                    create.window_size,
 316                                    create.start_addr);
 317     *pgsize = pagesize;
 318
 319     return 0;
 320 }
 321
 322 static int
 323 vfio_spapr_container_add_section_window(VFIOContainerBase *bcontainer,
 324                                         MemoryRegionSection *section,
 325                                         Error **errp)
 326 {
 327     VFIOContainer *container = container_of(bcontainer, VFIOContainer,
 328                                             bcontainer);
 329     VFIOHostDMAWindow *hostwin;
 330     hwaddr pgsize = 0;
 331     int ret;
 332
 333     /*
 334      * VFIO_SPAPR_TCE_IOMMU supports a single host window between
 335      * [dma32_window_start, dma32_window_size), we need to ensure
 336      * the section fall in this range.
 337      */
 338     if (container->iommu_type == VFIO_SPAPR_TCE_IOMMU) {
 339         hwaddr iova, end;
 340
 341         iova = section->offset_within_address_space;
 342         end = iova + int128_get64(section->size) - 1;
 343
 344         if (!vfio_find_hostwin(container, iova, end)) {
 345             error_setg(errp, "Container %p can't map guest IOVA region"
 346                        " 0x%"HWADDR_PRIx"..0x%"HWADDR_PRIx, container,
 347                        iova, end);
 348             return -EINVAL;
 349         }
 350         return 0;
 351     }
 352
 353     if (container->iommu_type != VFIO_SPAPR_TCE_v2_IOMMU) {
 354         return 0;
 355     }
 356
 357     /* For now intersections are not allowed, we may relax this later */
 358     QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) {
 359         if (ranges_overlap(hostwin->min_iova,
 360                            hostwin->max_iova - hostwin->min_iova + 1,
 361                            section->offset_within_address_space,
 362                            int128_get64(section->size))) {
 363             error_setg(errp,
 364                 "region [0x%"PRIx64",0x%"PRIx64"] overlaps with existing"
 365                 "host DMA window [0x%"PRIx64",0x%"PRIx64"]",
 366                 section->offset_within_address_space,
 367                 section->offset_within_address_space +
 368                     int128_get64(section->size) - 1,
 369                 hostwin->min_iova, hostwin->max_iova);
 370             return -EINVAL;
 371         }
 372     }
 373
 374     ret = vfio_spapr_create_window(container, section, &pgsize);
 375     if (ret) {
 376         error_setg_errno(errp, -ret, "Failed to create SPAPR window");
 377         return ret;
 378     }
 379
 380     vfio_host_win_add(container, section->offset_within_address_space,
 381                       section->offset_within_address_space +
 382                       int128_get64(section->size) - 1, pgsize);
 383 #ifdef CONFIG_KVM
 384     if (kvm_enabled()) {
 385         VFIOGroup *group;
 386         IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr);
 387         struct kvm_vfio_spapr_tce param;
 388         struct kvm_device_attr attr = {
 389             .group = KVM_DEV_VFIO_GROUP,
 390             .attr = KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE,
 391             .addr = (uint64_t)(unsigned long)&param,
 392         };
 393
 394         if (!memory_region_iommu_get_attr(iommu_mr, IOMMU_ATTR_SPAPR_TCE_FD,
 395                                           &param.tablefd)) {
 396             QLIST_FOREACH(group, &container->group_list, container_next) {
 397                 param.groupfd = group->fd;
 398                 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
 399                     error_setg_errno(errp, errno,
 400                                      "vfio: failed GROUP_SET_SPAPR_TCE for "
 401                                      "KVM VFIO device %d and group fd %d",
 402                                      param.tablefd, param.groupfd);
 403                     return -errno;
 404                 }
 405                 trace_vfio_spapr_group_attach(param.groupfd, param.tablefd);
 406             }
 407         }
 408     }
 409 #endif
 410     return 0;
 411 }
 412
 413 static void
 414 vfio_spapr_container_del_section_window(VFIOContainerBase *bcontainer,
 415                                         MemoryRegionSection *section)
 416 {
 417     VFIOContainer *container = container_of(bcontainer, VFIOContainer,
 418                                             bcontainer);
 419
 420     if (container->iommu_type != VFIO_SPAPR_TCE_v2_IOMMU) {
 421         return;
 422     }
 423
 424     vfio_spapr_remove_window(container,
 425                              section->offset_within_address_space);
 426     if (vfio_host_win_del(container,
 427                           section->offset_within_address_space,
 428                           section->offset_within_address_space +
 429                           int128_get64(section->size) - 1) < 0) {
 430         hw_error("%s: Cannot delete missing window at %"HWADDR_PRIx,
 431                  __func__, section->offset_within_address_space);
 432     }
 433 }
 434
 435 static VFIOIOMMUOps vfio_iommu_spapr_ops;
 436
 437 static void setup_spapr_ops(VFIOContainerBase *bcontainer)
 438 {
 439     vfio_iommu_spapr_ops = *bcontainer->ops;
 440     vfio_iommu_spapr_ops.add_window = vfio_spapr_container_add_section_window;
 441     vfio_iommu_spapr_ops.del_window = vfio_spapr_container_del_section_window;
 442     bcontainer->ops = &vfio_iommu_spapr_ops;
 443 }
 444
 445 int vfio_spapr_container_init(VFIOContainer *container, Error **errp)
 446 {
 447     VFIOContainerBase *bcontainer = &container->bcontainer;
 448     struct vfio_iommu_spapr_tce_info info;
 449     bool v2 = container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU;
 450     int ret, fd = container->fd;
 451
 452     QLIST_INIT(&container->hostwin_list);
 453
 454     /*
 455      * The host kernel code implementing VFIO_IOMMU_DISABLE is called
 456      * when container fd is closed so we do not call it explicitly
 457      * in this file.
 458      */
 459     if (!v2) {
 460         ret = ioctl(fd, VFIO_IOMMU_ENABLE);
 461         if (ret) {
 462             error_setg_errno(errp, errno, "failed to enable container");
 463             return -errno;
 464         }
 465     } else {
 466         container->prereg_listener = vfio_prereg_listener;
 467
 468         memory_listener_register(&container->prereg_listener,
 469                                  &address_space_memory);
 470         if (bcontainer->error) {
 471             ret = -1;
 472             error_propagate_prepend(errp, bcontainer->error,
 473                     "RAM memory listener initialization failed: ");
 474             goto listener_unregister_exit;
 475         }
 476     }
 477
 478     info.argsz = sizeof(info);
 479     ret = ioctl(fd, VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info);
 480     if (ret) {
 481         error_setg_errno(errp, errno,
 482                          "VFIO_IOMMU_SPAPR_TCE_GET_INFO failed");
 483         ret = -errno;
 484         goto listener_unregister_exit;
 485     }
 486
 487     if (v2) {
 488         bcontainer->pgsizes = info.ddw.pgsizes;
 489         /*
 490          * There is a default window in just created container.
 491          * To make region_add/del simpler, we better remove this
 492          * window now and let those iommu_listener callbacks
 493          * create/remove them when needed.
 494          */
 495         ret = vfio_spapr_remove_window(container, info.dma32_window_start);
 496         if (ret) {
 497             error_setg_errno(errp, -ret,
 498                              "failed to remove existing window");
 499             goto listener_unregister_exit;
 500         }
 501     } else {
 502         /* The default table uses 4K pages */
 503         bcontainer->pgsizes = 0x1000;
 504         vfio_host_win_add(container, info.dma32_window_start,
 505                           info.dma32_window_start +
 506                           info.dma32_window_size - 1,
 507                           0x1000);
 508     }
 509
 510     setup_spapr_ops(bcontainer);
 511
 512     return 0;
 513
 514 listener_unregister_exit:
 515     if (v2) {
 516         memory_listener_unregister(&container->prereg_listener);
 517     }
 518     return ret;
 519 }
 520
 521 void vfio_spapr_container_deinit(VFIOContainer *container)
 522 {
 523     VFIOHostDMAWindow *hostwin, *next;
 524
 525     if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) {
 526         memory_listener_unregister(&container->prereg_listener);
 527     }
 528     QLIST_FOREACH_SAFE(hostwin, &container->hostwin_list, hostwin_next,
 529                        next) {
 530         QLIST_REMOVE(hostwin, hostwin_next);
 531         g_free(hostwin);
 532     }
 533 }