hw/virtio/virtio-iommu.c

   1 /*
   2  * virtio-iommu device
   3  *
   4  * Copyright (c) 2020 Red Hat, Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2 or later, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along with
  16  * this program.  If not, see <http://www.gnu.org/licenses/>.
  17  *
  18  */
  19
  20 #include "qemu/osdep.h"
  21 #include "qemu/log.h"
  22 #include "qemu/iov.h"
  23 #include "qemu-common.h"
  24 #include "hw/qdev-properties.h"
  25 #include "hw/virtio/virtio.h"
  26 #include "sysemu/kvm.h"
  27 #include "qapi/error.h"
  28 #include "qemu/error-report.h"
  29 #include "trace.h"
  30
  31 #include "standard-headers/linux/virtio_ids.h"
  32
  33 #include "hw/virtio/virtio-bus.h"
  34 #include "hw/virtio/virtio-access.h"
  35 #include "hw/virtio/virtio-iommu.h"
  36 #include "hw/pci/pci_bus.h"
  37 #include "hw/pci/pci.h"
  38
  39 /* Max size */
  40 #define VIOMMU_DEFAULT_QUEUE_SIZE 256
  41 #define VIOMMU_PROBE_SIZE 512
  42
  43 typedef struct VirtIOIOMMUDomain {
  44     uint32_t id;
  45     GTree *mappings;
  46     QLIST_HEAD(, VirtIOIOMMUEndpoint) endpoint_list;
  47 } VirtIOIOMMUDomain;
  48
  49 typedef struct VirtIOIOMMUEndpoint {
  50     uint32_t id;
  51     VirtIOIOMMUDomain *domain;
  52     QLIST_ENTRY(VirtIOIOMMUEndpoint) next;
  53 } VirtIOIOMMUEndpoint;
  54
  55 typedef struct VirtIOIOMMUInterval {
  56     uint64_t low;
  57     uint64_t high;
  58 } VirtIOIOMMUInterval;
  59
  60 typedef struct VirtIOIOMMUMapping {
  61     uint64_t phys_addr;
  62     uint32_t flags;
  63 } VirtIOIOMMUMapping;
  64
  65 static inline uint16_t virtio_iommu_get_bdf(IOMMUDevice *dev)
  66 {
  67     return PCI_BUILD_BDF(pci_bus_num(dev->bus), dev->devfn);
  68 }
  69
  70 /**
  71  * The bus number is used for lookup when SID based operations occur.
  72  * In that case we lazily populate the IOMMUPciBus array from the bus hash
  73  * table. At the time the IOMMUPciBus is created (iommu_find_add_as), the bus
  74  * numbers may not be always initialized yet.
  75  */
  76 static IOMMUPciBus *iommu_find_iommu_pcibus(VirtIOIOMMU *s, uint8_t bus_num)
  77 {
  78     IOMMUPciBus *iommu_pci_bus = s->iommu_pcibus_by_bus_num[bus_num];
  79
  80     if (!iommu_pci_bus) {
  81         GHashTableIter iter;
  82
  83         g_hash_table_iter_init(&iter, s->as_by_busptr);
  84         while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) {
  85             if (pci_bus_num(iommu_pci_bus->bus) == bus_num) {
  86                 s->iommu_pcibus_by_bus_num[bus_num] = iommu_pci_bus;
  87                 return iommu_pci_bus;
  88             }
  89         }
  90         return NULL;
  91     }
  92     return iommu_pci_bus;
  93 }
  94
  95 static IOMMUMemoryRegion *virtio_iommu_mr(VirtIOIOMMU *s, uint32_t sid)
  96 {
  97     uint8_t bus_n, devfn;
  98     IOMMUPciBus *iommu_pci_bus;
  99     IOMMUDevice *dev;
 100
 101     bus_n = PCI_BUS_NUM(sid);
 102     iommu_pci_bus = iommu_find_iommu_pcibus(s, bus_n);
 103     if (iommu_pci_bus) {
 104         devfn = sid & PCI_DEVFN_MAX;
 105         dev = iommu_pci_bus->pbdev[devfn];
 106         if (dev) {
 107             return &dev->iommu_mr;
 108         }
 109     }
 110     return NULL;
 111 }
 112
 113 static gint interval_cmp(gconstpointer a, gconstpointer b, gpointer user_data)
 114 {
 115     VirtIOIOMMUInterval *inta = (VirtIOIOMMUInterval *)a;
 116     VirtIOIOMMUInterval *intb = (VirtIOIOMMUInterval *)b;
 117
 118     if (inta->high < intb->low) {
 119         return -1;
 120     } else if (intb->high < inta->low) {
 121         return 1;
 122     } else {
 123         return 0;
 124     }
 125 }
 126
 127 static void virtio_iommu_detach_endpoint_from_domain(VirtIOIOMMUEndpoint *ep)
 128 {
 129     if (!ep->domain) {
 130         return;
 131     }
 132     QLIST_REMOVE(ep, next);
 133     ep->domain = NULL;
 134 }
 135
 136 static VirtIOIOMMUEndpoint *virtio_iommu_get_endpoint(VirtIOIOMMU *s,
 137                                                       uint32_t ep_id)
 138 {
 139     VirtIOIOMMUEndpoint *ep;
 140
 141     ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id));
 142     if (ep) {
 143         return ep;
 144     }
 145     if (!virtio_iommu_mr(s, ep_id)) {
 146         return NULL;
 147     }
 148     ep = g_malloc0(sizeof(*ep));
 149     ep->id = ep_id;
 150     trace_virtio_iommu_get_endpoint(ep_id);
 151     g_tree_insert(s->endpoints, GUINT_TO_POINTER(ep_id), ep);
 152     return ep;
 153 }
 154
 155 static void virtio_iommu_put_endpoint(gpointer data)
 156 {
 157     VirtIOIOMMUEndpoint *ep = (VirtIOIOMMUEndpoint *)data;
 158
 159     if (ep->domain) {
 160         virtio_iommu_detach_endpoint_from_domain(ep);
 161     }
 162
 163     trace_virtio_iommu_put_endpoint(ep->id);
 164     g_free(ep);
 165 }
 166
 167 static VirtIOIOMMUDomain *virtio_iommu_get_domain(VirtIOIOMMU *s,
 168                                                   uint32_t domain_id)
 169 {
 170     VirtIOIOMMUDomain *domain;
 171
 172     domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
 173     if (domain) {
 174         return domain;
 175     }
 176     domain = g_malloc0(sizeof(*domain));
 177     domain->id = domain_id;
 178     domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp,
 179                                    NULL, (GDestroyNotify)g_free,
 180                                    (GDestroyNotify)g_free);
 181     g_tree_insert(s->domains, GUINT_TO_POINTER(domain_id), domain);
 182     QLIST_INIT(&domain->endpoint_list);
 183     trace_virtio_iommu_get_domain(domain_id);
 184     return domain;
 185 }
 186
 187 static void virtio_iommu_put_domain(gpointer data)
 188 {
 189     VirtIOIOMMUDomain *domain = (VirtIOIOMMUDomain *)data;
 190     VirtIOIOMMUEndpoint *iter, *tmp;
 191
 192     QLIST_FOREACH_SAFE(iter, &domain->endpoint_list, next, tmp) {
 193         virtio_iommu_detach_endpoint_from_domain(iter);
 194     }
 195     g_tree_destroy(domain->mappings);
 196     trace_virtio_iommu_put_domain(domain->id);
 197     g_free(domain);
 198 }
 199
 200 static AddressSpace *virtio_iommu_find_add_as(PCIBus *bus, void *opaque,
 201                                               int devfn)
 202 {
 203     VirtIOIOMMU *s = opaque;
 204     IOMMUPciBus *sbus = g_hash_table_lookup(s->as_by_busptr, bus);
 205     static uint32_t mr_index;
 206     IOMMUDevice *sdev;
 207
 208     if (!sbus) {
 209         sbus = g_malloc0(sizeof(IOMMUPciBus) +
 210                          sizeof(IOMMUDevice *) * PCI_DEVFN_MAX);
 211         sbus->bus = bus;
 212         g_hash_table_insert(s->as_by_busptr, bus, sbus);
 213     }
 214
 215     sdev = sbus->pbdev[devfn];
 216     if (!sdev) {
 217         char *name = g_strdup_printf("%s-%d-%d",
 218                                      TYPE_VIRTIO_IOMMU_MEMORY_REGION,
 219                                      mr_index++, devfn);
 220         sdev = sbus->pbdev[devfn] = g_malloc0(sizeof(IOMMUDevice));
 221
 222         sdev->viommu = s;
 223         sdev->bus = bus;
 224         sdev->devfn = devfn;
 225
 226         trace_virtio_iommu_init_iommu_mr(name);
 227
 228         memory_region_init_iommu(&sdev->iommu_mr, sizeof(sdev->iommu_mr),
 229                                  TYPE_VIRTIO_IOMMU_MEMORY_REGION,
 230                                  OBJECT(s), name,
 231                                  UINT64_MAX);
 232         address_space_init(&sdev->as,
 233                            MEMORY_REGION(&sdev->iommu_mr), TYPE_VIRTIO_IOMMU);
 234         g_free(name);
 235     }
 236     return &sdev->as;
 237 }
 238
 239 static int virtio_iommu_attach(VirtIOIOMMU *s,
 240                                struct virtio_iommu_req_attach *req)
 241 {
 242     uint32_t domain_id = le32_to_cpu(req->domain);
 243     uint32_t ep_id = le32_to_cpu(req->endpoint);
 244     VirtIOIOMMUDomain *domain;
 245     VirtIOIOMMUEndpoint *ep;
 246
 247     trace_virtio_iommu_attach(domain_id, ep_id);
 248
 249     ep = virtio_iommu_get_endpoint(s, ep_id);
 250     if (!ep) {
 251         return VIRTIO_IOMMU_S_NOENT;
 252     }
 253
 254     if (ep->domain) {
 255         VirtIOIOMMUDomain *previous_domain = ep->domain;
 256         /*
 257          * the device is already attached to a domain,
 258          * detach it first
 259          */
 260         virtio_iommu_detach_endpoint_from_domain(ep);
 261         if (QLIST_EMPTY(&previous_domain->endpoint_list)) {
 262             g_tree_remove(s->domains, GUINT_TO_POINTER(previous_domain->id));
 263         }
 264     }
 265
 266     domain = virtio_iommu_get_domain(s, domain_id);
 267     QLIST_INSERT_HEAD(&domain->endpoint_list, ep, next);
 268
 269     ep->domain = domain;
 270
 271     return VIRTIO_IOMMU_S_OK;
 272 }
 273
 274 static int virtio_iommu_detach(VirtIOIOMMU *s,
 275                                struct virtio_iommu_req_detach *req)
 276 {
 277     uint32_t domain_id = le32_to_cpu(req->domain);
 278     uint32_t ep_id = le32_to_cpu(req->endpoint);
 279     VirtIOIOMMUDomain *domain;
 280     VirtIOIOMMUEndpoint *ep;
 281
 282     trace_virtio_iommu_detach(domain_id, ep_id);
 283
 284     ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id));
 285     if (!ep) {
 286         return VIRTIO_IOMMU_S_NOENT;
 287     }
 288
 289     domain = ep->domain;
 290
 291     if (!domain || domain->id != domain_id) {
 292         return VIRTIO_IOMMU_S_INVAL;
 293     }
 294
 295     virtio_iommu_detach_endpoint_from_domain(ep);
 296
 297     if (QLIST_EMPTY(&domain->endpoint_list)) {
 298         g_tree_remove(s->domains, GUINT_TO_POINTER(domain->id));
 299     }
 300     return VIRTIO_IOMMU_S_OK;
 301 }
 302
 303 static int virtio_iommu_map(VirtIOIOMMU *s,
 304                             struct virtio_iommu_req_map *req)
 305 {
 306     uint32_t domain_id = le32_to_cpu(req->domain);
 307     uint64_t phys_start = le64_to_cpu(req->phys_start);
 308     uint64_t virt_start = le64_to_cpu(req->virt_start);
 309     uint64_t virt_end = le64_to_cpu(req->virt_end);
 310     uint32_t flags = le32_to_cpu(req->flags);
 311     VirtIOIOMMUDomain *domain;
 312     VirtIOIOMMUInterval *interval;
 313     VirtIOIOMMUMapping *mapping;
 314
 315     if (flags & ~VIRTIO_IOMMU_MAP_F_MASK) {
 316         return VIRTIO_IOMMU_S_INVAL;
 317     }
 318
 319     domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
 320     if (!domain) {
 321         return VIRTIO_IOMMU_S_NOENT;
 322     }
 323
 324     interval = g_malloc0(sizeof(*interval));
 325
 326     interval->low = virt_start;
 327     interval->high = virt_end;
 328
 329     mapping = g_tree_lookup(domain->mappings, (gpointer)interval);
 330     if (mapping) {
 331         g_free(interval);
 332         return VIRTIO_IOMMU_S_INVAL;
 333     }
 334
 335     trace_virtio_iommu_map(domain_id, virt_start, virt_end, phys_start, flags);
 336
 337     mapping = g_malloc0(sizeof(*mapping));
 338     mapping->phys_addr = phys_start;
 339     mapping->flags = flags;
 340
 341     g_tree_insert(domain->mappings, interval, mapping);
 342
 343     return VIRTIO_IOMMU_S_OK;
 344 }
 345
 346 static int virtio_iommu_unmap(VirtIOIOMMU *s,
 347                               struct virtio_iommu_req_unmap *req)
 348 {
 349     uint32_t domain_id = le32_to_cpu(req->domain);
 350     uint64_t virt_start = le64_to_cpu(req->virt_start);
 351     uint64_t virt_end = le64_to_cpu(req->virt_end);
 352     VirtIOIOMMUMapping *iter_val;
 353     VirtIOIOMMUInterval interval, *iter_key;
 354     VirtIOIOMMUDomain *domain;
 355     int ret = VIRTIO_IOMMU_S_OK;
 356
 357     trace_virtio_iommu_unmap(domain_id, virt_start, virt_end);
 358
 359     domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
 360     if (!domain) {
 361         return VIRTIO_IOMMU_S_NOENT;
 362     }
 363     interval.low = virt_start;
 364     interval.high = virt_end;
 365
 366     while (g_tree_lookup_extended(domain->mappings, &interval,
 367                                   (void **)&iter_key, (void**)&iter_val)) {
 368         uint64_t current_low = iter_key->low;
 369         uint64_t current_high = iter_key->high;
 370
 371         if (interval.low <= current_low && interval.high >= current_high) {
 372             g_tree_remove(domain->mappings, iter_key);
 373             trace_virtio_iommu_unmap_done(domain_id, current_low, current_high);
 374         } else {
 375             ret = VIRTIO_IOMMU_S_RANGE;
 376             break;
 377         }
 378     }
 379     return ret;
 380 }
 381
 382 static ssize_t virtio_iommu_fill_resv_mem_prop(VirtIOIOMMU *s, uint32_t ep,
 383                                                uint8_t *buf, size_t free)
 384 {
 385     struct virtio_iommu_probe_resv_mem prop = {};
 386     size_t size = sizeof(prop), length = size - sizeof(prop.head), total;
 387     int i;
 388
 389     total = size * s->nb_reserved_regions;
 390
 391     if (total > free) {
 392         return -ENOSPC;
 393     }
 394
 395     for (i = 0; i < s->nb_reserved_regions; i++) {
 396         unsigned subtype = s->reserved_regions[i].type;
 397
 398         assert(subtype == VIRTIO_IOMMU_RESV_MEM_T_RESERVED ||
 399                subtype == VIRTIO_IOMMU_RESV_MEM_T_MSI);
 400         prop.head.type = cpu_to_le16(VIRTIO_IOMMU_PROBE_T_RESV_MEM);
 401         prop.head.length = cpu_to_le16(length);
 402         prop.subtype = subtype;
 403         prop.start = cpu_to_le64(s->reserved_regions[i].low);
 404         prop.end = cpu_to_le64(s->reserved_regions[i].high);
 405
 406         memcpy(buf, &prop, size);
 407
 408         trace_virtio_iommu_fill_resv_property(ep, prop.subtype,
 409                                               prop.start, prop.end);
 410         buf += size;
 411     }
 412     return total;
 413 }
 414
 415 /**
 416  * virtio_iommu_probe - Fill the probe request buffer with
 417  * the properties the device is able to return
 418  */
 419 static int virtio_iommu_probe(VirtIOIOMMU *s,
 420                               struct virtio_iommu_req_probe *req,
 421                               uint8_t *buf)
 422 {
 423     uint32_t ep_id = le32_to_cpu(req->endpoint);
 424     size_t free = VIOMMU_PROBE_SIZE;
 425     ssize_t count;
 426
 427     if (!virtio_iommu_mr(s, ep_id)) {
 428         return VIRTIO_IOMMU_S_NOENT;
 429     }
 430
 431     count = virtio_iommu_fill_resv_mem_prop(s, ep_id, buf, free);
 432     if (count < 0) {
 433         return VIRTIO_IOMMU_S_INVAL;
 434     }
 435     buf += count;
 436     free -= count;
 437
 438     return VIRTIO_IOMMU_S_OK;
 439 }
 440
 441 static int virtio_iommu_iov_to_req(struct iovec *iov,
 442                                    unsigned int iov_cnt,
 443                                    void *req, size_t req_sz)
 444 {
 445     size_t sz, payload_sz = req_sz - sizeof(struct virtio_iommu_req_tail);
 446
 447     sz = iov_to_buf(iov, iov_cnt, 0, req, payload_sz);
 448     if (unlikely(sz != payload_sz)) {
 449         return VIRTIO_IOMMU_S_INVAL;
 450     }
 451     return 0;
 452 }
 453
 454 #define virtio_iommu_handle_req(__req)                                  \
 455 static int virtio_iommu_handle_ ## __req(VirtIOIOMMU *s,                \
 456                                          struct iovec *iov,             \
 457                                          unsigned int iov_cnt)          \
 458 {                                                                       \
 459     struct virtio_iommu_req_ ## __req req;                              \
 460     int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req)); \
 461                                                                         \
 462     return ret ? ret : virtio_iommu_ ## __req(s, &req);                 \
 463 }
 464
 465 virtio_iommu_handle_req(attach)
 466 virtio_iommu_handle_req(detach)
 467 virtio_iommu_handle_req(map)
 468 virtio_iommu_handle_req(unmap)
 469
 470 static int virtio_iommu_handle_probe(VirtIOIOMMU *s,
 471                                      struct iovec *iov,
 472                                      unsigned int iov_cnt,
 473                                      uint8_t *buf)
 474 {
 475     struct virtio_iommu_req_probe req;
 476     int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req));
 477
 478     return ret ? ret : virtio_iommu_probe(s, &req, buf);
 479 }
 480
 481 static void virtio_iommu_handle_command(VirtIODevice *vdev, VirtQueue *vq)
 482 {
 483     VirtIOIOMMU *s = VIRTIO_IOMMU(vdev);
 484     struct virtio_iommu_req_head head;
 485     struct virtio_iommu_req_tail tail = {};
 486     size_t output_size = sizeof(tail), sz;
 487     VirtQueueElement *elem;
 488     unsigned int iov_cnt;
 489     struct iovec *iov;
 490     void *buf = NULL;
 491
 492     for (;;) {
 493         elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
 494         if (!elem) {
 495             return;
 496         }
 497
 498         if (iov_size(elem->in_sg, elem->in_num) < sizeof(tail) ||
 499             iov_size(elem->out_sg, elem->out_num) < sizeof(head)) {
 500             virtio_error(vdev, "virtio-iommu bad head/tail size");
 501             virtqueue_detach_element(vq, elem, 0);
 502             g_free(elem);
 503             break;
 504         }
 505
 506         iov_cnt = elem->out_num;
 507         iov = elem->out_sg;
 508         sz = iov_to_buf(iov, iov_cnt, 0, &head, sizeof(head));
 509         if (unlikely(sz != sizeof(head))) {
 510             tail.status = VIRTIO_IOMMU_S_DEVERR;
 511             goto out;
 512         }
 513         qemu_mutex_lock(&s->mutex);
 514         switch (head.type) {
 515         case VIRTIO_IOMMU_T_ATTACH:
 516             tail.status = virtio_iommu_handle_attach(s, iov, iov_cnt);
 517             break;
 518         case VIRTIO_IOMMU_T_DETACH:
 519             tail.status = virtio_iommu_handle_detach(s, iov, iov_cnt);
 520             break;
 521         case VIRTIO_IOMMU_T_MAP:
 522             tail.status = virtio_iommu_handle_map(s, iov, iov_cnt);
 523             break;
 524         case VIRTIO_IOMMU_T_UNMAP:
 525             tail.status = virtio_iommu_handle_unmap(s, iov, iov_cnt);
 526             break;
 527         case VIRTIO_IOMMU_T_PROBE:
 528         {
 529             struct virtio_iommu_req_tail *ptail;
 530
 531             output_size = s->config.probe_size + sizeof(tail);
 532             buf = g_malloc0(output_size);
 533
 534             ptail = (struct virtio_iommu_req_tail *)
 535                         (buf + s->config.probe_size);
 536             ptail->status = virtio_iommu_handle_probe(s, iov, iov_cnt, buf);
 537         }
 538         default:
 539             tail.status = VIRTIO_IOMMU_S_UNSUPP;
 540         }
 541         qemu_mutex_unlock(&s->mutex);
 542
 543 out:
 544         sz = iov_from_buf(elem->in_sg, elem->in_num, 0,
 545                           buf ? buf : &tail, output_size);
 546         assert(sz == output_size);
 547
 548         virtqueue_push(vq, elem, sz);
 549         virtio_notify(vdev, vq);
 550         g_free(elem);
 551         g_free(buf);
 552     }
 553 }
 554
 555 static void virtio_iommu_report_fault(VirtIOIOMMU *viommu, uint8_t reason,
 556                                       int flags, uint32_t endpoint,
 557                                       uint64_t address)
 558 {
 559     VirtIODevice *vdev = &viommu->parent_obj;
 560     VirtQueue *vq = viommu->event_vq;
 561     struct virtio_iommu_fault fault;
 562     VirtQueueElement *elem;
 563     size_t sz;
 564
 565     memset(&fault, 0, sizeof(fault));
 566     fault.reason = reason;
 567     fault.flags = cpu_to_le32(flags);
 568     fault.endpoint = cpu_to_le32(endpoint);
 569     fault.address = cpu_to_le64(address);
 570
 571     elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
 572
 573     if (!elem) {
 574         error_report_once(
 575             "no buffer available in event queue to report event");
 576         return;
 577     }
 578
 579     if (iov_size(elem->in_sg, elem->in_num) < sizeof(fault)) {
 580         virtio_error(vdev, "error buffer of wrong size");
 581         virtqueue_detach_element(vq, elem, 0);
 582         g_free(elem);
 583         return;
 584     }
 585
 586     sz = iov_from_buf(elem->in_sg, elem->in_num, 0,
 587                       &fault, sizeof(fault));
 588     assert(sz == sizeof(fault));
 589
 590     trace_virtio_iommu_report_fault(reason, flags, endpoint, address);
 591     virtqueue_push(vq, elem, sz);
 592     virtio_notify(vdev, vq);
 593     g_free(elem);
 594
 595 }
 596
 597 static IOMMUTLBEntry virtio_iommu_translate(IOMMUMemoryRegion *mr, hwaddr addr,
 598                                             IOMMUAccessFlags flag,
 599                                             int iommu_idx)
 600 {
 601     IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr);
 602     VirtIOIOMMUInterval interval, *mapping_key;
 603     VirtIOIOMMUMapping *mapping_value;
 604     VirtIOIOMMU *s = sdev->viommu;
 605     bool read_fault, write_fault;
 606     VirtIOIOMMUEndpoint *ep;
 607     uint32_t sid, flags;
 608     bool bypass_allowed;
 609     bool found;
 610     int i;
 611
 612     interval.low = addr;
 613     interval.high = addr + 1;
 614
 615     IOMMUTLBEntry entry = {
 616         .target_as = &address_space_memory,
 617         .iova = addr,
 618         .translated_addr = addr,
 619         .addr_mask = (1 << ctz32(s->config.page_size_mask)) - 1,
 620         .perm = IOMMU_NONE,
 621     };
 622
 623     bypass_allowed = virtio_vdev_has_feature(&s->parent_obj,
 624                                              VIRTIO_IOMMU_F_BYPASS);
 625
 626     sid = virtio_iommu_get_bdf(sdev);
 627
 628     trace_virtio_iommu_translate(mr->parent_obj.name, sid, addr, flag);
 629     qemu_mutex_lock(&s->mutex);
 630
 631     ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid));
 632     if (!ep) {
 633         if (!bypass_allowed) {
 634             error_report_once("%s sid=%d is not known!!", __func__, sid);
 635             virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_UNKNOWN,
 636                                       VIRTIO_IOMMU_FAULT_F_ADDRESS,
 637                                       sid, addr);
 638         } else {
 639             entry.perm = flag;
 640         }
 641         goto unlock;
 642     }
 643
 644     for (i = 0; i < s->nb_reserved_regions; i++) {
 645         ReservedRegion *reg = &s->reserved_regions[i];
 646
 647         if (addr >= reg->low && addr <= reg->high) {
 648             switch (reg->type) {
 649             case VIRTIO_IOMMU_RESV_MEM_T_MSI:
 650                 entry.perm = flag;
 651                 break;
 652             case VIRTIO_IOMMU_RESV_MEM_T_RESERVED:
 653             default:
 654                 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
 655                                           VIRTIO_IOMMU_FAULT_F_ADDRESS,
 656                                           sid, addr);
 657                 break;
 658             }
 659             goto unlock;
 660         }
 661     }
 662
 663     if (!ep->domain) {
 664         if (!bypass_allowed) {
 665             error_report_once("%s %02x:%02x.%01x not attached to any domain",
 666                               __func__, PCI_BUS_NUM(sid),
 667                               PCI_SLOT(sid), PCI_FUNC(sid));
 668             virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_DOMAIN,
 669                                       VIRTIO_IOMMU_FAULT_F_ADDRESS,
 670                                       sid, addr);
 671         } else {
 672             entry.perm = flag;
 673         }
 674         goto unlock;
 675     }
 676
 677     found = g_tree_lookup_extended(ep->domain->mappings, (gpointer)(&interval),
 678                                    (void **)&mapping_key,
 679                                    (void **)&mapping_value);
 680     if (!found) {
 681         error_report_once("%s no mapping for 0x%"PRIx64" for sid=%d",
 682                           __func__, addr, sid);
 683         virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
 684                                   VIRTIO_IOMMU_FAULT_F_ADDRESS,
 685                                   sid, addr);
 686         goto unlock;
 687     }
 688
 689     read_fault = (flag & IOMMU_RO) &&
 690                     !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_READ);
 691     write_fault = (flag & IOMMU_WO) &&
 692                     !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_WRITE);
 693
 694     flags = read_fault ? VIRTIO_IOMMU_FAULT_F_READ : 0;
 695     flags |= write_fault ? VIRTIO_IOMMU_FAULT_F_WRITE : 0;
 696     if (flags) {
 697         error_report_once("%s permission error on 0x%"PRIx64"(%d): allowed=%d",
 698                           __func__, addr, flag, mapping_value->flags);
 699         flags |= VIRTIO_IOMMU_FAULT_F_ADDRESS;
 700         virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
 701                                   flags | VIRTIO_IOMMU_FAULT_F_ADDRESS,
 702                                   sid, addr);
 703         goto unlock;
 704     }
 705     entry.translated_addr = addr - mapping_key->low + mapping_value->phys_addr;
 706     entry.perm = flag;
 707     trace_virtio_iommu_translate_out(addr, entry.translated_addr, sid);
 708
 709 unlock:
 710     qemu_mutex_unlock(&s->mutex);
 711     return entry;
 712 }
 713
 714 static void virtio_iommu_get_config(VirtIODevice *vdev, uint8_t *config_data)
 715 {
 716     VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev);
 717     struct virtio_iommu_config *config = &dev->config;
 718
 719     trace_virtio_iommu_get_config(config->page_size_mask,
 720                                   config->input_range.start,
 721                                   config->input_range.end,
 722                                   config->domain_range.end,
 723                                   config->probe_size);
 724     memcpy(config_data, &dev->config, sizeof(struct virtio_iommu_config));
 725 }
 726
 727 static void virtio_iommu_set_config(VirtIODevice *vdev,
 728                                       const uint8_t *config_data)
 729 {
 730     struct virtio_iommu_config config;
 731
 732     memcpy(&config, config_data, sizeof(struct virtio_iommu_config));
 733     trace_virtio_iommu_set_config(config.page_size_mask,
 734                                   config.input_range.start,
 735                                   config.input_range.end,
 736                                   config.domain_range.end,
 737                                   config.probe_size);
 738 }
 739
 740 static uint64_t virtio_iommu_get_features(VirtIODevice *vdev, uint64_t f,
 741                                           Error **errp)
 742 {
 743     VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev);
 744
 745     f |= dev->features;
 746     trace_virtio_iommu_get_features(f);
 747     return f;
 748 }
 749
 750 static gint int_cmp(gconstpointer a, gconstpointer b, gpointer user_data)
 751 {
 752     guint ua = GPOINTER_TO_UINT(a);
 753     guint ub = GPOINTER_TO_UINT(b);
 754     return (ua > ub) - (ua < ub);
 755 }
 756
 757 static void virtio_iommu_device_realize(DeviceState *dev, Error **errp)
 758 {
 759     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
 760     VirtIOIOMMU *s = VIRTIO_IOMMU(dev);
 761
 762     virtio_init(vdev, "virtio-iommu", VIRTIO_ID_IOMMU,
 763                 sizeof(struct virtio_iommu_config));
 764
 765     memset(s->iommu_pcibus_by_bus_num, 0, sizeof(s->iommu_pcibus_by_bus_num));
 766
 767     s->req_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE,
 768                              virtio_iommu_handle_command);
 769     s->event_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE, NULL);
 770
 771     s->config.page_size_mask = TARGET_PAGE_MASK;
 772     s->config.input_range.end = -1UL;
 773     s->config.domain_range.end = 32;
 774     s->config.probe_size = VIOMMU_PROBE_SIZE;
 775
 776     virtio_add_feature(&s->features, VIRTIO_RING_F_EVENT_IDX);
 777     virtio_add_feature(&s->features, VIRTIO_RING_F_INDIRECT_DESC);
 778     virtio_add_feature(&s->features, VIRTIO_F_VERSION_1);
 779     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_INPUT_RANGE);
 780     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_DOMAIN_RANGE);
 781     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MAP_UNMAP);
 782     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_BYPASS);
 783     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MMIO);
 784     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_PROBE);
 785
 786     qemu_mutex_init(&s->mutex);
 787
 788     s->as_by_busptr = g_hash_table_new_full(NULL, NULL, NULL, g_free);
 789
 790     if (s->primary_bus) {
 791         pci_setup_iommu(s->primary_bus, virtio_iommu_find_add_as, s);
 792     } else {
 793         error_setg(errp, "VIRTIO-IOMMU is not attached to any PCI bus!");
 794     }
 795 }
 796
 797 static void virtio_iommu_device_unrealize(DeviceState *dev)
 798 {
 799     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
 800     VirtIOIOMMU *s = VIRTIO_IOMMU(dev);
 801
 802     g_hash_table_destroy(s->as_by_busptr);
 803     g_tree_destroy(s->domains);
 804     g_tree_destroy(s->endpoints);
 805
 806     virtio_delete_queue(s->req_vq);
 807     virtio_delete_queue(s->event_vq);
 808     virtio_cleanup(vdev);
 809 }
 810
 811 static void virtio_iommu_device_reset(VirtIODevice *vdev)
 812 {
 813     VirtIOIOMMU *s = VIRTIO_IOMMU(vdev);
 814
 815     trace_virtio_iommu_device_reset();
 816
 817     if (s->domains) {
 818         g_tree_destroy(s->domains);
 819     }
 820     if (s->endpoints) {
 821         g_tree_destroy(s->endpoints);
 822     }
 823     s->domains = g_tree_new_full((GCompareDataFunc)int_cmp,
 824                                  NULL, NULL, virtio_iommu_put_domain);
 825     s->endpoints = g_tree_new_full((GCompareDataFunc)int_cmp,
 826                                    NULL, NULL, virtio_iommu_put_endpoint);
 827 }
 828
 829 static void virtio_iommu_set_status(VirtIODevice *vdev, uint8_t status)
 830 {
 831     trace_virtio_iommu_device_status(status);
 832 }
 833
 834 static void virtio_iommu_instance_init(Object *obj)
 835 {
 836 }
 837
 838 #define VMSTATE_INTERVAL                               \
 839 {                                                      \
 840     .name = "interval",                                \
 841     .version_id = 1,                                   \
 842     .minimum_version_id = 1,                           \
 843     .fields = (VMStateField[]) {                       \
 844         VMSTATE_UINT64(low, VirtIOIOMMUInterval),      \
 845         VMSTATE_UINT64(high, VirtIOIOMMUInterval),     \
 846         VMSTATE_END_OF_LIST()                          \
 847     }                                                  \
 848 }
 849
 850 #define VMSTATE_MAPPING                               \
 851 {                                                     \
 852     .name = "mapping",                                \
 853     .version_id = 1,                                  \
 854     .minimum_version_id = 1,                          \
 855     .fields = (VMStateField[]) {                      \
 856         VMSTATE_UINT64(phys_addr, VirtIOIOMMUMapping),\
 857         VMSTATE_UINT32(flags, VirtIOIOMMUMapping),    \
 858         VMSTATE_END_OF_LIST()                         \
 859     },                                                \
 860 }
 861
 862 static const VMStateDescription vmstate_interval_mapping[2] = {
 863     VMSTATE_MAPPING,   /* value */
 864     VMSTATE_INTERVAL   /* key   */
 865 };
 866
 867 static int domain_preload(void *opaque)
 868 {
 869     VirtIOIOMMUDomain *domain = opaque;
 870
 871     domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp,
 872                                        NULL, g_free, g_free);
 873     return 0;
 874 }
 875
 876 static const VMStateDescription vmstate_endpoint = {
 877     .name = "endpoint",
 878     .version_id = 1,
 879     .minimum_version_id = 1,
 880     .fields = (VMStateField[]) {
 881         VMSTATE_UINT32(id, VirtIOIOMMUEndpoint),
 882         VMSTATE_END_OF_LIST()
 883     }
 884 };
 885
 886 static const VMStateDescription vmstate_domain = {
 887     .name = "domain",
 888     .version_id = 1,
 889     .minimum_version_id = 1,
 890     .pre_load = domain_preload,
 891     .fields = (VMStateField[]) {
 892         VMSTATE_UINT32(id, VirtIOIOMMUDomain),
 893         VMSTATE_GTREE_V(mappings, VirtIOIOMMUDomain, 1,
 894                         vmstate_interval_mapping,
 895                         VirtIOIOMMUInterval, VirtIOIOMMUMapping),
 896         VMSTATE_QLIST_V(endpoint_list, VirtIOIOMMUDomain, 1,
 897                         vmstate_endpoint, VirtIOIOMMUEndpoint, next),
 898         VMSTATE_END_OF_LIST()
 899     }
 900 };
 901
 902 static gboolean reconstruct_endpoints(gpointer key, gpointer value,
 903                                       gpointer data)
 904 {
 905     VirtIOIOMMU *s = (VirtIOIOMMU *)data;
 906     VirtIOIOMMUDomain *d = (VirtIOIOMMUDomain *)value;
 907     VirtIOIOMMUEndpoint *iter;
 908
 909     QLIST_FOREACH(iter, &d->endpoint_list, next) {
 910         iter->domain = d;
 911         g_tree_insert(s->endpoints, GUINT_TO_POINTER(iter->id), iter);
 912     }
 913     return false; /* continue the domain traversal */
 914 }
 915
 916 static int iommu_post_load(void *opaque, int version_id)
 917 {
 918     VirtIOIOMMU *s = opaque;
 919
 920     g_tree_foreach(s->domains, reconstruct_endpoints, s);
 921     return 0;
 922 }
 923
 924 static const VMStateDescription vmstate_virtio_iommu_device = {
 925     .name = "virtio-iommu-device",
 926     .minimum_version_id = 1,
 927     .version_id = 1,
 928     .post_load = iommu_post_load,
 929     .fields = (VMStateField[]) {
 930         VMSTATE_GTREE_DIRECT_KEY_V(domains, VirtIOIOMMU, 1,
 931                                    &vmstate_domain, VirtIOIOMMUDomain),
 932         VMSTATE_END_OF_LIST()
 933     },
 934 };
 935
 936 static const VMStateDescription vmstate_virtio_iommu = {
 937     .name = "virtio-iommu",
 938     .minimum_version_id = 1,
 939     .priority = MIG_PRI_IOMMU,
 940     .version_id = 1,
 941     .fields = (VMStateField[]) {
 942         VMSTATE_VIRTIO_DEVICE,
 943         VMSTATE_END_OF_LIST()
 944     },
 945 };
 946
 947 static Property virtio_iommu_properties[] = {
 948     DEFINE_PROP_LINK("primary-bus", VirtIOIOMMU, primary_bus, "PCI", PCIBus *),
 949     DEFINE_PROP_END_OF_LIST(),
 950 };
 951
 952 static void virtio_iommu_class_init(ObjectClass *klass, void *data)
 953 {
 954     DeviceClass *dc = DEVICE_CLASS(klass);
 955     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
 956
 957     device_class_set_props(dc, virtio_iommu_properties);
 958     dc->vmsd = &vmstate_virtio_iommu;
 959
 960     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 961     vdc->realize = virtio_iommu_device_realize;
 962     vdc->unrealize = virtio_iommu_device_unrealize;
 963     vdc->reset = virtio_iommu_device_reset;
 964     vdc->get_config = virtio_iommu_get_config;
 965     vdc->set_config = virtio_iommu_set_config;
 966     vdc->get_features = virtio_iommu_get_features;
 967     vdc->set_status = virtio_iommu_set_status;
 968     vdc->vmsd = &vmstate_virtio_iommu_device;
 969 }
 970
 971 static void virtio_iommu_memory_region_class_init(ObjectClass *klass,
 972                                                   void *data)
 973 {
 974     IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
 975
 976     imrc->translate = virtio_iommu_translate;
 977 }
 978
 979 static const TypeInfo virtio_iommu_info = {
 980     .name = TYPE_VIRTIO_IOMMU,
 981     .parent = TYPE_VIRTIO_DEVICE,
 982     .instance_size = sizeof(VirtIOIOMMU),
 983     .instance_init = virtio_iommu_instance_init,
 984     .class_init = virtio_iommu_class_init,
 985 };
 986
 987 static const TypeInfo virtio_iommu_memory_region_info = {
 988     .parent = TYPE_IOMMU_MEMORY_REGION,
 989     .name = TYPE_VIRTIO_IOMMU_MEMORY_REGION,
 990     .class_init = virtio_iommu_memory_region_class_init,
 991 };
 992
 993 static void virtio_register_types(void)
 994 {
 995     type_register_static(&virtio_iommu_info);
 996     type_register_static(&virtio_iommu_memory_region_info);
 997 }
 998
 999 type_init(virtio_register_types)