hw/msix.c

   1 /*
   2  * MSI-X device support
   3  *
   4  * This module includes support for MSI-X in pci devices.
   5  *
   6  * Author: Michael S. Tsirkin <mst@redhat.com>
   7  *
   8  *  Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin (mst@redhat.com)
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2.  See
  11  * the COPYING file in the top-level directory.
  12  */
  13
  14 #include "hw.h"
  15 #include "msix.h"
  16 #include "pci.h"
  17 #define QEMU_KVM_NO_CPU
  18 #include "qemu-kvm.h"
  19
  20 /* Declaration from linux/pci_regs.h */
  21 #define  PCI_CAP_ID_MSIX 0x11 /* MSI-X */
  22 #define  PCI_MSIX_FLAGS 2     /* Table at lower 11 bits */
  23 #define  PCI_MSIX_FLAGS_QSIZE   0x7FF
  24 #define  PCI_MSIX_FLAGS_ENABLE  (1 << 15)
  25 #define  PCI_MSIX_FLAGS_MASKALL (1 << 14)
  26 #define  PCI_MSIX_FLAGS_BIRMASK (7 << 0)
  27
  28 /* MSI-X capability structure */
  29 #define MSIX_TABLE_OFFSET 4
  30 #define MSIX_PBA_OFFSET 8
  31 #define MSIX_CAP_LENGTH 12
  32
  33 /* MSI enable bit and maskall bit are in byte 1 in FLAGS register */
  34 #define MSIX_CONTROL_OFFSET (PCI_MSIX_FLAGS + 1)
  35 #define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
  36 #define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8)
  37
  38 /* MSI-X table format */
  39 #define MSIX_MSG_ADDR 0
  40 #define MSIX_MSG_UPPER_ADDR 4
  41 #define MSIX_MSG_DATA 8
  42 #define MSIX_VECTOR_CTRL 12
  43 #define MSIX_ENTRY_SIZE 16
  44 #define MSIX_VECTOR_MASK 0x1
  45
  46 /* How much space does an MSIX table need. */
  47 /* The spec requires giving the table structure
  48  * a 4K aligned region all by itself. */
  49 #define MSIX_PAGE_SIZE 0x1000
  50 /* Reserve second half of the page for pending bits */
  51 #define MSIX_PAGE_PENDING (MSIX_PAGE_SIZE / 2)
  52 #define MSIX_MAX_ENTRIES 32
  53
  54
  55 #ifdef MSIX_DEBUG
  56 #define DEBUG(fmt, ...)                                       \
  57     do {                                                      \
  58       fprintf(stderr, "%s: " fmt, __func__ , __VA_ARGS__);    \
  59     } while (0)
  60 #else
  61 #define DEBUG(fmt, ...) do { } while(0)
  62 #endif
  63
  64 /* Flag for interrupt controller to declare MSI-X support */
  65 int msix_supported;
  66
  67 #ifdef CONFIG_KVM
  68 /* KVM specific MSIX helpers */
  69 static void kvm_msix_free(PCIDevice *dev)
  70 {
  71     int vector, changed = 0;
  72     for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
  73         if (dev->msix_entry_used[vector]) {
  74             kvm_del_routing_entry(kvm_context, &dev->msix_irq_entries[vector]);
  75             changed = 1;
  76         }
  77     }
  78     if (changed) {
  79         kvm_commit_irq_routes(kvm_context);
  80     }
  81 }
  82
  83 static void kvm_msix_routing_entry(PCIDevice *dev, unsigned vector,
  84                                    struct kvm_irq_routing_entry *entry)
  85 {
  86     uint8_t *table_entry = dev->msix_table_page + vector * MSIX_ENTRY_SIZE;
  87     entry->type = KVM_IRQ_ROUTING_MSI;
  88     entry->flags = 0;
  89     entry->u.msi.address_lo = pci_get_long(table_entry + MSIX_MSG_ADDR);
  90     entry->u.msi.address_hi = pci_get_long(table_entry + MSIX_MSG_UPPER_ADDR);
  91     entry->u.msi.data = pci_get_long(table_entry + MSIX_MSG_DATA);
  92 }
  93
  94 static void kvm_msix_update(PCIDevice *dev, int vector,
  95                             int was_masked, int is_masked)
  96 {
  97     struct kvm_irq_routing_entry e = {}, *entry;
  98     int mask_cleared = was_masked && !is_masked;
  99     /* It is only legal to change an entry when it is masked. Therefore, it is
 100      * enough to update the routing in kernel when mask is being cleared. */
 101     if (!mask_cleared) {
 102         return;
 103     }
 104     if (!dev->msix_entry_used[vector]) {
 105         return;
 106     }
 107     entry = dev->msix_irq_entries + vector;
 108     e.gsi = entry->gsi;
 109     kvm_msix_routing_entry(dev, vector, &e);
 110     if (memcmp(&entry->u.msi, &e.u.msi, sizeof entry->u.msi)) {
 111         int r;
 112         r = kvm_update_routing_entry(kvm_context, entry, &e);
 113         if (r) {
 114             fprintf(stderr, "%s: kvm_update_routing_entry failed: %s\n", __func__,
 115                     strerror(-r));
 116             exit(1);
 117         }
 118         memcpy(&entry->u.msi, &e.u.msi, sizeof entry->u.msi);
 119         r = kvm_commit_irq_routes(kvm_context);
 120         if (r) {
 121             fprintf(stderr, "%s: kvm_commit_irq_routes failed: %s\n", __func__,
 122                     strerror(-r));
 123             exit(1);
 124         }
 125     }
 126 }
 127
 128 static int kvm_msix_add(PCIDevice *dev, unsigned vector)
 129 {
 130     struct kvm_irq_routing_entry *entry = dev->msix_irq_entries + vector;
 131     int r;
 132
 133     if (!kvm_has_gsi_routing(kvm_context)) {
 134         fprintf(stderr, "Warning: no MSI-X support found. "
 135                 "At least kernel 2.6.30 is required for MSI-X support.\n"
 136                );
 137         return -EOPNOTSUPP;
 138     }
 139
 140     r = kvm_get_irq_route_gsi(kvm_context);
 141     if (r < 0) {
 142         fprintf(stderr, "%s: kvm_get_irq_route_gsi failed: %s\n", __func__, strerror(-r));
 143         return r;
 144     }
 145     entry->gsi = r;
 146     kvm_msix_routing_entry(dev, vector, entry);
 147     r = kvm_add_routing_entry(kvm_context, entry);
 148     if (r < 0) {
 149         fprintf(stderr, "%s: kvm_add_routing_entry failed: %s\n", __func__, strerror(-r));
 150         return r;
 151     }
 152
 153     r = kvm_commit_irq_routes(kvm_context);
 154     if (r < 0) {
 155         fprintf(stderr, "%s: kvm_commit_irq_routes failed: %s\n", __func__, strerror(-r));
 156         return r;
 157     }
 158     return 0;
 159 }
 160
 161 static void kvm_msix_del(PCIDevice *dev, unsigned vector)
 162 {
 163     if (dev->msix_entry_used[vector]) {
 164         return;
 165     }
 166     kvm_del_routing_entry(kvm_context, &dev->msix_irq_entries[vector]);
 167     kvm_commit_irq_routes(kvm_context);
 168 }
 169 #else
 170
 171 static void kvm_msix_free(PCIDevice *dev) {}
 172 static void kvm_msix_update(PCIDevice *dev, int vector,
 173                             int was_masked, int is_masked) {}
 174 static int kvm_msix_add(PCIDevice *dev, unsigned vector) { return -1; }
 175 static void kvm_msix_del(PCIDevice *dev, unsigned vector) {}
 176 #endif
 177
 178 /* Add MSI-X capability to the config space for the device. */
 179 /* Given a bar and its size, add MSI-X table on top of it
 180  * and fill MSI-X capability in the config space.
 181  * Original bar size must be a power of 2 or 0.
 182  * New bar size is returned. */
 183 static int msix_add_config(struct PCIDevice *pdev, unsigned short nentries,
 184                            unsigned bar_nr, unsigned bar_size)
 185 {
 186     int config_offset;
 187     uint8_t *config;
 188     uint32_t new_size;
 189
 190     if (nentries < 1 || nentries > PCI_MSIX_FLAGS_QSIZE + 1)
 191         return -EINVAL;
 192     if (bar_size > 0x80000000)
 193         return -ENOSPC;
 194
 195     /* Add space for MSI-X structures */
 196     if (!bar_size) {
 197         new_size = MSIX_PAGE_SIZE;
 198     } else if (bar_size < MSIX_PAGE_SIZE) {
 199         bar_size = MSIX_PAGE_SIZE;
 200         new_size = MSIX_PAGE_SIZE * 2;
 201     } else {
 202         new_size = bar_size * 2;
 203     }
 204
 205     pdev->msix_bar_size = new_size;
 206     config_offset = pci_add_capability(pdev, PCI_CAP_ID_MSIX, MSIX_CAP_LENGTH);
 207     if (config_offset < 0)
 208         return config_offset;
 209     config = pdev->config + config_offset;
 210
 211     pci_set_word(config + PCI_MSIX_FLAGS, nentries - 1);
 212     /* Table on top of BAR */
 213     pci_set_long(config + MSIX_TABLE_OFFSET, bar_size | bar_nr);
 214     /* Pending bits on top of that */
 215     pci_set_long(config + MSIX_PBA_OFFSET, (bar_size + MSIX_PAGE_PENDING) |
 216                  bar_nr);
 217     pdev->msix_cap = config_offset;
 218     /* Make flags bit writeable. */
 219     pdev->wmask[config_offset + MSIX_CONTROL_OFFSET] |= MSIX_ENABLE_MASK |
 220             MSIX_MASKALL_MASK;
 221     return 0;
 222 }
 223
 224 static uint32_t msix_mmio_readl(void *opaque, target_phys_addr_t addr)
 225 {
 226     PCIDevice *dev = opaque;
 227     unsigned int offset = addr & (MSIX_PAGE_SIZE - 1) & ~0x3;
 228     void *page = dev->msix_table_page;
 229
 230     return pci_get_long(page + offset);
 231 }
 232
 233 static uint32_t msix_mmio_read_unallowed(void *opaque, target_phys_addr_t addr)
 234 {
 235     fprintf(stderr, "MSI-X: only dword read is allowed!\n");
 236     return 0;
 237 }
 238
 239 static uint8_t msix_pending_mask(int vector)
 240 {
 241     return 1 << (vector % 8);
 242 }
 243
 244 static uint8_t *msix_pending_byte(PCIDevice *dev, int vector)
 245 {
 246     return dev->msix_table_page + MSIX_PAGE_PENDING + vector / 8;
 247 }
 248
 249 static int msix_is_pending(PCIDevice *dev, int vector)
 250 {
 251     return *msix_pending_byte(dev, vector) & msix_pending_mask(vector);
 252 }
 253
 254 static void msix_set_pending(PCIDevice *dev, int vector)
 255 {
 256     *msix_pending_byte(dev, vector) |= msix_pending_mask(vector);
 257 }
 258
 259 static void msix_clr_pending(PCIDevice *dev, int vector)
 260 {
 261     *msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector);
 262 }
 263
 264 static int msix_function_masked(PCIDevice *dev)
 265 {
 266     return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK;
 267 }
 268
 269 static int msix_is_masked(PCIDevice *dev, int vector)
 270 {
 271     unsigned offset = vector * MSIX_ENTRY_SIZE + MSIX_VECTOR_CTRL;
 272     return msix_function_masked(dev) ||
 273            dev->msix_table_page[offset] & MSIX_VECTOR_MASK;
 274 }
 275
 276 static void msix_handle_mask_update(PCIDevice *dev, int vector)
 277 {
 278     if (!msix_is_masked(dev, vector) && msix_is_pending(dev, vector)) {
 279         msix_clr_pending(dev, vector);
 280         msix_notify(dev, vector);
 281     }
 282 }
 283
 284 /* Handle MSI-X capability config write. */
 285 void msix_write_config(PCIDevice *dev, uint32_t addr,
 286                        uint32_t val, int len)
 287 {
 288     unsigned enable_pos = dev->msix_cap + MSIX_CONTROL_OFFSET;
 289     int vector;
 290
 291     if (!range_covers_byte(addr, len, enable_pos)) {
 292         return;
 293     }
 294
 295     if (!msix_enabled(dev)) {
 296         return;
 297     }
 298
 299     qemu_set_irq(dev->irq[0], 0);
 300
 301     if (msix_function_masked(dev)) {
 302         return;
 303     }
 304
 305     for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
 306         msix_handle_mask_update(dev, vector);
 307     }
 308 }
 309
 310 static void msix_mmio_writel(void *opaque, target_phys_addr_t addr,
 311                              uint32_t val)
 312 {
 313     PCIDevice *dev = opaque;
 314     unsigned int offset = addr & (MSIX_PAGE_SIZE - 1) & ~0x3;
 315     int vector = offset / MSIX_ENTRY_SIZE;
 316     int was_masked = msix_is_masked(dev, vector);
 317     pci_set_long(dev->msix_table_page + offset, val);
 318     if (kvm_enabled() && kvm_irqchip_in_kernel()) {
 319         kvm_msix_update(dev, vector, was_masked, msix_is_masked(dev, vector));
 320     }
 321     msix_handle_mask_update(dev, vector);
 322 }
 323
 324 static void msix_mmio_write_unallowed(void *opaque, target_phys_addr_t addr,
 325                                       uint32_t val)
 326 {
 327     fprintf(stderr, "MSI-X: only dword write is allowed!\n");
 328 }
 329
 330 static CPUWriteMemoryFunc * const msix_mmio_write[] = {
 331     msix_mmio_write_unallowed, msix_mmio_write_unallowed, msix_mmio_writel
 332 };
 333
 334 static CPUReadMemoryFunc * const msix_mmio_read[] = {
 335     msix_mmio_read_unallowed, msix_mmio_read_unallowed, msix_mmio_readl
 336 };
 337
 338 /* Should be called from device's map method. */
 339 void msix_mmio_map(PCIDevice *d, int region_num,
 340                    pcibus_t addr, pcibus_t size, int type)
 341 {
 342     uint8_t *config = d->config + d->msix_cap;
 343     uint32_t table = pci_get_long(config + MSIX_TABLE_OFFSET);
 344     uint32_t offset = table & ~(MSIX_PAGE_SIZE - 1);
 345     /* TODO: for assigned devices, we'll want to make it possible to map
 346      * pending bits separately in case they are in a separate bar. */
 347     int table_bir = table & PCI_MSIX_FLAGS_BIRMASK;
 348
 349     if (table_bir != region_num)
 350         return;
 351     if (size <= offset)
 352         return;
 353     cpu_register_physical_memory(addr + offset, size - offset,
 354                                  d->msix_mmio_index);
 355 }
 356
 357 static void msix_mask_all(struct PCIDevice *dev, unsigned nentries)
 358 {
 359     int vector;
 360     for (vector = 0; vector < nentries; ++vector) {
 361         unsigned offset = vector * MSIX_ENTRY_SIZE + MSIX_VECTOR_CTRL;
 362         dev->msix_table_page[offset] |= MSIX_VECTOR_MASK;
 363     }
 364 }
 365
 366 /* Initialize the MSI-X structures. Note: if MSI-X is supported, BAR size is
 367  * modified, it should be retrieved with msix_bar_size. */
 368 int msix_init(struct PCIDevice *dev, unsigned short nentries,
 369               unsigned bar_nr, unsigned bar_size)
 370 {
 371     int ret;
 372     /* Nothing to do if MSI is not supported by interrupt controller */
 373     if (!msix_supported)
 374         return -ENOTSUP;
 375
 376     if (nentries > MSIX_MAX_ENTRIES)
 377         return -EINVAL;
 378
 379 #ifdef KVM_CAP_IRQCHIP
 380     if (kvm_enabled() && kvm_irqchip_in_kernel()) {
 381         dev->msix_irq_entries = qemu_malloc(nentries *
 382                                             sizeof *dev->msix_irq_entries);
 383     }
 384 #endif
 385     dev->msix_entry_used = qemu_mallocz(MSIX_MAX_ENTRIES *
 386                                         sizeof *dev->msix_entry_used);
 387
 388     dev->msix_table_page = qemu_mallocz(MSIX_PAGE_SIZE);
 389     msix_mask_all(dev, nentries);
 390
 391     dev->msix_mmio_index = cpu_register_io_memory(msix_mmio_read,
 392                                                   msix_mmio_write, dev);
 393     if (dev->msix_mmio_index == -1) {
 394         ret = -EBUSY;
 395         goto err_index;
 396     }
 397
 398     dev->msix_entries_nr = nentries;
 399     ret = msix_add_config(dev, nentries, bar_nr, bar_size);
 400     if (ret)
 401         goto err_config;
 402
 403     dev->cap_present |= QEMU_PCI_CAP_MSIX;
 404     return 0;
 405
 406 err_config:
 407     dev->msix_entries_nr = 0;
 408     cpu_unregister_io_memory(dev->msix_mmio_index);
 409 err_index:
 410     qemu_free(dev->msix_table_page);
 411     dev->msix_table_page = NULL;
 412     qemu_free(dev->msix_entry_used);
 413     dev->msix_entry_used = NULL;
 414     return ret;
 415 }
 416
 417 static void msix_free_irq_entries(PCIDevice *dev)
 418 {
 419     int vector;
 420
 421     if (kvm_enabled() && kvm_irqchip_in_kernel()) {
 422         kvm_msix_free(dev);
 423     }
 424
 425     for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
 426         dev->msix_entry_used[vector] = 0;
 427         msix_clr_pending(dev, vector);
 428     }
 429 }
 430
 431 /* Clean up resources for the device. */
 432 int msix_uninit(PCIDevice *dev)
 433 {
 434     if (!(dev->cap_present & QEMU_PCI_CAP_MSIX))
 435         return 0;
 436     pci_del_capability(dev, PCI_CAP_ID_MSIX, MSIX_CAP_LENGTH);
 437     dev->msix_cap = 0;
 438     msix_free_irq_entries(dev);
 439     dev->msix_entries_nr = 0;
 440     cpu_unregister_io_memory(dev->msix_mmio_index);
 441     qemu_free(dev->msix_table_page);
 442     dev->msix_table_page = NULL;
 443     qemu_free(dev->msix_entry_used);
 444     dev->msix_entry_used = NULL;
 445     qemu_free(dev->msix_irq_entries);
 446     dev->msix_irq_entries = NULL;
 447     dev->cap_present &= ~QEMU_PCI_CAP_MSIX;
 448     return 0;
 449 }
 450
 451 void msix_save(PCIDevice *dev, QEMUFile *f)
 452 {
 453     unsigned n = dev->msix_entries_nr;
 454
 455     if (!msix_supported) {
 456         return;
 457     }
 458
 459     if (!(dev->cap_present & QEMU_PCI_CAP_MSIX)) {
 460         return;
 461     }
 462     qemu_put_buffer(f, dev->msix_table_page, n * MSIX_ENTRY_SIZE);
 463     qemu_put_buffer(f, dev->msix_table_page + MSIX_PAGE_PENDING, (n + 7) / 8);
 464 }
 465
 466 /* Should be called after restoring the config space. */
 467 void msix_load(PCIDevice *dev, QEMUFile *f)
 468 {
 469     unsigned n = dev->msix_entries_nr;
 470
 471     if (!msix_supported)
 472         return;
 473
 474     if (!(dev->cap_present & QEMU_PCI_CAP_MSIX)) {
 475         return;
 476     }
 477
 478     msix_free_irq_entries(dev);
 479     qemu_get_buffer(f, dev->msix_table_page, n * MSIX_ENTRY_SIZE);
 480     qemu_get_buffer(f, dev->msix_table_page + MSIX_PAGE_PENDING, (n + 7) / 8);
 481 }
 482
 483 /* Does device support MSI-X? */
 484 int msix_present(PCIDevice *dev)
 485 {
 486     return dev->cap_present & QEMU_PCI_CAP_MSIX;
 487 }
 488
 489 /* Is MSI-X enabled? */
 490 int msix_enabled(PCIDevice *dev)
 491 {
 492     return (dev->cap_present & QEMU_PCI_CAP_MSIX) &&
 493         (dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
 494          MSIX_ENABLE_MASK);
 495 }
 496
 497 /* Size of bar where MSI-X table resides, or 0 if MSI-X not supported. */
 498 uint32_t msix_bar_size(PCIDevice *dev)
 499 {
 500     return (dev->cap_present & QEMU_PCI_CAP_MSIX) ?
 501         dev->msix_bar_size : 0;
 502 }
 503
 504 /* Send an MSI-X message */
 505 void msix_notify(PCIDevice *dev, unsigned vector)
 506 {
 507     uint8_t *table_entry = dev->msix_table_page + vector * MSIX_ENTRY_SIZE;
 508     uint64_t address;
 509     uint32_t data;
 510
 511     if (vector >= dev->msix_entries_nr || !dev->msix_entry_used[vector])
 512         return;
 513     if (msix_is_masked(dev, vector)) {
 514         msix_set_pending(dev, vector);
 515         return;
 516     }
 517
 518 #ifdef KVM_CAP_IRQCHIP
 519     if (kvm_enabled() && kvm_irqchip_in_kernel()) {
 520         kvm_set_irq(dev->msix_irq_entries[vector].gsi, 1, NULL);
 521         return;
 522     }
 523 #endif
 524
 525     address = pci_get_long(table_entry + MSIX_MSG_UPPER_ADDR);
 526     address = (address << 32) | pci_get_long(table_entry + MSIX_MSG_ADDR);
 527     data = pci_get_long(table_entry + MSIX_MSG_DATA);
 528     stl_phys(address, data);
 529 }
 530
 531 void msix_reset(PCIDevice *dev)
 532 {
 533     if (!(dev->cap_present & QEMU_PCI_CAP_MSIX))
 534         return;
 535     msix_free_irq_entries(dev);
 536     dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &=
 537             ~dev->wmask[dev->msix_cap + MSIX_CONTROL_OFFSET];
 538     memset(dev->msix_table_page, 0, MSIX_PAGE_SIZE);
 539     msix_mask_all(dev, dev->msix_entries_nr);
 540 }
 541
 542 /* PCI spec suggests that devices make it possible for software to configure
 543  * less vectors than supported by the device, but does not specify a standard
 544  * mechanism for devices to do so.
 545  *
 546  * We support this by asking devices to declare vectors software is going to
 547  * actually use, and checking this on the notification path. Devices that
 548  * don't want to follow the spec suggestion can declare all vectors as used. */
 549
 550 /* Mark vector as used. */
 551 int msix_vector_use(PCIDevice *dev, unsigned vector)
 552 {
 553     int ret;
 554     if (vector >= dev->msix_entries_nr)
 555         return -EINVAL;
 556     if (dev->msix_entry_used[vector]) {
 557         return 0;
 558     }
 559     if (kvm_enabled() && kvm_irqchip_in_kernel()) {
 560         ret = kvm_msix_add(dev, vector);
 561         if (ret) {
 562             return ret;
 563         }
 564     }
 565     ++dev->msix_entry_used[vector];
 566     return 0;
 567 }
 568
 569 /* Mark vector as unused. */
 570 void msix_vector_unuse(PCIDevice *dev, unsigned vector)
 571 {
 572     if (vector >= dev->msix_entries_nr || !dev->msix_entry_used[vector]) {
 573         return;
 574     }
 575     if (--dev->msix_entry_used[vector]) {
 576         return;
 577     }
 578     if (kvm_enabled() && kvm_irqchip_in_kernel()) {
 579         kvm_msix_del(dev, vector);
 580     }
 581     msix_clr_pending(dev, vector);
 582 }
 583
 584 void msix_unuse_all_vectors(PCIDevice *dev)
 585 {
 586     if (!(dev->cap_present & QEMU_PCI_CAP_MSIX))
 587         return;
 588     msix_free_irq_entries(dev);
 589 }