hw/msix.c

   1 /*
   2  * MSI-X device support
   3  *
   4  * This module includes support for MSI-X in pci devices.
   5  *
   6  * Author: Michael S. Tsirkin <mst@redhat.com>
   7  *
   8  *  Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin (mst@redhat.com)
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2.  See
  11  * the COPYING file in the top-level directory.
  12  */
  13
  14 #include "hw.h"
  15 #include "msix.h"
  16 #include "pci.h"
  17 #include "range.h"
  18
  19 /* MSI-X capability structure */
  20 #define MSIX_TABLE_OFFSET 4
  21 #define MSIX_PBA_OFFSET 8
  22 #define MSIX_CAP_LENGTH 12
  23
  24 /* MSI enable bit and maskall bit are in byte 1 in FLAGS register */
  25 #define MSIX_CONTROL_OFFSET (PCI_MSIX_FLAGS + 1)
  26 #define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
  27 #define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8)
  28
  29 /* MSI-X table format */
  30 #define MSIX_MSG_ADDR 0
  31 #define MSIX_MSG_UPPER_ADDR 4
  32 #define MSIX_MSG_DATA 8
  33 #define MSIX_VECTOR_CTRL 12
  34 #define MSIX_ENTRY_SIZE 16
  35 #define MSIX_VECTOR_MASK 0x1
  36
  37 /* How much space does an MSIX table need. */
  38 /* The spec requires giving the table structure
  39  * a 4K aligned region all by itself. */
  40 #define MSIX_PAGE_SIZE 0x1000
  41 /* Reserve second half of the page for pending bits */
  42 #define MSIX_PAGE_PENDING (MSIX_PAGE_SIZE / 2)
  43 #define MSIX_MAX_ENTRIES 32
  44
  45
  46 /* Flag for interrupt controller to declare MSI-X support */
  47 int msix_supported;
  48
  49 /* Add MSI-X capability to the config space for the device. */
  50 /* Given a bar and its size, add MSI-X table on top of it
  51  * and fill MSI-X capability in the config space.
  52  * Original bar size must be a power of 2 or 0.
  53  * New bar size is returned. */
  54 static int msix_add_config(struct PCIDevice *pdev, unsigned short nentries,
  55                            unsigned bar_nr, unsigned bar_size)
  56 {
  57     int config_offset;
  58     uint8_t *config;
  59     uint32_t new_size;
  60
  61     if (nentries < 1 || nentries > PCI_MSIX_FLAGS_QSIZE + 1)
  62         return -EINVAL;
  63     if (bar_size > 0x80000000)
  64         return -ENOSPC;
  65
  66     /* Add space for MSI-X structures */
  67     if (!bar_size) {
  68         new_size = MSIX_PAGE_SIZE;
  69     } else if (bar_size < MSIX_PAGE_SIZE) {
  70         bar_size = MSIX_PAGE_SIZE;
  71         new_size = MSIX_PAGE_SIZE * 2;
  72     } else {
  73         new_size = bar_size * 2;
  74     }
  75
  76     pdev->msix_bar_size = new_size;
  77     config_offset = pci_add_capability(pdev, PCI_CAP_ID_MSIX, MSIX_CAP_LENGTH);
  78     if (config_offset < 0)
  79         return config_offset;
  80     config = pdev->config + config_offset;
  81
  82     pci_set_word(config + PCI_MSIX_FLAGS, nentries - 1);
  83     /* Table on top of BAR */
  84     pci_set_long(config + MSIX_TABLE_OFFSET, bar_size | bar_nr);
  85     /* Pending bits on top of that */
  86     pci_set_long(config + MSIX_PBA_OFFSET, (bar_size + MSIX_PAGE_PENDING) |
  87                  bar_nr);
  88     pdev->msix_cap = config_offset;
  89     /* Make flags bit writeable. */
  90     pdev->wmask[config_offset + MSIX_CONTROL_OFFSET] |= MSIX_ENABLE_MASK |
  91             MSIX_MASKALL_MASK;
  92     return 0;
  93 }
  94
  95 static uint32_t msix_mmio_readl(void *opaque, target_phys_addr_t addr)
  96 {
  97     PCIDevice *dev = opaque;
  98     unsigned int offset = addr & (MSIX_PAGE_SIZE - 1) & ~0x3;
  99     void *page = dev->msix_table_page;
 100
 101     return pci_get_long(page + offset);
 102 }
 103
 104 static uint32_t msix_mmio_read_unallowed(void *opaque, target_phys_addr_t addr)
 105 {
 106     fprintf(stderr, "MSI-X: only dword read is allowed!\n");
 107     return 0;
 108 }
 109
 110 static uint8_t msix_pending_mask(int vector)
 111 {
 112     return 1 << (vector % 8);
 113 }
 114
 115 static uint8_t *msix_pending_byte(PCIDevice *dev, int vector)
 116 {
 117     return dev->msix_table_page + MSIX_PAGE_PENDING + vector / 8;
 118 }
 119
 120 static int msix_is_pending(PCIDevice *dev, int vector)
 121 {
 122     return *msix_pending_byte(dev, vector) & msix_pending_mask(vector);
 123 }
 124
 125 static void msix_set_pending(PCIDevice *dev, int vector)
 126 {
 127     *msix_pending_byte(dev, vector) |= msix_pending_mask(vector);
 128 }
 129
 130 static void msix_clr_pending(PCIDevice *dev, int vector)
 131 {
 132     *msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector);
 133 }
 134
 135 static int msix_function_masked(PCIDevice *dev)
 136 {
 137     return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK;
 138 }
 139
 140 static int msix_is_masked(PCIDevice *dev, int vector)
 141 {
 142     unsigned offset = vector * MSIX_ENTRY_SIZE + MSIX_VECTOR_CTRL;
 143     return msix_function_masked(dev) ||
 144            dev->msix_table_page[offset] & MSIX_VECTOR_MASK;
 145 }
 146
 147 static void msix_handle_mask_update(PCIDevice *dev, int vector)
 148 {
 149     if (!msix_is_masked(dev, vector) && msix_is_pending(dev, vector)) {
 150         msix_clr_pending(dev, vector);
 151         msix_notify(dev, vector);
 152     }
 153 }
 154
 155 /* Handle MSI-X capability config write. */
 156 void msix_write_config(PCIDevice *dev, uint32_t addr,
 157                        uint32_t val, int len)
 158 {
 159     unsigned enable_pos = dev->msix_cap + MSIX_CONTROL_OFFSET;
 160     int vector;
 161
 162     if (!range_covers_byte(addr, len, enable_pos)) {
 163         return;
 164     }
 165
 166     if (!msix_enabled(dev)) {
 167         return;
 168     }
 169
 170     qemu_set_irq(dev->irq[0], 0);
 171
 172     if (msix_function_masked(dev)) {
 173         return;
 174     }
 175
 176     for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
 177         msix_handle_mask_update(dev, vector);
 178     }
 179 }
 180
 181 static void msix_mmio_writel(void *opaque, target_phys_addr_t addr,
 182                              uint32_t val)
 183 {
 184     PCIDevice *dev = opaque;
 185     unsigned int offset = addr & (MSIX_PAGE_SIZE - 1) & ~0x3;
 186     int vector = offset / MSIX_ENTRY_SIZE;
 187     pci_set_long(dev->msix_table_page + offset, val);
 188     msix_handle_mask_update(dev, vector);
 189 }
 190
 191 static void msix_mmio_write_unallowed(void *opaque, target_phys_addr_t addr,
 192                                       uint32_t val)
 193 {
 194     fprintf(stderr, "MSI-X: only dword write is allowed!\n");
 195 }
 196
 197 static CPUWriteMemoryFunc * const msix_mmio_write[] = {
 198     msix_mmio_write_unallowed, msix_mmio_write_unallowed, msix_mmio_writel
 199 };
 200
 201 static CPUReadMemoryFunc * const msix_mmio_read[] = {
 202     msix_mmio_read_unallowed, msix_mmio_read_unallowed, msix_mmio_readl
 203 };
 204
 205 /* Should be called from device's map method. */
 206 void msix_mmio_map(PCIDevice *d, int region_num,
 207                    pcibus_t addr, pcibus_t size, int type)
 208 {
 209     uint8_t *config = d->config + d->msix_cap;
 210     uint32_t table = pci_get_long(config + MSIX_TABLE_OFFSET);
 211     uint32_t offset = table & ~(MSIX_PAGE_SIZE - 1);
 212     /* TODO: for assigned devices, we'll want to make it possible to map
 213      * pending bits separately in case they are in a separate bar. */
 214     int table_bir = table & PCI_MSIX_FLAGS_BIRMASK;
 215
 216     if (table_bir != region_num)
 217         return;
 218     if (size <= offset)
 219         return;
 220     cpu_register_physical_memory(addr + offset, size - offset,
 221                                  d->msix_mmio_index);
 222 }
 223
 224 static void msix_mask_all(struct PCIDevice *dev, unsigned nentries)
 225 {
 226     int vector;
 227     for (vector = 0; vector < nentries; ++vector) {
 228         unsigned offset = vector * MSIX_ENTRY_SIZE + MSIX_VECTOR_CTRL;
 229         dev->msix_table_page[offset] |= MSIX_VECTOR_MASK;
 230     }
 231 }
 232
 233 /* Initialize the MSI-X structures. Note: if MSI-X is supported, BAR size is
 234  * modified, it should be retrieved with msix_bar_size. */
 235 int msix_init(struct PCIDevice *dev, unsigned short nentries,
 236               unsigned bar_nr, unsigned bar_size)
 237 {
 238     int ret;
 239     /* Nothing to do if MSI is not supported by interrupt controller */
 240     if (!msix_supported)
 241         return -ENOTSUP;
 242
 243     if (nentries > MSIX_MAX_ENTRIES)
 244         return -EINVAL;
 245
 246     dev->msix_entry_used = qemu_mallocz(MSIX_MAX_ENTRIES *
 247                                         sizeof *dev->msix_entry_used);
 248
 249     dev->msix_table_page = qemu_mallocz(MSIX_PAGE_SIZE);
 250     msix_mask_all(dev, nentries);
 251
 252     dev->msix_mmio_index = cpu_register_io_memory(msix_mmio_read,
 253                                                   msix_mmio_write, dev);
 254     if (dev->msix_mmio_index == -1) {
 255         ret = -EBUSY;
 256         goto err_index;
 257     }
 258
 259     dev->msix_entries_nr = nentries;
 260     ret = msix_add_config(dev, nentries, bar_nr, bar_size);
 261     if (ret)
 262         goto err_config;
 263
 264     dev->cap_present |= QEMU_PCI_CAP_MSIX;
 265     return 0;
 266
 267 err_config:
 268     dev->msix_entries_nr = 0;
 269     cpu_unregister_io_memory(dev->msix_mmio_index);
 270 err_index:
 271     qemu_free(dev->msix_table_page);
 272     dev->msix_table_page = NULL;
 273     qemu_free(dev->msix_entry_used);
 274     dev->msix_entry_used = NULL;
 275     return ret;
 276 }
 277
 278 static void msix_free_irq_entries(PCIDevice *dev)
 279 {
 280     int vector;
 281
 282     for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
 283         dev->msix_entry_used[vector] = 0;
 284         msix_clr_pending(dev, vector);
 285     }
 286 }
 287
 288 /* Clean up resources for the device. */
 289 int msix_uninit(PCIDevice *dev)
 290 {
 291     if (!(dev->cap_present & QEMU_PCI_CAP_MSIX))
 292         return 0;
 293     pci_del_capability(dev, PCI_CAP_ID_MSIX, MSIX_CAP_LENGTH);
 294     dev->msix_cap = 0;
 295     msix_free_irq_entries(dev);
 296     dev->msix_entries_nr = 0;
 297     cpu_unregister_io_memory(dev->msix_mmio_index);
 298     qemu_free(dev->msix_table_page);
 299     dev->msix_table_page = NULL;
 300     qemu_free(dev->msix_entry_used);
 301     dev->msix_entry_used = NULL;
 302     dev->cap_present &= ~QEMU_PCI_CAP_MSIX;
 303     return 0;
 304 }
 305
 306 void msix_save(PCIDevice *dev, QEMUFile *f)
 307 {
 308     unsigned n = dev->msix_entries_nr;
 309
 310     if (!(dev->cap_present & QEMU_PCI_CAP_MSIX)) {
 311         return;
 312     }
 313
 314     qemu_put_buffer(f, dev->msix_table_page, n * MSIX_ENTRY_SIZE);
 315     qemu_put_buffer(f, dev->msix_table_page + MSIX_PAGE_PENDING, (n + 7) / 8);
 316 }
 317
 318 /* Should be called after restoring the config space. */
 319 void msix_load(PCIDevice *dev, QEMUFile *f)
 320 {
 321     unsigned n = dev->msix_entries_nr;
 322
 323     if (!(dev->cap_present & QEMU_PCI_CAP_MSIX)) {
 324         return;
 325     }
 326
 327     msix_free_irq_entries(dev);
 328     qemu_get_buffer(f, dev->msix_table_page, n * MSIX_ENTRY_SIZE);
 329     qemu_get_buffer(f, dev->msix_table_page + MSIX_PAGE_PENDING, (n + 7) / 8);
 330 }
 331
 332 /* Does device support MSI-X? */
 333 int msix_present(PCIDevice *dev)
 334 {
 335     return dev->cap_present & QEMU_PCI_CAP_MSIX;
 336 }
 337
 338 /* Is MSI-X enabled? */
 339 int msix_enabled(PCIDevice *dev)
 340 {
 341     return (dev->cap_present & QEMU_PCI_CAP_MSIX) &&
 342         (dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
 343          MSIX_ENABLE_MASK);
 344 }
 345
 346 /* Size of bar where MSI-X table resides, or 0 if MSI-X not supported. */
 347 uint32_t msix_bar_size(PCIDevice *dev)
 348 {
 349     return (dev->cap_present & QEMU_PCI_CAP_MSIX) ?
 350         dev->msix_bar_size : 0;
 351 }
 352
 353 /* Send an MSI-X message */
 354 void msix_notify(PCIDevice *dev, unsigned vector)
 355 {
 356     uint8_t *table_entry = dev->msix_table_page + vector * MSIX_ENTRY_SIZE;
 357     uint64_t address;
 358     uint32_t data;
 359
 360     if (vector >= dev->msix_entries_nr || !dev->msix_entry_used[vector])
 361         return;
 362     if (msix_is_masked(dev, vector)) {
 363         msix_set_pending(dev, vector);
 364         return;
 365     }
 366
 367     address = pci_get_long(table_entry + MSIX_MSG_UPPER_ADDR);
 368     address = (address << 32) | pci_get_long(table_entry + MSIX_MSG_ADDR);
 369     data = pci_get_long(table_entry + MSIX_MSG_DATA);
 370     stl_phys(address, data);
 371 }
 372
 373 void msix_reset(PCIDevice *dev)
 374 {
 375     if (!(dev->cap_present & QEMU_PCI_CAP_MSIX))
 376         return;
 377     msix_free_irq_entries(dev);
 378     dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &=
 379             ~dev->wmask[dev->msix_cap + MSIX_CONTROL_OFFSET];
 380     memset(dev->msix_table_page, 0, MSIX_PAGE_SIZE);
 381     msix_mask_all(dev, dev->msix_entries_nr);
 382 }
 383
 384 /* PCI spec suggests that devices make it possible for software to configure
 385  * less vectors than supported by the device, but does not specify a standard
 386  * mechanism for devices to do so.
 387  *
 388  * We support this by asking devices to declare vectors software is going to
 389  * actually use, and checking this on the notification path. Devices that
 390  * don't want to follow the spec suggestion can declare all vectors as used. */
 391
 392 /* Mark vector as used. */
 393 int msix_vector_use(PCIDevice *dev, unsigned vector)
 394 {
 395     if (vector >= dev->msix_entries_nr)
 396         return -EINVAL;
 397     dev->msix_entry_used[vector]++;
 398     return 0;
 399 }
 400
 401 /* Mark vector as unused. */
 402 void msix_vector_unuse(PCIDevice *dev, unsigned vector)
 403 {
 404     if (vector >= dev->msix_entries_nr || !dev->msix_entry_used[vector]) {
 405         return;
 406     }
 407     if (--dev->msix_entry_used[vector]) {
 408         return;
 409     }
 410     msix_clr_pending(dev, vector);
 411 }
 412
 413 void msix_unuse_all_vectors(PCIDevice *dev)
 414 {
 415     if (!(dev->cap_present & QEMU_PCI_CAP_MSIX))
 416         return;
 417     msix_free_irq_entries(dev);
 418 }