hw/msi.c

   1 /*
   2  * msi.c
   3  *
   4  * Copyright (c) 2010 Isaku Yamahata <yamahata at valinux co jp>
   5  *                    VA Linux Systems Japan K.K.
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16
  17  * You should have received a copy of the GNU General Public License along
  18  * with this program; if not, see <http://www.gnu.org/licenses/>.
  19  */
  20
  21 #include "msi.h"
  22 #include "range.h"
  23 #include "kvm.h"
  24
  25 /* Eventually those constants should go to Linux pci_regs.h */
  26 #define PCI_MSI_PENDING_32      0x10
  27 #define PCI_MSI_PENDING_64      0x14
  28
  29 /* PCI_MSI_ADDRESS_LO */
  30 #define PCI_MSI_ADDRESS_LO_MASK         (~0x3)
  31
  32 /* If we get rid of cap allocator, we won't need those. */
  33 #define PCI_MSI_32_SIZEOF       0x0a
  34 #define PCI_MSI_64_SIZEOF       0x0e
  35 #define PCI_MSI_32M_SIZEOF      0x14
  36 #define PCI_MSI_64M_SIZEOF      0x18
  37
  38 #define PCI_MSI_VECTORS_MAX     32
  39
  40 /* Flag for interrupt controller to declare MSI/MSI-X support */
  41 bool msi_supported;
  42
  43 /* If we get rid of cap allocator, we won't need this. */
  44 static inline uint8_t msi_cap_sizeof(uint16_t flags)
  45 {
  46     switch (flags & (PCI_MSI_FLAGS_MASKBIT | PCI_MSI_FLAGS_64BIT)) {
  47     case PCI_MSI_FLAGS_MASKBIT | PCI_MSI_FLAGS_64BIT:
  48         return PCI_MSI_64M_SIZEOF;
  49     case PCI_MSI_FLAGS_64BIT:
  50         return PCI_MSI_64_SIZEOF;
  51     case PCI_MSI_FLAGS_MASKBIT:
  52         return PCI_MSI_32M_SIZEOF;
  53     case 0:
  54         return PCI_MSI_32_SIZEOF;
  55     default:
  56         abort();
  57         break;
  58     }
  59     return 0;
  60 }
  61
  62 //#define MSI_DEBUG
  63
  64 #ifdef MSI_DEBUG
  65 # define MSI_DPRINTF(fmt, ...)                                          \
  66     fprintf(stderr, "%s:%d " fmt, __func__, __LINE__, ## __VA_ARGS__)
  67 #else
  68 # define MSI_DPRINTF(fmt, ...)  do { } while (0)
  69 #endif
  70 #define MSI_DEV_PRINTF(dev, fmt, ...)                                   \
  71     MSI_DPRINTF("%s:%x " fmt, (dev)->name, (dev)->devfn, ## __VA_ARGS__)
  72
  73 static inline unsigned int msi_nr_vectors(uint16_t flags)
  74 {
  75     return 1U <<
  76         ((flags & PCI_MSI_FLAGS_QSIZE) >> (ffs(PCI_MSI_FLAGS_QSIZE) - 1));
  77 }
  78
  79 static inline uint8_t msi_flags_off(const PCIDevice* dev)
  80 {
  81     return dev->msi_cap + PCI_MSI_FLAGS;
  82 }
  83
  84 static inline uint8_t msi_address_lo_off(const PCIDevice* dev)
  85 {
  86     return dev->msi_cap + PCI_MSI_ADDRESS_LO;
  87 }
  88
  89 static inline uint8_t msi_address_hi_off(const PCIDevice* dev)
  90 {
  91     return dev->msi_cap + PCI_MSI_ADDRESS_HI;
  92 }
  93
  94 static inline uint8_t msi_data_off(const PCIDevice* dev, bool msi64bit)
  95 {
  96     return dev->msi_cap + (msi64bit ? PCI_MSI_DATA_64 : PCI_MSI_DATA_32);
  97 }
  98
  99 static inline uint8_t msi_mask_off(const PCIDevice* dev, bool msi64bit)
 100 {
 101     return dev->msi_cap + (msi64bit ? PCI_MSI_MASK_64 : PCI_MSI_MASK_32);
 102 }
 103
 104 static inline uint8_t msi_pending_off(const PCIDevice* dev, bool msi64bit)
 105 {
 106     return dev->msi_cap + (msi64bit ? PCI_MSI_PENDING_64 : PCI_MSI_PENDING_32);
 107 }
 108
 109 bool msi_enabled(const PCIDevice *dev)
 110 {
 111     return msi_present(dev) &&
 112         (pci_get_word(dev->config + msi_flags_off(dev)) &
 113          PCI_MSI_FLAGS_ENABLE);
 114 }
 115
 116 static void kvm_msi_message_from_vector(PCIDevice *dev, unsigned vector,
 117                                         KVMMsiMessage *kmm)
 118 {
 119     uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
 120     bool msi64bit = flags & PCI_MSI_FLAGS_64BIT;
 121     unsigned int nr_vectors = msi_nr_vectors(flags);
 122
 123     kmm->addr_lo = pci_get_long(dev->config + msi_address_lo_off(dev));
 124     if (msi64bit) {
 125         kmm->addr_hi = pci_get_long(dev->config + msi_address_hi_off(dev));
 126     } else {
 127         kmm->addr_hi = 0;
 128     }
 129
 130     kmm->data = pci_get_word(dev->config + msi_data_off(dev, msi64bit));
 131     if (nr_vectors > 1) {
 132         kmm->data &= ~(nr_vectors - 1);
 133         kmm->data |= vector;
 134     }
 135 }
 136
 137 static void kvm_msi_update(PCIDevice *dev)
 138 {
 139     uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
 140     unsigned int max_vectors = 1 <<
 141         ((flags & PCI_MSI_FLAGS_QMASK) >> (ffs(PCI_MSI_FLAGS_QMASK) - 1));
 142     unsigned int nr_vectors = msi_nr_vectors(flags);
 143     KVMMsiMessage new_entry, *entry;
 144     bool changed = false;
 145     unsigned int vector;
 146     int r;
 147
 148     for (vector = 0; vector < max_vectors; vector++) {
 149         entry = dev->msi_irq_entries + vector;
 150
 151         if (vector >= nr_vectors) {
 152             if (vector < dev->msi_entries_nr) {
 153                 kvm_msi_message_del(entry);
 154                 changed = true;
 155             }
 156         } else if (vector >= dev->msi_entries_nr) {
 157             kvm_msi_message_from_vector(dev, vector, entry);
 158             r = kvm_msi_message_add(entry);
 159             if (r) {
 160                 fprintf(stderr, "%s: kvm_msi_add failed: %s\n", __func__,
 161                         strerror(-r));
 162                 exit(1);
 163             }
 164             changed = true;
 165         } else {
 166             kvm_msi_message_from_vector(dev, vector, &new_entry);
 167             r = kvm_msi_message_update(entry, &new_entry);
 168             if (r < 0) {
 169                 fprintf(stderr, "%s: kvm_update_msi failed: %s\n",
 170                         __func__, strerror(-r));
 171                 exit(1);
 172             }
 173             if (r > 0) {
 174                 *entry = new_entry;
 175                 changed = true;
 176             }
 177         }
 178     }
 179     dev->msi_entries_nr = nr_vectors;
 180     if (changed) {
 181         r = kvm_commit_irq_routes();
 182         if (r) {
 183             fprintf(stderr, "%s: kvm_commit_irq_routes failed: %s\n", __func__,
 184                     strerror(-r));
 185             exit(1);
 186         }
 187     }
 188 }
 189
 190 /* KVM specific MSI helpers */
 191 static void kvm_msi_free(PCIDevice *dev)
 192 {
 193     unsigned int vector;
 194
 195     for (vector = 0; vector < dev->msi_entries_nr; ++vector) {
 196         kvm_msi_message_del(&dev->msi_irq_entries[vector]);
 197     }
 198     if (dev->msi_entries_nr > 0) {
 199         kvm_commit_irq_routes();
 200     }
 201     dev->msi_entries_nr = 0;
 202 }
 203
 204 int msi_init(struct PCIDevice *dev, uint8_t offset,
 205              unsigned int nr_vectors, bool msi64bit, bool msi_per_vector_mask)
 206 {
 207     unsigned int vectors_order;
 208     uint16_t flags;
 209     uint8_t cap_size;
 210     int config_offset;
 211
 212     if (!msi_supported) {
 213         return -ENOTSUP;
 214     }
 215
 216     MSI_DEV_PRINTF(dev,
 217                    "init offset: 0x%"PRIx8" vector: %"PRId8
 218                    " 64bit %d mask %d\n",
 219                    offset, nr_vectors, msi64bit, msi_per_vector_mask);
 220
 221     if (kvm_enabled() && kvm_irqchip_in_kernel() && !kvm_has_gsi_routing()) {
 222         return -ENOTSUP;
 223     }
 224
 225     assert(!(nr_vectors & (nr_vectors - 1)));   /* power of 2 */
 226     assert(nr_vectors > 0);
 227     assert(nr_vectors <= PCI_MSI_VECTORS_MAX);
 228     /* the nr of MSI vectors is up to 32 */
 229     vectors_order = ffs(nr_vectors) - 1;
 230
 231     flags = vectors_order << (ffs(PCI_MSI_FLAGS_QMASK) - 1);
 232     if (msi64bit) {
 233         flags |= PCI_MSI_FLAGS_64BIT;
 234     }
 235     if (msi_per_vector_mask) {
 236         flags |= PCI_MSI_FLAGS_MASKBIT;
 237     }
 238
 239     cap_size = msi_cap_sizeof(flags);
 240     config_offset = pci_add_capability(dev, PCI_CAP_ID_MSI, offset, cap_size);
 241     if (config_offset < 0) {
 242         return config_offset;
 243     }
 244
 245     dev->msi_cap = config_offset;
 246     dev->cap_present |= QEMU_PCI_CAP_MSI;
 247
 248     pci_set_word(dev->config + msi_flags_off(dev), flags);
 249     pci_set_word(dev->wmask + msi_flags_off(dev),
 250                  PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE);
 251     pci_set_long(dev->wmask + msi_address_lo_off(dev),
 252                  PCI_MSI_ADDRESS_LO_MASK);
 253     if (msi64bit) {
 254         pci_set_long(dev->wmask + msi_address_hi_off(dev), 0xffffffff);
 255     }
 256     pci_set_word(dev->wmask + msi_data_off(dev, msi64bit), 0xffff);
 257
 258     if (msi_per_vector_mask) {
 259         /* Make mask bits 0 to nr_vectors - 1 writable. */
 260         pci_set_long(dev->wmask + msi_mask_off(dev, msi64bit),
 261                      0xffffffff >> (PCI_MSI_VECTORS_MAX - nr_vectors));
 262     }
 263
 264     if (kvm_enabled() && kvm_irqchip_in_kernel()) {
 265         dev->msi_irq_entries = g_malloc(nr_vectors *
 266                                         sizeof(*dev->msix_irq_entries));
 267     }
 268
 269     return config_offset;
 270 }
 271
 272 void msi_uninit(struct PCIDevice *dev)
 273 {
 274     uint16_t flags;
 275     uint8_t cap_size;
 276
 277     if (!(dev->cap_present & QEMU_PCI_CAP_MSI)) {
 278         return;
 279     }
 280     flags = pci_get_word(dev->config + msi_flags_off(dev));
 281     cap_size = msi_cap_sizeof(flags);
 282
 283     if (kvm_enabled() && kvm_irqchip_in_kernel()) {
 284         kvm_msi_free(dev);
 285         g_free(dev->msi_irq_entries);
 286     }
 287
 288     pci_del_capability(dev, PCI_CAP_ID_MSI, cap_size);
 289     dev->cap_present &= ~QEMU_PCI_CAP_MSI;
 290
 291     MSI_DEV_PRINTF(dev, "uninit\n");
 292 }
 293
 294 void msi_reset(PCIDevice *dev)
 295 {
 296     uint16_t flags;
 297     bool msi64bit;
 298
 299     if (kvm_enabled() && kvm_irqchip_in_kernel()) {
 300         kvm_msi_free(dev);
 301     }
 302
 303     flags = pci_get_word(dev->config + msi_flags_off(dev));
 304     flags &= ~(PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE);
 305     msi64bit = flags & PCI_MSI_FLAGS_64BIT;
 306
 307     pci_set_word(dev->config + msi_flags_off(dev), flags);
 308     pci_set_long(dev->config + msi_address_lo_off(dev), 0);
 309     if (msi64bit) {
 310         pci_set_long(dev->config + msi_address_hi_off(dev), 0);
 311     }
 312     pci_set_word(dev->config + msi_data_off(dev, msi64bit), 0);
 313     if (flags & PCI_MSI_FLAGS_MASKBIT) {
 314         pci_set_long(dev->config + msi_mask_off(dev, msi64bit), 0);
 315         pci_set_long(dev->config + msi_pending_off(dev, msi64bit), 0);
 316     }
 317     MSI_DEV_PRINTF(dev, "reset\n");
 318 }
 319
 320 static bool msi_is_masked(const PCIDevice *dev, unsigned int vector)
 321 {
 322     uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
 323     uint32_t mask;
 324     assert(vector < PCI_MSI_VECTORS_MAX);
 325
 326     if (!(flags & PCI_MSI_FLAGS_MASKBIT)) {
 327         return false;
 328     }
 329
 330     mask = pci_get_long(dev->config +
 331                         msi_mask_off(dev, flags & PCI_MSI_FLAGS_64BIT));
 332     return mask & (1U << vector);
 333 }
 334
 335 void msi_notify(PCIDevice *dev, unsigned int vector)
 336 {
 337     uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
 338     bool msi64bit = flags & PCI_MSI_FLAGS_64BIT;
 339     unsigned int nr_vectors = msi_nr_vectors(flags);
 340     uint64_t address;
 341     uint32_t data;
 342
 343     assert(vector < nr_vectors);
 344     if (msi_is_masked(dev, vector)) {
 345         assert(flags & PCI_MSI_FLAGS_MASKBIT);
 346         pci_long_test_and_set_mask(
 347             dev->config + msi_pending_off(dev, msi64bit), 1U << vector);
 348         MSI_DEV_PRINTF(dev, "pending vector 0x%x\n", vector);
 349         return;
 350     }
 351
 352     if (kvm_enabled() && kvm_irqchip_in_kernel()) {
 353         kvm_set_irq(dev->msi_irq_entries[vector].gsi, 1, NULL);
 354         return;
 355     }
 356
 357     if (msi64bit) {
 358         address = pci_get_quad(dev->config + msi_address_lo_off(dev));
 359     } else {
 360         address = pci_get_long(dev->config + msi_address_lo_off(dev));
 361     }
 362
 363     /* upper bit 31:16 is zero */
 364     data = pci_get_word(dev->config + msi_data_off(dev, msi64bit));
 365     if (nr_vectors > 1) {
 366         data &= ~(nr_vectors - 1);
 367         data |= vector;
 368     }
 369
 370     MSI_DEV_PRINTF(dev,
 371                    "notify vector 0x%x"
 372                    " address: 0x%"PRIx64" data: 0x%"PRIx32"\n",
 373                    vector, address, data);
 374     stl_le_phys(address, data);
 375 }
 376
 377 /* call this function after updating configs by pci_default_write_config(). */
 378 void msi_write_config(PCIDevice *dev, uint32_t addr, uint32_t val, int len)
 379 {
 380     uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
 381     bool msi64bit = flags & PCI_MSI_FLAGS_64BIT;
 382     bool msi_per_vector_mask = flags & PCI_MSI_FLAGS_MASKBIT;
 383     unsigned int nr_vectors;
 384     uint8_t log_num_vecs;
 385     uint8_t log_max_vecs;
 386     unsigned int vector;
 387     uint32_t pending;
 388
 389     if (!ranges_overlap(addr, len, dev->msi_cap, msi_cap_sizeof(flags))) {
 390         return;
 391     }
 392
 393 #ifdef MSI_DEBUG
 394     MSI_DEV_PRINTF(dev, "addr 0x%"PRIx32" val 0x%"PRIx32" len %d\n",
 395                    addr, val, len);
 396     MSI_DEV_PRINTF(dev, "ctrl: 0x%"PRIx16" address: 0x%"PRIx32,
 397                    flags,
 398                    pci_get_long(dev->config + msi_address_lo_off(dev)));
 399     if (msi64bit) {
 400         fprintf(stderr, " address-hi: 0x%"PRIx32,
 401                 pci_get_long(dev->config + msi_address_hi_off(dev)));
 402     }
 403     fprintf(stderr, " data: 0x%"PRIx16,
 404             pci_get_word(dev->config + msi_data_off(dev, msi64bit)));
 405     if (flags & PCI_MSI_FLAGS_MASKBIT) {
 406         fprintf(stderr, " mask 0x%"PRIx32" pending 0x%"PRIx32,
 407                 pci_get_long(dev->config + msi_mask_off(dev, msi64bit)),
 408                 pci_get_long(dev->config + msi_pending_off(dev, msi64bit)));
 409     }
 410     fprintf(stderr, "\n");
 411 #endif
 412
 413     if (!(flags & PCI_MSI_FLAGS_ENABLE)) {
 414         return;
 415     }
 416
 417     /*
 418      * Now MSI is enabled, clear INTx# interrupts.
 419      * the driver is prohibited from writing enable bit to mask
 420      * a service request. But the guest OS could do this.
 421      * So we just discard the interrupts as moderate fallback.
 422      *
 423      * 6.8.3.3. Enabling Operation
 424      *   While enabled for MSI or MSI-X operation, a function is prohibited
 425      *   from using its INTx# pin (if implemented) to request
 426      *   service (MSI, MSI-X, and INTx# are mutually exclusive).
 427      */
 428     pci_device_deassert_intx(dev);
 429
 430     /*
 431      * nr_vectors might be set bigger than capable. So clamp it.
 432      * This is not legal by spec, so we can do anything we like,
 433      * just don't crash the host
 434      */
 435     log_num_vecs =
 436         (flags & PCI_MSI_FLAGS_QSIZE) >> (ffs(PCI_MSI_FLAGS_QSIZE) - 1);
 437     log_max_vecs =
 438         (flags & PCI_MSI_FLAGS_QMASK) >> (ffs(PCI_MSI_FLAGS_QMASK) - 1);
 439     if (log_num_vecs > log_max_vecs) {
 440         flags &= ~PCI_MSI_FLAGS_QSIZE;
 441         flags |= log_max_vecs << (ffs(PCI_MSI_FLAGS_QSIZE) - 1);
 442         pci_set_word(dev->config + msi_flags_off(dev), flags);
 443     }
 444
 445     if (kvm_enabled() && kvm_irqchip_in_kernel()) {
 446         kvm_msi_update(dev);
 447     }
 448
 449     if (!msi_per_vector_mask) {
 450         /* if per vector masking isn't supported,
 451            there is no pending interrupt. */
 452         return;
 453     }
 454
 455     nr_vectors = msi_nr_vectors(flags);
 456
 457     /* This will discard pending interrupts, if any. */
 458     pending = pci_get_long(dev->config + msi_pending_off(dev, msi64bit));
 459     pending &= 0xffffffff >> (PCI_MSI_VECTORS_MAX - nr_vectors);
 460     pci_set_long(dev->config + msi_pending_off(dev, msi64bit), pending);
 461
 462     /* deliver pending interrupts which are unmasked */
 463     for (vector = 0; vector < nr_vectors; ++vector) {
 464         if (msi_is_masked(dev, vector) || !(pending & (1U << vector))) {
 465             continue;
 466         }
 467
 468         pci_long_test_and_clear_mask(
 469             dev->config + msi_pending_off(dev, msi64bit), 1U << vector);
 470         msi_notify(dev, vector);
 471     }
 472 }
 473
 474 unsigned int msi_nr_vectors_allocated(const PCIDevice *dev)
 475 {
 476     uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
 477     return msi_nr_vectors(flags);
 478 }
 479
 480 void msi_post_load(PCIDevice *dev)
 481 {
 482     uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
 483
 484     if (kvm_enabled() && dev->msi_irq_entries) {
 485         kvm_msi_free(dev);
 486
 487         if (flags & PCI_MSI_FLAGS_ENABLE) {
 488             kvm_msi_update(dev);
 489         }
 490     }
 491 }