hw/virtio-pci.c

   1 /*
   2  * Virtio PCI Bindings
   3  *
   4  * Copyright IBM, Corp. 2007
   5  * Copyright (c) 2009 CodeSourcery
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Paul Brook        <paul@codesourcery.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2.  See
  12  * the COPYING file in the top-level directory.
  13  *
  14  * Contributions after 2012-01-13 are licensed under the terms of the
  15  * GNU GPL, version 2 or (at your option) any later version.
  16  */
  17
  18 #include <inttypes.h>
  19
  20 #include "virtio.h"
  21 #include "virtio-blk.h"
  22 #include "virtio-net.h"
  23 #include "virtio-serial.h"
  24 #include "virtio-scsi.h"
  25 #include "pci.h"
  26 #include "qemu-error.h"
  27 #include "msi.h"
  28 #include "msix.h"
  29 #include "net.h"
  30 #include "loader.h"
  31 #include "kvm.h"
  32 #include "blockdev.h"
  33 #include "virtio-pci.h"
  34 #include "range.h"
  35
  36 /* from Linux's linux/virtio_pci.h */
  37
  38 /* A 32-bit r/o bitmask of the features supported by the host */
  39 #define VIRTIO_PCI_HOST_FEATURES        0
  40
  41 /* A 32-bit r/w bitmask of features activated by the guest */
  42 #define VIRTIO_PCI_GUEST_FEATURES       4
  43
  44 /* A 32-bit r/w PFN for the currently selected queue */
  45 #define VIRTIO_PCI_QUEUE_PFN            8
  46
  47 /* A 16-bit r/o queue size for the currently selected queue */
  48 #define VIRTIO_PCI_QUEUE_NUM            12
  49
  50 /* A 16-bit r/w queue selector */
  51 #define VIRTIO_PCI_QUEUE_SEL            14
  52
  53 /* A 16-bit r/w queue notifier */
  54 #define VIRTIO_PCI_QUEUE_NOTIFY         16
  55
  56 /* An 8-bit device status register.  */
  57 #define VIRTIO_PCI_STATUS               18
  58
  59 /* An 8-bit r/o interrupt status register.  Reading the value will return the
  60  * current contents of the ISR and will also clear it.  This is effectively
  61  * a read-and-acknowledge. */
  62 #define VIRTIO_PCI_ISR                  19
  63
  64 /* MSI-X registers: only enabled if MSI-X is enabled. */
  65 /* A 16-bit vector for configuration changes. */
  66 #define VIRTIO_MSI_CONFIG_VECTOR        20
  67 /* A 16-bit vector for selected queue notifications. */
  68 #define VIRTIO_MSI_QUEUE_VECTOR         22
  69
  70 /* Config space size */
  71 #define VIRTIO_PCI_CONFIG_NOMSI         20
  72 #define VIRTIO_PCI_CONFIG_MSI           24
  73 #define VIRTIO_PCI_REGION_SIZE(dev)     (msix_present(dev) ? \
  74                                          VIRTIO_PCI_CONFIG_MSI : \
  75                                          VIRTIO_PCI_CONFIG_NOMSI)
  76
  77 /* The remaining space is defined by each driver as the per-driver
  78  * configuration space */
  79 #define VIRTIO_PCI_CONFIG(dev)          (msix_enabled(dev) ? \
  80                                          VIRTIO_PCI_CONFIG_MSI : \
  81                                          VIRTIO_PCI_CONFIG_NOMSI)
  82
  83 /* How many bits to shift physical queue address written to QUEUE_PFN.
  84  * 12 is historical, and due to x86 page size. */
  85 #define VIRTIO_PCI_QUEUE_ADDR_SHIFT    12
  86
  87 /* Flags track per-device state like workarounds for quirks in older guests. */
  88 #define VIRTIO_PCI_FLAG_BUS_MASTER_BUG  (1 << 0)
  89
  90 /* QEMU doesn't strictly need write barriers since everything runs in
  91  * lock-step.  We'll leave the calls to wmb() in though to make it obvious for
  92  * KVM or if kqemu gets SMP support.
  93  */
  94 #define wmb() do { } while (0)
  95
  96 /* HACK for virtio to determine if it's running a big endian guest */
  97 bool virtio_is_big_endian(void);
  98
  99 /* virtio device */
 100
 101 static void virtio_pci_notify(void *opaque, uint16_t vector)
 102 {
 103     VirtIOPCIProxy *proxy = opaque;
 104     if (msix_enabled(&proxy->pci_dev))
 105         msix_notify(&proxy->pci_dev, vector);
 106     else
 107         qemu_set_irq(proxy->pci_dev.irq[0], proxy->vdev->isr & 1);
 108 }
 109
 110 static void virtio_pci_save_config(void * opaque, QEMUFile *f)
 111 {
 112     VirtIOPCIProxy *proxy = opaque;
 113     pci_device_save(&proxy->pci_dev, f);
 114     msix_save(&proxy->pci_dev, f);
 115     if (msix_present(&proxy->pci_dev))
 116         qemu_put_be16(f, proxy->vdev->config_vector);
 117 }
 118
 119 static void virtio_pci_save_queue(void * opaque, int n, QEMUFile *f)
 120 {
 121     VirtIOPCIProxy *proxy = opaque;
 122     if (msix_present(&proxy->pci_dev))
 123         qemu_put_be16(f, virtio_queue_vector(proxy->vdev, n));
 124 }
 125
 126 static int virtio_pci_load_config(void * opaque, QEMUFile *f)
 127 {
 128     VirtIOPCIProxy *proxy = opaque;
 129     int ret;
 130     ret = pci_device_load(&proxy->pci_dev, f);
 131     if (ret) {
 132         return ret;
 133     }
 134     msix_load(&proxy->pci_dev, f);
 135     if (msix_present(&proxy->pci_dev)) {
 136         qemu_get_be16s(f, &proxy->vdev->config_vector);
 137     } else {
 138         proxy->vdev->config_vector = VIRTIO_NO_VECTOR;
 139     }
 140     if (proxy->vdev->config_vector != VIRTIO_NO_VECTOR) {
 141         return msix_vector_use(&proxy->pci_dev, proxy->vdev->config_vector);
 142     }
 143     return 0;
 144 }
 145
 146 static int virtio_pci_load_queue(void * opaque, int n, QEMUFile *f)
 147 {
 148     VirtIOPCIProxy *proxy = opaque;
 149     uint16_t vector;
 150     if (msix_present(&proxy->pci_dev)) {
 151         qemu_get_be16s(f, &vector);
 152     } else {
 153         vector = VIRTIO_NO_VECTOR;
 154     }
 155     virtio_queue_set_vector(proxy->vdev, n, vector);
 156     if (vector != VIRTIO_NO_VECTOR) {
 157         return msix_vector_use(&proxy->pci_dev, vector);
 158     }
 159     return 0;
 160 }
 161
 162 static int virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy,
 163                                                  int n, bool assign)
 164 {
 165     VirtQueue *vq = virtio_get_queue(proxy->vdev, n);
 166     EventNotifier *notifier = virtio_queue_get_host_notifier(vq);
 167     int r = 0;
 168
 169     if (assign) {
 170         r = event_notifier_init(notifier, 1);
 171         if (r < 0) {
 172             error_report("%s: unable to init event notifier: %d",
 173                          __func__, r);
 174             return r;
 175         }
 176         virtio_queue_set_host_notifier_fd_handler(vq, true);
 177         memory_region_add_eventfd(&proxy->bar, VIRTIO_PCI_QUEUE_NOTIFY, 2,
 178                                   true, n, notifier);
 179     } else {
 180         memory_region_del_eventfd(&proxy->bar, VIRTIO_PCI_QUEUE_NOTIFY, 2,
 181                                   true, n, notifier);
 182         virtio_queue_set_host_notifier_fd_handler(vq, false);
 183         event_notifier_cleanup(notifier);
 184     }
 185     return r;
 186 }
 187
 188 static void virtio_pci_start_ioeventfd(VirtIOPCIProxy *proxy)
 189 {
 190     int n, r;
 191
 192     if (!(proxy->flags & VIRTIO_PCI_FLAG_USE_IOEVENTFD) ||
 193         proxy->ioeventfd_disabled ||
 194         proxy->ioeventfd_started) {
 195         return;
 196     }
 197
 198     for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
 199         if (!virtio_queue_get_num(proxy->vdev, n)) {
 200             continue;
 201         }
 202
 203         r = virtio_pci_set_host_notifier_internal(proxy, n, true);
 204         if (r < 0) {
 205             goto assign_error;
 206         }
 207     }
 208     proxy->ioeventfd_started = true;
 209     return;
 210
 211 assign_error:
 212     while (--n >= 0) {
 213         if (!virtio_queue_get_num(proxy->vdev, n)) {
 214             continue;
 215         }
 216
 217         r = virtio_pci_set_host_notifier_internal(proxy, n, false);
 218         assert(r >= 0);
 219     }
 220     proxy->ioeventfd_started = false;
 221     error_report("%s: failed. Fallback to a userspace (slower).", __func__);
 222 }
 223
 224 static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy)
 225 {
 226     int r;
 227     int n;
 228
 229     if (!proxy->ioeventfd_started) {
 230         return;
 231     }
 232
 233     for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
 234         if (!virtio_queue_get_num(proxy->vdev, n)) {
 235             continue;
 236         }
 237
 238         r = virtio_pci_set_host_notifier_internal(proxy, n, false);
 239         assert(r >= 0);
 240     }
 241     proxy->ioeventfd_started = false;
 242 }
 243
 244 void virtio_pci_reset(DeviceState *d)
 245 {
 246     VirtIOPCIProxy *proxy = container_of(d, VirtIOPCIProxy, pci_dev.qdev);
 247     virtio_pci_stop_ioeventfd(proxy);
 248     virtio_reset(proxy->vdev);
 249     proxy->flags &= ~VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
 250 }
 251
 252 static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
 253 {
 254     VirtIOPCIProxy *proxy = opaque;
 255     VirtIODevice *vdev = proxy->vdev;
 256     target_phys_addr_t pa;
 257
 258     switch (addr) {
 259     case VIRTIO_PCI_GUEST_FEATURES:
 260         /* Guest does not negotiate properly?  We have to assume nothing. */
 261         if (val & (1 << VIRTIO_F_BAD_FEATURE)) {
 262             val = vdev->bad_features ? vdev->bad_features(vdev) : 0;
 263         }
 264         virtio_set_features(vdev, val);
 265         break;
 266     case VIRTIO_PCI_QUEUE_PFN:
 267         pa = (target_phys_addr_t)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT;
 268         if (pa == 0) {
 269             virtio_pci_stop_ioeventfd(proxy);
 270             virtio_reset(proxy->vdev);
 271             msix_unuse_all_vectors(&proxy->pci_dev);
 272         }
 273         else
 274             virtio_queue_set_addr(vdev, vdev->queue_sel, pa);
 275         break;
 276     case VIRTIO_PCI_QUEUE_SEL:
 277         if (val < VIRTIO_PCI_QUEUE_MAX)
 278             vdev->queue_sel = val;
 279         break;
 280     case VIRTIO_PCI_QUEUE_NOTIFY:
 281         if (val < VIRTIO_PCI_QUEUE_MAX) {
 282             virtio_queue_notify(vdev, val);
 283         }
 284         break;
 285     case VIRTIO_PCI_STATUS:
 286         if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) {
 287             virtio_pci_stop_ioeventfd(proxy);
 288         }
 289
 290         virtio_set_status(vdev, val & 0xFF);
 291
 292         if (val & VIRTIO_CONFIG_S_DRIVER_OK) {
 293             virtio_pci_start_ioeventfd(proxy);
 294         }
 295
 296         if (vdev->status == 0) {
 297             virtio_reset(proxy->vdev);
 298             msix_unuse_all_vectors(&proxy->pci_dev);
 299         }
 300
 301         /* Linux before 2.6.34 sets the device as OK without enabling
 302            the PCI device bus master bit. In this case we need to disable
 303            some safety checks. */
 304         if ((val & VIRTIO_CONFIG_S_DRIVER_OK) &&
 305             !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
 306             proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
 307         }
 308         break;
 309     case VIRTIO_MSI_CONFIG_VECTOR:
 310         msix_vector_unuse(&proxy->pci_dev, vdev->config_vector);
 311         /* Make it possible for guest to discover an error took place. */
 312         if (msix_vector_use(&proxy->pci_dev, val) < 0)
 313             val = VIRTIO_NO_VECTOR;
 314         vdev->config_vector = val;
 315         break;
 316     case VIRTIO_MSI_QUEUE_VECTOR:
 317         msix_vector_unuse(&proxy->pci_dev,
 318                           virtio_queue_vector(vdev, vdev->queue_sel));
 319         /* Make it possible for guest to discover an error took place. */
 320         if (msix_vector_use(&proxy->pci_dev, val) < 0)
 321             val = VIRTIO_NO_VECTOR;
 322         virtio_queue_set_vector(vdev, vdev->queue_sel, val);
 323         break;
 324     default:
 325         error_report("%s: unexpected address 0x%x value 0x%x",
 326                      __func__, addr, val);
 327         break;
 328     }
 329 }
 330
 331 static uint32_t virtio_ioport_read(VirtIOPCIProxy *proxy, uint32_t addr)
 332 {
 333     VirtIODevice *vdev = proxy->vdev;
 334     uint32_t ret = 0xFFFFFFFF;
 335
 336     switch (addr) {
 337     case VIRTIO_PCI_HOST_FEATURES:
 338         ret = proxy->host_features;
 339         break;
 340     case VIRTIO_PCI_GUEST_FEATURES:
 341         ret = vdev->guest_features;
 342         break;
 343     case VIRTIO_PCI_QUEUE_PFN:
 344         ret = virtio_queue_get_addr(vdev, vdev->queue_sel)
 345               >> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
 346         break;
 347     case VIRTIO_PCI_QUEUE_NUM:
 348         ret = virtio_queue_get_num(vdev, vdev->queue_sel);
 349         break;
 350     case VIRTIO_PCI_QUEUE_SEL:
 351         ret = vdev->queue_sel;
 352         break;
 353     case VIRTIO_PCI_STATUS:
 354         ret = vdev->status;
 355         break;
 356     case VIRTIO_PCI_ISR:
 357         /* reading from the ISR also clears it. */
 358         ret = vdev->isr;
 359         vdev->isr = 0;
 360         qemu_set_irq(proxy->pci_dev.irq[0], 0);
 361         break;
 362     case VIRTIO_MSI_CONFIG_VECTOR:
 363         ret = vdev->config_vector;
 364         break;
 365     case VIRTIO_MSI_QUEUE_VECTOR:
 366         ret = virtio_queue_vector(vdev, vdev->queue_sel);
 367         break;
 368     default:
 369         break;
 370     }
 371
 372     return ret;
 373 }
 374
 375 static uint32_t virtio_pci_config_readb(void *opaque, uint32_t addr)
 376 {
 377     VirtIOPCIProxy *proxy = opaque;
 378     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 379     if (addr < config)
 380         return virtio_ioport_read(proxy, addr);
 381     addr -= config;
 382     return virtio_config_readb(proxy->vdev, addr);
 383 }
 384
 385 static uint32_t virtio_pci_config_readw(void *opaque, uint32_t addr)
 386 {
 387     VirtIOPCIProxy *proxy = opaque;
 388     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 389     uint16_t val;
 390     if (addr < config)
 391         return virtio_ioport_read(proxy, addr);
 392     addr -= config;
 393     val = virtio_config_readw(proxy->vdev, addr);
 394     if (virtio_is_big_endian()) {
 395         /*
 396          * virtio is odd, ioports are LE but config space is target native
 397          * endian. However, in qemu, all PIO is LE, so we need to re-swap
 398          * on BE targets
 399          */
 400         val = bswap16(val);
 401     }
 402     return val;
 403 }
 404
 405 static uint32_t virtio_pci_config_readl(void *opaque, uint32_t addr)
 406 {
 407     VirtIOPCIProxy *proxy = opaque;
 408     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 409     uint32_t val;
 410     if (addr < config)
 411         return virtio_ioport_read(proxy, addr);
 412     addr -= config;
 413     val = virtio_config_readl(proxy->vdev, addr);
 414     if (virtio_is_big_endian()) {
 415         val = bswap32(val);
 416     }
 417     return val;
 418 }
 419
 420 static void virtio_pci_config_writeb(void *opaque, uint32_t addr, uint32_t val)
 421 {
 422     VirtIOPCIProxy *proxy = opaque;
 423     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 424     if (addr < config) {
 425         virtio_ioport_write(proxy, addr, val);
 426         return;
 427     }
 428     addr -= config;
 429     virtio_config_writeb(proxy->vdev, addr, val);
 430 }
 431
 432 static void virtio_pci_config_writew(void *opaque, uint32_t addr, uint32_t val)
 433 {
 434     VirtIOPCIProxy *proxy = opaque;
 435     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 436     if (addr < config) {
 437         virtio_ioport_write(proxy, addr, val);
 438         return;
 439     }
 440     addr -= config;
 441     if (virtio_is_big_endian()) {
 442         val = bswap16(val);
 443     }
 444     virtio_config_writew(proxy->vdev, addr, val);
 445 }
 446
 447 static void virtio_pci_config_writel(void *opaque, uint32_t addr, uint32_t val)
 448 {
 449     VirtIOPCIProxy *proxy = opaque;
 450     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 451     if (addr < config) {
 452         virtio_ioport_write(proxy, addr, val);
 453         return;
 454     }
 455     addr -= config;
 456     if (virtio_is_big_endian()) {
 457         val = bswap32(val);
 458     }
 459     virtio_config_writel(proxy->vdev, addr, val);
 460 }
 461
 462 static const MemoryRegionPortio virtio_portio[] = {
 463     { 0, 0x10000, 1, .write = virtio_pci_config_writeb, },
 464     { 0, 0x10000, 2, .write = virtio_pci_config_writew, },
 465     { 0, 0x10000, 4, .write = virtio_pci_config_writel, },
 466     { 0, 0x10000, 1, .read = virtio_pci_config_readb, },
 467     { 0, 0x10000, 2, .read = virtio_pci_config_readw, },
 468     { 0, 0x10000, 4, .read = virtio_pci_config_readl, },
 469     PORTIO_END_OF_LIST()
 470 };
 471
 472 static const MemoryRegionOps virtio_pci_config_ops = {
 473     .old_portio = virtio_portio,
 474     .endianness = DEVICE_LITTLE_ENDIAN,
 475 };
 476
 477 static void virtio_write_config(PCIDevice *pci_dev, uint32_t address,
 478                                 uint32_t val, int len)
 479 {
 480     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 481
 482     pci_default_write_config(pci_dev, address, val, len);
 483
 484     if (range_covers_byte(address, len, PCI_COMMAND) &&
 485         !(pci_dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER) &&
 486         !(proxy->flags & VIRTIO_PCI_FLAG_BUS_MASTER_BUG)) {
 487         virtio_pci_stop_ioeventfd(proxy);
 488         virtio_set_status(proxy->vdev,
 489                           proxy->vdev->status & ~VIRTIO_CONFIG_S_DRIVER_OK);
 490     }
 491 }
 492
 493 static unsigned virtio_pci_get_features(void *opaque)
 494 {
 495     VirtIOPCIProxy *proxy = opaque;
 496     return proxy->host_features;
 497 }
 498
 499 static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy,
 500                                         unsigned int queue_no,
 501                                         unsigned int vector,
 502                                         MSIMessage msg)
 503 {
 504     VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no);
 505     EventNotifier *n = virtio_queue_get_guest_notifier(vq);
 506     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
 507     int ret;
 508
 509     if (irqfd->users == 0) {
 510         ret = kvm_irqchip_add_msi_route(kvm_state, msg);
 511         if (ret < 0) {
 512             return ret;
 513         }
 514         irqfd->virq = ret;
 515     }
 516     irqfd->users++;
 517
 518     ret = kvm_irqchip_add_irq_notifier(kvm_state, n, irqfd->virq);
 519     if (ret < 0) {
 520         if (--irqfd->users == 0) {
 521             kvm_irqchip_release_virq(kvm_state, irqfd->virq);
 522         }
 523         return ret;
 524     }
 525
 526     virtio_queue_set_guest_notifier_fd_handler(vq, true, true);
 527     return 0;
 528 }
 529
 530 static void kvm_virtio_pci_vq_vector_release(VirtIOPCIProxy *proxy,
 531                                              unsigned int queue_no,
 532                                              unsigned int vector)
 533 {
 534     VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no);
 535     EventNotifier *n = virtio_queue_get_guest_notifier(vq);
 536     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
 537     int ret;
 538
 539     ret = kvm_irqchip_remove_irq_notifier(kvm_state, n, irqfd->virq);
 540     assert(ret == 0);
 541
 542     if (--irqfd->users == 0) {
 543         kvm_irqchip_release_virq(kvm_state, irqfd->virq);
 544     }
 545
 546     virtio_queue_set_guest_notifier_fd_handler(vq, true, false);
 547 }
 548
 549 static int kvm_virtio_pci_vector_use(PCIDevice *dev, unsigned vector,
 550                                      MSIMessage msg)
 551 {
 552     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
 553     VirtIODevice *vdev = proxy->vdev;
 554     int ret, queue_no;
 555
 556     for (queue_no = 0; queue_no < VIRTIO_PCI_QUEUE_MAX; queue_no++) {
 557         if (!virtio_queue_get_num(vdev, queue_no)) {
 558             break;
 559         }
 560         if (virtio_queue_vector(vdev, queue_no) != vector) {
 561             continue;
 562         }
 563         ret = kvm_virtio_pci_vq_vector_use(proxy, queue_no, vector, msg);
 564         if (ret < 0) {
 565             goto undo;
 566         }
 567     }
 568     return 0;
 569
 570 undo:
 571     while (--queue_no >= 0) {
 572         if (virtio_queue_vector(vdev, queue_no) != vector) {
 573             continue;
 574         }
 575         kvm_virtio_pci_vq_vector_release(proxy, queue_no, vector);
 576     }
 577     return ret;
 578 }
 579
 580 static void kvm_virtio_pci_vector_release(PCIDevice *dev, unsigned vector)
 581 {
 582     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
 583     VirtIODevice *vdev = proxy->vdev;
 584     int queue_no;
 585
 586     for (queue_no = 0; queue_no < VIRTIO_PCI_QUEUE_MAX; queue_no++) {
 587         if (!virtio_queue_get_num(vdev, queue_no)) {
 588             break;
 589         }
 590         if (virtio_queue_vector(vdev, queue_no) != vector) {
 591             continue;
 592         }
 593         kvm_virtio_pci_vq_vector_release(proxy, queue_no, vector);
 594     }
 595 }
 596
 597 static int virtio_pci_set_guest_notifier(void *opaque, int n, bool assign)
 598 {
 599     VirtIOPCIProxy *proxy = opaque;
 600     VirtQueue *vq = virtio_get_queue(proxy->vdev, n);
 601     EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
 602
 603     if (assign) {
 604         int r = event_notifier_init(notifier, 0);
 605         if (r < 0) {
 606             return r;
 607         }
 608         virtio_queue_set_guest_notifier_fd_handler(vq, true, false);
 609     } else {
 610         virtio_queue_set_guest_notifier_fd_handler(vq, false, false);
 611         event_notifier_cleanup(notifier);
 612     }
 613
 614     return 0;
 615 }
 616
 617 static bool virtio_pci_query_guest_notifiers(void *opaque)
 618 {
 619     VirtIOPCIProxy *proxy = opaque;
 620     return msix_enabled(&proxy->pci_dev);
 621 }
 622
 623 static int virtio_pci_set_guest_notifiers(void *opaque, bool assign)
 624 {
 625     VirtIOPCIProxy *proxy = opaque;
 626     VirtIODevice *vdev = proxy->vdev;
 627     int r, n;
 628
 629     /* Must unset vector notifier while guest notifier is still assigned */
 630     if (kvm_irqchip_in_kernel() && !assign) {
 631         msix_unset_vector_notifiers(&proxy->pci_dev);
 632         g_free(proxy->vector_irqfd);
 633         proxy->vector_irqfd = NULL;
 634     }
 635
 636     for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
 637         if (!virtio_queue_get_num(vdev, n)) {
 638             break;
 639         }
 640
 641         r = virtio_pci_set_guest_notifier(opaque, n, assign);
 642         if (r < 0) {
 643             goto assign_error;
 644         }
 645     }
 646
 647     /* Must set vector notifier after guest notifier has been assigned */
 648     if (kvm_irqchip_in_kernel() && assign) {
 649         proxy->vector_irqfd =
 650             g_malloc0(sizeof(*proxy->vector_irqfd) *
 651                       msix_nr_vectors_allocated(&proxy->pci_dev));
 652         r = msix_set_vector_notifiers(&proxy->pci_dev,
 653                                       kvm_virtio_pci_vector_use,
 654                                       kvm_virtio_pci_vector_release);
 655         if (r < 0) {
 656             goto assign_error;
 657         }
 658     }
 659
 660     return 0;
 661
 662 assign_error:
 663     /* We get here on assignment failure. Recover by undoing for VQs 0 .. n. */
 664     assert(assign);
 665     while (--n >= 0) {
 666         virtio_pci_set_guest_notifier(opaque, n, !assign);
 667     }
 668     return r;
 669 }
 670
 671 static int virtio_pci_set_host_notifier(void *opaque, int n, bool assign)
 672 {
 673     VirtIOPCIProxy *proxy = opaque;
 674
 675     /* Stop using ioeventfd for virtqueue kick if the device starts using host
 676      * notifiers.  This makes it easy to avoid stepping on each others' toes.
 677      */
 678     proxy->ioeventfd_disabled = assign;
 679     if (assign) {
 680         virtio_pci_stop_ioeventfd(proxy);
 681     }
 682     /* We don't need to start here: it's not needed because backend
 683      * currently only stops on status change away from ok,
 684      * reset, vmstop and such. If we do add code to start here,
 685      * need to check vmstate, device state etc. */
 686     return virtio_pci_set_host_notifier_internal(proxy, n, assign);
 687 }
 688
 689 static void virtio_pci_vmstate_change(void *opaque, bool running)
 690 {
 691     VirtIOPCIProxy *proxy = opaque;
 692
 693     if (running) {
 694         /* Try to find out if the guest has bus master disabled, but is
 695            in ready state. Then we have a buggy guest OS. */
 696         if ((proxy->vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) &&
 697             !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
 698             proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
 699         }
 700         virtio_pci_start_ioeventfd(proxy);
 701     } else {
 702         virtio_pci_stop_ioeventfd(proxy);
 703     }
 704 }
 705
 706 static const VirtIOBindings virtio_pci_bindings = {
 707     .notify = virtio_pci_notify,
 708     .save_config = virtio_pci_save_config,
 709     .load_config = virtio_pci_load_config,
 710     .save_queue = virtio_pci_save_queue,
 711     .load_queue = virtio_pci_load_queue,
 712     .get_features = virtio_pci_get_features,
 713     .query_guest_notifiers = virtio_pci_query_guest_notifiers,
 714     .set_host_notifier = virtio_pci_set_host_notifier,
 715     .set_guest_notifiers = virtio_pci_set_guest_notifiers,
 716     .vmstate_change = virtio_pci_vmstate_change,
 717 };
 718
 719 void virtio_init_pci(VirtIOPCIProxy *proxy, VirtIODevice *vdev)
 720 {
 721     uint8_t *config;
 722     uint32_t size;
 723
 724     proxy->vdev = vdev;
 725
 726     config = proxy->pci_dev.config;
 727
 728     if (proxy->class_code) {
 729         pci_config_set_class(config, proxy->class_code);
 730     }
 731     pci_set_word(config + PCI_SUBSYSTEM_VENDOR_ID,
 732                  pci_get_word(config + PCI_VENDOR_ID));
 733     pci_set_word(config + PCI_SUBSYSTEM_ID, vdev->device_id);
 734     config[PCI_INTERRUPT_PIN] = 1;
 735
 736     memory_region_init(&proxy->msix_bar, "virtio-msix", 4096);
 737     if (vdev->nvectors && !msix_init(&proxy->pci_dev, vdev->nvectors,
 738                                      &proxy->msix_bar, 1, 0)) {
 739         pci_register_bar(&proxy->pci_dev, 1, PCI_BASE_ADDRESS_SPACE_MEMORY,
 740                          &proxy->msix_bar);
 741     } else
 742         vdev->nvectors = 0;
 743
 744     proxy->pci_dev.config_write = virtio_write_config;
 745
 746     size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev) + vdev->config_len;
 747     if (size & (size-1))
 748         size = 1 << qemu_fls(size);
 749
 750     memory_region_init_io(&proxy->bar, &virtio_pci_config_ops, proxy,
 751                           "virtio-pci", size);
 752     pci_register_bar(&proxy->pci_dev, 0, PCI_BASE_ADDRESS_SPACE_IO,
 753                      &proxy->bar);
 754
 755     if (!kvm_has_many_ioeventfds()) {
 756         proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
 757     }
 758
 759     virtio_bind_device(vdev, &virtio_pci_bindings, proxy);
 760     proxy->host_features |= 0x1 << VIRTIO_F_NOTIFY_ON_EMPTY;
 761     proxy->host_features |= 0x1 << VIRTIO_F_BAD_FEATURE;
 762     proxy->host_features = vdev->get_features(vdev, proxy->host_features);
 763 }
 764
 765 static int virtio_blk_init_pci(PCIDevice *pci_dev)
 766 {
 767     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 768     VirtIODevice *vdev;
 769
 770     if (proxy->class_code != PCI_CLASS_STORAGE_SCSI &&
 771         proxy->class_code != PCI_CLASS_STORAGE_OTHER)
 772         proxy->class_code = PCI_CLASS_STORAGE_SCSI;
 773
 774     vdev = virtio_blk_init(&pci_dev->qdev, &proxy->blk);
 775     if (!vdev) {
 776         return -1;
 777     }
 778     vdev->nvectors = proxy->nvectors;
 779     virtio_init_pci(proxy, vdev);
 780     /* make the actual value visible */
 781     proxy->nvectors = vdev->nvectors;
 782     return 0;
 783 }
 784
 785 static int virtio_exit_pci(PCIDevice *pci_dev)
 786 {
 787     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 788     int r;
 789
 790     memory_region_destroy(&proxy->bar);
 791     r = msix_uninit(pci_dev, &proxy->msix_bar);
 792     memory_region_destroy(&proxy->msix_bar);
 793     return r;
 794 }
 795
 796 static int virtio_blk_exit_pci(PCIDevice *pci_dev)
 797 {
 798     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 799
 800     virtio_pci_stop_ioeventfd(proxy);
 801     virtio_blk_exit(proxy->vdev);
 802     return virtio_exit_pci(pci_dev);
 803 }
 804
 805 static int virtio_serial_init_pci(PCIDevice *pci_dev)
 806 {
 807     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 808     VirtIODevice *vdev;
 809
 810     if (proxy->class_code != PCI_CLASS_COMMUNICATION_OTHER &&
 811         proxy->class_code != PCI_CLASS_DISPLAY_OTHER && /* qemu 0.10 */
 812         proxy->class_code != PCI_CLASS_OTHERS)          /* qemu-kvm  */
 813         proxy->class_code = PCI_CLASS_COMMUNICATION_OTHER;
 814
 815     vdev = virtio_serial_init(&pci_dev->qdev, &proxy->serial);
 816     if (!vdev) {
 817         return -1;
 818     }
 819     vdev->nvectors = proxy->nvectors == DEV_NVECTORS_UNSPECIFIED
 820                                         ? proxy->serial.max_virtserial_ports + 1
 821                                         : proxy->nvectors;
 822     virtio_init_pci(proxy, vdev);
 823     proxy->nvectors = vdev->nvectors;
 824     return 0;
 825 }
 826
 827 static int virtio_serial_exit_pci(PCIDevice *pci_dev)
 828 {
 829     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 830
 831     virtio_pci_stop_ioeventfd(proxy);
 832     virtio_serial_exit(proxy->vdev);
 833     return virtio_exit_pci(pci_dev);
 834 }
 835
 836 static int virtio_net_init_pci(PCIDevice *pci_dev)
 837 {
 838     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 839     VirtIODevice *vdev;
 840
 841     vdev = virtio_net_init(&pci_dev->qdev, &proxy->nic, &proxy->net);
 842
 843     vdev->nvectors = proxy->nvectors;
 844     virtio_init_pci(proxy, vdev);
 845
 846     /* make the actual value visible */
 847     proxy->nvectors = vdev->nvectors;
 848     return 0;
 849 }
 850
 851 static int virtio_net_exit_pci(PCIDevice *pci_dev)
 852 {
 853     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 854
 855     virtio_pci_stop_ioeventfd(proxy);
 856     virtio_net_exit(proxy->vdev);
 857     return virtio_exit_pci(pci_dev);
 858 }
 859
 860 static int virtio_balloon_init_pci(PCIDevice *pci_dev)
 861 {
 862     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 863     VirtIODevice *vdev;
 864
 865     if (proxy->class_code != PCI_CLASS_OTHERS &&
 866         proxy->class_code != PCI_CLASS_MEMORY_RAM) { /* qemu < 1.1 */
 867         proxy->class_code = PCI_CLASS_OTHERS;
 868     }
 869
 870     vdev = virtio_balloon_init(&pci_dev->qdev);
 871     if (!vdev) {
 872         return -1;
 873     }
 874     virtio_init_pci(proxy, vdev);
 875     return 0;
 876 }
 877
 878 static int virtio_balloon_exit_pci(PCIDevice *pci_dev)
 879 {
 880     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 881
 882     virtio_pci_stop_ioeventfd(proxy);
 883     virtio_balloon_exit(proxy->vdev);
 884     return virtio_exit_pci(pci_dev);
 885 }
 886
 887 static Property virtio_blk_properties[] = {
 888     DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
 889     DEFINE_BLOCK_PROPERTIES(VirtIOPCIProxy, blk.conf),
 890     DEFINE_BLOCK_CHS_PROPERTIES(VirtIOPCIProxy, blk.conf),
 891     DEFINE_PROP_STRING("serial", VirtIOPCIProxy, blk.serial),
 892 #ifdef __linux__
 893     DEFINE_PROP_BIT("scsi", VirtIOPCIProxy, blk.scsi, 0, true),
 894 #endif
 895     DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
 896     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 2),
 897     DEFINE_VIRTIO_BLK_FEATURES(VirtIOPCIProxy, host_features),
 898     DEFINE_PROP_END_OF_LIST(),
 899 };
 900
 901 static void virtio_blk_class_init(ObjectClass *klass, void *data)
 902 {
 903     DeviceClass *dc = DEVICE_CLASS(klass);
 904     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 905
 906     k->init = virtio_blk_init_pci;
 907     k->exit = virtio_blk_exit_pci;
 908     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
 909     k->device_id = PCI_DEVICE_ID_VIRTIO_BLOCK;
 910     k->revision = VIRTIO_PCI_ABI_VERSION;
 911     k->class_id = PCI_CLASS_STORAGE_SCSI;
 912     dc->reset = virtio_pci_reset;
 913     dc->props = virtio_blk_properties;
 914 }
 915
 916 static TypeInfo virtio_blk_info = {
 917     .name          = "virtio-blk-pci",
 918     .parent        = TYPE_PCI_DEVICE,
 919     .instance_size = sizeof(VirtIOPCIProxy),
 920     .class_init    = virtio_blk_class_init,
 921 };
 922
 923 static Property virtio_net_properties[] = {
 924     DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, false),
 925     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 3),
 926     DEFINE_VIRTIO_NET_FEATURES(VirtIOPCIProxy, host_features),
 927     DEFINE_NIC_PROPERTIES(VirtIOPCIProxy, nic),
 928     DEFINE_PROP_UINT32("x-txtimer", VirtIOPCIProxy, net.txtimer, TX_TIMER_INTERVAL),
 929     DEFINE_PROP_INT32("x-txburst", VirtIOPCIProxy, net.txburst, TX_BURST),
 930     DEFINE_PROP_STRING("tx", VirtIOPCIProxy, net.tx),
 931     DEFINE_PROP_END_OF_LIST(),
 932 };
 933
 934 static void virtio_net_class_init(ObjectClass *klass, void *data)
 935 {
 936     DeviceClass *dc = DEVICE_CLASS(klass);
 937     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 938
 939     k->init = virtio_net_init_pci;
 940     k->exit = virtio_net_exit_pci;
 941     k->romfile = "pxe-virtio.rom";
 942     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
 943     k->device_id = PCI_DEVICE_ID_VIRTIO_NET;
 944     k->revision = VIRTIO_PCI_ABI_VERSION;
 945     k->class_id = PCI_CLASS_NETWORK_ETHERNET;
 946     dc->reset = virtio_pci_reset;
 947     dc->props = virtio_net_properties;
 948 }
 949
 950 static TypeInfo virtio_net_info = {
 951     .name          = "virtio-net-pci",
 952     .parent        = TYPE_PCI_DEVICE,
 953     .instance_size = sizeof(VirtIOPCIProxy),
 954     .class_init    = virtio_net_class_init,
 955 };
 956
 957 static Property virtio_serial_properties[] = {
 958     DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
 959     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, DEV_NVECTORS_UNSPECIFIED),
 960     DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
 961     DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
 962     DEFINE_PROP_UINT32("max_ports", VirtIOPCIProxy, serial.max_virtserial_ports, 31),
 963     DEFINE_PROP_END_OF_LIST(),
 964 };
 965
 966 static void virtio_serial_class_init(ObjectClass *klass, void *data)
 967 {
 968     DeviceClass *dc = DEVICE_CLASS(klass);
 969     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 970
 971     k->init = virtio_serial_init_pci;
 972     k->exit = virtio_serial_exit_pci;
 973     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
 974     k->device_id = PCI_DEVICE_ID_VIRTIO_CONSOLE;
 975     k->revision = VIRTIO_PCI_ABI_VERSION;
 976     k->class_id = PCI_CLASS_COMMUNICATION_OTHER;
 977     dc->reset = virtio_pci_reset;
 978     dc->props = virtio_serial_properties;
 979 }
 980
 981 static TypeInfo virtio_serial_info = {
 982     .name          = "virtio-serial-pci",
 983     .parent        = TYPE_PCI_DEVICE,
 984     .instance_size = sizeof(VirtIOPCIProxy),
 985     .class_init    = virtio_serial_class_init,
 986 };
 987
 988 static Property virtio_balloon_properties[] = {
 989     DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
 990     DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
 991     DEFINE_PROP_END_OF_LIST(),
 992 };
 993
 994 static void virtio_balloon_class_init(ObjectClass *klass, void *data)
 995 {
 996     DeviceClass *dc = DEVICE_CLASS(klass);
 997     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 998
 999     k->init = virtio_balloon_init_pci;
1000     k->exit = virtio_balloon_exit_pci;
1001     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
1002     k->device_id = PCI_DEVICE_ID_VIRTIO_BALLOON;
1003     k->revision = VIRTIO_PCI_ABI_VERSION;
1004     k->class_id = PCI_CLASS_OTHERS;
1005     dc->reset = virtio_pci_reset;
1006     dc->props = virtio_balloon_properties;
1007 }
1008
1009 static TypeInfo virtio_balloon_info = {
1010     .name          = "virtio-balloon-pci",
1011     .parent        = TYPE_PCI_DEVICE,
1012     .instance_size = sizeof(VirtIOPCIProxy),
1013     .class_init    = virtio_balloon_class_init,
1014 };
1015
1016 static int virtio_scsi_init_pci(PCIDevice *pci_dev)
1017 {
1018     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
1019     VirtIODevice *vdev;
1020
1021     vdev = virtio_scsi_init(&pci_dev->qdev, &proxy->scsi);
1022     if (!vdev) {
1023         return -EINVAL;
1024     }
1025
1026     vdev->nvectors = proxy->nvectors;
1027     virtio_init_pci(proxy, vdev);
1028
1029     /* make the actual value visible */
1030     proxy->nvectors = vdev->nvectors;
1031     return 0;
1032 }
1033
1034 static int virtio_scsi_exit_pci(PCIDevice *pci_dev)
1035 {
1036     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
1037
1038     virtio_scsi_exit(proxy->vdev);
1039     return virtio_exit_pci(pci_dev);
1040 }
1041
1042 static Property virtio_scsi_properties[] = {
1043     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 2),
1044     DEFINE_VIRTIO_SCSI_PROPERTIES(VirtIOPCIProxy, host_features, scsi),
1045     DEFINE_PROP_END_OF_LIST(),
1046 };
1047
1048 static void virtio_scsi_class_init(ObjectClass *klass, void *data)
1049 {
1050     DeviceClass *dc = DEVICE_CLASS(klass);
1051     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
1052
1053     k->init = virtio_scsi_init_pci;
1054     k->exit = virtio_scsi_exit_pci;
1055     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
1056     k->device_id = PCI_DEVICE_ID_VIRTIO_SCSI;
1057     k->revision = 0x00;
1058     k->class_id = PCI_CLASS_STORAGE_SCSI;
1059     dc->reset = virtio_pci_reset;
1060     dc->props = virtio_scsi_properties;
1061 }
1062
1063 static TypeInfo virtio_scsi_info = {
1064     .name          = "virtio-scsi-pci",
1065     .parent        = TYPE_PCI_DEVICE,
1066     .instance_size = sizeof(VirtIOPCIProxy),
1067     .class_init    = virtio_scsi_class_init,
1068 };
1069
1070 static void virtio_pci_register_types(void)
1071 {
1072     type_register_static(&virtio_blk_info);
1073     type_register_static(&virtio_net_info);
1074     type_register_static(&virtio_serial_info);
1075     type_register_static(&virtio_balloon_info);
1076     type_register_static(&virtio_scsi_info);
1077 }
1078
1079 type_init(virtio_pci_register_types)