hw/virtio-pci.c

   1 /*
   2  * Virtio PCI Bindings
   3  *
   4  * Copyright IBM, Corp. 2007
   5  * Copyright (c) 2009 CodeSourcery
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Paul Brook        <paul@codesourcery.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2.  See
  12  * the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include <inttypes.h>
  17
  18 #include "virtio.h"
  19 #include "virtio-blk.h"
  20 #include "virtio-net.h"
  21 #include "pci.h"
  22 #include "qemu-error.h"
  23 #include "msix.h"
  24 #include "net.h"
  25 #include "loader.h"
  26 #include "kvm.h"
  27 #include "blockdev.h"
  28
  29 /* from Linux's linux/virtio_pci.h */
  30
  31 /* A 32-bit r/o bitmask of the features supported by the host */
  32 #define VIRTIO_PCI_HOST_FEATURES        0
  33
  34 /* A 32-bit r/w bitmask of features activated by the guest */
  35 #define VIRTIO_PCI_GUEST_FEATURES       4
  36
  37 /* A 32-bit r/w PFN for the currently selected queue */
  38 #define VIRTIO_PCI_QUEUE_PFN            8
  39
  40 /* A 16-bit r/o queue size for the currently selected queue */
  41 #define VIRTIO_PCI_QUEUE_NUM            12
  42
  43 /* A 16-bit r/w queue selector */
  44 #define VIRTIO_PCI_QUEUE_SEL            14
  45
  46 /* A 16-bit r/w queue notifier */
  47 #define VIRTIO_PCI_QUEUE_NOTIFY         16
  48
  49 /* An 8-bit device status register.  */
  50 #define VIRTIO_PCI_STATUS               18
  51
  52 /* An 8-bit r/o interrupt status register.  Reading the value will return the
  53  * current contents of the ISR and will also clear it.  This is effectively
  54  * a read-and-acknowledge. */
  55 #define VIRTIO_PCI_ISR                  19
  56
  57 /* MSI-X registers: only enabled if MSI-X is enabled. */
  58 /* A 16-bit vector for configuration changes. */
  59 #define VIRTIO_MSI_CONFIG_VECTOR        20
  60 /* A 16-bit vector for selected queue notifications. */
  61 #define VIRTIO_MSI_QUEUE_VECTOR         22
  62
  63 /* Config space size */
  64 #define VIRTIO_PCI_CONFIG_NOMSI         20
  65 #define VIRTIO_PCI_CONFIG_MSI           24
  66 #define VIRTIO_PCI_REGION_SIZE(dev)     (msix_present(dev) ? \
  67                                          VIRTIO_PCI_CONFIG_MSI : \
  68                                          VIRTIO_PCI_CONFIG_NOMSI)
  69
  70 /* The remaining space is defined by each driver as the per-driver
  71  * configuration space */
  72 #define VIRTIO_PCI_CONFIG(dev)          (msix_enabled(dev) ? \
  73                                          VIRTIO_PCI_CONFIG_MSI : \
  74                                          VIRTIO_PCI_CONFIG_NOMSI)
  75
  76 /* Virtio ABI version, if we increment this, we break the guest driver. */
  77 #define VIRTIO_PCI_ABI_VERSION          0
  78
  79 /* How many bits to shift physical queue address written to QUEUE_PFN.
  80  * 12 is historical, and due to x86 page size. */
  81 #define VIRTIO_PCI_QUEUE_ADDR_SHIFT    12
  82
  83 /* Flags track per-device state like workarounds for quirks in older guests. */
  84 #define VIRTIO_PCI_FLAG_BUS_MASTER_BUG  (1 << 0)
  85
  86 /* Performance improves when virtqueue kick processing is decoupled from the
  87  * vcpu thread using ioeventfd for some devices. */
  88 #define VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT 1
  89 #define VIRTIO_PCI_FLAG_USE_IOEVENTFD   (1 << VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT)
  90
  91 /* QEMU doesn't strictly need write barriers since everything runs in
  92  * lock-step.  We'll leave the calls to wmb() in though to make it obvious for
  93  * KVM or if kqemu gets SMP support.
  94  */
  95 #define wmb() do { } while (0)
  96
  97 /* PCI bindings.  */
  98
  99 typedef struct {
 100     PCIDevice pci_dev;
 101     VirtIODevice *vdev;
 102     uint32_t flags;
 103     uint32_t addr;
 104     uint32_t class_code;
 105     uint32_t nvectors;
 106     BlockConf block;
 107     NICConf nic;
 108     uint32_t host_features;
 109 #ifdef CONFIG_LINUX
 110     V9fsConf fsconf;
 111 #endif
 112     /* Max. number of ports we can have for a the virtio-serial device */
 113     uint32_t max_virtserial_ports;
 114     virtio_net_conf net;
 115     bool ioeventfd_disabled;
 116     bool ioeventfd_started;
 117 } VirtIOPCIProxy;
 118
 119 /* virtio device */
 120
 121 static void virtio_pci_notify(void *opaque, uint16_t vector)
 122 {
 123     VirtIOPCIProxy *proxy = opaque;
 124     if (msix_enabled(&proxy->pci_dev))
 125         msix_notify(&proxy->pci_dev, vector);
 126     else
 127         qemu_set_irq(proxy->pci_dev.irq[0], proxy->vdev->isr & 1);
 128 }
 129
 130 static void virtio_pci_save_config(void * opaque, QEMUFile *f)
 131 {
 132     VirtIOPCIProxy *proxy = opaque;
 133     pci_device_save(&proxy->pci_dev, f);
 134     msix_save(&proxy->pci_dev, f);
 135     if (msix_present(&proxy->pci_dev))
 136         qemu_put_be16(f, proxy->vdev->config_vector);
 137 }
 138
 139 static void virtio_pci_save_queue(void * opaque, int n, QEMUFile *f)
 140 {
 141     VirtIOPCIProxy *proxy = opaque;
 142     if (msix_present(&proxy->pci_dev))
 143         qemu_put_be16(f, virtio_queue_vector(proxy->vdev, n));
 144 }
 145
 146 static int virtio_pci_load_config(void * opaque, QEMUFile *f)
 147 {
 148     VirtIOPCIProxy *proxy = opaque;
 149     int ret;
 150     ret = pci_device_load(&proxy->pci_dev, f);
 151     if (ret) {
 152         return ret;
 153     }
 154     msix_load(&proxy->pci_dev, f);
 155     if (msix_present(&proxy->pci_dev)) {
 156         qemu_get_be16s(f, &proxy->vdev->config_vector);
 157     } else {
 158         proxy->vdev->config_vector = VIRTIO_NO_VECTOR;
 159     }
 160     if (proxy->vdev->config_vector != VIRTIO_NO_VECTOR) {
 161         return msix_vector_use(&proxy->pci_dev, proxy->vdev->config_vector);
 162     }
 163
 164     /* Try to find out if the guest has bus master disabled, but is
 165        in ready state. Then we have a buggy guest OS. */
 166     if ((proxy->vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) &&
 167         !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
 168         proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
 169     }
 170     return 0;
 171 }
 172
 173 static int virtio_pci_load_queue(void * opaque, int n, QEMUFile *f)
 174 {
 175     VirtIOPCIProxy *proxy = opaque;
 176     uint16_t vector;
 177     if (msix_present(&proxy->pci_dev)) {
 178         qemu_get_be16s(f, &vector);
 179     } else {
 180         vector = VIRTIO_NO_VECTOR;
 181     }
 182     virtio_queue_set_vector(proxy->vdev, n, vector);
 183     if (vector != VIRTIO_NO_VECTOR) {
 184         return msix_vector_use(&proxy->pci_dev, vector);
 185     }
 186     return 0;
 187 }
 188
 189 static int virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy,
 190                                                  int n, bool assign)
 191 {
 192     VirtQueue *vq = virtio_get_queue(proxy->vdev, n);
 193     EventNotifier *notifier = virtio_queue_get_host_notifier(vq);
 194     int r;
 195     if (assign) {
 196         r = event_notifier_init(notifier, 1);
 197         if (r < 0) {
 198             error_report("%s: unable to init event notifier: %d",
 199                          __func__, r);
 200             return r;
 201         }
 202         r = kvm_set_ioeventfd_pio_word(event_notifier_get_fd(notifier),
 203                                        proxy->addr + VIRTIO_PCI_QUEUE_NOTIFY,
 204                                        n, assign);
 205         if (r < 0) {
 206             error_report("%s: unable to map ioeventfd: %d",
 207                          __func__, r);
 208             event_notifier_cleanup(notifier);
 209         }
 210     } else {
 211         r = kvm_set_ioeventfd_pio_word(event_notifier_get_fd(notifier),
 212                                        proxy->addr + VIRTIO_PCI_QUEUE_NOTIFY,
 213                                        n, assign);
 214         if (r < 0) {
 215             error_report("%s: unable to unmap ioeventfd: %d",
 216                          __func__, r);
 217             return r;
 218         }
 219
 220         /* Handle the race condition where the guest kicked and we deassigned
 221          * before we got around to handling the kick.
 222          */
 223         if (event_notifier_test_and_clear(notifier)) {
 224             virtio_queue_notify_vq(vq);
 225         }
 226
 227         event_notifier_cleanup(notifier);
 228     }
 229     return r;
 230 }
 231
 232 static void virtio_pci_host_notifier_read(void *opaque)
 233 {
 234     VirtQueue *vq = opaque;
 235     EventNotifier *n = virtio_queue_get_host_notifier(vq);
 236     if (event_notifier_test_and_clear(n)) {
 237         virtio_queue_notify_vq(vq);
 238     }
 239 }
 240
 241 static void virtio_pci_set_host_notifier_fd_handler(VirtIOPCIProxy *proxy,
 242                                                     int n, bool assign)
 243 {
 244     VirtQueue *vq = virtio_get_queue(proxy->vdev, n);
 245     EventNotifier *notifier = virtio_queue_get_host_notifier(vq);
 246     if (assign) {
 247         qemu_set_fd_handler(event_notifier_get_fd(notifier),
 248                             virtio_pci_host_notifier_read, NULL, vq);
 249     } else {
 250         qemu_set_fd_handler(event_notifier_get_fd(notifier),
 251                             NULL, NULL, NULL);
 252     }
 253 }
 254
 255 static void virtio_pci_start_ioeventfd(VirtIOPCIProxy *proxy)
 256 {
 257     int n, r;
 258
 259     if (!(proxy->flags & VIRTIO_PCI_FLAG_USE_IOEVENTFD) ||
 260         proxy->ioeventfd_disabled ||
 261         proxy->ioeventfd_started) {
 262         return;
 263     }
 264
 265     for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
 266         if (!virtio_queue_get_num(proxy->vdev, n)) {
 267             continue;
 268         }
 269
 270         r = virtio_pci_set_host_notifier_internal(proxy, n, true);
 271         if (r < 0) {
 272             goto assign_error;
 273         }
 274
 275         virtio_pci_set_host_notifier_fd_handler(proxy, n, true);
 276     }
 277     proxy->ioeventfd_started = true;
 278     return;
 279
 280 assign_error:
 281     while (--n >= 0) {
 282         if (!virtio_queue_get_num(proxy->vdev, n)) {
 283             continue;
 284         }
 285
 286         virtio_pci_set_host_notifier_fd_handler(proxy, n, false);
 287         r = virtio_pci_set_host_notifier_internal(proxy, n, false);
 288         assert(r >= 0);
 289     }
 290     proxy->ioeventfd_started = false;
 291     error_report("%s: failed. Fallback to a userspace (slower).", __func__);
 292 }
 293
 294 static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy)
 295 {
 296     int r;
 297     int n;
 298
 299     if (!proxy->ioeventfd_started) {
 300         return;
 301     }
 302
 303     for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
 304         if (!virtio_queue_get_num(proxy->vdev, n)) {
 305             continue;
 306         }
 307
 308         virtio_pci_set_host_notifier_fd_handler(proxy, n, false);
 309         r = virtio_pci_set_host_notifier_internal(proxy, n, false);
 310         assert(r >= 0);
 311     }
 312     proxy->ioeventfd_started = false;
 313 }
 314
 315 static void virtio_pci_reset(DeviceState *d)
 316 {
 317     VirtIOPCIProxy *proxy = container_of(d, VirtIOPCIProxy, pci_dev.qdev);
 318     virtio_pci_stop_ioeventfd(proxy);
 319     virtio_reset(proxy->vdev);
 320     msix_reset(&proxy->pci_dev);
 321     proxy->flags &= ~VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
 322 }
 323
 324 static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
 325 {
 326     VirtIOPCIProxy *proxy = opaque;
 327     VirtIODevice *vdev = proxy->vdev;
 328     target_phys_addr_t pa;
 329
 330     switch (addr) {
 331     case VIRTIO_PCI_GUEST_FEATURES:
 332         /* Guest does not negotiate properly?  We have to assume nothing. */
 333         if (val & (1 << VIRTIO_F_BAD_FEATURE)) {
 334             if (vdev->bad_features)
 335                 val = proxy->host_features & vdev->bad_features(vdev);
 336             else
 337                 val = 0;
 338         }
 339         if (vdev->set_features)
 340             vdev->set_features(vdev, val);
 341         vdev->guest_features = val;
 342         break;
 343     case VIRTIO_PCI_QUEUE_PFN:
 344         pa = (target_phys_addr_t)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT;
 345         if (pa == 0) {
 346             virtio_pci_stop_ioeventfd(proxy);
 347             virtio_reset(proxy->vdev);
 348             msix_unuse_all_vectors(&proxy->pci_dev);
 349         }
 350         else
 351             virtio_queue_set_addr(vdev, vdev->queue_sel, pa);
 352         break;
 353     case VIRTIO_PCI_QUEUE_SEL:
 354         if (val < VIRTIO_PCI_QUEUE_MAX)
 355             vdev->queue_sel = val;
 356         break;
 357     case VIRTIO_PCI_QUEUE_NOTIFY:
 358         virtio_queue_notify(vdev, val);
 359         break;
 360     case VIRTIO_PCI_STATUS:
 361         if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) {
 362             virtio_pci_stop_ioeventfd(proxy);
 363         }
 364
 365         virtio_set_status(vdev, val & 0xFF);
 366
 367         if (val & VIRTIO_CONFIG_S_DRIVER_OK) {
 368             virtio_pci_start_ioeventfd(proxy);
 369         }
 370
 371         if (vdev->status == 0) {
 372             virtio_reset(proxy->vdev);
 373             msix_unuse_all_vectors(&proxy->pci_dev);
 374         }
 375
 376         /* Linux before 2.6.34 sets the device as OK without enabling
 377            the PCI device bus master bit. In this case we need to disable
 378            some safety checks. */
 379         if ((val & VIRTIO_CONFIG_S_DRIVER_OK) &&
 380             !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
 381             proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
 382         }
 383         break;
 384     case VIRTIO_MSI_CONFIG_VECTOR:
 385         msix_vector_unuse(&proxy->pci_dev, vdev->config_vector);
 386         /* Make it possible for guest to discover an error took place. */
 387         if (msix_vector_use(&proxy->pci_dev, val) < 0)
 388             val = VIRTIO_NO_VECTOR;
 389         vdev->config_vector = val;
 390         break;
 391     case VIRTIO_MSI_QUEUE_VECTOR:
 392         msix_vector_unuse(&proxy->pci_dev,
 393                           virtio_queue_vector(vdev, vdev->queue_sel));
 394         /* Make it possible for guest to discover an error took place. */
 395         if (msix_vector_use(&proxy->pci_dev, val) < 0)
 396             val = VIRTIO_NO_VECTOR;
 397         virtio_queue_set_vector(vdev, vdev->queue_sel, val);
 398         break;
 399     default:
 400         error_report("%s: unexpected address 0x%x value 0x%x",
 401                      __func__, addr, val);
 402         break;
 403     }
 404 }
 405
 406 static uint32_t virtio_ioport_read(VirtIOPCIProxy *proxy, uint32_t addr)
 407 {
 408     VirtIODevice *vdev = proxy->vdev;
 409     uint32_t ret = 0xFFFFFFFF;
 410
 411     switch (addr) {
 412     case VIRTIO_PCI_HOST_FEATURES:
 413         ret = proxy->host_features;
 414         break;
 415     case VIRTIO_PCI_GUEST_FEATURES:
 416         ret = vdev->guest_features;
 417         break;
 418     case VIRTIO_PCI_QUEUE_PFN:
 419         ret = virtio_queue_get_addr(vdev, vdev->queue_sel)
 420               >> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
 421         break;
 422     case VIRTIO_PCI_QUEUE_NUM:
 423         ret = virtio_queue_get_num(vdev, vdev->queue_sel);
 424         break;
 425     case VIRTIO_PCI_QUEUE_SEL:
 426         ret = vdev->queue_sel;
 427         break;
 428     case VIRTIO_PCI_STATUS:
 429         ret = vdev->status;
 430         break;
 431     case VIRTIO_PCI_ISR:
 432         /* reading from the ISR also clears it. */
 433         ret = vdev->isr;
 434         vdev->isr = 0;
 435         qemu_set_irq(proxy->pci_dev.irq[0], 0);
 436         break;
 437     case VIRTIO_MSI_CONFIG_VECTOR:
 438         ret = vdev->config_vector;
 439         break;
 440     case VIRTIO_MSI_QUEUE_VECTOR:
 441         ret = virtio_queue_vector(vdev, vdev->queue_sel);
 442         break;
 443     default:
 444         break;
 445     }
 446
 447     return ret;
 448 }
 449
 450 static uint32_t virtio_pci_config_readb(void *opaque, uint32_t addr)
 451 {
 452     VirtIOPCIProxy *proxy = opaque;
 453     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 454     addr -= proxy->addr;
 455     if (addr < config)
 456         return virtio_ioport_read(proxy, addr);
 457     addr -= config;
 458     return virtio_config_readb(proxy->vdev, addr);
 459 }
 460
 461 static uint32_t virtio_pci_config_readw(void *opaque, uint32_t addr)
 462 {
 463     VirtIOPCIProxy *proxy = opaque;
 464     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 465     addr -= proxy->addr;
 466     if (addr < config)
 467         return virtio_ioport_read(proxy, addr);
 468     addr -= config;
 469     return virtio_config_readw(proxy->vdev, addr);
 470 }
 471
 472 static uint32_t virtio_pci_config_readl(void *opaque, uint32_t addr)
 473 {
 474     VirtIOPCIProxy *proxy = opaque;
 475     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 476     addr -= proxy->addr;
 477     if (addr < config)
 478         return virtio_ioport_read(proxy, addr);
 479     addr -= config;
 480     return virtio_config_readl(proxy->vdev, addr);
 481 }
 482
 483 static void virtio_pci_config_writeb(void *opaque, uint32_t addr, uint32_t val)
 484 {
 485     VirtIOPCIProxy *proxy = opaque;
 486     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 487     addr -= proxy->addr;
 488     if (addr < config) {
 489         virtio_ioport_write(proxy, addr, val);
 490         return;
 491     }
 492     addr -= config;
 493     virtio_config_writeb(proxy->vdev, addr, val);
 494 }
 495
 496 static void virtio_pci_config_writew(void *opaque, uint32_t addr, uint32_t val)
 497 {
 498     VirtIOPCIProxy *proxy = opaque;
 499     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 500     addr -= proxy->addr;
 501     if (addr < config) {
 502         virtio_ioport_write(proxy, addr, val);
 503         return;
 504     }
 505     addr -= config;
 506     virtio_config_writew(proxy->vdev, addr, val);
 507 }
 508
 509 static void virtio_pci_config_writel(void *opaque, uint32_t addr, uint32_t val)
 510 {
 511     VirtIOPCIProxy *proxy = opaque;
 512     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 513     addr -= proxy->addr;
 514     if (addr < config) {
 515         virtio_ioport_write(proxy, addr, val);
 516         return;
 517     }
 518     addr -= config;
 519     virtio_config_writel(proxy->vdev, addr, val);
 520 }
 521
 522 static void virtio_map(PCIDevice *pci_dev, int region_num,
 523                        pcibus_t addr, pcibus_t size, int type)
 524 {
 525     VirtIOPCIProxy *proxy = container_of(pci_dev, VirtIOPCIProxy, pci_dev);
 526     VirtIODevice *vdev = proxy->vdev;
 527     unsigned config_len = VIRTIO_PCI_REGION_SIZE(pci_dev) + vdev->config_len;
 528
 529     proxy->addr = addr;
 530
 531     register_ioport_write(addr, config_len, 1, virtio_pci_config_writeb, proxy);
 532     register_ioport_write(addr, config_len, 2, virtio_pci_config_writew, proxy);
 533     register_ioport_write(addr, config_len, 4, virtio_pci_config_writel, proxy);
 534     register_ioport_read(addr, config_len, 1, virtio_pci_config_readb, proxy);
 535     register_ioport_read(addr, config_len, 2, virtio_pci_config_readw, proxy);
 536     register_ioport_read(addr, config_len, 4, virtio_pci_config_readl, proxy);
 537
 538     if (vdev->config_len)
 539         vdev->get_config(vdev, vdev->config);
 540 }
 541
 542 static void virtio_write_config(PCIDevice *pci_dev, uint32_t address,
 543                                 uint32_t val, int len)
 544 {
 545     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 546
 547     if (PCI_COMMAND == address) {
 548         if (!(val & PCI_COMMAND_MASTER)) {
 549             if (!(proxy->flags & VIRTIO_PCI_FLAG_BUS_MASTER_BUG)) {
 550                 virtio_pci_stop_ioeventfd(proxy);
 551                 virtio_set_status(proxy->vdev,
 552                                   proxy->vdev->status & ~VIRTIO_CONFIG_S_DRIVER_OK);
 553             }
 554         }
 555     }
 556
 557     pci_default_write_config(pci_dev, address, val, len);
 558     msix_write_config(pci_dev, address, val, len);
 559 }
 560
 561 static unsigned virtio_pci_get_features(void *opaque)
 562 {
 563     VirtIOPCIProxy *proxy = opaque;
 564     return proxy->host_features;
 565 }
 566
 567 static void virtio_pci_guest_notifier_read(void *opaque)
 568 {
 569     VirtQueue *vq = opaque;
 570     EventNotifier *n = virtio_queue_get_guest_notifier(vq);
 571     if (event_notifier_test_and_clear(n)) {
 572         virtio_irq(vq);
 573     }
 574 }
 575
 576 static int virtio_pci_set_guest_notifier(void *opaque, int n, bool assign)
 577 {
 578     VirtIOPCIProxy *proxy = opaque;
 579     VirtQueue *vq = virtio_get_queue(proxy->vdev, n);
 580     EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
 581
 582     if (assign) {
 583         int r = event_notifier_init(notifier, 0);
 584         if (r < 0) {
 585             return r;
 586         }
 587         qemu_set_fd_handler(event_notifier_get_fd(notifier),
 588                             virtio_pci_guest_notifier_read, NULL, vq);
 589     } else {
 590         qemu_set_fd_handler(event_notifier_get_fd(notifier),
 591                             NULL, NULL, NULL);
 592         event_notifier_cleanup(notifier);
 593     }
 594
 595     return 0;
 596 }
 597
 598 static bool virtio_pci_query_guest_notifiers(void *opaque)
 599 {
 600     VirtIOPCIProxy *proxy = opaque;
 601     return msix_enabled(&proxy->pci_dev);
 602 }
 603
 604 static int virtio_pci_set_guest_notifiers(void *opaque, bool assign)
 605 {
 606     VirtIOPCIProxy *proxy = opaque;
 607     VirtIODevice *vdev = proxy->vdev;
 608     int r, n;
 609
 610     for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
 611         if (!virtio_queue_get_num(vdev, n)) {
 612             break;
 613         }
 614
 615         r = virtio_pci_set_guest_notifier(opaque, n, assign);
 616         if (r < 0) {
 617             goto assign_error;
 618         }
 619     }
 620
 621     return 0;
 622
 623 assign_error:
 624     /* We get here on assignment failure. Recover by undoing for VQs 0 .. n. */
 625     while (--n >= 0) {
 626         virtio_pci_set_guest_notifier(opaque, n, !assign);
 627     }
 628     return r;
 629 }
 630
 631 static int virtio_pci_set_host_notifier(void *opaque, int n, bool assign)
 632 {
 633     VirtIOPCIProxy *proxy = opaque;
 634
 635     /* Stop using ioeventfd for virtqueue kick if the device starts using host
 636      * notifiers.  This makes it easy to avoid stepping on each others' toes.
 637      */
 638     proxy->ioeventfd_disabled = assign;
 639     if (assign) {
 640         virtio_pci_stop_ioeventfd(proxy);
 641     }
 642     /* We don't need to start here: it's not needed because backend
 643      * currently only stops on status change away from ok,
 644      * reset, vmstop and such. If we do add code to start here,
 645      * need to check vmstate, device state etc. */
 646     return virtio_pci_set_host_notifier_internal(proxy, n, assign);
 647 }
 648
 649 static void virtio_pci_vmstate_change(void *opaque, bool running)
 650 {
 651     VirtIOPCIProxy *proxy = opaque;
 652
 653     if (running) {
 654         virtio_pci_start_ioeventfd(proxy);
 655     } else {
 656         virtio_pci_stop_ioeventfd(proxy);
 657     }
 658 }
 659
 660 static const VirtIOBindings virtio_pci_bindings = {
 661     .notify = virtio_pci_notify,
 662     .save_config = virtio_pci_save_config,
 663     .load_config = virtio_pci_load_config,
 664     .save_queue = virtio_pci_save_queue,
 665     .load_queue = virtio_pci_load_queue,
 666     .get_features = virtio_pci_get_features,
 667     .query_guest_notifiers = virtio_pci_query_guest_notifiers,
 668     .set_host_notifier = virtio_pci_set_host_notifier,
 669     .set_guest_notifiers = virtio_pci_set_guest_notifiers,
 670     .vmstate_change = virtio_pci_vmstate_change,
 671 };
 672
 673 static void virtio_init_pci(VirtIOPCIProxy *proxy, VirtIODevice *vdev,
 674                             uint16_t vendor, uint16_t device,
 675                             uint16_t class_code, uint8_t pif)
 676 {
 677     uint8_t *config;
 678     uint32_t size;
 679
 680     proxy->vdev = vdev;
 681
 682     config = proxy->pci_dev.config;
 683     pci_config_set_vendor_id(config, vendor);
 684     pci_config_set_device_id(config, device);
 685
 686     config[0x08] = VIRTIO_PCI_ABI_VERSION;
 687
 688     config[0x09] = pif;
 689     pci_config_set_class(config, class_code);
 690
 691     config[0x2c] = vendor & 0xFF;
 692     config[0x2d] = (vendor >> 8) & 0xFF;
 693     config[0x2e] = vdev->device_id & 0xFF;
 694     config[0x2f] = (vdev->device_id >> 8) & 0xFF;
 695
 696     config[0x3d] = 1;
 697
 698     if (vdev->nvectors && !msix_init(&proxy->pci_dev, vdev->nvectors, 1, 0)) {
 699         pci_register_bar(&proxy->pci_dev, 1,
 700                          msix_bar_size(&proxy->pci_dev),
 701                          PCI_BASE_ADDRESS_SPACE_MEMORY,
 702                          msix_mmio_map);
 703     } else
 704         vdev->nvectors = 0;
 705
 706     proxy->pci_dev.config_write = virtio_write_config;
 707
 708     size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev) + vdev->config_len;
 709     if (size & (size-1))
 710         size = 1 << qemu_fls(size);
 711
 712     pci_register_bar(&proxy->pci_dev, 0, size, PCI_BASE_ADDRESS_SPACE_IO,
 713                            virtio_map);
 714
 715     if (!kvm_has_many_ioeventfds()) {
 716         proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
 717     }
 718
 719     virtio_bind_device(vdev, &virtio_pci_bindings, proxy);
 720     proxy->host_features |= 0x1 << VIRTIO_F_NOTIFY_ON_EMPTY;
 721     proxy->host_features |= 0x1 << VIRTIO_F_BAD_FEATURE;
 722     proxy->host_features = vdev->get_features(vdev, proxy->host_features);
 723 }
 724
 725 static int virtio_blk_init_pci(PCIDevice *pci_dev)
 726 {
 727     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 728     VirtIODevice *vdev;
 729
 730     if (proxy->class_code != PCI_CLASS_STORAGE_SCSI &&
 731         proxy->class_code != PCI_CLASS_STORAGE_OTHER)
 732         proxy->class_code = PCI_CLASS_STORAGE_SCSI;
 733
 734     vdev = virtio_blk_init(&pci_dev->qdev, &proxy->block);
 735     if (!vdev) {
 736         return -1;
 737     }
 738     vdev->nvectors = proxy->nvectors;
 739     virtio_init_pci(proxy, vdev,
 740                     PCI_VENDOR_ID_REDHAT_QUMRANET,
 741                     PCI_DEVICE_ID_VIRTIO_BLOCK,
 742                     proxy->class_code, 0x00);
 743     /* make the actual value visible */
 744     proxy->nvectors = vdev->nvectors;
 745     return 0;
 746 }
 747
 748 static int virtio_exit_pci(PCIDevice *pci_dev)
 749 {
 750     return msix_uninit(pci_dev);
 751 }
 752
 753 static int virtio_blk_exit_pci(PCIDevice *pci_dev)
 754 {
 755     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 756
 757     virtio_pci_stop_ioeventfd(proxy);
 758     virtio_blk_exit(proxy->vdev);
 759     blockdev_mark_auto_del(proxy->block.bs);
 760     return virtio_exit_pci(pci_dev);
 761 }
 762
 763 static int virtio_serial_init_pci(PCIDevice *pci_dev)
 764 {
 765     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 766     VirtIODevice *vdev;
 767
 768     if (proxy->class_code != PCI_CLASS_COMMUNICATION_OTHER &&
 769         proxy->class_code != PCI_CLASS_DISPLAY_OTHER && /* qemu 0.10 */
 770         proxy->class_code != PCI_CLASS_OTHERS)          /* qemu-kvm  */
 771         proxy->class_code = PCI_CLASS_COMMUNICATION_OTHER;
 772
 773     vdev = virtio_serial_init(&pci_dev->qdev, proxy->max_virtserial_ports);
 774     if (!vdev) {
 775         return -1;
 776     }
 777     vdev->nvectors = proxy->nvectors == DEV_NVECTORS_UNSPECIFIED
 778                                         ? proxy->max_virtserial_ports + 1
 779                                         : proxy->nvectors;
 780     virtio_init_pci(proxy, vdev,
 781                     PCI_VENDOR_ID_REDHAT_QUMRANET,
 782                     PCI_DEVICE_ID_VIRTIO_CONSOLE,
 783                     proxy->class_code, 0x00);
 784     proxy->nvectors = vdev->nvectors;
 785     return 0;
 786 }
 787
 788 static int virtio_serial_exit_pci(PCIDevice *pci_dev)
 789 {
 790     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 791
 792     virtio_serial_exit(proxy->vdev);
 793     return virtio_exit_pci(pci_dev);
 794 }
 795
 796 static int virtio_net_init_pci(PCIDevice *pci_dev)
 797 {
 798     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 799     VirtIODevice *vdev;
 800
 801     vdev = virtio_net_init(&pci_dev->qdev, &proxy->nic, &proxy->net);
 802
 803     vdev->nvectors = proxy->nvectors;
 804     virtio_init_pci(proxy, vdev,
 805                     PCI_VENDOR_ID_REDHAT_QUMRANET,
 806                     PCI_DEVICE_ID_VIRTIO_NET,
 807                     PCI_CLASS_NETWORK_ETHERNET,
 808                     0x00);
 809
 810     /* make the actual value visible */
 811     proxy->nvectors = vdev->nvectors;
 812     return 0;
 813 }
 814
 815 static int virtio_net_exit_pci(PCIDevice *pci_dev)
 816 {
 817     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 818
 819     virtio_pci_stop_ioeventfd(proxy);
 820     virtio_net_exit(proxy->vdev);
 821     return virtio_exit_pci(pci_dev);
 822 }
 823
 824 static int virtio_balloon_init_pci(PCIDevice *pci_dev)
 825 {
 826     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 827     VirtIODevice *vdev;
 828
 829     vdev = virtio_balloon_init(&pci_dev->qdev);
 830     virtio_init_pci(proxy, vdev,
 831                     PCI_VENDOR_ID_REDHAT_QUMRANET,
 832                     PCI_DEVICE_ID_VIRTIO_BALLOON,
 833                     PCI_CLASS_MEMORY_RAM,
 834                     0x00);
 835     return 0;
 836 }
 837
 838 #ifdef CONFIG_VIRTFS
 839 static int virtio_9p_init_pci(PCIDevice *pci_dev)
 840 {
 841     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 842     VirtIODevice *vdev;
 843
 844     vdev = virtio_9p_init(&pci_dev->qdev, &proxy->fsconf);
 845     vdev->nvectors = proxy->nvectors;
 846     virtio_init_pci(proxy, vdev,
 847                     PCI_VENDOR_ID_REDHAT_QUMRANET,
 848                     0x1009,
 849                     0x2,
 850                     0x00);
 851     /* make the actual value visible */
 852     proxy->nvectors = vdev->nvectors;
 853     return 0;
 854 }
 855 #endif
 856
 857 static PCIDeviceInfo virtio_info[] = {
 858     {
 859         .qdev.name = "virtio-blk-pci",
 860         .qdev.alias = "virtio-blk",
 861         .qdev.size = sizeof(VirtIOPCIProxy),
 862         .init      = virtio_blk_init_pci,
 863         .exit      = virtio_blk_exit_pci,
 864         .qdev.props = (Property[]) {
 865             DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
 866             DEFINE_BLOCK_PROPERTIES(VirtIOPCIProxy, block),
 867             DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags,
 868                             VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
 869             DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 2),
 870             DEFINE_VIRTIO_BLK_FEATURES(VirtIOPCIProxy, host_features),
 871             DEFINE_PROP_END_OF_LIST(),
 872         },
 873         .qdev.reset = virtio_pci_reset,
 874     },{
 875         .qdev.name  = "virtio-net-pci",
 876         .qdev.size  = sizeof(VirtIOPCIProxy),
 877         .init       = virtio_net_init_pci,
 878         .exit       = virtio_net_exit_pci,
 879         .romfile    = "pxe-virtio.bin",
 880         .qdev.props = (Property[]) {
 881             DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags,
 882                             VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, false),
 883             DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 3),
 884             DEFINE_VIRTIO_NET_FEATURES(VirtIOPCIProxy, host_features),
 885             DEFINE_NIC_PROPERTIES(VirtIOPCIProxy, nic),
 886             DEFINE_PROP_UINT32("x-txtimer", VirtIOPCIProxy,
 887                                net.txtimer, TX_TIMER_INTERVAL),
 888             DEFINE_PROP_INT32("x-txburst", VirtIOPCIProxy,
 889                               net.txburst, TX_BURST),
 890             DEFINE_PROP_STRING("tx", VirtIOPCIProxy, net.tx),
 891             DEFINE_PROP_END_OF_LIST(),
 892         },
 893         .qdev.reset = virtio_pci_reset,
 894     },{
 895         .qdev.name = "virtio-serial-pci",
 896         .qdev.alias = "virtio-serial",
 897         .qdev.size = sizeof(VirtIOPCIProxy),
 898         .init      = virtio_serial_init_pci,
 899         .exit      = virtio_serial_exit_pci,
 900         .qdev.props = (Property[]) {
 901             DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors,
 902                                DEV_NVECTORS_UNSPECIFIED),
 903             DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
 904             DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
 905             DEFINE_PROP_UINT32("max_ports", VirtIOPCIProxy, max_virtserial_ports,
 906                                31),
 907             DEFINE_PROP_END_OF_LIST(),
 908         },
 909         .qdev.reset = virtio_pci_reset,
 910     },{
 911         .qdev.name = "virtio-balloon-pci",
 912         .qdev.size = sizeof(VirtIOPCIProxy),
 913         .init      = virtio_balloon_init_pci,
 914         .exit      = virtio_exit_pci,
 915         .qdev.props = (Property[]) {
 916             DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
 917             DEFINE_PROP_END_OF_LIST(),
 918         },
 919         .qdev.reset = virtio_pci_reset,
 920     },{
 921 #ifdef CONFIG_VIRTFS
 922         .qdev.name = "virtio-9p-pci",
 923         .qdev.size = sizeof(VirtIOPCIProxy),
 924         .init      = virtio_9p_init_pci,
 925         .qdev.props = (Property[]) {
 926             DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 2),
 927             DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
 928             DEFINE_PROP_STRING("mount_tag", VirtIOPCIProxy, fsconf.tag),
 929             DEFINE_PROP_STRING("fsdev", VirtIOPCIProxy, fsconf.fsdev_id),
 930             DEFINE_PROP_END_OF_LIST(),
 931         },
 932     }, {
 933 #endif
 934         /* end of list */
 935     }
 936 };
 937
 938 static void virtio_pci_register_devices(void)
 939 {
 940     pci_qdev_register_many(virtio_info);
 941 }
 942
 943 device_init(virtio_pci_register_devices)