hw/virtio.c

   1 /*
   2  * Virtio Support
   3  *
   4  * Copyright IBM, Corp. 2007
   5  *
   6  * Authors:
   7  *  Anthony Liguori   <aliguori@us.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  */
  13
  14 #include <inttypes.h>
  15
  16 #include "trace.h"
  17 #include "qemu-error.h"
  18 #include "virtio.h"
  19 #include "qemu-barrier.h"
  20
  21 /* The alignment to use between consumer and producer parts of vring.
  22  * x86 pagesize again. */
  23 #define VIRTIO_PCI_VRING_ALIGN         4096
  24
  25 typedef struct VRingDesc
  26 {
  27     uint64_t addr;
  28     uint32_t len;
  29     uint16_t flags;
  30     uint16_t next;
  31 } VRingDesc;
  32
  33 typedef struct VRingAvail
  34 {
  35     uint16_t flags;
  36     uint16_t idx;
  37     uint16_t ring[0];
  38 } VRingAvail;
  39
  40 typedef struct VRingUsedElem
  41 {
  42     uint32_t id;
  43     uint32_t len;
  44 } VRingUsedElem;
  45
  46 typedef struct VRingUsed
  47 {
  48     uint16_t flags;
  49     uint16_t idx;
  50     VRingUsedElem ring[0];
  51 } VRingUsed;
  52
  53 typedef struct VRing
  54 {
  55     unsigned int num;
  56     target_phys_addr_t desc;
  57     target_phys_addr_t avail;
  58     target_phys_addr_t used;
  59 } VRing;
  60
  61 struct VirtQueue
  62 {
  63     VRing vring;
  64     target_phys_addr_t pa;
  65     uint16_t last_avail_idx;
  66     /* Last used index value we have signalled on */
  67     uint16_t signalled_used;
  68
  69     /* Last used index value we have signalled on */
  70     bool signalled_used_valid;
  71
  72     /* Notification enabled? */
  73     bool notification;
  74
  75     int inuse;
  76
  77     uint16_t vector;
  78     void (*handle_output)(VirtIODevice *vdev, VirtQueue *vq);
  79     VirtIODevice *vdev;
  80     EventNotifier guest_notifier;
  81     EventNotifier host_notifier;
  82 };
  83
  84 /* virt queue functions */
  85 static void virtqueue_init(VirtQueue *vq)
  86 {
  87     target_phys_addr_t pa = vq->pa;
  88
  89     vq->vring.desc = pa;
  90     vq->vring.avail = pa + vq->vring.num * sizeof(VRingDesc);
  91     vq->vring.used = vring_align(vq->vring.avail +
  92                                  offsetof(VRingAvail, ring[vq->vring.num]),
  93                                  VIRTIO_PCI_VRING_ALIGN);
  94 }
  95
  96 static inline uint64_t vring_desc_addr(target_phys_addr_t desc_pa, int i)
  97 {
  98     target_phys_addr_t pa;
  99     pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, addr);
 100     return ldq_phys(pa);
 101 }
 102
 103 static inline uint32_t vring_desc_len(target_phys_addr_t desc_pa, int i)
 104 {
 105     target_phys_addr_t pa;
 106     pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, len);
 107     return ldl_phys(pa);
 108 }
 109
 110 static inline uint16_t vring_desc_flags(target_phys_addr_t desc_pa, int i)
 111 {
 112     target_phys_addr_t pa;
 113     pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, flags);
 114     return lduw_phys(pa);
 115 }
 116
 117 static inline uint16_t vring_desc_next(target_phys_addr_t desc_pa, int i)
 118 {
 119     target_phys_addr_t pa;
 120     pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, next);
 121     return lduw_phys(pa);
 122 }
 123
 124 static inline uint16_t vring_avail_flags(VirtQueue *vq)
 125 {
 126     target_phys_addr_t pa;
 127     pa = vq->vring.avail + offsetof(VRingAvail, flags);
 128     return lduw_phys(pa);
 129 }
 130
 131 static inline uint16_t vring_avail_idx(VirtQueue *vq)
 132 {
 133     target_phys_addr_t pa;
 134     pa = vq->vring.avail + offsetof(VRingAvail, idx);
 135     return lduw_phys(pa);
 136 }
 137
 138 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
 139 {
 140     target_phys_addr_t pa;
 141     pa = vq->vring.avail + offsetof(VRingAvail, ring[i]);
 142     return lduw_phys(pa);
 143 }
 144
 145 static inline uint16_t vring_used_event(VirtQueue *vq)
 146 {
 147     return vring_avail_ring(vq, vq->vring.num);
 148 }
 149
 150 static inline void vring_used_ring_id(VirtQueue *vq, int i, uint32_t val)
 151 {
 152     target_phys_addr_t pa;
 153     pa = vq->vring.used + offsetof(VRingUsed, ring[i].id);
 154     stl_phys(pa, val);
 155 }
 156
 157 static inline void vring_used_ring_len(VirtQueue *vq, int i, uint32_t val)
 158 {
 159     target_phys_addr_t pa;
 160     pa = vq->vring.used + offsetof(VRingUsed, ring[i].len);
 161     stl_phys(pa, val);
 162 }
 163
 164 static uint16_t vring_used_idx(VirtQueue *vq)
 165 {
 166     target_phys_addr_t pa;
 167     pa = vq->vring.used + offsetof(VRingUsed, idx);
 168     return lduw_phys(pa);
 169 }
 170
 171 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
 172 {
 173     target_phys_addr_t pa;
 174     pa = vq->vring.used + offsetof(VRingUsed, idx);
 175     stw_phys(pa, val);
 176 }
 177
 178 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
 179 {
 180     target_phys_addr_t pa;
 181     pa = vq->vring.used + offsetof(VRingUsed, flags);
 182     stw_phys(pa, lduw_phys(pa) | mask);
 183 }
 184
 185 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
 186 {
 187     target_phys_addr_t pa;
 188     pa = vq->vring.used + offsetof(VRingUsed, flags);
 189     stw_phys(pa, lduw_phys(pa) & ~mask);
 190 }
 191
 192 static inline void vring_avail_event(VirtQueue *vq, uint16_t val)
 193 {
 194     target_phys_addr_t pa;
 195     if (!vq->notification) {
 196         return;
 197     }
 198     pa = vq->vring.used + offsetof(VRingUsed, ring[vq->vring.num]);
 199     stw_phys(pa, val);
 200 }
 201
 202 void virtio_queue_set_notification(VirtQueue *vq, int enable)
 203 {
 204     vq->notification = enable;
 205     if (vq->vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX)) {
 206         vring_avail_event(vq, vring_avail_idx(vq));
 207     } else if (enable) {
 208         vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
 209     } else {
 210         vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
 211     }
 212     if (enable) {
 213         /* Expose avail event/used flags before caller checks the avail idx. */
 214         smp_mb();
 215     }
 216 }
 217
 218 int virtio_queue_ready(VirtQueue *vq)
 219 {
 220     return vq->vring.avail != 0;
 221 }
 222
 223 int virtio_queue_empty(VirtQueue *vq)
 224 {
 225     return vring_avail_idx(vq) == vq->last_avail_idx;
 226 }
 227
 228 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
 229                     unsigned int len, unsigned int idx)
 230 {
 231     unsigned int offset;
 232     int i;
 233
 234     trace_virtqueue_fill(vq, elem, len, idx);
 235
 236     offset = 0;
 237     for (i = 0; i < elem->in_num; i++) {
 238         size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
 239
 240         cpu_physical_memory_unmap(elem->in_sg[i].iov_base,
 241                                   elem->in_sg[i].iov_len,
 242                                   1, size);
 243
 244         offset += size;
 245     }
 246
 247     for (i = 0; i < elem->out_num; i++)
 248         cpu_physical_memory_unmap(elem->out_sg[i].iov_base,
 249                                   elem->out_sg[i].iov_len,
 250                                   0, elem->out_sg[i].iov_len);
 251
 252     idx = (idx + vring_used_idx(vq)) % vq->vring.num;
 253
 254     /* Get a pointer to the next entry in the used ring. */
 255     vring_used_ring_id(vq, idx, elem->index);
 256     vring_used_ring_len(vq, idx, len);
 257 }
 258
 259 void virtqueue_flush(VirtQueue *vq, unsigned int count)
 260 {
 261     uint16_t old, new;
 262     /* Make sure buffer is written before we update index. */
 263     smp_wmb();
 264     trace_virtqueue_flush(vq, count);
 265     old = vring_used_idx(vq);
 266     new = old + count;
 267     vring_used_idx_set(vq, new);
 268     vq->inuse -= count;
 269     if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)))
 270         vq->signalled_used_valid = false;
 271 }
 272
 273 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
 274                     unsigned int len)
 275 {
 276     virtqueue_fill(vq, elem, len, 0);
 277     virtqueue_flush(vq, 1);
 278 }
 279
 280 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
 281 {
 282     uint16_t num_heads = vring_avail_idx(vq) - idx;
 283
 284     /* Check it isn't doing very strange things with descriptor numbers. */
 285     if (num_heads > vq->vring.num) {
 286         error_report("Guest moved used index from %u to %u",
 287                      idx, vring_avail_idx(vq));
 288         exit(1);
 289     }
 290     /* On success, callers read a descriptor at vq->last_avail_idx.
 291      * Make sure descriptor read does not bypass avail index read. */
 292     if (num_heads) {
 293         smp_rmb();
 294     }
 295
 296     return num_heads;
 297 }
 298
 299 static unsigned int virtqueue_get_head(VirtQueue *vq, unsigned int idx)
 300 {
 301     unsigned int head;
 302
 303     /* Grab the next descriptor number they're advertising, and increment
 304      * the index we've seen. */
 305     head = vring_avail_ring(vq, idx % vq->vring.num);
 306
 307     /* If their number is silly, that's a fatal mistake. */
 308     if (head >= vq->vring.num) {
 309         error_report("Guest says index %u is available", head);
 310         exit(1);
 311     }
 312
 313     return head;
 314 }
 315
 316 static unsigned virtqueue_next_desc(target_phys_addr_t desc_pa,
 317                                     unsigned int i, unsigned int max)
 318 {
 319     unsigned int next;
 320
 321     /* If this descriptor says it doesn't chain, we're done. */
 322     if (!(vring_desc_flags(desc_pa, i) & VRING_DESC_F_NEXT))
 323         return max;
 324
 325     /* Check they're not leading us off end of descriptors. */
 326     next = vring_desc_next(desc_pa, i);
 327     /* Make sure compiler knows to grab that: we don't want it changing! */
 328     smp_wmb();
 329
 330     if (next >= max) {
 331         error_report("Desc next is %u", next);
 332         exit(1);
 333     }
 334
 335     return next;
 336 }
 337
 338 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
 339                                unsigned int *out_bytes)
 340 {
 341     unsigned int idx;
 342     unsigned int total_bufs, in_total, out_total;
 343
 344     idx = vq->last_avail_idx;
 345
 346     total_bufs = in_total = out_total = 0;
 347     while (virtqueue_num_heads(vq, idx)) {
 348         unsigned int max, num_bufs, indirect = 0;
 349         target_phys_addr_t desc_pa;
 350         int i;
 351
 352         max = vq->vring.num;
 353         num_bufs = total_bufs;
 354         i = virtqueue_get_head(vq, idx++);
 355         desc_pa = vq->vring.desc;
 356
 357         if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_INDIRECT) {
 358             if (vring_desc_len(desc_pa, i) % sizeof(VRingDesc)) {
 359                 error_report("Invalid size for indirect buffer table");
 360                 exit(1);
 361             }
 362
 363             /* If we've got too many, that implies a descriptor loop. */
 364             if (num_bufs >= max) {
 365                 error_report("Looped descriptor");
 366                 exit(1);
 367             }
 368
 369             /* loop over the indirect descriptor table */
 370             indirect = 1;
 371             max = vring_desc_len(desc_pa, i) / sizeof(VRingDesc);
 372             num_bufs = i = 0;
 373             desc_pa = vring_desc_addr(desc_pa, i);
 374         }
 375
 376         do {
 377             /* If we've got too many, that implies a descriptor loop. */
 378             if (++num_bufs > max) {
 379                 error_report("Looped descriptor");
 380                 exit(1);
 381             }
 382
 383             if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_WRITE) {
 384                 in_total += vring_desc_len(desc_pa, i);
 385             } else {
 386                 out_total += vring_desc_len(desc_pa, i);
 387             }
 388         } while ((i = virtqueue_next_desc(desc_pa, i, max)) != max);
 389
 390         if (!indirect)
 391             total_bufs = num_bufs;
 392         else
 393             total_bufs++;
 394     }
 395     if (in_bytes) {
 396         *in_bytes = in_total;
 397     }
 398     if (out_bytes) {
 399         *out_bytes = out_total;
 400     }
 401 }
 402
 403 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
 404                           unsigned int out_bytes)
 405 {
 406     unsigned int in_total, out_total;
 407
 408     virtqueue_get_avail_bytes(vq, &in_total, &out_total);
 409     if ((in_bytes && in_bytes < in_total)
 410         || (out_bytes && out_bytes < out_total)) {
 411         return 1;
 412     }
 413     return 0;
 414 }
 415
 416 void virtqueue_map_sg(struct iovec *sg, target_phys_addr_t *addr,
 417     size_t num_sg, int is_write)
 418 {
 419     unsigned int i;
 420     target_phys_addr_t len;
 421
 422     for (i = 0; i < num_sg; i++) {
 423         len = sg[i].iov_len;
 424         sg[i].iov_base = cpu_physical_memory_map(addr[i], &len, is_write);
 425         if (sg[i].iov_base == NULL || len != sg[i].iov_len) {
 426             error_report("virtio: trying to map MMIO memory");
 427             exit(1);
 428         }
 429     }
 430 }
 431
 432 int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem)
 433 {
 434     unsigned int i, head, max;
 435     target_phys_addr_t desc_pa = vq->vring.desc;
 436
 437     if (!virtqueue_num_heads(vq, vq->last_avail_idx))
 438         return 0;
 439
 440     /* When we start there are none of either input nor output. */
 441     elem->out_num = elem->in_num = 0;
 442
 443     max = vq->vring.num;
 444
 445     i = head = virtqueue_get_head(vq, vq->last_avail_idx++);
 446     if (vq->vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX)) {
 447         vring_avail_event(vq, vring_avail_idx(vq));
 448     }
 449
 450     if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_INDIRECT) {
 451         if (vring_desc_len(desc_pa, i) % sizeof(VRingDesc)) {
 452             error_report("Invalid size for indirect buffer table");
 453             exit(1);
 454         }
 455
 456         /* loop over the indirect descriptor table */
 457         max = vring_desc_len(desc_pa, i) / sizeof(VRingDesc);
 458         desc_pa = vring_desc_addr(desc_pa, i);
 459         i = 0;
 460     }
 461
 462     /* Collect all the descriptors */
 463     do {
 464         struct iovec *sg;
 465
 466         if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_WRITE) {
 467             if (elem->in_num >= ARRAY_SIZE(elem->in_sg)) {
 468                 error_report("Too many write descriptors in indirect table");
 469                 exit(1);
 470             }
 471             elem->in_addr[elem->in_num] = vring_desc_addr(desc_pa, i);
 472             sg = &elem->in_sg[elem->in_num++];
 473         } else {
 474             if (elem->out_num >= ARRAY_SIZE(elem->out_sg)) {
 475                 error_report("Too many read descriptors in indirect table");
 476                 exit(1);
 477             }
 478             elem->out_addr[elem->out_num] = vring_desc_addr(desc_pa, i);
 479             sg = &elem->out_sg[elem->out_num++];
 480         }
 481
 482         sg->iov_len = vring_desc_len(desc_pa, i);
 483
 484         /* If we've got too many, that implies a descriptor loop. */
 485         if ((elem->in_num + elem->out_num) > max) {
 486             error_report("Looped descriptor");
 487             exit(1);
 488         }
 489     } while ((i = virtqueue_next_desc(desc_pa, i, max)) != max);
 490
 491     /* Now map what we have collected */
 492     virtqueue_map_sg(elem->in_sg, elem->in_addr, elem->in_num, 1);
 493     virtqueue_map_sg(elem->out_sg, elem->out_addr, elem->out_num, 0);
 494
 495     elem->index = head;
 496
 497     vq->inuse++;
 498
 499     trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
 500     return elem->in_num + elem->out_num;
 501 }
 502
 503 /* virtio device */
 504 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector)
 505 {
 506     if (vdev->binding->notify) {
 507         vdev->binding->notify(vdev->binding_opaque, vector);
 508     }
 509 }
 510
 511 void virtio_update_irq(VirtIODevice *vdev)
 512 {
 513     virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
 514 }
 515
 516 void virtio_set_status(VirtIODevice *vdev, uint8_t val)
 517 {
 518     trace_virtio_set_status(vdev, val);
 519
 520     if (vdev->set_status) {
 521         vdev->set_status(vdev, val);
 522     }
 523     vdev->status = val;
 524 }
 525
 526 void virtio_reset(void *opaque)
 527 {
 528     VirtIODevice *vdev = opaque;
 529     int i;
 530
 531     virtio_set_status(vdev, 0);
 532
 533     if (vdev->reset)
 534         vdev->reset(vdev);
 535
 536     vdev->guest_features = 0;
 537     vdev->queue_sel = 0;
 538     vdev->status = 0;
 539     vdev->isr = 0;
 540     vdev->config_vector = VIRTIO_NO_VECTOR;
 541     virtio_notify_vector(vdev, vdev->config_vector);
 542
 543     for(i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
 544         vdev->vq[i].vring.desc = 0;
 545         vdev->vq[i].vring.avail = 0;
 546         vdev->vq[i].vring.used = 0;
 547         vdev->vq[i].last_avail_idx = 0;
 548         vdev->vq[i].pa = 0;
 549         vdev->vq[i].vector = VIRTIO_NO_VECTOR;
 550         vdev->vq[i].signalled_used = 0;
 551         vdev->vq[i].signalled_used_valid = false;
 552         vdev->vq[i].notification = true;
 553     }
 554 }
 555
 556 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr)
 557 {
 558     uint8_t val;
 559
 560     vdev->get_config(vdev, vdev->config);
 561
 562     if (addr > (vdev->config_len - sizeof(val)))
 563         return (uint32_t)-1;
 564
 565     val = ldub_p(vdev->config + addr);
 566     return val;
 567 }
 568
 569 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr)
 570 {
 571     uint16_t val;
 572
 573     vdev->get_config(vdev, vdev->config);
 574
 575     if (addr > (vdev->config_len - sizeof(val)))
 576         return (uint32_t)-1;
 577
 578     val = lduw_p(vdev->config + addr);
 579     return val;
 580 }
 581
 582 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr)
 583 {
 584     uint32_t val;
 585
 586     vdev->get_config(vdev, vdev->config);
 587
 588     if (addr > (vdev->config_len - sizeof(val)))
 589         return (uint32_t)-1;
 590
 591     val = ldl_p(vdev->config + addr);
 592     return val;
 593 }
 594
 595 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data)
 596 {
 597     uint8_t val = data;
 598
 599     if (addr > (vdev->config_len - sizeof(val)))
 600         return;
 601
 602     stb_p(vdev->config + addr, val);
 603
 604     if (vdev->set_config)
 605         vdev->set_config(vdev, vdev->config);
 606 }
 607
 608 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data)
 609 {
 610     uint16_t val = data;
 611
 612     if (addr > (vdev->config_len - sizeof(val)))
 613         return;
 614
 615     stw_p(vdev->config + addr, val);
 616
 617     if (vdev->set_config)
 618         vdev->set_config(vdev, vdev->config);
 619 }
 620
 621 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
 622 {
 623     uint32_t val = data;
 624
 625     if (addr > (vdev->config_len - sizeof(val)))
 626         return;
 627
 628     stl_p(vdev->config + addr, val);
 629
 630     if (vdev->set_config)
 631         vdev->set_config(vdev, vdev->config);
 632 }
 633
 634 void virtio_queue_set_addr(VirtIODevice *vdev, int n, target_phys_addr_t addr)
 635 {
 636     vdev->vq[n].pa = addr;
 637     virtqueue_init(&vdev->vq[n]);
 638 }
 639
 640 target_phys_addr_t virtio_queue_get_addr(VirtIODevice *vdev, int n)
 641 {
 642     return vdev->vq[n].pa;
 643 }
 644
 645 int virtio_queue_get_num(VirtIODevice *vdev, int n)
 646 {
 647     return vdev->vq[n].vring.num;
 648 }
 649
 650 int virtio_queue_get_id(VirtQueue *vq)
 651 {
 652     VirtIODevice *vdev = vq->vdev;
 653     assert(vq >= &vdev->vq[0] && vq < &vdev->vq[VIRTIO_PCI_QUEUE_MAX]);
 654     return vq - &vdev->vq[0];
 655 }
 656
 657 void virtio_queue_notify_vq(VirtQueue *vq)
 658 {
 659     if (vq->vring.desc) {
 660         VirtIODevice *vdev = vq->vdev;
 661         trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
 662         vq->handle_output(vdev, vq);
 663     }
 664 }
 665
 666 void virtio_queue_notify(VirtIODevice *vdev, int n)
 667 {
 668     virtio_queue_notify_vq(&vdev->vq[n]);
 669 }
 670
 671 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
 672 {
 673     return n < VIRTIO_PCI_QUEUE_MAX ? vdev->vq[n].vector :
 674         VIRTIO_NO_VECTOR;
 675 }
 676
 677 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
 678 {
 679     if (n < VIRTIO_PCI_QUEUE_MAX)
 680         vdev->vq[n].vector = vector;
 681 }
 682
 683 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
 684                             void (*handle_output)(VirtIODevice *, VirtQueue *))
 685 {
 686     int i;
 687
 688     for (i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
 689         if (vdev->vq[i].vring.num == 0)
 690             break;
 691     }
 692
 693     if (i == VIRTIO_PCI_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
 694         abort();
 695
 696     vdev->vq[i].vring.num = queue_size;
 697     vdev->vq[i].handle_output = handle_output;
 698
 699     return &vdev->vq[i];
 700 }
 701
 702 void virtio_irq(VirtQueue *vq)
 703 {
 704     trace_virtio_irq(vq);
 705     vq->vdev->isr |= 0x01;
 706     virtio_notify_vector(vq->vdev, vq->vector);
 707 }
 708
 709 /* Assuming a given event_idx value from the other size, if
 710  * we have just incremented index from old to new_idx,
 711  * should we trigger an event? */
 712 static inline int vring_need_event(uint16_t event, uint16_t new, uint16_t old)
 713 {
 714         /* Note: Xen has similar logic for notification hold-off
 715          * in include/xen/interface/io/ring.h with req_event and req_prod
 716          * corresponding to event_idx + 1 and new respectively.
 717          * Note also that req_event and req_prod in Xen start at 1,
 718          * event indexes in virtio start at 0. */
 719         return (uint16_t)(new - event - 1) < (uint16_t)(new - old);
 720 }
 721
 722 static bool vring_notify(VirtIODevice *vdev, VirtQueue *vq)
 723 {
 724     uint16_t old, new;
 725     bool v;
 726     /* We need to expose used array entries before checking used event. */
 727     smp_mb();
 728     /* Always notify when queue is empty (when feature acknowledge) */
 729     if (((vdev->guest_features & (1 << VIRTIO_F_NOTIFY_ON_EMPTY)) &&
 730          !vq->inuse && vring_avail_idx(vq) == vq->last_avail_idx)) {
 731         return true;
 732     }
 733
 734     if (!(vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX))) {
 735         return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
 736     }
 737
 738     v = vq->signalled_used_valid;
 739     vq->signalled_used_valid = true;
 740     old = vq->signalled_used;
 741     new = vq->signalled_used = vring_used_idx(vq);
 742     return !v || vring_need_event(vring_used_event(vq), new, old);
 743 }
 744
 745 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
 746 {
 747     if (!vring_notify(vdev, vq)) {
 748         return;
 749     }
 750
 751     trace_virtio_notify(vdev, vq);
 752     vdev->isr |= 0x01;
 753     virtio_notify_vector(vdev, vq->vector);
 754 }
 755
 756 void virtio_notify_config(VirtIODevice *vdev)
 757 {
 758     if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
 759         return;
 760
 761     vdev->isr |= 0x03;
 762     virtio_notify_vector(vdev, vdev->config_vector);
 763 }
 764
 765 void virtio_save(VirtIODevice *vdev, QEMUFile *f)
 766 {
 767     int i;
 768
 769     if (vdev->binding->save_config)
 770         vdev->binding->save_config(vdev->binding_opaque, f);
 771
 772     qemu_put_8s(f, &vdev->status);
 773     qemu_put_8s(f, &vdev->isr);
 774     qemu_put_be16s(f, &vdev->queue_sel);
 775     qemu_put_be32s(f, &vdev->guest_features);
 776     qemu_put_be32(f, vdev->config_len);
 777     qemu_put_buffer(f, vdev->config, vdev->config_len);
 778
 779     for (i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
 780         if (vdev->vq[i].vring.num == 0)
 781             break;
 782     }
 783
 784     qemu_put_be32(f, i);
 785
 786     for (i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
 787         if (vdev->vq[i].vring.num == 0)
 788             break;
 789
 790         qemu_put_be32(f, vdev->vq[i].vring.num);
 791         qemu_put_be64(f, vdev->vq[i].pa);
 792         qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
 793         if (vdev->binding->save_queue)
 794             vdev->binding->save_queue(vdev->binding_opaque, i, f);
 795     }
 796 }
 797
 798 int virtio_set_features(VirtIODevice *vdev, uint32_t val)
 799 {
 800     uint32_t supported_features =
 801         vdev->binding->get_features(vdev->binding_opaque);
 802     bool bad = (val & ~supported_features) != 0;
 803
 804     val &= supported_features;
 805     if (vdev->set_features) {
 806         vdev->set_features(vdev, val);
 807     }
 808     vdev->guest_features = val;
 809     return bad ? -1 : 0;
 810 }
 811
 812 int virtio_load(VirtIODevice *vdev, QEMUFile *f)
 813 {
 814     int num, i, ret;
 815     uint32_t features;
 816     uint32_t supported_features;
 817
 818     if (vdev->binding->load_config) {
 819         ret = vdev->binding->load_config(vdev->binding_opaque, f);
 820         if (ret)
 821             return ret;
 822     }
 823
 824     qemu_get_8s(f, &vdev->status);
 825     qemu_get_8s(f, &vdev->isr);
 826     qemu_get_be16s(f, &vdev->queue_sel);
 827     qemu_get_be32s(f, &features);
 828
 829     if (virtio_set_features(vdev, features) < 0) {
 830         supported_features = vdev->binding->get_features(vdev->binding_opaque);
 831         error_report("Features 0x%x unsupported. Allowed features: 0x%x",
 832                      features, supported_features);
 833         return -1;
 834     }
 835     vdev->config_len = qemu_get_be32(f);
 836     qemu_get_buffer(f, vdev->config, vdev->config_len);
 837
 838     num = qemu_get_be32(f);
 839
 840     for (i = 0; i < num; i++) {
 841         vdev->vq[i].vring.num = qemu_get_be32(f);
 842         vdev->vq[i].pa = qemu_get_be64(f);
 843         qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
 844         vdev->vq[i].signalled_used_valid = false;
 845         vdev->vq[i].notification = true;
 846
 847         if (vdev->vq[i].pa) {
 848             uint16_t nheads;
 849             virtqueue_init(&vdev->vq[i]);
 850             nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
 851             /* Check it isn't doing very strange things with descriptor numbers. */
 852             if (nheads > vdev->vq[i].vring.num) {
 853                 error_report("VQ %d size 0x%x Guest index 0x%x "
 854                              "inconsistent with Host index 0x%x: delta 0x%x",
 855                              i, vdev->vq[i].vring.num,
 856                              vring_avail_idx(&vdev->vq[i]),
 857                              vdev->vq[i].last_avail_idx, nheads);
 858                 return -1;
 859             }
 860         } else if (vdev->vq[i].last_avail_idx) {
 861             error_report("VQ %d address 0x0 "
 862                          "inconsistent with Host index 0x%x",
 863                          i, vdev->vq[i].last_avail_idx);
 864                 return -1;
 865         }
 866         if (vdev->binding->load_queue) {
 867             ret = vdev->binding->load_queue(vdev->binding_opaque, i, f);
 868             if (ret)
 869                 return ret;
 870         }
 871     }
 872
 873     virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
 874     return 0;
 875 }
 876
 877 void virtio_cleanup(VirtIODevice *vdev)
 878 {
 879     qemu_del_vm_change_state_handler(vdev->vmstate);
 880     g_free(vdev->config);
 881     g_free(vdev->vq);
 882     g_free(vdev);
 883 }
 884
 885 static void virtio_vmstate_change(void *opaque, int running, RunState state)
 886 {
 887     VirtIODevice *vdev = opaque;
 888     bool backend_run = running && (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK);
 889     vdev->vm_running = running;
 890
 891     if (backend_run) {
 892         virtio_set_status(vdev, vdev->status);
 893     }
 894
 895     if (vdev->binding->vmstate_change) {
 896         vdev->binding->vmstate_change(vdev->binding_opaque, backend_run);
 897     }
 898
 899     if (!backend_run) {
 900         virtio_set_status(vdev, vdev->status);
 901     }
 902 }
 903
 904 VirtIODevice *virtio_common_init(const char *name, uint16_t device_id,
 905                                  size_t config_size, size_t struct_size)
 906 {
 907     VirtIODevice *vdev;
 908     int i;
 909
 910     vdev = g_malloc0(struct_size);
 911
 912     vdev->device_id = device_id;
 913     vdev->status = 0;
 914     vdev->isr = 0;
 915     vdev->queue_sel = 0;
 916     vdev->config_vector = VIRTIO_NO_VECTOR;
 917     vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_PCI_QUEUE_MAX);
 918     vdev->vm_running = runstate_is_running();
 919     for(i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
 920         vdev->vq[i].vector = VIRTIO_NO_VECTOR;
 921         vdev->vq[i].vdev = vdev;
 922     }
 923
 924     vdev->name = name;
 925     vdev->config_len = config_size;
 926     if (vdev->config_len)
 927         vdev->config = g_malloc0(config_size);
 928     else
 929         vdev->config = NULL;
 930
 931     vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change, vdev);
 932
 933     return vdev;
 934 }
 935
 936 void virtio_bind_device(VirtIODevice *vdev, const VirtIOBindings *binding,
 937                         void *opaque)
 938 {
 939     vdev->binding = binding;
 940     vdev->binding_opaque = opaque;
 941 }
 942
 943 target_phys_addr_t virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
 944 {
 945     return vdev->vq[n].vring.desc;
 946 }
 947
 948 target_phys_addr_t virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
 949 {
 950     return vdev->vq[n].vring.avail;
 951 }
 952
 953 target_phys_addr_t virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
 954 {
 955     return vdev->vq[n].vring.used;
 956 }
 957
 958 target_phys_addr_t virtio_queue_get_ring_addr(VirtIODevice *vdev, int n)
 959 {
 960     return vdev->vq[n].vring.desc;
 961 }
 962
 963 target_phys_addr_t virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
 964 {
 965     return sizeof(VRingDesc) * vdev->vq[n].vring.num;
 966 }
 967
 968 target_phys_addr_t virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
 969 {
 970     return offsetof(VRingAvail, ring) +
 971         sizeof(uint64_t) * vdev->vq[n].vring.num;
 972 }
 973
 974 target_phys_addr_t virtio_queue_get_used_size(VirtIODevice *vdev, int n)
 975 {
 976     return offsetof(VRingUsed, ring) +
 977         sizeof(VRingUsedElem) * vdev->vq[n].vring.num;
 978 }
 979
 980 target_phys_addr_t virtio_queue_get_ring_size(VirtIODevice *vdev, int n)
 981 {
 982     return vdev->vq[n].vring.used - vdev->vq[n].vring.desc +
 983             virtio_queue_get_used_size(vdev, n);
 984 }
 985
 986 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
 987 {
 988     return vdev->vq[n].last_avail_idx;
 989 }
 990
 991 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx)
 992 {
 993     vdev->vq[n].last_avail_idx = idx;
 994 }
 995
 996 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)
 997 {
 998     return vdev->vq + n;
 999 }
1000
1001 static void virtio_queue_guest_notifier_read(EventNotifier *n)
1002 {
1003     VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
1004     if (event_notifier_test_and_clear(n)) {
1005         virtio_irq(vq);
1006     }
1007 }
1008
1009 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
1010                                                 bool with_irqfd)
1011 {
1012     if (assign && !with_irqfd) {
1013         event_notifier_set_handler(&vq->guest_notifier,
1014                                    virtio_queue_guest_notifier_read);
1015     } else {
1016         event_notifier_set_handler(&vq->guest_notifier, NULL);
1017     }
1018     if (!assign) {
1019         /* Test and clear notifier before closing it,
1020          * in case poll callback didn't have time to run. */
1021         virtio_queue_guest_notifier_read(&vq->guest_notifier);
1022     }
1023 }
1024
1025 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
1026 {
1027     return &vq->guest_notifier;
1028 }
1029
1030 static void virtio_queue_host_notifier_read(EventNotifier *n)
1031 {
1032     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
1033     if (event_notifier_test_and_clear(n)) {
1034         virtio_queue_notify_vq(vq);
1035     }
1036 }
1037
1038 void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
1039                                                bool set_handler)
1040 {
1041     if (assign && set_handler) {
1042         event_notifier_set_handler(&vq->host_notifier,
1043                                    virtio_queue_host_notifier_read);
1044     } else {
1045         event_notifier_set_handler(&vq->host_notifier, NULL);
1046     }
1047     if (!assign) {
1048         /* Test and clear notifier before after disabling event,
1049          * in case poll callback didn't have time to run. */
1050         virtio_queue_host_notifier_read(&vq->host_notifier);
1051     }
1052 }
1053
1054 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
1055 {
1056     return &vq->host_notifier;
1057 }