hw/virtio/virtio.c

   1 /*
   2  * Virtio Support
   3  *
   4  * Copyright IBM, Corp. 2007
   5  *
   6  * Authors:
   7  *  Anthony Liguori   <aliguori@us.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  */
  13
  14 #include "qemu/osdep.h"
  15
  16 #include "trace.h"
  17 #include "exec/address-spaces.h"
  18 #include "qemu/error-report.h"
  19 #include "hw/virtio/virtio.h"
  20 #include "qemu/atomic.h"
  21 #include "hw/virtio/virtio-bus.h"
  22 #include "migration/migration.h"
  23 #include "hw/virtio/virtio-access.h"
  24
  25 /*
  26  * The alignment to use between consumer and producer parts of vring.
  27  * x86 pagesize again. This is the default, used by transports like PCI
  28  * which don't provide a means for the guest to tell the host the alignment.
  29  */
  30 #define VIRTIO_PCI_VRING_ALIGN         4096
  31
  32 typedef struct VRingDesc
  33 {
  34     uint64_t addr;
  35     uint32_t len;
  36     uint16_t flags;
  37     uint16_t next;
  38 } VRingDesc;
  39
  40 typedef struct VRingAvail
  41 {
  42     uint16_t flags;
  43     uint16_t idx;
  44     uint16_t ring[0];
  45 } VRingAvail;
  46
  47 typedef struct VRingUsedElem
  48 {
  49     uint32_t id;
  50     uint32_t len;
  51 } VRingUsedElem;
  52
  53 typedef struct VRingUsed
  54 {
  55     uint16_t flags;
  56     uint16_t idx;
  57     VRingUsedElem ring[0];
  58 } VRingUsed;
  59
  60 typedef struct VRing
  61 {
  62     unsigned int num;
  63     unsigned int num_default;
  64     unsigned int align;
  65     hwaddr desc;
  66     hwaddr avail;
  67     hwaddr used;
  68 } VRing;
  69
  70 struct VirtQueue
  71 {
  72     VRing vring;
  73     uint16_t last_avail_idx;
  74     /* Last used index value we have signalled on */
  75     uint16_t signalled_used;
  76
  77     /* Last used index value we have signalled on */
  78     bool signalled_used_valid;
  79
  80     /* Notification enabled? */
  81     bool notification;
  82
  83     uint16_t queue_index;
  84
  85     int inuse;
  86
  87     uint16_t vector;
  88     void (*handle_output)(VirtIODevice *vdev, VirtQueue *vq);
  89     VirtIODevice *vdev;
  90     EventNotifier guest_notifier;
  91     EventNotifier host_notifier;
  92     QLIST_ENTRY(VirtQueue) node;
  93 };
  94
  95 /* virt queue functions */
  96 void virtio_queue_update_rings(VirtIODevice *vdev, int n)
  97 {
  98     VRing *vring = &vdev->vq[n].vring;
  99
 100     if (!vring->desc) {
 101         /* not yet setup -> nothing to do */
 102         return;
 103     }
 104     vring->avail = vring->desc + vring->num * sizeof(VRingDesc);
 105     vring->used = vring_align(vring->avail +
 106                               offsetof(VRingAvail, ring[vring->num]),
 107                               vring->align);
 108 }
 109
 110 static void vring_desc_read(VirtIODevice *vdev, VRingDesc *desc,
 111                             hwaddr desc_pa, int i)
 112 {
 113     address_space_read(&address_space_memory, desc_pa + i * sizeof(VRingDesc),
 114                        MEMTXATTRS_UNSPECIFIED, (void *)desc, sizeof(VRingDesc));
 115     virtio_tswap64s(vdev, &desc->addr);
 116     virtio_tswap32s(vdev, &desc->len);
 117     virtio_tswap16s(vdev, &desc->flags);
 118     virtio_tswap16s(vdev, &desc->next);
 119 }
 120
 121 static inline uint16_t vring_avail_flags(VirtQueue *vq)
 122 {
 123     hwaddr pa;
 124     pa = vq->vring.avail + offsetof(VRingAvail, flags);
 125     return virtio_lduw_phys(vq->vdev, pa);
 126 }
 127
 128 static inline uint16_t vring_avail_idx(VirtQueue *vq)
 129 {
 130     hwaddr pa;
 131     pa = vq->vring.avail + offsetof(VRingAvail, idx);
 132     return virtio_lduw_phys(vq->vdev, pa);
 133 }
 134
 135 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
 136 {
 137     hwaddr pa;
 138     pa = vq->vring.avail + offsetof(VRingAvail, ring[i]);
 139     return virtio_lduw_phys(vq->vdev, pa);
 140 }
 141
 142 static inline uint16_t vring_get_used_event(VirtQueue *vq)
 143 {
 144     return vring_avail_ring(vq, vq->vring.num);
 145 }
 146
 147 static inline void vring_used_ring_id(VirtQueue *vq, int i, uint32_t val)
 148 {
 149     hwaddr pa;
 150     pa = vq->vring.used + offsetof(VRingUsed, ring[i].id);
 151     virtio_stl_phys(vq->vdev, pa, val);
 152 }
 153
 154 static inline void vring_used_ring_len(VirtQueue *vq, int i, uint32_t val)
 155 {
 156     hwaddr pa;
 157     pa = vq->vring.used + offsetof(VRingUsed, ring[i].len);
 158     virtio_stl_phys(vq->vdev, pa, val);
 159 }
 160
 161 static uint16_t vring_used_idx(VirtQueue *vq)
 162 {
 163     hwaddr pa;
 164     pa = vq->vring.used + offsetof(VRingUsed, idx);
 165     return virtio_lduw_phys(vq->vdev, pa);
 166 }
 167
 168 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
 169 {
 170     hwaddr pa;
 171     pa = vq->vring.used + offsetof(VRingUsed, idx);
 172     virtio_stw_phys(vq->vdev, pa, val);
 173 }
 174
 175 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
 176 {
 177     VirtIODevice *vdev = vq->vdev;
 178     hwaddr pa;
 179     pa = vq->vring.used + offsetof(VRingUsed, flags);
 180     virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) | mask);
 181 }
 182
 183 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
 184 {
 185     VirtIODevice *vdev = vq->vdev;
 186     hwaddr pa;
 187     pa = vq->vring.used + offsetof(VRingUsed, flags);
 188     virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) & ~mask);
 189 }
 190
 191 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)
 192 {
 193     hwaddr pa;
 194     if (!vq->notification) {
 195         return;
 196     }
 197     pa = vq->vring.used + offsetof(VRingUsed, ring[vq->vring.num]);
 198     virtio_stw_phys(vq->vdev, pa, val);
 199 }
 200
 201 void virtio_queue_set_notification(VirtQueue *vq, int enable)
 202 {
 203     vq->notification = enable;
 204     if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
 205         vring_set_avail_event(vq, vring_avail_idx(vq));
 206     } else if (enable) {
 207         vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
 208     } else {
 209         vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
 210     }
 211     if (enable) {
 212         /* Expose avail event/used flags before caller checks the avail idx. */
 213         smp_mb();
 214     }
 215 }
 216
 217 int virtio_queue_ready(VirtQueue *vq)
 218 {
 219     return vq->vring.avail != 0;
 220 }
 221
 222 int virtio_queue_empty(VirtQueue *vq)
 223 {
 224     return vring_avail_idx(vq) == vq->last_avail_idx;
 225 }
 226
 227 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem,
 228                                unsigned int len)
 229 {
 230     unsigned int offset;
 231     int i;
 232
 233     offset = 0;
 234     for (i = 0; i < elem->in_num; i++) {
 235         size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
 236
 237         cpu_physical_memory_unmap(elem->in_sg[i].iov_base,
 238                                   elem->in_sg[i].iov_len,
 239                                   1, size);
 240
 241         offset += size;
 242     }
 243
 244     for (i = 0; i < elem->out_num; i++)
 245         cpu_physical_memory_unmap(elem->out_sg[i].iov_base,
 246                                   elem->out_sg[i].iov_len,
 247                                   0, elem->out_sg[i].iov_len);
 248 }
 249
 250 void virtqueue_discard(VirtQueue *vq, const VirtQueueElement *elem,
 251                        unsigned int len)
 252 {
 253     vq->last_avail_idx--;
 254     virtqueue_unmap_sg(vq, elem, len);
 255 }
 256
 257 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
 258                     unsigned int len, unsigned int idx)
 259 {
 260     trace_virtqueue_fill(vq, elem, len, idx);
 261
 262     virtqueue_unmap_sg(vq, elem, len);
 263
 264     idx = (idx + vring_used_idx(vq)) % vq->vring.num;
 265
 266     /* Get a pointer to the next entry in the used ring. */
 267     vring_used_ring_id(vq, idx, elem->index);
 268     vring_used_ring_len(vq, idx, len);
 269 }
 270
 271 void virtqueue_flush(VirtQueue *vq, unsigned int count)
 272 {
 273     uint16_t old, new;
 274     /* Make sure buffer is written before we update index. */
 275     smp_wmb();
 276     trace_virtqueue_flush(vq, count);
 277     old = vring_used_idx(vq);
 278     new = old + count;
 279     vring_used_idx_set(vq, new);
 280     vq->inuse -= count;
 281     if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)))
 282         vq->signalled_used_valid = false;
 283 }
 284
 285 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
 286                     unsigned int len)
 287 {
 288     virtqueue_fill(vq, elem, len, 0);
 289     virtqueue_flush(vq, 1);
 290 }
 291
 292 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
 293 {
 294     uint16_t num_heads = vring_avail_idx(vq) - idx;
 295
 296     /* Check it isn't doing very strange things with descriptor numbers. */
 297     if (num_heads > vq->vring.num) {
 298         error_report("Guest moved used index from %u to %u",
 299                      idx, vring_avail_idx(vq));
 300         exit(1);
 301     }
 302     /* On success, callers read a descriptor at vq->last_avail_idx.
 303      * Make sure descriptor read does not bypass avail index read. */
 304     if (num_heads) {
 305         smp_rmb();
 306     }
 307
 308     return num_heads;
 309 }
 310
 311 static unsigned int virtqueue_get_head(VirtQueue *vq, unsigned int idx)
 312 {
 313     unsigned int head;
 314
 315     /* Grab the next descriptor number they're advertising, and increment
 316      * the index we've seen. */
 317     head = vring_avail_ring(vq, idx % vq->vring.num);
 318
 319     /* If their number is silly, that's a fatal mistake. */
 320     if (head >= vq->vring.num) {
 321         error_report("Guest says index %u is available", head);
 322         exit(1);
 323     }
 324
 325     return head;
 326 }
 327
 328 static unsigned virtqueue_read_next_desc(VirtIODevice *vdev, VRingDesc *desc,
 329                                          hwaddr desc_pa, unsigned int max)
 330 {
 331     unsigned int next;
 332
 333     /* If this descriptor says it doesn't chain, we're done. */
 334     if (!(desc->flags & VRING_DESC_F_NEXT)) {
 335         return max;
 336     }
 337
 338     /* Check they're not leading us off end of descriptors. */
 339     next = desc->next;
 340     /* Make sure compiler knows to grab that: we don't want it changing! */
 341     smp_wmb();
 342
 343     if (next >= max) {
 344         error_report("Desc next is %u", next);
 345         exit(1);
 346     }
 347
 348     vring_desc_read(vdev, desc, desc_pa, next);
 349     return next;
 350 }
 351
 352 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
 353                                unsigned int *out_bytes,
 354                                unsigned max_in_bytes, unsigned max_out_bytes)
 355 {
 356     unsigned int idx;
 357     unsigned int total_bufs, in_total, out_total;
 358
 359     idx = vq->last_avail_idx;
 360
 361     total_bufs = in_total = out_total = 0;
 362     while (virtqueue_num_heads(vq, idx)) {
 363         VirtIODevice *vdev = vq->vdev;
 364         unsigned int max, num_bufs, indirect = 0;
 365         VRingDesc desc;
 366         hwaddr desc_pa;
 367         int i;
 368
 369         max = vq->vring.num;
 370         num_bufs = total_bufs;
 371         i = virtqueue_get_head(vq, idx++);
 372         desc_pa = vq->vring.desc;
 373         vring_desc_read(vdev, &desc, desc_pa, i);
 374
 375         if (desc.flags & VRING_DESC_F_INDIRECT) {
 376             if (desc.len % sizeof(VRingDesc)) {
 377                 error_report("Invalid size for indirect buffer table");
 378                 exit(1);
 379             }
 380
 381             /* If we've got too many, that implies a descriptor loop. */
 382             if (num_bufs >= max) {
 383                 error_report("Looped descriptor");
 384                 exit(1);
 385             }
 386
 387             /* loop over the indirect descriptor table */
 388             indirect = 1;
 389             max = desc.len / sizeof(VRingDesc);
 390             desc_pa = desc.addr;
 391             num_bufs = i = 0;
 392             vring_desc_read(vdev, &desc, desc_pa, i);
 393         }
 394
 395         do {
 396             /* If we've got too many, that implies a descriptor loop. */
 397             if (++num_bufs > max) {
 398                 error_report("Looped descriptor");
 399                 exit(1);
 400             }
 401
 402             if (desc.flags & VRING_DESC_F_WRITE) {
 403                 in_total += desc.len;
 404             } else {
 405                 out_total += desc.len;
 406             }
 407             if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
 408                 goto done;
 409             }
 410         } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max);
 411
 412         if (!indirect)
 413             total_bufs = num_bufs;
 414         else
 415             total_bufs++;
 416     }
 417 done:
 418     if (in_bytes) {
 419         *in_bytes = in_total;
 420     }
 421     if (out_bytes) {
 422         *out_bytes = out_total;
 423     }
 424 }
 425
 426 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
 427                           unsigned int out_bytes)
 428 {
 429     unsigned int in_total, out_total;
 430
 431     virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
 432     return in_bytes <= in_total && out_bytes <= out_total;
 433 }
 434
 435 static void virtqueue_map_desc(unsigned int *p_num_sg, hwaddr *addr, struct iovec *iov,
 436                                unsigned int max_num_sg, bool is_write,
 437                                hwaddr pa, size_t sz)
 438 {
 439     unsigned num_sg = *p_num_sg;
 440     assert(num_sg <= max_num_sg);
 441
 442     while (sz) {
 443         hwaddr len = sz;
 444
 445         if (num_sg == max_num_sg) {
 446             error_report("virtio: too many write descriptors in indirect table");
 447             exit(1);
 448         }
 449
 450         iov[num_sg].iov_base = cpu_physical_memory_map(pa, &len, is_write);
 451         iov[num_sg].iov_len = len;
 452         addr[num_sg] = pa;
 453
 454         sz -= len;
 455         pa += len;
 456         num_sg++;
 457     }
 458     *p_num_sg = num_sg;
 459 }
 460
 461 static void virtqueue_map_iovec(struct iovec *sg, hwaddr *addr,
 462                                 unsigned int *num_sg, unsigned int max_size,
 463                                 int is_write)
 464 {
 465     unsigned int i;
 466     hwaddr len;
 467
 468     /* Note: this function MUST validate input, some callers
 469      * are passing in num_sg values received over the network.
 470      */
 471     /* TODO: teach all callers that this can fail, and return failure instead
 472      * of asserting here.
 473      * When we do, we might be able to re-enable NDEBUG below.
 474      */
 475 #ifdef NDEBUG
 476 #error building with NDEBUG is not supported
 477 #endif
 478     assert(*num_sg <= max_size);
 479
 480     for (i = 0; i < *num_sg; i++) {
 481         len = sg[i].iov_len;
 482         sg[i].iov_base = cpu_physical_memory_map(addr[i], &len, is_write);
 483         if (!sg[i].iov_base) {
 484             error_report("virtio: error trying to map MMIO memory");
 485             exit(1);
 486         }
 487         if (len != sg[i].iov_len) {
 488             error_report("virtio: unexpected memory split");
 489             exit(1);
 490         }
 491     }
 492 }
 493
 494 void virtqueue_map(VirtQueueElement *elem)
 495 {
 496     virtqueue_map_iovec(elem->in_sg, elem->in_addr, &elem->in_num,
 497                         VIRTQUEUE_MAX_SIZE, 1);
 498     virtqueue_map_iovec(elem->out_sg, elem->out_addr, &elem->out_num,
 499                         VIRTQUEUE_MAX_SIZE, 0);
 500 }
 501
 502 void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num)
 503 {
 504     VirtQueueElement *elem;
 505     size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0]));
 506     size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]);
 507     size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]);
 508     size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0]));
 509     size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
 510     size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
 511
 512     assert(sz >= sizeof(VirtQueueElement));
 513     elem = g_malloc(out_sg_end);
 514     elem->out_num = out_num;
 515     elem->in_num = in_num;
 516     elem->in_addr = (void *)elem + in_addr_ofs;
 517     elem->out_addr = (void *)elem + out_addr_ofs;
 518     elem->in_sg = (void *)elem + in_sg_ofs;
 519     elem->out_sg = (void *)elem + out_sg_ofs;
 520     return elem;
 521 }
 522
 523 void *virtqueue_pop(VirtQueue *vq, size_t sz)
 524 {
 525     unsigned int i, head, max;
 526     hwaddr desc_pa = vq->vring.desc;
 527     VirtIODevice *vdev = vq->vdev;
 528     VirtQueueElement *elem;
 529     unsigned out_num, in_num;
 530     hwaddr addr[VIRTQUEUE_MAX_SIZE];
 531     struct iovec iov[VIRTQUEUE_MAX_SIZE];
 532     VRingDesc desc;
 533
 534     if (!virtqueue_num_heads(vq, vq->last_avail_idx)) {
 535         return NULL;
 536     }
 537
 538     /* When we start there are none of either input nor output. */
 539     out_num = in_num = 0;
 540
 541     max = vq->vring.num;
 542
 543     i = head = virtqueue_get_head(vq, vq->last_avail_idx++);
 544     if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
 545         vring_set_avail_event(vq, vq->last_avail_idx);
 546     }
 547
 548     vring_desc_read(vdev, &desc, desc_pa, i);
 549     if (desc.flags & VRING_DESC_F_INDIRECT) {
 550         if (desc.len % sizeof(VRingDesc)) {
 551             error_report("Invalid size for indirect buffer table");
 552             exit(1);
 553         }
 554
 555         /* loop over the indirect descriptor table */
 556         max = desc.len / sizeof(VRingDesc);
 557         desc_pa = desc.addr;
 558         i = 0;
 559         vring_desc_read(vdev, &desc, desc_pa, i);
 560     }
 561
 562     /* Collect all the descriptors */
 563     do {
 564         if (desc.flags & VRING_DESC_F_WRITE) {
 565             virtqueue_map_desc(&in_num, addr + out_num, iov + out_num,
 566                                VIRTQUEUE_MAX_SIZE - out_num, true, desc.addr, desc.len);
 567         } else {
 568             if (in_num) {
 569                 error_report("Incorrect order for descriptors");
 570                 exit(1);
 571             }
 572             virtqueue_map_desc(&out_num, addr, iov,
 573                                VIRTQUEUE_MAX_SIZE, false, desc.addr, desc.len);
 574         }
 575
 576         /* If we've got too many, that implies a descriptor loop. */
 577         if ((in_num + out_num) > max) {
 578             error_report("Looped descriptor");
 579             exit(1);
 580         }
 581     } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max);
 582
 583     /* Now copy what we have collected and mapped */
 584     elem = virtqueue_alloc_element(sz, out_num, in_num);
 585     elem->index = head;
 586     for (i = 0; i < out_num; i++) {
 587         elem->out_addr[i] = addr[i];
 588         elem->out_sg[i] = iov[i];
 589     }
 590     for (i = 0; i < in_num; i++) {
 591         elem->in_addr[i] = addr[out_num + i];
 592         elem->in_sg[i] = iov[out_num + i];
 593     }
 594
 595     vq->inuse++;
 596
 597     trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
 598     return elem;
 599 }
 600
 601 /* Reading and writing a structure directly to QEMUFile is *awful*, but
 602  * it is what QEMU has always done by mistake.  We can change it sooner
 603  * or later by bumping the version number of the affected vm states.
 604  * In the meanwhile, since the in-memory layout of VirtQueueElement
 605  * has changed, we need to marshal to and from the layout that was
 606  * used before the change.
 607  */
 608 typedef struct VirtQueueElementOld {
 609     unsigned int index;
 610     unsigned int out_num;
 611     unsigned int in_num;
 612     hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
 613     hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
 614     struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
 615     struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
 616 } VirtQueueElementOld;
 617
 618 void *qemu_get_virtqueue_element(QEMUFile *f, size_t sz)
 619 {
 620     VirtQueueElement *elem;
 621     VirtQueueElementOld data;
 622     int i;
 623
 624     qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
 625
 626     elem = virtqueue_alloc_element(sz, data.out_num, data.in_num);
 627     elem->index = data.index;
 628
 629     for (i = 0; i < elem->in_num; i++) {
 630         elem->in_addr[i] = data.in_addr[i];
 631     }
 632
 633     for (i = 0; i < elem->out_num; i++) {
 634         elem->out_addr[i] = data.out_addr[i];
 635     }
 636
 637     for (i = 0; i < elem->in_num; i++) {
 638         /* Base is overwritten by virtqueue_map.  */
 639         elem->in_sg[i].iov_base = 0;
 640         elem->in_sg[i].iov_len = data.in_sg[i].iov_len;
 641     }
 642
 643     for (i = 0; i < elem->out_num; i++) {
 644         /* Base is overwritten by virtqueue_map.  */
 645         elem->out_sg[i].iov_base = 0;
 646         elem->out_sg[i].iov_len = data.out_sg[i].iov_len;
 647     }
 648
 649     virtqueue_map(elem);
 650     return elem;
 651 }
 652
 653 void qemu_put_virtqueue_element(QEMUFile *f, VirtQueueElement *elem)
 654 {
 655     VirtQueueElementOld data;
 656     int i;
 657
 658     memset(&data, 0, sizeof(data));
 659     data.index = elem->index;
 660     data.in_num = elem->in_num;
 661     data.out_num = elem->out_num;
 662
 663     for (i = 0; i < elem->in_num; i++) {
 664         data.in_addr[i] = elem->in_addr[i];
 665     }
 666
 667     for (i = 0; i < elem->out_num; i++) {
 668         data.out_addr[i] = elem->out_addr[i];
 669     }
 670
 671     for (i = 0; i < elem->in_num; i++) {
 672         /* Base is overwritten by virtqueue_map when loading.  Do not
 673          * save it, as it would leak the QEMU address space layout.  */
 674         data.in_sg[i].iov_len = elem->in_sg[i].iov_len;
 675     }
 676
 677     for (i = 0; i < elem->out_num; i++) {
 678         /* Do not save iov_base as above.  */
 679         data.out_sg[i].iov_len = elem->out_sg[i].iov_len;
 680     }
 681     qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
 682 }
 683
 684 /* virtio device */
 685 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector)
 686 {
 687     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
 688     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
 689
 690     if (k->notify) {
 691         k->notify(qbus->parent, vector);
 692     }
 693 }
 694
 695 void virtio_update_irq(VirtIODevice *vdev)
 696 {
 697     virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
 698 }
 699
 700 static int virtio_validate_features(VirtIODevice *vdev)
 701 {
 702     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 703
 704     if (k->validate_features) {
 705         return k->validate_features(vdev);
 706     } else {
 707         return 0;
 708     }
 709 }
 710
 711 int virtio_set_status(VirtIODevice *vdev, uint8_t val)
 712 {
 713     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 714     trace_virtio_set_status(vdev, val);
 715
 716     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
 717         if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) &&
 718             val & VIRTIO_CONFIG_S_FEATURES_OK) {
 719             int ret = virtio_validate_features(vdev);
 720
 721             if (ret) {
 722                 return ret;
 723             }
 724         }
 725     }
 726     if (k->set_status) {
 727         k->set_status(vdev, val);
 728     }
 729     vdev->status = val;
 730     return 0;
 731 }
 732
 733 bool target_words_bigendian(void);
 734 static enum virtio_device_endian virtio_default_endian(void)
 735 {
 736     if (target_words_bigendian()) {
 737         return VIRTIO_DEVICE_ENDIAN_BIG;
 738     } else {
 739         return VIRTIO_DEVICE_ENDIAN_LITTLE;
 740     }
 741 }
 742
 743 static enum virtio_device_endian virtio_current_cpu_endian(void)
 744 {
 745     CPUClass *cc = CPU_GET_CLASS(current_cpu);
 746
 747     if (cc->virtio_is_big_endian(current_cpu)) {
 748         return VIRTIO_DEVICE_ENDIAN_BIG;
 749     } else {
 750         return VIRTIO_DEVICE_ENDIAN_LITTLE;
 751     }
 752 }
 753
 754 void virtio_reset(void *opaque)
 755 {
 756     VirtIODevice *vdev = opaque;
 757     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 758     int i;
 759
 760     virtio_set_status(vdev, 0);
 761     if (current_cpu) {
 762         /* Guest initiated reset */
 763         vdev->device_endian = virtio_current_cpu_endian();
 764     } else {
 765         /* System reset */
 766         vdev->device_endian = virtio_default_endian();
 767     }
 768
 769     if (k->reset) {
 770         k->reset(vdev);
 771     }
 772
 773     vdev->guest_features = 0;
 774     vdev->queue_sel = 0;
 775     vdev->status = 0;
 776     vdev->isr = 0;
 777     vdev->config_vector = VIRTIO_NO_VECTOR;
 778     virtio_notify_vector(vdev, vdev->config_vector);
 779
 780     for(i = 0; i < VIRTIO_QUEUE_MAX; i++) {
 781         vdev->vq[i].vring.desc = 0;
 782         vdev->vq[i].vring.avail = 0;
 783         vdev->vq[i].vring.used = 0;
 784         vdev->vq[i].last_avail_idx = 0;
 785         virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
 786         vdev->vq[i].signalled_used = 0;
 787         vdev->vq[i].signalled_used_valid = false;
 788         vdev->vq[i].notification = true;
 789         vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
 790     }
 791 }
 792
 793 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr)
 794 {
 795     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 796     uint8_t val;
 797
 798     if (addr + sizeof(val) > vdev->config_len) {
 799         return (uint32_t)-1;
 800     }
 801
 802     k->get_config(vdev, vdev->config);
 803
 804     val = ldub_p(vdev->config + addr);
 805     return val;
 806 }
 807
 808 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr)
 809 {
 810     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 811     uint16_t val;
 812
 813     if (addr + sizeof(val) > vdev->config_len) {
 814         return (uint32_t)-1;
 815     }
 816
 817     k->get_config(vdev, vdev->config);
 818
 819     val = lduw_p(vdev->config + addr);
 820     return val;
 821 }
 822
 823 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr)
 824 {
 825     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 826     uint32_t val;
 827
 828     if (addr + sizeof(val) > vdev->config_len) {
 829         return (uint32_t)-1;
 830     }
 831
 832     k->get_config(vdev, vdev->config);
 833
 834     val = ldl_p(vdev->config + addr);
 835     return val;
 836 }
 837
 838 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data)
 839 {
 840     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 841     uint8_t val = data;
 842
 843     if (addr + sizeof(val) > vdev->config_len) {
 844         return;
 845     }
 846
 847     stb_p(vdev->config + addr, val);
 848
 849     if (k->set_config) {
 850         k->set_config(vdev, vdev->config);
 851     }
 852 }
 853
 854 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data)
 855 {
 856     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 857     uint16_t val = data;
 858
 859     if (addr + sizeof(val) > vdev->config_len) {
 860         return;
 861     }
 862
 863     stw_p(vdev->config + addr, val);
 864
 865     if (k->set_config) {
 866         k->set_config(vdev, vdev->config);
 867     }
 868 }
 869
 870 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
 871 {
 872     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 873     uint32_t val = data;
 874
 875     if (addr + sizeof(val) > vdev->config_len) {
 876         return;
 877     }
 878
 879     stl_p(vdev->config + addr, val);
 880
 881     if (k->set_config) {
 882         k->set_config(vdev, vdev->config);
 883     }
 884 }
 885
 886 uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr)
 887 {
 888     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 889     uint8_t val;
 890
 891     if (addr + sizeof(val) > vdev->config_len) {
 892         return (uint32_t)-1;
 893     }
 894
 895     k->get_config(vdev, vdev->config);
 896
 897     val = ldub_p(vdev->config + addr);
 898     return val;
 899 }
 900
 901 uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr)
 902 {
 903     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 904     uint16_t val;
 905
 906     if (addr + sizeof(val) > vdev->config_len) {
 907         return (uint32_t)-1;
 908     }
 909
 910     k->get_config(vdev, vdev->config);
 911
 912     val = lduw_le_p(vdev->config + addr);
 913     return val;
 914 }
 915
 916 uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr)
 917 {
 918     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 919     uint32_t val;
 920
 921     if (addr + sizeof(val) > vdev->config_len) {
 922         return (uint32_t)-1;
 923     }
 924
 925     k->get_config(vdev, vdev->config);
 926
 927     val = ldl_le_p(vdev->config + addr);
 928     return val;
 929 }
 930
 931 void virtio_config_modern_writeb(VirtIODevice *vdev,
 932                                  uint32_t addr, uint32_t data)
 933 {
 934     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 935     uint8_t val = data;
 936
 937     if (addr + sizeof(val) > vdev->config_len) {
 938         return;
 939     }
 940
 941     stb_p(vdev->config + addr, val);
 942
 943     if (k->set_config) {
 944         k->set_config(vdev, vdev->config);
 945     }
 946 }
 947
 948 void virtio_config_modern_writew(VirtIODevice *vdev,
 949                                  uint32_t addr, uint32_t data)
 950 {
 951     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 952     uint16_t val = data;
 953
 954     if (addr + sizeof(val) > vdev->config_len) {
 955         return;
 956     }
 957
 958     stw_le_p(vdev->config + addr, val);
 959
 960     if (k->set_config) {
 961         k->set_config(vdev, vdev->config);
 962     }
 963 }
 964
 965 void virtio_config_modern_writel(VirtIODevice *vdev,
 966                                  uint32_t addr, uint32_t data)
 967 {
 968     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 969     uint32_t val = data;
 970
 971     if (addr + sizeof(val) > vdev->config_len) {
 972         return;
 973     }
 974
 975     stl_le_p(vdev->config + addr, val);
 976
 977     if (k->set_config) {
 978         k->set_config(vdev, vdev->config);
 979     }
 980 }
 981
 982 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
 983 {
 984     vdev->vq[n].vring.desc = addr;
 985     virtio_queue_update_rings(vdev, n);
 986 }
 987
 988 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
 989 {
 990     return vdev->vq[n].vring.desc;
 991 }
 992
 993 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
 994                             hwaddr avail, hwaddr used)
 995 {
 996     vdev->vq[n].vring.desc = desc;
 997     vdev->vq[n].vring.avail = avail;
 998     vdev->vq[n].vring.used = used;
 999 }
1000
1001 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num)
1002 {
1003     /* Don't allow guest to flip queue between existent and
1004      * nonexistent states, or to set it to an invalid size.
1005      */
1006     if (!!num != !!vdev->vq[n].vring.num ||
1007         num > VIRTQUEUE_MAX_SIZE ||
1008         num < 0) {
1009         return;
1010     }
1011     vdev->vq[n].vring.num = num;
1012 }
1013
1014 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector)
1015 {
1016     return QLIST_FIRST(&vdev->vector_queues[vector]);
1017 }
1018
1019 VirtQueue *virtio_vector_next_queue(VirtQueue *vq)
1020 {
1021     return QLIST_NEXT(vq, node);
1022 }
1023
1024 int virtio_queue_get_num(VirtIODevice *vdev, int n)
1025 {
1026     return vdev->vq[n].vring.num;
1027 }
1028
1029 int virtio_get_num_queues(VirtIODevice *vdev)
1030 {
1031     int i;
1032
1033     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1034         if (!virtio_queue_get_num(vdev, i)) {
1035             break;
1036         }
1037     }
1038
1039     return i;
1040 }
1041
1042 int virtio_queue_get_id(VirtQueue *vq)
1043 {
1044     VirtIODevice *vdev = vq->vdev;
1045     assert(vq >= &vdev->vq[0] && vq < &vdev->vq[VIRTIO_QUEUE_MAX]);
1046     return vq - &vdev->vq[0];
1047 }
1048
1049 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
1050 {
1051     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1052     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1053
1054     /* virtio-1 compliant devices cannot change the alignment */
1055     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1056         error_report("tried to modify queue alignment for virtio-1 device");
1057         return;
1058     }
1059     /* Check that the transport told us it was going to do this
1060      * (so a buggy transport will immediately assert rather than
1061      * silently failing to migrate this state)
1062      */
1063     assert(k->has_variable_vring_alignment);
1064
1065     vdev->vq[n].vring.align = align;
1066     virtio_queue_update_rings(vdev, n);
1067 }
1068
1069 void virtio_queue_notify_vq(VirtQueue *vq)
1070 {
1071     if (vq->vring.desc && vq->handle_output) {
1072         VirtIODevice *vdev = vq->vdev;
1073
1074         trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1075         vq->handle_output(vdev, vq);
1076     }
1077 }
1078
1079 void virtio_queue_notify(VirtIODevice *vdev, int n)
1080 {
1081     virtio_queue_notify_vq(&vdev->vq[n]);
1082 }
1083
1084 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
1085 {
1086     return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector :
1087         VIRTIO_NO_VECTOR;
1088 }
1089
1090 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
1091 {
1092     VirtQueue *vq = &vdev->vq[n];
1093
1094     if (n < VIRTIO_QUEUE_MAX) {
1095         if (vdev->vector_queues &&
1096             vdev->vq[n].vector != VIRTIO_NO_VECTOR) {
1097             QLIST_REMOVE(vq, node);
1098         }
1099         vdev->vq[n].vector = vector;
1100         if (vdev->vector_queues &&
1101             vector != VIRTIO_NO_VECTOR) {
1102             QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node);
1103         }
1104     }
1105 }
1106
1107 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
1108                             void (*handle_output)(VirtIODevice *, VirtQueue *))
1109 {
1110     int i;
1111
1112     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1113         if (vdev->vq[i].vring.num == 0)
1114             break;
1115     }
1116
1117     if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
1118         abort();
1119
1120     vdev->vq[i].vring.num = queue_size;
1121     vdev->vq[i].vring.num_default = queue_size;
1122     vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
1123     vdev->vq[i].handle_output = handle_output;
1124
1125     return &vdev->vq[i];
1126 }
1127
1128 void virtio_del_queue(VirtIODevice *vdev, int n)
1129 {
1130     if (n < 0 || n >= VIRTIO_QUEUE_MAX) {
1131         abort();
1132     }
1133
1134     vdev->vq[n].vring.num = 0;
1135     vdev->vq[n].vring.num_default = 0;
1136 }
1137
1138 void virtio_irq(VirtQueue *vq)
1139 {
1140     trace_virtio_irq(vq);
1141     vq->vdev->isr |= 0x01;
1142     virtio_notify_vector(vq->vdev, vq->vector);
1143 }
1144
1145 static bool vring_notify(VirtIODevice *vdev, VirtQueue *vq)
1146 {
1147     uint16_t old, new;
1148     bool v;
1149     /* We need to expose used array entries before checking used event. */
1150     smp_mb();
1151     /* Always notify when queue is empty (when feature acknowledge) */
1152     if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
1153         !vq->inuse && vring_avail_idx(vq) == vq->last_avail_idx) {
1154         return true;
1155     }
1156
1157     if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
1158         return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
1159     }
1160
1161     v = vq->signalled_used_valid;
1162     vq->signalled_used_valid = true;
1163     old = vq->signalled_used;
1164     new = vq->signalled_used = vring_used_idx(vq);
1165     return !v || vring_need_event(vring_get_used_event(vq), new, old);
1166 }
1167
1168 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
1169 {
1170     if (!vring_notify(vdev, vq)) {
1171         return;
1172     }
1173
1174     trace_virtio_notify(vdev, vq);
1175     vdev->isr |= 0x01;
1176     virtio_notify_vector(vdev, vq->vector);
1177 }
1178
1179 void virtio_notify_config(VirtIODevice *vdev)
1180 {
1181     if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
1182         return;
1183
1184     vdev->isr |= 0x03;
1185     vdev->generation++;
1186     virtio_notify_vector(vdev, vdev->config_vector);
1187 }
1188
1189 static bool virtio_device_endian_needed(void *opaque)
1190 {
1191     VirtIODevice *vdev = opaque;
1192
1193     assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
1194     if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1195         return vdev->device_endian != virtio_default_endian();
1196     }
1197     /* Devices conforming to VIRTIO 1.0 or later are always LE. */
1198     return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE;
1199 }
1200
1201 static bool virtio_64bit_features_needed(void *opaque)
1202 {
1203     VirtIODevice *vdev = opaque;
1204
1205     return (vdev->host_features >> 32) != 0;
1206 }
1207
1208 static bool virtio_virtqueue_needed(void *opaque)
1209 {
1210     VirtIODevice *vdev = opaque;
1211
1212     return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
1213 }
1214
1215 static bool virtio_ringsize_needed(void *opaque)
1216 {
1217     VirtIODevice *vdev = opaque;
1218     int i;
1219
1220     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1221         if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) {
1222             return true;
1223         }
1224     }
1225     return false;
1226 }
1227
1228 static bool virtio_extra_state_needed(void *opaque)
1229 {
1230     VirtIODevice *vdev = opaque;
1231     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1232     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1233
1234     return k->has_extra_state &&
1235         k->has_extra_state(qbus->parent);
1236 }
1237
1238 static const VMStateDescription vmstate_virtqueue = {
1239     .name = "virtqueue_state",
1240     .version_id = 1,
1241     .minimum_version_id = 1,
1242     .fields = (VMStateField[]) {
1243         VMSTATE_UINT64(vring.avail, struct VirtQueue),
1244         VMSTATE_UINT64(vring.used, struct VirtQueue),
1245         VMSTATE_END_OF_LIST()
1246     }
1247 };
1248
1249 static const VMStateDescription vmstate_virtio_virtqueues = {
1250     .name = "virtio/virtqueues",
1251     .version_id = 1,
1252     .minimum_version_id = 1,
1253     .needed = &virtio_virtqueue_needed,
1254     .fields = (VMStateField[]) {
1255         VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1256                       VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
1257         VMSTATE_END_OF_LIST()
1258     }
1259 };
1260
1261 static const VMStateDescription vmstate_ringsize = {
1262     .name = "ringsize_state",
1263     .version_id = 1,
1264     .minimum_version_id = 1,
1265     .fields = (VMStateField[]) {
1266         VMSTATE_UINT32(vring.num_default, struct VirtQueue),
1267         VMSTATE_END_OF_LIST()
1268     }
1269 };
1270
1271 static const VMStateDescription vmstate_virtio_ringsize = {
1272     .name = "virtio/ringsize",
1273     .version_id = 1,
1274     .minimum_version_id = 1,
1275     .needed = &virtio_ringsize_needed,
1276     .fields = (VMStateField[]) {
1277         VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1278                       VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
1279         VMSTATE_END_OF_LIST()
1280     }
1281 };
1282
1283 static int get_extra_state(QEMUFile *f, void *pv, size_t size)
1284 {
1285     VirtIODevice *vdev = pv;
1286     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1287     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1288
1289     if (!k->load_extra_state) {
1290         return -1;
1291     } else {
1292         return k->load_extra_state(qbus->parent, f);
1293     }
1294 }
1295
1296 static void put_extra_state(QEMUFile *f, void *pv, size_t size)
1297 {
1298     VirtIODevice *vdev = pv;
1299     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1300     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1301
1302     k->save_extra_state(qbus->parent, f);
1303 }
1304
1305 static const VMStateInfo vmstate_info_extra_state = {
1306     .name = "virtqueue_extra_state",
1307     .get = get_extra_state,
1308     .put = put_extra_state,
1309 };
1310
1311 static const VMStateDescription vmstate_virtio_extra_state = {
1312     .name = "virtio/extra_state",
1313     .version_id = 1,
1314     .minimum_version_id = 1,
1315     .needed = &virtio_extra_state_needed,
1316     .fields = (VMStateField[]) {
1317         {
1318             .name         = "extra_state",
1319             .version_id   = 0,
1320             .field_exists = NULL,
1321             .size         = 0,
1322             .info         = &vmstate_info_extra_state,
1323             .flags        = VMS_SINGLE,
1324             .offset       = 0,
1325         },
1326         VMSTATE_END_OF_LIST()
1327     }
1328 };
1329
1330 static const VMStateDescription vmstate_virtio_device_endian = {
1331     .name = "virtio/device_endian",
1332     .version_id = 1,
1333     .minimum_version_id = 1,
1334     .needed = &virtio_device_endian_needed,
1335     .fields = (VMStateField[]) {
1336         VMSTATE_UINT8(device_endian, VirtIODevice),
1337         VMSTATE_END_OF_LIST()
1338     }
1339 };
1340
1341 static const VMStateDescription vmstate_virtio_64bit_features = {
1342     .name = "virtio/64bit_features",
1343     .version_id = 1,
1344     .minimum_version_id = 1,
1345     .needed = &virtio_64bit_features_needed,
1346     .fields = (VMStateField[]) {
1347         VMSTATE_UINT64(guest_features, VirtIODevice),
1348         VMSTATE_END_OF_LIST()
1349     }
1350 };
1351
1352 static const VMStateDescription vmstate_virtio = {
1353     .name = "virtio",
1354     .version_id = 1,
1355     .minimum_version_id = 1,
1356     .minimum_version_id_old = 1,
1357     .fields = (VMStateField[]) {
1358         VMSTATE_END_OF_LIST()
1359     },
1360     .subsections = (const VMStateDescription*[]) {
1361         &vmstate_virtio_device_endian,
1362         &vmstate_virtio_64bit_features,
1363         &vmstate_virtio_virtqueues,
1364         &vmstate_virtio_ringsize,
1365         &vmstate_virtio_extra_state,
1366         NULL
1367     }
1368 };
1369
1370 void virtio_save(VirtIODevice *vdev, QEMUFile *f)
1371 {
1372     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1373     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1374     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1375     uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff);
1376     int i;
1377
1378     if (k->save_config) {
1379         k->save_config(qbus->parent, f);
1380     }
1381
1382     qemu_put_8s(f, &vdev->status);
1383     qemu_put_8s(f, &vdev->isr);
1384     qemu_put_be16s(f, &vdev->queue_sel);
1385     qemu_put_be32s(f, &guest_features_lo);
1386     qemu_put_be32(f, vdev->config_len);
1387     qemu_put_buffer(f, vdev->config, vdev->config_len);
1388
1389     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1390         if (vdev->vq[i].vring.num == 0)
1391             break;
1392     }
1393
1394     qemu_put_be32(f, i);
1395
1396     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1397         if (vdev->vq[i].vring.num == 0)
1398             break;
1399
1400         qemu_put_be32(f, vdev->vq[i].vring.num);
1401         if (k->has_variable_vring_alignment) {
1402             qemu_put_be32(f, vdev->vq[i].vring.align);
1403         }
1404         /* XXX virtio-1 devices */
1405         qemu_put_be64(f, vdev->vq[i].vring.desc);
1406         qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
1407         if (k->save_queue) {
1408             k->save_queue(qbus->parent, i, f);
1409         }
1410     }
1411
1412     if (vdc->save != NULL) {
1413         vdc->save(vdev, f);
1414     }
1415
1416     /* Subsections */
1417     vmstate_save_state(f, &vmstate_virtio, vdev, NULL);
1418 }
1419
1420 static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val)
1421 {
1422     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1423     bool bad = (val & ~(vdev->host_features)) != 0;
1424
1425     val &= vdev->host_features;
1426     if (k->set_features) {
1427         k->set_features(vdev, val);
1428     }
1429     vdev->guest_features = val;
1430     return bad ? -1 : 0;
1431 }
1432
1433 int virtio_set_features(VirtIODevice *vdev, uint64_t val)
1434 {
1435    /*
1436      * The driver must not attempt to set features after feature negotiation
1437      * has finished.
1438      */
1439     if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) {
1440         return -EINVAL;
1441     }
1442     return virtio_set_features_nocheck(vdev, val);
1443 }
1444
1445 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
1446 {
1447     int i, ret;
1448     int32_t config_len;
1449     uint32_t num;
1450     uint32_t features;
1451     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1452     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1453     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1454
1455     /*
1456      * We poison the endianness to ensure it does not get used before
1457      * subsections have been loaded.
1458      */
1459     vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN;
1460
1461     if (k->load_config) {
1462         ret = k->load_config(qbus->parent, f);
1463         if (ret)
1464             return ret;
1465     }
1466
1467     qemu_get_8s(f, &vdev->status);
1468     qemu_get_8s(f, &vdev->isr);
1469     qemu_get_be16s(f, &vdev->queue_sel);
1470     if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) {
1471         return -1;
1472     }
1473     qemu_get_be32s(f, &features);
1474
1475     config_len = qemu_get_be32(f);
1476
1477     /*
1478      * There are cases where the incoming config can be bigger or smaller
1479      * than what we have; so load what we have space for, and skip
1480      * any excess that's in the stream.
1481      */
1482     qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len));
1483
1484     while (config_len > vdev->config_len) {
1485         qemu_get_byte(f);
1486         config_len--;
1487     }
1488
1489     num = qemu_get_be32(f);
1490
1491     if (num > VIRTIO_QUEUE_MAX) {
1492         error_report("Invalid number of virtqueues: 0x%x", num);
1493         return -1;
1494     }
1495
1496     for (i = 0; i < num; i++) {
1497         vdev->vq[i].vring.num = qemu_get_be32(f);
1498         if (k->has_variable_vring_alignment) {
1499             vdev->vq[i].vring.align = qemu_get_be32(f);
1500         }
1501         vdev->vq[i].vring.desc = qemu_get_be64(f);
1502         qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
1503         vdev->vq[i].signalled_used_valid = false;
1504         vdev->vq[i].notification = true;
1505
1506         if (vdev->vq[i].vring.desc) {
1507             /* XXX virtio-1 devices */
1508             virtio_queue_update_rings(vdev, i);
1509         } else if (vdev->vq[i].last_avail_idx) {
1510             error_report("VQ %d address 0x0 "
1511                          "inconsistent with Host index 0x%x",
1512                          i, vdev->vq[i].last_avail_idx);
1513                 return -1;
1514         }
1515         if (k->load_queue) {
1516             ret = k->load_queue(qbus->parent, i, f);
1517             if (ret)
1518                 return ret;
1519         }
1520     }
1521
1522     virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
1523
1524     if (vdc->load != NULL) {
1525         ret = vdc->load(vdev, f, version_id);
1526         if (ret) {
1527             return ret;
1528         }
1529     }
1530
1531     /* Subsections */
1532     ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1);
1533     if (ret) {
1534         return ret;
1535     }
1536
1537     if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) {
1538         vdev->device_endian = virtio_default_endian();
1539     }
1540
1541     if (virtio_64bit_features_needed(vdev)) {
1542         /*
1543          * Subsection load filled vdev->guest_features.  Run them
1544          * through virtio_set_features to sanity-check them against
1545          * host_features.
1546          */
1547         uint64_t features64 = vdev->guest_features;
1548         if (virtio_set_features_nocheck(vdev, features64) < 0) {
1549             error_report("Features 0x%" PRIx64 " unsupported. "
1550                          "Allowed features: 0x%" PRIx64,
1551                          features64, vdev->host_features);
1552             return -1;
1553         }
1554     } else {
1555         if (virtio_set_features_nocheck(vdev, features) < 0) {
1556             error_report("Features 0x%x unsupported. "
1557                          "Allowed features: 0x%" PRIx64,
1558                          features, vdev->host_features);
1559             return -1;
1560         }
1561     }
1562
1563     for (i = 0; i < num; i++) {
1564         if (vdev->vq[i].vring.desc) {
1565             uint16_t nheads;
1566             nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
1567             /* Check it isn't doing strange things with descriptor numbers. */
1568             if (nheads > vdev->vq[i].vring.num) {
1569                 error_report("VQ %d size 0x%x Guest index 0x%x "
1570                              "inconsistent with Host index 0x%x: delta 0x%x",
1571                              i, vdev->vq[i].vring.num,
1572                              vring_avail_idx(&vdev->vq[i]),
1573                              vdev->vq[i].last_avail_idx, nheads);
1574                 return -1;
1575             }
1576         }
1577     }
1578
1579     return 0;
1580 }
1581
1582 void virtio_cleanup(VirtIODevice *vdev)
1583 {
1584     qemu_del_vm_change_state_handler(vdev->vmstate);
1585     g_free(vdev->config);
1586     g_free(vdev->vq);
1587     g_free(vdev->vector_queues);
1588 }
1589
1590 static void virtio_vmstate_change(void *opaque, int running, RunState state)
1591 {
1592     VirtIODevice *vdev = opaque;
1593     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1594     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1595     bool backend_run = running && (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK);
1596     vdev->vm_running = running;
1597
1598     if (backend_run) {
1599         virtio_set_status(vdev, vdev->status);
1600     }
1601
1602     if (k->vmstate_change) {
1603         k->vmstate_change(qbus->parent, backend_run);
1604     }
1605
1606     if (!backend_run) {
1607         virtio_set_status(vdev, vdev->status);
1608     }
1609 }
1610
1611 void virtio_instance_init_common(Object *proxy_obj, void *data,
1612                                  size_t vdev_size, const char *vdev_name)
1613 {
1614     DeviceState *vdev = data;
1615
1616     object_initialize(vdev, vdev_size, vdev_name);
1617     object_property_add_child(proxy_obj, "virtio-backend", OBJECT(vdev), NULL);
1618     object_unref(OBJECT(vdev));
1619     qdev_alias_all_properties(vdev, proxy_obj);
1620 }
1621
1622 void virtio_init(VirtIODevice *vdev, const char *name,
1623                  uint16_t device_id, size_t config_size)
1624 {
1625     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1626     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1627     int i;
1628     int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0;
1629
1630     if (nvectors) {
1631         vdev->vector_queues =
1632             g_malloc0(sizeof(*vdev->vector_queues) * nvectors);
1633     }
1634
1635     vdev->device_id = device_id;
1636     vdev->status = 0;
1637     vdev->isr = 0;
1638     vdev->queue_sel = 0;
1639     vdev->config_vector = VIRTIO_NO_VECTOR;
1640     vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX);
1641     vdev->vm_running = runstate_is_running();
1642     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1643         vdev->vq[i].vector = VIRTIO_NO_VECTOR;
1644         vdev->vq[i].vdev = vdev;
1645         vdev->vq[i].queue_index = i;
1646     }
1647
1648     vdev->name = name;
1649     vdev->config_len = config_size;
1650     if (vdev->config_len) {
1651         vdev->config = g_malloc0(config_size);
1652     } else {
1653         vdev->config = NULL;
1654     }
1655     vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change,
1656                                                      vdev);
1657     vdev->device_endian = virtio_default_endian();
1658 }
1659
1660 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
1661 {
1662     return vdev->vq[n].vring.desc;
1663 }
1664
1665 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
1666 {
1667     return vdev->vq[n].vring.avail;
1668 }
1669
1670 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
1671 {
1672     return vdev->vq[n].vring.used;
1673 }
1674
1675 hwaddr virtio_queue_get_ring_addr(VirtIODevice *vdev, int n)
1676 {
1677     return vdev->vq[n].vring.desc;
1678 }
1679
1680 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
1681 {
1682     return sizeof(VRingDesc) * vdev->vq[n].vring.num;
1683 }
1684
1685 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
1686 {
1687     return offsetof(VRingAvail, ring) +
1688         sizeof(uint16_t) * vdev->vq[n].vring.num;
1689 }
1690
1691 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
1692 {
1693     return offsetof(VRingUsed, ring) +
1694         sizeof(VRingUsedElem) * vdev->vq[n].vring.num;
1695 }
1696
1697 hwaddr virtio_queue_get_ring_size(VirtIODevice *vdev, int n)
1698 {
1699     return vdev->vq[n].vring.used - vdev->vq[n].vring.desc +
1700             virtio_queue_get_used_size(vdev, n);
1701 }
1702
1703 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
1704 {
1705     return vdev->vq[n].last_avail_idx;
1706 }
1707
1708 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx)
1709 {
1710     vdev->vq[n].last_avail_idx = idx;
1711 }
1712
1713 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
1714 {
1715     vdev->vq[n].signalled_used_valid = false;
1716 }
1717
1718 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)
1719 {
1720     return vdev->vq + n;
1721 }
1722
1723 uint16_t virtio_get_queue_index(VirtQueue *vq)
1724 {
1725     return vq->queue_index;
1726 }
1727
1728 static void virtio_queue_guest_notifier_read(EventNotifier *n)
1729 {
1730     VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
1731     if (event_notifier_test_and_clear(n)) {
1732         virtio_irq(vq);
1733     }
1734 }
1735
1736 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
1737                                                 bool with_irqfd)
1738 {
1739     if (assign && !with_irqfd) {
1740         event_notifier_set_handler(&vq->guest_notifier,
1741                                    virtio_queue_guest_notifier_read);
1742     } else {
1743         event_notifier_set_handler(&vq->guest_notifier, NULL);
1744     }
1745     if (!assign) {
1746         /* Test and clear notifier before closing it,
1747          * in case poll callback didn't have time to run. */
1748         virtio_queue_guest_notifier_read(&vq->guest_notifier);
1749     }
1750 }
1751
1752 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
1753 {
1754     return &vq->guest_notifier;
1755 }
1756
1757 static void virtio_queue_host_notifier_read(EventNotifier *n)
1758 {
1759     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
1760     if (event_notifier_test_and_clear(n)) {
1761         virtio_queue_notify_vq(vq);
1762     }
1763 }
1764
1765 void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
1766                                                bool set_handler)
1767 {
1768     if (assign && set_handler) {
1769         event_notifier_set_handler(&vq->host_notifier,
1770                                    virtio_queue_host_notifier_read);
1771     } else {
1772         event_notifier_set_handler(&vq->host_notifier, NULL);
1773     }
1774     if (!assign) {
1775         /* Test and clear notifier before after disabling event,
1776          * in case poll callback didn't have time to run. */
1777         virtio_queue_host_notifier_read(&vq->host_notifier);
1778     }
1779 }
1780
1781 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
1782 {
1783     return &vq->host_notifier;
1784 }
1785
1786 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name)
1787 {
1788     g_free(vdev->bus_name);
1789     vdev->bus_name = g_strdup(bus_name);
1790 }
1791
1792 static void virtio_device_realize(DeviceState *dev, Error **errp)
1793 {
1794     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1795     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
1796     Error *err = NULL;
1797
1798     if (vdc->realize != NULL) {
1799         vdc->realize(dev, &err);
1800         if (err != NULL) {
1801             error_propagate(errp, err);
1802             return;
1803         }
1804     }
1805
1806     virtio_bus_device_plugged(vdev, &err);
1807     if (err != NULL) {
1808         error_propagate(errp, err);
1809         return;
1810     }
1811 }
1812
1813 static void virtio_device_unrealize(DeviceState *dev, Error **errp)
1814 {
1815     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1816     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
1817     Error *err = NULL;
1818
1819     virtio_bus_device_unplugged(vdev);
1820
1821     if (vdc->unrealize != NULL) {
1822         vdc->unrealize(dev, &err);
1823         if (err != NULL) {
1824             error_propagate(errp, err);
1825             return;
1826         }
1827     }
1828
1829     g_free(vdev->bus_name);
1830     vdev->bus_name = NULL;
1831 }
1832
1833 static Property virtio_properties[] = {
1834     DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features),
1835     DEFINE_PROP_END_OF_LIST(),
1836 };
1837
1838 static void virtio_device_class_init(ObjectClass *klass, void *data)
1839 {
1840     /* Set the default value here. */
1841     DeviceClass *dc = DEVICE_CLASS(klass);
1842
1843     dc->realize = virtio_device_realize;
1844     dc->unrealize = virtio_device_unrealize;
1845     dc->bus_type = TYPE_VIRTIO_BUS;
1846     dc->props = virtio_properties;
1847 }
1848
1849 static const TypeInfo virtio_device_info = {
1850     .name = TYPE_VIRTIO_DEVICE,
1851     .parent = TYPE_DEVICE,
1852     .instance_size = sizeof(VirtIODevice),
1853     .class_init = virtio_device_class_init,
1854     .abstract = true,
1855     .class_size = sizeof(VirtioDeviceClass),
1856 };
1857
1858 static void virtio_register_types(void)
1859 {
1860     type_register_static(&virtio_device_info);
1861 }
1862
1863 type_init(virtio_register_types)