hw/virtio/virtio.c

   1 /*
   2  * Virtio Support
   3  *
   4  * Copyright IBM, Corp. 2007
   5  *
   6  * Authors:
   7  *  Anthony Liguori   <aliguori@us.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  */
  13
  14 #include "qemu/osdep.h"
  15
  16 #include "trace.h"
  17 #include "exec/address-spaces.h"
  18 #include "qemu/error-report.h"
  19 #include "hw/virtio/virtio.h"
  20 #include "qemu/atomic.h"
  21 #include "hw/virtio/virtio-bus.h"
  22 #include "migration/migration.h"
  23 #include "hw/virtio/virtio-access.h"
  24
  25 /*
  26  * The alignment to use between consumer and producer parts of vring.
  27  * x86 pagesize again. This is the default, used by transports like PCI
  28  * which don't provide a means for the guest to tell the host the alignment.
  29  */
  30 #define VIRTIO_PCI_VRING_ALIGN         4096
  31
  32 typedef struct VRingDesc
  33 {
  34     uint64_t addr;
  35     uint32_t len;
  36     uint16_t flags;
  37     uint16_t next;
  38 } VRingDesc;
  39
  40 typedef struct VRingAvail
  41 {
  42     uint16_t flags;
  43     uint16_t idx;
  44     uint16_t ring[0];
  45 } VRingAvail;
  46
  47 typedef struct VRingUsedElem
  48 {
  49     uint32_t id;
  50     uint32_t len;
  51 } VRingUsedElem;
  52
  53 typedef struct VRingUsed
  54 {
  55     uint16_t flags;
  56     uint16_t idx;
  57     VRingUsedElem ring[0];
  58 } VRingUsed;
  59
  60 typedef struct VRing
  61 {
  62     unsigned int num;
  63     unsigned int num_default;
  64     unsigned int align;
  65     hwaddr desc;
  66     hwaddr avail;
  67     hwaddr used;
  68 } VRing;
  69
  70 struct VirtQueue
  71 {
  72     VRing vring;
  73     uint16_t last_avail_idx;
  74
  75     uint16_t used_idx;
  76
  77     /* Last used index value we have signalled on */
  78     uint16_t signalled_used;
  79
  80     /* Last used index value we have signalled on */
  81     bool signalled_used_valid;
  82
  83     /* Notification enabled? */
  84     bool notification;
  85
  86     uint16_t queue_index;
  87
  88     int inuse;
  89
  90     uint16_t vector;
  91     void (*handle_output)(VirtIODevice *vdev, VirtQueue *vq);
  92     VirtIODevice *vdev;
  93     EventNotifier guest_notifier;
  94     EventNotifier host_notifier;
  95     QLIST_ENTRY(VirtQueue) node;
  96 };
  97
  98 /* virt queue functions */
  99 void virtio_queue_update_rings(VirtIODevice *vdev, int n)
 100 {
 101     VRing *vring = &vdev->vq[n].vring;
 102
 103     if (!vring->desc) {
 104         /* not yet setup -> nothing to do */
 105         return;
 106     }
 107     vring->avail = vring->desc + vring->num * sizeof(VRingDesc);
 108     vring->used = vring_align(vring->avail +
 109                               offsetof(VRingAvail, ring[vring->num]),
 110                               vring->align);
 111 }
 112
 113 static void vring_desc_read(VirtIODevice *vdev, VRingDesc *desc,
 114                             hwaddr desc_pa, int i)
 115 {
 116     address_space_read(&address_space_memory, desc_pa + i * sizeof(VRingDesc),
 117                        MEMTXATTRS_UNSPECIFIED, (void *)desc, sizeof(VRingDesc));
 118     virtio_tswap64s(vdev, &desc->addr);
 119     virtio_tswap32s(vdev, &desc->len);
 120     virtio_tswap16s(vdev, &desc->flags);
 121     virtio_tswap16s(vdev, &desc->next);
 122 }
 123
 124 static inline uint16_t vring_avail_flags(VirtQueue *vq)
 125 {
 126     hwaddr pa;
 127     pa = vq->vring.avail + offsetof(VRingAvail, flags);
 128     return virtio_lduw_phys(vq->vdev, pa);
 129 }
 130
 131 static inline uint16_t vring_avail_idx(VirtQueue *vq)
 132 {
 133     hwaddr pa;
 134     pa = vq->vring.avail + offsetof(VRingAvail, idx);
 135     return virtio_lduw_phys(vq->vdev, pa);
 136 }
 137
 138 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
 139 {
 140     hwaddr pa;
 141     pa = vq->vring.avail + offsetof(VRingAvail, ring[i]);
 142     return virtio_lduw_phys(vq->vdev, pa);
 143 }
 144
 145 static inline uint16_t vring_get_used_event(VirtQueue *vq)
 146 {
 147     return vring_avail_ring(vq, vq->vring.num);
 148 }
 149
 150 static inline void vring_used_ring_id(VirtQueue *vq, int i, uint32_t val)
 151 {
 152     hwaddr pa;
 153     pa = vq->vring.used + offsetof(VRingUsed, ring[i].id);
 154     virtio_stl_phys(vq->vdev, pa, val);
 155 }
 156
 157 static inline void vring_used_ring_len(VirtQueue *vq, int i, uint32_t val)
 158 {
 159     hwaddr pa;
 160     pa = vq->vring.used + offsetof(VRingUsed, ring[i].len);
 161     virtio_stl_phys(vq->vdev, pa, val);
 162 }
 163
 164 static uint16_t vring_used_idx(VirtQueue *vq)
 165 {
 166     hwaddr pa;
 167     pa = vq->vring.used + offsetof(VRingUsed, idx);
 168     return virtio_lduw_phys(vq->vdev, pa);
 169 }
 170
 171 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
 172 {
 173     hwaddr pa;
 174     pa = vq->vring.used + offsetof(VRingUsed, idx);
 175     virtio_stw_phys(vq->vdev, pa, val);
 176     vq->used_idx = val;
 177 }
 178
 179 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
 180 {
 181     VirtIODevice *vdev = vq->vdev;
 182     hwaddr pa;
 183     pa = vq->vring.used + offsetof(VRingUsed, flags);
 184     virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) | mask);
 185 }
 186
 187 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
 188 {
 189     VirtIODevice *vdev = vq->vdev;
 190     hwaddr pa;
 191     pa = vq->vring.used + offsetof(VRingUsed, flags);
 192     virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) & ~mask);
 193 }
 194
 195 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)
 196 {
 197     hwaddr pa;
 198     if (!vq->notification) {
 199         return;
 200     }
 201     pa = vq->vring.used + offsetof(VRingUsed, ring[vq->vring.num]);
 202     virtio_stw_phys(vq->vdev, pa, val);
 203 }
 204
 205 void virtio_queue_set_notification(VirtQueue *vq, int enable)
 206 {
 207     vq->notification = enable;
 208     if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
 209         vring_set_avail_event(vq, vring_avail_idx(vq));
 210     } else if (enable) {
 211         vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
 212     } else {
 213         vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
 214     }
 215     if (enable) {
 216         /* Expose avail event/used flags before caller checks the avail idx. */
 217         smp_mb();
 218     }
 219 }
 220
 221 int virtio_queue_ready(VirtQueue *vq)
 222 {
 223     return vq->vring.avail != 0;
 224 }
 225
 226 int virtio_queue_empty(VirtQueue *vq)
 227 {
 228     return vring_avail_idx(vq) == vq->last_avail_idx;
 229 }
 230
 231 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem,
 232                                unsigned int len)
 233 {
 234     unsigned int offset;
 235     int i;
 236
 237     offset = 0;
 238     for (i = 0; i < elem->in_num; i++) {
 239         size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
 240
 241         cpu_physical_memory_unmap(elem->in_sg[i].iov_base,
 242                                   elem->in_sg[i].iov_len,
 243                                   1, size);
 244
 245         offset += size;
 246     }
 247
 248     for (i = 0; i < elem->out_num; i++)
 249         cpu_physical_memory_unmap(elem->out_sg[i].iov_base,
 250                                   elem->out_sg[i].iov_len,
 251                                   0, elem->out_sg[i].iov_len);
 252 }
 253
 254 void virtqueue_discard(VirtQueue *vq, const VirtQueueElement *elem,
 255                        unsigned int len)
 256 {
 257     vq->last_avail_idx--;
 258     virtqueue_unmap_sg(vq, elem, len);
 259 }
 260
 261 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
 262                     unsigned int len, unsigned int idx)
 263 {
 264     trace_virtqueue_fill(vq, elem, len, idx);
 265
 266     virtqueue_unmap_sg(vq, elem, len);
 267
 268     idx = (idx + vq->used_idx) % vq->vring.num;
 269
 270     /* Get a pointer to the next entry in the used ring. */
 271     vring_used_ring_id(vq, idx, elem->index);
 272     vring_used_ring_len(vq, idx, len);
 273 }
 274
 275 void virtqueue_flush(VirtQueue *vq, unsigned int count)
 276 {
 277     uint16_t old, new;
 278     /* Make sure buffer is written before we update index. */
 279     smp_wmb();
 280     trace_virtqueue_flush(vq, count);
 281     old = vq->used_idx;
 282     new = old + count;
 283     vring_used_idx_set(vq, new);
 284     vq->inuse -= count;
 285     if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)))
 286         vq->signalled_used_valid = false;
 287 }
 288
 289 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
 290                     unsigned int len)
 291 {
 292     virtqueue_fill(vq, elem, len, 0);
 293     virtqueue_flush(vq, 1);
 294 }
 295
 296 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
 297 {
 298     uint16_t num_heads = vring_avail_idx(vq) - idx;
 299
 300     /* Check it isn't doing very strange things with descriptor numbers. */
 301     if (num_heads > vq->vring.num) {
 302         error_report("Guest moved used index from %u to %u",
 303                      idx, vring_avail_idx(vq));
 304         exit(1);
 305     }
 306     /* On success, callers read a descriptor at vq->last_avail_idx.
 307      * Make sure descriptor read does not bypass avail index read. */
 308     if (num_heads) {
 309         smp_rmb();
 310     }
 311
 312     return num_heads;
 313 }
 314
 315 static unsigned int virtqueue_get_head(VirtQueue *vq, unsigned int idx)
 316 {
 317     unsigned int head;
 318
 319     /* Grab the next descriptor number they're advertising, and increment
 320      * the index we've seen. */
 321     head = vring_avail_ring(vq, idx % vq->vring.num);
 322
 323     /* If their number is silly, that's a fatal mistake. */
 324     if (head >= vq->vring.num) {
 325         error_report("Guest says index %u is available", head);
 326         exit(1);
 327     }
 328
 329     return head;
 330 }
 331
 332 static unsigned virtqueue_read_next_desc(VirtIODevice *vdev, VRingDesc *desc,
 333                                          hwaddr desc_pa, unsigned int max)
 334 {
 335     unsigned int next;
 336
 337     /* If this descriptor says it doesn't chain, we're done. */
 338     if (!(desc->flags & VRING_DESC_F_NEXT)) {
 339         return max;
 340     }
 341
 342     /* Check they're not leading us off end of descriptors. */
 343     next = desc->next;
 344     /* Make sure compiler knows to grab that: we don't want it changing! */
 345     smp_wmb();
 346
 347     if (next >= max) {
 348         error_report("Desc next is %u", next);
 349         exit(1);
 350     }
 351
 352     vring_desc_read(vdev, desc, desc_pa, next);
 353     return next;
 354 }
 355
 356 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
 357                                unsigned int *out_bytes,
 358                                unsigned max_in_bytes, unsigned max_out_bytes)
 359 {
 360     unsigned int idx;
 361     unsigned int total_bufs, in_total, out_total;
 362
 363     idx = vq->last_avail_idx;
 364
 365     total_bufs = in_total = out_total = 0;
 366     while (virtqueue_num_heads(vq, idx)) {
 367         VirtIODevice *vdev = vq->vdev;
 368         unsigned int max, num_bufs, indirect = 0;
 369         VRingDesc desc;
 370         hwaddr desc_pa;
 371         int i;
 372
 373         max = vq->vring.num;
 374         num_bufs = total_bufs;
 375         i = virtqueue_get_head(vq, idx++);
 376         desc_pa = vq->vring.desc;
 377         vring_desc_read(vdev, &desc, desc_pa, i);
 378
 379         if (desc.flags & VRING_DESC_F_INDIRECT) {
 380             if (desc.len % sizeof(VRingDesc)) {
 381                 error_report("Invalid size for indirect buffer table");
 382                 exit(1);
 383             }
 384
 385             /* If we've got too many, that implies a descriptor loop. */
 386             if (num_bufs >= max) {
 387                 error_report("Looped descriptor");
 388                 exit(1);
 389             }
 390
 391             /* loop over the indirect descriptor table */
 392             indirect = 1;
 393             max = desc.len / sizeof(VRingDesc);
 394             desc_pa = desc.addr;
 395             num_bufs = i = 0;
 396             vring_desc_read(vdev, &desc, desc_pa, i);
 397         }
 398
 399         do {
 400             /* If we've got too many, that implies a descriptor loop. */
 401             if (++num_bufs > max) {
 402                 error_report("Looped descriptor");
 403                 exit(1);
 404             }
 405
 406             if (desc.flags & VRING_DESC_F_WRITE) {
 407                 in_total += desc.len;
 408             } else {
 409                 out_total += desc.len;
 410             }
 411             if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
 412                 goto done;
 413             }
 414         } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max);
 415
 416         if (!indirect)
 417             total_bufs = num_bufs;
 418         else
 419             total_bufs++;
 420     }
 421 done:
 422     if (in_bytes) {
 423         *in_bytes = in_total;
 424     }
 425     if (out_bytes) {
 426         *out_bytes = out_total;
 427     }
 428 }
 429
 430 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
 431                           unsigned int out_bytes)
 432 {
 433     unsigned int in_total, out_total;
 434
 435     virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
 436     return in_bytes <= in_total && out_bytes <= out_total;
 437 }
 438
 439 static void virtqueue_map_desc(unsigned int *p_num_sg, hwaddr *addr, struct iovec *iov,
 440                                unsigned int max_num_sg, bool is_write,
 441                                hwaddr pa, size_t sz)
 442 {
 443     unsigned num_sg = *p_num_sg;
 444     assert(num_sg <= max_num_sg);
 445
 446     while (sz) {
 447         hwaddr len = sz;
 448
 449         if (num_sg == max_num_sg) {
 450             error_report("virtio: too many write descriptors in indirect table");
 451             exit(1);
 452         }
 453
 454         iov[num_sg].iov_base = cpu_physical_memory_map(pa, &len, is_write);
 455         iov[num_sg].iov_len = len;
 456         addr[num_sg] = pa;
 457
 458         sz -= len;
 459         pa += len;
 460         num_sg++;
 461     }
 462     *p_num_sg = num_sg;
 463 }
 464
 465 static void virtqueue_map_iovec(struct iovec *sg, hwaddr *addr,
 466                                 unsigned int *num_sg, unsigned int max_size,
 467                                 int is_write)
 468 {
 469     unsigned int i;
 470     hwaddr len;
 471
 472     /* Note: this function MUST validate input, some callers
 473      * are passing in num_sg values received over the network.
 474      */
 475     /* TODO: teach all callers that this can fail, and return failure instead
 476      * of asserting here.
 477      * When we do, we might be able to re-enable NDEBUG below.
 478      */
 479 #ifdef NDEBUG
 480 #error building with NDEBUG is not supported
 481 #endif
 482     assert(*num_sg <= max_size);
 483
 484     for (i = 0; i < *num_sg; i++) {
 485         len = sg[i].iov_len;
 486         sg[i].iov_base = cpu_physical_memory_map(addr[i], &len, is_write);
 487         if (!sg[i].iov_base) {
 488             error_report("virtio: error trying to map MMIO memory");
 489             exit(1);
 490         }
 491         if (len != sg[i].iov_len) {
 492             error_report("virtio: unexpected memory split");
 493             exit(1);
 494         }
 495     }
 496 }
 497
 498 void virtqueue_map(VirtQueueElement *elem)
 499 {
 500     virtqueue_map_iovec(elem->in_sg, elem->in_addr, &elem->in_num,
 501                         VIRTQUEUE_MAX_SIZE, 1);
 502     virtqueue_map_iovec(elem->out_sg, elem->out_addr, &elem->out_num,
 503                         VIRTQUEUE_MAX_SIZE, 0);
 504 }
 505
 506 void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num)
 507 {
 508     VirtQueueElement *elem;
 509     size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0]));
 510     size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]);
 511     size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]);
 512     size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0]));
 513     size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
 514     size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
 515
 516     assert(sz >= sizeof(VirtQueueElement));
 517     elem = g_malloc(out_sg_end);
 518     elem->out_num = out_num;
 519     elem->in_num = in_num;
 520     elem->in_addr = (void *)elem + in_addr_ofs;
 521     elem->out_addr = (void *)elem + out_addr_ofs;
 522     elem->in_sg = (void *)elem + in_sg_ofs;
 523     elem->out_sg = (void *)elem + out_sg_ofs;
 524     return elem;
 525 }
 526
 527 void *virtqueue_pop(VirtQueue *vq, size_t sz)
 528 {
 529     unsigned int i, head, max;
 530     hwaddr desc_pa = vq->vring.desc;
 531     VirtIODevice *vdev = vq->vdev;
 532     VirtQueueElement *elem;
 533     unsigned out_num, in_num;
 534     hwaddr addr[VIRTQUEUE_MAX_SIZE];
 535     struct iovec iov[VIRTQUEUE_MAX_SIZE];
 536     VRingDesc desc;
 537
 538     if (!virtqueue_num_heads(vq, vq->last_avail_idx)) {
 539         return NULL;
 540     }
 541
 542     /* When we start there are none of either input nor output. */
 543     out_num = in_num = 0;
 544
 545     max = vq->vring.num;
 546
 547     i = head = virtqueue_get_head(vq, vq->last_avail_idx++);
 548     if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
 549         vring_set_avail_event(vq, vq->last_avail_idx);
 550     }
 551
 552     vring_desc_read(vdev, &desc, desc_pa, i);
 553     if (desc.flags & VRING_DESC_F_INDIRECT) {
 554         if (desc.len % sizeof(VRingDesc)) {
 555             error_report("Invalid size for indirect buffer table");
 556             exit(1);
 557         }
 558
 559         /* loop over the indirect descriptor table */
 560         max = desc.len / sizeof(VRingDesc);
 561         desc_pa = desc.addr;
 562         i = 0;
 563         vring_desc_read(vdev, &desc, desc_pa, i);
 564     }
 565
 566     /* Collect all the descriptors */
 567     do {
 568         if (desc.flags & VRING_DESC_F_WRITE) {
 569             virtqueue_map_desc(&in_num, addr + out_num, iov + out_num,
 570                                VIRTQUEUE_MAX_SIZE - out_num, true, desc.addr, desc.len);
 571         } else {
 572             if (in_num) {
 573                 error_report("Incorrect order for descriptors");
 574                 exit(1);
 575             }
 576             virtqueue_map_desc(&out_num, addr, iov,
 577                                VIRTQUEUE_MAX_SIZE, false, desc.addr, desc.len);
 578         }
 579
 580         /* If we've got too many, that implies a descriptor loop. */
 581         if ((in_num + out_num) > max) {
 582             error_report("Looped descriptor");
 583             exit(1);
 584         }
 585     } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max);
 586
 587     /* Now copy what we have collected and mapped */
 588     elem = virtqueue_alloc_element(sz, out_num, in_num);
 589     elem->index = head;
 590     for (i = 0; i < out_num; i++) {
 591         elem->out_addr[i] = addr[i];
 592         elem->out_sg[i] = iov[i];
 593     }
 594     for (i = 0; i < in_num; i++) {
 595         elem->in_addr[i] = addr[out_num + i];
 596         elem->in_sg[i] = iov[out_num + i];
 597     }
 598
 599     vq->inuse++;
 600
 601     trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
 602     return elem;
 603 }
 604
 605 /* Reading and writing a structure directly to QEMUFile is *awful*, but
 606  * it is what QEMU has always done by mistake.  We can change it sooner
 607  * or later by bumping the version number of the affected vm states.
 608  * In the meanwhile, since the in-memory layout of VirtQueueElement
 609  * has changed, we need to marshal to and from the layout that was
 610  * used before the change.
 611  */
 612 typedef struct VirtQueueElementOld {
 613     unsigned int index;
 614     unsigned int out_num;
 615     unsigned int in_num;
 616     hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
 617     hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
 618     struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
 619     struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
 620 } VirtQueueElementOld;
 621
 622 void *qemu_get_virtqueue_element(QEMUFile *f, size_t sz)
 623 {
 624     VirtQueueElement *elem;
 625     VirtQueueElementOld data;
 626     int i;
 627
 628     qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
 629
 630     elem = virtqueue_alloc_element(sz, data.out_num, data.in_num);
 631     elem->index = data.index;
 632
 633     for (i = 0; i < elem->in_num; i++) {
 634         elem->in_addr[i] = data.in_addr[i];
 635     }
 636
 637     for (i = 0; i < elem->out_num; i++) {
 638         elem->out_addr[i] = data.out_addr[i];
 639     }
 640
 641     for (i = 0; i < elem->in_num; i++) {
 642         /* Base is overwritten by virtqueue_map.  */
 643         elem->in_sg[i].iov_base = 0;
 644         elem->in_sg[i].iov_len = data.in_sg[i].iov_len;
 645     }
 646
 647     for (i = 0; i < elem->out_num; i++) {
 648         /* Base is overwritten by virtqueue_map.  */
 649         elem->out_sg[i].iov_base = 0;
 650         elem->out_sg[i].iov_len = data.out_sg[i].iov_len;
 651     }
 652
 653     virtqueue_map(elem);
 654     return elem;
 655 }
 656
 657 void qemu_put_virtqueue_element(QEMUFile *f, VirtQueueElement *elem)
 658 {
 659     VirtQueueElementOld data;
 660     int i;
 661
 662     memset(&data, 0, sizeof(data));
 663     data.index = elem->index;
 664     data.in_num = elem->in_num;
 665     data.out_num = elem->out_num;
 666
 667     for (i = 0; i < elem->in_num; i++) {
 668         data.in_addr[i] = elem->in_addr[i];
 669     }
 670
 671     for (i = 0; i < elem->out_num; i++) {
 672         data.out_addr[i] = elem->out_addr[i];
 673     }
 674
 675     for (i = 0; i < elem->in_num; i++) {
 676         /* Base is overwritten by virtqueue_map when loading.  Do not
 677          * save it, as it would leak the QEMU address space layout.  */
 678         data.in_sg[i].iov_len = elem->in_sg[i].iov_len;
 679     }
 680
 681     for (i = 0; i < elem->out_num; i++) {
 682         /* Do not save iov_base as above.  */
 683         data.out_sg[i].iov_len = elem->out_sg[i].iov_len;
 684     }
 685     qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
 686 }
 687
 688 /* virtio device */
 689 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector)
 690 {
 691     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
 692     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
 693
 694     if (k->notify) {
 695         k->notify(qbus->parent, vector);
 696     }
 697 }
 698
 699 void virtio_update_irq(VirtIODevice *vdev)
 700 {
 701     virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
 702 }
 703
 704 static int virtio_validate_features(VirtIODevice *vdev)
 705 {
 706     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 707
 708     if (k->validate_features) {
 709         return k->validate_features(vdev);
 710     } else {
 711         return 0;
 712     }
 713 }
 714
 715 int virtio_set_status(VirtIODevice *vdev, uint8_t val)
 716 {
 717     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 718     trace_virtio_set_status(vdev, val);
 719
 720     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
 721         if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) &&
 722             val & VIRTIO_CONFIG_S_FEATURES_OK) {
 723             int ret = virtio_validate_features(vdev);
 724
 725             if (ret) {
 726                 return ret;
 727             }
 728         }
 729     }
 730     if (k->set_status) {
 731         k->set_status(vdev, val);
 732     }
 733     vdev->status = val;
 734     return 0;
 735 }
 736
 737 bool target_words_bigendian(void);
 738 static enum virtio_device_endian virtio_default_endian(void)
 739 {
 740     if (target_words_bigendian()) {
 741         return VIRTIO_DEVICE_ENDIAN_BIG;
 742     } else {
 743         return VIRTIO_DEVICE_ENDIAN_LITTLE;
 744     }
 745 }
 746
 747 static enum virtio_device_endian virtio_current_cpu_endian(void)
 748 {
 749     CPUClass *cc = CPU_GET_CLASS(current_cpu);
 750
 751     if (cc->virtio_is_big_endian(current_cpu)) {
 752         return VIRTIO_DEVICE_ENDIAN_BIG;
 753     } else {
 754         return VIRTIO_DEVICE_ENDIAN_LITTLE;
 755     }
 756 }
 757
 758 void virtio_reset(void *opaque)
 759 {
 760     VirtIODevice *vdev = opaque;
 761     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 762     int i;
 763
 764     virtio_set_status(vdev, 0);
 765     if (current_cpu) {
 766         /* Guest initiated reset */
 767         vdev->device_endian = virtio_current_cpu_endian();
 768     } else {
 769         /* System reset */
 770         vdev->device_endian = virtio_default_endian();
 771     }
 772
 773     if (k->reset) {
 774         k->reset(vdev);
 775     }
 776
 777     vdev->guest_features = 0;
 778     vdev->queue_sel = 0;
 779     vdev->status = 0;
 780     vdev->isr = 0;
 781     vdev->config_vector = VIRTIO_NO_VECTOR;
 782     virtio_notify_vector(vdev, vdev->config_vector);
 783
 784     for(i = 0; i < VIRTIO_QUEUE_MAX; i++) {
 785         vdev->vq[i].vring.desc = 0;
 786         vdev->vq[i].vring.avail = 0;
 787         vdev->vq[i].vring.used = 0;
 788         vdev->vq[i].last_avail_idx = 0;
 789         vdev->vq[i].used_idx = 0;
 790         virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
 791         vdev->vq[i].signalled_used = 0;
 792         vdev->vq[i].signalled_used_valid = false;
 793         vdev->vq[i].notification = true;
 794         vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
 795     }
 796 }
 797
 798 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr)
 799 {
 800     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 801     uint8_t val;
 802
 803     if (addr + sizeof(val) > vdev->config_len) {
 804         return (uint32_t)-1;
 805     }
 806
 807     k->get_config(vdev, vdev->config);
 808
 809     val = ldub_p(vdev->config + addr);
 810     return val;
 811 }
 812
 813 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr)
 814 {
 815     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 816     uint16_t val;
 817
 818     if (addr + sizeof(val) > vdev->config_len) {
 819         return (uint32_t)-1;
 820     }
 821
 822     k->get_config(vdev, vdev->config);
 823
 824     val = lduw_p(vdev->config + addr);
 825     return val;
 826 }
 827
 828 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr)
 829 {
 830     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 831     uint32_t val;
 832
 833     if (addr + sizeof(val) > vdev->config_len) {
 834         return (uint32_t)-1;
 835     }
 836
 837     k->get_config(vdev, vdev->config);
 838
 839     val = ldl_p(vdev->config + addr);
 840     return val;
 841 }
 842
 843 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data)
 844 {
 845     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 846     uint8_t val = data;
 847
 848     if (addr + sizeof(val) > vdev->config_len) {
 849         return;
 850     }
 851
 852     stb_p(vdev->config + addr, val);
 853
 854     if (k->set_config) {
 855         k->set_config(vdev, vdev->config);
 856     }
 857 }
 858
 859 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data)
 860 {
 861     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 862     uint16_t val = data;
 863
 864     if (addr + sizeof(val) > vdev->config_len) {
 865         return;
 866     }
 867
 868     stw_p(vdev->config + addr, val);
 869
 870     if (k->set_config) {
 871         k->set_config(vdev, vdev->config);
 872     }
 873 }
 874
 875 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
 876 {
 877     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 878     uint32_t val = data;
 879
 880     if (addr + sizeof(val) > vdev->config_len) {
 881         return;
 882     }
 883
 884     stl_p(vdev->config + addr, val);
 885
 886     if (k->set_config) {
 887         k->set_config(vdev, vdev->config);
 888     }
 889 }
 890
 891 uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr)
 892 {
 893     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 894     uint8_t val;
 895
 896     if (addr + sizeof(val) > vdev->config_len) {
 897         return (uint32_t)-1;
 898     }
 899
 900     k->get_config(vdev, vdev->config);
 901
 902     val = ldub_p(vdev->config + addr);
 903     return val;
 904 }
 905
 906 uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr)
 907 {
 908     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 909     uint16_t val;
 910
 911     if (addr + sizeof(val) > vdev->config_len) {
 912         return (uint32_t)-1;
 913     }
 914
 915     k->get_config(vdev, vdev->config);
 916
 917     val = lduw_le_p(vdev->config + addr);
 918     return val;
 919 }
 920
 921 uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr)
 922 {
 923     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 924     uint32_t val;
 925
 926     if (addr + sizeof(val) > vdev->config_len) {
 927         return (uint32_t)-1;
 928     }
 929
 930     k->get_config(vdev, vdev->config);
 931
 932     val = ldl_le_p(vdev->config + addr);
 933     return val;
 934 }
 935
 936 void virtio_config_modern_writeb(VirtIODevice *vdev,
 937                                  uint32_t addr, uint32_t data)
 938 {
 939     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 940     uint8_t val = data;
 941
 942     if (addr + sizeof(val) > vdev->config_len) {
 943         return;
 944     }
 945
 946     stb_p(vdev->config + addr, val);
 947
 948     if (k->set_config) {
 949         k->set_config(vdev, vdev->config);
 950     }
 951 }
 952
 953 void virtio_config_modern_writew(VirtIODevice *vdev,
 954                                  uint32_t addr, uint32_t data)
 955 {
 956     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 957     uint16_t val = data;
 958
 959     if (addr + sizeof(val) > vdev->config_len) {
 960         return;
 961     }
 962
 963     stw_le_p(vdev->config + addr, val);
 964
 965     if (k->set_config) {
 966         k->set_config(vdev, vdev->config);
 967     }
 968 }
 969
 970 void virtio_config_modern_writel(VirtIODevice *vdev,
 971                                  uint32_t addr, uint32_t data)
 972 {
 973     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 974     uint32_t val = data;
 975
 976     if (addr + sizeof(val) > vdev->config_len) {
 977         return;
 978     }
 979
 980     stl_le_p(vdev->config + addr, val);
 981
 982     if (k->set_config) {
 983         k->set_config(vdev, vdev->config);
 984     }
 985 }
 986
 987 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
 988 {
 989     vdev->vq[n].vring.desc = addr;
 990     virtio_queue_update_rings(vdev, n);
 991 }
 992
 993 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
 994 {
 995     return vdev->vq[n].vring.desc;
 996 }
 997
 998 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
 999                             hwaddr avail, hwaddr used)
1000 {
1001     vdev->vq[n].vring.desc = desc;
1002     vdev->vq[n].vring.avail = avail;
1003     vdev->vq[n].vring.used = used;
1004 }
1005
1006 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num)
1007 {
1008     /* Don't allow guest to flip queue between existent and
1009      * nonexistent states, or to set it to an invalid size.
1010      */
1011     if (!!num != !!vdev->vq[n].vring.num ||
1012         num > VIRTQUEUE_MAX_SIZE ||
1013         num < 0) {
1014         return;
1015     }
1016     vdev->vq[n].vring.num = num;
1017 }
1018
1019 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector)
1020 {
1021     return QLIST_FIRST(&vdev->vector_queues[vector]);
1022 }
1023
1024 VirtQueue *virtio_vector_next_queue(VirtQueue *vq)
1025 {
1026     return QLIST_NEXT(vq, node);
1027 }
1028
1029 int virtio_queue_get_num(VirtIODevice *vdev, int n)
1030 {
1031     return vdev->vq[n].vring.num;
1032 }
1033
1034 int virtio_get_num_queues(VirtIODevice *vdev)
1035 {
1036     int i;
1037
1038     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1039         if (!virtio_queue_get_num(vdev, i)) {
1040             break;
1041         }
1042     }
1043
1044     return i;
1045 }
1046
1047 int virtio_queue_get_id(VirtQueue *vq)
1048 {
1049     VirtIODevice *vdev = vq->vdev;
1050     assert(vq >= &vdev->vq[0] && vq < &vdev->vq[VIRTIO_QUEUE_MAX]);
1051     return vq - &vdev->vq[0];
1052 }
1053
1054 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
1055 {
1056     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1057     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1058
1059     /* virtio-1 compliant devices cannot change the alignment */
1060     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1061         error_report("tried to modify queue alignment for virtio-1 device");
1062         return;
1063     }
1064     /* Check that the transport told us it was going to do this
1065      * (so a buggy transport will immediately assert rather than
1066      * silently failing to migrate this state)
1067      */
1068     assert(k->has_variable_vring_alignment);
1069
1070     vdev->vq[n].vring.align = align;
1071     virtio_queue_update_rings(vdev, n);
1072 }
1073
1074 void virtio_queue_notify_vq(VirtQueue *vq)
1075 {
1076     if (vq->vring.desc && vq->handle_output) {
1077         VirtIODevice *vdev = vq->vdev;
1078
1079         trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1080         vq->handle_output(vdev, vq);
1081     }
1082 }
1083
1084 void virtio_queue_notify(VirtIODevice *vdev, int n)
1085 {
1086     virtio_queue_notify_vq(&vdev->vq[n]);
1087 }
1088
1089 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
1090 {
1091     return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector :
1092         VIRTIO_NO_VECTOR;
1093 }
1094
1095 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
1096 {
1097     VirtQueue *vq = &vdev->vq[n];
1098
1099     if (n < VIRTIO_QUEUE_MAX) {
1100         if (vdev->vector_queues &&
1101             vdev->vq[n].vector != VIRTIO_NO_VECTOR) {
1102             QLIST_REMOVE(vq, node);
1103         }
1104         vdev->vq[n].vector = vector;
1105         if (vdev->vector_queues &&
1106             vector != VIRTIO_NO_VECTOR) {
1107             QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node);
1108         }
1109     }
1110 }
1111
1112 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
1113                             void (*handle_output)(VirtIODevice *, VirtQueue *))
1114 {
1115     int i;
1116
1117     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1118         if (vdev->vq[i].vring.num == 0)
1119             break;
1120     }
1121
1122     if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
1123         abort();
1124
1125     vdev->vq[i].vring.num = queue_size;
1126     vdev->vq[i].vring.num_default = queue_size;
1127     vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
1128     vdev->vq[i].handle_output = handle_output;
1129
1130     return &vdev->vq[i];
1131 }
1132
1133 void virtio_del_queue(VirtIODevice *vdev, int n)
1134 {
1135     if (n < 0 || n >= VIRTIO_QUEUE_MAX) {
1136         abort();
1137     }
1138
1139     vdev->vq[n].vring.num = 0;
1140     vdev->vq[n].vring.num_default = 0;
1141 }
1142
1143 void virtio_irq(VirtQueue *vq)
1144 {
1145     trace_virtio_irq(vq);
1146     vq->vdev->isr |= 0x01;
1147     virtio_notify_vector(vq->vdev, vq->vector);
1148 }
1149
1150 static bool vring_notify(VirtIODevice *vdev, VirtQueue *vq)
1151 {
1152     uint16_t old, new;
1153     bool v;
1154     /* We need to expose used array entries before checking used event. */
1155     smp_mb();
1156     /* Always notify when queue is empty (when feature acknowledge) */
1157     if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
1158         !vq->inuse && vring_avail_idx(vq) == vq->last_avail_idx) {
1159         return true;
1160     }
1161
1162     if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
1163         return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
1164     }
1165
1166     v = vq->signalled_used_valid;
1167     vq->signalled_used_valid = true;
1168     old = vq->signalled_used;
1169     new = vq->signalled_used = vq->used_idx;
1170     return !v || vring_need_event(vring_get_used_event(vq), new, old);
1171 }
1172
1173 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
1174 {
1175     if (!vring_notify(vdev, vq)) {
1176         return;
1177     }
1178
1179     trace_virtio_notify(vdev, vq);
1180     vdev->isr |= 0x01;
1181     virtio_notify_vector(vdev, vq->vector);
1182 }
1183
1184 void virtio_notify_config(VirtIODevice *vdev)
1185 {
1186     if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
1187         return;
1188
1189     vdev->isr |= 0x03;
1190     vdev->generation++;
1191     virtio_notify_vector(vdev, vdev->config_vector);
1192 }
1193
1194 static bool virtio_device_endian_needed(void *opaque)
1195 {
1196     VirtIODevice *vdev = opaque;
1197
1198     assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
1199     if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1200         return vdev->device_endian != virtio_default_endian();
1201     }
1202     /* Devices conforming to VIRTIO 1.0 or later are always LE. */
1203     return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE;
1204 }
1205
1206 static bool virtio_64bit_features_needed(void *opaque)
1207 {
1208     VirtIODevice *vdev = opaque;
1209
1210     return (vdev->host_features >> 32) != 0;
1211 }
1212
1213 static bool virtio_virtqueue_needed(void *opaque)
1214 {
1215     VirtIODevice *vdev = opaque;
1216
1217     return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
1218 }
1219
1220 static bool virtio_ringsize_needed(void *opaque)
1221 {
1222     VirtIODevice *vdev = opaque;
1223     int i;
1224
1225     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1226         if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) {
1227             return true;
1228         }
1229     }
1230     return false;
1231 }
1232
1233 static bool virtio_extra_state_needed(void *opaque)
1234 {
1235     VirtIODevice *vdev = opaque;
1236     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1237     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1238
1239     return k->has_extra_state &&
1240         k->has_extra_state(qbus->parent);
1241 }
1242
1243 static const VMStateDescription vmstate_virtqueue = {
1244     .name = "virtqueue_state",
1245     .version_id = 1,
1246     .minimum_version_id = 1,
1247     .fields = (VMStateField[]) {
1248         VMSTATE_UINT64(vring.avail, struct VirtQueue),
1249         VMSTATE_UINT64(vring.used, struct VirtQueue),
1250         VMSTATE_END_OF_LIST()
1251     }
1252 };
1253
1254 static const VMStateDescription vmstate_virtio_virtqueues = {
1255     .name = "virtio/virtqueues",
1256     .version_id = 1,
1257     .minimum_version_id = 1,
1258     .needed = &virtio_virtqueue_needed,
1259     .fields = (VMStateField[]) {
1260         VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1261                       VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
1262         VMSTATE_END_OF_LIST()
1263     }
1264 };
1265
1266 static const VMStateDescription vmstate_ringsize = {
1267     .name = "ringsize_state",
1268     .version_id = 1,
1269     .minimum_version_id = 1,
1270     .fields = (VMStateField[]) {
1271         VMSTATE_UINT32(vring.num_default, struct VirtQueue),
1272         VMSTATE_END_OF_LIST()
1273     }
1274 };
1275
1276 static const VMStateDescription vmstate_virtio_ringsize = {
1277     .name = "virtio/ringsize",
1278     .version_id = 1,
1279     .minimum_version_id = 1,
1280     .needed = &virtio_ringsize_needed,
1281     .fields = (VMStateField[]) {
1282         VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1283                       VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
1284         VMSTATE_END_OF_LIST()
1285     }
1286 };
1287
1288 static int get_extra_state(QEMUFile *f, void *pv, size_t size)
1289 {
1290     VirtIODevice *vdev = pv;
1291     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1292     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1293
1294     if (!k->load_extra_state) {
1295         return -1;
1296     } else {
1297         return k->load_extra_state(qbus->parent, f);
1298     }
1299 }
1300
1301 static void put_extra_state(QEMUFile *f, void *pv, size_t size)
1302 {
1303     VirtIODevice *vdev = pv;
1304     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1305     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1306
1307     k->save_extra_state(qbus->parent, f);
1308 }
1309
1310 static const VMStateInfo vmstate_info_extra_state = {
1311     .name = "virtqueue_extra_state",
1312     .get = get_extra_state,
1313     .put = put_extra_state,
1314 };
1315
1316 static const VMStateDescription vmstate_virtio_extra_state = {
1317     .name = "virtio/extra_state",
1318     .version_id = 1,
1319     .minimum_version_id = 1,
1320     .needed = &virtio_extra_state_needed,
1321     .fields = (VMStateField[]) {
1322         {
1323             .name         = "extra_state",
1324             .version_id   = 0,
1325             .field_exists = NULL,
1326             .size         = 0,
1327             .info         = &vmstate_info_extra_state,
1328             .flags        = VMS_SINGLE,
1329             .offset       = 0,
1330         },
1331         VMSTATE_END_OF_LIST()
1332     }
1333 };
1334
1335 static const VMStateDescription vmstate_virtio_device_endian = {
1336     .name = "virtio/device_endian",
1337     .version_id = 1,
1338     .minimum_version_id = 1,
1339     .needed = &virtio_device_endian_needed,
1340     .fields = (VMStateField[]) {
1341         VMSTATE_UINT8(device_endian, VirtIODevice),
1342         VMSTATE_END_OF_LIST()
1343     }
1344 };
1345
1346 static const VMStateDescription vmstate_virtio_64bit_features = {
1347     .name = "virtio/64bit_features",
1348     .version_id = 1,
1349     .minimum_version_id = 1,
1350     .needed = &virtio_64bit_features_needed,
1351     .fields = (VMStateField[]) {
1352         VMSTATE_UINT64(guest_features, VirtIODevice),
1353         VMSTATE_END_OF_LIST()
1354     }
1355 };
1356
1357 static const VMStateDescription vmstate_virtio = {
1358     .name = "virtio",
1359     .version_id = 1,
1360     .minimum_version_id = 1,
1361     .minimum_version_id_old = 1,
1362     .fields = (VMStateField[]) {
1363         VMSTATE_END_OF_LIST()
1364     },
1365     .subsections = (const VMStateDescription*[]) {
1366         &vmstate_virtio_device_endian,
1367         &vmstate_virtio_64bit_features,
1368         &vmstate_virtio_virtqueues,
1369         &vmstate_virtio_ringsize,
1370         &vmstate_virtio_extra_state,
1371         NULL
1372     }
1373 };
1374
1375 void virtio_save(VirtIODevice *vdev, QEMUFile *f)
1376 {
1377     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1378     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1379     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1380     uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff);
1381     int i;
1382
1383     if (k->save_config) {
1384         k->save_config(qbus->parent, f);
1385     }
1386
1387     qemu_put_8s(f, &vdev->status);
1388     qemu_put_8s(f, &vdev->isr);
1389     qemu_put_be16s(f, &vdev->queue_sel);
1390     qemu_put_be32s(f, &guest_features_lo);
1391     qemu_put_be32(f, vdev->config_len);
1392     qemu_put_buffer(f, vdev->config, vdev->config_len);
1393
1394     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1395         if (vdev->vq[i].vring.num == 0)
1396             break;
1397     }
1398
1399     qemu_put_be32(f, i);
1400
1401     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1402         if (vdev->vq[i].vring.num == 0)
1403             break;
1404
1405         qemu_put_be32(f, vdev->vq[i].vring.num);
1406         if (k->has_variable_vring_alignment) {
1407             qemu_put_be32(f, vdev->vq[i].vring.align);
1408         }
1409         /* XXX virtio-1 devices */
1410         qemu_put_be64(f, vdev->vq[i].vring.desc);
1411         qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
1412         if (k->save_queue) {
1413             k->save_queue(qbus->parent, i, f);
1414         }
1415     }
1416
1417     if (vdc->save != NULL) {
1418         vdc->save(vdev, f);
1419     }
1420
1421     /* Subsections */
1422     vmstate_save_state(f, &vmstate_virtio, vdev, NULL);
1423 }
1424
1425 static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val)
1426 {
1427     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1428     bool bad = (val & ~(vdev->host_features)) != 0;
1429
1430     val &= vdev->host_features;
1431     if (k->set_features) {
1432         k->set_features(vdev, val);
1433     }
1434     vdev->guest_features = val;
1435     return bad ? -1 : 0;
1436 }
1437
1438 int virtio_set_features(VirtIODevice *vdev, uint64_t val)
1439 {
1440    /*
1441      * The driver must not attempt to set features after feature negotiation
1442      * has finished.
1443      */
1444     if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) {
1445         return -EINVAL;
1446     }
1447     return virtio_set_features_nocheck(vdev, val);
1448 }
1449
1450 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
1451 {
1452     int i, ret;
1453     int32_t config_len;
1454     uint32_t num;
1455     uint32_t features;
1456     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1457     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1458     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1459
1460     /*
1461      * We poison the endianness to ensure it does not get used before
1462      * subsections have been loaded.
1463      */
1464     vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN;
1465
1466     if (k->load_config) {
1467         ret = k->load_config(qbus->parent, f);
1468         if (ret)
1469             return ret;
1470     }
1471
1472     qemu_get_8s(f, &vdev->status);
1473     qemu_get_8s(f, &vdev->isr);
1474     qemu_get_be16s(f, &vdev->queue_sel);
1475     if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) {
1476         return -1;
1477     }
1478     qemu_get_be32s(f, &features);
1479
1480     config_len = qemu_get_be32(f);
1481
1482     /*
1483      * There are cases where the incoming config can be bigger or smaller
1484      * than what we have; so load what we have space for, and skip
1485      * any excess that's in the stream.
1486      */
1487     qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len));
1488
1489     while (config_len > vdev->config_len) {
1490         qemu_get_byte(f);
1491         config_len--;
1492     }
1493
1494     num = qemu_get_be32(f);
1495
1496     if (num > VIRTIO_QUEUE_MAX) {
1497         error_report("Invalid number of virtqueues: 0x%x", num);
1498         return -1;
1499     }
1500
1501     for (i = 0; i < num; i++) {
1502         vdev->vq[i].vring.num = qemu_get_be32(f);
1503         if (k->has_variable_vring_alignment) {
1504             vdev->vq[i].vring.align = qemu_get_be32(f);
1505         }
1506         vdev->vq[i].vring.desc = qemu_get_be64(f);
1507         qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
1508         vdev->vq[i].signalled_used_valid = false;
1509         vdev->vq[i].notification = true;
1510
1511         if (vdev->vq[i].vring.desc) {
1512             /* XXX virtio-1 devices */
1513             virtio_queue_update_rings(vdev, i);
1514         } else if (vdev->vq[i].last_avail_idx) {
1515             error_report("VQ %d address 0x0 "
1516                          "inconsistent with Host index 0x%x",
1517                          i, vdev->vq[i].last_avail_idx);
1518                 return -1;
1519         }
1520         if (k->load_queue) {
1521             ret = k->load_queue(qbus->parent, i, f);
1522             if (ret)
1523                 return ret;
1524         }
1525     }
1526
1527     virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
1528
1529     if (vdc->load != NULL) {
1530         ret = vdc->load(vdev, f, version_id);
1531         if (ret) {
1532             return ret;
1533         }
1534     }
1535
1536     /* Subsections */
1537     ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1);
1538     if (ret) {
1539         return ret;
1540     }
1541
1542     if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) {
1543         vdev->device_endian = virtio_default_endian();
1544     }
1545
1546     if (virtio_64bit_features_needed(vdev)) {
1547         /*
1548          * Subsection load filled vdev->guest_features.  Run them
1549          * through virtio_set_features to sanity-check them against
1550          * host_features.
1551          */
1552         uint64_t features64 = vdev->guest_features;
1553         if (virtio_set_features_nocheck(vdev, features64) < 0) {
1554             error_report("Features 0x%" PRIx64 " unsupported. "
1555                          "Allowed features: 0x%" PRIx64,
1556                          features64, vdev->host_features);
1557             return -1;
1558         }
1559     } else {
1560         if (virtio_set_features_nocheck(vdev, features) < 0) {
1561             error_report("Features 0x%x unsupported. "
1562                          "Allowed features: 0x%" PRIx64,
1563                          features, vdev->host_features);
1564             return -1;
1565         }
1566     }
1567
1568     for (i = 0; i < num; i++) {
1569         if (vdev->vq[i].vring.desc) {
1570             uint16_t nheads;
1571             nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
1572             /* Check it isn't doing strange things with descriptor numbers. */
1573             if (nheads > vdev->vq[i].vring.num) {
1574                 error_report("VQ %d size 0x%x Guest index 0x%x "
1575                              "inconsistent with Host index 0x%x: delta 0x%x",
1576                              i, vdev->vq[i].vring.num,
1577                              vring_avail_idx(&vdev->vq[i]),
1578                              vdev->vq[i].last_avail_idx, nheads);
1579                 return -1;
1580             }
1581             vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]);
1582         }
1583     }
1584
1585     return 0;
1586 }
1587
1588 void virtio_cleanup(VirtIODevice *vdev)
1589 {
1590     qemu_del_vm_change_state_handler(vdev->vmstate);
1591     g_free(vdev->config);
1592     g_free(vdev->vq);
1593     g_free(vdev->vector_queues);
1594 }
1595
1596 static void virtio_vmstate_change(void *opaque, int running, RunState state)
1597 {
1598     VirtIODevice *vdev = opaque;
1599     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1600     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1601     bool backend_run = running && (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK);
1602     vdev->vm_running = running;
1603
1604     if (backend_run) {
1605         virtio_set_status(vdev, vdev->status);
1606     }
1607
1608     if (k->vmstate_change) {
1609         k->vmstate_change(qbus->parent, backend_run);
1610     }
1611
1612     if (!backend_run) {
1613         virtio_set_status(vdev, vdev->status);
1614     }
1615 }
1616
1617 void virtio_instance_init_common(Object *proxy_obj, void *data,
1618                                  size_t vdev_size, const char *vdev_name)
1619 {
1620     DeviceState *vdev = data;
1621
1622     object_initialize(vdev, vdev_size, vdev_name);
1623     object_property_add_child(proxy_obj, "virtio-backend", OBJECT(vdev), NULL);
1624     object_unref(OBJECT(vdev));
1625     qdev_alias_all_properties(vdev, proxy_obj);
1626 }
1627
1628 void virtio_init(VirtIODevice *vdev, const char *name,
1629                  uint16_t device_id, size_t config_size)
1630 {
1631     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1632     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1633     int i;
1634     int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0;
1635
1636     if (nvectors) {
1637         vdev->vector_queues =
1638             g_malloc0(sizeof(*vdev->vector_queues) * nvectors);
1639     }
1640
1641     vdev->device_id = device_id;
1642     vdev->status = 0;
1643     vdev->isr = 0;
1644     vdev->queue_sel = 0;
1645     vdev->config_vector = VIRTIO_NO_VECTOR;
1646     vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX);
1647     vdev->vm_running = runstate_is_running();
1648     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1649         vdev->vq[i].vector = VIRTIO_NO_VECTOR;
1650         vdev->vq[i].vdev = vdev;
1651         vdev->vq[i].queue_index = i;
1652     }
1653
1654     vdev->name = name;
1655     vdev->config_len = config_size;
1656     if (vdev->config_len) {
1657         vdev->config = g_malloc0(config_size);
1658     } else {
1659         vdev->config = NULL;
1660     }
1661     vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change,
1662                                                      vdev);
1663     vdev->device_endian = virtio_default_endian();
1664 }
1665
1666 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
1667 {
1668     return vdev->vq[n].vring.desc;
1669 }
1670
1671 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
1672 {
1673     return vdev->vq[n].vring.avail;
1674 }
1675
1676 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
1677 {
1678     return vdev->vq[n].vring.used;
1679 }
1680
1681 hwaddr virtio_queue_get_ring_addr(VirtIODevice *vdev, int n)
1682 {
1683     return vdev->vq[n].vring.desc;
1684 }
1685
1686 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
1687 {
1688     return sizeof(VRingDesc) * vdev->vq[n].vring.num;
1689 }
1690
1691 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
1692 {
1693     return offsetof(VRingAvail, ring) +
1694         sizeof(uint16_t) * vdev->vq[n].vring.num;
1695 }
1696
1697 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
1698 {
1699     return offsetof(VRingUsed, ring) +
1700         sizeof(VRingUsedElem) * vdev->vq[n].vring.num;
1701 }
1702
1703 hwaddr virtio_queue_get_ring_size(VirtIODevice *vdev, int n)
1704 {
1705     return vdev->vq[n].vring.used - vdev->vq[n].vring.desc +
1706             virtio_queue_get_used_size(vdev, n);
1707 }
1708
1709 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
1710 {
1711     return vdev->vq[n].last_avail_idx;
1712 }
1713
1714 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx)
1715 {
1716     vdev->vq[n].last_avail_idx = idx;
1717 }
1718
1719 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
1720 {
1721     vdev->vq[n].signalled_used_valid = false;
1722 }
1723
1724 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)
1725 {
1726     return vdev->vq + n;
1727 }
1728
1729 uint16_t virtio_get_queue_index(VirtQueue *vq)
1730 {
1731     return vq->queue_index;
1732 }
1733
1734 static void virtio_queue_guest_notifier_read(EventNotifier *n)
1735 {
1736     VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
1737     if (event_notifier_test_and_clear(n)) {
1738         virtio_irq(vq);
1739     }
1740 }
1741
1742 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
1743                                                 bool with_irqfd)
1744 {
1745     if (assign && !with_irqfd) {
1746         event_notifier_set_handler(&vq->guest_notifier,
1747                                    virtio_queue_guest_notifier_read);
1748     } else {
1749         event_notifier_set_handler(&vq->guest_notifier, NULL);
1750     }
1751     if (!assign) {
1752         /* Test and clear notifier before closing it,
1753          * in case poll callback didn't have time to run. */
1754         virtio_queue_guest_notifier_read(&vq->guest_notifier);
1755     }
1756 }
1757
1758 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
1759 {
1760     return &vq->guest_notifier;
1761 }
1762
1763 static void virtio_queue_host_notifier_read(EventNotifier *n)
1764 {
1765     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
1766     if (event_notifier_test_and_clear(n)) {
1767         virtio_queue_notify_vq(vq);
1768     }
1769 }
1770
1771 void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
1772                                                bool set_handler)
1773 {
1774     if (assign && set_handler) {
1775         event_notifier_set_handler(&vq->host_notifier,
1776                                    virtio_queue_host_notifier_read);
1777     } else {
1778         event_notifier_set_handler(&vq->host_notifier, NULL);
1779     }
1780     if (!assign) {
1781         /* Test and clear notifier before after disabling event,
1782          * in case poll callback didn't have time to run. */
1783         virtio_queue_host_notifier_read(&vq->host_notifier);
1784     }
1785 }
1786
1787 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
1788 {
1789     return &vq->host_notifier;
1790 }
1791
1792 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name)
1793 {
1794     g_free(vdev->bus_name);
1795     vdev->bus_name = g_strdup(bus_name);
1796 }
1797
1798 static void virtio_device_realize(DeviceState *dev, Error **errp)
1799 {
1800     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1801     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
1802     Error *err = NULL;
1803
1804     if (vdc->realize != NULL) {
1805         vdc->realize(dev, &err);
1806         if (err != NULL) {
1807             error_propagate(errp, err);
1808             return;
1809         }
1810     }
1811
1812     virtio_bus_device_plugged(vdev, &err);
1813     if (err != NULL) {
1814         error_propagate(errp, err);
1815         return;
1816     }
1817 }
1818
1819 static void virtio_device_unrealize(DeviceState *dev, Error **errp)
1820 {
1821     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1822     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
1823     Error *err = NULL;
1824
1825     virtio_bus_device_unplugged(vdev);
1826
1827     if (vdc->unrealize != NULL) {
1828         vdc->unrealize(dev, &err);
1829         if (err != NULL) {
1830             error_propagate(errp, err);
1831             return;
1832         }
1833     }
1834
1835     g_free(vdev->bus_name);
1836     vdev->bus_name = NULL;
1837 }
1838
1839 static Property virtio_properties[] = {
1840     DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features),
1841     DEFINE_PROP_END_OF_LIST(),
1842 };
1843
1844 static void virtio_device_class_init(ObjectClass *klass, void *data)
1845 {
1846     /* Set the default value here. */
1847     DeviceClass *dc = DEVICE_CLASS(klass);
1848
1849     dc->realize = virtio_device_realize;
1850     dc->unrealize = virtio_device_unrealize;
1851     dc->bus_type = TYPE_VIRTIO_BUS;
1852     dc->props = virtio_properties;
1853 }
1854
1855 static const TypeInfo virtio_device_info = {
1856     .name = TYPE_VIRTIO_DEVICE,
1857     .parent = TYPE_DEVICE,
1858     .instance_size = sizeof(VirtIODevice),
1859     .class_init = virtio_device_class_init,
1860     .abstract = true,
1861     .class_size = sizeof(VirtioDeviceClass),
1862 };
1863
1864 static void virtio_register_types(void)
1865 {
1866     type_register_static(&virtio_device_info);
1867 }
1868
1869 type_init(virtio_register_types)