hw/virtio/virtio.c

   1 /*
   2  * Virtio Support
   3  *
   4  * Copyright IBM, Corp. 2007
   5  *
   6  * Authors:
   7  *  Anthony Liguori   <aliguori@us.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  */
  13
  14 #include <inttypes.h>
  15
  16 #include "trace.h"
  17 #include "exec/address-spaces.h"
  18 #include "qemu/error-report.h"
  19 #include "hw/virtio/virtio.h"
  20 #include "qemu/atomic.h"
  21 #include "hw/virtio/virtio-bus.h"
  22 #include "migration/migration.h"
  23 #include "hw/virtio/virtio-access.h"
  24
  25 /*
  26  * The alignment to use between consumer and producer parts of vring.
  27  * x86 pagesize again. This is the default, used by transports like PCI
  28  * which don't provide a means for the guest to tell the host the alignment.
  29  */
  30 #define VIRTIO_PCI_VRING_ALIGN         4096
  31
  32 typedef struct VRingDesc
  33 {
  34     uint64_t addr;
  35     uint32_t len;
  36     uint16_t flags;
  37     uint16_t next;
  38 } VRingDesc;
  39
  40 typedef struct VRingAvail
  41 {
  42     uint16_t flags;
  43     uint16_t idx;
  44     uint16_t ring[0];
  45 } VRingAvail;
  46
  47 typedef struct VRingUsedElem
  48 {
  49     uint32_t id;
  50     uint32_t len;
  51 } VRingUsedElem;
  52
  53 typedef struct VRingUsed
  54 {
  55     uint16_t flags;
  56     uint16_t idx;
  57     VRingUsedElem ring[0];
  58 } VRingUsed;
  59
  60 typedef struct VRing
  61 {
  62     unsigned int num;
  63     unsigned int num_default;
  64     unsigned int align;
  65     hwaddr desc;
  66     hwaddr avail;
  67     hwaddr used;
  68 } VRing;
  69
  70 struct VirtQueue
  71 {
  72     VRing vring;
  73     uint16_t last_avail_idx;
  74     /* Last used index value we have signalled on */
  75     uint16_t signalled_used;
  76
  77     /* Last used index value we have signalled on */
  78     bool signalled_used_valid;
  79
  80     /* Notification enabled? */
  81     bool notification;
  82
  83     uint16_t queue_index;
  84
  85     int inuse;
  86
  87     uint16_t vector;
  88     void (*handle_output)(VirtIODevice *vdev, VirtQueue *vq);
  89     VirtIODevice *vdev;
  90     EventNotifier guest_notifier;
  91     EventNotifier host_notifier;
  92     QLIST_ENTRY(VirtQueue) node;
  93 };
  94
  95 /* virt queue functions */
  96 void virtio_queue_update_rings(VirtIODevice *vdev, int n)
  97 {
  98     VRing *vring = &vdev->vq[n].vring;
  99
 100     if (!vring->desc) {
 101         /* not yet setup -> nothing to do */
 102         return;
 103     }
 104     vring->avail = vring->desc + vring->num * sizeof(VRingDesc);
 105     vring->used = vring_align(vring->avail +
 106                               offsetof(VRingAvail, ring[vring->num]),
 107                               vring->align);
 108 }
 109
 110 static inline uint64_t vring_desc_addr(VirtIODevice *vdev, hwaddr desc_pa,
 111                                        int i)
 112 {
 113     hwaddr pa;
 114     pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, addr);
 115     return virtio_ldq_phys(vdev, pa);
 116 }
 117
 118 static inline uint32_t vring_desc_len(VirtIODevice *vdev, hwaddr desc_pa, int i)
 119 {
 120     hwaddr pa;
 121     pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, len);
 122     return virtio_ldl_phys(vdev, pa);
 123 }
 124
 125 static inline uint16_t vring_desc_flags(VirtIODevice *vdev, hwaddr desc_pa,
 126                                         int i)
 127 {
 128     hwaddr pa;
 129     pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, flags);
 130     return virtio_lduw_phys(vdev, pa);
 131 }
 132
 133 static inline uint16_t vring_desc_next(VirtIODevice *vdev, hwaddr desc_pa,
 134                                        int i)
 135 {
 136     hwaddr pa;
 137     pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, next);
 138     return virtio_lduw_phys(vdev, pa);
 139 }
 140
 141 static inline uint16_t vring_avail_flags(VirtQueue *vq)
 142 {
 143     hwaddr pa;
 144     pa = vq->vring.avail + offsetof(VRingAvail, flags);
 145     return virtio_lduw_phys(vq->vdev, pa);
 146 }
 147
 148 static inline uint16_t vring_avail_idx(VirtQueue *vq)
 149 {
 150     hwaddr pa;
 151     pa = vq->vring.avail + offsetof(VRingAvail, idx);
 152     return virtio_lduw_phys(vq->vdev, pa);
 153 }
 154
 155 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
 156 {
 157     hwaddr pa;
 158     pa = vq->vring.avail + offsetof(VRingAvail, ring[i]);
 159     return virtio_lduw_phys(vq->vdev, pa);
 160 }
 161
 162 static inline uint16_t vring_get_used_event(VirtQueue *vq)
 163 {
 164     return vring_avail_ring(vq, vq->vring.num);
 165 }
 166
 167 static inline void vring_used_ring_id(VirtQueue *vq, int i, uint32_t val)
 168 {
 169     hwaddr pa;
 170     pa = vq->vring.used + offsetof(VRingUsed, ring[i].id);
 171     virtio_stl_phys(vq->vdev, pa, val);
 172 }
 173
 174 static inline void vring_used_ring_len(VirtQueue *vq, int i, uint32_t val)
 175 {
 176     hwaddr pa;
 177     pa = vq->vring.used + offsetof(VRingUsed, ring[i].len);
 178     virtio_stl_phys(vq->vdev, pa, val);
 179 }
 180
 181 static uint16_t vring_used_idx(VirtQueue *vq)
 182 {
 183     hwaddr pa;
 184     pa = vq->vring.used + offsetof(VRingUsed, idx);
 185     return virtio_lduw_phys(vq->vdev, pa);
 186 }
 187
 188 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
 189 {
 190     hwaddr pa;
 191     pa = vq->vring.used + offsetof(VRingUsed, idx);
 192     virtio_stw_phys(vq->vdev, pa, val);
 193 }
 194
 195 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
 196 {
 197     VirtIODevice *vdev = vq->vdev;
 198     hwaddr pa;
 199     pa = vq->vring.used + offsetof(VRingUsed, flags);
 200     virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) | mask);
 201 }
 202
 203 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
 204 {
 205     VirtIODevice *vdev = vq->vdev;
 206     hwaddr pa;
 207     pa = vq->vring.used + offsetof(VRingUsed, flags);
 208     virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) & ~mask);
 209 }
 210
 211 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)
 212 {
 213     hwaddr pa;
 214     if (!vq->notification) {
 215         return;
 216     }
 217     pa = vq->vring.used + offsetof(VRingUsed, ring[vq->vring.num]);
 218     virtio_stw_phys(vq->vdev, pa, val);
 219 }
 220
 221 void virtio_queue_set_notification(VirtQueue *vq, int enable)
 222 {
 223     vq->notification = enable;
 224     if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
 225         vring_set_avail_event(vq, vring_avail_idx(vq));
 226     } else if (enable) {
 227         vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
 228     } else {
 229         vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
 230     }
 231     if (enable) {
 232         /* Expose avail event/used flags before caller checks the avail idx. */
 233         smp_mb();
 234     }
 235 }
 236
 237 int virtio_queue_ready(VirtQueue *vq)
 238 {
 239     return vq->vring.avail != 0;
 240 }
 241
 242 int virtio_queue_empty(VirtQueue *vq)
 243 {
 244     return vring_avail_idx(vq) == vq->last_avail_idx;
 245 }
 246
 247 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem,
 248                                unsigned int len)
 249 {
 250     unsigned int offset;
 251     int i;
 252
 253     offset = 0;
 254     for (i = 0; i < elem->in_num; i++) {
 255         size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
 256
 257         cpu_physical_memory_unmap(elem->in_sg[i].iov_base,
 258                                   elem->in_sg[i].iov_len,
 259                                   1, size);
 260
 261         offset += size;
 262     }
 263
 264     for (i = 0; i < elem->out_num; i++)
 265         cpu_physical_memory_unmap(elem->out_sg[i].iov_base,
 266                                   elem->out_sg[i].iov_len,
 267                                   0, elem->out_sg[i].iov_len);
 268 }
 269
 270 void virtqueue_discard(VirtQueue *vq, const VirtQueueElement *elem,
 271                        unsigned int len)
 272 {
 273     vq->last_avail_idx--;
 274     virtqueue_unmap_sg(vq, elem, len);
 275 }
 276
 277 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
 278                     unsigned int len, unsigned int idx)
 279 {
 280     trace_virtqueue_fill(vq, elem, len, idx);
 281
 282     virtqueue_unmap_sg(vq, elem, len);
 283
 284     idx = (idx + vring_used_idx(vq)) % vq->vring.num;
 285
 286     /* Get a pointer to the next entry in the used ring. */
 287     vring_used_ring_id(vq, idx, elem->index);
 288     vring_used_ring_len(vq, idx, len);
 289 }
 290
 291 void virtqueue_flush(VirtQueue *vq, unsigned int count)
 292 {
 293     uint16_t old, new;
 294     /* Make sure buffer is written before we update index. */
 295     smp_wmb();
 296     trace_virtqueue_flush(vq, count);
 297     old = vring_used_idx(vq);
 298     new = old + count;
 299     vring_used_idx_set(vq, new);
 300     vq->inuse -= count;
 301     if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)))
 302         vq->signalled_used_valid = false;
 303 }
 304
 305 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
 306                     unsigned int len)
 307 {
 308     virtqueue_fill(vq, elem, len, 0);
 309     virtqueue_flush(vq, 1);
 310 }
 311
 312 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
 313 {
 314     uint16_t num_heads = vring_avail_idx(vq) - idx;
 315
 316     /* Check it isn't doing very strange things with descriptor numbers. */
 317     if (num_heads > vq->vring.num) {
 318         error_report("Guest moved used index from %u to %u",
 319                      idx, vring_avail_idx(vq));
 320         exit(1);
 321     }
 322     /* On success, callers read a descriptor at vq->last_avail_idx.
 323      * Make sure descriptor read does not bypass avail index read. */
 324     if (num_heads) {
 325         smp_rmb();
 326     }
 327
 328     return num_heads;
 329 }
 330
 331 static unsigned int virtqueue_get_head(VirtQueue *vq, unsigned int idx)
 332 {
 333     unsigned int head;
 334
 335     /* Grab the next descriptor number they're advertising, and increment
 336      * the index we've seen. */
 337     head = vring_avail_ring(vq, idx % vq->vring.num);
 338
 339     /* If their number is silly, that's a fatal mistake. */
 340     if (head >= vq->vring.num) {
 341         error_report("Guest says index %u is available", head);
 342         exit(1);
 343     }
 344
 345     return head;
 346 }
 347
 348 static unsigned virtqueue_next_desc(VirtIODevice *vdev, hwaddr desc_pa,
 349                                     unsigned int i, unsigned int max)
 350 {
 351     unsigned int next;
 352
 353     /* If this descriptor says it doesn't chain, we're done. */
 354     if (!(vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_NEXT)) {
 355         return max;
 356     }
 357
 358     /* Check they're not leading us off end of descriptors. */
 359     next = vring_desc_next(vdev, desc_pa, i);
 360     /* Make sure compiler knows to grab that: we don't want it changing! */
 361     smp_wmb();
 362
 363     if (next >= max) {
 364         error_report("Desc next is %u", next);
 365         exit(1);
 366     }
 367
 368     return next;
 369 }
 370
 371 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
 372                                unsigned int *out_bytes,
 373                                unsigned max_in_bytes, unsigned max_out_bytes)
 374 {
 375     unsigned int idx;
 376     unsigned int total_bufs, in_total, out_total;
 377
 378     idx = vq->last_avail_idx;
 379
 380     total_bufs = in_total = out_total = 0;
 381     while (virtqueue_num_heads(vq, idx)) {
 382         VirtIODevice *vdev = vq->vdev;
 383         unsigned int max, num_bufs, indirect = 0;
 384         hwaddr desc_pa;
 385         int i;
 386
 387         max = vq->vring.num;
 388         num_bufs = total_bufs;
 389         i = virtqueue_get_head(vq, idx++);
 390         desc_pa = vq->vring.desc;
 391
 392         if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_INDIRECT) {
 393             if (vring_desc_len(vdev, desc_pa, i) % sizeof(VRingDesc)) {
 394                 error_report("Invalid size for indirect buffer table");
 395                 exit(1);
 396             }
 397
 398             /* If we've got too many, that implies a descriptor loop. */
 399             if (num_bufs >= max) {
 400                 error_report("Looped descriptor");
 401                 exit(1);
 402             }
 403
 404             /* loop over the indirect descriptor table */
 405             indirect = 1;
 406             max = vring_desc_len(vdev, desc_pa, i) / sizeof(VRingDesc);
 407             desc_pa = vring_desc_addr(vdev, desc_pa, i);
 408             num_bufs = i = 0;
 409         }
 410
 411         do {
 412             /* If we've got too many, that implies a descriptor loop. */
 413             if (++num_bufs > max) {
 414                 error_report("Looped descriptor");
 415                 exit(1);
 416             }
 417
 418             if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_WRITE) {
 419                 in_total += vring_desc_len(vdev, desc_pa, i);
 420             } else {
 421                 out_total += vring_desc_len(vdev, desc_pa, i);
 422             }
 423             if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
 424                 goto done;
 425             }
 426         } while ((i = virtqueue_next_desc(vdev, desc_pa, i, max)) != max);
 427
 428         if (!indirect)
 429             total_bufs = num_bufs;
 430         else
 431             total_bufs++;
 432     }
 433 done:
 434     if (in_bytes) {
 435         *in_bytes = in_total;
 436     }
 437     if (out_bytes) {
 438         *out_bytes = out_total;
 439     }
 440 }
 441
 442 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
 443                           unsigned int out_bytes)
 444 {
 445     unsigned int in_total, out_total;
 446
 447     virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
 448     return in_bytes <= in_total && out_bytes <= out_total;
 449 }
 450
 451 void virtqueue_map_sg(struct iovec *sg, hwaddr *addr,
 452     size_t num_sg, int is_write)
 453 {
 454     unsigned int i;
 455     hwaddr len;
 456
 457     if (num_sg > VIRTQUEUE_MAX_SIZE) {
 458         error_report("virtio: map attempt out of bounds: %zd > %d",
 459                      num_sg, VIRTQUEUE_MAX_SIZE);
 460         exit(1);
 461     }
 462
 463     for (i = 0; i < num_sg; i++) {
 464         len = sg[i].iov_len;
 465         sg[i].iov_base = cpu_physical_memory_map(addr[i], &len, is_write);
 466         if (sg[i].iov_base == NULL || len != sg[i].iov_len) {
 467             error_report("virtio: error trying to map MMIO memory");
 468             exit(1);
 469         }
 470     }
 471 }
 472
 473 int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem)
 474 {
 475     unsigned int i, head, max;
 476     hwaddr desc_pa = vq->vring.desc;
 477     VirtIODevice *vdev = vq->vdev;
 478
 479     if (!virtqueue_num_heads(vq, vq->last_avail_idx))
 480         return 0;
 481
 482     /* When we start there are none of either input nor output. */
 483     elem->out_num = elem->in_num = 0;
 484
 485     max = vq->vring.num;
 486
 487     i = head = virtqueue_get_head(vq, vq->last_avail_idx++);
 488     if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
 489         vring_set_avail_event(vq, vq->last_avail_idx);
 490     }
 491
 492     if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_INDIRECT) {
 493         if (vring_desc_len(vdev, desc_pa, i) % sizeof(VRingDesc)) {
 494             error_report("Invalid size for indirect buffer table");
 495             exit(1);
 496         }
 497
 498         /* loop over the indirect descriptor table */
 499         max = vring_desc_len(vdev, desc_pa, i) / sizeof(VRingDesc);
 500         desc_pa = vring_desc_addr(vdev, desc_pa, i);
 501         i = 0;
 502     }
 503
 504     /* Collect all the descriptors */
 505     do {
 506         struct iovec *sg;
 507
 508         if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_WRITE) {
 509             if (elem->in_num >= ARRAY_SIZE(elem->in_sg)) {
 510                 error_report("Too many write descriptors in indirect table");
 511                 exit(1);
 512             }
 513             elem->in_addr[elem->in_num] = vring_desc_addr(vdev, desc_pa, i);
 514             sg = &elem->in_sg[elem->in_num++];
 515         } else {
 516             if (elem->out_num >= ARRAY_SIZE(elem->out_sg)) {
 517                 error_report("Too many read descriptors in indirect table");
 518                 exit(1);
 519             }
 520             elem->out_addr[elem->out_num] = vring_desc_addr(vdev, desc_pa, i);
 521             sg = &elem->out_sg[elem->out_num++];
 522         }
 523
 524         sg->iov_len = vring_desc_len(vdev, desc_pa, i);
 525
 526         /* If we've got too many, that implies a descriptor loop. */
 527         if ((elem->in_num + elem->out_num) > max) {
 528             error_report("Looped descriptor");
 529             exit(1);
 530         }
 531     } while ((i = virtqueue_next_desc(vdev, desc_pa, i, max)) != max);
 532
 533     /* Now map what we have collected */
 534     virtqueue_map_sg(elem->in_sg, elem->in_addr, elem->in_num, 1);
 535     virtqueue_map_sg(elem->out_sg, elem->out_addr, elem->out_num, 0);
 536
 537     elem->index = head;
 538
 539     vq->inuse++;
 540
 541     trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
 542     return elem->in_num + elem->out_num;
 543 }
 544
 545 /* virtio device */
 546 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector)
 547 {
 548     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
 549     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
 550
 551     if (k->notify) {
 552         k->notify(qbus->parent, vector);
 553     }
 554 }
 555
 556 void virtio_update_irq(VirtIODevice *vdev)
 557 {
 558     virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
 559 }
 560
 561 static int virtio_validate_features(VirtIODevice *vdev)
 562 {
 563     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 564
 565     if (k->validate_features) {
 566         return k->validate_features(vdev);
 567     } else {
 568         return 0;
 569     }
 570 }
 571
 572 int virtio_set_status(VirtIODevice *vdev, uint8_t val)
 573 {
 574     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 575     trace_virtio_set_status(vdev, val);
 576
 577     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
 578         if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) &&
 579             val & VIRTIO_CONFIG_S_FEATURES_OK) {
 580             int ret = virtio_validate_features(vdev);
 581
 582             if (ret) {
 583                 return ret;
 584             }
 585         }
 586     }
 587     if (k->set_status) {
 588         k->set_status(vdev, val);
 589     }
 590     vdev->status = val;
 591     return 0;
 592 }
 593
 594 bool target_words_bigendian(void);
 595 static enum virtio_device_endian virtio_default_endian(void)
 596 {
 597     if (target_words_bigendian()) {
 598         return VIRTIO_DEVICE_ENDIAN_BIG;
 599     } else {
 600         return VIRTIO_DEVICE_ENDIAN_LITTLE;
 601     }
 602 }
 603
 604 static enum virtio_device_endian virtio_current_cpu_endian(void)
 605 {
 606     CPUClass *cc = CPU_GET_CLASS(current_cpu);
 607
 608     if (cc->virtio_is_big_endian(current_cpu)) {
 609         return VIRTIO_DEVICE_ENDIAN_BIG;
 610     } else {
 611         return VIRTIO_DEVICE_ENDIAN_LITTLE;
 612     }
 613 }
 614
 615 void virtio_reset(void *opaque)
 616 {
 617     VirtIODevice *vdev = opaque;
 618     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 619     int i;
 620
 621     virtio_set_status(vdev, 0);
 622     if (current_cpu) {
 623         /* Guest initiated reset */
 624         vdev->device_endian = virtio_current_cpu_endian();
 625     } else {
 626         /* System reset */
 627         vdev->device_endian = virtio_default_endian();
 628     }
 629
 630     if (k->reset) {
 631         k->reset(vdev);
 632     }
 633
 634     vdev->guest_features = 0;
 635     vdev->queue_sel = 0;
 636     vdev->status = 0;
 637     vdev->isr = 0;
 638     vdev->config_vector = VIRTIO_NO_VECTOR;
 639     virtio_notify_vector(vdev, vdev->config_vector);
 640
 641     for(i = 0; i < VIRTIO_QUEUE_MAX; i++) {
 642         vdev->vq[i].vring.desc = 0;
 643         vdev->vq[i].vring.avail = 0;
 644         vdev->vq[i].vring.used = 0;
 645         vdev->vq[i].last_avail_idx = 0;
 646         virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
 647         vdev->vq[i].signalled_used = 0;
 648         vdev->vq[i].signalled_used_valid = false;
 649         vdev->vq[i].notification = true;
 650         vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
 651     }
 652 }
 653
 654 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr)
 655 {
 656     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 657     uint8_t val;
 658
 659     if (addr + sizeof(val) > vdev->config_len) {
 660         return (uint32_t)-1;
 661     }
 662
 663     k->get_config(vdev, vdev->config);
 664
 665     val = ldub_p(vdev->config + addr);
 666     return val;
 667 }
 668
 669 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr)
 670 {
 671     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 672     uint16_t val;
 673
 674     if (addr + sizeof(val) > vdev->config_len) {
 675         return (uint32_t)-1;
 676     }
 677
 678     k->get_config(vdev, vdev->config);
 679
 680     val = lduw_p(vdev->config + addr);
 681     return val;
 682 }
 683
 684 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr)
 685 {
 686     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 687     uint32_t val;
 688
 689     if (addr + sizeof(val) > vdev->config_len) {
 690         return (uint32_t)-1;
 691     }
 692
 693     k->get_config(vdev, vdev->config);
 694
 695     val = ldl_p(vdev->config + addr);
 696     return val;
 697 }
 698
 699 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data)
 700 {
 701     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 702     uint8_t val = data;
 703
 704     if (addr + sizeof(val) > vdev->config_len) {
 705         return;
 706     }
 707
 708     stb_p(vdev->config + addr, val);
 709
 710     if (k->set_config) {
 711         k->set_config(vdev, vdev->config);
 712     }
 713 }
 714
 715 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data)
 716 {
 717     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 718     uint16_t val = data;
 719
 720     if (addr + sizeof(val) > vdev->config_len) {
 721         return;
 722     }
 723
 724     stw_p(vdev->config + addr, val);
 725
 726     if (k->set_config) {
 727         k->set_config(vdev, vdev->config);
 728     }
 729 }
 730
 731 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
 732 {
 733     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 734     uint32_t val = data;
 735
 736     if (addr + sizeof(val) > vdev->config_len) {
 737         return;
 738     }
 739
 740     stl_p(vdev->config + addr, val);
 741
 742     if (k->set_config) {
 743         k->set_config(vdev, vdev->config);
 744     }
 745 }
 746
 747 uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr)
 748 {
 749     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 750     uint8_t val;
 751
 752     if (addr + sizeof(val) > vdev->config_len) {
 753         return (uint32_t)-1;
 754     }
 755
 756     k->get_config(vdev, vdev->config);
 757
 758     val = ldub_p(vdev->config + addr);
 759     return val;
 760 }
 761
 762 uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr)
 763 {
 764     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 765     uint16_t val;
 766
 767     if (addr + sizeof(val) > vdev->config_len) {
 768         return (uint32_t)-1;
 769     }
 770
 771     k->get_config(vdev, vdev->config);
 772
 773     val = lduw_le_p(vdev->config + addr);
 774     return val;
 775 }
 776
 777 uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr)
 778 {
 779     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 780     uint32_t val;
 781
 782     if (addr + sizeof(val) > vdev->config_len) {
 783         return (uint32_t)-1;
 784     }
 785
 786     k->get_config(vdev, vdev->config);
 787
 788     val = ldl_le_p(vdev->config + addr);
 789     return val;
 790 }
 791
 792 void virtio_config_modern_writeb(VirtIODevice *vdev,
 793                                  uint32_t addr, uint32_t data)
 794 {
 795     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 796     uint8_t val = data;
 797
 798     if (addr + sizeof(val) > vdev->config_len) {
 799         return;
 800     }
 801
 802     stb_p(vdev->config + addr, val);
 803
 804     if (k->set_config) {
 805         k->set_config(vdev, vdev->config);
 806     }
 807 }
 808
 809 void virtio_config_modern_writew(VirtIODevice *vdev,
 810                                  uint32_t addr, uint32_t data)
 811 {
 812     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 813     uint16_t val = data;
 814
 815     if (addr + sizeof(val) > vdev->config_len) {
 816         return;
 817     }
 818
 819     stw_le_p(vdev->config + addr, val);
 820
 821     if (k->set_config) {
 822         k->set_config(vdev, vdev->config);
 823     }
 824 }
 825
 826 void virtio_config_modern_writel(VirtIODevice *vdev,
 827                                  uint32_t addr, uint32_t data)
 828 {
 829     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 830     uint32_t val = data;
 831
 832     if (addr + sizeof(val) > vdev->config_len) {
 833         return;
 834     }
 835
 836     stl_le_p(vdev->config + addr, val);
 837
 838     if (k->set_config) {
 839         k->set_config(vdev, vdev->config);
 840     }
 841 }
 842
 843 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
 844 {
 845     vdev->vq[n].vring.desc = addr;
 846     virtio_queue_update_rings(vdev, n);
 847 }
 848
 849 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
 850 {
 851     return vdev->vq[n].vring.desc;
 852 }
 853
 854 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
 855                             hwaddr avail, hwaddr used)
 856 {
 857     vdev->vq[n].vring.desc = desc;
 858     vdev->vq[n].vring.avail = avail;
 859     vdev->vq[n].vring.used = used;
 860 }
 861
 862 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num)
 863 {
 864     /* Don't allow guest to flip queue between existent and
 865      * nonexistent states, or to set it to an invalid size.
 866      */
 867     if (!!num != !!vdev->vq[n].vring.num ||
 868         num > VIRTQUEUE_MAX_SIZE ||
 869         num < 0) {
 870         return;
 871     }
 872     vdev->vq[n].vring.num = num;
 873 }
 874
 875 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector)
 876 {
 877     return QLIST_FIRST(&vdev->vector_queues[vector]);
 878 }
 879
 880 VirtQueue *virtio_vector_next_queue(VirtQueue *vq)
 881 {
 882     return QLIST_NEXT(vq, node);
 883 }
 884
 885 int virtio_queue_get_num(VirtIODevice *vdev, int n)
 886 {
 887     return vdev->vq[n].vring.num;
 888 }
 889
 890 int virtio_get_num_queues(VirtIODevice *vdev)
 891 {
 892     int i;
 893
 894     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
 895         if (!virtio_queue_get_num(vdev, i)) {
 896             break;
 897         }
 898     }
 899
 900     return i;
 901 }
 902
 903 int virtio_queue_get_id(VirtQueue *vq)
 904 {
 905     VirtIODevice *vdev = vq->vdev;
 906     assert(vq >= &vdev->vq[0] && vq < &vdev->vq[VIRTIO_QUEUE_MAX]);
 907     return vq - &vdev->vq[0];
 908 }
 909
 910 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
 911 {
 912     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
 913     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
 914
 915     /* virtio-1 compliant devices cannot change the alignment */
 916     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
 917         error_report("tried to modify queue alignment for virtio-1 device");
 918         return;
 919     }
 920     /* Check that the transport told us it was going to do this
 921      * (so a buggy transport will immediately assert rather than
 922      * silently failing to migrate this state)
 923      */
 924     assert(k->has_variable_vring_alignment);
 925
 926     vdev->vq[n].vring.align = align;
 927     virtio_queue_update_rings(vdev, n);
 928 }
 929
 930 void virtio_queue_notify_vq(VirtQueue *vq)
 931 {
 932     if (vq->vring.desc && vq->handle_output) {
 933         VirtIODevice *vdev = vq->vdev;
 934
 935         trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
 936         vq->handle_output(vdev, vq);
 937     }
 938 }
 939
 940 void virtio_queue_notify(VirtIODevice *vdev, int n)
 941 {
 942     virtio_queue_notify_vq(&vdev->vq[n]);
 943 }
 944
 945 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
 946 {
 947     return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector :
 948         VIRTIO_NO_VECTOR;
 949 }
 950
 951 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
 952 {
 953     VirtQueue *vq = &vdev->vq[n];
 954
 955     if (n < VIRTIO_QUEUE_MAX) {
 956         if (vdev->vector_queues &&
 957             vdev->vq[n].vector != VIRTIO_NO_VECTOR) {
 958             QLIST_REMOVE(vq, node);
 959         }
 960         vdev->vq[n].vector = vector;
 961         if (vdev->vector_queues &&
 962             vector != VIRTIO_NO_VECTOR) {
 963             QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node);
 964         }
 965     }
 966 }
 967
 968 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
 969                             void (*handle_output)(VirtIODevice *, VirtQueue *))
 970 {
 971     int i;
 972
 973     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
 974         if (vdev->vq[i].vring.num == 0)
 975             break;
 976     }
 977
 978     if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
 979         abort();
 980
 981     vdev->vq[i].vring.num = queue_size;
 982     vdev->vq[i].vring.num_default = queue_size;
 983     vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
 984     vdev->vq[i].handle_output = handle_output;
 985
 986     return &vdev->vq[i];
 987 }
 988
 989 void virtio_del_queue(VirtIODevice *vdev, int n)
 990 {
 991     if (n < 0 || n >= VIRTIO_QUEUE_MAX) {
 992         abort();
 993     }
 994
 995     vdev->vq[n].vring.num = 0;
 996     vdev->vq[n].vring.num_default = 0;
 997 }
 998
 999 void virtio_irq(VirtQueue *vq)
1000 {
1001     trace_virtio_irq(vq);
1002     vq->vdev->isr |= 0x01;
1003     virtio_notify_vector(vq->vdev, vq->vector);
1004 }
1005
1006 static bool vring_notify(VirtIODevice *vdev, VirtQueue *vq)
1007 {
1008     uint16_t old, new;
1009     bool v;
1010     /* We need to expose used array entries before checking used event. */
1011     smp_mb();
1012     /* Always notify when queue is empty (when feature acknowledge) */
1013     if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
1014         !vq->inuse && vring_avail_idx(vq) == vq->last_avail_idx) {
1015         return true;
1016     }
1017
1018     if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
1019         return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
1020     }
1021
1022     v = vq->signalled_used_valid;
1023     vq->signalled_used_valid = true;
1024     old = vq->signalled_used;
1025     new = vq->signalled_used = vring_used_idx(vq);
1026     return !v || vring_need_event(vring_get_used_event(vq), new, old);
1027 }
1028
1029 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
1030 {
1031     if (!vring_notify(vdev, vq)) {
1032         return;
1033     }
1034
1035     trace_virtio_notify(vdev, vq);
1036     vdev->isr |= 0x01;
1037     virtio_notify_vector(vdev, vq->vector);
1038 }
1039
1040 void virtio_notify_config(VirtIODevice *vdev)
1041 {
1042     if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
1043         return;
1044
1045     vdev->isr |= 0x03;
1046     vdev->generation++;
1047     virtio_notify_vector(vdev, vdev->config_vector);
1048 }
1049
1050 static bool virtio_device_endian_needed(void *opaque)
1051 {
1052     VirtIODevice *vdev = opaque;
1053
1054     assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
1055     if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1056         return vdev->device_endian != virtio_default_endian();
1057     }
1058     /* Devices conforming to VIRTIO 1.0 or later are always LE. */
1059     return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE;
1060 }
1061
1062 static bool virtio_64bit_features_needed(void *opaque)
1063 {
1064     VirtIODevice *vdev = opaque;
1065
1066     return (vdev->host_features >> 32) != 0;
1067 }
1068
1069 static bool virtio_virtqueue_needed(void *opaque)
1070 {
1071     VirtIODevice *vdev = opaque;
1072
1073     return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
1074 }
1075
1076 static bool virtio_ringsize_needed(void *opaque)
1077 {
1078     VirtIODevice *vdev = opaque;
1079     int i;
1080
1081     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1082         if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) {
1083             return true;
1084         }
1085     }
1086     return false;
1087 }
1088
1089 static void put_virtqueue_state(QEMUFile *f, void *pv, size_t size)
1090 {
1091     VirtIODevice *vdev = pv;
1092     int i;
1093
1094     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1095         qemu_put_be64(f, vdev->vq[i].vring.avail);
1096         qemu_put_be64(f, vdev->vq[i].vring.used);
1097     }
1098 }
1099
1100 static int get_virtqueue_state(QEMUFile *f, void *pv, size_t size)
1101 {
1102     VirtIODevice *vdev = pv;
1103     int i;
1104
1105     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1106         vdev->vq[i].vring.avail = qemu_get_be64(f);
1107         vdev->vq[i].vring.used = qemu_get_be64(f);
1108     }
1109     return 0;
1110 }
1111
1112 static VMStateInfo vmstate_info_virtqueue = {
1113     .name = "virtqueue_state",
1114     .get = get_virtqueue_state,
1115     .put = put_virtqueue_state,
1116 };
1117
1118 static const VMStateDescription vmstate_virtio_virtqueues = {
1119     .name = "virtio/virtqueues",
1120     .version_id = 1,
1121     .minimum_version_id = 1,
1122     .needed = &virtio_virtqueue_needed,
1123     .fields = (VMStateField[]) {
1124         {
1125             .name         = "virtqueues",
1126             .version_id   = 0,
1127             .field_exists = NULL,
1128             .size         = 0,
1129             .info         = &vmstate_info_virtqueue,
1130             .flags        = VMS_SINGLE,
1131             .offset       = 0,
1132         },
1133         VMSTATE_END_OF_LIST()
1134     }
1135 };
1136
1137 static void put_ringsize_state(QEMUFile *f, void *pv, size_t size)
1138 {
1139     VirtIODevice *vdev = pv;
1140     int i;
1141
1142     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1143         qemu_put_be32(f, vdev->vq[i].vring.num_default);
1144     }
1145 }
1146
1147 static int get_ringsize_state(QEMUFile *f, void *pv, size_t size)
1148 {
1149     VirtIODevice *vdev = pv;
1150     int i;
1151
1152     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1153         vdev->vq[i].vring.num_default = qemu_get_be32(f);
1154     }
1155     return 0;
1156 }
1157
1158 static VMStateInfo vmstate_info_ringsize = {
1159     .name = "ringsize_state",
1160     .get = get_ringsize_state,
1161     .put = put_ringsize_state,
1162 };
1163
1164 static const VMStateDescription vmstate_virtio_ringsize = {
1165     .name = "virtio/ringsize",
1166     .version_id = 1,
1167     .minimum_version_id = 1,
1168     .needed = &virtio_ringsize_needed,
1169     .fields = (VMStateField[]) {
1170         {
1171             .name         = "ringsize",
1172             .version_id   = 0,
1173             .field_exists = NULL,
1174             .size         = 0,
1175             .info         = &vmstate_info_ringsize,
1176             .flags        = VMS_SINGLE,
1177             .offset       = 0,
1178         },
1179         VMSTATE_END_OF_LIST()
1180     }
1181 };
1182
1183 static const VMStateDescription vmstate_virtio_device_endian = {
1184     .name = "virtio/device_endian",
1185     .version_id = 1,
1186     .minimum_version_id = 1,
1187     .needed = &virtio_device_endian_needed,
1188     .fields = (VMStateField[]) {
1189         VMSTATE_UINT8(device_endian, VirtIODevice),
1190         VMSTATE_END_OF_LIST()
1191     }
1192 };
1193
1194 static const VMStateDescription vmstate_virtio_64bit_features = {
1195     .name = "virtio/64bit_features",
1196     .version_id = 1,
1197     .minimum_version_id = 1,
1198     .needed = &virtio_64bit_features_needed,
1199     .fields = (VMStateField[]) {
1200         VMSTATE_UINT64(guest_features, VirtIODevice),
1201         VMSTATE_END_OF_LIST()
1202     }
1203 };
1204
1205 static const VMStateDescription vmstate_virtio = {
1206     .name = "virtio",
1207     .version_id = 1,
1208     .minimum_version_id = 1,
1209     .minimum_version_id_old = 1,
1210     .fields = (VMStateField[]) {
1211         VMSTATE_END_OF_LIST()
1212     },
1213     .subsections = (const VMStateDescription*[]) {
1214         &vmstate_virtio_device_endian,
1215         &vmstate_virtio_64bit_features,
1216         &vmstate_virtio_virtqueues,
1217         &vmstate_virtio_ringsize,
1218         NULL
1219     }
1220 };
1221
1222 void virtio_save(VirtIODevice *vdev, QEMUFile *f)
1223 {
1224     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1225     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1226     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1227     uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff);
1228     int i;
1229
1230     if (k->save_config) {
1231         k->save_config(qbus->parent, f);
1232     }
1233
1234     qemu_put_8s(f, &vdev->status);
1235     qemu_put_8s(f, &vdev->isr);
1236     qemu_put_be16s(f, &vdev->queue_sel);
1237     qemu_put_be32s(f, &guest_features_lo);
1238     qemu_put_be32(f, vdev->config_len);
1239     qemu_put_buffer(f, vdev->config, vdev->config_len);
1240
1241     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1242         if (vdev->vq[i].vring.num == 0)
1243             break;
1244     }
1245
1246     qemu_put_be32(f, i);
1247
1248     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1249         if (vdev->vq[i].vring.num == 0)
1250             break;
1251
1252         qemu_put_be32(f, vdev->vq[i].vring.num);
1253         if (k->has_variable_vring_alignment) {
1254             qemu_put_be32(f, vdev->vq[i].vring.align);
1255         }
1256         /* XXX virtio-1 devices */
1257         qemu_put_be64(f, vdev->vq[i].vring.desc);
1258         qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
1259         if (k->save_queue) {
1260             k->save_queue(qbus->parent, i, f);
1261         }
1262     }
1263
1264     if (vdc->save != NULL) {
1265         vdc->save(vdev, f);
1266     }
1267
1268     /* Subsections */
1269     vmstate_save_state(f, &vmstate_virtio, vdev, NULL);
1270 }
1271
1272 static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val)
1273 {
1274     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1275     bool bad = (val & ~(vdev->host_features)) != 0;
1276
1277     val &= vdev->host_features;
1278     if (k->set_features) {
1279         k->set_features(vdev, val);
1280     }
1281     vdev->guest_features = val;
1282     return bad ? -1 : 0;
1283 }
1284
1285 int virtio_set_features(VirtIODevice *vdev, uint64_t val)
1286 {
1287    /*
1288      * The driver must not attempt to set features after feature negotiation
1289      * has finished.
1290      */
1291     if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) {
1292         return -EINVAL;
1293     }
1294     return virtio_set_features_nocheck(vdev, val);
1295 }
1296
1297 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
1298 {
1299     int i, ret;
1300     int32_t config_len;
1301     uint32_t num;
1302     uint32_t features;
1303     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1304     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1305     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1306
1307     /*
1308      * We poison the endianness to ensure it does not get used before
1309      * subsections have been loaded.
1310      */
1311     vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN;
1312
1313     if (k->load_config) {
1314         ret = k->load_config(qbus->parent, f);
1315         if (ret)
1316             return ret;
1317     }
1318
1319     qemu_get_8s(f, &vdev->status);
1320     qemu_get_8s(f, &vdev->isr);
1321     qemu_get_be16s(f, &vdev->queue_sel);
1322     if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) {
1323         return -1;
1324     }
1325     qemu_get_be32s(f, &features);
1326
1327     config_len = qemu_get_be32(f);
1328
1329     /*
1330      * There are cases where the incoming config can be bigger or smaller
1331      * than what we have; so load what we have space for, and skip
1332      * any excess that's in the stream.
1333      */
1334     qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len));
1335
1336     while (config_len > vdev->config_len) {
1337         qemu_get_byte(f);
1338         config_len--;
1339     }
1340
1341     num = qemu_get_be32(f);
1342
1343     if (num > VIRTIO_QUEUE_MAX) {
1344         error_report("Invalid number of PCI queues: 0x%x", num);
1345         return -1;
1346     }
1347
1348     for (i = 0; i < num; i++) {
1349         vdev->vq[i].vring.num = qemu_get_be32(f);
1350         if (k->has_variable_vring_alignment) {
1351             vdev->vq[i].vring.align = qemu_get_be32(f);
1352         }
1353         vdev->vq[i].vring.desc = qemu_get_be64(f);
1354         qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
1355         vdev->vq[i].signalled_used_valid = false;
1356         vdev->vq[i].notification = true;
1357
1358         if (vdev->vq[i].vring.desc) {
1359             /* XXX virtio-1 devices */
1360             virtio_queue_update_rings(vdev, i);
1361         } else if (vdev->vq[i].last_avail_idx) {
1362             error_report("VQ %d address 0x0 "
1363                          "inconsistent with Host index 0x%x",
1364                          i, vdev->vq[i].last_avail_idx);
1365                 return -1;
1366         }
1367         if (k->load_queue) {
1368             ret = k->load_queue(qbus->parent, i, f);
1369             if (ret)
1370                 return ret;
1371         }
1372     }
1373
1374     virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
1375
1376     if (vdc->load != NULL) {
1377         ret = vdc->load(vdev, f, version_id);
1378         if (ret) {
1379             return ret;
1380         }
1381     }
1382
1383     /* Subsections */
1384     ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1);
1385     if (ret) {
1386         return ret;
1387     }
1388
1389     if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) {
1390         vdev->device_endian = virtio_default_endian();
1391     }
1392
1393     if (virtio_64bit_features_needed(vdev)) {
1394         /*
1395          * Subsection load filled vdev->guest_features.  Run them
1396          * through virtio_set_features to sanity-check them against
1397          * host_features.
1398          */
1399         uint64_t features64 = vdev->guest_features;
1400         if (virtio_set_features_nocheck(vdev, features64) < 0) {
1401             error_report("Features 0x%" PRIx64 " unsupported. "
1402                          "Allowed features: 0x%" PRIx64,
1403                          features64, vdev->host_features);
1404             return -1;
1405         }
1406     } else {
1407         if (virtio_set_features_nocheck(vdev, features) < 0) {
1408             error_report("Features 0x%x unsupported. "
1409                          "Allowed features: 0x%" PRIx64,
1410                          features, vdev->host_features);
1411             return -1;
1412         }
1413     }
1414
1415     for (i = 0; i < num; i++) {
1416         if (vdev->vq[i].vring.desc) {
1417             uint16_t nheads;
1418             nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
1419             /* Check it isn't doing strange things with descriptor numbers. */
1420             if (nheads > vdev->vq[i].vring.num) {
1421                 error_report("VQ %d size 0x%x Guest index 0x%x "
1422                              "inconsistent with Host index 0x%x: delta 0x%x",
1423                              i, vdev->vq[i].vring.num,
1424                              vring_avail_idx(&vdev->vq[i]),
1425                              vdev->vq[i].last_avail_idx, nheads);
1426                 return -1;
1427             }
1428         }
1429     }
1430
1431     return 0;
1432 }
1433
1434 void virtio_cleanup(VirtIODevice *vdev)
1435 {
1436     qemu_del_vm_change_state_handler(vdev->vmstate);
1437     g_free(vdev->config);
1438     g_free(vdev->vq);
1439     g_free(vdev->vector_queues);
1440 }
1441
1442 static void virtio_vmstate_change(void *opaque, int running, RunState state)
1443 {
1444     VirtIODevice *vdev = opaque;
1445     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1446     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1447     bool backend_run = running && (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK);
1448     vdev->vm_running = running;
1449
1450     if (backend_run) {
1451         virtio_set_status(vdev, vdev->status);
1452     }
1453
1454     if (k->vmstate_change) {
1455         k->vmstate_change(qbus->parent, backend_run);
1456     }
1457
1458     if (!backend_run) {
1459         virtio_set_status(vdev, vdev->status);
1460     }
1461 }
1462
1463 void virtio_instance_init_common(Object *proxy_obj, void *data,
1464                                  size_t vdev_size, const char *vdev_name)
1465 {
1466     DeviceState *vdev = data;
1467
1468     object_initialize(vdev, vdev_size, vdev_name);
1469     object_property_add_child(proxy_obj, "virtio-backend", OBJECT(vdev), NULL);
1470     object_unref(OBJECT(vdev));
1471     qdev_alias_all_properties(vdev, proxy_obj);
1472 }
1473
1474 void virtio_init(VirtIODevice *vdev, const char *name,
1475                  uint16_t device_id, size_t config_size)
1476 {
1477     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1478     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1479     int i;
1480     int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0;
1481
1482     if (nvectors) {
1483         vdev->vector_queues =
1484             g_malloc0(sizeof(*vdev->vector_queues) * nvectors);
1485     }
1486
1487     vdev->device_id = device_id;
1488     vdev->status = 0;
1489     vdev->isr = 0;
1490     vdev->queue_sel = 0;
1491     vdev->config_vector = VIRTIO_NO_VECTOR;
1492     vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX);
1493     vdev->vm_running = runstate_is_running();
1494     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1495         vdev->vq[i].vector = VIRTIO_NO_VECTOR;
1496         vdev->vq[i].vdev = vdev;
1497         vdev->vq[i].queue_index = i;
1498     }
1499
1500     vdev->name = name;
1501     vdev->config_len = config_size;
1502     if (vdev->config_len) {
1503         vdev->config = g_malloc0(config_size);
1504     } else {
1505         vdev->config = NULL;
1506     }
1507     vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change,
1508                                                      vdev);
1509     vdev->device_endian = virtio_default_endian();
1510 }
1511
1512 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
1513 {
1514     return vdev->vq[n].vring.desc;
1515 }
1516
1517 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
1518 {
1519     return vdev->vq[n].vring.avail;
1520 }
1521
1522 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
1523 {
1524     return vdev->vq[n].vring.used;
1525 }
1526
1527 hwaddr virtio_queue_get_ring_addr(VirtIODevice *vdev, int n)
1528 {
1529     return vdev->vq[n].vring.desc;
1530 }
1531
1532 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
1533 {
1534     return sizeof(VRingDesc) * vdev->vq[n].vring.num;
1535 }
1536
1537 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
1538 {
1539     return offsetof(VRingAvail, ring) +
1540         sizeof(uint16_t) * vdev->vq[n].vring.num;
1541 }
1542
1543 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
1544 {
1545     return offsetof(VRingUsed, ring) +
1546         sizeof(VRingUsedElem) * vdev->vq[n].vring.num;
1547 }
1548
1549 hwaddr virtio_queue_get_ring_size(VirtIODevice *vdev, int n)
1550 {
1551     return vdev->vq[n].vring.used - vdev->vq[n].vring.desc +
1552             virtio_queue_get_used_size(vdev, n);
1553 }
1554
1555 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
1556 {
1557     return vdev->vq[n].last_avail_idx;
1558 }
1559
1560 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx)
1561 {
1562     vdev->vq[n].last_avail_idx = idx;
1563 }
1564
1565 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
1566 {
1567     vdev->vq[n].signalled_used_valid = false;
1568 }
1569
1570 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)
1571 {
1572     return vdev->vq + n;
1573 }
1574
1575 uint16_t virtio_get_queue_index(VirtQueue *vq)
1576 {
1577     return vq->queue_index;
1578 }
1579
1580 static void virtio_queue_guest_notifier_read(EventNotifier *n)
1581 {
1582     VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
1583     if (event_notifier_test_and_clear(n)) {
1584         virtio_irq(vq);
1585     }
1586 }
1587
1588 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
1589                                                 bool with_irqfd)
1590 {
1591     if (assign && !with_irqfd) {
1592         event_notifier_set_handler(&vq->guest_notifier,
1593                                    virtio_queue_guest_notifier_read);
1594     } else {
1595         event_notifier_set_handler(&vq->guest_notifier, NULL);
1596     }
1597     if (!assign) {
1598         /* Test and clear notifier before closing it,
1599          * in case poll callback didn't have time to run. */
1600         virtio_queue_guest_notifier_read(&vq->guest_notifier);
1601     }
1602 }
1603
1604 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
1605 {
1606     return &vq->guest_notifier;
1607 }
1608
1609 static void virtio_queue_host_notifier_read(EventNotifier *n)
1610 {
1611     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
1612     if (event_notifier_test_and_clear(n)) {
1613         virtio_queue_notify_vq(vq);
1614     }
1615 }
1616
1617 void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
1618                                                bool set_handler)
1619 {
1620     if (assign && set_handler) {
1621         event_notifier_set_handler(&vq->host_notifier,
1622                                    virtio_queue_host_notifier_read);
1623     } else {
1624         event_notifier_set_handler(&vq->host_notifier, NULL);
1625     }
1626     if (!assign) {
1627         /* Test and clear notifier before after disabling event,
1628          * in case poll callback didn't have time to run. */
1629         virtio_queue_host_notifier_read(&vq->host_notifier);
1630     }
1631 }
1632
1633 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
1634 {
1635     return &vq->host_notifier;
1636 }
1637
1638 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name)
1639 {
1640     g_free(vdev->bus_name);
1641     vdev->bus_name = g_strdup(bus_name);
1642 }
1643
1644 static void virtio_device_realize(DeviceState *dev, Error **errp)
1645 {
1646     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1647     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
1648     Error *err = NULL;
1649
1650     if (vdc->realize != NULL) {
1651         vdc->realize(dev, &err);
1652         if (err != NULL) {
1653             error_propagate(errp, err);
1654             return;
1655         }
1656     }
1657
1658     virtio_bus_device_plugged(vdev, &err);
1659     if (err != NULL) {
1660         error_propagate(errp, err);
1661         return;
1662     }
1663 }
1664
1665 static void virtio_device_unrealize(DeviceState *dev, Error **errp)
1666 {
1667     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1668     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
1669     Error *err = NULL;
1670
1671     virtio_bus_device_unplugged(vdev);
1672
1673     if (vdc->unrealize != NULL) {
1674         vdc->unrealize(dev, &err);
1675         if (err != NULL) {
1676             error_propagate(errp, err);
1677             return;
1678         }
1679     }
1680
1681     g_free(vdev->bus_name);
1682     vdev->bus_name = NULL;
1683 }
1684
1685 static Property virtio_properties[] = {
1686     DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features),
1687     DEFINE_PROP_END_OF_LIST(),
1688 };
1689
1690 static void virtio_device_class_init(ObjectClass *klass, void *data)
1691 {
1692     /* Set the default value here. */
1693     DeviceClass *dc = DEVICE_CLASS(klass);
1694
1695     dc->realize = virtio_device_realize;
1696     dc->unrealize = virtio_device_unrealize;
1697     dc->bus_type = TYPE_VIRTIO_BUS;
1698     dc->props = virtio_properties;
1699 }
1700
1701 static const TypeInfo virtio_device_info = {
1702     .name = TYPE_VIRTIO_DEVICE,
1703     .parent = TYPE_DEVICE,
1704     .instance_size = sizeof(VirtIODevice),
1705     .class_init = virtio_device_class_init,
1706     .abstract = true,
1707     .class_size = sizeof(VirtioDeviceClass),
1708 };
1709
1710 static void virtio_register_types(void)
1711 {
1712     type_register_static(&virtio_device_info);
1713 }
1714
1715 type_init(virtio_register_types)