hw/virtio/dataplane/vring.c

   1 /* Copyright 2012 Red Hat, Inc.
   2  * Copyright IBM, Corp. 2012
   3  *
   4  * Based on Linux 2.6.39 vhost code:
   5  * Copyright (C) 2009 Red Hat, Inc.
   6  * Copyright (C) 2006 Rusty Russell IBM Corporation
   7  *
   8  * Author: Michael S. Tsirkin <mst@redhat.com>
   9  *         Stefan Hajnoczi <stefanha@redhat.com>
  10  *
  11  * Inspiration, some code, and most witty comments come from
  12  * Documentation/virtual/lguest/lguest.c, by Rusty Russell
  13  *
  14  * This work is licensed under the terms of the GNU GPL, version 2.
  15  */
  16
  17 #include "qemu/osdep.h"
  18 #include "trace.h"
  19 #include "hw/hw.h"
  20 #include "exec/memory.h"
  21 #include "exec/address-spaces.h"
  22 #include "hw/virtio/virtio-access.h"
  23 #include "hw/virtio/dataplane/vring.h"
  24 #include "hw/virtio/dataplane/vring-accessors.h"
  25 #include "qemu/error-report.h"
  26
  27 /* vring_map can be coupled with vring_unmap or (if you still have the
  28  * value returned in *mr) memory_region_unref.
  29  * Returns NULL on failure.
  30  * Callers that can handle a partial mapping must supply mapped_len pointer to
  31  * get the actual length mapped.
  32  * Passing mapped_len == NULL requires either a full mapping or a failure.
  33  */
  34 static void *vring_map(MemoryRegion **mr, hwaddr phys,
  35                        hwaddr len, hwaddr *mapped_len,
  36                        bool is_write)
  37 {
  38     MemoryRegionSection section = memory_region_find(get_system_memory(), phys, len);
  39     uint64_t size;
  40
  41     if (!section.mr) {
  42         goto out;
  43     }
  44
  45     size = int128_get64(section.size);
  46     assert(size);
  47
  48     /* Passing mapped_len == NULL requires either a full mapping or a failure. */
  49     if (!mapped_len && size < len) {
  50         goto out;
  51     }
  52
  53     if (is_write && section.readonly) {
  54         goto out;
  55     }
  56     if (!memory_region_is_ram(section.mr)) {
  57         goto out;
  58     }
  59
  60     /* Ignore regions with dirty logging, we cannot mark them dirty */
  61     if (memory_region_get_dirty_log_mask(section.mr)) {
  62         goto out;
  63     }
  64
  65     if (mapped_len) {
  66         *mapped_len = MIN(size, len);
  67     }
  68
  69     *mr = section.mr;
  70     return memory_region_get_ram_ptr(section.mr) + section.offset_within_region;
  71
  72 out:
  73     memory_region_unref(section.mr);
  74     *mr = NULL;
  75     return NULL;
  76 }
  77
  78 static void vring_unmap(void *buffer, bool is_write)
  79 {
  80     ram_addr_t addr;
  81     MemoryRegion *mr;
  82
  83     mr = qemu_ram_addr_from_host(buffer, &addr);
  84     memory_region_unref(mr);
  85 }
  86
  87 /* Map the guest's vring to host memory */
  88 bool vring_setup(Vring *vring, VirtIODevice *vdev, int n)
  89 {
  90     struct vring *vr = &vring->vr;
  91     hwaddr addr;
  92     hwaddr size;
  93     void *ptr;
  94
  95     vring->broken = false;
  96     vr->num = virtio_queue_get_num(vdev, n);
  97
  98     addr = virtio_queue_get_desc_addr(vdev, n);
  99     size = virtio_queue_get_desc_size(vdev, n);
 100     /* Map the descriptor area as read only */
 101     ptr = vring_map(&vring->mr_desc, addr, size, NULL, false);
 102     if (!ptr) {
 103         error_report("Failed to map 0x%" HWADDR_PRIx " byte for vring desc "
 104                      "at 0x%" HWADDR_PRIx,
 105                       size, addr);
 106         goto out_err_desc;
 107     }
 108     vr->desc = ptr;
 109
 110     addr = virtio_queue_get_avail_addr(vdev, n);
 111     size = virtio_queue_get_avail_size(vdev, n);
 112     /* Add the size of the used_event_idx */
 113     size += sizeof(uint16_t);
 114     /* Map the driver area as read only */
 115     ptr = vring_map(&vring->mr_avail, addr, size, NULL, false);
 116     if (!ptr) {
 117         error_report("Failed to map 0x%" HWADDR_PRIx " byte for vring avail "
 118                      "at 0x%" HWADDR_PRIx,
 119                       size, addr);
 120         goto out_err_avail;
 121     }
 122     vr->avail = ptr;
 123
 124     addr = virtio_queue_get_used_addr(vdev, n);
 125     size = virtio_queue_get_used_size(vdev, n);
 126     /* Add the size of the avail_event_idx */
 127     size += sizeof(uint16_t);
 128     /* Map the device area as read-write */
 129     ptr = vring_map(&vring->mr_used, addr, size, NULL, true);
 130     if (!ptr) {
 131         error_report("Failed to map 0x%" HWADDR_PRIx " byte for vring used "
 132                      "at 0x%" HWADDR_PRIx,
 133                       size, addr);
 134         goto out_err_used;
 135     }
 136     vr->used = ptr;
 137
 138     vring->last_avail_idx = virtio_queue_get_last_avail_idx(vdev, n);
 139     vring->last_used_idx = vring_get_used_idx(vdev, vring);
 140     vring->signalled_used = 0;
 141     vring->signalled_used_valid = false;
 142
 143     trace_vring_setup(virtio_queue_get_ring_addr(vdev, n),
 144                       vring->vr.desc, vring->vr.avail, vring->vr.used);
 145     return true;
 146
 147 out_err_used:
 148     memory_region_unref(vring->mr_avail);
 149 out_err_avail:
 150     memory_region_unref(vring->mr_desc);
 151 out_err_desc:
 152     vring->broken = true;
 153     return false;
 154 }
 155
 156 void vring_teardown(Vring *vring, VirtIODevice *vdev, int n)
 157 {
 158     virtio_queue_set_last_avail_idx(vdev, n, vring->last_avail_idx);
 159     virtio_queue_invalidate_signalled_used(vdev, n);
 160
 161     memory_region_unref(vring->mr_desc);
 162     memory_region_unref(vring->mr_avail);
 163     memory_region_unref(vring->mr_used);
 164 }
 165
 166 /* Disable guest->host notifies */
 167 void vring_disable_notification(VirtIODevice *vdev, Vring *vring)
 168 {
 169     if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
 170         vring_set_used_flags(vdev, vring, VRING_USED_F_NO_NOTIFY);
 171     }
 172 }
 173
 174 /* Enable guest->host notifies
 175  *
 176  * Return true if the vring is empty, false if there are more requests.
 177  */
 178 bool vring_enable_notification(VirtIODevice *vdev, Vring *vring)
 179 {
 180     if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
 181         vring_avail_event(&vring->vr) = vring->vr.avail->idx;
 182     } else {
 183         vring_clear_used_flags(vdev, vring, VRING_USED_F_NO_NOTIFY);
 184     }
 185     smp_mb(); /* ensure update is seen before reading avail_idx */
 186     return !vring_more_avail(vdev, vring);
 187 }
 188
 189 /* This is stolen from linux/drivers/vhost/vhost.c:vhost_notify() */
 190 bool vring_should_notify(VirtIODevice *vdev, Vring *vring)
 191 {
 192     uint16_t old, new;
 193     bool v;
 194     /* Flush out used index updates. This is paired
 195      * with the barrier that the Guest executes when enabling
 196      * interrupts. */
 197     smp_mb();
 198
 199     if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
 200         unlikely(!vring_more_avail(vdev, vring))) {
 201         return true;
 202     }
 203
 204     if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
 205         return !(vring_get_avail_flags(vdev, vring) &
 206                  VRING_AVAIL_F_NO_INTERRUPT);
 207     }
 208     old = vring->signalled_used;
 209     v = vring->signalled_used_valid;
 210     new = vring->signalled_used = vring->last_used_idx;
 211     vring->signalled_used_valid = true;
 212
 213     if (unlikely(!v)) {
 214         return true;
 215     }
 216
 217     return vring_need_event(virtio_tswap16(vdev, vring_used_event(&vring->vr)),
 218                             new, old);
 219 }
 220
 221
 222 static int get_desc(Vring *vring, VirtQueueElement *elem,
 223                     struct vring_desc *desc)
 224 {
 225     unsigned *num;
 226     struct iovec *iov;
 227     hwaddr *addr;
 228     MemoryRegion *mr;
 229     hwaddr len;
 230
 231     if (desc->flags & VRING_DESC_F_WRITE) {
 232         num = &elem->in_num;
 233         iov = &elem->in_sg[*num];
 234         addr = &elem->in_addr[*num];
 235     } else {
 236         num = &elem->out_num;
 237         iov = &elem->out_sg[*num];
 238         addr = &elem->out_addr[*num];
 239
 240         /* If it's an output descriptor, they're all supposed
 241          * to come before any input descriptors. */
 242         if (unlikely(elem->in_num)) {
 243             error_report("Descriptor has out after in");
 244             return -EFAULT;
 245         }
 246     }
 247
 248     while (desc->len) {
 249         /* Stop for now if there are not enough iovecs available. */
 250         if (*num >= VIRTQUEUE_MAX_SIZE) {
 251             error_report("Invalid SG num: %u", *num);
 252             return -EFAULT;
 253         }
 254
 255         iov->iov_base = vring_map(&mr, desc->addr, desc->len, &len,
 256                                   desc->flags & VRING_DESC_F_WRITE);
 257         if (!iov->iov_base) {
 258             error_report("Failed to map descriptor addr %#" PRIx64 " len %u",
 259                          (uint64_t)desc->addr, desc->len);
 260             return -EFAULT;
 261         }
 262
 263         /* The MemoryRegion is looked up again and unref'ed later, leave the
 264          * ref in place.  */
 265         (iov++)->iov_len = len;
 266         *addr++ = desc->addr;
 267         desc->len -= len;
 268         desc->addr += len;
 269         *num += 1;
 270     }
 271
 272     return 0;
 273 }
 274
 275 static void copy_in_vring_desc(VirtIODevice *vdev,
 276                                const struct vring_desc *guest,
 277                                struct vring_desc *host)
 278 {
 279     host->addr = virtio_ldq_p(vdev, &guest->addr);
 280     host->len = virtio_ldl_p(vdev, &guest->len);
 281     host->flags = virtio_lduw_p(vdev, &guest->flags);
 282     host->next = virtio_lduw_p(vdev, &guest->next);
 283 }
 284
 285 static bool read_vring_desc(VirtIODevice *vdev,
 286                             hwaddr guest,
 287                             struct vring_desc *host)
 288 {
 289     if (address_space_read(&address_space_memory, guest, MEMTXATTRS_UNSPECIFIED,
 290                            (uint8_t *)host, sizeof *host)) {
 291         return false;
 292     }
 293     host->addr = virtio_tswap64(vdev, host->addr);
 294     host->len = virtio_tswap32(vdev, host->len);
 295     host->flags = virtio_tswap16(vdev, host->flags);
 296     host->next = virtio_tswap16(vdev, host->next);
 297     return true;
 298 }
 299
 300 /* This is stolen from linux/drivers/vhost/vhost.c. */
 301 static int get_indirect(VirtIODevice *vdev, Vring *vring,
 302                         VirtQueueElement *elem, struct vring_desc *indirect)
 303 {
 304     struct vring_desc desc;
 305     unsigned int i = 0, count, found = 0;
 306     int ret;
 307
 308     /* Sanity check */
 309     if (unlikely(indirect->len % sizeof(desc))) {
 310         error_report("Invalid length in indirect descriptor: "
 311                      "len %#x not multiple of %#zx",
 312                      indirect->len, sizeof(desc));
 313         vring->broken = true;
 314         return -EFAULT;
 315     }
 316
 317     count = indirect->len / sizeof(desc);
 318     /* Buffers are chained via a 16 bit next field, so
 319      * we can have at most 2^16 of these. */
 320     if (unlikely(count > USHRT_MAX + 1)) {
 321         error_report("Indirect buffer length too big: %d", indirect->len);
 322         vring->broken = true;
 323         return -EFAULT;
 324     }
 325
 326     do {
 327         /* Translate indirect descriptor */
 328         if (!read_vring_desc(vdev, indirect->addr + found * sizeof(desc),
 329                              &desc)) {
 330             error_report("Failed to read indirect descriptor "
 331                          "addr %#" PRIx64 " len %zu",
 332                          (uint64_t)indirect->addr + found * sizeof(desc),
 333                          sizeof(desc));
 334             vring->broken = true;
 335             return -EFAULT;
 336         }
 337
 338         /* Ensure descriptor has been loaded before accessing fields */
 339         barrier(); /* read_barrier_depends(); */
 340
 341         if (unlikely(++found > count)) {
 342             error_report("Loop detected: last one at %u "
 343                          "indirect size %u", i, count);
 344             vring->broken = true;
 345             return -EFAULT;
 346         }
 347
 348         if (unlikely(desc.flags & VRING_DESC_F_INDIRECT)) {
 349             error_report("Nested indirect descriptor");
 350             vring->broken = true;
 351             return -EFAULT;
 352         }
 353
 354         ret = get_desc(vring, elem, &desc);
 355         if (ret < 0) {
 356             vring->broken |= (ret == -EFAULT);
 357             return ret;
 358         }
 359         i = desc.next;
 360     } while (desc.flags & VRING_DESC_F_NEXT);
 361     return 0;
 362 }
 363
 364 static void vring_unmap_element(VirtQueueElement *elem)
 365 {
 366     int i;
 367
 368     /* This assumes that the iovecs, if changed, are never moved past
 369      * the end of the valid area.  This is true if iovec manipulations
 370      * are done with iov_discard_front and iov_discard_back.
 371      */
 372     for (i = 0; i < elem->out_num; i++) {
 373         vring_unmap(elem->out_sg[i].iov_base, false);
 374     }
 375
 376     for (i = 0; i < elem->in_num; i++) {
 377         vring_unmap(elem->in_sg[i].iov_base, true);
 378     }
 379 }
 380
 381 /* This looks in the virtqueue and for the first available buffer, and converts
 382  * it to an iovec for convenient access.  Since descriptors consist of some
 383  * number of output then some number of input descriptors, it's actually two
 384  * iovecs, but we pack them into one and note how many of each there were.
 385  *
 386  * This function returns the descriptor number found, or vq->num (which is
 387  * never a valid descriptor number) if none was found.  A negative code is
 388  * returned on error.
 389  *
 390  * Stolen from linux/drivers/vhost/vhost.c.
 391  */
 392 void *vring_pop(VirtIODevice *vdev, Vring *vring, size_t sz)
 393 {
 394     struct vring_desc desc;
 395     unsigned int i, head, found = 0, num = vring->vr.num;
 396     uint16_t avail_idx, last_avail_idx;
 397     VirtQueueElement *elem = NULL;
 398     int ret;
 399
 400     /* If there was a fatal error then refuse operation */
 401     if (vring->broken) {
 402         ret = -EFAULT;
 403         goto out;
 404     }
 405
 406     elem = virtqueue_alloc_element(sz, VIRTQUEUE_MAX_SIZE, VIRTQUEUE_MAX_SIZE);
 407
 408     /* Initialize elem so it can be safely unmapped */
 409     elem->in_num = elem->out_num = 0;
 410
 411     /* Check it isn't doing very strange things with descriptor numbers. */
 412     last_avail_idx = vring->last_avail_idx;
 413     avail_idx = vring_get_avail_idx(vdev, vring);
 414     barrier(); /* load indices now and not again later */
 415
 416     if (unlikely((uint16_t)(avail_idx - last_avail_idx) > num)) {
 417         error_report("Guest moved used index from %u to %u",
 418                      last_avail_idx, avail_idx);
 419         ret = -EFAULT;
 420         goto out;
 421     }
 422
 423     /* If there's nothing new since last we looked. */
 424     if (avail_idx == last_avail_idx) {
 425         ret = -EAGAIN;
 426         goto out;
 427     }
 428
 429     /* Only get avail ring entries after they have been exposed by guest. */
 430     smp_rmb();
 431
 432     /* Grab the next descriptor number they're advertising, and increment
 433      * the index we've seen. */
 434     head = vring_get_avail_ring(vdev, vring, last_avail_idx % num);
 435
 436     elem->index = head;
 437
 438     /* If their number is silly, that's an error. */
 439     if (unlikely(head >= num)) {
 440         error_report("Guest says index %u > %u is available", head, num);
 441         ret = -EFAULT;
 442         goto out;
 443     }
 444
 445     i = head;
 446     do {
 447         if (unlikely(i >= num)) {
 448             error_report("Desc index is %u > %u, head = %u", i, num, head);
 449             ret = -EFAULT;
 450             goto out;
 451         }
 452         if (unlikely(++found > num)) {
 453             error_report("Loop detected: last one at %u vq size %u head %u",
 454                          i, num, head);
 455             ret = -EFAULT;
 456             goto out;
 457         }
 458         copy_in_vring_desc(vdev, &vring->vr.desc[i], &desc);
 459
 460         /* Ensure descriptor is loaded before accessing fields */
 461         barrier();
 462
 463         if (desc.flags & VRING_DESC_F_INDIRECT) {
 464             ret = get_indirect(vdev, vring, elem, &desc);
 465             if (ret < 0) {
 466                 goto out;
 467             }
 468             continue;
 469         }
 470
 471         ret = get_desc(vring, elem, &desc);
 472         if (ret < 0) {
 473             goto out;
 474         }
 475
 476         i = desc.next;
 477     } while (desc.flags & VRING_DESC_F_NEXT);
 478
 479     /* On success, increment avail index. */
 480     vring->last_avail_idx++;
 481     if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
 482         vring_avail_event(&vring->vr) =
 483             virtio_tswap16(vdev, vring->last_avail_idx);
 484     }
 485
 486     return elem;
 487
 488 out:
 489     assert(ret < 0);
 490     if (ret == -EFAULT) {
 491         vring->broken = true;
 492     }
 493     vring_unmap_element(elem);
 494     g_free(elem);
 495     return NULL;
 496 }
 497
 498 /* After we've used one of their buffers, we tell them about it.
 499  *
 500  * Stolen from linux/drivers/vhost/vhost.c.
 501  */
 502 void vring_push(VirtIODevice *vdev, Vring *vring, VirtQueueElement *elem,
 503                 int len)
 504 {
 505     unsigned int head = elem->index;
 506     uint16_t new;
 507
 508     vring_unmap_element(elem);
 509
 510     /* Don't touch vring if a fatal error occurred */
 511     if (vring->broken) {
 512         return;
 513     }
 514
 515     /* The virtqueue contains a ring of used buffers.  Get a pointer to the
 516      * next entry in that used ring. */
 517     vring_set_used_ring_id(vdev, vring, vring->last_used_idx % vring->vr.num,
 518                            head);
 519     vring_set_used_ring_len(vdev, vring, vring->last_used_idx % vring->vr.num,
 520                             len);
 521
 522     /* Make sure buffer is written before we update index. */
 523     smp_wmb();
 524
 525     new = ++vring->last_used_idx;
 526     vring_set_used_idx(vdev, vring, new);
 527     if (unlikely((int16_t)(new - vring->signalled_used) < (uint16_t)1)) {
 528         vring->signalled_used_valid = false;
 529     }
 530 }