hw/vhost.c

   1 /*
   2  * vhost support
   3  *
   4  * Copyright Red Hat, Inc. 2010
   5  *
   6  * Authors:
   7  *  Michael S. Tsirkin <mst@redhat.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include <sys/ioctl.h>
  17 #include "vhost.h"
  18 #include "hw/hw.h"
  19 #include "range.h"
  20 #include <linux/vhost.h>
  21 #include "exec-memory.h"
  22
  23 static void vhost_dev_sync_region(struct vhost_dev *dev,
  24                                   MemoryRegionSection *section,
  25                                   uint64_t mfirst, uint64_t mlast,
  26                                   uint64_t rfirst, uint64_t rlast)
  27 {
  28     uint64_t start = MAX(mfirst, rfirst);
  29     uint64_t end = MIN(mlast, rlast);
  30     vhost_log_chunk_t *from = dev->log + start / VHOST_LOG_CHUNK;
  31     vhost_log_chunk_t *to = dev->log + end / VHOST_LOG_CHUNK + 1;
  32     uint64_t addr = (start / VHOST_LOG_CHUNK) * VHOST_LOG_CHUNK;
  33
  34     assert(end / VHOST_LOG_CHUNK < dev->log_size);
  35     assert(start / VHOST_LOG_CHUNK < dev->log_size);
  36     if (end < start) {
  37         return;
  38     }
  39     for (;from < to; ++from) {
  40         vhost_log_chunk_t log;
  41         int bit;
  42         /* We first check with non-atomic: much cheaper,
  43          * and we expect non-dirty to be the common case. */
  44         if (!*from) {
  45             addr += VHOST_LOG_CHUNK;
  46             continue;
  47         }
  48         /* Data must be read atomically. We don't really
  49          * need the barrier semantics of __sync
  50          * builtins, but it's easier to use them than
  51          * roll our own. */
  52         log = __sync_fetch_and_and(from, 0);
  53         while ((bit = sizeof(log) > sizeof(int) ?
  54                 ffsll(log) : ffs(log))) {
  55             ram_addr_t ram_addr;
  56             bit -= 1;
  57             ram_addr = section->offset_within_region + bit * VHOST_LOG_PAGE;
  58             memory_region_set_dirty(section->mr, ram_addr, VHOST_LOG_PAGE);
  59             log &= ~(0x1ull << bit);
  60         }
  61         addr += VHOST_LOG_CHUNK;
  62     }
  63 }
  64
  65 static int vhost_sync_dirty_bitmap(struct vhost_dev *dev,
  66                                    MemoryRegionSection *section,
  67                                    target_phys_addr_t start_addr,
  68                                    target_phys_addr_t end_addr)
  69 {
  70     int i;
  71
  72     if (!dev->log_enabled || !dev->started) {
  73         return 0;
  74     }
  75     for (i = 0; i < dev->mem->nregions; ++i) {
  76         struct vhost_memory_region *reg = dev->mem->regions + i;
  77         vhost_dev_sync_region(dev, section, start_addr, end_addr,
  78                               reg->guest_phys_addr,
  79                               range_get_last(reg->guest_phys_addr,
  80                                              reg->memory_size));
  81     }
  82     for (i = 0; i < dev->nvqs; ++i) {
  83         struct vhost_virtqueue *vq = dev->vqs + i;
  84         vhost_dev_sync_region(dev, section, start_addr, end_addr, vq->used_phys,
  85                               range_get_last(vq->used_phys, vq->used_size));
  86     }
  87     return 0;
  88 }
  89
  90 static void vhost_log_sync(MemoryListener *listener,
  91                           MemoryRegionSection *section)
  92 {
  93     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
  94                                          memory_listener);
  95     target_phys_addr_t start_addr = section->offset_within_address_space;
  96     target_phys_addr_t end_addr = start_addr + section->size;
  97
  98     vhost_sync_dirty_bitmap(dev, section, start_addr, end_addr);
  99 }
 100
 101 /* Assign/unassign. Keep an unsorted array of non-overlapping
 102  * memory regions in dev->mem. */
 103 static void vhost_dev_unassign_memory(struct vhost_dev *dev,
 104                                       uint64_t start_addr,
 105                                       uint64_t size)
 106 {
 107     int from, to, n = dev->mem->nregions;
 108     /* Track overlapping/split regions for sanity checking. */
 109     int overlap_start = 0, overlap_end = 0, overlap_middle = 0, split = 0;
 110
 111     for (from = 0, to = 0; from < n; ++from, ++to) {
 112         struct vhost_memory_region *reg = dev->mem->regions + to;
 113         uint64_t reglast;
 114         uint64_t memlast;
 115         uint64_t change;
 116
 117         /* clone old region */
 118         if (to != from) {
 119             memcpy(reg, dev->mem->regions + from, sizeof *reg);
 120         }
 121
 122         /* No overlap is simple */
 123         if (!ranges_overlap(reg->guest_phys_addr, reg->memory_size,
 124                             start_addr, size)) {
 125             continue;
 126         }
 127
 128         /* Split only happens if supplied region
 129          * is in the middle of an existing one. Thus it can not
 130          * overlap with any other existing region. */
 131         assert(!split);
 132
 133         reglast = range_get_last(reg->guest_phys_addr, reg->memory_size);
 134         memlast = range_get_last(start_addr, size);
 135
 136         /* Remove whole region */
 137         if (start_addr <= reg->guest_phys_addr && memlast >= reglast) {
 138             --dev->mem->nregions;
 139             --to;
 140             ++overlap_middle;
 141             continue;
 142         }
 143
 144         /* Shrink region */
 145         if (memlast >= reglast) {
 146             reg->memory_size = start_addr - reg->guest_phys_addr;
 147             assert(reg->memory_size);
 148             assert(!overlap_end);
 149             ++overlap_end;
 150             continue;
 151         }
 152
 153         /* Shift region */
 154         if (start_addr <= reg->guest_phys_addr) {
 155             change = memlast + 1 - reg->guest_phys_addr;
 156             reg->memory_size -= change;
 157             reg->guest_phys_addr += change;
 158             reg->userspace_addr += change;
 159             assert(reg->memory_size);
 160             assert(!overlap_start);
 161             ++overlap_start;
 162             continue;
 163         }
 164
 165         /* This only happens if supplied region
 166          * is in the middle of an existing one. Thus it can not
 167          * overlap with any other existing region. */
 168         assert(!overlap_start);
 169         assert(!overlap_end);
 170         assert(!overlap_middle);
 171         /* Split region: shrink first part, shift second part. */
 172         memcpy(dev->mem->regions + n, reg, sizeof *reg);
 173         reg->memory_size = start_addr - reg->guest_phys_addr;
 174         assert(reg->memory_size);
 175         change = memlast + 1 - reg->guest_phys_addr;
 176         reg = dev->mem->regions + n;
 177         reg->memory_size -= change;
 178         assert(reg->memory_size);
 179         reg->guest_phys_addr += change;
 180         reg->userspace_addr += change;
 181         /* Never add more than 1 region */
 182         assert(dev->mem->nregions == n);
 183         ++dev->mem->nregions;
 184         ++split;
 185     }
 186 }
 187
 188 /* Called after unassign, so no regions overlap the given range. */
 189 static void vhost_dev_assign_memory(struct vhost_dev *dev,
 190                                     uint64_t start_addr,
 191                                     uint64_t size,
 192                                     uint64_t uaddr)
 193 {
 194     int from, to;
 195     struct vhost_memory_region *merged = NULL;
 196     for (from = 0, to = 0; from < dev->mem->nregions; ++from, ++to) {
 197         struct vhost_memory_region *reg = dev->mem->regions + to;
 198         uint64_t prlast, urlast;
 199         uint64_t pmlast, umlast;
 200         uint64_t s, e, u;
 201
 202         /* clone old region */
 203         if (to != from) {
 204             memcpy(reg, dev->mem->regions + from, sizeof *reg);
 205         }
 206         prlast = range_get_last(reg->guest_phys_addr, reg->memory_size);
 207         pmlast = range_get_last(start_addr, size);
 208         urlast = range_get_last(reg->userspace_addr, reg->memory_size);
 209         umlast = range_get_last(uaddr, size);
 210
 211         /* check for overlapping regions: should never happen. */
 212         assert(prlast < start_addr || pmlast < reg->guest_phys_addr);
 213         /* Not an adjacent or overlapping region - do not merge. */
 214         if ((prlast + 1 != start_addr || urlast + 1 != uaddr) &&
 215             (pmlast + 1 != reg->guest_phys_addr ||
 216              umlast + 1 != reg->userspace_addr)) {
 217             continue;
 218         }
 219
 220         if (merged) {
 221             --to;
 222             assert(to >= 0);
 223         } else {
 224             merged = reg;
 225         }
 226         u = MIN(uaddr, reg->userspace_addr);
 227         s = MIN(start_addr, reg->guest_phys_addr);
 228         e = MAX(pmlast, prlast);
 229         uaddr = merged->userspace_addr = u;
 230         start_addr = merged->guest_phys_addr = s;
 231         size = merged->memory_size = e - s + 1;
 232         assert(merged->memory_size);
 233     }
 234
 235     if (!merged) {
 236         struct vhost_memory_region *reg = dev->mem->regions + to;
 237         memset(reg, 0, sizeof *reg);
 238         reg->memory_size = size;
 239         assert(reg->memory_size);
 240         reg->guest_phys_addr = start_addr;
 241         reg->userspace_addr = uaddr;
 242         ++to;
 243     }
 244     assert(to <= dev->mem->nregions + 1);
 245     dev->mem->nregions = to;
 246 }
 247
 248 static uint64_t vhost_get_log_size(struct vhost_dev *dev)
 249 {
 250     uint64_t log_size = 0;
 251     int i;
 252     for (i = 0; i < dev->mem->nregions; ++i) {
 253         struct vhost_memory_region *reg = dev->mem->regions + i;
 254         uint64_t last = range_get_last(reg->guest_phys_addr,
 255                                        reg->memory_size);
 256         log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1);
 257     }
 258     for (i = 0; i < dev->nvqs; ++i) {
 259         struct vhost_virtqueue *vq = dev->vqs + i;
 260         uint64_t last = vq->used_phys + vq->used_size - 1;
 261         log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1);
 262     }
 263     return log_size;
 264 }
 265
 266 static inline void vhost_dev_log_resize(struct vhost_dev* dev, uint64_t size)
 267 {
 268     vhost_log_chunk_t *log;
 269     uint64_t log_base;
 270     int r, i;
 271     if (size) {
 272         log = g_malloc0(size * sizeof *log);
 273     } else {
 274         log = NULL;
 275     }
 276     log_base = (uint64_t)(unsigned long)log;
 277     r = ioctl(dev->control, VHOST_SET_LOG_BASE, &log_base);
 278     assert(r >= 0);
 279     for (i = 0; i < dev->n_mem_sections; ++i) {
 280         vhost_sync_dirty_bitmap(dev, &dev->mem_sections[i],
 281                                 0, (target_phys_addr_t)~0x0ull);
 282     }
 283     if (dev->log) {
 284         g_free(dev->log);
 285     }
 286     dev->log = log;
 287     dev->log_size = size;
 288 }
 289
 290 static int vhost_verify_ring_mappings(struct vhost_dev *dev,
 291                                       uint64_t start_addr,
 292                                       uint64_t size)
 293 {
 294     int i;
 295     for (i = 0; i < dev->nvqs; ++i) {
 296         struct vhost_virtqueue *vq = dev->vqs + i;
 297         target_phys_addr_t l;
 298         void *p;
 299
 300         if (!ranges_overlap(start_addr, size, vq->ring_phys, vq->ring_size)) {
 301             continue;
 302         }
 303         l = vq->ring_size;
 304         p = cpu_physical_memory_map(vq->ring_phys, &l, 1);
 305         if (!p || l != vq->ring_size) {
 306             fprintf(stderr, "Unable to map ring buffer for ring %d\n", i);
 307             return -ENOMEM;
 308         }
 309         if (p != vq->ring) {
 310             fprintf(stderr, "Ring buffer relocated for ring %d\n", i);
 311             return -EBUSY;
 312         }
 313         cpu_physical_memory_unmap(p, l, 0, 0);
 314     }
 315     return 0;
 316 }
 317
 318 static struct vhost_memory_region *vhost_dev_find_reg(struct vhost_dev *dev,
 319                                                       uint64_t start_addr,
 320                                                       uint64_t size)
 321 {
 322     int i, n = dev->mem->nregions;
 323     for (i = 0; i < n; ++i) {
 324         struct vhost_memory_region *reg = dev->mem->regions + i;
 325         if (ranges_overlap(reg->guest_phys_addr, reg->memory_size,
 326                            start_addr, size)) {
 327             return reg;
 328         }
 329     }
 330     return NULL;
 331 }
 332
 333 static bool vhost_dev_cmp_memory(struct vhost_dev *dev,
 334                                  uint64_t start_addr,
 335                                  uint64_t size,
 336                                  uint64_t uaddr)
 337 {
 338     struct vhost_memory_region *reg = vhost_dev_find_reg(dev, start_addr, size);
 339     uint64_t reglast;
 340     uint64_t memlast;
 341
 342     if (!reg) {
 343         return true;
 344     }
 345
 346     reglast = range_get_last(reg->guest_phys_addr, reg->memory_size);
 347     memlast = range_get_last(start_addr, size);
 348
 349     /* Need to extend region? */
 350     if (start_addr < reg->guest_phys_addr || memlast > reglast) {
 351         return true;
 352     }
 353     /* userspace_addr changed? */
 354     return uaddr != reg->userspace_addr + start_addr - reg->guest_phys_addr;
 355 }
 356
 357 static void vhost_set_memory(MemoryListener *listener,
 358                              MemoryRegionSection *section,
 359                              bool add)
 360 {
 361     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 362                                          memory_listener);
 363     target_phys_addr_t start_addr = section->offset_within_address_space;
 364     ram_addr_t size = section->size;
 365     bool log_dirty = memory_region_is_logging(section->mr);
 366     int s = offsetof(struct vhost_memory, regions) +
 367         (dev->mem->nregions + 1) * sizeof dev->mem->regions[0];
 368     uint64_t log_size;
 369     int r;
 370     void *ram;
 371
 372     dev->mem = g_realloc(dev->mem, s);
 373
 374     if (log_dirty) {
 375         add = false;
 376     }
 377
 378     assert(size);
 379
 380     /* Optimize no-change case. At least cirrus_vga does this a lot at this time. */
 381     ram = memory_region_get_ram_ptr(section->mr) + section->offset_within_region;
 382     if (add) {
 383         if (!vhost_dev_cmp_memory(dev, start_addr, size, (uintptr_t)ram)) {
 384             /* Region exists with same address. Nothing to do. */
 385             return;
 386         }
 387     } else {
 388         if (!vhost_dev_find_reg(dev, start_addr, size)) {
 389             /* Removing region that we don't access. Nothing to do. */
 390             return;
 391         }
 392     }
 393
 394     vhost_dev_unassign_memory(dev, start_addr, size);
 395     if (add) {
 396         /* Add given mapping, merging adjacent regions if any */
 397         vhost_dev_assign_memory(dev, start_addr, size, (uintptr_t)ram);
 398     } else {
 399         /* Remove old mapping for this memory, if any. */
 400         vhost_dev_unassign_memory(dev, start_addr, size);
 401     }
 402
 403     if (!dev->started) {
 404         return;
 405     }
 406
 407     if (dev->started) {
 408         r = vhost_verify_ring_mappings(dev, start_addr, size);
 409         assert(r >= 0);
 410     }
 411
 412     if (!dev->log_enabled) {
 413         r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem);
 414         assert(r >= 0);
 415         return;
 416     }
 417     log_size = vhost_get_log_size(dev);
 418     /* We allocate an extra 4K bytes to log,
 419      * to reduce the * number of reallocations. */
 420 #define VHOST_LOG_BUFFER (0x1000 / sizeof *dev->log)
 421     /* To log more, must increase log size before table update. */
 422     if (dev->log_size < log_size) {
 423         vhost_dev_log_resize(dev, log_size + VHOST_LOG_BUFFER);
 424     }
 425     r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem);
 426     assert(r >= 0);
 427     /* To log less, can only decrease log size after table update. */
 428     if (dev->log_size > log_size + VHOST_LOG_BUFFER) {
 429         vhost_dev_log_resize(dev, log_size);
 430     }
 431 }
 432
 433 static bool vhost_section(MemoryRegionSection *section)
 434 {
 435     return section->address_space == get_system_memory()
 436         && memory_region_is_ram(section->mr);
 437 }
 438
 439 static void vhost_region_add(MemoryListener *listener,
 440                              MemoryRegionSection *section)
 441 {
 442     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 443                                          memory_listener);
 444
 445     if (!vhost_section(section)) {
 446         return;
 447     }
 448
 449     ++dev->n_mem_sections;
 450     dev->mem_sections = g_renew(MemoryRegionSection, dev->mem_sections,
 451                                 dev->n_mem_sections);
 452     dev->mem_sections[dev->n_mem_sections - 1] = *section;
 453     vhost_set_memory(listener, section, true);
 454 }
 455
 456 static void vhost_region_del(MemoryListener *listener,
 457                              MemoryRegionSection *section)
 458 {
 459     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 460                                          memory_listener);
 461     int i;
 462
 463     if (!vhost_section(section)) {
 464         return;
 465     }
 466
 467     vhost_set_memory(listener, section, false);
 468     for (i = 0; i < dev->n_mem_sections; ++i) {
 469         if (dev->mem_sections[i].offset_within_address_space
 470             == section->offset_within_address_space) {
 471             --dev->n_mem_sections;
 472             memmove(&dev->mem_sections[i], &dev->mem_sections[i+1],
 473                     (dev->n_mem_sections - i) * sizeof(*dev->mem_sections));
 474             break;
 475         }
 476     }
 477 }
 478
 479 static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
 480                                     struct vhost_virtqueue *vq,
 481                                     unsigned idx, bool enable_log)
 482 {
 483     struct vhost_vring_addr addr = {
 484         .index = idx,
 485         .desc_user_addr = (uint64_t)(unsigned long)vq->desc,
 486         .avail_user_addr = (uint64_t)(unsigned long)vq->avail,
 487         .used_user_addr = (uint64_t)(unsigned long)vq->used,
 488         .log_guest_addr = vq->used_phys,
 489         .flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0,
 490     };
 491     int r = ioctl(dev->control, VHOST_SET_VRING_ADDR, &addr);
 492     if (r < 0) {
 493         return -errno;
 494     }
 495     return 0;
 496 }
 497
 498 static int vhost_dev_set_features(struct vhost_dev *dev, bool enable_log)
 499 {
 500     uint64_t features = dev->acked_features;
 501     int r;
 502     if (enable_log) {
 503         features |= 0x1 << VHOST_F_LOG_ALL;
 504     }
 505     r = ioctl(dev->control, VHOST_SET_FEATURES, &features);
 506     return r < 0 ? -errno : 0;
 507 }
 508
 509 static int vhost_dev_set_log(struct vhost_dev *dev, bool enable_log)
 510 {
 511     int r, t, i;
 512     r = vhost_dev_set_features(dev, enable_log);
 513     if (r < 0) {
 514         goto err_features;
 515     }
 516     for (i = 0; i < dev->nvqs; ++i) {
 517         r = vhost_virtqueue_set_addr(dev, dev->vqs + i, i,
 518                                      enable_log);
 519         if (r < 0) {
 520             goto err_vq;
 521         }
 522     }
 523     return 0;
 524 err_vq:
 525     for (; i >= 0; --i) {
 526         t = vhost_virtqueue_set_addr(dev, dev->vqs + i, i,
 527                                      dev->log_enabled);
 528         assert(t >= 0);
 529     }
 530     t = vhost_dev_set_features(dev, dev->log_enabled);
 531     assert(t >= 0);
 532 err_features:
 533     return r;
 534 }
 535
 536 static int vhost_migration_log(MemoryListener *listener, int enable)
 537 {
 538     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 539                                          memory_listener);
 540     int r;
 541     if (!!enable == dev->log_enabled) {
 542         return 0;
 543     }
 544     if (!dev->started) {
 545         dev->log_enabled = enable;
 546         return 0;
 547     }
 548     if (!enable) {
 549         r = vhost_dev_set_log(dev, false);
 550         if (r < 0) {
 551             return r;
 552         }
 553         if (dev->log) {
 554             g_free(dev->log);
 555         }
 556         dev->log = NULL;
 557         dev->log_size = 0;
 558     } else {
 559         vhost_dev_log_resize(dev, vhost_get_log_size(dev));
 560         r = vhost_dev_set_log(dev, true);
 561         if (r < 0) {
 562             return r;
 563         }
 564     }
 565     dev->log_enabled = enable;
 566     return 0;
 567 }
 568
 569 static void vhost_log_global_start(MemoryListener *listener)
 570 {
 571     int r;
 572
 573     r = vhost_migration_log(listener, true);
 574     if (r < 0) {
 575         abort();
 576     }
 577 }
 578
 579 static void vhost_log_global_stop(MemoryListener *listener)
 580 {
 581     int r;
 582
 583     r = vhost_migration_log(listener, false);
 584     if (r < 0) {
 585         abort();
 586     }
 587 }
 588
 589 static void vhost_log_start(MemoryListener *listener,
 590                             MemoryRegionSection *section)
 591 {
 592     /* FIXME: implement */
 593 }
 594
 595 static void vhost_log_stop(MemoryListener *listener,
 596                            MemoryRegionSection *section)
 597 {
 598     /* FIXME: implement */
 599 }
 600
 601 static int vhost_virtqueue_init(struct vhost_dev *dev,
 602                                 struct VirtIODevice *vdev,
 603                                 struct vhost_virtqueue *vq,
 604                                 unsigned idx)
 605 {
 606     target_phys_addr_t s, l, a;
 607     int r;
 608     struct vhost_vring_file file = {
 609         .index = idx,
 610     };
 611     struct vhost_vring_state state = {
 612         .index = idx,
 613     };
 614     struct VirtQueue *vvq = virtio_get_queue(vdev, idx);
 615
 616     vq->num = state.num = virtio_queue_get_num(vdev, idx);
 617     r = ioctl(dev->control, VHOST_SET_VRING_NUM, &state);
 618     if (r) {
 619         return -errno;
 620     }
 621
 622     state.num = virtio_queue_get_last_avail_idx(vdev, idx);
 623     r = ioctl(dev->control, VHOST_SET_VRING_BASE, &state);
 624     if (r) {
 625         return -errno;
 626     }
 627
 628     s = l = virtio_queue_get_desc_size(vdev, idx);
 629     a = virtio_queue_get_desc_addr(vdev, idx);
 630     vq->desc = cpu_physical_memory_map(a, &l, 0);
 631     if (!vq->desc || l != s) {
 632         r = -ENOMEM;
 633         goto fail_alloc_desc;
 634     }
 635     s = l = virtio_queue_get_avail_size(vdev, idx);
 636     a = virtio_queue_get_avail_addr(vdev, idx);
 637     vq->avail = cpu_physical_memory_map(a, &l, 0);
 638     if (!vq->avail || l != s) {
 639         r = -ENOMEM;
 640         goto fail_alloc_avail;
 641     }
 642     vq->used_size = s = l = virtio_queue_get_used_size(vdev, idx);
 643     vq->used_phys = a = virtio_queue_get_used_addr(vdev, idx);
 644     vq->used = cpu_physical_memory_map(a, &l, 1);
 645     if (!vq->used || l != s) {
 646         r = -ENOMEM;
 647         goto fail_alloc_used;
 648     }
 649
 650     vq->ring_size = s = l = virtio_queue_get_ring_size(vdev, idx);
 651     vq->ring_phys = a = virtio_queue_get_ring_addr(vdev, idx);
 652     vq->ring = cpu_physical_memory_map(a, &l, 1);
 653     if (!vq->ring || l != s) {
 654         r = -ENOMEM;
 655         goto fail_alloc_ring;
 656     }
 657
 658     r = vhost_virtqueue_set_addr(dev, vq, idx, dev->log_enabled);
 659     if (r < 0) {
 660         r = -errno;
 661         goto fail_alloc;
 662     }
 663     file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq));
 664     r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file);
 665     if (r) {
 666         r = -errno;
 667         goto fail_kick;
 668     }
 669
 670     file.fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq));
 671     r = ioctl(dev->control, VHOST_SET_VRING_CALL, &file);
 672     if (r) {
 673         r = -errno;
 674         goto fail_call;
 675     }
 676
 677     return 0;
 678
 679 fail_call:
 680 fail_kick:
 681 fail_alloc:
 682     cpu_physical_memory_unmap(vq->ring, virtio_queue_get_ring_size(vdev, idx),
 683                               0, 0);
 684 fail_alloc_ring:
 685     cpu_physical_memory_unmap(vq->used, virtio_queue_get_used_size(vdev, idx),
 686                               0, 0);
 687 fail_alloc_used:
 688     cpu_physical_memory_unmap(vq->avail, virtio_queue_get_avail_size(vdev, idx),
 689                               0, 0);
 690 fail_alloc_avail:
 691     cpu_physical_memory_unmap(vq->desc, virtio_queue_get_desc_size(vdev, idx),
 692                               0, 0);
 693 fail_alloc_desc:
 694     return r;
 695 }
 696
 697 static void vhost_virtqueue_cleanup(struct vhost_dev *dev,
 698                                     struct VirtIODevice *vdev,
 699                                     struct vhost_virtqueue *vq,
 700                                     unsigned idx)
 701 {
 702     struct vhost_vring_state state = {
 703         .index = idx,
 704     };
 705     int r;
 706     r = ioctl(dev->control, VHOST_GET_VRING_BASE, &state);
 707     if (r < 0) {
 708         fprintf(stderr, "vhost VQ %d ring restore failed: %d\n", idx, r);
 709         fflush(stderr);
 710     }
 711     virtio_queue_set_last_avail_idx(vdev, idx, state.num);
 712     assert (r >= 0);
 713     cpu_physical_memory_unmap(vq->ring, virtio_queue_get_ring_size(vdev, idx),
 714                               0, virtio_queue_get_ring_size(vdev, idx));
 715     cpu_physical_memory_unmap(vq->used, virtio_queue_get_used_size(vdev, idx),
 716                               1, virtio_queue_get_used_size(vdev, idx));
 717     cpu_physical_memory_unmap(vq->avail, virtio_queue_get_avail_size(vdev, idx),
 718                               0, virtio_queue_get_avail_size(vdev, idx));
 719     cpu_physical_memory_unmap(vq->desc, virtio_queue_get_desc_size(vdev, idx),
 720                               0, virtio_queue_get_desc_size(vdev, idx));
 721 }
 722
 723 int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force)
 724 {
 725     uint64_t features;
 726     int r;
 727     if (devfd >= 0) {
 728         hdev->control = devfd;
 729     } else {
 730         hdev->control = open("/dev/vhost-net", O_RDWR);
 731         if (hdev->control < 0) {
 732             return -errno;
 733         }
 734     }
 735     r = ioctl(hdev->control, VHOST_SET_OWNER, NULL);
 736     if (r < 0) {
 737         goto fail;
 738     }
 739
 740     r = ioctl(hdev->control, VHOST_GET_FEATURES, &features);
 741     if (r < 0) {
 742         goto fail;
 743     }
 744     hdev->features = features;
 745
 746     hdev->memory_listener = (MemoryListener) {
 747         .region_add = vhost_region_add,
 748         .region_del = vhost_region_del,
 749         .log_start = vhost_log_start,
 750         .log_stop = vhost_log_stop,
 751         .log_sync = vhost_log_sync,
 752         .log_global_start = vhost_log_global_start,
 753         .log_global_stop = vhost_log_global_stop,
 754     };
 755     hdev->mem = g_malloc0(offsetof(struct vhost_memory, regions));
 756     hdev->n_mem_sections = 0;
 757     hdev->mem_sections = NULL;
 758     hdev->log = NULL;
 759     hdev->log_size = 0;
 760     hdev->log_enabled = false;
 761     hdev->started = false;
 762     memory_listener_register(&hdev->memory_listener);
 763     hdev->force = force;
 764     return 0;
 765 fail:
 766     r = -errno;
 767     close(hdev->control);
 768     return r;
 769 }
 770
 771 void vhost_dev_cleanup(struct vhost_dev *hdev)
 772 {
 773     memory_listener_unregister(&hdev->memory_listener);
 774     g_free(hdev->mem);
 775     g_free(hdev->mem_sections);
 776     close(hdev->control);
 777 }
 778
 779 bool vhost_dev_query(struct vhost_dev *hdev, VirtIODevice *vdev)
 780 {
 781     return !vdev->binding->query_guest_notifiers ||
 782         vdev->binding->query_guest_notifiers(vdev->binding_opaque) ||
 783         hdev->force;
 784 }
 785
 786 /* Stop processing guest IO notifications in qemu.
 787  * Start processing them in vhost in kernel.
 788  */
 789 int vhost_dev_enable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
 790 {
 791     int i, r;
 792     if (!vdev->binding->set_host_notifier) {
 793         fprintf(stderr, "binding does not support host notifiers\n");
 794         r = -ENOSYS;
 795         goto fail;
 796     }
 797
 798     for (i = 0; i < hdev->nvqs; ++i) {
 799         r = vdev->binding->set_host_notifier(vdev->binding_opaque, i, true);
 800         if (r < 0) {
 801             fprintf(stderr, "vhost VQ %d notifier binding failed: %d\n", i, -r);
 802             goto fail_vq;
 803         }
 804     }
 805
 806     return 0;
 807 fail_vq:
 808     while (--i >= 0) {
 809         r = vdev->binding->set_host_notifier(vdev->binding_opaque, i, false);
 810         if (r < 0) {
 811             fprintf(stderr, "vhost VQ %d notifier cleanup error: %d\n", i, -r);
 812             fflush(stderr);
 813         }
 814         assert (r >= 0);
 815     }
 816 fail:
 817     return r;
 818 }
 819
 820 /* Stop processing guest IO notifications in vhost.
 821  * Start processing them in qemu.
 822  * This might actually run the qemu handlers right away,
 823  * so virtio in qemu must be completely setup when this is called.
 824  */
 825 void vhost_dev_disable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
 826 {
 827     int i, r;
 828
 829     for (i = 0; i < hdev->nvqs; ++i) {
 830         r = vdev->binding->set_host_notifier(vdev->binding_opaque, i, false);
 831         if (r < 0) {
 832             fprintf(stderr, "vhost VQ %d notifier cleanup failed: %d\n", i, -r);
 833             fflush(stderr);
 834         }
 835         assert (r >= 0);
 836     }
 837 }
 838
 839 /* Host notifiers must be enabled at this point. */
 840 int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
 841 {
 842     int i, r;
 843     if (!vdev->binding->set_guest_notifiers) {
 844         fprintf(stderr, "binding does not support guest notifiers\n");
 845         r = -ENOSYS;
 846         goto fail;
 847     }
 848
 849     r = vdev->binding->set_guest_notifiers(vdev->binding_opaque, true);
 850     if (r < 0) {
 851         fprintf(stderr, "Error binding guest notifier: %d\n", -r);
 852         goto fail_notifiers;
 853     }
 854
 855     r = vhost_dev_set_features(hdev, hdev->log_enabled);
 856     if (r < 0) {
 857         goto fail_features;
 858     }
 859     r = ioctl(hdev->control, VHOST_SET_MEM_TABLE, hdev->mem);
 860     if (r < 0) {
 861         r = -errno;
 862         goto fail_mem;
 863     }
 864     for (i = 0; i < hdev->nvqs; ++i) {
 865         r = vhost_virtqueue_init(hdev,
 866                                  vdev,
 867                                  hdev->vqs + i,
 868                                  i);
 869         if (r < 0) {
 870             goto fail_vq;
 871         }
 872     }
 873
 874     if (hdev->log_enabled) {
 875         hdev->log_size = vhost_get_log_size(hdev);
 876         hdev->log = hdev->log_size ?
 877             g_malloc0(hdev->log_size * sizeof *hdev->log) : NULL;
 878         r = ioctl(hdev->control, VHOST_SET_LOG_BASE,
 879                   (uint64_t)(unsigned long)hdev->log);
 880         if (r < 0) {
 881             r = -errno;
 882             goto fail_log;
 883         }
 884     }
 885
 886     hdev->started = true;
 887
 888     return 0;
 889 fail_log:
 890 fail_vq:
 891     while (--i >= 0) {
 892         vhost_virtqueue_cleanup(hdev,
 893                                 vdev,
 894                                 hdev->vqs + i,
 895                                 i);
 896     }
 897 fail_mem:
 898 fail_features:
 899     vdev->binding->set_guest_notifiers(vdev->binding_opaque, false);
 900 fail_notifiers:
 901 fail:
 902     return r;
 903 }
 904
 905 /* Host notifiers must be enabled at this point. */
 906 void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
 907 {
 908     int i, r;
 909
 910     for (i = 0; i < hdev->nvqs; ++i) {
 911         vhost_virtqueue_cleanup(hdev,
 912                                 vdev,
 913                                 hdev->vqs + i,
 914                                 i);
 915     }
 916     for (i = 0; i < hdev->n_mem_sections; ++i) {
 917         vhost_sync_dirty_bitmap(hdev, &hdev->mem_sections[i],
 918                                 0, (target_phys_addr_t)~0x0ull);
 919     }
 920     r = vdev->binding->set_guest_notifiers(vdev->binding_opaque, false);
 921     if (r < 0) {
 922         fprintf(stderr, "vhost guest notifier cleanup failed: %d\n", r);
 923         fflush(stderr);
 924     }
 925     assert (r >= 0);
 926
 927     hdev->started = false;
 928     g_free(hdev->log);
 929     hdev->log = NULL;
 930     hdev->log_size = 0;
 931 }