xen-mapcache.c

   1 /*
   2  * Copyright (C) 2011       Citrix Ltd.
   3  *
   4  * This work is licensed under the terms of the GNU GPL, version 2.  See
   5  * the COPYING file in the top-level directory.
   6  *
   7  */
   8
   9 #include "config.h"
  10
  11 #include <sys/resource.h>
  12
  13 #include "hw/xen_backend.h"
  14 #include "blockdev.h"
  15 #include "bitmap.h"
  16
  17 #include <xen/hvm/params.h>
  18 #include <sys/mman.h>
  19
  20 #include "xen-mapcache.h"
  21 #include "trace.h"
  22
  23
  24 //#define MAPCACHE_DEBUG
  25
  26 #ifdef MAPCACHE_DEBUG
  27 #  define DPRINTF(fmt, ...) do { \
  28     fprintf(stderr, "xen_mapcache: " fmt, ## __VA_ARGS__); \
  29 } while (0)
  30 #else
  31 #  define DPRINTF(fmt, ...) do { } while (0)
  32 #endif
  33
  34 #if defined(__i386__)
  35 #  define MCACHE_BUCKET_SHIFT 16
  36 #  define MCACHE_MAX_SIZE     (1UL<<31) /* 2GB Cap */
  37 #elif defined(__x86_64__)
  38 #  define MCACHE_BUCKET_SHIFT 20
  39 #  define MCACHE_MAX_SIZE     (1UL<<35) /* 32GB Cap */
  40 #endif
  41 #define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)
  42
  43 /* This is the size of the virtual address space reserve to QEMU that will not
  44  * be use by MapCache.
  45  * From empirical tests I observed that qemu use 75MB more than the
  46  * max_mcache_size.
  47  */
  48 #define NON_MCACHE_MEMORY_SIZE (80 * 1024 * 1024)
  49
  50 #define mapcache_lock()   ((void)0)
  51 #define mapcache_unlock() ((void)0)
  52
  53 typedef struct MapCacheEntry {
  54     target_phys_addr_t paddr_index;
  55     uint8_t *vaddr_base;
  56     unsigned long *valid_mapping;
  57     uint8_t lock;
  58     target_phys_addr_t size;
  59     struct MapCacheEntry *next;
  60 } MapCacheEntry;
  61
  62 typedef struct MapCacheRev {
  63     uint8_t *vaddr_req;
  64     target_phys_addr_t paddr_index;
  65     target_phys_addr_t size;
  66     QTAILQ_ENTRY(MapCacheRev) next;
  67 } MapCacheRev;
  68
  69 typedef struct MapCache {
  70     MapCacheEntry *entry;
  71     unsigned long nr_buckets;
  72     QTAILQ_HEAD(map_cache_head, MapCacheRev) locked_entries;
  73
  74     /* For most cases (>99.9%), the page address is the same. */
  75     target_phys_addr_t last_address_index;
  76     uint8_t *last_address_vaddr;
  77     unsigned long max_mcache_size;
  78     unsigned int mcache_bucket_shift;
  79 } MapCache;
  80
  81 static MapCache *mapcache;
  82
  83 static inline int test_bits(int nr, int size, const unsigned long *addr)
  84 {
  85     unsigned long res = find_next_zero_bit(addr, size + nr, nr);
  86     if (res >= nr + size)
  87         return 1;
  88     else
  89         return 0;
  90 }
  91
  92 void xen_map_cache_init(void)
  93 {
  94     unsigned long size;
  95     struct rlimit rlimit_as;
  96
  97     mapcache = g_malloc0(sizeof (MapCache));
  98
  99     QTAILQ_INIT(&mapcache->locked_entries);
 100     mapcache->last_address_index = -1;
 101
 102     if (geteuid() == 0) {
 103         rlimit_as.rlim_cur = RLIM_INFINITY;
 104         rlimit_as.rlim_max = RLIM_INFINITY;
 105         mapcache->max_mcache_size = MCACHE_MAX_SIZE;
 106     } else {
 107         getrlimit(RLIMIT_AS, &rlimit_as);
 108         rlimit_as.rlim_cur = rlimit_as.rlim_max;
 109
 110         if (rlimit_as.rlim_max != RLIM_INFINITY) {
 111             fprintf(stderr, "Warning: QEMU's maximum size of virtual"
 112                     " memory is not infinity.\n");
 113         }
 114         if (rlimit_as.rlim_max < MCACHE_MAX_SIZE + NON_MCACHE_MEMORY_SIZE) {
 115             mapcache->max_mcache_size = rlimit_as.rlim_max -
 116                 NON_MCACHE_MEMORY_SIZE;
 117         } else {
 118             mapcache->max_mcache_size = MCACHE_MAX_SIZE;
 119         }
 120     }
 121
 122     setrlimit(RLIMIT_AS, &rlimit_as);
 123
 124     mapcache->nr_buckets =
 125         (((mapcache->max_mcache_size >> XC_PAGE_SHIFT) +
 126           (1UL << (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT)) - 1) >>
 127          (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT));
 128
 129     size = mapcache->nr_buckets * sizeof (MapCacheEntry);
 130     size = (size + XC_PAGE_SIZE - 1) & ~(XC_PAGE_SIZE - 1);
 131     DPRINTF("%s, nr_buckets = %lx size %lu\n", __func__,
 132             mapcache->nr_buckets, size);
 133     mapcache->entry = g_malloc0(size);
 134 }
 135
 136 static void xen_remap_bucket(MapCacheEntry *entry,
 137                              target_phys_addr_t size,
 138                              target_phys_addr_t address_index)
 139 {
 140     uint8_t *vaddr_base;
 141     xen_pfn_t *pfns;
 142     int *err;
 143     unsigned int i;
 144     target_phys_addr_t nb_pfn = size >> XC_PAGE_SHIFT;
 145
 146     trace_xen_remap_bucket(address_index);
 147
 148     pfns = g_malloc0(nb_pfn * sizeof (xen_pfn_t));
 149     err = g_malloc0(nb_pfn * sizeof (int));
 150
 151     if (entry->vaddr_base != NULL) {
 152         if (munmap(entry->vaddr_base, entry->size) != 0) {
 153             perror("unmap fails");
 154             exit(-1);
 155         }
 156     }
 157     if (entry->valid_mapping != NULL) {
 158         g_free(entry->valid_mapping);
 159         entry->valid_mapping = NULL;
 160     }
 161
 162     for (i = 0; i < nb_pfn; i++) {
 163         pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;
 164     }
 165
 166     vaddr_base = xc_map_foreign_bulk(xen_xc, xen_domid, PROT_READ|PROT_WRITE,
 167                                      pfns, err, nb_pfn);
 168     if (vaddr_base == NULL) {
 169         perror("xc_map_foreign_bulk");
 170         exit(-1);
 171     }
 172
 173     entry->vaddr_base = vaddr_base;
 174     entry->paddr_index = address_index;
 175     entry->size = size;
 176     entry->valid_mapping = (unsigned long *) g_malloc0(sizeof(unsigned long) *
 177             BITS_TO_LONGS(size >> XC_PAGE_SHIFT));
 178
 179     bitmap_zero(entry->valid_mapping, nb_pfn);
 180     for (i = 0; i < nb_pfn; i++) {
 181         if (!err[i]) {
 182             bitmap_set(entry->valid_mapping, i, 1);
 183         }
 184     }
 185
 186     g_free(pfns);
 187     g_free(err);
 188 }
 189
 190 uint8_t *xen_map_cache(target_phys_addr_t phys_addr, target_phys_addr_t size,
 191                        uint8_t lock)
 192 {
 193     MapCacheEntry *entry, *pentry = NULL;
 194     target_phys_addr_t address_index  = phys_addr >> MCACHE_BUCKET_SHIFT;
 195     target_phys_addr_t address_offset = phys_addr & (MCACHE_BUCKET_SIZE - 1);
 196     target_phys_addr_t __size = size;
 197
 198     trace_xen_map_cache(phys_addr);
 199
 200     if (address_index == mapcache->last_address_index && !lock && !__size) {
 201         trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
 202         return mapcache->last_address_vaddr + address_offset;
 203     }
 204
 205     /* size is always a multiple of MCACHE_BUCKET_SIZE */
 206     if ((address_offset + (__size % MCACHE_BUCKET_SIZE)) > MCACHE_BUCKET_SIZE)
 207         __size += MCACHE_BUCKET_SIZE;
 208     if (__size % MCACHE_BUCKET_SIZE)
 209         __size += MCACHE_BUCKET_SIZE - (__size % MCACHE_BUCKET_SIZE);
 210     if (!__size)
 211         __size = MCACHE_BUCKET_SIZE;
 212
 213     entry = &mapcache->entry[address_index % mapcache->nr_buckets];
 214
 215     while (entry && entry->lock && entry->vaddr_base &&
 216             (entry->paddr_index != address_index || entry->size != __size ||
 217              !test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
 218                  entry->valid_mapping))) {
 219         pentry = entry;
 220         entry = entry->next;
 221     }
 222     if (!entry) {
 223         entry = g_malloc0(sizeof (MapCacheEntry));
 224         pentry->next = entry;
 225         xen_remap_bucket(entry, __size, address_index);
 226     } else if (!entry->lock) {
 227         if (!entry->vaddr_base || entry->paddr_index != address_index ||
 228                 entry->size != __size ||
 229                 !test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
 230                     entry->valid_mapping)) {
 231             xen_remap_bucket(entry, __size, address_index);
 232         }
 233     }
 234
 235     if(!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
 236                 entry->valid_mapping)) {
 237         mapcache->last_address_index = -1;
 238         trace_xen_map_cache_return(NULL);
 239         return NULL;
 240     }
 241
 242     mapcache->last_address_index = address_index;
 243     mapcache->last_address_vaddr = entry->vaddr_base;
 244     if (lock) {
 245         MapCacheRev *reventry = g_malloc0(sizeof(MapCacheRev));
 246         entry->lock++;
 247         reventry->vaddr_req = mapcache->last_address_vaddr + address_offset;
 248         reventry->paddr_index = mapcache->last_address_index;
 249         reventry->size = entry->size;
 250         QTAILQ_INSERT_HEAD(&mapcache->locked_entries, reventry, next);
 251     }
 252
 253     trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
 254     return mapcache->last_address_vaddr + address_offset;
 255 }
 256
 257 ram_addr_t xen_ram_addr_from_mapcache(void *ptr)
 258 {
 259     MapCacheEntry *entry = NULL;
 260     MapCacheRev *reventry;
 261     target_phys_addr_t paddr_index;
 262     target_phys_addr_t size;
 263     int found = 0;
 264
 265     QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
 266         if (reventry->vaddr_req == ptr) {
 267             paddr_index = reventry->paddr_index;
 268             size = reventry->size;
 269             found = 1;
 270             break;
 271         }
 272     }
 273     if (!found) {
 274         fprintf(stderr, "%s, could not find %p\n", __func__, ptr);
 275         QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
 276             DPRINTF("   "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index,
 277                     reventry->vaddr_req);
 278         }
 279         abort();
 280         return 0;
 281     }
 282
 283     entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
 284     while (entry && (entry->paddr_index != paddr_index || entry->size != size)) {
 285         entry = entry->next;
 286     }
 287     if (!entry) {
 288         DPRINTF("Trying to find address %p that is not in the mapcache!\n", ptr);
 289         return 0;
 290     }
 291     return (reventry->paddr_index << MCACHE_BUCKET_SHIFT) +
 292         ((unsigned long) ptr - (unsigned long) entry->vaddr_base);
 293 }
 294
 295 void xen_invalidate_map_cache_entry(uint8_t *buffer)
 296 {
 297     MapCacheEntry *entry = NULL, *pentry = NULL;
 298     MapCacheRev *reventry;
 299     target_phys_addr_t paddr_index;
 300     target_phys_addr_t size;
 301     int found = 0;
 302
 303     if (mapcache->last_address_vaddr == buffer) {
 304         mapcache->last_address_index = -1;
 305     }
 306
 307     QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
 308         if (reventry->vaddr_req == buffer) {
 309             paddr_index = reventry->paddr_index;
 310             size = reventry->size;
 311             found = 1;
 312             break;
 313         }
 314     }
 315     if (!found) {
 316         DPRINTF("%s, could not find %p\n", __func__, buffer);
 317         QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
 318             DPRINTF("   "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index, reventry->vaddr_req);
 319         }
 320         return;
 321     }
 322     QTAILQ_REMOVE(&mapcache->locked_entries, reventry, next);
 323     g_free(reventry);
 324
 325     entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
 326     while (entry && (entry->paddr_index != paddr_index || entry->size != size)) {
 327         pentry = entry;
 328         entry = entry->next;
 329     }
 330     if (!entry) {
 331         DPRINTF("Trying to unmap address %p that is not in the mapcache!\n", buffer);
 332         return;
 333     }
 334     entry->lock--;
 335     if (entry->lock > 0 || pentry == NULL) {
 336         return;
 337     }
 338
 339     pentry->next = entry->next;
 340     if (munmap(entry->vaddr_base, entry->size) != 0) {
 341         perror("unmap fails");
 342         exit(-1);
 343     }
 344     g_free(entry->valid_mapping);
 345     g_free(entry);
 346 }
 347
 348 void xen_invalidate_map_cache(void)
 349 {
 350     unsigned long i;
 351     MapCacheRev *reventry;
 352
 353     /* Flush pending AIO before destroying the mapcache */
 354     bdrv_drain_all();
 355
 356     QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
 357         DPRINTF("There should be no locked mappings at this time, "
 358                 "but "TARGET_FMT_plx" -> %p is present\n",
 359                 reventry->paddr_index, reventry->vaddr_req);
 360     }
 361
 362     mapcache_lock();
 363
 364     for (i = 0; i < mapcache->nr_buckets; i++) {
 365         MapCacheEntry *entry = &mapcache->entry[i];
 366
 367         if (entry->vaddr_base == NULL) {
 368             continue;
 369         }
 370
 371         if (munmap(entry->vaddr_base, entry->size) != 0) {
 372             perror("unmap fails");
 373             exit(-1);
 374         }
 375
 376         entry->paddr_index = 0;
 377         entry->vaddr_base = NULL;
 378         entry->size = 0;
 379         g_free(entry->valid_mapping);
 380         entry->valid_mapping = NULL;
 381     }
 382
 383     mapcache->last_address_index = -1;
 384     mapcache->last_address_vaddr = NULL;
 385
 386     mapcache_unlock();
 387 }