include/exec/ram_addr.h

   1 /*
   2  * Declarations for cpu physical memory functions
   3  *
   4  * Copyright 2011 Red Hat, Inc. and/or its affiliates
   5  *
   6  * Authors:
   7  *  Avi Kivity <avi@redhat.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 or
  10  * later.  See the COPYING file in the top-level directory.
  11  *
  12  */
  13
  14 /*
  15  * This header is for use by exec.c and memory.c ONLY.  Do not include it.
  16  * The functions declared here will be removed soon.
  17  */
  18
  19 #ifndef RAM_ADDR_H
  20 #define RAM_ADDR_H
  21
  22 #ifndef CONFIG_USER_ONLY
  23 #include "hw/xen/xen.h"
  24
  25 struct RAMBlock {
  26     struct rcu_head rcu;
  27     struct MemoryRegion *mr;
  28     uint8_t *host;
  29     ram_addr_t offset;
  30     ram_addr_t used_length;
  31     ram_addr_t max_length;
  32     void (*resized)(const char*, uint64_t length, void *host);
  33     uint32_t flags;
  34     /* Protected by iothread lock.  */
  35     char idstr[256];
  36     /* RCU-enabled, writes protected by the ramlist lock */
  37     QLIST_ENTRY(RAMBlock) next;
  38     int fd;
  39 };
  40
  41 static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset)
  42 {
  43     return (b && b->host && offset < b->used_length) ? true : false;
  44 }
  45
  46 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
  47 {
  48     assert(offset_in_ramblock(block, offset));
  49     return (char *)block->host + offset;
  50 }
  51
  52 /* The dirty memory bitmap is split into fixed-size blocks to allow growth
  53  * under RCU.  The bitmap for a block can be accessed as follows:
  54  *
  55  *   rcu_read_lock();
  56  *
  57  *   DirtyMemoryBlocks *blocks =
  58  *       atomic_rcu_read(&ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION]);
  59  *
  60  *   ram_addr_t idx = (addr >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
  61  *   unsigned long *block = blocks.blocks[idx];
  62  *   ...access block bitmap...
  63  *
  64  *   rcu_read_unlock();
  65  *
  66  * Remember to check for the end of the block when accessing a range of
  67  * addresses.  Move on to the next block if you reach the end.
  68  *
  69  * Organization into blocks allows dirty memory to grow (but not shrink) under
  70  * RCU.  When adding new RAMBlocks requires the dirty memory to grow, a new
  71  * DirtyMemoryBlocks array is allocated with pointers to existing blocks kept
  72  * the same.  Other threads can safely access existing blocks while dirty
  73  * memory is being grown.  When no threads are using the old DirtyMemoryBlocks
  74  * anymore it is freed by RCU (but the underlying blocks stay because they are
  75  * pointed to from the new DirtyMemoryBlocks).
  76  */
  77 #define DIRTY_MEMORY_BLOCK_SIZE ((ram_addr_t)256 * 1024 * 8)
  78 typedef struct {
  79     struct rcu_head rcu;
  80     unsigned long *blocks[];
  81 } DirtyMemoryBlocks;
  82
  83 typedef struct RAMList {
  84     QemuMutex mutex;
  85     RAMBlock *mru_block;
  86     /* RCU-enabled, writes protected by the ramlist lock. */
  87     QLIST_HEAD(, RAMBlock) blocks;
  88     DirtyMemoryBlocks *dirty_memory[DIRTY_MEMORY_NUM];
  89     uint32_t version;
  90 } RAMList;
  91 extern RAMList ram_list;
  92
  93 ram_addr_t last_ram_offset(void);
  94 void qemu_mutex_lock_ramlist(void);
  95 void qemu_mutex_unlock_ramlist(void);
  96
  97 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
  98                                    bool share, const char *mem_path,
  99                                    Error **errp);
 100 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
 101                                   MemoryRegion *mr, Error **errp);
 102 RAMBlock *qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp);
 103 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
 104                                     void (*resized)(const char*,
 105                                                     uint64_t length,
 106                                                     void *host),
 107                                     MemoryRegion *mr, Error **errp);
 108 int qemu_get_ram_fd(ram_addr_t addr);
 109 void qemu_set_ram_fd(ram_addr_t addr, int fd);
 110 void *qemu_get_ram_block_host_ptr(ram_addr_t addr);
 111 void qemu_ram_free(RAMBlock *block);
 112
 113 int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp);
 114
 115 #define DIRTY_CLIENTS_ALL     ((1 << DIRTY_MEMORY_NUM) - 1)
 116 #define DIRTY_CLIENTS_NOCODE  (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
 117
 118 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
 119                                                  ram_addr_t length,
 120                                                  unsigned client)
 121 {
 122     DirtyMemoryBlocks *blocks;
 123     unsigned long end, page;
 124     unsigned long idx, offset, base;
 125     bool dirty = false;
 126
 127     assert(client < DIRTY_MEMORY_NUM);
 128
 129     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 130     page = start >> TARGET_PAGE_BITS;
 131
 132     rcu_read_lock();
 133
 134     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 135
 136     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 137     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 138     base = page - offset;
 139     while (page < end) {
 140         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
 141         unsigned long num = next - base;
 142         unsigned long found = find_next_bit(blocks->blocks[idx], num, offset);
 143         if (found < num) {
 144             dirty = true;
 145             break;
 146         }
 147
 148         page = next;
 149         idx++;
 150         offset = 0;
 151         base += DIRTY_MEMORY_BLOCK_SIZE;
 152     }
 153
 154     rcu_read_unlock();
 155
 156     return dirty;
 157 }
 158
 159 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
 160                                                  ram_addr_t length,
 161                                                  unsigned client)
 162 {
 163     DirtyMemoryBlocks *blocks;
 164     unsigned long end, page;
 165     unsigned long idx, offset, base;
 166     bool dirty = true;
 167
 168     assert(client < DIRTY_MEMORY_NUM);
 169
 170     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 171     page = start >> TARGET_PAGE_BITS;
 172
 173     rcu_read_lock();
 174
 175     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 176
 177     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 178     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 179     base = page - offset;
 180     while (page < end) {
 181         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
 182         unsigned long num = next - base;
 183         unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset);
 184         if (found < num) {
 185             dirty = false;
 186             break;
 187         }
 188
 189         page = next;
 190         idx++;
 191         offset = 0;
 192         base += DIRTY_MEMORY_BLOCK_SIZE;
 193     }
 194
 195     rcu_read_unlock();
 196
 197     return dirty;
 198 }
 199
 200 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
 201                                                       unsigned client)
 202 {
 203     return cpu_physical_memory_get_dirty(addr, 1, client);
 204 }
 205
 206 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
 207 {
 208     bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
 209     bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
 210     bool migration =
 211         cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
 212     return !(vga && code && migration);
 213 }
 214
 215 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
 216                                                                ram_addr_t length,
 217                                                                uint8_t mask)
 218 {
 219     uint8_t ret = 0;
 220
 221     if (mask & (1 << DIRTY_MEMORY_VGA) &&
 222         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) {
 223         ret |= (1 << DIRTY_MEMORY_VGA);
 224     }
 225     if (mask & (1 << DIRTY_MEMORY_CODE) &&
 226         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) {
 227         ret |= (1 << DIRTY_MEMORY_CODE);
 228     }
 229     if (mask & (1 << DIRTY_MEMORY_MIGRATION) &&
 230         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) {
 231         ret |= (1 << DIRTY_MEMORY_MIGRATION);
 232     }
 233     return ret;
 234 }
 235
 236 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
 237                                                       unsigned client)
 238 {
 239     unsigned long page, idx, offset;
 240     DirtyMemoryBlocks *blocks;
 241
 242     assert(client < DIRTY_MEMORY_NUM);
 243
 244     page = addr >> TARGET_PAGE_BITS;
 245     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 246     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 247
 248     rcu_read_lock();
 249
 250     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 251
 252     set_bit_atomic(offset, blocks->blocks[idx]);
 253
 254     rcu_read_unlock();
 255 }
 256
 257 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
 258                                                        ram_addr_t length,
 259                                                        uint8_t mask)
 260 {
 261     DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
 262     unsigned long end, page;
 263     unsigned long idx, offset, base;
 264     int i;
 265
 266     if (!mask && !xen_enabled()) {
 267         return;
 268     }
 269
 270     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 271     page = start >> TARGET_PAGE_BITS;
 272
 273     rcu_read_lock();
 274
 275     for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
 276         blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i]);
 277     }
 278
 279     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 280     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 281     base = page - offset;
 282     while (page < end) {
 283         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
 284
 285         if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
 286             bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
 287                               offset, next - page);
 288         }
 289         if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
 290             bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
 291                               offset, next - page);
 292         }
 293         if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
 294             bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
 295                               offset, next - page);
 296         }
 297
 298         page = next;
 299         idx++;
 300         offset = 0;
 301         base += DIRTY_MEMORY_BLOCK_SIZE;
 302     }
 303
 304     rcu_read_unlock();
 305
 306     xen_modified_memory(start, length);
 307 }
 308
 309 #if !defined(_WIN32)
 310 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
 311                                                           ram_addr_t start,
 312                                                           ram_addr_t pages)
 313 {
 314     unsigned long i, j;
 315     unsigned long page_number, c;
 316     hwaddr addr;
 317     ram_addr_t ram_addr;
 318     unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
 319     unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
 320     unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
 321
 322     /* start address is aligned at the start of a word? */
 323     if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
 324         (hpratio == 1)) {
 325         unsigned long **blocks[DIRTY_MEMORY_NUM];
 326         unsigned long idx;
 327         unsigned long offset;
 328         long k;
 329         long nr = BITS_TO_LONGS(pages);
 330
 331         idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
 332         offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
 333                           DIRTY_MEMORY_BLOCK_SIZE);
 334
 335         rcu_read_lock();
 336
 337         for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
 338             blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
 339         }
 340
 341         for (k = 0; k < nr; k++) {
 342             if (bitmap[k]) {
 343                 unsigned long temp = leul_to_cpu(bitmap[k]);
 344
 345                 atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset], temp);
 346                 atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
 347                 if (tcg_enabled()) {
 348                     atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], temp);
 349                 }
 350             }
 351
 352             if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
 353                 offset = 0;
 354                 idx++;
 355             }
 356         }
 357
 358         rcu_read_unlock();
 359
 360         xen_modified_memory(start, pages << TARGET_PAGE_BITS);
 361     } else {
 362         uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
 363         /*
 364          * bitmap-traveling is faster than memory-traveling (for addr...)
 365          * especially when most of the memory is not dirty.
 366          */
 367         for (i = 0; i < len; i++) {
 368             if (bitmap[i] != 0) {
 369                 c = leul_to_cpu(bitmap[i]);
 370                 do {
 371                     j = ctzl(c);
 372                     c &= ~(1ul << j);
 373                     page_number = (i * HOST_LONG_BITS + j) * hpratio;
 374                     addr = page_number * TARGET_PAGE_SIZE;
 375                     ram_addr = start + addr;
 376                     cpu_physical_memory_set_dirty_range(ram_addr,
 377                                        TARGET_PAGE_SIZE * hpratio, clients);
 378                 } while (c != 0);
 379             }
 380         }
 381     }
 382 }
 383 #endif /* not _WIN32 */
 384
 385 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
 386                                               ram_addr_t length,
 387                                               unsigned client);
 388
 389 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
 390                                                          ram_addr_t length)
 391 {
 392     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION);
 393     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA);
 394     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE);
 395 }
 396
 397
 398 static inline
 399 uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest,
 400                                                ram_addr_t start,
 401                                                ram_addr_t length)
 402 {
 403     ram_addr_t addr;
 404     unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
 405     uint64_t num_dirty = 0;
 406
 407     /* start address is aligned at the start of a word? */
 408     if (((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) {
 409         int k;
 410         int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
 411         unsigned long * const *src;
 412         unsigned long idx = (page * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
 413         unsigned long offset = BIT_WORD((page * BITS_PER_LONG) %
 414                                         DIRTY_MEMORY_BLOCK_SIZE);
 415
 416         rcu_read_lock();
 417
 418         src = atomic_rcu_read(
 419                 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
 420
 421         for (k = page; k < page + nr; k++) {
 422             if (src[idx][offset]) {
 423                 unsigned long bits = atomic_xchg(&src[idx][offset], 0);
 424                 unsigned long new_dirty;
 425                 new_dirty = ~dest[k];
 426                 dest[k] |= bits;
 427                 new_dirty &= bits;
 428                 num_dirty += ctpopl(new_dirty);
 429             }
 430
 431             if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
 432                 offset = 0;
 433                 idx++;
 434             }
 435         }
 436
 437         rcu_read_unlock();
 438     } else {
 439         for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
 440             if (cpu_physical_memory_test_and_clear_dirty(
 441                         start + addr,
 442                         TARGET_PAGE_SIZE,
 443                         DIRTY_MEMORY_MIGRATION)) {
 444                 long k = (start + addr) >> TARGET_PAGE_BITS;
 445                 if (!test_and_set_bit(k, dest)) {
 446                     num_dirty++;
 447                 }
 448             }
 449         }
 450     }
 451
 452     return num_dirty;
 453 }
 454
 455 void migration_bitmap_extend(ram_addr_t old, ram_addr_t new);
 456 #endif
 457 #endif