include/exec/ram_addr.h

   1 /*
   2  * Declarations for cpu physical memory functions
   3  *
   4  * Copyright 2011 Red Hat, Inc. and/or its affiliates
   5  *
   6  * Authors:
   7  *  Avi Kivity <avi@redhat.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 or
  10  * later.  See the COPYING file in the top-level directory.
  11  *
  12  */
  13
  14 /*
  15  * This header is for use by exec.c and memory.c ONLY.  Do not include it.
  16  * The functions declared here will be removed soon.
  17  */
  18
  19 #ifndef RAM_ADDR_H
  20 #define RAM_ADDR_H
  21
  22 #ifndef CONFIG_USER_ONLY
  23 #include "hw/xen/xen.h"
  24
  25 struct RAMBlock {
  26     struct rcu_head rcu;
  27     struct MemoryRegion *mr;
  28     uint8_t *host;
  29     ram_addr_t offset;
  30     ram_addr_t used_length;
  31     ram_addr_t max_length;
  32     void (*resized)(const char*, uint64_t length, void *host);
  33     uint32_t flags;
  34     /* Protected by iothread lock.  */
  35     char idstr[256];
  36     /* RCU-enabled, writes protected by the ramlist lock */
  37     QLIST_ENTRY(RAMBlock) next;
  38     int fd;
  39 };
  40
  41 static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset)
  42 {
  43     return (b && b->host && offset < b->used_length) ? true : false;
  44 }
  45
  46 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
  47 {
  48     assert(offset_in_ramblock(block, offset));
  49     return (char *)block->host + offset;
  50 }
  51
  52 /* The dirty memory bitmap is split into fixed-size blocks to allow growth
  53  * under RCU.  The bitmap for a block can be accessed as follows:
  54  *
  55  *   rcu_read_lock();
  56  *
  57  *   DirtyMemoryBlocks *blocks =
  58  *       atomic_rcu_read(&ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION]);
  59  *
  60  *   ram_addr_t idx = (addr >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
  61  *   unsigned long *block = blocks.blocks[idx];
  62  *   ...access block bitmap...
  63  *
  64  *   rcu_read_unlock();
  65  *
  66  * Remember to check for the end of the block when accessing a range of
  67  * addresses.  Move on to the next block if you reach the end.
  68  *
  69  * Organization into blocks allows dirty memory to grow (but not shrink) under
  70  * RCU.  When adding new RAMBlocks requires the dirty memory to grow, a new
  71  * DirtyMemoryBlocks array is allocated with pointers to existing blocks kept
  72  * the same.  Other threads can safely access existing blocks while dirty
  73  * memory is being grown.  When no threads are using the old DirtyMemoryBlocks
  74  * anymore it is freed by RCU (but the underlying blocks stay because they are
  75  * pointed to from the new DirtyMemoryBlocks).
  76  */
  77 #define DIRTY_MEMORY_BLOCK_SIZE ((ram_addr_t)256 * 1024 * 8)
  78 typedef struct {
  79     struct rcu_head rcu;
  80     unsigned long *blocks[];
  81 } DirtyMemoryBlocks;
  82
  83 typedef struct RAMList {
  84     QemuMutex mutex;
  85     RAMBlock *mru_block;
  86     /* RCU-enabled, writes protected by the ramlist lock. */
  87     QLIST_HEAD(, RAMBlock) blocks;
  88     DirtyMemoryBlocks *dirty_memory[DIRTY_MEMORY_NUM];
  89     uint32_t version;
  90 } RAMList;
  91 extern RAMList ram_list;
  92
  93 ram_addr_t last_ram_offset(void);
  94 void qemu_mutex_lock_ramlist(void);
  95 void qemu_mutex_unlock_ramlist(void);
  96
  97 ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
  98                                     bool share, const char *mem_path,
  99                                     Error **errp);
 100 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
 101                                    MemoryRegion *mr, Error **errp);
 102 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp);
 103 ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
 104                                      void (*resized)(const char*,
 105                                                      uint64_t length,
 106                                                      void *host),
 107                                      MemoryRegion *mr, Error **errp);
 108 int qemu_get_ram_fd(ram_addr_t addr);
 109 void qemu_set_ram_fd(ram_addr_t addr, int fd);
 110 void *qemu_get_ram_block_host_ptr(ram_addr_t addr);
 111 void qemu_ram_free(ram_addr_t addr);
 112
 113 int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp);
 114
 115 #define DIRTY_CLIENTS_ALL     ((1 << DIRTY_MEMORY_NUM) - 1)
 116 #define DIRTY_CLIENTS_NOCODE  (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
 117
 118 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
 119                                                  ram_addr_t length,
 120                                                  unsigned client)
 121 {
 122     DirtyMemoryBlocks *blocks;
 123     unsigned long end, page;
 124     bool dirty = false;
 125
 126     assert(client < DIRTY_MEMORY_NUM);
 127
 128     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 129     page = start >> TARGET_PAGE_BITS;
 130
 131     rcu_read_lock();
 132
 133     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 134
 135     while (page < end) {
 136         unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 137         unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 138         unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
 139
 140         if (find_next_bit(blocks->blocks[idx], offset, num) < num) {
 141             dirty = true;
 142             break;
 143         }
 144
 145         page += num;
 146     }
 147
 148     rcu_read_unlock();
 149
 150     return dirty;
 151 }
 152
 153 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
 154                                                  ram_addr_t length,
 155                                                  unsigned client)
 156 {
 157     DirtyMemoryBlocks *blocks;
 158     unsigned long end, page;
 159     bool dirty = true;
 160
 161     assert(client < DIRTY_MEMORY_NUM);
 162
 163     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 164     page = start >> TARGET_PAGE_BITS;
 165
 166     rcu_read_lock();
 167
 168     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 169
 170     while (page < end) {
 171         unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 172         unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 173         unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
 174
 175         if (find_next_zero_bit(blocks->blocks[idx], offset, num) < num) {
 176             dirty = false;
 177             break;
 178         }
 179
 180         page += num;
 181     }
 182
 183     rcu_read_unlock();
 184
 185     return dirty;
 186 }
 187
 188 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
 189                                                       unsigned client)
 190 {
 191     return cpu_physical_memory_get_dirty(addr, 1, client);
 192 }
 193
 194 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
 195 {
 196     bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
 197     bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
 198     bool migration =
 199         cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
 200     return !(vga && code && migration);
 201 }
 202
 203 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
 204                                                                ram_addr_t length,
 205                                                                uint8_t mask)
 206 {
 207     uint8_t ret = 0;
 208
 209     if (mask & (1 << DIRTY_MEMORY_VGA) &&
 210         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) {
 211         ret |= (1 << DIRTY_MEMORY_VGA);
 212     }
 213     if (mask & (1 << DIRTY_MEMORY_CODE) &&
 214         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) {
 215         ret |= (1 << DIRTY_MEMORY_CODE);
 216     }
 217     if (mask & (1 << DIRTY_MEMORY_MIGRATION) &&
 218         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) {
 219         ret |= (1 << DIRTY_MEMORY_MIGRATION);
 220     }
 221     return ret;
 222 }
 223
 224 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
 225                                                       unsigned client)
 226 {
 227     unsigned long page, idx, offset;
 228     DirtyMemoryBlocks *blocks;
 229
 230     assert(client < DIRTY_MEMORY_NUM);
 231
 232     page = addr >> TARGET_PAGE_BITS;
 233     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 234     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 235
 236     rcu_read_lock();
 237
 238     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 239
 240     set_bit_atomic(offset, blocks->blocks[idx]);
 241
 242     rcu_read_unlock();
 243 }
 244
 245 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
 246                                                        ram_addr_t length,
 247                                                        uint8_t mask)
 248 {
 249     DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
 250     unsigned long end, page;
 251     int i;
 252
 253     if (!mask && !xen_enabled()) {
 254         return;
 255     }
 256
 257     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 258     page = start >> TARGET_PAGE_BITS;
 259
 260     rcu_read_lock();
 261
 262     for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
 263         blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i]);
 264     }
 265
 266     while (page < end) {
 267         unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 268         unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 269         unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
 270
 271         if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
 272             bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
 273                               offset, num);
 274         }
 275         if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
 276             bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
 277                               offset, num);
 278         }
 279         if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
 280             bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
 281                               offset, num);
 282         }
 283
 284         page += num;
 285     }
 286
 287     rcu_read_unlock();
 288
 289     xen_modified_memory(start, length);
 290 }
 291
 292 #if !defined(_WIN32)
 293 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
 294                                                           ram_addr_t start,
 295                                                           ram_addr_t pages)
 296 {
 297     unsigned long i, j;
 298     unsigned long page_number, c;
 299     hwaddr addr;
 300     ram_addr_t ram_addr;
 301     unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
 302     unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
 303     unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
 304
 305     /* start address is aligned at the start of a word? */
 306     if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
 307         (hpratio == 1)) {
 308         unsigned long **blocks[DIRTY_MEMORY_NUM];
 309         unsigned long idx;
 310         unsigned long offset;
 311         long k;
 312         long nr = BITS_TO_LONGS(pages);
 313
 314         idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
 315         offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
 316                           DIRTY_MEMORY_BLOCK_SIZE);
 317
 318         rcu_read_lock();
 319
 320         for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
 321             blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
 322         }
 323
 324         for (k = 0; k < nr; k++) {
 325             if (bitmap[k]) {
 326                 unsigned long temp = leul_to_cpu(bitmap[k]);
 327
 328                 atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset], temp);
 329                 atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
 330                 if (tcg_enabled()) {
 331                     atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], temp);
 332                 }
 333             }
 334
 335             if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
 336                 offset = 0;
 337                 idx++;
 338             }
 339         }
 340
 341         rcu_read_unlock();
 342
 343         xen_modified_memory(start, pages << TARGET_PAGE_BITS);
 344     } else {
 345         uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
 346         /*
 347          * bitmap-traveling is faster than memory-traveling (for addr...)
 348          * especially when most of the memory is not dirty.
 349          */
 350         for (i = 0; i < len; i++) {
 351             if (bitmap[i] != 0) {
 352                 c = leul_to_cpu(bitmap[i]);
 353                 do {
 354                     j = ctzl(c);
 355                     c &= ~(1ul << j);
 356                     page_number = (i * HOST_LONG_BITS + j) * hpratio;
 357                     addr = page_number * TARGET_PAGE_SIZE;
 358                     ram_addr = start + addr;
 359                     cpu_physical_memory_set_dirty_range(ram_addr,
 360                                        TARGET_PAGE_SIZE * hpratio, clients);
 361                 } while (c != 0);
 362             }
 363         }
 364     }
 365 }
 366 #endif /* not _WIN32 */
 367
 368 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
 369                                               ram_addr_t length,
 370                                               unsigned client);
 371
 372 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
 373                                                          ram_addr_t length)
 374 {
 375     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION);
 376     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA);
 377     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE);
 378 }
 379
 380
 381 static inline
 382 uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest,
 383                                                ram_addr_t start,
 384                                                ram_addr_t length)
 385 {
 386     ram_addr_t addr;
 387     unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
 388     uint64_t num_dirty = 0;
 389
 390     /* start address is aligned at the start of a word? */
 391     if (((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) {
 392         int k;
 393         int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
 394         unsigned long * const *src;
 395         unsigned long idx = (page * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
 396         unsigned long offset = BIT_WORD((page * BITS_PER_LONG) %
 397                                         DIRTY_MEMORY_BLOCK_SIZE);
 398
 399         rcu_read_lock();
 400
 401         src = atomic_rcu_read(
 402                 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
 403
 404         for (k = page; k < page + nr; k++) {
 405             if (src[idx][offset]) {
 406                 unsigned long bits = atomic_xchg(&src[idx][offset], 0);
 407                 unsigned long new_dirty;
 408                 new_dirty = ~dest[k];
 409                 dest[k] |= bits;
 410                 new_dirty &= bits;
 411                 num_dirty += ctpopl(new_dirty);
 412             }
 413
 414             if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
 415                 offset = 0;
 416                 idx++;
 417             }
 418         }
 419
 420         rcu_read_unlock();
 421     } else {
 422         for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
 423             if (cpu_physical_memory_test_and_clear_dirty(
 424                         start + addr,
 425                         TARGET_PAGE_SIZE,
 426                         DIRTY_MEMORY_MIGRATION)) {
 427                 long k = (start + addr) >> TARGET_PAGE_BITS;
 428                 if (!test_and_set_bit(k, dest)) {
 429                     num_dirty++;
 430                 }
 431             }
 432         }
 433     }
 434
 435     return num_dirty;
 436 }
 437
 438 void migration_bitmap_extend(ram_addr_t old, ram_addr_t new);
 439 #endif
 440 #endif