include/exec/ram_addr.h

   1 /*
   2  * Declarations for cpu physical memory functions
   3  *
   4  * Copyright 2011 Red Hat, Inc. and/or its affiliates
   5  *
   6  * Authors:
   7  *  Avi Kivity <avi@redhat.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 or
  10  * later.  See the COPYING file in the top-level directory.
  11  *
  12  */
  13
  14 /*
  15  * This header is for use by exec.c and memory.c ONLY.  Do not include it.
  16  * The functions declared here will be removed soon.
  17  */
  18
  19 #ifndef RAM_ADDR_H
  20 #define RAM_ADDR_H
  21
  22 #ifndef CONFIG_USER_ONLY
  23 #include "hw/xen/xen.h"
  24 #include "exec/ramlist.h"
  25
  26 struct RAMBlock {
  27     struct rcu_head rcu;
  28     struct MemoryRegion *mr;
  29     uint8_t *host;
  30     ram_addr_t offset;
  31     ram_addr_t used_length;
  32     ram_addr_t max_length;
  33     void (*resized)(const char*, uint64_t length, void *host);
  34     uint32_t flags;
  35     /* Protected by iothread lock.  */
  36     char idstr[256];
  37     /* RCU-enabled, writes protected by the ramlist lock */
  38     QLIST_ENTRY(RAMBlock) next;
  39     QLIST_HEAD(, RAMBlockNotifier) ramblock_notifiers;
  40     int fd;
  41     size_t page_size;
  42     /* dirty bitmap used during migration */
  43     unsigned long *bmap;
  44     /* bitmap of pages that haven't been sent even once
  45      * only maintained and used in postcopy at the moment
  46      * where it's used to send the dirtymap at the start
  47      * of the postcopy phase
  48      */
  49     unsigned long *unsentmap;
  50     /* bitmap of already received pages in postcopy */
  51     unsigned long *receivedmap;
  52 };
  53
  54 static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset)
  55 {
  56     return (b && b->host && offset < b->used_length) ? true : false;
  57 }
  58
  59 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
  60 {
  61     assert(offset_in_ramblock(block, offset));
  62     return (char *)block->host + offset;
  63 }
  64
  65 static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr,
  66                                                             RAMBlock *rb)
  67 {
  68     uint64_t host_addr_offset =
  69             (uint64_t)(uintptr_t)(host_addr - (void *)rb->host);
  70     return host_addr_offset >> TARGET_PAGE_BITS;
  71 }
  72
  73 long qemu_getrampagesize(void);
  74 unsigned long last_ram_page(void);
  75 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
  76                                    bool share, const char *mem_path,
  77                                    Error **errp);
  78 RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
  79                                  bool share, int fd,
  80                                  Error **errp);
  81 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
  82                                   MemoryRegion *mr, Error **errp);
  83 RAMBlock *qemu_ram_alloc(ram_addr_t size, bool share, MemoryRegion *mr,
  84                          Error **errp);
  85 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
  86                                     void (*resized)(const char*,
  87                                                     uint64_t length,
  88                                                     void *host),
  89                                     MemoryRegion *mr, Error **errp);
  90 void qemu_ram_free(RAMBlock *block);
  91
  92 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp);
  93
  94 #define DIRTY_CLIENTS_ALL     ((1 << DIRTY_MEMORY_NUM) - 1)
  95 #define DIRTY_CLIENTS_NOCODE  (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
  96
  97 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
  98                                                  ram_addr_t length,
  99                                                  unsigned client)
 100 {
 101     DirtyMemoryBlocks *blocks;
 102     unsigned long end, page;
 103     unsigned long idx, offset, base;
 104     bool dirty = false;
 105
 106     assert(client < DIRTY_MEMORY_NUM);
 107
 108     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 109     page = start >> TARGET_PAGE_BITS;
 110
 111     rcu_read_lock();
 112
 113     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 114
 115     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 116     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 117     base = page - offset;
 118     while (page < end) {
 119         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
 120         unsigned long num = next - base;
 121         unsigned long found = find_next_bit(blocks->blocks[idx], num, offset);
 122         if (found < num) {
 123             dirty = true;
 124             break;
 125         }
 126
 127         page = next;
 128         idx++;
 129         offset = 0;
 130         base += DIRTY_MEMORY_BLOCK_SIZE;
 131     }
 132
 133     rcu_read_unlock();
 134
 135     return dirty;
 136 }
 137
 138 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
 139                                                  ram_addr_t length,
 140                                                  unsigned client)
 141 {
 142     DirtyMemoryBlocks *blocks;
 143     unsigned long end, page;
 144     unsigned long idx, offset, base;
 145     bool dirty = true;
 146
 147     assert(client < DIRTY_MEMORY_NUM);
 148
 149     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 150     page = start >> TARGET_PAGE_BITS;
 151
 152     rcu_read_lock();
 153
 154     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 155
 156     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 157     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 158     base = page - offset;
 159     while (page < end) {
 160         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
 161         unsigned long num = next - base;
 162         unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset);
 163         if (found < num) {
 164             dirty = false;
 165             break;
 166         }
 167
 168         page = next;
 169         idx++;
 170         offset = 0;
 171         base += DIRTY_MEMORY_BLOCK_SIZE;
 172     }
 173
 174     rcu_read_unlock();
 175
 176     return dirty;
 177 }
 178
 179 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
 180                                                       unsigned client)
 181 {
 182     return cpu_physical_memory_get_dirty(addr, 1, client);
 183 }
 184
 185 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
 186 {
 187     bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
 188     bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
 189     bool migration =
 190         cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
 191     return !(vga && code && migration);
 192 }
 193
 194 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
 195                                                                ram_addr_t length,
 196                                                                uint8_t mask)
 197 {
 198     uint8_t ret = 0;
 199
 200     if (mask & (1 << DIRTY_MEMORY_VGA) &&
 201         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) {
 202         ret |= (1 << DIRTY_MEMORY_VGA);
 203     }
 204     if (mask & (1 << DIRTY_MEMORY_CODE) &&
 205         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) {
 206         ret |= (1 << DIRTY_MEMORY_CODE);
 207     }
 208     if (mask & (1 << DIRTY_MEMORY_MIGRATION) &&
 209         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) {
 210         ret |= (1 << DIRTY_MEMORY_MIGRATION);
 211     }
 212     return ret;
 213 }
 214
 215 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
 216                                                       unsigned client)
 217 {
 218     unsigned long page, idx, offset;
 219     DirtyMemoryBlocks *blocks;
 220
 221     assert(client < DIRTY_MEMORY_NUM);
 222
 223     page = addr >> TARGET_PAGE_BITS;
 224     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 225     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 226
 227     rcu_read_lock();
 228
 229     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 230
 231     set_bit_atomic(offset, blocks->blocks[idx]);
 232
 233     rcu_read_unlock();
 234 }
 235
 236 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
 237                                                        ram_addr_t length,
 238                                                        uint8_t mask)
 239 {
 240     DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
 241     unsigned long end, page;
 242     unsigned long idx, offset, base;
 243     int i;
 244
 245     if (!mask && !xen_enabled()) {
 246         return;
 247     }
 248
 249     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 250     page = start >> TARGET_PAGE_BITS;
 251
 252     rcu_read_lock();
 253
 254     for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
 255         blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i]);
 256     }
 257
 258     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 259     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 260     base = page - offset;
 261     while (page < end) {
 262         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
 263
 264         if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
 265             bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
 266                               offset, next - page);
 267         }
 268         if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
 269             bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
 270                               offset, next - page);
 271         }
 272         if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
 273             bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
 274                               offset, next - page);
 275         }
 276
 277         page = next;
 278         idx++;
 279         offset = 0;
 280         base += DIRTY_MEMORY_BLOCK_SIZE;
 281     }
 282
 283     rcu_read_unlock();
 284
 285     xen_hvm_modified_memory(start, length);
 286 }
 287
 288 #if !defined(_WIN32)
 289 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
 290                                                           ram_addr_t start,
 291                                                           ram_addr_t pages)
 292 {
 293     unsigned long i, j;
 294     unsigned long page_number, c;
 295     hwaddr addr;
 296     ram_addr_t ram_addr;
 297     unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
 298     unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
 299     unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
 300
 301     /* start address is aligned at the start of a word? */
 302     if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
 303         (hpratio == 1)) {
 304         unsigned long **blocks[DIRTY_MEMORY_NUM];
 305         unsigned long idx;
 306         unsigned long offset;
 307         long k;
 308         long nr = BITS_TO_LONGS(pages);
 309
 310         idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
 311         offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
 312                           DIRTY_MEMORY_BLOCK_SIZE);
 313
 314         rcu_read_lock();
 315
 316         for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
 317             blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
 318         }
 319
 320         for (k = 0; k < nr; k++) {
 321             if (bitmap[k]) {
 322                 unsigned long temp = leul_to_cpu(bitmap[k]);
 323
 324                 atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset], temp);
 325                 atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
 326                 if (tcg_enabled()) {
 327                     atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], temp);
 328                 }
 329             }
 330
 331             if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
 332                 offset = 0;
 333                 idx++;
 334             }
 335         }
 336
 337         rcu_read_unlock();
 338
 339         xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS);
 340     } else {
 341         uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
 342         /*
 343          * bitmap-traveling is faster than memory-traveling (for addr...)
 344          * especially when most of the memory is not dirty.
 345          */
 346         for (i = 0; i < len; i++) {
 347             if (bitmap[i] != 0) {
 348                 c = leul_to_cpu(bitmap[i]);
 349                 do {
 350                     j = ctzl(c);
 351                     c &= ~(1ul << j);
 352                     page_number = (i * HOST_LONG_BITS + j) * hpratio;
 353                     addr = page_number * TARGET_PAGE_SIZE;
 354                     ram_addr = start + addr;
 355                     cpu_physical_memory_set_dirty_range(ram_addr,
 356                                        TARGET_PAGE_SIZE * hpratio, clients);
 357                 } while (c != 0);
 358             }
 359         }
 360     }
 361 }
 362 #endif /* not _WIN32 */
 363
 364 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
 365                                               ram_addr_t length,
 366                                               unsigned client);
 367
 368 DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
 369     (ram_addr_t start, ram_addr_t length, unsigned client);
 370
 371 bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
 372                                             ram_addr_t start,
 373                                             ram_addr_t length);
 374
 375 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
 376                                                          ram_addr_t length)
 377 {
 378     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION);
 379     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA);
 380     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE);
 381 }
 382
 383
 384 static inline
 385 uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb,
 386                                                ram_addr_t start,
 387                                                ram_addr_t length,
 388                                                uint64_t *real_dirty_pages)
 389 {
 390     ram_addr_t addr;
 391     unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS);
 392     uint64_t num_dirty = 0;
 393     unsigned long *dest = rb->bmap;
 394
 395     /* start address and length is aligned at the start of a word? */
 396     if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) ==
 397          (start + rb->offset) &&
 398         !(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) {
 399         int k;
 400         int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
 401         unsigned long * const *src;
 402         unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
 403         unsigned long offset = BIT_WORD((word * BITS_PER_LONG) %
 404                                         DIRTY_MEMORY_BLOCK_SIZE);
 405         unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
 406
 407         rcu_read_lock();
 408
 409         src = atomic_rcu_read(
 410                 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
 411
 412         for (k = page; k < page + nr; k++) {
 413             if (src[idx][offset]) {
 414                 unsigned long bits = atomic_xchg(&src[idx][offset], 0);
 415                 unsigned long new_dirty;
 416                 *real_dirty_pages += ctpopl(bits);
 417                 new_dirty = ~dest[k];
 418                 dest[k] |= bits;
 419                 new_dirty &= bits;
 420                 num_dirty += ctpopl(new_dirty);
 421             }
 422
 423             if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
 424                 offset = 0;
 425                 idx++;
 426             }
 427         }
 428
 429         rcu_read_unlock();
 430     } else {
 431         ram_addr_t offset = rb->offset;
 432
 433         for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
 434             if (cpu_physical_memory_test_and_clear_dirty(
 435                         start + addr + offset,
 436                         TARGET_PAGE_SIZE,
 437                         DIRTY_MEMORY_MIGRATION)) {
 438                 *real_dirty_pages += 1;
 439                 long k = (start + addr) >> TARGET_PAGE_BITS;
 440                 if (!test_and_set_bit(k, dest)) {
 441                     num_dirty++;
 442                 }
 443             }
 444         }
 445     }
 446
 447     return num_dirty;
 448 }
 449 #endif
 450 #endif