4 * Copyright (C) 2020 Red Hat, Inc.
7 * David Hildenbrand <david@redhat.com>
9 * This work is licensed under the terms of the GNU GPL, version 2.
10 * See the COPYING file in the top-level directory.
13 #include "qemu/osdep.h"
15 #include "qemu/cutils.h"
16 #include "qemu/error-report.h"
17 #include "qemu/units.h"
18 #include "sysemu/numa.h"
19 #include "sysemu/sysemu.h"
20 #include "sysemu/reset.h"
21 #include "sysemu/runstate.h"
22 #include "hw/virtio/virtio.h"
23 #include "hw/virtio/virtio-bus.h"
24 #include "hw/virtio/virtio-mem.h"
25 #include "qapi/error.h"
26 #include "qapi/visitor.h"
27 #include "exec/ram_addr.h"
28 #include "migration/misc.h"
29 #include "hw/boards.h"
30 #include "hw/qdev-properties.h"
31 #include CONFIG_DEVICES
34 static const VMStateDescription vmstate_virtio_mem_device_early
;
37 * We only had legacy x86 guests that did not support
38 * VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE. Other targets don't have legacy guests.
40 #if defined(TARGET_X86_64) || defined(TARGET_I386)
41 #define VIRTIO_MEM_HAS_LEGACY_GUESTS
45 * Let's not allow blocks smaller than 1 MiB, for example, to keep the tracking
48 #define VIRTIO_MEM_MIN_BLOCK_SIZE ((uint32_t)(1 * MiB))
50 static uint32_t virtio_mem_default_thp_size(void)
52 uint32_t default_thp_size
= VIRTIO_MEM_MIN_BLOCK_SIZE
;
54 #if defined(__x86_64__) || defined(__arm__) || defined(__powerpc64__)
55 default_thp_size
= 2 * MiB
;
56 #elif defined(__aarch64__)
57 if (qemu_real_host_page_size() == 4 * KiB
) {
58 default_thp_size
= 2 * MiB
;
59 } else if (qemu_real_host_page_size() == 16 * KiB
) {
60 default_thp_size
= 32 * MiB
;
61 } else if (qemu_real_host_page_size() == 64 * KiB
) {
62 default_thp_size
= 512 * MiB
;
66 return default_thp_size
;
70 * The minimum memslot size depends on this setting ("sane default"), the
71 * device block size, and the memory backend page size. The last (or single)
72 * memslot might be smaller than this constant.
74 #define VIRTIO_MEM_MIN_MEMSLOT_SIZE (1 * GiB)
77 * We want to have a reasonable default block size such that
78 * 1. We avoid splitting THPs when unplugging memory, which degrades
80 * 2. We avoid placing THPs for plugged blocks that also cover unplugged
83 * The actual THP size might differ between Linux kernels, so we try to probe
84 * it. In the future (if we ever run into issues regarding 2.), we might want
85 * to disable THP in case we fail to properly probe the THP size, or if the
86 * block size is configured smaller than the THP size.
88 static uint32_t thp_size
;
90 #define HPAGE_PMD_SIZE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size"
91 static uint32_t virtio_mem_thp_size(void)
93 gchar
*content
= NULL
;
102 * Try to probe the actual THP size, fallback to (sane but eventually
103 * incorrect) default sizes.
105 if (g_file_get_contents(HPAGE_PMD_SIZE_PATH
, &content
, NULL
, NULL
) &&
106 !qemu_strtou64(content
, &endptr
, 0, &tmp
) &&
107 (!endptr
|| *endptr
== '\n')) {
108 /* Sanity-check the value and fallback to something reasonable. */
109 if (!tmp
|| !is_power_of_2(tmp
)) {
110 warn_report("Read unsupported THP size: %" PRIx64
, tmp
);
117 thp_size
= virtio_mem_default_thp_size();
118 warn_report("Could not detect THP size, falling back to %" PRIx64
119 " MiB.", thp_size
/ MiB
);
126 static uint64_t virtio_mem_default_block_size(RAMBlock
*rb
)
128 const uint64_t page_size
= qemu_ram_pagesize(rb
);
130 /* We can have hugetlbfs with a page size smaller than the THP size. */
131 if (page_size
== qemu_real_host_page_size()) {
132 return MAX(page_size
, virtio_mem_thp_size());
134 return MAX(page_size
, VIRTIO_MEM_MIN_BLOCK_SIZE
);
137 #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS)
138 static bool virtio_mem_has_shared_zeropage(RAMBlock
*rb
)
141 * We only have a guaranteed shared zeropage on ordinary MAP_PRIVATE
142 * anonymous RAM. In any other case, reading unplugged *can* populate a
143 * fresh page, consuming actual memory.
145 return !qemu_ram_is_shared(rb
) && qemu_ram_get_fd(rb
) < 0 &&
146 qemu_ram_pagesize(rb
) == qemu_real_host_page_size();
148 #endif /* VIRTIO_MEM_HAS_LEGACY_GUESTS */
151 * Size the usable region bigger than the requested size if possible. Esp.
152 * Linux guests will only add (aligned) memory blocks in case they fully
153 * fit into the usable region, but plug+online only a subset of the pages.
154 * The memory block size corresponds mostly to the section size.
156 * This allows e.g., to add 20MB with a section size of 128MB on x86_64, and
157 * a section size of 512MB on arm64 (as long as the start address is properly
158 * aligned, similar to ordinary DIMMs).
160 * We can change this at any time and maybe even make it configurable if
161 * necessary (as the section size can change). But it's more likely that the
162 * section size will rather get smaller and not bigger over time.
164 #if defined(TARGET_X86_64) || defined(TARGET_I386)
165 #define VIRTIO_MEM_USABLE_EXTENT (2 * (128 * MiB))
166 #elif defined(TARGET_ARM)
167 #define VIRTIO_MEM_USABLE_EXTENT (2 * (512 * MiB))
169 #error VIRTIO_MEM_USABLE_EXTENT not defined
172 static bool virtio_mem_is_busy(void)
175 * Postcopy cannot handle concurrent discards and we don't want to migrate
176 * pages on-demand with stale content when plugging new blocks.
178 * For precopy, we don't want unplugged blocks in our migration stream, and
179 * when plugging new blocks, the page content might differ between source
180 * and destination (observable by the guest when not initializing pages
181 * after plugging them) until we're running on the destination (as we didn't
182 * migrate these blocks when they were unplugged).
184 return migration_in_incoming_postcopy() || !migration_is_idle();
187 typedef int (*virtio_mem_range_cb
)(VirtIOMEM
*vmem
, void *arg
,
188 uint64_t offset
, uint64_t size
);
190 static int virtio_mem_for_each_unplugged_range(VirtIOMEM
*vmem
, void *arg
,
191 virtio_mem_range_cb cb
)
193 unsigned long first_zero_bit
, last_zero_bit
;
194 uint64_t offset
, size
;
197 first_zero_bit
= find_first_zero_bit(vmem
->bitmap
, vmem
->bitmap_size
);
198 while (first_zero_bit
< vmem
->bitmap_size
) {
199 offset
= first_zero_bit
* vmem
->block_size
;
200 last_zero_bit
= find_next_bit(vmem
->bitmap
, vmem
->bitmap_size
,
201 first_zero_bit
+ 1) - 1;
202 size
= (last_zero_bit
- first_zero_bit
+ 1) * vmem
->block_size
;
204 ret
= cb(vmem
, arg
, offset
, size
);
208 first_zero_bit
= find_next_zero_bit(vmem
->bitmap
, vmem
->bitmap_size
,
214 static int virtio_mem_for_each_plugged_range(VirtIOMEM
*vmem
, void *arg
,
215 virtio_mem_range_cb cb
)
217 unsigned long first_bit
, last_bit
;
218 uint64_t offset
, size
;
221 first_bit
= find_first_bit(vmem
->bitmap
, vmem
->bitmap_size
);
222 while (first_bit
< vmem
->bitmap_size
) {
223 offset
= first_bit
* vmem
->block_size
;
224 last_bit
= find_next_zero_bit(vmem
->bitmap
, vmem
->bitmap_size
,
226 size
= (last_bit
- first_bit
+ 1) * vmem
->block_size
;
228 ret
= cb(vmem
, arg
, offset
, size
);
232 first_bit
= find_next_bit(vmem
->bitmap
, vmem
->bitmap_size
,
239 * Adjust the memory section to cover the intersection with the given range.
241 * Returns false if the intersection is empty, otherwise returns true.
243 static bool virtio_mem_intersect_memory_section(MemoryRegionSection
*s
,
244 uint64_t offset
, uint64_t size
)
246 uint64_t start
= MAX(s
->offset_within_region
, offset
);
247 uint64_t end
= MIN(s
->offset_within_region
+ int128_get64(s
->size
),
254 s
->offset_within_address_space
+= start
- s
->offset_within_region
;
255 s
->offset_within_region
= start
;
256 s
->size
= int128_make64(end
- start
);
260 typedef int (*virtio_mem_section_cb
)(MemoryRegionSection
*s
, void *arg
);
262 static int virtio_mem_for_each_plugged_section(const VirtIOMEM
*vmem
,
263 MemoryRegionSection
*s
,
265 virtio_mem_section_cb cb
)
267 unsigned long first_bit
, last_bit
;
268 uint64_t offset
, size
;
271 first_bit
= s
->offset_within_region
/ vmem
->block_size
;
272 first_bit
= find_next_bit(vmem
->bitmap
, vmem
->bitmap_size
, first_bit
);
273 while (first_bit
< vmem
->bitmap_size
) {
274 MemoryRegionSection tmp
= *s
;
276 offset
= first_bit
* vmem
->block_size
;
277 last_bit
= find_next_zero_bit(vmem
->bitmap
, vmem
->bitmap_size
,
279 size
= (last_bit
- first_bit
+ 1) * vmem
->block_size
;
281 if (!virtio_mem_intersect_memory_section(&tmp
, offset
, size
)) {
288 first_bit
= find_next_bit(vmem
->bitmap
, vmem
->bitmap_size
,
294 static int virtio_mem_for_each_unplugged_section(const VirtIOMEM
*vmem
,
295 MemoryRegionSection
*s
,
297 virtio_mem_section_cb cb
)
299 unsigned long first_bit
, last_bit
;
300 uint64_t offset
, size
;
303 first_bit
= s
->offset_within_region
/ vmem
->block_size
;
304 first_bit
= find_next_zero_bit(vmem
->bitmap
, vmem
->bitmap_size
, first_bit
);
305 while (first_bit
< vmem
->bitmap_size
) {
306 MemoryRegionSection tmp
= *s
;
308 offset
= first_bit
* vmem
->block_size
;
309 last_bit
= find_next_bit(vmem
->bitmap
, vmem
->bitmap_size
,
311 size
= (last_bit
- first_bit
+ 1) * vmem
->block_size
;
313 if (!virtio_mem_intersect_memory_section(&tmp
, offset
, size
)) {
320 first_bit
= find_next_zero_bit(vmem
->bitmap
, vmem
->bitmap_size
,
326 static int virtio_mem_notify_populate_cb(MemoryRegionSection
*s
, void *arg
)
328 RamDiscardListener
*rdl
= arg
;
330 return rdl
->notify_populate(rdl
, s
);
333 static int virtio_mem_notify_discard_cb(MemoryRegionSection
*s
, void *arg
)
335 RamDiscardListener
*rdl
= arg
;
337 rdl
->notify_discard(rdl
, s
);
341 static void virtio_mem_notify_unplug(VirtIOMEM
*vmem
, uint64_t offset
,
344 RamDiscardListener
*rdl
;
346 QLIST_FOREACH(rdl
, &vmem
->rdl_list
, next
) {
347 MemoryRegionSection tmp
= *rdl
->section
;
349 if (!virtio_mem_intersect_memory_section(&tmp
, offset
, size
)) {
352 rdl
->notify_discard(rdl
, &tmp
);
356 static int virtio_mem_notify_plug(VirtIOMEM
*vmem
, uint64_t offset
,
359 RamDiscardListener
*rdl
, *rdl2
;
362 QLIST_FOREACH(rdl
, &vmem
->rdl_list
, next
) {
363 MemoryRegionSection tmp
= *rdl
->section
;
365 if (!virtio_mem_intersect_memory_section(&tmp
, offset
, size
)) {
368 ret
= rdl
->notify_populate(rdl
, &tmp
);
375 /* Notify all already-notified listeners. */
376 QLIST_FOREACH(rdl2
, &vmem
->rdl_list
, next
) {
377 MemoryRegionSection tmp
= *rdl2
->section
;
382 if (!virtio_mem_intersect_memory_section(&tmp
, offset
, size
)) {
385 rdl2
->notify_discard(rdl2
, &tmp
);
391 static void virtio_mem_notify_unplug_all(VirtIOMEM
*vmem
)
393 RamDiscardListener
*rdl
;
399 QLIST_FOREACH(rdl
, &vmem
->rdl_list
, next
) {
400 if (rdl
->double_discard_supported
) {
401 rdl
->notify_discard(rdl
, rdl
->section
);
403 virtio_mem_for_each_plugged_section(vmem
, rdl
->section
, rdl
,
404 virtio_mem_notify_discard_cb
);
409 static bool virtio_mem_is_range_plugged(const VirtIOMEM
*vmem
,
410 uint64_t start_gpa
, uint64_t size
)
412 const unsigned long first_bit
= (start_gpa
- vmem
->addr
) / vmem
->block_size
;
413 const unsigned long last_bit
= first_bit
+ (size
/ vmem
->block_size
) - 1;
414 unsigned long found_bit
;
416 /* We fake a shorter bitmap to avoid searching too far. */
417 found_bit
= find_next_zero_bit(vmem
->bitmap
, last_bit
+ 1, first_bit
);
418 return found_bit
> last_bit
;
421 static bool virtio_mem_is_range_unplugged(const VirtIOMEM
*vmem
,
422 uint64_t start_gpa
, uint64_t size
)
424 const unsigned long first_bit
= (start_gpa
- vmem
->addr
) / vmem
->block_size
;
425 const unsigned long last_bit
= first_bit
+ (size
/ vmem
->block_size
) - 1;
426 unsigned long found_bit
;
428 /* We fake a shorter bitmap to avoid searching too far. */
429 found_bit
= find_next_bit(vmem
->bitmap
, last_bit
+ 1, first_bit
);
430 return found_bit
> last_bit
;
433 static void virtio_mem_set_range_plugged(VirtIOMEM
*vmem
, uint64_t start_gpa
,
436 const unsigned long bit
= (start_gpa
- vmem
->addr
) / vmem
->block_size
;
437 const unsigned long nbits
= size
/ vmem
->block_size
;
439 bitmap_set(vmem
->bitmap
, bit
, nbits
);
442 static void virtio_mem_set_range_unplugged(VirtIOMEM
*vmem
, uint64_t start_gpa
,
445 const unsigned long bit
= (start_gpa
- vmem
->addr
) / vmem
->block_size
;
446 const unsigned long nbits
= size
/ vmem
->block_size
;
448 bitmap_clear(vmem
->bitmap
, bit
, nbits
);
451 static void virtio_mem_send_response(VirtIOMEM
*vmem
, VirtQueueElement
*elem
,
452 struct virtio_mem_resp
*resp
)
454 VirtIODevice
*vdev
= VIRTIO_DEVICE(vmem
);
455 VirtQueue
*vq
= vmem
->vq
;
457 trace_virtio_mem_send_response(le16_to_cpu(resp
->type
));
458 iov_from_buf(elem
->in_sg
, elem
->in_num
, 0, resp
, sizeof(*resp
));
460 virtqueue_push(vq
, elem
, sizeof(*resp
));
461 virtio_notify(vdev
, vq
);
464 static void virtio_mem_send_response_simple(VirtIOMEM
*vmem
,
465 VirtQueueElement
*elem
,
468 struct virtio_mem_resp resp
= {
469 .type
= cpu_to_le16(type
),
472 virtio_mem_send_response(vmem
, elem
, &resp
);
475 static bool virtio_mem_valid_range(const VirtIOMEM
*vmem
, uint64_t gpa
,
478 if (!QEMU_IS_ALIGNED(gpa
, vmem
->block_size
)) {
481 if (gpa
+ size
< gpa
|| !size
) {
484 if (gpa
< vmem
->addr
|| gpa
>= vmem
->addr
+ vmem
->usable_region_size
) {
487 if (gpa
+ size
> vmem
->addr
+ vmem
->usable_region_size
) {
493 static void virtio_mem_activate_memslot(VirtIOMEM
*vmem
, unsigned int idx
)
495 const uint64_t memslot_offset
= idx
* vmem
->memslot_size
;
497 assert(vmem
->memslots
);
500 * Instead of enabling/disabling memslots, we add/remove them. This should
501 * make address space updates faster, because we don't have to loop over
502 * many disabled subregions.
504 if (memory_region_is_mapped(&vmem
->memslots
[idx
])) {
507 memory_region_add_subregion(vmem
->mr
, memslot_offset
, &vmem
->memslots
[idx
]);
510 static void virtio_mem_deactivate_memslot(VirtIOMEM
*vmem
, unsigned int idx
)
512 assert(vmem
->memslots
);
514 if (!memory_region_is_mapped(&vmem
->memslots
[idx
])) {
517 memory_region_del_subregion(vmem
->mr
, &vmem
->memslots
[idx
]);
520 static void virtio_mem_activate_memslots_to_plug(VirtIOMEM
*vmem
,
521 uint64_t offset
, uint64_t size
)
523 const unsigned int start_idx
= offset
/ vmem
->memslot_size
;
524 const unsigned int end_idx
= (offset
+ size
+ vmem
->memslot_size
- 1) /
528 assert(vmem
->dynamic_memslots
);
530 /* Activate all involved memslots in a single transaction. */
531 memory_region_transaction_begin();
532 for (idx
= start_idx
; idx
< end_idx
; idx
++) {
533 virtio_mem_activate_memslot(vmem
, idx
);
535 memory_region_transaction_commit();
538 static void virtio_mem_deactivate_unplugged_memslots(VirtIOMEM
*vmem
,
542 const uint64_t region_size
= memory_region_size(&vmem
->memdev
->mr
);
543 const unsigned int start_idx
= offset
/ vmem
->memslot_size
;
544 const unsigned int end_idx
= (offset
+ size
+ vmem
->memslot_size
- 1) /
548 assert(vmem
->dynamic_memslots
);
550 /* Deactivate all memslots with unplugged blocks in a single transaction. */
551 memory_region_transaction_begin();
552 for (idx
= start_idx
; idx
< end_idx
; idx
++) {
553 const uint64_t memslot_offset
= idx
* vmem
->memslot_size
;
554 uint64_t memslot_size
= vmem
->memslot_size
;
556 /* The size of the last memslot might be smaller. */
557 if (idx
== vmem
->nb_memslots
- 1) {
558 memslot_size
= region_size
- memslot_offset
;
562 * Partially covered memslots might still have some blocks plugged and
563 * have to remain active if that's the case.
565 if (offset
> memslot_offset
||
566 offset
+ size
< memslot_offset
+ memslot_size
) {
567 const uint64_t gpa
= vmem
->addr
+ memslot_offset
;
569 if (!virtio_mem_is_range_unplugged(vmem
, gpa
, memslot_size
)) {
574 virtio_mem_deactivate_memslot(vmem
, idx
);
576 memory_region_transaction_commit();
579 static int virtio_mem_set_block_state(VirtIOMEM
*vmem
, uint64_t start_gpa
,
580 uint64_t size
, bool plug
)
582 const uint64_t offset
= start_gpa
- vmem
->addr
;
583 RAMBlock
*rb
= vmem
->memdev
->mr
.ram_block
;
586 if (virtio_mem_is_busy()) {
591 if (ram_block_discard_range(rb
, offset
, size
)) {
594 virtio_mem_notify_unplug(vmem
, offset
, size
);
595 virtio_mem_set_range_unplugged(vmem
, start_gpa
, size
);
596 /* Deactivate completely unplugged memslots after updating the state. */
597 if (vmem
->dynamic_memslots
) {
598 virtio_mem_deactivate_unplugged_memslots(vmem
, offset
, size
);
603 if (vmem
->prealloc
) {
604 void *area
= memory_region_get_ram_ptr(&vmem
->memdev
->mr
) + offset
;
605 int fd
= memory_region_get_fd(&vmem
->memdev
->mr
);
606 Error
*local_err
= NULL
;
608 if (!qemu_prealloc_mem(fd
, area
, size
, 1, NULL
, false, &local_err
)) {
612 * Warn only once, we don't want to fill the log with these
616 warn_report_err(local_err
);
619 error_free(local_err
);
627 * Activate before notifying and rollback in case of any errors.
629 * When activating a yet inactive memslot, memory notifiers will get
630 * notified about the added memory region and can register with the
631 * RamDiscardManager; this will traverse all plugged blocks and skip the
632 * blocks we are plugging here. The following notification will inform
633 * registered listeners about the blocks we're plugging.
635 if (vmem
->dynamic_memslots
) {
636 virtio_mem_activate_memslots_to_plug(vmem
, offset
, size
);
638 ret
= virtio_mem_notify_plug(vmem
, offset
, size
);
639 if (ret
&& vmem
->dynamic_memslots
) {
640 virtio_mem_deactivate_unplugged_memslots(vmem
, offset
, size
);
644 /* Could be preallocation or a notifier populated memory. */
645 ram_block_discard_range(vmem
->memdev
->mr
.ram_block
, offset
, size
);
649 virtio_mem_set_range_plugged(vmem
, start_gpa
, size
);
653 static int virtio_mem_state_change_request(VirtIOMEM
*vmem
, uint64_t gpa
,
654 uint16_t nb_blocks
, bool plug
)
656 const uint64_t size
= nb_blocks
* vmem
->block_size
;
659 if (!virtio_mem_valid_range(vmem
, gpa
, size
)) {
660 return VIRTIO_MEM_RESP_ERROR
;
663 if (plug
&& (vmem
->size
+ size
> vmem
->requested_size
)) {
664 return VIRTIO_MEM_RESP_NACK
;
667 /* test if really all blocks are in the opposite state */
668 if ((plug
&& !virtio_mem_is_range_unplugged(vmem
, gpa
, size
)) ||
669 (!plug
&& !virtio_mem_is_range_plugged(vmem
, gpa
, size
))) {
670 return VIRTIO_MEM_RESP_ERROR
;
673 ret
= virtio_mem_set_block_state(vmem
, gpa
, size
, plug
);
675 return VIRTIO_MEM_RESP_BUSY
;
682 notifier_list_notify(&vmem
->size_change_notifiers
, &vmem
->size
);
683 return VIRTIO_MEM_RESP_ACK
;
686 static void virtio_mem_plug_request(VirtIOMEM
*vmem
, VirtQueueElement
*elem
,
687 struct virtio_mem_req
*req
)
689 const uint64_t gpa
= le64_to_cpu(req
->u
.plug
.addr
);
690 const uint16_t nb_blocks
= le16_to_cpu(req
->u
.plug
.nb_blocks
);
693 trace_virtio_mem_plug_request(gpa
, nb_blocks
);
694 type
= virtio_mem_state_change_request(vmem
, gpa
, nb_blocks
, true);
695 virtio_mem_send_response_simple(vmem
, elem
, type
);
698 static void virtio_mem_unplug_request(VirtIOMEM
*vmem
, VirtQueueElement
*elem
,
699 struct virtio_mem_req
*req
)
701 const uint64_t gpa
= le64_to_cpu(req
->u
.unplug
.addr
);
702 const uint16_t nb_blocks
= le16_to_cpu(req
->u
.unplug
.nb_blocks
);
705 trace_virtio_mem_unplug_request(gpa
, nb_blocks
);
706 type
= virtio_mem_state_change_request(vmem
, gpa
, nb_blocks
, false);
707 virtio_mem_send_response_simple(vmem
, elem
, type
);
710 static void virtio_mem_resize_usable_region(VirtIOMEM
*vmem
,
711 uint64_t requested_size
,
714 uint64_t newsize
= MIN(memory_region_size(&vmem
->memdev
->mr
),
715 requested_size
+ VIRTIO_MEM_USABLE_EXTENT
);
717 /* The usable region size always has to be multiples of the block size. */
718 newsize
= QEMU_ALIGN_UP(newsize
, vmem
->block_size
);
720 if (!requested_size
) {
724 if (newsize
< vmem
->usable_region_size
&& !can_shrink
) {
728 trace_virtio_mem_resized_usable_region(vmem
->usable_region_size
, newsize
);
729 vmem
->usable_region_size
= newsize
;
732 static int virtio_mem_unplug_all(VirtIOMEM
*vmem
)
734 const uint64_t region_size
= memory_region_size(&vmem
->memdev
->mr
);
735 RAMBlock
*rb
= vmem
->memdev
->mr
.ram_block
;
738 if (virtio_mem_is_busy()) {
741 if (ram_block_discard_range(rb
, 0, qemu_ram_get_used_length(rb
))) {
744 virtio_mem_notify_unplug_all(vmem
);
746 bitmap_clear(vmem
->bitmap
, 0, vmem
->bitmap_size
);
748 notifier_list_notify(&vmem
->size_change_notifiers
, &vmem
->size
);
750 /* Deactivate all memslots after updating the state. */
751 if (vmem
->dynamic_memslots
) {
752 virtio_mem_deactivate_unplugged_memslots(vmem
, 0, region_size
);
756 trace_virtio_mem_unplugged_all();
757 virtio_mem_resize_usable_region(vmem
, vmem
->requested_size
, true);
761 static void virtio_mem_unplug_all_request(VirtIOMEM
*vmem
,
762 VirtQueueElement
*elem
)
764 trace_virtio_mem_unplug_all_request();
765 if (virtio_mem_unplug_all(vmem
)) {
766 virtio_mem_send_response_simple(vmem
, elem
, VIRTIO_MEM_RESP_BUSY
);
768 virtio_mem_send_response_simple(vmem
, elem
, VIRTIO_MEM_RESP_ACK
);
772 static void virtio_mem_state_request(VirtIOMEM
*vmem
, VirtQueueElement
*elem
,
773 struct virtio_mem_req
*req
)
775 const uint16_t nb_blocks
= le16_to_cpu(req
->u
.state
.nb_blocks
);
776 const uint64_t gpa
= le64_to_cpu(req
->u
.state
.addr
);
777 const uint64_t size
= nb_blocks
* vmem
->block_size
;
778 struct virtio_mem_resp resp
= {
779 .type
= cpu_to_le16(VIRTIO_MEM_RESP_ACK
),
782 trace_virtio_mem_state_request(gpa
, nb_blocks
);
783 if (!virtio_mem_valid_range(vmem
, gpa
, size
)) {
784 virtio_mem_send_response_simple(vmem
, elem
, VIRTIO_MEM_RESP_ERROR
);
788 if (virtio_mem_is_range_plugged(vmem
, gpa
, size
)) {
789 resp
.u
.state
.state
= cpu_to_le16(VIRTIO_MEM_STATE_PLUGGED
);
790 } else if (virtio_mem_is_range_unplugged(vmem
, gpa
, size
)) {
791 resp
.u
.state
.state
= cpu_to_le16(VIRTIO_MEM_STATE_UNPLUGGED
);
793 resp
.u
.state
.state
= cpu_to_le16(VIRTIO_MEM_STATE_MIXED
);
795 trace_virtio_mem_state_response(le16_to_cpu(resp
.u
.state
.state
));
796 virtio_mem_send_response(vmem
, elem
, &resp
);
799 static void virtio_mem_handle_request(VirtIODevice
*vdev
, VirtQueue
*vq
)
801 const int len
= sizeof(struct virtio_mem_req
);
802 VirtIOMEM
*vmem
= VIRTIO_MEM(vdev
);
803 VirtQueueElement
*elem
;
804 struct virtio_mem_req req
;
808 elem
= virtqueue_pop(vq
, sizeof(VirtQueueElement
));
813 if (iov_to_buf(elem
->out_sg
, elem
->out_num
, 0, &req
, len
) < len
) {
814 virtio_error(vdev
, "virtio-mem protocol violation: invalid request"
816 virtqueue_detach_element(vq
, elem
, 0);
821 if (iov_size(elem
->in_sg
, elem
->in_num
) <
822 sizeof(struct virtio_mem_resp
)) {
823 virtio_error(vdev
, "virtio-mem protocol violation: not enough space"
824 " for response: %zu",
825 iov_size(elem
->in_sg
, elem
->in_num
));
826 virtqueue_detach_element(vq
, elem
, 0);
831 type
= le16_to_cpu(req
.type
);
833 case VIRTIO_MEM_REQ_PLUG
:
834 virtio_mem_plug_request(vmem
, elem
, &req
);
836 case VIRTIO_MEM_REQ_UNPLUG
:
837 virtio_mem_unplug_request(vmem
, elem
, &req
);
839 case VIRTIO_MEM_REQ_UNPLUG_ALL
:
840 virtio_mem_unplug_all_request(vmem
, elem
);
842 case VIRTIO_MEM_REQ_STATE
:
843 virtio_mem_state_request(vmem
, elem
, &req
);
846 virtio_error(vdev
, "virtio-mem protocol violation: unknown request"
848 virtqueue_detach_element(vq
, elem
, 0);
857 static void virtio_mem_get_config(VirtIODevice
*vdev
, uint8_t *config_data
)
859 VirtIOMEM
*vmem
= VIRTIO_MEM(vdev
);
860 struct virtio_mem_config
*config
= (void *) config_data
;
862 config
->block_size
= cpu_to_le64(vmem
->block_size
);
863 config
->node_id
= cpu_to_le16(vmem
->node
);
864 config
->requested_size
= cpu_to_le64(vmem
->requested_size
);
865 config
->plugged_size
= cpu_to_le64(vmem
->size
);
866 config
->addr
= cpu_to_le64(vmem
->addr
);
867 config
->region_size
= cpu_to_le64(memory_region_size(&vmem
->memdev
->mr
));
868 config
->usable_region_size
= cpu_to_le64(vmem
->usable_region_size
);
871 static uint64_t virtio_mem_get_features(VirtIODevice
*vdev
, uint64_t features
,
874 MachineState
*ms
= MACHINE(qdev_get_machine());
875 VirtIOMEM
*vmem
= VIRTIO_MEM(vdev
);
877 if (ms
->numa_state
) {
878 #if defined(CONFIG_ACPI)
879 virtio_add_feature(&features
, VIRTIO_MEM_F_ACPI_PXM
);
882 assert(vmem
->unplugged_inaccessible
!= ON_OFF_AUTO_AUTO
);
883 if (vmem
->unplugged_inaccessible
== ON_OFF_AUTO_ON
) {
884 virtio_add_feature(&features
, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE
);
889 static int virtio_mem_validate_features(VirtIODevice
*vdev
)
891 if (virtio_host_has_feature(vdev
, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE
) &&
892 !virtio_vdev_has_feature(vdev
, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE
)) {
898 static void virtio_mem_system_reset(void *opaque
)
900 VirtIOMEM
*vmem
= VIRTIO_MEM(opaque
);
903 * During usual resets, we will unplug all memory and shrink the usable
904 * region size. This is, however, not possible in all scenarios. Then,
905 * the guest has to deal with this manually (VIRTIO_MEM_REQ_UNPLUG_ALL).
907 virtio_mem_unplug_all(vmem
);
910 static void virtio_mem_prepare_mr(VirtIOMEM
*vmem
)
912 const uint64_t region_size
= memory_region_size(&vmem
->memdev
->mr
);
914 assert(!vmem
->mr
&& vmem
->dynamic_memslots
);
915 vmem
->mr
= g_new0(MemoryRegion
, 1);
916 memory_region_init(vmem
->mr
, OBJECT(vmem
), "virtio-mem",
918 vmem
->mr
->align
= memory_region_get_alignment(&vmem
->memdev
->mr
);
921 static void virtio_mem_prepare_memslots(VirtIOMEM
*vmem
)
923 const uint64_t region_size
= memory_region_size(&vmem
->memdev
->mr
);
926 g_assert(!vmem
->memslots
&& vmem
->nb_memslots
&& vmem
->dynamic_memslots
);
927 vmem
->memslots
= g_new0(MemoryRegion
, vmem
->nb_memslots
);
929 /* Initialize our memslots, but don't map them yet. */
930 for (idx
= 0; idx
< vmem
->nb_memslots
; idx
++) {
931 const uint64_t memslot_offset
= idx
* vmem
->memslot_size
;
932 uint64_t memslot_size
= vmem
->memslot_size
;
935 /* The size of the last memslot might be smaller. */
936 if (idx
== vmem
->nb_memslots
- 1) {
937 memslot_size
= region_size
- memslot_offset
;
940 snprintf(name
, sizeof(name
), "memslot-%u", idx
);
941 memory_region_init_alias(&vmem
->memslots
[idx
], OBJECT(vmem
), name
,
942 &vmem
->memdev
->mr
, memslot_offset
,
945 * We want to be able to atomically and efficiently activate/deactivate
946 * individual memslots without affecting adjacent memslots in memory
949 memory_region_set_unmergeable(&vmem
->memslots
[idx
], true);
953 static void virtio_mem_device_realize(DeviceState
*dev
, Error
**errp
)
955 MachineState
*ms
= MACHINE(qdev_get_machine());
956 int nb_numa_nodes
= ms
->numa_state
? ms
->numa_state
->num_nodes
: 0;
957 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
958 VirtIOMEM
*vmem
= VIRTIO_MEM(dev
);
964 error_setg(errp
, "'%s' property is not set", VIRTIO_MEM_MEMDEV_PROP
);
966 } else if (host_memory_backend_is_mapped(vmem
->memdev
)) {
967 error_setg(errp
, "'%s' property specifies a busy memdev: %s",
968 VIRTIO_MEM_MEMDEV_PROP
,
969 object_get_canonical_path_component(OBJECT(vmem
->memdev
)));
971 } else if (!memory_region_is_ram(&vmem
->memdev
->mr
) ||
972 memory_region_is_rom(&vmem
->memdev
->mr
) ||
973 !vmem
->memdev
->mr
.ram_block
) {
974 error_setg(errp
, "'%s' property specifies an unsupported memdev",
975 VIRTIO_MEM_MEMDEV_PROP
);
977 } else if (vmem
->memdev
->prealloc
) {
978 error_setg(errp
, "'%s' property specifies a memdev with preallocation"
979 " enabled: %s. Instead, specify 'prealloc=on' for the"
980 " virtio-mem device. ", VIRTIO_MEM_MEMDEV_PROP
,
981 object_get_canonical_path_component(OBJECT(vmem
->memdev
)));
985 if ((nb_numa_nodes
&& vmem
->node
>= nb_numa_nodes
) ||
986 (!nb_numa_nodes
&& vmem
->node
)) {
987 error_setg(errp
, "'%s' property has value '%" PRIu32
"', which exceeds"
988 "the number of numa nodes: %d", VIRTIO_MEM_NODE_PROP
,
989 vmem
->node
, nb_numa_nodes
? nb_numa_nodes
: 1);
994 error_setg(errp
, "Incompatible with mlock");
998 rb
= vmem
->memdev
->mr
.ram_block
;
999 page_size
= qemu_ram_pagesize(rb
);
1001 #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS)
1002 switch (vmem
->unplugged_inaccessible
) {
1003 case ON_OFF_AUTO_AUTO
:
1004 if (virtio_mem_has_shared_zeropage(rb
)) {
1005 vmem
->unplugged_inaccessible
= ON_OFF_AUTO_OFF
;
1007 vmem
->unplugged_inaccessible
= ON_OFF_AUTO_ON
;
1010 case ON_OFF_AUTO_OFF
:
1011 if (!virtio_mem_has_shared_zeropage(rb
)) {
1012 warn_report("'%s' property set to 'off' with a memdev that does"
1013 " not support the shared zeropage.",
1014 VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP
);
1020 #else /* VIRTIO_MEM_HAS_LEGACY_GUESTS */
1021 vmem
->unplugged_inaccessible
= ON_OFF_AUTO_ON
;
1022 #endif /* VIRTIO_MEM_HAS_LEGACY_GUESTS */
1024 if (vmem
->dynamic_memslots
&&
1025 vmem
->unplugged_inaccessible
!= ON_OFF_AUTO_ON
) {
1026 error_setg(errp
, "'%s' property set to 'on' requires '%s' to be 'on'",
1027 VIRTIO_MEM_DYNAMIC_MEMSLOTS_PROP
,
1028 VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP
);
1033 * If the block size wasn't configured by the user, use a sane default. This
1034 * allows using hugetlbfs backends of any page size without manual
1037 if (!vmem
->block_size
) {
1038 vmem
->block_size
= virtio_mem_default_block_size(rb
);
1041 if (vmem
->block_size
< page_size
) {
1042 error_setg(errp
, "'%s' property has to be at least the page size (0x%"
1043 PRIx64
")", VIRTIO_MEM_BLOCK_SIZE_PROP
, page_size
);
1045 } else if (vmem
->block_size
< virtio_mem_default_block_size(rb
)) {
1046 warn_report("'%s' property is smaller than the default block size (%"
1047 PRIx64
" MiB)", VIRTIO_MEM_BLOCK_SIZE_PROP
,
1048 virtio_mem_default_block_size(rb
) / MiB
);
1050 if (!QEMU_IS_ALIGNED(vmem
->requested_size
, vmem
->block_size
)) {
1051 error_setg(errp
, "'%s' property has to be multiples of '%s' (0x%" PRIx64
1052 ")", VIRTIO_MEM_REQUESTED_SIZE_PROP
,
1053 VIRTIO_MEM_BLOCK_SIZE_PROP
, vmem
->block_size
);
1055 } else if (!QEMU_IS_ALIGNED(vmem
->addr
, vmem
->block_size
)) {
1056 error_setg(errp
, "'%s' property has to be multiples of '%s' (0x%" PRIx64
1057 ")", VIRTIO_MEM_ADDR_PROP
, VIRTIO_MEM_BLOCK_SIZE_PROP
,
1060 } else if (!QEMU_IS_ALIGNED(memory_region_size(&vmem
->memdev
->mr
),
1061 vmem
->block_size
)) {
1062 error_setg(errp
, "'%s' property memdev size has to be multiples of"
1063 "'%s' (0x%" PRIx64
")", VIRTIO_MEM_MEMDEV_PROP
,
1064 VIRTIO_MEM_BLOCK_SIZE_PROP
, vmem
->block_size
);
1068 if (ram_block_coordinated_discard_require(true)) {
1069 error_setg(errp
, "Discarding RAM is disabled");
1074 * We don't know at this point whether shared RAM is migrated using
1075 * QEMU or migrated using the file content. "x-ignore-shared" will be
1076 * configured after realizing the device. So in case we have an
1077 * incoming migration, simply always skip the discard step.
1079 * Otherwise, make sure that we start with a clean slate: either the
1080 * memory backend might get reused or the shared file might still have
1083 if (!runstate_check(RUN_STATE_INMIGRATE
)) {
1084 ret
= ram_block_discard_range(rb
, 0, qemu_ram_get_used_length(rb
));
1086 error_setg_errno(errp
, -ret
, "Unexpected error discarding RAM");
1087 ram_block_coordinated_discard_require(false);
1092 virtio_mem_resize_usable_region(vmem
, vmem
->requested_size
, true);
1094 vmem
->bitmap_size
= memory_region_size(&vmem
->memdev
->mr
) /
1096 vmem
->bitmap
= bitmap_new(vmem
->bitmap_size
);
1098 virtio_init(vdev
, VIRTIO_ID_MEM
, sizeof(struct virtio_mem_config
));
1099 vmem
->vq
= virtio_add_queue(vdev
, 128, virtio_mem_handle_request
);
1102 * With "dynamic-memslots=off" (old behavior) we always map the whole
1103 * RAM memory region directly.
1105 if (vmem
->dynamic_memslots
) {
1107 virtio_mem_prepare_mr(vmem
);
1109 if (vmem
->nb_memslots
<= 1) {
1110 vmem
->nb_memslots
= 1;
1111 vmem
->memslot_size
= memory_region_size(&vmem
->memdev
->mr
);
1113 if (!vmem
->memslots
) {
1114 virtio_mem_prepare_memslots(vmem
);
1117 assert(!vmem
->mr
&& !vmem
->nb_memslots
&& !vmem
->memslots
);
1120 host_memory_backend_set_mapped(vmem
->memdev
, true);
1121 vmstate_register_ram(&vmem
->memdev
->mr
, DEVICE(vmem
));
1122 if (vmem
->early_migration
) {
1123 vmstate_register_any(VMSTATE_IF(vmem
),
1124 &vmstate_virtio_mem_device_early
, vmem
);
1126 qemu_register_reset(virtio_mem_system_reset
, vmem
);
1129 * Set ourselves as RamDiscardManager before the plug handler maps the
1130 * memory region and exposes it via an address space.
1132 memory_region_set_ram_discard_manager(&vmem
->memdev
->mr
,
1133 RAM_DISCARD_MANAGER(vmem
));
1136 static void virtio_mem_device_unrealize(DeviceState
*dev
)
1138 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
1139 VirtIOMEM
*vmem
= VIRTIO_MEM(dev
);
1142 * The unplug handler unmapped the memory region, it cannot be
1143 * found via an address space anymore. Unset ourselves.
1145 memory_region_set_ram_discard_manager(&vmem
->memdev
->mr
, NULL
);
1146 qemu_unregister_reset(virtio_mem_system_reset
, vmem
);
1147 if (vmem
->early_migration
) {
1148 vmstate_unregister(VMSTATE_IF(vmem
), &vmstate_virtio_mem_device_early
,
1151 vmstate_unregister_ram(&vmem
->memdev
->mr
, DEVICE(vmem
));
1152 host_memory_backend_set_mapped(vmem
->memdev
, false);
1153 virtio_del_queue(vdev
, 0);
1154 virtio_cleanup(vdev
);
1155 g_free(vmem
->bitmap
);
1156 ram_block_coordinated_discard_require(false);
1159 static int virtio_mem_discard_range_cb(VirtIOMEM
*vmem
, void *arg
,
1160 uint64_t offset
, uint64_t size
)
1162 RAMBlock
*rb
= vmem
->memdev
->mr
.ram_block
;
1164 return ram_block_discard_range(rb
, offset
, size
) ? -EINVAL
: 0;
1167 static int virtio_mem_restore_unplugged(VirtIOMEM
*vmem
)
1169 /* Make sure all memory is really discarded after migration. */
1170 return virtio_mem_for_each_unplugged_range(vmem
, NULL
,
1171 virtio_mem_discard_range_cb
);
1174 static int virtio_mem_activate_memslot_range_cb(VirtIOMEM
*vmem
, void *arg
,
1175 uint64_t offset
, uint64_t size
)
1177 virtio_mem_activate_memslots_to_plug(vmem
, offset
, size
);
1181 static int virtio_mem_post_load_bitmap(VirtIOMEM
*vmem
)
1183 RamDiscardListener
*rdl
;
1187 * We restored the bitmap and updated the requested size; activate all
1188 * memslots (so listeners register) before notifying about plugged blocks.
1190 if (vmem
->dynamic_memslots
) {
1192 * We don't expect any active memslots at this point to deactivate: no
1193 * memory was plugged on the migration destination.
1195 virtio_mem_for_each_plugged_range(vmem
, NULL
,
1196 virtio_mem_activate_memslot_range_cb
);
1200 * We started out with all memory discarded and our memory region is mapped
1201 * into an address space. Replay, now that we updated the bitmap.
1203 QLIST_FOREACH(rdl
, &vmem
->rdl_list
, next
) {
1204 ret
= virtio_mem_for_each_plugged_section(vmem
, rdl
->section
, rdl
,
1205 virtio_mem_notify_populate_cb
);
1213 static int virtio_mem_post_load(void *opaque
, int version_id
)
1215 VirtIOMEM
*vmem
= VIRTIO_MEM(opaque
);
1218 if (!vmem
->early_migration
) {
1219 ret
= virtio_mem_post_load_bitmap(vmem
);
1226 * If shared RAM is migrated using the file content and not using QEMU,
1227 * don't mess with preallocation and postcopy.
1229 if (migrate_ram_is_ignored(vmem
->memdev
->mr
.ram_block
)) {
1233 if (vmem
->prealloc
&& !vmem
->early_migration
) {
1234 warn_report("Proper preallocation with migration requires a newer QEMU machine");
1237 if (migration_in_incoming_postcopy()) {
1241 return virtio_mem_restore_unplugged(vmem
);
1244 static int virtio_mem_prealloc_range_cb(VirtIOMEM
*vmem
, void *arg
,
1245 uint64_t offset
, uint64_t size
)
1247 void *area
= memory_region_get_ram_ptr(&vmem
->memdev
->mr
) + offset
;
1248 int fd
= memory_region_get_fd(&vmem
->memdev
->mr
);
1249 Error
*local_err
= NULL
;
1251 if (!qemu_prealloc_mem(fd
, area
, size
, 1, NULL
, false, &local_err
)) {
1252 error_report_err(local_err
);
1258 static int virtio_mem_post_load_early(void *opaque
, int version_id
)
1260 VirtIOMEM
*vmem
= VIRTIO_MEM(opaque
);
1261 RAMBlock
*rb
= vmem
->memdev
->mr
.ram_block
;
1264 if (!vmem
->prealloc
) {
1265 goto post_load_bitmap
;
1269 * If shared RAM is migrated using the file content and not using QEMU,
1270 * don't mess with preallocation and postcopy.
1272 if (migrate_ram_is_ignored(rb
)) {
1273 goto post_load_bitmap
;
1277 * We restored the bitmap and verified that the basic properties
1278 * match on source and destination, so we can go ahead and preallocate
1279 * memory for all plugged memory blocks, before actual RAM migration starts
1280 * touching this memory.
1282 ret
= virtio_mem_for_each_plugged_range(vmem
, NULL
,
1283 virtio_mem_prealloc_range_cb
);
1289 * This is tricky: postcopy wants to start with a clean slate. On
1290 * POSTCOPY_INCOMING_ADVISE, postcopy code discards all (ordinarily
1291 * preallocated) RAM such that postcopy will work as expected later.
1293 * However, we run after POSTCOPY_INCOMING_ADVISE -- but before actual
1294 * RAM migration. So let's discard all memory again. This looks like an
1295 * expensive NOP, but actually serves a purpose: we made sure that we
1296 * were able to allocate all required backend memory once. We cannot
1297 * guarantee that the backend memory we will free will remain free
1298 * until we need it during postcopy, but at least we can catch the
1299 * obvious setup issues this way.
1301 if (migration_incoming_postcopy_advised()) {
1302 if (ram_block_discard_range(rb
, 0, qemu_ram_get_used_length(rb
))) {
1308 /* Finally, update any other state to be consistent with the new bitmap. */
1309 return virtio_mem_post_load_bitmap(vmem
);
1312 typedef struct VirtIOMEMMigSanityChecks
{
1315 uint64_t region_size
;
1316 uint64_t block_size
;
1318 } VirtIOMEMMigSanityChecks
;
1320 static int virtio_mem_mig_sanity_checks_pre_save(void *opaque
)
1322 VirtIOMEMMigSanityChecks
*tmp
= opaque
;
1323 VirtIOMEM
*vmem
= tmp
->parent
;
1325 tmp
->addr
= vmem
->addr
;
1326 tmp
->region_size
= memory_region_size(&vmem
->memdev
->mr
);
1327 tmp
->block_size
= vmem
->block_size
;
1328 tmp
->node
= vmem
->node
;
1332 static int virtio_mem_mig_sanity_checks_post_load(void *opaque
, int version_id
)
1334 VirtIOMEMMigSanityChecks
*tmp
= opaque
;
1335 VirtIOMEM
*vmem
= tmp
->parent
;
1336 const uint64_t new_region_size
= memory_region_size(&vmem
->memdev
->mr
);
1338 if (tmp
->addr
!= vmem
->addr
) {
1339 error_report("Property '%s' changed from 0x%" PRIx64
" to 0x%" PRIx64
,
1340 VIRTIO_MEM_ADDR_PROP
, tmp
->addr
, vmem
->addr
);
1344 * Note: Preparation for resizable memory regions. The maximum size
1345 * of the memory region must not change during migration.
1347 if (tmp
->region_size
!= new_region_size
) {
1348 error_report("Property '%s' size changed from 0x%" PRIx64
" to 0x%"
1349 PRIx64
, VIRTIO_MEM_MEMDEV_PROP
, tmp
->region_size
,
1353 if (tmp
->block_size
!= vmem
->block_size
) {
1354 error_report("Property '%s' changed from 0x%" PRIx64
" to 0x%" PRIx64
,
1355 VIRTIO_MEM_BLOCK_SIZE_PROP
, tmp
->block_size
,
1359 if (tmp
->node
!= vmem
->node
) {
1360 error_report("Property '%s' changed from %" PRIu32
" to %" PRIu32
,
1361 VIRTIO_MEM_NODE_PROP
, tmp
->node
, vmem
->node
);
1367 static const VMStateDescription vmstate_virtio_mem_sanity_checks
= {
1368 .name
= "virtio-mem-device/sanity-checks",
1369 .pre_save
= virtio_mem_mig_sanity_checks_pre_save
,
1370 .post_load
= virtio_mem_mig_sanity_checks_post_load
,
1371 .fields
= (const VMStateField
[]) {
1372 VMSTATE_UINT64(addr
, VirtIOMEMMigSanityChecks
),
1373 VMSTATE_UINT64(region_size
, VirtIOMEMMigSanityChecks
),
1374 VMSTATE_UINT64(block_size
, VirtIOMEMMigSanityChecks
),
1375 VMSTATE_UINT32(node
, VirtIOMEMMigSanityChecks
),
1376 VMSTATE_END_OF_LIST(),
1380 static bool virtio_mem_vmstate_field_exists(void *opaque
, int version_id
)
1382 const VirtIOMEM
*vmem
= VIRTIO_MEM(opaque
);
1384 /* With early migration, these fields were already migrated. */
1385 return !vmem
->early_migration
;
1388 static const VMStateDescription vmstate_virtio_mem_device
= {
1389 .name
= "virtio-mem-device",
1390 .minimum_version_id
= 1,
1392 .priority
= MIG_PRI_VIRTIO_MEM
,
1393 .post_load
= virtio_mem_post_load
,
1394 .fields
= (const VMStateField
[]) {
1395 VMSTATE_WITH_TMP_TEST(VirtIOMEM
, virtio_mem_vmstate_field_exists
,
1396 VirtIOMEMMigSanityChecks
,
1397 vmstate_virtio_mem_sanity_checks
),
1398 VMSTATE_UINT64(usable_region_size
, VirtIOMEM
),
1399 VMSTATE_UINT64_TEST(size
, VirtIOMEM
, virtio_mem_vmstate_field_exists
),
1400 VMSTATE_UINT64(requested_size
, VirtIOMEM
),
1401 VMSTATE_BITMAP_TEST(bitmap
, VirtIOMEM
, virtio_mem_vmstate_field_exists
,
1403 VMSTATE_END_OF_LIST()
1408 * Transfer properties that are immutable while migration is active early,
1409 * such that we have have this information around before migrating any RAM
1412 * Note that virtio_mem_is_busy() makes sure these properties can no longer
1413 * change on the migration source until migration completed.
1415 * With QEMU compat machines, we transmit these properties later, via
1416 * vmstate_virtio_mem_device instead -- see virtio_mem_vmstate_field_exists().
1418 static const VMStateDescription vmstate_virtio_mem_device_early
= {
1419 .name
= "virtio-mem-device-early",
1420 .minimum_version_id
= 1,
1422 .early_setup
= true,
1423 .post_load
= virtio_mem_post_load_early
,
1424 .fields
= (const VMStateField
[]) {
1425 VMSTATE_WITH_TMP(VirtIOMEM
, VirtIOMEMMigSanityChecks
,
1426 vmstate_virtio_mem_sanity_checks
),
1427 VMSTATE_UINT64(size
, VirtIOMEM
),
1428 VMSTATE_BITMAP(bitmap
, VirtIOMEM
, 0, bitmap_size
),
1429 VMSTATE_END_OF_LIST()
1433 static const VMStateDescription vmstate_virtio_mem
= {
1434 .name
= "virtio-mem",
1435 .minimum_version_id
= 1,
1437 .fields
= (const VMStateField
[]) {
1438 VMSTATE_VIRTIO_DEVICE
,
1439 VMSTATE_END_OF_LIST()
1443 static void virtio_mem_fill_device_info(const VirtIOMEM
*vmem
,
1444 VirtioMEMDeviceInfo
*vi
)
1446 vi
->memaddr
= vmem
->addr
;
1447 vi
->node
= vmem
->node
;
1448 vi
->requested_size
= vmem
->requested_size
;
1449 vi
->size
= vmem
->size
;
1450 vi
->max_size
= memory_region_size(&vmem
->memdev
->mr
);
1451 vi
->block_size
= vmem
->block_size
;
1452 vi
->memdev
= object_get_canonical_path(OBJECT(vmem
->memdev
));
1455 static MemoryRegion
*virtio_mem_get_memory_region(VirtIOMEM
*vmem
, Error
**errp
)
1457 if (!vmem
->memdev
) {
1458 error_setg(errp
, "'%s' property must be set", VIRTIO_MEM_MEMDEV_PROP
);
1460 } else if (vmem
->dynamic_memslots
) {
1462 virtio_mem_prepare_mr(vmem
);
1467 return &vmem
->memdev
->mr
;
1470 static void virtio_mem_decide_memslots(VirtIOMEM
*vmem
, unsigned int limit
)
1472 uint64_t region_size
, memslot_size
, min_memslot_size
;
1473 unsigned int memslots
;
1476 if (!vmem
->dynamic_memslots
) {
1480 /* We're called exactly once, before realizing the device. */
1481 assert(!vmem
->nb_memslots
);
1483 /* If realizing the device will fail, just assume a single memslot. */
1484 if (limit
<= 1 || !vmem
->memdev
|| !vmem
->memdev
->mr
.ram_block
) {
1485 vmem
->nb_memslots
= 1;
1489 rb
= vmem
->memdev
->mr
.ram_block
;
1490 region_size
= memory_region_size(&vmem
->memdev
->mr
);
1493 * Determine the default block size now, to determine the minimum memslot
1494 * size. We want the minimum slot size to be at least the device block size.
1496 if (!vmem
->block_size
) {
1497 vmem
->block_size
= virtio_mem_default_block_size(rb
);
1499 /* If realizing the device will fail, just assume a single memslot. */
1500 if (vmem
->block_size
< qemu_ram_pagesize(rb
) ||
1501 !QEMU_IS_ALIGNED(region_size
, vmem
->block_size
)) {
1502 vmem
->nb_memslots
= 1;
1507 * All memslots except the last one have a reasonable minimum size, and
1508 * and all memslot sizes are aligned to the device block size.
1510 memslot_size
= QEMU_ALIGN_UP(region_size
/ limit
, vmem
->block_size
);
1511 min_memslot_size
= MAX(vmem
->block_size
, VIRTIO_MEM_MIN_MEMSLOT_SIZE
);
1512 memslot_size
= MAX(memslot_size
, min_memslot_size
);
1514 memslots
= QEMU_ALIGN_UP(region_size
, memslot_size
) / memslot_size
;
1515 if (memslots
!= 1) {
1516 vmem
->memslot_size
= memslot_size
;
1518 vmem
->nb_memslots
= memslots
;
1521 static unsigned int virtio_mem_get_memslots(VirtIOMEM
*vmem
)
1523 if (!vmem
->dynamic_memslots
) {
1524 /* Exactly one static RAM memory region. */
1528 /* We're called after instructed to make a decision. */
1529 g_assert(vmem
->nb_memslots
);
1530 return vmem
->nb_memslots
;
1533 static void virtio_mem_add_size_change_notifier(VirtIOMEM
*vmem
,
1536 notifier_list_add(&vmem
->size_change_notifiers
, notifier
);
1539 static void virtio_mem_remove_size_change_notifier(VirtIOMEM
*vmem
,
1542 notifier_remove(notifier
);
1545 static void virtio_mem_get_size(Object
*obj
, Visitor
*v
, const char *name
,
1546 void *opaque
, Error
**errp
)
1548 const VirtIOMEM
*vmem
= VIRTIO_MEM(obj
);
1549 uint64_t value
= vmem
->size
;
1551 visit_type_size(v
, name
, &value
, errp
);
1554 static void virtio_mem_get_requested_size(Object
*obj
, Visitor
*v
,
1555 const char *name
, void *opaque
,
1558 const VirtIOMEM
*vmem
= VIRTIO_MEM(obj
);
1559 uint64_t value
= vmem
->requested_size
;
1561 visit_type_size(v
, name
, &value
, errp
);
1564 static void virtio_mem_set_requested_size(Object
*obj
, Visitor
*v
,
1565 const char *name
, void *opaque
,
1568 VirtIOMEM
*vmem
= VIRTIO_MEM(obj
);
1571 if (!visit_type_size(v
, name
, &value
, errp
)) {
1576 * The block size and memory backend are not fixed until the device was
1577 * realized. realize() will verify these properties then.
1579 if (DEVICE(obj
)->realized
) {
1580 if (!QEMU_IS_ALIGNED(value
, vmem
->block_size
)) {
1581 error_setg(errp
, "'%s' has to be multiples of '%s' (0x%" PRIx64
1582 ")", name
, VIRTIO_MEM_BLOCK_SIZE_PROP
,
1585 } else if (value
> memory_region_size(&vmem
->memdev
->mr
)) {
1586 error_setg(errp
, "'%s' cannot exceed the memory backend size"
1587 "(0x%" PRIx64
")", name
,
1588 memory_region_size(&vmem
->memdev
->mr
));
1592 if (value
!= vmem
->requested_size
) {
1593 virtio_mem_resize_usable_region(vmem
, value
, false);
1594 vmem
->requested_size
= value
;
1597 * Trigger a config update so the guest gets notified. We trigger
1598 * even if the size didn't change (especially helpful for debugging).
1600 virtio_notify_config(VIRTIO_DEVICE(vmem
));
1602 vmem
->requested_size
= value
;
1606 static void virtio_mem_get_block_size(Object
*obj
, Visitor
*v
, const char *name
,
1607 void *opaque
, Error
**errp
)
1609 const VirtIOMEM
*vmem
= VIRTIO_MEM(obj
);
1610 uint64_t value
= vmem
->block_size
;
1613 * If not configured by the user (and we're not realized yet), use the
1614 * default block size we would use with the current memory backend.
1617 if (vmem
->memdev
&& memory_region_is_ram(&vmem
->memdev
->mr
)) {
1618 value
= virtio_mem_default_block_size(vmem
->memdev
->mr
.ram_block
);
1620 value
= virtio_mem_thp_size();
1624 visit_type_size(v
, name
, &value
, errp
);
1627 static void virtio_mem_set_block_size(Object
*obj
, Visitor
*v
, const char *name
,
1628 void *opaque
, Error
**errp
)
1630 VirtIOMEM
*vmem
= VIRTIO_MEM(obj
);
1633 if (DEVICE(obj
)->realized
) {
1634 error_setg(errp
, "'%s' cannot be changed", name
);
1638 if (!visit_type_size(v
, name
, &value
, errp
)) {
1642 if (value
< VIRTIO_MEM_MIN_BLOCK_SIZE
) {
1643 error_setg(errp
, "'%s' property has to be at least 0x%" PRIx32
, name
,
1644 VIRTIO_MEM_MIN_BLOCK_SIZE
);
1646 } else if (!is_power_of_2(value
)) {
1647 error_setg(errp
, "'%s' property has to be a power of two", name
);
1650 vmem
->block_size
= value
;
1653 static void virtio_mem_instance_init(Object
*obj
)
1655 VirtIOMEM
*vmem
= VIRTIO_MEM(obj
);
1657 notifier_list_init(&vmem
->size_change_notifiers
);
1658 QLIST_INIT(&vmem
->rdl_list
);
1660 object_property_add(obj
, VIRTIO_MEM_SIZE_PROP
, "size", virtio_mem_get_size
,
1662 object_property_add(obj
, VIRTIO_MEM_REQUESTED_SIZE_PROP
, "size",
1663 virtio_mem_get_requested_size
,
1664 virtio_mem_set_requested_size
, NULL
, NULL
);
1665 object_property_add(obj
, VIRTIO_MEM_BLOCK_SIZE_PROP
, "size",
1666 virtio_mem_get_block_size
, virtio_mem_set_block_size
,
1670 static void virtio_mem_instance_finalize(Object
*obj
)
1672 VirtIOMEM
*vmem
= VIRTIO_MEM(obj
);
1675 * Note: the core already dropped the references on all memory regions
1676 * (it's passed as the owner to memory_region_init_*()) and finalized
1677 * these objects. We can simply free the memory.
1679 g_free(vmem
->memslots
);
1680 vmem
->memslots
= NULL
;
1685 static Property virtio_mem_properties
[] = {
1686 DEFINE_PROP_UINT64(VIRTIO_MEM_ADDR_PROP
, VirtIOMEM
, addr
, 0),
1687 DEFINE_PROP_UINT32(VIRTIO_MEM_NODE_PROP
, VirtIOMEM
, node
, 0),
1688 DEFINE_PROP_BOOL(VIRTIO_MEM_PREALLOC_PROP
, VirtIOMEM
, prealloc
, false),
1689 DEFINE_PROP_LINK(VIRTIO_MEM_MEMDEV_PROP
, VirtIOMEM
, memdev
,
1690 TYPE_MEMORY_BACKEND
, HostMemoryBackend
*),
1691 #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS)
1692 DEFINE_PROP_ON_OFF_AUTO(VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP
, VirtIOMEM
,
1693 unplugged_inaccessible
, ON_OFF_AUTO_ON
),
1695 DEFINE_PROP_BOOL(VIRTIO_MEM_EARLY_MIGRATION_PROP
, VirtIOMEM
,
1696 early_migration
, true),
1697 DEFINE_PROP_BOOL(VIRTIO_MEM_DYNAMIC_MEMSLOTS_PROP
, VirtIOMEM
,
1698 dynamic_memslots
, false),
1699 DEFINE_PROP_END_OF_LIST(),
1702 static uint64_t virtio_mem_rdm_get_min_granularity(const RamDiscardManager
*rdm
,
1703 const MemoryRegion
*mr
)
1705 const VirtIOMEM
*vmem
= VIRTIO_MEM(rdm
);
1707 g_assert(mr
== &vmem
->memdev
->mr
);
1708 return vmem
->block_size
;
1711 static bool virtio_mem_rdm_is_populated(const RamDiscardManager
*rdm
,
1712 const MemoryRegionSection
*s
)
1714 const VirtIOMEM
*vmem
= VIRTIO_MEM(rdm
);
1715 uint64_t start_gpa
= vmem
->addr
+ s
->offset_within_region
;
1716 uint64_t end_gpa
= start_gpa
+ int128_get64(s
->size
);
1718 g_assert(s
->mr
== &vmem
->memdev
->mr
);
1720 start_gpa
= QEMU_ALIGN_DOWN(start_gpa
, vmem
->block_size
);
1721 end_gpa
= QEMU_ALIGN_UP(end_gpa
, vmem
->block_size
);
1723 if (!virtio_mem_valid_range(vmem
, start_gpa
, end_gpa
- start_gpa
)) {
1727 return virtio_mem_is_range_plugged(vmem
, start_gpa
, end_gpa
- start_gpa
);
1730 struct VirtIOMEMReplayData
{
1735 static int virtio_mem_rdm_replay_populated_cb(MemoryRegionSection
*s
, void *arg
)
1737 struct VirtIOMEMReplayData
*data
= arg
;
1739 return ((ReplayRamPopulate
)data
->fn
)(s
, data
->opaque
);
1742 static int virtio_mem_rdm_replay_populated(const RamDiscardManager
*rdm
,
1743 MemoryRegionSection
*s
,
1744 ReplayRamPopulate replay_fn
,
1747 const VirtIOMEM
*vmem
= VIRTIO_MEM(rdm
);
1748 struct VirtIOMEMReplayData data
= {
1753 g_assert(s
->mr
== &vmem
->memdev
->mr
);
1754 return virtio_mem_for_each_plugged_section(vmem
, s
, &data
,
1755 virtio_mem_rdm_replay_populated_cb
);
1758 static int virtio_mem_rdm_replay_discarded_cb(MemoryRegionSection
*s
,
1761 struct VirtIOMEMReplayData
*data
= arg
;
1763 ((ReplayRamDiscard
)data
->fn
)(s
, data
->opaque
);
1767 static void virtio_mem_rdm_replay_discarded(const RamDiscardManager
*rdm
,
1768 MemoryRegionSection
*s
,
1769 ReplayRamDiscard replay_fn
,
1772 const VirtIOMEM
*vmem
= VIRTIO_MEM(rdm
);
1773 struct VirtIOMEMReplayData data
= {
1778 g_assert(s
->mr
== &vmem
->memdev
->mr
);
1779 virtio_mem_for_each_unplugged_section(vmem
, s
, &data
,
1780 virtio_mem_rdm_replay_discarded_cb
);
1783 static void virtio_mem_rdm_register_listener(RamDiscardManager
*rdm
,
1784 RamDiscardListener
*rdl
,
1785 MemoryRegionSection
*s
)
1787 VirtIOMEM
*vmem
= VIRTIO_MEM(rdm
);
1790 g_assert(s
->mr
== &vmem
->memdev
->mr
);
1791 rdl
->section
= memory_region_section_new_copy(s
);
1793 QLIST_INSERT_HEAD(&vmem
->rdl_list
, rdl
, next
);
1794 ret
= virtio_mem_for_each_plugged_section(vmem
, rdl
->section
, rdl
,
1795 virtio_mem_notify_populate_cb
);
1797 error_report("%s: Replaying plugged ranges failed: %s", __func__
,
1802 static void virtio_mem_rdm_unregister_listener(RamDiscardManager
*rdm
,
1803 RamDiscardListener
*rdl
)
1805 VirtIOMEM
*vmem
= VIRTIO_MEM(rdm
);
1807 g_assert(rdl
->section
->mr
== &vmem
->memdev
->mr
);
1809 if (rdl
->double_discard_supported
) {
1810 rdl
->notify_discard(rdl
, rdl
->section
);
1812 virtio_mem_for_each_plugged_section(vmem
, rdl
->section
, rdl
,
1813 virtio_mem_notify_discard_cb
);
1817 memory_region_section_free_copy(rdl
->section
);
1818 rdl
->section
= NULL
;
1819 QLIST_REMOVE(rdl
, next
);
1822 static void virtio_mem_unplug_request_check(VirtIOMEM
*vmem
, Error
**errp
)
1824 if (vmem
->unplugged_inaccessible
== ON_OFF_AUTO_OFF
) {
1826 * We could allow it with a usable region size of 0, but let's just
1827 * not care about that legacy setting.
1829 error_setg(errp
, "virtio-mem device cannot get unplugged while"
1830 " '" VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP
"' != 'on'");
1835 error_setg(errp
, "virtio-mem device cannot get unplugged while some"
1836 " of its memory is still plugged");
1839 if (vmem
->requested_size
) {
1840 error_setg(errp
, "virtio-mem device cannot get unplugged while"
1841 " '" VIRTIO_MEM_REQUESTED_SIZE_PROP
"' != '0'");
1846 static void virtio_mem_class_init(ObjectClass
*klass
, void *data
)
1848 DeviceClass
*dc
= DEVICE_CLASS(klass
);
1849 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_CLASS(klass
);
1850 VirtIOMEMClass
*vmc
= VIRTIO_MEM_CLASS(klass
);
1851 RamDiscardManagerClass
*rdmc
= RAM_DISCARD_MANAGER_CLASS(klass
);
1853 device_class_set_props(dc
, virtio_mem_properties
);
1854 dc
->vmsd
= &vmstate_virtio_mem
;
1856 set_bit(DEVICE_CATEGORY_MISC
, dc
->categories
);
1857 vdc
->realize
= virtio_mem_device_realize
;
1858 vdc
->unrealize
= virtio_mem_device_unrealize
;
1859 vdc
->get_config
= virtio_mem_get_config
;
1860 vdc
->get_features
= virtio_mem_get_features
;
1861 vdc
->validate_features
= virtio_mem_validate_features
;
1862 vdc
->vmsd
= &vmstate_virtio_mem_device
;
1864 vmc
->fill_device_info
= virtio_mem_fill_device_info
;
1865 vmc
->get_memory_region
= virtio_mem_get_memory_region
;
1866 vmc
->decide_memslots
= virtio_mem_decide_memslots
;
1867 vmc
->get_memslots
= virtio_mem_get_memslots
;
1868 vmc
->add_size_change_notifier
= virtio_mem_add_size_change_notifier
;
1869 vmc
->remove_size_change_notifier
= virtio_mem_remove_size_change_notifier
;
1870 vmc
->unplug_request_check
= virtio_mem_unplug_request_check
;
1872 rdmc
->get_min_granularity
= virtio_mem_rdm_get_min_granularity
;
1873 rdmc
->is_populated
= virtio_mem_rdm_is_populated
;
1874 rdmc
->replay_populated
= virtio_mem_rdm_replay_populated
;
1875 rdmc
->replay_discarded
= virtio_mem_rdm_replay_discarded
;
1876 rdmc
->register_listener
= virtio_mem_rdm_register_listener
;
1877 rdmc
->unregister_listener
= virtio_mem_rdm_unregister_listener
;
1880 static const TypeInfo virtio_mem_info
= {
1881 .name
= TYPE_VIRTIO_MEM
,
1882 .parent
= TYPE_VIRTIO_DEVICE
,
1883 .instance_size
= sizeof(VirtIOMEM
),
1884 .instance_init
= virtio_mem_instance_init
,
1885 .instance_finalize
= virtio_mem_instance_finalize
,
1886 .class_init
= virtio_mem_class_init
,
1887 .class_size
= sizeof(VirtIOMEMClass
),
1888 .interfaces
= (InterfaceInfo
[]) {
1889 { TYPE_RAM_DISCARD_MANAGER
},
1894 static void virtio_register_types(void)
1896 type_register_static(&virtio_mem_info
);
1899 type_init(virtio_register_types
)