4 * Copyright (C) 2020 Red Hat, Inc.
7 * David Hildenbrand <david@redhat.com>
9 * This work is licensed under the terms of the GNU GPL, version 2.
10 * See the COPYING file in the top-level directory.
13 #include "qemu/osdep.h"
14 #include "qemu-common.h"
16 #include "qemu/cutils.h"
17 #include "qemu/error-report.h"
18 #include "qemu/units.h"
19 #include "sysemu/numa.h"
20 #include "sysemu/sysemu.h"
21 #include "sysemu/reset.h"
22 #include "hw/virtio/virtio.h"
23 #include "hw/virtio/virtio-bus.h"
24 #include "hw/virtio/virtio-access.h"
25 #include "hw/virtio/virtio-mem.h"
26 #include "qapi/error.h"
27 #include "qapi/visitor.h"
28 #include "exec/ram_addr.h"
29 #include "migration/misc.h"
30 #include "hw/boards.h"
31 #include "hw/qdev-properties.h"
32 #include CONFIG_DEVICES
36 * Let's not allow blocks smaller than 1 MiB, for example, to keep the tracking
39 #define VIRTIO_MEM_MIN_BLOCK_SIZE ((uint32_t)(1 * MiB))
41 #if defined(__x86_64__) || defined(__arm__) || defined(__aarch64__) || \
42 defined(__powerpc64__)
43 #define VIRTIO_MEM_DEFAULT_THP_SIZE ((uint32_t)(2 * MiB))
45 /* fallback to 1 MiB (e.g., the THP size on s390x) */
46 #define VIRTIO_MEM_DEFAULT_THP_SIZE VIRTIO_MEM_MIN_BLOCK_SIZE
50 * We want to have a reasonable default block size such that
51 * 1. We avoid splitting THPs when unplugging memory, which degrades
53 * 2. We avoid placing THPs for plugged blocks that also cover unplugged
56 * The actual THP size might differ between Linux kernels, so we try to probe
57 * it. In the future (if we ever run into issues regarding 2.), we might want
58 * to disable THP in case we fail to properly probe the THP size, or if the
59 * block size is configured smaller than the THP size.
61 static uint32_t thp_size
;
63 #define HPAGE_PMD_SIZE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size"
64 static uint32_t virtio_mem_thp_size(void)
66 gchar
*content
= NULL
;
75 * Try to probe the actual THP size, fallback to (sane but eventually
76 * incorrect) default sizes.
78 if (g_file_get_contents(HPAGE_PMD_SIZE_PATH
, &content
, NULL
, NULL
) &&
79 !qemu_strtou64(content
, &endptr
, 0, &tmp
) &&
80 (!endptr
|| *endptr
== '\n')) {
82 * Sanity-check the value, if it's too big (e.g., aarch64 with 64k base
83 * pages) or weird, fallback to something smaller.
85 if (!tmp
|| !is_power_of_2(tmp
) || tmp
> 16 * MiB
) {
86 warn_report("Read unsupported THP size: %" PRIx64
, tmp
);
93 thp_size
= VIRTIO_MEM_DEFAULT_THP_SIZE
;
94 warn_report("Could not detect THP size, falling back to %" PRIx64
95 " MiB.", thp_size
/ MiB
);
102 static uint64_t virtio_mem_default_block_size(RAMBlock
*rb
)
104 const uint64_t page_size
= qemu_ram_pagesize(rb
);
106 /* We can have hugetlbfs with a page size smaller than the THP size. */
107 if (page_size
== qemu_real_host_page_size
) {
108 return MAX(page_size
, virtio_mem_thp_size());
110 return MAX(page_size
, VIRTIO_MEM_MIN_BLOCK_SIZE
);
114 * Size the usable region bigger than the requested size if possible. Esp.
115 * Linux guests will only add (aligned) memory blocks in case they fully
116 * fit into the usable region, but plug+online only a subset of the pages.
117 * The memory block size corresponds mostly to the section size.
119 * This allows e.g., to add 20MB with a section size of 128MB on x86_64, and
120 * a section size of 1GB on arm64 (as long as the start address is properly
121 * aligned, similar to ordinary DIMMs).
123 * We can change this at any time and maybe even make it configurable if
124 * necessary (as the section size can change). But it's more likely that the
125 * section size will rather get smaller and not bigger over time.
127 #if defined(TARGET_X86_64) || defined(TARGET_I386)
128 #define VIRTIO_MEM_USABLE_EXTENT (2 * (128 * MiB))
130 #error VIRTIO_MEM_USABLE_EXTENT not defined
133 static bool virtio_mem_is_busy(void)
136 * Postcopy cannot handle concurrent discards and we don't want to migrate
137 * pages on-demand with stale content when plugging new blocks.
139 * For precopy, we don't want unplugged blocks in our migration stream, and
140 * when plugging new blocks, the page content might differ between source
141 * and destination (observable by the guest when not initializing pages
142 * after plugging them) until we're running on the destination (as we didn't
143 * migrate these blocks when they were unplugged).
145 return migration_in_incoming_postcopy() || !migration_is_idle();
148 typedef int (*virtio_mem_range_cb
)(const VirtIOMEM
*vmem
, void *arg
,
149 uint64_t offset
, uint64_t size
);
151 static int virtio_mem_for_each_unplugged_range(const VirtIOMEM
*vmem
, void *arg
,
152 virtio_mem_range_cb cb
)
154 unsigned long first_zero_bit
, last_zero_bit
;
155 uint64_t offset
, size
;
158 first_zero_bit
= find_first_zero_bit(vmem
->bitmap
, vmem
->bitmap_size
);
159 while (first_zero_bit
< vmem
->bitmap_size
) {
160 offset
= first_zero_bit
* vmem
->block_size
;
161 last_zero_bit
= find_next_bit(vmem
->bitmap
, vmem
->bitmap_size
,
162 first_zero_bit
+ 1) - 1;
163 size
= (last_zero_bit
- first_zero_bit
+ 1) * vmem
->block_size
;
165 ret
= cb(vmem
, arg
, offset
, size
);
169 first_zero_bit
= find_next_zero_bit(vmem
->bitmap
, vmem
->bitmap_size
,
176 * Adjust the memory section to cover the intersection with the given range.
178 * Returns false if the intersection is empty, otherwise returns true.
180 static bool virito_mem_intersect_memory_section(MemoryRegionSection
*s
,
181 uint64_t offset
, uint64_t size
)
183 uint64_t start
= MAX(s
->offset_within_region
, offset
);
184 uint64_t end
= MIN(s
->offset_within_region
+ int128_get64(s
->size
),
191 s
->offset_within_address_space
+= start
- s
->offset_within_region
;
192 s
->offset_within_region
= start
;
193 s
->size
= int128_make64(end
- start
);
197 typedef int (*virtio_mem_section_cb
)(MemoryRegionSection
*s
, void *arg
);
199 static int virtio_mem_for_each_plugged_section(const VirtIOMEM
*vmem
,
200 MemoryRegionSection
*s
,
202 virtio_mem_section_cb cb
)
204 unsigned long first_bit
, last_bit
;
205 uint64_t offset
, size
;
208 first_bit
= s
->offset_within_region
/ vmem
->bitmap_size
;
209 first_bit
= find_next_bit(vmem
->bitmap
, vmem
->bitmap_size
, first_bit
);
210 while (first_bit
< vmem
->bitmap_size
) {
211 MemoryRegionSection tmp
= *s
;
213 offset
= first_bit
* vmem
->block_size
;
214 last_bit
= find_next_zero_bit(vmem
->bitmap
, vmem
->bitmap_size
,
216 size
= (last_bit
- first_bit
+ 1) * vmem
->block_size
;
218 if (!virito_mem_intersect_memory_section(&tmp
, offset
, size
)) {
225 first_bit
= find_next_bit(vmem
->bitmap
, vmem
->bitmap_size
,
231 static int virtio_mem_for_each_unplugged_section(const VirtIOMEM
*vmem
,
232 MemoryRegionSection
*s
,
234 virtio_mem_section_cb cb
)
236 unsigned long first_bit
, last_bit
;
237 uint64_t offset
, size
;
240 first_bit
= s
->offset_within_region
/ vmem
->bitmap_size
;
241 first_bit
= find_next_zero_bit(vmem
->bitmap
, vmem
->bitmap_size
, first_bit
);
242 while (first_bit
< vmem
->bitmap_size
) {
243 MemoryRegionSection tmp
= *s
;
245 offset
= first_bit
* vmem
->block_size
;
246 last_bit
= find_next_bit(vmem
->bitmap
, vmem
->bitmap_size
,
248 size
= (last_bit
- first_bit
+ 1) * vmem
->block_size
;
250 if (!virito_mem_intersect_memory_section(&tmp
, offset
, size
)) {
257 first_bit
= find_next_zero_bit(vmem
->bitmap
, vmem
->bitmap_size
,
263 static int virtio_mem_notify_populate_cb(MemoryRegionSection
*s
, void *arg
)
265 RamDiscardListener
*rdl
= arg
;
267 return rdl
->notify_populate(rdl
, s
);
270 static int virtio_mem_notify_discard_cb(MemoryRegionSection
*s
, void *arg
)
272 RamDiscardListener
*rdl
= arg
;
274 rdl
->notify_discard(rdl
, s
);
278 static void virtio_mem_notify_unplug(VirtIOMEM
*vmem
, uint64_t offset
,
281 RamDiscardListener
*rdl
;
283 QLIST_FOREACH(rdl
, &vmem
->rdl_list
, next
) {
284 MemoryRegionSection tmp
= *rdl
->section
;
286 if (!virito_mem_intersect_memory_section(&tmp
, offset
, size
)) {
289 rdl
->notify_discard(rdl
, &tmp
);
293 static int virtio_mem_notify_plug(VirtIOMEM
*vmem
, uint64_t offset
,
296 RamDiscardListener
*rdl
, *rdl2
;
299 QLIST_FOREACH(rdl
, &vmem
->rdl_list
, next
) {
300 MemoryRegionSection tmp
= *rdl
->section
;
302 if (!virito_mem_intersect_memory_section(&tmp
, offset
, size
)) {
305 ret
= rdl
->notify_populate(rdl
, &tmp
);
312 /* Notify all already-notified listeners. */
313 QLIST_FOREACH(rdl2
, &vmem
->rdl_list
, next
) {
314 MemoryRegionSection tmp
= *rdl
->section
;
319 if (!virito_mem_intersect_memory_section(&tmp
, offset
, size
)) {
322 rdl2
->notify_discard(rdl2
, &tmp
);
328 static void virtio_mem_notify_unplug_all(VirtIOMEM
*vmem
)
330 RamDiscardListener
*rdl
;
336 QLIST_FOREACH(rdl
, &vmem
->rdl_list
, next
) {
337 if (rdl
->double_discard_supported
) {
338 rdl
->notify_discard(rdl
, rdl
->section
);
340 virtio_mem_for_each_plugged_section(vmem
, rdl
->section
, rdl
,
341 virtio_mem_notify_discard_cb
);
346 static bool virtio_mem_test_bitmap(const VirtIOMEM
*vmem
, uint64_t start_gpa
,
347 uint64_t size
, bool plugged
)
349 const unsigned long first_bit
= (start_gpa
- vmem
->addr
) / vmem
->block_size
;
350 const unsigned long last_bit
= first_bit
+ (size
/ vmem
->block_size
) - 1;
351 unsigned long found_bit
;
353 /* We fake a shorter bitmap to avoid searching too far. */
355 found_bit
= find_next_zero_bit(vmem
->bitmap
, last_bit
+ 1, first_bit
);
357 found_bit
= find_next_bit(vmem
->bitmap
, last_bit
+ 1, first_bit
);
359 return found_bit
> last_bit
;
362 static void virtio_mem_set_bitmap(VirtIOMEM
*vmem
, uint64_t start_gpa
,
363 uint64_t size
, bool plugged
)
365 const unsigned long bit
= (start_gpa
- vmem
->addr
) / vmem
->block_size
;
366 const unsigned long nbits
= size
/ vmem
->block_size
;
369 bitmap_set(vmem
->bitmap
, bit
, nbits
);
371 bitmap_clear(vmem
->bitmap
, bit
, nbits
);
375 static void virtio_mem_send_response(VirtIOMEM
*vmem
, VirtQueueElement
*elem
,
376 struct virtio_mem_resp
*resp
)
378 VirtIODevice
*vdev
= VIRTIO_DEVICE(vmem
);
379 VirtQueue
*vq
= vmem
->vq
;
381 trace_virtio_mem_send_response(le16_to_cpu(resp
->type
));
382 iov_from_buf(elem
->in_sg
, elem
->in_num
, 0, resp
, sizeof(*resp
));
384 virtqueue_push(vq
, elem
, sizeof(*resp
));
385 virtio_notify(vdev
, vq
);
388 static void virtio_mem_send_response_simple(VirtIOMEM
*vmem
,
389 VirtQueueElement
*elem
,
392 struct virtio_mem_resp resp
= {
393 .type
= cpu_to_le16(type
),
396 virtio_mem_send_response(vmem
, elem
, &resp
);
399 static bool virtio_mem_valid_range(const VirtIOMEM
*vmem
, uint64_t gpa
,
402 if (!QEMU_IS_ALIGNED(gpa
, vmem
->block_size
)) {
405 if (gpa
+ size
< gpa
|| !size
) {
408 if (gpa
< vmem
->addr
|| gpa
>= vmem
->addr
+ vmem
->usable_region_size
) {
411 if (gpa
+ size
> vmem
->addr
+ vmem
->usable_region_size
) {
417 static int virtio_mem_set_block_state(VirtIOMEM
*vmem
, uint64_t start_gpa
,
418 uint64_t size
, bool plug
)
420 const uint64_t offset
= start_gpa
- vmem
->addr
;
421 RAMBlock
*rb
= vmem
->memdev
->mr
.ram_block
;
423 if (virtio_mem_is_busy()) {
428 if (ram_block_discard_range(rb
, offset
, size
)) {
431 virtio_mem_notify_unplug(vmem
, offset
, size
);
432 } else if (virtio_mem_notify_plug(vmem
, offset
, size
)) {
433 /* Could be a mapping attempt resulted in memory getting populated. */
434 ram_block_discard_range(vmem
->memdev
->mr
.ram_block
, offset
, size
);
437 virtio_mem_set_bitmap(vmem
, start_gpa
, size
, plug
);
441 static int virtio_mem_state_change_request(VirtIOMEM
*vmem
, uint64_t gpa
,
442 uint16_t nb_blocks
, bool plug
)
444 const uint64_t size
= nb_blocks
* vmem
->block_size
;
447 if (!virtio_mem_valid_range(vmem
, gpa
, size
)) {
448 return VIRTIO_MEM_RESP_ERROR
;
451 if (plug
&& (vmem
->size
+ size
> vmem
->requested_size
)) {
452 return VIRTIO_MEM_RESP_NACK
;
455 /* test if really all blocks are in the opposite state */
456 if (!virtio_mem_test_bitmap(vmem
, gpa
, size
, !plug
)) {
457 return VIRTIO_MEM_RESP_ERROR
;
460 ret
= virtio_mem_set_block_state(vmem
, gpa
, size
, plug
);
462 return VIRTIO_MEM_RESP_BUSY
;
469 notifier_list_notify(&vmem
->size_change_notifiers
, &vmem
->size
);
470 return VIRTIO_MEM_RESP_ACK
;
473 static void virtio_mem_plug_request(VirtIOMEM
*vmem
, VirtQueueElement
*elem
,
474 struct virtio_mem_req
*req
)
476 const uint64_t gpa
= le64_to_cpu(req
->u
.plug
.addr
);
477 const uint16_t nb_blocks
= le16_to_cpu(req
->u
.plug
.nb_blocks
);
480 trace_virtio_mem_plug_request(gpa
, nb_blocks
);
481 type
= virtio_mem_state_change_request(vmem
, gpa
, nb_blocks
, true);
482 virtio_mem_send_response_simple(vmem
, elem
, type
);
485 static void virtio_mem_unplug_request(VirtIOMEM
*vmem
, VirtQueueElement
*elem
,
486 struct virtio_mem_req
*req
)
488 const uint64_t gpa
= le64_to_cpu(req
->u
.unplug
.addr
);
489 const uint16_t nb_blocks
= le16_to_cpu(req
->u
.unplug
.nb_blocks
);
492 trace_virtio_mem_unplug_request(gpa
, nb_blocks
);
493 type
= virtio_mem_state_change_request(vmem
, gpa
, nb_blocks
, false);
494 virtio_mem_send_response_simple(vmem
, elem
, type
);
497 static void virtio_mem_resize_usable_region(VirtIOMEM
*vmem
,
498 uint64_t requested_size
,
501 uint64_t newsize
= MIN(memory_region_size(&vmem
->memdev
->mr
),
502 requested_size
+ VIRTIO_MEM_USABLE_EXTENT
);
504 /* The usable region size always has to be multiples of the block size. */
505 newsize
= QEMU_ALIGN_UP(newsize
, vmem
->block_size
);
507 if (!requested_size
) {
511 if (newsize
< vmem
->usable_region_size
&& !can_shrink
) {
515 trace_virtio_mem_resized_usable_region(vmem
->usable_region_size
, newsize
);
516 vmem
->usable_region_size
= newsize
;
519 static int virtio_mem_unplug_all(VirtIOMEM
*vmem
)
521 RAMBlock
*rb
= vmem
->memdev
->mr
.ram_block
;
523 if (virtio_mem_is_busy()) {
527 if (ram_block_discard_range(rb
, 0, qemu_ram_get_used_length(rb
))) {
530 virtio_mem_notify_unplug_all(vmem
);
532 bitmap_clear(vmem
->bitmap
, 0, vmem
->bitmap_size
);
535 notifier_list_notify(&vmem
->size_change_notifiers
, &vmem
->size
);
537 trace_virtio_mem_unplugged_all();
538 virtio_mem_resize_usable_region(vmem
, vmem
->requested_size
, true);
542 static void virtio_mem_unplug_all_request(VirtIOMEM
*vmem
,
543 VirtQueueElement
*elem
)
545 trace_virtio_mem_unplug_all_request();
546 if (virtio_mem_unplug_all(vmem
)) {
547 virtio_mem_send_response_simple(vmem
, elem
, VIRTIO_MEM_RESP_BUSY
);
549 virtio_mem_send_response_simple(vmem
, elem
, VIRTIO_MEM_RESP_ACK
);
553 static void virtio_mem_state_request(VirtIOMEM
*vmem
, VirtQueueElement
*elem
,
554 struct virtio_mem_req
*req
)
556 const uint16_t nb_blocks
= le16_to_cpu(req
->u
.state
.nb_blocks
);
557 const uint64_t gpa
= le64_to_cpu(req
->u
.state
.addr
);
558 const uint64_t size
= nb_blocks
* vmem
->block_size
;
559 struct virtio_mem_resp resp
= {
560 .type
= cpu_to_le16(VIRTIO_MEM_RESP_ACK
),
563 trace_virtio_mem_state_request(gpa
, nb_blocks
);
564 if (!virtio_mem_valid_range(vmem
, gpa
, size
)) {
565 virtio_mem_send_response_simple(vmem
, elem
, VIRTIO_MEM_RESP_ERROR
);
569 if (virtio_mem_test_bitmap(vmem
, gpa
, size
, true)) {
570 resp
.u
.state
.state
= cpu_to_le16(VIRTIO_MEM_STATE_PLUGGED
);
571 } else if (virtio_mem_test_bitmap(vmem
, gpa
, size
, false)) {
572 resp
.u
.state
.state
= cpu_to_le16(VIRTIO_MEM_STATE_UNPLUGGED
);
574 resp
.u
.state
.state
= cpu_to_le16(VIRTIO_MEM_STATE_MIXED
);
576 trace_virtio_mem_state_response(le16_to_cpu(resp
.u
.state
.state
));
577 virtio_mem_send_response(vmem
, elem
, &resp
);
580 static void virtio_mem_handle_request(VirtIODevice
*vdev
, VirtQueue
*vq
)
582 const int len
= sizeof(struct virtio_mem_req
);
583 VirtIOMEM
*vmem
= VIRTIO_MEM(vdev
);
584 VirtQueueElement
*elem
;
585 struct virtio_mem_req req
;
589 elem
= virtqueue_pop(vq
, sizeof(VirtQueueElement
));
594 if (iov_to_buf(elem
->out_sg
, elem
->out_num
, 0, &req
, len
) < len
) {
595 virtio_error(vdev
, "virtio-mem protocol violation: invalid request"
597 virtqueue_detach_element(vq
, elem
, 0);
602 if (iov_size(elem
->in_sg
, elem
->in_num
) <
603 sizeof(struct virtio_mem_resp
)) {
604 virtio_error(vdev
, "virtio-mem protocol violation: not enough space"
605 " for response: %zu",
606 iov_size(elem
->in_sg
, elem
->in_num
));
607 virtqueue_detach_element(vq
, elem
, 0);
612 type
= le16_to_cpu(req
.type
);
614 case VIRTIO_MEM_REQ_PLUG
:
615 virtio_mem_plug_request(vmem
, elem
, &req
);
617 case VIRTIO_MEM_REQ_UNPLUG
:
618 virtio_mem_unplug_request(vmem
, elem
, &req
);
620 case VIRTIO_MEM_REQ_UNPLUG_ALL
:
621 virtio_mem_unplug_all_request(vmem
, elem
);
623 case VIRTIO_MEM_REQ_STATE
:
624 virtio_mem_state_request(vmem
, elem
, &req
);
627 virtio_error(vdev
, "virtio-mem protocol violation: unknown request"
629 virtqueue_detach_element(vq
, elem
, 0);
638 static void virtio_mem_get_config(VirtIODevice
*vdev
, uint8_t *config_data
)
640 VirtIOMEM
*vmem
= VIRTIO_MEM(vdev
);
641 struct virtio_mem_config
*config
= (void *) config_data
;
643 config
->block_size
= cpu_to_le64(vmem
->block_size
);
644 config
->node_id
= cpu_to_le16(vmem
->node
);
645 config
->requested_size
= cpu_to_le64(vmem
->requested_size
);
646 config
->plugged_size
= cpu_to_le64(vmem
->size
);
647 config
->addr
= cpu_to_le64(vmem
->addr
);
648 config
->region_size
= cpu_to_le64(memory_region_size(&vmem
->memdev
->mr
));
649 config
->usable_region_size
= cpu_to_le64(vmem
->usable_region_size
);
652 static uint64_t virtio_mem_get_features(VirtIODevice
*vdev
, uint64_t features
,
655 MachineState
*ms
= MACHINE(qdev_get_machine());
657 if (ms
->numa_state
) {
658 #if defined(CONFIG_ACPI)
659 virtio_add_feature(&features
, VIRTIO_MEM_F_ACPI_PXM
);
665 static void virtio_mem_system_reset(void *opaque
)
667 VirtIOMEM
*vmem
= VIRTIO_MEM(opaque
);
670 * During usual resets, we will unplug all memory and shrink the usable
671 * region size. This is, however, not possible in all scenarios. Then,
672 * the guest has to deal with this manually (VIRTIO_MEM_REQ_UNPLUG_ALL).
674 virtio_mem_unplug_all(vmem
);
677 static void virtio_mem_device_realize(DeviceState
*dev
, Error
**errp
)
679 MachineState
*ms
= MACHINE(qdev_get_machine());
680 int nb_numa_nodes
= ms
->numa_state
? ms
->numa_state
->num_nodes
: 0;
681 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
682 VirtIOMEM
*vmem
= VIRTIO_MEM(dev
);
688 error_setg(errp
, "'%s' property is not set", VIRTIO_MEM_MEMDEV_PROP
);
690 } else if (host_memory_backend_is_mapped(vmem
->memdev
)) {
691 error_setg(errp
, "'%s' property specifies a busy memdev: %s",
692 VIRTIO_MEM_MEMDEV_PROP
,
693 object_get_canonical_path_component(OBJECT(vmem
->memdev
)));
695 } else if (!memory_region_is_ram(&vmem
->memdev
->mr
) ||
696 memory_region_is_rom(&vmem
->memdev
->mr
) ||
697 !vmem
->memdev
->mr
.ram_block
) {
698 error_setg(errp
, "'%s' property specifies an unsupported memdev",
699 VIRTIO_MEM_MEMDEV_PROP
);
703 if ((nb_numa_nodes
&& vmem
->node
>= nb_numa_nodes
) ||
704 (!nb_numa_nodes
&& vmem
->node
)) {
705 error_setg(errp
, "'%s' property has value '%" PRIu32
"', which exceeds"
706 "the number of numa nodes: %d", VIRTIO_MEM_NODE_PROP
,
707 vmem
->node
, nb_numa_nodes
? nb_numa_nodes
: 1);
712 error_setg(errp
, "Incompatible with mlock");
716 rb
= vmem
->memdev
->mr
.ram_block
;
717 page_size
= qemu_ram_pagesize(rb
);
720 * If the block size wasn't configured by the user, use a sane default. This
721 * allows using hugetlbfs backends of any page size without manual
724 if (!vmem
->block_size
) {
725 vmem
->block_size
= virtio_mem_default_block_size(rb
);
728 if (vmem
->block_size
< page_size
) {
729 error_setg(errp
, "'%s' property has to be at least the page size (0x%"
730 PRIx64
")", VIRTIO_MEM_BLOCK_SIZE_PROP
, page_size
);
732 } else if (vmem
->block_size
< virtio_mem_default_block_size(rb
)) {
733 warn_report("'%s' property is smaller than the default block size (%"
734 PRIx64
" MiB)", VIRTIO_MEM_BLOCK_SIZE_PROP
,
735 virtio_mem_default_block_size(rb
) / MiB
);
736 } else if (!QEMU_IS_ALIGNED(vmem
->requested_size
, vmem
->block_size
)) {
737 error_setg(errp
, "'%s' property has to be multiples of '%s' (0x%" PRIx64
738 ")", VIRTIO_MEM_REQUESTED_SIZE_PROP
,
739 VIRTIO_MEM_BLOCK_SIZE_PROP
, vmem
->block_size
);
741 } else if (!QEMU_IS_ALIGNED(vmem
->addr
, vmem
->block_size
)) {
742 error_setg(errp
, "'%s' property has to be multiples of '%s' (0x%" PRIx64
743 ")", VIRTIO_MEM_ADDR_PROP
, VIRTIO_MEM_BLOCK_SIZE_PROP
,
746 } else if (!QEMU_IS_ALIGNED(memory_region_size(&vmem
->memdev
->mr
),
748 error_setg(errp
, "'%s' property memdev size has to be multiples of"
749 "'%s' (0x%" PRIx64
")", VIRTIO_MEM_MEMDEV_PROP
,
750 VIRTIO_MEM_BLOCK_SIZE_PROP
, vmem
->block_size
);
754 if (ram_block_coordinated_discard_require(true)) {
755 error_setg(errp
, "Discarding RAM is disabled");
759 ret
= ram_block_discard_range(rb
, 0, qemu_ram_get_used_length(rb
));
761 error_setg_errno(errp
, -ret
, "Unexpected error discarding RAM");
762 ram_block_coordinated_discard_require(false);
766 virtio_mem_resize_usable_region(vmem
, vmem
->requested_size
, true);
768 vmem
->bitmap_size
= memory_region_size(&vmem
->memdev
->mr
) /
770 vmem
->bitmap
= bitmap_new(vmem
->bitmap_size
);
772 virtio_init(vdev
, TYPE_VIRTIO_MEM
, VIRTIO_ID_MEM
,
773 sizeof(struct virtio_mem_config
));
774 vmem
->vq
= virtio_add_queue(vdev
, 128, virtio_mem_handle_request
);
776 host_memory_backend_set_mapped(vmem
->memdev
, true);
777 vmstate_register_ram(&vmem
->memdev
->mr
, DEVICE(vmem
));
778 qemu_register_reset(virtio_mem_system_reset
, vmem
);
779 precopy_add_notifier(&vmem
->precopy_notifier
);
782 * Set ourselves as RamDiscardManager before the plug handler maps the
783 * memory region and exposes it via an address space.
785 memory_region_set_ram_discard_manager(&vmem
->memdev
->mr
,
786 RAM_DISCARD_MANAGER(vmem
));
789 static void virtio_mem_device_unrealize(DeviceState
*dev
)
791 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
792 VirtIOMEM
*vmem
= VIRTIO_MEM(dev
);
795 * The unplug handler unmapped the memory region, it cannot be
796 * found via an address space anymore. Unset ourselves.
798 memory_region_set_ram_discard_manager(&vmem
->memdev
->mr
, NULL
);
799 precopy_remove_notifier(&vmem
->precopy_notifier
);
800 qemu_unregister_reset(virtio_mem_system_reset
, vmem
);
801 vmstate_unregister_ram(&vmem
->memdev
->mr
, DEVICE(vmem
));
802 host_memory_backend_set_mapped(vmem
->memdev
, false);
803 virtio_del_queue(vdev
, 0);
804 virtio_cleanup(vdev
);
805 g_free(vmem
->bitmap
);
806 ram_block_coordinated_discard_require(false);
809 static int virtio_mem_discard_range_cb(const VirtIOMEM
*vmem
, void *arg
,
810 uint64_t offset
, uint64_t size
)
812 RAMBlock
*rb
= vmem
->memdev
->mr
.ram_block
;
814 return ram_block_discard_range(rb
, offset
, size
) ? -EINVAL
: 0;
817 static int virtio_mem_restore_unplugged(VirtIOMEM
*vmem
)
819 /* Make sure all memory is really discarded after migration. */
820 return virtio_mem_for_each_unplugged_range(vmem
, NULL
,
821 virtio_mem_discard_range_cb
);
824 static int virtio_mem_post_load(void *opaque
, int version_id
)
826 VirtIOMEM
*vmem
= VIRTIO_MEM(opaque
);
827 RamDiscardListener
*rdl
;
831 * We started out with all memory discarded and our memory region is mapped
832 * into an address space. Replay, now that we updated the bitmap.
834 QLIST_FOREACH(rdl
, &vmem
->rdl_list
, next
) {
835 ret
= virtio_mem_for_each_plugged_section(vmem
, rdl
->section
, rdl
,
836 virtio_mem_notify_populate_cb
);
842 if (migration_in_incoming_postcopy()) {
846 return virtio_mem_restore_unplugged(vmem
);
849 typedef struct VirtIOMEMMigSanityChecks
{
852 uint64_t region_size
;
855 } VirtIOMEMMigSanityChecks
;
857 static int virtio_mem_mig_sanity_checks_pre_save(void *opaque
)
859 VirtIOMEMMigSanityChecks
*tmp
= opaque
;
860 VirtIOMEM
*vmem
= tmp
->parent
;
862 tmp
->addr
= vmem
->addr
;
863 tmp
->region_size
= memory_region_size(&vmem
->memdev
->mr
);
864 tmp
->block_size
= vmem
->block_size
;
865 tmp
->node
= vmem
->node
;
869 static int virtio_mem_mig_sanity_checks_post_load(void *opaque
, int version_id
)
871 VirtIOMEMMigSanityChecks
*tmp
= opaque
;
872 VirtIOMEM
*vmem
= tmp
->parent
;
873 const uint64_t new_region_size
= memory_region_size(&vmem
->memdev
->mr
);
875 if (tmp
->addr
!= vmem
->addr
) {
876 error_report("Property '%s' changed from 0x%" PRIx64
" to 0x%" PRIx64
,
877 VIRTIO_MEM_ADDR_PROP
, tmp
->addr
, vmem
->addr
);
881 * Note: Preparation for resizeable memory regions. The maximum size
882 * of the memory region must not change during migration.
884 if (tmp
->region_size
!= new_region_size
) {
885 error_report("Property '%s' size changed from 0x%" PRIx64
" to 0x%"
886 PRIx64
, VIRTIO_MEM_MEMDEV_PROP
, tmp
->region_size
,
890 if (tmp
->block_size
!= vmem
->block_size
) {
891 error_report("Property '%s' changed from 0x%" PRIx64
" to 0x%" PRIx64
,
892 VIRTIO_MEM_BLOCK_SIZE_PROP
, tmp
->block_size
,
896 if (tmp
->node
!= vmem
->node
) {
897 error_report("Property '%s' changed from %" PRIu32
" to %" PRIu32
,
898 VIRTIO_MEM_NODE_PROP
, tmp
->node
, vmem
->node
);
904 static const VMStateDescription vmstate_virtio_mem_sanity_checks
= {
905 .name
= "virtio-mem-device/sanity-checks",
906 .pre_save
= virtio_mem_mig_sanity_checks_pre_save
,
907 .post_load
= virtio_mem_mig_sanity_checks_post_load
,
908 .fields
= (VMStateField
[]) {
909 VMSTATE_UINT64(addr
, VirtIOMEMMigSanityChecks
),
910 VMSTATE_UINT64(region_size
, VirtIOMEMMigSanityChecks
),
911 VMSTATE_UINT64(block_size
, VirtIOMEMMigSanityChecks
),
912 VMSTATE_UINT32(node
, VirtIOMEMMigSanityChecks
),
913 VMSTATE_END_OF_LIST(),
917 static const VMStateDescription vmstate_virtio_mem_device
= {
918 .name
= "virtio-mem-device",
919 .minimum_version_id
= 1,
921 .priority
= MIG_PRI_VIRTIO_MEM
,
922 .post_load
= virtio_mem_post_load
,
923 .fields
= (VMStateField
[]) {
924 VMSTATE_WITH_TMP(VirtIOMEM
, VirtIOMEMMigSanityChecks
,
925 vmstate_virtio_mem_sanity_checks
),
926 VMSTATE_UINT64(usable_region_size
, VirtIOMEM
),
927 VMSTATE_UINT64(size
, VirtIOMEM
),
928 VMSTATE_UINT64(requested_size
, VirtIOMEM
),
929 VMSTATE_BITMAP(bitmap
, VirtIOMEM
, 0, bitmap_size
),
930 VMSTATE_END_OF_LIST()
934 static const VMStateDescription vmstate_virtio_mem
= {
935 .name
= "virtio-mem",
936 .minimum_version_id
= 1,
938 .fields
= (VMStateField
[]) {
939 VMSTATE_VIRTIO_DEVICE
,
940 VMSTATE_END_OF_LIST()
944 static void virtio_mem_fill_device_info(const VirtIOMEM
*vmem
,
945 VirtioMEMDeviceInfo
*vi
)
947 vi
->memaddr
= vmem
->addr
;
948 vi
->node
= vmem
->node
;
949 vi
->requested_size
= vmem
->requested_size
;
950 vi
->size
= vmem
->size
;
951 vi
->max_size
= memory_region_size(&vmem
->memdev
->mr
);
952 vi
->block_size
= vmem
->block_size
;
953 vi
->memdev
= object_get_canonical_path(OBJECT(vmem
->memdev
));
956 static MemoryRegion
*virtio_mem_get_memory_region(VirtIOMEM
*vmem
, Error
**errp
)
959 error_setg(errp
, "'%s' property must be set", VIRTIO_MEM_MEMDEV_PROP
);
963 return &vmem
->memdev
->mr
;
966 static void virtio_mem_add_size_change_notifier(VirtIOMEM
*vmem
,
969 notifier_list_add(&vmem
->size_change_notifiers
, notifier
);
972 static void virtio_mem_remove_size_change_notifier(VirtIOMEM
*vmem
,
975 notifier_remove(notifier
);
978 static void virtio_mem_get_size(Object
*obj
, Visitor
*v
, const char *name
,
979 void *opaque
, Error
**errp
)
981 const VirtIOMEM
*vmem
= VIRTIO_MEM(obj
);
982 uint64_t value
= vmem
->size
;
984 visit_type_size(v
, name
, &value
, errp
);
987 static void virtio_mem_get_requested_size(Object
*obj
, Visitor
*v
,
988 const char *name
, void *opaque
,
991 const VirtIOMEM
*vmem
= VIRTIO_MEM(obj
);
992 uint64_t value
= vmem
->requested_size
;
994 visit_type_size(v
, name
, &value
, errp
);
997 static void virtio_mem_set_requested_size(Object
*obj
, Visitor
*v
,
998 const char *name
, void *opaque
,
1001 VirtIOMEM
*vmem
= VIRTIO_MEM(obj
);
1005 visit_type_size(v
, name
, &value
, &err
);
1007 error_propagate(errp
, err
);
1012 * The block size and memory backend are not fixed until the device was
1013 * realized. realize() will verify these properties then.
1015 if (DEVICE(obj
)->realized
) {
1016 if (!QEMU_IS_ALIGNED(value
, vmem
->block_size
)) {
1017 error_setg(errp
, "'%s' has to be multiples of '%s' (0x%" PRIx64
1018 ")", name
, VIRTIO_MEM_BLOCK_SIZE_PROP
,
1021 } else if (value
> memory_region_size(&vmem
->memdev
->mr
)) {
1022 error_setg(errp
, "'%s' cannot exceed the memory backend size"
1023 "(0x%" PRIx64
")", name
,
1024 memory_region_size(&vmem
->memdev
->mr
));
1028 if (value
!= vmem
->requested_size
) {
1029 virtio_mem_resize_usable_region(vmem
, value
, false);
1030 vmem
->requested_size
= value
;
1033 * Trigger a config update so the guest gets notified. We trigger
1034 * even if the size didn't change (especially helpful for debugging).
1036 virtio_notify_config(VIRTIO_DEVICE(vmem
));
1038 vmem
->requested_size
= value
;
1042 static void virtio_mem_get_block_size(Object
*obj
, Visitor
*v
, const char *name
,
1043 void *opaque
, Error
**errp
)
1045 const VirtIOMEM
*vmem
= VIRTIO_MEM(obj
);
1046 uint64_t value
= vmem
->block_size
;
1049 * If not configured by the user (and we're not realized yet), use the
1050 * default block size we would use with the current memory backend.
1053 if (vmem
->memdev
&& memory_region_is_ram(&vmem
->memdev
->mr
)) {
1054 value
= virtio_mem_default_block_size(vmem
->memdev
->mr
.ram_block
);
1056 value
= virtio_mem_thp_size();
1060 visit_type_size(v
, name
, &value
, errp
);
1063 static void virtio_mem_set_block_size(Object
*obj
, Visitor
*v
, const char *name
,
1064 void *opaque
, Error
**errp
)
1066 VirtIOMEM
*vmem
= VIRTIO_MEM(obj
);
1070 if (DEVICE(obj
)->realized
) {
1071 error_setg(errp
, "'%s' cannot be changed", name
);
1075 visit_type_size(v
, name
, &value
, &err
);
1077 error_propagate(errp
, err
);
1081 if (value
< VIRTIO_MEM_MIN_BLOCK_SIZE
) {
1082 error_setg(errp
, "'%s' property has to be at least 0x%" PRIx32
, name
,
1083 VIRTIO_MEM_MIN_BLOCK_SIZE
);
1085 } else if (!is_power_of_2(value
)) {
1086 error_setg(errp
, "'%s' property has to be a power of two", name
);
1089 vmem
->block_size
= value
;
1092 static int virtio_mem_precopy_exclude_range_cb(const VirtIOMEM
*vmem
, void *arg
,
1093 uint64_t offset
, uint64_t size
)
1095 void * const host
= qemu_ram_get_host_addr(vmem
->memdev
->mr
.ram_block
);
1097 qemu_guest_free_page_hint(host
+ offset
, size
);
1101 static void virtio_mem_precopy_exclude_unplugged(VirtIOMEM
*vmem
)
1103 virtio_mem_for_each_unplugged_range(vmem
, NULL
,
1104 virtio_mem_precopy_exclude_range_cb
);
1107 static int virtio_mem_precopy_notify(NotifierWithReturn
*n
, void *data
)
1109 VirtIOMEM
*vmem
= container_of(n
, VirtIOMEM
, precopy_notifier
);
1110 PrecopyNotifyData
*pnd
= data
;
1112 switch (pnd
->reason
) {
1113 case PRECOPY_NOTIFY_AFTER_BITMAP_SYNC
:
1114 virtio_mem_precopy_exclude_unplugged(vmem
);
1123 static void virtio_mem_instance_init(Object
*obj
)
1125 VirtIOMEM
*vmem
= VIRTIO_MEM(obj
);
1127 notifier_list_init(&vmem
->size_change_notifiers
);
1128 vmem
->precopy_notifier
.notify
= virtio_mem_precopy_notify
;
1129 QLIST_INIT(&vmem
->rdl_list
);
1131 object_property_add(obj
, VIRTIO_MEM_SIZE_PROP
, "size", virtio_mem_get_size
,
1133 object_property_add(obj
, VIRTIO_MEM_REQUESTED_SIZE_PROP
, "size",
1134 virtio_mem_get_requested_size
,
1135 virtio_mem_set_requested_size
, NULL
, NULL
);
1136 object_property_add(obj
, VIRTIO_MEM_BLOCK_SIZE_PROP
, "size",
1137 virtio_mem_get_block_size
, virtio_mem_set_block_size
,
1141 static Property virtio_mem_properties
[] = {
1142 DEFINE_PROP_UINT64(VIRTIO_MEM_ADDR_PROP
, VirtIOMEM
, addr
, 0),
1143 DEFINE_PROP_UINT32(VIRTIO_MEM_NODE_PROP
, VirtIOMEM
, node
, 0),
1144 DEFINE_PROP_LINK(VIRTIO_MEM_MEMDEV_PROP
, VirtIOMEM
, memdev
,
1145 TYPE_MEMORY_BACKEND
, HostMemoryBackend
*),
1146 DEFINE_PROP_END_OF_LIST(),
1149 static uint64_t virtio_mem_rdm_get_min_granularity(const RamDiscardManager
*rdm
,
1150 const MemoryRegion
*mr
)
1152 const VirtIOMEM
*vmem
= VIRTIO_MEM(rdm
);
1154 g_assert(mr
== &vmem
->memdev
->mr
);
1155 return vmem
->block_size
;
1158 static bool virtio_mem_rdm_is_populated(const RamDiscardManager
*rdm
,
1159 const MemoryRegionSection
*s
)
1161 const VirtIOMEM
*vmem
= VIRTIO_MEM(rdm
);
1162 uint64_t start_gpa
= vmem
->addr
+ s
->offset_within_region
;
1163 uint64_t end_gpa
= start_gpa
+ int128_get64(s
->size
);
1165 g_assert(s
->mr
== &vmem
->memdev
->mr
);
1167 start_gpa
= QEMU_ALIGN_DOWN(start_gpa
, vmem
->block_size
);
1168 end_gpa
= QEMU_ALIGN_UP(end_gpa
, vmem
->block_size
);
1170 if (!virtio_mem_valid_range(vmem
, start_gpa
, end_gpa
- start_gpa
)) {
1174 return virtio_mem_test_bitmap(vmem
, start_gpa
, end_gpa
- start_gpa
, true);
1177 struct VirtIOMEMReplayData
{
1182 static int virtio_mem_rdm_replay_populated_cb(MemoryRegionSection
*s
, void *arg
)
1184 struct VirtIOMEMReplayData
*data
= arg
;
1186 return ((ReplayRamPopulate
)data
->fn
)(s
, data
->opaque
);
1189 static int virtio_mem_rdm_replay_populated(const RamDiscardManager
*rdm
,
1190 MemoryRegionSection
*s
,
1191 ReplayRamPopulate replay_fn
,
1194 const VirtIOMEM
*vmem
= VIRTIO_MEM(rdm
);
1195 struct VirtIOMEMReplayData data
= {
1200 g_assert(s
->mr
== &vmem
->memdev
->mr
);
1201 return virtio_mem_for_each_plugged_section(vmem
, s
, &data
,
1202 virtio_mem_rdm_replay_populated_cb
);
1205 static int virtio_mem_rdm_replay_discarded_cb(MemoryRegionSection
*s
,
1208 struct VirtIOMEMReplayData
*data
= arg
;
1210 ((ReplayRamDiscard
)data
->fn
)(s
, data
->opaque
);
1214 static void virtio_mem_rdm_replay_discarded(const RamDiscardManager
*rdm
,
1215 MemoryRegionSection
*s
,
1216 ReplayRamDiscard replay_fn
,
1219 const VirtIOMEM
*vmem
= VIRTIO_MEM(rdm
);
1220 struct VirtIOMEMReplayData data
= {
1225 g_assert(s
->mr
== &vmem
->memdev
->mr
);
1226 virtio_mem_for_each_unplugged_section(vmem
, s
, &data
,
1227 virtio_mem_rdm_replay_discarded_cb
);
1230 static void virtio_mem_rdm_register_listener(RamDiscardManager
*rdm
,
1231 RamDiscardListener
*rdl
,
1232 MemoryRegionSection
*s
)
1234 VirtIOMEM
*vmem
= VIRTIO_MEM(rdm
);
1237 g_assert(s
->mr
== &vmem
->memdev
->mr
);
1238 rdl
->section
= memory_region_section_new_copy(s
);
1240 QLIST_INSERT_HEAD(&vmem
->rdl_list
, rdl
, next
);
1241 ret
= virtio_mem_for_each_plugged_section(vmem
, rdl
->section
, rdl
,
1242 virtio_mem_notify_populate_cb
);
1244 error_report("%s: Replaying plugged ranges failed: %s", __func__
,
1249 static void virtio_mem_rdm_unregister_listener(RamDiscardManager
*rdm
,
1250 RamDiscardListener
*rdl
)
1252 VirtIOMEM
*vmem
= VIRTIO_MEM(rdm
);
1254 g_assert(rdl
->section
->mr
== &vmem
->memdev
->mr
);
1256 if (rdl
->double_discard_supported
) {
1257 rdl
->notify_discard(rdl
, rdl
->section
);
1259 virtio_mem_for_each_plugged_section(vmem
, rdl
->section
, rdl
,
1260 virtio_mem_notify_discard_cb
);
1264 memory_region_section_free_copy(rdl
->section
);
1265 rdl
->section
= NULL
;
1266 QLIST_REMOVE(rdl
, next
);
1269 static void virtio_mem_class_init(ObjectClass
*klass
, void *data
)
1271 DeviceClass
*dc
= DEVICE_CLASS(klass
);
1272 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_CLASS(klass
);
1273 VirtIOMEMClass
*vmc
= VIRTIO_MEM_CLASS(klass
);
1274 RamDiscardManagerClass
*rdmc
= RAM_DISCARD_MANAGER_CLASS(klass
);
1276 device_class_set_props(dc
, virtio_mem_properties
);
1277 dc
->vmsd
= &vmstate_virtio_mem
;
1279 set_bit(DEVICE_CATEGORY_MISC
, dc
->categories
);
1280 vdc
->realize
= virtio_mem_device_realize
;
1281 vdc
->unrealize
= virtio_mem_device_unrealize
;
1282 vdc
->get_config
= virtio_mem_get_config
;
1283 vdc
->get_features
= virtio_mem_get_features
;
1284 vdc
->vmsd
= &vmstate_virtio_mem_device
;
1286 vmc
->fill_device_info
= virtio_mem_fill_device_info
;
1287 vmc
->get_memory_region
= virtio_mem_get_memory_region
;
1288 vmc
->add_size_change_notifier
= virtio_mem_add_size_change_notifier
;
1289 vmc
->remove_size_change_notifier
= virtio_mem_remove_size_change_notifier
;
1291 rdmc
->get_min_granularity
= virtio_mem_rdm_get_min_granularity
;
1292 rdmc
->is_populated
= virtio_mem_rdm_is_populated
;
1293 rdmc
->replay_populated
= virtio_mem_rdm_replay_populated
;
1294 rdmc
->replay_discarded
= virtio_mem_rdm_replay_discarded
;
1295 rdmc
->register_listener
= virtio_mem_rdm_register_listener
;
1296 rdmc
->unregister_listener
= virtio_mem_rdm_unregister_listener
;
1299 static const TypeInfo virtio_mem_info
= {
1300 .name
= TYPE_VIRTIO_MEM
,
1301 .parent
= TYPE_VIRTIO_DEVICE
,
1302 .instance_size
= sizeof(VirtIOMEM
),
1303 .instance_init
= virtio_mem_instance_init
,
1304 .class_init
= virtio_mem_class_init
,
1305 .class_size
= sizeof(VirtIOMEMClass
),
1306 .interfaces
= (InterfaceInfo
[]) {
1307 { TYPE_RAM_DISCARD_MANAGER
},
1312 static void virtio_register_types(void)
1314 type_register_static(&virtio_mem_info
);
1317 type_init(virtio_register_types
)