4 * Copyright (c) 2003-2008 Fabrice Bellard
5 * Copyright (c) 2011-2015 Red Hat Inc
8 * Juan Quintela <quintela@redhat.com>
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29 #include "qemu/osdep.h"
30 #include "qemu/cutils.h"
31 #include "qemu/bitops.h"
32 #include "qemu/bitmap.h"
33 #include "qemu/madvise.h"
34 #include "qemu/main-loop.h"
35 #include "io/channel-null.h"
38 #include "migration.h"
39 #include "migration/register.h"
40 #include "migration/misc.h"
41 #include "qemu-file.h"
42 #include "postcopy-ram.h"
43 #include "page_cache.h"
44 #include "qemu/error-report.h"
45 #include "qapi/error.h"
46 #include "qapi/qapi-types-migration.h"
47 #include "qapi/qapi-events-migration.h"
48 #include "qapi/qmp/qerror.h"
50 #include "exec/ram_addr.h"
51 #include "exec/target_page.h"
52 #include "qemu/rcu_queue.h"
53 #include "migration/colo.h"
55 #include "sysemu/cpu-throttle.h"
59 #include "sysemu/runstate.h"
61 #include "hw/boards.h" /* for machine_dump_guest_core() */
63 #if defined(__linux__)
64 #include "qemu/userfaultfd.h"
65 #endif /* defined(__linux__) */
67 /***********************************************************/
68 /* ram save/restore */
70 /* RAM_SAVE_FLAG_ZERO used to be named RAM_SAVE_FLAG_COMPRESS, it
71 * worked for pages that where filled with the same char. We switched
72 * it to only search for the zero value. And to avoid confusion with
73 * RAM_SSAVE_FLAG_COMPRESS_PAGE just rename it.
76 #define RAM_SAVE_FLAG_FULL 0x01 /* Obsolete, not used anymore */
77 #define RAM_SAVE_FLAG_ZERO 0x02
78 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
79 #define RAM_SAVE_FLAG_PAGE 0x08
80 #define RAM_SAVE_FLAG_EOS 0x10
81 #define RAM_SAVE_FLAG_CONTINUE 0x20
82 #define RAM_SAVE_FLAG_XBZRLE 0x40
83 /* 0x80 is reserved in migration.h start with 0x100 next */
84 #define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
86 XBZRLECacheStats xbzrle_counters
;
88 /* struct contains XBZRLE cache and a static page
89 used by the compression */
91 /* buffer used for XBZRLE encoding */
93 /* buffer for storing page content */
95 /* Cache for XBZRLE, Protected by lock. */
98 /* it will store a page full of zeros */
99 uint8_t *zero_target_page
;
100 /* buffer used for XBZRLE decoding */
101 uint8_t *decoded_buf
;
104 static void XBZRLE_cache_lock(void)
106 if (migrate_use_xbzrle()) {
107 qemu_mutex_lock(&XBZRLE
.lock
);
111 static void XBZRLE_cache_unlock(void)
113 if (migrate_use_xbzrle()) {
114 qemu_mutex_unlock(&XBZRLE
.lock
);
119 * xbzrle_cache_resize: resize the xbzrle cache
121 * This function is called from migrate_params_apply in main
122 * thread, possibly while a migration is in progress. A running
123 * migration may be using the cache and might finish during this call,
124 * hence changes to the cache are protected by XBZRLE.lock().
126 * Returns 0 for success or -1 for error
128 * @new_size: new cache size
129 * @errp: set *errp if the check failed, with reason
131 int xbzrle_cache_resize(uint64_t new_size
, Error
**errp
)
133 PageCache
*new_cache
;
136 /* Check for truncation */
137 if (new_size
!= (size_t)new_size
) {
138 error_setg(errp
, QERR_INVALID_PARAMETER_VALUE
, "cache size",
139 "exceeding address space");
143 if (new_size
== migrate_xbzrle_cache_size()) {
150 if (XBZRLE
.cache
!= NULL
) {
151 new_cache
= cache_init(new_size
, TARGET_PAGE_SIZE
, errp
);
157 cache_fini(XBZRLE
.cache
);
158 XBZRLE
.cache
= new_cache
;
161 XBZRLE_cache_unlock();
165 bool ramblock_is_ignored(RAMBlock
*block
)
167 return !qemu_ram_is_migratable(block
) ||
168 (migrate_ignore_shared() && qemu_ram_is_shared(block
));
171 #undef RAMBLOCK_FOREACH
173 int foreach_not_ignored_block(RAMBlockIterFunc func
, void *opaque
)
178 RCU_READ_LOCK_GUARD();
180 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
181 ret
= func(block
, opaque
);
189 static void ramblock_recv_map_init(void)
193 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
194 assert(!rb
->receivedmap
);
195 rb
->receivedmap
= bitmap_new(rb
->max_length
>> qemu_target_page_bits());
199 int ramblock_recv_bitmap_test(RAMBlock
*rb
, void *host_addr
)
201 return test_bit(ramblock_recv_bitmap_offset(host_addr
, rb
),
205 bool ramblock_recv_bitmap_test_byte_offset(RAMBlock
*rb
, uint64_t byte_offset
)
207 return test_bit(byte_offset
>> TARGET_PAGE_BITS
, rb
->receivedmap
);
210 void ramblock_recv_bitmap_set(RAMBlock
*rb
, void *host_addr
)
212 set_bit_atomic(ramblock_recv_bitmap_offset(host_addr
, rb
), rb
->receivedmap
);
215 void ramblock_recv_bitmap_set_range(RAMBlock
*rb
, void *host_addr
,
218 bitmap_set_atomic(rb
->receivedmap
,
219 ramblock_recv_bitmap_offset(host_addr
, rb
),
223 #define RAMBLOCK_RECV_BITMAP_ENDING (0x0123456789abcdefULL)
226 * Format: bitmap_size (8 bytes) + whole_bitmap (N bytes).
228 * Returns >0 if success with sent bytes, or <0 if error.
230 int64_t ramblock_recv_bitmap_send(QEMUFile
*file
,
231 const char *block_name
)
233 RAMBlock
*block
= qemu_ram_block_by_name(block_name
);
234 unsigned long *le_bitmap
, nbits
;
238 error_report("%s: invalid block name: %s", __func__
, block_name
);
242 nbits
= block
->postcopy_length
>> TARGET_PAGE_BITS
;
245 * Make sure the tmp bitmap buffer is big enough, e.g., on 32bit
246 * machines we may need 4 more bytes for padding (see below
247 * comment). So extend it a bit before hand.
249 le_bitmap
= bitmap_new(nbits
+ BITS_PER_LONG
);
252 * Always use little endian when sending the bitmap. This is
253 * required that when source and destination VMs are not using the
254 * same endianness. (Note: big endian won't work.)
256 bitmap_to_le(le_bitmap
, block
->receivedmap
, nbits
);
258 /* Size of the bitmap, in bytes */
259 size
= DIV_ROUND_UP(nbits
, 8);
262 * size is always aligned to 8 bytes for 64bit machines, but it
263 * may not be true for 32bit machines. We need this padding to
264 * make sure the migration can survive even between 32bit and
267 size
= ROUND_UP(size
, 8);
269 qemu_put_be64(file
, size
);
270 qemu_put_buffer(file
, (const uint8_t *)le_bitmap
, size
);
272 * Mark as an end, in case the middle part is screwed up due to
273 * some "mysterious" reason.
275 qemu_put_be64(file
, RAMBLOCK_RECV_BITMAP_ENDING
);
280 if (qemu_file_get_error(file
)) {
281 return qemu_file_get_error(file
);
284 return size
+ sizeof(size
);
288 * An outstanding page request, on the source, having been received
291 struct RAMSrcPageRequest
{
296 QSIMPLEQ_ENTRY(RAMSrcPageRequest
) next_req
;
301 * Cached ramblock/offset values if preempted. They're only meaningful if
302 * preempted==true below.
305 unsigned long ram_page
;
307 * Whether a postcopy preemption just happened. Will be reset after
308 * precopy recovered to background migration.
311 } PostcopyPreemptState
;
313 /* State of RAM for migration */
315 /* QEMUFile used for this migration */
317 /* UFFD file descriptor, used in 'write-tracking' migration */
319 /* Last block that we have visited searching for dirty pages */
320 RAMBlock
*last_seen_block
;
321 /* Last block from where we have sent data */
322 RAMBlock
*last_sent_block
;
323 /* Last dirty target page we have sent */
324 ram_addr_t last_page
;
325 /* last ram version we have seen */
326 uint32_t last_version
;
327 /* How many times we have dirty too many pages */
328 int dirty_rate_high_cnt
;
329 /* these variables are used for bitmap sync */
330 /* last time we did a full bitmap_sync */
331 int64_t time_last_bitmap_sync
;
332 /* bytes transferred at start_time */
333 uint64_t bytes_xfer_prev
;
334 /* number of dirty pages since start_time */
335 uint64_t num_dirty_pages_period
;
336 /* xbzrle misses since the beginning of the period */
337 uint64_t xbzrle_cache_miss_prev
;
338 /* Amount of xbzrle pages since the beginning of the period */
339 uint64_t xbzrle_pages_prev
;
340 /* Amount of xbzrle encoded bytes since the beginning of the period */
341 uint64_t xbzrle_bytes_prev
;
342 /* Start using XBZRLE (e.g., after the first round). */
344 /* Are we on the last stage of migration */
346 /* compression statistics since the beginning of the period */
347 /* amount of count that no free thread to compress data */
348 uint64_t compress_thread_busy_prev
;
349 /* amount bytes after compression */
350 uint64_t compressed_size_prev
;
351 /* amount of compressed pages */
352 uint64_t compress_pages_prev
;
354 /* total handled target pages at the beginning of period */
355 uint64_t target_page_count_prev
;
356 /* total handled target pages since start */
357 uint64_t target_page_count
;
358 /* number of dirty bits in the bitmap */
359 uint64_t migration_dirty_pages
;
360 /* Protects modification of the bitmap and migration dirty pages */
361 QemuMutex bitmap_mutex
;
362 /* The RAMBlock used in the last src_page_requests */
363 RAMBlock
*last_req_rb
;
364 /* Queue of outstanding page requests from the destination */
365 QemuMutex src_page_req_mutex
;
366 QSIMPLEQ_HEAD(, RAMSrcPageRequest
) src_page_requests
;
368 /* Postcopy preemption informations */
369 PostcopyPreemptState postcopy_preempt_state
;
371 * Current channel we're using on src VM. Only valid if postcopy-preempt
374 unsigned int postcopy_channel
;
376 typedef struct RAMState RAMState
;
378 static RAMState
*ram_state
;
380 static NotifierWithReturnList precopy_notifier_list
;
382 static void postcopy_preempt_reset(RAMState
*rs
)
384 memset(&rs
->postcopy_preempt_state
, 0, sizeof(PostcopyPreemptState
));
387 /* Whether postcopy has queued requests? */
388 static bool postcopy_has_request(RAMState
*rs
)
390 return !QSIMPLEQ_EMPTY_ATOMIC(&rs
->src_page_requests
);
393 void precopy_infrastructure_init(void)
395 notifier_with_return_list_init(&precopy_notifier_list
);
398 void precopy_add_notifier(NotifierWithReturn
*n
)
400 notifier_with_return_list_add(&precopy_notifier_list
, n
);
403 void precopy_remove_notifier(NotifierWithReturn
*n
)
405 notifier_with_return_remove(n
);
408 int precopy_notify(PrecopyNotifyReason reason
, Error
**errp
)
410 PrecopyNotifyData pnd
;
414 return notifier_with_return_list_notify(&precopy_notifier_list
, &pnd
);
417 uint64_t ram_bytes_remaining(void)
419 return ram_state
? (ram_state
->migration_dirty_pages
* TARGET_PAGE_SIZE
) :
423 MigrationStats ram_counters
;
425 static void ram_transferred_add(uint64_t bytes
)
427 if (runstate_is_running()) {
428 ram_counters
.precopy_bytes
+= bytes
;
429 } else if (migration_in_postcopy()) {
430 ram_counters
.postcopy_bytes
+= bytes
;
432 ram_counters
.downtime_bytes
+= bytes
;
434 ram_counters
.transferred
+= bytes
;
437 void dirty_sync_missed_zero_copy(void)
439 ram_counters
.dirty_sync_missed_zero_copy
++;
442 /* used by the search for pages to send */
443 struct PageSearchStatus
{
444 /* Current block being searched */
446 /* Current page to search from */
448 /* Set once we wrap around */
451 * [POSTCOPY-ONLY] Whether current page is explicitly requested by
452 * postcopy. When set, the request is "urgent" because the dest QEMU
453 * threads are waiting for us.
455 bool postcopy_requested
;
457 * [POSTCOPY-ONLY] The target channel to use to send current page.
459 * Note: This may _not_ match with the value in postcopy_requested
460 * above. Let's imagine the case where the postcopy request is exactly
461 * the page that we're sending in progress during precopy. In this case
462 * we'll have postcopy_requested set to true but the target channel
463 * will be the precopy channel (so that we don't split brain on that
464 * specific page since the precopy channel already contains partial of
467 * Besides that specific use case, postcopy_target_channel should
468 * always be equal to postcopy_requested, because by default we send
469 * postcopy pages via postcopy preempt channel.
471 bool postcopy_target_channel
;
473 typedef struct PageSearchStatus PageSearchStatus
;
475 CompressionStats compression_counters
;
477 struct CompressParam
{
487 /* internally used fields */
491 typedef struct CompressParam CompressParam
;
493 struct DecompressParam
{
503 typedef struct DecompressParam DecompressParam
;
505 static CompressParam
*comp_param
;
506 static QemuThread
*compress_threads
;
507 /* comp_done_cond is used to wake up the migration thread when
508 * one of the compression threads has finished the compression.
509 * comp_done_lock is used to co-work with comp_done_cond.
511 static QemuMutex comp_done_lock
;
512 static QemuCond comp_done_cond
;
514 static QEMUFile
*decomp_file
;
515 static DecompressParam
*decomp_param
;
516 static QemuThread
*decompress_threads
;
517 static QemuMutex decomp_done_lock
;
518 static QemuCond decomp_done_cond
;
520 static bool do_compress_ram_page(QEMUFile
*f
, z_stream
*stream
, RAMBlock
*block
,
521 ram_addr_t offset
, uint8_t *source_buf
);
523 static void postcopy_preempt_restore(RAMState
*rs
, PageSearchStatus
*pss
,
524 bool postcopy_requested
);
526 static void *do_data_compress(void *opaque
)
528 CompressParam
*param
= opaque
;
533 qemu_mutex_lock(¶m
->mutex
);
534 while (!param
->quit
) {
536 block
= param
->block
;
537 offset
= param
->offset
;
539 qemu_mutex_unlock(¶m
->mutex
);
541 zero_page
= do_compress_ram_page(param
->file
, ¶m
->stream
,
542 block
, offset
, param
->originbuf
);
544 qemu_mutex_lock(&comp_done_lock
);
546 param
->zero_page
= zero_page
;
547 qemu_cond_signal(&comp_done_cond
);
548 qemu_mutex_unlock(&comp_done_lock
);
550 qemu_mutex_lock(¶m
->mutex
);
552 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
555 qemu_mutex_unlock(¶m
->mutex
);
560 static void compress_threads_save_cleanup(void)
564 if (!migrate_use_compression() || !comp_param
) {
568 thread_count
= migrate_compress_threads();
569 for (i
= 0; i
< thread_count
; i
++) {
571 * we use it as a indicator which shows if the thread is
572 * properly init'd or not
574 if (!comp_param
[i
].file
) {
578 qemu_mutex_lock(&comp_param
[i
].mutex
);
579 comp_param
[i
].quit
= true;
580 qemu_cond_signal(&comp_param
[i
].cond
);
581 qemu_mutex_unlock(&comp_param
[i
].mutex
);
583 qemu_thread_join(compress_threads
+ i
);
584 qemu_mutex_destroy(&comp_param
[i
].mutex
);
585 qemu_cond_destroy(&comp_param
[i
].cond
);
586 deflateEnd(&comp_param
[i
].stream
);
587 g_free(comp_param
[i
].originbuf
);
588 qemu_fclose(comp_param
[i
].file
);
589 comp_param
[i
].file
= NULL
;
591 qemu_mutex_destroy(&comp_done_lock
);
592 qemu_cond_destroy(&comp_done_cond
);
593 g_free(compress_threads
);
595 compress_threads
= NULL
;
599 static int compress_threads_save_setup(void)
603 if (!migrate_use_compression()) {
606 thread_count
= migrate_compress_threads();
607 compress_threads
= g_new0(QemuThread
, thread_count
);
608 comp_param
= g_new0(CompressParam
, thread_count
);
609 qemu_cond_init(&comp_done_cond
);
610 qemu_mutex_init(&comp_done_lock
);
611 for (i
= 0; i
< thread_count
; i
++) {
612 comp_param
[i
].originbuf
= g_try_malloc(TARGET_PAGE_SIZE
);
613 if (!comp_param
[i
].originbuf
) {
617 if (deflateInit(&comp_param
[i
].stream
,
618 migrate_compress_level()) != Z_OK
) {
619 g_free(comp_param
[i
].originbuf
);
623 /* comp_param[i].file is just used as a dummy buffer to save data,
624 * set its ops to empty.
626 comp_param
[i
].file
= qemu_file_new_output(
627 QIO_CHANNEL(qio_channel_null_new()));
628 comp_param
[i
].done
= true;
629 comp_param
[i
].quit
= false;
630 qemu_mutex_init(&comp_param
[i
].mutex
);
631 qemu_cond_init(&comp_param
[i
].cond
);
632 qemu_thread_create(compress_threads
+ i
, "compress",
633 do_data_compress
, comp_param
+ i
,
634 QEMU_THREAD_JOINABLE
);
639 compress_threads_save_cleanup();
644 * save_page_header: write page header to wire
646 * If this is the 1st block, it also writes the block identification
648 * Returns the number of bytes written
650 * @f: QEMUFile where to send the data
651 * @block: block that contains the page we want to send
652 * @offset: offset inside the block for the page
653 * in the lower bits, it contains flags
655 static size_t save_page_header(RAMState
*rs
, QEMUFile
*f
, RAMBlock
*block
,
660 if (block
== rs
->last_sent_block
) {
661 offset
|= RAM_SAVE_FLAG_CONTINUE
;
663 qemu_put_be64(f
, offset
);
666 if (!(offset
& RAM_SAVE_FLAG_CONTINUE
)) {
667 len
= strlen(block
->idstr
);
668 qemu_put_byte(f
, len
);
669 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, len
);
671 rs
->last_sent_block
= block
;
677 * mig_throttle_guest_down: throttle down the guest
679 * Reduce amount of guest cpu execution to hopefully slow down memory
680 * writes. If guest dirty memory rate is reduced below the rate at
681 * which we can transfer pages to the destination then we should be
682 * able to complete migration. Some workloads dirty memory way too
683 * fast and will not effectively converge, even with auto-converge.
685 static void mig_throttle_guest_down(uint64_t bytes_dirty_period
,
686 uint64_t bytes_dirty_threshold
)
688 MigrationState
*s
= migrate_get_current();
689 uint64_t pct_initial
= s
->parameters
.cpu_throttle_initial
;
690 uint64_t pct_increment
= s
->parameters
.cpu_throttle_increment
;
691 bool pct_tailslow
= s
->parameters
.cpu_throttle_tailslow
;
692 int pct_max
= s
->parameters
.max_cpu_throttle
;
694 uint64_t throttle_now
= cpu_throttle_get_percentage();
695 uint64_t cpu_now
, cpu_ideal
, throttle_inc
;
697 /* We have not started throttling yet. Let's start it. */
698 if (!cpu_throttle_active()) {
699 cpu_throttle_set(pct_initial
);
701 /* Throttling already on, just increase the rate */
703 throttle_inc
= pct_increment
;
705 /* Compute the ideal CPU percentage used by Guest, which may
706 * make the dirty rate match the dirty rate threshold. */
707 cpu_now
= 100 - throttle_now
;
708 cpu_ideal
= cpu_now
* (bytes_dirty_threshold
* 1.0 /
710 throttle_inc
= MIN(cpu_now
- cpu_ideal
, pct_increment
);
712 cpu_throttle_set(MIN(throttle_now
+ throttle_inc
, pct_max
));
716 void mig_throttle_counter_reset(void)
718 RAMState
*rs
= ram_state
;
720 rs
->time_last_bitmap_sync
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
721 rs
->num_dirty_pages_period
= 0;
722 rs
->bytes_xfer_prev
= ram_counters
.transferred
;
726 * xbzrle_cache_zero_page: insert a zero page in the XBZRLE cache
728 * @rs: current RAM state
729 * @current_addr: address for the zero page
731 * Update the xbzrle cache to reflect a page that's been sent as all 0.
732 * The important thing is that a stale (not-yet-0'd) page be replaced
734 * As a bonus, if the page wasn't in the cache it gets added so that
735 * when a small write is made into the 0'd page it gets XBZRLE sent.
737 static void xbzrle_cache_zero_page(RAMState
*rs
, ram_addr_t current_addr
)
739 if (!rs
->xbzrle_enabled
) {
743 /* We don't care if this fails to allocate a new cache page
744 * as long as it updated an old one */
745 cache_insert(XBZRLE
.cache
, current_addr
, XBZRLE
.zero_target_page
,
746 ram_counters
.dirty_sync_count
);
749 #define ENCODING_FLAG_XBZRLE 0x1
752 * save_xbzrle_page: compress and send current page
754 * Returns: 1 means that we wrote the page
755 * 0 means that page is identical to the one already sent
756 * -1 means that xbzrle would be longer than normal
758 * @rs: current RAM state
759 * @current_data: pointer to the address of the page contents
760 * @current_addr: addr of the page
761 * @block: block that contains the page we want to send
762 * @offset: offset inside the block for the page
764 static int save_xbzrle_page(RAMState
*rs
, uint8_t **current_data
,
765 ram_addr_t current_addr
, RAMBlock
*block
,
768 int encoded_len
= 0, bytes_xbzrle
;
769 uint8_t *prev_cached_page
;
771 if (!cache_is_cached(XBZRLE
.cache
, current_addr
,
772 ram_counters
.dirty_sync_count
)) {
773 xbzrle_counters
.cache_miss
++;
774 if (!rs
->last_stage
) {
775 if (cache_insert(XBZRLE
.cache
, current_addr
, *current_data
,
776 ram_counters
.dirty_sync_count
) == -1) {
779 /* update *current_data when the page has been
780 inserted into cache */
781 *current_data
= get_cached_data(XBZRLE
.cache
, current_addr
);
788 * Reaching here means the page has hit the xbzrle cache, no matter what
789 * encoding result it is (normal encoding, overflow or skipping the page),
790 * count the page as encoded. This is used to calculate the encoding rate.
792 * Example: 2 pages (8KB) being encoded, first page encoding generates 2KB,
793 * 2nd page turns out to be skipped (i.e. no new bytes written to the
794 * page), the overall encoding rate will be 8KB / 2KB = 4, which has the
795 * skipped page included. In this way, the encoding rate can tell if the
796 * guest page is good for xbzrle encoding.
798 xbzrle_counters
.pages
++;
799 prev_cached_page
= get_cached_data(XBZRLE
.cache
, current_addr
);
801 /* save current buffer into memory */
802 memcpy(XBZRLE
.current_buf
, *current_data
, TARGET_PAGE_SIZE
);
804 /* XBZRLE encoding (if there is no overflow) */
805 encoded_len
= xbzrle_encode_buffer(prev_cached_page
, XBZRLE
.current_buf
,
806 TARGET_PAGE_SIZE
, XBZRLE
.encoded_buf
,
810 * Update the cache contents, so that it corresponds to the data
811 * sent, in all cases except where we skip the page.
813 if (!rs
->last_stage
&& encoded_len
!= 0) {
814 memcpy(prev_cached_page
, XBZRLE
.current_buf
, TARGET_PAGE_SIZE
);
816 * In the case where we couldn't compress, ensure that the caller
817 * sends the data from the cache, since the guest might have
818 * changed the RAM since we copied it.
820 *current_data
= prev_cached_page
;
823 if (encoded_len
== 0) {
824 trace_save_xbzrle_page_skipping();
826 } else if (encoded_len
== -1) {
827 trace_save_xbzrle_page_overflow();
828 xbzrle_counters
.overflow
++;
829 xbzrle_counters
.bytes
+= TARGET_PAGE_SIZE
;
833 /* Send XBZRLE based compressed page */
834 bytes_xbzrle
= save_page_header(rs
, rs
->f
, block
,
835 offset
| RAM_SAVE_FLAG_XBZRLE
);
836 qemu_put_byte(rs
->f
, ENCODING_FLAG_XBZRLE
);
837 qemu_put_be16(rs
->f
, encoded_len
);
838 qemu_put_buffer(rs
->f
, XBZRLE
.encoded_buf
, encoded_len
);
839 bytes_xbzrle
+= encoded_len
+ 1 + 2;
841 * Like compressed_size (please see update_compress_thread_counts),
842 * the xbzrle encoded bytes don't count the 8 byte header with
843 * RAM_SAVE_FLAG_CONTINUE.
845 xbzrle_counters
.bytes
+= bytes_xbzrle
- 8;
846 ram_transferred_add(bytes_xbzrle
);
852 * migration_bitmap_find_dirty: find the next dirty page from start
854 * Returns the page offset within memory region of the start of a dirty page
856 * @rs: current RAM state
857 * @rb: RAMBlock where to search for dirty pages
858 * @start: page where we start the search
861 unsigned long migration_bitmap_find_dirty(RAMState
*rs
, RAMBlock
*rb
,
864 unsigned long size
= rb
->used_length
>> TARGET_PAGE_BITS
;
865 unsigned long *bitmap
= rb
->bmap
;
867 if (ramblock_is_ignored(rb
)) {
871 return find_next_bit(bitmap
, size
, start
);
874 static void migration_clear_memory_region_dirty_bitmap(RAMBlock
*rb
,
880 if (!rb
->clear_bmap
|| !clear_bmap_test_and_clear(rb
, page
)) {
884 shift
= rb
->clear_bmap_shift
;
886 * CLEAR_BITMAP_SHIFT_MIN should always guarantee this... this
887 * can make things easier sometimes since then start address
888 * of the small chunk will always be 64 pages aligned so the
889 * bitmap will always be aligned to unsigned long. We should
890 * even be able to remove this restriction but I'm simply
895 size
= 1ULL << (TARGET_PAGE_BITS
+ shift
);
896 start
= QEMU_ALIGN_DOWN((ram_addr_t
)page
<< TARGET_PAGE_BITS
, size
);
897 trace_migration_bitmap_clear_dirty(rb
->idstr
, start
, size
, page
);
898 memory_region_clear_dirty_bitmap(rb
->mr
, start
, size
);
902 migration_clear_memory_region_dirty_bitmap_range(RAMBlock
*rb
,
904 unsigned long npages
)
906 unsigned long i
, chunk_pages
= 1UL << rb
->clear_bmap_shift
;
907 unsigned long chunk_start
= QEMU_ALIGN_DOWN(start
, chunk_pages
);
908 unsigned long chunk_end
= QEMU_ALIGN_UP(start
+ npages
, chunk_pages
);
911 * Clear pages from start to start + npages - 1, so the end boundary is
914 for (i
= chunk_start
; i
< chunk_end
; i
+= chunk_pages
) {
915 migration_clear_memory_region_dirty_bitmap(rb
, i
);
920 * colo_bitmap_find_diry:find contiguous dirty pages from start
922 * Returns the page offset within memory region of the start of the contiguout
925 * @rs: current RAM state
926 * @rb: RAMBlock where to search for dirty pages
927 * @start: page where we start the search
928 * @num: the number of contiguous dirty pages
931 unsigned long colo_bitmap_find_dirty(RAMState
*rs
, RAMBlock
*rb
,
932 unsigned long start
, unsigned long *num
)
934 unsigned long size
= rb
->used_length
>> TARGET_PAGE_BITS
;
935 unsigned long *bitmap
= rb
->bmap
;
936 unsigned long first
, next
;
940 if (ramblock_is_ignored(rb
)) {
944 first
= find_next_bit(bitmap
, size
, start
);
948 next
= find_next_zero_bit(bitmap
, size
, first
+ 1);
949 assert(next
>= first
);
954 static inline bool migration_bitmap_clear_dirty(RAMState
*rs
,
961 * Clear dirty bitmap if needed. This _must_ be called before we
962 * send any of the page in the chunk because we need to make sure
963 * we can capture further page content changes when we sync dirty
964 * log the next time. So as long as we are going to send any of
965 * the page in the chunk we clear the remote dirty bitmap for all.
966 * Clearing it earlier won't be a problem, but too late will.
968 migration_clear_memory_region_dirty_bitmap(rb
, page
);
970 ret
= test_and_clear_bit(page
, rb
->bmap
);
972 rs
->migration_dirty_pages
--;
978 static void dirty_bitmap_clear_section(MemoryRegionSection
*section
,
981 const hwaddr offset
= section
->offset_within_region
;
982 const hwaddr size
= int128_get64(section
->size
);
983 const unsigned long start
= offset
>> TARGET_PAGE_BITS
;
984 const unsigned long npages
= size
>> TARGET_PAGE_BITS
;
985 RAMBlock
*rb
= section
->mr
->ram_block
;
986 uint64_t *cleared_bits
= opaque
;
989 * We don't grab ram_state->bitmap_mutex because we expect to run
990 * only when starting migration or during postcopy recovery where
991 * we don't have concurrent access.
993 if (!migration_in_postcopy() && !migrate_background_snapshot()) {
994 migration_clear_memory_region_dirty_bitmap_range(rb
, start
, npages
);
996 *cleared_bits
+= bitmap_count_one_with_offset(rb
->bmap
, start
, npages
);
997 bitmap_clear(rb
->bmap
, start
, npages
);
1001 * Exclude all dirty pages from migration that fall into a discarded range as
1002 * managed by a RamDiscardManager responsible for the mapped memory region of
1003 * the RAMBlock. Clear the corresponding bits in the dirty bitmaps.
1005 * Discarded pages ("logically unplugged") have undefined content and must
1006 * not get migrated, because even reading these pages for migration might
1007 * result in undesired behavior.
1009 * Returns the number of cleared bits in the RAMBlock dirty bitmap.
1011 * Note: The result is only stable while migrating (precopy/postcopy).
1013 static uint64_t ramblock_dirty_bitmap_clear_discarded_pages(RAMBlock
*rb
)
1015 uint64_t cleared_bits
= 0;
1017 if (rb
->mr
&& rb
->bmap
&& memory_region_has_ram_discard_manager(rb
->mr
)) {
1018 RamDiscardManager
*rdm
= memory_region_get_ram_discard_manager(rb
->mr
);
1019 MemoryRegionSection section
= {
1021 .offset_within_region
= 0,
1022 .size
= int128_make64(qemu_ram_get_used_length(rb
)),
1025 ram_discard_manager_replay_discarded(rdm
, §ion
,
1026 dirty_bitmap_clear_section
,
1029 return cleared_bits
;
1033 * Check if a host-page aligned page falls into a discarded range as managed by
1034 * a RamDiscardManager responsible for the mapped memory region of the RAMBlock.
1036 * Note: The result is only stable while migrating (precopy/postcopy).
1038 bool ramblock_page_is_discarded(RAMBlock
*rb
, ram_addr_t start
)
1040 if (rb
->mr
&& memory_region_has_ram_discard_manager(rb
->mr
)) {
1041 RamDiscardManager
*rdm
= memory_region_get_ram_discard_manager(rb
->mr
);
1042 MemoryRegionSection section
= {
1044 .offset_within_region
= start
,
1045 .size
= int128_make64(qemu_ram_pagesize(rb
)),
1048 return !ram_discard_manager_is_populated(rdm
, §ion
);
1053 /* Called with RCU critical section */
1054 static void ramblock_sync_dirty_bitmap(RAMState
*rs
, RAMBlock
*rb
)
1056 uint64_t new_dirty_pages
=
1057 cpu_physical_memory_sync_dirty_bitmap(rb
, 0, rb
->used_length
);
1059 rs
->migration_dirty_pages
+= new_dirty_pages
;
1060 rs
->num_dirty_pages_period
+= new_dirty_pages
;
1064 * ram_pagesize_summary: calculate all the pagesizes of a VM
1066 * Returns a summary bitmap of the page sizes of all RAMBlocks
1068 * For VMs with just normal pages this is equivalent to the host page
1069 * size. If it's got some huge pages then it's the OR of all the
1070 * different page sizes.
1072 uint64_t ram_pagesize_summary(void)
1075 uint64_t summary
= 0;
1077 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1078 summary
|= block
->page_size
;
1084 uint64_t ram_get_total_transferred_pages(void)
1086 return ram_counters
.normal
+ ram_counters
.duplicate
+
1087 compression_counters
.pages
+ xbzrle_counters
.pages
;
1090 static void migration_update_rates(RAMState
*rs
, int64_t end_time
)
1092 uint64_t page_count
= rs
->target_page_count
- rs
->target_page_count_prev
;
1093 double compressed_size
;
1095 /* calculate period counters */
1096 ram_counters
.dirty_pages_rate
= rs
->num_dirty_pages_period
* 1000
1097 / (end_time
- rs
->time_last_bitmap_sync
);
1103 if (migrate_use_xbzrle()) {
1104 double encoded_size
, unencoded_size
;
1106 xbzrle_counters
.cache_miss_rate
= (double)(xbzrle_counters
.cache_miss
-
1107 rs
->xbzrle_cache_miss_prev
) / page_count
;
1108 rs
->xbzrle_cache_miss_prev
= xbzrle_counters
.cache_miss
;
1109 unencoded_size
= (xbzrle_counters
.pages
- rs
->xbzrle_pages_prev
) *
1111 encoded_size
= xbzrle_counters
.bytes
- rs
->xbzrle_bytes_prev
;
1112 if (xbzrle_counters
.pages
== rs
->xbzrle_pages_prev
|| !encoded_size
) {
1113 xbzrle_counters
.encoding_rate
= 0;
1115 xbzrle_counters
.encoding_rate
= unencoded_size
/ encoded_size
;
1117 rs
->xbzrle_pages_prev
= xbzrle_counters
.pages
;
1118 rs
->xbzrle_bytes_prev
= xbzrle_counters
.bytes
;
1121 if (migrate_use_compression()) {
1122 compression_counters
.busy_rate
= (double)(compression_counters
.busy
-
1123 rs
->compress_thread_busy_prev
) / page_count
;
1124 rs
->compress_thread_busy_prev
= compression_counters
.busy
;
1126 compressed_size
= compression_counters
.compressed_size
-
1127 rs
->compressed_size_prev
;
1128 if (compressed_size
) {
1129 double uncompressed_size
= (compression_counters
.pages
-
1130 rs
->compress_pages_prev
) * TARGET_PAGE_SIZE
;
1132 /* Compression-Ratio = Uncompressed-size / Compressed-size */
1133 compression_counters
.compression_rate
=
1134 uncompressed_size
/ compressed_size
;
1136 rs
->compress_pages_prev
= compression_counters
.pages
;
1137 rs
->compressed_size_prev
= compression_counters
.compressed_size
;
1142 static void migration_trigger_throttle(RAMState
*rs
)
1144 MigrationState
*s
= migrate_get_current();
1145 uint64_t threshold
= s
->parameters
.throttle_trigger_threshold
;
1147 uint64_t bytes_xfer_period
= ram_counters
.transferred
- rs
->bytes_xfer_prev
;
1148 uint64_t bytes_dirty_period
= rs
->num_dirty_pages_period
* TARGET_PAGE_SIZE
;
1149 uint64_t bytes_dirty_threshold
= bytes_xfer_period
* threshold
/ 100;
1151 /* During block migration the auto-converge logic incorrectly detects
1152 * that ram migration makes no progress. Avoid this by disabling the
1153 * throttling logic during the bulk phase of block migration. */
1154 if (migrate_auto_converge() && !blk_mig_bulk_active()) {
1155 /* The following detection logic can be refined later. For now:
1156 Check to see if the ratio between dirtied bytes and the approx.
1157 amount of bytes that just got transferred since the last time
1158 we were in this routine reaches the threshold. If that happens
1159 twice, start or increase throttling. */
1161 if ((bytes_dirty_period
> bytes_dirty_threshold
) &&
1162 (++rs
->dirty_rate_high_cnt
>= 2)) {
1163 trace_migration_throttle();
1164 rs
->dirty_rate_high_cnt
= 0;
1165 mig_throttle_guest_down(bytes_dirty_period
,
1166 bytes_dirty_threshold
);
1171 static void migration_bitmap_sync(RAMState
*rs
)
1176 ram_counters
.dirty_sync_count
++;
1178 if (!rs
->time_last_bitmap_sync
) {
1179 rs
->time_last_bitmap_sync
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
1182 trace_migration_bitmap_sync_start();
1183 memory_global_dirty_log_sync();
1185 qemu_mutex_lock(&rs
->bitmap_mutex
);
1186 WITH_RCU_READ_LOCK_GUARD() {
1187 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1188 ramblock_sync_dirty_bitmap(rs
, block
);
1190 ram_counters
.remaining
= ram_bytes_remaining();
1192 qemu_mutex_unlock(&rs
->bitmap_mutex
);
1194 memory_global_after_dirty_log_sync();
1195 trace_migration_bitmap_sync_end(rs
->num_dirty_pages_period
);
1197 end_time
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
1199 /* more than 1 second = 1000 millisecons */
1200 if (end_time
> rs
->time_last_bitmap_sync
+ 1000) {
1201 migration_trigger_throttle(rs
);
1203 migration_update_rates(rs
, end_time
);
1205 rs
->target_page_count_prev
= rs
->target_page_count
;
1207 /* reset period counters */
1208 rs
->time_last_bitmap_sync
= end_time
;
1209 rs
->num_dirty_pages_period
= 0;
1210 rs
->bytes_xfer_prev
= ram_counters
.transferred
;
1212 if (migrate_use_events()) {
1213 qapi_event_send_migration_pass(ram_counters
.dirty_sync_count
);
1217 static void migration_bitmap_sync_precopy(RAMState
*rs
)
1219 Error
*local_err
= NULL
;
1222 * The current notifier usage is just an optimization to migration, so we
1223 * don't stop the normal migration process in the error case.
1225 if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC
, &local_err
)) {
1226 error_report_err(local_err
);
1230 migration_bitmap_sync(rs
);
1232 if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC
, &local_err
)) {
1233 error_report_err(local_err
);
1237 static void ram_release_page(const char *rbname
, uint64_t offset
)
1239 if (!migrate_release_ram() || !migration_in_postcopy()) {
1243 ram_discard_range(rbname
, offset
, TARGET_PAGE_SIZE
);
1247 * save_zero_page_to_file: send the zero page to the file
1249 * Returns the size of data written to the file, 0 means the page is not
1252 * @rs: current RAM state
1253 * @file: the file where the data is saved
1254 * @block: block that contains the page we want to send
1255 * @offset: offset inside the block for the page
1257 static int save_zero_page_to_file(RAMState
*rs
, QEMUFile
*file
,
1258 RAMBlock
*block
, ram_addr_t offset
)
1260 uint8_t *p
= block
->host
+ offset
;
1263 if (buffer_is_zero(p
, TARGET_PAGE_SIZE
)) {
1264 len
+= save_page_header(rs
, file
, block
, offset
| RAM_SAVE_FLAG_ZERO
);
1265 qemu_put_byte(file
, 0);
1267 ram_release_page(block
->idstr
, offset
);
1273 * save_zero_page: send the zero page to the stream
1275 * Returns the number of pages written.
1277 * @rs: current RAM state
1278 * @block: block that contains the page we want to send
1279 * @offset: offset inside the block for the page
1281 static int save_zero_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
)
1283 int len
= save_zero_page_to_file(rs
, rs
->f
, block
, offset
);
1286 ram_counters
.duplicate
++;
1287 ram_transferred_add(len
);
1294 * @pages: the number of pages written by the control path,
1296 * > 0 - number of pages written
1298 * Return true if the pages has been saved, otherwise false is returned.
1300 static bool control_save_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
,
1303 uint64_t bytes_xmit
= 0;
1307 ret
= ram_control_save_page(rs
->f
, block
->offset
, offset
, TARGET_PAGE_SIZE
,
1309 if (ret
== RAM_SAVE_CONTROL_NOT_SUPP
) {
1314 ram_transferred_add(bytes_xmit
);
1318 if (ret
== RAM_SAVE_CONTROL_DELAYED
) {
1322 if (bytes_xmit
> 0) {
1323 ram_counters
.normal
++;
1324 } else if (bytes_xmit
== 0) {
1325 ram_counters
.duplicate
++;
1332 * directly send the page to the stream
1334 * Returns the number of pages written.
1336 * @rs: current RAM state
1337 * @block: block that contains the page we want to send
1338 * @offset: offset inside the block for the page
1339 * @buf: the page to be sent
1340 * @async: send to page asyncly
1342 static int save_normal_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
,
1343 uint8_t *buf
, bool async
)
1345 ram_transferred_add(save_page_header(rs
, rs
->f
, block
,
1346 offset
| RAM_SAVE_FLAG_PAGE
));
1348 qemu_put_buffer_async(rs
->f
, buf
, TARGET_PAGE_SIZE
,
1349 migrate_release_ram() &&
1350 migration_in_postcopy());
1352 qemu_put_buffer(rs
->f
, buf
, TARGET_PAGE_SIZE
);
1354 ram_transferred_add(TARGET_PAGE_SIZE
);
1355 ram_counters
.normal
++;
1360 * ram_save_page: send the given page to the stream
1362 * Returns the number of pages written.
1364 * >=0 - Number of pages written - this might legally be 0
1365 * if xbzrle noticed the page was the same.
1367 * @rs: current RAM state
1368 * @block: block that contains the page we want to send
1369 * @offset: offset inside the block for the page
1371 static int ram_save_page(RAMState
*rs
, PageSearchStatus
*pss
)
1375 bool send_async
= true;
1376 RAMBlock
*block
= pss
->block
;
1377 ram_addr_t offset
= ((ram_addr_t
)pss
->page
) << TARGET_PAGE_BITS
;
1378 ram_addr_t current_addr
= block
->offset
+ offset
;
1380 p
= block
->host
+ offset
;
1381 trace_ram_save_page(block
->idstr
, (uint64_t)offset
, p
);
1383 XBZRLE_cache_lock();
1384 if (rs
->xbzrle_enabled
&& !migration_in_postcopy()) {
1385 pages
= save_xbzrle_page(rs
, &p
, current_addr
, block
,
1387 if (!rs
->last_stage
) {
1388 /* Can't send this cached data async, since the cache page
1389 * might get updated before it gets to the wire
1395 /* XBZRLE overflow or normal page */
1397 pages
= save_normal_page(rs
, block
, offset
, p
, send_async
);
1400 XBZRLE_cache_unlock();
1405 static int ram_save_multifd_page(RAMState
*rs
, RAMBlock
*block
,
1408 if (multifd_queue_page(rs
->f
, block
, offset
) < 0) {
1411 ram_counters
.normal
++;
1416 static bool do_compress_ram_page(QEMUFile
*f
, z_stream
*stream
, RAMBlock
*block
,
1417 ram_addr_t offset
, uint8_t *source_buf
)
1419 RAMState
*rs
= ram_state
;
1420 uint8_t *p
= block
->host
+ offset
;
1423 if (save_zero_page_to_file(rs
, f
, block
, offset
)) {
1427 save_page_header(rs
, f
, block
, offset
| RAM_SAVE_FLAG_COMPRESS_PAGE
);
1430 * copy it to a internal buffer to avoid it being modified by VM
1431 * so that we can catch up the error during compression and
1434 memcpy(source_buf
, p
, TARGET_PAGE_SIZE
);
1435 ret
= qemu_put_compression_data(f
, stream
, source_buf
, TARGET_PAGE_SIZE
);
1437 qemu_file_set_error(migrate_get_current()->to_dst_file
, ret
);
1438 error_report("compressed data failed!");
1444 update_compress_thread_counts(const CompressParam
*param
, int bytes_xmit
)
1446 ram_transferred_add(bytes_xmit
);
1448 if (param
->zero_page
) {
1449 ram_counters
.duplicate
++;
1453 /* 8 means a header with RAM_SAVE_FLAG_CONTINUE. */
1454 compression_counters
.compressed_size
+= bytes_xmit
- 8;
1455 compression_counters
.pages
++;
1458 static bool save_page_use_compression(RAMState
*rs
);
1460 static void flush_compressed_data(RAMState
*rs
)
1462 int idx
, len
, thread_count
;
1464 if (!save_page_use_compression(rs
)) {
1467 thread_count
= migrate_compress_threads();
1469 qemu_mutex_lock(&comp_done_lock
);
1470 for (idx
= 0; idx
< thread_count
; idx
++) {
1471 while (!comp_param
[idx
].done
) {
1472 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
1475 qemu_mutex_unlock(&comp_done_lock
);
1477 for (idx
= 0; idx
< thread_count
; idx
++) {
1478 qemu_mutex_lock(&comp_param
[idx
].mutex
);
1479 if (!comp_param
[idx
].quit
) {
1480 len
= qemu_put_qemu_file(rs
->f
, comp_param
[idx
].file
);
1482 * it's safe to fetch zero_page without holding comp_done_lock
1483 * as there is no further request submitted to the thread,
1484 * i.e, the thread should be waiting for a request at this point.
1486 update_compress_thread_counts(&comp_param
[idx
], len
);
1488 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
1492 static inline void set_compress_params(CompressParam
*param
, RAMBlock
*block
,
1495 param
->block
= block
;
1496 param
->offset
= offset
;
1499 static int compress_page_with_multi_thread(RAMState
*rs
, RAMBlock
*block
,
1502 int idx
, thread_count
, bytes_xmit
= -1, pages
= -1;
1503 bool wait
= migrate_compress_wait_thread();
1505 thread_count
= migrate_compress_threads();
1506 qemu_mutex_lock(&comp_done_lock
);
1508 for (idx
= 0; idx
< thread_count
; idx
++) {
1509 if (comp_param
[idx
].done
) {
1510 comp_param
[idx
].done
= false;
1511 bytes_xmit
= qemu_put_qemu_file(rs
->f
, comp_param
[idx
].file
);
1512 qemu_mutex_lock(&comp_param
[idx
].mutex
);
1513 set_compress_params(&comp_param
[idx
], block
, offset
);
1514 qemu_cond_signal(&comp_param
[idx
].cond
);
1515 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
1517 update_compress_thread_counts(&comp_param
[idx
], bytes_xmit
);
1523 * wait for the free thread if the user specifies 'compress-wait-thread',
1524 * otherwise we will post the page out in the main thread as normal page.
1526 if (pages
< 0 && wait
) {
1527 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
1530 qemu_mutex_unlock(&comp_done_lock
);
1536 * find_dirty_block: find the next dirty page and update any state
1537 * associated with the search process.
1539 * Returns true if a page is found
1541 * @rs: current RAM state
1542 * @pss: data about the state of the current dirty page scan
1543 * @again: set to false if the search has scanned the whole of RAM
1545 static bool find_dirty_block(RAMState
*rs
, PageSearchStatus
*pss
, bool *again
)
1548 * This is not a postcopy requested page, mark it "not urgent", and use
1549 * precopy channel to send it.
1551 pss
->postcopy_requested
= false;
1552 pss
->postcopy_target_channel
= RAM_CHANNEL_PRECOPY
;
1554 pss
->page
= migration_bitmap_find_dirty(rs
, pss
->block
, pss
->page
);
1555 if (pss
->complete_round
&& pss
->block
== rs
->last_seen_block
&&
1556 pss
->page
>= rs
->last_page
) {
1558 * We've been once around the RAM and haven't found anything.
1564 if (!offset_in_ramblock(pss
->block
,
1565 ((ram_addr_t
)pss
->page
) << TARGET_PAGE_BITS
)) {
1566 /* Didn't find anything in this RAM Block */
1568 pss
->block
= QLIST_NEXT_RCU(pss
->block
, next
);
1571 * If memory migration starts over, we will meet a dirtied page
1572 * which may still exists in compression threads's ring, so we
1573 * should flush the compressed data to make sure the new page
1574 * is not overwritten by the old one in the destination.
1576 * Also If xbzrle is on, stop using the data compression at this
1577 * point. In theory, xbzrle can do better than compression.
1579 flush_compressed_data(rs
);
1581 /* Hit the end of the list */
1582 pss
->block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
1583 /* Flag that we've looped */
1584 pss
->complete_round
= true;
1585 /* After the first round, enable XBZRLE. */
1586 if (migrate_use_xbzrle()) {
1587 rs
->xbzrle_enabled
= true;
1590 /* Didn't find anything this time, but try again on the new block */
1594 /* Can go around again, but... */
1596 /* We've found something so probably don't need to */
1602 * unqueue_page: gets a page of the queue
1604 * Helper for 'get_queued_page' - gets a page off the queue
1606 * Returns the block of the page (or NULL if none available)
1608 * @rs: current RAM state
1609 * @offset: used to return the offset within the RAMBlock
1611 static RAMBlock
*unqueue_page(RAMState
*rs
, ram_addr_t
*offset
)
1613 struct RAMSrcPageRequest
*entry
;
1614 RAMBlock
*block
= NULL
;
1617 if (!postcopy_has_request(rs
)) {
1621 QEMU_LOCK_GUARD(&rs
->src_page_req_mutex
);
1624 * This should _never_ change even after we take the lock, because no one
1625 * should be taking anything off the request list other than us.
1627 assert(postcopy_has_request(rs
));
1629 entry
= QSIMPLEQ_FIRST(&rs
->src_page_requests
);
1631 *offset
= entry
->offset
;
1632 page_size
= qemu_ram_pagesize(block
);
1633 /* Each page request should only be multiple page size of the ramblock */
1634 assert((entry
->len
% page_size
) == 0);
1636 if (entry
->len
> page_size
) {
1637 entry
->len
-= page_size
;
1638 entry
->offset
+= page_size
;
1640 memory_region_unref(block
->mr
);
1641 QSIMPLEQ_REMOVE_HEAD(&rs
->src_page_requests
, next_req
);
1643 migration_consume_urgent_request();
1646 trace_unqueue_page(block
->idstr
, *offset
,
1647 test_bit((*offset
>> TARGET_PAGE_BITS
), block
->bmap
));
1652 #if defined(__linux__)
1654 * poll_fault_page: try to get next UFFD write fault page and, if pending fault
1655 * is found, return RAM block pointer and page offset
1657 * Returns pointer to the RAMBlock containing faulting page,
1658 * NULL if no write faults are pending
1660 * @rs: current RAM state
1661 * @offset: page offset from the beginning of the block
1663 static RAMBlock
*poll_fault_page(RAMState
*rs
, ram_addr_t
*offset
)
1665 struct uffd_msg uffd_msg
;
1670 if (!migrate_background_snapshot()) {
1674 res
= uffd_read_events(rs
->uffdio_fd
, &uffd_msg
, 1);
1679 page_address
= (void *)(uintptr_t) uffd_msg
.arg
.pagefault
.address
;
1680 block
= qemu_ram_block_from_host(page_address
, false, offset
);
1681 assert(block
&& (block
->flags
& RAM_UF_WRITEPROTECT
) != 0);
1686 * ram_save_release_protection: release UFFD write protection after
1687 * a range of pages has been saved
1689 * @rs: current RAM state
1690 * @pss: page-search-status structure
1691 * @start_page: index of the first page in the range relative to pss->block
1693 * Returns 0 on success, negative value in case of an error
1695 static int ram_save_release_protection(RAMState
*rs
, PageSearchStatus
*pss
,
1696 unsigned long start_page
)
1700 /* Check if page is from UFFD-managed region. */
1701 if (pss
->block
->flags
& RAM_UF_WRITEPROTECT
) {
1702 void *page_address
= pss
->block
->host
+ (start_page
<< TARGET_PAGE_BITS
);
1703 uint64_t run_length
= (pss
->page
- start_page
) << TARGET_PAGE_BITS
;
1705 /* Flush async buffers before un-protect. */
1707 /* Un-protect memory range. */
1708 res
= uffd_change_protection(rs
->uffdio_fd
, page_address
, run_length
,
1715 /* ram_write_tracking_available: check if kernel supports required UFFD features
1717 * Returns true if supports, false otherwise
1719 bool ram_write_tracking_available(void)
1721 uint64_t uffd_features
;
1724 res
= uffd_query_features(&uffd_features
);
1726 (uffd_features
& UFFD_FEATURE_PAGEFAULT_FLAG_WP
) != 0);
1729 /* ram_write_tracking_compatible: check if guest configuration is
1730 * compatible with 'write-tracking'
1732 * Returns true if compatible, false otherwise
1734 bool ram_write_tracking_compatible(void)
1736 const uint64_t uffd_ioctls_mask
= BIT(_UFFDIO_WRITEPROTECT
);
1741 /* Open UFFD file descriptor */
1742 uffd_fd
= uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP
, false);
1747 RCU_READ_LOCK_GUARD();
1749 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1750 uint64_t uffd_ioctls
;
1752 /* Nothing to do with read-only and MMIO-writable regions */
1753 if (block
->mr
->readonly
|| block
->mr
->rom_device
) {
1756 /* Try to register block memory via UFFD-IO to track writes */
1757 if (uffd_register_memory(uffd_fd
, block
->host
, block
->max_length
,
1758 UFFDIO_REGISTER_MODE_WP
, &uffd_ioctls
)) {
1761 if ((uffd_ioctls
& uffd_ioctls_mask
) != uffd_ioctls_mask
) {
1768 uffd_close_fd(uffd_fd
);
1772 static inline void populate_read_range(RAMBlock
*block
, ram_addr_t offset
,
1776 * We read one byte of each page; this will preallocate page tables if
1777 * required and populate the shared zeropage on MAP_PRIVATE anonymous memory
1778 * where no page was populated yet. This might require adaption when
1779 * supporting other mappings, like shmem.
1781 for (; offset
< size
; offset
+= block
->page_size
) {
1782 char tmp
= *((char *)block
->host
+ offset
);
1784 /* Don't optimize the read out */
1785 asm volatile("" : "+r" (tmp
));
1789 static inline int populate_read_section(MemoryRegionSection
*section
,
1792 const hwaddr size
= int128_get64(section
->size
);
1793 hwaddr offset
= section
->offset_within_region
;
1794 RAMBlock
*block
= section
->mr
->ram_block
;
1796 populate_read_range(block
, offset
, size
);
1801 * ram_block_populate_read: preallocate page tables and populate pages in the
1802 * RAM block by reading a byte of each page.
1804 * Since it's solely used for userfault_fd WP feature, here we just
1805 * hardcode page size to qemu_real_host_page_size.
1807 * @block: RAM block to populate
1809 static void ram_block_populate_read(RAMBlock
*rb
)
1812 * Skip populating all pages that fall into a discarded range as managed by
1813 * a RamDiscardManager responsible for the mapped memory region of the
1814 * RAMBlock. Such discarded ("logically unplugged") parts of a RAMBlock
1815 * must not get populated automatically. We don't have to track
1816 * modifications via userfaultfd WP reliably, because these pages will
1817 * not be part of the migration stream either way -- see
1818 * ramblock_dirty_bitmap_exclude_discarded_pages().
1820 * Note: The result is only stable while migrating (precopy/postcopy).
1822 if (rb
->mr
&& memory_region_has_ram_discard_manager(rb
->mr
)) {
1823 RamDiscardManager
*rdm
= memory_region_get_ram_discard_manager(rb
->mr
);
1824 MemoryRegionSection section
= {
1826 .offset_within_region
= 0,
1827 .size
= rb
->mr
->size
,
1830 ram_discard_manager_replay_populated(rdm
, §ion
,
1831 populate_read_section
, NULL
);
1833 populate_read_range(rb
, 0, rb
->used_length
);
1838 * ram_write_tracking_prepare: prepare for UFFD-WP memory tracking
1840 void ram_write_tracking_prepare(void)
1844 RCU_READ_LOCK_GUARD();
1846 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1847 /* Nothing to do with read-only and MMIO-writable regions */
1848 if (block
->mr
->readonly
|| block
->mr
->rom_device
) {
1853 * Populate pages of the RAM block before enabling userfault_fd
1856 * This stage is required since ioctl(UFFDIO_WRITEPROTECT) with
1857 * UFFDIO_WRITEPROTECT_MODE_WP mode setting would silently skip
1858 * pages with pte_none() entries in page table.
1860 ram_block_populate_read(block
);
1865 * ram_write_tracking_start: start UFFD-WP memory tracking
1867 * Returns 0 for success or negative value in case of error
1869 int ram_write_tracking_start(void)
1872 RAMState
*rs
= ram_state
;
1875 /* Open UFFD file descriptor */
1876 uffd_fd
= uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP
, true);
1880 rs
->uffdio_fd
= uffd_fd
;
1882 RCU_READ_LOCK_GUARD();
1884 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1885 /* Nothing to do with read-only and MMIO-writable regions */
1886 if (block
->mr
->readonly
|| block
->mr
->rom_device
) {
1890 /* Register block memory with UFFD to track writes */
1891 if (uffd_register_memory(rs
->uffdio_fd
, block
->host
,
1892 block
->max_length
, UFFDIO_REGISTER_MODE_WP
, NULL
)) {
1895 /* Apply UFFD write protection to the block memory range */
1896 if (uffd_change_protection(rs
->uffdio_fd
, block
->host
,
1897 block
->max_length
, true, false)) {
1900 block
->flags
|= RAM_UF_WRITEPROTECT
;
1901 memory_region_ref(block
->mr
);
1903 trace_ram_write_tracking_ramblock_start(block
->idstr
, block
->page_size
,
1904 block
->host
, block
->max_length
);
1910 error_report("ram_write_tracking_start() failed: restoring initial memory state");
1912 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1913 if ((block
->flags
& RAM_UF_WRITEPROTECT
) == 0) {
1917 * In case some memory block failed to be write-protected
1918 * remove protection and unregister all succeeded RAM blocks
1920 uffd_change_protection(rs
->uffdio_fd
, block
->host
, block
->max_length
,
1922 uffd_unregister_memory(rs
->uffdio_fd
, block
->host
, block
->max_length
);
1923 /* Cleanup flags and remove reference */
1924 block
->flags
&= ~RAM_UF_WRITEPROTECT
;
1925 memory_region_unref(block
->mr
);
1928 uffd_close_fd(uffd_fd
);
1934 * ram_write_tracking_stop: stop UFFD-WP memory tracking and remove protection
1936 void ram_write_tracking_stop(void)
1938 RAMState
*rs
= ram_state
;
1941 RCU_READ_LOCK_GUARD();
1943 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1944 if ((block
->flags
& RAM_UF_WRITEPROTECT
) == 0) {
1947 /* Remove protection and unregister all affected RAM blocks */
1948 uffd_change_protection(rs
->uffdio_fd
, block
->host
, block
->max_length
,
1950 uffd_unregister_memory(rs
->uffdio_fd
, block
->host
, block
->max_length
);
1952 trace_ram_write_tracking_ramblock_stop(block
->idstr
, block
->page_size
,
1953 block
->host
, block
->max_length
);
1955 /* Cleanup flags and remove reference */
1956 block
->flags
&= ~RAM_UF_WRITEPROTECT
;
1957 memory_region_unref(block
->mr
);
1960 /* Finally close UFFD file descriptor */
1961 uffd_close_fd(rs
->uffdio_fd
);
1966 /* No target OS support, stubs just fail or ignore */
1968 static RAMBlock
*poll_fault_page(RAMState
*rs
, ram_addr_t
*offset
)
1976 static int ram_save_release_protection(RAMState
*rs
, PageSearchStatus
*pss
,
1977 unsigned long start_page
)
1986 bool ram_write_tracking_available(void)
1991 bool ram_write_tracking_compatible(void)
1997 int ram_write_tracking_start(void)
2003 void ram_write_tracking_stop(void)
2007 #endif /* defined(__linux__) */
2010 * Check whether two addr/offset of the ramblock falls onto the same host huge
2011 * page. Returns true if so, false otherwise.
2013 static bool offset_on_same_huge_page(RAMBlock
*rb
, uint64_t addr1
,
2016 size_t page_size
= qemu_ram_pagesize(rb
);
2018 addr1
= ROUND_DOWN(addr1
, page_size
);
2019 addr2
= ROUND_DOWN(addr2
, page_size
);
2021 return addr1
== addr2
;
2025 * Whether a previous preempted precopy huge page contains current requested
2026 * page? Returns true if so, false otherwise.
2028 * This should really happen very rarely, because it means when we were sending
2029 * during background migration for postcopy we're sending exactly the page that
2030 * some vcpu got faulted on on dest node. When it happens, we probably don't
2031 * need to do much but drop the request, because we know right after we restore
2032 * the precopy stream it'll be serviced. It'll slightly affect the order of
2033 * postcopy requests to be serviced (e.g. it'll be the same as we move current
2034 * request to the end of the queue) but it shouldn't be a big deal. The most
2035 * imporant thing is we can _never_ try to send a partial-sent huge page on the
2036 * POSTCOPY channel again, otherwise that huge page will got "split brain" on
2037 * two channels (PRECOPY, POSTCOPY).
2039 static bool postcopy_preempted_contains(RAMState
*rs
, RAMBlock
*block
,
2042 PostcopyPreemptState
*state
= &rs
->postcopy_preempt_state
;
2044 /* No preemption at all? */
2045 if (!state
->preempted
) {
2049 /* Not even the same ramblock? */
2050 if (state
->ram_block
!= block
) {
2054 return offset_on_same_huge_page(block
, offset
,
2055 state
->ram_page
<< TARGET_PAGE_BITS
);
2059 * get_queued_page: unqueue a page from the postcopy requests
2061 * Skips pages that are already sent (!dirty)
2063 * Returns true if a queued page is found
2065 * @rs: current RAM state
2066 * @pss: data about the state of the current dirty page scan
2068 static bool get_queued_page(RAMState
*rs
, PageSearchStatus
*pss
)
2073 block
= unqueue_page(rs
, &offset
);
2076 /* See comment above postcopy_preempted_contains() */
2077 if (postcopy_preempted_contains(rs
, block
, offset
)) {
2078 trace_postcopy_preempt_hit(block
->idstr
, offset
);
2080 * If what we preempted previously was exactly what we're
2081 * requesting right now, restore the preempted precopy
2082 * immediately, boosting its priority as it's requested by
2085 postcopy_preempt_restore(rs
, pss
, true);
2090 * Poll write faults too if background snapshot is enabled; that's
2091 * when we have vcpus got blocked by the write protected pages.
2093 block
= poll_fault_page(rs
, &offset
);
2098 * We want the background search to continue from the queued page
2099 * since the guest is likely to want other pages near to the page
2100 * it just requested.
2103 pss
->page
= offset
>> TARGET_PAGE_BITS
;
2106 * This unqueued page would break the "one round" check, even is
2109 pss
->complete_round
= false;
2110 /* Mark it an urgent request, meanwhile using POSTCOPY channel */
2111 pss
->postcopy_requested
= true;
2112 pss
->postcopy_target_channel
= RAM_CHANNEL_POSTCOPY
;
2119 * migration_page_queue_free: drop any remaining pages in the ram
2122 * It should be empty at the end anyway, but in error cases there may
2123 * be some left. in case that there is any page left, we drop it.
2126 static void migration_page_queue_free(RAMState
*rs
)
2128 struct RAMSrcPageRequest
*mspr
, *next_mspr
;
2129 /* This queue generally should be empty - but in the case of a failed
2130 * migration might have some droppings in.
2132 RCU_READ_LOCK_GUARD();
2133 QSIMPLEQ_FOREACH_SAFE(mspr
, &rs
->src_page_requests
, next_req
, next_mspr
) {
2134 memory_region_unref(mspr
->rb
->mr
);
2135 QSIMPLEQ_REMOVE_HEAD(&rs
->src_page_requests
, next_req
);
2141 * ram_save_queue_pages: queue the page for transmission
2143 * A request from postcopy destination for example.
2145 * Returns zero on success or negative on error
2147 * @rbname: Name of the RAMBLock of the request. NULL means the
2148 * same that last one.
2149 * @start: starting address from the start of the RAMBlock
2150 * @len: length (in bytes) to send
2152 int ram_save_queue_pages(const char *rbname
, ram_addr_t start
, ram_addr_t len
)
2155 RAMState
*rs
= ram_state
;
2157 ram_counters
.postcopy_requests
++;
2158 RCU_READ_LOCK_GUARD();
2161 /* Reuse last RAMBlock */
2162 ramblock
= rs
->last_req_rb
;
2166 * Shouldn't happen, we can't reuse the last RAMBlock if
2167 * it's the 1st request.
2169 error_report("ram_save_queue_pages no previous block");
2173 ramblock
= qemu_ram_block_by_name(rbname
);
2176 /* We shouldn't be asked for a non-existent RAMBlock */
2177 error_report("ram_save_queue_pages no block '%s'", rbname
);
2180 rs
->last_req_rb
= ramblock
;
2182 trace_ram_save_queue_pages(ramblock
->idstr
, start
, len
);
2183 if (!offset_in_ramblock(ramblock
, start
+ len
- 1)) {
2184 error_report("%s request overrun start=" RAM_ADDR_FMT
" len="
2185 RAM_ADDR_FMT
" blocklen=" RAM_ADDR_FMT
,
2186 __func__
, start
, len
, ramblock
->used_length
);
2190 struct RAMSrcPageRequest
*new_entry
=
2191 g_new0(struct RAMSrcPageRequest
, 1);
2192 new_entry
->rb
= ramblock
;
2193 new_entry
->offset
= start
;
2194 new_entry
->len
= len
;
2196 memory_region_ref(ramblock
->mr
);
2197 qemu_mutex_lock(&rs
->src_page_req_mutex
);
2198 QSIMPLEQ_INSERT_TAIL(&rs
->src_page_requests
, new_entry
, next_req
);
2199 migration_make_urgent_request();
2200 qemu_mutex_unlock(&rs
->src_page_req_mutex
);
2205 static bool save_page_use_compression(RAMState
*rs
)
2207 if (!migrate_use_compression()) {
2212 * If xbzrle is enabled (e.g., after first round of migration), stop
2213 * using the data compression. In theory, xbzrle can do better than
2216 if (rs
->xbzrle_enabled
) {
2224 * try to compress the page before posting it out, return true if the page
2225 * has been properly handled by compression, otherwise needs other
2226 * paths to handle it
2228 static bool save_compress_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
)
2230 if (!save_page_use_compression(rs
)) {
2235 * When starting the process of a new block, the first page of
2236 * the block should be sent out before other pages in the same
2237 * block, and all the pages in last block should have been sent
2238 * out, keeping this order is important, because the 'cont' flag
2239 * is used to avoid resending the block name.
2241 * We post the fist page as normal page as compression will take
2242 * much CPU resource.
2244 if (block
!= rs
->last_sent_block
) {
2245 flush_compressed_data(rs
);
2249 if (compress_page_with_multi_thread(rs
, block
, offset
) > 0) {
2253 compression_counters
.busy
++;
2258 * ram_save_target_page: save one target page
2260 * Returns the number of pages written
2262 * @rs: current RAM state
2263 * @pss: data about the page we want to send
2265 static int ram_save_target_page(RAMState
*rs
, PageSearchStatus
*pss
)
2267 RAMBlock
*block
= pss
->block
;
2268 ram_addr_t offset
= ((ram_addr_t
)pss
->page
) << TARGET_PAGE_BITS
;
2271 if (control_save_page(rs
, block
, offset
, &res
)) {
2275 if (save_compress_page(rs
, block
, offset
)) {
2279 res
= save_zero_page(rs
, block
, offset
);
2281 /* Must let xbzrle know, otherwise a previous (now 0'd) cached
2282 * page would be stale
2284 if (!save_page_use_compression(rs
)) {
2285 XBZRLE_cache_lock();
2286 xbzrle_cache_zero_page(rs
, block
->offset
+ offset
);
2287 XBZRLE_cache_unlock();
2293 * Do not use multifd for:
2294 * 1. Compression as the first page in the new block should be posted out
2295 * before sending the compressed page
2296 * 2. In postcopy as one whole host page should be placed
2298 if (!save_page_use_compression(rs
) && migrate_use_multifd()
2299 && !migration_in_postcopy()) {
2300 return ram_save_multifd_page(rs
, block
, offset
);
2303 return ram_save_page(rs
, pss
);
2306 static bool postcopy_needs_preempt(RAMState
*rs
, PageSearchStatus
*pss
)
2308 MigrationState
*ms
= migrate_get_current();
2310 /* Not enabled eager preempt? Then never do that. */
2311 if (!migrate_postcopy_preempt()) {
2315 /* If the user explicitly disabled breaking of huge page, skip */
2316 if (!ms
->postcopy_preempt_break_huge
) {
2320 /* If the ramblock we're sending is a small page? Never bother. */
2321 if (qemu_ram_pagesize(pss
->block
) == TARGET_PAGE_SIZE
) {
2325 /* Not in postcopy at all? */
2326 if (!migration_in_postcopy()) {
2331 * If we're already handling a postcopy request, don't preempt as this page
2332 * has got the same high priority.
2334 if (pss
->postcopy_requested
) {
2338 /* If there's postcopy requests, then check it up! */
2339 return postcopy_has_request(rs
);
2342 /* Returns true if we preempted precopy, false otherwise */
2343 static void postcopy_do_preempt(RAMState
*rs
, PageSearchStatus
*pss
)
2345 PostcopyPreemptState
*p_state
= &rs
->postcopy_preempt_state
;
2347 trace_postcopy_preempt_triggered(pss
->block
->idstr
, pss
->page
);
2350 * Time to preempt precopy. Cache current PSS into preempt state, so that
2351 * after handling the postcopy pages we can recover to it. We need to do
2352 * so because the dest VM will have partial of the precopy huge page kept
2353 * over in its tmp huge page caches; better move on with it when we can.
2355 p_state
->ram_block
= pss
->block
;
2356 p_state
->ram_page
= pss
->page
;
2357 p_state
->preempted
= true;
2360 /* Whether we're preempted by a postcopy request during sending a huge page */
2361 static bool postcopy_preempt_triggered(RAMState
*rs
)
2363 return rs
->postcopy_preempt_state
.preempted
;
2366 static void postcopy_preempt_restore(RAMState
*rs
, PageSearchStatus
*pss
,
2367 bool postcopy_requested
)
2369 PostcopyPreemptState
*state
= &rs
->postcopy_preempt_state
;
2371 assert(state
->preempted
);
2373 pss
->block
= state
->ram_block
;
2374 pss
->page
= state
->ram_page
;
2376 /* Whether this is a postcopy request? */
2377 pss
->postcopy_requested
= postcopy_requested
;
2379 * When restoring a preempted page, the old data resides in PRECOPY
2380 * slow channel, even if postcopy_requested is set. So always use
2381 * PRECOPY channel here.
2383 pss
->postcopy_target_channel
= RAM_CHANNEL_PRECOPY
;
2385 trace_postcopy_preempt_restored(pss
->block
->idstr
, pss
->page
);
2387 /* Reset preempt state, most importantly, set preempted==false */
2388 postcopy_preempt_reset(rs
);
2391 static void postcopy_preempt_choose_channel(RAMState
*rs
, PageSearchStatus
*pss
)
2393 MigrationState
*s
= migrate_get_current();
2394 unsigned int channel
= pss
->postcopy_target_channel
;
2397 if (channel
!= rs
->postcopy_channel
) {
2398 if (channel
== RAM_CHANNEL_PRECOPY
) {
2399 next
= s
->to_dst_file
;
2401 next
= s
->postcopy_qemufile_src
;
2403 /* Update and cache the current channel */
2405 rs
->postcopy_channel
= channel
;
2408 * If channel switched, reset last_sent_block since the old sent block
2409 * may not be on the same channel.
2411 rs
->last_sent_block
= NULL
;
2413 trace_postcopy_preempt_switch_channel(channel
);
2416 trace_postcopy_preempt_send_host_page(pss
->block
->idstr
, pss
->page
);
2419 /* We need to make sure rs->f always points to the default channel elsewhere */
2420 static void postcopy_preempt_reset_channel(RAMState
*rs
)
2422 if (migrate_postcopy_preempt() && migration_in_postcopy()) {
2423 rs
->postcopy_channel
= RAM_CHANNEL_PRECOPY
;
2424 rs
->f
= migrate_get_current()->to_dst_file
;
2425 trace_postcopy_preempt_reset_channel();
2430 * ram_save_host_page: save a whole host page
2432 * Starting at *offset send pages up to the end of the current host
2433 * page. It's valid for the initial offset to point into the middle of
2434 * a host page in which case the remainder of the hostpage is sent.
2435 * Only dirty target pages are sent. Note that the host page size may
2436 * be a huge page for this block.
2437 * The saving stops at the boundary of the used_length of the block
2438 * if the RAMBlock isn't a multiple of the host page size.
2440 * Returns the number of pages written or negative on error
2442 * @rs: current RAM state
2443 * @pss: data about the page we want to send
2445 static int ram_save_host_page(RAMState
*rs
, PageSearchStatus
*pss
)
2447 int tmppages
, pages
= 0;
2448 size_t pagesize_bits
=
2449 qemu_ram_pagesize(pss
->block
) >> TARGET_PAGE_BITS
;
2450 unsigned long hostpage_boundary
=
2451 QEMU_ALIGN_UP(pss
->page
+ 1, pagesize_bits
);
2452 unsigned long start_page
= pss
->page
;
2455 if (ramblock_is_ignored(pss
->block
)) {
2456 error_report("block %s should not be migrated !", pss
->block
->idstr
);
2460 if (migrate_postcopy_preempt() && migration_in_postcopy()) {
2461 postcopy_preempt_choose_channel(rs
, pss
);
2465 if (postcopy_needs_preempt(rs
, pss
)) {
2466 postcopy_do_preempt(rs
, pss
);
2470 /* Check the pages is dirty and if it is send it */
2471 if (migration_bitmap_clear_dirty(rs
, pss
->block
, pss
->page
)) {
2472 tmppages
= ram_save_target_page(rs
, pss
);
2479 * Allow rate limiting to happen in the middle of huge pages if
2480 * something is sent in the current iteration.
2482 if (pagesize_bits
> 1 && tmppages
> 0) {
2483 migration_rate_limit();
2486 pss
->page
= migration_bitmap_find_dirty(rs
, pss
->block
, pss
->page
);
2487 } while ((pss
->page
< hostpage_boundary
) &&
2488 offset_in_ramblock(pss
->block
,
2489 ((ram_addr_t
)pss
->page
) << TARGET_PAGE_BITS
));
2490 /* The offset we leave with is the min boundary of host page and block */
2491 pss
->page
= MIN(pss
->page
, hostpage_boundary
);
2494 * When with postcopy preempt mode, flush the data as soon as possible for
2495 * postcopy requests, because we've already sent a whole huge page, so the
2496 * dst node should already have enough resource to atomically filling in
2497 * the current missing page.
2499 * More importantly, when using separate postcopy channel, we must do
2500 * explicit flush or it won't flush until the buffer is full.
2502 if (migrate_postcopy_preempt() && pss
->postcopy_requested
) {
2506 res
= ram_save_release_protection(rs
, pss
, start_page
);
2507 return (res
< 0 ? res
: pages
);
2511 * ram_find_and_save_block: finds a dirty page and sends it to f
2513 * Called within an RCU critical section.
2515 * Returns the number of pages written where zero means no dirty pages,
2516 * or negative on error
2518 * @rs: current RAM state
2520 * On systems where host-page-size > target-page-size it will send all the
2521 * pages in a host page that are dirty.
2523 static int ram_find_and_save_block(RAMState
*rs
)
2525 PageSearchStatus pss
;
2529 /* No dirty page as there is zero RAM */
2530 if (!ram_bytes_total()) {
2534 pss
.block
= rs
->last_seen_block
;
2535 pss
.page
= rs
->last_page
;
2536 pss
.complete_round
= false;
2539 pss
.block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
2544 found
= get_queued_page(rs
, &pss
);
2548 * Recover previous precopy ramblock/offset if postcopy has
2549 * preempted precopy. Otherwise find the next dirty bit.
2551 if (postcopy_preempt_triggered(rs
)) {
2552 postcopy_preempt_restore(rs
, &pss
, false);
2555 /* priority queue empty, so just search for something dirty */
2556 found
= find_dirty_block(rs
, &pss
, &again
);
2561 pages
= ram_save_host_page(rs
, &pss
);
2563 } while (!pages
&& again
);
2565 rs
->last_seen_block
= pss
.block
;
2566 rs
->last_page
= pss
.page
;
2571 void acct_update_position(QEMUFile
*f
, size_t size
, bool zero
)
2573 uint64_t pages
= size
/ TARGET_PAGE_SIZE
;
2576 ram_counters
.duplicate
+= pages
;
2578 ram_counters
.normal
+= pages
;
2579 ram_transferred_add(size
);
2580 qemu_file_credit_transfer(f
, size
);
2584 static uint64_t ram_bytes_total_common(bool count_ignored
)
2589 RCU_READ_LOCK_GUARD();
2591 if (count_ignored
) {
2592 RAMBLOCK_FOREACH_MIGRATABLE(block
) {
2593 total
+= block
->used_length
;
2596 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2597 total
+= block
->used_length
;
2603 uint64_t ram_bytes_total(void)
2605 return ram_bytes_total_common(false);
2608 static void xbzrle_load_setup(void)
2610 XBZRLE
.decoded_buf
= g_malloc(TARGET_PAGE_SIZE
);
2613 static void xbzrle_load_cleanup(void)
2615 g_free(XBZRLE
.decoded_buf
);
2616 XBZRLE
.decoded_buf
= NULL
;
2619 static void ram_state_cleanup(RAMState
**rsp
)
2622 migration_page_queue_free(*rsp
);
2623 qemu_mutex_destroy(&(*rsp
)->bitmap_mutex
);
2624 qemu_mutex_destroy(&(*rsp
)->src_page_req_mutex
);
2630 static void xbzrle_cleanup(void)
2632 XBZRLE_cache_lock();
2634 cache_fini(XBZRLE
.cache
);
2635 g_free(XBZRLE
.encoded_buf
);
2636 g_free(XBZRLE
.current_buf
);
2637 g_free(XBZRLE
.zero_target_page
);
2638 XBZRLE
.cache
= NULL
;
2639 XBZRLE
.encoded_buf
= NULL
;
2640 XBZRLE
.current_buf
= NULL
;
2641 XBZRLE
.zero_target_page
= NULL
;
2643 XBZRLE_cache_unlock();
2646 static void ram_save_cleanup(void *opaque
)
2648 RAMState
**rsp
= opaque
;
2651 /* We don't use dirty log with background snapshots */
2652 if (!migrate_background_snapshot()) {
2653 /* caller have hold iothread lock or is in a bh, so there is
2654 * no writing race against the migration bitmap
2656 if (global_dirty_tracking
& GLOBAL_DIRTY_MIGRATION
) {
2658 * do not stop dirty log without starting it, since
2659 * memory_global_dirty_log_stop will assert that
2660 * memory_global_dirty_log_start/stop used in pairs
2662 memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION
);
2666 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2667 g_free(block
->clear_bmap
);
2668 block
->clear_bmap
= NULL
;
2669 g_free(block
->bmap
);
2674 compress_threads_save_cleanup();
2675 ram_state_cleanup(rsp
);
2678 static void ram_state_reset(RAMState
*rs
)
2680 rs
->last_seen_block
= NULL
;
2681 rs
->last_sent_block
= NULL
;
2683 rs
->last_version
= ram_list
.version
;
2684 rs
->xbzrle_enabled
= false;
2685 postcopy_preempt_reset(rs
);
2686 rs
->postcopy_channel
= RAM_CHANNEL_PRECOPY
;
2689 #define MAX_WAIT 50 /* ms, half buffered_file limit */
2691 /* **** functions for postcopy ***** */
2693 void ram_postcopy_migrated_memory_release(MigrationState
*ms
)
2695 struct RAMBlock
*block
;
2697 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2698 unsigned long *bitmap
= block
->bmap
;
2699 unsigned long range
= block
->used_length
>> TARGET_PAGE_BITS
;
2700 unsigned long run_start
= find_next_zero_bit(bitmap
, range
, 0);
2702 while (run_start
< range
) {
2703 unsigned long run_end
= find_next_bit(bitmap
, range
, run_start
+ 1);
2704 ram_discard_range(block
->idstr
,
2705 ((ram_addr_t
)run_start
) << TARGET_PAGE_BITS
,
2706 ((ram_addr_t
)(run_end
- run_start
))
2707 << TARGET_PAGE_BITS
);
2708 run_start
= find_next_zero_bit(bitmap
, range
, run_end
+ 1);
2714 * postcopy_send_discard_bm_ram: discard a RAMBlock
2716 * Callback from postcopy_each_ram_send_discard for each RAMBlock
2718 * @ms: current migration state
2719 * @block: RAMBlock to discard
2721 static void postcopy_send_discard_bm_ram(MigrationState
*ms
, RAMBlock
*block
)
2723 unsigned long end
= block
->used_length
>> TARGET_PAGE_BITS
;
2724 unsigned long current
;
2725 unsigned long *bitmap
= block
->bmap
;
2727 for (current
= 0; current
< end
; ) {
2728 unsigned long one
= find_next_bit(bitmap
, end
, current
);
2729 unsigned long zero
, discard_length
;
2735 zero
= find_next_zero_bit(bitmap
, end
, one
+ 1);
2738 discard_length
= end
- one
;
2740 discard_length
= zero
- one
;
2742 postcopy_discard_send_range(ms
, one
, discard_length
);
2743 current
= one
+ discard_length
;
2747 static void postcopy_chunk_hostpages_pass(MigrationState
*ms
, RAMBlock
*block
);
2750 * postcopy_each_ram_send_discard: discard all RAMBlocks
2752 * Utility for the outgoing postcopy code.
2753 * Calls postcopy_send_discard_bm_ram for each RAMBlock
2754 * passing it bitmap indexes and name.
2755 * (qemu_ram_foreach_block ends up passing unscaled lengths
2756 * which would mean postcopy code would have to deal with target page)
2758 * @ms: current migration state
2760 static void postcopy_each_ram_send_discard(MigrationState
*ms
)
2762 struct RAMBlock
*block
;
2764 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2765 postcopy_discard_send_init(ms
, block
->idstr
);
2768 * Deal with TPS != HPS and huge pages. It discard any partially sent
2769 * host-page size chunks, mark any partially dirty host-page size
2770 * chunks as all dirty. In this case the host-page is the host-page
2771 * for the particular RAMBlock, i.e. it might be a huge page.
2773 postcopy_chunk_hostpages_pass(ms
, block
);
2776 * Postcopy sends chunks of bitmap over the wire, but it
2777 * just needs indexes at this point, avoids it having
2778 * target page specific code.
2780 postcopy_send_discard_bm_ram(ms
, block
);
2781 postcopy_discard_send_finish(ms
);
2786 * postcopy_chunk_hostpages_pass: canonicalize bitmap in hostpages
2788 * Helper for postcopy_chunk_hostpages; it's called twice to
2789 * canonicalize the two bitmaps, that are similar, but one is
2792 * Postcopy requires that all target pages in a hostpage are dirty or
2793 * clean, not a mix. This function canonicalizes the bitmaps.
2795 * @ms: current migration state
2796 * @block: block that contains the page we want to canonicalize
2798 static void postcopy_chunk_hostpages_pass(MigrationState
*ms
, RAMBlock
*block
)
2800 RAMState
*rs
= ram_state
;
2801 unsigned long *bitmap
= block
->bmap
;
2802 unsigned int host_ratio
= block
->page_size
/ TARGET_PAGE_SIZE
;
2803 unsigned long pages
= block
->used_length
>> TARGET_PAGE_BITS
;
2804 unsigned long run_start
;
2806 if (block
->page_size
== TARGET_PAGE_SIZE
) {
2807 /* Easy case - TPS==HPS for a non-huge page RAMBlock */
2811 /* Find a dirty page */
2812 run_start
= find_next_bit(bitmap
, pages
, 0);
2814 while (run_start
< pages
) {
2817 * If the start of this run of pages is in the middle of a host
2818 * page, then we need to fixup this host page.
2820 if (QEMU_IS_ALIGNED(run_start
, host_ratio
)) {
2821 /* Find the end of this run */
2822 run_start
= find_next_zero_bit(bitmap
, pages
, run_start
+ 1);
2824 * If the end isn't at the start of a host page, then the
2825 * run doesn't finish at the end of a host page
2826 * and we need to discard.
2830 if (!QEMU_IS_ALIGNED(run_start
, host_ratio
)) {
2832 unsigned long fixup_start_addr
= QEMU_ALIGN_DOWN(run_start
,
2834 run_start
= QEMU_ALIGN_UP(run_start
, host_ratio
);
2836 /* Clean up the bitmap */
2837 for (page
= fixup_start_addr
;
2838 page
< fixup_start_addr
+ host_ratio
; page
++) {
2840 * Remark them as dirty, updating the count for any pages
2841 * that weren't previously dirty.
2843 rs
->migration_dirty_pages
+= !test_and_set_bit(page
, bitmap
);
2847 /* Find the next dirty page for the next iteration */
2848 run_start
= find_next_bit(bitmap
, pages
, run_start
);
2853 * ram_postcopy_send_discard_bitmap: transmit the discard bitmap
2855 * Transmit the set of pages to be discarded after precopy to the target
2856 * these are pages that:
2857 * a) Have been previously transmitted but are now dirty again
2858 * b) Pages that have never been transmitted, this ensures that
2859 * any pages on the destination that have been mapped by background
2860 * tasks get discarded (transparent huge pages is the specific concern)
2861 * Hopefully this is pretty sparse
2863 * @ms: current migration state
2865 void ram_postcopy_send_discard_bitmap(MigrationState
*ms
)
2867 RAMState
*rs
= ram_state
;
2869 RCU_READ_LOCK_GUARD();
2871 /* This should be our last sync, the src is now paused */
2872 migration_bitmap_sync(rs
);
2874 /* Easiest way to make sure we don't resume in the middle of a host-page */
2875 rs
->last_seen_block
= NULL
;
2876 rs
->last_sent_block
= NULL
;
2879 postcopy_each_ram_send_discard(ms
);
2881 trace_ram_postcopy_send_discard_bitmap();
2885 * ram_discard_range: discard dirtied pages at the beginning of postcopy
2887 * Returns zero on success
2889 * @rbname: name of the RAMBlock of the request. NULL means the
2890 * same that last one.
2891 * @start: RAMBlock starting page
2892 * @length: RAMBlock size
2894 int ram_discard_range(const char *rbname
, uint64_t start
, size_t length
)
2896 trace_ram_discard_range(rbname
, start
, length
);
2898 RCU_READ_LOCK_GUARD();
2899 RAMBlock
*rb
= qemu_ram_block_by_name(rbname
);
2902 error_report("ram_discard_range: Failed to find block '%s'", rbname
);
2907 * On source VM, we don't need to update the received bitmap since
2908 * we don't even have one.
2910 if (rb
->receivedmap
) {
2911 bitmap_clear(rb
->receivedmap
, start
>> qemu_target_page_bits(),
2912 length
>> qemu_target_page_bits());
2915 return ram_block_discard_range(rb
, start
, length
);
2919 * For every allocation, we will try not to crash the VM if the
2920 * allocation failed.
2922 static int xbzrle_init(void)
2924 Error
*local_err
= NULL
;
2926 if (!migrate_use_xbzrle()) {
2930 XBZRLE_cache_lock();
2932 XBZRLE
.zero_target_page
= g_try_malloc0(TARGET_PAGE_SIZE
);
2933 if (!XBZRLE
.zero_target_page
) {
2934 error_report("%s: Error allocating zero page", __func__
);
2938 XBZRLE
.cache
= cache_init(migrate_xbzrle_cache_size(),
2939 TARGET_PAGE_SIZE
, &local_err
);
2940 if (!XBZRLE
.cache
) {
2941 error_report_err(local_err
);
2942 goto free_zero_page
;
2945 XBZRLE
.encoded_buf
= g_try_malloc0(TARGET_PAGE_SIZE
);
2946 if (!XBZRLE
.encoded_buf
) {
2947 error_report("%s: Error allocating encoded_buf", __func__
);
2951 XBZRLE
.current_buf
= g_try_malloc(TARGET_PAGE_SIZE
);
2952 if (!XBZRLE
.current_buf
) {
2953 error_report("%s: Error allocating current_buf", __func__
);
2954 goto free_encoded_buf
;
2957 /* We are all good */
2958 XBZRLE_cache_unlock();
2962 g_free(XBZRLE
.encoded_buf
);
2963 XBZRLE
.encoded_buf
= NULL
;
2965 cache_fini(XBZRLE
.cache
);
2966 XBZRLE
.cache
= NULL
;
2968 g_free(XBZRLE
.zero_target_page
);
2969 XBZRLE
.zero_target_page
= NULL
;
2971 XBZRLE_cache_unlock();
2975 static int ram_state_init(RAMState
**rsp
)
2977 *rsp
= g_try_new0(RAMState
, 1);
2980 error_report("%s: Init ramstate fail", __func__
);
2984 qemu_mutex_init(&(*rsp
)->bitmap_mutex
);
2985 qemu_mutex_init(&(*rsp
)->src_page_req_mutex
);
2986 QSIMPLEQ_INIT(&(*rsp
)->src_page_requests
);
2989 * Count the total number of pages used by ram blocks not including any
2990 * gaps due to alignment or unplugs.
2991 * This must match with the initial values of dirty bitmap.
2993 (*rsp
)->migration_dirty_pages
= ram_bytes_total() >> TARGET_PAGE_BITS
;
2994 ram_state_reset(*rsp
);
2999 static void ram_list_init_bitmaps(void)
3001 MigrationState
*ms
= migrate_get_current();
3003 unsigned long pages
;
3006 /* Skip setting bitmap if there is no RAM */
3007 if (ram_bytes_total()) {
3008 shift
= ms
->clear_bitmap_shift
;
3009 if (shift
> CLEAR_BITMAP_SHIFT_MAX
) {
3010 error_report("clear_bitmap_shift (%u) too big, using "
3011 "max value (%u)", shift
, CLEAR_BITMAP_SHIFT_MAX
);
3012 shift
= CLEAR_BITMAP_SHIFT_MAX
;
3013 } else if (shift
< CLEAR_BITMAP_SHIFT_MIN
) {
3014 error_report("clear_bitmap_shift (%u) too small, using "
3015 "min value (%u)", shift
, CLEAR_BITMAP_SHIFT_MIN
);
3016 shift
= CLEAR_BITMAP_SHIFT_MIN
;
3019 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3020 pages
= block
->max_length
>> TARGET_PAGE_BITS
;
3022 * The initial dirty bitmap for migration must be set with all
3023 * ones to make sure we'll migrate every guest RAM page to
3025 * Here we set RAMBlock.bmap all to 1 because when rebegin a
3026 * new migration after a failed migration, ram_list.
3027 * dirty_memory[DIRTY_MEMORY_MIGRATION] don't include the whole
3030 block
->bmap
= bitmap_new(pages
);
3031 bitmap_set(block
->bmap
, 0, pages
);
3032 block
->clear_bmap_shift
= shift
;
3033 block
->clear_bmap
= bitmap_new(clear_bmap_size(pages
, shift
));
3038 static void migration_bitmap_clear_discarded_pages(RAMState
*rs
)
3040 unsigned long pages
;
3043 RCU_READ_LOCK_GUARD();
3045 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
3046 pages
= ramblock_dirty_bitmap_clear_discarded_pages(rb
);
3047 rs
->migration_dirty_pages
-= pages
;
3051 static void ram_init_bitmaps(RAMState
*rs
)
3053 /* For memory_global_dirty_log_start below. */
3054 qemu_mutex_lock_iothread();
3055 qemu_mutex_lock_ramlist();
3057 WITH_RCU_READ_LOCK_GUARD() {
3058 ram_list_init_bitmaps();
3059 /* We don't use dirty log with background snapshots */
3060 if (!migrate_background_snapshot()) {
3061 memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION
);
3062 migration_bitmap_sync_precopy(rs
);
3065 qemu_mutex_unlock_ramlist();
3066 qemu_mutex_unlock_iothread();
3069 * After an eventual first bitmap sync, fixup the initial bitmap
3070 * containing all 1s to exclude any discarded pages from migration.
3072 migration_bitmap_clear_discarded_pages(rs
);
3075 static int ram_init_all(RAMState
**rsp
)
3077 if (ram_state_init(rsp
)) {
3081 if (xbzrle_init()) {
3082 ram_state_cleanup(rsp
);
3086 ram_init_bitmaps(*rsp
);
3091 static void ram_state_resume_prepare(RAMState
*rs
, QEMUFile
*out
)
3097 * Postcopy is not using xbzrle/compression, so no need for that.
3098 * Also, since source are already halted, we don't need to care
3099 * about dirty page logging as well.
3102 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3103 pages
+= bitmap_count_one(block
->bmap
,
3104 block
->used_length
>> TARGET_PAGE_BITS
);
3107 /* This may not be aligned with current bitmaps. Recalculate. */
3108 rs
->migration_dirty_pages
= pages
;
3110 ram_state_reset(rs
);
3112 /* Update RAMState cache of output QEMUFile */
3115 trace_ram_state_resume_prepare(pages
);
3119 * This function clears bits of the free pages reported by the caller from the
3120 * migration dirty bitmap. @addr is the host address corresponding to the
3121 * start of the continuous guest free pages, and @len is the total bytes of
3124 void qemu_guest_free_page_hint(void *addr
, size_t len
)
3128 size_t used_len
, start
, npages
;
3129 MigrationState
*s
= migrate_get_current();
3131 /* This function is currently expected to be used during live migration */
3132 if (!migration_is_setup_or_active(s
->state
)) {
3136 for (; len
> 0; len
-= used_len
, addr
+= used_len
) {
3137 block
= qemu_ram_block_from_host(addr
, false, &offset
);
3138 if (unlikely(!block
|| offset
>= block
->used_length
)) {
3140 * The implementation might not support RAMBlock resize during
3141 * live migration, but it could happen in theory with future
3142 * updates. So we add a check here to capture that case.
3144 error_report_once("%s unexpected error", __func__
);
3148 if (len
<= block
->used_length
- offset
) {
3151 used_len
= block
->used_length
- offset
;
3154 start
= offset
>> TARGET_PAGE_BITS
;
3155 npages
= used_len
>> TARGET_PAGE_BITS
;
3157 qemu_mutex_lock(&ram_state
->bitmap_mutex
);
3159 * The skipped free pages are equavalent to be sent from clear_bmap's
3160 * perspective, so clear the bits from the memory region bitmap which
3161 * are initially set. Otherwise those skipped pages will be sent in
3162 * the next round after syncing from the memory region bitmap.
3164 migration_clear_memory_region_dirty_bitmap_range(block
, start
, npages
);
3165 ram_state
->migration_dirty_pages
-=
3166 bitmap_count_one_with_offset(block
->bmap
, start
, npages
);
3167 bitmap_clear(block
->bmap
, start
, npages
);
3168 qemu_mutex_unlock(&ram_state
->bitmap_mutex
);
3173 * Each of ram_save_setup, ram_save_iterate and ram_save_complete has
3174 * long-running RCU critical section. When rcu-reclaims in the code
3175 * start to become numerous it will be necessary to reduce the
3176 * granularity of these critical sections.
3180 * ram_save_setup: Setup RAM for migration
3182 * Returns zero to indicate success and negative for error
3184 * @f: QEMUFile where to send the data
3185 * @opaque: RAMState pointer
3187 static int ram_save_setup(QEMUFile
*f
, void *opaque
)
3189 RAMState
**rsp
= opaque
;
3193 if (compress_threads_save_setup()) {
3197 /* migration has already setup the bitmap, reuse it. */
3198 if (!migration_in_colo_state()) {
3199 if (ram_init_all(rsp
) != 0) {
3200 compress_threads_save_cleanup();
3206 WITH_RCU_READ_LOCK_GUARD() {
3207 qemu_put_be64(f
, ram_bytes_total_common(true) | RAM_SAVE_FLAG_MEM_SIZE
);
3209 RAMBLOCK_FOREACH_MIGRATABLE(block
) {
3210 qemu_put_byte(f
, strlen(block
->idstr
));
3211 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, strlen(block
->idstr
));
3212 qemu_put_be64(f
, block
->used_length
);
3213 if (migrate_postcopy_ram() && block
->page_size
!=
3214 qemu_host_page_size
) {
3215 qemu_put_be64(f
, block
->page_size
);
3217 if (migrate_ignore_shared()) {
3218 qemu_put_be64(f
, block
->mr
->addr
);
3223 ram_control_before_iterate(f
, RAM_CONTROL_SETUP
);
3224 ram_control_after_iterate(f
, RAM_CONTROL_SETUP
);
3226 ret
= multifd_send_sync_main(f
);
3231 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
3238 * ram_save_iterate: iterative stage for migration
3240 * Returns zero to indicate success and negative for error
3242 * @f: QEMUFile where to send the data
3243 * @opaque: RAMState pointer
3245 static int ram_save_iterate(QEMUFile
*f
, void *opaque
)
3247 RAMState
**temp
= opaque
;
3248 RAMState
*rs
= *temp
;
3254 if (blk_mig_bulk_active()) {
3255 /* Avoid transferring ram during bulk phase of block migration as
3256 * the bulk phase will usually take a long time and transferring
3257 * ram updates during that time is pointless. */
3262 * We'll take this lock a little bit long, but it's okay for two reasons.
3263 * Firstly, the only possible other thread to take it is who calls
3264 * qemu_guest_free_page_hint(), which should be rare; secondly, see
3265 * MAX_WAIT (if curious, further see commit 4508bd9ed8053ce) below, which
3266 * guarantees that we'll at least released it in a regular basis.
3268 qemu_mutex_lock(&rs
->bitmap_mutex
);
3269 WITH_RCU_READ_LOCK_GUARD() {
3270 if (ram_list
.version
!= rs
->last_version
) {
3271 ram_state_reset(rs
);
3274 /* Read version before ram_list.blocks */
3277 ram_control_before_iterate(f
, RAM_CONTROL_ROUND
);
3279 t0
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
);
3281 while ((ret
= qemu_file_rate_limit(f
)) == 0 ||
3282 postcopy_has_request(rs
)) {
3285 if (qemu_file_get_error(f
)) {
3289 pages
= ram_find_and_save_block(rs
);
3290 /* no more pages to sent */
3297 qemu_file_set_error(f
, pages
);
3301 rs
->target_page_count
+= pages
;
3304 * During postcopy, it is necessary to make sure one whole host
3305 * page is sent in one chunk.
3307 if (migrate_postcopy_ram()) {
3308 flush_compressed_data(rs
);
3312 * we want to check in the 1st loop, just in case it was the 1st
3313 * time and we had to sync the dirty bitmap.
3314 * qemu_clock_get_ns() is a bit expensive, so we only check each
3317 if ((i
& 63) == 0) {
3318 uint64_t t1
= (qemu_clock_get_ns(QEMU_CLOCK_REALTIME
) - t0
) /
3320 if (t1
> MAX_WAIT
) {
3321 trace_ram_save_iterate_big_wait(t1
, i
);
3328 qemu_mutex_unlock(&rs
->bitmap_mutex
);
3330 postcopy_preempt_reset_channel(rs
);
3333 * Must occur before EOS (or any QEMUFile operation)
3334 * because of RDMA protocol.
3336 ram_control_after_iterate(f
, RAM_CONTROL_ROUND
);
3340 && migration_is_setup_or_active(migrate_get_current()->state
)) {
3341 ret
= multifd_send_sync_main(rs
->f
);
3346 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
3348 ram_transferred_add(8);
3350 ret
= qemu_file_get_error(f
);
3360 * ram_save_complete: function called to send the remaining amount of ram
3362 * Returns zero to indicate success or negative on error
3364 * Called with iothread lock
3366 * @f: QEMUFile where to send the data
3367 * @opaque: RAMState pointer
3369 static int ram_save_complete(QEMUFile
*f
, void *opaque
)
3371 RAMState
**temp
= opaque
;
3372 RAMState
*rs
= *temp
;
3375 rs
->last_stage
= !migration_in_colo_state();
3377 WITH_RCU_READ_LOCK_GUARD() {
3378 if (!migration_in_postcopy()) {
3379 migration_bitmap_sync_precopy(rs
);
3382 ram_control_before_iterate(f
, RAM_CONTROL_FINISH
);
3384 /* try transferring iterative blocks of memory */
3386 /* flush all remaining blocks regardless of rate limiting */
3390 pages
= ram_find_and_save_block(rs
);
3391 /* no more blocks to sent */
3401 flush_compressed_data(rs
);
3402 ram_control_after_iterate(f
, RAM_CONTROL_FINISH
);
3409 postcopy_preempt_reset_channel(rs
);
3411 ret
= multifd_send_sync_main(rs
->f
);
3416 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
3422 static void ram_save_pending(QEMUFile
*f
, void *opaque
, uint64_t max_size
,
3423 uint64_t *res_precopy_only
,
3424 uint64_t *res_compatible
,
3425 uint64_t *res_postcopy_only
)
3427 RAMState
**temp
= opaque
;
3428 RAMState
*rs
= *temp
;
3429 uint64_t remaining_size
;
3431 remaining_size
= rs
->migration_dirty_pages
* TARGET_PAGE_SIZE
;
3433 if (!migration_in_postcopy() &&
3434 remaining_size
< max_size
) {
3435 qemu_mutex_lock_iothread();
3436 WITH_RCU_READ_LOCK_GUARD() {
3437 migration_bitmap_sync_precopy(rs
);
3439 qemu_mutex_unlock_iothread();
3440 remaining_size
= rs
->migration_dirty_pages
* TARGET_PAGE_SIZE
;
3443 if (migrate_postcopy_ram()) {
3444 /* We can do postcopy, and all the data is postcopiable */
3445 *res_compatible
+= remaining_size
;
3447 *res_precopy_only
+= remaining_size
;
3451 static int load_xbzrle(QEMUFile
*f
, ram_addr_t addr
, void *host
)
3453 unsigned int xh_len
;
3455 uint8_t *loaded_data
;
3457 /* extract RLE header */
3458 xh_flags
= qemu_get_byte(f
);
3459 xh_len
= qemu_get_be16(f
);
3461 if (xh_flags
!= ENCODING_FLAG_XBZRLE
) {
3462 error_report("Failed to load XBZRLE page - wrong compression!");
3466 if (xh_len
> TARGET_PAGE_SIZE
) {
3467 error_report("Failed to load XBZRLE page - len overflow!");
3470 loaded_data
= XBZRLE
.decoded_buf
;
3471 /* load data and decode */
3472 /* it can change loaded_data to point to an internal buffer */
3473 qemu_get_buffer_in_place(f
, &loaded_data
, xh_len
);
3476 if (xbzrle_decode_buffer(loaded_data
, xh_len
, host
,
3477 TARGET_PAGE_SIZE
) == -1) {
3478 error_report("Failed to load XBZRLE page - decode error!");
3486 * ram_block_from_stream: read a RAMBlock id from the migration stream
3488 * Must be called from within a rcu critical section.
3490 * Returns a pointer from within the RCU-protected ram_list.
3492 * @mis: the migration incoming state pointer
3493 * @f: QEMUFile where to read the data from
3494 * @flags: Page flags (mostly to see if it's a continuation of previous block)
3495 * @channel: the channel we're using
3497 static inline RAMBlock
*ram_block_from_stream(MigrationIncomingState
*mis
,
3498 QEMUFile
*f
, int flags
,
3501 RAMBlock
*block
= mis
->last_recv_block
[channel
];
3505 if (flags
& RAM_SAVE_FLAG_CONTINUE
) {
3507 error_report("Ack, bad migration stream!");
3513 len
= qemu_get_byte(f
);
3514 qemu_get_buffer(f
, (uint8_t *)id
, len
);
3517 block
= qemu_ram_block_by_name(id
);
3519 error_report("Can't find block %s", id
);
3523 if (ramblock_is_ignored(block
)) {
3524 error_report("block %s should not be migrated !", id
);
3528 mis
->last_recv_block
[channel
] = block
;
3533 static inline void *host_from_ram_block_offset(RAMBlock
*block
,
3536 if (!offset_in_ramblock(block
, offset
)) {
3540 return block
->host
+ offset
;
3543 static void *host_page_from_ram_block_offset(RAMBlock
*block
,
3546 /* Note: Explicitly no check against offset_in_ramblock(). */
3547 return (void *)QEMU_ALIGN_DOWN((uintptr_t)(block
->host
+ offset
),
3551 static ram_addr_t
host_page_offset_from_ram_block_offset(RAMBlock
*block
,
3554 return ((uintptr_t)block
->host
+ offset
) & (block
->page_size
- 1);
3557 static inline void *colo_cache_from_block_offset(RAMBlock
*block
,
3558 ram_addr_t offset
, bool record_bitmap
)
3560 if (!offset_in_ramblock(block
, offset
)) {
3563 if (!block
->colo_cache
) {
3564 error_report("%s: colo_cache is NULL in block :%s",
3565 __func__
, block
->idstr
);
3570 * During colo checkpoint, we need bitmap of these migrated pages.
3571 * It help us to decide which pages in ram cache should be flushed
3572 * into VM's RAM later.
3574 if (record_bitmap
&&
3575 !test_and_set_bit(offset
>> TARGET_PAGE_BITS
, block
->bmap
)) {
3576 ram_state
->migration_dirty_pages
++;
3578 return block
->colo_cache
+ offset
;
3582 * ram_handle_compressed: handle the zero page case
3584 * If a page (or a whole RDMA chunk) has been
3585 * determined to be zero, then zap it.
3587 * @host: host address for the zero page
3588 * @ch: what the page is filled from. We only support zero
3589 * @size: size of the zero page
3591 void ram_handle_compressed(void *host
, uint8_t ch
, uint64_t size
)
3593 if (ch
!= 0 || !buffer_is_zero(host
, size
)) {
3594 memset(host
, ch
, size
);
3598 /* return the size after decompression, or negative value on error */
3600 qemu_uncompress_data(z_stream
*stream
, uint8_t *dest
, size_t dest_len
,
3601 const uint8_t *source
, size_t source_len
)
3605 err
= inflateReset(stream
);
3610 stream
->avail_in
= source_len
;
3611 stream
->next_in
= (uint8_t *)source
;
3612 stream
->avail_out
= dest_len
;
3613 stream
->next_out
= dest
;
3615 err
= inflate(stream
, Z_NO_FLUSH
);
3616 if (err
!= Z_STREAM_END
) {
3620 return stream
->total_out
;
3623 static void *do_data_decompress(void *opaque
)
3625 DecompressParam
*param
= opaque
;
3626 unsigned long pagesize
;
3630 qemu_mutex_lock(¶m
->mutex
);
3631 while (!param
->quit
) {
3636 qemu_mutex_unlock(¶m
->mutex
);
3638 pagesize
= TARGET_PAGE_SIZE
;
3640 ret
= qemu_uncompress_data(¶m
->stream
, des
, pagesize
,
3641 param
->compbuf
, len
);
3642 if (ret
< 0 && migrate_get_current()->decompress_error_check
) {
3643 error_report("decompress data failed");
3644 qemu_file_set_error(decomp_file
, ret
);
3647 qemu_mutex_lock(&decomp_done_lock
);
3649 qemu_cond_signal(&decomp_done_cond
);
3650 qemu_mutex_unlock(&decomp_done_lock
);
3652 qemu_mutex_lock(¶m
->mutex
);
3654 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
3657 qemu_mutex_unlock(¶m
->mutex
);
3662 static int wait_for_decompress_done(void)
3664 int idx
, thread_count
;
3666 if (!migrate_use_compression()) {
3670 thread_count
= migrate_decompress_threads();
3671 qemu_mutex_lock(&decomp_done_lock
);
3672 for (idx
= 0; idx
< thread_count
; idx
++) {
3673 while (!decomp_param
[idx
].done
) {
3674 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
3677 qemu_mutex_unlock(&decomp_done_lock
);
3678 return qemu_file_get_error(decomp_file
);
3681 static void compress_threads_load_cleanup(void)
3683 int i
, thread_count
;
3685 if (!migrate_use_compression()) {
3688 thread_count
= migrate_decompress_threads();
3689 for (i
= 0; i
< thread_count
; i
++) {
3691 * we use it as a indicator which shows if the thread is
3692 * properly init'd or not
3694 if (!decomp_param
[i
].compbuf
) {
3698 qemu_mutex_lock(&decomp_param
[i
].mutex
);
3699 decomp_param
[i
].quit
= true;
3700 qemu_cond_signal(&decomp_param
[i
].cond
);
3701 qemu_mutex_unlock(&decomp_param
[i
].mutex
);
3703 for (i
= 0; i
< thread_count
; i
++) {
3704 if (!decomp_param
[i
].compbuf
) {
3708 qemu_thread_join(decompress_threads
+ i
);
3709 qemu_mutex_destroy(&decomp_param
[i
].mutex
);
3710 qemu_cond_destroy(&decomp_param
[i
].cond
);
3711 inflateEnd(&decomp_param
[i
].stream
);
3712 g_free(decomp_param
[i
].compbuf
);
3713 decomp_param
[i
].compbuf
= NULL
;
3715 g_free(decompress_threads
);
3716 g_free(decomp_param
);
3717 decompress_threads
= NULL
;
3718 decomp_param
= NULL
;
3722 static int compress_threads_load_setup(QEMUFile
*f
)
3724 int i
, thread_count
;
3726 if (!migrate_use_compression()) {
3730 thread_count
= migrate_decompress_threads();
3731 decompress_threads
= g_new0(QemuThread
, thread_count
);
3732 decomp_param
= g_new0(DecompressParam
, thread_count
);
3733 qemu_mutex_init(&decomp_done_lock
);
3734 qemu_cond_init(&decomp_done_cond
);
3736 for (i
= 0; i
< thread_count
; i
++) {
3737 if (inflateInit(&decomp_param
[i
].stream
) != Z_OK
) {
3741 decomp_param
[i
].compbuf
= g_malloc0(compressBound(TARGET_PAGE_SIZE
));
3742 qemu_mutex_init(&decomp_param
[i
].mutex
);
3743 qemu_cond_init(&decomp_param
[i
].cond
);
3744 decomp_param
[i
].done
= true;
3745 decomp_param
[i
].quit
= false;
3746 qemu_thread_create(decompress_threads
+ i
, "decompress",
3747 do_data_decompress
, decomp_param
+ i
,
3748 QEMU_THREAD_JOINABLE
);
3752 compress_threads_load_cleanup();
3756 static void decompress_data_with_multi_threads(QEMUFile
*f
,
3757 void *host
, int len
)
3759 int idx
, thread_count
;
3761 thread_count
= migrate_decompress_threads();
3762 QEMU_LOCK_GUARD(&decomp_done_lock
);
3764 for (idx
= 0; idx
< thread_count
; idx
++) {
3765 if (decomp_param
[idx
].done
) {
3766 decomp_param
[idx
].done
= false;
3767 qemu_mutex_lock(&decomp_param
[idx
].mutex
);
3768 qemu_get_buffer(f
, decomp_param
[idx
].compbuf
, len
);
3769 decomp_param
[idx
].des
= host
;
3770 decomp_param
[idx
].len
= len
;
3771 qemu_cond_signal(&decomp_param
[idx
].cond
);
3772 qemu_mutex_unlock(&decomp_param
[idx
].mutex
);
3776 if (idx
< thread_count
) {
3779 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
3784 static void colo_init_ram_state(void)
3786 ram_state_init(&ram_state
);
3790 * colo cache: this is for secondary VM, we cache the whole
3791 * memory of the secondary VM, it is need to hold the global lock
3792 * to call this helper.
3794 int colo_init_ram_cache(void)
3798 WITH_RCU_READ_LOCK_GUARD() {
3799 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3800 block
->colo_cache
= qemu_anon_ram_alloc(block
->used_length
,
3801 NULL
, false, false);
3802 if (!block
->colo_cache
) {
3803 error_report("%s: Can't alloc memory for COLO cache of block %s,"
3804 "size 0x" RAM_ADDR_FMT
, __func__
, block
->idstr
,
3805 block
->used_length
);
3806 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3807 if (block
->colo_cache
) {
3808 qemu_anon_ram_free(block
->colo_cache
, block
->used_length
);
3809 block
->colo_cache
= NULL
;
3814 if (!machine_dump_guest_core(current_machine
)) {
3815 qemu_madvise(block
->colo_cache
, block
->used_length
,
3816 QEMU_MADV_DONTDUMP
);
3822 * Record the dirty pages that sent by PVM, we use this dirty bitmap together
3823 * with to decide which page in cache should be flushed into SVM's RAM. Here
3824 * we use the same name 'ram_bitmap' as for migration.
3826 if (ram_bytes_total()) {
3829 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3830 unsigned long pages
= block
->max_length
>> TARGET_PAGE_BITS
;
3831 block
->bmap
= bitmap_new(pages
);
3835 colo_init_ram_state();
3839 /* TODO: duplicated with ram_init_bitmaps */
3840 void colo_incoming_start_dirty_log(void)
3842 RAMBlock
*block
= NULL
;
3843 /* For memory_global_dirty_log_start below. */
3844 qemu_mutex_lock_iothread();
3845 qemu_mutex_lock_ramlist();
3847 memory_global_dirty_log_sync();
3848 WITH_RCU_READ_LOCK_GUARD() {
3849 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3850 ramblock_sync_dirty_bitmap(ram_state
, block
);
3851 /* Discard this dirty bitmap record */
3852 bitmap_zero(block
->bmap
, block
->max_length
>> TARGET_PAGE_BITS
);
3854 memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION
);
3856 ram_state
->migration_dirty_pages
= 0;
3857 qemu_mutex_unlock_ramlist();
3858 qemu_mutex_unlock_iothread();
3861 /* It is need to hold the global lock to call this helper */
3862 void colo_release_ram_cache(void)
3866 memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION
);
3867 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3868 g_free(block
->bmap
);
3872 WITH_RCU_READ_LOCK_GUARD() {
3873 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3874 if (block
->colo_cache
) {
3875 qemu_anon_ram_free(block
->colo_cache
, block
->used_length
);
3876 block
->colo_cache
= NULL
;
3880 ram_state_cleanup(&ram_state
);
3884 * ram_load_setup: Setup RAM for migration incoming side
3886 * Returns zero to indicate success and negative for error
3888 * @f: QEMUFile where to receive the data
3889 * @opaque: RAMState pointer
3891 static int ram_load_setup(QEMUFile
*f
, void *opaque
)
3893 if (compress_threads_load_setup(f
)) {
3897 xbzrle_load_setup();
3898 ramblock_recv_map_init();
3903 static int ram_load_cleanup(void *opaque
)
3907 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
3908 qemu_ram_block_writeback(rb
);
3911 xbzrle_load_cleanup();
3912 compress_threads_load_cleanup();
3914 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
3915 g_free(rb
->receivedmap
);
3916 rb
->receivedmap
= NULL
;
3923 * ram_postcopy_incoming_init: allocate postcopy data structures
3925 * Returns 0 for success and negative if there was one error
3927 * @mis: current migration incoming state
3929 * Allocate data structures etc needed by incoming migration with
3930 * postcopy-ram. postcopy-ram's similarly names
3931 * postcopy_ram_incoming_init does the work.
3933 int ram_postcopy_incoming_init(MigrationIncomingState
*mis
)
3935 return postcopy_ram_incoming_init(mis
);
3939 * ram_load_postcopy: load a page in postcopy case
3941 * Returns 0 for success or -errno in case of error
3943 * Called in postcopy mode by ram_load().
3944 * rcu_read_lock is taken prior to this being called.
3946 * @f: QEMUFile where to send the data
3947 * @channel: the channel to use for loading
3949 int ram_load_postcopy(QEMUFile
*f
, int channel
)
3951 int flags
= 0, ret
= 0;
3952 bool place_needed
= false;
3953 bool matches_target_page_size
= false;
3954 MigrationIncomingState
*mis
= migration_incoming_get_current();
3955 PostcopyTmpPage
*tmp_page
= &mis
->postcopy_tmp_pages
[channel
];
3957 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
3959 void *page_buffer
= NULL
;
3960 void *place_source
= NULL
;
3961 RAMBlock
*block
= NULL
;
3965 addr
= qemu_get_be64(f
);
3968 * If qemu file error, we should stop here, and then "addr"
3971 ret
= qemu_file_get_error(f
);
3976 flags
= addr
& ~TARGET_PAGE_MASK
;
3977 addr
&= TARGET_PAGE_MASK
;
3979 trace_ram_load_postcopy_loop(channel
, (uint64_t)addr
, flags
);
3980 if (flags
& (RAM_SAVE_FLAG_ZERO
| RAM_SAVE_FLAG_PAGE
|
3981 RAM_SAVE_FLAG_COMPRESS_PAGE
)) {
3982 block
= ram_block_from_stream(mis
, f
, flags
, channel
);
3989 * Relying on used_length is racy and can result in false positives.
3990 * We might place pages beyond used_length in case RAM was shrunk
3991 * while in postcopy, which is fine - trying to place via
3992 * UFFDIO_COPY/UFFDIO_ZEROPAGE will never segfault.
3994 if (!block
->host
|| addr
>= block
->postcopy_length
) {
3995 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
3999 tmp_page
->target_pages
++;
4000 matches_target_page_size
= block
->page_size
== TARGET_PAGE_SIZE
;
4002 * Postcopy requires that we place whole host pages atomically;
4003 * these may be huge pages for RAMBlocks that are backed by
4005 * To make it atomic, the data is read into a temporary page
4006 * that's moved into place later.
4007 * The migration protocol uses, possibly smaller, target-pages
4008 * however the source ensures it always sends all the components
4009 * of a host page in one chunk.
4011 page_buffer
= tmp_page
->tmp_huge_page
+
4012 host_page_offset_from_ram_block_offset(block
, addr
);
4013 /* If all TP are zero then we can optimise the place */
4014 if (tmp_page
->target_pages
== 1) {
4015 tmp_page
->host_addr
=
4016 host_page_from_ram_block_offset(block
, addr
);
4017 } else if (tmp_page
->host_addr
!=
4018 host_page_from_ram_block_offset(block
, addr
)) {
4019 /* not the 1st TP within the HP */
4020 error_report("Non-same host page detected on channel %d: "
4021 "Target host page %p, received host page %p "
4022 "(rb %s offset 0x"RAM_ADDR_FMT
" target_pages %d)",
4023 channel
, tmp_page
->host_addr
,
4024 host_page_from_ram_block_offset(block
, addr
),
4025 block
->idstr
, addr
, tmp_page
->target_pages
);
4031 * If it's the last part of a host page then we place the host
4034 if (tmp_page
->target_pages
==
4035 (block
->page_size
/ TARGET_PAGE_SIZE
)) {
4036 place_needed
= true;
4038 place_source
= tmp_page
->tmp_huge_page
;
4041 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
4042 case RAM_SAVE_FLAG_ZERO
:
4043 ch
= qemu_get_byte(f
);
4045 * Can skip to set page_buffer when
4046 * this is a zero page and (block->page_size == TARGET_PAGE_SIZE).
4048 if (ch
|| !matches_target_page_size
) {
4049 memset(page_buffer
, ch
, TARGET_PAGE_SIZE
);
4052 tmp_page
->all_zero
= false;
4056 case RAM_SAVE_FLAG_PAGE
:
4057 tmp_page
->all_zero
= false;
4058 if (!matches_target_page_size
) {
4059 /* For huge pages, we always use temporary buffer */
4060 qemu_get_buffer(f
, page_buffer
, TARGET_PAGE_SIZE
);
4063 * For small pages that matches target page size, we
4064 * avoid the qemu_file copy. Instead we directly use
4065 * the buffer of QEMUFile to place the page. Note: we
4066 * cannot do any QEMUFile operation before using that
4067 * buffer to make sure the buffer is valid when
4070 qemu_get_buffer_in_place(f
, (uint8_t **)&place_source
,
4074 case RAM_SAVE_FLAG_COMPRESS_PAGE
:
4075 tmp_page
->all_zero
= false;
4076 len
= qemu_get_be32(f
);
4077 if (len
< 0 || len
> compressBound(TARGET_PAGE_SIZE
)) {
4078 error_report("Invalid compressed data length: %d", len
);
4082 decompress_data_with_multi_threads(f
, page_buffer
, len
);
4085 case RAM_SAVE_FLAG_EOS
:
4087 multifd_recv_sync_main();
4090 error_report("Unknown combination of migration flags: 0x%x"
4091 " (postcopy mode)", flags
);
4096 /* Got the whole host page, wait for decompress before placing. */
4098 ret
|= wait_for_decompress_done();
4101 /* Detect for any possible file errors */
4102 if (!ret
&& qemu_file_get_error(f
)) {
4103 ret
= qemu_file_get_error(f
);
4106 if (!ret
&& place_needed
) {
4107 if (tmp_page
->all_zero
) {
4108 ret
= postcopy_place_page_zero(mis
, tmp_page
->host_addr
, block
);
4110 ret
= postcopy_place_page(mis
, tmp_page
->host_addr
,
4111 place_source
, block
);
4113 place_needed
= false;
4114 postcopy_temp_page_reset(tmp_page
);
4121 static bool postcopy_is_advised(void)
4123 PostcopyState ps
= postcopy_state_get();
4124 return ps
>= POSTCOPY_INCOMING_ADVISE
&& ps
< POSTCOPY_INCOMING_END
;
4127 static bool postcopy_is_running(void)
4129 PostcopyState ps
= postcopy_state_get();
4130 return ps
>= POSTCOPY_INCOMING_LISTENING
&& ps
< POSTCOPY_INCOMING_END
;
4134 * Flush content of RAM cache into SVM's memory.
4135 * Only flush the pages that be dirtied by PVM or SVM or both.
4137 void colo_flush_ram_cache(void)
4139 RAMBlock
*block
= NULL
;
4142 unsigned long offset
= 0;
4144 memory_global_dirty_log_sync();
4145 WITH_RCU_READ_LOCK_GUARD() {
4146 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
4147 ramblock_sync_dirty_bitmap(ram_state
, block
);
4151 trace_colo_flush_ram_cache_begin(ram_state
->migration_dirty_pages
);
4152 WITH_RCU_READ_LOCK_GUARD() {
4153 block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
4156 unsigned long num
= 0;
4158 offset
= colo_bitmap_find_dirty(ram_state
, block
, offset
, &num
);
4159 if (!offset_in_ramblock(block
,
4160 ((ram_addr_t
)offset
) << TARGET_PAGE_BITS
)) {
4163 block
= QLIST_NEXT_RCU(block
, next
);
4165 unsigned long i
= 0;
4167 for (i
= 0; i
< num
; i
++) {
4168 migration_bitmap_clear_dirty(ram_state
, block
, offset
+ i
);
4170 dst_host
= block
->host
4171 + (((ram_addr_t
)offset
) << TARGET_PAGE_BITS
);
4172 src_host
= block
->colo_cache
4173 + (((ram_addr_t
)offset
) << TARGET_PAGE_BITS
);
4174 memcpy(dst_host
, src_host
, TARGET_PAGE_SIZE
* num
);
4179 trace_colo_flush_ram_cache_end();
4183 * ram_load_precopy: load pages in precopy case
4185 * Returns 0 for success or -errno in case of error
4187 * Called in precopy mode by ram_load().
4188 * rcu_read_lock is taken prior to this being called.
4190 * @f: QEMUFile where to send the data
4192 static int ram_load_precopy(QEMUFile
*f
)
4194 MigrationIncomingState
*mis
= migration_incoming_get_current();
4195 int flags
= 0, ret
= 0, invalid_flags
= 0, len
= 0, i
= 0;
4196 /* ADVISE is earlier, it shows the source has the postcopy capability on */
4197 bool postcopy_advised
= postcopy_is_advised();
4198 if (!migrate_use_compression()) {
4199 invalid_flags
|= RAM_SAVE_FLAG_COMPRESS_PAGE
;
4202 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
4203 ram_addr_t addr
, total_ram_bytes
;
4204 void *host
= NULL
, *host_bak
= NULL
;
4208 * Yield periodically to let main loop run, but an iteration of
4209 * the main loop is expensive, so do it each some iterations
4211 if ((i
& 32767) == 0 && qemu_in_coroutine()) {
4212 aio_co_schedule(qemu_get_current_aio_context(),
4213 qemu_coroutine_self());
4214 qemu_coroutine_yield();
4218 addr
= qemu_get_be64(f
);
4219 flags
= addr
& ~TARGET_PAGE_MASK
;
4220 addr
&= TARGET_PAGE_MASK
;
4222 if (flags
& invalid_flags
) {
4223 if (flags
& invalid_flags
& RAM_SAVE_FLAG_COMPRESS_PAGE
) {
4224 error_report("Received an unexpected compressed page");
4231 if (flags
& (RAM_SAVE_FLAG_ZERO
| RAM_SAVE_FLAG_PAGE
|
4232 RAM_SAVE_FLAG_COMPRESS_PAGE
| RAM_SAVE_FLAG_XBZRLE
)) {
4233 RAMBlock
*block
= ram_block_from_stream(mis
, f
, flags
,
4234 RAM_CHANNEL_PRECOPY
);
4236 host
= host_from_ram_block_offset(block
, addr
);
4238 * After going into COLO stage, we should not load the page
4239 * into SVM's memory directly, we put them into colo_cache firstly.
4240 * NOTE: We need to keep a copy of SVM's ram in colo_cache.
4241 * Previously, we copied all these memory in preparing stage of COLO
4242 * while we need to stop VM, which is a time-consuming process.
4243 * Here we optimize it by a trick, back-up every page while in
4244 * migration process while COLO is enabled, though it affects the
4245 * speed of the migration, but it obviously reduce the downtime of
4246 * back-up all SVM'S memory in COLO preparing stage.
4248 if (migration_incoming_colo_enabled()) {
4249 if (migration_incoming_in_colo_state()) {
4250 /* In COLO stage, put all pages into cache temporarily */
4251 host
= colo_cache_from_block_offset(block
, addr
, true);
4254 * In migration stage but before COLO stage,
4255 * Put all pages into both cache and SVM's memory.
4257 host_bak
= colo_cache_from_block_offset(block
, addr
, false);
4261 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
4265 if (!migration_incoming_in_colo_state()) {
4266 ramblock_recv_bitmap_set(block
, host
);
4269 trace_ram_load_loop(block
->idstr
, (uint64_t)addr
, flags
, host
);
4272 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
4273 case RAM_SAVE_FLAG_MEM_SIZE
:
4274 /* Synchronize RAM block list */
4275 total_ram_bytes
= addr
;
4276 while (!ret
&& total_ram_bytes
) {
4281 len
= qemu_get_byte(f
);
4282 qemu_get_buffer(f
, (uint8_t *)id
, len
);
4284 length
= qemu_get_be64(f
);
4286 block
= qemu_ram_block_by_name(id
);
4287 if (block
&& !qemu_ram_is_migratable(block
)) {
4288 error_report("block %s should not be migrated !", id
);
4291 if (length
!= block
->used_length
) {
4292 Error
*local_err
= NULL
;
4294 ret
= qemu_ram_resize(block
, length
,
4297 error_report_err(local_err
);
4300 /* For postcopy we need to check hugepage sizes match */
4301 if (postcopy_advised
&& migrate_postcopy_ram() &&
4302 block
->page_size
!= qemu_host_page_size
) {
4303 uint64_t remote_page_size
= qemu_get_be64(f
);
4304 if (remote_page_size
!= block
->page_size
) {
4305 error_report("Mismatched RAM page size %s "
4306 "(local) %zd != %" PRId64
,
4307 id
, block
->page_size
,
4312 if (migrate_ignore_shared()) {
4313 hwaddr addr
= qemu_get_be64(f
);
4314 if (ramblock_is_ignored(block
) &&
4315 block
->mr
->addr
!= addr
) {
4316 error_report("Mismatched GPAs for block %s "
4317 "%" PRId64
"!= %" PRId64
,
4319 (uint64_t)block
->mr
->addr
);
4323 ram_control_load_hook(f
, RAM_CONTROL_BLOCK_REG
,
4326 error_report("Unknown ramblock \"%s\", cannot "
4327 "accept migration", id
);
4331 total_ram_bytes
-= length
;
4335 case RAM_SAVE_FLAG_ZERO
:
4336 ch
= qemu_get_byte(f
);
4337 ram_handle_compressed(host
, ch
, TARGET_PAGE_SIZE
);
4340 case RAM_SAVE_FLAG_PAGE
:
4341 qemu_get_buffer(f
, host
, TARGET_PAGE_SIZE
);
4344 case RAM_SAVE_FLAG_COMPRESS_PAGE
:
4345 len
= qemu_get_be32(f
);
4346 if (len
< 0 || len
> compressBound(TARGET_PAGE_SIZE
)) {
4347 error_report("Invalid compressed data length: %d", len
);
4351 decompress_data_with_multi_threads(f
, host
, len
);
4354 case RAM_SAVE_FLAG_XBZRLE
:
4355 if (load_xbzrle(f
, addr
, host
) < 0) {
4356 error_report("Failed to decompress XBZRLE page at "
4357 RAM_ADDR_FMT
, addr
);
4362 case RAM_SAVE_FLAG_EOS
:
4364 multifd_recv_sync_main();
4367 if (flags
& RAM_SAVE_FLAG_HOOK
) {
4368 ram_control_load_hook(f
, RAM_CONTROL_HOOK
, NULL
);
4370 error_report("Unknown combination of migration flags: 0x%x",
4376 ret
= qemu_file_get_error(f
);
4378 if (!ret
&& host_bak
) {
4379 memcpy(host_bak
, host
, TARGET_PAGE_SIZE
);
4383 ret
|= wait_for_decompress_done();
4387 static int ram_load(QEMUFile
*f
, void *opaque
, int version_id
)
4390 static uint64_t seq_iter
;
4392 * If system is running in postcopy mode, page inserts to host memory must
4395 bool postcopy_running
= postcopy_is_running();
4399 if (version_id
!= 4) {
4404 * This RCU critical section can be very long running.
4405 * When RCU reclaims in the code start to become numerous,
4406 * it will be necessary to reduce the granularity of this
4409 WITH_RCU_READ_LOCK_GUARD() {
4410 if (postcopy_running
) {
4412 * Note! Here RAM_CHANNEL_PRECOPY is the precopy channel of
4413 * postcopy migration, we have another RAM_CHANNEL_POSTCOPY to
4414 * service fast page faults.
4416 ret
= ram_load_postcopy(f
, RAM_CHANNEL_PRECOPY
);
4418 ret
= ram_load_precopy(f
);
4421 trace_ram_load_complete(ret
, seq_iter
);
4426 static bool ram_has_postcopy(void *opaque
)
4429 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
4430 if (ramblock_is_pmem(rb
)) {
4431 info_report("Block: %s, host: %p is a nvdimm memory, postcopy"
4432 "is not supported now!", rb
->idstr
, rb
->host
);
4437 return migrate_postcopy_ram();
4440 /* Sync all the dirty bitmap with destination VM. */
4441 static int ram_dirty_bitmap_sync_all(MigrationState
*s
, RAMState
*rs
)
4444 QEMUFile
*file
= s
->to_dst_file
;
4445 int ramblock_count
= 0;
4447 trace_ram_dirty_bitmap_sync_start();
4449 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
4450 qemu_savevm_send_recv_bitmap(file
, block
->idstr
);
4451 trace_ram_dirty_bitmap_request(block
->idstr
);
4455 trace_ram_dirty_bitmap_sync_wait();
4457 /* Wait until all the ramblocks' dirty bitmap synced */
4458 while (ramblock_count
--) {
4459 qemu_sem_wait(&s
->rp_state
.rp_sem
);
4462 trace_ram_dirty_bitmap_sync_complete();
4467 static void ram_dirty_bitmap_reload_notify(MigrationState
*s
)
4469 qemu_sem_post(&s
->rp_state
.rp_sem
);
4473 * Read the received bitmap, revert it as the initial dirty bitmap.
4474 * This is only used when the postcopy migration is paused but wants
4475 * to resume from a middle point.
4477 int ram_dirty_bitmap_reload(MigrationState
*s
, RAMBlock
*block
)
4480 /* from_dst_file is always valid because we're within rp_thread */
4481 QEMUFile
*file
= s
->rp_state
.from_dst_file
;
4482 unsigned long *le_bitmap
, nbits
= block
->used_length
>> TARGET_PAGE_BITS
;
4483 uint64_t local_size
= DIV_ROUND_UP(nbits
, 8);
4484 uint64_t size
, end_mark
;
4486 trace_ram_dirty_bitmap_reload_begin(block
->idstr
);
4488 if (s
->state
!= MIGRATION_STATUS_POSTCOPY_RECOVER
) {
4489 error_report("%s: incorrect state %s", __func__
,
4490 MigrationStatus_str(s
->state
));
4495 * Note: see comments in ramblock_recv_bitmap_send() on why we
4496 * need the endianness conversion, and the paddings.
4498 local_size
= ROUND_UP(local_size
, 8);
4501 le_bitmap
= bitmap_new(nbits
+ BITS_PER_LONG
);
4503 size
= qemu_get_be64(file
);
4505 /* The size of the bitmap should match with our ramblock */
4506 if (size
!= local_size
) {
4507 error_report("%s: ramblock '%s' bitmap size mismatch "
4508 "(0x%"PRIx64
" != 0x%"PRIx64
")", __func__
,
4509 block
->idstr
, size
, local_size
);
4514 size
= qemu_get_buffer(file
, (uint8_t *)le_bitmap
, local_size
);
4515 end_mark
= qemu_get_be64(file
);
4517 ret
= qemu_file_get_error(file
);
4518 if (ret
|| size
!= local_size
) {
4519 error_report("%s: read bitmap failed for ramblock '%s': %d"
4520 " (size 0x%"PRIx64
", got: 0x%"PRIx64
")",
4521 __func__
, block
->idstr
, ret
, local_size
, size
);
4526 if (end_mark
!= RAMBLOCK_RECV_BITMAP_ENDING
) {
4527 error_report("%s: ramblock '%s' end mark incorrect: 0x%"PRIx64
,
4528 __func__
, block
->idstr
, end_mark
);
4534 * Endianness conversion. We are during postcopy (though paused).
4535 * The dirty bitmap won't change. We can directly modify it.
4537 bitmap_from_le(block
->bmap
, le_bitmap
, nbits
);
4540 * What we received is "received bitmap". Revert it as the initial
4541 * dirty bitmap for this ramblock.
4543 bitmap_complement(block
->bmap
, block
->bmap
, nbits
);
4545 /* Clear dirty bits of discarded ranges that we don't want to migrate. */
4546 ramblock_dirty_bitmap_clear_discarded_pages(block
);
4548 /* We'll recalculate migration_dirty_pages in ram_state_resume_prepare(). */
4549 trace_ram_dirty_bitmap_reload_complete(block
->idstr
);
4552 * We succeeded to sync bitmap for current ramblock. If this is
4553 * the last one to sync, we need to notify the main send thread.
4555 ram_dirty_bitmap_reload_notify(s
);
4563 static int ram_resume_prepare(MigrationState
*s
, void *opaque
)
4565 RAMState
*rs
= *(RAMState
**)opaque
;
4568 ret
= ram_dirty_bitmap_sync_all(s
, rs
);
4573 ram_state_resume_prepare(rs
, s
->to_dst_file
);
4578 void postcopy_preempt_shutdown_file(MigrationState
*s
)
4580 qemu_put_be64(s
->postcopy_qemufile_src
, RAM_SAVE_FLAG_EOS
);
4581 qemu_fflush(s
->postcopy_qemufile_src
);
4584 static SaveVMHandlers savevm_ram_handlers
= {
4585 .save_setup
= ram_save_setup
,
4586 .save_live_iterate
= ram_save_iterate
,
4587 .save_live_complete_postcopy
= ram_save_complete
,
4588 .save_live_complete_precopy
= ram_save_complete
,
4589 .has_postcopy
= ram_has_postcopy
,
4590 .save_live_pending
= ram_save_pending
,
4591 .load_state
= ram_load
,
4592 .save_cleanup
= ram_save_cleanup
,
4593 .load_setup
= ram_load_setup
,
4594 .load_cleanup
= ram_load_cleanup
,
4595 .resume_prepare
= ram_resume_prepare
,
4598 static void ram_mig_ram_block_resized(RAMBlockNotifier
*n
, void *host
,
4599 size_t old_size
, size_t new_size
)
4601 PostcopyState ps
= postcopy_state_get();
4603 RAMBlock
*rb
= qemu_ram_block_from_host(host
, false, &offset
);
4606 if (ramblock_is_ignored(rb
)) {
4610 if (!migration_is_idle()) {
4612 * Precopy code on the source cannot deal with the size of RAM blocks
4613 * changing at random points in time - especially after sending the
4614 * RAM block sizes in the migration stream, they must no longer change.
4615 * Abort and indicate a proper reason.
4617 error_setg(&err
, "RAM block '%s' resized during precopy.", rb
->idstr
);
4618 migration_cancel(err
);
4623 case POSTCOPY_INCOMING_ADVISE
:
4625 * Update what ram_postcopy_incoming_init()->init_range() does at the
4626 * time postcopy was advised. Syncing RAM blocks with the source will
4627 * result in RAM resizes.
4629 if (old_size
< new_size
) {
4630 if (ram_discard_range(rb
->idstr
, old_size
, new_size
- old_size
)) {
4631 error_report("RAM block '%s' discard of resized RAM failed",
4635 rb
->postcopy_length
= new_size
;
4637 case POSTCOPY_INCOMING_NONE
:
4638 case POSTCOPY_INCOMING_RUNNING
:
4639 case POSTCOPY_INCOMING_END
:
4641 * Once our guest is running, postcopy does no longer care about
4642 * resizes. When growing, the new memory was not available on the
4643 * source, no handler needed.
4647 error_report("RAM block '%s' resized during postcopy state: %d",
4653 static RAMBlockNotifier ram_mig_ram_notifier
= {
4654 .ram_block_resized
= ram_mig_ram_block_resized
,
4657 void ram_mig_init(void)
4659 qemu_mutex_init(&XBZRLE
.lock
);
4660 register_savevm_live("ram", 0, 4, &savevm_ram_handlers
, &ram_state
);
4661 ram_block_notifier_add(&ram_mig_ram_notifier
);