4 * Copyright (c) 2003-2008 Fabrice Bellard
5 * Copyright (c) 2011-2015 Red Hat Inc
8 * Juan Quintela <quintela@redhat.com>
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29 #include "qemu/osdep.h"
31 #include "qemu/cutils.h"
32 #include "qemu/bitops.h"
33 #include "qemu/bitmap.h"
34 #include "qemu/main-loop.h"
37 #include "migration.h"
38 #include "migration/register.h"
39 #include "migration/misc.h"
40 #include "qemu-file.h"
41 #include "postcopy-ram.h"
42 #include "page_cache.h"
43 #include "qemu/error-report.h"
44 #include "qapi/error.h"
45 #include "qapi/qapi-types-migration.h"
46 #include "qapi/qapi-events-migration.h"
47 #include "qapi/qmp/qerror.h"
49 #include "exec/ram_addr.h"
50 #include "exec/target_page.h"
51 #include "qemu/rcu_queue.h"
52 #include "migration/colo.h"
54 #include "sysemu/sysemu.h"
59 /***********************************************************/
60 /* ram save/restore */
62 /* RAM_SAVE_FLAG_ZERO used to be named RAM_SAVE_FLAG_COMPRESS, it
63 * worked for pages that where filled with the same char. We switched
64 * it to only search for the zero value. And to avoid confusion with
65 * RAM_SSAVE_FLAG_COMPRESS_PAGE just rename it.
68 #define RAM_SAVE_FLAG_FULL 0x01 /* Obsolete, not used anymore */
69 #define RAM_SAVE_FLAG_ZERO 0x02
70 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
71 #define RAM_SAVE_FLAG_PAGE 0x08
72 #define RAM_SAVE_FLAG_EOS 0x10
73 #define RAM_SAVE_FLAG_CONTINUE 0x20
74 #define RAM_SAVE_FLAG_XBZRLE 0x40
75 /* 0x80 is reserved in migration.h start with 0x100 next */
76 #define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
78 static inline bool is_zero_range(uint8_t *p
, uint64_t size
)
80 return buffer_is_zero(p
, size
);
83 XBZRLECacheStats xbzrle_counters
;
85 /* struct contains XBZRLE cache and a static page
86 used by the compression */
88 /* buffer used for XBZRLE encoding */
90 /* buffer for storing page content */
92 /* Cache for XBZRLE, Protected by lock. */
95 /* it will store a page full of zeros */
96 uint8_t *zero_target_page
;
97 /* buffer used for XBZRLE decoding */
101 static void XBZRLE_cache_lock(void)
103 if (migrate_use_xbzrle())
104 qemu_mutex_lock(&XBZRLE
.lock
);
107 static void XBZRLE_cache_unlock(void)
109 if (migrate_use_xbzrle())
110 qemu_mutex_unlock(&XBZRLE
.lock
);
114 * xbzrle_cache_resize: resize the xbzrle cache
116 * This function is called from qmp_migrate_set_cache_size in main
117 * thread, possibly while a migration is in progress. A running
118 * migration may be using the cache and might finish during this call,
119 * hence changes to the cache are protected by XBZRLE.lock().
121 * Returns 0 for success or -1 for error
123 * @new_size: new cache size
124 * @errp: set *errp if the check failed, with reason
126 int xbzrle_cache_resize(int64_t new_size
, Error
**errp
)
128 PageCache
*new_cache
;
131 /* Check for truncation */
132 if (new_size
!= (size_t)new_size
) {
133 error_setg(errp
, QERR_INVALID_PARAMETER_VALUE
, "cache size",
134 "exceeding address space");
138 if (new_size
== migrate_xbzrle_cache_size()) {
145 if (XBZRLE
.cache
!= NULL
) {
146 new_cache
= cache_init(new_size
, TARGET_PAGE_SIZE
, errp
);
152 cache_fini(XBZRLE
.cache
);
153 XBZRLE
.cache
= new_cache
;
156 XBZRLE_cache_unlock();
160 static bool ramblock_is_ignored(RAMBlock
*block
)
162 return !qemu_ram_is_migratable(block
) ||
163 (migrate_ignore_shared() && qemu_ram_is_shared(block
));
166 /* Should be holding either ram_list.mutex, or the RCU lock. */
167 #define RAMBLOCK_FOREACH_NOT_IGNORED(block) \
168 INTERNAL_RAMBLOCK_FOREACH(block) \
169 if (ramblock_is_ignored(block)) {} else
171 #define RAMBLOCK_FOREACH_MIGRATABLE(block) \
172 INTERNAL_RAMBLOCK_FOREACH(block) \
173 if (!qemu_ram_is_migratable(block)) {} else
175 #undef RAMBLOCK_FOREACH
177 int foreach_not_ignored_block(RAMBlockIterFunc func
, void *opaque
)
182 RCU_READ_LOCK_GUARD();
184 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
185 ret
= func(block
, opaque
);
193 static void ramblock_recv_map_init(void)
197 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
198 assert(!rb
->receivedmap
);
199 rb
->receivedmap
= bitmap_new(rb
->max_length
>> qemu_target_page_bits());
203 int ramblock_recv_bitmap_test(RAMBlock
*rb
, void *host_addr
)
205 return test_bit(ramblock_recv_bitmap_offset(host_addr
, rb
),
209 bool ramblock_recv_bitmap_test_byte_offset(RAMBlock
*rb
, uint64_t byte_offset
)
211 return test_bit(byte_offset
>> TARGET_PAGE_BITS
, rb
->receivedmap
);
214 void ramblock_recv_bitmap_set(RAMBlock
*rb
, void *host_addr
)
216 set_bit_atomic(ramblock_recv_bitmap_offset(host_addr
, rb
), rb
->receivedmap
);
219 void ramblock_recv_bitmap_set_range(RAMBlock
*rb
, void *host_addr
,
222 bitmap_set_atomic(rb
->receivedmap
,
223 ramblock_recv_bitmap_offset(host_addr
, rb
),
227 #define RAMBLOCK_RECV_BITMAP_ENDING (0x0123456789abcdefULL)
230 * Format: bitmap_size (8 bytes) + whole_bitmap (N bytes).
232 * Returns >0 if success with sent bytes, or <0 if error.
234 int64_t ramblock_recv_bitmap_send(QEMUFile
*file
,
235 const char *block_name
)
237 RAMBlock
*block
= qemu_ram_block_by_name(block_name
);
238 unsigned long *le_bitmap
, nbits
;
242 error_report("%s: invalid block name: %s", __func__
, block_name
);
246 nbits
= block
->used_length
>> TARGET_PAGE_BITS
;
249 * Make sure the tmp bitmap buffer is big enough, e.g., on 32bit
250 * machines we may need 4 more bytes for padding (see below
251 * comment). So extend it a bit before hand.
253 le_bitmap
= bitmap_new(nbits
+ BITS_PER_LONG
);
256 * Always use little endian when sending the bitmap. This is
257 * required that when source and destination VMs are not using the
258 * same endianess. (Note: big endian won't work.)
260 bitmap_to_le(le_bitmap
, block
->receivedmap
, nbits
);
262 /* Size of the bitmap, in bytes */
263 size
= DIV_ROUND_UP(nbits
, 8);
266 * size is always aligned to 8 bytes for 64bit machines, but it
267 * may not be true for 32bit machines. We need this padding to
268 * make sure the migration can survive even between 32bit and
271 size
= ROUND_UP(size
, 8);
273 qemu_put_be64(file
, size
);
274 qemu_put_buffer(file
, (const uint8_t *)le_bitmap
, size
);
276 * Mark as an end, in case the middle part is screwed up due to
277 * some "misterious" reason.
279 qemu_put_be64(file
, RAMBLOCK_RECV_BITMAP_ENDING
);
284 if (qemu_file_get_error(file
)) {
285 return qemu_file_get_error(file
);
288 return size
+ sizeof(size
);
292 * An outstanding page request, on the source, having been received
295 struct RAMSrcPageRequest
{
300 QSIMPLEQ_ENTRY(RAMSrcPageRequest
) next_req
;
303 /* State of RAM for migration */
305 /* QEMUFile used for this migration */
307 /* Last block that we have visited searching for dirty pages */
308 RAMBlock
*last_seen_block
;
309 /* Last block from where we have sent data */
310 RAMBlock
*last_sent_block
;
311 /* Last dirty target page we have sent */
312 ram_addr_t last_page
;
313 /* last ram version we have seen */
314 uint32_t last_version
;
315 /* We are in the first round */
317 /* The free page optimization is enabled */
319 /* How many times we have dirty too many pages */
320 int dirty_rate_high_cnt
;
321 /* these variables are used for bitmap sync */
322 /* last time we did a full bitmap_sync */
323 int64_t time_last_bitmap_sync
;
324 /* bytes transferred at start_time */
325 uint64_t bytes_xfer_prev
;
326 /* number of dirty pages since start_time */
327 uint64_t num_dirty_pages_period
;
328 /* xbzrle misses since the beginning of the period */
329 uint64_t xbzrle_cache_miss_prev
;
331 /* compression statistics since the beginning of the period */
332 /* amount of count that no free thread to compress data */
333 uint64_t compress_thread_busy_prev
;
334 /* amount bytes after compression */
335 uint64_t compressed_size_prev
;
336 /* amount of compressed pages */
337 uint64_t compress_pages_prev
;
339 /* total handled target pages at the beginning of period */
340 uint64_t target_page_count_prev
;
341 /* total handled target pages since start */
342 uint64_t target_page_count
;
343 /* number of dirty bits in the bitmap */
344 uint64_t migration_dirty_pages
;
345 /* Protects modification of the bitmap and migration dirty pages */
346 QemuMutex bitmap_mutex
;
347 /* The RAMBlock used in the last src_page_requests */
348 RAMBlock
*last_req_rb
;
349 /* Queue of outstanding page requests from the destination */
350 QemuMutex src_page_req_mutex
;
351 QSIMPLEQ_HEAD(, RAMSrcPageRequest
) src_page_requests
;
353 typedef struct RAMState RAMState
;
355 static RAMState
*ram_state
;
357 static NotifierWithReturnList precopy_notifier_list
;
359 void precopy_infrastructure_init(void)
361 notifier_with_return_list_init(&precopy_notifier_list
);
364 void precopy_add_notifier(NotifierWithReturn
*n
)
366 notifier_with_return_list_add(&precopy_notifier_list
, n
);
369 void precopy_remove_notifier(NotifierWithReturn
*n
)
371 notifier_with_return_remove(n
);
374 int precopy_notify(PrecopyNotifyReason reason
, Error
**errp
)
376 PrecopyNotifyData pnd
;
380 return notifier_with_return_list_notify(&precopy_notifier_list
, &pnd
);
383 void precopy_enable_free_page_optimization(void)
389 ram_state
->fpo_enabled
= true;
392 uint64_t ram_bytes_remaining(void)
394 return ram_state
? (ram_state
->migration_dirty_pages
* TARGET_PAGE_SIZE
) :
398 MigrationStats ram_counters
;
400 /* used by the search for pages to send */
401 struct PageSearchStatus
{
402 /* Current block being searched */
404 /* Current page to search from */
406 /* Set once we wrap around */
409 typedef struct PageSearchStatus PageSearchStatus
;
411 CompressionStats compression_counters
;
413 struct CompressParam
{
423 /* internally used fields */
427 typedef struct CompressParam CompressParam
;
429 struct DecompressParam
{
439 typedef struct DecompressParam DecompressParam
;
441 static CompressParam
*comp_param
;
442 static QemuThread
*compress_threads
;
443 /* comp_done_cond is used to wake up the migration thread when
444 * one of the compression threads has finished the compression.
445 * comp_done_lock is used to co-work with comp_done_cond.
447 static QemuMutex comp_done_lock
;
448 static QemuCond comp_done_cond
;
449 /* The empty QEMUFileOps will be used by file in CompressParam */
450 static const QEMUFileOps empty_ops
= { };
452 static QEMUFile
*decomp_file
;
453 static DecompressParam
*decomp_param
;
454 static QemuThread
*decompress_threads
;
455 static QemuMutex decomp_done_lock
;
456 static QemuCond decomp_done_cond
;
458 static bool do_compress_ram_page(QEMUFile
*f
, z_stream
*stream
, RAMBlock
*block
,
459 ram_addr_t offset
, uint8_t *source_buf
);
461 static void *do_data_compress(void *opaque
)
463 CompressParam
*param
= opaque
;
468 qemu_mutex_lock(¶m
->mutex
);
469 while (!param
->quit
) {
471 block
= param
->block
;
472 offset
= param
->offset
;
474 qemu_mutex_unlock(¶m
->mutex
);
476 zero_page
= do_compress_ram_page(param
->file
, ¶m
->stream
,
477 block
, offset
, param
->originbuf
);
479 qemu_mutex_lock(&comp_done_lock
);
481 param
->zero_page
= zero_page
;
482 qemu_cond_signal(&comp_done_cond
);
483 qemu_mutex_unlock(&comp_done_lock
);
485 qemu_mutex_lock(¶m
->mutex
);
487 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
490 qemu_mutex_unlock(¶m
->mutex
);
495 static void compress_threads_save_cleanup(void)
499 if (!migrate_use_compression() || !comp_param
) {
503 thread_count
= migrate_compress_threads();
504 for (i
= 0; i
< thread_count
; i
++) {
506 * we use it as a indicator which shows if the thread is
507 * properly init'd or not
509 if (!comp_param
[i
].file
) {
513 qemu_mutex_lock(&comp_param
[i
].mutex
);
514 comp_param
[i
].quit
= true;
515 qemu_cond_signal(&comp_param
[i
].cond
);
516 qemu_mutex_unlock(&comp_param
[i
].mutex
);
518 qemu_thread_join(compress_threads
+ i
);
519 qemu_mutex_destroy(&comp_param
[i
].mutex
);
520 qemu_cond_destroy(&comp_param
[i
].cond
);
521 deflateEnd(&comp_param
[i
].stream
);
522 g_free(comp_param
[i
].originbuf
);
523 qemu_fclose(comp_param
[i
].file
);
524 comp_param
[i
].file
= NULL
;
526 qemu_mutex_destroy(&comp_done_lock
);
527 qemu_cond_destroy(&comp_done_cond
);
528 g_free(compress_threads
);
530 compress_threads
= NULL
;
534 static int compress_threads_save_setup(void)
538 if (!migrate_use_compression()) {
541 thread_count
= migrate_compress_threads();
542 compress_threads
= g_new0(QemuThread
, thread_count
);
543 comp_param
= g_new0(CompressParam
, thread_count
);
544 qemu_cond_init(&comp_done_cond
);
545 qemu_mutex_init(&comp_done_lock
);
546 for (i
= 0; i
< thread_count
; i
++) {
547 comp_param
[i
].originbuf
= g_try_malloc(TARGET_PAGE_SIZE
);
548 if (!comp_param
[i
].originbuf
) {
552 if (deflateInit(&comp_param
[i
].stream
,
553 migrate_compress_level()) != Z_OK
) {
554 g_free(comp_param
[i
].originbuf
);
558 /* comp_param[i].file is just used as a dummy buffer to save data,
559 * set its ops to empty.
561 comp_param
[i
].file
= qemu_fopen_ops(NULL
, &empty_ops
);
562 comp_param
[i
].done
= true;
563 comp_param
[i
].quit
= false;
564 qemu_mutex_init(&comp_param
[i
].mutex
);
565 qemu_cond_init(&comp_param
[i
].cond
);
566 qemu_thread_create(compress_threads
+ i
, "compress",
567 do_data_compress
, comp_param
+ i
,
568 QEMU_THREAD_JOINABLE
);
573 compress_threads_save_cleanup();
578 * save_page_header: write page header to wire
580 * If this is the 1st block, it also writes the block identification
582 * Returns the number of bytes written
584 * @f: QEMUFile where to send the data
585 * @block: block that contains the page we want to send
586 * @offset: offset inside the block for the page
587 * in the lower bits, it contains flags
589 static size_t save_page_header(RAMState
*rs
, QEMUFile
*f
, RAMBlock
*block
,
594 if (block
== rs
->last_sent_block
) {
595 offset
|= RAM_SAVE_FLAG_CONTINUE
;
597 qemu_put_be64(f
, offset
);
600 if (!(offset
& RAM_SAVE_FLAG_CONTINUE
)) {
601 len
= strlen(block
->idstr
);
602 qemu_put_byte(f
, len
);
603 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, len
);
605 rs
->last_sent_block
= block
;
611 * mig_throttle_guest_down: throotle down the guest
613 * Reduce amount of guest cpu execution to hopefully slow down memory
614 * writes. If guest dirty memory rate is reduced below the rate at
615 * which we can transfer pages to the destination then we should be
616 * able to complete migration. Some workloads dirty memory way too
617 * fast and will not effectively converge, even with auto-converge.
619 static void mig_throttle_guest_down(void)
621 MigrationState
*s
= migrate_get_current();
622 uint64_t pct_initial
= s
->parameters
.cpu_throttle_initial
;
623 uint64_t pct_icrement
= s
->parameters
.cpu_throttle_increment
;
624 int pct_max
= s
->parameters
.max_cpu_throttle
;
626 /* We have not started throttling yet. Let's start it. */
627 if (!cpu_throttle_active()) {
628 cpu_throttle_set(pct_initial
);
630 /* Throttling already on, just increase the rate */
631 cpu_throttle_set(MIN(cpu_throttle_get_percentage() + pct_icrement
,
637 * xbzrle_cache_zero_page: insert a zero page in the XBZRLE cache
639 * @rs: current RAM state
640 * @current_addr: address for the zero page
642 * Update the xbzrle cache to reflect a page that's been sent as all 0.
643 * The important thing is that a stale (not-yet-0'd) page be replaced
645 * As a bonus, if the page wasn't in the cache it gets added so that
646 * when a small write is made into the 0'd page it gets XBZRLE sent.
648 static void xbzrle_cache_zero_page(RAMState
*rs
, ram_addr_t current_addr
)
650 if (rs
->ram_bulk_stage
|| !migrate_use_xbzrle()) {
654 /* We don't care if this fails to allocate a new cache page
655 * as long as it updated an old one */
656 cache_insert(XBZRLE
.cache
, current_addr
, XBZRLE
.zero_target_page
,
657 ram_counters
.dirty_sync_count
);
660 #define ENCODING_FLAG_XBZRLE 0x1
663 * save_xbzrle_page: compress and send current page
665 * Returns: 1 means that we wrote the page
666 * 0 means that page is identical to the one already sent
667 * -1 means that xbzrle would be longer than normal
669 * @rs: current RAM state
670 * @current_data: pointer to the address of the page contents
671 * @current_addr: addr of the page
672 * @block: block that contains the page we want to send
673 * @offset: offset inside the block for the page
674 * @last_stage: if we are at the completion stage
676 static int save_xbzrle_page(RAMState
*rs
, uint8_t **current_data
,
677 ram_addr_t current_addr
, RAMBlock
*block
,
678 ram_addr_t offset
, bool last_stage
)
680 int encoded_len
= 0, bytes_xbzrle
;
681 uint8_t *prev_cached_page
;
683 if (!cache_is_cached(XBZRLE
.cache
, current_addr
,
684 ram_counters
.dirty_sync_count
)) {
685 xbzrle_counters
.cache_miss
++;
687 if (cache_insert(XBZRLE
.cache
, current_addr
, *current_data
,
688 ram_counters
.dirty_sync_count
) == -1) {
691 /* update *current_data when the page has been
692 inserted into cache */
693 *current_data
= get_cached_data(XBZRLE
.cache
, current_addr
);
699 prev_cached_page
= get_cached_data(XBZRLE
.cache
, current_addr
);
701 /* save current buffer into memory */
702 memcpy(XBZRLE
.current_buf
, *current_data
, TARGET_PAGE_SIZE
);
704 /* XBZRLE encoding (if there is no overflow) */
705 encoded_len
= xbzrle_encode_buffer(prev_cached_page
, XBZRLE
.current_buf
,
706 TARGET_PAGE_SIZE
, XBZRLE
.encoded_buf
,
710 * Update the cache contents, so that it corresponds to the data
711 * sent, in all cases except where we skip the page.
713 if (!last_stage
&& encoded_len
!= 0) {
714 memcpy(prev_cached_page
, XBZRLE
.current_buf
, TARGET_PAGE_SIZE
);
716 * In the case where we couldn't compress, ensure that the caller
717 * sends the data from the cache, since the guest might have
718 * changed the RAM since we copied it.
720 *current_data
= prev_cached_page
;
723 if (encoded_len
== 0) {
724 trace_save_xbzrle_page_skipping();
726 } else if (encoded_len
== -1) {
727 trace_save_xbzrle_page_overflow();
728 xbzrle_counters
.overflow
++;
732 /* Send XBZRLE based compressed page */
733 bytes_xbzrle
= save_page_header(rs
, rs
->f
, block
,
734 offset
| RAM_SAVE_FLAG_XBZRLE
);
735 qemu_put_byte(rs
->f
, ENCODING_FLAG_XBZRLE
);
736 qemu_put_be16(rs
->f
, encoded_len
);
737 qemu_put_buffer(rs
->f
, XBZRLE
.encoded_buf
, encoded_len
);
738 bytes_xbzrle
+= encoded_len
+ 1 + 2;
739 xbzrle_counters
.pages
++;
740 xbzrle_counters
.bytes
+= bytes_xbzrle
;
741 ram_counters
.transferred
+= bytes_xbzrle
;
747 * migration_bitmap_find_dirty: find the next dirty page from start
749 * Returns the page offset within memory region of the start of a dirty page
751 * @rs: current RAM state
752 * @rb: RAMBlock where to search for dirty pages
753 * @start: page where we start the search
756 unsigned long migration_bitmap_find_dirty(RAMState
*rs
, RAMBlock
*rb
,
759 unsigned long size
= rb
->used_length
>> TARGET_PAGE_BITS
;
760 unsigned long *bitmap
= rb
->bmap
;
763 if (ramblock_is_ignored(rb
)) {
768 * When the free page optimization is enabled, we need to check the bitmap
769 * to send the non-free pages rather than all the pages in the bulk stage.
771 if (!rs
->fpo_enabled
&& rs
->ram_bulk_stage
&& start
> 0) {
774 next
= find_next_bit(bitmap
, size
, start
);
780 static inline bool migration_bitmap_clear_dirty(RAMState
*rs
,
786 qemu_mutex_lock(&rs
->bitmap_mutex
);
789 * Clear dirty bitmap if needed. This _must_ be called before we
790 * send any of the page in the chunk because we need to make sure
791 * we can capture further page content changes when we sync dirty
792 * log the next time. So as long as we are going to send any of
793 * the page in the chunk we clear the remote dirty bitmap for all.
794 * Clearing it earlier won't be a problem, but too late will.
796 if (rb
->clear_bmap
&& clear_bmap_test_and_clear(rb
, page
)) {
797 uint8_t shift
= rb
->clear_bmap_shift
;
798 hwaddr size
= 1ULL << (TARGET_PAGE_BITS
+ shift
);
799 hwaddr start
= (((ram_addr_t
)page
) << TARGET_PAGE_BITS
) & (-size
);
802 * CLEAR_BITMAP_SHIFT_MIN should always guarantee this... this
803 * can make things easier sometimes since then start address
804 * of the small chunk will always be 64 pages aligned so the
805 * bitmap will always be aligned to unsigned long. We should
806 * even be able to remove this restriction but I'm simply
810 trace_migration_bitmap_clear_dirty(rb
->idstr
, start
, size
, page
);
811 memory_region_clear_dirty_bitmap(rb
->mr
, start
, size
);
814 ret
= test_and_clear_bit(page
, rb
->bmap
);
817 rs
->migration_dirty_pages
--;
819 qemu_mutex_unlock(&rs
->bitmap_mutex
);
824 /* Called with RCU critical section */
825 static void ramblock_sync_dirty_bitmap(RAMState
*rs
, RAMBlock
*rb
)
827 rs
->migration_dirty_pages
+=
828 cpu_physical_memory_sync_dirty_bitmap(rb
, 0, rb
->used_length
,
829 &rs
->num_dirty_pages_period
);
833 * ram_pagesize_summary: calculate all the pagesizes of a VM
835 * Returns a summary bitmap of the page sizes of all RAMBlocks
837 * For VMs with just normal pages this is equivalent to the host page
838 * size. If it's got some huge pages then it's the OR of all the
839 * different page sizes.
841 uint64_t ram_pagesize_summary(void)
844 uint64_t summary
= 0;
846 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
847 summary
|= block
->page_size
;
853 uint64_t ram_get_total_transferred_pages(void)
855 return ram_counters
.normal
+ ram_counters
.duplicate
+
856 compression_counters
.pages
+ xbzrle_counters
.pages
;
859 static void migration_update_rates(RAMState
*rs
, int64_t end_time
)
861 uint64_t page_count
= rs
->target_page_count
- rs
->target_page_count_prev
;
862 double compressed_size
;
864 /* calculate period counters */
865 ram_counters
.dirty_pages_rate
= rs
->num_dirty_pages_period
* 1000
866 / (end_time
- rs
->time_last_bitmap_sync
);
872 if (migrate_use_xbzrle()) {
873 xbzrle_counters
.cache_miss_rate
= (double)(xbzrle_counters
.cache_miss
-
874 rs
->xbzrle_cache_miss_prev
) / page_count
;
875 rs
->xbzrle_cache_miss_prev
= xbzrle_counters
.cache_miss
;
878 if (migrate_use_compression()) {
879 compression_counters
.busy_rate
= (double)(compression_counters
.busy
-
880 rs
->compress_thread_busy_prev
) / page_count
;
881 rs
->compress_thread_busy_prev
= compression_counters
.busy
;
883 compressed_size
= compression_counters
.compressed_size
-
884 rs
->compressed_size_prev
;
885 if (compressed_size
) {
886 double uncompressed_size
= (compression_counters
.pages
-
887 rs
->compress_pages_prev
) * TARGET_PAGE_SIZE
;
889 /* Compression-Ratio = Uncompressed-size / Compressed-size */
890 compression_counters
.compression_rate
=
891 uncompressed_size
/ compressed_size
;
893 rs
->compress_pages_prev
= compression_counters
.pages
;
894 rs
->compressed_size_prev
= compression_counters
.compressed_size
;
899 static void migration_bitmap_sync(RAMState
*rs
)
903 uint64_t bytes_xfer_now
;
905 ram_counters
.dirty_sync_count
++;
907 if (!rs
->time_last_bitmap_sync
) {
908 rs
->time_last_bitmap_sync
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
911 trace_migration_bitmap_sync_start();
912 memory_global_dirty_log_sync();
914 qemu_mutex_lock(&rs
->bitmap_mutex
);
915 WITH_RCU_READ_LOCK_GUARD() {
916 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
917 ramblock_sync_dirty_bitmap(rs
, block
);
919 ram_counters
.remaining
= ram_bytes_remaining();
921 qemu_mutex_unlock(&rs
->bitmap_mutex
);
923 memory_global_after_dirty_log_sync();
924 trace_migration_bitmap_sync_end(rs
->num_dirty_pages_period
);
926 end_time
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
928 /* more than 1 second = 1000 millisecons */
929 if (end_time
> rs
->time_last_bitmap_sync
+ 1000) {
930 bytes_xfer_now
= ram_counters
.transferred
;
932 /* During block migration the auto-converge logic incorrectly detects
933 * that ram migration makes no progress. Avoid this by disabling the
934 * throttling logic during the bulk phase of block migration. */
935 if (migrate_auto_converge() && !blk_mig_bulk_active()) {
936 /* The following detection logic can be refined later. For now:
937 Check to see if the dirtied bytes is 50% more than the approx.
938 amount of bytes that just got transferred since the last time we
939 were in this routine. If that happens twice, start or increase
942 if ((rs
->num_dirty_pages_period
* TARGET_PAGE_SIZE
>
943 (bytes_xfer_now
- rs
->bytes_xfer_prev
) / 2) &&
944 (++rs
->dirty_rate_high_cnt
>= 2)) {
945 trace_migration_throttle();
946 rs
->dirty_rate_high_cnt
= 0;
947 mig_throttle_guest_down();
951 migration_update_rates(rs
, end_time
);
953 rs
->target_page_count_prev
= rs
->target_page_count
;
955 /* reset period counters */
956 rs
->time_last_bitmap_sync
= end_time
;
957 rs
->num_dirty_pages_period
= 0;
958 rs
->bytes_xfer_prev
= bytes_xfer_now
;
960 if (migrate_use_events()) {
961 qapi_event_send_migration_pass(ram_counters
.dirty_sync_count
);
965 static void migration_bitmap_sync_precopy(RAMState
*rs
)
967 Error
*local_err
= NULL
;
970 * The current notifier usage is just an optimization to migration, so we
971 * don't stop the normal migration process in the error case.
973 if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC
, &local_err
)) {
974 error_report_err(local_err
);
977 migration_bitmap_sync(rs
);
979 if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC
, &local_err
)) {
980 error_report_err(local_err
);
985 * save_zero_page_to_file: send the zero page to the file
987 * Returns the size of data written to the file, 0 means the page is not
990 * @rs: current RAM state
991 * @file: the file where the data is saved
992 * @block: block that contains the page we want to send
993 * @offset: offset inside the block for the page
995 static int save_zero_page_to_file(RAMState
*rs
, QEMUFile
*file
,
996 RAMBlock
*block
, ram_addr_t offset
)
998 uint8_t *p
= block
->host
+ offset
;
1001 if (is_zero_range(p
, TARGET_PAGE_SIZE
)) {
1002 len
+= save_page_header(rs
, file
, block
, offset
| RAM_SAVE_FLAG_ZERO
);
1003 qemu_put_byte(file
, 0);
1010 * save_zero_page: send the zero page to the stream
1012 * Returns the number of pages written.
1014 * @rs: current RAM state
1015 * @block: block that contains the page we want to send
1016 * @offset: offset inside the block for the page
1018 static int save_zero_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
)
1020 int len
= save_zero_page_to_file(rs
, rs
->f
, block
, offset
);
1023 ram_counters
.duplicate
++;
1024 ram_counters
.transferred
+= len
;
1030 static void ram_release_pages(const char *rbname
, uint64_t offset
, int pages
)
1032 if (!migrate_release_ram() || !migration_in_postcopy()) {
1036 ram_discard_range(rbname
, offset
, ((ram_addr_t
)pages
) << TARGET_PAGE_BITS
);
1040 * @pages: the number of pages written by the control path,
1042 * > 0 - number of pages written
1044 * Return true if the pages has been saved, otherwise false is returned.
1046 static bool control_save_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
,
1049 uint64_t bytes_xmit
= 0;
1053 ret
= ram_control_save_page(rs
->f
, block
->offset
, offset
, TARGET_PAGE_SIZE
,
1055 if (ret
== RAM_SAVE_CONTROL_NOT_SUPP
) {
1060 ram_counters
.transferred
+= bytes_xmit
;
1064 if (ret
== RAM_SAVE_CONTROL_DELAYED
) {
1068 if (bytes_xmit
> 0) {
1069 ram_counters
.normal
++;
1070 } else if (bytes_xmit
== 0) {
1071 ram_counters
.duplicate
++;
1078 * directly send the page to the stream
1080 * Returns the number of pages written.
1082 * @rs: current RAM state
1083 * @block: block that contains the page we want to send
1084 * @offset: offset inside the block for the page
1085 * @buf: the page to be sent
1086 * @async: send to page asyncly
1088 static int save_normal_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
,
1089 uint8_t *buf
, bool async
)
1091 ram_counters
.transferred
+= save_page_header(rs
, rs
->f
, block
,
1092 offset
| RAM_SAVE_FLAG_PAGE
);
1094 qemu_put_buffer_async(rs
->f
, buf
, TARGET_PAGE_SIZE
,
1095 migrate_release_ram() &
1096 migration_in_postcopy());
1098 qemu_put_buffer(rs
->f
, buf
, TARGET_PAGE_SIZE
);
1100 ram_counters
.transferred
+= TARGET_PAGE_SIZE
;
1101 ram_counters
.normal
++;
1106 * ram_save_page: send the given page to the stream
1108 * Returns the number of pages written.
1110 * >=0 - Number of pages written - this might legally be 0
1111 * if xbzrle noticed the page was the same.
1113 * @rs: current RAM state
1114 * @block: block that contains the page we want to send
1115 * @offset: offset inside the block for the page
1116 * @last_stage: if we are at the completion stage
1118 static int ram_save_page(RAMState
*rs
, PageSearchStatus
*pss
, bool last_stage
)
1122 bool send_async
= true;
1123 RAMBlock
*block
= pss
->block
;
1124 ram_addr_t offset
= ((ram_addr_t
)pss
->page
) << TARGET_PAGE_BITS
;
1125 ram_addr_t current_addr
= block
->offset
+ offset
;
1127 p
= block
->host
+ offset
;
1128 trace_ram_save_page(block
->idstr
, (uint64_t)offset
, p
);
1130 XBZRLE_cache_lock();
1131 if (!rs
->ram_bulk_stage
&& !migration_in_postcopy() &&
1132 migrate_use_xbzrle()) {
1133 pages
= save_xbzrle_page(rs
, &p
, current_addr
, block
,
1134 offset
, last_stage
);
1136 /* Can't send this cached data async, since the cache page
1137 * might get updated before it gets to the wire
1143 /* XBZRLE overflow or normal page */
1145 pages
= save_normal_page(rs
, block
, offset
, p
, send_async
);
1148 XBZRLE_cache_unlock();
1153 static int ram_save_multifd_page(RAMState
*rs
, RAMBlock
*block
,
1156 if (multifd_queue_page(rs
->f
, block
, offset
) < 0) {
1159 ram_counters
.normal
++;
1164 static bool do_compress_ram_page(QEMUFile
*f
, z_stream
*stream
, RAMBlock
*block
,
1165 ram_addr_t offset
, uint8_t *source_buf
)
1167 RAMState
*rs
= ram_state
;
1168 uint8_t *p
= block
->host
+ (offset
& TARGET_PAGE_MASK
);
1169 bool zero_page
= false;
1172 if (save_zero_page_to_file(rs
, f
, block
, offset
)) {
1177 save_page_header(rs
, f
, block
, offset
| RAM_SAVE_FLAG_COMPRESS_PAGE
);
1180 * copy it to a internal buffer to avoid it being modified by VM
1181 * so that we can catch up the error during compression and
1184 memcpy(source_buf
, p
, TARGET_PAGE_SIZE
);
1185 ret
= qemu_put_compression_data(f
, stream
, source_buf
, TARGET_PAGE_SIZE
);
1187 qemu_file_set_error(migrate_get_current()->to_dst_file
, ret
);
1188 error_report("compressed data failed!");
1193 ram_release_pages(block
->idstr
, offset
& TARGET_PAGE_MASK
, 1);
1198 update_compress_thread_counts(const CompressParam
*param
, int bytes_xmit
)
1200 ram_counters
.transferred
+= bytes_xmit
;
1202 if (param
->zero_page
) {
1203 ram_counters
.duplicate
++;
1207 /* 8 means a header with RAM_SAVE_FLAG_CONTINUE. */
1208 compression_counters
.compressed_size
+= bytes_xmit
- 8;
1209 compression_counters
.pages
++;
1212 static bool save_page_use_compression(RAMState
*rs
);
1214 static void flush_compressed_data(RAMState
*rs
)
1216 int idx
, len
, thread_count
;
1218 if (!save_page_use_compression(rs
)) {
1221 thread_count
= migrate_compress_threads();
1223 qemu_mutex_lock(&comp_done_lock
);
1224 for (idx
= 0; idx
< thread_count
; idx
++) {
1225 while (!comp_param
[idx
].done
) {
1226 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
1229 qemu_mutex_unlock(&comp_done_lock
);
1231 for (idx
= 0; idx
< thread_count
; idx
++) {
1232 qemu_mutex_lock(&comp_param
[idx
].mutex
);
1233 if (!comp_param
[idx
].quit
) {
1234 len
= qemu_put_qemu_file(rs
->f
, comp_param
[idx
].file
);
1236 * it's safe to fetch zero_page without holding comp_done_lock
1237 * as there is no further request submitted to the thread,
1238 * i.e, the thread should be waiting for a request at this point.
1240 update_compress_thread_counts(&comp_param
[idx
], len
);
1242 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
1246 static inline void set_compress_params(CompressParam
*param
, RAMBlock
*block
,
1249 param
->block
= block
;
1250 param
->offset
= offset
;
1253 static int compress_page_with_multi_thread(RAMState
*rs
, RAMBlock
*block
,
1256 int idx
, thread_count
, bytes_xmit
= -1, pages
= -1;
1257 bool wait
= migrate_compress_wait_thread();
1259 thread_count
= migrate_compress_threads();
1260 qemu_mutex_lock(&comp_done_lock
);
1262 for (idx
= 0; idx
< thread_count
; idx
++) {
1263 if (comp_param
[idx
].done
) {
1264 comp_param
[idx
].done
= false;
1265 bytes_xmit
= qemu_put_qemu_file(rs
->f
, comp_param
[idx
].file
);
1266 qemu_mutex_lock(&comp_param
[idx
].mutex
);
1267 set_compress_params(&comp_param
[idx
], block
, offset
);
1268 qemu_cond_signal(&comp_param
[idx
].cond
);
1269 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
1271 update_compress_thread_counts(&comp_param
[idx
], bytes_xmit
);
1277 * wait for the free thread if the user specifies 'compress-wait-thread',
1278 * otherwise we will post the page out in the main thread as normal page.
1280 if (pages
< 0 && wait
) {
1281 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
1284 qemu_mutex_unlock(&comp_done_lock
);
1290 * find_dirty_block: find the next dirty page and update any state
1291 * associated with the search process.
1293 * Returns true if a page is found
1295 * @rs: current RAM state
1296 * @pss: data about the state of the current dirty page scan
1297 * @again: set to false if the search has scanned the whole of RAM
1299 static bool find_dirty_block(RAMState
*rs
, PageSearchStatus
*pss
, bool *again
)
1301 pss
->page
= migration_bitmap_find_dirty(rs
, pss
->block
, pss
->page
);
1302 if (pss
->complete_round
&& pss
->block
== rs
->last_seen_block
&&
1303 pss
->page
>= rs
->last_page
) {
1305 * We've been once around the RAM and haven't found anything.
1311 if ((((ram_addr_t
)pss
->page
) << TARGET_PAGE_BITS
)
1312 >= pss
->block
->used_length
) {
1313 /* Didn't find anything in this RAM Block */
1315 pss
->block
= QLIST_NEXT_RCU(pss
->block
, next
);
1318 * If memory migration starts over, we will meet a dirtied page
1319 * which may still exists in compression threads's ring, so we
1320 * should flush the compressed data to make sure the new page
1321 * is not overwritten by the old one in the destination.
1323 * Also If xbzrle is on, stop using the data compression at this
1324 * point. In theory, xbzrle can do better than compression.
1326 flush_compressed_data(rs
);
1328 /* Hit the end of the list */
1329 pss
->block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
1330 /* Flag that we've looped */
1331 pss
->complete_round
= true;
1332 rs
->ram_bulk_stage
= false;
1334 /* Didn't find anything this time, but try again on the new block */
1338 /* Can go around again, but... */
1340 /* We've found something so probably don't need to */
1346 * unqueue_page: gets a page of the queue
1348 * Helper for 'get_queued_page' - gets a page off the queue
1350 * Returns the block of the page (or NULL if none available)
1352 * @rs: current RAM state
1353 * @offset: used to return the offset within the RAMBlock
1355 static RAMBlock
*unqueue_page(RAMState
*rs
, ram_addr_t
*offset
)
1357 RAMBlock
*block
= NULL
;
1359 if (QSIMPLEQ_EMPTY_ATOMIC(&rs
->src_page_requests
)) {
1363 qemu_mutex_lock(&rs
->src_page_req_mutex
);
1364 if (!QSIMPLEQ_EMPTY(&rs
->src_page_requests
)) {
1365 struct RAMSrcPageRequest
*entry
=
1366 QSIMPLEQ_FIRST(&rs
->src_page_requests
);
1368 *offset
= entry
->offset
;
1370 if (entry
->len
> TARGET_PAGE_SIZE
) {
1371 entry
->len
-= TARGET_PAGE_SIZE
;
1372 entry
->offset
+= TARGET_PAGE_SIZE
;
1374 memory_region_unref(block
->mr
);
1375 QSIMPLEQ_REMOVE_HEAD(&rs
->src_page_requests
, next_req
);
1377 migration_consume_urgent_request();
1380 qemu_mutex_unlock(&rs
->src_page_req_mutex
);
1386 * get_queued_page: unqueue a page from the postcopy requests
1388 * Skips pages that are already sent (!dirty)
1390 * Returns true if a queued page is found
1392 * @rs: current RAM state
1393 * @pss: data about the state of the current dirty page scan
1395 static bool get_queued_page(RAMState
*rs
, PageSearchStatus
*pss
)
1402 block
= unqueue_page(rs
, &offset
);
1404 * We're sending this page, and since it's postcopy nothing else
1405 * will dirty it, and we must make sure it doesn't get sent again
1406 * even if this queue request was received after the background
1407 * search already sent it.
1412 page
= offset
>> TARGET_PAGE_BITS
;
1413 dirty
= test_bit(page
, block
->bmap
);
1415 trace_get_queued_page_not_dirty(block
->idstr
, (uint64_t)offset
,
1418 trace_get_queued_page(block
->idstr
, (uint64_t)offset
, page
);
1422 } while (block
&& !dirty
);
1426 * As soon as we start servicing pages out of order, then we have
1427 * to kill the bulk stage, since the bulk stage assumes
1428 * in (migration_bitmap_find_and_reset_dirty) that every page is
1429 * dirty, that's no longer true.
1431 rs
->ram_bulk_stage
= false;
1434 * We want the background search to continue from the queued page
1435 * since the guest is likely to want other pages near to the page
1436 * it just requested.
1439 pss
->page
= offset
>> TARGET_PAGE_BITS
;
1442 * This unqueued page would break the "one round" check, even is
1445 pss
->complete_round
= false;
1452 * migration_page_queue_free: drop any remaining pages in the ram
1455 * It should be empty at the end anyway, but in error cases there may
1456 * be some left. in case that there is any page left, we drop it.
1459 static void migration_page_queue_free(RAMState
*rs
)
1461 struct RAMSrcPageRequest
*mspr
, *next_mspr
;
1462 /* This queue generally should be empty - but in the case of a failed
1463 * migration might have some droppings in.
1465 RCU_READ_LOCK_GUARD();
1466 QSIMPLEQ_FOREACH_SAFE(mspr
, &rs
->src_page_requests
, next_req
, next_mspr
) {
1467 memory_region_unref(mspr
->rb
->mr
);
1468 QSIMPLEQ_REMOVE_HEAD(&rs
->src_page_requests
, next_req
);
1474 * ram_save_queue_pages: queue the page for transmission
1476 * A request from postcopy destination for example.
1478 * Returns zero on success or negative on error
1480 * @rbname: Name of the RAMBLock of the request. NULL means the
1481 * same that last one.
1482 * @start: starting address from the start of the RAMBlock
1483 * @len: length (in bytes) to send
1485 int ram_save_queue_pages(const char *rbname
, ram_addr_t start
, ram_addr_t len
)
1488 RAMState
*rs
= ram_state
;
1490 ram_counters
.postcopy_requests
++;
1491 RCU_READ_LOCK_GUARD();
1494 /* Reuse last RAMBlock */
1495 ramblock
= rs
->last_req_rb
;
1499 * Shouldn't happen, we can't reuse the last RAMBlock if
1500 * it's the 1st request.
1502 error_report("ram_save_queue_pages no previous block");
1506 ramblock
= qemu_ram_block_by_name(rbname
);
1509 /* We shouldn't be asked for a non-existent RAMBlock */
1510 error_report("ram_save_queue_pages no block '%s'", rbname
);
1513 rs
->last_req_rb
= ramblock
;
1515 trace_ram_save_queue_pages(ramblock
->idstr
, start
, len
);
1516 if (start
+len
> ramblock
->used_length
) {
1517 error_report("%s request overrun start=" RAM_ADDR_FMT
" len="
1518 RAM_ADDR_FMT
" blocklen=" RAM_ADDR_FMT
,
1519 __func__
, start
, len
, ramblock
->used_length
);
1523 struct RAMSrcPageRequest
*new_entry
=
1524 g_malloc0(sizeof(struct RAMSrcPageRequest
));
1525 new_entry
->rb
= ramblock
;
1526 new_entry
->offset
= start
;
1527 new_entry
->len
= len
;
1529 memory_region_ref(ramblock
->mr
);
1530 qemu_mutex_lock(&rs
->src_page_req_mutex
);
1531 QSIMPLEQ_INSERT_TAIL(&rs
->src_page_requests
, new_entry
, next_req
);
1532 migration_make_urgent_request();
1533 qemu_mutex_unlock(&rs
->src_page_req_mutex
);
1538 static bool save_page_use_compression(RAMState
*rs
)
1540 if (!migrate_use_compression()) {
1545 * If xbzrle is on, stop using the data compression after first
1546 * round of migration even if compression is enabled. In theory,
1547 * xbzrle can do better than compression.
1549 if (rs
->ram_bulk_stage
|| !migrate_use_xbzrle()) {
1557 * try to compress the page before posting it out, return true if the page
1558 * has been properly handled by compression, otherwise needs other
1559 * paths to handle it
1561 static bool save_compress_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
)
1563 if (!save_page_use_compression(rs
)) {
1568 * When starting the process of a new block, the first page of
1569 * the block should be sent out before other pages in the same
1570 * block, and all the pages in last block should have been sent
1571 * out, keeping this order is important, because the 'cont' flag
1572 * is used to avoid resending the block name.
1574 * We post the fist page as normal page as compression will take
1575 * much CPU resource.
1577 if (block
!= rs
->last_sent_block
) {
1578 flush_compressed_data(rs
);
1582 if (compress_page_with_multi_thread(rs
, block
, offset
) > 0) {
1586 compression_counters
.busy
++;
1591 * ram_save_target_page: save one target page
1593 * Returns the number of pages written
1595 * @rs: current RAM state
1596 * @pss: data about the page we want to send
1597 * @last_stage: if we are at the completion stage
1599 static int ram_save_target_page(RAMState
*rs
, PageSearchStatus
*pss
,
1602 RAMBlock
*block
= pss
->block
;
1603 ram_addr_t offset
= ((ram_addr_t
)pss
->page
) << TARGET_PAGE_BITS
;
1606 if (control_save_page(rs
, block
, offset
, &res
)) {
1610 if (save_compress_page(rs
, block
, offset
)) {
1614 res
= save_zero_page(rs
, block
, offset
);
1616 /* Must let xbzrle know, otherwise a previous (now 0'd) cached
1617 * page would be stale
1619 if (!save_page_use_compression(rs
)) {
1620 XBZRLE_cache_lock();
1621 xbzrle_cache_zero_page(rs
, block
->offset
+ offset
);
1622 XBZRLE_cache_unlock();
1624 ram_release_pages(block
->idstr
, offset
, res
);
1629 * Do not use multifd for:
1630 * 1. Compression as the first page in the new block should be posted out
1631 * before sending the compressed page
1632 * 2. In postcopy as one whole host page should be placed
1634 if (!save_page_use_compression(rs
) && migrate_use_multifd()
1635 && !migration_in_postcopy()) {
1636 return ram_save_multifd_page(rs
, block
, offset
);
1639 return ram_save_page(rs
, pss
, last_stage
);
1643 * ram_save_host_page: save a whole host page
1645 * Starting at *offset send pages up to the end of the current host
1646 * page. It's valid for the initial offset to point into the middle of
1647 * a host page in which case the remainder of the hostpage is sent.
1648 * Only dirty target pages are sent. Note that the host page size may
1649 * be a huge page for this block.
1650 * The saving stops at the boundary of the used_length of the block
1651 * if the RAMBlock isn't a multiple of the host page size.
1653 * Returns the number of pages written or negative on error
1655 * @rs: current RAM state
1656 * @ms: current migration state
1657 * @pss: data about the page we want to send
1658 * @last_stage: if we are at the completion stage
1660 static int ram_save_host_page(RAMState
*rs
, PageSearchStatus
*pss
,
1663 int tmppages
, pages
= 0;
1664 size_t pagesize_bits
=
1665 qemu_ram_pagesize(pss
->block
) >> TARGET_PAGE_BITS
;
1667 if (ramblock_is_ignored(pss
->block
)) {
1668 error_report("block %s should not be migrated !", pss
->block
->idstr
);
1673 /* Check the pages is dirty and if it is send it */
1674 if (!migration_bitmap_clear_dirty(rs
, pss
->block
, pss
->page
)) {
1679 tmppages
= ram_save_target_page(rs
, pss
, last_stage
);
1686 /* Allow rate limiting to happen in the middle of huge pages */
1687 migration_rate_limit();
1688 } while ((pss
->page
& (pagesize_bits
- 1)) &&
1689 offset_in_ramblock(pss
->block
,
1690 ((ram_addr_t
)pss
->page
) << TARGET_PAGE_BITS
));
1692 /* The offset we leave with is the last one we looked at */
1698 * ram_find_and_save_block: finds a dirty page and sends it to f
1700 * Called within an RCU critical section.
1702 * Returns the number of pages written where zero means no dirty pages,
1703 * or negative on error
1705 * @rs: current RAM state
1706 * @last_stage: if we are at the completion stage
1708 * On systems where host-page-size > target-page-size it will send all the
1709 * pages in a host page that are dirty.
1712 static int ram_find_and_save_block(RAMState
*rs
, bool last_stage
)
1714 PageSearchStatus pss
;
1718 /* No dirty page as there is zero RAM */
1719 if (!ram_bytes_total()) {
1723 pss
.block
= rs
->last_seen_block
;
1724 pss
.page
= rs
->last_page
;
1725 pss
.complete_round
= false;
1728 pss
.block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
1733 found
= get_queued_page(rs
, &pss
);
1736 /* priority queue empty, so just search for something dirty */
1737 found
= find_dirty_block(rs
, &pss
, &again
);
1741 pages
= ram_save_host_page(rs
, &pss
, last_stage
);
1743 } while (!pages
&& again
);
1745 rs
->last_seen_block
= pss
.block
;
1746 rs
->last_page
= pss
.page
;
1751 void acct_update_position(QEMUFile
*f
, size_t size
, bool zero
)
1753 uint64_t pages
= size
/ TARGET_PAGE_SIZE
;
1756 ram_counters
.duplicate
+= pages
;
1758 ram_counters
.normal
+= pages
;
1759 ram_counters
.transferred
+= size
;
1760 qemu_update_position(f
, size
);
1764 static uint64_t ram_bytes_total_common(bool count_ignored
)
1769 RCU_READ_LOCK_GUARD();
1771 if (count_ignored
) {
1772 RAMBLOCK_FOREACH_MIGRATABLE(block
) {
1773 total
+= block
->used_length
;
1776 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1777 total
+= block
->used_length
;
1783 uint64_t ram_bytes_total(void)
1785 return ram_bytes_total_common(false);
1788 static void xbzrle_load_setup(void)
1790 XBZRLE
.decoded_buf
= g_malloc(TARGET_PAGE_SIZE
);
1793 static void xbzrle_load_cleanup(void)
1795 g_free(XBZRLE
.decoded_buf
);
1796 XBZRLE
.decoded_buf
= NULL
;
1799 static void ram_state_cleanup(RAMState
**rsp
)
1802 migration_page_queue_free(*rsp
);
1803 qemu_mutex_destroy(&(*rsp
)->bitmap_mutex
);
1804 qemu_mutex_destroy(&(*rsp
)->src_page_req_mutex
);
1810 static void xbzrle_cleanup(void)
1812 XBZRLE_cache_lock();
1814 cache_fini(XBZRLE
.cache
);
1815 g_free(XBZRLE
.encoded_buf
);
1816 g_free(XBZRLE
.current_buf
);
1817 g_free(XBZRLE
.zero_target_page
);
1818 XBZRLE
.cache
= NULL
;
1819 XBZRLE
.encoded_buf
= NULL
;
1820 XBZRLE
.current_buf
= NULL
;
1821 XBZRLE
.zero_target_page
= NULL
;
1823 XBZRLE_cache_unlock();
1826 static void ram_save_cleanup(void *opaque
)
1828 RAMState
**rsp
= opaque
;
1831 /* caller have hold iothread lock or is in a bh, so there is
1832 * no writing race against the migration bitmap
1834 memory_global_dirty_log_stop();
1836 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1837 g_free(block
->clear_bmap
);
1838 block
->clear_bmap
= NULL
;
1839 g_free(block
->bmap
);
1844 compress_threads_save_cleanup();
1845 ram_state_cleanup(rsp
);
1848 static void ram_state_reset(RAMState
*rs
)
1850 rs
->last_seen_block
= NULL
;
1851 rs
->last_sent_block
= NULL
;
1853 rs
->last_version
= ram_list
.version
;
1854 rs
->ram_bulk_stage
= true;
1855 rs
->fpo_enabled
= false;
1858 #define MAX_WAIT 50 /* ms, half buffered_file limit */
1861 * 'expected' is the value you expect the bitmap mostly to be full
1862 * of; it won't bother printing lines that are all this value.
1863 * If 'todump' is null the migration bitmap is dumped.
1865 void ram_debug_dump_bitmap(unsigned long *todump
, bool expected
,
1866 unsigned long pages
)
1869 int64_t linelen
= 128;
1872 for (cur
= 0; cur
< pages
; cur
+= linelen
) {
1876 * Last line; catch the case where the line length
1877 * is longer than remaining ram
1879 if (cur
+ linelen
> pages
) {
1880 linelen
= pages
- cur
;
1882 for (curb
= 0; curb
< linelen
; curb
++) {
1883 bool thisbit
= test_bit(cur
+ curb
, todump
);
1884 linebuf
[curb
] = thisbit
? '1' : '.';
1885 found
= found
|| (thisbit
!= expected
);
1888 linebuf
[curb
] = '\0';
1889 fprintf(stderr
, "0x%08" PRIx64
" : %s\n", cur
, linebuf
);
1894 /* **** functions for postcopy ***** */
1896 void ram_postcopy_migrated_memory_release(MigrationState
*ms
)
1898 struct RAMBlock
*block
;
1900 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1901 unsigned long *bitmap
= block
->bmap
;
1902 unsigned long range
= block
->used_length
>> TARGET_PAGE_BITS
;
1903 unsigned long run_start
= find_next_zero_bit(bitmap
, range
, 0);
1905 while (run_start
< range
) {
1906 unsigned long run_end
= find_next_bit(bitmap
, range
, run_start
+ 1);
1907 ram_discard_range(block
->idstr
,
1908 ((ram_addr_t
)run_start
) << TARGET_PAGE_BITS
,
1909 ((ram_addr_t
)(run_end
- run_start
))
1910 << TARGET_PAGE_BITS
);
1911 run_start
= find_next_zero_bit(bitmap
, range
, run_end
+ 1);
1917 * postcopy_send_discard_bm_ram: discard a RAMBlock
1919 * Returns zero on success
1921 * Callback from postcopy_each_ram_send_discard for each RAMBlock
1923 * @ms: current migration state
1924 * @block: RAMBlock to discard
1926 static int postcopy_send_discard_bm_ram(MigrationState
*ms
, RAMBlock
*block
)
1928 unsigned long end
= block
->used_length
>> TARGET_PAGE_BITS
;
1929 unsigned long current
;
1930 unsigned long *bitmap
= block
->bmap
;
1932 for (current
= 0; current
< end
; ) {
1933 unsigned long one
= find_next_bit(bitmap
, end
, current
);
1934 unsigned long zero
, discard_length
;
1940 zero
= find_next_zero_bit(bitmap
, end
, one
+ 1);
1943 discard_length
= end
- one
;
1945 discard_length
= zero
- one
;
1947 postcopy_discard_send_range(ms
, one
, discard_length
);
1948 current
= one
+ discard_length
;
1955 * postcopy_each_ram_send_discard: discard all RAMBlocks
1957 * Returns 0 for success or negative for error
1959 * Utility for the outgoing postcopy code.
1960 * Calls postcopy_send_discard_bm_ram for each RAMBlock
1961 * passing it bitmap indexes and name.
1962 * (qemu_ram_foreach_block ends up passing unscaled lengths
1963 * which would mean postcopy code would have to deal with target page)
1965 * @ms: current migration state
1967 static int postcopy_each_ram_send_discard(MigrationState
*ms
)
1969 struct RAMBlock
*block
;
1972 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1973 postcopy_discard_send_init(ms
, block
->idstr
);
1976 * Postcopy sends chunks of bitmap over the wire, but it
1977 * just needs indexes at this point, avoids it having
1978 * target page specific code.
1980 ret
= postcopy_send_discard_bm_ram(ms
, block
);
1981 postcopy_discard_send_finish(ms
);
1991 * postcopy_chunk_hostpages_pass: canonicalize bitmap in hostpages
1993 * Helper for postcopy_chunk_hostpages; it's called twice to
1994 * canonicalize the two bitmaps, that are similar, but one is
1997 * Postcopy requires that all target pages in a hostpage are dirty or
1998 * clean, not a mix. This function canonicalizes the bitmaps.
2000 * @ms: current migration state
2001 * @block: block that contains the page we want to canonicalize
2003 static void postcopy_chunk_hostpages_pass(MigrationState
*ms
, RAMBlock
*block
)
2005 RAMState
*rs
= ram_state
;
2006 unsigned long *bitmap
= block
->bmap
;
2007 unsigned int host_ratio
= block
->page_size
/ TARGET_PAGE_SIZE
;
2008 unsigned long pages
= block
->used_length
>> TARGET_PAGE_BITS
;
2009 unsigned long run_start
;
2011 if (block
->page_size
== TARGET_PAGE_SIZE
) {
2012 /* Easy case - TPS==HPS for a non-huge page RAMBlock */
2016 /* Find a dirty page */
2017 run_start
= find_next_bit(bitmap
, pages
, 0);
2019 while (run_start
< pages
) {
2022 * If the start of this run of pages is in the middle of a host
2023 * page, then we need to fixup this host page.
2025 if (QEMU_IS_ALIGNED(run_start
, host_ratio
)) {
2026 /* Find the end of this run */
2027 run_start
= find_next_zero_bit(bitmap
, pages
, run_start
+ 1);
2029 * If the end isn't at the start of a host page, then the
2030 * run doesn't finish at the end of a host page
2031 * and we need to discard.
2035 if (!QEMU_IS_ALIGNED(run_start
, host_ratio
)) {
2037 unsigned long fixup_start_addr
= QEMU_ALIGN_DOWN(run_start
,
2039 run_start
= QEMU_ALIGN_UP(run_start
, host_ratio
);
2041 /* Clean up the bitmap */
2042 for (page
= fixup_start_addr
;
2043 page
< fixup_start_addr
+ host_ratio
; page
++) {
2045 * Remark them as dirty, updating the count for any pages
2046 * that weren't previously dirty.
2048 rs
->migration_dirty_pages
+= !test_and_set_bit(page
, bitmap
);
2052 /* Find the next dirty page for the next iteration */
2053 run_start
= find_next_bit(bitmap
, pages
, run_start
);
2058 * postcopy_chunk_hostpages: discard any partially sent host page
2060 * Utility for the outgoing postcopy code.
2062 * Discard any partially sent host-page size chunks, mark any partially
2063 * dirty host-page size chunks as all dirty. In this case the host-page
2064 * is the host-page for the particular RAMBlock, i.e. it might be a huge page
2066 * Returns zero on success
2068 * @ms: current migration state
2069 * @block: block we want to work with
2071 static int postcopy_chunk_hostpages(MigrationState
*ms
, RAMBlock
*block
)
2073 postcopy_discard_send_init(ms
, block
->idstr
);
2076 * Ensure that all partially dirty host pages are made fully dirty.
2078 postcopy_chunk_hostpages_pass(ms
, block
);
2080 postcopy_discard_send_finish(ms
);
2085 * ram_postcopy_send_discard_bitmap: transmit the discard bitmap
2087 * Returns zero on success
2089 * Transmit the set of pages to be discarded after precopy to the target
2090 * these are pages that:
2091 * a) Have been previously transmitted but are now dirty again
2092 * b) Pages that have never been transmitted, this ensures that
2093 * any pages on the destination that have been mapped by background
2094 * tasks get discarded (transparent huge pages is the specific concern)
2095 * Hopefully this is pretty sparse
2097 * @ms: current migration state
2099 int ram_postcopy_send_discard_bitmap(MigrationState
*ms
)
2101 RAMState
*rs
= ram_state
;
2105 RCU_READ_LOCK_GUARD();
2107 /* This should be our last sync, the src is now paused */
2108 migration_bitmap_sync(rs
);
2110 /* Easiest way to make sure we don't resume in the middle of a host-page */
2111 rs
->last_seen_block
= NULL
;
2112 rs
->last_sent_block
= NULL
;
2115 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2116 /* Deal with TPS != HPS and huge pages */
2117 ret
= postcopy_chunk_hostpages(ms
, block
);
2122 #ifdef DEBUG_POSTCOPY
2123 ram_debug_dump_bitmap(block
->bmap
, true,
2124 block
->used_length
>> TARGET_PAGE_BITS
);
2127 trace_ram_postcopy_send_discard_bitmap();
2129 ret
= postcopy_each_ram_send_discard(ms
);
2135 * ram_discard_range: discard dirtied pages at the beginning of postcopy
2137 * Returns zero on success
2139 * @rbname: name of the RAMBlock of the request. NULL means the
2140 * same that last one.
2141 * @start: RAMBlock starting page
2142 * @length: RAMBlock size
2144 int ram_discard_range(const char *rbname
, uint64_t start
, size_t length
)
2146 trace_ram_discard_range(rbname
, start
, length
);
2148 RCU_READ_LOCK_GUARD();
2149 RAMBlock
*rb
= qemu_ram_block_by_name(rbname
);
2152 error_report("ram_discard_range: Failed to find block '%s'", rbname
);
2157 * On source VM, we don't need to update the received bitmap since
2158 * we don't even have one.
2160 if (rb
->receivedmap
) {
2161 bitmap_clear(rb
->receivedmap
, start
>> qemu_target_page_bits(),
2162 length
>> qemu_target_page_bits());
2165 return ram_block_discard_range(rb
, start
, length
);
2169 * For every allocation, we will try not to crash the VM if the
2170 * allocation failed.
2172 static int xbzrle_init(void)
2174 Error
*local_err
= NULL
;
2176 if (!migrate_use_xbzrle()) {
2180 XBZRLE_cache_lock();
2182 XBZRLE
.zero_target_page
= g_try_malloc0(TARGET_PAGE_SIZE
);
2183 if (!XBZRLE
.zero_target_page
) {
2184 error_report("%s: Error allocating zero page", __func__
);
2188 XBZRLE
.cache
= cache_init(migrate_xbzrle_cache_size(),
2189 TARGET_PAGE_SIZE
, &local_err
);
2190 if (!XBZRLE
.cache
) {
2191 error_report_err(local_err
);
2192 goto free_zero_page
;
2195 XBZRLE
.encoded_buf
= g_try_malloc0(TARGET_PAGE_SIZE
);
2196 if (!XBZRLE
.encoded_buf
) {
2197 error_report("%s: Error allocating encoded_buf", __func__
);
2201 XBZRLE
.current_buf
= g_try_malloc(TARGET_PAGE_SIZE
);
2202 if (!XBZRLE
.current_buf
) {
2203 error_report("%s: Error allocating current_buf", __func__
);
2204 goto free_encoded_buf
;
2207 /* We are all good */
2208 XBZRLE_cache_unlock();
2212 g_free(XBZRLE
.encoded_buf
);
2213 XBZRLE
.encoded_buf
= NULL
;
2215 cache_fini(XBZRLE
.cache
);
2216 XBZRLE
.cache
= NULL
;
2218 g_free(XBZRLE
.zero_target_page
);
2219 XBZRLE
.zero_target_page
= NULL
;
2221 XBZRLE_cache_unlock();
2225 static int ram_state_init(RAMState
**rsp
)
2227 *rsp
= g_try_new0(RAMState
, 1);
2230 error_report("%s: Init ramstate fail", __func__
);
2234 qemu_mutex_init(&(*rsp
)->bitmap_mutex
);
2235 qemu_mutex_init(&(*rsp
)->src_page_req_mutex
);
2236 QSIMPLEQ_INIT(&(*rsp
)->src_page_requests
);
2239 * Count the total number of pages used by ram blocks not including any
2240 * gaps due to alignment or unplugs.
2241 * This must match with the initial values of dirty bitmap.
2243 (*rsp
)->migration_dirty_pages
= ram_bytes_total() >> TARGET_PAGE_BITS
;
2244 ram_state_reset(*rsp
);
2249 static void ram_list_init_bitmaps(void)
2251 MigrationState
*ms
= migrate_get_current();
2253 unsigned long pages
;
2256 /* Skip setting bitmap if there is no RAM */
2257 if (ram_bytes_total()) {
2258 shift
= ms
->clear_bitmap_shift
;
2259 if (shift
> CLEAR_BITMAP_SHIFT_MAX
) {
2260 error_report("clear_bitmap_shift (%u) too big, using "
2261 "max value (%u)", shift
, CLEAR_BITMAP_SHIFT_MAX
);
2262 shift
= CLEAR_BITMAP_SHIFT_MAX
;
2263 } else if (shift
< CLEAR_BITMAP_SHIFT_MIN
) {
2264 error_report("clear_bitmap_shift (%u) too small, using "
2265 "min value (%u)", shift
, CLEAR_BITMAP_SHIFT_MIN
);
2266 shift
= CLEAR_BITMAP_SHIFT_MIN
;
2269 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2270 pages
= block
->max_length
>> TARGET_PAGE_BITS
;
2272 * The initial dirty bitmap for migration must be set with all
2273 * ones to make sure we'll migrate every guest RAM page to
2275 * Here we set RAMBlock.bmap all to 1 because when rebegin a
2276 * new migration after a failed migration, ram_list.
2277 * dirty_memory[DIRTY_MEMORY_MIGRATION] don't include the whole
2280 block
->bmap
= bitmap_new(pages
);
2281 bitmap_set(block
->bmap
, 0, pages
);
2282 block
->clear_bmap_shift
= shift
;
2283 block
->clear_bmap
= bitmap_new(clear_bmap_size(pages
, shift
));
2288 static void ram_init_bitmaps(RAMState
*rs
)
2290 /* For memory_global_dirty_log_start below. */
2291 qemu_mutex_lock_iothread();
2292 qemu_mutex_lock_ramlist();
2294 WITH_RCU_READ_LOCK_GUARD() {
2295 ram_list_init_bitmaps();
2296 memory_global_dirty_log_start();
2297 migration_bitmap_sync_precopy(rs
);
2299 qemu_mutex_unlock_ramlist();
2300 qemu_mutex_unlock_iothread();
2303 static int ram_init_all(RAMState
**rsp
)
2305 if (ram_state_init(rsp
)) {
2309 if (xbzrle_init()) {
2310 ram_state_cleanup(rsp
);
2314 ram_init_bitmaps(*rsp
);
2319 static void ram_state_resume_prepare(RAMState
*rs
, QEMUFile
*out
)
2325 * Postcopy is not using xbzrle/compression, so no need for that.
2326 * Also, since source are already halted, we don't need to care
2327 * about dirty page logging as well.
2330 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2331 pages
+= bitmap_count_one(block
->bmap
,
2332 block
->used_length
>> TARGET_PAGE_BITS
);
2335 /* This may not be aligned with current bitmaps. Recalculate. */
2336 rs
->migration_dirty_pages
= pages
;
2338 rs
->last_seen_block
= NULL
;
2339 rs
->last_sent_block
= NULL
;
2341 rs
->last_version
= ram_list
.version
;
2343 * Disable the bulk stage, otherwise we'll resend the whole RAM no
2344 * matter what we have sent.
2346 rs
->ram_bulk_stage
= false;
2348 /* Update RAMState cache of output QEMUFile */
2351 trace_ram_state_resume_prepare(pages
);
2355 * This function clears bits of the free pages reported by the caller from the
2356 * migration dirty bitmap. @addr is the host address corresponding to the
2357 * start of the continuous guest free pages, and @len is the total bytes of
2360 void qemu_guest_free_page_hint(void *addr
, size_t len
)
2364 size_t used_len
, start
, npages
;
2365 MigrationState
*s
= migrate_get_current();
2367 /* This function is currently expected to be used during live migration */
2368 if (!migration_is_setup_or_active(s
->state
)) {
2372 for (; len
> 0; len
-= used_len
, addr
+= used_len
) {
2373 block
= qemu_ram_block_from_host(addr
, false, &offset
);
2374 if (unlikely(!block
|| offset
>= block
->used_length
)) {
2376 * The implementation might not support RAMBlock resize during
2377 * live migration, but it could happen in theory with future
2378 * updates. So we add a check here to capture that case.
2380 error_report_once("%s unexpected error", __func__
);
2384 if (len
<= block
->used_length
- offset
) {
2387 used_len
= block
->used_length
- offset
;
2390 start
= offset
>> TARGET_PAGE_BITS
;
2391 npages
= used_len
>> TARGET_PAGE_BITS
;
2393 qemu_mutex_lock(&ram_state
->bitmap_mutex
);
2394 ram_state
->migration_dirty_pages
-=
2395 bitmap_count_one_with_offset(block
->bmap
, start
, npages
);
2396 bitmap_clear(block
->bmap
, start
, npages
);
2397 qemu_mutex_unlock(&ram_state
->bitmap_mutex
);
2402 * Each of ram_save_setup, ram_save_iterate and ram_save_complete has
2403 * long-running RCU critical section. When rcu-reclaims in the code
2404 * start to become numerous it will be necessary to reduce the
2405 * granularity of these critical sections.
2409 * ram_save_setup: Setup RAM for migration
2411 * Returns zero to indicate success and negative for error
2413 * @f: QEMUFile where to send the data
2414 * @opaque: RAMState pointer
2416 static int ram_save_setup(QEMUFile
*f
, void *opaque
)
2418 RAMState
**rsp
= opaque
;
2421 if (compress_threads_save_setup()) {
2425 /* migration has already setup the bitmap, reuse it. */
2426 if (!migration_in_colo_state()) {
2427 if (ram_init_all(rsp
) != 0) {
2428 compress_threads_save_cleanup();
2434 WITH_RCU_READ_LOCK_GUARD() {
2435 qemu_put_be64(f
, ram_bytes_total_common(true) | RAM_SAVE_FLAG_MEM_SIZE
);
2437 RAMBLOCK_FOREACH_MIGRATABLE(block
) {
2438 qemu_put_byte(f
, strlen(block
->idstr
));
2439 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, strlen(block
->idstr
));
2440 qemu_put_be64(f
, block
->used_length
);
2441 if (migrate_postcopy_ram() && block
->page_size
!=
2442 qemu_host_page_size
) {
2443 qemu_put_be64(f
, block
->page_size
);
2445 if (migrate_ignore_shared()) {
2446 qemu_put_be64(f
, block
->mr
->addr
);
2451 ram_control_before_iterate(f
, RAM_CONTROL_SETUP
);
2452 ram_control_after_iterate(f
, RAM_CONTROL_SETUP
);
2454 multifd_send_sync_main(f
);
2455 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
2462 * ram_save_iterate: iterative stage for migration
2464 * Returns zero to indicate success and negative for error
2466 * @f: QEMUFile where to send the data
2467 * @opaque: RAMState pointer
2469 static int ram_save_iterate(QEMUFile
*f
, void *opaque
)
2471 RAMState
**temp
= opaque
;
2472 RAMState
*rs
= *temp
;
2478 if (blk_mig_bulk_active()) {
2479 /* Avoid transferring ram during bulk phase of block migration as
2480 * the bulk phase will usually take a long time and transferring
2481 * ram updates during that time is pointless. */
2485 WITH_RCU_READ_LOCK_GUARD() {
2486 if (ram_list
.version
!= rs
->last_version
) {
2487 ram_state_reset(rs
);
2490 /* Read version before ram_list.blocks */
2493 ram_control_before_iterate(f
, RAM_CONTROL_ROUND
);
2495 t0
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
);
2497 while ((ret
= qemu_file_rate_limit(f
)) == 0 ||
2498 !QSIMPLEQ_EMPTY(&rs
->src_page_requests
)) {
2501 if (qemu_file_get_error(f
)) {
2505 pages
= ram_find_and_save_block(rs
, false);
2506 /* no more pages to sent */
2513 qemu_file_set_error(f
, pages
);
2517 rs
->target_page_count
+= pages
;
2520 * During postcopy, it is necessary to make sure one whole host
2521 * page is sent in one chunk.
2523 if (migrate_postcopy_ram()) {
2524 flush_compressed_data(rs
);
2528 * we want to check in the 1st loop, just in case it was the 1st
2529 * time and we had to sync the dirty bitmap.
2530 * qemu_clock_get_ns() is a bit expensive, so we only check each
2533 if ((i
& 63) == 0) {
2534 uint64_t t1
= (qemu_clock_get_ns(QEMU_CLOCK_REALTIME
) - t0
) /
2536 if (t1
> MAX_WAIT
) {
2537 trace_ram_save_iterate_big_wait(t1
, i
);
2546 * Must occur before EOS (or any QEMUFile operation)
2547 * because of RDMA protocol.
2549 ram_control_after_iterate(f
, RAM_CONTROL_ROUND
);
2553 && migration_is_setup_or_active(migrate_get_current()->state
)) {
2554 multifd_send_sync_main(rs
->f
);
2555 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
2557 ram_counters
.transferred
+= 8;
2559 ret
= qemu_file_get_error(f
);
2569 * ram_save_complete: function called to send the remaining amount of ram
2571 * Returns zero to indicate success or negative on error
2573 * Called with iothread lock
2575 * @f: QEMUFile where to send the data
2576 * @opaque: RAMState pointer
2578 static int ram_save_complete(QEMUFile
*f
, void *opaque
)
2580 RAMState
**temp
= opaque
;
2581 RAMState
*rs
= *temp
;
2584 WITH_RCU_READ_LOCK_GUARD() {
2585 if (!migration_in_postcopy()) {
2586 migration_bitmap_sync_precopy(rs
);
2589 ram_control_before_iterate(f
, RAM_CONTROL_FINISH
);
2591 /* try transferring iterative blocks of memory */
2593 /* flush all remaining blocks regardless of rate limiting */
2597 pages
= ram_find_and_save_block(rs
, !migration_in_colo_state());
2598 /* no more blocks to sent */
2608 flush_compressed_data(rs
);
2609 ram_control_after_iterate(f
, RAM_CONTROL_FINISH
);
2613 multifd_send_sync_main(rs
->f
);
2614 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
2621 static void ram_save_pending(QEMUFile
*f
, void *opaque
, uint64_t max_size
,
2622 uint64_t *res_precopy_only
,
2623 uint64_t *res_compatible
,
2624 uint64_t *res_postcopy_only
)
2626 RAMState
**temp
= opaque
;
2627 RAMState
*rs
= *temp
;
2628 uint64_t remaining_size
;
2630 remaining_size
= rs
->migration_dirty_pages
* TARGET_PAGE_SIZE
;
2632 if (!migration_in_postcopy() &&
2633 remaining_size
< max_size
) {
2634 qemu_mutex_lock_iothread();
2635 WITH_RCU_READ_LOCK_GUARD() {
2636 migration_bitmap_sync_precopy(rs
);
2638 qemu_mutex_unlock_iothread();
2639 remaining_size
= rs
->migration_dirty_pages
* TARGET_PAGE_SIZE
;
2642 if (migrate_postcopy_ram()) {
2643 /* We can do postcopy, and all the data is postcopiable */
2644 *res_compatible
+= remaining_size
;
2646 *res_precopy_only
+= remaining_size
;
2650 static int load_xbzrle(QEMUFile
*f
, ram_addr_t addr
, void *host
)
2652 unsigned int xh_len
;
2654 uint8_t *loaded_data
;
2656 /* extract RLE header */
2657 xh_flags
= qemu_get_byte(f
);
2658 xh_len
= qemu_get_be16(f
);
2660 if (xh_flags
!= ENCODING_FLAG_XBZRLE
) {
2661 error_report("Failed to load XBZRLE page - wrong compression!");
2665 if (xh_len
> TARGET_PAGE_SIZE
) {
2666 error_report("Failed to load XBZRLE page - len overflow!");
2669 loaded_data
= XBZRLE
.decoded_buf
;
2670 /* load data and decode */
2671 /* it can change loaded_data to point to an internal buffer */
2672 qemu_get_buffer_in_place(f
, &loaded_data
, xh_len
);
2675 if (xbzrle_decode_buffer(loaded_data
, xh_len
, host
,
2676 TARGET_PAGE_SIZE
) == -1) {
2677 error_report("Failed to load XBZRLE page - decode error!");
2685 * ram_block_from_stream: read a RAMBlock id from the migration stream
2687 * Must be called from within a rcu critical section.
2689 * Returns a pointer from within the RCU-protected ram_list.
2691 * @f: QEMUFile where to read the data from
2692 * @flags: Page flags (mostly to see if it's a continuation of previous block)
2694 static inline RAMBlock
*ram_block_from_stream(QEMUFile
*f
, int flags
)
2696 static RAMBlock
*block
= NULL
;
2700 if (flags
& RAM_SAVE_FLAG_CONTINUE
) {
2702 error_report("Ack, bad migration stream!");
2708 len
= qemu_get_byte(f
);
2709 qemu_get_buffer(f
, (uint8_t *)id
, len
);
2712 block
= qemu_ram_block_by_name(id
);
2714 error_report("Can't find block %s", id
);
2718 if (ramblock_is_ignored(block
)) {
2719 error_report("block %s should not be migrated !", id
);
2726 static inline void *host_from_ram_block_offset(RAMBlock
*block
,
2729 if (!offset_in_ramblock(block
, offset
)) {
2733 return block
->host
+ offset
;
2736 static inline void *colo_cache_from_block_offset(RAMBlock
*block
,
2739 if (!offset_in_ramblock(block
, offset
)) {
2742 if (!block
->colo_cache
) {
2743 error_report("%s: colo_cache is NULL in block :%s",
2744 __func__
, block
->idstr
);
2749 * During colo checkpoint, we need bitmap of these migrated pages.
2750 * It help us to decide which pages in ram cache should be flushed
2751 * into VM's RAM later.
2753 if (!test_and_set_bit(offset
>> TARGET_PAGE_BITS
, block
->bmap
)) {
2754 ram_state
->migration_dirty_pages
++;
2756 return block
->colo_cache
+ offset
;
2760 * ram_handle_compressed: handle the zero page case
2762 * If a page (or a whole RDMA chunk) has been
2763 * determined to be zero, then zap it.
2765 * @host: host address for the zero page
2766 * @ch: what the page is filled from. We only support zero
2767 * @size: size of the zero page
2769 void ram_handle_compressed(void *host
, uint8_t ch
, uint64_t size
)
2771 if (ch
!= 0 || !is_zero_range(host
, size
)) {
2772 memset(host
, ch
, size
);
2776 /* return the size after decompression, or negative value on error */
2778 qemu_uncompress_data(z_stream
*stream
, uint8_t *dest
, size_t dest_len
,
2779 const uint8_t *source
, size_t source_len
)
2783 err
= inflateReset(stream
);
2788 stream
->avail_in
= source_len
;
2789 stream
->next_in
= (uint8_t *)source
;
2790 stream
->avail_out
= dest_len
;
2791 stream
->next_out
= dest
;
2793 err
= inflate(stream
, Z_NO_FLUSH
);
2794 if (err
!= Z_STREAM_END
) {
2798 return stream
->total_out
;
2801 static void *do_data_decompress(void *opaque
)
2803 DecompressParam
*param
= opaque
;
2804 unsigned long pagesize
;
2808 qemu_mutex_lock(¶m
->mutex
);
2809 while (!param
->quit
) {
2814 qemu_mutex_unlock(¶m
->mutex
);
2816 pagesize
= TARGET_PAGE_SIZE
;
2818 ret
= qemu_uncompress_data(¶m
->stream
, des
, pagesize
,
2819 param
->compbuf
, len
);
2820 if (ret
< 0 && migrate_get_current()->decompress_error_check
) {
2821 error_report("decompress data failed");
2822 qemu_file_set_error(decomp_file
, ret
);
2825 qemu_mutex_lock(&decomp_done_lock
);
2827 qemu_cond_signal(&decomp_done_cond
);
2828 qemu_mutex_unlock(&decomp_done_lock
);
2830 qemu_mutex_lock(¶m
->mutex
);
2832 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
2835 qemu_mutex_unlock(¶m
->mutex
);
2840 static int wait_for_decompress_done(void)
2842 int idx
, thread_count
;
2844 if (!migrate_use_compression()) {
2848 thread_count
= migrate_decompress_threads();
2849 qemu_mutex_lock(&decomp_done_lock
);
2850 for (idx
= 0; idx
< thread_count
; idx
++) {
2851 while (!decomp_param
[idx
].done
) {
2852 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
2855 qemu_mutex_unlock(&decomp_done_lock
);
2856 return qemu_file_get_error(decomp_file
);
2859 static void compress_threads_load_cleanup(void)
2861 int i
, thread_count
;
2863 if (!migrate_use_compression()) {
2866 thread_count
= migrate_decompress_threads();
2867 for (i
= 0; i
< thread_count
; i
++) {
2869 * we use it as a indicator which shows if the thread is
2870 * properly init'd or not
2872 if (!decomp_param
[i
].compbuf
) {
2876 qemu_mutex_lock(&decomp_param
[i
].mutex
);
2877 decomp_param
[i
].quit
= true;
2878 qemu_cond_signal(&decomp_param
[i
].cond
);
2879 qemu_mutex_unlock(&decomp_param
[i
].mutex
);
2881 for (i
= 0; i
< thread_count
; i
++) {
2882 if (!decomp_param
[i
].compbuf
) {
2886 qemu_thread_join(decompress_threads
+ i
);
2887 qemu_mutex_destroy(&decomp_param
[i
].mutex
);
2888 qemu_cond_destroy(&decomp_param
[i
].cond
);
2889 inflateEnd(&decomp_param
[i
].stream
);
2890 g_free(decomp_param
[i
].compbuf
);
2891 decomp_param
[i
].compbuf
= NULL
;
2893 g_free(decompress_threads
);
2894 g_free(decomp_param
);
2895 decompress_threads
= NULL
;
2896 decomp_param
= NULL
;
2900 static int compress_threads_load_setup(QEMUFile
*f
)
2902 int i
, thread_count
;
2904 if (!migrate_use_compression()) {
2908 thread_count
= migrate_decompress_threads();
2909 decompress_threads
= g_new0(QemuThread
, thread_count
);
2910 decomp_param
= g_new0(DecompressParam
, thread_count
);
2911 qemu_mutex_init(&decomp_done_lock
);
2912 qemu_cond_init(&decomp_done_cond
);
2914 for (i
= 0; i
< thread_count
; i
++) {
2915 if (inflateInit(&decomp_param
[i
].stream
) != Z_OK
) {
2919 decomp_param
[i
].compbuf
= g_malloc0(compressBound(TARGET_PAGE_SIZE
));
2920 qemu_mutex_init(&decomp_param
[i
].mutex
);
2921 qemu_cond_init(&decomp_param
[i
].cond
);
2922 decomp_param
[i
].done
= true;
2923 decomp_param
[i
].quit
= false;
2924 qemu_thread_create(decompress_threads
+ i
, "decompress",
2925 do_data_decompress
, decomp_param
+ i
,
2926 QEMU_THREAD_JOINABLE
);
2930 compress_threads_load_cleanup();
2934 static void decompress_data_with_multi_threads(QEMUFile
*f
,
2935 void *host
, int len
)
2937 int idx
, thread_count
;
2939 thread_count
= migrate_decompress_threads();
2940 qemu_mutex_lock(&decomp_done_lock
);
2942 for (idx
= 0; idx
< thread_count
; idx
++) {
2943 if (decomp_param
[idx
].done
) {
2944 decomp_param
[idx
].done
= false;
2945 qemu_mutex_lock(&decomp_param
[idx
].mutex
);
2946 qemu_get_buffer(f
, decomp_param
[idx
].compbuf
, len
);
2947 decomp_param
[idx
].des
= host
;
2948 decomp_param
[idx
].len
= len
;
2949 qemu_cond_signal(&decomp_param
[idx
].cond
);
2950 qemu_mutex_unlock(&decomp_param
[idx
].mutex
);
2954 if (idx
< thread_count
) {
2957 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
2960 qemu_mutex_unlock(&decomp_done_lock
);
2964 * colo cache: this is for secondary VM, we cache the whole
2965 * memory of the secondary VM, it is need to hold the global lock
2966 * to call this helper.
2968 int colo_init_ram_cache(void)
2972 WITH_RCU_READ_LOCK_GUARD() {
2973 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2974 block
->colo_cache
= qemu_anon_ram_alloc(block
->used_length
,
2977 if (!block
->colo_cache
) {
2978 error_report("%s: Can't alloc memory for COLO cache of block %s,"
2979 "size 0x" RAM_ADDR_FMT
, __func__
, block
->idstr
,
2980 block
->used_length
);
2981 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2982 if (block
->colo_cache
) {
2983 qemu_anon_ram_free(block
->colo_cache
, block
->used_length
);
2984 block
->colo_cache
= NULL
;
2989 memcpy(block
->colo_cache
, block
->host
, block
->used_length
);
2994 * Record the dirty pages that sent by PVM, we use this dirty bitmap together
2995 * with to decide which page in cache should be flushed into SVM's RAM. Here
2996 * we use the same name 'ram_bitmap' as for migration.
2998 if (ram_bytes_total()) {
3001 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3002 unsigned long pages
= block
->max_length
>> TARGET_PAGE_BITS
;
3004 block
->bmap
= bitmap_new(pages
);
3005 bitmap_set(block
->bmap
, 0, pages
);
3008 ram_state
= g_new0(RAMState
, 1);
3009 ram_state
->migration_dirty_pages
= 0;
3010 qemu_mutex_init(&ram_state
->bitmap_mutex
);
3011 memory_global_dirty_log_start();
3016 /* It is need to hold the global lock to call this helper */
3017 void colo_release_ram_cache(void)
3021 memory_global_dirty_log_stop();
3022 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3023 g_free(block
->bmap
);
3027 WITH_RCU_READ_LOCK_GUARD() {
3028 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3029 if (block
->colo_cache
) {
3030 qemu_anon_ram_free(block
->colo_cache
, block
->used_length
);
3031 block
->colo_cache
= NULL
;
3035 qemu_mutex_destroy(&ram_state
->bitmap_mutex
);
3041 * ram_load_setup: Setup RAM for migration incoming side
3043 * Returns zero to indicate success and negative for error
3045 * @f: QEMUFile where to receive the data
3046 * @opaque: RAMState pointer
3048 static int ram_load_setup(QEMUFile
*f
, void *opaque
)
3050 if (compress_threads_load_setup(f
)) {
3054 xbzrle_load_setup();
3055 ramblock_recv_map_init();
3060 static int ram_load_cleanup(void *opaque
)
3064 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
3065 qemu_ram_block_writeback(rb
);
3068 xbzrle_load_cleanup();
3069 compress_threads_load_cleanup();
3071 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
3072 g_free(rb
->receivedmap
);
3073 rb
->receivedmap
= NULL
;
3080 * ram_postcopy_incoming_init: allocate postcopy data structures
3082 * Returns 0 for success and negative if there was one error
3084 * @mis: current migration incoming state
3086 * Allocate data structures etc needed by incoming migration with
3087 * postcopy-ram. postcopy-ram's similarly names
3088 * postcopy_ram_incoming_init does the work.
3090 int ram_postcopy_incoming_init(MigrationIncomingState
*mis
)
3092 return postcopy_ram_incoming_init(mis
);
3096 * ram_load_postcopy: load a page in postcopy case
3098 * Returns 0 for success or -errno in case of error
3100 * Called in postcopy mode by ram_load().
3101 * rcu_read_lock is taken prior to this being called.
3103 * @f: QEMUFile where to send the data
3105 static int ram_load_postcopy(QEMUFile
*f
)
3107 int flags
= 0, ret
= 0;
3108 bool place_needed
= false;
3109 bool matches_target_page_size
= false;
3110 MigrationIncomingState
*mis
= migration_incoming_get_current();
3111 /* Temporary page that is later 'placed' */
3112 void *postcopy_host_page
= mis
->postcopy_tmp_page
;
3113 void *this_host
= NULL
;
3114 bool all_zero
= false;
3115 int target_pages
= 0;
3117 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
3120 void *page_buffer
= NULL
;
3121 void *place_source
= NULL
;
3122 RAMBlock
*block
= NULL
;
3126 addr
= qemu_get_be64(f
);
3129 * If qemu file error, we should stop here, and then "addr"
3132 ret
= qemu_file_get_error(f
);
3137 flags
= addr
& ~TARGET_PAGE_MASK
;
3138 addr
&= TARGET_PAGE_MASK
;
3140 trace_ram_load_postcopy_loop((uint64_t)addr
, flags
);
3141 place_needed
= false;
3142 if (flags
& (RAM_SAVE_FLAG_ZERO
| RAM_SAVE_FLAG_PAGE
|
3143 RAM_SAVE_FLAG_COMPRESS_PAGE
)) {
3144 block
= ram_block_from_stream(f
, flags
);
3146 host
= host_from_ram_block_offset(block
, addr
);
3148 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
3153 matches_target_page_size
= block
->page_size
== TARGET_PAGE_SIZE
;
3155 * Postcopy requires that we place whole host pages atomically;
3156 * these may be huge pages for RAMBlocks that are backed by
3158 * To make it atomic, the data is read into a temporary page
3159 * that's moved into place later.
3160 * The migration protocol uses, possibly smaller, target-pages
3161 * however the source ensures it always sends all the components
3162 * of a host page in one chunk.
3164 page_buffer
= postcopy_host_page
+
3165 ((uintptr_t)host
& (block
->page_size
- 1));
3166 /* If all TP are zero then we can optimise the place */
3167 if (target_pages
== 1) {
3169 this_host
= (void *)QEMU_ALIGN_DOWN((uintptr_t)host
,
3172 /* not the 1st TP within the HP */
3173 if (QEMU_ALIGN_DOWN((uintptr_t)host
, block
->page_size
) !=
3174 (uintptr_t)this_host
) {
3175 error_report("Non-same host page %p/%p",
3183 * If it's the last part of a host page then we place the host
3186 if (target_pages
== (block
->page_size
/ TARGET_PAGE_SIZE
)) {
3187 place_needed
= true;
3190 place_source
= postcopy_host_page
;
3193 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
3194 case RAM_SAVE_FLAG_ZERO
:
3195 ch
= qemu_get_byte(f
);
3197 * Can skip to set page_buffer when
3198 * this is a zero page and (block->page_size == TARGET_PAGE_SIZE).
3200 if (ch
|| !matches_target_page_size
) {
3201 memset(page_buffer
, ch
, TARGET_PAGE_SIZE
);
3208 case RAM_SAVE_FLAG_PAGE
:
3210 if (!matches_target_page_size
) {
3211 /* For huge pages, we always use temporary buffer */
3212 qemu_get_buffer(f
, page_buffer
, TARGET_PAGE_SIZE
);
3215 * For small pages that matches target page size, we
3216 * avoid the qemu_file copy. Instead we directly use
3217 * the buffer of QEMUFile to place the page. Note: we
3218 * cannot do any QEMUFile operation before using that
3219 * buffer to make sure the buffer is valid when
3222 qemu_get_buffer_in_place(f
, (uint8_t **)&place_source
,
3226 case RAM_SAVE_FLAG_COMPRESS_PAGE
:
3228 len
= qemu_get_be32(f
);
3229 if (len
< 0 || len
> compressBound(TARGET_PAGE_SIZE
)) {
3230 error_report("Invalid compressed data length: %d", len
);
3234 decompress_data_with_multi_threads(f
, page_buffer
, len
);
3237 case RAM_SAVE_FLAG_EOS
:
3239 multifd_recv_sync_main();
3242 error_report("Unknown combination of migration flags: %#x"
3243 " (postcopy mode)", flags
);
3248 /* Got the whole host page, wait for decompress before placing. */
3250 ret
|= wait_for_decompress_done();
3253 /* Detect for any possible file errors */
3254 if (!ret
&& qemu_file_get_error(f
)) {
3255 ret
= qemu_file_get_error(f
);
3258 if (!ret
&& place_needed
) {
3259 /* This gets called at the last target page in the host page */
3260 void *place_dest
= (void *)QEMU_ALIGN_DOWN((uintptr_t)host
,
3264 ret
= postcopy_place_page_zero(mis
, place_dest
,
3267 ret
= postcopy_place_page(mis
, place_dest
,
3268 place_source
, block
);
3276 static bool postcopy_is_advised(void)
3278 PostcopyState ps
= postcopy_state_get();
3279 return ps
>= POSTCOPY_INCOMING_ADVISE
&& ps
< POSTCOPY_INCOMING_END
;
3282 static bool postcopy_is_running(void)
3284 PostcopyState ps
= postcopy_state_get();
3285 return ps
>= POSTCOPY_INCOMING_LISTENING
&& ps
< POSTCOPY_INCOMING_END
;
3289 * Flush content of RAM cache into SVM's memory.
3290 * Only flush the pages that be dirtied by PVM or SVM or both.
3292 static void colo_flush_ram_cache(void)
3294 RAMBlock
*block
= NULL
;
3297 unsigned long offset
= 0;
3299 memory_global_dirty_log_sync();
3300 WITH_RCU_READ_LOCK_GUARD() {
3301 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3302 ramblock_sync_dirty_bitmap(ram_state
, block
);
3306 trace_colo_flush_ram_cache_begin(ram_state
->migration_dirty_pages
);
3307 WITH_RCU_READ_LOCK_GUARD() {
3308 block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
3311 offset
= migration_bitmap_find_dirty(ram_state
, block
, offset
);
3313 if (((ram_addr_t
)offset
) << TARGET_PAGE_BITS
3314 >= block
->used_length
) {
3316 block
= QLIST_NEXT_RCU(block
, next
);
3318 migration_bitmap_clear_dirty(ram_state
, block
, offset
);
3319 dst_host
= block
->host
3320 + (((ram_addr_t
)offset
) << TARGET_PAGE_BITS
);
3321 src_host
= block
->colo_cache
3322 + (((ram_addr_t
)offset
) << TARGET_PAGE_BITS
);
3323 memcpy(dst_host
, src_host
, TARGET_PAGE_SIZE
);
3327 trace_colo_flush_ram_cache_end();
3331 * ram_load_precopy: load pages in precopy case
3333 * Returns 0 for success or -errno in case of error
3335 * Called in precopy mode by ram_load().
3336 * rcu_read_lock is taken prior to this being called.
3338 * @f: QEMUFile where to send the data
3340 static int ram_load_precopy(QEMUFile
*f
)
3342 int flags
= 0, ret
= 0, invalid_flags
= 0, len
= 0, i
= 0;
3343 /* ADVISE is earlier, it shows the source has the postcopy capability on */
3344 bool postcopy_advised
= postcopy_is_advised();
3345 if (!migrate_use_compression()) {
3346 invalid_flags
|= RAM_SAVE_FLAG_COMPRESS_PAGE
;
3349 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
3350 ram_addr_t addr
, total_ram_bytes
;
3355 * Yield periodically to let main loop run, but an iteration of
3356 * the main loop is expensive, so do it each some iterations
3358 if ((i
& 32767) == 0 && qemu_in_coroutine()) {
3359 aio_co_schedule(qemu_get_current_aio_context(),
3360 qemu_coroutine_self());
3361 qemu_coroutine_yield();
3365 addr
= qemu_get_be64(f
);
3366 flags
= addr
& ~TARGET_PAGE_MASK
;
3367 addr
&= TARGET_PAGE_MASK
;
3369 if (flags
& invalid_flags
) {
3370 if (flags
& invalid_flags
& RAM_SAVE_FLAG_COMPRESS_PAGE
) {
3371 error_report("Received an unexpected compressed page");
3378 if (flags
& (RAM_SAVE_FLAG_ZERO
| RAM_SAVE_FLAG_PAGE
|
3379 RAM_SAVE_FLAG_COMPRESS_PAGE
| RAM_SAVE_FLAG_XBZRLE
)) {
3380 RAMBlock
*block
= ram_block_from_stream(f
, flags
);
3383 * After going into COLO, we should load the Page into colo_cache.
3385 if (migration_incoming_in_colo_state()) {
3386 host
= colo_cache_from_block_offset(block
, addr
);
3388 host
= host_from_ram_block_offset(block
, addr
);
3391 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
3396 if (!migration_incoming_in_colo_state()) {
3397 ramblock_recv_bitmap_set(block
, host
);
3400 trace_ram_load_loop(block
->idstr
, (uint64_t)addr
, flags
, host
);
3403 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
3404 case RAM_SAVE_FLAG_MEM_SIZE
:
3405 /* Synchronize RAM block list */
3406 total_ram_bytes
= addr
;
3407 while (!ret
&& total_ram_bytes
) {
3412 len
= qemu_get_byte(f
);
3413 qemu_get_buffer(f
, (uint8_t *)id
, len
);
3415 length
= qemu_get_be64(f
);
3417 block
= qemu_ram_block_by_name(id
);
3418 if (block
&& !qemu_ram_is_migratable(block
)) {
3419 error_report("block %s should not be migrated !", id
);
3422 if (length
!= block
->used_length
) {
3423 Error
*local_err
= NULL
;
3425 ret
= qemu_ram_resize(block
, length
,
3428 error_report_err(local_err
);
3431 /* For postcopy we need to check hugepage sizes match */
3432 if (postcopy_advised
&&
3433 block
->page_size
!= qemu_host_page_size
) {
3434 uint64_t remote_page_size
= qemu_get_be64(f
);
3435 if (remote_page_size
!= block
->page_size
) {
3436 error_report("Mismatched RAM page size %s "
3437 "(local) %zd != %" PRId64
,
3438 id
, block
->page_size
,
3443 if (migrate_ignore_shared()) {
3444 hwaddr addr
= qemu_get_be64(f
);
3445 if (ramblock_is_ignored(block
) &&
3446 block
->mr
->addr
!= addr
) {
3447 error_report("Mismatched GPAs for block %s "
3448 "%" PRId64
"!= %" PRId64
,
3450 (uint64_t)block
->mr
->addr
);
3454 ram_control_load_hook(f
, RAM_CONTROL_BLOCK_REG
,
3457 error_report("Unknown ramblock \"%s\", cannot "
3458 "accept migration", id
);
3462 total_ram_bytes
-= length
;
3466 case RAM_SAVE_FLAG_ZERO
:
3467 ch
= qemu_get_byte(f
);
3468 ram_handle_compressed(host
, ch
, TARGET_PAGE_SIZE
);
3471 case RAM_SAVE_FLAG_PAGE
:
3472 qemu_get_buffer(f
, host
, TARGET_PAGE_SIZE
);
3475 case RAM_SAVE_FLAG_COMPRESS_PAGE
:
3476 len
= qemu_get_be32(f
);
3477 if (len
< 0 || len
> compressBound(TARGET_PAGE_SIZE
)) {
3478 error_report("Invalid compressed data length: %d", len
);
3482 decompress_data_with_multi_threads(f
, host
, len
);
3485 case RAM_SAVE_FLAG_XBZRLE
:
3486 if (load_xbzrle(f
, addr
, host
) < 0) {
3487 error_report("Failed to decompress XBZRLE page at "
3488 RAM_ADDR_FMT
, addr
);
3493 case RAM_SAVE_FLAG_EOS
:
3495 multifd_recv_sync_main();
3498 if (flags
& RAM_SAVE_FLAG_HOOK
) {
3499 ram_control_load_hook(f
, RAM_CONTROL_HOOK
, NULL
);
3501 error_report("Unknown combination of migration flags: %#x",
3507 ret
= qemu_file_get_error(f
);
3511 ret
|= wait_for_decompress_done();
3515 static int ram_load(QEMUFile
*f
, void *opaque
, int version_id
)
3518 static uint64_t seq_iter
;
3520 * If system is running in postcopy mode, page inserts to host memory must
3523 bool postcopy_running
= postcopy_is_running();
3527 if (version_id
!= 4) {
3532 * This RCU critical section can be very long running.
3533 * When RCU reclaims in the code start to become numerous,
3534 * it will be necessary to reduce the granularity of this
3537 WITH_RCU_READ_LOCK_GUARD() {
3538 if (postcopy_running
) {
3539 ret
= ram_load_postcopy(f
);
3541 ret
= ram_load_precopy(f
);
3544 trace_ram_load_complete(ret
, seq_iter
);
3546 if (!ret
&& migration_incoming_in_colo_state()) {
3547 colo_flush_ram_cache();
3552 static bool ram_has_postcopy(void *opaque
)
3555 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
3556 if (ramblock_is_pmem(rb
)) {
3557 info_report("Block: %s, host: %p is a nvdimm memory, postcopy"
3558 "is not supported now!", rb
->idstr
, rb
->host
);
3563 return migrate_postcopy_ram();
3566 /* Sync all the dirty bitmap with destination VM. */
3567 static int ram_dirty_bitmap_sync_all(MigrationState
*s
, RAMState
*rs
)
3570 QEMUFile
*file
= s
->to_dst_file
;
3571 int ramblock_count
= 0;
3573 trace_ram_dirty_bitmap_sync_start();
3575 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3576 qemu_savevm_send_recv_bitmap(file
, block
->idstr
);
3577 trace_ram_dirty_bitmap_request(block
->idstr
);
3581 trace_ram_dirty_bitmap_sync_wait();
3583 /* Wait until all the ramblocks' dirty bitmap synced */
3584 while (ramblock_count
--) {
3585 qemu_sem_wait(&s
->rp_state
.rp_sem
);
3588 trace_ram_dirty_bitmap_sync_complete();
3593 static void ram_dirty_bitmap_reload_notify(MigrationState
*s
)
3595 qemu_sem_post(&s
->rp_state
.rp_sem
);
3599 * Read the received bitmap, revert it as the initial dirty bitmap.
3600 * This is only used when the postcopy migration is paused but wants
3601 * to resume from a middle point.
3603 int ram_dirty_bitmap_reload(MigrationState
*s
, RAMBlock
*block
)
3606 QEMUFile
*file
= s
->rp_state
.from_dst_file
;
3607 unsigned long *le_bitmap
, nbits
= block
->used_length
>> TARGET_PAGE_BITS
;
3608 uint64_t local_size
= DIV_ROUND_UP(nbits
, 8);
3609 uint64_t size
, end_mark
;
3611 trace_ram_dirty_bitmap_reload_begin(block
->idstr
);
3613 if (s
->state
!= MIGRATION_STATUS_POSTCOPY_RECOVER
) {
3614 error_report("%s: incorrect state %s", __func__
,
3615 MigrationStatus_str(s
->state
));
3620 * Note: see comments in ramblock_recv_bitmap_send() on why we
3621 * need the endianess convertion, and the paddings.
3623 local_size
= ROUND_UP(local_size
, 8);
3626 le_bitmap
= bitmap_new(nbits
+ BITS_PER_LONG
);
3628 size
= qemu_get_be64(file
);
3630 /* The size of the bitmap should match with our ramblock */
3631 if (size
!= local_size
) {
3632 error_report("%s: ramblock '%s' bitmap size mismatch "
3633 "(0x%"PRIx64
" != 0x%"PRIx64
")", __func__
,
3634 block
->idstr
, size
, local_size
);
3639 size
= qemu_get_buffer(file
, (uint8_t *)le_bitmap
, local_size
);
3640 end_mark
= qemu_get_be64(file
);
3642 ret
= qemu_file_get_error(file
);
3643 if (ret
|| size
!= local_size
) {
3644 error_report("%s: read bitmap failed for ramblock '%s': %d"
3645 " (size 0x%"PRIx64
", got: 0x%"PRIx64
")",
3646 __func__
, block
->idstr
, ret
, local_size
, size
);
3651 if (end_mark
!= RAMBLOCK_RECV_BITMAP_ENDING
) {
3652 error_report("%s: ramblock '%s' end mark incorrect: 0x%"PRIu64
,
3653 __func__
, block
->idstr
, end_mark
);
3659 * Endianess convertion. We are during postcopy (though paused).
3660 * The dirty bitmap won't change. We can directly modify it.
3662 bitmap_from_le(block
->bmap
, le_bitmap
, nbits
);
3665 * What we received is "received bitmap". Revert it as the initial
3666 * dirty bitmap for this ramblock.
3668 bitmap_complement(block
->bmap
, block
->bmap
, nbits
);
3670 trace_ram_dirty_bitmap_reload_complete(block
->idstr
);
3673 * We succeeded to sync bitmap for current ramblock. If this is
3674 * the last one to sync, we need to notify the main send thread.
3676 ram_dirty_bitmap_reload_notify(s
);
3684 static int ram_resume_prepare(MigrationState
*s
, void *opaque
)
3686 RAMState
*rs
= *(RAMState
**)opaque
;
3689 ret
= ram_dirty_bitmap_sync_all(s
, rs
);
3694 ram_state_resume_prepare(rs
, s
->to_dst_file
);
3699 static SaveVMHandlers savevm_ram_handlers
= {
3700 .save_setup
= ram_save_setup
,
3701 .save_live_iterate
= ram_save_iterate
,
3702 .save_live_complete_postcopy
= ram_save_complete
,
3703 .save_live_complete_precopy
= ram_save_complete
,
3704 .has_postcopy
= ram_has_postcopy
,
3705 .save_live_pending
= ram_save_pending
,
3706 .load_state
= ram_load
,
3707 .save_cleanup
= ram_save_cleanup
,
3708 .load_setup
= ram_load_setup
,
3709 .load_cleanup
= ram_load_cleanup
,
3710 .resume_prepare
= ram_resume_prepare
,
3713 void ram_mig_init(void)
3715 qemu_mutex_init(&XBZRLE
.lock
);
3716 register_savevm_live("ram", 0, 4, &savevm_ram_handlers
, &ram_state
);