4 * Copyright (c) 2003-2008 Fabrice Bellard
5 * Copyright (c) 2011-2015 Red Hat Inc
8 * Juan Quintela <quintela@redhat.com>
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
28 #include "qemu/osdep.h"
29 #include "qemu-common.h"
32 #include "qapi-event.h"
33 #include "qemu/cutils.h"
34 #include "qemu/bitops.h"
35 #include "qemu/bitmap.h"
36 #include "qemu/timer.h"
37 #include "qemu/main-loop.h"
38 #include "migration/migration.h"
39 #include "migration/postcopy-ram.h"
40 #include "exec/address-spaces.h"
41 #include "migration/page_cache.h"
42 #include "qemu/error-report.h"
44 #include "exec/ram_addr.h"
45 #include "qemu/rcu_queue.h"
46 #include "migration/colo.h"
48 static int dirty_rate_high_cnt
;
50 static uint64_t bitmap_sync_count
;
52 /***********************************************************/
53 /* ram save/restore */
55 #define RAM_SAVE_FLAG_FULL 0x01 /* Obsolete, not used anymore */
56 #define RAM_SAVE_FLAG_COMPRESS 0x02
57 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
58 #define RAM_SAVE_FLAG_PAGE 0x08
59 #define RAM_SAVE_FLAG_EOS 0x10
60 #define RAM_SAVE_FLAG_CONTINUE 0x20
61 #define RAM_SAVE_FLAG_XBZRLE 0x40
62 /* 0x80 is reserved in migration.h start with 0x100 next */
63 #define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
65 static uint8_t *ZERO_TARGET_PAGE
;
67 static inline bool is_zero_range(uint8_t *p
, uint64_t size
)
69 return buffer_is_zero(p
, size
);
72 /* struct contains XBZRLE cache and a static page
73 used by the compression */
75 /* buffer used for XBZRLE encoding */
77 /* buffer for storing page content */
79 /* Cache for XBZRLE, Protected by lock. */
84 /* buffer used for XBZRLE decoding */
85 static uint8_t *xbzrle_decoded_buf
;
87 static void XBZRLE_cache_lock(void)
89 if (migrate_use_xbzrle())
90 qemu_mutex_lock(&XBZRLE
.lock
);
93 static void XBZRLE_cache_unlock(void)
95 if (migrate_use_xbzrle())
96 qemu_mutex_unlock(&XBZRLE
.lock
);
100 * called from qmp_migrate_set_cache_size in main thread, possibly while
101 * a migration is in progress.
102 * A running migration maybe using the cache and might finish during this
103 * call, hence changes to the cache are protected by XBZRLE.lock().
105 int64_t xbzrle_cache_resize(int64_t new_size
)
107 PageCache
*new_cache
;
110 if (new_size
< TARGET_PAGE_SIZE
) {
116 if (XBZRLE
.cache
!= NULL
) {
117 if (pow2floor(new_size
) == migrate_xbzrle_cache_size()) {
120 new_cache
= cache_init(new_size
/ TARGET_PAGE_SIZE
,
123 error_report("Error creating cache");
128 cache_fini(XBZRLE
.cache
);
129 XBZRLE
.cache
= new_cache
;
133 ret
= pow2floor(new_size
);
135 XBZRLE_cache_unlock();
139 /* accounting for migration statistics */
140 typedef struct AccountingInfo
{
142 uint64_t skipped_pages
;
145 uint64_t xbzrle_bytes
;
146 uint64_t xbzrle_pages
;
147 uint64_t xbzrle_cache_miss
;
148 double xbzrle_cache_miss_rate
;
149 uint64_t xbzrle_overflows
;
152 static AccountingInfo acct_info
;
154 static void acct_clear(void)
156 memset(&acct_info
, 0, sizeof(acct_info
));
159 uint64_t dup_mig_bytes_transferred(void)
161 return acct_info
.dup_pages
* TARGET_PAGE_SIZE
;
164 uint64_t dup_mig_pages_transferred(void)
166 return acct_info
.dup_pages
;
169 uint64_t skipped_mig_bytes_transferred(void)
171 return acct_info
.skipped_pages
* TARGET_PAGE_SIZE
;
174 uint64_t skipped_mig_pages_transferred(void)
176 return acct_info
.skipped_pages
;
179 uint64_t norm_mig_bytes_transferred(void)
181 return acct_info
.norm_pages
* TARGET_PAGE_SIZE
;
184 uint64_t norm_mig_pages_transferred(void)
186 return acct_info
.norm_pages
;
189 uint64_t xbzrle_mig_bytes_transferred(void)
191 return acct_info
.xbzrle_bytes
;
194 uint64_t xbzrle_mig_pages_transferred(void)
196 return acct_info
.xbzrle_pages
;
199 uint64_t xbzrle_mig_pages_cache_miss(void)
201 return acct_info
.xbzrle_cache_miss
;
204 double xbzrle_mig_cache_miss_rate(void)
206 return acct_info
.xbzrle_cache_miss_rate
;
209 uint64_t xbzrle_mig_pages_overflow(void)
211 return acct_info
.xbzrle_overflows
;
214 /* This is the last block that we have visited serching for dirty pages
216 static RAMBlock
*last_seen_block
;
217 /* This is the last block from where we have sent data */
218 static RAMBlock
*last_sent_block
;
219 static ram_addr_t last_offset
;
220 static QemuMutex migration_bitmap_mutex
;
221 static uint64_t migration_dirty_pages
;
222 static uint32_t last_version
;
223 static bool ram_bulk_stage
;
225 /* used by the search for pages to send */
226 struct PageSearchStatus
{
227 /* Current block being searched */
229 /* Current offset to search from */
231 /* Set once we wrap around */
234 typedef struct PageSearchStatus PageSearchStatus
;
236 static struct BitmapRcu
{
238 /* Main migration bitmap */
240 /* bitmap of pages that haven't been sent even once
241 * only maintained and used in postcopy at the moment
242 * where it's used to send the dirtymap at the start
243 * of the postcopy phase
245 unsigned long *unsentmap
;
246 } *migration_bitmap_rcu
;
248 struct CompressParam
{
257 typedef struct CompressParam CompressParam
;
259 struct DecompressParam
{
268 typedef struct DecompressParam DecompressParam
;
270 static CompressParam
*comp_param
;
271 static QemuThread
*compress_threads
;
272 /* comp_done_cond is used to wake up the migration thread when
273 * one of the compression threads has finished the compression.
274 * comp_done_lock is used to co-work with comp_done_cond.
276 static QemuMutex comp_done_lock
;
277 static QemuCond comp_done_cond
;
278 /* The empty QEMUFileOps will be used by file in CompressParam */
279 static const QEMUFileOps empty_ops
= { };
281 static bool compression_switch
;
282 static DecompressParam
*decomp_param
;
283 static QemuThread
*decompress_threads
;
284 static QemuMutex decomp_done_lock
;
285 static QemuCond decomp_done_cond
;
287 static int do_compress_ram_page(QEMUFile
*f
, RAMBlock
*block
,
290 static void *do_data_compress(void *opaque
)
292 CompressParam
*param
= opaque
;
296 qemu_mutex_lock(¶m
->mutex
);
297 while (!param
->quit
) {
299 block
= param
->block
;
300 offset
= param
->offset
;
302 qemu_mutex_unlock(¶m
->mutex
);
304 do_compress_ram_page(param
->file
, block
, offset
);
306 qemu_mutex_lock(&comp_done_lock
);
308 qemu_cond_signal(&comp_done_cond
);
309 qemu_mutex_unlock(&comp_done_lock
);
311 qemu_mutex_lock(¶m
->mutex
);
313 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
316 qemu_mutex_unlock(¶m
->mutex
);
321 static inline void terminate_compression_threads(void)
323 int idx
, thread_count
;
325 thread_count
= migrate_compress_threads();
326 for (idx
= 0; idx
< thread_count
; idx
++) {
327 qemu_mutex_lock(&comp_param
[idx
].mutex
);
328 comp_param
[idx
].quit
= true;
329 qemu_cond_signal(&comp_param
[idx
].cond
);
330 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
334 void migrate_compress_threads_join(void)
338 if (!migrate_use_compression()) {
341 terminate_compression_threads();
342 thread_count
= migrate_compress_threads();
343 for (i
= 0; i
< thread_count
; i
++) {
344 qemu_thread_join(compress_threads
+ i
);
345 qemu_fclose(comp_param
[i
].file
);
346 qemu_mutex_destroy(&comp_param
[i
].mutex
);
347 qemu_cond_destroy(&comp_param
[i
].cond
);
349 qemu_mutex_destroy(&comp_done_lock
);
350 qemu_cond_destroy(&comp_done_cond
);
351 g_free(compress_threads
);
353 compress_threads
= NULL
;
357 void migrate_compress_threads_create(void)
361 if (!migrate_use_compression()) {
364 compression_switch
= true;
365 thread_count
= migrate_compress_threads();
366 compress_threads
= g_new0(QemuThread
, thread_count
);
367 comp_param
= g_new0(CompressParam
, thread_count
);
368 qemu_cond_init(&comp_done_cond
);
369 qemu_mutex_init(&comp_done_lock
);
370 for (i
= 0; i
< thread_count
; i
++) {
371 /* comp_param[i].file is just used as a dummy buffer to save data,
372 * set its ops to empty.
374 comp_param
[i
].file
= qemu_fopen_ops(NULL
, &empty_ops
);
375 comp_param
[i
].done
= true;
376 comp_param
[i
].quit
= false;
377 qemu_mutex_init(&comp_param
[i
].mutex
);
378 qemu_cond_init(&comp_param
[i
].cond
);
379 qemu_thread_create(compress_threads
+ i
, "compress",
380 do_data_compress
, comp_param
+ i
,
381 QEMU_THREAD_JOINABLE
);
386 * save_page_header: Write page header to wire
388 * If this is the 1st block, it also writes the block identification
390 * Returns: Number of bytes written
392 * @f: QEMUFile where to send the data
393 * @block: block that contains the page we want to send
394 * @offset: offset inside the block for the page
395 * in the lower bits, it contains flags
397 static size_t save_page_header(QEMUFile
*f
, RAMBlock
*block
, ram_addr_t offset
)
401 qemu_put_be64(f
, offset
);
404 if (!(offset
& RAM_SAVE_FLAG_CONTINUE
)) {
405 len
= strlen(block
->idstr
);
406 qemu_put_byte(f
, len
);
407 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, len
);
413 /* Reduce amount of guest cpu execution to hopefully slow down memory writes.
414 * If guest dirty memory rate is reduced below the rate at which we can
415 * transfer pages to the destination then we should be able to complete
416 * migration. Some workloads dirty memory way too fast and will not effectively
417 * converge, even with auto-converge.
419 static void mig_throttle_guest_down(void)
421 MigrationState
*s
= migrate_get_current();
422 uint64_t pct_initial
= s
->parameters
.cpu_throttle_initial
;
423 uint64_t pct_icrement
= s
->parameters
.cpu_throttle_increment
;
425 /* We have not started throttling yet. Let's start it. */
426 if (!cpu_throttle_active()) {
427 cpu_throttle_set(pct_initial
);
429 /* Throttling already on, just increase the rate */
430 cpu_throttle_set(cpu_throttle_get_percentage() + pct_icrement
);
434 /* Update the xbzrle cache to reflect a page that's been sent as all 0.
435 * The important thing is that a stale (not-yet-0'd) page be replaced
437 * As a bonus, if the page wasn't in the cache it gets added so that
438 * when a small write is made into the 0'd page it gets XBZRLE sent
440 static void xbzrle_cache_zero_page(ram_addr_t current_addr
)
442 if (ram_bulk_stage
|| !migrate_use_xbzrle()) {
446 /* We don't care if this fails to allocate a new cache page
447 * as long as it updated an old one */
448 cache_insert(XBZRLE
.cache
, current_addr
, ZERO_TARGET_PAGE
,
452 #define ENCODING_FLAG_XBZRLE 0x1
455 * save_xbzrle_page: compress and send current page
457 * Returns: 1 means that we wrote the page
458 * 0 means that page is identical to the one already sent
459 * -1 means that xbzrle would be longer than normal
461 * @f: QEMUFile where to send the data
464 * @block: block that contains the page we want to send
465 * @offset: offset inside the block for the page
466 * @last_stage: if we are at the completion stage
467 * @bytes_transferred: increase it with the number of transferred bytes
469 static int save_xbzrle_page(QEMUFile
*f
, uint8_t **current_data
,
470 ram_addr_t current_addr
, RAMBlock
*block
,
471 ram_addr_t offset
, bool last_stage
,
472 uint64_t *bytes_transferred
)
474 int encoded_len
= 0, bytes_xbzrle
;
475 uint8_t *prev_cached_page
;
477 if (!cache_is_cached(XBZRLE
.cache
, current_addr
, bitmap_sync_count
)) {
478 acct_info
.xbzrle_cache_miss
++;
480 if (cache_insert(XBZRLE
.cache
, current_addr
, *current_data
,
481 bitmap_sync_count
) == -1) {
484 /* update *current_data when the page has been
485 inserted into cache */
486 *current_data
= get_cached_data(XBZRLE
.cache
, current_addr
);
492 prev_cached_page
= get_cached_data(XBZRLE
.cache
, current_addr
);
494 /* save current buffer into memory */
495 memcpy(XBZRLE
.current_buf
, *current_data
, TARGET_PAGE_SIZE
);
497 /* XBZRLE encoding (if there is no overflow) */
498 encoded_len
= xbzrle_encode_buffer(prev_cached_page
, XBZRLE
.current_buf
,
499 TARGET_PAGE_SIZE
, XBZRLE
.encoded_buf
,
501 if (encoded_len
== 0) {
502 trace_save_xbzrle_page_skipping();
504 } else if (encoded_len
== -1) {
505 trace_save_xbzrle_page_overflow();
506 acct_info
.xbzrle_overflows
++;
507 /* update data in the cache */
509 memcpy(prev_cached_page
, *current_data
, TARGET_PAGE_SIZE
);
510 *current_data
= prev_cached_page
;
515 /* we need to update the data in the cache, in order to get the same data */
517 memcpy(prev_cached_page
, XBZRLE
.current_buf
, TARGET_PAGE_SIZE
);
520 /* Send XBZRLE based compressed page */
521 bytes_xbzrle
= save_page_header(f
, block
, offset
| RAM_SAVE_FLAG_XBZRLE
);
522 qemu_put_byte(f
, ENCODING_FLAG_XBZRLE
);
523 qemu_put_be16(f
, encoded_len
);
524 qemu_put_buffer(f
, XBZRLE
.encoded_buf
, encoded_len
);
525 bytes_xbzrle
+= encoded_len
+ 1 + 2;
526 acct_info
.xbzrle_pages
++;
527 acct_info
.xbzrle_bytes
+= bytes_xbzrle
;
528 *bytes_transferred
+= bytes_xbzrle
;
533 /* Called with rcu_read_lock() to protect migration_bitmap
534 * rb: The RAMBlock to search for dirty pages in
535 * start: Start address (typically so we can continue from previous page)
536 * ram_addr_abs: Pointer into which to store the address of the dirty page
537 * within the global ram_addr space
539 * Returns: byte offset within memory region of the start of a dirty page
542 ram_addr_t
migration_bitmap_find_dirty(RAMBlock
*rb
,
544 ram_addr_t
*ram_addr_abs
)
546 unsigned long base
= rb
->offset
>> TARGET_PAGE_BITS
;
547 unsigned long nr
= base
+ (start
>> TARGET_PAGE_BITS
);
548 uint64_t rb_size
= rb
->used_length
;
549 unsigned long size
= base
+ (rb_size
>> TARGET_PAGE_BITS
);
550 unsigned long *bitmap
;
554 bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
555 if (ram_bulk_stage
&& nr
> base
) {
558 next
= find_next_bit(bitmap
, size
, nr
);
561 *ram_addr_abs
= next
<< TARGET_PAGE_BITS
;
562 return (next
- base
) << TARGET_PAGE_BITS
;
565 static inline bool migration_bitmap_clear_dirty(ram_addr_t addr
)
568 int nr
= addr
>> TARGET_PAGE_BITS
;
569 unsigned long *bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
571 ret
= test_and_clear_bit(nr
, bitmap
);
574 migration_dirty_pages
--;
579 static void migration_bitmap_sync_range(ram_addr_t start
, ram_addr_t length
)
581 unsigned long *bitmap
;
582 bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
583 migration_dirty_pages
+=
584 cpu_physical_memory_sync_dirty_bitmap(bitmap
, start
, length
);
587 /* Fix me: there are too many global variables used in migration process. */
588 static int64_t start_time
;
589 static int64_t bytes_xfer_prev
;
590 static int64_t num_dirty_pages_period
;
591 static uint64_t xbzrle_cache_miss_prev
;
592 static uint64_t iterations_prev
;
594 static void migration_bitmap_sync_init(void)
598 num_dirty_pages_period
= 0;
599 xbzrle_cache_miss_prev
= 0;
603 /* Returns a summary bitmap of the page sizes of all RAMBlocks;
604 * for VMs with just normal pages this is equivalent to the
605 * host page size. If it's got some huge pages then it's the OR
606 * of all the different page sizes.
608 uint64_t ram_pagesize_summary(void)
611 uint64_t summary
= 0;
613 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
614 summary
|= block
->page_size
;
620 static void migration_bitmap_sync(void)
623 uint64_t num_dirty_pages_init
= migration_dirty_pages
;
624 MigrationState
*s
= migrate_get_current();
626 int64_t bytes_xfer_now
;
630 if (!bytes_xfer_prev
) {
631 bytes_xfer_prev
= ram_bytes_transferred();
635 start_time
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
638 trace_migration_bitmap_sync_start();
639 memory_global_dirty_log_sync();
641 qemu_mutex_lock(&migration_bitmap_mutex
);
643 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
644 migration_bitmap_sync_range(block
->offset
, block
->used_length
);
647 qemu_mutex_unlock(&migration_bitmap_mutex
);
649 trace_migration_bitmap_sync_end(migration_dirty_pages
650 - num_dirty_pages_init
);
651 num_dirty_pages_period
+= migration_dirty_pages
- num_dirty_pages_init
;
652 end_time
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
654 /* more than 1 second = 1000 millisecons */
655 if (end_time
> start_time
+ 1000) {
656 if (migrate_auto_converge()) {
657 /* The following detection logic can be refined later. For now:
658 Check to see if the dirtied bytes is 50% more than the approx.
659 amount of bytes that just got transferred since the last time we
660 were in this routine. If that happens twice, start or increase
662 bytes_xfer_now
= ram_bytes_transferred();
664 if (s
->dirty_pages_rate
&&
665 (num_dirty_pages_period
* TARGET_PAGE_SIZE
>
666 (bytes_xfer_now
- bytes_xfer_prev
)/2) &&
667 (dirty_rate_high_cnt
++ >= 2)) {
668 trace_migration_throttle();
669 dirty_rate_high_cnt
= 0;
670 mig_throttle_guest_down();
672 bytes_xfer_prev
= bytes_xfer_now
;
675 if (migrate_use_xbzrle()) {
676 if (iterations_prev
!= acct_info
.iterations
) {
677 acct_info
.xbzrle_cache_miss_rate
=
678 (double)(acct_info
.xbzrle_cache_miss
-
679 xbzrle_cache_miss_prev
) /
680 (acct_info
.iterations
- iterations_prev
);
682 iterations_prev
= acct_info
.iterations
;
683 xbzrle_cache_miss_prev
= acct_info
.xbzrle_cache_miss
;
685 s
->dirty_pages_rate
= num_dirty_pages_period
* 1000
686 / (end_time
- start_time
);
687 s
->dirty_bytes_rate
= s
->dirty_pages_rate
* TARGET_PAGE_SIZE
;
688 start_time
= end_time
;
689 num_dirty_pages_period
= 0;
691 s
->dirty_sync_count
= bitmap_sync_count
;
692 if (migrate_use_events()) {
693 qapi_event_send_migration_pass(bitmap_sync_count
, NULL
);
698 * save_zero_page: Send the zero page to the stream
700 * Returns: Number of pages written.
702 * @f: QEMUFile where to send the data
703 * @block: block that contains the page we want to send
704 * @offset: offset inside the block for the page
705 * @p: pointer to the page
706 * @bytes_transferred: increase it with the number of transferred bytes
708 static int save_zero_page(QEMUFile
*f
, RAMBlock
*block
, ram_addr_t offset
,
709 uint8_t *p
, uint64_t *bytes_transferred
)
713 if (is_zero_range(p
, TARGET_PAGE_SIZE
)) {
714 acct_info
.dup_pages
++;
715 *bytes_transferred
+= save_page_header(f
, block
,
716 offset
| RAM_SAVE_FLAG_COMPRESS
);
718 *bytes_transferred
+= 1;
725 static void ram_release_pages(MigrationState
*ms
, const char *block_name
,
726 uint64_t offset
, int pages
)
728 if (!migrate_release_ram() || !migration_in_postcopy(ms
)) {
732 ram_discard_range(NULL
, block_name
, offset
, pages
<< TARGET_PAGE_BITS
);
736 * ram_save_page: Send the given page to the stream
738 * Returns: Number of pages written.
740 * >=0 - Number of pages written - this might legally be 0
741 * if xbzrle noticed the page was the same.
743 * @ms: The current migration state.
744 * @f: QEMUFile where to send the data
745 * @block: block that contains the page we want to send
746 * @offset: offset inside the block for the page
747 * @last_stage: if we are at the completion stage
748 * @bytes_transferred: increase it with the number of transferred bytes
750 static int ram_save_page(MigrationState
*ms
, QEMUFile
*f
, PageSearchStatus
*pss
,
751 bool last_stage
, uint64_t *bytes_transferred
)
755 ram_addr_t current_addr
;
758 bool send_async
= true;
759 RAMBlock
*block
= pss
->block
;
760 ram_addr_t offset
= pss
->offset
;
762 p
= block
->host
+ offset
;
764 /* In doubt sent page as normal */
766 ret
= ram_control_save_page(f
, block
->offset
,
767 offset
, TARGET_PAGE_SIZE
, &bytes_xmit
);
769 *bytes_transferred
+= bytes_xmit
;
775 current_addr
= block
->offset
+ offset
;
777 if (block
== last_sent_block
) {
778 offset
|= RAM_SAVE_FLAG_CONTINUE
;
780 if (ret
!= RAM_SAVE_CONTROL_NOT_SUPP
) {
781 if (ret
!= RAM_SAVE_CONTROL_DELAYED
) {
782 if (bytes_xmit
> 0) {
783 acct_info
.norm_pages
++;
784 } else if (bytes_xmit
== 0) {
785 acct_info
.dup_pages
++;
789 pages
= save_zero_page(f
, block
, offset
, p
, bytes_transferred
);
791 /* Must let xbzrle know, otherwise a previous (now 0'd) cached
792 * page would be stale
794 xbzrle_cache_zero_page(current_addr
);
795 ram_release_pages(ms
, block
->idstr
, pss
->offset
, pages
);
796 } else if (!ram_bulk_stage
&&
797 !migration_in_postcopy(ms
) && migrate_use_xbzrle()) {
798 pages
= save_xbzrle_page(f
, &p
, current_addr
, block
,
799 offset
, last_stage
, bytes_transferred
);
801 /* Can't send this cached data async, since the cache page
802 * might get updated before it gets to the wire
809 /* XBZRLE overflow or normal page */
811 *bytes_transferred
+= save_page_header(f
, block
,
812 offset
| RAM_SAVE_FLAG_PAGE
);
814 qemu_put_buffer_async(f
, p
, TARGET_PAGE_SIZE
,
815 migrate_release_ram() &
816 migration_in_postcopy(ms
));
818 qemu_put_buffer(f
, p
, TARGET_PAGE_SIZE
);
820 *bytes_transferred
+= TARGET_PAGE_SIZE
;
822 acct_info
.norm_pages
++;
825 XBZRLE_cache_unlock();
830 static int do_compress_ram_page(QEMUFile
*f
, RAMBlock
*block
,
833 int bytes_sent
, blen
;
834 uint8_t *p
= block
->host
+ (offset
& TARGET_PAGE_MASK
);
836 bytes_sent
= save_page_header(f
, block
, offset
|
837 RAM_SAVE_FLAG_COMPRESS_PAGE
);
838 blen
= qemu_put_compression_data(f
, p
, TARGET_PAGE_SIZE
,
839 migrate_compress_level());
842 qemu_file_set_error(migrate_get_current()->to_dst_file
, blen
);
843 error_report("compressed data failed!");
846 ram_release_pages(migrate_get_current(), block
->idstr
,
847 offset
& TARGET_PAGE_MASK
, 1);
853 static uint64_t bytes_transferred
;
855 static void flush_compressed_data(QEMUFile
*f
)
857 int idx
, len
, thread_count
;
859 if (!migrate_use_compression()) {
862 thread_count
= migrate_compress_threads();
864 qemu_mutex_lock(&comp_done_lock
);
865 for (idx
= 0; idx
< thread_count
; idx
++) {
866 while (!comp_param
[idx
].done
) {
867 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
870 qemu_mutex_unlock(&comp_done_lock
);
872 for (idx
= 0; idx
< thread_count
; idx
++) {
873 qemu_mutex_lock(&comp_param
[idx
].mutex
);
874 if (!comp_param
[idx
].quit
) {
875 len
= qemu_put_qemu_file(f
, comp_param
[idx
].file
);
876 bytes_transferred
+= len
;
878 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
882 static inline void set_compress_params(CompressParam
*param
, RAMBlock
*block
,
885 param
->block
= block
;
886 param
->offset
= offset
;
889 static int compress_page_with_multi_thread(QEMUFile
*f
, RAMBlock
*block
,
891 uint64_t *bytes_transferred
)
893 int idx
, thread_count
, bytes_xmit
= -1, pages
= -1;
895 thread_count
= migrate_compress_threads();
896 qemu_mutex_lock(&comp_done_lock
);
898 for (idx
= 0; idx
< thread_count
; idx
++) {
899 if (comp_param
[idx
].done
) {
900 comp_param
[idx
].done
= false;
901 bytes_xmit
= qemu_put_qemu_file(f
, comp_param
[idx
].file
);
902 qemu_mutex_lock(&comp_param
[idx
].mutex
);
903 set_compress_params(&comp_param
[idx
], block
, offset
);
904 qemu_cond_signal(&comp_param
[idx
].cond
);
905 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
907 acct_info
.norm_pages
++;
908 *bytes_transferred
+= bytes_xmit
;
915 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
918 qemu_mutex_unlock(&comp_done_lock
);
924 * ram_save_compressed_page: compress the given page and send it to the stream
926 * Returns: Number of pages written.
928 * @ms: The current migration state.
929 * @f: QEMUFile where to send the data
930 * @block: block that contains the page we want to send
931 * @offset: offset inside the block for the page
932 * @last_stage: if we are at the completion stage
933 * @bytes_transferred: increase it with the number of transferred bytes
935 static int ram_save_compressed_page(MigrationState
*ms
, QEMUFile
*f
,
936 PageSearchStatus
*pss
, bool last_stage
,
937 uint64_t *bytes_transferred
)
940 uint64_t bytes_xmit
= 0;
943 RAMBlock
*block
= pss
->block
;
944 ram_addr_t offset
= pss
->offset
;
946 p
= block
->host
+ offset
;
948 ret
= ram_control_save_page(f
, block
->offset
,
949 offset
, TARGET_PAGE_SIZE
, &bytes_xmit
);
951 *bytes_transferred
+= bytes_xmit
;
954 if (ret
!= RAM_SAVE_CONTROL_NOT_SUPP
) {
955 if (ret
!= RAM_SAVE_CONTROL_DELAYED
) {
956 if (bytes_xmit
> 0) {
957 acct_info
.norm_pages
++;
958 } else if (bytes_xmit
== 0) {
959 acct_info
.dup_pages
++;
963 /* When starting the process of a new block, the first page of
964 * the block should be sent out before other pages in the same
965 * block, and all the pages in last block should have been sent
966 * out, keeping this order is important, because the 'cont' flag
967 * is used to avoid resending the block name.
969 if (block
!= last_sent_block
) {
970 flush_compressed_data(f
);
971 pages
= save_zero_page(f
, block
, offset
, p
, bytes_transferred
);
973 /* Make sure the first page is sent out before other pages */
974 bytes_xmit
= save_page_header(f
, block
, offset
|
975 RAM_SAVE_FLAG_COMPRESS_PAGE
);
976 blen
= qemu_put_compression_data(f
, p
, TARGET_PAGE_SIZE
,
977 migrate_compress_level());
979 *bytes_transferred
+= bytes_xmit
+ blen
;
980 acct_info
.norm_pages
++;
983 qemu_file_set_error(f
, blen
);
984 error_report("compressed data failed!");
988 ram_release_pages(ms
, block
->idstr
, pss
->offset
, pages
);
991 offset
|= RAM_SAVE_FLAG_CONTINUE
;
992 pages
= save_zero_page(f
, block
, offset
, p
, bytes_transferred
);
994 pages
= compress_page_with_multi_thread(f
, block
, offset
,
997 ram_release_pages(ms
, block
->idstr
, pss
->offset
, pages
);
1006 * Find the next dirty page and update any state associated with
1007 * the search process.
1009 * Returns: True if a page is found
1011 * @f: Current migration stream.
1012 * @pss: Data about the state of the current dirty page scan.
1013 * @*again: Set to false if the search has scanned the whole of RAM
1014 * *ram_addr_abs: Pointer into which to store the address of the dirty page
1015 * within the global ram_addr space
1017 static bool find_dirty_block(QEMUFile
*f
, PageSearchStatus
*pss
,
1018 bool *again
, ram_addr_t
*ram_addr_abs
)
1020 pss
->offset
= migration_bitmap_find_dirty(pss
->block
, pss
->offset
,
1022 if (pss
->complete_round
&& pss
->block
== last_seen_block
&&
1023 pss
->offset
>= last_offset
) {
1025 * We've been once around the RAM and haven't found anything.
1031 if (pss
->offset
>= pss
->block
->used_length
) {
1032 /* Didn't find anything in this RAM Block */
1034 pss
->block
= QLIST_NEXT_RCU(pss
->block
, next
);
1036 /* Hit the end of the list */
1037 pss
->block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
1038 /* Flag that we've looped */
1039 pss
->complete_round
= true;
1040 ram_bulk_stage
= false;
1041 if (migrate_use_xbzrle()) {
1042 /* If xbzrle is on, stop using the data compression at this
1043 * point. In theory, xbzrle can do better than compression.
1045 flush_compressed_data(f
);
1046 compression_switch
= false;
1049 /* Didn't find anything this time, but try again on the new block */
1053 /* Can go around again, but... */
1055 /* We've found something so probably don't need to */
1061 * Helper for 'get_queued_page' - gets a page off the queue
1062 * ms: MigrationState in
1063 * *offset: Used to return the offset within the RAMBlock
1064 * ram_addr_abs: global offset in the dirty/sent bitmaps
1066 * Returns: block (or NULL if none available)
1068 static RAMBlock
*unqueue_page(MigrationState
*ms
, ram_addr_t
*offset
,
1069 ram_addr_t
*ram_addr_abs
)
1071 RAMBlock
*block
= NULL
;
1073 qemu_mutex_lock(&ms
->src_page_req_mutex
);
1074 if (!QSIMPLEQ_EMPTY(&ms
->src_page_requests
)) {
1075 struct MigrationSrcPageRequest
*entry
=
1076 QSIMPLEQ_FIRST(&ms
->src_page_requests
);
1078 *offset
= entry
->offset
;
1079 *ram_addr_abs
= (entry
->offset
+ entry
->rb
->offset
) &
1082 if (entry
->len
> TARGET_PAGE_SIZE
) {
1083 entry
->len
-= TARGET_PAGE_SIZE
;
1084 entry
->offset
+= TARGET_PAGE_SIZE
;
1086 memory_region_unref(block
->mr
);
1087 QSIMPLEQ_REMOVE_HEAD(&ms
->src_page_requests
, next_req
);
1091 qemu_mutex_unlock(&ms
->src_page_req_mutex
);
1097 * Unqueue a page from the queue fed by postcopy page requests; skips pages
1098 * that are already sent (!dirty)
1100 * ms: MigrationState in
1101 * pss: PageSearchStatus structure updated with found block/offset
1102 * ram_addr_abs: global offset in the dirty/sent bitmaps
1104 * Returns: true if a queued page is found
1106 static bool get_queued_page(MigrationState
*ms
, PageSearchStatus
*pss
,
1107 ram_addr_t
*ram_addr_abs
)
1114 block
= unqueue_page(ms
, &offset
, ram_addr_abs
);
1116 * We're sending this page, and since it's postcopy nothing else
1117 * will dirty it, and we must make sure it doesn't get sent again
1118 * even if this queue request was received after the background
1119 * search already sent it.
1122 unsigned long *bitmap
;
1123 bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
1124 dirty
= test_bit(*ram_addr_abs
>> TARGET_PAGE_BITS
, bitmap
);
1126 trace_get_queued_page_not_dirty(
1127 block
->idstr
, (uint64_t)offset
,
1128 (uint64_t)*ram_addr_abs
,
1129 test_bit(*ram_addr_abs
>> TARGET_PAGE_BITS
,
1130 atomic_rcu_read(&migration_bitmap_rcu
)->unsentmap
));
1132 trace_get_queued_page(block
->idstr
,
1134 (uint64_t)*ram_addr_abs
);
1138 } while (block
&& !dirty
);
1142 * As soon as we start servicing pages out of order, then we have
1143 * to kill the bulk stage, since the bulk stage assumes
1144 * in (migration_bitmap_find_and_reset_dirty) that every page is
1145 * dirty, that's no longer true.
1147 ram_bulk_stage
= false;
1150 * We want the background search to continue from the queued page
1151 * since the guest is likely to want other pages near to the page
1152 * it just requested.
1155 pss
->offset
= offset
;
1162 * flush_page_queue: Flush any remaining pages in the ram request queue
1163 * it should be empty at the end anyway, but in error cases there may be
1166 * ms: MigrationState
1168 void flush_page_queue(MigrationState
*ms
)
1170 struct MigrationSrcPageRequest
*mspr
, *next_mspr
;
1171 /* This queue generally should be empty - but in the case of a failed
1172 * migration might have some droppings in.
1175 QSIMPLEQ_FOREACH_SAFE(mspr
, &ms
->src_page_requests
, next_req
, next_mspr
) {
1176 memory_region_unref(mspr
->rb
->mr
);
1177 QSIMPLEQ_REMOVE_HEAD(&ms
->src_page_requests
, next_req
);
1184 * Queue the pages for transmission, e.g. a request from postcopy destination
1185 * ms: MigrationStatus in which the queue is held
1186 * rbname: The RAMBlock the request is for - may be NULL (to mean reuse last)
1187 * start: Offset from the start of the RAMBlock
1188 * len: Length (in bytes) to send
1189 * Return: 0 on success
1191 int ram_save_queue_pages(MigrationState
*ms
, const char *rbname
,
1192 ram_addr_t start
, ram_addr_t len
)
1196 ms
->postcopy_requests
++;
1199 /* Reuse last RAMBlock */
1200 ramblock
= ms
->last_req_rb
;
1204 * Shouldn't happen, we can't reuse the last RAMBlock if
1205 * it's the 1st request.
1207 error_report("ram_save_queue_pages no previous block");
1211 ramblock
= qemu_ram_block_by_name(rbname
);
1214 /* We shouldn't be asked for a non-existent RAMBlock */
1215 error_report("ram_save_queue_pages no block '%s'", rbname
);
1218 ms
->last_req_rb
= ramblock
;
1220 trace_ram_save_queue_pages(ramblock
->idstr
, start
, len
);
1221 if (start
+len
> ramblock
->used_length
) {
1222 error_report("%s request overrun start=" RAM_ADDR_FMT
" len="
1223 RAM_ADDR_FMT
" blocklen=" RAM_ADDR_FMT
,
1224 __func__
, start
, len
, ramblock
->used_length
);
1228 struct MigrationSrcPageRequest
*new_entry
=
1229 g_malloc0(sizeof(struct MigrationSrcPageRequest
));
1230 new_entry
->rb
= ramblock
;
1231 new_entry
->offset
= start
;
1232 new_entry
->len
= len
;
1234 memory_region_ref(ramblock
->mr
);
1235 qemu_mutex_lock(&ms
->src_page_req_mutex
);
1236 QSIMPLEQ_INSERT_TAIL(&ms
->src_page_requests
, new_entry
, next_req
);
1237 qemu_mutex_unlock(&ms
->src_page_req_mutex
);
1248 * ram_save_target_page: Save one target page
1251 * @f: QEMUFile where to send the data
1252 * @block: pointer to block that contains the page we want to send
1253 * @offset: offset inside the block for the page;
1254 * @last_stage: if we are at the completion stage
1255 * @bytes_transferred: increase it with the number of transferred bytes
1256 * @dirty_ram_abs: Address of the start of the dirty page in ram_addr_t space
1258 * Returns: Number of pages written.
1260 static int ram_save_target_page(MigrationState
*ms
, QEMUFile
*f
,
1261 PageSearchStatus
*pss
,
1263 uint64_t *bytes_transferred
,
1264 ram_addr_t dirty_ram_abs
)
1268 /* Check the pages is dirty and if it is send it */
1269 if (migration_bitmap_clear_dirty(dirty_ram_abs
)) {
1270 unsigned long *unsentmap
;
1271 if (compression_switch
&& migrate_use_compression()) {
1272 res
= ram_save_compressed_page(ms
, f
, pss
,
1276 res
= ram_save_page(ms
, f
, pss
, last_stage
,
1283 unsentmap
= atomic_rcu_read(&migration_bitmap_rcu
)->unsentmap
;
1285 clear_bit(dirty_ram_abs
>> TARGET_PAGE_BITS
, unsentmap
);
1287 /* Only update last_sent_block if a block was actually sent; xbzrle
1288 * might have decided the page was identical so didn't bother writing
1292 last_sent_block
= pss
->block
;
1300 * ram_save_host_page: Starting at *offset send pages up to the end
1301 * of the current host page. It's valid for the initial
1302 * offset to point into the middle of a host page
1303 * in which case the remainder of the hostpage is sent.
1304 * Only dirty target pages are sent.
1305 * Note that the host page size may be a huge page for this
1308 * Returns: Number of pages written.
1310 * @f: QEMUFile where to send the data
1311 * @block: pointer to block that contains the page we want to send
1312 * @offset: offset inside the block for the page; updated to last target page
1314 * @last_stage: if we are at the completion stage
1315 * @bytes_transferred: increase it with the number of transferred bytes
1316 * @dirty_ram_abs: Address of the start of the dirty page in ram_addr_t space
1318 static int ram_save_host_page(MigrationState
*ms
, QEMUFile
*f
,
1319 PageSearchStatus
*pss
,
1321 uint64_t *bytes_transferred
,
1322 ram_addr_t dirty_ram_abs
)
1324 int tmppages
, pages
= 0;
1325 size_t pagesize
= qemu_ram_pagesize(pss
->block
);
1328 tmppages
= ram_save_target_page(ms
, f
, pss
, last_stage
,
1329 bytes_transferred
, dirty_ram_abs
);
1335 pss
->offset
+= TARGET_PAGE_SIZE
;
1336 dirty_ram_abs
+= TARGET_PAGE_SIZE
;
1337 } while (pss
->offset
& (pagesize
- 1));
1339 /* The offset we leave with is the last one we looked at */
1340 pss
->offset
-= TARGET_PAGE_SIZE
;
1345 * ram_find_and_save_block: Finds a dirty page and sends it to f
1347 * Called within an RCU critical section.
1349 * Returns: The number of pages written
1350 * 0 means no dirty pages
1352 * @f: QEMUFile where to send the data
1353 * @last_stage: if we are at the completion stage
1354 * @bytes_transferred: increase it with the number of transferred bytes
1356 * On systems where host-page-size > target-page-size it will send all the
1357 * pages in a host page that are dirty.
1360 static int ram_find_and_save_block(QEMUFile
*f
, bool last_stage
,
1361 uint64_t *bytes_transferred
)
1363 PageSearchStatus pss
;
1364 MigrationState
*ms
= migrate_get_current();
1367 ram_addr_t dirty_ram_abs
; /* Address of the start of the dirty page in
1370 /* No dirty page as there is zero RAM */
1371 if (!ram_bytes_total()) {
1375 pss
.block
= last_seen_block
;
1376 pss
.offset
= last_offset
;
1377 pss
.complete_round
= false;
1380 pss
.block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
1385 found
= get_queued_page(ms
, &pss
, &dirty_ram_abs
);
1388 /* priority queue empty, so just search for something dirty */
1389 found
= find_dirty_block(f
, &pss
, &again
, &dirty_ram_abs
);
1393 pages
= ram_save_host_page(ms
, f
, &pss
,
1394 last_stage
, bytes_transferred
,
1397 } while (!pages
&& again
);
1399 last_seen_block
= pss
.block
;
1400 last_offset
= pss
.offset
;
1405 void acct_update_position(QEMUFile
*f
, size_t size
, bool zero
)
1407 uint64_t pages
= size
/ TARGET_PAGE_SIZE
;
1409 acct_info
.dup_pages
+= pages
;
1411 acct_info
.norm_pages
+= pages
;
1412 bytes_transferred
+= size
;
1413 qemu_update_position(f
, size
);
1417 static ram_addr_t
ram_save_remaining(void)
1419 return migration_dirty_pages
;
1422 uint64_t ram_bytes_remaining(void)
1424 return ram_save_remaining() * TARGET_PAGE_SIZE
;
1427 uint64_t ram_bytes_transferred(void)
1429 return bytes_transferred
;
1432 uint64_t ram_bytes_total(void)
1438 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
)
1439 total
+= block
->used_length
;
1444 void free_xbzrle_decoded_buf(void)
1446 g_free(xbzrle_decoded_buf
);
1447 xbzrle_decoded_buf
= NULL
;
1450 static void migration_bitmap_free(struct BitmapRcu
*bmap
)
1453 g_free(bmap
->unsentmap
);
1457 static void ram_migration_cleanup(void *opaque
)
1459 /* caller have hold iothread lock or is in a bh, so there is
1460 * no writing race against this migration_bitmap
1462 struct BitmapRcu
*bitmap
= migration_bitmap_rcu
;
1463 atomic_rcu_set(&migration_bitmap_rcu
, NULL
);
1465 memory_global_dirty_log_stop();
1466 call_rcu(bitmap
, migration_bitmap_free
, rcu
);
1469 XBZRLE_cache_lock();
1471 cache_fini(XBZRLE
.cache
);
1472 g_free(XBZRLE
.encoded_buf
);
1473 g_free(XBZRLE
.current_buf
);
1474 g_free(ZERO_TARGET_PAGE
);
1475 XBZRLE
.cache
= NULL
;
1476 XBZRLE
.encoded_buf
= NULL
;
1477 XBZRLE
.current_buf
= NULL
;
1479 XBZRLE_cache_unlock();
1482 static void reset_ram_globals(void)
1484 last_seen_block
= NULL
;
1485 last_sent_block
= NULL
;
1487 last_version
= ram_list
.version
;
1488 ram_bulk_stage
= true;
1491 #define MAX_WAIT 50 /* ms, half buffered_file limit */
1493 void migration_bitmap_extend(ram_addr_t old
, ram_addr_t
new)
1495 /* called in qemu main thread, so there is
1496 * no writing race against this migration_bitmap
1498 if (migration_bitmap_rcu
) {
1499 struct BitmapRcu
*old_bitmap
= migration_bitmap_rcu
, *bitmap
;
1500 bitmap
= g_new(struct BitmapRcu
, 1);
1501 bitmap
->bmap
= bitmap_new(new);
1503 /* prevent migration_bitmap content from being set bit
1504 * by migration_bitmap_sync_range() at the same time.
1505 * it is safe to migration if migration_bitmap is cleared bit
1508 qemu_mutex_lock(&migration_bitmap_mutex
);
1509 bitmap_copy(bitmap
->bmap
, old_bitmap
->bmap
, old
);
1510 bitmap_set(bitmap
->bmap
, old
, new - old
);
1512 /* We don't have a way to safely extend the sentmap
1513 * with RCU; so mark it as missing, entry to postcopy
1516 bitmap
->unsentmap
= NULL
;
1518 atomic_rcu_set(&migration_bitmap_rcu
, bitmap
);
1519 qemu_mutex_unlock(&migration_bitmap_mutex
);
1520 migration_dirty_pages
+= new - old
;
1521 call_rcu(old_bitmap
, migration_bitmap_free
, rcu
);
1526 * 'expected' is the value you expect the bitmap mostly to be full
1527 * of; it won't bother printing lines that are all this value.
1528 * If 'todump' is null the migration bitmap is dumped.
1530 void ram_debug_dump_bitmap(unsigned long *todump
, bool expected
)
1532 int64_t ram_pages
= last_ram_offset() >> TARGET_PAGE_BITS
;
1535 int64_t linelen
= 128;
1539 todump
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
1542 for (cur
= 0; cur
< ram_pages
; cur
+= linelen
) {
1546 * Last line; catch the case where the line length
1547 * is longer than remaining ram
1549 if (cur
+ linelen
> ram_pages
) {
1550 linelen
= ram_pages
- cur
;
1552 for (curb
= 0; curb
< linelen
; curb
++) {
1553 bool thisbit
= test_bit(cur
+ curb
, todump
);
1554 linebuf
[curb
] = thisbit
? '1' : '.';
1555 found
= found
|| (thisbit
!= expected
);
1558 linebuf
[curb
] = '\0';
1559 fprintf(stderr
, "0x%08" PRIx64
" : %s\n", cur
, linebuf
);
1564 /* **** functions for postcopy ***** */
1566 void ram_postcopy_migrated_memory_release(MigrationState
*ms
)
1568 struct RAMBlock
*block
;
1569 unsigned long *bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
1571 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1572 unsigned long first
= block
->offset
>> TARGET_PAGE_BITS
;
1573 unsigned long range
= first
+ (block
->used_length
>> TARGET_PAGE_BITS
);
1574 unsigned long run_start
= find_next_zero_bit(bitmap
, range
, first
);
1576 while (run_start
< range
) {
1577 unsigned long run_end
= find_next_bit(bitmap
, range
, run_start
+ 1);
1578 ram_discard_range(NULL
, block
->idstr
, run_start
<< TARGET_PAGE_BITS
,
1579 (run_end
- run_start
) << TARGET_PAGE_BITS
);
1580 run_start
= find_next_zero_bit(bitmap
, range
, run_end
+ 1);
1586 * Callback from postcopy_each_ram_send_discard for each RAMBlock
1587 * Note: At this point the 'unsentmap' is the processed bitmap combined
1588 * with the dirtymap; so a '1' means it's either dirty or unsent.
1589 * start,length: Indexes into the bitmap for the first bit
1590 * representing the named block and length in target-pages
1592 static int postcopy_send_discard_bm_ram(MigrationState
*ms
,
1593 PostcopyDiscardState
*pds
,
1594 unsigned long start
,
1595 unsigned long length
)
1597 unsigned long end
= start
+ length
; /* one after the end */
1598 unsigned long current
;
1599 unsigned long *unsentmap
;
1601 unsentmap
= atomic_rcu_read(&migration_bitmap_rcu
)->unsentmap
;
1602 for (current
= start
; current
< end
; ) {
1603 unsigned long one
= find_next_bit(unsentmap
, end
, current
);
1606 unsigned long zero
= find_next_zero_bit(unsentmap
, end
, one
+ 1);
1607 unsigned long discard_length
;
1610 discard_length
= end
- one
;
1612 discard_length
= zero
- one
;
1614 if (discard_length
) {
1615 postcopy_discard_send_range(ms
, pds
, one
, discard_length
);
1617 current
= one
+ discard_length
;
1627 * Utility for the outgoing postcopy code.
1628 * Calls postcopy_send_discard_bm_ram for each RAMBlock
1629 * passing it bitmap indexes and name.
1630 * Returns: 0 on success
1631 * (qemu_ram_foreach_block ends up passing unscaled lengths
1632 * which would mean postcopy code would have to deal with target page)
1634 static int postcopy_each_ram_send_discard(MigrationState
*ms
)
1636 struct RAMBlock
*block
;
1639 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1640 unsigned long first
= block
->offset
>> TARGET_PAGE_BITS
;
1641 PostcopyDiscardState
*pds
= postcopy_discard_send_init(ms
,
1646 * Postcopy sends chunks of bitmap over the wire, but it
1647 * just needs indexes at this point, avoids it having
1648 * target page specific code.
1650 ret
= postcopy_send_discard_bm_ram(ms
, pds
, first
,
1651 block
->used_length
>> TARGET_PAGE_BITS
);
1652 postcopy_discard_send_finish(ms
, pds
);
1662 * Helper for postcopy_chunk_hostpages; it's called twice to cleanup
1663 * the two bitmaps, that are similar, but one is inverted.
1665 * We search for runs of target-pages that don't start or end on a
1666 * host page boundary;
1667 * unsent_pass=true: Cleans up partially unsent host pages by searching
1669 * unsent_pass=false: Cleans up partially dirty host pages by searching
1670 * the main migration bitmap
1673 static void postcopy_chunk_hostpages_pass(MigrationState
*ms
, bool unsent_pass
,
1675 PostcopyDiscardState
*pds
)
1677 unsigned long *bitmap
;
1678 unsigned long *unsentmap
;
1679 unsigned int host_ratio
= block
->page_size
/ TARGET_PAGE_SIZE
;
1680 unsigned long first
= block
->offset
>> TARGET_PAGE_BITS
;
1681 unsigned long len
= block
->used_length
>> TARGET_PAGE_BITS
;
1682 unsigned long last
= first
+ (len
- 1);
1683 unsigned long run_start
;
1685 if (block
->page_size
== TARGET_PAGE_SIZE
) {
1686 /* Easy case - TPS==HPS for a non-huge page RAMBlock */
1690 bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
1691 unsentmap
= atomic_rcu_read(&migration_bitmap_rcu
)->unsentmap
;
1694 /* Find a sent page */
1695 run_start
= find_next_zero_bit(unsentmap
, last
+ 1, first
);
1697 /* Find a dirty page */
1698 run_start
= find_next_bit(bitmap
, last
+ 1, first
);
1701 while (run_start
<= last
) {
1702 bool do_fixup
= false;
1703 unsigned long fixup_start_addr
;
1704 unsigned long host_offset
;
1707 * If the start of this run of pages is in the middle of a host
1708 * page, then we need to fixup this host page.
1710 host_offset
= run_start
% host_ratio
;
1713 run_start
-= host_offset
;
1714 fixup_start_addr
= run_start
;
1715 /* For the next pass */
1716 run_start
= run_start
+ host_ratio
;
1718 /* Find the end of this run */
1719 unsigned long run_end
;
1721 run_end
= find_next_bit(unsentmap
, last
+ 1, run_start
+ 1);
1723 run_end
= find_next_zero_bit(bitmap
, last
+ 1, run_start
+ 1);
1726 * If the end isn't at the start of a host page, then the
1727 * run doesn't finish at the end of a host page
1728 * and we need to discard.
1730 host_offset
= run_end
% host_ratio
;
1733 fixup_start_addr
= run_end
- host_offset
;
1735 * This host page has gone, the next loop iteration starts
1736 * from after the fixup
1738 run_start
= fixup_start_addr
+ host_ratio
;
1741 * No discards on this iteration, next loop starts from
1742 * next sent/dirty page
1744 run_start
= run_end
+ 1;
1751 /* Tell the destination to discard this page */
1752 if (unsent_pass
|| !test_bit(fixup_start_addr
, unsentmap
)) {
1753 /* For the unsent_pass we:
1754 * discard partially sent pages
1755 * For the !unsent_pass (dirty) we:
1756 * discard partially dirty pages that were sent
1757 * (any partially sent pages were already discarded
1758 * by the previous unsent_pass)
1760 postcopy_discard_send_range(ms
, pds
, fixup_start_addr
,
1764 /* Clean up the bitmap */
1765 for (page
= fixup_start_addr
;
1766 page
< fixup_start_addr
+ host_ratio
; page
++) {
1767 /* All pages in this host page are now not sent */
1768 set_bit(page
, unsentmap
);
1771 * Remark them as dirty, updating the count for any pages
1772 * that weren't previously dirty.
1774 migration_dirty_pages
+= !test_and_set_bit(page
, bitmap
);
1779 /* Find the next sent page for the next iteration */
1780 run_start
= find_next_zero_bit(unsentmap
, last
+ 1,
1783 /* Find the next dirty page for the next iteration */
1784 run_start
= find_next_bit(bitmap
, last
+ 1, run_start
);
1790 * Utility for the outgoing postcopy code.
1792 * Discard any partially sent host-page size chunks, mark any partially
1793 * dirty host-page size chunks as all dirty. In this case the host-page
1794 * is the host-page for the particular RAMBlock, i.e. it might be a huge page
1796 * Returns: 0 on success
1798 static int postcopy_chunk_hostpages(MigrationState
*ms
)
1800 struct RAMBlock
*block
;
1802 /* Easiest way to make sure we don't resume in the middle of a host-page */
1803 last_seen_block
= NULL
;
1804 last_sent_block
= NULL
;
1807 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1808 unsigned long first
= block
->offset
>> TARGET_PAGE_BITS
;
1810 PostcopyDiscardState
*pds
=
1811 postcopy_discard_send_init(ms
, first
, block
->idstr
);
1813 /* First pass: Discard all partially sent host pages */
1814 postcopy_chunk_hostpages_pass(ms
, true, block
, pds
);
1816 * Second pass: Ensure that all partially dirty host pages are made
1819 postcopy_chunk_hostpages_pass(ms
, false, block
, pds
);
1821 postcopy_discard_send_finish(ms
, pds
);
1822 } /* ram_list loop */
1828 * Transmit the set of pages to be discarded after precopy to the target
1829 * these are pages that:
1830 * a) Have been previously transmitted but are now dirty again
1831 * b) Pages that have never been transmitted, this ensures that
1832 * any pages on the destination that have been mapped by background
1833 * tasks get discarded (transparent huge pages is the specific concern)
1834 * Hopefully this is pretty sparse
1836 int ram_postcopy_send_discard_bitmap(MigrationState
*ms
)
1839 unsigned long *bitmap
, *unsentmap
;
1843 /* This should be our last sync, the src is now paused */
1844 migration_bitmap_sync();
1846 unsentmap
= atomic_rcu_read(&migration_bitmap_rcu
)->unsentmap
;
1848 /* We don't have a safe way to resize the sentmap, so
1849 * if the bitmap was resized it will be NULL at this
1852 error_report("migration ram resized during precopy phase");
1857 /* Deal with TPS != HPS and huge pages */
1858 ret
= postcopy_chunk_hostpages(ms
);
1865 * Update the unsentmap to be unsentmap = unsentmap | dirty
1867 bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
1868 bitmap_or(unsentmap
, unsentmap
, bitmap
,
1869 last_ram_offset() >> TARGET_PAGE_BITS
);
1872 trace_ram_postcopy_send_discard_bitmap();
1873 #ifdef DEBUG_POSTCOPY
1874 ram_debug_dump_bitmap(unsentmap
, true);
1877 ret
= postcopy_each_ram_send_discard(ms
);
1884 * At the start of the postcopy phase of migration, any now-dirty
1885 * precopied pages are discarded.
1887 * start, length describe a byte address range within the RAMBlock
1889 * Returns 0 on success.
1891 int ram_discard_range(MigrationIncomingState
*mis
,
1892 const char *block_name
,
1893 uint64_t start
, size_t length
)
1897 trace_ram_discard_range(block_name
, start
, length
);
1900 RAMBlock
*rb
= qemu_ram_block_by_name(block_name
);
1903 error_report("ram_discard_range: Failed to find block '%s'",
1908 ret
= ram_block_discard_range(rb
, start
, length
);
1916 static int ram_save_init_globals(void)
1918 int64_t ram_bitmap_pages
; /* Size of bitmap in pages, including gaps */
1920 dirty_rate_high_cnt
= 0;
1921 bitmap_sync_count
= 0;
1922 migration_bitmap_sync_init();
1923 qemu_mutex_init(&migration_bitmap_mutex
);
1925 if (migrate_use_xbzrle()) {
1926 XBZRLE_cache_lock();
1927 ZERO_TARGET_PAGE
= g_malloc0(TARGET_PAGE_SIZE
);
1928 XBZRLE
.cache
= cache_init(migrate_xbzrle_cache_size() /
1931 if (!XBZRLE
.cache
) {
1932 XBZRLE_cache_unlock();
1933 error_report("Error creating cache");
1936 XBZRLE_cache_unlock();
1938 /* We prefer not to abort if there is no memory */
1939 XBZRLE
.encoded_buf
= g_try_malloc0(TARGET_PAGE_SIZE
);
1940 if (!XBZRLE
.encoded_buf
) {
1941 error_report("Error allocating encoded_buf");
1945 XBZRLE
.current_buf
= g_try_malloc(TARGET_PAGE_SIZE
);
1946 if (!XBZRLE
.current_buf
) {
1947 error_report("Error allocating current_buf");
1948 g_free(XBZRLE
.encoded_buf
);
1949 XBZRLE
.encoded_buf
= NULL
;
1956 /* For memory_global_dirty_log_start below. */
1957 qemu_mutex_lock_iothread();
1959 qemu_mutex_lock_ramlist();
1961 bytes_transferred
= 0;
1962 reset_ram_globals();
1964 migration_bitmap_rcu
= g_new0(struct BitmapRcu
, 1);
1965 /* Skip setting bitmap if there is no RAM */
1966 if (ram_bytes_total()) {
1967 ram_bitmap_pages
= last_ram_offset() >> TARGET_PAGE_BITS
;
1968 migration_bitmap_rcu
->bmap
= bitmap_new(ram_bitmap_pages
);
1969 bitmap_set(migration_bitmap_rcu
->bmap
, 0, ram_bitmap_pages
);
1971 if (migrate_postcopy_ram()) {
1972 migration_bitmap_rcu
->unsentmap
= bitmap_new(ram_bitmap_pages
);
1973 bitmap_set(migration_bitmap_rcu
->unsentmap
, 0, ram_bitmap_pages
);
1978 * Count the total number of pages used by ram blocks not including any
1979 * gaps due to alignment or unplugs.
1981 migration_dirty_pages
= ram_bytes_total() >> TARGET_PAGE_BITS
;
1983 memory_global_dirty_log_start();
1984 migration_bitmap_sync();
1985 qemu_mutex_unlock_ramlist();
1986 qemu_mutex_unlock_iothread();
1992 /* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
1993 * long-running RCU critical section. When rcu-reclaims in the code
1994 * start to become numerous it will be necessary to reduce the
1995 * granularity of these critical sections.
1998 static int ram_save_setup(QEMUFile
*f
, void *opaque
)
2002 /* migration has already setup the bitmap, reuse it. */
2003 if (!migration_in_colo_state()) {
2004 if (ram_save_init_globals() < 0) {
2011 qemu_put_be64(f
, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE
);
2013 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
2014 qemu_put_byte(f
, strlen(block
->idstr
));
2015 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, strlen(block
->idstr
));
2016 qemu_put_be64(f
, block
->used_length
);
2017 if (migrate_postcopy_ram() && block
->page_size
!= qemu_host_page_size
) {
2018 qemu_put_be64(f
, block
->page_size
);
2024 ram_control_before_iterate(f
, RAM_CONTROL_SETUP
);
2025 ram_control_after_iterate(f
, RAM_CONTROL_SETUP
);
2027 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
2032 static int ram_save_iterate(QEMUFile
*f
, void *opaque
)
2040 if (ram_list
.version
!= last_version
) {
2041 reset_ram_globals();
2044 /* Read version before ram_list.blocks */
2047 ram_control_before_iterate(f
, RAM_CONTROL_ROUND
);
2049 t0
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
);
2051 while ((ret
= qemu_file_rate_limit(f
)) == 0) {
2054 pages
= ram_find_and_save_block(f
, false, &bytes_transferred
);
2055 /* no more pages to sent */
2060 acct_info
.iterations
++;
2062 /* we want to check in the 1st loop, just in case it was the 1st time
2063 and we had to sync the dirty bitmap.
2064 qemu_get_clock_ns() is a bit expensive, so we only check each some
2067 if ((i
& 63) == 0) {
2068 uint64_t t1
= (qemu_clock_get_ns(QEMU_CLOCK_REALTIME
) - t0
) / 1000000;
2069 if (t1
> MAX_WAIT
) {
2070 trace_ram_save_iterate_big_wait(t1
, i
);
2076 flush_compressed_data(f
);
2080 * Must occur before EOS (or any QEMUFile operation)
2081 * because of RDMA protocol.
2083 ram_control_after_iterate(f
, RAM_CONTROL_ROUND
);
2085 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
2086 bytes_transferred
+= 8;
2088 ret
= qemu_file_get_error(f
);
2096 /* Called with iothread lock */
2097 static int ram_save_complete(QEMUFile
*f
, void *opaque
)
2101 if (!migration_in_postcopy(migrate_get_current())) {
2102 migration_bitmap_sync();
2105 ram_control_before_iterate(f
, RAM_CONTROL_FINISH
);
2107 /* try transferring iterative blocks of memory */
2109 /* flush all remaining blocks regardless of rate limiting */
2113 pages
= ram_find_and_save_block(f
, !migration_in_colo_state(),
2114 &bytes_transferred
);
2115 /* no more blocks to sent */
2121 flush_compressed_data(f
);
2122 ram_control_after_iterate(f
, RAM_CONTROL_FINISH
);
2126 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
2131 static void ram_save_pending(QEMUFile
*f
, void *opaque
, uint64_t max_size
,
2132 uint64_t *non_postcopiable_pending
,
2133 uint64_t *postcopiable_pending
)
2135 uint64_t remaining_size
;
2137 remaining_size
= ram_save_remaining() * TARGET_PAGE_SIZE
;
2139 if (!migration_in_postcopy(migrate_get_current()) &&
2140 remaining_size
< max_size
) {
2141 qemu_mutex_lock_iothread();
2143 migration_bitmap_sync();
2145 qemu_mutex_unlock_iothread();
2146 remaining_size
= ram_save_remaining() * TARGET_PAGE_SIZE
;
2149 /* We can do postcopy, and all the data is postcopiable */
2150 *postcopiable_pending
+= remaining_size
;
2153 static int load_xbzrle(QEMUFile
*f
, ram_addr_t addr
, void *host
)
2155 unsigned int xh_len
;
2157 uint8_t *loaded_data
;
2159 if (!xbzrle_decoded_buf
) {
2160 xbzrle_decoded_buf
= g_malloc(TARGET_PAGE_SIZE
);
2162 loaded_data
= xbzrle_decoded_buf
;
2164 /* extract RLE header */
2165 xh_flags
= qemu_get_byte(f
);
2166 xh_len
= qemu_get_be16(f
);
2168 if (xh_flags
!= ENCODING_FLAG_XBZRLE
) {
2169 error_report("Failed to load XBZRLE page - wrong compression!");
2173 if (xh_len
> TARGET_PAGE_SIZE
) {
2174 error_report("Failed to load XBZRLE page - len overflow!");
2177 /* load data and decode */
2178 qemu_get_buffer_in_place(f
, &loaded_data
, xh_len
);
2181 if (xbzrle_decode_buffer(loaded_data
, xh_len
, host
,
2182 TARGET_PAGE_SIZE
) == -1) {
2183 error_report("Failed to load XBZRLE page - decode error!");
2190 /* Must be called from within a rcu critical section.
2191 * Returns a pointer from within the RCU-protected ram_list.
2194 * Read a RAMBlock ID from the stream f.
2196 * f: Stream to read from
2197 * flags: Page flags (mostly to see if it's a continuation of previous block)
2199 static inline RAMBlock
*ram_block_from_stream(QEMUFile
*f
,
2202 static RAMBlock
*block
= NULL
;
2206 if (flags
& RAM_SAVE_FLAG_CONTINUE
) {
2208 error_report("Ack, bad migration stream!");
2214 len
= qemu_get_byte(f
);
2215 qemu_get_buffer(f
, (uint8_t *)id
, len
);
2218 block
= qemu_ram_block_by_name(id
);
2220 error_report("Can't find block %s", id
);
2227 static inline void *host_from_ram_block_offset(RAMBlock
*block
,
2230 if (!offset_in_ramblock(block
, offset
)) {
2234 return block
->host
+ offset
;
2238 * If a page (or a whole RDMA chunk) has been
2239 * determined to be zero, then zap it.
2241 void ram_handle_compressed(void *host
, uint8_t ch
, uint64_t size
)
2243 if (ch
!= 0 || !is_zero_range(host
, size
)) {
2244 memset(host
, ch
, size
);
2248 static void *do_data_decompress(void *opaque
)
2250 DecompressParam
*param
= opaque
;
2251 unsigned long pagesize
;
2255 qemu_mutex_lock(¶m
->mutex
);
2256 while (!param
->quit
) {
2261 qemu_mutex_unlock(¶m
->mutex
);
2263 pagesize
= TARGET_PAGE_SIZE
;
2264 /* uncompress() will return failed in some case, especially
2265 * when the page is dirted when doing the compression, it's
2266 * not a problem because the dirty page will be retransferred
2267 * and uncompress() won't break the data in other pages.
2269 uncompress((Bytef
*)des
, &pagesize
,
2270 (const Bytef
*)param
->compbuf
, len
);
2272 qemu_mutex_lock(&decomp_done_lock
);
2274 qemu_cond_signal(&decomp_done_cond
);
2275 qemu_mutex_unlock(&decomp_done_lock
);
2277 qemu_mutex_lock(¶m
->mutex
);
2279 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
2282 qemu_mutex_unlock(¶m
->mutex
);
2287 static void wait_for_decompress_done(void)
2289 int idx
, thread_count
;
2291 if (!migrate_use_compression()) {
2295 thread_count
= migrate_decompress_threads();
2296 qemu_mutex_lock(&decomp_done_lock
);
2297 for (idx
= 0; idx
< thread_count
; idx
++) {
2298 while (!decomp_param
[idx
].done
) {
2299 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
2302 qemu_mutex_unlock(&decomp_done_lock
);
2305 void migrate_decompress_threads_create(void)
2307 int i
, thread_count
;
2309 thread_count
= migrate_decompress_threads();
2310 decompress_threads
= g_new0(QemuThread
, thread_count
);
2311 decomp_param
= g_new0(DecompressParam
, thread_count
);
2312 qemu_mutex_init(&decomp_done_lock
);
2313 qemu_cond_init(&decomp_done_cond
);
2314 for (i
= 0; i
< thread_count
; i
++) {
2315 qemu_mutex_init(&decomp_param
[i
].mutex
);
2316 qemu_cond_init(&decomp_param
[i
].cond
);
2317 decomp_param
[i
].compbuf
= g_malloc0(compressBound(TARGET_PAGE_SIZE
));
2318 decomp_param
[i
].done
= true;
2319 decomp_param
[i
].quit
= false;
2320 qemu_thread_create(decompress_threads
+ i
, "decompress",
2321 do_data_decompress
, decomp_param
+ i
,
2322 QEMU_THREAD_JOINABLE
);
2326 void migrate_decompress_threads_join(void)
2328 int i
, thread_count
;
2330 thread_count
= migrate_decompress_threads();
2331 for (i
= 0; i
< thread_count
; i
++) {
2332 qemu_mutex_lock(&decomp_param
[i
].mutex
);
2333 decomp_param
[i
].quit
= true;
2334 qemu_cond_signal(&decomp_param
[i
].cond
);
2335 qemu_mutex_unlock(&decomp_param
[i
].mutex
);
2337 for (i
= 0; i
< thread_count
; i
++) {
2338 qemu_thread_join(decompress_threads
+ i
);
2339 qemu_mutex_destroy(&decomp_param
[i
].mutex
);
2340 qemu_cond_destroy(&decomp_param
[i
].cond
);
2341 g_free(decomp_param
[i
].compbuf
);
2343 g_free(decompress_threads
);
2344 g_free(decomp_param
);
2345 decompress_threads
= NULL
;
2346 decomp_param
= NULL
;
2349 static void decompress_data_with_multi_threads(QEMUFile
*f
,
2350 void *host
, int len
)
2352 int idx
, thread_count
;
2354 thread_count
= migrate_decompress_threads();
2355 qemu_mutex_lock(&decomp_done_lock
);
2357 for (idx
= 0; idx
< thread_count
; idx
++) {
2358 if (decomp_param
[idx
].done
) {
2359 decomp_param
[idx
].done
= false;
2360 qemu_mutex_lock(&decomp_param
[idx
].mutex
);
2361 qemu_get_buffer(f
, decomp_param
[idx
].compbuf
, len
);
2362 decomp_param
[idx
].des
= host
;
2363 decomp_param
[idx
].len
= len
;
2364 qemu_cond_signal(&decomp_param
[idx
].cond
);
2365 qemu_mutex_unlock(&decomp_param
[idx
].mutex
);
2369 if (idx
< thread_count
) {
2372 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
2375 qemu_mutex_unlock(&decomp_done_lock
);
2379 * Allocate data structures etc needed by incoming migration with postcopy-ram
2380 * postcopy-ram's similarly names postcopy_ram_incoming_init does the work
2382 int ram_postcopy_incoming_init(MigrationIncomingState
*mis
)
2384 size_t ram_pages
= last_ram_offset() >> TARGET_PAGE_BITS
;
2386 return postcopy_ram_incoming_init(mis
, ram_pages
);
2390 * Called in postcopy mode by ram_load().
2391 * rcu_read_lock is taken prior to this being called.
2393 static int ram_load_postcopy(QEMUFile
*f
)
2395 int flags
= 0, ret
= 0;
2396 bool place_needed
= false;
2397 bool matching_page_sizes
= false;
2398 MigrationIncomingState
*mis
= migration_incoming_get_current();
2399 /* Temporary page that is later 'placed' */
2400 void *postcopy_host_page
= postcopy_get_tmp_page(mis
);
2401 void *last_host
= NULL
;
2402 bool all_zero
= false;
2404 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
2407 void *page_buffer
= NULL
;
2408 void *place_source
= NULL
;
2409 RAMBlock
*block
= NULL
;
2412 addr
= qemu_get_be64(f
);
2413 flags
= addr
& ~TARGET_PAGE_MASK
;
2414 addr
&= TARGET_PAGE_MASK
;
2416 trace_ram_load_postcopy_loop((uint64_t)addr
, flags
);
2417 place_needed
= false;
2418 if (flags
& (RAM_SAVE_FLAG_COMPRESS
| RAM_SAVE_FLAG_PAGE
)) {
2419 block
= ram_block_from_stream(f
, flags
);
2421 host
= host_from_ram_block_offset(block
, addr
);
2423 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
2427 matching_page_sizes
= block
->page_size
== TARGET_PAGE_SIZE
;
2429 * Postcopy requires that we place whole host pages atomically;
2430 * these may be huge pages for RAMBlocks that are backed by
2432 * To make it atomic, the data is read into a temporary page
2433 * that's moved into place later.
2434 * The migration protocol uses, possibly smaller, target-pages
2435 * however the source ensures it always sends all the components
2436 * of a host page in order.
2438 page_buffer
= postcopy_host_page
+
2439 ((uintptr_t)host
& (block
->page_size
- 1));
2440 /* If all TP are zero then we can optimise the place */
2441 if (!((uintptr_t)host
& (block
->page_size
- 1))) {
2444 /* not the 1st TP within the HP */
2445 if (host
!= (last_host
+ TARGET_PAGE_SIZE
)) {
2446 error_report("Non-sequential target page %p/%p",
2455 * If it's the last part of a host page then we place the host
2458 place_needed
= (((uintptr_t)host
+ TARGET_PAGE_SIZE
) &
2459 (block
->page_size
- 1)) == 0;
2460 place_source
= postcopy_host_page
;
2464 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
2465 case RAM_SAVE_FLAG_COMPRESS
:
2466 ch
= qemu_get_byte(f
);
2467 memset(page_buffer
, ch
, TARGET_PAGE_SIZE
);
2473 case RAM_SAVE_FLAG_PAGE
:
2475 if (!place_needed
|| !matching_page_sizes
) {
2476 qemu_get_buffer(f
, page_buffer
, TARGET_PAGE_SIZE
);
2478 /* Avoids the qemu_file copy during postcopy, which is
2479 * going to do a copy later; can only do it when we
2480 * do this read in one go (matching page sizes)
2482 qemu_get_buffer_in_place(f
, (uint8_t **)&place_source
,
2486 case RAM_SAVE_FLAG_EOS
:
2490 error_report("Unknown combination of migration flags: %#x"
2491 " (postcopy mode)", flags
);
2496 /* This gets called at the last target page in the host page */
2497 void *place_dest
= host
+ TARGET_PAGE_SIZE
- block
->page_size
;
2500 ret
= postcopy_place_page_zero(mis
, place_dest
,
2503 ret
= postcopy_place_page(mis
, place_dest
,
2504 place_source
, block
->page_size
);
2508 ret
= qemu_file_get_error(f
);
2515 static int ram_load(QEMUFile
*f
, void *opaque
, int version_id
)
2517 int flags
= 0, ret
= 0;
2518 static uint64_t seq_iter
;
2521 * If system is running in postcopy mode, page inserts to host memory must
2524 bool postcopy_running
= postcopy_state_get() >= POSTCOPY_INCOMING_LISTENING
;
2525 /* ADVISE is earlier, it shows the source has the postcopy capability on */
2526 bool postcopy_advised
= postcopy_state_get() >= POSTCOPY_INCOMING_ADVISE
;
2530 if (version_id
!= 4) {
2534 /* This RCU critical section can be very long running.
2535 * When RCU reclaims in the code start to become numerous,
2536 * it will be necessary to reduce the granularity of this
2541 if (postcopy_running
) {
2542 ret
= ram_load_postcopy(f
);
2545 while (!postcopy_running
&& !ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
2546 ram_addr_t addr
, total_ram_bytes
;
2550 addr
= qemu_get_be64(f
);
2551 flags
= addr
& ~TARGET_PAGE_MASK
;
2552 addr
&= TARGET_PAGE_MASK
;
2554 if (flags
& (RAM_SAVE_FLAG_COMPRESS
| RAM_SAVE_FLAG_PAGE
|
2555 RAM_SAVE_FLAG_COMPRESS_PAGE
| RAM_SAVE_FLAG_XBZRLE
)) {
2556 RAMBlock
*block
= ram_block_from_stream(f
, flags
);
2558 host
= host_from_ram_block_offset(block
, addr
);
2560 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
2566 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
2567 case RAM_SAVE_FLAG_MEM_SIZE
:
2568 /* Synchronize RAM block list */
2569 total_ram_bytes
= addr
;
2570 while (!ret
&& total_ram_bytes
) {
2575 len
= qemu_get_byte(f
);
2576 qemu_get_buffer(f
, (uint8_t *)id
, len
);
2578 length
= qemu_get_be64(f
);
2580 block
= qemu_ram_block_by_name(id
);
2582 if (length
!= block
->used_length
) {
2583 Error
*local_err
= NULL
;
2585 ret
= qemu_ram_resize(block
, length
,
2588 error_report_err(local_err
);
2591 /* For postcopy we need to check hugepage sizes match */
2592 if (postcopy_advised
&&
2593 block
->page_size
!= qemu_host_page_size
) {
2594 uint64_t remote_page_size
= qemu_get_be64(f
);
2595 if (remote_page_size
!= block
->page_size
) {
2596 error_report("Mismatched RAM page size %s "
2597 "(local) %zd != %" PRId64
,
2598 id
, block
->page_size
,
2603 ram_control_load_hook(f
, RAM_CONTROL_BLOCK_REG
,
2606 error_report("Unknown ramblock \"%s\", cannot "
2607 "accept migration", id
);
2611 total_ram_bytes
-= length
;
2615 case RAM_SAVE_FLAG_COMPRESS
:
2616 ch
= qemu_get_byte(f
);
2617 ram_handle_compressed(host
, ch
, TARGET_PAGE_SIZE
);
2620 case RAM_SAVE_FLAG_PAGE
:
2621 qemu_get_buffer(f
, host
, TARGET_PAGE_SIZE
);
2624 case RAM_SAVE_FLAG_COMPRESS_PAGE
:
2625 len
= qemu_get_be32(f
);
2626 if (len
< 0 || len
> compressBound(TARGET_PAGE_SIZE
)) {
2627 error_report("Invalid compressed data length: %d", len
);
2631 decompress_data_with_multi_threads(f
, host
, len
);
2634 case RAM_SAVE_FLAG_XBZRLE
:
2635 if (load_xbzrle(f
, addr
, host
) < 0) {
2636 error_report("Failed to decompress XBZRLE page at "
2637 RAM_ADDR_FMT
, addr
);
2642 case RAM_SAVE_FLAG_EOS
:
2646 if (flags
& RAM_SAVE_FLAG_HOOK
) {
2647 ram_control_load_hook(f
, RAM_CONTROL_HOOK
, NULL
);
2649 error_report("Unknown combination of migration flags: %#x",
2655 ret
= qemu_file_get_error(f
);
2659 wait_for_decompress_done();
2661 trace_ram_load_complete(ret
, seq_iter
);
2665 static SaveVMHandlers savevm_ram_handlers
= {
2666 .save_live_setup
= ram_save_setup
,
2667 .save_live_iterate
= ram_save_iterate
,
2668 .save_live_complete_postcopy
= ram_save_complete
,
2669 .save_live_complete_precopy
= ram_save_complete
,
2670 .save_live_pending
= ram_save_pending
,
2671 .load_state
= ram_load
,
2672 .cleanup
= ram_migration_cleanup
,
2675 void ram_mig_init(void)
2677 qemu_mutex_init(&XBZRLE
.lock
);
2678 register_savevm_live(NULL
, "ram", 0, 4, &savevm_ram_handlers
, NULL
);