4 * Copyright (c) 2003-2008 Fabrice Bellard
5 * Copyright (c) 2011-2015 Red Hat Inc
8 * Juan Quintela <quintela@redhat.com>
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29 #include "qemu/osdep.h"
32 #include "qemu/cutils.h"
33 #include "qemu/bitops.h"
34 #include "qemu/bitmap.h"
35 #include "qemu/main-loop.h"
36 #include "qemu/pmem.h"
39 #include "migration.h"
41 #include "migration/register.h"
42 #include "migration/misc.h"
43 #include "qemu-file.h"
44 #include "postcopy-ram.h"
45 #include "page_cache.h"
46 #include "qemu/error-report.h"
47 #include "qapi/error.h"
48 #include "qapi/qapi-events-migration.h"
49 #include "qapi/qmp/qerror.h"
51 #include "exec/ram_addr.h"
52 #include "exec/target_page.h"
53 #include "qemu/rcu_queue.h"
54 #include "migration/colo.h"
56 #include "sysemu/sysemu.h"
57 #include "qemu/uuid.h"
61 /***********************************************************/
62 /* ram save/restore */
64 /* RAM_SAVE_FLAG_ZERO used to be named RAM_SAVE_FLAG_COMPRESS, it
65 * worked for pages that where filled with the same char. We switched
66 * it to only search for the zero value. And to avoid confusion with
67 * RAM_SSAVE_FLAG_COMPRESS_PAGE just rename it.
70 #define RAM_SAVE_FLAG_FULL 0x01 /* Obsolete, not used anymore */
71 #define RAM_SAVE_FLAG_ZERO 0x02
72 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
73 #define RAM_SAVE_FLAG_PAGE 0x08
74 #define RAM_SAVE_FLAG_EOS 0x10
75 #define RAM_SAVE_FLAG_CONTINUE 0x20
76 #define RAM_SAVE_FLAG_XBZRLE 0x40
77 /* 0x80 is reserved in migration.h start with 0x100 next */
78 #define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
80 static inline bool is_zero_range(uint8_t *p
, uint64_t size
)
82 return buffer_is_zero(p
, size
);
85 XBZRLECacheStats xbzrle_counters
;
87 /* struct contains XBZRLE cache and a static page
88 used by the compression */
90 /* buffer used for XBZRLE encoding */
92 /* buffer for storing page content */
94 /* Cache for XBZRLE, Protected by lock. */
97 /* it will store a page full of zeros */
98 uint8_t *zero_target_page
;
99 /* buffer used for XBZRLE decoding */
100 uint8_t *decoded_buf
;
103 static void XBZRLE_cache_lock(void)
105 if (migrate_use_xbzrle())
106 qemu_mutex_lock(&XBZRLE
.lock
);
109 static void XBZRLE_cache_unlock(void)
111 if (migrate_use_xbzrle())
112 qemu_mutex_unlock(&XBZRLE
.lock
);
116 * xbzrle_cache_resize: resize the xbzrle cache
118 * This function is called from qmp_migrate_set_cache_size in main
119 * thread, possibly while a migration is in progress. A running
120 * migration may be using the cache and might finish during this call,
121 * hence changes to the cache are protected by XBZRLE.lock().
123 * Returns 0 for success or -1 for error
125 * @new_size: new cache size
126 * @errp: set *errp if the check failed, with reason
128 int xbzrle_cache_resize(int64_t new_size
, Error
**errp
)
130 PageCache
*new_cache
;
133 /* Check for truncation */
134 if (new_size
!= (size_t)new_size
) {
135 error_setg(errp
, QERR_INVALID_PARAMETER_VALUE
, "cache size",
136 "exceeding address space");
140 if (new_size
== migrate_xbzrle_cache_size()) {
147 if (XBZRLE
.cache
!= NULL
) {
148 new_cache
= cache_init(new_size
, TARGET_PAGE_SIZE
, errp
);
154 cache_fini(XBZRLE
.cache
);
155 XBZRLE
.cache
= new_cache
;
158 XBZRLE_cache_unlock();
162 static bool ramblock_is_ignored(RAMBlock
*block
)
164 return !qemu_ram_is_migratable(block
) ||
165 (migrate_ignore_shared() && qemu_ram_is_shared(block
));
168 /* Should be holding either ram_list.mutex, or the RCU lock. */
169 #define RAMBLOCK_FOREACH_NOT_IGNORED(block) \
170 INTERNAL_RAMBLOCK_FOREACH(block) \
171 if (ramblock_is_ignored(block)) {} else
173 #define RAMBLOCK_FOREACH_MIGRATABLE(block) \
174 INTERNAL_RAMBLOCK_FOREACH(block) \
175 if (!qemu_ram_is_migratable(block)) {} else
177 #undef RAMBLOCK_FOREACH
179 int foreach_not_ignored_block(RAMBlockIterFunc func
, void *opaque
)
185 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
186 ret
= func(block
, opaque
);
195 static void ramblock_recv_map_init(void)
199 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
200 assert(!rb
->receivedmap
);
201 rb
->receivedmap
= bitmap_new(rb
->max_length
>> qemu_target_page_bits());
205 int ramblock_recv_bitmap_test(RAMBlock
*rb
, void *host_addr
)
207 return test_bit(ramblock_recv_bitmap_offset(host_addr
, rb
),
211 bool ramblock_recv_bitmap_test_byte_offset(RAMBlock
*rb
, uint64_t byte_offset
)
213 return test_bit(byte_offset
>> TARGET_PAGE_BITS
, rb
->receivedmap
);
216 void ramblock_recv_bitmap_set(RAMBlock
*rb
, void *host_addr
)
218 set_bit_atomic(ramblock_recv_bitmap_offset(host_addr
, rb
), rb
->receivedmap
);
221 void ramblock_recv_bitmap_set_range(RAMBlock
*rb
, void *host_addr
,
224 bitmap_set_atomic(rb
->receivedmap
,
225 ramblock_recv_bitmap_offset(host_addr
, rb
),
229 #define RAMBLOCK_RECV_BITMAP_ENDING (0x0123456789abcdefULL)
232 * Format: bitmap_size (8 bytes) + whole_bitmap (N bytes).
234 * Returns >0 if success with sent bytes, or <0 if error.
236 int64_t ramblock_recv_bitmap_send(QEMUFile
*file
,
237 const char *block_name
)
239 RAMBlock
*block
= qemu_ram_block_by_name(block_name
);
240 unsigned long *le_bitmap
, nbits
;
244 error_report("%s: invalid block name: %s", __func__
, block_name
);
248 nbits
= block
->used_length
>> TARGET_PAGE_BITS
;
251 * Make sure the tmp bitmap buffer is big enough, e.g., on 32bit
252 * machines we may need 4 more bytes for padding (see below
253 * comment). So extend it a bit before hand.
255 le_bitmap
= bitmap_new(nbits
+ BITS_PER_LONG
);
258 * Always use little endian when sending the bitmap. This is
259 * required that when source and destination VMs are not using the
260 * same endianess. (Note: big endian won't work.)
262 bitmap_to_le(le_bitmap
, block
->receivedmap
, nbits
);
264 /* Size of the bitmap, in bytes */
265 size
= DIV_ROUND_UP(nbits
, 8);
268 * size is always aligned to 8 bytes for 64bit machines, but it
269 * may not be true for 32bit machines. We need this padding to
270 * make sure the migration can survive even between 32bit and
273 size
= ROUND_UP(size
, 8);
275 qemu_put_be64(file
, size
);
276 qemu_put_buffer(file
, (const uint8_t *)le_bitmap
, size
);
278 * Mark as an end, in case the middle part is screwed up due to
279 * some "misterious" reason.
281 qemu_put_be64(file
, RAMBLOCK_RECV_BITMAP_ENDING
);
286 if (qemu_file_get_error(file
)) {
287 return qemu_file_get_error(file
);
290 return size
+ sizeof(size
);
294 * An outstanding page request, on the source, having been received
297 struct RAMSrcPageRequest
{
302 QSIMPLEQ_ENTRY(RAMSrcPageRequest
) next_req
;
305 /* State of RAM for migration */
307 /* QEMUFile used for this migration */
309 /* Last block that we have visited searching for dirty pages */
310 RAMBlock
*last_seen_block
;
311 /* Last block from where we have sent data */
312 RAMBlock
*last_sent_block
;
313 /* Last dirty target page we have sent */
314 ram_addr_t last_page
;
315 /* last ram version we have seen */
316 uint32_t last_version
;
317 /* We are in the first round */
319 /* The free page optimization is enabled */
321 /* How many times we have dirty too many pages */
322 int dirty_rate_high_cnt
;
323 /* these variables are used for bitmap sync */
324 /* last time we did a full bitmap_sync */
325 int64_t time_last_bitmap_sync
;
326 /* bytes transferred at start_time */
327 uint64_t bytes_xfer_prev
;
328 /* number of dirty pages since start_time */
329 uint64_t num_dirty_pages_period
;
330 /* xbzrle misses since the beginning of the period */
331 uint64_t xbzrle_cache_miss_prev
;
333 /* compression statistics since the beginning of the period */
334 /* amount of count that no free thread to compress data */
335 uint64_t compress_thread_busy_prev
;
336 /* amount bytes after compression */
337 uint64_t compressed_size_prev
;
338 /* amount of compressed pages */
339 uint64_t compress_pages_prev
;
341 /* total handled target pages at the beginning of period */
342 uint64_t target_page_count_prev
;
343 /* total handled target pages since start */
344 uint64_t target_page_count
;
345 /* number of dirty bits in the bitmap */
346 uint64_t migration_dirty_pages
;
347 /* Protects modification of the bitmap and migration dirty pages */
348 QemuMutex bitmap_mutex
;
349 /* The RAMBlock used in the last src_page_requests */
350 RAMBlock
*last_req_rb
;
351 /* Queue of outstanding page requests from the destination */
352 QemuMutex src_page_req_mutex
;
353 QSIMPLEQ_HEAD(, RAMSrcPageRequest
) src_page_requests
;
355 typedef struct RAMState RAMState
;
357 static RAMState
*ram_state
;
359 static NotifierWithReturnList precopy_notifier_list
;
361 void precopy_infrastructure_init(void)
363 notifier_with_return_list_init(&precopy_notifier_list
);
366 void precopy_add_notifier(NotifierWithReturn
*n
)
368 notifier_with_return_list_add(&precopy_notifier_list
, n
);
371 void precopy_remove_notifier(NotifierWithReturn
*n
)
373 notifier_with_return_remove(n
);
376 int precopy_notify(PrecopyNotifyReason reason
, Error
**errp
)
378 PrecopyNotifyData pnd
;
382 return notifier_with_return_list_notify(&precopy_notifier_list
, &pnd
);
385 void precopy_enable_free_page_optimization(void)
391 ram_state
->fpo_enabled
= true;
394 uint64_t ram_bytes_remaining(void)
396 return ram_state
? (ram_state
->migration_dirty_pages
* TARGET_PAGE_SIZE
) :
400 MigrationStats ram_counters
;
402 /* used by the search for pages to send */
403 struct PageSearchStatus
{
404 /* Current block being searched */
406 /* Current page to search from */
408 /* Set once we wrap around */
411 typedef struct PageSearchStatus PageSearchStatus
;
413 CompressionStats compression_counters
;
415 struct CompressParam
{
425 /* internally used fields */
429 typedef struct CompressParam CompressParam
;
431 struct DecompressParam
{
441 typedef struct DecompressParam DecompressParam
;
443 static CompressParam
*comp_param
;
444 static QemuThread
*compress_threads
;
445 /* comp_done_cond is used to wake up the migration thread when
446 * one of the compression threads has finished the compression.
447 * comp_done_lock is used to co-work with comp_done_cond.
449 static QemuMutex comp_done_lock
;
450 static QemuCond comp_done_cond
;
451 /* The empty QEMUFileOps will be used by file in CompressParam */
452 static const QEMUFileOps empty_ops
= { };
454 static QEMUFile
*decomp_file
;
455 static DecompressParam
*decomp_param
;
456 static QemuThread
*decompress_threads
;
457 static QemuMutex decomp_done_lock
;
458 static QemuCond decomp_done_cond
;
460 static bool do_compress_ram_page(QEMUFile
*f
, z_stream
*stream
, RAMBlock
*block
,
461 ram_addr_t offset
, uint8_t *source_buf
);
463 static void *do_data_compress(void *opaque
)
465 CompressParam
*param
= opaque
;
470 qemu_mutex_lock(¶m
->mutex
);
471 while (!param
->quit
) {
473 block
= param
->block
;
474 offset
= param
->offset
;
476 qemu_mutex_unlock(¶m
->mutex
);
478 zero_page
= do_compress_ram_page(param
->file
, ¶m
->stream
,
479 block
, offset
, param
->originbuf
);
481 qemu_mutex_lock(&comp_done_lock
);
483 param
->zero_page
= zero_page
;
484 qemu_cond_signal(&comp_done_cond
);
485 qemu_mutex_unlock(&comp_done_lock
);
487 qemu_mutex_lock(¶m
->mutex
);
489 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
492 qemu_mutex_unlock(¶m
->mutex
);
497 static void compress_threads_save_cleanup(void)
501 if (!migrate_use_compression() || !comp_param
) {
505 thread_count
= migrate_compress_threads();
506 for (i
= 0; i
< thread_count
; i
++) {
508 * we use it as a indicator which shows if the thread is
509 * properly init'd or not
511 if (!comp_param
[i
].file
) {
515 qemu_mutex_lock(&comp_param
[i
].mutex
);
516 comp_param
[i
].quit
= true;
517 qemu_cond_signal(&comp_param
[i
].cond
);
518 qemu_mutex_unlock(&comp_param
[i
].mutex
);
520 qemu_thread_join(compress_threads
+ i
);
521 qemu_mutex_destroy(&comp_param
[i
].mutex
);
522 qemu_cond_destroy(&comp_param
[i
].cond
);
523 deflateEnd(&comp_param
[i
].stream
);
524 g_free(comp_param
[i
].originbuf
);
525 qemu_fclose(comp_param
[i
].file
);
526 comp_param
[i
].file
= NULL
;
528 qemu_mutex_destroy(&comp_done_lock
);
529 qemu_cond_destroy(&comp_done_cond
);
530 g_free(compress_threads
);
532 compress_threads
= NULL
;
536 static int compress_threads_save_setup(void)
540 if (!migrate_use_compression()) {
543 thread_count
= migrate_compress_threads();
544 compress_threads
= g_new0(QemuThread
, thread_count
);
545 comp_param
= g_new0(CompressParam
, thread_count
);
546 qemu_cond_init(&comp_done_cond
);
547 qemu_mutex_init(&comp_done_lock
);
548 for (i
= 0; i
< thread_count
; i
++) {
549 comp_param
[i
].originbuf
= g_try_malloc(TARGET_PAGE_SIZE
);
550 if (!comp_param
[i
].originbuf
) {
554 if (deflateInit(&comp_param
[i
].stream
,
555 migrate_compress_level()) != Z_OK
) {
556 g_free(comp_param
[i
].originbuf
);
560 /* comp_param[i].file is just used as a dummy buffer to save data,
561 * set its ops to empty.
563 comp_param
[i
].file
= qemu_fopen_ops(NULL
, &empty_ops
);
564 comp_param
[i
].done
= true;
565 comp_param
[i
].quit
= false;
566 qemu_mutex_init(&comp_param
[i
].mutex
);
567 qemu_cond_init(&comp_param
[i
].cond
);
568 qemu_thread_create(compress_threads
+ i
, "compress",
569 do_data_compress
, comp_param
+ i
,
570 QEMU_THREAD_JOINABLE
);
575 compress_threads_save_cleanup();
581 #define MULTIFD_MAGIC 0x11223344U
582 #define MULTIFD_VERSION 1
584 #define MULTIFD_FLAG_SYNC (1 << 0)
586 /* This value needs to be a multiple of qemu_target_page_size() */
587 #define MULTIFD_PACKET_SIZE (512 * 1024)
592 unsigned char uuid
[16]; /* QemuUUID */
594 uint8_t unused1
[7]; /* Reserved for future use */
595 uint64_t unused2
[4]; /* Reserved for future use */
596 } __attribute__((packed
)) MultiFDInit_t
;
602 /* maximum number of allocated pages */
603 uint32_t pages_alloc
;
605 /* size of the next packet that contains pages */
606 uint32_t next_packet_size
;
608 uint64_t unused
[4]; /* Reserved for future use */
611 } __attribute__((packed
)) MultiFDPacket_t
;
614 /* number of used pages */
616 /* number of allocated pages */
618 /* global number of generated multifd packets */
620 /* offset of each page */
622 /* pointer to each page */
628 /* this fields are not changed once the thread is created */
631 /* channel thread name */
633 /* channel thread id */
635 /* communication channel */
637 /* sem where to wait for more work */
639 /* this mutex protects the following parameters */
641 /* is this channel thread running */
643 /* should this thread finish */
645 /* thread has work to do */
647 /* array of pages to sent */
648 MultiFDPages_t
*pages
;
649 /* packet allocated len */
651 /* pointer to the packet */
652 MultiFDPacket_t
*packet
;
653 /* multifd flags for each packet */
655 /* size of the next packet that contains pages */
656 uint32_t next_packet_size
;
657 /* global number of generated multifd packets */
659 /* thread local variables */
660 /* packets sent through this channel */
661 uint64_t num_packets
;
662 /* pages sent through this channel */
667 /* this fields are not changed once the thread is created */
670 /* channel thread name */
672 /* channel thread id */
674 /* communication channel */
676 /* this mutex protects the following parameters */
678 /* is this channel thread running */
680 /* array of pages to receive */
681 MultiFDPages_t
*pages
;
682 /* packet allocated len */
684 /* pointer to the packet */
685 MultiFDPacket_t
*packet
;
686 /* multifd flags for each packet */
688 /* global number of generated multifd packets */
690 /* thread local variables */
691 /* size of the next packet that contains pages */
692 uint32_t next_packet_size
;
693 /* packets sent through this channel */
694 uint64_t num_packets
;
695 /* pages sent through this channel */
697 /* syncs main thread and channels */
698 QemuSemaphore sem_sync
;
701 static int multifd_send_initial_packet(MultiFDSendParams
*p
, Error
**errp
)
706 msg
.magic
= cpu_to_be32(MULTIFD_MAGIC
);
707 msg
.version
= cpu_to_be32(MULTIFD_VERSION
);
709 memcpy(msg
.uuid
, &qemu_uuid
.data
, sizeof(msg
.uuid
));
711 ret
= qio_channel_write_all(p
->c
, (char *)&msg
, sizeof(msg
), errp
);
718 static int multifd_recv_initial_packet(QIOChannel
*c
, Error
**errp
)
723 ret
= qio_channel_read_all(c
, (char *)&msg
, sizeof(msg
), errp
);
728 msg
.magic
= be32_to_cpu(msg
.magic
);
729 msg
.version
= be32_to_cpu(msg
.version
);
731 if (msg
.magic
!= MULTIFD_MAGIC
) {
732 error_setg(errp
, "multifd: received packet magic %x "
733 "expected %x", msg
.magic
, MULTIFD_MAGIC
);
737 if (msg
.version
!= MULTIFD_VERSION
) {
738 error_setg(errp
, "multifd: received packet version %d "
739 "expected %d", msg
.version
, MULTIFD_VERSION
);
743 if (memcmp(msg
.uuid
, &qemu_uuid
, sizeof(qemu_uuid
))) {
744 char *uuid
= qemu_uuid_unparse_strdup(&qemu_uuid
);
745 char *msg_uuid
= qemu_uuid_unparse_strdup((const QemuUUID
*)msg
.uuid
);
747 error_setg(errp
, "multifd: received uuid '%s' and expected "
748 "uuid '%s' for channel %hhd", msg_uuid
, uuid
, msg
.id
);
754 if (msg
.id
> migrate_multifd_channels()) {
755 error_setg(errp
, "multifd: received channel version %d "
756 "expected %d", msg
.version
, MULTIFD_VERSION
);
763 static MultiFDPages_t
*multifd_pages_init(size_t size
)
765 MultiFDPages_t
*pages
= g_new0(MultiFDPages_t
, 1);
767 pages
->allocated
= size
;
768 pages
->iov
= g_new0(struct iovec
, size
);
769 pages
->offset
= g_new0(ram_addr_t
, size
);
774 static void multifd_pages_clear(MultiFDPages_t
*pages
)
777 pages
->allocated
= 0;
778 pages
->packet_num
= 0;
782 g_free(pages
->offset
);
783 pages
->offset
= NULL
;
787 static void multifd_send_fill_packet(MultiFDSendParams
*p
)
789 MultiFDPacket_t
*packet
= p
->packet
;
790 uint32_t page_max
= MULTIFD_PACKET_SIZE
/ qemu_target_page_size();
793 packet
->magic
= cpu_to_be32(MULTIFD_MAGIC
);
794 packet
->version
= cpu_to_be32(MULTIFD_VERSION
);
795 packet
->flags
= cpu_to_be32(p
->flags
);
796 packet
->pages_alloc
= cpu_to_be32(page_max
);
797 packet
->pages_used
= cpu_to_be32(p
->pages
->used
);
798 packet
->next_packet_size
= cpu_to_be32(p
->next_packet_size
);
799 packet
->packet_num
= cpu_to_be64(p
->packet_num
);
801 if (p
->pages
->block
) {
802 strncpy(packet
->ramblock
, p
->pages
->block
->idstr
, 256);
805 for (i
= 0; i
< p
->pages
->used
; i
++) {
806 packet
->offset
[i
] = cpu_to_be64(p
->pages
->offset
[i
]);
810 static int multifd_recv_unfill_packet(MultiFDRecvParams
*p
, Error
**errp
)
812 MultiFDPacket_t
*packet
= p
->packet
;
813 uint32_t pages_max
= MULTIFD_PACKET_SIZE
/ qemu_target_page_size();
817 packet
->magic
= be32_to_cpu(packet
->magic
);
818 if (packet
->magic
!= MULTIFD_MAGIC
) {
819 error_setg(errp
, "multifd: received packet "
820 "magic %x and expected magic %x",
821 packet
->magic
, MULTIFD_MAGIC
);
825 packet
->version
= be32_to_cpu(packet
->version
);
826 if (packet
->version
!= MULTIFD_VERSION
) {
827 error_setg(errp
, "multifd: received packet "
828 "version %d and expected version %d",
829 packet
->version
, MULTIFD_VERSION
);
833 p
->flags
= be32_to_cpu(packet
->flags
);
835 packet
->pages_alloc
= be32_to_cpu(packet
->pages_alloc
);
837 * If we recevied a packet that is 100 times bigger than expected
838 * just stop migration. It is a magic number.
840 if (packet
->pages_alloc
> pages_max
* 100) {
841 error_setg(errp
, "multifd: received packet "
842 "with size %d and expected a maximum size of %d",
843 packet
->pages_alloc
, pages_max
* 100) ;
847 * We received a packet that is bigger than expected but inside
848 * reasonable limits (see previous comment). Just reallocate.
850 if (packet
->pages_alloc
> p
->pages
->allocated
) {
851 multifd_pages_clear(p
->pages
);
852 p
->pages
= multifd_pages_init(packet
->pages_alloc
);
855 p
->pages
->used
= be32_to_cpu(packet
->pages_used
);
856 if (p
->pages
->used
> packet
->pages_alloc
) {
857 error_setg(errp
, "multifd: received packet "
858 "with %d pages and expected maximum pages are %d",
859 p
->pages
->used
, packet
->pages_alloc
) ;
863 p
->next_packet_size
= be32_to_cpu(packet
->next_packet_size
);
864 p
->packet_num
= be64_to_cpu(packet
->packet_num
);
866 if (p
->pages
->used
) {
867 /* make sure that ramblock is 0 terminated */
868 packet
->ramblock
[255] = 0;
869 block
= qemu_ram_block_by_name(packet
->ramblock
);
871 error_setg(errp
, "multifd: unknown ram block %s",
877 for (i
= 0; i
< p
->pages
->used
; i
++) {
878 ram_addr_t offset
= be64_to_cpu(packet
->offset
[i
]);
880 if (offset
> (block
->used_length
- TARGET_PAGE_SIZE
)) {
881 error_setg(errp
, "multifd: offset too long " RAM_ADDR_FMT
882 " (max " RAM_ADDR_FMT
")",
883 offset
, block
->max_length
);
886 p
->pages
->iov
[i
].iov_base
= block
->host
+ offset
;
887 p
->pages
->iov
[i
].iov_len
= TARGET_PAGE_SIZE
;
894 MultiFDSendParams
*params
;
895 /* array of pages to sent */
896 MultiFDPages_t
*pages
;
897 /* syncs main thread and channels */
898 QemuSemaphore sem_sync
;
899 /* global number of generated multifd packets */
901 /* send channels ready */
902 QemuSemaphore channels_ready
;
903 } *multifd_send_state
;
906 * How we use multifd_send_state->pages and channel->pages?
908 * We create a pages for each channel, and a main one. Each time that
909 * we need to send a batch of pages we interchange the ones between
910 * multifd_send_state and the channel that is sending it. There are
911 * two reasons for that:
912 * - to not have to do so many mallocs during migration
913 * - to make easier to know what to free at the end of migration
915 * This way we always know who is the owner of each "pages" struct,
916 * and we don't need any locking. It belongs to the migration thread
917 * or to the channel thread. Switching is safe because the migration
918 * thread is using the channel mutex when changing it, and the channel
919 * have to had finish with its own, otherwise pending_job can't be
923 static void multifd_send_pages(void)
926 static int next_channel
;
927 MultiFDSendParams
*p
= NULL
; /* make happy gcc */
928 MultiFDPages_t
*pages
= multifd_send_state
->pages
;
929 uint64_t transferred
;
931 qemu_sem_wait(&multifd_send_state
->channels_ready
);
932 for (i
= next_channel
;; i
= (i
+ 1) % migrate_multifd_channels()) {
933 p
= &multifd_send_state
->params
[i
];
935 qemu_mutex_lock(&p
->mutex
);
936 if (!p
->pending_job
) {
938 next_channel
= (i
+ 1) % migrate_multifd_channels();
941 qemu_mutex_unlock(&p
->mutex
);
945 p
->packet_num
= multifd_send_state
->packet_num
++;
946 p
->pages
->block
= NULL
;
947 multifd_send_state
->pages
= p
->pages
;
949 transferred
= ((uint64_t) pages
->used
) * TARGET_PAGE_SIZE
+ p
->packet_len
;
950 ram_counters
.multifd_bytes
+= transferred
;
951 ram_counters
.transferred
+= transferred
;;
952 qemu_mutex_unlock(&p
->mutex
);
953 qemu_sem_post(&p
->sem
);
956 static void multifd_queue_page(RAMBlock
*block
, ram_addr_t offset
)
958 MultiFDPages_t
*pages
= multifd_send_state
->pages
;
961 pages
->block
= block
;
964 if (pages
->block
== block
) {
965 pages
->offset
[pages
->used
] = offset
;
966 pages
->iov
[pages
->used
].iov_base
= block
->host
+ offset
;
967 pages
->iov
[pages
->used
].iov_len
= TARGET_PAGE_SIZE
;
970 if (pages
->used
< pages
->allocated
) {
975 multifd_send_pages();
977 if (pages
->block
!= block
) {
978 multifd_queue_page(block
, offset
);
982 static void multifd_send_terminate_threads(Error
*err
)
987 MigrationState
*s
= migrate_get_current();
988 migrate_set_error(s
, err
);
989 if (s
->state
== MIGRATION_STATUS_SETUP
||
990 s
->state
== MIGRATION_STATUS_PRE_SWITCHOVER
||
991 s
->state
== MIGRATION_STATUS_DEVICE
||
992 s
->state
== MIGRATION_STATUS_ACTIVE
) {
993 migrate_set_state(&s
->state
, s
->state
,
994 MIGRATION_STATUS_FAILED
);
998 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
999 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1001 qemu_mutex_lock(&p
->mutex
);
1003 qemu_sem_post(&p
->sem
);
1004 qemu_mutex_unlock(&p
->mutex
);
1008 void multifd_save_cleanup(void)
1012 if (!migrate_use_multifd()) {
1015 multifd_send_terminate_threads(NULL
);
1016 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1017 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1020 qemu_thread_join(&p
->thread
);
1022 socket_send_channel_destroy(p
->c
);
1024 qemu_mutex_destroy(&p
->mutex
);
1025 qemu_sem_destroy(&p
->sem
);
1028 multifd_pages_clear(p
->pages
);
1034 qemu_sem_destroy(&multifd_send_state
->channels_ready
);
1035 qemu_sem_destroy(&multifd_send_state
->sem_sync
);
1036 g_free(multifd_send_state
->params
);
1037 multifd_send_state
->params
= NULL
;
1038 multifd_pages_clear(multifd_send_state
->pages
);
1039 multifd_send_state
->pages
= NULL
;
1040 g_free(multifd_send_state
);
1041 multifd_send_state
= NULL
;
1044 static void multifd_send_sync_main(void)
1048 if (!migrate_use_multifd()) {
1051 if (multifd_send_state
->pages
->used
) {
1052 multifd_send_pages();
1054 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1055 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1057 trace_multifd_send_sync_main_signal(p
->id
);
1059 qemu_mutex_lock(&p
->mutex
);
1061 p
->packet_num
= multifd_send_state
->packet_num
++;
1062 p
->flags
|= MULTIFD_FLAG_SYNC
;
1064 qemu_mutex_unlock(&p
->mutex
);
1065 qemu_sem_post(&p
->sem
);
1067 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1068 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1070 trace_multifd_send_sync_main_wait(p
->id
);
1071 qemu_sem_wait(&multifd_send_state
->sem_sync
);
1073 trace_multifd_send_sync_main(multifd_send_state
->packet_num
);
1076 static void *multifd_send_thread(void *opaque
)
1078 MultiFDSendParams
*p
= opaque
;
1079 Error
*local_err
= NULL
;
1082 trace_multifd_send_thread_start(p
->id
);
1083 rcu_register_thread();
1085 if (multifd_send_initial_packet(p
, &local_err
) < 0) {
1088 /* initial packet */
1092 qemu_sem_wait(&p
->sem
);
1093 qemu_mutex_lock(&p
->mutex
);
1095 if (p
->pending_job
) {
1096 uint32_t used
= p
->pages
->used
;
1097 uint64_t packet_num
= p
->packet_num
;
1098 uint32_t flags
= p
->flags
;
1100 p
->next_packet_size
= used
* qemu_target_page_size();
1101 multifd_send_fill_packet(p
);
1104 p
->num_pages
+= used
;
1106 qemu_mutex_unlock(&p
->mutex
);
1108 trace_multifd_send(p
->id
, packet_num
, used
, flags
,
1109 p
->next_packet_size
);
1111 ret
= qio_channel_write_all(p
->c
, (void *)p
->packet
,
1112 p
->packet_len
, &local_err
);
1118 ret
= qio_channel_writev_all(p
->c
, p
->pages
->iov
,
1125 qemu_mutex_lock(&p
->mutex
);
1127 qemu_mutex_unlock(&p
->mutex
);
1129 if (flags
& MULTIFD_FLAG_SYNC
) {
1130 qemu_sem_post(&multifd_send_state
->sem_sync
);
1132 qemu_sem_post(&multifd_send_state
->channels_ready
);
1133 } else if (p
->quit
) {
1134 qemu_mutex_unlock(&p
->mutex
);
1137 qemu_mutex_unlock(&p
->mutex
);
1138 /* sometimes there are spurious wakeups */
1144 multifd_send_terminate_threads(local_err
);
1147 qemu_mutex_lock(&p
->mutex
);
1149 qemu_mutex_unlock(&p
->mutex
);
1151 rcu_unregister_thread();
1152 trace_multifd_send_thread_end(p
->id
, p
->num_packets
, p
->num_pages
);
1157 static void multifd_new_send_channel_async(QIOTask
*task
, gpointer opaque
)
1159 MultiFDSendParams
*p
= opaque
;
1160 QIOChannel
*sioc
= QIO_CHANNEL(qio_task_get_source(task
));
1161 Error
*local_err
= NULL
;
1163 if (qio_task_propagate_error(task
, &local_err
)) {
1164 migrate_set_error(migrate_get_current(), local_err
);
1165 multifd_save_cleanup();
1167 p
->c
= QIO_CHANNEL(sioc
);
1168 qio_channel_set_delay(p
->c
, false);
1170 qemu_thread_create(&p
->thread
, p
->name
, multifd_send_thread
, p
,
1171 QEMU_THREAD_JOINABLE
);
1175 int multifd_save_setup(void)
1178 uint32_t page_count
= MULTIFD_PACKET_SIZE
/ qemu_target_page_size();
1181 if (!migrate_use_multifd()) {
1184 thread_count
= migrate_multifd_channels();
1185 multifd_send_state
= g_malloc0(sizeof(*multifd_send_state
));
1186 multifd_send_state
->params
= g_new0(MultiFDSendParams
, thread_count
);
1187 multifd_send_state
->pages
= multifd_pages_init(page_count
);
1188 qemu_sem_init(&multifd_send_state
->sem_sync
, 0);
1189 qemu_sem_init(&multifd_send_state
->channels_ready
, 0);
1191 for (i
= 0; i
< thread_count
; i
++) {
1192 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1194 qemu_mutex_init(&p
->mutex
);
1195 qemu_sem_init(&p
->sem
, 0);
1199 p
->pages
= multifd_pages_init(page_count
);
1200 p
->packet_len
= sizeof(MultiFDPacket_t
)
1201 + sizeof(ram_addr_t
) * page_count
;
1202 p
->packet
= g_malloc0(p
->packet_len
);
1203 p
->name
= g_strdup_printf("multifdsend_%d", i
);
1204 socket_send_channel_create(multifd_new_send_channel_async
, p
);
1210 MultiFDRecvParams
*params
;
1211 /* number of created threads */
1213 /* syncs main thread and channels */
1214 QemuSemaphore sem_sync
;
1215 /* global number of generated multifd packets */
1216 uint64_t packet_num
;
1217 } *multifd_recv_state
;
1219 static void multifd_recv_terminate_threads(Error
*err
)
1224 MigrationState
*s
= migrate_get_current();
1225 migrate_set_error(s
, err
);
1226 if (s
->state
== MIGRATION_STATUS_SETUP
||
1227 s
->state
== MIGRATION_STATUS_ACTIVE
) {
1228 migrate_set_state(&s
->state
, s
->state
,
1229 MIGRATION_STATUS_FAILED
);
1233 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1234 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1236 qemu_mutex_lock(&p
->mutex
);
1237 /* We could arrive here for two reasons:
1238 - normal quit, i.e. everything went fine, just finished
1239 - error quit: We close the channels so the channel threads
1240 finish the qio_channel_read_all_eof() */
1241 qio_channel_shutdown(p
->c
, QIO_CHANNEL_SHUTDOWN_BOTH
, NULL
);
1242 qemu_mutex_unlock(&p
->mutex
);
1246 int multifd_load_cleanup(Error
**errp
)
1251 if (!migrate_use_multifd()) {
1254 multifd_recv_terminate_threads(NULL
);
1255 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1256 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1259 qemu_thread_join(&p
->thread
);
1261 object_unref(OBJECT(p
->c
));
1263 qemu_mutex_destroy(&p
->mutex
);
1264 qemu_sem_destroy(&p
->sem_sync
);
1267 multifd_pages_clear(p
->pages
);
1273 qemu_sem_destroy(&multifd_recv_state
->sem_sync
);
1274 g_free(multifd_recv_state
->params
);
1275 multifd_recv_state
->params
= NULL
;
1276 g_free(multifd_recv_state
);
1277 multifd_recv_state
= NULL
;
1282 static void multifd_recv_sync_main(void)
1286 if (!migrate_use_multifd()) {
1289 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1290 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1292 trace_multifd_recv_sync_main_wait(p
->id
);
1293 qemu_sem_wait(&multifd_recv_state
->sem_sync
);
1294 qemu_mutex_lock(&p
->mutex
);
1295 if (multifd_recv_state
->packet_num
< p
->packet_num
) {
1296 multifd_recv_state
->packet_num
= p
->packet_num
;
1298 qemu_mutex_unlock(&p
->mutex
);
1300 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1301 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1303 trace_multifd_recv_sync_main_signal(p
->id
);
1304 qemu_sem_post(&p
->sem_sync
);
1306 trace_multifd_recv_sync_main(multifd_recv_state
->packet_num
);
1309 static void *multifd_recv_thread(void *opaque
)
1311 MultiFDRecvParams
*p
= opaque
;
1312 Error
*local_err
= NULL
;
1315 trace_multifd_recv_thread_start(p
->id
);
1316 rcu_register_thread();
1322 ret
= qio_channel_read_all_eof(p
->c
, (void *)p
->packet
,
1323 p
->packet_len
, &local_err
);
1324 if (ret
== 0) { /* EOF */
1327 if (ret
== -1) { /* Error */
1331 qemu_mutex_lock(&p
->mutex
);
1332 ret
= multifd_recv_unfill_packet(p
, &local_err
);
1334 qemu_mutex_unlock(&p
->mutex
);
1338 used
= p
->pages
->used
;
1340 trace_multifd_recv(p
->id
, p
->packet_num
, used
, flags
,
1341 p
->next_packet_size
);
1343 p
->num_pages
+= used
;
1344 qemu_mutex_unlock(&p
->mutex
);
1347 ret
= qio_channel_readv_all(p
->c
, p
->pages
->iov
,
1354 if (flags
& MULTIFD_FLAG_SYNC
) {
1355 qemu_sem_post(&multifd_recv_state
->sem_sync
);
1356 qemu_sem_wait(&p
->sem_sync
);
1361 multifd_recv_terminate_threads(local_err
);
1363 qemu_mutex_lock(&p
->mutex
);
1365 qemu_mutex_unlock(&p
->mutex
);
1367 rcu_unregister_thread();
1368 trace_multifd_recv_thread_end(p
->id
, p
->num_packets
, p
->num_pages
);
1373 int multifd_load_setup(void)
1376 uint32_t page_count
= MULTIFD_PACKET_SIZE
/ qemu_target_page_size();
1379 if (!migrate_use_multifd()) {
1382 thread_count
= migrate_multifd_channels();
1383 multifd_recv_state
= g_malloc0(sizeof(*multifd_recv_state
));
1384 multifd_recv_state
->params
= g_new0(MultiFDRecvParams
, thread_count
);
1385 atomic_set(&multifd_recv_state
->count
, 0);
1386 qemu_sem_init(&multifd_recv_state
->sem_sync
, 0);
1388 for (i
= 0; i
< thread_count
; i
++) {
1389 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1391 qemu_mutex_init(&p
->mutex
);
1392 qemu_sem_init(&p
->sem_sync
, 0);
1394 p
->pages
= multifd_pages_init(page_count
);
1395 p
->packet_len
= sizeof(MultiFDPacket_t
)
1396 + sizeof(ram_addr_t
) * page_count
;
1397 p
->packet
= g_malloc0(p
->packet_len
);
1398 p
->name
= g_strdup_printf("multifdrecv_%d", i
);
1403 bool multifd_recv_all_channels_created(void)
1405 int thread_count
= migrate_multifd_channels();
1407 if (!migrate_use_multifd()) {
1411 return thread_count
== atomic_read(&multifd_recv_state
->count
);
1415 * Try to receive all multifd channels to get ready for the migration.
1416 * - Return true and do not set @errp when correctly receving all channels;
1417 * - Return false and do not set @errp when correctly receiving the current one;
1418 * - Return false and set @errp when failing to receive the current channel.
1420 bool multifd_recv_new_channel(QIOChannel
*ioc
, Error
**errp
)
1422 MultiFDRecvParams
*p
;
1423 Error
*local_err
= NULL
;
1426 id
= multifd_recv_initial_packet(ioc
, &local_err
);
1428 multifd_recv_terminate_threads(local_err
);
1429 error_propagate_prepend(errp
, local_err
,
1430 "failed to receive packet"
1431 " via multifd channel %d: ",
1432 atomic_read(&multifd_recv_state
->count
));
1436 p
= &multifd_recv_state
->params
[id
];
1438 error_setg(&local_err
, "multifd: received id '%d' already setup'",
1440 multifd_recv_terminate_threads(local_err
);
1441 error_propagate(errp
, local_err
);
1445 object_ref(OBJECT(ioc
));
1446 /* initial packet */
1450 qemu_thread_create(&p
->thread
, p
->name
, multifd_recv_thread
, p
,
1451 QEMU_THREAD_JOINABLE
);
1452 atomic_inc(&multifd_recv_state
->count
);
1453 return atomic_read(&multifd_recv_state
->count
) ==
1454 migrate_multifd_channels();
1458 * save_page_header: write page header to wire
1460 * If this is the 1st block, it also writes the block identification
1462 * Returns the number of bytes written
1464 * @f: QEMUFile where to send the data
1465 * @block: block that contains the page we want to send
1466 * @offset: offset inside the block for the page
1467 * in the lower bits, it contains flags
1469 static size_t save_page_header(RAMState
*rs
, QEMUFile
*f
, RAMBlock
*block
,
1474 if (block
== rs
->last_sent_block
) {
1475 offset
|= RAM_SAVE_FLAG_CONTINUE
;
1477 qemu_put_be64(f
, offset
);
1480 if (!(offset
& RAM_SAVE_FLAG_CONTINUE
)) {
1481 len
= strlen(block
->idstr
);
1482 qemu_put_byte(f
, len
);
1483 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, len
);
1485 rs
->last_sent_block
= block
;
1491 * mig_throttle_guest_down: throotle down the guest
1493 * Reduce amount of guest cpu execution to hopefully slow down memory
1494 * writes. If guest dirty memory rate is reduced below the rate at
1495 * which we can transfer pages to the destination then we should be
1496 * able to complete migration. Some workloads dirty memory way too
1497 * fast and will not effectively converge, even with auto-converge.
1499 static void mig_throttle_guest_down(void)
1501 MigrationState
*s
= migrate_get_current();
1502 uint64_t pct_initial
= s
->parameters
.cpu_throttle_initial
;
1503 uint64_t pct_icrement
= s
->parameters
.cpu_throttle_increment
;
1504 int pct_max
= s
->parameters
.max_cpu_throttle
;
1506 /* We have not started throttling yet. Let's start it. */
1507 if (!cpu_throttle_active()) {
1508 cpu_throttle_set(pct_initial
);
1510 /* Throttling already on, just increase the rate */
1511 cpu_throttle_set(MIN(cpu_throttle_get_percentage() + pct_icrement
,
1517 * xbzrle_cache_zero_page: insert a zero page in the XBZRLE cache
1519 * @rs: current RAM state
1520 * @current_addr: address for the zero page
1522 * Update the xbzrle cache to reflect a page that's been sent as all 0.
1523 * The important thing is that a stale (not-yet-0'd) page be replaced
1525 * As a bonus, if the page wasn't in the cache it gets added so that
1526 * when a small write is made into the 0'd page it gets XBZRLE sent.
1528 static void xbzrle_cache_zero_page(RAMState
*rs
, ram_addr_t current_addr
)
1530 if (rs
->ram_bulk_stage
|| !migrate_use_xbzrle()) {
1534 /* We don't care if this fails to allocate a new cache page
1535 * as long as it updated an old one */
1536 cache_insert(XBZRLE
.cache
, current_addr
, XBZRLE
.zero_target_page
,
1537 ram_counters
.dirty_sync_count
);
1540 #define ENCODING_FLAG_XBZRLE 0x1
1543 * save_xbzrle_page: compress and send current page
1545 * Returns: 1 means that we wrote the page
1546 * 0 means that page is identical to the one already sent
1547 * -1 means that xbzrle would be longer than normal
1549 * @rs: current RAM state
1550 * @current_data: pointer to the address of the page contents
1551 * @current_addr: addr of the page
1552 * @block: block that contains the page we want to send
1553 * @offset: offset inside the block for the page
1554 * @last_stage: if we are at the completion stage
1556 static int save_xbzrle_page(RAMState
*rs
, uint8_t **current_data
,
1557 ram_addr_t current_addr
, RAMBlock
*block
,
1558 ram_addr_t offset
, bool last_stage
)
1560 int encoded_len
= 0, bytes_xbzrle
;
1561 uint8_t *prev_cached_page
;
1563 if (!cache_is_cached(XBZRLE
.cache
, current_addr
,
1564 ram_counters
.dirty_sync_count
)) {
1565 xbzrle_counters
.cache_miss
++;
1567 if (cache_insert(XBZRLE
.cache
, current_addr
, *current_data
,
1568 ram_counters
.dirty_sync_count
) == -1) {
1571 /* update *current_data when the page has been
1572 inserted into cache */
1573 *current_data
= get_cached_data(XBZRLE
.cache
, current_addr
);
1579 prev_cached_page
= get_cached_data(XBZRLE
.cache
, current_addr
);
1581 /* save current buffer into memory */
1582 memcpy(XBZRLE
.current_buf
, *current_data
, TARGET_PAGE_SIZE
);
1584 /* XBZRLE encoding (if there is no overflow) */
1585 encoded_len
= xbzrle_encode_buffer(prev_cached_page
, XBZRLE
.current_buf
,
1586 TARGET_PAGE_SIZE
, XBZRLE
.encoded_buf
,
1588 if (encoded_len
== 0) {
1589 trace_save_xbzrle_page_skipping();
1591 } else if (encoded_len
== -1) {
1592 trace_save_xbzrle_page_overflow();
1593 xbzrle_counters
.overflow
++;
1594 /* update data in the cache */
1596 memcpy(prev_cached_page
, *current_data
, TARGET_PAGE_SIZE
);
1597 *current_data
= prev_cached_page
;
1602 /* we need to update the data in the cache, in order to get the same data */
1604 memcpy(prev_cached_page
, XBZRLE
.current_buf
, TARGET_PAGE_SIZE
);
1607 /* Send XBZRLE based compressed page */
1608 bytes_xbzrle
= save_page_header(rs
, rs
->f
, block
,
1609 offset
| RAM_SAVE_FLAG_XBZRLE
);
1610 qemu_put_byte(rs
->f
, ENCODING_FLAG_XBZRLE
);
1611 qemu_put_be16(rs
->f
, encoded_len
);
1612 qemu_put_buffer(rs
->f
, XBZRLE
.encoded_buf
, encoded_len
);
1613 bytes_xbzrle
+= encoded_len
+ 1 + 2;
1614 xbzrle_counters
.pages
++;
1615 xbzrle_counters
.bytes
+= bytes_xbzrle
;
1616 ram_counters
.transferred
+= bytes_xbzrle
;
1622 * migration_bitmap_find_dirty: find the next dirty page from start
1624 * Returns the page offset within memory region of the start of a dirty page
1626 * @rs: current RAM state
1627 * @rb: RAMBlock where to search for dirty pages
1628 * @start: page where we start the search
1631 unsigned long migration_bitmap_find_dirty(RAMState
*rs
, RAMBlock
*rb
,
1632 unsigned long start
)
1634 unsigned long size
= rb
->used_length
>> TARGET_PAGE_BITS
;
1635 unsigned long *bitmap
= rb
->bmap
;
1638 if (ramblock_is_ignored(rb
)) {
1643 * When the free page optimization is enabled, we need to check the bitmap
1644 * to send the non-free pages rather than all the pages in the bulk stage.
1646 if (!rs
->fpo_enabled
&& rs
->ram_bulk_stage
&& start
> 0) {
1649 next
= find_next_bit(bitmap
, size
, start
);
1655 static inline bool migration_bitmap_clear_dirty(RAMState
*rs
,
1661 qemu_mutex_lock(&rs
->bitmap_mutex
);
1662 ret
= test_and_clear_bit(page
, rb
->bmap
);
1665 rs
->migration_dirty_pages
--;
1667 qemu_mutex_unlock(&rs
->bitmap_mutex
);
1672 static void migration_bitmap_sync_range(RAMState
*rs
, RAMBlock
*rb
,
1675 rs
->migration_dirty_pages
+=
1676 cpu_physical_memory_sync_dirty_bitmap(rb
, 0, length
,
1677 &rs
->num_dirty_pages_period
);
1681 * ram_pagesize_summary: calculate all the pagesizes of a VM
1683 * Returns a summary bitmap of the page sizes of all RAMBlocks
1685 * For VMs with just normal pages this is equivalent to the host page
1686 * size. If it's got some huge pages then it's the OR of all the
1687 * different page sizes.
1689 uint64_t ram_pagesize_summary(void)
1692 uint64_t summary
= 0;
1694 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1695 summary
|= block
->page_size
;
1701 uint64_t ram_get_total_transferred_pages(void)
1703 return ram_counters
.normal
+ ram_counters
.duplicate
+
1704 compression_counters
.pages
+ xbzrle_counters
.pages
;
1707 static void migration_update_rates(RAMState
*rs
, int64_t end_time
)
1709 uint64_t page_count
= rs
->target_page_count
- rs
->target_page_count_prev
;
1710 double compressed_size
;
1712 /* calculate period counters */
1713 ram_counters
.dirty_pages_rate
= rs
->num_dirty_pages_period
* 1000
1714 / (end_time
- rs
->time_last_bitmap_sync
);
1720 if (migrate_use_xbzrle()) {
1721 xbzrle_counters
.cache_miss_rate
= (double)(xbzrle_counters
.cache_miss
-
1722 rs
->xbzrle_cache_miss_prev
) / page_count
;
1723 rs
->xbzrle_cache_miss_prev
= xbzrle_counters
.cache_miss
;
1726 if (migrate_use_compression()) {
1727 compression_counters
.busy_rate
= (double)(compression_counters
.busy
-
1728 rs
->compress_thread_busy_prev
) / page_count
;
1729 rs
->compress_thread_busy_prev
= compression_counters
.busy
;
1731 compressed_size
= compression_counters
.compressed_size
-
1732 rs
->compressed_size_prev
;
1733 if (compressed_size
) {
1734 double uncompressed_size
= (compression_counters
.pages
-
1735 rs
->compress_pages_prev
) * TARGET_PAGE_SIZE
;
1737 /* Compression-Ratio = Uncompressed-size / Compressed-size */
1738 compression_counters
.compression_rate
=
1739 uncompressed_size
/ compressed_size
;
1741 rs
->compress_pages_prev
= compression_counters
.pages
;
1742 rs
->compressed_size_prev
= compression_counters
.compressed_size
;
1747 static void migration_bitmap_sync(RAMState
*rs
)
1751 uint64_t bytes_xfer_now
;
1753 ram_counters
.dirty_sync_count
++;
1755 if (!rs
->time_last_bitmap_sync
) {
1756 rs
->time_last_bitmap_sync
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
1759 trace_migration_bitmap_sync_start();
1760 memory_global_dirty_log_sync();
1762 qemu_mutex_lock(&rs
->bitmap_mutex
);
1764 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1765 migration_bitmap_sync_range(rs
, block
, block
->used_length
);
1767 ram_counters
.remaining
= ram_bytes_remaining();
1769 qemu_mutex_unlock(&rs
->bitmap_mutex
);
1771 trace_migration_bitmap_sync_end(rs
->num_dirty_pages_period
);
1773 end_time
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
1775 /* more than 1 second = 1000 millisecons */
1776 if (end_time
> rs
->time_last_bitmap_sync
+ 1000) {
1777 bytes_xfer_now
= ram_counters
.transferred
;
1779 /* During block migration the auto-converge logic incorrectly detects
1780 * that ram migration makes no progress. Avoid this by disabling the
1781 * throttling logic during the bulk phase of block migration. */
1782 if (migrate_auto_converge() && !blk_mig_bulk_active()) {
1783 /* The following detection logic can be refined later. For now:
1784 Check to see if the dirtied bytes is 50% more than the approx.
1785 amount of bytes that just got transferred since the last time we
1786 were in this routine. If that happens twice, start or increase
1789 if ((rs
->num_dirty_pages_period
* TARGET_PAGE_SIZE
>
1790 (bytes_xfer_now
- rs
->bytes_xfer_prev
) / 2) &&
1791 (++rs
->dirty_rate_high_cnt
>= 2)) {
1792 trace_migration_throttle();
1793 rs
->dirty_rate_high_cnt
= 0;
1794 mig_throttle_guest_down();
1798 migration_update_rates(rs
, end_time
);
1800 rs
->target_page_count_prev
= rs
->target_page_count
;
1802 /* reset period counters */
1803 rs
->time_last_bitmap_sync
= end_time
;
1804 rs
->num_dirty_pages_period
= 0;
1805 rs
->bytes_xfer_prev
= bytes_xfer_now
;
1807 if (migrate_use_events()) {
1808 qapi_event_send_migration_pass(ram_counters
.dirty_sync_count
);
1812 static void migration_bitmap_sync_precopy(RAMState
*rs
)
1814 Error
*local_err
= NULL
;
1817 * The current notifier usage is just an optimization to migration, so we
1818 * don't stop the normal migration process in the error case.
1820 if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC
, &local_err
)) {
1821 error_report_err(local_err
);
1824 migration_bitmap_sync(rs
);
1826 if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC
, &local_err
)) {
1827 error_report_err(local_err
);
1832 * save_zero_page_to_file: send the zero page to the file
1834 * Returns the size of data written to the file, 0 means the page is not
1837 * @rs: current RAM state
1838 * @file: the file where the data is saved
1839 * @block: block that contains the page we want to send
1840 * @offset: offset inside the block for the page
1842 static int save_zero_page_to_file(RAMState
*rs
, QEMUFile
*file
,
1843 RAMBlock
*block
, ram_addr_t offset
)
1845 uint8_t *p
= block
->host
+ offset
;
1848 if (is_zero_range(p
, TARGET_PAGE_SIZE
)) {
1849 len
+= save_page_header(rs
, file
, block
, offset
| RAM_SAVE_FLAG_ZERO
);
1850 qemu_put_byte(file
, 0);
1857 * save_zero_page: send the zero page to the stream
1859 * Returns the number of pages written.
1861 * @rs: current RAM state
1862 * @block: block that contains the page we want to send
1863 * @offset: offset inside the block for the page
1865 static int save_zero_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
)
1867 int len
= save_zero_page_to_file(rs
, rs
->f
, block
, offset
);
1870 ram_counters
.duplicate
++;
1871 ram_counters
.transferred
+= len
;
1877 static void ram_release_pages(const char *rbname
, uint64_t offset
, int pages
)
1879 if (!migrate_release_ram() || !migration_in_postcopy()) {
1883 ram_discard_range(rbname
, offset
, pages
<< TARGET_PAGE_BITS
);
1887 * @pages: the number of pages written by the control path,
1889 * > 0 - number of pages written
1891 * Return true if the pages has been saved, otherwise false is returned.
1893 static bool control_save_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
,
1896 uint64_t bytes_xmit
= 0;
1900 ret
= ram_control_save_page(rs
->f
, block
->offset
, offset
, TARGET_PAGE_SIZE
,
1902 if (ret
== RAM_SAVE_CONTROL_NOT_SUPP
) {
1907 ram_counters
.transferred
+= bytes_xmit
;
1911 if (ret
== RAM_SAVE_CONTROL_DELAYED
) {
1915 if (bytes_xmit
> 0) {
1916 ram_counters
.normal
++;
1917 } else if (bytes_xmit
== 0) {
1918 ram_counters
.duplicate
++;
1925 * directly send the page to the stream
1927 * Returns the number of pages written.
1929 * @rs: current RAM state
1930 * @block: block that contains the page we want to send
1931 * @offset: offset inside the block for the page
1932 * @buf: the page to be sent
1933 * @async: send to page asyncly
1935 static int save_normal_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
,
1936 uint8_t *buf
, bool async
)
1938 ram_counters
.transferred
+= save_page_header(rs
, rs
->f
, block
,
1939 offset
| RAM_SAVE_FLAG_PAGE
);
1941 qemu_put_buffer_async(rs
->f
, buf
, TARGET_PAGE_SIZE
,
1942 migrate_release_ram() &
1943 migration_in_postcopy());
1945 qemu_put_buffer(rs
->f
, buf
, TARGET_PAGE_SIZE
);
1947 ram_counters
.transferred
+= TARGET_PAGE_SIZE
;
1948 ram_counters
.normal
++;
1953 * ram_save_page: send the given page to the stream
1955 * Returns the number of pages written.
1957 * >=0 - Number of pages written - this might legally be 0
1958 * if xbzrle noticed the page was the same.
1960 * @rs: current RAM state
1961 * @block: block that contains the page we want to send
1962 * @offset: offset inside the block for the page
1963 * @last_stage: if we are at the completion stage
1965 static int ram_save_page(RAMState
*rs
, PageSearchStatus
*pss
, bool last_stage
)
1969 bool send_async
= true;
1970 RAMBlock
*block
= pss
->block
;
1971 ram_addr_t offset
= pss
->page
<< TARGET_PAGE_BITS
;
1972 ram_addr_t current_addr
= block
->offset
+ offset
;
1974 p
= block
->host
+ offset
;
1975 trace_ram_save_page(block
->idstr
, (uint64_t)offset
, p
);
1977 XBZRLE_cache_lock();
1978 if (!rs
->ram_bulk_stage
&& !migration_in_postcopy() &&
1979 migrate_use_xbzrle()) {
1980 pages
= save_xbzrle_page(rs
, &p
, current_addr
, block
,
1981 offset
, last_stage
);
1983 /* Can't send this cached data async, since the cache page
1984 * might get updated before it gets to the wire
1990 /* XBZRLE overflow or normal page */
1992 pages
= save_normal_page(rs
, block
, offset
, p
, send_async
);
1995 XBZRLE_cache_unlock();
2000 static int ram_save_multifd_page(RAMState
*rs
, RAMBlock
*block
,
2003 multifd_queue_page(block
, offset
);
2004 ram_counters
.normal
++;
2009 static bool do_compress_ram_page(QEMUFile
*f
, z_stream
*stream
, RAMBlock
*block
,
2010 ram_addr_t offset
, uint8_t *source_buf
)
2012 RAMState
*rs
= ram_state
;
2013 uint8_t *p
= block
->host
+ (offset
& TARGET_PAGE_MASK
);
2014 bool zero_page
= false;
2017 if (save_zero_page_to_file(rs
, f
, block
, offset
)) {
2022 save_page_header(rs
, f
, block
, offset
| RAM_SAVE_FLAG_COMPRESS_PAGE
);
2025 * copy it to a internal buffer to avoid it being modified by VM
2026 * so that we can catch up the error during compression and
2029 memcpy(source_buf
, p
, TARGET_PAGE_SIZE
);
2030 ret
= qemu_put_compression_data(f
, stream
, source_buf
, TARGET_PAGE_SIZE
);
2032 qemu_file_set_error(migrate_get_current()->to_dst_file
, ret
);
2033 error_report("compressed data failed!");
2038 ram_release_pages(block
->idstr
, offset
& TARGET_PAGE_MASK
, 1);
2043 update_compress_thread_counts(const CompressParam
*param
, int bytes_xmit
)
2045 ram_counters
.transferred
+= bytes_xmit
;
2047 if (param
->zero_page
) {
2048 ram_counters
.duplicate
++;
2052 /* 8 means a header with RAM_SAVE_FLAG_CONTINUE. */
2053 compression_counters
.compressed_size
+= bytes_xmit
- 8;
2054 compression_counters
.pages
++;
2057 static bool save_page_use_compression(RAMState
*rs
);
2059 static void flush_compressed_data(RAMState
*rs
)
2061 int idx
, len
, thread_count
;
2063 if (!save_page_use_compression(rs
)) {
2066 thread_count
= migrate_compress_threads();
2068 qemu_mutex_lock(&comp_done_lock
);
2069 for (idx
= 0; idx
< thread_count
; idx
++) {
2070 while (!comp_param
[idx
].done
) {
2071 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
2074 qemu_mutex_unlock(&comp_done_lock
);
2076 for (idx
= 0; idx
< thread_count
; idx
++) {
2077 qemu_mutex_lock(&comp_param
[idx
].mutex
);
2078 if (!comp_param
[idx
].quit
) {
2079 len
= qemu_put_qemu_file(rs
->f
, comp_param
[idx
].file
);
2081 * it's safe to fetch zero_page without holding comp_done_lock
2082 * as there is no further request submitted to the thread,
2083 * i.e, the thread should be waiting for a request at this point.
2085 update_compress_thread_counts(&comp_param
[idx
], len
);
2087 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
2091 static inline void set_compress_params(CompressParam
*param
, RAMBlock
*block
,
2094 param
->block
= block
;
2095 param
->offset
= offset
;
2098 static int compress_page_with_multi_thread(RAMState
*rs
, RAMBlock
*block
,
2101 int idx
, thread_count
, bytes_xmit
= -1, pages
= -1;
2102 bool wait
= migrate_compress_wait_thread();
2104 thread_count
= migrate_compress_threads();
2105 qemu_mutex_lock(&comp_done_lock
);
2107 for (idx
= 0; idx
< thread_count
; idx
++) {
2108 if (comp_param
[idx
].done
) {
2109 comp_param
[idx
].done
= false;
2110 bytes_xmit
= qemu_put_qemu_file(rs
->f
, comp_param
[idx
].file
);
2111 qemu_mutex_lock(&comp_param
[idx
].mutex
);
2112 set_compress_params(&comp_param
[idx
], block
, offset
);
2113 qemu_cond_signal(&comp_param
[idx
].cond
);
2114 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
2116 update_compress_thread_counts(&comp_param
[idx
], bytes_xmit
);
2122 * wait for the free thread if the user specifies 'compress-wait-thread',
2123 * otherwise we will post the page out in the main thread as normal page.
2125 if (pages
< 0 && wait
) {
2126 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
2129 qemu_mutex_unlock(&comp_done_lock
);
2135 * find_dirty_block: find the next dirty page and update any state
2136 * associated with the search process.
2138 * Returns true if a page is found
2140 * @rs: current RAM state
2141 * @pss: data about the state of the current dirty page scan
2142 * @again: set to false if the search has scanned the whole of RAM
2144 static bool find_dirty_block(RAMState
*rs
, PageSearchStatus
*pss
, bool *again
)
2146 pss
->page
= migration_bitmap_find_dirty(rs
, pss
->block
, pss
->page
);
2147 if (pss
->complete_round
&& pss
->block
== rs
->last_seen_block
&&
2148 pss
->page
>= rs
->last_page
) {
2150 * We've been once around the RAM and haven't found anything.
2156 if ((pss
->page
<< TARGET_PAGE_BITS
) >= pss
->block
->used_length
) {
2157 /* Didn't find anything in this RAM Block */
2159 pss
->block
= QLIST_NEXT_RCU(pss
->block
, next
);
2162 * If memory migration starts over, we will meet a dirtied page
2163 * which may still exists in compression threads's ring, so we
2164 * should flush the compressed data to make sure the new page
2165 * is not overwritten by the old one in the destination.
2167 * Also If xbzrle is on, stop using the data compression at this
2168 * point. In theory, xbzrle can do better than compression.
2170 flush_compressed_data(rs
);
2172 /* Hit the end of the list */
2173 pss
->block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
2174 /* Flag that we've looped */
2175 pss
->complete_round
= true;
2176 rs
->ram_bulk_stage
= false;
2178 /* Didn't find anything this time, but try again on the new block */
2182 /* Can go around again, but... */
2184 /* We've found something so probably don't need to */
2190 * unqueue_page: gets a page of the queue
2192 * Helper for 'get_queued_page' - gets a page off the queue
2194 * Returns the block of the page (or NULL if none available)
2196 * @rs: current RAM state
2197 * @offset: used to return the offset within the RAMBlock
2199 static RAMBlock
*unqueue_page(RAMState
*rs
, ram_addr_t
*offset
)
2201 RAMBlock
*block
= NULL
;
2203 if (QSIMPLEQ_EMPTY_ATOMIC(&rs
->src_page_requests
)) {
2207 qemu_mutex_lock(&rs
->src_page_req_mutex
);
2208 if (!QSIMPLEQ_EMPTY(&rs
->src_page_requests
)) {
2209 struct RAMSrcPageRequest
*entry
=
2210 QSIMPLEQ_FIRST(&rs
->src_page_requests
);
2212 *offset
= entry
->offset
;
2214 if (entry
->len
> TARGET_PAGE_SIZE
) {
2215 entry
->len
-= TARGET_PAGE_SIZE
;
2216 entry
->offset
+= TARGET_PAGE_SIZE
;
2218 memory_region_unref(block
->mr
);
2219 QSIMPLEQ_REMOVE_HEAD(&rs
->src_page_requests
, next_req
);
2221 migration_consume_urgent_request();
2224 qemu_mutex_unlock(&rs
->src_page_req_mutex
);
2230 * get_queued_page: unqueue a page from the postocpy requests
2232 * Skips pages that are already sent (!dirty)
2234 * Returns true if a queued page is found
2236 * @rs: current RAM state
2237 * @pss: data about the state of the current dirty page scan
2239 static bool get_queued_page(RAMState
*rs
, PageSearchStatus
*pss
)
2246 block
= unqueue_page(rs
, &offset
);
2248 * We're sending this page, and since it's postcopy nothing else
2249 * will dirty it, and we must make sure it doesn't get sent again
2250 * even if this queue request was received after the background
2251 * search already sent it.
2256 page
= offset
>> TARGET_PAGE_BITS
;
2257 dirty
= test_bit(page
, block
->bmap
);
2259 trace_get_queued_page_not_dirty(block
->idstr
, (uint64_t)offset
,
2260 page
, test_bit(page
, block
->unsentmap
));
2262 trace_get_queued_page(block
->idstr
, (uint64_t)offset
, page
);
2266 } while (block
&& !dirty
);
2270 * As soon as we start servicing pages out of order, then we have
2271 * to kill the bulk stage, since the bulk stage assumes
2272 * in (migration_bitmap_find_and_reset_dirty) that every page is
2273 * dirty, that's no longer true.
2275 rs
->ram_bulk_stage
= false;
2278 * We want the background search to continue from the queued page
2279 * since the guest is likely to want other pages near to the page
2280 * it just requested.
2283 pss
->page
= offset
>> TARGET_PAGE_BITS
;
2290 * migration_page_queue_free: drop any remaining pages in the ram
2293 * It should be empty at the end anyway, but in error cases there may
2294 * be some left. in case that there is any page left, we drop it.
2297 static void migration_page_queue_free(RAMState
*rs
)
2299 struct RAMSrcPageRequest
*mspr
, *next_mspr
;
2300 /* This queue generally should be empty - but in the case of a failed
2301 * migration might have some droppings in.
2304 QSIMPLEQ_FOREACH_SAFE(mspr
, &rs
->src_page_requests
, next_req
, next_mspr
) {
2305 memory_region_unref(mspr
->rb
->mr
);
2306 QSIMPLEQ_REMOVE_HEAD(&rs
->src_page_requests
, next_req
);
2313 * ram_save_queue_pages: queue the page for transmission
2315 * A request from postcopy destination for example.
2317 * Returns zero on success or negative on error
2319 * @rbname: Name of the RAMBLock of the request. NULL means the
2320 * same that last one.
2321 * @start: starting address from the start of the RAMBlock
2322 * @len: length (in bytes) to send
2324 int ram_save_queue_pages(const char *rbname
, ram_addr_t start
, ram_addr_t len
)
2327 RAMState
*rs
= ram_state
;
2329 ram_counters
.postcopy_requests
++;
2332 /* Reuse last RAMBlock */
2333 ramblock
= rs
->last_req_rb
;
2337 * Shouldn't happen, we can't reuse the last RAMBlock if
2338 * it's the 1st request.
2340 error_report("ram_save_queue_pages no previous block");
2344 ramblock
= qemu_ram_block_by_name(rbname
);
2347 /* We shouldn't be asked for a non-existent RAMBlock */
2348 error_report("ram_save_queue_pages no block '%s'", rbname
);
2351 rs
->last_req_rb
= ramblock
;
2353 trace_ram_save_queue_pages(ramblock
->idstr
, start
, len
);
2354 if (start
+len
> ramblock
->used_length
) {
2355 error_report("%s request overrun start=" RAM_ADDR_FMT
" len="
2356 RAM_ADDR_FMT
" blocklen=" RAM_ADDR_FMT
,
2357 __func__
, start
, len
, ramblock
->used_length
);
2361 struct RAMSrcPageRequest
*new_entry
=
2362 g_malloc0(sizeof(struct RAMSrcPageRequest
));
2363 new_entry
->rb
= ramblock
;
2364 new_entry
->offset
= start
;
2365 new_entry
->len
= len
;
2367 memory_region_ref(ramblock
->mr
);
2368 qemu_mutex_lock(&rs
->src_page_req_mutex
);
2369 QSIMPLEQ_INSERT_TAIL(&rs
->src_page_requests
, new_entry
, next_req
);
2370 migration_make_urgent_request();
2371 qemu_mutex_unlock(&rs
->src_page_req_mutex
);
2381 static bool save_page_use_compression(RAMState
*rs
)
2383 if (!migrate_use_compression()) {
2388 * If xbzrle is on, stop using the data compression after first
2389 * round of migration even if compression is enabled. In theory,
2390 * xbzrle can do better than compression.
2392 if (rs
->ram_bulk_stage
|| !migrate_use_xbzrle()) {
2400 * try to compress the page before posting it out, return true if the page
2401 * has been properly handled by compression, otherwise needs other
2402 * paths to handle it
2404 static bool save_compress_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
)
2406 if (!save_page_use_compression(rs
)) {
2411 * When starting the process of a new block, the first page of
2412 * the block should be sent out before other pages in the same
2413 * block, and all the pages in last block should have been sent
2414 * out, keeping this order is important, because the 'cont' flag
2415 * is used to avoid resending the block name.
2417 * We post the fist page as normal page as compression will take
2418 * much CPU resource.
2420 if (block
!= rs
->last_sent_block
) {
2421 flush_compressed_data(rs
);
2425 if (compress_page_with_multi_thread(rs
, block
, offset
) > 0) {
2429 compression_counters
.busy
++;
2434 * ram_save_target_page: save one target page
2436 * Returns the number of pages written
2438 * @rs: current RAM state
2439 * @pss: data about the page we want to send
2440 * @last_stage: if we are at the completion stage
2442 static int ram_save_target_page(RAMState
*rs
, PageSearchStatus
*pss
,
2445 RAMBlock
*block
= pss
->block
;
2446 ram_addr_t offset
= pss
->page
<< TARGET_PAGE_BITS
;
2449 if (control_save_page(rs
, block
, offset
, &res
)) {
2453 if (save_compress_page(rs
, block
, offset
)) {
2457 res
= save_zero_page(rs
, block
, offset
);
2459 /* Must let xbzrle know, otherwise a previous (now 0'd) cached
2460 * page would be stale
2462 if (!save_page_use_compression(rs
)) {
2463 XBZRLE_cache_lock();
2464 xbzrle_cache_zero_page(rs
, block
->offset
+ offset
);
2465 XBZRLE_cache_unlock();
2467 ram_release_pages(block
->idstr
, offset
, res
);
2472 * do not use multifd for compression as the first page in the new
2473 * block should be posted out before sending the compressed page
2475 if (!save_page_use_compression(rs
) && migrate_use_multifd()) {
2476 return ram_save_multifd_page(rs
, block
, offset
);
2479 return ram_save_page(rs
, pss
, last_stage
);
2483 * ram_save_host_page: save a whole host page
2485 * Starting at *offset send pages up to the end of the current host
2486 * page. It's valid for the initial offset to point into the middle of
2487 * a host page in which case the remainder of the hostpage is sent.
2488 * Only dirty target pages are sent. Note that the host page size may
2489 * be a huge page for this block.
2490 * The saving stops at the boundary of the used_length of the block
2491 * if the RAMBlock isn't a multiple of the host page size.
2493 * Returns the number of pages written or negative on error
2495 * @rs: current RAM state
2496 * @ms: current migration state
2497 * @pss: data about the page we want to send
2498 * @last_stage: if we are at the completion stage
2500 static int ram_save_host_page(RAMState
*rs
, PageSearchStatus
*pss
,
2503 int tmppages
, pages
= 0;
2504 size_t pagesize_bits
=
2505 qemu_ram_pagesize(pss
->block
) >> TARGET_PAGE_BITS
;
2507 if (ramblock_is_ignored(pss
->block
)) {
2508 error_report("block %s should not be migrated !", pss
->block
->idstr
);
2513 /* Check the pages is dirty and if it is send it */
2514 if (!migration_bitmap_clear_dirty(rs
, pss
->block
, pss
->page
)) {
2519 tmppages
= ram_save_target_page(rs
, pss
, last_stage
);
2525 if (pss
->block
->unsentmap
) {
2526 clear_bit(pss
->page
, pss
->block
->unsentmap
);
2530 } while ((pss
->page
& (pagesize_bits
- 1)) &&
2531 offset_in_ramblock(pss
->block
, pss
->page
<< TARGET_PAGE_BITS
));
2533 /* The offset we leave with is the last one we looked at */
2539 * ram_find_and_save_block: finds a dirty page and sends it to f
2541 * Called within an RCU critical section.
2543 * Returns the number of pages written where zero means no dirty pages,
2544 * or negative on error
2546 * @rs: current RAM state
2547 * @last_stage: if we are at the completion stage
2549 * On systems where host-page-size > target-page-size it will send all the
2550 * pages in a host page that are dirty.
2553 static int ram_find_and_save_block(RAMState
*rs
, bool last_stage
)
2555 PageSearchStatus pss
;
2559 /* No dirty page as there is zero RAM */
2560 if (!ram_bytes_total()) {
2564 pss
.block
= rs
->last_seen_block
;
2565 pss
.page
= rs
->last_page
;
2566 pss
.complete_round
= false;
2569 pss
.block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
2574 found
= get_queued_page(rs
, &pss
);
2577 /* priority queue empty, so just search for something dirty */
2578 found
= find_dirty_block(rs
, &pss
, &again
);
2582 pages
= ram_save_host_page(rs
, &pss
, last_stage
);
2584 } while (!pages
&& again
);
2586 rs
->last_seen_block
= pss
.block
;
2587 rs
->last_page
= pss
.page
;
2592 void acct_update_position(QEMUFile
*f
, size_t size
, bool zero
)
2594 uint64_t pages
= size
/ TARGET_PAGE_SIZE
;
2597 ram_counters
.duplicate
+= pages
;
2599 ram_counters
.normal
+= pages
;
2600 ram_counters
.transferred
+= size
;
2601 qemu_update_position(f
, size
);
2605 static uint64_t ram_bytes_total_common(bool count_ignored
)
2611 if (count_ignored
) {
2612 RAMBLOCK_FOREACH_MIGRATABLE(block
) {
2613 total
+= block
->used_length
;
2616 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2617 total
+= block
->used_length
;
2624 uint64_t ram_bytes_total(void)
2626 return ram_bytes_total_common(false);
2629 static void xbzrle_load_setup(void)
2631 XBZRLE
.decoded_buf
= g_malloc(TARGET_PAGE_SIZE
);
2634 static void xbzrle_load_cleanup(void)
2636 g_free(XBZRLE
.decoded_buf
);
2637 XBZRLE
.decoded_buf
= NULL
;
2640 static void ram_state_cleanup(RAMState
**rsp
)
2643 migration_page_queue_free(*rsp
);
2644 qemu_mutex_destroy(&(*rsp
)->bitmap_mutex
);
2645 qemu_mutex_destroy(&(*rsp
)->src_page_req_mutex
);
2651 static void xbzrle_cleanup(void)
2653 XBZRLE_cache_lock();
2655 cache_fini(XBZRLE
.cache
);
2656 g_free(XBZRLE
.encoded_buf
);
2657 g_free(XBZRLE
.current_buf
);
2658 g_free(XBZRLE
.zero_target_page
);
2659 XBZRLE
.cache
= NULL
;
2660 XBZRLE
.encoded_buf
= NULL
;
2661 XBZRLE
.current_buf
= NULL
;
2662 XBZRLE
.zero_target_page
= NULL
;
2664 XBZRLE_cache_unlock();
2667 static void ram_save_cleanup(void *opaque
)
2669 RAMState
**rsp
= opaque
;
2672 /* caller have hold iothread lock or is in a bh, so there is
2673 * no writing race against the migration bitmap
2675 memory_global_dirty_log_stop();
2677 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2678 g_free(block
->bmap
);
2680 g_free(block
->unsentmap
);
2681 block
->unsentmap
= NULL
;
2685 compress_threads_save_cleanup();
2686 ram_state_cleanup(rsp
);
2689 static void ram_state_reset(RAMState
*rs
)
2691 rs
->last_seen_block
= NULL
;
2692 rs
->last_sent_block
= NULL
;
2694 rs
->last_version
= ram_list
.version
;
2695 rs
->ram_bulk_stage
= true;
2696 rs
->fpo_enabled
= false;
2699 #define MAX_WAIT 50 /* ms, half buffered_file limit */
2702 * 'expected' is the value you expect the bitmap mostly to be full
2703 * of; it won't bother printing lines that are all this value.
2704 * If 'todump' is null the migration bitmap is dumped.
2706 void ram_debug_dump_bitmap(unsigned long *todump
, bool expected
,
2707 unsigned long pages
)
2710 int64_t linelen
= 128;
2713 for (cur
= 0; cur
< pages
; cur
+= linelen
) {
2717 * Last line; catch the case where the line length
2718 * is longer than remaining ram
2720 if (cur
+ linelen
> pages
) {
2721 linelen
= pages
- cur
;
2723 for (curb
= 0; curb
< linelen
; curb
++) {
2724 bool thisbit
= test_bit(cur
+ curb
, todump
);
2725 linebuf
[curb
] = thisbit
? '1' : '.';
2726 found
= found
|| (thisbit
!= expected
);
2729 linebuf
[curb
] = '\0';
2730 fprintf(stderr
, "0x%08" PRIx64
" : %s\n", cur
, linebuf
);
2735 /* **** functions for postcopy ***** */
2737 void ram_postcopy_migrated_memory_release(MigrationState
*ms
)
2739 struct RAMBlock
*block
;
2741 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2742 unsigned long *bitmap
= block
->bmap
;
2743 unsigned long range
= block
->used_length
>> TARGET_PAGE_BITS
;
2744 unsigned long run_start
= find_next_zero_bit(bitmap
, range
, 0);
2746 while (run_start
< range
) {
2747 unsigned long run_end
= find_next_bit(bitmap
, range
, run_start
+ 1);
2748 ram_discard_range(block
->idstr
, run_start
<< TARGET_PAGE_BITS
,
2749 (run_end
- run_start
) << TARGET_PAGE_BITS
);
2750 run_start
= find_next_zero_bit(bitmap
, range
, run_end
+ 1);
2756 * postcopy_send_discard_bm_ram: discard a RAMBlock
2758 * Returns zero on success
2760 * Callback from postcopy_each_ram_send_discard for each RAMBlock
2761 * Note: At this point the 'unsentmap' is the processed bitmap combined
2762 * with the dirtymap; so a '1' means it's either dirty or unsent.
2764 * @ms: current migration state
2765 * @pds: state for postcopy
2766 * @start: RAMBlock starting page
2767 * @length: RAMBlock size
2769 static int postcopy_send_discard_bm_ram(MigrationState
*ms
,
2770 PostcopyDiscardState
*pds
,
2773 unsigned long end
= block
->used_length
>> TARGET_PAGE_BITS
;
2774 unsigned long current
;
2775 unsigned long *unsentmap
= block
->unsentmap
;
2777 for (current
= 0; current
< end
; ) {
2778 unsigned long one
= find_next_bit(unsentmap
, end
, current
);
2781 unsigned long zero
= find_next_zero_bit(unsentmap
, end
, one
+ 1);
2782 unsigned long discard_length
;
2785 discard_length
= end
- one
;
2787 discard_length
= zero
- one
;
2789 if (discard_length
) {
2790 postcopy_discard_send_range(ms
, pds
, one
, discard_length
);
2792 current
= one
+ discard_length
;
2802 * postcopy_each_ram_send_discard: discard all RAMBlocks
2804 * Returns 0 for success or negative for error
2806 * Utility for the outgoing postcopy code.
2807 * Calls postcopy_send_discard_bm_ram for each RAMBlock
2808 * passing it bitmap indexes and name.
2809 * (qemu_ram_foreach_block ends up passing unscaled lengths
2810 * which would mean postcopy code would have to deal with target page)
2812 * @ms: current migration state
2814 static int postcopy_each_ram_send_discard(MigrationState
*ms
)
2816 struct RAMBlock
*block
;
2819 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2820 PostcopyDiscardState
*pds
=
2821 postcopy_discard_send_init(ms
, block
->idstr
);
2824 * Postcopy sends chunks of bitmap over the wire, but it
2825 * just needs indexes at this point, avoids it having
2826 * target page specific code.
2828 ret
= postcopy_send_discard_bm_ram(ms
, pds
, block
);
2829 postcopy_discard_send_finish(ms
, pds
);
2839 * postcopy_chunk_hostpages_pass: canocalize bitmap in hostpages
2841 * Helper for postcopy_chunk_hostpages; it's called twice to
2842 * canonicalize the two bitmaps, that are similar, but one is
2845 * Postcopy requires that all target pages in a hostpage are dirty or
2846 * clean, not a mix. This function canonicalizes the bitmaps.
2848 * @ms: current migration state
2849 * @unsent_pass: if true we need to canonicalize partially unsent host pages
2850 * otherwise we need to canonicalize partially dirty host pages
2851 * @block: block that contains the page we want to canonicalize
2852 * @pds: state for postcopy
2854 static void postcopy_chunk_hostpages_pass(MigrationState
*ms
, bool unsent_pass
,
2856 PostcopyDiscardState
*pds
)
2858 RAMState
*rs
= ram_state
;
2859 unsigned long *bitmap
= block
->bmap
;
2860 unsigned long *unsentmap
= block
->unsentmap
;
2861 unsigned int host_ratio
= block
->page_size
/ TARGET_PAGE_SIZE
;
2862 unsigned long pages
= block
->used_length
>> TARGET_PAGE_BITS
;
2863 unsigned long run_start
;
2865 if (block
->page_size
== TARGET_PAGE_SIZE
) {
2866 /* Easy case - TPS==HPS for a non-huge page RAMBlock */
2871 /* Find a sent page */
2872 run_start
= find_next_zero_bit(unsentmap
, pages
, 0);
2874 /* Find a dirty page */
2875 run_start
= find_next_bit(bitmap
, pages
, 0);
2878 while (run_start
< pages
) {
2879 bool do_fixup
= false;
2880 unsigned long fixup_start_addr
;
2881 unsigned long host_offset
;
2884 * If the start of this run of pages is in the middle of a host
2885 * page, then we need to fixup this host page.
2887 host_offset
= run_start
% host_ratio
;
2890 run_start
-= host_offset
;
2891 fixup_start_addr
= run_start
;
2892 /* For the next pass */
2893 run_start
= run_start
+ host_ratio
;
2895 /* Find the end of this run */
2896 unsigned long run_end
;
2898 run_end
= find_next_bit(unsentmap
, pages
, run_start
+ 1);
2900 run_end
= find_next_zero_bit(bitmap
, pages
, run_start
+ 1);
2903 * If the end isn't at the start of a host page, then the
2904 * run doesn't finish at the end of a host page
2905 * and we need to discard.
2907 host_offset
= run_end
% host_ratio
;
2910 fixup_start_addr
= run_end
- host_offset
;
2912 * This host page has gone, the next loop iteration starts
2913 * from after the fixup
2915 run_start
= fixup_start_addr
+ host_ratio
;
2918 * No discards on this iteration, next loop starts from
2919 * next sent/dirty page
2921 run_start
= run_end
+ 1;
2928 /* Tell the destination to discard this page */
2929 if (unsent_pass
|| !test_bit(fixup_start_addr
, unsentmap
)) {
2930 /* For the unsent_pass we:
2931 * discard partially sent pages
2932 * For the !unsent_pass (dirty) we:
2933 * discard partially dirty pages that were sent
2934 * (any partially sent pages were already discarded
2935 * by the previous unsent_pass)
2937 postcopy_discard_send_range(ms
, pds
, fixup_start_addr
,
2941 /* Clean up the bitmap */
2942 for (page
= fixup_start_addr
;
2943 page
< fixup_start_addr
+ host_ratio
; page
++) {
2944 /* All pages in this host page are now not sent */
2945 set_bit(page
, unsentmap
);
2948 * Remark them as dirty, updating the count for any pages
2949 * that weren't previously dirty.
2951 rs
->migration_dirty_pages
+= !test_and_set_bit(page
, bitmap
);
2956 /* Find the next sent page for the next iteration */
2957 run_start
= find_next_zero_bit(unsentmap
, pages
, run_start
);
2959 /* Find the next dirty page for the next iteration */
2960 run_start
= find_next_bit(bitmap
, pages
, run_start
);
2966 * postcopy_chuck_hostpages: discrad any partially sent host page
2968 * Utility for the outgoing postcopy code.
2970 * Discard any partially sent host-page size chunks, mark any partially
2971 * dirty host-page size chunks as all dirty. In this case the host-page
2972 * is the host-page for the particular RAMBlock, i.e. it might be a huge page
2974 * Returns zero on success
2976 * @ms: current migration state
2977 * @block: block we want to work with
2979 static int postcopy_chunk_hostpages(MigrationState
*ms
, RAMBlock
*block
)
2981 PostcopyDiscardState
*pds
=
2982 postcopy_discard_send_init(ms
, block
->idstr
);
2984 /* First pass: Discard all partially sent host pages */
2985 postcopy_chunk_hostpages_pass(ms
, true, block
, pds
);
2987 * Second pass: Ensure that all partially dirty host pages are made
2990 postcopy_chunk_hostpages_pass(ms
, false, block
, pds
);
2992 postcopy_discard_send_finish(ms
, pds
);
2997 * ram_postcopy_send_discard_bitmap: transmit the discard bitmap
2999 * Returns zero on success
3001 * Transmit the set of pages to be discarded after precopy to the target
3002 * these are pages that:
3003 * a) Have been previously transmitted but are now dirty again
3004 * b) Pages that have never been transmitted, this ensures that
3005 * any pages on the destination that have been mapped by background
3006 * tasks get discarded (transparent huge pages is the specific concern)
3007 * Hopefully this is pretty sparse
3009 * @ms: current migration state
3011 int ram_postcopy_send_discard_bitmap(MigrationState
*ms
)
3013 RAMState
*rs
= ram_state
;
3019 /* This should be our last sync, the src is now paused */
3020 migration_bitmap_sync(rs
);
3022 /* Easiest way to make sure we don't resume in the middle of a host-page */
3023 rs
->last_seen_block
= NULL
;
3024 rs
->last_sent_block
= NULL
;
3027 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3028 unsigned long pages
= block
->used_length
>> TARGET_PAGE_BITS
;
3029 unsigned long *bitmap
= block
->bmap
;
3030 unsigned long *unsentmap
= block
->unsentmap
;
3033 /* We don't have a safe way to resize the sentmap, so
3034 * if the bitmap was resized it will be NULL at this
3037 error_report("migration ram resized during precopy phase");
3041 /* Deal with TPS != HPS and huge pages */
3042 ret
= postcopy_chunk_hostpages(ms
, block
);
3049 * Update the unsentmap to be unsentmap = unsentmap | dirty
3051 bitmap_or(unsentmap
, unsentmap
, bitmap
, pages
);
3052 #ifdef DEBUG_POSTCOPY
3053 ram_debug_dump_bitmap(unsentmap
, true, pages
);
3056 trace_ram_postcopy_send_discard_bitmap();
3058 ret
= postcopy_each_ram_send_discard(ms
);
3065 * ram_discard_range: discard dirtied pages at the beginning of postcopy
3067 * Returns zero on success
3069 * @rbname: name of the RAMBlock of the request. NULL means the
3070 * same that last one.
3071 * @start: RAMBlock starting page
3072 * @length: RAMBlock size
3074 int ram_discard_range(const char *rbname
, uint64_t start
, size_t length
)
3078 trace_ram_discard_range(rbname
, start
, length
);
3081 RAMBlock
*rb
= qemu_ram_block_by_name(rbname
);
3084 error_report("ram_discard_range: Failed to find block '%s'", rbname
);
3089 * On source VM, we don't need to update the received bitmap since
3090 * we don't even have one.
3092 if (rb
->receivedmap
) {
3093 bitmap_clear(rb
->receivedmap
, start
>> qemu_target_page_bits(),
3094 length
>> qemu_target_page_bits());
3097 ret
= ram_block_discard_range(rb
, start
, length
);
3106 * For every allocation, we will try not to crash the VM if the
3107 * allocation failed.
3109 static int xbzrle_init(void)
3111 Error
*local_err
= NULL
;
3113 if (!migrate_use_xbzrle()) {
3117 XBZRLE_cache_lock();
3119 XBZRLE
.zero_target_page
= g_try_malloc0(TARGET_PAGE_SIZE
);
3120 if (!XBZRLE
.zero_target_page
) {
3121 error_report("%s: Error allocating zero page", __func__
);
3125 XBZRLE
.cache
= cache_init(migrate_xbzrle_cache_size(),
3126 TARGET_PAGE_SIZE
, &local_err
);
3127 if (!XBZRLE
.cache
) {
3128 error_report_err(local_err
);
3129 goto free_zero_page
;
3132 XBZRLE
.encoded_buf
= g_try_malloc0(TARGET_PAGE_SIZE
);
3133 if (!XBZRLE
.encoded_buf
) {
3134 error_report("%s: Error allocating encoded_buf", __func__
);
3138 XBZRLE
.current_buf
= g_try_malloc(TARGET_PAGE_SIZE
);
3139 if (!XBZRLE
.current_buf
) {
3140 error_report("%s: Error allocating current_buf", __func__
);
3141 goto free_encoded_buf
;
3144 /* We are all good */
3145 XBZRLE_cache_unlock();
3149 g_free(XBZRLE
.encoded_buf
);
3150 XBZRLE
.encoded_buf
= NULL
;
3152 cache_fini(XBZRLE
.cache
);
3153 XBZRLE
.cache
= NULL
;
3155 g_free(XBZRLE
.zero_target_page
);
3156 XBZRLE
.zero_target_page
= NULL
;
3158 XBZRLE_cache_unlock();
3162 static int ram_state_init(RAMState
**rsp
)
3164 *rsp
= g_try_new0(RAMState
, 1);
3167 error_report("%s: Init ramstate fail", __func__
);
3171 qemu_mutex_init(&(*rsp
)->bitmap_mutex
);
3172 qemu_mutex_init(&(*rsp
)->src_page_req_mutex
);
3173 QSIMPLEQ_INIT(&(*rsp
)->src_page_requests
);
3176 * Count the total number of pages used by ram blocks not including any
3177 * gaps due to alignment or unplugs.
3179 (*rsp
)->migration_dirty_pages
= ram_bytes_total() >> TARGET_PAGE_BITS
;
3181 ram_state_reset(*rsp
);
3186 static void ram_list_init_bitmaps(void)
3189 unsigned long pages
;
3191 /* Skip setting bitmap if there is no RAM */
3192 if (ram_bytes_total()) {
3193 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3194 pages
= block
->max_length
>> TARGET_PAGE_BITS
;
3195 block
->bmap
= bitmap_new(pages
);
3196 bitmap_set(block
->bmap
, 0, pages
);
3197 if (migrate_postcopy_ram()) {
3198 block
->unsentmap
= bitmap_new(pages
);
3199 bitmap_set(block
->unsentmap
, 0, pages
);
3205 static void ram_init_bitmaps(RAMState
*rs
)
3207 /* For memory_global_dirty_log_start below. */
3208 qemu_mutex_lock_iothread();
3209 qemu_mutex_lock_ramlist();
3212 ram_list_init_bitmaps();
3213 memory_global_dirty_log_start();
3214 migration_bitmap_sync_precopy(rs
);
3217 qemu_mutex_unlock_ramlist();
3218 qemu_mutex_unlock_iothread();
3221 static int ram_init_all(RAMState
**rsp
)
3223 if (ram_state_init(rsp
)) {
3227 if (xbzrle_init()) {
3228 ram_state_cleanup(rsp
);
3232 ram_init_bitmaps(*rsp
);
3237 static void ram_state_resume_prepare(RAMState
*rs
, QEMUFile
*out
)
3243 * Postcopy is not using xbzrle/compression, so no need for that.
3244 * Also, since source are already halted, we don't need to care
3245 * about dirty page logging as well.
3248 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3249 pages
+= bitmap_count_one(block
->bmap
,
3250 block
->used_length
>> TARGET_PAGE_BITS
);
3253 /* This may not be aligned with current bitmaps. Recalculate. */
3254 rs
->migration_dirty_pages
= pages
;
3256 rs
->last_seen_block
= NULL
;
3257 rs
->last_sent_block
= NULL
;
3259 rs
->last_version
= ram_list
.version
;
3261 * Disable the bulk stage, otherwise we'll resend the whole RAM no
3262 * matter what we have sent.
3264 rs
->ram_bulk_stage
= false;
3266 /* Update RAMState cache of output QEMUFile */
3269 trace_ram_state_resume_prepare(pages
);
3273 * This function clears bits of the free pages reported by the caller from the
3274 * migration dirty bitmap. @addr is the host address corresponding to the
3275 * start of the continuous guest free pages, and @len is the total bytes of
3278 void qemu_guest_free_page_hint(void *addr
, size_t len
)
3282 size_t used_len
, start
, npages
;
3283 MigrationState
*s
= migrate_get_current();
3285 /* This function is currently expected to be used during live migration */
3286 if (!migration_is_setup_or_active(s
->state
)) {
3290 for (; len
> 0; len
-= used_len
, addr
+= used_len
) {
3291 block
= qemu_ram_block_from_host(addr
, false, &offset
);
3292 if (unlikely(!block
|| offset
>= block
->used_length
)) {
3294 * The implementation might not support RAMBlock resize during
3295 * live migration, but it could happen in theory with future
3296 * updates. So we add a check here to capture that case.
3298 error_report_once("%s unexpected error", __func__
);
3302 if (len
<= block
->used_length
- offset
) {
3305 used_len
= block
->used_length
- offset
;
3308 start
= offset
>> TARGET_PAGE_BITS
;
3309 npages
= used_len
>> TARGET_PAGE_BITS
;
3311 qemu_mutex_lock(&ram_state
->bitmap_mutex
);
3312 ram_state
->migration_dirty_pages
-=
3313 bitmap_count_one_with_offset(block
->bmap
, start
, npages
);
3314 bitmap_clear(block
->bmap
, start
, npages
);
3315 qemu_mutex_unlock(&ram_state
->bitmap_mutex
);
3320 * Each of ram_save_setup, ram_save_iterate and ram_save_complete has
3321 * long-running RCU critical section. When rcu-reclaims in the code
3322 * start to become numerous it will be necessary to reduce the
3323 * granularity of these critical sections.
3327 * ram_save_setup: Setup RAM for migration
3329 * Returns zero to indicate success and negative for error
3331 * @f: QEMUFile where to send the data
3332 * @opaque: RAMState pointer
3334 static int ram_save_setup(QEMUFile
*f
, void *opaque
)
3336 RAMState
**rsp
= opaque
;
3339 if (compress_threads_save_setup()) {
3343 /* migration has already setup the bitmap, reuse it. */
3344 if (!migration_in_colo_state()) {
3345 if (ram_init_all(rsp
) != 0) {
3346 compress_threads_save_cleanup();
3354 qemu_put_be64(f
, ram_bytes_total_common(true) | RAM_SAVE_FLAG_MEM_SIZE
);
3356 RAMBLOCK_FOREACH_MIGRATABLE(block
) {
3357 qemu_put_byte(f
, strlen(block
->idstr
));
3358 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, strlen(block
->idstr
));
3359 qemu_put_be64(f
, block
->used_length
);
3360 if (migrate_postcopy_ram() && block
->page_size
!= qemu_host_page_size
) {
3361 qemu_put_be64(f
, block
->page_size
);
3363 if (migrate_ignore_shared()) {
3364 qemu_put_be64(f
, block
->mr
->addr
);
3365 qemu_put_byte(f
, ramblock_is_ignored(block
) ? 1 : 0);
3371 ram_control_before_iterate(f
, RAM_CONTROL_SETUP
);
3372 ram_control_after_iterate(f
, RAM_CONTROL_SETUP
);
3374 multifd_send_sync_main();
3375 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
3382 * ram_save_iterate: iterative stage for migration
3384 * Returns zero to indicate success and negative for error
3386 * @f: QEMUFile where to send the data
3387 * @opaque: RAMState pointer
3389 static int ram_save_iterate(QEMUFile
*f
, void *opaque
)
3391 RAMState
**temp
= opaque
;
3392 RAMState
*rs
= *temp
;
3398 if (blk_mig_bulk_active()) {
3399 /* Avoid transferring ram during bulk phase of block migration as
3400 * the bulk phase will usually take a long time and transferring
3401 * ram updates during that time is pointless. */
3406 if (ram_list
.version
!= rs
->last_version
) {
3407 ram_state_reset(rs
);
3410 /* Read version before ram_list.blocks */
3413 ram_control_before_iterate(f
, RAM_CONTROL_ROUND
);
3415 t0
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
);
3417 while ((ret
= qemu_file_rate_limit(f
)) == 0 ||
3418 !QSIMPLEQ_EMPTY(&rs
->src_page_requests
)) {
3421 if (qemu_file_get_error(f
)) {
3425 pages
= ram_find_and_save_block(rs
, false);
3426 /* no more pages to sent */
3433 qemu_file_set_error(f
, pages
);
3437 rs
->target_page_count
+= pages
;
3439 /* we want to check in the 1st loop, just in case it was the 1st time
3440 and we had to sync the dirty bitmap.
3441 qemu_clock_get_ns() is a bit expensive, so we only check each some
3444 if ((i
& 63) == 0) {
3445 uint64_t t1
= (qemu_clock_get_ns(QEMU_CLOCK_REALTIME
) - t0
) / 1000000;
3446 if (t1
> MAX_WAIT
) {
3447 trace_ram_save_iterate_big_wait(t1
, i
);
3456 * Must occur before EOS (or any QEMUFile operation)
3457 * because of RDMA protocol.
3459 ram_control_after_iterate(f
, RAM_CONTROL_ROUND
);
3461 multifd_send_sync_main();
3463 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
3465 ram_counters
.transferred
+= 8;
3467 ret
= qemu_file_get_error(f
);
3476 * ram_save_complete: function called to send the remaining amount of ram
3478 * Returns zero to indicate success or negative on error
3480 * Called with iothread lock
3482 * @f: QEMUFile where to send the data
3483 * @opaque: RAMState pointer
3485 static int ram_save_complete(QEMUFile
*f
, void *opaque
)
3487 RAMState
**temp
= opaque
;
3488 RAMState
*rs
= *temp
;
3493 if (!migration_in_postcopy()) {
3494 migration_bitmap_sync_precopy(rs
);
3497 ram_control_before_iterate(f
, RAM_CONTROL_FINISH
);
3499 /* try transferring iterative blocks of memory */
3501 /* flush all remaining blocks regardless of rate limiting */
3505 pages
= ram_find_and_save_block(rs
, !migration_in_colo_state());
3506 /* no more blocks to sent */
3516 flush_compressed_data(rs
);
3517 ram_control_after_iterate(f
, RAM_CONTROL_FINISH
);
3521 multifd_send_sync_main();
3522 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
3528 static void ram_save_pending(QEMUFile
*f
, void *opaque
, uint64_t max_size
,
3529 uint64_t *res_precopy_only
,
3530 uint64_t *res_compatible
,
3531 uint64_t *res_postcopy_only
)
3533 RAMState
**temp
= opaque
;
3534 RAMState
*rs
= *temp
;
3535 uint64_t remaining_size
;
3537 remaining_size
= rs
->migration_dirty_pages
* TARGET_PAGE_SIZE
;
3539 if (!migration_in_postcopy() &&
3540 remaining_size
< max_size
) {
3541 qemu_mutex_lock_iothread();
3543 migration_bitmap_sync_precopy(rs
);
3545 qemu_mutex_unlock_iothread();
3546 remaining_size
= rs
->migration_dirty_pages
* TARGET_PAGE_SIZE
;
3549 if (migrate_postcopy_ram()) {
3550 /* We can do postcopy, and all the data is postcopiable */
3551 *res_compatible
+= remaining_size
;
3553 *res_precopy_only
+= remaining_size
;
3557 static int load_xbzrle(QEMUFile
*f
, ram_addr_t addr
, void *host
)
3559 unsigned int xh_len
;
3561 uint8_t *loaded_data
;
3563 /* extract RLE header */
3564 xh_flags
= qemu_get_byte(f
);
3565 xh_len
= qemu_get_be16(f
);
3567 if (xh_flags
!= ENCODING_FLAG_XBZRLE
) {
3568 error_report("Failed to load XBZRLE page - wrong compression!");
3572 if (xh_len
> TARGET_PAGE_SIZE
) {
3573 error_report("Failed to load XBZRLE page - len overflow!");
3576 loaded_data
= XBZRLE
.decoded_buf
;
3577 /* load data and decode */
3578 /* it can change loaded_data to point to an internal buffer */
3579 qemu_get_buffer_in_place(f
, &loaded_data
, xh_len
);
3582 if (xbzrle_decode_buffer(loaded_data
, xh_len
, host
,
3583 TARGET_PAGE_SIZE
) == -1) {
3584 error_report("Failed to load XBZRLE page - decode error!");
3592 * ram_block_from_stream: read a RAMBlock id from the migration stream
3594 * Must be called from within a rcu critical section.
3596 * Returns a pointer from within the RCU-protected ram_list.
3598 * @f: QEMUFile where to read the data from
3599 * @flags: Page flags (mostly to see if it's a continuation of previous block)
3601 static inline RAMBlock
*ram_block_from_stream(QEMUFile
*f
, int flags
)
3603 static RAMBlock
*block
= NULL
;
3607 if (flags
& RAM_SAVE_FLAG_CONTINUE
) {
3609 error_report("Ack, bad migration stream!");
3615 len
= qemu_get_byte(f
);
3616 qemu_get_buffer(f
, (uint8_t *)id
, len
);
3619 block
= qemu_ram_block_by_name(id
);
3621 error_report("Can't find block %s", id
);
3625 if (ramblock_is_ignored(block
)) {
3626 error_report("block %s should not be migrated !", id
);
3633 static inline void *host_from_ram_block_offset(RAMBlock
*block
,
3636 if (!offset_in_ramblock(block
, offset
)) {
3640 return block
->host
+ offset
;
3643 static inline void *colo_cache_from_block_offset(RAMBlock
*block
,
3646 if (!offset_in_ramblock(block
, offset
)) {
3649 if (!block
->colo_cache
) {
3650 error_report("%s: colo_cache is NULL in block :%s",
3651 __func__
, block
->idstr
);
3656 * During colo checkpoint, we need bitmap of these migrated pages.
3657 * It help us to decide which pages in ram cache should be flushed
3658 * into VM's RAM later.
3660 if (!test_and_set_bit(offset
>> TARGET_PAGE_BITS
, block
->bmap
)) {
3661 ram_state
->migration_dirty_pages
++;
3663 return block
->colo_cache
+ offset
;
3667 * ram_handle_compressed: handle the zero page case
3669 * If a page (or a whole RDMA chunk) has been
3670 * determined to be zero, then zap it.
3672 * @host: host address for the zero page
3673 * @ch: what the page is filled from. We only support zero
3674 * @size: size of the zero page
3676 void ram_handle_compressed(void *host
, uint8_t ch
, uint64_t size
)
3678 if (ch
!= 0 || !is_zero_range(host
, size
)) {
3679 memset(host
, ch
, size
);
3683 /* return the size after decompression, or negative value on error */
3685 qemu_uncompress_data(z_stream
*stream
, uint8_t *dest
, size_t dest_len
,
3686 const uint8_t *source
, size_t source_len
)
3690 err
= inflateReset(stream
);
3695 stream
->avail_in
= source_len
;
3696 stream
->next_in
= (uint8_t *)source
;
3697 stream
->avail_out
= dest_len
;
3698 stream
->next_out
= dest
;
3700 err
= inflate(stream
, Z_NO_FLUSH
);
3701 if (err
!= Z_STREAM_END
) {
3705 return stream
->total_out
;
3708 static void *do_data_decompress(void *opaque
)
3710 DecompressParam
*param
= opaque
;
3711 unsigned long pagesize
;
3715 qemu_mutex_lock(¶m
->mutex
);
3716 while (!param
->quit
) {
3721 qemu_mutex_unlock(¶m
->mutex
);
3723 pagesize
= TARGET_PAGE_SIZE
;
3725 ret
= qemu_uncompress_data(¶m
->stream
, des
, pagesize
,
3726 param
->compbuf
, len
);
3727 if (ret
< 0 && migrate_get_current()->decompress_error_check
) {
3728 error_report("decompress data failed");
3729 qemu_file_set_error(decomp_file
, ret
);
3732 qemu_mutex_lock(&decomp_done_lock
);
3734 qemu_cond_signal(&decomp_done_cond
);
3735 qemu_mutex_unlock(&decomp_done_lock
);
3737 qemu_mutex_lock(¶m
->mutex
);
3739 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
3742 qemu_mutex_unlock(¶m
->mutex
);
3747 static int wait_for_decompress_done(void)
3749 int idx
, thread_count
;
3751 if (!migrate_use_compression()) {
3755 thread_count
= migrate_decompress_threads();
3756 qemu_mutex_lock(&decomp_done_lock
);
3757 for (idx
= 0; idx
< thread_count
; idx
++) {
3758 while (!decomp_param
[idx
].done
) {
3759 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
3762 qemu_mutex_unlock(&decomp_done_lock
);
3763 return qemu_file_get_error(decomp_file
);
3766 static void compress_threads_load_cleanup(void)
3768 int i
, thread_count
;
3770 if (!migrate_use_compression()) {
3773 thread_count
= migrate_decompress_threads();
3774 for (i
= 0; i
< thread_count
; i
++) {
3776 * we use it as a indicator which shows if the thread is
3777 * properly init'd or not
3779 if (!decomp_param
[i
].compbuf
) {
3783 qemu_mutex_lock(&decomp_param
[i
].mutex
);
3784 decomp_param
[i
].quit
= true;
3785 qemu_cond_signal(&decomp_param
[i
].cond
);
3786 qemu_mutex_unlock(&decomp_param
[i
].mutex
);
3788 for (i
= 0; i
< thread_count
; i
++) {
3789 if (!decomp_param
[i
].compbuf
) {
3793 qemu_thread_join(decompress_threads
+ i
);
3794 qemu_mutex_destroy(&decomp_param
[i
].mutex
);
3795 qemu_cond_destroy(&decomp_param
[i
].cond
);
3796 inflateEnd(&decomp_param
[i
].stream
);
3797 g_free(decomp_param
[i
].compbuf
);
3798 decomp_param
[i
].compbuf
= NULL
;
3800 g_free(decompress_threads
);
3801 g_free(decomp_param
);
3802 decompress_threads
= NULL
;
3803 decomp_param
= NULL
;
3807 static int compress_threads_load_setup(QEMUFile
*f
)
3809 int i
, thread_count
;
3811 if (!migrate_use_compression()) {
3815 thread_count
= migrate_decompress_threads();
3816 decompress_threads
= g_new0(QemuThread
, thread_count
);
3817 decomp_param
= g_new0(DecompressParam
, thread_count
);
3818 qemu_mutex_init(&decomp_done_lock
);
3819 qemu_cond_init(&decomp_done_cond
);
3821 for (i
= 0; i
< thread_count
; i
++) {
3822 if (inflateInit(&decomp_param
[i
].stream
) != Z_OK
) {
3826 decomp_param
[i
].compbuf
= g_malloc0(compressBound(TARGET_PAGE_SIZE
));
3827 qemu_mutex_init(&decomp_param
[i
].mutex
);
3828 qemu_cond_init(&decomp_param
[i
].cond
);
3829 decomp_param
[i
].done
= true;
3830 decomp_param
[i
].quit
= false;
3831 qemu_thread_create(decompress_threads
+ i
, "decompress",
3832 do_data_decompress
, decomp_param
+ i
,
3833 QEMU_THREAD_JOINABLE
);
3837 compress_threads_load_cleanup();
3841 static void decompress_data_with_multi_threads(QEMUFile
*f
,
3842 void *host
, int len
)
3844 int idx
, thread_count
;
3846 thread_count
= migrate_decompress_threads();
3847 qemu_mutex_lock(&decomp_done_lock
);
3849 for (idx
= 0; idx
< thread_count
; idx
++) {
3850 if (decomp_param
[idx
].done
) {
3851 decomp_param
[idx
].done
= false;
3852 qemu_mutex_lock(&decomp_param
[idx
].mutex
);
3853 qemu_get_buffer(f
, decomp_param
[idx
].compbuf
, len
);
3854 decomp_param
[idx
].des
= host
;
3855 decomp_param
[idx
].len
= len
;
3856 qemu_cond_signal(&decomp_param
[idx
].cond
);
3857 qemu_mutex_unlock(&decomp_param
[idx
].mutex
);
3861 if (idx
< thread_count
) {
3864 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
3867 qemu_mutex_unlock(&decomp_done_lock
);
3871 * colo cache: this is for secondary VM, we cache the whole
3872 * memory of the secondary VM, it is need to hold the global lock
3873 * to call this helper.
3875 int colo_init_ram_cache(void)
3880 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3881 block
->colo_cache
= qemu_anon_ram_alloc(block
->used_length
,
3884 if (!block
->colo_cache
) {
3885 error_report("%s: Can't alloc memory for COLO cache of block %s,"
3886 "size 0x" RAM_ADDR_FMT
, __func__
, block
->idstr
,
3887 block
->used_length
);
3890 memcpy(block
->colo_cache
, block
->host
, block
->used_length
);
3894 * Record the dirty pages that sent by PVM, we use this dirty bitmap together
3895 * with to decide which page in cache should be flushed into SVM's RAM. Here
3896 * we use the same name 'ram_bitmap' as for migration.
3898 if (ram_bytes_total()) {
3901 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3902 unsigned long pages
= block
->max_length
>> TARGET_PAGE_BITS
;
3904 block
->bmap
= bitmap_new(pages
);
3905 bitmap_set(block
->bmap
, 0, pages
);
3908 ram_state
= g_new0(RAMState
, 1);
3909 ram_state
->migration_dirty_pages
= 0;
3910 qemu_mutex_init(&ram_state
->bitmap_mutex
);
3911 memory_global_dirty_log_start();
3917 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3918 if (block
->colo_cache
) {
3919 qemu_anon_ram_free(block
->colo_cache
, block
->used_length
);
3920 block
->colo_cache
= NULL
;
3928 /* It is need to hold the global lock to call this helper */
3929 void colo_release_ram_cache(void)
3933 memory_global_dirty_log_stop();
3934 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3935 g_free(block
->bmap
);
3941 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3942 if (block
->colo_cache
) {
3943 qemu_anon_ram_free(block
->colo_cache
, block
->used_length
);
3944 block
->colo_cache
= NULL
;
3949 qemu_mutex_destroy(&ram_state
->bitmap_mutex
);
3955 * ram_load_setup: Setup RAM for migration incoming side
3957 * Returns zero to indicate success and negative for error
3959 * @f: QEMUFile where to receive the data
3960 * @opaque: RAMState pointer
3962 static int ram_load_setup(QEMUFile
*f
, void *opaque
)
3964 if (compress_threads_load_setup(f
)) {
3968 xbzrle_load_setup();
3969 ramblock_recv_map_init();
3974 static int ram_load_cleanup(void *opaque
)
3978 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
3979 if (ramblock_is_pmem(rb
)) {
3980 pmem_persist(rb
->host
, rb
->used_length
);
3984 xbzrle_load_cleanup();
3985 compress_threads_load_cleanup();
3987 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
3988 g_free(rb
->receivedmap
);
3989 rb
->receivedmap
= NULL
;
3996 * ram_postcopy_incoming_init: allocate postcopy data structures
3998 * Returns 0 for success and negative if there was one error
4000 * @mis: current migration incoming state
4002 * Allocate data structures etc needed by incoming migration with
4003 * postcopy-ram. postcopy-ram's similarly names
4004 * postcopy_ram_incoming_init does the work.
4006 int ram_postcopy_incoming_init(MigrationIncomingState
*mis
)
4008 return postcopy_ram_incoming_init(mis
);
4012 * ram_load_postcopy: load a page in postcopy case
4014 * Returns 0 for success or -errno in case of error
4016 * Called in postcopy mode by ram_load().
4017 * rcu_read_lock is taken prior to this being called.
4019 * @f: QEMUFile where to send the data
4021 static int ram_load_postcopy(QEMUFile
*f
)
4023 int flags
= 0, ret
= 0;
4024 bool place_needed
= false;
4025 bool matches_target_page_size
= false;
4026 MigrationIncomingState
*mis
= migration_incoming_get_current();
4027 /* Temporary page that is later 'placed' */
4028 void *postcopy_host_page
= postcopy_get_tmp_page(mis
);
4029 void *last_host
= NULL
;
4030 bool all_zero
= false;
4032 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
4035 void *page_buffer
= NULL
;
4036 void *place_source
= NULL
;
4037 RAMBlock
*block
= NULL
;
4040 addr
= qemu_get_be64(f
);
4043 * If qemu file error, we should stop here, and then "addr"
4046 ret
= qemu_file_get_error(f
);
4051 flags
= addr
& ~TARGET_PAGE_MASK
;
4052 addr
&= TARGET_PAGE_MASK
;
4054 trace_ram_load_postcopy_loop((uint64_t)addr
, flags
);
4055 place_needed
= false;
4056 if (flags
& (RAM_SAVE_FLAG_ZERO
| RAM_SAVE_FLAG_PAGE
)) {
4057 block
= ram_block_from_stream(f
, flags
);
4059 host
= host_from_ram_block_offset(block
, addr
);
4061 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
4065 matches_target_page_size
= block
->page_size
== TARGET_PAGE_SIZE
;
4067 * Postcopy requires that we place whole host pages atomically;
4068 * these may be huge pages for RAMBlocks that are backed by
4070 * To make it atomic, the data is read into a temporary page
4071 * that's moved into place later.
4072 * The migration protocol uses, possibly smaller, target-pages
4073 * however the source ensures it always sends all the components
4074 * of a host page in order.
4076 page_buffer
= postcopy_host_page
+
4077 ((uintptr_t)host
& (block
->page_size
- 1));
4078 /* If all TP are zero then we can optimise the place */
4079 if (!((uintptr_t)host
& (block
->page_size
- 1))) {
4082 /* not the 1st TP within the HP */
4083 if (host
!= (last_host
+ TARGET_PAGE_SIZE
)) {
4084 error_report("Non-sequential target page %p/%p",
4093 * If it's the last part of a host page then we place the host
4096 place_needed
= (((uintptr_t)host
+ TARGET_PAGE_SIZE
) &
4097 (block
->page_size
- 1)) == 0;
4098 place_source
= postcopy_host_page
;
4102 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
4103 case RAM_SAVE_FLAG_ZERO
:
4104 ch
= qemu_get_byte(f
);
4105 memset(page_buffer
, ch
, TARGET_PAGE_SIZE
);
4111 case RAM_SAVE_FLAG_PAGE
:
4113 if (!matches_target_page_size
) {
4114 /* For huge pages, we always use temporary buffer */
4115 qemu_get_buffer(f
, page_buffer
, TARGET_PAGE_SIZE
);
4118 * For small pages that matches target page size, we
4119 * avoid the qemu_file copy. Instead we directly use
4120 * the buffer of QEMUFile to place the page. Note: we
4121 * cannot do any QEMUFile operation before using that
4122 * buffer to make sure the buffer is valid when
4125 qemu_get_buffer_in_place(f
, (uint8_t **)&place_source
,
4129 case RAM_SAVE_FLAG_EOS
:
4131 multifd_recv_sync_main();
4134 error_report("Unknown combination of migration flags: %#x"
4135 " (postcopy mode)", flags
);
4140 /* Detect for any possible file errors */
4141 if (!ret
&& qemu_file_get_error(f
)) {
4142 ret
= qemu_file_get_error(f
);
4145 if (!ret
&& place_needed
) {
4146 /* This gets called at the last target page in the host page */
4147 void *place_dest
= host
+ TARGET_PAGE_SIZE
- block
->page_size
;
4150 ret
= postcopy_place_page_zero(mis
, place_dest
,
4153 ret
= postcopy_place_page(mis
, place_dest
,
4154 place_source
, block
);
4162 static bool postcopy_is_advised(void)
4164 PostcopyState ps
= postcopy_state_get();
4165 return ps
>= POSTCOPY_INCOMING_ADVISE
&& ps
< POSTCOPY_INCOMING_END
;
4168 static bool postcopy_is_running(void)
4170 PostcopyState ps
= postcopy_state_get();
4171 return ps
>= POSTCOPY_INCOMING_LISTENING
&& ps
< POSTCOPY_INCOMING_END
;
4175 * Flush content of RAM cache into SVM's memory.
4176 * Only flush the pages that be dirtied by PVM or SVM or both.
4178 static void colo_flush_ram_cache(void)
4180 RAMBlock
*block
= NULL
;
4183 unsigned long offset
= 0;
4185 memory_global_dirty_log_sync();
4187 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
4188 migration_bitmap_sync_range(ram_state
, block
, block
->used_length
);
4192 trace_colo_flush_ram_cache_begin(ram_state
->migration_dirty_pages
);
4194 block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
4197 offset
= migration_bitmap_find_dirty(ram_state
, block
, offset
);
4199 if (offset
<< TARGET_PAGE_BITS
>= block
->used_length
) {
4201 block
= QLIST_NEXT_RCU(block
, next
);
4203 migration_bitmap_clear_dirty(ram_state
, block
, offset
);
4204 dst_host
= block
->host
+ (offset
<< TARGET_PAGE_BITS
);
4205 src_host
= block
->colo_cache
+ (offset
<< TARGET_PAGE_BITS
);
4206 memcpy(dst_host
, src_host
, TARGET_PAGE_SIZE
);
4211 trace_colo_flush_ram_cache_end();
4214 static int ram_load(QEMUFile
*f
, void *opaque
, int version_id
)
4216 int flags
= 0, ret
= 0, invalid_flags
= 0;
4217 static uint64_t seq_iter
;
4220 * If system is running in postcopy mode, page inserts to host memory must
4223 bool postcopy_running
= postcopy_is_running();
4224 /* ADVISE is earlier, it shows the source has the postcopy capability on */
4225 bool postcopy_advised
= postcopy_is_advised();
4229 if (version_id
!= 4) {
4233 if (!migrate_use_compression()) {
4234 invalid_flags
|= RAM_SAVE_FLAG_COMPRESS_PAGE
;
4236 /* This RCU critical section can be very long running.
4237 * When RCU reclaims in the code start to become numerous,
4238 * it will be necessary to reduce the granularity of this
4243 if (postcopy_running
) {
4244 ret
= ram_load_postcopy(f
);
4247 while (!postcopy_running
&& !ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
4248 ram_addr_t addr
, total_ram_bytes
;
4252 addr
= qemu_get_be64(f
);
4253 flags
= addr
& ~TARGET_PAGE_MASK
;
4254 addr
&= TARGET_PAGE_MASK
;
4256 if (flags
& invalid_flags
) {
4257 if (flags
& invalid_flags
& RAM_SAVE_FLAG_COMPRESS_PAGE
) {
4258 error_report("Received an unexpected compressed page");
4265 if (flags
& (RAM_SAVE_FLAG_ZERO
| RAM_SAVE_FLAG_PAGE
|
4266 RAM_SAVE_FLAG_COMPRESS_PAGE
| RAM_SAVE_FLAG_XBZRLE
)) {
4267 RAMBlock
*block
= ram_block_from_stream(f
, flags
);
4270 * After going into COLO, we should load the Page into colo_cache.
4272 if (migration_incoming_in_colo_state()) {
4273 host
= colo_cache_from_block_offset(block
, addr
);
4275 host
= host_from_ram_block_offset(block
, addr
);
4278 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
4283 if (!migration_incoming_in_colo_state()) {
4284 ramblock_recv_bitmap_set(block
, host
);
4287 trace_ram_load_loop(block
->idstr
, (uint64_t)addr
, flags
, host
);
4290 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
4291 case RAM_SAVE_FLAG_MEM_SIZE
:
4292 /* Synchronize RAM block list */
4293 total_ram_bytes
= addr
;
4294 while (!ret
&& total_ram_bytes
) {
4299 len
= qemu_get_byte(f
);
4300 qemu_get_buffer(f
, (uint8_t *)id
, len
);
4302 length
= qemu_get_be64(f
);
4304 block
= qemu_ram_block_by_name(id
);
4305 if (block
&& !qemu_ram_is_migratable(block
)) {
4306 error_report("block %s should not be migrated !", id
);
4309 if (length
!= block
->used_length
) {
4310 Error
*local_err
= NULL
;
4312 ret
= qemu_ram_resize(block
, length
,
4315 error_report_err(local_err
);
4318 /* For postcopy we need to check hugepage sizes match */
4319 if (postcopy_advised
&&
4320 block
->page_size
!= qemu_host_page_size
) {
4321 uint64_t remote_page_size
= qemu_get_be64(f
);
4322 if (remote_page_size
!= block
->page_size
) {
4323 error_report("Mismatched RAM page size %s "
4324 "(local) %zd != %" PRId64
,
4325 id
, block
->page_size
,
4330 if (migrate_ignore_shared()) {
4331 hwaddr addr
= qemu_get_be64(f
);
4332 bool ignored
= qemu_get_byte(f
);
4333 if (ignored
!= ramblock_is_ignored(block
)) {
4334 error_report("RAM block %s should %s be migrated",
4335 id
, ignored
? "" : "not");
4338 if (ramblock_is_ignored(block
) &&
4339 block
->mr
->addr
!= addr
) {
4340 error_report("Mismatched GPAs for block %s "
4341 "%" PRId64
"!= %" PRId64
,
4343 (uint64_t)block
->mr
->addr
);
4347 ram_control_load_hook(f
, RAM_CONTROL_BLOCK_REG
,
4350 error_report("Unknown ramblock \"%s\", cannot "
4351 "accept migration", id
);
4355 total_ram_bytes
-= length
;
4359 case RAM_SAVE_FLAG_ZERO
:
4360 ch
= qemu_get_byte(f
);
4361 ram_handle_compressed(host
, ch
, TARGET_PAGE_SIZE
);
4364 case RAM_SAVE_FLAG_PAGE
:
4365 qemu_get_buffer(f
, host
, TARGET_PAGE_SIZE
);
4368 case RAM_SAVE_FLAG_COMPRESS_PAGE
:
4369 len
= qemu_get_be32(f
);
4370 if (len
< 0 || len
> compressBound(TARGET_PAGE_SIZE
)) {
4371 error_report("Invalid compressed data length: %d", len
);
4375 decompress_data_with_multi_threads(f
, host
, len
);
4378 case RAM_SAVE_FLAG_XBZRLE
:
4379 if (load_xbzrle(f
, addr
, host
) < 0) {
4380 error_report("Failed to decompress XBZRLE page at "
4381 RAM_ADDR_FMT
, addr
);
4386 case RAM_SAVE_FLAG_EOS
:
4388 multifd_recv_sync_main();
4391 if (flags
& RAM_SAVE_FLAG_HOOK
) {
4392 ram_control_load_hook(f
, RAM_CONTROL_HOOK
, NULL
);
4394 error_report("Unknown combination of migration flags: %#x",
4400 ret
= qemu_file_get_error(f
);
4404 ret
|= wait_for_decompress_done();
4406 trace_ram_load_complete(ret
, seq_iter
);
4408 if (!ret
&& migration_incoming_in_colo_state()) {
4409 colo_flush_ram_cache();
4414 static bool ram_has_postcopy(void *opaque
)
4417 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
4418 if (ramblock_is_pmem(rb
)) {
4419 info_report("Block: %s, host: %p is a nvdimm memory, postcopy"
4420 "is not supported now!", rb
->idstr
, rb
->host
);
4425 return migrate_postcopy_ram();
4428 /* Sync all the dirty bitmap with destination VM. */
4429 static int ram_dirty_bitmap_sync_all(MigrationState
*s
, RAMState
*rs
)
4432 QEMUFile
*file
= s
->to_dst_file
;
4433 int ramblock_count
= 0;
4435 trace_ram_dirty_bitmap_sync_start();
4437 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
4438 qemu_savevm_send_recv_bitmap(file
, block
->idstr
);
4439 trace_ram_dirty_bitmap_request(block
->idstr
);
4443 trace_ram_dirty_bitmap_sync_wait();
4445 /* Wait until all the ramblocks' dirty bitmap synced */
4446 while (ramblock_count
--) {
4447 qemu_sem_wait(&s
->rp_state
.rp_sem
);
4450 trace_ram_dirty_bitmap_sync_complete();
4455 static void ram_dirty_bitmap_reload_notify(MigrationState
*s
)
4457 qemu_sem_post(&s
->rp_state
.rp_sem
);
4461 * Read the received bitmap, revert it as the initial dirty bitmap.
4462 * This is only used when the postcopy migration is paused but wants
4463 * to resume from a middle point.
4465 int ram_dirty_bitmap_reload(MigrationState
*s
, RAMBlock
*block
)
4468 QEMUFile
*file
= s
->rp_state
.from_dst_file
;
4469 unsigned long *le_bitmap
, nbits
= block
->used_length
>> TARGET_PAGE_BITS
;
4470 uint64_t local_size
= DIV_ROUND_UP(nbits
, 8);
4471 uint64_t size
, end_mark
;
4473 trace_ram_dirty_bitmap_reload_begin(block
->idstr
);
4475 if (s
->state
!= MIGRATION_STATUS_POSTCOPY_RECOVER
) {
4476 error_report("%s: incorrect state %s", __func__
,
4477 MigrationStatus_str(s
->state
));
4482 * Note: see comments in ramblock_recv_bitmap_send() on why we
4483 * need the endianess convertion, and the paddings.
4485 local_size
= ROUND_UP(local_size
, 8);
4488 le_bitmap
= bitmap_new(nbits
+ BITS_PER_LONG
);
4490 size
= qemu_get_be64(file
);
4492 /* The size of the bitmap should match with our ramblock */
4493 if (size
!= local_size
) {
4494 error_report("%s: ramblock '%s' bitmap size mismatch "
4495 "(0x%"PRIx64
" != 0x%"PRIx64
")", __func__
,
4496 block
->idstr
, size
, local_size
);
4501 size
= qemu_get_buffer(file
, (uint8_t *)le_bitmap
, local_size
);
4502 end_mark
= qemu_get_be64(file
);
4504 ret
= qemu_file_get_error(file
);
4505 if (ret
|| size
!= local_size
) {
4506 error_report("%s: read bitmap failed for ramblock '%s': %d"
4507 " (size 0x%"PRIx64
", got: 0x%"PRIx64
")",
4508 __func__
, block
->idstr
, ret
, local_size
, size
);
4513 if (end_mark
!= RAMBLOCK_RECV_BITMAP_ENDING
) {
4514 error_report("%s: ramblock '%s' end mark incorrect: 0x%"PRIu64
,
4515 __func__
, block
->idstr
, end_mark
);
4521 * Endianess convertion. We are during postcopy (though paused).
4522 * The dirty bitmap won't change. We can directly modify it.
4524 bitmap_from_le(block
->bmap
, le_bitmap
, nbits
);
4527 * What we received is "received bitmap". Revert it as the initial
4528 * dirty bitmap for this ramblock.
4530 bitmap_complement(block
->bmap
, block
->bmap
, nbits
);
4532 trace_ram_dirty_bitmap_reload_complete(block
->idstr
);
4535 * We succeeded to sync bitmap for current ramblock. If this is
4536 * the last one to sync, we need to notify the main send thread.
4538 ram_dirty_bitmap_reload_notify(s
);
4546 static int ram_resume_prepare(MigrationState
*s
, void *opaque
)
4548 RAMState
*rs
= *(RAMState
**)opaque
;
4551 ret
= ram_dirty_bitmap_sync_all(s
, rs
);
4556 ram_state_resume_prepare(rs
, s
->to_dst_file
);
4561 static SaveVMHandlers savevm_ram_handlers
= {
4562 .save_setup
= ram_save_setup
,
4563 .save_live_iterate
= ram_save_iterate
,
4564 .save_live_complete_postcopy
= ram_save_complete
,
4565 .save_live_complete_precopy
= ram_save_complete
,
4566 .has_postcopy
= ram_has_postcopy
,
4567 .save_live_pending
= ram_save_pending
,
4568 .load_state
= ram_load
,
4569 .save_cleanup
= ram_save_cleanup
,
4570 .load_setup
= ram_load_setup
,
4571 .load_cleanup
= ram_load_cleanup
,
4572 .resume_prepare
= ram_resume_prepare
,
4575 void ram_mig_init(void)
4577 qemu_mutex_init(&XBZRLE
.lock
);
4578 register_savevm_live(NULL
, "ram", 0, 4, &savevm_ram_handlers
, &ram_state
);