4 * Copyright (c) 2003-2008 Fabrice Bellard
5 * Copyright (c) 2011-2015 Red Hat Inc
8 * Juan Quintela <quintela@redhat.com>
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29 #include "qemu/osdep.h"
32 #include "qemu/cutils.h"
33 #include "qemu/bitops.h"
34 #include "qemu/bitmap.h"
35 #include "qemu/main-loop.h"
36 #include "qemu/pmem.h"
39 #include "migration.h"
41 #include "migration/register.h"
42 #include "migration/misc.h"
43 #include "qemu-file.h"
44 #include "postcopy-ram.h"
45 #include "page_cache.h"
46 #include "qemu/error-report.h"
47 #include "qapi/error.h"
48 #include "qapi/qapi-events-migration.h"
49 #include "qapi/qmp/qerror.h"
51 #include "exec/ram_addr.h"
52 #include "exec/target_page.h"
53 #include "qemu/rcu_queue.h"
54 #include "migration/colo.h"
56 #include "sysemu/sysemu.h"
57 #include "qemu/uuid.h"
61 /***********************************************************/
62 /* ram save/restore */
64 /* RAM_SAVE_FLAG_ZERO used to be named RAM_SAVE_FLAG_COMPRESS, it
65 * worked for pages that where filled with the same char. We switched
66 * it to only search for the zero value. And to avoid confusion with
67 * RAM_SSAVE_FLAG_COMPRESS_PAGE just rename it.
70 #define RAM_SAVE_FLAG_FULL 0x01 /* Obsolete, not used anymore */
71 #define RAM_SAVE_FLAG_ZERO 0x02
72 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
73 #define RAM_SAVE_FLAG_PAGE 0x08
74 #define RAM_SAVE_FLAG_EOS 0x10
75 #define RAM_SAVE_FLAG_CONTINUE 0x20
76 #define RAM_SAVE_FLAG_XBZRLE 0x40
77 /* 0x80 is reserved in migration.h start with 0x100 next */
78 #define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
80 static inline bool is_zero_range(uint8_t *p
, uint64_t size
)
82 return buffer_is_zero(p
, size
);
85 XBZRLECacheStats xbzrle_counters
;
87 /* struct contains XBZRLE cache and a static page
88 used by the compression */
90 /* buffer used for XBZRLE encoding */
92 /* buffer for storing page content */
94 /* Cache for XBZRLE, Protected by lock. */
97 /* it will store a page full of zeros */
98 uint8_t *zero_target_page
;
99 /* buffer used for XBZRLE decoding */
100 uint8_t *decoded_buf
;
103 static void XBZRLE_cache_lock(void)
105 if (migrate_use_xbzrle())
106 qemu_mutex_lock(&XBZRLE
.lock
);
109 static void XBZRLE_cache_unlock(void)
111 if (migrate_use_xbzrle())
112 qemu_mutex_unlock(&XBZRLE
.lock
);
116 * xbzrle_cache_resize: resize the xbzrle cache
118 * This function is called from qmp_migrate_set_cache_size in main
119 * thread, possibly while a migration is in progress. A running
120 * migration may be using the cache and might finish during this call,
121 * hence changes to the cache are protected by XBZRLE.lock().
123 * Returns 0 for success or -1 for error
125 * @new_size: new cache size
126 * @errp: set *errp if the check failed, with reason
128 int xbzrle_cache_resize(int64_t new_size
, Error
**errp
)
130 PageCache
*new_cache
;
133 /* Check for truncation */
134 if (new_size
!= (size_t)new_size
) {
135 error_setg(errp
, QERR_INVALID_PARAMETER_VALUE
, "cache size",
136 "exceeding address space");
140 if (new_size
== migrate_xbzrle_cache_size()) {
147 if (XBZRLE
.cache
!= NULL
) {
148 new_cache
= cache_init(new_size
, TARGET_PAGE_SIZE
, errp
);
154 cache_fini(XBZRLE
.cache
);
155 XBZRLE
.cache
= new_cache
;
158 XBZRLE_cache_unlock();
162 static bool ramblock_is_ignored(RAMBlock
*block
)
164 return !qemu_ram_is_migratable(block
) ||
165 (migrate_ignore_shared() && qemu_ram_is_shared(block
));
168 /* Should be holding either ram_list.mutex, or the RCU lock. */
169 #define RAMBLOCK_FOREACH_NOT_IGNORED(block) \
170 INTERNAL_RAMBLOCK_FOREACH(block) \
171 if (ramblock_is_ignored(block)) {} else
173 #define RAMBLOCK_FOREACH_MIGRATABLE(block) \
174 INTERNAL_RAMBLOCK_FOREACH(block) \
175 if (!qemu_ram_is_migratable(block)) {} else
177 #undef RAMBLOCK_FOREACH
179 int foreach_not_ignored_block(RAMBlockIterFunc func
, void *opaque
)
185 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
186 ret
= func(block
, opaque
);
195 static void ramblock_recv_map_init(void)
199 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
200 assert(!rb
->receivedmap
);
201 rb
->receivedmap
= bitmap_new(rb
->max_length
>> qemu_target_page_bits());
205 int ramblock_recv_bitmap_test(RAMBlock
*rb
, void *host_addr
)
207 return test_bit(ramblock_recv_bitmap_offset(host_addr
, rb
),
211 bool ramblock_recv_bitmap_test_byte_offset(RAMBlock
*rb
, uint64_t byte_offset
)
213 return test_bit(byte_offset
>> TARGET_PAGE_BITS
, rb
->receivedmap
);
216 void ramblock_recv_bitmap_set(RAMBlock
*rb
, void *host_addr
)
218 set_bit_atomic(ramblock_recv_bitmap_offset(host_addr
, rb
), rb
->receivedmap
);
221 void ramblock_recv_bitmap_set_range(RAMBlock
*rb
, void *host_addr
,
224 bitmap_set_atomic(rb
->receivedmap
,
225 ramblock_recv_bitmap_offset(host_addr
, rb
),
229 #define RAMBLOCK_RECV_BITMAP_ENDING (0x0123456789abcdefULL)
232 * Format: bitmap_size (8 bytes) + whole_bitmap (N bytes).
234 * Returns >0 if success with sent bytes, or <0 if error.
236 int64_t ramblock_recv_bitmap_send(QEMUFile
*file
,
237 const char *block_name
)
239 RAMBlock
*block
= qemu_ram_block_by_name(block_name
);
240 unsigned long *le_bitmap
, nbits
;
244 error_report("%s: invalid block name: %s", __func__
, block_name
);
248 nbits
= block
->used_length
>> TARGET_PAGE_BITS
;
251 * Make sure the tmp bitmap buffer is big enough, e.g., on 32bit
252 * machines we may need 4 more bytes for padding (see below
253 * comment). So extend it a bit before hand.
255 le_bitmap
= bitmap_new(nbits
+ BITS_PER_LONG
);
258 * Always use little endian when sending the bitmap. This is
259 * required that when source and destination VMs are not using the
260 * same endianess. (Note: big endian won't work.)
262 bitmap_to_le(le_bitmap
, block
->receivedmap
, nbits
);
264 /* Size of the bitmap, in bytes */
265 size
= DIV_ROUND_UP(nbits
, 8);
268 * size is always aligned to 8 bytes for 64bit machines, but it
269 * may not be true for 32bit machines. We need this padding to
270 * make sure the migration can survive even between 32bit and
273 size
= ROUND_UP(size
, 8);
275 qemu_put_be64(file
, size
);
276 qemu_put_buffer(file
, (const uint8_t *)le_bitmap
, size
);
278 * Mark as an end, in case the middle part is screwed up due to
279 * some "misterious" reason.
281 qemu_put_be64(file
, RAMBLOCK_RECV_BITMAP_ENDING
);
286 if (qemu_file_get_error(file
)) {
287 return qemu_file_get_error(file
);
290 return size
+ sizeof(size
);
294 * An outstanding page request, on the source, having been received
297 struct RAMSrcPageRequest
{
302 QSIMPLEQ_ENTRY(RAMSrcPageRequest
) next_req
;
305 /* State of RAM for migration */
307 /* QEMUFile used for this migration */
309 /* Last block that we have visited searching for dirty pages */
310 RAMBlock
*last_seen_block
;
311 /* Last block from where we have sent data */
312 RAMBlock
*last_sent_block
;
313 /* Last dirty target page we have sent */
314 ram_addr_t last_page
;
315 /* last ram version we have seen */
316 uint32_t last_version
;
317 /* We are in the first round */
319 /* The free page optimization is enabled */
321 /* How many times we have dirty too many pages */
322 int dirty_rate_high_cnt
;
323 /* these variables are used for bitmap sync */
324 /* last time we did a full bitmap_sync */
325 int64_t time_last_bitmap_sync
;
326 /* bytes transferred at start_time */
327 uint64_t bytes_xfer_prev
;
328 /* number of dirty pages since start_time */
329 uint64_t num_dirty_pages_period
;
330 /* xbzrle misses since the beginning of the period */
331 uint64_t xbzrle_cache_miss_prev
;
333 /* compression statistics since the beginning of the period */
334 /* amount of count that no free thread to compress data */
335 uint64_t compress_thread_busy_prev
;
336 /* amount bytes after compression */
337 uint64_t compressed_size_prev
;
338 /* amount of compressed pages */
339 uint64_t compress_pages_prev
;
341 /* total handled target pages at the beginning of period */
342 uint64_t target_page_count_prev
;
343 /* total handled target pages since start */
344 uint64_t target_page_count
;
345 /* number of dirty bits in the bitmap */
346 uint64_t migration_dirty_pages
;
347 /* Protects modification of the bitmap and migration dirty pages */
348 QemuMutex bitmap_mutex
;
349 /* The RAMBlock used in the last src_page_requests */
350 RAMBlock
*last_req_rb
;
351 /* Queue of outstanding page requests from the destination */
352 QemuMutex src_page_req_mutex
;
353 QSIMPLEQ_HEAD(, RAMSrcPageRequest
) src_page_requests
;
355 typedef struct RAMState RAMState
;
357 static RAMState
*ram_state
;
359 static NotifierWithReturnList precopy_notifier_list
;
361 void precopy_infrastructure_init(void)
363 notifier_with_return_list_init(&precopy_notifier_list
);
366 void precopy_add_notifier(NotifierWithReturn
*n
)
368 notifier_with_return_list_add(&precopy_notifier_list
, n
);
371 void precopy_remove_notifier(NotifierWithReturn
*n
)
373 notifier_with_return_remove(n
);
376 int precopy_notify(PrecopyNotifyReason reason
, Error
**errp
)
378 PrecopyNotifyData pnd
;
382 return notifier_with_return_list_notify(&precopy_notifier_list
, &pnd
);
385 void precopy_enable_free_page_optimization(void)
391 ram_state
->fpo_enabled
= true;
394 uint64_t ram_bytes_remaining(void)
396 return ram_state
? (ram_state
->migration_dirty_pages
* TARGET_PAGE_SIZE
) :
400 MigrationStats ram_counters
;
402 /* used by the search for pages to send */
403 struct PageSearchStatus
{
404 /* Current block being searched */
406 /* Current page to search from */
408 /* Set once we wrap around */
411 typedef struct PageSearchStatus PageSearchStatus
;
413 CompressionStats compression_counters
;
415 struct CompressParam
{
425 /* internally used fields */
429 typedef struct CompressParam CompressParam
;
431 struct DecompressParam
{
441 typedef struct DecompressParam DecompressParam
;
443 static CompressParam
*comp_param
;
444 static QemuThread
*compress_threads
;
445 /* comp_done_cond is used to wake up the migration thread when
446 * one of the compression threads has finished the compression.
447 * comp_done_lock is used to co-work with comp_done_cond.
449 static QemuMutex comp_done_lock
;
450 static QemuCond comp_done_cond
;
451 /* The empty QEMUFileOps will be used by file in CompressParam */
452 static const QEMUFileOps empty_ops
= { };
454 static QEMUFile
*decomp_file
;
455 static DecompressParam
*decomp_param
;
456 static QemuThread
*decompress_threads
;
457 static QemuMutex decomp_done_lock
;
458 static QemuCond decomp_done_cond
;
460 static bool do_compress_ram_page(QEMUFile
*f
, z_stream
*stream
, RAMBlock
*block
,
461 ram_addr_t offset
, uint8_t *source_buf
);
463 static void *do_data_compress(void *opaque
)
465 CompressParam
*param
= opaque
;
470 qemu_mutex_lock(¶m
->mutex
);
471 while (!param
->quit
) {
473 block
= param
->block
;
474 offset
= param
->offset
;
476 qemu_mutex_unlock(¶m
->mutex
);
478 zero_page
= do_compress_ram_page(param
->file
, ¶m
->stream
,
479 block
, offset
, param
->originbuf
);
481 qemu_mutex_lock(&comp_done_lock
);
483 param
->zero_page
= zero_page
;
484 qemu_cond_signal(&comp_done_cond
);
485 qemu_mutex_unlock(&comp_done_lock
);
487 qemu_mutex_lock(¶m
->mutex
);
489 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
492 qemu_mutex_unlock(¶m
->mutex
);
497 static void compress_threads_save_cleanup(void)
501 if (!migrate_use_compression() || !comp_param
) {
505 thread_count
= migrate_compress_threads();
506 for (i
= 0; i
< thread_count
; i
++) {
508 * we use it as a indicator which shows if the thread is
509 * properly init'd or not
511 if (!comp_param
[i
].file
) {
515 qemu_mutex_lock(&comp_param
[i
].mutex
);
516 comp_param
[i
].quit
= true;
517 qemu_cond_signal(&comp_param
[i
].cond
);
518 qemu_mutex_unlock(&comp_param
[i
].mutex
);
520 qemu_thread_join(compress_threads
+ i
);
521 qemu_mutex_destroy(&comp_param
[i
].mutex
);
522 qemu_cond_destroy(&comp_param
[i
].cond
);
523 deflateEnd(&comp_param
[i
].stream
);
524 g_free(comp_param
[i
].originbuf
);
525 qemu_fclose(comp_param
[i
].file
);
526 comp_param
[i
].file
= NULL
;
528 qemu_mutex_destroy(&comp_done_lock
);
529 qemu_cond_destroy(&comp_done_cond
);
530 g_free(compress_threads
);
532 compress_threads
= NULL
;
536 static int compress_threads_save_setup(void)
540 if (!migrate_use_compression()) {
543 thread_count
= migrate_compress_threads();
544 compress_threads
= g_new0(QemuThread
, thread_count
);
545 comp_param
= g_new0(CompressParam
, thread_count
);
546 qemu_cond_init(&comp_done_cond
);
547 qemu_mutex_init(&comp_done_lock
);
548 for (i
= 0; i
< thread_count
; i
++) {
549 comp_param
[i
].originbuf
= g_try_malloc(TARGET_PAGE_SIZE
);
550 if (!comp_param
[i
].originbuf
) {
554 if (deflateInit(&comp_param
[i
].stream
,
555 migrate_compress_level()) != Z_OK
) {
556 g_free(comp_param
[i
].originbuf
);
560 /* comp_param[i].file is just used as a dummy buffer to save data,
561 * set its ops to empty.
563 comp_param
[i
].file
= qemu_fopen_ops(NULL
, &empty_ops
);
564 comp_param
[i
].done
= true;
565 comp_param
[i
].quit
= false;
566 qemu_mutex_init(&comp_param
[i
].mutex
);
567 qemu_cond_init(&comp_param
[i
].cond
);
568 qemu_thread_create(compress_threads
+ i
, "compress",
569 do_data_compress
, comp_param
+ i
,
570 QEMU_THREAD_JOINABLE
);
575 compress_threads_save_cleanup();
581 #define MULTIFD_MAGIC 0x11223344U
582 #define MULTIFD_VERSION 1
584 #define MULTIFD_FLAG_SYNC (1 << 0)
586 /* This value needs to be a multiple of qemu_target_page_size() */
587 #define MULTIFD_PACKET_SIZE (512 * 1024)
592 unsigned char uuid
[16]; /* QemuUUID */
594 uint8_t unused1
[7]; /* Reserved for future use */
595 uint64_t unused2
[4]; /* Reserved for future use */
596 } __attribute__((packed
)) MultiFDInit_t
;
602 /* maximum number of allocated pages */
603 uint32_t pages_alloc
;
605 /* size of the next packet that contains pages */
606 uint32_t next_packet_size
;
608 uint64_t unused
[4]; /* Reserved for future use */
611 } __attribute__((packed
)) MultiFDPacket_t
;
614 /* number of used pages */
616 /* number of allocated pages */
618 /* global number of generated multifd packets */
620 /* offset of each page */
622 /* pointer to each page */
628 /* this fields are not changed once the thread is created */
631 /* channel thread name */
633 /* channel thread id */
635 /* communication channel */
637 /* sem where to wait for more work */
639 /* this mutex protects the following parameters */
641 /* is this channel thread running */
643 /* should this thread finish */
645 /* thread has work to do */
647 /* array of pages to sent */
648 MultiFDPages_t
*pages
;
649 /* packet allocated len */
651 /* pointer to the packet */
652 MultiFDPacket_t
*packet
;
653 /* multifd flags for each packet */
655 /* size of the next packet that contains pages */
656 uint32_t next_packet_size
;
657 /* global number of generated multifd packets */
659 /* thread local variables */
660 /* packets sent through this channel */
661 uint64_t num_packets
;
662 /* pages sent through this channel */
664 /* syncs main thread and channels */
665 QemuSemaphore sem_sync
;
669 /* this fields are not changed once the thread is created */
672 /* channel thread name */
674 /* channel thread id */
676 /* communication channel */
678 /* this mutex protects the following parameters */
680 /* is this channel thread running */
682 /* should this thread finish */
684 /* array of pages to receive */
685 MultiFDPages_t
*pages
;
686 /* packet allocated len */
688 /* pointer to the packet */
689 MultiFDPacket_t
*packet
;
690 /* multifd flags for each packet */
692 /* global number of generated multifd packets */
694 /* thread local variables */
695 /* size of the next packet that contains pages */
696 uint32_t next_packet_size
;
697 /* packets sent through this channel */
698 uint64_t num_packets
;
699 /* pages sent through this channel */
701 /* syncs main thread and channels */
702 QemuSemaphore sem_sync
;
705 static int multifd_send_initial_packet(MultiFDSendParams
*p
, Error
**errp
)
710 msg
.magic
= cpu_to_be32(MULTIFD_MAGIC
);
711 msg
.version
= cpu_to_be32(MULTIFD_VERSION
);
713 memcpy(msg
.uuid
, &qemu_uuid
.data
, sizeof(msg
.uuid
));
715 ret
= qio_channel_write_all(p
->c
, (char *)&msg
, sizeof(msg
), errp
);
722 static int multifd_recv_initial_packet(QIOChannel
*c
, Error
**errp
)
727 ret
= qio_channel_read_all(c
, (char *)&msg
, sizeof(msg
), errp
);
732 msg
.magic
= be32_to_cpu(msg
.magic
);
733 msg
.version
= be32_to_cpu(msg
.version
);
735 if (msg
.magic
!= MULTIFD_MAGIC
) {
736 error_setg(errp
, "multifd: received packet magic %x "
737 "expected %x", msg
.magic
, MULTIFD_MAGIC
);
741 if (msg
.version
!= MULTIFD_VERSION
) {
742 error_setg(errp
, "multifd: received packet version %d "
743 "expected %d", msg
.version
, MULTIFD_VERSION
);
747 if (memcmp(msg
.uuid
, &qemu_uuid
, sizeof(qemu_uuid
))) {
748 char *uuid
= qemu_uuid_unparse_strdup(&qemu_uuid
);
749 char *msg_uuid
= qemu_uuid_unparse_strdup((const QemuUUID
*)msg
.uuid
);
751 error_setg(errp
, "multifd: received uuid '%s' and expected "
752 "uuid '%s' for channel %hhd", msg_uuid
, uuid
, msg
.id
);
758 if (msg
.id
> migrate_multifd_channels()) {
759 error_setg(errp
, "multifd: received channel version %d "
760 "expected %d", msg
.version
, MULTIFD_VERSION
);
767 static MultiFDPages_t
*multifd_pages_init(size_t size
)
769 MultiFDPages_t
*pages
= g_new0(MultiFDPages_t
, 1);
771 pages
->allocated
= size
;
772 pages
->iov
= g_new0(struct iovec
, size
);
773 pages
->offset
= g_new0(ram_addr_t
, size
);
778 static void multifd_pages_clear(MultiFDPages_t
*pages
)
781 pages
->allocated
= 0;
782 pages
->packet_num
= 0;
786 g_free(pages
->offset
);
787 pages
->offset
= NULL
;
791 static void multifd_send_fill_packet(MultiFDSendParams
*p
)
793 MultiFDPacket_t
*packet
= p
->packet
;
794 uint32_t page_max
= MULTIFD_PACKET_SIZE
/ qemu_target_page_size();
797 packet
->magic
= cpu_to_be32(MULTIFD_MAGIC
);
798 packet
->version
= cpu_to_be32(MULTIFD_VERSION
);
799 packet
->flags
= cpu_to_be32(p
->flags
);
800 packet
->pages_alloc
= cpu_to_be32(page_max
);
801 packet
->pages_used
= cpu_to_be32(p
->pages
->used
);
802 packet
->next_packet_size
= cpu_to_be32(p
->next_packet_size
);
803 packet
->packet_num
= cpu_to_be64(p
->packet_num
);
805 if (p
->pages
->block
) {
806 strncpy(packet
->ramblock
, p
->pages
->block
->idstr
, 256);
809 for (i
= 0; i
< p
->pages
->used
; i
++) {
810 packet
->offset
[i
] = cpu_to_be64(p
->pages
->offset
[i
]);
814 static int multifd_recv_unfill_packet(MultiFDRecvParams
*p
, Error
**errp
)
816 MultiFDPacket_t
*packet
= p
->packet
;
817 uint32_t pages_max
= MULTIFD_PACKET_SIZE
/ qemu_target_page_size();
821 packet
->magic
= be32_to_cpu(packet
->magic
);
822 if (packet
->magic
!= MULTIFD_MAGIC
) {
823 error_setg(errp
, "multifd: received packet "
824 "magic %x and expected magic %x",
825 packet
->magic
, MULTIFD_MAGIC
);
829 packet
->version
= be32_to_cpu(packet
->version
);
830 if (packet
->version
!= MULTIFD_VERSION
) {
831 error_setg(errp
, "multifd: received packet "
832 "version %d and expected version %d",
833 packet
->version
, MULTIFD_VERSION
);
837 p
->flags
= be32_to_cpu(packet
->flags
);
839 packet
->pages_alloc
= be32_to_cpu(packet
->pages_alloc
);
841 * If we recevied a packet that is 100 times bigger than expected
842 * just stop migration. It is a magic number.
844 if (packet
->pages_alloc
> pages_max
* 100) {
845 error_setg(errp
, "multifd: received packet "
846 "with size %d and expected a maximum size of %d",
847 packet
->pages_alloc
, pages_max
* 100) ;
851 * We received a packet that is bigger than expected but inside
852 * reasonable limits (see previous comment). Just reallocate.
854 if (packet
->pages_alloc
> p
->pages
->allocated
) {
855 multifd_pages_clear(p
->pages
);
856 p
->pages
= multifd_pages_init(packet
->pages_alloc
);
859 p
->pages
->used
= be32_to_cpu(packet
->pages_used
);
860 if (p
->pages
->used
> packet
->pages_alloc
) {
861 error_setg(errp
, "multifd: received packet "
862 "with %d pages and expected maximum pages are %d",
863 p
->pages
->used
, packet
->pages_alloc
) ;
867 p
->next_packet_size
= be32_to_cpu(packet
->next_packet_size
);
868 p
->packet_num
= be64_to_cpu(packet
->packet_num
);
870 if (p
->pages
->used
) {
871 /* make sure that ramblock is 0 terminated */
872 packet
->ramblock
[255] = 0;
873 block
= qemu_ram_block_by_name(packet
->ramblock
);
875 error_setg(errp
, "multifd: unknown ram block %s",
881 for (i
= 0; i
< p
->pages
->used
; i
++) {
882 ram_addr_t offset
= be64_to_cpu(packet
->offset
[i
]);
884 if (offset
> (block
->used_length
- TARGET_PAGE_SIZE
)) {
885 error_setg(errp
, "multifd: offset too long " RAM_ADDR_FMT
886 " (max " RAM_ADDR_FMT
")",
887 offset
, block
->max_length
);
890 p
->pages
->iov
[i
].iov_base
= block
->host
+ offset
;
891 p
->pages
->iov
[i
].iov_len
= TARGET_PAGE_SIZE
;
898 MultiFDSendParams
*params
;
899 /* array of pages to sent */
900 MultiFDPages_t
*pages
;
901 /* global number of generated multifd packets */
903 /* send channels ready */
904 QemuSemaphore channels_ready
;
905 } *multifd_send_state
;
908 * How we use multifd_send_state->pages and channel->pages?
910 * We create a pages for each channel, and a main one. Each time that
911 * we need to send a batch of pages we interchange the ones between
912 * multifd_send_state and the channel that is sending it. There are
913 * two reasons for that:
914 * - to not have to do so many mallocs during migration
915 * - to make easier to know what to free at the end of migration
917 * This way we always know who is the owner of each "pages" struct,
918 * and we don't need any locking. It belongs to the migration thread
919 * or to the channel thread. Switching is safe because the migration
920 * thread is using the channel mutex when changing it, and the channel
921 * have to had finish with its own, otherwise pending_job can't be
925 static int multifd_send_pages(RAMState
*rs
)
928 static int next_channel
;
929 MultiFDSendParams
*p
= NULL
; /* make happy gcc */
930 MultiFDPages_t
*pages
= multifd_send_state
->pages
;
931 uint64_t transferred
;
933 qemu_sem_wait(&multifd_send_state
->channels_ready
);
934 for (i
= next_channel
;; i
= (i
+ 1) % migrate_multifd_channels()) {
935 p
= &multifd_send_state
->params
[i
];
937 qemu_mutex_lock(&p
->mutex
);
939 error_report("%s: channel %d has already quit!", __func__
, i
);
940 qemu_mutex_unlock(&p
->mutex
);
943 if (!p
->pending_job
) {
945 next_channel
= (i
+ 1) % migrate_multifd_channels();
948 qemu_mutex_unlock(&p
->mutex
);
952 p
->packet_num
= multifd_send_state
->packet_num
++;
953 p
->pages
->block
= NULL
;
954 multifd_send_state
->pages
= p
->pages
;
956 transferred
= ((uint64_t) pages
->used
) * TARGET_PAGE_SIZE
+ p
->packet_len
;
957 qemu_file_update_transfer(rs
->f
, transferred
);
958 ram_counters
.multifd_bytes
+= transferred
;
959 ram_counters
.transferred
+= transferred
;;
960 qemu_mutex_unlock(&p
->mutex
);
961 qemu_sem_post(&p
->sem
);
966 static int multifd_queue_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
)
968 MultiFDPages_t
*pages
= multifd_send_state
->pages
;
971 pages
->block
= block
;
974 if (pages
->block
== block
) {
975 pages
->offset
[pages
->used
] = offset
;
976 pages
->iov
[pages
->used
].iov_base
= block
->host
+ offset
;
977 pages
->iov
[pages
->used
].iov_len
= TARGET_PAGE_SIZE
;
980 if (pages
->used
< pages
->allocated
) {
985 if (multifd_send_pages(rs
) < 0) {
989 if (pages
->block
!= block
) {
990 return multifd_queue_page(rs
, block
, offset
);
996 static void multifd_send_terminate_threads(Error
*err
)
1000 trace_multifd_send_terminate_threads(err
!= NULL
);
1003 MigrationState
*s
= migrate_get_current();
1004 migrate_set_error(s
, err
);
1005 if (s
->state
== MIGRATION_STATUS_SETUP
||
1006 s
->state
== MIGRATION_STATUS_PRE_SWITCHOVER
||
1007 s
->state
== MIGRATION_STATUS_DEVICE
||
1008 s
->state
== MIGRATION_STATUS_ACTIVE
) {
1009 migrate_set_state(&s
->state
, s
->state
,
1010 MIGRATION_STATUS_FAILED
);
1014 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1015 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1017 qemu_mutex_lock(&p
->mutex
);
1019 qemu_sem_post(&p
->sem
);
1020 qemu_mutex_unlock(&p
->mutex
);
1024 void multifd_save_cleanup(void)
1028 if (!migrate_use_multifd()) {
1031 multifd_send_terminate_threads(NULL
);
1032 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1033 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1036 qemu_thread_join(&p
->thread
);
1038 socket_send_channel_destroy(p
->c
);
1040 qemu_mutex_destroy(&p
->mutex
);
1041 qemu_sem_destroy(&p
->sem
);
1042 qemu_sem_destroy(&p
->sem_sync
);
1045 multifd_pages_clear(p
->pages
);
1051 qemu_sem_destroy(&multifd_send_state
->channels_ready
);
1052 g_free(multifd_send_state
->params
);
1053 multifd_send_state
->params
= NULL
;
1054 multifd_pages_clear(multifd_send_state
->pages
);
1055 multifd_send_state
->pages
= NULL
;
1056 g_free(multifd_send_state
);
1057 multifd_send_state
= NULL
;
1060 static void multifd_send_sync_main(RAMState
*rs
)
1064 if (!migrate_use_multifd()) {
1067 if (multifd_send_state
->pages
->used
) {
1068 if (multifd_send_pages(rs
) < 0) {
1069 error_report("%s: multifd_send_pages fail", __func__
);
1073 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1074 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1076 trace_multifd_send_sync_main_signal(p
->id
);
1078 qemu_mutex_lock(&p
->mutex
);
1081 error_report("%s: channel %d has already quit", __func__
, i
);
1082 qemu_mutex_unlock(&p
->mutex
);
1086 p
->packet_num
= multifd_send_state
->packet_num
++;
1087 p
->flags
|= MULTIFD_FLAG_SYNC
;
1089 qemu_file_update_transfer(rs
->f
, p
->packet_len
);
1090 ram_counters
.multifd_bytes
+= p
->packet_len
;
1091 ram_counters
.transferred
+= p
->packet_len
;
1092 qemu_mutex_unlock(&p
->mutex
);
1093 qemu_sem_post(&p
->sem
);
1095 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1096 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1098 trace_multifd_send_sync_main_wait(p
->id
);
1099 qemu_sem_wait(&p
->sem_sync
);
1101 trace_multifd_send_sync_main(multifd_send_state
->packet_num
);
1104 static void *multifd_send_thread(void *opaque
)
1106 MultiFDSendParams
*p
= opaque
;
1107 Error
*local_err
= NULL
;
1111 trace_multifd_send_thread_start(p
->id
);
1112 rcu_register_thread();
1114 if (multifd_send_initial_packet(p
, &local_err
) < 0) {
1118 /* initial packet */
1122 qemu_sem_wait(&p
->sem
);
1123 qemu_mutex_lock(&p
->mutex
);
1125 if (p
->pending_job
) {
1126 uint32_t used
= p
->pages
->used
;
1127 uint64_t packet_num
= p
->packet_num
;
1130 p
->next_packet_size
= used
* qemu_target_page_size();
1131 multifd_send_fill_packet(p
);
1134 p
->num_pages
+= used
;
1136 qemu_mutex_unlock(&p
->mutex
);
1138 trace_multifd_send(p
->id
, packet_num
, used
, flags
,
1139 p
->next_packet_size
);
1141 ret
= qio_channel_write_all(p
->c
, (void *)p
->packet
,
1142 p
->packet_len
, &local_err
);
1148 ret
= qio_channel_writev_all(p
->c
, p
->pages
->iov
,
1155 qemu_mutex_lock(&p
->mutex
);
1157 qemu_mutex_unlock(&p
->mutex
);
1159 if (flags
& MULTIFD_FLAG_SYNC
) {
1160 qemu_sem_post(&p
->sem_sync
);
1162 qemu_sem_post(&multifd_send_state
->channels_ready
);
1163 } else if (p
->quit
) {
1164 qemu_mutex_unlock(&p
->mutex
);
1167 qemu_mutex_unlock(&p
->mutex
);
1168 /* sometimes there are spurious wakeups */
1174 trace_multifd_send_error(p
->id
);
1175 multifd_send_terminate_threads(local_err
);
1179 * Error happen, I will exit, but I can't just leave, tell
1180 * who pay attention to me.
1183 qemu_sem_post(&p
->sem_sync
);
1184 qemu_sem_post(&multifd_send_state
->channels_ready
);
1187 qemu_mutex_lock(&p
->mutex
);
1189 qemu_mutex_unlock(&p
->mutex
);
1191 rcu_unregister_thread();
1192 trace_multifd_send_thread_end(p
->id
, p
->num_packets
, p
->num_pages
);
1197 static void multifd_new_send_channel_async(QIOTask
*task
, gpointer opaque
)
1199 MultiFDSendParams
*p
= opaque
;
1200 QIOChannel
*sioc
= QIO_CHANNEL(qio_task_get_source(task
));
1201 Error
*local_err
= NULL
;
1203 trace_multifd_new_send_channel_async(p
->id
);
1204 if (qio_task_propagate_error(task
, &local_err
)) {
1205 migrate_set_error(migrate_get_current(), local_err
);
1206 multifd_save_cleanup();
1208 p
->c
= QIO_CHANNEL(sioc
);
1209 qio_channel_set_delay(p
->c
, false);
1211 qemu_thread_create(&p
->thread
, p
->name
, multifd_send_thread
, p
,
1212 QEMU_THREAD_JOINABLE
);
1216 int multifd_save_setup(void)
1219 uint32_t page_count
= MULTIFD_PACKET_SIZE
/ qemu_target_page_size();
1222 if (!migrate_use_multifd()) {
1225 thread_count
= migrate_multifd_channels();
1226 multifd_send_state
= g_malloc0(sizeof(*multifd_send_state
));
1227 multifd_send_state
->params
= g_new0(MultiFDSendParams
, thread_count
);
1228 multifd_send_state
->pages
= multifd_pages_init(page_count
);
1229 qemu_sem_init(&multifd_send_state
->channels_ready
, 0);
1231 for (i
= 0; i
< thread_count
; i
++) {
1232 MultiFDSendParams
*p
= &multifd_send_state
->params
[i
];
1234 qemu_mutex_init(&p
->mutex
);
1235 qemu_sem_init(&p
->sem
, 0);
1236 qemu_sem_init(&p
->sem_sync
, 0);
1240 p
->pages
= multifd_pages_init(page_count
);
1241 p
->packet_len
= sizeof(MultiFDPacket_t
)
1242 + sizeof(ram_addr_t
) * page_count
;
1243 p
->packet
= g_malloc0(p
->packet_len
);
1244 p
->name
= g_strdup_printf("multifdsend_%d", i
);
1245 socket_send_channel_create(multifd_new_send_channel_async
, p
);
1251 MultiFDRecvParams
*params
;
1252 /* number of created threads */
1254 /* syncs main thread and channels */
1255 QemuSemaphore sem_sync
;
1256 /* global number of generated multifd packets */
1257 uint64_t packet_num
;
1258 } *multifd_recv_state
;
1260 static void multifd_recv_terminate_threads(Error
*err
)
1264 trace_multifd_recv_terminate_threads(err
!= NULL
);
1267 MigrationState
*s
= migrate_get_current();
1268 migrate_set_error(s
, err
);
1269 if (s
->state
== MIGRATION_STATUS_SETUP
||
1270 s
->state
== MIGRATION_STATUS_ACTIVE
) {
1271 migrate_set_state(&s
->state
, s
->state
,
1272 MIGRATION_STATUS_FAILED
);
1276 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1277 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1279 qemu_mutex_lock(&p
->mutex
);
1281 /* We could arrive here for two reasons:
1282 - normal quit, i.e. everything went fine, just finished
1283 - error quit: We close the channels so the channel threads
1284 finish the qio_channel_read_all_eof() */
1285 qio_channel_shutdown(p
->c
, QIO_CHANNEL_SHUTDOWN_BOTH
, NULL
);
1286 qemu_mutex_unlock(&p
->mutex
);
1290 int multifd_load_cleanup(Error
**errp
)
1295 if (!migrate_use_multifd()) {
1298 multifd_recv_terminate_threads(NULL
);
1299 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1300 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1305 * multifd_recv_thread may hung at MULTIFD_FLAG_SYNC handle code,
1306 * however try to wakeup it without harm in cleanup phase.
1308 qemu_sem_post(&p
->sem_sync
);
1309 qemu_thread_join(&p
->thread
);
1311 object_unref(OBJECT(p
->c
));
1313 qemu_mutex_destroy(&p
->mutex
);
1314 qemu_sem_destroy(&p
->sem_sync
);
1317 multifd_pages_clear(p
->pages
);
1323 qemu_sem_destroy(&multifd_recv_state
->sem_sync
);
1324 g_free(multifd_recv_state
->params
);
1325 multifd_recv_state
->params
= NULL
;
1326 g_free(multifd_recv_state
);
1327 multifd_recv_state
= NULL
;
1332 static void multifd_recv_sync_main(void)
1336 if (!migrate_use_multifd()) {
1339 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1340 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1342 trace_multifd_recv_sync_main_wait(p
->id
);
1343 qemu_sem_wait(&multifd_recv_state
->sem_sync
);
1345 for (i
= 0; i
< migrate_multifd_channels(); i
++) {
1346 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1348 qemu_mutex_lock(&p
->mutex
);
1349 if (multifd_recv_state
->packet_num
< p
->packet_num
) {
1350 multifd_recv_state
->packet_num
= p
->packet_num
;
1352 qemu_mutex_unlock(&p
->mutex
);
1353 trace_multifd_recv_sync_main_signal(p
->id
);
1354 qemu_sem_post(&p
->sem_sync
);
1356 trace_multifd_recv_sync_main(multifd_recv_state
->packet_num
);
1359 static void *multifd_recv_thread(void *opaque
)
1361 MultiFDRecvParams
*p
= opaque
;
1362 Error
*local_err
= NULL
;
1365 trace_multifd_recv_thread_start(p
->id
);
1366 rcu_register_thread();
1376 ret
= qio_channel_read_all_eof(p
->c
, (void *)p
->packet
,
1377 p
->packet_len
, &local_err
);
1378 if (ret
== 0) { /* EOF */
1381 if (ret
== -1) { /* Error */
1385 qemu_mutex_lock(&p
->mutex
);
1386 ret
= multifd_recv_unfill_packet(p
, &local_err
);
1388 qemu_mutex_unlock(&p
->mutex
);
1392 used
= p
->pages
->used
;
1394 trace_multifd_recv(p
->id
, p
->packet_num
, used
, flags
,
1395 p
->next_packet_size
);
1397 p
->num_pages
+= used
;
1398 qemu_mutex_unlock(&p
->mutex
);
1401 ret
= qio_channel_readv_all(p
->c
, p
->pages
->iov
,
1408 if (flags
& MULTIFD_FLAG_SYNC
) {
1409 qemu_sem_post(&multifd_recv_state
->sem_sync
);
1410 qemu_sem_wait(&p
->sem_sync
);
1415 multifd_recv_terminate_threads(local_err
);
1417 qemu_mutex_lock(&p
->mutex
);
1419 qemu_mutex_unlock(&p
->mutex
);
1421 rcu_unregister_thread();
1422 trace_multifd_recv_thread_end(p
->id
, p
->num_packets
, p
->num_pages
);
1427 int multifd_load_setup(void)
1430 uint32_t page_count
= MULTIFD_PACKET_SIZE
/ qemu_target_page_size();
1433 if (!migrate_use_multifd()) {
1436 thread_count
= migrate_multifd_channels();
1437 multifd_recv_state
= g_malloc0(sizeof(*multifd_recv_state
));
1438 multifd_recv_state
->params
= g_new0(MultiFDRecvParams
, thread_count
);
1439 atomic_set(&multifd_recv_state
->count
, 0);
1440 qemu_sem_init(&multifd_recv_state
->sem_sync
, 0);
1442 for (i
= 0; i
< thread_count
; i
++) {
1443 MultiFDRecvParams
*p
= &multifd_recv_state
->params
[i
];
1445 qemu_mutex_init(&p
->mutex
);
1446 qemu_sem_init(&p
->sem_sync
, 0);
1449 p
->pages
= multifd_pages_init(page_count
);
1450 p
->packet_len
= sizeof(MultiFDPacket_t
)
1451 + sizeof(ram_addr_t
) * page_count
;
1452 p
->packet
= g_malloc0(p
->packet_len
);
1453 p
->name
= g_strdup_printf("multifdrecv_%d", i
);
1458 bool multifd_recv_all_channels_created(void)
1460 int thread_count
= migrate_multifd_channels();
1462 if (!migrate_use_multifd()) {
1466 return thread_count
== atomic_read(&multifd_recv_state
->count
);
1470 * Try to receive all multifd channels to get ready for the migration.
1471 * - Return true and do not set @errp when correctly receving all channels;
1472 * - Return false and do not set @errp when correctly receiving the current one;
1473 * - Return false and set @errp when failing to receive the current channel.
1475 bool multifd_recv_new_channel(QIOChannel
*ioc
, Error
**errp
)
1477 MultiFDRecvParams
*p
;
1478 Error
*local_err
= NULL
;
1481 id
= multifd_recv_initial_packet(ioc
, &local_err
);
1483 multifd_recv_terminate_threads(local_err
);
1484 error_propagate_prepend(errp
, local_err
,
1485 "failed to receive packet"
1486 " via multifd channel %d: ",
1487 atomic_read(&multifd_recv_state
->count
));
1490 trace_multifd_recv_new_channel(id
);
1492 p
= &multifd_recv_state
->params
[id
];
1494 error_setg(&local_err
, "multifd: received id '%d' already setup'",
1496 multifd_recv_terminate_threads(local_err
);
1497 error_propagate(errp
, local_err
);
1501 object_ref(OBJECT(ioc
));
1502 /* initial packet */
1506 qemu_thread_create(&p
->thread
, p
->name
, multifd_recv_thread
, p
,
1507 QEMU_THREAD_JOINABLE
);
1508 atomic_inc(&multifd_recv_state
->count
);
1509 return atomic_read(&multifd_recv_state
->count
) ==
1510 migrate_multifd_channels();
1514 * save_page_header: write page header to wire
1516 * If this is the 1st block, it also writes the block identification
1518 * Returns the number of bytes written
1520 * @f: QEMUFile where to send the data
1521 * @block: block that contains the page we want to send
1522 * @offset: offset inside the block for the page
1523 * in the lower bits, it contains flags
1525 static size_t save_page_header(RAMState
*rs
, QEMUFile
*f
, RAMBlock
*block
,
1530 if (block
== rs
->last_sent_block
) {
1531 offset
|= RAM_SAVE_FLAG_CONTINUE
;
1533 qemu_put_be64(f
, offset
);
1536 if (!(offset
& RAM_SAVE_FLAG_CONTINUE
)) {
1537 len
= strlen(block
->idstr
);
1538 qemu_put_byte(f
, len
);
1539 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, len
);
1541 rs
->last_sent_block
= block
;
1547 * mig_throttle_guest_down: throotle down the guest
1549 * Reduce amount of guest cpu execution to hopefully slow down memory
1550 * writes. If guest dirty memory rate is reduced below the rate at
1551 * which we can transfer pages to the destination then we should be
1552 * able to complete migration. Some workloads dirty memory way too
1553 * fast and will not effectively converge, even with auto-converge.
1555 static void mig_throttle_guest_down(void)
1557 MigrationState
*s
= migrate_get_current();
1558 uint64_t pct_initial
= s
->parameters
.cpu_throttle_initial
;
1559 uint64_t pct_icrement
= s
->parameters
.cpu_throttle_increment
;
1560 int pct_max
= s
->parameters
.max_cpu_throttle
;
1562 /* We have not started throttling yet. Let's start it. */
1563 if (!cpu_throttle_active()) {
1564 cpu_throttle_set(pct_initial
);
1566 /* Throttling already on, just increase the rate */
1567 cpu_throttle_set(MIN(cpu_throttle_get_percentage() + pct_icrement
,
1573 * xbzrle_cache_zero_page: insert a zero page in the XBZRLE cache
1575 * @rs: current RAM state
1576 * @current_addr: address for the zero page
1578 * Update the xbzrle cache to reflect a page that's been sent as all 0.
1579 * The important thing is that a stale (not-yet-0'd) page be replaced
1581 * As a bonus, if the page wasn't in the cache it gets added so that
1582 * when a small write is made into the 0'd page it gets XBZRLE sent.
1584 static void xbzrle_cache_zero_page(RAMState
*rs
, ram_addr_t current_addr
)
1586 if (rs
->ram_bulk_stage
|| !migrate_use_xbzrle()) {
1590 /* We don't care if this fails to allocate a new cache page
1591 * as long as it updated an old one */
1592 cache_insert(XBZRLE
.cache
, current_addr
, XBZRLE
.zero_target_page
,
1593 ram_counters
.dirty_sync_count
);
1596 #define ENCODING_FLAG_XBZRLE 0x1
1599 * save_xbzrle_page: compress and send current page
1601 * Returns: 1 means that we wrote the page
1602 * 0 means that page is identical to the one already sent
1603 * -1 means that xbzrle would be longer than normal
1605 * @rs: current RAM state
1606 * @current_data: pointer to the address of the page contents
1607 * @current_addr: addr of the page
1608 * @block: block that contains the page we want to send
1609 * @offset: offset inside the block for the page
1610 * @last_stage: if we are at the completion stage
1612 static int save_xbzrle_page(RAMState
*rs
, uint8_t **current_data
,
1613 ram_addr_t current_addr
, RAMBlock
*block
,
1614 ram_addr_t offset
, bool last_stage
)
1616 int encoded_len
= 0, bytes_xbzrle
;
1617 uint8_t *prev_cached_page
;
1619 if (!cache_is_cached(XBZRLE
.cache
, current_addr
,
1620 ram_counters
.dirty_sync_count
)) {
1621 xbzrle_counters
.cache_miss
++;
1623 if (cache_insert(XBZRLE
.cache
, current_addr
, *current_data
,
1624 ram_counters
.dirty_sync_count
) == -1) {
1627 /* update *current_data when the page has been
1628 inserted into cache */
1629 *current_data
= get_cached_data(XBZRLE
.cache
, current_addr
);
1635 prev_cached_page
= get_cached_data(XBZRLE
.cache
, current_addr
);
1637 /* save current buffer into memory */
1638 memcpy(XBZRLE
.current_buf
, *current_data
, TARGET_PAGE_SIZE
);
1640 /* XBZRLE encoding (if there is no overflow) */
1641 encoded_len
= xbzrle_encode_buffer(prev_cached_page
, XBZRLE
.current_buf
,
1642 TARGET_PAGE_SIZE
, XBZRLE
.encoded_buf
,
1646 * Update the cache contents, so that it corresponds to the data
1647 * sent, in all cases except where we skip the page.
1649 if (!last_stage
&& encoded_len
!= 0) {
1650 memcpy(prev_cached_page
, XBZRLE
.current_buf
, TARGET_PAGE_SIZE
);
1652 * In the case where we couldn't compress, ensure that the caller
1653 * sends the data from the cache, since the guest might have
1654 * changed the RAM since we copied it.
1656 *current_data
= prev_cached_page
;
1659 if (encoded_len
== 0) {
1660 trace_save_xbzrle_page_skipping();
1662 } else if (encoded_len
== -1) {
1663 trace_save_xbzrle_page_overflow();
1664 xbzrle_counters
.overflow
++;
1668 /* Send XBZRLE based compressed page */
1669 bytes_xbzrle
= save_page_header(rs
, rs
->f
, block
,
1670 offset
| RAM_SAVE_FLAG_XBZRLE
);
1671 qemu_put_byte(rs
->f
, ENCODING_FLAG_XBZRLE
);
1672 qemu_put_be16(rs
->f
, encoded_len
);
1673 qemu_put_buffer(rs
->f
, XBZRLE
.encoded_buf
, encoded_len
);
1674 bytes_xbzrle
+= encoded_len
+ 1 + 2;
1675 xbzrle_counters
.pages
++;
1676 xbzrle_counters
.bytes
+= bytes_xbzrle
;
1677 ram_counters
.transferred
+= bytes_xbzrle
;
1683 * migration_bitmap_find_dirty: find the next dirty page from start
1685 * Returns the page offset within memory region of the start of a dirty page
1687 * @rs: current RAM state
1688 * @rb: RAMBlock where to search for dirty pages
1689 * @start: page where we start the search
1692 unsigned long migration_bitmap_find_dirty(RAMState
*rs
, RAMBlock
*rb
,
1693 unsigned long start
)
1695 unsigned long size
= rb
->used_length
>> TARGET_PAGE_BITS
;
1696 unsigned long *bitmap
= rb
->bmap
;
1699 if (ramblock_is_ignored(rb
)) {
1704 * When the free page optimization is enabled, we need to check the bitmap
1705 * to send the non-free pages rather than all the pages in the bulk stage.
1707 if (!rs
->fpo_enabled
&& rs
->ram_bulk_stage
&& start
> 0) {
1710 next
= find_next_bit(bitmap
, size
, start
);
1716 static inline bool migration_bitmap_clear_dirty(RAMState
*rs
,
1722 qemu_mutex_lock(&rs
->bitmap_mutex
);
1725 * Clear dirty bitmap if needed. This _must_ be called before we
1726 * send any of the page in the chunk because we need to make sure
1727 * we can capture further page content changes when we sync dirty
1728 * log the next time. So as long as we are going to send any of
1729 * the page in the chunk we clear the remote dirty bitmap for all.
1730 * Clearing it earlier won't be a problem, but too late will.
1732 if (rb
->clear_bmap
&& clear_bmap_test_and_clear(rb
, page
)) {
1733 uint8_t shift
= rb
->clear_bmap_shift
;
1734 hwaddr size
= 1ULL << (TARGET_PAGE_BITS
+ shift
);
1735 hwaddr start
= (page
<< TARGET_PAGE_BITS
) & (-size
);
1738 * CLEAR_BITMAP_SHIFT_MIN should always guarantee this... this
1739 * can make things easier sometimes since then start address
1740 * of the small chunk will always be 64 pages aligned so the
1741 * bitmap will always be aligned to unsigned long. We should
1742 * even be able to remove this restriction but I'm simply
1746 trace_migration_bitmap_clear_dirty(rb
->idstr
, start
, size
, page
);
1747 memory_region_clear_dirty_bitmap(rb
->mr
, start
, size
);
1750 ret
= test_and_clear_bit(page
, rb
->bmap
);
1753 rs
->migration_dirty_pages
--;
1755 qemu_mutex_unlock(&rs
->bitmap_mutex
);
1760 /* Called with RCU critical section */
1761 static void ramblock_sync_dirty_bitmap(RAMState
*rs
, RAMBlock
*rb
)
1763 rs
->migration_dirty_pages
+=
1764 cpu_physical_memory_sync_dirty_bitmap(rb
, 0, rb
->used_length
,
1765 &rs
->num_dirty_pages_period
);
1769 * ram_pagesize_summary: calculate all the pagesizes of a VM
1771 * Returns a summary bitmap of the page sizes of all RAMBlocks
1773 * For VMs with just normal pages this is equivalent to the host page
1774 * size. If it's got some huge pages then it's the OR of all the
1775 * different page sizes.
1777 uint64_t ram_pagesize_summary(void)
1780 uint64_t summary
= 0;
1782 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1783 summary
|= block
->page_size
;
1789 uint64_t ram_get_total_transferred_pages(void)
1791 return ram_counters
.normal
+ ram_counters
.duplicate
+
1792 compression_counters
.pages
+ xbzrle_counters
.pages
;
1795 static void migration_update_rates(RAMState
*rs
, int64_t end_time
)
1797 uint64_t page_count
= rs
->target_page_count
- rs
->target_page_count_prev
;
1798 double compressed_size
;
1800 /* calculate period counters */
1801 ram_counters
.dirty_pages_rate
= rs
->num_dirty_pages_period
* 1000
1802 / (end_time
- rs
->time_last_bitmap_sync
);
1808 if (migrate_use_xbzrle()) {
1809 xbzrle_counters
.cache_miss_rate
= (double)(xbzrle_counters
.cache_miss
-
1810 rs
->xbzrle_cache_miss_prev
) / page_count
;
1811 rs
->xbzrle_cache_miss_prev
= xbzrle_counters
.cache_miss
;
1814 if (migrate_use_compression()) {
1815 compression_counters
.busy_rate
= (double)(compression_counters
.busy
-
1816 rs
->compress_thread_busy_prev
) / page_count
;
1817 rs
->compress_thread_busy_prev
= compression_counters
.busy
;
1819 compressed_size
= compression_counters
.compressed_size
-
1820 rs
->compressed_size_prev
;
1821 if (compressed_size
) {
1822 double uncompressed_size
= (compression_counters
.pages
-
1823 rs
->compress_pages_prev
) * TARGET_PAGE_SIZE
;
1825 /* Compression-Ratio = Uncompressed-size / Compressed-size */
1826 compression_counters
.compression_rate
=
1827 uncompressed_size
/ compressed_size
;
1829 rs
->compress_pages_prev
= compression_counters
.pages
;
1830 rs
->compressed_size_prev
= compression_counters
.compressed_size
;
1835 static void migration_bitmap_sync(RAMState
*rs
)
1839 uint64_t bytes_xfer_now
;
1841 ram_counters
.dirty_sync_count
++;
1843 if (!rs
->time_last_bitmap_sync
) {
1844 rs
->time_last_bitmap_sync
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
1847 trace_migration_bitmap_sync_start();
1848 memory_global_dirty_log_sync();
1850 qemu_mutex_lock(&rs
->bitmap_mutex
);
1852 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
1853 ramblock_sync_dirty_bitmap(rs
, block
);
1855 ram_counters
.remaining
= ram_bytes_remaining();
1857 qemu_mutex_unlock(&rs
->bitmap_mutex
);
1859 memory_global_after_dirty_log_sync();
1860 trace_migration_bitmap_sync_end(rs
->num_dirty_pages_period
);
1862 end_time
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
1864 /* more than 1 second = 1000 millisecons */
1865 if (end_time
> rs
->time_last_bitmap_sync
+ 1000) {
1866 bytes_xfer_now
= ram_counters
.transferred
;
1868 /* During block migration the auto-converge logic incorrectly detects
1869 * that ram migration makes no progress. Avoid this by disabling the
1870 * throttling logic during the bulk phase of block migration. */
1871 if (migrate_auto_converge() && !blk_mig_bulk_active()) {
1872 /* The following detection logic can be refined later. For now:
1873 Check to see if the dirtied bytes is 50% more than the approx.
1874 amount of bytes that just got transferred since the last time we
1875 were in this routine. If that happens twice, start or increase
1878 if ((rs
->num_dirty_pages_period
* TARGET_PAGE_SIZE
>
1879 (bytes_xfer_now
- rs
->bytes_xfer_prev
) / 2) &&
1880 (++rs
->dirty_rate_high_cnt
>= 2)) {
1881 trace_migration_throttle();
1882 rs
->dirty_rate_high_cnt
= 0;
1883 mig_throttle_guest_down();
1887 migration_update_rates(rs
, end_time
);
1889 rs
->target_page_count_prev
= rs
->target_page_count
;
1891 /* reset period counters */
1892 rs
->time_last_bitmap_sync
= end_time
;
1893 rs
->num_dirty_pages_period
= 0;
1894 rs
->bytes_xfer_prev
= bytes_xfer_now
;
1896 if (migrate_use_events()) {
1897 qapi_event_send_migration_pass(ram_counters
.dirty_sync_count
);
1901 static void migration_bitmap_sync_precopy(RAMState
*rs
)
1903 Error
*local_err
= NULL
;
1906 * The current notifier usage is just an optimization to migration, so we
1907 * don't stop the normal migration process in the error case.
1909 if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC
, &local_err
)) {
1910 error_report_err(local_err
);
1913 migration_bitmap_sync(rs
);
1915 if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC
, &local_err
)) {
1916 error_report_err(local_err
);
1921 * save_zero_page_to_file: send the zero page to the file
1923 * Returns the size of data written to the file, 0 means the page is not
1926 * @rs: current RAM state
1927 * @file: the file where the data is saved
1928 * @block: block that contains the page we want to send
1929 * @offset: offset inside the block for the page
1931 static int save_zero_page_to_file(RAMState
*rs
, QEMUFile
*file
,
1932 RAMBlock
*block
, ram_addr_t offset
)
1934 uint8_t *p
= block
->host
+ offset
;
1937 if (is_zero_range(p
, TARGET_PAGE_SIZE
)) {
1938 len
+= save_page_header(rs
, file
, block
, offset
| RAM_SAVE_FLAG_ZERO
);
1939 qemu_put_byte(file
, 0);
1946 * save_zero_page: send the zero page to the stream
1948 * Returns the number of pages written.
1950 * @rs: current RAM state
1951 * @block: block that contains the page we want to send
1952 * @offset: offset inside the block for the page
1954 static int save_zero_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
)
1956 int len
= save_zero_page_to_file(rs
, rs
->f
, block
, offset
);
1959 ram_counters
.duplicate
++;
1960 ram_counters
.transferred
+= len
;
1966 static void ram_release_pages(const char *rbname
, uint64_t offset
, int pages
)
1968 if (!migrate_release_ram() || !migration_in_postcopy()) {
1972 ram_discard_range(rbname
, offset
, pages
<< TARGET_PAGE_BITS
);
1976 * @pages: the number of pages written by the control path,
1978 * > 0 - number of pages written
1980 * Return true if the pages has been saved, otherwise false is returned.
1982 static bool control_save_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
,
1985 uint64_t bytes_xmit
= 0;
1989 ret
= ram_control_save_page(rs
->f
, block
->offset
, offset
, TARGET_PAGE_SIZE
,
1991 if (ret
== RAM_SAVE_CONTROL_NOT_SUPP
) {
1996 ram_counters
.transferred
+= bytes_xmit
;
2000 if (ret
== RAM_SAVE_CONTROL_DELAYED
) {
2004 if (bytes_xmit
> 0) {
2005 ram_counters
.normal
++;
2006 } else if (bytes_xmit
== 0) {
2007 ram_counters
.duplicate
++;
2014 * directly send the page to the stream
2016 * Returns the number of pages written.
2018 * @rs: current RAM state
2019 * @block: block that contains the page we want to send
2020 * @offset: offset inside the block for the page
2021 * @buf: the page to be sent
2022 * @async: send to page asyncly
2024 static int save_normal_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
,
2025 uint8_t *buf
, bool async
)
2027 ram_counters
.transferred
+= save_page_header(rs
, rs
->f
, block
,
2028 offset
| RAM_SAVE_FLAG_PAGE
);
2030 qemu_put_buffer_async(rs
->f
, buf
, TARGET_PAGE_SIZE
,
2031 migrate_release_ram() &
2032 migration_in_postcopy());
2034 qemu_put_buffer(rs
->f
, buf
, TARGET_PAGE_SIZE
);
2036 ram_counters
.transferred
+= TARGET_PAGE_SIZE
;
2037 ram_counters
.normal
++;
2042 * ram_save_page: send the given page to the stream
2044 * Returns the number of pages written.
2046 * >=0 - Number of pages written - this might legally be 0
2047 * if xbzrle noticed the page was the same.
2049 * @rs: current RAM state
2050 * @block: block that contains the page we want to send
2051 * @offset: offset inside the block for the page
2052 * @last_stage: if we are at the completion stage
2054 static int ram_save_page(RAMState
*rs
, PageSearchStatus
*pss
, bool last_stage
)
2058 bool send_async
= true;
2059 RAMBlock
*block
= pss
->block
;
2060 ram_addr_t offset
= pss
->page
<< TARGET_PAGE_BITS
;
2061 ram_addr_t current_addr
= block
->offset
+ offset
;
2063 p
= block
->host
+ offset
;
2064 trace_ram_save_page(block
->idstr
, (uint64_t)offset
, p
);
2066 XBZRLE_cache_lock();
2067 if (!rs
->ram_bulk_stage
&& !migration_in_postcopy() &&
2068 migrate_use_xbzrle()) {
2069 pages
= save_xbzrle_page(rs
, &p
, current_addr
, block
,
2070 offset
, last_stage
);
2072 /* Can't send this cached data async, since the cache page
2073 * might get updated before it gets to the wire
2079 /* XBZRLE overflow or normal page */
2081 pages
= save_normal_page(rs
, block
, offset
, p
, send_async
);
2084 XBZRLE_cache_unlock();
2089 static int ram_save_multifd_page(RAMState
*rs
, RAMBlock
*block
,
2092 if (multifd_queue_page(rs
, block
, offset
) < 0) {
2095 ram_counters
.normal
++;
2100 static bool do_compress_ram_page(QEMUFile
*f
, z_stream
*stream
, RAMBlock
*block
,
2101 ram_addr_t offset
, uint8_t *source_buf
)
2103 RAMState
*rs
= ram_state
;
2104 uint8_t *p
= block
->host
+ (offset
& TARGET_PAGE_MASK
);
2105 bool zero_page
= false;
2108 if (save_zero_page_to_file(rs
, f
, block
, offset
)) {
2113 save_page_header(rs
, f
, block
, offset
| RAM_SAVE_FLAG_COMPRESS_PAGE
);
2116 * copy it to a internal buffer to avoid it being modified by VM
2117 * so that we can catch up the error during compression and
2120 memcpy(source_buf
, p
, TARGET_PAGE_SIZE
);
2121 ret
= qemu_put_compression_data(f
, stream
, source_buf
, TARGET_PAGE_SIZE
);
2123 qemu_file_set_error(migrate_get_current()->to_dst_file
, ret
);
2124 error_report("compressed data failed!");
2129 ram_release_pages(block
->idstr
, offset
& TARGET_PAGE_MASK
, 1);
2134 update_compress_thread_counts(const CompressParam
*param
, int bytes_xmit
)
2136 ram_counters
.transferred
+= bytes_xmit
;
2138 if (param
->zero_page
) {
2139 ram_counters
.duplicate
++;
2143 /* 8 means a header with RAM_SAVE_FLAG_CONTINUE. */
2144 compression_counters
.compressed_size
+= bytes_xmit
- 8;
2145 compression_counters
.pages
++;
2148 static bool save_page_use_compression(RAMState
*rs
);
2150 static void flush_compressed_data(RAMState
*rs
)
2152 int idx
, len
, thread_count
;
2154 if (!save_page_use_compression(rs
)) {
2157 thread_count
= migrate_compress_threads();
2159 qemu_mutex_lock(&comp_done_lock
);
2160 for (idx
= 0; idx
< thread_count
; idx
++) {
2161 while (!comp_param
[idx
].done
) {
2162 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
2165 qemu_mutex_unlock(&comp_done_lock
);
2167 for (idx
= 0; idx
< thread_count
; idx
++) {
2168 qemu_mutex_lock(&comp_param
[idx
].mutex
);
2169 if (!comp_param
[idx
].quit
) {
2170 len
= qemu_put_qemu_file(rs
->f
, comp_param
[idx
].file
);
2172 * it's safe to fetch zero_page without holding comp_done_lock
2173 * as there is no further request submitted to the thread,
2174 * i.e, the thread should be waiting for a request at this point.
2176 update_compress_thread_counts(&comp_param
[idx
], len
);
2178 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
2182 static inline void set_compress_params(CompressParam
*param
, RAMBlock
*block
,
2185 param
->block
= block
;
2186 param
->offset
= offset
;
2189 static int compress_page_with_multi_thread(RAMState
*rs
, RAMBlock
*block
,
2192 int idx
, thread_count
, bytes_xmit
= -1, pages
= -1;
2193 bool wait
= migrate_compress_wait_thread();
2195 thread_count
= migrate_compress_threads();
2196 qemu_mutex_lock(&comp_done_lock
);
2198 for (idx
= 0; idx
< thread_count
; idx
++) {
2199 if (comp_param
[idx
].done
) {
2200 comp_param
[idx
].done
= false;
2201 bytes_xmit
= qemu_put_qemu_file(rs
->f
, comp_param
[idx
].file
);
2202 qemu_mutex_lock(&comp_param
[idx
].mutex
);
2203 set_compress_params(&comp_param
[idx
], block
, offset
);
2204 qemu_cond_signal(&comp_param
[idx
].cond
);
2205 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
2207 update_compress_thread_counts(&comp_param
[idx
], bytes_xmit
);
2213 * wait for the free thread if the user specifies 'compress-wait-thread',
2214 * otherwise we will post the page out in the main thread as normal page.
2216 if (pages
< 0 && wait
) {
2217 qemu_cond_wait(&comp_done_cond
, &comp_done_lock
);
2220 qemu_mutex_unlock(&comp_done_lock
);
2226 * find_dirty_block: find the next dirty page and update any state
2227 * associated with the search process.
2229 * Returns true if a page is found
2231 * @rs: current RAM state
2232 * @pss: data about the state of the current dirty page scan
2233 * @again: set to false if the search has scanned the whole of RAM
2235 static bool find_dirty_block(RAMState
*rs
, PageSearchStatus
*pss
, bool *again
)
2237 pss
->page
= migration_bitmap_find_dirty(rs
, pss
->block
, pss
->page
);
2238 if (pss
->complete_round
&& pss
->block
== rs
->last_seen_block
&&
2239 pss
->page
>= rs
->last_page
) {
2241 * We've been once around the RAM and haven't found anything.
2247 if ((pss
->page
<< TARGET_PAGE_BITS
) >= pss
->block
->used_length
) {
2248 /* Didn't find anything in this RAM Block */
2250 pss
->block
= QLIST_NEXT_RCU(pss
->block
, next
);
2253 * If memory migration starts over, we will meet a dirtied page
2254 * which may still exists in compression threads's ring, so we
2255 * should flush the compressed data to make sure the new page
2256 * is not overwritten by the old one in the destination.
2258 * Also If xbzrle is on, stop using the data compression at this
2259 * point. In theory, xbzrle can do better than compression.
2261 flush_compressed_data(rs
);
2263 /* Hit the end of the list */
2264 pss
->block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
2265 /* Flag that we've looped */
2266 pss
->complete_round
= true;
2267 rs
->ram_bulk_stage
= false;
2269 /* Didn't find anything this time, but try again on the new block */
2273 /* Can go around again, but... */
2275 /* We've found something so probably don't need to */
2281 * unqueue_page: gets a page of the queue
2283 * Helper for 'get_queued_page' - gets a page off the queue
2285 * Returns the block of the page (or NULL if none available)
2287 * @rs: current RAM state
2288 * @offset: used to return the offset within the RAMBlock
2290 static RAMBlock
*unqueue_page(RAMState
*rs
, ram_addr_t
*offset
)
2292 RAMBlock
*block
= NULL
;
2294 if (QSIMPLEQ_EMPTY_ATOMIC(&rs
->src_page_requests
)) {
2298 qemu_mutex_lock(&rs
->src_page_req_mutex
);
2299 if (!QSIMPLEQ_EMPTY(&rs
->src_page_requests
)) {
2300 struct RAMSrcPageRequest
*entry
=
2301 QSIMPLEQ_FIRST(&rs
->src_page_requests
);
2303 *offset
= entry
->offset
;
2305 if (entry
->len
> TARGET_PAGE_SIZE
) {
2306 entry
->len
-= TARGET_PAGE_SIZE
;
2307 entry
->offset
+= TARGET_PAGE_SIZE
;
2309 memory_region_unref(block
->mr
);
2310 QSIMPLEQ_REMOVE_HEAD(&rs
->src_page_requests
, next_req
);
2312 migration_consume_urgent_request();
2315 qemu_mutex_unlock(&rs
->src_page_req_mutex
);
2321 * get_queued_page: unqueue a page from the postcopy requests
2323 * Skips pages that are already sent (!dirty)
2325 * Returns true if a queued page is found
2327 * @rs: current RAM state
2328 * @pss: data about the state of the current dirty page scan
2330 static bool get_queued_page(RAMState
*rs
, PageSearchStatus
*pss
)
2337 block
= unqueue_page(rs
, &offset
);
2339 * We're sending this page, and since it's postcopy nothing else
2340 * will dirty it, and we must make sure it doesn't get sent again
2341 * even if this queue request was received after the background
2342 * search already sent it.
2347 page
= offset
>> TARGET_PAGE_BITS
;
2348 dirty
= test_bit(page
, block
->bmap
);
2350 trace_get_queued_page_not_dirty(block
->idstr
, (uint64_t)offset
,
2351 page
, test_bit(page
, block
->unsentmap
));
2353 trace_get_queued_page(block
->idstr
, (uint64_t)offset
, page
);
2357 } while (block
&& !dirty
);
2361 * As soon as we start servicing pages out of order, then we have
2362 * to kill the bulk stage, since the bulk stage assumes
2363 * in (migration_bitmap_find_and_reset_dirty) that every page is
2364 * dirty, that's no longer true.
2366 rs
->ram_bulk_stage
= false;
2369 * We want the background search to continue from the queued page
2370 * since the guest is likely to want other pages near to the page
2371 * it just requested.
2374 pss
->page
= offset
>> TARGET_PAGE_BITS
;
2377 * This unqueued page would break the "one round" check, even is
2380 pss
->complete_round
= false;
2387 * migration_page_queue_free: drop any remaining pages in the ram
2390 * It should be empty at the end anyway, but in error cases there may
2391 * be some left. in case that there is any page left, we drop it.
2394 static void migration_page_queue_free(RAMState
*rs
)
2396 struct RAMSrcPageRequest
*mspr
, *next_mspr
;
2397 /* This queue generally should be empty - but in the case of a failed
2398 * migration might have some droppings in.
2401 QSIMPLEQ_FOREACH_SAFE(mspr
, &rs
->src_page_requests
, next_req
, next_mspr
) {
2402 memory_region_unref(mspr
->rb
->mr
);
2403 QSIMPLEQ_REMOVE_HEAD(&rs
->src_page_requests
, next_req
);
2410 * ram_save_queue_pages: queue the page for transmission
2412 * A request from postcopy destination for example.
2414 * Returns zero on success or negative on error
2416 * @rbname: Name of the RAMBLock of the request. NULL means the
2417 * same that last one.
2418 * @start: starting address from the start of the RAMBlock
2419 * @len: length (in bytes) to send
2421 int ram_save_queue_pages(const char *rbname
, ram_addr_t start
, ram_addr_t len
)
2424 RAMState
*rs
= ram_state
;
2426 ram_counters
.postcopy_requests
++;
2429 /* Reuse last RAMBlock */
2430 ramblock
= rs
->last_req_rb
;
2434 * Shouldn't happen, we can't reuse the last RAMBlock if
2435 * it's the 1st request.
2437 error_report("ram_save_queue_pages no previous block");
2441 ramblock
= qemu_ram_block_by_name(rbname
);
2444 /* We shouldn't be asked for a non-existent RAMBlock */
2445 error_report("ram_save_queue_pages no block '%s'", rbname
);
2448 rs
->last_req_rb
= ramblock
;
2450 trace_ram_save_queue_pages(ramblock
->idstr
, start
, len
);
2451 if (start
+len
> ramblock
->used_length
) {
2452 error_report("%s request overrun start=" RAM_ADDR_FMT
" len="
2453 RAM_ADDR_FMT
" blocklen=" RAM_ADDR_FMT
,
2454 __func__
, start
, len
, ramblock
->used_length
);
2458 struct RAMSrcPageRequest
*new_entry
=
2459 g_malloc0(sizeof(struct RAMSrcPageRequest
));
2460 new_entry
->rb
= ramblock
;
2461 new_entry
->offset
= start
;
2462 new_entry
->len
= len
;
2464 memory_region_ref(ramblock
->mr
);
2465 qemu_mutex_lock(&rs
->src_page_req_mutex
);
2466 QSIMPLEQ_INSERT_TAIL(&rs
->src_page_requests
, new_entry
, next_req
);
2467 migration_make_urgent_request();
2468 qemu_mutex_unlock(&rs
->src_page_req_mutex
);
2478 static bool save_page_use_compression(RAMState
*rs
)
2480 if (!migrate_use_compression()) {
2485 * If xbzrle is on, stop using the data compression after first
2486 * round of migration even if compression is enabled. In theory,
2487 * xbzrle can do better than compression.
2489 if (rs
->ram_bulk_stage
|| !migrate_use_xbzrle()) {
2497 * try to compress the page before posting it out, return true if the page
2498 * has been properly handled by compression, otherwise needs other
2499 * paths to handle it
2501 static bool save_compress_page(RAMState
*rs
, RAMBlock
*block
, ram_addr_t offset
)
2503 if (!save_page_use_compression(rs
)) {
2508 * When starting the process of a new block, the first page of
2509 * the block should be sent out before other pages in the same
2510 * block, and all the pages in last block should have been sent
2511 * out, keeping this order is important, because the 'cont' flag
2512 * is used to avoid resending the block name.
2514 * We post the fist page as normal page as compression will take
2515 * much CPU resource.
2517 if (block
!= rs
->last_sent_block
) {
2518 flush_compressed_data(rs
);
2522 if (compress_page_with_multi_thread(rs
, block
, offset
) > 0) {
2526 compression_counters
.busy
++;
2531 * ram_save_target_page: save one target page
2533 * Returns the number of pages written
2535 * @rs: current RAM state
2536 * @pss: data about the page we want to send
2537 * @last_stage: if we are at the completion stage
2539 static int ram_save_target_page(RAMState
*rs
, PageSearchStatus
*pss
,
2542 RAMBlock
*block
= pss
->block
;
2543 ram_addr_t offset
= pss
->page
<< TARGET_PAGE_BITS
;
2546 if (control_save_page(rs
, block
, offset
, &res
)) {
2550 if (save_compress_page(rs
, block
, offset
)) {
2554 res
= save_zero_page(rs
, block
, offset
);
2556 /* Must let xbzrle know, otherwise a previous (now 0'd) cached
2557 * page would be stale
2559 if (!save_page_use_compression(rs
)) {
2560 XBZRLE_cache_lock();
2561 xbzrle_cache_zero_page(rs
, block
->offset
+ offset
);
2562 XBZRLE_cache_unlock();
2564 ram_release_pages(block
->idstr
, offset
, res
);
2569 * do not use multifd for compression as the first page in the new
2570 * block should be posted out before sending the compressed page
2572 if (!save_page_use_compression(rs
) && migrate_use_multifd()) {
2573 return ram_save_multifd_page(rs
, block
, offset
);
2576 return ram_save_page(rs
, pss
, last_stage
);
2580 * ram_save_host_page: save a whole host page
2582 * Starting at *offset send pages up to the end of the current host
2583 * page. It's valid for the initial offset to point into the middle of
2584 * a host page in which case the remainder of the hostpage is sent.
2585 * Only dirty target pages are sent. Note that the host page size may
2586 * be a huge page for this block.
2587 * The saving stops at the boundary of the used_length of the block
2588 * if the RAMBlock isn't a multiple of the host page size.
2590 * Returns the number of pages written or negative on error
2592 * @rs: current RAM state
2593 * @ms: current migration state
2594 * @pss: data about the page we want to send
2595 * @last_stage: if we are at the completion stage
2597 static int ram_save_host_page(RAMState
*rs
, PageSearchStatus
*pss
,
2600 int tmppages
, pages
= 0;
2601 size_t pagesize_bits
=
2602 qemu_ram_pagesize(pss
->block
) >> TARGET_PAGE_BITS
;
2604 if (ramblock_is_ignored(pss
->block
)) {
2605 error_report("block %s should not be migrated !", pss
->block
->idstr
);
2610 /* Check the pages is dirty and if it is send it */
2611 if (!migration_bitmap_clear_dirty(rs
, pss
->block
, pss
->page
)) {
2616 tmppages
= ram_save_target_page(rs
, pss
, last_stage
);
2622 if (pss
->block
->unsentmap
) {
2623 clear_bit(pss
->page
, pss
->block
->unsentmap
);
2627 } while ((pss
->page
& (pagesize_bits
- 1)) &&
2628 offset_in_ramblock(pss
->block
, pss
->page
<< TARGET_PAGE_BITS
));
2630 /* The offset we leave with is the last one we looked at */
2636 * ram_find_and_save_block: finds a dirty page and sends it to f
2638 * Called within an RCU critical section.
2640 * Returns the number of pages written where zero means no dirty pages,
2641 * or negative on error
2643 * @rs: current RAM state
2644 * @last_stage: if we are at the completion stage
2646 * On systems where host-page-size > target-page-size it will send all the
2647 * pages in a host page that are dirty.
2650 static int ram_find_and_save_block(RAMState
*rs
, bool last_stage
)
2652 PageSearchStatus pss
;
2656 /* No dirty page as there is zero RAM */
2657 if (!ram_bytes_total()) {
2661 pss
.block
= rs
->last_seen_block
;
2662 pss
.page
= rs
->last_page
;
2663 pss
.complete_round
= false;
2666 pss
.block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
2671 found
= get_queued_page(rs
, &pss
);
2674 /* priority queue empty, so just search for something dirty */
2675 found
= find_dirty_block(rs
, &pss
, &again
);
2679 pages
= ram_save_host_page(rs
, &pss
, last_stage
);
2681 } while (!pages
&& again
);
2683 rs
->last_seen_block
= pss
.block
;
2684 rs
->last_page
= pss
.page
;
2689 void acct_update_position(QEMUFile
*f
, size_t size
, bool zero
)
2691 uint64_t pages
= size
/ TARGET_PAGE_SIZE
;
2694 ram_counters
.duplicate
+= pages
;
2696 ram_counters
.normal
+= pages
;
2697 ram_counters
.transferred
+= size
;
2698 qemu_update_position(f
, size
);
2702 static uint64_t ram_bytes_total_common(bool count_ignored
)
2708 if (count_ignored
) {
2709 RAMBLOCK_FOREACH_MIGRATABLE(block
) {
2710 total
+= block
->used_length
;
2713 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2714 total
+= block
->used_length
;
2721 uint64_t ram_bytes_total(void)
2723 return ram_bytes_total_common(false);
2726 static void xbzrle_load_setup(void)
2728 XBZRLE
.decoded_buf
= g_malloc(TARGET_PAGE_SIZE
);
2731 static void xbzrle_load_cleanup(void)
2733 g_free(XBZRLE
.decoded_buf
);
2734 XBZRLE
.decoded_buf
= NULL
;
2737 static void ram_state_cleanup(RAMState
**rsp
)
2740 migration_page_queue_free(*rsp
);
2741 qemu_mutex_destroy(&(*rsp
)->bitmap_mutex
);
2742 qemu_mutex_destroy(&(*rsp
)->src_page_req_mutex
);
2748 static void xbzrle_cleanup(void)
2750 XBZRLE_cache_lock();
2752 cache_fini(XBZRLE
.cache
);
2753 g_free(XBZRLE
.encoded_buf
);
2754 g_free(XBZRLE
.current_buf
);
2755 g_free(XBZRLE
.zero_target_page
);
2756 XBZRLE
.cache
= NULL
;
2757 XBZRLE
.encoded_buf
= NULL
;
2758 XBZRLE
.current_buf
= NULL
;
2759 XBZRLE
.zero_target_page
= NULL
;
2761 XBZRLE_cache_unlock();
2764 static void ram_save_cleanup(void *opaque
)
2766 RAMState
**rsp
= opaque
;
2769 /* caller have hold iothread lock or is in a bh, so there is
2770 * no writing race against the migration bitmap
2772 memory_global_dirty_log_stop();
2774 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2775 g_free(block
->clear_bmap
);
2776 block
->clear_bmap
= NULL
;
2777 g_free(block
->bmap
);
2779 g_free(block
->unsentmap
);
2780 block
->unsentmap
= NULL
;
2784 compress_threads_save_cleanup();
2785 ram_state_cleanup(rsp
);
2788 static void ram_state_reset(RAMState
*rs
)
2790 rs
->last_seen_block
= NULL
;
2791 rs
->last_sent_block
= NULL
;
2793 rs
->last_version
= ram_list
.version
;
2794 rs
->ram_bulk_stage
= true;
2795 rs
->fpo_enabled
= false;
2798 #define MAX_WAIT 50 /* ms, half buffered_file limit */
2801 * 'expected' is the value you expect the bitmap mostly to be full
2802 * of; it won't bother printing lines that are all this value.
2803 * If 'todump' is null the migration bitmap is dumped.
2805 void ram_debug_dump_bitmap(unsigned long *todump
, bool expected
,
2806 unsigned long pages
)
2809 int64_t linelen
= 128;
2812 for (cur
= 0; cur
< pages
; cur
+= linelen
) {
2816 * Last line; catch the case where the line length
2817 * is longer than remaining ram
2819 if (cur
+ linelen
> pages
) {
2820 linelen
= pages
- cur
;
2822 for (curb
= 0; curb
< linelen
; curb
++) {
2823 bool thisbit
= test_bit(cur
+ curb
, todump
);
2824 linebuf
[curb
] = thisbit
? '1' : '.';
2825 found
= found
|| (thisbit
!= expected
);
2828 linebuf
[curb
] = '\0';
2829 fprintf(stderr
, "0x%08" PRIx64
" : %s\n", cur
, linebuf
);
2834 /* **** functions for postcopy ***** */
2836 void ram_postcopy_migrated_memory_release(MigrationState
*ms
)
2838 struct RAMBlock
*block
;
2840 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2841 unsigned long *bitmap
= block
->bmap
;
2842 unsigned long range
= block
->used_length
>> TARGET_PAGE_BITS
;
2843 unsigned long run_start
= find_next_zero_bit(bitmap
, range
, 0);
2845 while (run_start
< range
) {
2846 unsigned long run_end
= find_next_bit(bitmap
, range
, run_start
+ 1);
2847 ram_discard_range(block
->idstr
, run_start
<< TARGET_PAGE_BITS
,
2848 (run_end
- run_start
) << TARGET_PAGE_BITS
);
2849 run_start
= find_next_zero_bit(bitmap
, range
, run_end
+ 1);
2855 * postcopy_send_discard_bm_ram: discard a RAMBlock
2857 * Returns zero on success
2859 * Callback from postcopy_each_ram_send_discard for each RAMBlock
2860 * Note: At this point the 'unsentmap' is the processed bitmap combined
2861 * with the dirtymap; so a '1' means it's either dirty or unsent.
2863 * @ms: current migration state
2864 * @block: RAMBlock to discard
2866 static int postcopy_send_discard_bm_ram(MigrationState
*ms
, RAMBlock
*block
)
2868 unsigned long end
= block
->used_length
>> TARGET_PAGE_BITS
;
2869 unsigned long current
;
2870 unsigned long *unsentmap
= block
->unsentmap
;
2872 for (current
= 0; current
< end
; ) {
2873 unsigned long one
= find_next_bit(unsentmap
, end
, current
);
2874 unsigned long zero
, discard_length
;
2880 zero
= find_next_zero_bit(unsentmap
, end
, one
+ 1);
2883 discard_length
= end
- one
;
2885 discard_length
= zero
- one
;
2887 postcopy_discard_send_range(ms
, one
, discard_length
);
2888 current
= one
+ discard_length
;
2895 * postcopy_each_ram_send_discard: discard all RAMBlocks
2897 * Returns 0 for success or negative for error
2899 * Utility for the outgoing postcopy code.
2900 * Calls postcopy_send_discard_bm_ram for each RAMBlock
2901 * passing it bitmap indexes and name.
2902 * (qemu_ram_foreach_block ends up passing unscaled lengths
2903 * which would mean postcopy code would have to deal with target page)
2905 * @ms: current migration state
2907 static int postcopy_each_ram_send_discard(MigrationState
*ms
)
2909 struct RAMBlock
*block
;
2912 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
2913 postcopy_discard_send_init(ms
, block
->idstr
);
2916 * Postcopy sends chunks of bitmap over the wire, but it
2917 * just needs indexes at this point, avoids it having
2918 * target page specific code.
2920 ret
= postcopy_send_discard_bm_ram(ms
, block
);
2921 postcopy_discard_send_finish(ms
);
2931 * postcopy_chunk_hostpages_pass: canonicalize bitmap in hostpages
2933 * Helper for postcopy_chunk_hostpages; it's called twice to
2934 * canonicalize the two bitmaps, that are similar, but one is
2937 * Postcopy requires that all target pages in a hostpage are dirty or
2938 * clean, not a mix. This function canonicalizes the bitmaps.
2940 * @ms: current migration state
2941 * @unsent_pass: if true we need to canonicalize partially unsent host pages
2942 * otherwise we need to canonicalize partially dirty host pages
2943 * @block: block that contains the page we want to canonicalize
2945 static void postcopy_chunk_hostpages_pass(MigrationState
*ms
, bool unsent_pass
,
2948 RAMState
*rs
= ram_state
;
2949 unsigned long *bitmap
= block
->bmap
;
2950 unsigned long *unsentmap
= block
->unsentmap
;
2951 unsigned int host_ratio
= block
->page_size
/ TARGET_PAGE_SIZE
;
2952 unsigned long pages
= block
->used_length
>> TARGET_PAGE_BITS
;
2953 unsigned long run_start
;
2955 if (block
->page_size
== TARGET_PAGE_SIZE
) {
2956 /* Easy case - TPS==HPS for a non-huge page RAMBlock */
2961 /* Find a sent page */
2962 run_start
= find_next_zero_bit(unsentmap
, pages
, 0);
2964 /* Find a dirty page */
2965 run_start
= find_next_bit(bitmap
, pages
, 0);
2968 while (run_start
< pages
) {
2971 * If the start of this run of pages is in the middle of a host
2972 * page, then we need to fixup this host page.
2974 if (QEMU_IS_ALIGNED(run_start
, host_ratio
)) {
2975 /* Find the end of this run */
2977 run_start
= find_next_bit(unsentmap
, pages
, run_start
+ 1);
2979 run_start
= find_next_zero_bit(bitmap
, pages
, run_start
+ 1);
2982 * If the end isn't at the start of a host page, then the
2983 * run doesn't finish at the end of a host page
2984 * and we need to discard.
2988 if (!QEMU_IS_ALIGNED(run_start
, host_ratio
)) {
2990 unsigned long fixup_start_addr
= QEMU_ALIGN_DOWN(run_start
,
2992 run_start
= QEMU_ALIGN_UP(run_start
, host_ratio
);
2994 /* Clean up the bitmap */
2995 for (page
= fixup_start_addr
;
2996 page
< fixup_start_addr
+ host_ratio
; page
++) {
2997 /* All pages in this host page are now not sent */
2998 set_bit(page
, unsentmap
);
3001 * Remark them as dirty, updating the count for any pages
3002 * that weren't previously dirty.
3004 rs
->migration_dirty_pages
+= !test_and_set_bit(page
, bitmap
);
3009 /* Find the next sent page for the next iteration */
3010 run_start
= find_next_zero_bit(unsentmap
, pages
, run_start
);
3012 /* Find the next dirty page for the next iteration */
3013 run_start
= find_next_bit(bitmap
, pages
, run_start
);
3019 * postcopy_chunk_hostpages: discard any partially sent host page
3021 * Utility for the outgoing postcopy code.
3023 * Discard any partially sent host-page size chunks, mark any partially
3024 * dirty host-page size chunks as all dirty. In this case the host-page
3025 * is the host-page for the particular RAMBlock, i.e. it might be a huge page
3027 * Returns zero on success
3029 * @ms: current migration state
3030 * @block: block we want to work with
3032 static int postcopy_chunk_hostpages(MigrationState
*ms
, RAMBlock
*block
)
3034 postcopy_discard_send_init(ms
, block
->idstr
);
3036 /* First pass: Discard all partially sent host pages */
3037 postcopy_chunk_hostpages_pass(ms
, true, block
);
3039 * Second pass: Ensure that all partially dirty host pages are made
3042 postcopy_chunk_hostpages_pass(ms
, false, block
);
3044 postcopy_discard_send_finish(ms
);
3049 * ram_postcopy_send_discard_bitmap: transmit the discard bitmap
3051 * Returns zero on success
3053 * Transmit the set of pages to be discarded after precopy to the target
3054 * these are pages that:
3055 * a) Have been previously transmitted but are now dirty again
3056 * b) Pages that have never been transmitted, this ensures that
3057 * any pages on the destination that have been mapped by background
3058 * tasks get discarded (transparent huge pages is the specific concern)
3059 * Hopefully this is pretty sparse
3061 * @ms: current migration state
3063 int ram_postcopy_send_discard_bitmap(MigrationState
*ms
)
3065 RAMState
*rs
= ram_state
;
3071 /* This should be our last sync, the src is now paused */
3072 migration_bitmap_sync(rs
);
3074 /* Easiest way to make sure we don't resume in the middle of a host-page */
3075 rs
->last_seen_block
= NULL
;
3076 rs
->last_sent_block
= NULL
;
3079 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3080 unsigned long pages
= block
->used_length
>> TARGET_PAGE_BITS
;
3081 unsigned long *bitmap
= block
->bmap
;
3082 unsigned long *unsentmap
= block
->unsentmap
;
3085 /* We don't have a safe way to resize the sentmap, so
3086 * if the bitmap was resized it will be NULL at this
3089 error_report("migration ram resized during precopy phase");
3093 /* Deal with TPS != HPS and huge pages */
3094 ret
= postcopy_chunk_hostpages(ms
, block
);
3101 * Update the unsentmap to be unsentmap = unsentmap | dirty
3103 bitmap_or(unsentmap
, unsentmap
, bitmap
, pages
);
3104 #ifdef DEBUG_POSTCOPY
3105 ram_debug_dump_bitmap(unsentmap
, true, pages
);
3108 trace_ram_postcopy_send_discard_bitmap();
3110 ret
= postcopy_each_ram_send_discard(ms
);
3117 * ram_discard_range: discard dirtied pages at the beginning of postcopy
3119 * Returns zero on success
3121 * @rbname: name of the RAMBlock of the request. NULL means the
3122 * same that last one.
3123 * @start: RAMBlock starting page
3124 * @length: RAMBlock size
3126 int ram_discard_range(const char *rbname
, uint64_t start
, size_t length
)
3130 trace_ram_discard_range(rbname
, start
, length
);
3133 RAMBlock
*rb
= qemu_ram_block_by_name(rbname
);
3136 error_report("ram_discard_range: Failed to find block '%s'", rbname
);
3141 * On source VM, we don't need to update the received bitmap since
3142 * we don't even have one.
3144 if (rb
->receivedmap
) {
3145 bitmap_clear(rb
->receivedmap
, start
>> qemu_target_page_bits(),
3146 length
>> qemu_target_page_bits());
3149 ret
= ram_block_discard_range(rb
, start
, length
);
3158 * For every allocation, we will try not to crash the VM if the
3159 * allocation failed.
3161 static int xbzrle_init(void)
3163 Error
*local_err
= NULL
;
3165 if (!migrate_use_xbzrle()) {
3169 XBZRLE_cache_lock();
3171 XBZRLE
.zero_target_page
= g_try_malloc0(TARGET_PAGE_SIZE
);
3172 if (!XBZRLE
.zero_target_page
) {
3173 error_report("%s: Error allocating zero page", __func__
);
3177 XBZRLE
.cache
= cache_init(migrate_xbzrle_cache_size(),
3178 TARGET_PAGE_SIZE
, &local_err
);
3179 if (!XBZRLE
.cache
) {
3180 error_report_err(local_err
);
3181 goto free_zero_page
;
3184 XBZRLE
.encoded_buf
= g_try_malloc0(TARGET_PAGE_SIZE
);
3185 if (!XBZRLE
.encoded_buf
) {
3186 error_report("%s: Error allocating encoded_buf", __func__
);
3190 XBZRLE
.current_buf
= g_try_malloc(TARGET_PAGE_SIZE
);
3191 if (!XBZRLE
.current_buf
) {
3192 error_report("%s: Error allocating current_buf", __func__
);
3193 goto free_encoded_buf
;
3196 /* We are all good */
3197 XBZRLE_cache_unlock();
3201 g_free(XBZRLE
.encoded_buf
);
3202 XBZRLE
.encoded_buf
= NULL
;
3204 cache_fini(XBZRLE
.cache
);
3205 XBZRLE
.cache
= NULL
;
3207 g_free(XBZRLE
.zero_target_page
);
3208 XBZRLE
.zero_target_page
= NULL
;
3210 XBZRLE_cache_unlock();
3214 static int ram_state_init(RAMState
**rsp
)
3216 *rsp
= g_try_new0(RAMState
, 1);
3219 error_report("%s: Init ramstate fail", __func__
);
3223 qemu_mutex_init(&(*rsp
)->bitmap_mutex
);
3224 qemu_mutex_init(&(*rsp
)->src_page_req_mutex
);
3225 QSIMPLEQ_INIT(&(*rsp
)->src_page_requests
);
3228 * Count the total number of pages used by ram blocks not including any
3229 * gaps due to alignment or unplugs.
3230 * This must match with the initial values of dirty bitmap.
3232 (*rsp
)->migration_dirty_pages
= ram_bytes_total() >> TARGET_PAGE_BITS
;
3233 ram_state_reset(*rsp
);
3238 static void ram_list_init_bitmaps(void)
3240 MigrationState
*ms
= migrate_get_current();
3242 unsigned long pages
;
3245 /* Skip setting bitmap if there is no RAM */
3246 if (ram_bytes_total()) {
3247 shift
= ms
->clear_bitmap_shift
;
3248 if (shift
> CLEAR_BITMAP_SHIFT_MAX
) {
3249 error_report("clear_bitmap_shift (%u) too big, using "
3250 "max value (%u)", shift
, CLEAR_BITMAP_SHIFT_MAX
);
3251 shift
= CLEAR_BITMAP_SHIFT_MAX
;
3252 } else if (shift
< CLEAR_BITMAP_SHIFT_MIN
) {
3253 error_report("clear_bitmap_shift (%u) too small, using "
3254 "min value (%u)", shift
, CLEAR_BITMAP_SHIFT_MIN
);
3255 shift
= CLEAR_BITMAP_SHIFT_MIN
;
3258 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3259 pages
= block
->max_length
>> TARGET_PAGE_BITS
;
3261 * The initial dirty bitmap for migration must be set with all
3262 * ones to make sure we'll migrate every guest RAM page to
3264 * Here we set RAMBlock.bmap all to 1 because when rebegin a
3265 * new migration after a failed migration, ram_list.
3266 * dirty_memory[DIRTY_MEMORY_MIGRATION] don't include the whole
3269 block
->bmap
= bitmap_new(pages
);
3270 bitmap_set(block
->bmap
, 0, pages
);
3271 block
->clear_bmap_shift
= shift
;
3272 block
->clear_bmap
= bitmap_new(clear_bmap_size(pages
, shift
));
3273 if (migrate_postcopy_ram()) {
3274 block
->unsentmap
= bitmap_new(pages
);
3275 bitmap_set(block
->unsentmap
, 0, pages
);
3281 static void ram_init_bitmaps(RAMState
*rs
)
3283 /* For memory_global_dirty_log_start below. */
3284 qemu_mutex_lock_iothread();
3285 qemu_mutex_lock_ramlist();
3288 ram_list_init_bitmaps();
3289 memory_global_dirty_log_start();
3290 migration_bitmap_sync_precopy(rs
);
3293 qemu_mutex_unlock_ramlist();
3294 qemu_mutex_unlock_iothread();
3297 static int ram_init_all(RAMState
**rsp
)
3299 if (ram_state_init(rsp
)) {
3303 if (xbzrle_init()) {
3304 ram_state_cleanup(rsp
);
3308 ram_init_bitmaps(*rsp
);
3313 static void ram_state_resume_prepare(RAMState
*rs
, QEMUFile
*out
)
3319 * Postcopy is not using xbzrle/compression, so no need for that.
3320 * Also, since source are already halted, we don't need to care
3321 * about dirty page logging as well.
3324 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3325 pages
+= bitmap_count_one(block
->bmap
,
3326 block
->used_length
>> TARGET_PAGE_BITS
);
3329 /* This may not be aligned with current bitmaps. Recalculate. */
3330 rs
->migration_dirty_pages
= pages
;
3332 rs
->last_seen_block
= NULL
;
3333 rs
->last_sent_block
= NULL
;
3335 rs
->last_version
= ram_list
.version
;
3337 * Disable the bulk stage, otherwise we'll resend the whole RAM no
3338 * matter what we have sent.
3340 rs
->ram_bulk_stage
= false;
3342 /* Update RAMState cache of output QEMUFile */
3345 trace_ram_state_resume_prepare(pages
);
3349 * This function clears bits of the free pages reported by the caller from the
3350 * migration dirty bitmap. @addr is the host address corresponding to the
3351 * start of the continuous guest free pages, and @len is the total bytes of
3354 void qemu_guest_free_page_hint(void *addr
, size_t len
)
3358 size_t used_len
, start
, npages
;
3359 MigrationState
*s
= migrate_get_current();
3361 /* This function is currently expected to be used during live migration */
3362 if (!migration_is_setup_or_active(s
->state
)) {
3366 for (; len
> 0; len
-= used_len
, addr
+= used_len
) {
3367 block
= qemu_ram_block_from_host(addr
, false, &offset
);
3368 if (unlikely(!block
|| offset
>= block
->used_length
)) {
3370 * The implementation might not support RAMBlock resize during
3371 * live migration, but it could happen in theory with future
3372 * updates. So we add a check here to capture that case.
3374 error_report_once("%s unexpected error", __func__
);
3378 if (len
<= block
->used_length
- offset
) {
3381 used_len
= block
->used_length
- offset
;
3384 start
= offset
>> TARGET_PAGE_BITS
;
3385 npages
= used_len
>> TARGET_PAGE_BITS
;
3387 qemu_mutex_lock(&ram_state
->bitmap_mutex
);
3388 ram_state
->migration_dirty_pages
-=
3389 bitmap_count_one_with_offset(block
->bmap
, start
, npages
);
3390 bitmap_clear(block
->bmap
, start
, npages
);
3391 qemu_mutex_unlock(&ram_state
->bitmap_mutex
);
3396 * Each of ram_save_setup, ram_save_iterate and ram_save_complete has
3397 * long-running RCU critical section. When rcu-reclaims in the code
3398 * start to become numerous it will be necessary to reduce the
3399 * granularity of these critical sections.
3403 * ram_save_setup: Setup RAM for migration
3405 * Returns zero to indicate success and negative for error
3407 * @f: QEMUFile where to send the data
3408 * @opaque: RAMState pointer
3410 static int ram_save_setup(QEMUFile
*f
, void *opaque
)
3412 RAMState
**rsp
= opaque
;
3415 if (compress_threads_save_setup()) {
3419 /* migration has already setup the bitmap, reuse it. */
3420 if (!migration_in_colo_state()) {
3421 if (ram_init_all(rsp
) != 0) {
3422 compress_threads_save_cleanup();
3430 qemu_put_be64(f
, ram_bytes_total_common(true) | RAM_SAVE_FLAG_MEM_SIZE
);
3432 RAMBLOCK_FOREACH_MIGRATABLE(block
) {
3433 qemu_put_byte(f
, strlen(block
->idstr
));
3434 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, strlen(block
->idstr
));
3435 qemu_put_be64(f
, block
->used_length
);
3436 if (migrate_postcopy_ram() && block
->page_size
!= qemu_host_page_size
) {
3437 qemu_put_be64(f
, block
->page_size
);
3439 if (migrate_ignore_shared()) {
3440 qemu_put_be64(f
, block
->mr
->addr
);
3446 ram_control_before_iterate(f
, RAM_CONTROL_SETUP
);
3447 ram_control_after_iterate(f
, RAM_CONTROL_SETUP
);
3449 multifd_send_sync_main(*rsp
);
3450 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
3457 * ram_save_iterate: iterative stage for migration
3459 * Returns zero to indicate success and negative for error
3461 * @f: QEMUFile where to send the data
3462 * @opaque: RAMState pointer
3464 static int ram_save_iterate(QEMUFile
*f
, void *opaque
)
3466 RAMState
**temp
= opaque
;
3467 RAMState
*rs
= *temp
;
3473 if (blk_mig_bulk_active()) {
3474 /* Avoid transferring ram during bulk phase of block migration as
3475 * the bulk phase will usually take a long time and transferring
3476 * ram updates during that time is pointless. */
3481 if (ram_list
.version
!= rs
->last_version
) {
3482 ram_state_reset(rs
);
3485 /* Read version before ram_list.blocks */
3488 ram_control_before_iterate(f
, RAM_CONTROL_ROUND
);
3490 t0
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
);
3492 while ((ret
= qemu_file_rate_limit(f
)) == 0 ||
3493 !QSIMPLEQ_EMPTY(&rs
->src_page_requests
)) {
3496 if (qemu_file_get_error(f
)) {
3500 pages
= ram_find_and_save_block(rs
, false);
3501 /* no more pages to sent */
3508 qemu_file_set_error(f
, pages
);
3512 rs
->target_page_count
+= pages
;
3514 /* we want to check in the 1st loop, just in case it was the 1st time
3515 and we had to sync the dirty bitmap.
3516 qemu_clock_get_ns() is a bit expensive, so we only check each some
3519 if ((i
& 63) == 0) {
3520 uint64_t t1
= (qemu_clock_get_ns(QEMU_CLOCK_REALTIME
) - t0
) / 1000000;
3521 if (t1
> MAX_WAIT
) {
3522 trace_ram_save_iterate_big_wait(t1
, i
);
3531 * Must occur before EOS (or any QEMUFile operation)
3532 * because of RDMA protocol.
3534 ram_control_after_iterate(f
, RAM_CONTROL_ROUND
);
3537 multifd_send_sync_main(rs
);
3538 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
3540 ram_counters
.transferred
+= 8;
3542 ret
= qemu_file_get_error(f
);
3551 * ram_save_complete: function called to send the remaining amount of ram
3553 * Returns zero to indicate success or negative on error
3555 * Called with iothread lock
3557 * @f: QEMUFile where to send the data
3558 * @opaque: RAMState pointer
3560 static int ram_save_complete(QEMUFile
*f
, void *opaque
)
3562 RAMState
**temp
= opaque
;
3563 RAMState
*rs
= *temp
;
3568 if (!migration_in_postcopy()) {
3569 migration_bitmap_sync_precopy(rs
);
3572 ram_control_before_iterate(f
, RAM_CONTROL_FINISH
);
3574 /* try transferring iterative blocks of memory */
3576 /* flush all remaining blocks regardless of rate limiting */
3580 pages
= ram_find_and_save_block(rs
, !migration_in_colo_state());
3581 /* no more blocks to sent */
3591 flush_compressed_data(rs
);
3592 ram_control_after_iterate(f
, RAM_CONTROL_FINISH
);
3596 multifd_send_sync_main(rs
);
3597 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
3603 static void ram_save_pending(QEMUFile
*f
, void *opaque
, uint64_t max_size
,
3604 uint64_t *res_precopy_only
,
3605 uint64_t *res_compatible
,
3606 uint64_t *res_postcopy_only
)
3608 RAMState
**temp
= opaque
;
3609 RAMState
*rs
= *temp
;
3610 uint64_t remaining_size
;
3612 remaining_size
= rs
->migration_dirty_pages
* TARGET_PAGE_SIZE
;
3614 if (!migration_in_postcopy() &&
3615 remaining_size
< max_size
) {
3616 qemu_mutex_lock_iothread();
3618 migration_bitmap_sync_precopy(rs
);
3620 qemu_mutex_unlock_iothread();
3621 remaining_size
= rs
->migration_dirty_pages
* TARGET_PAGE_SIZE
;
3624 if (migrate_postcopy_ram()) {
3625 /* We can do postcopy, and all the data is postcopiable */
3626 *res_compatible
+= remaining_size
;
3628 *res_precopy_only
+= remaining_size
;
3632 static int load_xbzrle(QEMUFile
*f
, ram_addr_t addr
, void *host
)
3634 unsigned int xh_len
;
3636 uint8_t *loaded_data
;
3638 /* extract RLE header */
3639 xh_flags
= qemu_get_byte(f
);
3640 xh_len
= qemu_get_be16(f
);
3642 if (xh_flags
!= ENCODING_FLAG_XBZRLE
) {
3643 error_report("Failed to load XBZRLE page - wrong compression!");
3647 if (xh_len
> TARGET_PAGE_SIZE
) {
3648 error_report("Failed to load XBZRLE page - len overflow!");
3651 loaded_data
= XBZRLE
.decoded_buf
;
3652 /* load data and decode */
3653 /* it can change loaded_data to point to an internal buffer */
3654 qemu_get_buffer_in_place(f
, &loaded_data
, xh_len
);
3657 if (xbzrle_decode_buffer(loaded_data
, xh_len
, host
,
3658 TARGET_PAGE_SIZE
) == -1) {
3659 error_report("Failed to load XBZRLE page - decode error!");
3667 * ram_block_from_stream: read a RAMBlock id from the migration stream
3669 * Must be called from within a rcu critical section.
3671 * Returns a pointer from within the RCU-protected ram_list.
3673 * @f: QEMUFile where to read the data from
3674 * @flags: Page flags (mostly to see if it's a continuation of previous block)
3676 static inline RAMBlock
*ram_block_from_stream(QEMUFile
*f
, int flags
)
3678 static RAMBlock
*block
= NULL
;
3682 if (flags
& RAM_SAVE_FLAG_CONTINUE
) {
3684 error_report("Ack, bad migration stream!");
3690 len
= qemu_get_byte(f
);
3691 qemu_get_buffer(f
, (uint8_t *)id
, len
);
3694 block
= qemu_ram_block_by_name(id
);
3696 error_report("Can't find block %s", id
);
3700 if (ramblock_is_ignored(block
)) {
3701 error_report("block %s should not be migrated !", id
);
3708 static inline void *host_from_ram_block_offset(RAMBlock
*block
,
3711 if (!offset_in_ramblock(block
, offset
)) {
3715 return block
->host
+ offset
;
3718 static inline void *colo_cache_from_block_offset(RAMBlock
*block
,
3721 if (!offset_in_ramblock(block
, offset
)) {
3724 if (!block
->colo_cache
) {
3725 error_report("%s: colo_cache is NULL in block :%s",
3726 __func__
, block
->idstr
);
3731 * During colo checkpoint, we need bitmap of these migrated pages.
3732 * It help us to decide which pages in ram cache should be flushed
3733 * into VM's RAM later.
3735 if (!test_and_set_bit(offset
>> TARGET_PAGE_BITS
, block
->bmap
)) {
3736 ram_state
->migration_dirty_pages
++;
3738 return block
->colo_cache
+ offset
;
3742 * ram_handle_compressed: handle the zero page case
3744 * If a page (or a whole RDMA chunk) has been
3745 * determined to be zero, then zap it.
3747 * @host: host address for the zero page
3748 * @ch: what the page is filled from. We only support zero
3749 * @size: size of the zero page
3751 void ram_handle_compressed(void *host
, uint8_t ch
, uint64_t size
)
3753 if (ch
!= 0 || !is_zero_range(host
, size
)) {
3754 memset(host
, ch
, size
);
3758 /* return the size after decompression, or negative value on error */
3760 qemu_uncompress_data(z_stream
*stream
, uint8_t *dest
, size_t dest_len
,
3761 const uint8_t *source
, size_t source_len
)
3765 err
= inflateReset(stream
);
3770 stream
->avail_in
= source_len
;
3771 stream
->next_in
= (uint8_t *)source
;
3772 stream
->avail_out
= dest_len
;
3773 stream
->next_out
= dest
;
3775 err
= inflate(stream
, Z_NO_FLUSH
);
3776 if (err
!= Z_STREAM_END
) {
3780 return stream
->total_out
;
3783 static void *do_data_decompress(void *opaque
)
3785 DecompressParam
*param
= opaque
;
3786 unsigned long pagesize
;
3790 qemu_mutex_lock(¶m
->mutex
);
3791 while (!param
->quit
) {
3796 qemu_mutex_unlock(¶m
->mutex
);
3798 pagesize
= TARGET_PAGE_SIZE
;
3800 ret
= qemu_uncompress_data(¶m
->stream
, des
, pagesize
,
3801 param
->compbuf
, len
);
3802 if (ret
< 0 && migrate_get_current()->decompress_error_check
) {
3803 error_report("decompress data failed");
3804 qemu_file_set_error(decomp_file
, ret
);
3807 qemu_mutex_lock(&decomp_done_lock
);
3809 qemu_cond_signal(&decomp_done_cond
);
3810 qemu_mutex_unlock(&decomp_done_lock
);
3812 qemu_mutex_lock(¶m
->mutex
);
3814 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
3817 qemu_mutex_unlock(¶m
->mutex
);
3822 static int wait_for_decompress_done(void)
3824 int idx
, thread_count
;
3826 if (!migrate_use_compression()) {
3830 thread_count
= migrate_decompress_threads();
3831 qemu_mutex_lock(&decomp_done_lock
);
3832 for (idx
= 0; idx
< thread_count
; idx
++) {
3833 while (!decomp_param
[idx
].done
) {
3834 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
3837 qemu_mutex_unlock(&decomp_done_lock
);
3838 return qemu_file_get_error(decomp_file
);
3841 static void compress_threads_load_cleanup(void)
3843 int i
, thread_count
;
3845 if (!migrate_use_compression()) {
3848 thread_count
= migrate_decompress_threads();
3849 for (i
= 0; i
< thread_count
; i
++) {
3851 * we use it as a indicator which shows if the thread is
3852 * properly init'd or not
3854 if (!decomp_param
[i
].compbuf
) {
3858 qemu_mutex_lock(&decomp_param
[i
].mutex
);
3859 decomp_param
[i
].quit
= true;
3860 qemu_cond_signal(&decomp_param
[i
].cond
);
3861 qemu_mutex_unlock(&decomp_param
[i
].mutex
);
3863 for (i
= 0; i
< thread_count
; i
++) {
3864 if (!decomp_param
[i
].compbuf
) {
3868 qemu_thread_join(decompress_threads
+ i
);
3869 qemu_mutex_destroy(&decomp_param
[i
].mutex
);
3870 qemu_cond_destroy(&decomp_param
[i
].cond
);
3871 inflateEnd(&decomp_param
[i
].stream
);
3872 g_free(decomp_param
[i
].compbuf
);
3873 decomp_param
[i
].compbuf
= NULL
;
3875 g_free(decompress_threads
);
3876 g_free(decomp_param
);
3877 decompress_threads
= NULL
;
3878 decomp_param
= NULL
;
3882 static int compress_threads_load_setup(QEMUFile
*f
)
3884 int i
, thread_count
;
3886 if (!migrate_use_compression()) {
3890 thread_count
= migrate_decompress_threads();
3891 decompress_threads
= g_new0(QemuThread
, thread_count
);
3892 decomp_param
= g_new0(DecompressParam
, thread_count
);
3893 qemu_mutex_init(&decomp_done_lock
);
3894 qemu_cond_init(&decomp_done_cond
);
3896 for (i
= 0; i
< thread_count
; i
++) {
3897 if (inflateInit(&decomp_param
[i
].stream
) != Z_OK
) {
3901 decomp_param
[i
].compbuf
= g_malloc0(compressBound(TARGET_PAGE_SIZE
));
3902 qemu_mutex_init(&decomp_param
[i
].mutex
);
3903 qemu_cond_init(&decomp_param
[i
].cond
);
3904 decomp_param
[i
].done
= true;
3905 decomp_param
[i
].quit
= false;
3906 qemu_thread_create(decompress_threads
+ i
, "decompress",
3907 do_data_decompress
, decomp_param
+ i
,
3908 QEMU_THREAD_JOINABLE
);
3912 compress_threads_load_cleanup();
3916 static void decompress_data_with_multi_threads(QEMUFile
*f
,
3917 void *host
, int len
)
3919 int idx
, thread_count
;
3921 thread_count
= migrate_decompress_threads();
3922 qemu_mutex_lock(&decomp_done_lock
);
3924 for (idx
= 0; idx
< thread_count
; idx
++) {
3925 if (decomp_param
[idx
].done
) {
3926 decomp_param
[idx
].done
= false;
3927 qemu_mutex_lock(&decomp_param
[idx
].mutex
);
3928 qemu_get_buffer(f
, decomp_param
[idx
].compbuf
, len
);
3929 decomp_param
[idx
].des
= host
;
3930 decomp_param
[idx
].len
= len
;
3931 qemu_cond_signal(&decomp_param
[idx
].cond
);
3932 qemu_mutex_unlock(&decomp_param
[idx
].mutex
);
3936 if (idx
< thread_count
) {
3939 qemu_cond_wait(&decomp_done_cond
, &decomp_done_lock
);
3942 qemu_mutex_unlock(&decomp_done_lock
);
3946 * colo cache: this is for secondary VM, we cache the whole
3947 * memory of the secondary VM, it is need to hold the global lock
3948 * to call this helper.
3950 int colo_init_ram_cache(void)
3955 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3956 block
->colo_cache
= qemu_anon_ram_alloc(block
->used_length
,
3959 if (!block
->colo_cache
) {
3960 error_report("%s: Can't alloc memory for COLO cache of block %s,"
3961 "size 0x" RAM_ADDR_FMT
, __func__
, block
->idstr
,
3962 block
->used_length
);
3965 memcpy(block
->colo_cache
, block
->host
, block
->used_length
);
3969 * Record the dirty pages that sent by PVM, we use this dirty bitmap together
3970 * with to decide which page in cache should be flushed into SVM's RAM. Here
3971 * we use the same name 'ram_bitmap' as for migration.
3973 if (ram_bytes_total()) {
3976 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3977 unsigned long pages
= block
->max_length
>> TARGET_PAGE_BITS
;
3979 block
->bmap
= bitmap_new(pages
);
3980 bitmap_set(block
->bmap
, 0, pages
);
3983 ram_state
= g_new0(RAMState
, 1);
3984 ram_state
->migration_dirty_pages
= 0;
3985 qemu_mutex_init(&ram_state
->bitmap_mutex
);
3986 memory_global_dirty_log_start();
3992 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
3993 if (block
->colo_cache
) {
3994 qemu_anon_ram_free(block
->colo_cache
, block
->used_length
);
3995 block
->colo_cache
= NULL
;
4003 /* It is need to hold the global lock to call this helper */
4004 void colo_release_ram_cache(void)
4008 memory_global_dirty_log_stop();
4009 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
4010 g_free(block
->bmap
);
4016 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
4017 if (block
->colo_cache
) {
4018 qemu_anon_ram_free(block
->colo_cache
, block
->used_length
);
4019 block
->colo_cache
= NULL
;
4024 qemu_mutex_destroy(&ram_state
->bitmap_mutex
);
4030 * ram_load_setup: Setup RAM for migration incoming side
4032 * Returns zero to indicate success and negative for error
4034 * @f: QEMUFile where to receive the data
4035 * @opaque: RAMState pointer
4037 static int ram_load_setup(QEMUFile
*f
, void *opaque
)
4039 if (compress_threads_load_setup(f
)) {
4043 xbzrle_load_setup();
4044 ramblock_recv_map_init();
4049 static int ram_load_cleanup(void *opaque
)
4053 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
4054 if (ramblock_is_pmem(rb
)) {
4055 pmem_persist(rb
->host
, rb
->used_length
);
4059 xbzrle_load_cleanup();
4060 compress_threads_load_cleanup();
4062 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
4063 g_free(rb
->receivedmap
);
4064 rb
->receivedmap
= NULL
;
4071 * ram_postcopy_incoming_init: allocate postcopy data structures
4073 * Returns 0 for success and negative if there was one error
4075 * @mis: current migration incoming state
4077 * Allocate data structures etc needed by incoming migration with
4078 * postcopy-ram. postcopy-ram's similarly names
4079 * postcopy_ram_incoming_init does the work.
4081 int ram_postcopy_incoming_init(MigrationIncomingState
*mis
)
4083 return postcopy_ram_incoming_init(mis
);
4087 * ram_load_postcopy: load a page in postcopy case
4089 * Returns 0 for success or -errno in case of error
4091 * Called in postcopy mode by ram_load().
4092 * rcu_read_lock is taken prior to this being called.
4094 * @f: QEMUFile where to send the data
4096 static int ram_load_postcopy(QEMUFile
*f
)
4098 int flags
= 0, ret
= 0;
4099 bool place_needed
= false;
4100 bool matches_target_page_size
= false;
4101 MigrationIncomingState
*mis
= migration_incoming_get_current();
4102 /* Temporary page that is later 'placed' */
4103 void *postcopy_host_page
= postcopy_get_tmp_page(mis
);
4104 void *last_host
= NULL
;
4105 bool all_zero
= false;
4107 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
4110 void *page_buffer
= NULL
;
4111 void *place_source
= NULL
;
4112 RAMBlock
*block
= NULL
;
4115 addr
= qemu_get_be64(f
);
4118 * If qemu file error, we should stop here, and then "addr"
4121 ret
= qemu_file_get_error(f
);
4126 flags
= addr
& ~TARGET_PAGE_MASK
;
4127 addr
&= TARGET_PAGE_MASK
;
4129 trace_ram_load_postcopy_loop((uint64_t)addr
, flags
);
4130 place_needed
= false;
4131 if (flags
& (RAM_SAVE_FLAG_ZERO
| RAM_SAVE_FLAG_PAGE
)) {
4132 block
= ram_block_from_stream(f
, flags
);
4134 host
= host_from_ram_block_offset(block
, addr
);
4136 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
4140 matches_target_page_size
= block
->page_size
== TARGET_PAGE_SIZE
;
4142 * Postcopy requires that we place whole host pages atomically;
4143 * these may be huge pages for RAMBlocks that are backed by
4145 * To make it atomic, the data is read into a temporary page
4146 * that's moved into place later.
4147 * The migration protocol uses, possibly smaller, target-pages
4148 * however the source ensures it always sends all the components
4149 * of a host page in order.
4151 page_buffer
= postcopy_host_page
+
4152 ((uintptr_t)host
& (block
->page_size
- 1));
4153 /* If all TP are zero then we can optimise the place */
4154 if (!((uintptr_t)host
& (block
->page_size
- 1))) {
4157 /* not the 1st TP within the HP */
4158 if (host
!= (last_host
+ TARGET_PAGE_SIZE
)) {
4159 error_report("Non-sequential target page %p/%p",
4168 * If it's the last part of a host page then we place the host
4171 place_needed
= (((uintptr_t)host
+ TARGET_PAGE_SIZE
) &
4172 (block
->page_size
- 1)) == 0;
4173 place_source
= postcopy_host_page
;
4177 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
4178 case RAM_SAVE_FLAG_ZERO
:
4179 ch
= qemu_get_byte(f
);
4180 memset(page_buffer
, ch
, TARGET_PAGE_SIZE
);
4186 case RAM_SAVE_FLAG_PAGE
:
4188 if (!matches_target_page_size
) {
4189 /* For huge pages, we always use temporary buffer */
4190 qemu_get_buffer(f
, page_buffer
, TARGET_PAGE_SIZE
);
4193 * For small pages that matches target page size, we
4194 * avoid the qemu_file copy. Instead we directly use
4195 * the buffer of QEMUFile to place the page. Note: we
4196 * cannot do any QEMUFile operation before using that
4197 * buffer to make sure the buffer is valid when
4200 qemu_get_buffer_in_place(f
, (uint8_t **)&place_source
,
4204 case RAM_SAVE_FLAG_EOS
:
4206 multifd_recv_sync_main();
4209 error_report("Unknown combination of migration flags: %#x"
4210 " (postcopy mode)", flags
);
4215 /* Detect for any possible file errors */
4216 if (!ret
&& qemu_file_get_error(f
)) {
4217 ret
= qemu_file_get_error(f
);
4220 if (!ret
&& place_needed
) {
4221 /* This gets called at the last target page in the host page */
4222 void *place_dest
= host
+ TARGET_PAGE_SIZE
- block
->page_size
;
4225 ret
= postcopy_place_page_zero(mis
, place_dest
,
4228 ret
= postcopy_place_page(mis
, place_dest
,
4229 place_source
, block
);
4237 static bool postcopy_is_advised(void)
4239 PostcopyState ps
= postcopy_state_get();
4240 return ps
>= POSTCOPY_INCOMING_ADVISE
&& ps
< POSTCOPY_INCOMING_END
;
4243 static bool postcopy_is_running(void)
4245 PostcopyState ps
= postcopy_state_get();
4246 return ps
>= POSTCOPY_INCOMING_LISTENING
&& ps
< POSTCOPY_INCOMING_END
;
4250 * Flush content of RAM cache into SVM's memory.
4251 * Only flush the pages that be dirtied by PVM or SVM or both.
4253 static void colo_flush_ram_cache(void)
4255 RAMBlock
*block
= NULL
;
4258 unsigned long offset
= 0;
4260 memory_global_dirty_log_sync();
4262 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
4263 ramblock_sync_dirty_bitmap(ram_state
, block
);
4267 trace_colo_flush_ram_cache_begin(ram_state
->migration_dirty_pages
);
4269 block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
4272 offset
= migration_bitmap_find_dirty(ram_state
, block
, offset
);
4274 if (offset
<< TARGET_PAGE_BITS
>= block
->used_length
) {
4276 block
= QLIST_NEXT_RCU(block
, next
);
4278 migration_bitmap_clear_dirty(ram_state
, block
, offset
);
4279 dst_host
= block
->host
+ (offset
<< TARGET_PAGE_BITS
);
4280 src_host
= block
->colo_cache
+ (offset
<< TARGET_PAGE_BITS
);
4281 memcpy(dst_host
, src_host
, TARGET_PAGE_SIZE
);
4286 trace_colo_flush_ram_cache_end();
4290 * ram_load_precopy: load pages in precopy case
4292 * Returns 0 for success or -errno in case of error
4294 * Called in precopy mode by ram_load().
4295 * rcu_read_lock is taken prior to this being called.
4297 * @f: QEMUFile where to send the data
4299 static int ram_load_precopy(QEMUFile
*f
)
4301 int flags
= 0, ret
= 0, invalid_flags
= 0, len
= 0;
4302 /* ADVISE is earlier, it shows the source has the postcopy capability on */
4303 bool postcopy_advised
= postcopy_is_advised();
4304 if (!migrate_use_compression()) {
4305 invalid_flags
|= RAM_SAVE_FLAG_COMPRESS_PAGE
;
4308 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
4309 ram_addr_t addr
, total_ram_bytes
;
4313 addr
= qemu_get_be64(f
);
4314 flags
= addr
& ~TARGET_PAGE_MASK
;
4315 addr
&= TARGET_PAGE_MASK
;
4317 if (flags
& invalid_flags
) {
4318 if (flags
& invalid_flags
& RAM_SAVE_FLAG_COMPRESS_PAGE
) {
4319 error_report("Received an unexpected compressed page");
4326 if (flags
& (RAM_SAVE_FLAG_ZERO
| RAM_SAVE_FLAG_PAGE
|
4327 RAM_SAVE_FLAG_COMPRESS_PAGE
| RAM_SAVE_FLAG_XBZRLE
)) {
4328 RAMBlock
*block
= ram_block_from_stream(f
, flags
);
4331 * After going into COLO, we should load the Page into colo_cache.
4333 if (migration_incoming_in_colo_state()) {
4334 host
= colo_cache_from_block_offset(block
, addr
);
4336 host
= host_from_ram_block_offset(block
, addr
);
4339 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
4344 if (!migration_incoming_in_colo_state()) {
4345 ramblock_recv_bitmap_set(block
, host
);
4348 trace_ram_load_loop(block
->idstr
, (uint64_t)addr
, flags
, host
);
4351 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
4352 case RAM_SAVE_FLAG_MEM_SIZE
:
4353 /* Synchronize RAM block list */
4354 total_ram_bytes
= addr
;
4355 while (!ret
&& total_ram_bytes
) {
4360 len
= qemu_get_byte(f
);
4361 qemu_get_buffer(f
, (uint8_t *)id
, len
);
4363 length
= qemu_get_be64(f
);
4365 block
= qemu_ram_block_by_name(id
);
4366 if (block
&& !qemu_ram_is_migratable(block
)) {
4367 error_report("block %s should not be migrated !", id
);
4370 if (length
!= block
->used_length
) {
4371 Error
*local_err
= NULL
;
4373 ret
= qemu_ram_resize(block
, length
,
4376 error_report_err(local_err
);
4379 /* For postcopy we need to check hugepage sizes match */
4380 if (postcopy_advised
&&
4381 block
->page_size
!= qemu_host_page_size
) {
4382 uint64_t remote_page_size
= qemu_get_be64(f
);
4383 if (remote_page_size
!= block
->page_size
) {
4384 error_report("Mismatched RAM page size %s "
4385 "(local) %zd != %" PRId64
,
4386 id
, block
->page_size
,
4391 if (migrate_ignore_shared()) {
4392 hwaddr addr
= qemu_get_be64(f
);
4393 if (ramblock_is_ignored(block
) &&
4394 block
->mr
->addr
!= addr
) {
4395 error_report("Mismatched GPAs for block %s "
4396 "%" PRId64
"!= %" PRId64
,
4398 (uint64_t)block
->mr
->addr
);
4402 ram_control_load_hook(f
, RAM_CONTROL_BLOCK_REG
,
4405 error_report("Unknown ramblock \"%s\", cannot "
4406 "accept migration", id
);
4410 total_ram_bytes
-= length
;
4414 case RAM_SAVE_FLAG_ZERO
:
4415 ch
= qemu_get_byte(f
);
4416 ram_handle_compressed(host
, ch
, TARGET_PAGE_SIZE
);
4419 case RAM_SAVE_FLAG_PAGE
:
4420 qemu_get_buffer(f
, host
, TARGET_PAGE_SIZE
);
4423 case RAM_SAVE_FLAG_COMPRESS_PAGE
:
4424 len
= qemu_get_be32(f
);
4425 if (len
< 0 || len
> compressBound(TARGET_PAGE_SIZE
)) {
4426 error_report("Invalid compressed data length: %d", len
);
4430 decompress_data_with_multi_threads(f
, host
, len
);
4433 case RAM_SAVE_FLAG_XBZRLE
:
4434 if (load_xbzrle(f
, addr
, host
) < 0) {
4435 error_report("Failed to decompress XBZRLE page at "
4436 RAM_ADDR_FMT
, addr
);
4441 case RAM_SAVE_FLAG_EOS
:
4443 multifd_recv_sync_main();
4446 if (flags
& RAM_SAVE_FLAG_HOOK
) {
4447 ram_control_load_hook(f
, RAM_CONTROL_HOOK
, NULL
);
4449 error_report("Unknown combination of migration flags: %#x",
4455 ret
= qemu_file_get_error(f
);
4462 static int ram_load(QEMUFile
*f
, void *opaque
, int version_id
)
4465 static uint64_t seq_iter
;
4467 * If system is running in postcopy mode, page inserts to host memory must
4470 bool postcopy_running
= postcopy_is_running();
4474 if (version_id
!= 4) {
4479 * This RCU critical section can be very long running.
4480 * When RCU reclaims in the code start to become numerous,
4481 * it will be necessary to reduce the granularity of this
4486 if (postcopy_running
) {
4487 ret
= ram_load_postcopy(f
);
4489 ret
= ram_load_precopy(f
);
4492 ret
|= wait_for_decompress_done();
4494 trace_ram_load_complete(ret
, seq_iter
);
4496 if (!ret
&& migration_incoming_in_colo_state()) {
4497 colo_flush_ram_cache();
4502 static bool ram_has_postcopy(void *opaque
)
4505 RAMBLOCK_FOREACH_NOT_IGNORED(rb
) {
4506 if (ramblock_is_pmem(rb
)) {
4507 info_report("Block: %s, host: %p is a nvdimm memory, postcopy"
4508 "is not supported now!", rb
->idstr
, rb
->host
);
4513 return migrate_postcopy_ram();
4516 /* Sync all the dirty bitmap with destination VM. */
4517 static int ram_dirty_bitmap_sync_all(MigrationState
*s
, RAMState
*rs
)
4520 QEMUFile
*file
= s
->to_dst_file
;
4521 int ramblock_count
= 0;
4523 trace_ram_dirty_bitmap_sync_start();
4525 RAMBLOCK_FOREACH_NOT_IGNORED(block
) {
4526 qemu_savevm_send_recv_bitmap(file
, block
->idstr
);
4527 trace_ram_dirty_bitmap_request(block
->idstr
);
4531 trace_ram_dirty_bitmap_sync_wait();
4533 /* Wait until all the ramblocks' dirty bitmap synced */
4534 while (ramblock_count
--) {
4535 qemu_sem_wait(&s
->rp_state
.rp_sem
);
4538 trace_ram_dirty_bitmap_sync_complete();
4543 static void ram_dirty_bitmap_reload_notify(MigrationState
*s
)
4545 qemu_sem_post(&s
->rp_state
.rp_sem
);
4549 * Read the received bitmap, revert it as the initial dirty bitmap.
4550 * This is only used when the postcopy migration is paused but wants
4551 * to resume from a middle point.
4553 int ram_dirty_bitmap_reload(MigrationState
*s
, RAMBlock
*block
)
4556 QEMUFile
*file
= s
->rp_state
.from_dst_file
;
4557 unsigned long *le_bitmap
, nbits
= block
->used_length
>> TARGET_PAGE_BITS
;
4558 uint64_t local_size
= DIV_ROUND_UP(nbits
, 8);
4559 uint64_t size
, end_mark
;
4561 trace_ram_dirty_bitmap_reload_begin(block
->idstr
);
4563 if (s
->state
!= MIGRATION_STATUS_POSTCOPY_RECOVER
) {
4564 error_report("%s: incorrect state %s", __func__
,
4565 MigrationStatus_str(s
->state
));
4570 * Note: see comments in ramblock_recv_bitmap_send() on why we
4571 * need the endianess convertion, and the paddings.
4573 local_size
= ROUND_UP(local_size
, 8);
4576 le_bitmap
= bitmap_new(nbits
+ BITS_PER_LONG
);
4578 size
= qemu_get_be64(file
);
4580 /* The size of the bitmap should match with our ramblock */
4581 if (size
!= local_size
) {
4582 error_report("%s: ramblock '%s' bitmap size mismatch "
4583 "(0x%"PRIx64
" != 0x%"PRIx64
")", __func__
,
4584 block
->idstr
, size
, local_size
);
4589 size
= qemu_get_buffer(file
, (uint8_t *)le_bitmap
, local_size
);
4590 end_mark
= qemu_get_be64(file
);
4592 ret
= qemu_file_get_error(file
);
4593 if (ret
|| size
!= local_size
) {
4594 error_report("%s: read bitmap failed for ramblock '%s': %d"
4595 " (size 0x%"PRIx64
", got: 0x%"PRIx64
")",
4596 __func__
, block
->idstr
, ret
, local_size
, size
);
4601 if (end_mark
!= RAMBLOCK_RECV_BITMAP_ENDING
) {
4602 error_report("%s: ramblock '%s' end mark incorrect: 0x%"PRIu64
,
4603 __func__
, block
->idstr
, end_mark
);
4609 * Endianess convertion. We are during postcopy (though paused).
4610 * The dirty bitmap won't change. We can directly modify it.
4612 bitmap_from_le(block
->bmap
, le_bitmap
, nbits
);
4615 * What we received is "received bitmap". Revert it as the initial
4616 * dirty bitmap for this ramblock.
4618 bitmap_complement(block
->bmap
, block
->bmap
, nbits
);
4620 trace_ram_dirty_bitmap_reload_complete(block
->idstr
);
4623 * We succeeded to sync bitmap for current ramblock. If this is
4624 * the last one to sync, we need to notify the main send thread.
4626 ram_dirty_bitmap_reload_notify(s
);
4634 static int ram_resume_prepare(MigrationState
*s
, void *opaque
)
4636 RAMState
*rs
= *(RAMState
**)opaque
;
4639 ret
= ram_dirty_bitmap_sync_all(s
, rs
);
4644 ram_state_resume_prepare(rs
, s
->to_dst_file
);
4649 static SaveVMHandlers savevm_ram_handlers
= {
4650 .save_setup
= ram_save_setup
,
4651 .save_live_iterate
= ram_save_iterate
,
4652 .save_live_complete_postcopy
= ram_save_complete
,
4653 .save_live_complete_precopy
= ram_save_complete
,
4654 .has_postcopy
= ram_has_postcopy
,
4655 .save_live_pending
= ram_save_pending
,
4656 .load_state
= ram_load
,
4657 .save_cleanup
= ram_save_cleanup
,
4658 .load_setup
= ram_load_setup
,
4659 .load_cleanup
= ram_load_cleanup
,
4660 .resume_prepare
= ram_resume_prepare
,
4663 void ram_mig_init(void)
4665 qemu_mutex_init(&XBZRLE
.lock
);
4666 register_savevm_live("ram", 0, 4, &savevm_ram_handlers
, &ram_state
);