4 * Copyright (c) 2003-2008 Fabrice Bellard
5 * Copyright (c) 2011-2015 Red Hat Inc
8 * Juan Quintela <quintela@redhat.com>
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
30 #include "qemu/bitops.h"
31 #include "qemu/bitmap.h"
32 #include "qemu/timer.h"
33 #include "qemu/main-loop.h"
34 #include "migration/migration.h"
35 #include "migration/postcopy-ram.h"
36 #include "exec/address-spaces.h"
37 #include "migration/page_cache.h"
38 #include "qemu/error-report.h"
40 #include "exec/ram_addr.h"
41 #include "qemu/rcu_queue.h"
43 #ifdef DEBUG_MIGRATION_RAM
44 #define DPRINTF(fmt, ...) \
45 do { fprintf(stdout, "migration_ram: " fmt, ## __VA_ARGS__); } while (0)
47 #define DPRINTF(fmt, ...) \
51 static int dirty_rate_high_cnt
;
53 static uint64_t bitmap_sync_count
;
55 /***********************************************************/
56 /* ram save/restore */
58 #define RAM_SAVE_FLAG_FULL 0x01 /* Obsolete, not used anymore */
59 #define RAM_SAVE_FLAG_COMPRESS 0x02
60 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
61 #define RAM_SAVE_FLAG_PAGE 0x08
62 #define RAM_SAVE_FLAG_EOS 0x10
63 #define RAM_SAVE_FLAG_CONTINUE 0x20
64 #define RAM_SAVE_FLAG_XBZRLE 0x40
65 /* 0x80 is reserved in migration.h start with 0x100 next */
66 #define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
68 static const uint8_t ZERO_TARGET_PAGE
[TARGET_PAGE_SIZE
];
70 static inline bool is_zero_range(uint8_t *p
, uint64_t size
)
72 return buffer_find_nonzero_offset(p
, size
) == size
;
75 /* struct contains XBZRLE cache and a static page
76 used by the compression */
78 /* buffer used for XBZRLE encoding */
80 /* buffer for storing page content */
82 /* Cache for XBZRLE, Protected by lock. */
87 /* buffer used for XBZRLE decoding */
88 static uint8_t *xbzrle_decoded_buf
;
90 static void XBZRLE_cache_lock(void)
92 if (migrate_use_xbzrle())
93 qemu_mutex_lock(&XBZRLE
.lock
);
96 static void XBZRLE_cache_unlock(void)
98 if (migrate_use_xbzrle())
99 qemu_mutex_unlock(&XBZRLE
.lock
);
103 * called from qmp_migrate_set_cache_size in main thread, possibly while
104 * a migration is in progress.
105 * A running migration maybe using the cache and might finish during this
106 * call, hence changes to the cache are protected by XBZRLE.lock().
108 int64_t xbzrle_cache_resize(int64_t new_size
)
110 PageCache
*new_cache
;
113 if (new_size
< TARGET_PAGE_SIZE
) {
119 if (XBZRLE
.cache
!= NULL
) {
120 if (pow2floor(new_size
) == migrate_xbzrle_cache_size()) {
123 new_cache
= cache_init(new_size
/ TARGET_PAGE_SIZE
,
126 error_report("Error creating cache");
131 cache_fini(XBZRLE
.cache
);
132 XBZRLE
.cache
= new_cache
;
136 ret
= pow2floor(new_size
);
138 XBZRLE_cache_unlock();
142 /* accounting for migration statistics */
143 typedef struct AccountingInfo
{
145 uint64_t skipped_pages
;
148 uint64_t xbzrle_bytes
;
149 uint64_t xbzrle_pages
;
150 uint64_t xbzrle_cache_miss
;
151 double xbzrle_cache_miss_rate
;
152 uint64_t xbzrle_overflows
;
155 static AccountingInfo acct_info
;
157 static void acct_clear(void)
159 memset(&acct_info
, 0, sizeof(acct_info
));
162 uint64_t dup_mig_bytes_transferred(void)
164 return acct_info
.dup_pages
* TARGET_PAGE_SIZE
;
167 uint64_t dup_mig_pages_transferred(void)
169 return acct_info
.dup_pages
;
172 uint64_t skipped_mig_bytes_transferred(void)
174 return acct_info
.skipped_pages
* TARGET_PAGE_SIZE
;
177 uint64_t skipped_mig_pages_transferred(void)
179 return acct_info
.skipped_pages
;
182 uint64_t norm_mig_bytes_transferred(void)
184 return acct_info
.norm_pages
* TARGET_PAGE_SIZE
;
187 uint64_t norm_mig_pages_transferred(void)
189 return acct_info
.norm_pages
;
192 uint64_t xbzrle_mig_bytes_transferred(void)
194 return acct_info
.xbzrle_bytes
;
197 uint64_t xbzrle_mig_pages_transferred(void)
199 return acct_info
.xbzrle_pages
;
202 uint64_t xbzrle_mig_pages_cache_miss(void)
204 return acct_info
.xbzrle_cache_miss
;
207 double xbzrle_mig_cache_miss_rate(void)
209 return acct_info
.xbzrle_cache_miss_rate
;
212 uint64_t xbzrle_mig_pages_overflow(void)
214 return acct_info
.xbzrle_overflows
;
217 /* This is the last block that we have visited serching for dirty pages
219 static RAMBlock
*last_seen_block
;
220 /* This is the last block from where we have sent data */
221 static RAMBlock
*last_sent_block
;
222 static ram_addr_t last_offset
;
223 static QemuMutex migration_bitmap_mutex
;
224 static uint64_t migration_dirty_pages
;
225 static uint32_t last_version
;
226 static bool ram_bulk_stage
;
228 /* used by the search for pages to send */
229 struct PageSearchStatus
{
230 /* Current block being searched */
232 /* Current offset to search from */
234 /* Set once we wrap around */
237 typedef struct PageSearchStatus PageSearchStatus
;
239 static struct BitmapRcu
{
241 /* Main migration bitmap */
243 /* bitmap of pages that haven't been sent even once
244 * only maintained and used in postcopy at the moment
245 * where it's used to send the dirtymap at the start
246 * of the postcopy phase
248 unsigned long *unsentmap
;
249 } *migration_bitmap_rcu
;
251 struct CompressParam
{
260 typedef struct CompressParam CompressParam
;
262 struct DecompressParam
{
270 typedef struct DecompressParam DecompressParam
;
272 static CompressParam
*comp_param
;
273 static QemuThread
*compress_threads
;
274 /* comp_done_cond is used to wake up the migration thread when
275 * one of the compression threads has finished the compression.
276 * comp_done_lock is used to co-work with comp_done_cond.
278 static QemuMutex
*comp_done_lock
;
279 static QemuCond
*comp_done_cond
;
280 /* The empty QEMUFileOps will be used by file in CompressParam */
281 static const QEMUFileOps empty_ops
= { };
283 static bool compression_switch
;
284 static bool quit_comp_thread
;
285 static bool quit_decomp_thread
;
286 static DecompressParam
*decomp_param
;
287 static QemuThread
*decompress_threads
;
288 static uint8_t *compressed_data_buf
;
290 static int do_compress_ram_page(CompressParam
*param
);
292 static void *do_data_compress(void *opaque
)
294 CompressParam
*param
= opaque
;
296 while (!quit_comp_thread
) {
297 qemu_mutex_lock(¶m
->mutex
);
298 /* Re-check the quit_comp_thread in case of
299 * terminate_compression_threads is called just before
300 * qemu_mutex_lock(¶m->mutex) and after
301 * while(!quit_comp_thread), re-check it here can make
302 * sure the compression thread terminate as expected.
304 while (!param
->start
&& !quit_comp_thread
) {
305 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
307 if (!quit_comp_thread
) {
308 do_compress_ram_page(param
);
310 param
->start
= false;
311 qemu_mutex_unlock(¶m
->mutex
);
313 qemu_mutex_lock(comp_done_lock
);
315 qemu_cond_signal(comp_done_cond
);
316 qemu_mutex_unlock(comp_done_lock
);
322 static inline void terminate_compression_threads(void)
324 int idx
, thread_count
;
326 thread_count
= migrate_compress_threads();
327 quit_comp_thread
= true;
328 for (idx
= 0; idx
< thread_count
; idx
++) {
329 qemu_mutex_lock(&comp_param
[idx
].mutex
);
330 qemu_cond_signal(&comp_param
[idx
].cond
);
331 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
335 void migrate_compress_threads_join(void)
339 if (!migrate_use_compression()) {
342 terminate_compression_threads();
343 thread_count
= migrate_compress_threads();
344 for (i
= 0; i
< thread_count
; i
++) {
345 qemu_thread_join(compress_threads
+ i
);
346 qemu_fclose(comp_param
[i
].file
);
347 qemu_mutex_destroy(&comp_param
[i
].mutex
);
348 qemu_cond_destroy(&comp_param
[i
].cond
);
350 qemu_mutex_destroy(comp_done_lock
);
351 qemu_cond_destroy(comp_done_cond
);
352 g_free(compress_threads
);
354 g_free(comp_done_cond
);
355 g_free(comp_done_lock
);
356 compress_threads
= NULL
;
358 comp_done_cond
= NULL
;
359 comp_done_lock
= NULL
;
362 void migrate_compress_threads_create(void)
366 if (!migrate_use_compression()) {
369 quit_comp_thread
= false;
370 compression_switch
= true;
371 thread_count
= migrate_compress_threads();
372 compress_threads
= g_new0(QemuThread
, thread_count
);
373 comp_param
= g_new0(CompressParam
, thread_count
);
374 comp_done_cond
= g_new0(QemuCond
, 1);
375 comp_done_lock
= g_new0(QemuMutex
, 1);
376 qemu_cond_init(comp_done_cond
);
377 qemu_mutex_init(comp_done_lock
);
378 for (i
= 0; i
< thread_count
; i
++) {
379 /* com_param[i].file is just used as a dummy buffer to save data, set
382 comp_param
[i
].file
= qemu_fopen_ops(NULL
, &empty_ops
);
383 comp_param
[i
].done
= true;
384 qemu_mutex_init(&comp_param
[i
].mutex
);
385 qemu_cond_init(&comp_param
[i
].cond
);
386 qemu_thread_create(compress_threads
+ i
, "compress",
387 do_data_compress
, comp_param
+ i
,
388 QEMU_THREAD_JOINABLE
);
393 * save_page_header: Write page header to wire
395 * If this is the 1st block, it also writes the block identification
397 * Returns: Number of bytes written
399 * @f: QEMUFile where to send the data
400 * @block: block that contains the page we want to send
401 * @offset: offset inside the block for the page
402 * in the lower bits, it contains flags
404 static size_t save_page_header(QEMUFile
*f
, RAMBlock
*block
, ram_addr_t offset
)
408 qemu_put_be64(f
, offset
);
411 if (!(offset
& RAM_SAVE_FLAG_CONTINUE
)) {
412 len
= strlen(block
->idstr
);
413 qemu_put_byte(f
, len
);
414 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, len
);
420 /* Reduce amount of guest cpu execution to hopefully slow down memory writes.
421 * If guest dirty memory rate is reduced below the rate at which we can
422 * transfer pages to the destination then we should be able to complete
423 * migration. Some workloads dirty memory way too fast and will not effectively
424 * converge, even with auto-converge.
426 static void mig_throttle_guest_down(void)
428 MigrationState
*s
= migrate_get_current();
429 uint64_t pct_initial
=
430 s
->parameters
[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL
];
431 uint64_t pct_icrement
=
432 s
->parameters
[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT
];
434 /* We have not started throttling yet. Let's start it. */
435 if (!cpu_throttle_active()) {
436 cpu_throttle_set(pct_initial
);
438 /* Throttling already on, just increase the rate */
439 cpu_throttle_set(cpu_throttle_get_percentage() + pct_icrement
);
443 /* Update the xbzrle cache to reflect a page that's been sent as all 0.
444 * The important thing is that a stale (not-yet-0'd) page be replaced
446 * As a bonus, if the page wasn't in the cache it gets added so that
447 * when a small write is made into the 0'd page it gets XBZRLE sent
449 static void xbzrle_cache_zero_page(ram_addr_t current_addr
)
451 if (ram_bulk_stage
|| !migrate_use_xbzrle()) {
455 /* We don't care if this fails to allocate a new cache page
456 * as long as it updated an old one */
457 cache_insert(XBZRLE
.cache
, current_addr
, ZERO_TARGET_PAGE
,
461 #define ENCODING_FLAG_XBZRLE 0x1
464 * save_xbzrle_page: compress and send current page
466 * Returns: 1 means that we wrote the page
467 * 0 means that page is identical to the one already sent
468 * -1 means that xbzrle would be longer than normal
470 * @f: QEMUFile where to send the data
473 * @block: block that contains the page we want to send
474 * @offset: offset inside the block for the page
475 * @last_stage: if we are at the completion stage
476 * @bytes_transferred: increase it with the number of transferred bytes
478 static int save_xbzrle_page(QEMUFile
*f
, uint8_t **current_data
,
479 ram_addr_t current_addr
, RAMBlock
*block
,
480 ram_addr_t offset
, bool last_stage
,
481 uint64_t *bytes_transferred
)
483 int encoded_len
= 0, bytes_xbzrle
;
484 uint8_t *prev_cached_page
;
486 if (!cache_is_cached(XBZRLE
.cache
, current_addr
, bitmap_sync_count
)) {
487 acct_info
.xbzrle_cache_miss
++;
489 if (cache_insert(XBZRLE
.cache
, current_addr
, *current_data
,
490 bitmap_sync_count
) == -1) {
493 /* update *current_data when the page has been
494 inserted into cache */
495 *current_data
= get_cached_data(XBZRLE
.cache
, current_addr
);
501 prev_cached_page
= get_cached_data(XBZRLE
.cache
, current_addr
);
503 /* save current buffer into memory */
504 memcpy(XBZRLE
.current_buf
, *current_data
, TARGET_PAGE_SIZE
);
506 /* XBZRLE encoding (if there is no overflow) */
507 encoded_len
= xbzrle_encode_buffer(prev_cached_page
, XBZRLE
.current_buf
,
508 TARGET_PAGE_SIZE
, XBZRLE
.encoded_buf
,
510 if (encoded_len
== 0) {
511 DPRINTF("Skipping unmodified page\n");
513 } else if (encoded_len
== -1) {
514 DPRINTF("Overflow\n");
515 acct_info
.xbzrle_overflows
++;
516 /* update data in the cache */
518 memcpy(prev_cached_page
, *current_data
, TARGET_PAGE_SIZE
);
519 *current_data
= prev_cached_page
;
524 /* we need to update the data in the cache, in order to get the same data */
526 memcpy(prev_cached_page
, XBZRLE
.current_buf
, TARGET_PAGE_SIZE
);
529 /* Send XBZRLE based compressed page */
530 bytes_xbzrle
= save_page_header(f
, block
, offset
| RAM_SAVE_FLAG_XBZRLE
);
531 qemu_put_byte(f
, ENCODING_FLAG_XBZRLE
);
532 qemu_put_be16(f
, encoded_len
);
533 qemu_put_buffer(f
, XBZRLE
.encoded_buf
, encoded_len
);
534 bytes_xbzrle
+= encoded_len
+ 1 + 2;
535 acct_info
.xbzrle_pages
++;
536 acct_info
.xbzrle_bytes
+= bytes_xbzrle
;
537 *bytes_transferred
+= bytes_xbzrle
;
542 /* Called with rcu_read_lock() to protect migration_bitmap
543 * rb: The RAMBlock to search for dirty pages in
544 * start: Start address (typically so we can continue from previous page)
545 * ram_addr_abs: Pointer into which to store the address of the dirty page
546 * within the global ram_addr space
548 * Returns: byte offset within memory region of the start of a dirty page
551 ram_addr_t
migration_bitmap_find_dirty(RAMBlock
*rb
,
553 ram_addr_t
*ram_addr_abs
)
555 unsigned long base
= rb
->offset
>> TARGET_PAGE_BITS
;
556 unsigned long nr
= base
+ (start
>> TARGET_PAGE_BITS
);
557 uint64_t rb_size
= rb
->used_length
;
558 unsigned long size
= base
+ (rb_size
>> TARGET_PAGE_BITS
);
559 unsigned long *bitmap
;
563 bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
564 if (ram_bulk_stage
&& nr
> base
) {
567 next
= find_next_bit(bitmap
, size
, nr
);
570 *ram_addr_abs
= next
<< TARGET_PAGE_BITS
;
571 return (next
- base
) << TARGET_PAGE_BITS
;
574 static inline bool migration_bitmap_clear_dirty(ram_addr_t addr
)
577 int nr
= addr
>> TARGET_PAGE_BITS
;
578 unsigned long *bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
580 ret
= test_and_clear_bit(nr
, bitmap
);
583 migration_dirty_pages
--;
588 static void migration_bitmap_sync_range(ram_addr_t start
, ram_addr_t length
)
590 unsigned long *bitmap
;
591 bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
592 migration_dirty_pages
+=
593 cpu_physical_memory_sync_dirty_bitmap(bitmap
, start
, length
);
596 /* Fix me: there are too many global variables used in migration process. */
597 static int64_t start_time
;
598 static int64_t bytes_xfer_prev
;
599 static int64_t num_dirty_pages_period
;
600 static uint64_t xbzrle_cache_miss_prev
;
601 static uint64_t iterations_prev
;
603 static void migration_bitmap_sync_init(void)
607 num_dirty_pages_period
= 0;
608 xbzrle_cache_miss_prev
= 0;
612 /* Called with iothread lock held, to protect ram_list.dirty_memory[] */
613 static void migration_bitmap_sync(void)
616 uint64_t num_dirty_pages_init
= migration_dirty_pages
;
617 MigrationState
*s
= migrate_get_current();
619 int64_t bytes_xfer_now
;
623 if (!bytes_xfer_prev
) {
624 bytes_xfer_prev
= ram_bytes_transferred();
628 start_time
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
631 trace_migration_bitmap_sync_start();
632 address_space_sync_dirty_bitmap(&address_space_memory
);
634 qemu_mutex_lock(&migration_bitmap_mutex
);
636 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
637 migration_bitmap_sync_range(block
->offset
, block
->used_length
);
640 qemu_mutex_unlock(&migration_bitmap_mutex
);
642 trace_migration_bitmap_sync_end(migration_dirty_pages
643 - num_dirty_pages_init
);
644 num_dirty_pages_period
+= migration_dirty_pages
- num_dirty_pages_init
;
645 end_time
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
647 /* more than 1 second = 1000 millisecons */
648 if (end_time
> start_time
+ 1000) {
649 if (migrate_auto_converge()) {
650 /* The following detection logic can be refined later. For now:
651 Check to see if the dirtied bytes is 50% more than the approx.
652 amount of bytes that just got transferred since the last time we
653 were in this routine. If that happens twice, start or increase
655 bytes_xfer_now
= ram_bytes_transferred();
657 if (s
->dirty_pages_rate
&&
658 (num_dirty_pages_period
* TARGET_PAGE_SIZE
>
659 (bytes_xfer_now
- bytes_xfer_prev
)/2) &&
660 (dirty_rate_high_cnt
++ >= 2)) {
661 trace_migration_throttle();
662 dirty_rate_high_cnt
= 0;
663 mig_throttle_guest_down();
665 bytes_xfer_prev
= bytes_xfer_now
;
668 if (migrate_use_xbzrle()) {
669 if (iterations_prev
!= acct_info
.iterations
) {
670 acct_info
.xbzrle_cache_miss_rate
=
671 (double)(acct_info
.xbzrle_cache_miss
-
672 xbzrle_cache_miss_prev
) /
673 (acct_info
.iterations
- iterations_prev
);
675 iterations_prev
= acct_info
.iterations
;
676 xbzrle_cache_miss_prev
= acct_info
.xbzrle_cache_miss
;
678 s
->dirty_pages_rate
= num_dirty_pages_period
* 1000
679 / (end_time
- start_time
);
680 s
->dirty_bytes_rate
= s
->dirty_pages_rate
* TARGET_PAGE_SIZE
;
681 start_time
= end_time
;
682 num_dirty_pages_period
= 0;
684 s
->dirty_sync_count
= bitmap_sync_count
;
688 * save_zero_page: Send the zero page to the stream
690 * Returns: Number of pages written.
692 * @f: QEMUFile where to send the data
693 * @block: block that contains the page we want to send
694 * @offset: offset inside the block for the page
695 * @p: pointer to the page
696 * @bytes_transferred: increase it with the number of transferred bytes
698 static int save_zero_page(QEMUFile
*f
, RAMBlock
*block
, ram_addr_t offset
,
699 uint8_t *p
, uint64_t *bytes_transferred
)
703 if (is_zero_range(p
, TARGET_PAGE_SIZE
)) {
704 acct_info
.dup_pages
++;
705 *bytes_transferred
+= save_page_header(f
, block
,
706 offset
| RAM_SAVE_FLAG_COMPRESS
);
708 *bytes_transferred
+= 1;
716 * ram_save_page: Send the given page to the stream
718 * Returns: Number of pages written.
720 * @f: QEMUFile where to send the data
721 * @block: block that contains the page we want to send
722 * @offset: offset inside the block for the page
723 * @last_stage: if we are at the completion stage
724 * @bytes_transferred: increase it with the number of transferred bytes
726 static int ram_save_page(QEMUFile
*f
, RAMBlock
* block
, ram_addr_t offset
,
727 bool last_stage
, uint64_t *bytes_transferred
)
731 ram_addr_t current_addr
;
734 bool send_async
= true;
736 p
= block
->host
+ offset
;
738 /* In doubt sent page as normal */
740 ret
= ram_control_save_page(f
, block
->offset
,
741 offset
, TARGET_PAGE_SIZE
, &bytes_xmit
);
743 *bytes_transferred
+= bytes_xmit
;
749 current_addr
= block
->offset
+ offset
;
751 if (block
== last_sent_block
) {
752 offset
|= RAM_SAVE_FLAG_CONTINUE
;
754 if (ret
!= RAM_SAVE_CONTROL_NOT_SUPP
) {
755 if (ret
!= RAM_SAVE_CONTROL_DELAYED
) {
756 if (bytes_xmit
> 0) {
757 acct_info
.norm_pages
++;
758 } else if (bytes_xmit
== 0) {
759 acct_info
.dup_pages
++;
763 pages
= save_zero_page(f
, block
, offset
, p
, bytes_transferred
);
765 /* Must let xbzrle know, otherwise a previous (now 0'd) cached
766 * page would be stale
768 xbzrle_cache_zero_page(current_addr
);
769 } else if (!ram_bulk_stage
&& migrate_use_xbzrle()) {
770 pages
= save_xbzrle_page(f
, &p
, current_addr
, block
,
771 offset
, last_stage
, bytes_transferred
);
773 /* Can't send this cached data async, since the cache page
774 * might get updated before it gets to the wire
781 /* XBZRLE overflow or normal page */
783 *bytes_transferred
+= save_page_header(f
, block
,
784 offset
| RAM_SAVE_FLAG_PAGE
);
786 qemu_put_buffer_async(f
, p
, TARGET_PAGE_SIZE
);
788 qemu_put_buffer(f
, p
, TARGET_PAGE_SIZE
);
790 *bytes_transferred
+= TARGET_PAGE_SIZE
;
792 acct_info
.norm_pages
++;
795 XBZRLE_cache_unlock();
800 static int do_compress_ram_page(CompressParam
*param
)
802 int bytes_sent
, blen
;
804 RAMBlock
*block
= param
->block
;
805 ram_addr_t offset
= param
->offset
;
807 p
= block
->host
+ (offset
& TARGET_PAGE_MASK
);
809 bytes_sent
= save_page_header(param
->file
, block
, offset
|
810 RAM_SAVE_FLAG_COMPRESS_PAGE
);
811 blen
= qemu_put_compression_data(param
->file
, p
, TARGET_PAGE_SIZE
,
812 migrate_compress_level());
818 static inline void start_compression(CompressParam
*param
)
821 qemu_mutex_lock(¶m
->mutex
);
823 qemu_cond_signal(¶m
->cond
);
824 qemu_mutex_unlock(¶m
->mutex
);
827 static inline void start_decompression(DecompressParam
*param
)
829 qemu_mutex_lock(¶m
->mutex
);
831 qemu_cond_signal(¶m
->cond
);
832 qemu_mutex_unlock(¶m
->mutex
);
835 static uint64_t bytes_transferred
;
837 static void flush_compressed_data(QEMUFile
*f
)
839 int idx
, len
, thread_count
;
841 if (!migrate_use_compression()) {
844 thread_count
= migrate_compress_threads();
845 for (idx
= 0; idx
< thread_count
; idx
++) {
846 if (!comp_param
[idx
].done
) {
847 qemu_mutex_lock(comp_done_lock
);
848 while (!comp_param
[idx
].done
&& !quit_comp_thread
) {
849 qemu_cond_wait(comp_done_cond
, comp_done_lock
);
851 qemu_mutex_unlock(comp_done_lock
);
853 if (!quit_comp_thread
) {
854 len
= qemu_put_qemu_file(f
, comp_param
[idx
].file
);
855 bytes_transferred
+= len
;
860 static inline void set_compress_params(CompressParam
*param
, RAMBlock
*block
,
863 param
->block
= block
;
864 param
->offset
= offset
;
867 static int compress_page_with_multi_thread(QEMUFile
*f
, RAMBlock
*block
,
869 uint64_t *bytes_transferred
)
871 int idx
, thread_count
, bytes_xmit
= -1, pages
= -1;
873 thread_count
= migrate_compress_threads();
874 qemu_mutex_lock(comp_done_lock
);
876 for (idx
= 0; idx
< thread_count
; idx
++) {
877 if (comp_param
[idx
].done
) {
878 bytes_xmit
= qemu_put_qemu_file(f
, comp_param
[idx
].file
);
879 set_compress_params(&comp_param
[idx
], block
, offset
);
880 start_compression(&comp_param
[idx
]);
882 acct_info
.norm_pages
++;
883 *bytes_transferred
+= bytes_xmit
;
890 qemu_cond_wait(comp_done_cond
, comp_done_lock
);
893 qemu_mutex_unlock(comp_done_lock
);
899 * ram_save_compressed_page: compress the given page and send it to the stream
901 * Returns: Number of pages written.
903 * @f: QEMUFile where to send the data
904 * @block: block that contains the page we want to send
905 * @offset: offset inside the block for the page
906 * @last_stage: if we are at the completion stage
907 * @bytes_transferred: increase it with the number of transferred bytes
909 static int ram_save_compressed_page(QEMUFile
*f
, RAMBlock
*block
,
910 ram_addr_t offset
, bool last_stage
,
911 uint64_t *bytes_transferred
)
918 p
= block
->host
+ offset
;
921 ret
= ram_control_save_page(f
, block
->offset
,
922 offset
, TARGET_PAGE_SIZE
, &bytes_xmit
);
924 *bytes_transferred
+= bytes_xmit
;
927 if (block
== last_sent_block
) {
928 offset
|= RAM_SAVE_FLAG_CONTINUE
;
930 if (ret
!= RAM_SAVE_CONTROL_NOT_SUPP
) {
931 if (ret
!= RAM_SAVE_CONTROL_DELAYED
) {
932 if (bytes_xmit
> 0) {
933 acct_info
.norm_pages
++;
934 } else if (bytes_xmit
== 0) {
935 acct_info
.dup_pages
++;
939 /* When starting the process of a new block, the first page of
940 * the block should be sent out before other pages in the same
941 * block, and all the pages in last block should have been sent
942 * out, keeping this order is important, because the 'cont' flag
943 * is used to avoid resending the block name.
945 if (block
!= last_sent_block
) {
946 flush_compressed_data(f
);
947 pages
= save_zero_page(f
, block
, offset
, p
, bytes_transferred
);
949 set_compress_params(&comp_param
[0], block
, offset
);
950 /* Use the qemu thread to compress the data to make sure the
951 * first page is sent out before other pages
953 bytes_xmit
= do_compress_ram_page(&comp_param
[0]);
954 acct_info
.norm_pages
++;
955 qemu_put_qemu_file(f
, comp_param
[0].file
);
956 *bytes_transferred
+= bytes_xmit
;
960 pages
= save_zero_page(f
, block
, offset
, p
, bytes_transferred
);
962 pages
= compress_page_with_multi_thread(f
, block
, offset
,
972 * Find the next dirty page and update any state associated with
973 * the search process.
975 * Returns: True if a page is found
977 * @f: Current migration stream.
978 * @pss: Data about the state of the current dirty page scan.
979 * @*again: Set to false if the search has scanned the whole of RAM
980 * *ram_addr_abs: Pointer into which to store the address of the dirty page
981 * within the global ram_addr space
983 static bool find_dirty_block(QEMUFile
*f
, PageSearchStatus
*pss
,
984 bool *again
, ram_addr_t
*ram_addr_abs
)
986 pss
->offset
= migration_bitmap_find_dirty(pss
->block
, pss
->offset
,
988 if (pss
->complete_round
&& pss
->block
== last_seen_block
&&
989 pss
->offset
>= last_offset
) {
991 * We've been once around the RAM and haven't found anything.
997 if (pss
->offset
>= pss
->block
->used_length
) {
998 /* Didn't find anything in this RAM Block */
1000 pss
->block
= QLIST_NEXT_RCU(pss
->block
, next
);
1002 /* Hit the end of the list */
1003 pss
->block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
1004 /* Flag that we've looped */
1005 pss
->complete_round
= true;
1006 ram_bulk_stage
= false;
1007 if (migrate_use_xbzrle()) {
1008 /* If xbzrle is on, stop using the data compression at this
1009 * point. In theory, xbzrle can do better than compression.
1011 flush_compressed_data(f
);
1012 compression_switch
= false;
1015 /* Didn't find anything this time, but try again on the new block */
1019 /* Can go around again, but... */
1021 /* We've found something so probably don't need to */
1027 * Helper for 'get_queued_page' - gets a page off the queue
1028 * ms: MigrationState in
1029 * *offset: Used to return the offset within the RAMBlock
1030 * ram_addr_abs: global offset in the dirty/sent bitmaps
1032 * Returns: block (or NULL if none available)
1034 static RAMBlock
*unqueue_page(MigrationState
*ms
, ram_addr_t
*offset
,
1035 ram_addr_t
*ram_addr_abs
)
1037 RAMBlock
*block
= NULL
;
1039 qemu_mutex_lock(&ms
->src_page_req_mutex
);
1040 if (!QSIMPLEQ_EMPTY(&ms
->src_page_requests
)) {
1041 struct MigrationSrcPageRequest
*entry
=
1042 QSIMPLEQ_FIRST(&ms
->src_page_requests
);
1044 *offset
= entry
->offset
;
1045 *ram_addr_abs
= (entry
->offset
+ entry
->rb
->offset
) &
1048 if (entry
->len
> TARGET_PAGE_SIZE
) {
1049 entry
->len
-= TARGET_PAGE_SIZE
;
1050 entry
->offset
+= TARGET_PAGE_SIZE
;
1052 memory_region_unref(block
->mr
);
1053 QSIMPLEQ_REMOVE_HEAD(&ms
->src_page_requests
, next_req
);
1057 qemu_mutex_unlock(&ms
->src_page_req_mutex
);
1063 * Unqueue a page from the queue fed by postcopy page requests; skips pages
1064 * that are already sent (!dirty)
1066 * ms: MigrationState in
1067 * pss: PageSearchStatus structure updated with found block/offset
1068 * ram_addr_abs: global offset in the dirty/sent bitmaps
1070 * Returns: true if a queued page is found
1072 static bool get_queued_page(MigrationState
*ms
, PageSearchStatus
*pss
,
1073 ram_addr_t
*ram_addr_abs
)
1080 block
= unqueue_page(ms
, &offset
, ram_addr_abs
);
1082 * We're sending this page, and since it's postcopy nothing else
1083 * will dirty it, and we must make sure it doesn't get sent again
1084 * even if this queue request was received after the background
1085 * search already sent it.
1088 unsigned long *bitmap
;
1089 bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
1090 dirty
= test_bit(*ram_addr_abs
>> TARGET_PAGE_BITS
, bitmap
);
1092 trace_get_queued_page_not_dirty(
1093 block
->idstr
, (uint64_t)offset
,
1094 (uint64_t)*ram_addr_abs
,
1095 test_bit(*ram_addr_abs
>> TARGET_PAGE_BITS
,
1096 atomic_rcu_read(&migration_bitmap_rcu
)->unsentmap
));
1098 trace_get_queued_page(block
->idstr
,
1100 (uint64_t)*ram_addr_abs
);
1104 } while (block
&& !dirty
);
1108 * As soon as we start servicing pages out of order, then we have
1109 * to kill the bulk stage, since the bulk stage assumes
1110 * in (migration_bitmap_find_and_reset_dirty) that every page is
1111 * dirty, that's no longer true.
1113 ram_bulk_stage
= false;
1116 * We want the background search to continue from the queued page
1117 * since the guest is likely to want other pages near to the page
1118 * it just requested.
1121 pss
->offset
= offset
;
1128 * flush_page_queue: Flush any remaining pages in the ram request queue
1129 * it should be empty at the end anyway, but in error cases there may be
1132 * ms: MigrationState
1134 void flush_page_queue(MigrationState
*ms
)
1136 struct MigrationSrcPageRequest
*mspr
, *next_mspr
;
1137 /* This queue generally should be empty - but in the case of a failed
1138 * migration might have some droppings in.
1141 QSIMPLEQ_FOREACH_SAFE(mspr
, &ms
->src_page_requests
, next_req
, next_mspr
) {
1142 memory_region_unref(mspr
->rb
->mr
);
1143 QSIMPLEQ_REMOVE_HEAD(&ms
->src_page_requests
, next_req
);
1150 * Queue the pages for transmission, e.g. a request from postcopy destination
1151 * ms: MigrationStatus in which the queue is held
1152 * rbname: The RAMBlock the request is for - may be NULL (to mean reuse last)
1153 * start: Offset from the start of the RAMBlock
1154 * len: Length (in bytes) to send
1155 * Return: 0 on success
1157 int ram_save_queue_pages(MigrationState
*ms
, const char *rbname
,
1158 ram_addr_t start
, ram_addr_t len
)
1164 /* Reuse last RAMBlock */
1165 ramblock
= ms
->last_req_rb
;
1169 * Shouldn't happen, we can't reuse the last RAMBlock if
1170 * it's the 1st request.
1172 error_report("ram_save_queue_pages no previous block");
1176 ramblock
= qemu_ram_block_by_name(rbname
);
1179 /* We shouldn't be asked for a non-existent RAMBlock */
1180 error_report("ram_save_queue_pages no block '%s'", rbname
);
1183 ms
->last_req_rb
= ramblock
;
1185 trace_ram_save_queue_pages(ramblock
->idstr
, start
, len
);
1186 if (start
+len
> ramblock
->used_length
) {
1187 error_report("%s request overrun start=%zx len=%zx blocklen=%zx",
1188 __func__
, start
, len
, ramblock
->used_length
);
1192 struct MigrationSrcPageRequest
*new_entry
=
1193 g_malloc0(sizeof(struct MigrationSrcPageRequest
));
1194 new_entry
->rb
= ramblock
;
1195 new_entry
->offset
= start
;
1196 new_entry
->len
= len
;
1198 memory_region_ref(ramblock
->mr
);
1199 qemu_mutex_lock(&ms
->src_page_req_mutex
);
1200 QSIMPLEQ_INSERT_TAIL(&ms
->src_page_requests
, new_entry
, next_req
);
1201 qemu_mutex_unlock(&ms
->src_page_req_mutex
);
1212 * ram_save_target_page: Save one target page
1215 * @f: QEMUFile where to send the data
1216 * @block: pointer to block that contains the page we want to send
1217 * @offset: offset inside the block for the page;
1218 * @last_stage: if we are at the completion stage
1219 * @bytes_transferred: increase it with the number of transferred bytes
1220 * @dirty_ram_abs: Address of the start of the dirty page in ram_addr_t space
1222 * Returns: Number of pages written.
1224 static int ram_save_target_page(MigrationState
*ms
, QEMUFile
*f
,
1225 RAMBlock
*block
, ram_addr_t offset
,
1227 uint64_t *bytes_transferred
,
1228 ram_addr_t dirty_ram_abs
)
1232 /* Check the pages is dirty and if it is send it */
1233 if (migration_bitmap_clear_dirty(dirty_ram_abs
)) {
1234 unsigned long *unsentmap
;
1235 if (compression_switch
&& migrate_use_compression()) {
1236 res
= ram_save_compressed_page(f
, block
, offset
,
1240 res
= ram_save_page(f
, block
, offset
, last_stage
,
1247 unsentmap
= atomic_rcu_read(&migration_bitmap_rcu
)->unsentmap
;
1249 clear_bit(dirty_ram_abs
>> TARGET_PAGE_BITS
, unsentmap
);
1257 * ram_save_host_page: Starting at *offset send pages upto the end
1258 * of the current host page. It's valid for the initial
1259 * offset to point into the middle of a host page
1260 * in which case the remainder of the hostpage is sent.
1261 * Only dirty target pages are sent.
1263 * Returns: Number of pages written.
1265 * @f: QEMUFile where to send the data
1266 * @block: pointer to block that contains the page we want to send
1267 * @offset: offset inside the block for the page; updated to last target page
1269 * @last_stage: if we are at the completion stage
1270 * @bytes_transferred: increase it with the number of transferred bytes
1271 * @dirty_ram_abs: Address of the start of the dirty page in ram_addr_t space
1273 static int ram_save_host_page(MigrationState
*ms
, QEMUFile
*f
, RAMBlock
*block
,
1274 ram_addr_t
*offset
, bool last_stage
,
1275 uint64_t *bytes_transferred
,
1276 ram_addr_t dirty_ram_abs
)
1278 int tmppages
, pages
= 0;
1280 tmppages
= ram_save_target_page(ms
, f
, block
, *offset
, last_stage
,
1281 bytes_transferred
, dirty_ram_abs
);
1287 *offset
+= TARGET_PAGE_SIZE
;
1288 dirty_ram_abs
+= TARGET_PAGE_SIZE
;
1289 } while (*offset
& (qemu_host_page_size
- 1));
1291 /* The offset we leave with is the last one we looked at */
1292 *offset
-= TARGET_PAGE_SIZE
;
1297 * ram_find_and_save_block: Finds a dirty page and sends it to f
1299 * Called within an RCU critical section.
1301 * Returns: The number of pages written
1302 * 0 means no dirty pages
1304 * @f: QEMUFile where to send the data
1305 * @last_stage: if we are at the completion stage
1306 * @bytes_transferred: increase it with the number of transferred bytes
1308 * On systems where host-page-size > target-page-size it will send all the
1309 * pages in a host page that are dirty.
1312 static int ram_find_and_save_block(QEMUFile
*f
, bool last_stage
,
1313 uint64_t *bytes_transferred
)
1315 PageSearchStatus pss
;
1316 MigrationState
*ms
= migrate_get_current();
1319 ram_addr_t dirty_ram_abs
; /* Address of the start of the dirty page in
1322 pss
.block
= last_seen_block
;
1323 pss
.offset
= last_offset
;
1324 pss
.complete_round
= false;
1327 pss
.block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
1332 found
= get_queued_page(ms
, &pss
, &dirty_ram_abs
);
1335 /* priority queue empty, so just search for something dirty */
1336 found
= find_dirty_block(f
, &pss
, &again
, &dirty_ram_abs
);
1340 pages
= ram_save_host_page(ms
, f
, pss
.block
, &pss
.offset
,
1341 last_stage
, bytes_transferred
,
1344 } while (!pages
&& again
);
1346 last_seen_block
= pss
.block
;
1347 last_offset
= pss
.offset
;
1352 void acct_update_position(QEMUFile
*f
, size_t size
, bool zero
)
1354 uint64_t pages
= size
/ TARGET_PAGE_SIZE
;
1356 acct_info
.dup_pages
+= pages
;
1358 acct_info
.norm_pages
+= pages
;
1359 bytes_transferred
+= size
;
1360 qemu_update_position(f
, size
);
1364 static ram_addr_t
ram_save_remaining(void)
1366 return migration_dirty_pages
;
1369 uint64_t ram_bytes_remaining(void)
1371 return ram_save_remaining() * TARGET_PAGE_SIZE
;
1374 uint64_t ram_bytes_transferred(void)
1376 return bytes_transferred
;
1379 uint64_t ram_bytes_total(void)
1385 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
)
1386 total
+= block
->used_length
;
1391 void free_xbzrle_decoded_buf(void)
1393 g_free(xbzrle_decoded_buf
);
1394 xbzrle_decoded_buf
= NULL
;
1397 static void migration_bitmap_free(struct BitmapRcu
*bmap
)
1400 g_free(bmap
->unsentmap
);
1404 static void ram_migration_cleanup(void *opaque
)
1406 /* caller have hold iothread lock or is in a bh, so there is
1407 * no writing race against this migration_bitmap
1409 struct BitmapRcu
*bitmap
= migration_bitmap_rcu
;
1410 atomic_rcu_set(&migration_bitmap_rcu
, NULL
);
1412 memory_global_dirty_log_stop();
1413 call_rcu(bitmap
, migration_bitmap_free
, rcu
);
1416 XBZRLE_cache_lock();
1418 cache_fini(XBZRLE
.cache
);
1419 g_free(XBZRLE
.encoded_buf
);
1420 g_free(XBZRLE
.current_buf
);
1421 XBZRLE
.cache
= NULL
;
1422 XBZRLE
.encoded_buf
= NULL
;
1423 XBZRLE
.current_buf
= NULL
;
1425 XBZRLE_cache_unlock();
1428 static void reset_ram_globals(void)
1430 last_seen_block
= NULL
;
1431 last_sent_block
= NULL
;
1433 last_version
= ram_list
.version
;
1434 ram_bulk_stage
= true;
1437 #define MAX_WAIT 50 /* ms, half buffered_file limit */
1439 void migration_bitmap_extend(ram_addr_t old
, ram_addr_t
new)
1441 /* called in qemu main thread, so there is
1442 * no writing race against this migration_bitmap
1444 if (migration_bitmap_rcu
) {
1445 struct BitmapRcu
*old_bitmap
= migration_bitmap_rcu
, *bitmap
;
1446 bitmap
= g_new(struct BitmapRcu
, 1);
1447 bitmap
->bmap
= bitmap_new(new);
1449 /* prevent migration_bitmap content from being set bit
1450 * by migration_bitmap_sync_range() at the same time.
1451 * it is safe to migration if migration_bitmap is cleared bit
1454 qemu_mutex_lock(&migration_bitmap_mutex
);
1455 bitmap_copy(bitmap
->bmap
, old_bitmap
->bmap
, old
);
1456 bitmap_set(bitmap
->bmap
, old
, new - old
);
1458 /* We don't have a way to safely extend the sentmap
1459 * with RCU; so mark it as missing, entry to postcopy
1462 bitmap
->unsentmap
= NULL
;
1464 atomic_rcu_set(&migration_bitmap_rcu
, bitmap
);
1465 qemu_mutex_unlock(&migration_bitmap_mutex
);
1466 migration_dirty_pages
+= new - old
;
1467 call_rcu(old_bitmap
, migration_bitmap_free
, rcu
);
1472 * 'expected' is the value you expect the bitmap mostly to be full
1473 * of; it won't bother printing lines that are all this value.
1474 * If 'todump' is null the migration bitmap is dumped.
1476 void ram_debug_dump_bitmap(unsigned long *todump
, bool expected
)
1478 int64_t ram_pages
= last_ram_offset() >> TARGET_PAGE_BITS
;
1481 int64_t linelen
= 128;
1485 todump
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
1488 for (cur
= 0; cur
< ram_pages
; cur
+= linelen
) {
1492 * Last line; catch the case where the line length
1493 * is longer than remaining ram
1495 if (cur
+ linelen
> ram_pages
) {
1496 linelen
= ram_pages
- cur
;
1498 for (curb
= 0; curb
< linelen
; curb
++) {
1499 bool thisbit
= test_bit(cur
+ curb
, todump
);
1500 linebuf
[curb
] = thisbit
? '1' : '.';
1501 found
= found
|| (thisbit
!= expected
);
1504 linebuf
[curb
] = '\0';
1505 fprintf(stderr
, "0x%08" PRIx64
" : %s\n", cur
, linebuf
);
1510 /* **** functions for postcopy ***** */
1513 * Callback from postcopy_each_ram_send_discard for each RAMBlock
1514 * Note: At this point the 'unsentmap' is the processed bitmap combined
1515 * with the dirtymap; so a '1' means it's either dirty or unsent.
1516 * start,length: Indexes into the bitmap for the first bit
1517 * representing the named block and length in target-pages
1519 static int postcopy_send_discard_bm_ram(MigrationState
*ms
,
1520 PostcopyDiscardState
*pds
,
1521 unsigned long start
,
1522 unsigned long length
)
1524 unsigned long end
= start
+ length
; /* one after the end */
1525 unsigned long current
;
1526 unsigned long *unsentmap
;
1528 unsentmap
= atomic_rcu_read(&migration_bitmap_rcu
)->unsentmap
;
1529 for (current
= start
; current
< end
; ) {
1530 unsigned long one
= find_next_bit(unsentmap
, end
, current
);
1533 unsigned long zero
= find_next_zero_bit(unsentmap
, end
, one
+ 1);
1534 unsigned long discard_length
;
1537 discard_length
= end
- one
;
1539 discard_length
= zero
- one
;
1541 postcopy_discard_send_range(ms
, pds
, one
, discard_length
);
1542 current
= one
+ discard_length
;
1552 * Utility for the outgoing postcopy code.
1553 * Calls postcopy_send_discard_bm_ram for each RAMBlock
1554 * passing it bitmap indexes and name.
1555 * Returns: 0 on success
1556 * (qemu_ram_foreach_block ends up passing unscaled lengths
1557 * which would mean postcopy code would have to deal with target page)
1559 static int postcopy_each_ram_send_discard(MigrationState
*ms
)
1561 struct RAMBlock
*block
;
1564 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1565 unsigned long first
= block
->offset
>> TARGET_PAGE_BITS
;
1566 PostcopyDiscardState
*pds
= postcopy_discard_send_init(ms
,
1571 * Postcopy sends chunks of bitmap over the wire, but it
1572 * just needs indexes at this point, avoids it having
1573 * target page specific code.
1575 ret
= postcopy_send_discard_bm_ram(ms
, pds
, first
,
1576 block
->used_length
>> TARGET_PAGE_BITS
);
1577 postcopy_discard_send_finish(ms
, pds
);
1587 * Helper for postcopy_chunk_hostpages; it's called twice to cleanup
1588 * the two bitmaps, that are similar, but one is inverted.
1590 * We search for runs of target-pages that don't start or end on a
1591 * host page boundary;
1592 * unsent_pass=true: Cleans up partially unsent host pages by searching
1594 * unsent_pass=false: Cleans up partially dirty host pages by searching
1595 * the main migration bitmap
1598 static void postcopy_chunk_hostpages_pass(MigrationState
*ms
, bool unsent_pass
,
1600 PostcopyDiscardState
*pds
)
1602 unsigned long *bitmap
;
1603 unsigned long *unsentmap
;
1604 unsigned int host_ratio
= qemu_host_page_size
/ TARGET_PAGE_SIZE
;
1605 unsigned long first
= block
->offset
>> TARGET_PAGE_BITS
;
1606 unsigned long len
= block
->used_length
>> TARGET_PAGE_BITS
;
1607 unsigned long last
= first
+ (len
- 1);
1608 unsigned long run_start
;
1610 bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
1611 unsentmap
= atomic_rcu_read(&migration_bitmap_rcu
)->unsentmap
;
1614 /* Find a sent page */
1615 run_start
= find_next_zero_bit(unsentmap
, last
+ 1, first
);
1617 /* Find a dirty page */
1618 run_start
= find_next_bit(bitmap
, last
+ 1, first
);
1621 while (run_start
<= last
) {
1622 bool do_fixup
= false;
1623 unsigned long fixup_start_addr
;
1624 unsigned long host_offset
;
1627 * If the start of this run of pages is in the middle of a host
1628 * page, then we need to fixup this host page.
1630 host_offset
= run_start
% host_ratio
;
1633 run_start
-= host_offset
;
1634 fixup_start_addr
= run_start
;
1635 /* For the next pass */
1636 run_start
= run_start
+ host_ratio
;
1638 /* Find the end of this run */
1639 unsigned long run_end
;
1641 run_end
= find_next_bit(unsentmap
, last
+ 1, run_start
+ 1);
1643 run_end
= find_next_zero_bit(bitmap
, last
+ 1, run_start
+ 1);
1646 * If the end isn't at the start of a host page, then the
1647 * run doesn't finish at the end of a host page
1648 * and we need to discard.
1650 host_offset
= run_end
% host_ratio
;
1653 fixup_start_addr
= run_end
- host_offset
;
1655 * This host page has gone, the next loop iteration starts
1656 * from after the fixup
1658 run_start
= fixup_start_addr
+ host_ratio
;
1661 * No discards on this iteration, next loop starts from
1662 * next sent/dirty page
1664 run_start
= run_end
+ 1;
1671 /* Tell the destination to discard this page */
1672 if (unsent_pass
|| !test_bit(fixup_start_addr
, unsentmap
)) {
1673 /* For the unsent_pass we:
1674 * discard partially sent pages
1675 * For the !unsent_pass (dirty) we:
1676 * discard partially dirty pages that were sent
1677 * (any partially sent pages were already discarded
1678 * by the previous unsent_pass)
1680 postcopy_discard_send_range(ms
, pds
, fixup_start_addr
,
1684 /* Clean up the bitmap */
1685 for (page
= fixup_start_addr
;
1686 page
< fixup_start_addr
+ host_ratio
; page
++) {
1687 /* All pages in this host page are now not sent */
1688 set_bit(page
, unsentmap
);
1691 * Remark them as dirty, updating the count for any pages
1692 * that weren't previously dirty.
1694 migration_dirty_pages
+= !test_and_set_bit(page
, bitmap
);
1699 /* Find the next sent page for the next iteration */
1700 run_start
= find_next_zero_bit(unsentmap
, last
+ 1,
1703 /* Find the next dirty page for the next iteration */
1704 run_start
= find_next_bit(bitmap
, last
+ 1, run_start
);
1710 * Utility for the outgoing postcopy code.
1712 * Discard any partially sent host-page size chunks, mark any partially
1713 * dirty host-page size chunks as all dirty.
1715 * Returns: 0 on success
1717 static int postcopy_chunk_hostpages(MigrationState
*ms
)
1719 struct RAMBlock
*block
;
1721 if (qemu_host_page_size
== TARGET_PAGE_SIZE
) {
1722 /* Easy case - TPS==HPS - nothing to be done */
1726 /* Easiest way to make sure we don't resume in the middle of a host-page */
1727 last_seen_block
= NULL
;
1728 last_sent_block
= NULL
;
1731 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1732 unsigned long first
= block
->offset
>> TARGET_PAGE_BITS
;
1734 PostcopyDiscardState
*pds
=
1735 postcopy_discard_send_init(ms
, first
, block
->idstr
);
1737 /* First pass: Discard all partially sent host pages */
1738 postcopy_chunk_hostpages_pass(ms
, true, block
, pds
);
1740 * Second pass: Ensure that all partially dirty host pages are made
1743 postcopy_chunk_hostpages_pass(ms
, false, block
, pds
);
1745 postcopy_discard_send_finish(ms
, pds
);
1746 } /* ram_list loop */
1752 * Transmit the set of pages to be discarded after precopy to the target
1753 * these are pages that:
1754 * a) Have been previously transmitted but are now dirty again
1755 * b) Pages that have never been transmitted, this ensures that
1756 * any pages on the destination that have been mapped by background
1757 * tasks get discarded (transparent huge pages is the specific concern)
1758 * Hopefully this is pretty sparse
1760 int ram_postcopy_send_discard_bitmap(MigrationState
*ms
)
1763 unsigned long *bitmap
, *unsentmap
;
1767 /* This should be our last sync, the src is now paused */
1768 migration_bitmap_sync();
1770 unsentmap
= atomic_rcu_read(&migration_bitmap_rcu
)->unsentmap
;
1772 /* We don't have a safe way to resize the sentmap, so
1773 * if the bitmap was resized it will be NULL at this
1776 error_report("migration ram resized during precopy phase");
1781 /* Deal with TPS != HPS */
1782 ret
= postcopy_chunk_hostpages(ms
);
1789 * Update the unsentmap to be unsentmap = unsentmap | dirty
1791 bitmap
= atomic_rcu_read(&migration_bitmap_rcu
)->bmap
;
1792 bitmap_or(unsentmap
, unsentmap
, bitmap
,
1793 last_ram_offset() >> TARGET_PAGE_BITS
);
1796 trace_ram_postcopy_send_discard_bitmap();
1797 #ifdef DEBUG_POSTCOPY
1798 ram_debug_dump_bitmap(unsentmap
, true);
1801 ret
= postcopy_each_ram_send_discard(ms
);
1808 * At the start of the postcopy phase of migration, any now-dirty
1809 * precopied pages are discarded.
1811 * start, length describe a byte address range within the RAMBlock
1813 * Returns 0 on success.
1815 int ram_discard_range(MigrationIncomingState
*mis
,
1816 const char *block_name
,
1817 uint64_t start
, size_t length
)
1822 RAMBlock
*rb
= qemu_ram_block_by_name(block_name
);
1825 error_report("ram_discard_range: Failed to find block '%s'",
1830 uint8_t *host_startaddr
= rb
->host
+ start
;
1832 if ((uintptr_t)host_startaddr
& (qemu_host_page_size
- 1)) {
1833 error_report("ram_discard_range: Unaligned start address: %p",
1838 if ((start
+ length
) <= rb
->used_length
) {
1839 uint8_t *host_endaddr
= host_startaddr
+ length
;
1840 if ((uintptr_t)host_endaddr
& (qemu_host_page_size
- 1)) {
1841 error_report("ram_discard_range: Unaligned end address: %p",
1845 ret
= postcopy_ram_discard_range(mis
, host_startaddr
, length
);
1847 error_report("ram_discard_range: Overrun block '%s' (%" PRIu64
1849 block_name
, start
, length
, rb
->used_length
);
1859 /* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
1860 * long-running RCU critical section. When rcu-reclaims in the code
1861 * start to become numerous it will be necessary to reduce the
1862 * granularity of these critical sections.
1865 static int ram_save_setup(QEMUFile
*f
, void *opaque
)
1868 int64_t ram_bitmap_pages
; /* Size of bitmap in pages, including gaps */
1870 dirty_rate_high_cnt
= 0;
1871 bitmap_sync_count
= 0;
1872 migration_bitmap_sync_init();
1873 qemu_mutex_init(&migration_bitmap_mutex
);
1875 if (migrate_use_xbzrle()) {
1876 XBZRLE_cache_lock();
1877 XBZRLE
.cache
= cache_init(migrate_xbzrle_cache_size() /
1880 if (!XBZRLE
.cache
) {
1881 XBZRLE_cache_unlock();
1882 error_report("Error creating cache");
1885 XBZRLE_cache_unlock();
1887 /* We prefer not to abort if there is no memory */
1888 XBZRLE
.encoded_buf
= g_try_malloc0(TARGET_PAGE_SIZE
);
1889 if (!XBZRLE
.encoded_buf
) {
1890 error_report("Error allocating encoded_buf");
1894 XBZRLE
.current_buf
= g_try_malloc(TARGET_PAGE_SIZE
);
1895 if (!XBZRLE
.current_buf
) {
1896 error_report("Error allocating current_buf");
1897 g_free(XBZRLE
.encoded_buf
);
1898 XBZRLE
.encoded_buf
= NULL
;
1905 /* iothread lock needed for ram_list.dirty_memory[] */
1906 qemu_mutex_lock_iothread();
1907 qemu_mutex_lock_ramlist();
1909 bytes_transferred
= 0;
1910 reset_ram_globals();
1912 ram_bitmap_pages
= last_ram_offset() >> TARGET_PAGE_BITS
;
1913 migration_bitmap_rcu
= g_new0(struct BitmapRcu
, 1);
1914 migration_bitmap_rcu
->bmap
= bitmap_new(ram_bitmap_pages
);
1915 bitmap_set(migration_bitmap_rcu
->bmap
, 0, ram_bitmap_pages
);
1917 if (migrate_postcopy_ram()) {
1918 migration_bitmap_rcu
->unsentmap
= bitmap_new(ram_bitmap_pages
);
1919 bitmap_set(migration_bitmap_rcu
->unsentmap
, 0, ram_bitmap_pages
);
1923 * Count the total number of pages used by ram blocks not including any
1924 * gaps due to alignment or unplugs.
1926 migration_dirty_pages
= ram_bytes_total() >> TARGET_PAGE_BITS
;
1928 memory_global_dirty_log_start();
1929 migration_bitmap_sync();
1930 qemu_mutex_unlock_ramlist();
1931 qemu_mutex_unlock_iothread();
1933 qemu_put_be64(f
, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE
);
1935 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1936 qemu_put_byte(f
, strlen(block
->idstr
));
1937 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, strlen(block
->idstr
));
1938 qemu_put_be64(f
, block
->used_length
);
1943 ram_control_before_iterate(f
, RAM_CONTROL_SETUP
);
1944 ram_control_after_iterate(f
, RAM_CONTROL_SETUP
);
1946 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
1951 static int ram_save_iterate(QEMUFile
*f
, void *opaque
)
1959 if (ram_list
.version
!= last_version
) {
1960 reset_ram_globals();
1963 /* Read version before ram_list.blocks */
1966 ram_control_before_iterate(f
, RAM_CONTROL_ROUND
);
1968 t0
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
);
1970 while ((ret
= qemu_file_rate_limit(f
)) == 0) {
1973 pages
= ram_find_and_save_block(f
, false, &bytes_transferred
);
1974 /* no more pages to sent */
1978 pages_sent
+= pages
;
1979 acct_info
.iterations
++;
1981 /* we want to check in the 1st loop, just in case it was the 1st time
1982 and we had to sync the dirty bitmap.
1983 qemu_get_clock_ns() is a bit expensive, so we only check each some
1986 if ((i
& 63) == 0) {
1987 uint64_t t1
= (qemu_clock_get_ns(QEMU_CLOCK_REALTIME
) - t0
) / 1000000;
1988 if (t1
> MAX_WAIT
) {
1989 DPRINTF("big wait: %" PRIu64
" milliseconds, %d iterations\n",
1996 flush_compressed_data(f
);
2000 * Must occur before EOS (or any QEMUFile operation)
2001 * because of RDMA protocol.
2003 ram_control_after_iterate(f
, RAM_CONTROL_ROUND
);
2005 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
2006 bytes_transferred
+= 8;
2008 ret
= qemu_file_get_error(f
);
2016 /* Called with iothread lock */
2017 static int ram_save_complete(QEMUFile
*f
, void *opaque
)
2021 if (!migration_in_postcopy(migrate_get_current())) {
2022 migration_bitmap_sync();
2025 ram_control_before_iterate(f
, RAM_CONTROL_FINISH
);
2027 /* try transferring iterative blocks of memory */
2029 /* flush all remaining blocks regardless of rate limiting */
2033 pages
= ram_find_and_save_block(f
, true, &bytes_transferred
);
2034 /* no more blocks to sent */
2040 flush_compressed_data(f
);
2041 ram_control_after_iterate(f
, RAM_CONTROL_FINISH
);
2045 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
2050 static void ram_save_pending(QEMUFile
*f
, void *opaque
, uint64_t max_size
,
2051 uint64_t *non_postcopiable_pending
,
2052 uint64_t *postcopiable_pending
)
2054 uint64_t remaining_size
;
2056 remaining_size
= ram_save_remaining() * TARGET_PAGE_SIZE
;
2058 if (!migration_in_postcopy(migrate_get_current()) &&
2059 remaining_size
< max_size
) {
2060 qemu_mutex_lock_iothread();
2062 migration_bitmap_sync();
2064 qemu_mutex_unlock_iothread();
2065 remaining_size
= ram_save_remaining() * TARGET_PAGE_SIZE
;
2068 /* We can do postcopy, and all the data is postcopiable */
2069 *postcopiable_pending
+= remaining_size
;
2072 static int load_xbzrle(QEMUFile
*f
, ram_addr_t addr
, void *host
)
2074 unsigned int xh_len
;
2077 if (!xbzrle_decoded_buf
) {
2078 xbzrle_decoded_buf
= g_malloc(TARGET_PAGE_SIZE
);
2081 /* extract RLE header */
2082 xh_flags
= qemu_get_byte(f
);
2083 xh_len
= qemu_get_be16(f
);
2085 if (xh_flags
!= ENCODING_FLAG_XBZRLE
) {
2086 error_report("Failed to load XBZRLE page - wrong compression!");
2090 if (xh_len
> TARGET_PAGE_SIZE
) {
2091 error_report("Failed to load XBZRLE page - len overflow!");
2094 /* load data and decode */
2095 qemu_get_buffer(f
, xbzrle_decoded_buf
, xh_len
);
2098 if (xbzrle_decode_buffer(xbzrle_decoded_buf
, xh_len
, host
,
2099 TARGET_PAGE_SIZE
) == -1) {
2100 error_report("Failed to load XBZRLE page - decode error!");
2107 /* Must be called from within a rcu critical section.
2108 * Returns a pointer from within the RCU-protected ram_list.
2111 * Read a RAMBlock ID from the stream f, find the host address of the
2112 * start of that block and add on 'offset'
2114 * f: Stream to read from
2115 * offset: Offset within the block
2116 * flags: Page flags (mostly to see if it's a continuation of previous block)
2118 static inline void *host_from_stream_offset(QEMUFile
*f
,
2122 static RAMBlock
*block
= NULL
;
2126 if (flags
& RAM_SAVE_FLAG_CONTINUE
) {
2127 if (!block
|| block
->max_length
<= offset
) {
2128 error_report("Ack, bad migration stream!");
2132 return block
->host
+ offset
;
2135 len
= qemu_get_byte(f
);
2136 qemu_get_buffer(f
, (uint8_t *)id
, len
);
2139 block
= qemu_ram_block_by_name(id
);
2140 if (block
&& block
->max_length
> offset
) {
2141 return block
->host
+ offset
;
2144 error_report("Can't find block %s", id
);
2149 * If a page (or a whole RDMA chunk) has been
2150 * determined to be zero, then zap it.
2152 void ram_handle_compressed(void *host
, uint8_t ch
, uint64_t size
)
2154 if (ch
!= 0 || !is_zero_range(host
, size
)) {
2155 memset(host
, ch
, size
);
2159 static void *do_data_decompress(void *opaque
)
2161 DecompressParam
*param
= opaque
;
2162 unsigned long pagesize
;
2164 while (!quit_decomp_thread
) {
2165 qemu_mutex_lock(¶m
->mutex
);
2166 while (!param
->start
&& !quit_decomp_thread
) {
2167 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
2168 pagesize
= TARGET_PAGE_SIZE
;
2169 if (!quit_decomp_thread
) {
2170 /* uncompress() will return failed in some case, especially
2171 * when the page is dirted when doing the compression, it's
2172 * not a problem because the dirty page will be retransferred
2173 * and uncompress() won't break the data in other pages.
2175 uncompress((Bytef
*)param
->des
, &pagesize
,
2176 (const Bytef
*)param
->compbuf
, param
->len
);
2178 param
->start
= false;
2180 qemu_mutex_unlock(¶m
->mutex
);
2186 void migrate_decompress_threads_create(void)
2188 int i
, thread_count
;
2190 thread_count
= migrate_decompress_threads();
2191 decompress_threads
= g_new0(QemuThread
, thread_count
);
2192 decomp_param
= g_new0(DecompressParam
, thread_count
);
2193 compressed_data_buf
= g_malloc0(compressBound(TARGET_PAGE_SIZE
));
2194 quit_decomp_thread
= false;
2195 for (i
= 0; i
< thread_count
; i
++) {
2196 qemu_mutex_init(&decomp_param
[i
].mutex
);
2197 qemu_cond_init(&decomp_param
[i
].cond
);
2198 decomp_param
[i
].compbuf
= g_malloc0(compressBound(TARGET_PAGE_SIZE
));
2199 qemu_thread_create(decompress_threads
+ i
, "decompress",
2200 do_data_decompress
, decomp_param
+ i
,
2201 QEMU_THREAD_JOINABLE
);
2205 void migrate_decompress_threads_join(void)
2207 int i
, thread_count
;
2209 quit_decomp_thread
= true;
2210 thread_count
= migrate_decompress_threads();
2211 for (i
= 0; i
< thread_count
; i
++) {
2212 qemu_mutex_lock(&decomp_param
[i
].mutex
);
2213 qemu_cond_signal(&decomp_param
[i
].cond
);
2214 qemu_mutex_unlock(&decomp_param
[i
].mutex
);
2216 for (i
= 0; i
< thread_count
; i
++) {
2217 qemu_thread_join(decompress_threads
+ i
);
2218 qemu_mutex_destroy(&decomp_param
[i
].mutex
);
2219 qemu_cond_destroy(&decomp_param
[i
].cond
);
2220 g_free(decomp_param
[i
].compbuf
);
2222 g_free(decompress_threads
);
2223 g_free(decomp_param
);
2224 g_free(compressed_data_buf
);
2225 decompress_threads
= NULL
;
2226 decomp_param
= NULL
;
2227 compressed_data_buf
= NULL
;
2230 static void decompress_data_with_multi_threads(uint8_t *compbuf
,
2231 void *host
, int len
)
2233 int idx
, thread_count
;
2235 thread_count
= migrate_decompress_threads();
2237 for (idx
= 0; idx
< thread_count
; idx
++) {
2238 if (!decomp_param
[idx
].start
) {
2239 memcpy(decomp_param
[idx
].compbuf
, compbuf
, len
);
2240 decomp_param
[idx
].des
= host
;
2241 decomp_param
[idx
].len
= len
;
2242 start_decompression(&decomp_param
[idx
]);
2246 if (idx
< thread_count
) {
2253 * Allocate data structures etc needed by incoming migration with postcopy-ram
2254 * postcopy-ram's similarly names postcopy_ram_incoming_init does the work
2256 int ram_postcopy_incoming_init(MigrationIncomingState
*mis
)
2258 size_t ram_pages
= last_ram_offset() >> TARGET_PAGE_BITS
;
2260 return postcopy_ram_incoming_init(mis
, ram_pages
);
2264 * Called in postcopy mode by ram_load().
2265 * rcu_read_lock is taken prior to this being called.
2267 static int ram_load_postcopy(QEMUFile
*f
)
2269 int flags
= 0, ret
= 0;
2270 bool place_needed
= false;
2271 bool matching_page_sizes
= qemu_host_page_size
== TARGET_PAGE_SIZE
;
2272 MigrationIncomingState
*mis
= migration_incoming_get_current();
2273 /* Temporary page that is later 'placed' */
2274 void *postcopy_host_page
= postcopy_get_tmp_page(mis
);
2275 void *last_host
= NULL
;
2277 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
2280 void *page_buffer
= NULL
;
2281 void *place_source
= NULL
;
2283 bool all_zero
= false;
2285 addr
= qemu_get_be64(f
);
2286 flags
= addr
& ~TARGET_PAGE_MASK
;
2287 addr
&= TARGET_PAGE_MASK
;
2289 trace_ram_load_postcopy_loop((uint64_t)addr
, flags
);
2290 place_needed
= false;
2291 if (flags
& (RAM_SAVE_FLAG_COMPRESS
| RAM_SAVE_FLAG_PAGE
)) {
2292 host
= host_from_stream_offset(f
, addr
, flags
);
2294 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
2300 * Postcopy requires that we place whole host pages atomically.
2301 * To make it atomic, the data is read into a temporary page
2302 * that's moved into place later.
2303 * The migration protocol uses, possibly smaller, target-pages
2304 * however the source ensures it always sends all the components
2305 * of a host page in order.
2307 page_buffer
= postcopy_host_page
+
2308 ((uintptr_t)host
& ~qemu_host_page_mask
);
2309 /* If all TP are zero then we can optimise the place */
2310 if (!((uintptr_t)host
& ~qemu_host_page_mask
)) {
2313 /* not the 1st TP within the HP */
2314 if (host
!= (last_host
+ TARGET_PAGE_SIZE
)) {
2315 error_report("Non-sequential target page %p/%p\n",
2324 * If it's the last part of a host page then we place the host
2327 place_needed
= (((uintptr_t)host
+ TARGET_PAGE_SIZE
) &
2328 ~qemu_host_page_mask
) == 0;
2329 place_source
= postcopy_host_page
;
2333 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
2334 case RAM_SAVE_FLAG_COMPRESS
:
2335 ch
= qemu_get_byte(f
);
2336 memset(page_buffer
, ch
, TARGET_PAGE_SIZE
);
2342 case RAM_SAVE_FLAG_PAGE
:
2344 if (!place_needed
|| !matching_page_sizes
) {
2345 qemu_get_buffer(f
, page_buffer
, TARGET_PAGE_SIZE
);
2347 /* Avoids the qemu_file copy during postcopy, which is
2348 * going to do a copy later; can only do it when we
2349 * do this read in one go (matching page sizes)
2351 qemu_get_buffer_in_place(f
, (uint8_t **)&place_source
,
2355 case RAM_SAVE_FLAG_EOS
:
2359 error_report("Unknown combination of migration flags: %#x"
2360 " (postcopy mode)", flags
);
2365 /* This gets called at the last target page in the host page */
2367 ret
= postcopy_place_page_zero(mis
,
2368 host
+ TARGET_PAGE_SIZE
-
2369 qemu_host_page_size
);
2371 ret
= postcopy_place_page(mis
, host
+ TARGET_PAGE_SIZE
-
2372 qemu_host_page_size
,
2377 ret
= qemu_file_get_error(f
);
2384 static int ram_load(QEMUFile
*f
, void *opaque
, int version_id
)
2386 int flags
= 0, ret
= 0;
2387 static uint64_t seq_iter
;
2390 * If system is running in postcopy mode, page inserts to host memory must
2393 bool postcopy_running
= postcopy_state_get() >= POSTCOPY_INCOMING_LISTENING
;
2397 if (version_id
!= 4) {
2401 /* This RCU critical section can be very long running.
2402 * When RCU reclaims in the code start to become numerous,
2403 * it will be necessary to reduce the granularity of this
2408 if (postcopy_running
) {
2409 ret
= ram_load_postcopy(f
);
2412 while (!postcopy_running
&& !ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
2413 ram_addr_t addr
, total_ram_bytes
;
2417 addr
= qemu_get_be64(f
);
2418 flags
= addr
& ~TARGET_PAGE_MASK
;
2419 addr
&= TARGET_PAGE_MASK
;
2421 if (flags
& (RAM_SAVE_FLAG_COMPRESS
| RAM_SAVE_FLAG_PAGE
|
2422 RAM_SAVE_FLAG_COMPRESS_PAGE
| RAM_SAVE_FLAG_XBZRLE
)) {
2423 host
= host_from_stream_offset(f
, addr
, flags
);
2425 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
2431 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
2432 case RAM_SAVE_FLAG_MEM_SIZE
:
2433 /* Synchronize RAM block list */
2434 total_ram_bytes
= addr
;
2435 while (!ret
&& total_ram_bytes
) {
2440 len
= qemu_get_byte(f
);
2441 qemu_get_buffer(f
, (uint8_t *)id
, len
);
2443 length
= qemu_get_be64(f
);
2445 block
= qemu_ram_block_by_name(id
);
2447 if (length
!= block
->used_length
) {
2448 Error
*local_err
= NULL
;
2450 ret
= qemu_ram_resize(block
->offset
, length
,
2453 error_report_err(local_err
);
2456 ram_control_load_hook(f
, RAM_CONTROL_BLOCK_REG
,
2459 error_report("Unknown ramblock \"%s\", cannot "
2460 "accept migration", id
);
2464 total_ram_bytes
-= length
;
2468 case RAM_SAVE_FLAG_COMPRESS
:
2469 ch
= qemu_get_byte(f
);
2470 ram_handle_compressed(host
, ch
, TARGET_PAGE_SIZE
);
2473 case RAM_SAVE_FLAG_PAGE
:
2474 qemu_get_buffer(f
, host
, TARGET_PAGE_SIZE
);
2477 case RAM_SAVE_FLAG_COMPRESS_PAGE
:
2478 len
= qemu_get_be32(f
);
2479 if (len
< 0 || len
> compressBound(TARGET_PAGE_SIZE
)) {
2480 error_report("Invalid compressed data length: %d", len
);
2484 qemu_get_buffer(f
, compressed_data_buf
, len
);
2485 decompress_data_with_multi_threads(compressed_data_buf
, host
, len
);
2488 case RAM_SAVE_FLAG_XBZRLE
:
2489 if (load_xbzrle(f
, addr
, host
) < 0) {
2490 error_report("Failed to decompress XBZRLE page at "
2491 RAM_ADDR_FMT
, addr
);
2496 case RAM_SAVE_FLAG_EOS
:
2500 if (flags
& RAM_SAVE_FLAG_HOOK
) {
2501 ram_control_load_hook(f
, RAM_CONTROL_HOOK
, NULL
);
2503 error_report("Unknown combination of migration flags: %#x",
2509 ret
= qemu_file_get_error(f
);
2514 DPRINTF("Completed load of VM with exit code %d seq iteration "
2515 "%" PRIu64
"\n", ret
, seq_iter
);
2519 static SaveVMHandlers savevm_ram_handlers
= {
2520 .save_live_setup
= ram_save_setup
,
2521 .save_live_iterate
= ram_save_iterate
,
2522 .save_live_complete_postcopy
= ram_save_complete
,
2523 .save_live_complete_precopy
= ram_save_complete
,
2524 .save_live_pending
= ram_save_pending
,
2525 .load_state
= ram_load
,
2526 .cleanup
= ram_migration_cleanup
,
2529 void ram_mig_init(void)
2531 qemu_mutex_init(&XBZRLE
.lock
);
2532 register_savevm_live(NULL
, "ram", 0, 4, &savevm_ram_handlers
, NULL
);