4 * Copyright (c) 2003-2008 Fabrice Bellard
5 * Copyright (c) 2011-2015 Red Hat Inc
8 * Juan Quintela <quintela@redhat.com>
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
30 #include "qemu/bitops.h"
31 #include "qemu/bitmap.h"
32 #include "qemu/timer.h"
33 #include "qemu/main-loop.h"
34 #include "migration/migration.h"
35 #include "exec/address-spaces.h"
36 #include "migration/page_cache.h"
37 #include "qemu/error-report.h"
39 #include "exec/ram_addr.h"
40 #include "qemu/rcu_queue.h"
42 #ifdef DEBUG_MIGRATION_RAM
43 #define DPRINTF(fmt, ...) \
44 do { fprintf(stdout, "migration_ram: " fmt, ## __VA_ARGS__); } while (0)
46 #define DPRINTF(fmt, ...) \
50 static bool mig_throttle_on
;
51 static int dirty_rate_high_cnt
;
52 static void check_guest_throttling(void);
54 static uint64_t bitmap_sync_count
;
56 /***********************************************************/
57 /* ram save/restore */
59 #define RAM_SAVE_FLAG_FULL 0x01 /* Obsolete, not used anymore */
60 #define RAM_SAVE_FLAG_COMPRESS 0x02
61 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
62 #define RAM_SAVE_FLAG_PAGE 0x08
63 #define RAM_SAVE_FLAG_EOS 0x10
64 #define RAM_SAVE_FLAG_CONTINUE 0x20
65 #define RAM_SAVE_FLAG_XBZRLE 0x40
66 /* 0x80 is reserved in migration.h start with 0x100 next */
67 #define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
69 static const uint8_t ZERO_TARGET_PAGE
[TARGET_PAGE_SIZE
];
71 static inline bool is_zero_range(uint8_t *p
, uint64_t size
)
73 return buffer_find_nonzero_offset(p
, size
) == size
;
76 /* struct contains XBZRLE cache and a static page
77 used by the compression */
79 /* buffer used for XBZRLE encoding */
81 /* buffer for storing page content */
83 /* Cache for XBZRLE, Protected by lock. */
88 /* buffer used for XBZRLE decoding */
89 static uint8_t *xbzrle_decoded_buf
;
91 static void XBZRLE_cache_lock(void)
93 if (migrate_use_xbzrle())
94 qemu_mutex_lock(&XBZRLE
.lock
);
97 static void XBZRLE_cache_unlock(void)
99 if (migrate_use_xbzrle())
100 qemu_mutex_unlock(&XBZRLE
.lock
);
104 * called from qmp_migrate_set_cache_size in main thread, possibly while
105 * a migration is in progress.
106 * A running migration maybe using the cache and might finish during this
107 * call, hence changes to the cache are protected by XBZRLE.lock().
109 int64_t xbzrle_cache_resize(int64_t new_size
)
111 PageCache
*new_cache
;
114 if (new_size
< TARGET_PAGE_SIZE
) {
120 if (XBZRLE
.cache
!= NULL
) {
121 if (pow2floor(new_size
) == migrate_xbzrle_cache_size()) {
124 new_cache
= cache_init(new_size
/ TARGET_PAGE_SIZE
,
127 error_report("Error creating cache");
132 cache_fini(XBZRLE
.cache
);
133 XBZRLE
.cache
= new_cache
;
137 ret
= pow2floor(new_size
);
139 XBZRLE_cache_unlock();
143 /* accounting for migration statistics */
144 typedef struct AccountingInfo
{
146 uint64_t skipped_pages
;
149 uint64_t xbzrle_bytes
;
150 uint64_t xbzrle_pages
;
151 uint64_t xbzrle_cache_miss
;
152 double xbzrle_cache_miss_rate
;
153 uint64_t xbzrle_overflows
;
156 static AccountingInfo acct_info
;
158 static void acct_clear(void)
160 memset(&acct_info
, 0, sizeof(acct_info
));
163 uint64_t dup_mig_bytes_transferred(void)
165 return acct_info
.dup_pages
* TARGET_PAGE_SIZE
;
168 uint64_t dup_mig_pages_transferred(void)
170 return acct_info
.dup_pages
;
173 uint64_t skipped_mig_bytes_transferred(void)
175 return acct_info
.skipped_pages
* TARGET_PAGE_SIZE
;
178 uint64_t skipped_mig_pages_transferred(void)
180 return acct_info
.skipped_pages
;
183 uint64_t norm_mig_bytes_transferred(void)
185 return acct_info
.norm_pages
* TARGET_PAGE_SIZE
;
188 uint64_t norm_mig_pages_transferred(void)
190 return acct_info
.norm_pages
;
193 uint64_t xbzrle_mig_bytes_transferred(void)
195 return acct_info
.xbzrle_bytes
;
198 uint64_t xbzrle_mig_pages_transferred(void)
200 return acct_info
.xbzrle_pages
;
203 uint64_t xbzrle_mig_pages_cache_miss(void)
205 return acct_info
.xbzrle_cache_miss
;
208 double xbzrle_mig_cache_miss_rate(void)
210 return acct_info
.xbzrle_cache_miss_rate
;
213 uint64_t xbzrle_mig_pages_overflow(void)
215 return acct_info
.xbzrle_overflows
;
218 /* This is the last block that we have visited serching for dirty pages
220 static RAMBlock
*last_seen_block
;
221 /* This is the last block from where we have sent data */
222 static RAMBlock
*last_sent_block
;
223 static ram_addr_t last_offset
;
224 static unsigned long *migration_bitmap
;
225 static uint64_t migration_dirty_pages
;
226 static uint32_t last_version
;
227 static bool ram_bulk_stage
;
229 struct CompressParam
{
238 typedef struct CompressParam CompressParam
;
240 struct DecompressParam
{
248 typedef struct DecompressParam DecompressParam
;
250 static CompressParam
*comp_param
;
251 static QemuThread
*compress_threads
;
252 /* comp_done_cond is used to wake up the migration thread when
253 * one of the compression threads has finished the compression.
254 * comp_done_lock is used to co-work with comp_done_cond.
256 static QemuMutex
*comp_done_lock
;
257 static QemuCond
*comp_done_cond
;
258 /* The empty QEMUFileOps will be used by file in CompressParam */
259 static const QEMUFileOps empty_ops
= { };
261 static bool compression_switch
;
262 static bool quit_comp_thread
;
263 static bool quit_decomp_thread
;
264 static DecompressParam
*decomp_param
;
265 static QemuThread
*decompress_threads
;
266 static uint8_t *compressed_data_buf
;
268 static int do_compress_ram_page(CompressParam
*param
);
270 static void *do_data_compress(void *opaque
)
272 CompressParam
*param
= opaque
;
274 while (!quit_comp_thread
) {
275 qemu_mutex_lock(¶m
->mutex
);
276 /* Re-check the quit_comp_thread in case of
277 * terminate_compression_threads is called just before
278 * qemu_mutex_lock(¶m->mutex) and after
279 * while(!quit_comp_thread), re-check it here can make
280 * sure the compression thread terminate as expected.
282 while (!param
->start
&& !quit_comp_thread
) {
283 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
285 if (!quit_comp_thread
) {
286 do_compress_ram_page(param
);
288 param
->start
= false;
289 qemu_mutex_unlock(¶m
->mutex
);
291 qemu_mutex_lock(comp_done_lock
);
293 qemu_cond_signal(comp_done_cond
);
294 qemu_mutex_unlock(comp_done_lock
);
300 static inline void terminate_compression_threads(void)
302 int idx
, thread_count
;
304 thread_count
= migrate_compress_threads();
305 quit_comp_thread
= true;
306 for (idx
= 0; idx
< thread_count
; idx
++) {
307 qemu_mutex_lock(&comp_param
[idx
].mutex
);
308 qemu_cond_signal(&comp_param
[idx
].cond
);
309 qemu_mutex_unlock(&comp_param
[idx
].mutex
);
313 void migrate_compress_threads_join(void)
317 if (!migrate_use_compression()) {
320 terminate_compression_threads();
321 thread_count
= migrate_compress_threads();
322 for (i
= 0; i
< thread_count
; i
++) {
323 qemu_thread_join(compress_threads
+ i
);
324 qemu_fclose(comp_param
[i
].file
);
325 qemu_mutex_destroy(&comp_param
[i
].mutex
);
326 qemu_cond_destroy(&comp_param
[i
].cond
);
328 qemu_mutex_destroy(comp_done_lock
);
329 qemu_cond_destroy(comp_done_cond
);
330 g_free(compress_threads
);
332 g_free(comp_done_cond
);
333 g_free(comp_done_lock
);
334 compress_threads
= NULL
;
336 comp_done_cond
= NULL
;
337 comp_done_lock
= NULL
;
340 void migrate_compress_threads_create(void)
344 if (!migrate_use_compression()) {
347 quit_comp_thread
= false;
348 compression_switch
= true;
349 thread_count
= migrate_compress_threads();
350 compress_threads
= g_new0(QemuThread
, thread_count
);
351 comp_param
= g_new0(CompressParam
, thread_count
);
352 comp_done_cond
= g_new0(QemuCond
, 1);
353 comp_done_lock
= g_new0(QemuMutex
, 1);
354 qemu_cond_init(comp_done_cond
);
355 qemu_mutex_init(comp_done_lock
);
356 for (i
= 0; i
< thread_count
; i
++) {
357 /* com_param[i].file is just used as a dummy buffer to save data, set
360 comp_param
[i
].file
= qemu_fopen_ops(NULL
, &empty_ops
);
361 comp_param
[i
].done
= true;
362 qemu_mutex_init(&comp_param
[i
].mutex
);
363 qemu_cond_init(&comp_param
[i
].cond
);
364 qemu_thread_create(compress_threads
+ i
, "compress",
365 do_data_compress
, comp_param
+ i
,
366 QEMU_THREAD_JOINABLE
);
371 * save_page_header: Write page header to wire
373 * If this is the 1st block, it also writes the block identification
375 * Returns: Number of bytes written
377 * @f: QEMUFile where to send the data
378 * @block: block that contains the page we want to send
379 * @offset: offset inside the block for the page
380 * in the lower bits, it contains flags
382 static size_t save_page_header(QEMUFile
*f
, RAMBlock
*block
, ram_addr_t offset
)
386 qemu_put_be64(f
, offset
);
389 if (!(offset
& RAM_SAVE_FLAG_CONTINUE
)) {
390 qemu_put_byte(f
, strlen(block
->idstr
));
391 qemu_put_buffer(f
, (uint8_t *)block
->idstr
,
392 strlen(block
->idstr
));
393 size
+= 1 + strlen(block
->idstr
);
398 /* Update the xbzrle cache to reflect a page that's been sent as all 0.
399 * The important thing is that a stale (not-yet-0'd) page be replaced
401 * As a bonus, if the page wasn't in the cache it gets added so that
402 * when a small write is made into the 0'd page it gets XBZRLE sent
404 static void xbzrle_cache_zero_page(ram_addr_t current_addr
)
406 if (ram_bulk_stage
|| !migrate_use_xbzrle()) {
410 /* We don't care if this fails to allocate a new cache page
411 * as long as it updated an old one */
412 cache_insert(XBZRLE
.cache
, current_addr
, ZERO_TARGET_PAGE
,
416 #define ENCODING_FLAG_XBZRLE 0x1
419 * save_xbzrle_page: compress and send current page
421 * Returns: 1 means that we wrote the page
422 * 0 means that page is identical to the one already sent
423 * -1 means that xbzrle would be longer than normal
425 * @f: QEMUFile where to send the data
428 * @block: block that contains the page we want to send
429 * @offset: offset inside the block for the page
430 * @last_stage: if we are at the completion stage
431 * @bytes_transferred: increase it with the number of transferred bytes
433 static int save_xbzrle_page(QEMUFile
*f
, uint8_t **current_data
,
434 ram_addr_t current_addr
, RAMBlock
*block
,
435 ram_addr_t offset
, bool last_stage
,
436 uint64_t *bytes_transferred
)
438 int encoded_len
= 0, bytes_xbzrle
;
439 uint8_t *prev_cached_page
;
441 if (!cache_is_cached(XBZRLE
.cache
, current_addr
, bitmap_sync_count
)) {
442 acct_info
.xbzrle_cache_miss
++;
444 if (cache_insert(XBZRLE
.cache
, current_addr
, *current_data
,
445 bitmap_sync_count
) == -1) {
448 /* update *current_data when the page has been
449 inserted into cache */
450 *current_data
= get_cached_data(XBZRLE
.cache
, current_addr
);
456 prev_cached_page
= get_cached_data(XBZRLE
.cache
, current_addr
);
458 /* save current buffer into memory */
459 memcpy(XBZRLE
.current_buf
, *current_data
, TARGET_PAGE_SIZE
);
461 /* XBZRLE encoding (if there is no overflow) */
462 encoded_len
= xbzrle_encode_buffer(prev_cached_page
, XBZRLE
.current_buf
,
463 TARGET_PAGE_SIZE
, XBZRLE
.encoded_buf
,
465 if (encoded_len
== 0) {
466 DPRINTF("Skipping unmodified page\n");
468 } else if (encoded_len
== -1) {
469 DPRINTF("Overflow\n");
470 acct_info
.xbzrle_overflows
++;
471 /* update data in the cache */
473 memcpy(prev_cached_page
, *current_data
, TARGET_PAGE_SIZE
);
474 *current_data
= prev_cached_page
;
479 /* we need to update the data in the cache, in order to get the same data */
481 memcpy(prev_cached_page
, XBZRLE
.current_buf
, TARGET_PAGE_SIZE
);
484 /* Send XBZRLE based compressed page */
485 bytes_xbzrle
= save_page_header(f
, block
, offset
| RAM_SAVE_FLAG_XBZRLE
);
486 qemu_put_byte(f
, ENCODING_FLAG_XBZRLE
);
487 qemu_put_be16(f
, encoded_len
);
488 qemu_put_buffer(f
, XBZRLE
.encoded_buf
, encoded_len
);
489 bytes_xbzrle
+= encoded_len
+ 1 + 2;
490 acct_info
.xbzrle_pages
++;
491 acct_info
.xbzrle_bytes
+= bytes_xbzrle
;
492 *bytes_transferred
+= bytes_xbzrle
;
497 /* Called with rcu_read_lock() to protect migration_bitmap */
499 ram_addr_t
migration_bitmap_find_and_reset_dirty(MemoryRegion
*mr
,
502 unsigned long base
= mr
->ram_addr
>> TARGET_PAGE_BITS
;
503 unsigned long nr
= base
+ (start
>> TARGET_PAGE_BITS
);
504 uint64_t mr_size
= TARGET_PAGE_ALIGN(memory_region_size(mr
));
505 unsigned long size
= base
+ (mr_size
>> TARGET_PAGE_BITS
);
506 unsigned long *bitmap
;
510 bitmap
= atomic_rcu_read(&migration_bitmap
);
511 if (ram_bulk_stage
&& nr
> base
) {
514 next
= find_next_bit(bitmap
, size
, nr
);
518 clear_bit(next
, bitmap
);
519 migration_dirty_pages
--;
521 return (next
- base
) << TARGET_PAGE_BITS
;
524 /* Called with rcu_read_lock() to protect migration_bitmap */
525 static void migration_bitmap_sync_range(ram_addr_t start
, ram_addr_t length
)
527 unsigned long *bitmap
;
528 bitmap
= atomic_rcu_read(&migration_bitmap
);
529 migration_dirty_pages
+=
530 cpu_physical_memory_sync_dirty_bitmap(bitmap
, start
, length
);
534 /* Fix me: there are too many global variables used in migration process. */
535 static int64_t start_time
;
536 static int64_t bytes_xfer_prev
;
537 static int64_t num_dirty_pages_period
;
538 static uint64_t xbzrle_cache_miss_prev
;
539 static uint64_t iterations_prev
;
541 static void migration_bitmap_sync_init(void)
545 num_dirty_pages_period
= 0;
546 xbzrle_cache_miss_prev
= 0;
550 /* Called with iothread lock held, to protect ram_list.dirty_memory[] */
551 static void migration_bitmap_sync(void)
554 uint64_t num_dirty_pages_init
= migration_dirty_pages
;
555 MigrationState
*s
= migrate_get_current();
557 int64_t bytes_xfer_now
;
561 if (!bytes_xfer_prev
) {
562 bytes_xfer_prev
= ram_bytes_transferred();
566 start_time
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
569 trace_migration_bitmap_sync_start();
570 address_space_sync_dirty_bitmap(&address_space_memory
);
573 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
574 migration_bitmap_sync_range(block
->mr
->ram_addr
, block
->used_length
);
578 trace_migration_bitmap_sync_end(migration_dirty_pages
579 - num_dirty_pages_init
);
580 num_dirty_pages_period
+= migration_dirty_pages
- num_dirty_pages_init
;
581 end_time
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
583 /* more than 1 second = 1000 millisecons */
584 if (end_time
> start_time
+ 1000) {
585 if (migrate_auto_converge()) {
586 /* The following detection logic can be refined later. For now:
587 Check to see if the dirtied bytes is 50% more than the approx.
588 amount of bytes that just got transferred since the last time we
589 were in this routine. If that happens >N times (for now N==4)
590 we turn on the throttle down logic */
591 bytes_xfer_now
= ram_bytes_transferred();
592 if (s
->dirty_pages_rate
&&
593 (num_dirty_pages_period
* TARGET_PAGE_SIZE
>
594 (bytes_xfer_now
- bytes_xfer_prev
)/2) &&
595 (dirty_rate_high_cnt
++ > 4)) {
596 trace_migration_throttle();
597 mig_throttle_on
= true;
598 dirty_rate_high_cnt
= 0;
600 bytes_xfer_prev
= bytes_xfer_now
;
602 mig_throttle_on
= false;
604 if (migrate_use_xbzrle()) {
605 if (iterations_prev
!= acct_info
.iterations
) {
606 acct_info
.xbzrle_cache_miss_rate
=
607 (double)(acct_info
.xbzrle_cache_miss
-
608 xbzrle_cache_miss_prev
) /
609 (acct_info
.iterations
- iterations_prev
);
611 iterations_prev
= acct_info
.iterations
;
612 xbzrle_cache_miss_prev
= acct_info
.xbzrle_cache_miss
;
614 s
->dirty_pages_rate
= num_dirty_pages_period
* 1000
615 / (end_time
- start_time
);
616 s
->dirty_bytes_rate
= s
->dirty_pages_rate
* TARGET_PAGE_SIZE
;
617 start_time
= end_time
;
618 num_dirty_pages_period
= 0;
620 s
->dirty_sync_count
= bitmap_sync_count
;
624 * save_zero_page: Send the zero page to the stream
626 * Returns: Number of pages written.
628 * @f: QEMUFile where to send the data
629 * @block: block that contains the page we want to send
630 * @offset: offset inside the block for the page
631 * @p: pointer to the page
632 * @bytes_transferred: increase it with the number of transferred bytes
634 static int save_zero_page(QEMUFile
*f
, RAMBlock
*block
, ram_addr_t offset
,
635 uint8_t *p
, uint64_t *bytes_transferred
)
639 if (is_zero_range(p
, TARGET_PAGE_SIZE
)) {
640 acct_info
.dup_pages
++;
641 *bytes_transferred
+= save_page_header(f
, block
,
642 offset
| RAM_SAVE_FLAG_COMPRESS
);
644 *bytes_transferred
+= 1;
652 * ram_save_page: Send the given page to the stream
654 * Returns: Number of pages written.
656 * @f: QEMUFile where to send the data
657 * @block: block that contains the page we want to send
658 * @offset: offset inside the block for the page
659 * @last_stage: if we are at the completion stage
660 * @bytes_transferred: increase it with the number of transferred bytes
662 static int ram_save_page(QEMUFile
*f
, RAMBlock
* block
, ram_addr_t offset
,
663 bool last_stage
, uint64_t *bytes_transferred
)
667 ram_addr_t current_addr
;
668 MemoryRegion
*mr
= block
->mr
;
671 bool send_async
= true;
673 p
= memory_region_get_ram_ptr(mr
) + offset
;
675 /* In doubt sent page as normal */
677 ret
= ram_control_save_page(f
, block
->offset
,
678 offset
, TARGET_PAGE_SIZE
, &bytes_xmit
);
680 *bytes_transferred
+= bytes_xmit
;
686 current_addr
= block
->offset
+ offset
;
688 if (block
== last_sent_block
) {
689 offset
|= RAM_SAVE_FLAG_CONTINUE
;
691 if (ret
!= RAM_SAVE_CONTROL_NOT_SUPP
) {
692 if (ret
!= RAM_SAVE_CONTROL_DELAYED
) {
693 if (bytes_xmit
> 0) {
694 acct_info
.norm_pages
++;
695 } else if (bytes_xmit
== 0) {
696 acct_info
.dup_pages
++;
700 pages
= save_zero_page(f
, block
, offset
, p
, bytes_transferred
);
702 /* Must let xbzrle know, otherwise a previous (now 0'd) cached
703 * page would be stale
705 xbzrle_cache_zero_page(current_addr
);
706 } else if (!ram_bulk_stage
&& migrate_use_xbzrle()) {
707 pages
= save_xbzrle_page(f
, &p
, current_addr
, block
,
708 offset
, last_stage
, bytes_transferred
);
710 /* Can't send this cached data async, since the cache page
711 * might get updated before it gets to the wire
718 /* XBZRLE overflow or normal page */
720 *bytes_transferred
+= save_page_header(f
, block
,
721 offset
| RAM_SAVE_FLAG_PAGE
);
723 qemu_put_buffer_async(f
, p
, TARGET_PAGE_SIZE
);
725 qemu_put_buffer(f
, p
, TARGET_PAGE_SIZE
);
727 *bytes_transferred
+= TARGET_PAGE_SIZE
;
729 acct_info
.norm_pages
++;
732 XBZRLE_cache_unlock();
737 static int do_compress_ram_page(CompressParam
*param
)
739 int bytes_sent
, blen
;
741 RAMBlock
*block
= param
->block
;
742 ram_addr_t offset
= param
->offset
;
744 p
= memory_region_get_ram_ptr(block
->mr
) + (offset
& TARGET_PAGE_MASK
);
746 bytes_sent
= save_page_header(param
->file
, block
, offset
|
747 RAM_SAVE_FLAG_COMPRESS_PAGE
);
748 blen
= qemu_put_compression_data(param
->file
, p
, TARGET_PAGE_SIZE
,
749 migrate_compress_level());
755 static inline void start_compression(CompressParam
*param
)
758 qemu_mutex_lock(¶m
->mutex
);
760 qemu_cond_signal(¶m
->cond
);
761 qemu_mutex_unlock(¶m
->mutex
);
764 static inline void start_decompression(DecompressParam
*param
)
766 qemu_mutex_lock(¶m
->mutex
);
768 qemu_cond_signal(¶m
->cond
);
769 qemu_mutex_unlock(¶m
->mutex
);
772 static uint64_t bytes_transferred
;
774 static void flush_compressed_data(QEMUFile
*f
)
776 int idx
, len
, thread_count
;
778 if (!migrate_use_compression()) {
781 thread_count
= migrate_compress_threads();
782 for (idx
= 0; idx
< thread_count
; idx
++) {
783 if (!comp_param
[idx
].done
) {
784 qemu_mutex_lock(comp_done_lock
);
785 while (!comp_param
[idx
].done
&& !quit_comp_thread
) {
786 qemu_cond_wait(comp_done_cond
, comp_done_lock
);
788 qemu_mutex_unlock(comp_done_lock
);
790 if (!quit_comp_thread
) {
791 len
= qemu_put_qemu_file(f
, comp_param
[idx
].file
);
792 bytes_transferred
+= len
;
797 static inline void set_compress_params(CompressParam
*param
, RAMBlock
*block
,
800 param
->block
= block
;
801 param
->offset
= offset
;
804 static int compress_page_with_multi_thread(QEMUFile
*f
, RAMBlock
*block
,
806 uint64_t *bytes_transferred
)
808 int idx
, thread_count
, bytes_xmit
= -1, pages
= -1;
810 thread_count
= migrate_compress_threads();
811 qemu_mutex_lock(comp_done_lock
);
813 for (idx
= 0; idx
< thread_count
; idx
++) {
814 if (comp_param
[idx
].done
) {
815 bytes_xmit
= qemu_put_qemu_file(f
, comp_param
[idx
].file
);
816 set_compress_params(&comp_param
[idx
], block
, offset
);
817 start_compression(&comp_param
[idx
]);
819 acct_info
.norm_pages
++;
820 *bytes_transferred
+= bytes_xmit
;
827 qemu_cond_wait(comp_done_cond
, comp_done_lock
);
830 qemu_mutex_unlock(comp_done_lock
);
836 * ram_save_compressed_page: compress the given page and send it to the stream
838 * Returns: Number of pages written.
840 * @f: QEMUFile where to send the data
841 * @block: block that contains the page we want to send
842 * @offset: offset inside the block for the page
843 * @last_stage: if we are at the completion stage
844 * @bytes_transferred: increase it with the number of transferred bytes
846 static int ram_save_compressed_page(QEMUFile
*f
, RAMBlock
*block
,
847 ram_addr_t offset
, bool last_stage
,
848 uint64_t *bytes_transferred
)
852 MemoryRegion
*mr
= block
->mr
;
856 p
= memory_region_get_ram_ptr(mr
) + offset
;
859 ret
= ram_control_save_page(f
, block
->offset
,
860 offset
, TARGET_PAGE_SIZE
, &bytes_xmit
);
862 *bytes_transferred
+= bytes_xmit
;
865 if (block
== last_sent_block
) {
866 offset
|= RAM_SAVE_FLAG_CONTINUE
;
868 if (ret
!= RAM_SAVE_CONTROL_NOT_SUPP
) {
869 if (ret
!= RAM_SAVE_CONTROL_DELAYED
) {
870 if (bytes_xmit
> 0) {
871 acct_info
.norm_pages
++;
872 } else if (bytes_xmit
== 0) {
873 acct_info
.dup_pages
++;
877 /* When starting the process of a new block, the first page of
878 * the block should be sent out before other pages in the same
879 * block, and all the pages in last block should have been sent
880 * out, keeping this order is important, because the 'cont' flag
881 * is used to avoid resending the block name.
883 if (block
!= last_sent_block
) {
884 flush_compressed_data(f
);
885 pages
= save_zero_page(f
, block
, offset
, p
, bytes_transferred
);
887 set_compress_params(&comp_param
[0], block
, offset
);
888 /* Use the qemu thread to compress the data to make sure the
889 * first page is sent out before other pages
891 bytes_xmit
= do_compress_ram_page(&comp_param
[0]);
892 acct_info
.norm_pages
++;
893 qemu_put_qemu_file(f
, comp_param
[0].file
);
894 *bytes_transferred
+= bytes_xmit
;
898 pages
= save_zero_page(f
, block
, offset
, p
, bytes_transferred
);
900 pages
= compress_page_with_multi_thread(f
, block
, offset
,
910 * ram_find_and_save_block: Finds a dirty page and sends it to f
912 * Called within an RCU critical section.
914 * Returns: The number of pages written
915 * 0 means no dirty pages
917 * @f: QEMUFile where to send the data
918 * @last_stage: if we are at the completion stage
919 * @bytes_transferred: increase it with the number of transferred bytes
922 static int ram_find_and_save_block(QEMUFile
*f
, bool last_stage
,
923 uint64_t *bytes_transferred
)
925 RAMBlock
*block
= last_seen_block
;
926 ram_addr_t offset
= last_offset
;
927 bool complete_round
= false;
932 block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
936 offset
= migration_bitmap_find_and_reset_dirty(mr
, offset
);
937 if (complete_round
&& block
== last_seen_block
&&
938 offset
>= last_offset
) {
941 if (offset
>= block
->used_length
) {
943 block
= QLIST_NEXT_RCU(block
, next
);
945 block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
946 complete_round
= true;
947 ram_bulk_stage
= false;
948 if (migrate_use_xbzrle()) {
949 /* If xbzrle is on, stop using the data compression at this
950 * point. In theory, xbzrle can do better than compression.
952 flush_compressed_data(f
);
953 compression_switch
= false;
957 if (compression_switch
&& migrate_use_compression()) {
958 pages
= ram_save_compressed_page(f
, block
, offset
, last_stage
,
961 pages
= ram_save_page(f
, block
, offset
, last_stage
,
965 /* if page is unmodified, continue to the next */
967 last_sent_block
= block
;
973 last_seen_block
= block
;
974 last_offset
= offset
;
979 void acct_update_position(QEMUFile
*f
, size_t size
, bool zero
)
981 uint64_t pages
= size
/ TARGET_PAGE_SIZE
;
983 acct_info
.dup_pages
+= pages
;
985 acct_info
.norm_pages
+= pages
;
986 bytes_transferred
+= size
;
987 qemu_update_position(f
, size
);
991 static ram_addr_t
ram_save_remaining(void)
993 return migration_dirty_pages
;
996 uint64_t ram_bytes_remaining(void)
998 return ram_save_remaining() * TARGET_PAGE_SIZE
;
1001 uint64_t ram_bytes_transferred(void)
1003 return bytes_transferred
;
1006 uint64_t ram_bytes_total(void)
1012 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
)
1013 total
+= block
->used_length
;
1018 void free_xbzrle_decoded_buf(void)
1020 g_free(xbzrle_decoded_buf
);
1021 xbzrle_decoded_buf
= NULL
;
1024 static void migration_end(void)
1026 /* caller have hold iothread lock or is in a bh, so there is
1027 * no writing race against this migration_bitmap
1029 unsigned long *bitmap
= migration_bitmap
;
1030 atomic_rcu_set(&migration_bitmap
, NULL
);
1032 memory_global_dirty_log_stop();
1037 XBZRLE_cache_lock();
1039 cache_fini(XBZRLE
.cache
);
1040 g_free(XBZRLE
.encoded_buf
);
1041 g_free(XBZRLE
.current_buf
);
1042 XBZRLE
.cache
= NULL
;
1043 XBZRLE
.encoded_buf
= NULL
;
1044 XBZRLE
.current_buf
= NULL
;
1046 XBZRLE_cache_unlock();
1049 static void ram_migration_cancel(void *opaque
)
1054 static void reset_ram_globals(void)
1056 last_seen_block
= NULL
;
1057 last_sent_block
= NULL
;
1059 last_version
= ram_list
.version
;
1060 ram_bulk_stage
= true;
1063 #define MAX_WAIT 50 /* ms, half buffered_file limit */
1066 /* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
1067 * long-running RCU critical section. When rcu-reclaims in the code
1068 * start to become numerous it will be necessary to reduce the
1069 * granularity of these critical sections.
1072 static int ram_save_setup(QEMUFile
*f
, void *opaque
)
1075 int64_t ram_bitmap_pages
; /* Size of bitmap in pages, including gaps */
1077 mig_throttle_on
= false;
1078 dirty_rate_high_cnt
= 0;
1079 bitmap_sync_count
= 0;
1080 migration_bitmap_sync_init();
1082 if (migrate_use_xbzrle()) {
1083 XBZRLE_cache_lock();
1084 XBZRLE
.cache
= cache_init(migrate_xbzrle_cache_size() /
1087 if (!XBZRLE
.cache
) {
1088 XBZRLE_cache_unlock();
1089 error_report("Error creating cache");
1092 XBZRLE_cache_unlock();
1094 /* We prefer not to abort if there is no memory */
1095 XBZRLE
.encoded_buf
= g_try_malloc0(TARGET_PAGE_SIZE
);
1096 if (!XBZRLE
.encoded_buf
) {
1097 error_report("Error allocating encoded_buf");
1101 XBZRLE
.current_buf
= g_try_malloc(TARGET_PAGE_SIZE
);
1102 if (!XBZRLE
.current_buf
) {
1103 error_report("Error allocating current_buf");
1104 g_free(XBZRLE
.encoded_buf
);
1105 XBZRLE
.encoded_buf
= NULL
;
1112 /* iothread lock needed for ram_list.dirty_memory[] */
1113 qemu_mutex_lock_iothread();
1114 qemu_mutex_lock_ramlist();
1116 bytes_transferred
= 0;
1117 reset_ram_globals();
1119 ram_bitmap_pages
= last_ram_offset() >> TARGET_PAGE_BITS
;
1120 migration_bitmap
= bitmap_new(ram_bitmap_pages
);
1121 bitmap_set(migration_bitmap
, 0, ram_bitmap_pages
);
1124 * Count the total number of pages used by ram blocks not including any
1125 * gaps due to alignment or unplugs.
1127 migration_dirty_pages
= ram_bytes_total() >> TARGET_PAGE_BITS
;
1129 memory_global_dirty_log_start();
1130 migration_bitmap_sync();
1131 qemu_mutex_unlock_ramlist();
1132 qemu_mutex_unlock_iothread();
1134 qemu_put_be64(f
, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE
);
1136 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1137 qemu_put_byte(f
, strlen(block
->idstr
));
1138 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, strlen(block
->idstr
));
1139 qemu_put_be64(f
, block
->used_length
);
1144 ram_control_before_iterate(f
, RAM_CONTROL_SETUP
);
1145 ram_control_after_iterate(f
, RAM_CONTROL_SETUP
);
1147 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
1152 static int ram_save_iterate(QEMUFile
*f
, void *opaque
)
1160 if (ram_list
.version
!= last_version
) {
1161 reset_ram_globals();
1164 /* Read version before ram_list.blocks */
1167 ram_control_before_iterate(f
, RAM_CONTROL_ROUND
);
1169 t0
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
);
1171 while ((ret
= qemu_file_rate_limit(f
)) == 0) {
1174 pages
= ram_find_and_save_block(f
, false, &bytes_transferred
);
1175 /* no more pages to sent */
1179 pages_sent
+= pages
;
1180 acct_info
.iterations
++;
1181 check_guest_throttling();
1182 /* we want to check in the 1st loop, just in case it was the 1st time
1183 and we had to sync the dirty bitmap.
1184 qemu_get_clock_ns() is a bit expensive, so we only check each some
1187 if ((i
& 63) == 0) {
1188 uint64_t t1
= (qemu_clock_get_ns(QEMU_CLOCK_REALTIME
) - t0
) / 1000000;
1189 if (t1
> MAX_WAIT
) {
1190 DPRINTF("big wait: %" PRIu64
" milliseconds, %d iterations\n",
1197 flush_compressed_data(f
);
1201 * Must occur before EOS (or any QEMUFile operation)
1202 * because of RDMA protocol.
1204 ram_control_after_iterate(f
, RAM_CONTROL_ROUND
);
1206 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
1207 bytes_transferred
+= 8;
1209 ret
= qemu_file_get_error(f
);
1217 /* Called with iothread lock */
1218 static int ram_save_complete(QEMUFile
*f
, void *opaque
)
1222 migration_bitmap_sync();
1224 ram_control_before_iterate(f
, RAM_CONTROL_FINISH
);
1226 /* try transferring iterative blocks of memory */
1228 /* flush all remaining blocks regardless of rate limiting */
1232 pages
= ram_find_and_save_block(f
, true, &bytes_transferred
);
1233 /* no more blocks to sent */
1239 flush_compressed_data(f
);
1240 ram_control_after_iterate(f
, RAM_CONTROL_FINISH
);
1244 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
1249 static uint64_t ram_save_pending(QEMUFile
*f
, void *opaque
, uint64_t max_size
)
1251 uint64_t remaining_size
;
1253 remaining_size
= ram_save_remaining() * TARGET_PAGE_SIZE
;
1255 if (remaining_size
< max_size
) {
1256 qemu_mutex_lock_iothread();
1258 migration_bitmap_sync();
1260 qemu_mutex_unlock_iothread();
1261 remaining_size
= ram_save_remaining() * TARGET_PAGE_SIZE
;
1263 return remaining_size
;
1266 static int load_xbzrle(QEMUFile
*f
, ram_addr_t addr
, void *host
)
1268 unsigned int xh_len
;
1271 if (!xbzrle_decoded_buf
) {
1272 xbzrle_decoded_buf
= g_malloc(TARGET_PAGE_SIZE
);
1275 /* extract RLE header */
1276 xh_flags
= qemu_get_byte(f
);
1277 xh_len
= qemu_get_be16(f
);
1279 if (xh_flags
!= ENCODING_FLAG_XBZRLE
) {
1280 error_report("Failed to load XBZRLE page - wrong compression!");
1284 if (xh_len
> TARGET_PAGE_SIZE
) {
1285 error_report("Failed to load XBZRLE page - len overflow!");
1288 /* load data and decode */
1289 qemu_get_buffer(f
, xbzrle_decoded_buf
, xh_len
);
1292 if (xbzrle_decode_buffer(xbzrle_decoded_buf
, xh_len
, host
,
1293 TARGET_PAGE_SIZE
) == -1) {
1294 error_report("Failed to load XBZRLE page - decode error!");
1301 /* Must be called from within a rcu critical section.
1302 * Returns a pointer from within the RCU-protected ram_list.
1304 static inline void *host_from_stream_offset(QEMUFile
*f
,
1308 static RAMBlock
*block
= NULL
;
1312 if (flags
& RAM_SAVE_FLAG_CONTINUE
) {
1313 if (!block
|| block
->max_length
<= offset
) {
1314 error_report("Ack, bad migration stream!");
1318 return memory_region_get_ram_ptr(block
->mr
) + offset
;
1321 len
= qemu_get_byte(f
);
1322 qemu_get_buffer(f
, (uint8_t *)id
, len
);
1325 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1326 if (!strncmp(id
, block
->idstr
, sizeof(id
)) &&
1327 block
->max_length
> offset
) {
1328 return memory_region_get_ram_ptr(block
->mr
) + offset
;
1332 error_report("Can't find block %s!", id
);
1337 * If a page (or a whole RDMA chunk) has been
1338 * determined to be zero, then zap it.
1340 void ram_handle_compressed(void *host
, uint8_t ch
, uint64_t size
)
1342 if (ch
!= 0 || !is_zero_range(host
, size
)) {
1343 memset(host
, ch
, size
);
1347 static void *do_data_decompress(void *opaque
)
1349 DecompressParam
*param
= opaque
;
1350 unsigned long pagesize
;
1352 while (!quit_decomp_thread
) {
1353 qemu_mutex_lock(¶m
->mutex
);
1354 while (!param
->start
&& !quit_decomp_thread
) {
1355 qemu_cond_wait(¶m
->cond
, ¶m
->mutex
);
1356 pagesize
= TARGET_PAGE_SIZE
;
1357 if (!quit_decomp_thread
) {
1358 /* uncompress() will return failed in some case, especially
1359 * when the page is dirted when doing the compression, it's
1360 * not a problem because the dirty page will be retransferred
1361 * and uncompress() won't break the data in other pages.
1363 uncompress((Bytef
*)param
->des
, &pagesize
,
1364 (const Bytef
*)param
->compbuf
, param
->len
);
1366 param
->start
= false;
1368 qemu_mutex_unlock(¶m
->mutex
);
1374 void migrate_decompress_threads_create(void)
1376 int i
, thread_count
;
1378 thread_count
= migrate_decompress_threads();
1379 decompress_threads
= g_new0(QemuThread
, thread_count
);
1380 decomp_param
= g_new0(DecompressParam
, thread_count
);
1381 compressed_data_buf
= g_malloc0(compressBound(TARGET_PAGE_SIZE
));
1382 quit_decomp_thread
= false;
1383 for (i
= 0; i
< thread_count
; i
++) {
1384 qemu_mutex_init(&decomp_param
[i
].mutex
);
1385 qemu_cond_init(&decomp_param
[i
].cond
);
1386 decomp_param
[i
].compbuf
= g_malloc0(compressBound(TARGET_PAGE_SIZE
));
1387 qemu_thread_create(decompress_threads
+ i
, "decompress",
1388 do_data_decompress
, decomp_param
+ i
,
1389 QEMU_THREAD_JOINABLE
);
1393 void migrate_decompress_threads_join(void)
1395 int i
, thread_count
;
1397 quit_decomp_thread
= true;
1398 thread_count
= migrate_decompress_threads();
1399 for (i
= 0; i
< thread_count
; i
++) {
1400 qemu_mutex_lock(&decomp_param
[i
].mutex
);
1401 qemu_cond_signal(&decomp_param
[i
].cond
);
1402 qemu_mutex_unlock(&decomp_param
[i
].mutex
);
1404 for (i
= 0; i
< thread_count
; i
++) {
1405 qemu_thread_join(decompress_threads
+ i
);
1406 qemu_mutex_destroy(&decomp_param
[i
].mutex
);
1407 qemu_cond_destroy(&decomp_param
[i
].cond
);
1408 g_free(decomp_param
[i
].compbuf
);
1410 g_free(decompress_threads
);
1411 g_free(decomp_param
);
1412 g_free(compressed_data_buf
);
1413 decompress_threads
= NULL
;
1414 decomp_param
= NULL
;
1415 compressed_data_buf
= NULL
;
1418 static void decompress_data_with_multi_threads(uint8_t *compbuf
,
1419 void *host
, int len
)
1421 int idx
, thread_count
;
1423 thread_count
= migrate_decompress_threads();
1425 for (idx
= 0; idx
< thread_count
; idx
++) {
1426 if (!decomp_param
[idx
].start
) {
1427 memcpy(decomp_param
[idx
].compbuf
, compbuf
, len
);
1428 decomp_param
[idx
].des
= host
;
1429 decomp_param
[idx
].len
= len
;
1430 start_decompression(&decomp_param
[idx
]);
1434 if (idx
< thread_count
) {
1440 static int ram_load(QEMUFile
*f
, void *opaque
, int version_id
)
1442 int flags
= 0, ret
= 0;
1443 static uint64_t seq_iter
;
1448 if (version_id
!= 4) {
1452 /* This RCU critical section can be very long running.
1453 * When RCU reclaims in the code start to become numerous,
1454 * it will be necessary to reduce the granularity of this
1458 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
1459 ram_addr_t addr
, total_ram_bytes
;
1463 addr
= qemu_get_be64(f
);
1464 flags
= addr
& ~TARGET_PAGE_MASK
;
1465 addr
&= TARGET_PAGE_MASK
;
1467 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
1468 case RAM_SAVE_FLAG_MEM_SIZE
:
1469 /* Synchronize RAM block list */
1470 total_ram_bytes
= addr
;
1471 while (!ret
&& total_ram_bytes
) {
1476 len
= qemu_get_byte(f
);
1477 qemu_get_buffer(f
, (uint8_t *)id
, len
);
1479 length
= qemu_get_be64(f
);
1481 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1482 if (!strncmp(id
, block
->idstr
, sizeof(id
))) {
1483 if (length
!= block
->used_length
) {
1484 Error
*local_err
= NULL
;
1486 ret
= qemu_ram_resize(block
->offset
, length
, &local_err
);
1488 error_report_err(local_err
);
1491 ram_control_load_hook(f
, RAM_CONTROL_BLOCK_REG
,
1498 error_report("Unknown ramblock \"%s\", cannot "
1499 "accept migration", id
);
1503 total_ram_bytes
-= length
;
1506 case RAM_SAVE_FLAG_COMPRESS
:
1507 host
= host_from_stream_offset(f
, addr
, flags
);
1509 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
1513 ch
= qemu_get_byte(f
);
1514 ram_handle_compressed(host
, ch
, TARGET_PAGE_SIZE
);
1516 case RAM_SAVE_FLAG_PAGE
:
1517 host
= host_from_stream_offset(f
, addr
, flags
);
1519 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
1523 qemu_get_buffer(f
, host
, TARGET_PAGE_SIZE
);
1525 case RAM_SAVE_FLAG_COMPRESS_PAGE
:
1526 host
= host_from_stream_offset(f
, addr
, flags
);
1528 error_report("Invalid RAM offset " RAM_ADDR_FMT
, addr
);
1533 len
= qemu_get_be32(f
);
1534 if (len
< 0 || len
> compressBound(TARGET_PAGE_SIZE
)) {
1535 error_report("Invalid compressed data length: %d", len
);
1539 qemu_get_buffer(f
, compressed_data_buf
, len
);
1540 decompress_data_with_multi_threads(compressed_data_buf
, host
, len
);
1542 case RAM_SAVE_FLAG_XBZRLE
:
1543 host
= host_from_stream_offset(f
, addr
, flags
);
1545 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
1549 if (load_xbzrle(f
, addr
, host
) < 0) {
1550 error_report("Failed to decompress XBZRLE page at "
1551 RAM_ADDR_FMT
, addr
);
1556 case RAM_SAVE_FLAG_EOS
:
1560 if (flags
& RAM_SAVE_FLAG_HOOK
) {
1561 ram_control_load_hook(f
, RAM_CONTROL_HOOK
, NULL
);
1563 error_report("Unknown combination of migration flags: %#x",
1569 ret
= qemu_file_get_error(f
);
1574 DPRINTF("Completed load of VM with exit code %d seq iteration "
1575 "%" PRIu64
"\n", ret
, seq_iter
);
1579 static SaveVMHandlers savevm_ram_handlers
= {
1580 .save_live_setup
= ram_save_setup
,
1581 .save_live_iterate
= ram_save_iterate
,
1582 .save_live_complete
= ram_save_complete
,
1583 .save_live_pending
= ram_save_pending
,
1584 .load_state
= ram_load
,
1585 .cancel
= ram_migration_cancel
,
1588 void ram_mig_init(void)
1590 qemu_mutex_init(&XBZRLE
.lock
);
1591 register_savevm_live(NULL
, "ram", 0, 4, &savevm_ram_handlers
, NULL
);
1593 /* Stub function that's gets run on the vcpu when its brought out of the
1594 VM to run inside qemu via async_run_on_cpu()*/
1596 static void mig_sleep_cpu(void *opq
)
1598 qemu_mutex_unlock_iothread();
1600 qemu_mutex_lock_iothread();
1603 /* To reduce the dirty rate explicitly disallow the VCPUs from spending
1604 much time in the VM. The migration thread will try to catchup.
1605 Workload will experience a performance drop.
1607 static void mig_throttle_guest_down(void)
1611 qemu_mutex_lock_iothread();
1613 async_run_on_cpu(cpu
, mig_sleep_cpu
, NULL
);
1615 qemu_mutex_unlock_iothread();
1618 static void check_guest_throttling(void)
1623 if (!mig_throttle_on
) {
1628 t0
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
);
1632 t1
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
);
1634 /* If it has been more than 40 ms since the last time the guest
1635 * was throttled then do it again.
1637 if (40 < (t1
-t0
)/1000000) {
1638 mig_throttle_guest_down();