4 * Copyright (c) 2003-2008 Fabrice Bellard
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29 #include <sys/types.h>
33 #include "monitor/monitor.h"
34 #include "sysemu/sysemu.h"
35 #include "qemu/bitops.h"
36 #include "qemu/bitmap.h"
37 #include "sysemu/arch_init.h"
38 #include "audio/audio.h"
39 #include "hw/i386/pc.h"
40 #include "hw/pci/pci.h"
41 #include "hw/audio/audio.h"
42 #include "sysemu/kvm.h"
43 #include "migration/migration.h"
44 #include "hw/i386/smbios.h"
45 #include "exec/address-spaces.h"
46 #include "hw/audio/pcspk.h"
47 #include "migration/page_cache.h"
48 #include "qemu/config-file.h"
49 #include "qemu/error-report.h"
50 #include "qmp-commands.h"
52 #include "exec/cpu-all.h"
53 #include "exec/ram_addr.h"
54 #include "hw/acpi/acpi.h"
55 #include "qemu/host-utils.h"
56 #include "qemu/rcu_queue.h"
58 #ifdef DEBUG_ARCH_INIT
59 #define DPRINTF(fmt, ...) \
60 do { fprintf(stdout, "arch_init: " fmt, ## __VA_ARGS__); } while (0)
62 #define DPRINTF(fmt, ...) \
67 int graphic_width
= 1024;
68 int graphic_height
= 768;
69 int graphic_depth
= 8;
71 int graphic_width
= 800;
72 int graphic_height
= 600;
73 int graphic_depth
= 32;
77 #if defined(TARGET_ALPHA)
78 #define QEMU_ARCH QEMU_ARCH_ALPHA
79 #elif defined(TARGET_ARM)
80 #define QEMU_ARCH QEMU_ARCH_ARM
81 #elif defined(TARGET_CRIS)
82 #define QEMU_ARCH QEMU_ARCH_CRIS
83 #elif defined(TARGET_I386)
84 #define QEMU_ARCH QEMU_ARCH_I386
85 #elif defined(TARGET_M68K)
86 #define QEMU_ARCH QEMU_ARCH_M68K
87 #elif defined(TARGET_LM32)
88 #define QEMU_ARCH QEMU_ARCH_LM32
89 #elif defined(TARGET_MICROBLAZE)
90 #define QEMU_ARCH QEMU_ARCH_MICROBLAZE
91 #elif defined(TARGET_MIPS)
92 #define QEMU_ARCH QEMU_ARCH_MIPS
93 #elif defined(TARGET_MOXIE)
94 #define QEMU_ARCH QEMU_ARCH_MOXIE
95 #elif defined(TARGET_OPENRISC)
96 #define QEMU_ARCH QEMU_ARCH_OPENRISC
97 #elif defined(TARGET_PPC)
98 #define QEMU_ARCH QEMU_ARCH_PPC
99 #elif defined(TARGET_S390X)
100 #define QEMU_ARCH QEMU_ARCH_S390X
101 #elif defined(TARGET_SH4)
102 #define QEMU_ARCH QEMU_ARCH_SH4
103 #elif defined(TARGET_SPARC)
104 #define QEMU_ARCH QEMU_ARCH_SPARC
105 #elif defined(TARGET_XTENSA)
106 #define QEMU_ARCH QEMU_ARCH_XTENSA
107 #elif defined(TARGET_UNICORE32)
108 #define QEMU_ARCH QEMU_ARCH_UNICORE32
109 #elif defined(TARGET_TRICORE)
110 #define QEMU_ARCH QEMU_ARCH_TRICORE
113 const uint32_t arch_type
= QEMU_ARCH
;
114 static bool mig_throttle_on
;
115 static int dirty_rate_high_cnt
;
116 static void check_guest_throttling(void);
118 static uint64_t bitmap_sync_count
;
120 /***********************************************************/
121 /* ram save/restore */
123 #define RAM_SAVE_FLAG_FULL 0x01 /* Obsolete, not used anymore */
124 #define RAM_SAVE_FLAG_COMPRESS 0x02
125 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
126 #define RAM_SAVE_FLAG_PAGE 0x08
127 #define RAM_SAVE_FLAG_EOS 0x10
128 #define RAM_SAVE_FLAG_CONTINUE 0x20
129 #define RAM_SAVE_FLAG_XBZRLE 0x40
130 /* 0x80 is reserved in migration.h start with 0x100 next */
131 #define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
133 static struct defconfig_file
{
134 const char *filename
;
135 /* Indicates it is an user config file (disabled by -no-user-config) */
137 } default_config_files
[] = {
138 { CONFIG_QEMU_CONFDIR
"/qemu.conf", true },
139 { CONFIG_QEMU_CONFDIR
"/target-" TARGET_NAME
".conf", true },
140 { NULL
}, /* end of list */
143 static const uint8_t ZERO_TARGET_PAGE
[TARGET_PAGE_SIZE
];
145 int qemu_read_default_config_files(bool userconfig
)
148 struct defconfig_file
*f
;
150 for (f
= default_config_files
; f
->filename
; f
++) {
151 if (!userconfig
&& f
->userconfig
) {
154 ret
= qemu_read_config_file(f
->filename
);
155 if (ret
< 0 && ret
!= -ENOENT
) {
163 static inline bool is_zero_range(uint8_t *p
, uint64_t size
)
165 return buffer_find_nonzero_offset(p
, size
) == size
;
168 /* struct contains XBZRLE cache and a static page
169 used by the compression */
171 /* buffer used for XBZRLE encoding */
172 uint8_t *encoded_buf
;
173 /* buffer for storing page content */
174 uint8_t *current_buf
;
175 /* Cache for XBZRLE, Protected by lock. */
180 /* buffer used for XBZRLE decoding */
181 static uint8_t *xbzrle_decoded_buf
;
183 static void XBZRLE_cache_lock(void)
185 if (migrate_use_xbzrle())
186 qemu_mutex_lock(&XBZRLE
.lock
);
189 static void XBZRLE_cache_unlock(void)
191 if (migrate_use_xbzrle())
192 qemu_mutex_unlock(&XBZRLE
.lock
);
196 * called from qmp_migrate_set_cache_size in main thread, possibly while
197 * a migration is in progress.
198 * A running migration maybe using the cache and might finish during this
199 * call, hence changes to the cache are protected by XBZRLE.lock().
201 int64_t xbzrle_cache_resize(int64_t new_size
)
203 PageCache
*new_cache
;
206 if (new_size
< TARGET_PAGE_SIZE
) {
212 if (XBZRLE
.cache
!= NULL
) {
213 if (pow2floor(new_size
) == migrate_xbzrle_cache_size()) {
216 new_cache
= cache_init(new_size
/ TARGET_PAGE_SIZE
,
219 error_report("Error creating cache");
224 cache_fini(XBZRLE
.cache
);
225 XBZRLE
.cache
= new_cache
;
229 ret
= pow2floor(new_size
);
231 XBZRLE_cache_unlock();
235 /* accounting for migration statistics */
236 typedef struct AccountingInfo
{
238 uint64_t skipped_pages
;
241 uint64_t xbzrle_bytes
;
242 uint64_t xbzrle_pages
;
243 uint64_t xbzrle_cache_miss
;
244 double xbzrle_cache_miss_rate
;
245 uint64_t xbzrle_overflows
;
248 static AccountingInfo acct_info
;
250 static void acct_clear(void)
252 memset(&acct_info
, 0, sizeof(acct_info
));
255 uint64_t dup_mig_bytes_transferred(void)
257 return acct_info
.dup_pages
* TARGET_PAGE_SIZE
;
260 uint64_t dup_mig_pages_transferred(void)
262 return acct_info
.dup_pages
;
265 uint64_t skipped_mig_bytes_transferred(void)
267 return acct_info
.skipped_pages
* TARGET_PAGE_SIZE
;
270 uint64_t skipped_mig_pages_transferred(void)
272 return acct_info
.skipped_pages
;
275 uint64_t norm_mig_bytes_transferred(void)
277 return acct_info
.norm_pages
* TARGET_PAGE_SIZE
;
280 uint64_t norm_mig_pages_transferred(void)
282 return acct_info
.norm_pages
;
285 uint64_t xbzrle_mig_bytes_transferred(void)
287 return acct_info
.xbzrle_bytes
;
290 uint64_t xbzrle_mig_pages_transferred(void)
292 return acct_info
.xbzrle_pages
;
295 uint64_t xbzrle_mig_pages_cache_miss(void)
297 return acct_info
.xbzrle_cache_miss
;
300 double xbzrle_mig_cache_miss_rate(void)
302 return acct_info
.xbzrle_cache_miss_rate
;
305 uint64_t xbzrle_mig_pages_overflow(void)
307 return acct_info
.xbzrle_overflows
;
310 /* This is the last block that we have visited serching for dirty pages
312 static RAMBlock
*last_seen_block
;
313 /* This is the last block from where we have sent data */
314 static RAMBlock
*last_sent_block
;
315 static ram_addr_t last_offset
;
316 static unsigned long *migration_bitmap
;
317 static uint64_t migration_dirty_pages
;
318 static uint32_t last_version
;
319 static bool ram_bulk_stage
;
321 struct CompressParam
{
330 typedef struct CompressParam CompressParam
;
332 struct DecompressParam
{
335 typedef struct DecompressParam DecompressParam
;
337 static CompressParam
*comp_param
;
338 static QemuThread
*compress_threads
;
339 /* comp_done_cond is used to wake up the migration thread when
340 * one of the compression threads has finished the compression.
341 * comp_done_lock is used to co-work with comp_done_cond.
343 static QemuMutex
*comp_done_lock
;
344 static QemuCond
*comp_done_cond
;
345 /* The empty QEMUFileOps will be used by file in CompressParam */
346 static const QEMUFileOps empty_ops
= { };
347 static bool quit_comp_thread
;
348 static bool quit_decomp_thread
;
349 static DecompressParam
*decomp_param
;
350 static QemuThread
*decompress_threads
;
351 static uint8_t *compressed_data_buf
;
353 static void *do_data_compress(void *opaque
)
355 while (!quit_comp_thread
) {
364 static inline void terminate_compression_threads(void)
366 quit_comp_thread
= true;
371 void migrate_compress_threads_join(void)
375 if (!migrate_use_compression()) {
378 terminate_compression_threads();
379 thread_count
= migrate_compress_threads();
380 for (i
= 0; i
< thread_count
; i
++) {
381 qemu_thread_join(compress_threads
+ i
);
382 qemu_fclose(comp_param
[i
].file
);
383 qemu_mutex_destroy(&comp_param
[i
].mutex
);
384 qemu_cond_destroy(&comp_param
[i
].cond
);
386 qemu_mutex_destroy(comp_done_lock
);
387 qemu_cond_destroy(comp_done_cond
);
388 g_free(compress_threads
);
390 g_free(comp_done_cond
);
391 g_free(comp_done_lock
);
392 compress_threads
= NULL
;
394 comp_done_cond
= NULL
;
395 comp_done_lock
= NULL
;
398 void migrate_compress_threads_create(void)
402 if (!migrate_use_compression()) {
405 quit_comp_thread
= false;
406 thread_count
= migrate_compress_threads();
407 compress_threads
= g_new0(QemuThread
, thread_count
);
408 comp_param
= g_new0(CompressParam
, thread_count
);
409 comp_done_cond
= g_new0(QemuCond
, 1);
410 comp_done_lock
= g_new0(QemuMutex
, 1);
411 qemu_cond_init(comp_done_cond
);
412 qemu_mutex_init(comp_done_lock
);
413 for (i
= 0; i
< thread_count
; i
++) {
414 /* com_param[i].file is just used as a dummy buffer to save data, set
417 comp_param
[i
].file
= qemu_fopen_ops(NULL
, &empty_ops
);
418 qemu_mutex_init(&comp_param
[i
].mutex
);
419 qemu_cond_init(&comp_param
[i
].cond
);
420 qemu_thread_create(compress_threads
+ i
, "compress",
421 do_data_compress
, comp_param
+ i
,
422 QEMU_THREAD_JOINABLE
);
427 * save_page_header: Write page header to wire
429 * If this is the 1st block, it also writes the block identification
431 * Returns: Number of bytes written
433 * @f: QEMUFile where to send the data
434 * @block: block that contains the page we want to send
435 * @offset: offset inside the block for the page
436 * in the lower bits, it contains flags
438 static size_t save_page_header(QEMUFile
*f
, RAMBlock
*block
, ram_addr_t offset
)
442 qemu_put_be64(f
, offset
);
445 if (!(offset
& RAM_SAVE_FLAG_CONTINUE
)) {
446 qemu_put_byte(f
, strlen(block
->idstr
));
447 qemu_put_buffer(f
, (uint8_t *)block
->idstr
,
448 strlen(block
->idstr
));
449 size
+= 1 + strlen(block
->idstr
);
454 /* Update the xbzrle cache to reflect a page that's been sent as all 0.
455 * The important thing is that a stale (not-yet-0'd) page be replaced
457 * As a bonus, if the page wasn't in the cache it gets added so that
458 * when a small write is made into the 0'd page it gets XBZRLE sent
460 static void xbzrle_cache_zero_page(ram_addr_t current_addr
)
462 if (ram_bulk_stage
|| !migrate_use_xbzrle()) {
466 /* We don't care if this fails to allocate a new cache page
467 * as long as it updated an old one */
468 cache_insert(XBZRLE
.cache
, current_addr
, ZERO_TARGET_PAGE
,
472 #define ENCODING_FLAG_XBZRLE 0x1
475 * save_xbzrle_page: compress and send current page
477 * Returns: 1 means that we wrote the page
478 * 0 means that page is identical to the one already sent
479 * -1 means that xbzrle would be longer than normal
481 * @f: QEMUFile where to send the data
484 * @block: block that contains the page we want to send
485 * @offset: offset inside the block for the page
486 * @last_stage: if we are at the completion stage
487 * @bytes_transferred: increase it with the number of transferred bytes
489 static int save_xbzrle_page(QEMUFile
*f
, uint8_t **current_data
,
490 ram_addr_t current_addr
, RAMBlock
*block
,
491 ram_addr_t offset
, bool last_stage
,
492 uint64_t *bytes_transferred
)
494 int encoded_len
= 0, bytes_xbzrle
;
495 uint8_t *prev_cached_page
;
497 if (!cache_is_cached(XBZRLE
.cache
, current_addr
, bitmap_sync_count
)) {
498 acct_info
.xbzrle_cache_miss
++;
500 if (cache_insert(XBZRLE
.cache
, current_addr
, *current_data
,
501 bitmap_sync_count
) == -1) {
504 /* update *current_data when the page has been
505 inserted into cache */
506 *current_data
= get_cached_data(XBZRLE
.cache
, current_addr
);
512 prev_cached_page
= get_cached_data(XBZRLE
.cache
, current_addr
);
514 /* save current buffer into memory */
515 memcpy(XBZRLE
.current_buf
, *current_data
, TARGET_PAGE_SIZE
);
517 /* XBZRLE encoding (if there is no overflow) */
518 encoded_len
= xbzrle_encode_buffer(prev_cached_page
, XBZRLE
.current_buf
,
519 TARGET_PAGE_SIZE
, XBZRLE
.encoded_buf
,
521 if (encoded_len
== 0) {
522 DPRINTF("Skipping unmodified page\n");
524 } else if (encoded_len
== -1) {
525 DPRINTF("Overflow\n");
526 acct_info
.xbzrle_overflows
++;
527 /* update data in the cache */
529 memcpy(prev_cached_page
, *current_data
, TARGET_PAGE_SIZE
);
530 *current_data
= prev_cached_page
;
535 /* we need to update the data in the cache, in order to get the same data */
537 memcpy(prev_cached_page
, XBZRLE
.current_buf
, TARGET_PAGE_SIZE
);
540 /* Send XBZRLE based compressed page */
541 bytes_xbzrle
= save_page_header(f
, block
, offset
| RAM_SAVE_FLAG_XBZRLE
);
542 qemu_put_byte(f
, ENCODING_FLAG_XBZRLE
);
543 qemu_put_be16(f
, encoded_len
);
544 qemu_put_buffer(f
, XBZRLE
.encoded_buf
, encoded_len
);
545 bytes_xbzrle
+= encoded_len
+ 1 + 2;
546 acct_info
.xbzrle_pages
++;
547 acct_info
.xbzrle_bytes
+= bytes_xbzrle
;
548 *bytes_transferred
+= bytes_xbzrle
;
554 ram_addr_t
migration_bitmap_find_and_reset_dirty(MemoryRegion
*mr
,
557 unsigned long base
= mr
->ram_addr
>> TARGET_PAGE_BITS
;
558 unsigned long nr
= base
+ (start
>> TARGET_PAGE_BITS
);
559 uint64_t mr_size
= TARGET_PAGE_ALIGN(memory_region_size(mr
));
560 unsigned long size
= base
+ (mr_size
>> TARGET_PAGE_BITS
);
564 if (ram_bulk_stage
&& nr
> base
) {
567 next
= find_next_bit(migration_bitmap
, size
, nr
);
571 clear_bit(next
, migration_bitmap
);
572 migration_dirty_pages
--;
574 return (next
- base
) << TARGET_PAGE_BITS
;
577 static inline bool migration_bitmap_set_dirty(ram_addr_t addr
)
580 int nr
= addr
>> TARGET_PAGE_BITS
;
582 ret
= test_and_set_bit(nr
, migration_bitmap
);
585 migration_dirty_pages
++;
590 static void migration_bitmap_sync_range(ram_addr_t start
, ram_addr_t length
)
593 unsigned long page
= BIT_WORD(start
>> TARGET_PAGE_BITS
);
595 /* start address is aligned at the start of a word? */
596 if (((page
* BITS_PER_LONG
) << TARGET_PAGE_BITS
) == start
) {
598 int nr
= BITS_TO_LONGS(length
>> TARGET_PAGE_BITS
);
599 unsigned long *src
= ram_list
.dirty_memory
[DIRTY_MEMORY_MIGRATION
];
601 for (k
= page
; k
< page
+ nr
; k
++) {
603 unsigned long new_dirty
;
604 new_dirty
= ~migration_bitmap
[k
];
605 migration_bitmap
[k
] |= src
[k
];
607 migration_dirty_pages
+= ctpopl(new_dirty
);
612 for (addr
= 0; addr
< length
; addr
+= TARGET_PAGE_SIZE
) {
613 if (cpu_physical_memory_get_dirty(start
+ addr
,
615 DIRTY_MEMORY_MIGRATION
)) {
616 cpu_physical_memory_reset_dirty(start
+ addr
,
618 DIRTY_MEMORY_MIGRATION
);
619 migration_bitmap_set_dirty(start
+ addr
);
626 /* Fix me: there are too many global variables used in migration process. */
627 static int64_t start_time
;
628 static int64_t bytes_xfer_prev
;
629 static int64_t num_dirty_pages_period
;
631 static void migration_bitmap_sync_init(void)
635 num_dirty_pages_period
= 0;
638 /* Called with iothread lock held, to protect ram_list.dirty_memory[] */
639 static void migration_bitmap_sync(void)
642 uint64_t num_dirty_pages_init
= migration_dirty_pages
;
643 MigrationState
*s
= migrate_get_current();
645 int64_t bytes_xfer_now
;
646 static uint64_t xbzrle_cache_miss_prev
;
647 static uint64_t iterations_prev
;
651 if (!bytes_xfer_prev
) {
652 bytes_xfer_prev
= ram_bytes_transferred();
656 start_time
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
659 trace_migration_bitmap_sync_start();
660 address_space_sync_dirty_bitmap(&address_space_memory
);
663 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
664 migration_bitmap_sync_range(block
->mr
->ram_addr
, block
->used_length
);
668 trace_migration_bitmap_sync_end(migration_dirty_pages
669 - num_dirty_pages_init
);
670 num_dirty_pages_period
+= migration_dirty_pages
- num_dirty_pages_init
;
671 end_time
= qemu_clock_get_ms(QEMU_CLOCK_REALTIME
);
673 /* more than 1 second = 1000 millisecons */
674 if (end_time
> start_time
+ 1000) {
675 if (migrate_auto_converge()) {
676 /* The following detection logic can be refined later. For now:
677 Check to see if the dirtied bytes is 50% more than the approx.
678 amount of bytes that just got transferred since the last time we
679 were in this routine. If that happens >N times (for now N==4)
680 we turn on the throttle down logic */
681 bytes_xfer_now
= ram_bytes_transferred();
682 if (s
->dirty_pages_rate
&&
683 (num_dirty_pages_period
* TARGET_PAGE_SIZE
>
684 (bytes_xfer_now
- bytes_xfer_prev
)/2) &&
685 (dirty_rate_high_cnt
++ > 4)) {
686 trace_migration_throttle();
687 mig_throttle_on
= true;
688 dirty_rate_high_cnt
= 0;
690 bytes_xfer_prev
= bytes_xfer_now
;
692 mig_throttle_on
= false;
694 if (migrate_use_xbzrle()) {
695 if (iterations_prev
!= 0) {
696 acct_info
.xbzrle_cache_miss_rate
=
697 (double)(acct_info
.xbzrle_cache_miss
-
698 xbzrle_cache_miss_prev
) /
699 (acct_info
.iterations
- iterations_prev
);
701 iterations_prev
= acct_info
.iterations
;
702 xbzrle_cache_miss_prev
= acct_info
.xbzrle_cache_miss
;
704 s
->dirty_pages_rate
= num_dirty_pages_period
* 1000
705 / (end_time
- start_time
);
706 s
->dirty_bytes_rate
= s
->dirty_pages_rate
* TARGET_PAGE_SIZE
;
707 start_time
= end_time
;
708 num_dirty_pages_period
= 0;
709 s
->dirty_sync_count
= bitmap_sync_count
;
714 * ram_save_page: Send the given page to the stream
716 * Returns: Number of pages written.
718 * @f: QEMUFile where to send the data
719 * @block: block that contains the page we want to send
720 * @offset: offset inside the block for the page
721 * @last_stage: if we are at the completion stage
722 * @bytes_transferred: increase it with the number of transferred bytes
724 static int ram_save_page(QEMUFile
*f
, RAMBlock
* block
, ram_addr_t offset
,
725 bool last_stage
, uint64_t *bytes_transferred
)
729 ram_addr_t current_addr
;
730 MemoryRegion
*mr
= block
->mr
;
733 bool send_async
= true;
735 p
= memory_region_get_ram_ptr(mr
) + offset
;
737 /* In doubt sent page as normal */
739 ret
= ram_control_save_page(f
, block
->offset
,
740 offset
, TARGET_PAGE_SIZE
, &bytes_xmit
);
742 *bytes_transferred
+= bytes_xmit
;
748 current_addr
= block
->offset
+ offset
;
750 if (block
== last_sent_block
) {
751 offset
|= RAM_SAVE_FLAG_CONTINUE
;
753 if (ret
!= RAM_SAVE_CONTROL_NOT_SUPP
) {
754 if (ret
!= RAM_SAVE_CONTROL_DELAYED
) {
755 if (bytes_xmit
> 0) {
756 acct_info
.norm_pages
++;
757 } else if (bytes_xmit
== 0) {
758 acct_info
.dup_pages
++;
761 } else if (is_zero_range(p
, TARGET_PAGE_SIZE
)) {
762 acct_info
.dup_pages
++;
763 *bytes_transferred
+= save_page_header(f
, block
,
764 offset
| RAM_SAVE_FLAG_COMPRESS
);
766 *bytes_transferred
+= 1;
768 /* Must let xbzrle know, otherwise a previous (now 0'd) cached
769 * page would be stale
771 xbzrle_cache_zero_page(current_addr
);
772 } else if (!ram_bulk_stage
&& migrate_use_xbzrle()) {
773 pages
= save_xbzrle_page(f
, &p
, current_addr
, block
,
774 offset
, last_stage
, bytes_transferred
);
776 /* Can't send this cached data async, since the cache page
777 * might get updated before it gets to the wire
783 /* XBZRLE overflow or normal page */
785 *bytes_transferred
+= save_page_header(f
, block
,
786 offset
| RAM_SAVE_FLAG_PAGE
);
788 qemu_put_buffer_async(f
, p
, TARGET_PAGE_SIZE
);
790 qemu_put_buffer(f
, p
, TARGET_PAGE_SIZE
);
792 *bytes_transferred
+= TARGET_PAGE_SIZE
;
794 acct_info
.norm_pages
++;
797 XBZRLE_cache_unlock();
803 * ram_save_compressed_page: compress the given page and send it to the stream
805 * Returns: Number of pages written.
807 * @f: QEMUFile where to send the data
808 * @block: block that contains the page we want to send
809 * @offset: offset inside the block for the page
810 * @last_stage: if we are at the completion stage
811 * @bytes_transferred: increase it with the number of transferred bytes
813 static int ram_save_compressed_page(QEMUFile
*f
, RAMBlock
*block
,
814 ram_addr_t offset
, bool last_stage
,
815 uint64_t *bytes_transferred
)
825 * ram_find_and_save_block: Finds a dirty page and sends it to f
827 * Called within an RCU critical section.
829 * Returns: The number of pages written
830 * 0 means no dirty pages
832 * @f: QEMUFile where to send the data
833 * @last_stage: if we are at the completion stage
834 * @bytes_transferred: increase it with the number of transferred bytes
837 static int ram_find_and_save_block(QEMUFile
*f
, bool last_stage
,
838 uint64_t *bytes_transferred
)
840 RAMBlock
*block
= last_seen_block
;
841 ram_addr_t offset
= last_offset
;
842 bool complete_round
= false;
847 block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
851 offset
= migration_bitmap_find_and_reset_dirty(mr
, offset
);
852 if (complete_round
&& block
== last_seen_block
&&
853 offset
>= last_offset
) {
856 if (offset
>= block
->used_length
) {
858 block
= QLIST_NEXT_RCU(block
, next
);
860 block
= QLIST_FIRST_RCU(&ram_list
.blocks
);
861 complete_round
= true;
862 ram_bulk_stage
= false;
865 if (migrate_use_compression()) {
866 pages
= ram_save_compressed_page(f
, block
, offset
, last_stage
,
869 pages
= ram_save_page(f
, block
, offset
, last_stage
,
873 /* if page is unmodified, continue to the next */
875 last_sent_block
= block
;
881 last_seen_block
= block
;
882 last_offset
= offset
;
887 static uint64_t bytes_transferred
;
889 void acct_update_position(QEMUFile
*f
, size_t size
, bool zero
)
891 uint64_t pages
= size
/ TARGET_PAGE_SIZE
;
893 acct_info
.dup_pages
+= pages
;
895 acct_info
.norm_pages
+= pages
;
896 bytes_transferred
+= size
;
897 qemu_update_position(f
, size
);
901 static ram_addr_t
ram_save_remaining(void)
903 return migration_dirty_pages
;
906 uint64_t ram_bytes_remaining(void)
908 return ram_save_remaining() * TARGET_PAGE_SIZE
;
911 uint64_t ram_bytes_transferred(void)
913 return bytes_transferred
;
916 uint64_t ram_bytes_total(void)
922 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
)
923 total
+= block
->used_length
;
928 void free_xbzrle_decoded_buf(void)
930 g_free(xbzrle_decoded_buf
);
931 xbzrle_decoded_buf
= NULL
;
934 static void migration_end(void)
936 if (migration_bitmap
) {
937 memory_global_dirty_log_stop();
938 g_free(migration_bitmap
);
939 migration_bitmap
= NULL
;
944 cache_fini(XBZRLE
.cache
);
945 g_free(XBZRLE
.encoded_buf
);
946 g_free(XBZRLE
.current_buf
);
948 XBZRLE
.encoded_buf
= NULL
;
949 XBZRLE
.current_buf
= NULL
;
951 XBZRLE_cache_unlock();
954 static void ram_migration_cancel(void *opaque
)
959 static void reset_ram_globals(void)
961 last_seen_block
= NULL
;
962 last_sent_block
= NULL
;
964 last_version
= ram_list
.version
;
965 ram_bulk_stage
= true;
968 #define MAX_WAIT 50 /* ms, half buffered_file limit */
971 /* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
972 * long-running RCU critical section. When rcu-reclaims in the code
973 * start to become numerous it will be necessary to reduce the
974 * granularity of these critical sections.
977 static int ram_save_setup(QEMUFile
*f
, void *opaque
)
980 int64_t ram_bitmap_pages
; /* Size of bitmap in pages, including gaps */
982 mig_throttle_on
= false;
983 dirty_rate_high_cnt
= 0;
984 bitmap_sync_count
= 0;
985 migration_bitmap_sync_init();
987 if (migrate_use_xbzrle()) {
989 XBZRLE
.cache
= cache_init(migrate_xbzrle_cache_size() /
993 XBZRLE_cache_unlock();
994 error_report("Error creating cache");
997 XBZRLE_cache_unlock();
999 /* We prefer not to abort if there is no memory */
1000 XBZRLE
.encoded_buf
= g_try_malloc0(TARGET_PAGE_SIZE
);
1001 if (!XBZRLE
.encoded_buf
) {
1002 error_report("Error allocating encoded_buf");
1006 XBZRLE
.current_buf
= g_try_malloc(TARGET_PAGE_SIZE
);
1007 if (!XBZRLE
.current_buf
) {
1008 error_report("Error allocating current_buf");
1009 g_free(XBZRLE
.encoded_buf
);
1010 XBZRLE
.encoded_buf
= NULL
;
1017 /* iothread lock needed for ram_list.dirty_memory[] */
1018 qemu_mutex_lock_iothread();
1019 qemu_mutex_lock_ramlist();
1021 bytes_transferred
= 0;
1022 reset_ram_globals();
1024 ram_bitmap_pages
= last_ram_offset() >> TARGET_PAGE_BITS
;
1025 migration_bitmap
= bitmap_new(ram_bitmap_pages
);
1026 bitmap_set(migration_bitmap
, 0, ram_bitmap_pages
);
1029 * Count the total number of pages used by ram blocks not including any
1030 * gaps due to alignment or unplugs.
1032 migration_dirty_pages
= ram_bytes_total() >> TARGET_PAGE_BITS
;
1034 memory_global_dirty_log_start();
1035 migration_bitmap_sync();
1036 qemu_mutex_unlock_ramlist();
1037 qemu_mutex_unlock_iothread();
1039 qemu_put_be64(f
, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE
);
1041 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1042 qemu_put_byte(f
, strlen(block
->idstr
));
1043 qemu_put_buffer(f
, (uint8_t *)block
->idstr
, strlen(block
->idstr
));
1044 qemu_put_be64(f
, block
->used_length
);
1049 ram_control_before_iterate(f
, RAM_CONTROL_SETUP
);
1050 ram_control_after_iterate(f
, RAM_CONTROL_SETUP
);
1052 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
1057 static int ram_save_iterate(QEMUFile
*f
, void *opaque
)
1065 if (ram_list
.version
!= last_version
) {
1066 reset_ram_globals();
1069 /* Read version before ram_list.blocks */
1072 ram_control_before_iterate(f
, RAM_CONTROL_ROUND
);
1074 t0
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
);
1076 while ((ret
= qemu_file_rate_limit(f
)) == 0) {
1079 pages
= ram_find_and_save_block(f
, false, &bytes_transferred
);
1080 /* no more pages to sent */
1084 pages_sent
+= pages
;
1085 acct_info
.iterations
++;
1086 check_guest_throttling();
1087 /* we want to check in the 1st loop, just in case it was the 1st time
1088 and we had to sync the dirty bitmap.
1089 qemu_get_clock_ns() is a bit expensive, so we only check each some
1092 if ((i
& 63) == 0) {
1093 uint64_t t1
= (qemu_clock_get_ns(QEMU_CLOCK_REALTIME
) - t0
) / 1000000;
1094 if (t1
> MAX_WAIT
) {
1095 DPRINTF("big wait: %" PRIu64
" milliseconds, %d iterations\n",
1105 * Must occur before EOS (or any QEMUFile operation)
1106 * because of RDMA protocol.
1108 ram_control_after_iterate(f
, RAM_CONTROL_ROUND
);
1110 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
1111 bytes_transferred
+= 8;
1113 ret
= qemu_file_get_error(f
);
1121 /* Called with iothread lock */
1122 static int ram_save_complete(QEMUFile
*f
, void *opaque
)
1126 migration_bitmap_sync();
1128 ram_control_before_iterate(f
, RAM_CONTROL_FINISH
);
1130 /* try transferring iterative blocks of memory */
1132 /* flush all remaining blocks regardless of rate limiting */
1136 pages
= ram_find_and_save_block(f
, true, &bytes_transferred
);
1137 /* no more blocks to sent */
1143 ram_control_after_iterate(f
, RAM_CONTROL_FINISH
);
1147 qemu_put_be64(f
, RAM_SAVE_FLAG_EOS
);
1152 static uint64_t ram_save_pending(QEMUFile
*f
, void *opaque
, uint64_t max_size
)
1154 uint64_t remaining_size
;
1156 remaining_size
= ram_save_remaining() * TARGET_PAGE_SIZE
;
1158 if (remaining_size
< max_size
) {
1159 qemu_mutex_lock_iothread();
1161 migration_bitmap_sync();
1163 qemu_mutex_unlock_iothread();
1164 remaining_size
= ram_save_remaining() * TARGET_PAGE_SIZE
;
1166 return remaining_size
;
1169 static int load_xbzrle(QEMUFile
*f
, ram_addr_t addr
, void *host
)
1171 unsigned int xh_len
;
1174 if (!xbzrle_decoded_buf
) {
1175 xbzrle_decoded_buf
= g_malloc(TARGET_PAGE_SIZE
);
1178 /* extract RLE header */
1179 xh_flags
= qemu_get_byte(f
);
1180 xh_len
= qemu_get_be16(f
);
1182 if (xh_flags
!= ENCODING_FLAG_XBZRLE
) {
1183 error_report("Failed to load XBZRLE page - wrong compression!");
1187 if (xh_len
> TARGET_PAGE_SIZE
) {
1188 error_report("Failed to load XBZRLE page - len overflow!");
1191 /* load data and decode */
1192 qemu_get_buffer(f
, xbzrle_decoded_buf
, xh_len
);
1195 if (xbzrle_decode_buffer(xbzrle_decoded_buf
, xh_len
, host
,
1196 TARGET_PAGE_SIZE
) == -1) {
1197 error_report("Failed to load XBZRLE page - decode error!");
1204 /* Must be called from within a rcu critical section.
1205 * Returns a pointer from within the RCU-protected ram_list.
1207 static inline void *host_from_stream_offset(QEMUFile
*f
,
1211 static RAMBlock
*block
= NULL
;
1215 if (flags
& RAM_SAVE_FLAG_CONTINUE
) {
1216 if (!block
|| block
->max_length
<= offset
) {
1217 error_report("Ack, bad migration stream!");
1221 return memory_region_get_ram_ptr(block
->mr
) + offset
;
1224 len
= qemu_get_byte(f
);
1225 qemu_get_buffer(f
, (uint8_t *)id
, len
);
1228 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1229 if (!strncmp(id
, block
->idstr
, sizeof(id
)) &&
1230 block
->max_length
> offset
) {
1231 return memory_region_get_ram_ptr(block
->mr
) + offset
;
1235 error_report("Can't find block %s!", id
);
1240 * If a page (or a whole RDMA chunk) has been
1241 * determined to be zero, then zap it.
1243 void ram_handle_compressed(void *host
, uint8_t ch
, uint64_t size
)
1245 if (ch
!= 0 || !is_zero_range(host
, size
)) {
1246 memset(host
, ch
, size
);
1250 static void *do_data_decompress(void *opaque
)
1252 while (!quit_decomp_thread
) {
1259 void migrate_decompress_threads_create(void)
1261 int i
, thread_count
;
1263 thread_count
= migrate_decompress_threads();
1264 decompress_threads
= g_new0(QemuThread
, thread_count
);
1265 decomp_param
= g_new0(DecompressParam
, thread_count
);
1266 compressed_data_buf
= g_malloc0(compressBound(TARGET_PAGE_SIZE
));
1267 quit_decomp_thread
= false;
1268 for (i
= 0; i
< thread_count
; i
++) {
1269 qemu_thread_create(decompress_threads
+ i
, "decompress",
1270 do_data_decompress
, decomp_param
+ i
,
1271 QEMU_THREAD_JOINABLE
);
1275 void migrate_decompress_threads_join(void)
1277 int i
, thread_count
;
1279 quit_decomp_thread
= true;
1280 thread_count
= migrate_decompress_threads();
1281 for (i
= 0; i
< thread_count
; i
++) {
1282 qemu_thread_join(decompress_threads
+ i
);
1284 g_free(decompress_threads
);
1285 g_free(decomp_param
);
1286 g_free(compressed_data_buf
);
1287 decompress_threads
= NULL
;
1288 decomp_param
= NULL
;
1289 compressed_data_buf
= NULL
;
1292 static void decompress_data_with_multi_threads(uint8_t *compbuf
,
1293 void *host
, int len
)
1298 static int ram_load(QEMUFile
*f
, void *opaque
, int version_id
)
1300 int flags
= 0, ret
= 0;
1301 static uint64_t seq_iter
;
1306 if (version_id
!= 4) {
1310 /* This RCU critical section can be very long running.
1311 * When RCU reclaims in the code start to become numerous,
1312 * it will be necessary to reduce the granularity of this
1316 while (!ret
&& !(flags
& RAM_SAVE_FLAG_EOS
)) {
1317 ram_addr_t addr
, total_ram_bytes
;
1321 addr
= qemu_get_be64(f
);
1322 flags
= addr
& ~TARGET_PAGE_MASK
;
1323 addr
&= TARGET_PAGE_MASK
;
1325 switch (flags
& ~RAM_SAVE_FLAG_CONTINUE
) {
1326 case RAM_SAVE_FLAG_MEM_SIZE
:
1327 /* Synchronize RAM block list */
1328 total_ram_bytes
= addr
;
1329 while (!ret
&& total_ram_bytes
) {
1335 len
= qemu_get_byte(f
);
1336 qemu_get_buffer(f
, (uint8_t *)id
, len
);
1338 length
= qemu_get_be64(f
);
1340 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1341 if (!strncmp(id
, block
->idstr
, sizeof(id
))) {
1342 if (length
!= block
->used_length
) {
1343 Error
*local_err
= NULL
;
1345 ret
= qemu_ram_resize(block
->offset
, length
, &local_err
);
1347 error_report_err(local_err
);
1355 error_report("Unknown ramblock \"%s\", cannot "
1356 "accept migration", id
);
1360 total_ram_bytes
-= length
;
1363 case RAM_SAVE_FLAG_COMPRESS
:
1364 host
= host_from_stream_offset(f
, addr
, flags
);
1366 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
1370 ch
= qemu_get_byte(f
);
1371 ram_handle_compressed(host
, ch
, TARGET_PAGE_SIZE
);
1373 case RAM_SAVE_FLAG_PAGE
:
1374 host
= host_from_stream_offset(f
, addr
, flags
);
1376 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
1380 qemu_get_buffer(f
, host
, TARGET_PAGE_SIZE
);
1382 case RAM_SAVE_FLAG_COMPRESS_PAGE
:
1383 host
= host_from_stream_offset(f
, addr
, flags
);
1385 error_report("Invalid RAM offset " RAM_ADDR_FMT
, addr
);
1390 len
= qemu_get_be32(f
);
1391 if (len
< 0 || len
> compressBound(TARGET_PAGE_SIZE
)) {
1392 error_report("Invalid compressed data length: %d", len
);
1396 qemu_get_buffer(f
, compressed_data_buf
, len
);
1397 decompress_data_with_multi_threads(compressed_data_buf
, host
, len
);
1399 case RAM_SAVE_FLAG_XBZRLE
:
1400 host
= host_from_stream_offset(f
, addr
, flags
);
1402 error_report("Illegal RAM offset " RAM_ADDR_FMT
, addr
);
1406 if (load_xbzrle(f
, addr
, host
) < 0) {
1407 error_report("Failed to decompress XBZRLE page at "
1408 RAM_ADDR_FMT
, addr
);
1413 case RAM_SAVE_FLAG_EOS
:
1417 if (flags
& RAM_SAVE_FLAG_HOOK
) {
1418 ram_control_load_hook(f
, flags
);
1420 error_report("Unknown combination of migration flags: %#x",
1426 ret
= qemu_file_get_error(f
);
1431 DPRINTF("Completed load of VM with exit code %d seq iteration "
1432 "%" PRIu64
"\n", ret
, seq_iter
);
1436 static SaveVMHandlers savevm_ram_handlers
= {
1437 .save_live_setup
= ram_save_setup
,
1438 .save_live_iterate
= ram_save_iterate
,
1439 .save_live_complete
= ram_save_complete
,
1440 .save_live_pending
= ram_save_pending
,
1441 .load_state
= ram_load
,
1442 .cancel
= ram_migration_cancel
,
1445 void ram_mig_init(void)
1447 qemu_mutex_init(&XBZRLE
.lock
);
1448 register_savevm_live(NULL
, "ram", 0, 4, &savevm_ram_handlers
, NULL
);
1457 int (*init_isa
) (ISABus
*bus
);
1458 int (*init_pci
) (PCIBus
*bus
);
1462 static struct soundhw soundhw
[9];
1463 static int soundhw_count
;
1465 void isa_register_soundhw(const char *name
, const char *descr
,
1466 int (*init_isa
)(ISABus
*bus
))
1468 assert(soundhw_count
< ARRAY_SIZE(soundhw
) - 1);
1469 soundhw
[soundhw_count
].name
= name
;
1470 soundhw
[soundhw_count
].descr
= descr
;
1471 soundhw
[soundhw_count
].isa
= 1;
1472 soundhw
[soundhw_count
].init
.init_isa
= init_isa
;
1476 void pci_register_soundhw(const char *name
, const char *descr
,
1477 int (*init_pci
)(PCIBus
*bus
))
1479 assert(soundhw_count
< ARRAY_SIZE(soundhw
) - 1);
1480 soundhw
[soundhw_count
].name
= name
;
1481 soundhw
[soundhw_count
].descr
= descr
;
1482 soundhw
[soundhw_count
].isa
= 0;
1483 soundhw
[soundhw_count
].init
.init_pci
= init_pci
;
1487 void select_soundhw(const char *optarg
)
1491 if (is_help_option(optarg
)) {
1494 if (soundhw_count
) {
1495 printf("Valid sound card names (comma separated):\n");
1496 for (c
= soundhw
; c
->name
; ++c
) {
1497 printf ("%-11s %s\n", c
->name
, c
->descr
);
1499 printf("\n-soundhw all will enable all of the above\n");
1501 printf("Machine has no user-selectable audio hardware "
1502 "(it may or may not have always-present audio hardware).\n");
1504 exit(!is_help_option(optarg
));
1512 if (!strcmp(optarg
, "all")) {
1513 for (c
= soundhw
; c
->name
; ++c
) {
1522 l
= !e
? strlen(p
) : (size_t) (e
- p
);
1524 for (c
= soundhw
; c
->name
; ++c
) {
1525 if (!strncmp(c
->name
, p
, l
) && !c
->name
[l
]) {
1533 error_report("Unknown sound card name (too big to show)");
1536 error_report("Unknown sound card name `%.*s'",
1541 p
+= l
+ (e
!= NULL
);
1545 goto show_valid_cards
;
1550 void audio_init(void)
1553 ISABus
*isa_bus
= (ISABus
*) object_resolve_path_type("", TYPE_ISA_BUS
, NULL
);
1554 PCIBus
*pci_bus
= (PCIBus
*) object_resolve_path_type("", TYPE_PCI_BUS
, NULL
);
1556 for (c
= soundhw
; c
->name
; ++c
) {
1560 error_report("ISA bus not available for %s", c
->name
);
1563 c
->init
.init_isa(isa_bus
);
1566 error_report("PCI bus not available for %s", c
->name
);
1569 c
->init
.init_pci(pci_bus
);
1575 int qemu_uuid_parse(const char *str
, uint8_t *uuid
)
1579 if (strlen(str
) != 36) {
1583 ret
= sscanf(str
, UUID_FMT
, &uuid
[0], &uuid
[1], &uuid
[2], &uuid
[3],
1584 &uuid
[4], &uuid
[5], &uuid
[6], &uuid
[7], &uuid
[8], &uuid
[9],
1585 &uuid
[10], &uuid
[11], &uuid
[12], &uuid
[13], &uuid
[14],
1594 void do_acpitable_option(const QemuOpts
*opts
)
1599 acpi_table_add(opts
, &err
);
1601 error_report("Wrong acpi table provided: %s",
1602 error_get_pretty(err
));
1609 void do_smbios_option(QemuOpts
*opts
)
1612 smbios_entry_add(opts
);
1616 void cpudef_init(void)
1618 #if defined(cpudef_setup)
1619 cpudef_setup(); /* parse cpu definitions in target config file */
1623 int kvm_available(void)
1632 int xen_available(void)
1642 TargetInfo
*qmp_query_target(Error
**errp
)
1644 TargetInfo
*info
= g_malloc0(sizeof(*info
));
1646 info
->arch
= g_strdup(TARGET_NAME
);
1651 /* Stub function that's gets run on the vcpu when its brought out of the
1652 VM to run inside qemu via async_run_on_cpu()*/
1653 static void mig_sleep_cpu(void *opq
)
1655 qemu_mutex_unlock_iothread();
1657 qemu_mutex_lock_iothread();
1660 /* To reduce the dirty rate explicitly disallow the VCPUs from spending
1661 much time in the VM. The migration thread will try to catchup.
1662 Workload will experience a performance drop.
1664 static void mig_throttle_guest_down(void)
1668 qemu_mutex_lock_iothread();
1670 async_run_on_cpu(cpu
, mig_sleep_cpu
, NULL
);
1672 qemu_mutex_unlock_iothread();
1675 static void check_guest_throttling(void)
1680 if (!mig_throttle_on
) {
1685 t0
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
);
1689 t1
= qemu_clock_get_ns(QEMU_CLOCK_REALTIME
);
1691 /* If it has been more than 40 ms since the last time the guest
1692 * was throttled then do it again.
1694 if (40 < (t1
-t0
)/1000000) {
1695 mig_throttle_guest_down();