qjson: replace QString in JSONLexer with GString
[qemu/ar7.git] / migration / ram.c
blob1eb155aedd57035eef2978e496e1ac0444204431
1 /*
2 * QEMU System Emulator
4 * Copyright (c) 2003-2008 Fabrice Bellard
5 * Copyright (c) 2011-2015 Red Hat Inc
7 * Authors:
8 * Juan Quintela <quintela@redhat.com>
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26 * THE SOFTWARE.
28 #include <stdint.h>
29 #include <zlib.h>
30 #include "qemu/bitops.h"
31 #include "qemu/bitmap.h"
32 #include "qemu/timer.h"
33 #include "qemu/main-loop.h"
34 #include "migration/migration.h"
35 #include "migration/postcopy-ram.h"
36 #include "exec/address-spaces.h"
37 #include "migration/page_cache.h"
38 #include "qemu/error-report.h"
39 #include "trace.h"
40 #include "exec/ram_addr.h"
41 #include "qemu/rcu_queue.h"
43 #ifdef DEBUG_MIGRATION_RAM
44 #define DPRINTF(fmt, ...) \
45 do { fprintf(stdout, "migration_ram: " fmt, ## __VA_ARGS__); } while (0)
46 #else
47 #define DPRINTF(fmt, ...) \
48 do { } while (0)
49 #endif
51 static int dirty_rate_high_cnt;
53 static uint64_t bitmap_sync_count;
55 /***********************************************************/
56 /* ram save/restore */
58 #define RAM_SAVE_FLAG_FULL 0x01 /* Obsolete, not used anymore */
59 #define RAM_SAVE_FLAG_COMPRESS 0x02
60 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
61 #define RAM_SAVE_FLAG_PAGE 0x08
62 #define RAM_SAVE_FLAG_EOS 0x10
63 #define RAM_SAVE_FLAG_CONTINUE 0x20
64 #define RAM_SAVE_FLAG_XBZRLE 0x40
65 /* 0x80 is reserved in migration.h start with 0x100 next */
66 #define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
68 static const uint8_t ZERO_TARGET_PAGE[TARGET_PAGE_SIZE];
70 static inline bool is_zero_range(uint8_t *p, uint64_t size)
72 return buffer_find_nonzero_offset(p, size) == size;
75 /* struct contains XBZRLE cache and a static page
76 used by the compression */
77 static struct {
78 /* buffer used for XBZRLE encoding */
79 uint8_t *encoded_buf;
80 /* buffer for storing page content */
81 uint8_t *current_buf;
82 /* Cache for XBZRLE, Protected by lock. */
83 PageCache *cache;
84 QemuMutex lock;
85 } XBZRLE;
87 /* buffer used for XBZRLE decoding */
88 static uint8_t *xbzrle_decoded_buf;
90 static void XBZRLE_cache_lock(void)
92 if (migrate_use_xbzrle())
93 qemu_mutex_lock(&XBZRLE.lock);
96 static void XBZRLE_cache_unlock(void)
98 if (migrate_use_xbzrle())
99 qemu_mutex_unlock(&XBZRLE.lock);
103 * called from qmp_migrate_set_cache_size in main thread, possibly while
104 * a migration is in progress.
105 * A running migration maybe using the cache and might finish during this
106 * call, hence changes to the cache are protected by XBZRLE.lock().
108 int64_t xbzrle_cache_resize(int64_t new_size)
110 PageCache *new_cache;
111 int64_t ret;
113 if (new_size < TARGET_PAGE_SIZE) {
114 return -1;
117 XBZRLE_cache_lock();
119 if (XBZRLE.cache != NULL) {
120 if (pow2floor(new_size) == migrate_xbzrle_cache_size()) {
121 goto out_new_size;
123 new_cache = cache_init(new_size / TARGET_PAGE_SIZE,
124 TARGET_PAGE_SIZE);
125 if (!new_cache) {
126 error_report("Error creating cache");
127 ret = -1;
128 goto out;
131 cache_fini(XBZRLE.cache);
132 XBZRLE.cache = new_cache;
135 out_new_size:
136 ret = pow2floor(new_size);
137 out:
138 XBZRLE_cache_unlock();
139 return ret;
142 /* accounting for migration statistics */
143 typedef struct AccountingInfo {
144 uint64_t dup_pages;
145 uint64_t skipped_pages;
146 uint64_t norm_pages;
147 uint64_t iterations;
148 uint64_t xbzrle_bytes;
149 uint64_t xbzrle_pages;
150 uint64_t xbzrle_cache_miss;
151 double xbzrle_cache_miss_rate;
152 uint64_t xbzrle_overflows;
153 } AccountingInfo;
155 static AccountingInfo acct_info;
157 static void acct_clear(void)
159 memset(&acct_info, 0, sizeof(acct_info));
162 uint64_t dup_mig_bytes_transferred(void)
164 return acct_info.dup_pages * TARGET_PAGE_SIZE;
167 uint64_t dup_mig_pages_transferred(void)
169 return acct_info.dup_pages;
172 uint64_t skipped_mig_bytes_transferred(void)
174 return acct_info.skipped_pages * TARGET_PAGE_SIZE;
177 uint64_t skipped_mig_pages_transferred(void)
179 return acct_info.skipped_pages;
182 uint64_t norm_mig_bytes_transferred(void)
184 return acct_info.norm_pages * TARGET_PAGE_SIZE;
187 uint64_t norm_mig_pages_transferred(void)
189 return acct_info.norm_pages;
192 uint64_t xbzrle_mig_bytes_transferred(void)
194 return acct_info.xbzrle_bytes;
197 uint64_t xbzrle_mig_pages_transferred(void)
199 return acct_info.xbzrle_pages;
202 uint64_t xbzrle_mig_pages_cache_miss(void)
204 return acct_info.xbzrle_cache_miss;
207 double xbzrle_mig_cache_miss_rate(void)
209 return acct_info.xbzrle_cache_miss_rate;
212 uint64_t xbzrle_mig_pages_overflow(void)
214 return acct_info.xbzrle_overflows;
217 /* This is the last block that we have visited serching for dirty pages
219 static RAMBlock *last_seen_block;
220 /* This is the last block from where we have sent data */
221 static RAMBlock *last_sent_block;
222 static ram_addr_t last_offset;
223 static QemuMutex migration_bitmap_mutex;
224 static uint64_t migration_dirty_pages;
225 static uint32_t last_version;
226 static bool ram_bulk_stage;
228 /* used by the search for pages to send */
229 struct PageSearchStatus {
230 /* Current block being searched */
231 RAMBlock *block;
232 /* Current offset to search from */
233 ram_addr_t offset;
234 /* Set once we wrap around */
235 bool complete_round;
237 typedef struct PageSearchStatus PageSearchStatus;
239 static struct BitmapRcu {
240 struct rcu_head rcu;
241 /* Main migration bitmap */
242 unsigned long *bmap;
243 /* bitmap of pages that haven't been sent even once
244 * only maintained and used in postcopy at the moment
245 * where it's used to send the dirtymap at the start
246 * of the postcopy phase
248 unsigned long *unsentmap;
249 } *migration_bitmap_rcu;
251 struct CompressParam {
252 bool start;
253 bool done;
254 QEMUFile *file;
255 QemuMutex mutex;
256 QemuCond cond;
257 RAMBlock *block;
258 ram_addr_t offset;
260 typedef struct CompressParam CompressParam;
262 struct DecompressParam {
263 bool start;
264 QemuMutex mutex;
265 QemuCond cond;
266 void *des;
267 uint8 *compbuf;
268 int len;
270 typedef struct DecompressParam DecompressParam;
272 static CompressParam *comp_param;
273 static QemuThread *compress_threads;
274 /* comp_done_cond is used to wake up the migration thread when
275 * one of the compression threads has finished the compression.
276 * comp_done_lock is used to co-work with comp_done_cond.
278 static QemuMutex *comp_done_lock;
279 static QemuCond *comp_done_cond;
280 /* The empty QEMUFileOps will be used by file in CompressParam */
281 static const QEMUFileOps empty_ops = { };
283 static bool compression_switch;
284 static bool quit_comp_thread;
285 static bool quit_decomp_thread;
286 static DecompressParam *decomp_param;
287 static QemuThread *decompress_threads;
288 static uint8_t *compressed_data_buf;
290 static int do_compress_ram_page(CompressParam *param);
292 static void *do_data_compress(void *opaque)
294 CompressParam *param = opaque;
296 while (!quit_comp_thread) {
297 qemu_mutex_lock(&param->mutex);
298 /* Re-check the quit_comp_thread in case of
299 * terminate_compression_threads is called just before
300 * qemu_mutex_lock(&param->mutex) and after
301 * while(!quit_comp_thread), re-check it here can make
302 * sure the compression thread terminate as expected.
304 while (!param->start && !quit_comp_thread) {
305 qemu_cond_wait(&param->cond, &param->mutex);
307 if (!quit_comp_thread) {
308 do_compress_ram_page(param);
310 param->start = false;
311 qemu_mutex_unlock(&param->mutex);
313 qemu_mutex_lock(comp_done_lock);
314 param->done = true;
315 qemu_cond_signal(comp_done_cond);
316 qemu_mutex_unlock(comp_done_lock);
319 return NULL;
322 static inline void terminate_compression_threads(void)
324 int idx, thread_count;
326 thread_count = migrate_compress_threads();
327 quit_comp_thread = true;
328 for (idx = 0; idx < thread_count; idx++) {
329 qemu_mutex_lock(&comp_param[idx].mutex);
330 qemu_cond_signal(&comp_param[idx].cond);
331 qemu_mutex_unlock(&comp_param[idx].mutex);
335 void migrate_compress_threads_join(void)
337 int i, thread_count;
339 if (!migrate_use_compression()) {
340 return;
342 terminate_compression_threads();
343 thread_count = migrate_compress_threads();
344 for (i = 0; i < thread_count; i++) {
345 qemu_thread_join(compress_threads + i);
346 qemu_fclose(comp_param[i].file);
347 qemu_mutex_destroy(&comp_param[i].mutex);
348 qemu_cond_destroy(&comp_param[i].cond);
350 qemu_mutex_destroy(comp_done_lock);
351 qemu_cond_destroy(comp_done_cond);
352 g_free(compress_threads);
353 g_free(comp_param);
354 g_free(comp_done_cond);
355 g_free(comp_done_lock);
356 compress_threads = NULL;
357 comp_param = NULL;
358 comp_done_cond = NULL;
359 comp_done_lock = NULL;
362 void migrate_compress_threads_create(void)
364 int i, thread_count;
366 if (!migrate_use_compression()) {
367 return;
369 quit_comp_thread = false;
370 compression_switch = true;
371 thread_count = migrate_compress_threads();
372 compress_threads = g_new0(QemuThread, thread_count);
373 comp_param = g_new0(CompressParam, thread_count);
374 comp_done_cond = g_new0(QemuCond, 1);
375 comp_done_lock = g_new0(QemuMutex, 1);
376 qemu_cond_init(comp_done_cond);
377 qemu_mutex_init(comp_done_lock);
378 for (i = 0; i < thread_count; i++) {
379 /* com_param[i].file is just used as a dummy buffer to save data, set
380 * it's ops to empty.
382 comp_param[i].file = qemu_fopen_ops(NULL, &empty_ops);
383 comp_param[i].done = true;
384 qemu_mutex_init(&comp_param[i].mutex);
385 qemu_cond_init(&comp_param[i].cond);
386 qemu_thread_create(compress_threads + i, "compress",
387 do_data_compress, comp_param + i,
388 QEMU_THREAD_JOINABLE);
393 * save_page_header: Write page header to wire
395 * If this is the 1st block, it also writes the block identification
397 * Returns: Number of bytes written
399 * @f: QEMUFile where to send the data
400 * @block: block that contains the page we want to send
401 * @offset: offset inside the block for the page
402 * in the lower bits, it contains flags
404 static size_t save_page_header(QEMUFile *f, RAMBlock *block, ram_addr_t offset)
406 size_t size, len;
408 qemu_put_be64(f, offset);
409 size = 8;
411 if (!(offset & RAM_SAVE_FLAG_CONTINUE)) {
412 len = strlen(block->idstr);
413 qemu_put_byte(f, len);
414 qemu_put_buffer(f, (uint8_t *)block->idstr, len);
415 size += 1 + len;
417 return size;
420 /* Reduce amount of guest cpu execution to hopefully slow down memory writes.
421 * If guest dirty memory rate is reduced below the rate at which we can
422 * transfer pages to the destination then we should be able to complete
423 * migration. Some workloads dirty memory way too fast and will not effectively
424 * converge, even with auto-converge.
426 static void mig_throttle_guest_down(void)
428 MigrationState *s = migrate_get_current();
429 uint64_t pct_initial =
430 s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL];
431 uint64_t pct_icrement =
432 s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT];
434 /* We have not started throttling yet. Let's start it. */
435 if (!cpu_throttle_active()) {
436 cpu_throttle_set(pct_initial);
437 } else {
438 /* Throttling already on, just increase the rate */
439 cpu_throttle_set(cpu_throttle_get_percentage() + pct_icrement);
443 /* Update the xbzrle cache to reflect a page that's been sent as all 0.
444 * The important thing is that a stale (not-yet-0'd) page be replaced
445 * by the new data.
446 * As a bonus, if the page wasn't in the cache it gets added so that
447 * when a small write is made into the 0'd page it gets XBZRLE sent
449 static void xbzrle_cache_zero_page(ram_addr_t current_addr)
451 if (ram_bulk_stage || !migrate_use_xbzrle()) {
452 return;
455 /* We don't care if this fails to allocate a new cache page
456 * as long as it updated an old one */
457 cache_insert(XBZRLE.cache, current_addr, ZERO_TARGET_PAGE,
458 bitmap_sync_count);
461 #define ENCODING_FLAG_XBZRLE 0x1
464 * save_xbzrle_page: compress and send current page
466 * Returns: 1 means that we wrote the page
467 * 0 means that page is identical to the one already sent
468 * -1 means that xbzrle would be longer than normal
470 * @f: QEMUFile where to send the data
471 * @current_data:
472 * @current_addr:
473 * @block: block that contains the page we want to send
474 * @offset: offset inside the block for the page
475 * @last_stage: if we are at the completion stage
476 * @bytes_transferred: increase it with the number of transferred bytes
478 static int save_xbzrle_page(QEMUFile *f, uint8_t **current_data,
479 ram_addr_t current_addr, RAMBlock *block,
480 ram_addr_t offset, bool last_stage,
481 uint64_t *bytes_transferred)
483 int encoded_len = 0, bytes_xbzrle;
484 uint8_t *prev_cached_page;
486 if (!cache_is_cached(XBZRLE.cache, current_addr, bitmap_sync_count)) {
487 acct_info.xbzrle_cache_miss++;
488 if (!last_stage) {
489 if (cache_insert(XBZRLE.cache, current_addr, *current_data,
490 bitmap_sync_count) == -1) {
491 return -1;
492 } else {
493 /* update *current_data when the page has been
494 inserted into cache */
495 *current_data = get_cached_data(XBZRLE.cache, current_addr);
498 return -1;
501 prev_cached_page = get_cached_data(XBZRLE.cache, current_addr);
503 /* save current buffer into memory */
504 memcpy(XBZRLE.current_buf, *current_data, TARGET_PAGE_SIZE);
506 /* XBZRLE encoding (if there is no overflow) */
507 encoded_len = xbzrle_encode_buffer(prev_cached_page, XBZRLE.current_buf,
508 TARGET_PAGE_SIZE, XBZRLE.encoded_buf,
509 TARGET_PAGE_SIZE);
510 if (encoded_len == 0) {
511 DPRINTF("Skipping unmodified page\n");
512 return 0;
513 } else if (encoded_len == -1) {
514 DPRINTF("Overflow\n");
515 acct_info.xbzrle_overflows++;
516 /* update data in the cache */
517 if (!last_stage) {
518 memcpy(prev_cached_page, *current_data, TARGET_PAGE_SIZE);
519 *current_data = prev_cached_page;
521 return -1;
524 /* we need to update the data in the cache, in order to get the same data */
525 if (!last_stage) {
526 memcpy(prev_cached_page, XBZRLE.current_buf, TARGET_PAGE_SIZE);
529 /* Send XBZRLE based compressed page */
530 bytes_xbzrle = save_page_header(f, block, offset | RAM_SAVE_FLAG_XBZRLE);
531 qemu_put_byte(f, ENCODING_FLAG_XBZRLE);
532 qemu_put_be16(f, encoded_len);
533 qemu_put_buffer(f, XBZRLE.encoded_buf, encoded_len);
534 bytes_xbzrle += encoded_len + 1 + 2;
535 acct_info.xbzrle_pages++;
536 acct_info.xbzrle_bytes += bytes_xbzrle;
537 *bytes_transferred += bytes_xbzrle;
539 return 1;
542 /* Called with rcu_read_lock() to protect migration_bitmap
543 * rb: The RAMBlock to search for dirty pages in
544 * start: Start address (typically so we can continue from previous page)
545 * ram_addr_abs: Pointer into which to store the address of the dirty page
546 * within the global ram_addr space
548 * Returns: byte offset within memory region of the start of a dirty page
550 static inline
551 ram_addr_t migration_bitmap_find_dirty(RAMBlock *rb,
552 ram_addr_t start,
553 ram_addr_t *ram_addr_abs)
555 unsigned long base = rb->offset >> TARGET_PAGE_BITS;
556 unsigned long nr = base + (start >> TARGET_PAGE_BITS);
557 uint64_t rb_size = rb->used_length;
558 unsigned long size = base + (rb_size >> TARGET_PAGE_BITS);
559 unsigned long *bitmap;
561 unsigned long next;
563 bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
564 if (ram_bulk_stage && nr > base) {
565 next = nr + 1;
566 } else {
567 next = find_next_bit(bitmap, size, nr);
570 *ram_addr_abs = next << TARGET_PAGE_BITS;
571 return (next - base) << TARGET_PAGE_BITS;
574 static inline bool migration_bitmap_clear_dirty(ram_addr_t addr)
576 bool ret;
577 int nr = addr >> TARGET_PAGE_BITS;
578 unsigned long *bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
580 ret = test_and_clear_bit(nr, bitmap);
582 if (ret) {
583 migration_dirty_pages--;
585 return ret;
588 static void migration_bitmap_sync_range(ram_addr_t start, ram_addr_t length)
590 unsigned long *bitmap;
591 bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
592 migration_dirty_pages +=
593 cpu_physical_memory_sync_dirty_bitmap(bitmap, start, length);
596 /* Fix me: there are too many global variables used in migration process. */
597 static int64_t start_time;
598 static int64_t bytes_xfer_prev;
599 static int64_t num_dirty_pages_period;
600 static uint64_t xbzrle_cache_miss_prev;
601 static uint64_t iterations_prev;
603 static void migration_bitmap_sync_init(void)
605 start_time = 0;
606 bytes_xfer_prev = 0;
607 num_dirty_pages_period = 0;
608 xbzrle_cache_miss_prev = 0;
609 iterations_prev = 0;
612 /* Called with iothread lock held, to protect ram_list.dirty_memory[] */
613 static void migration_bitmap_sync(void)
615 RAMBlock *block;
616 uint64_t num_dirty_pages_init = migration_dirty_pages;
617 MigrationState *s = migrate_get_current();
618 int64_t end_time;
619 int64_t bytes_xfer_now;
621 bitmap_sync_count++;
623 if (!bytes_xfer_prev) {
624 bytes_xfer_prev = ram_bytes_transferred();
627 if (!start_time) {
628 start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
631 trace_migration_bitmap_sync_start();
632 address_space_sync_dirty_bitmap(&address_space_memory);
634 qemu_mutex_lock(&migration_bitmap_mutex);
635 rcu_read_lock();
636 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
637 migration_bitmap_sync_range(block->offset, block->used_length);
639 rcu_read_unlock();
640 qemu_mutex_unlock(&migration_bitmap_mutex);
642 trace_migration_bitmap_sync_end(migration_dirty_pages
643 - num_dirty_pages_init);
644 num_dirty_pages_period += migration_dirty_pages - num_dirty_pages_init;
645 end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
647 /* more than 1 second = 1000 millisecons */
648 if (end_time > start_time + 1000) {
649 if (migrate_auto_converge()) {
650 /* The following detection logic can be refined later. For now:
651 Check to see if the dirtied bytes is 50% more than the approx.
652 amount of bytes that just got transferred since the last time we
653 were in this routine. If that happens twice, start or increase
654 throttling */
655 bytes_xfer_now = ram_bytes_transferred();
657 if (s->dirty_pages_rate &&
658 (num_dirty_pages_period * TARGET_PAGE_SIZE >
659 (bytes_xfer_now - bytes_xfer_prev)/2) &&
660 (dirty_rate_high_cnt++ >= 2)) {
661 trace_migration_throttle();
662 dirty_rate_high_cnt = 0;
663 mig_throttle_guest_down();
665 bytes_xfer_prev = bytes_xfer_now;
668 if (migrate_use_xbzrle()) {
669 if (iterations_prev != acct_info.iterations) {
670 acct_info.xbzrle_cache_miss_rate =
671 (double)(acct_info.xbzrle_cache_miss -
672 xbzrle_cache_miss_prev) /
673 (acct_info.iterations - iterations_prev);
675 iterations_prev = acct_info.iterations;
676 xbzrle_cache_miss_prev = acct_info.xbzrle_cache_miss;
678 s->dirty_pages_rate = num_dirty_pages_period * 1000
679 / (end_time - start_time);
680 s->dirty_bytes_rate = s->dirty_pages_rate * TARGET_PAGE_SIZE;
681 start_time = end_time;
682 num_dirty_pages_period = 0;
684 s->dirty_sync_count = bitmap_sync_count;
688 * save_zero_page: Send the zero page to the stream
690 * Returns: Number of pages written.
692 * @f: QEMUFile where to send the data
693 * @block: block that contains the page we want to send
694 * @offset: offset inside the block for the page
695 * @p: pointer to the page
696 * @bytes_transferred: increase it with the number of transferred bytes
698 static int save_zero_page(QEMUFile *f, RAMBlock *block, ram_addr_t offset,
699 uint8_t *p, uint64_t *bytes_transferred)
701 int pages = -1;
703 if (is_zero_range(p, TARGET_PAGE_SIZE)) {
704 acct_info.dup_pages++;
705 *bytes_transferred += save_page_header(f, block,
706 offset | RAM_SAVE_FLAG_COMPRESS);
707 qemu_put_byte(f, 0);
708 *bytes_transferred += 1;
709 pages = 1;
712 return pages;
716 * ram_save_page: Send the given page to the stream
718 * Returns: Number of pages written.
720 * @f: QEMUFile where to send the data
721 * @block: block that contains the page we want to send
722 * @offset: offset inside the block for the page
723 * @last_stage: if we are at the completion stage
724 * @bytes_transferred: increase it with the number of transferred bytes
726 static int ram_save_page(QEMUFile *f, RAMBlock* block, ram_addr_t offset,
727 bool last_stage, uint64_t *bytes_transferred)
729 int pages = -1;
730 uint64_t bytes_xmit;
731 ram_addr_t current_addr;
732 uint8_t *p;
733 int ret;
734 bool send_async = true;
736 p = block->host + offset;
738 /* In doubt sent page as normal */
739 bytes_xmit = 0;
740 ret = ram_control_save_page(f, block->offset,
741 offset, TARGET_PAGE_SIZE, &bytes_xmit);
742 if (bytes_xmit) {
743 *bytes_transferred += bytes_xmit;
744 pages = 1;
747 XBZRLE_cache_lock();
749 current_addr = block->offset + offset;
751 if (block == last_sent_block) {
752 offset |= RAM_SAVE_FLAG_CONTINUE;
754 if (ret != RAM_SAVE_CONTROL_NOT_SUPP) {
755 if (ret != RAM_SAVE_CONTROL_DELAYED) {
756 if (bytes_xmit > 0) {
757 acct_info.norm_pages++;
758 } else if (bytes_xmit == 0) {
759 acct_info.dup_pages++;
762 } else {
763 pages = save_zero_page(f, block, offset, p, bytes_transferred);
764 if (pages > 0) {
765 /* Must let xbzrle know, otherwise a previous (now 0'd) cached
766 * page would be stale
768 xbzrle_cache_zero_page(current_addr);
769 } else if (!ram_bulk_stage && migrate_use_xbzrle()) {
770 pages = save_xbzrle_page(f, &p, current_addr, block,
771 offset, last_stage, bytes_transferred);
772 if (!last_stage) {
773 /* Can't send this cached data async, since the cache page
774 * might get updated before it gets to the wire
776 send_async = false;
781 /* XBZRLE overflow or normal page */
782 if (pages == -1) {
783 *bytes_transferred += save_page_header(f, block,
784 offset | RAM_SAVE_FLAG_PAGE);
785 if (send_async) {
786 qemu_put_buffer_async(f, p, TARGET_PAGE_SIZE);
787 } else {
788 qemu_put_buffer(f, p, TARGET_PAGE_SIZE);
790 *bytes_transferred += TARGET_PAGE_SIZE;
791 pages = 1;
792 acct_info.norm_pages++;
795 XBZRLE_cache_unlock();
797 return pages;
800 static int do_compress_ram_page(CompressParam *param)
802 int bytes_sent, blen;
803 uint8_t *p;
804 RAMBlock *block = param->block;
805 ram_addr_t offset = param->offset;
807 p = block->host + (offset & TARGET_PAGE_MASK);
809 bytes_sent = save_page_header(param->file, block, offset |
810 RAM_SAVE_FLAG_COMPRESS_PAGE);
811 blen = qemu_put_compression_data(param->file, p, TARGET_PAGE_SIZE,
812 migrate_compress_level());
813 bytes_sent += blen;
815 return bytes_sent;
818 static inline void start_compression(CompressParam *param)
820 param->done = false;
821 qemu_mutex_lock(&param->mutex);
822 param->start = true;
823 qemu_cond_signal(&param->cond);
824 qemu_mutex_unlock(&param->mutex);
827 static inline void start_decompression(DecompressParam *param)
829 qemu_mutex_lock(&param->mutex);
830 param->start = true;
831 qemu_cond_signal(&param->cond);
832 qemu_mutex_unlock(&param->mutex);
835 static uint64_t bytes_transferred;
837 static void flush_compressed_data(QEMUFile *f)
839 int idx, len, thread_count;
841 if (!migrate_use_compression()) {
842 return;
844 thread_count = migrate_compress_threads();
845 for (idx = 0; idx < thread_count; idx++) {
846 if (!comp_param[idx].done) {
847 qemu_mutex_lock(comp_done_lock);
848 while (!comp_param[idx].done && !quit_comp_thread) {
849 qemu_cond_wait(comp_done_cond, comp_done_lock);
851 qemu_mutex_unlock(comp_done_lock);
853 if (!quit_comp_thread) {
854 len = qemu_put_qemu_file(f, comp_param[idx].file);
855 bytes_transferred += len;
860 static inline void set_compress_params(CompressParam *param, RAMBlock *block,
861 ram_addr_t offset)
863 param->block = block;
864 param->offset = offset;
867 static int compress_page_with_multi_thread(QEMUFile *f, RAMBlock *block,
868 ram_addr_t offset,
869 uint64_t *bytes_transferred)
871 int idx, thread_count, bytes_xmit = -1, pages = -1;
873 thread_count = migrate_compress_threads();
874 qemu_mutex_lock(comp_done_lock);
875 while (true) {
876 for (idx = 0; idx < thread_count; idx++) {
877 if (comp_param[idx].done) {
878 bytes_xmit = qemu_put_qemu_file(f, comp_param[idx].file);
879 set_compress_params(&comp_param[idx], block, offset);
880 start_compression(&comp_param[idx]);
881 pages = 1;
882 acct_info.norm_pages++;
883 *bytes_transferred += bytes_xmit;
884 break;
887 if (pages > 0) {
888 break;
889 } else {
890 qemu_cond_wait(comp_done_cond, comp_done_lock);
893 qemu_mutex_unlock(comp_done_lock);
895 return pages;
899 * ram_save_compressed_page: compress the given page and send it to the stream
901 * Returns: Number of pages written.
903 * @f: QEMUFile where to send the data
904 * @block: block that contains the page we want to send
905 * @offset: offset inside the block for the page
906 * @last_stage: if we are at the completion stage
907 * @bytes_transferred: increase it with the number of transferred bytes
909 static int ram_save_compressed_page(QEMUFile *f, RAMBlock *block,
910 ram_addr_t offset, bool last_stage,
911 uint64_t *bytes_transferred)
913 int pages = -1;
914 uint64_t bytes_xmit;
915 uint8_t *p;
916 int ret;
918 p = block->host + offset;
920 bytes_xmit = 0;
921 ret = ram_control_save_page(f, block->offset,
922 offset, TARGET_PAGE_SIZE, &bytes_xmit);
923 if (bytes_xmit) {
924 *bytes_transferred += bytes_xmit;
925 pages = 1;
927 if (block == last_sent_block) {
928 offset |= RAM_SAVE_FLAG_CONTINUE;
930 if (ret != RAM_SAVE_CONTROL_NOT_SUPP) {
931 if (ret != RAM_SAVE_CONTROL_DELAYED) {
932 if (bytes_xmit > 0) {
933 acct_info.norm_pages++;
934 } else if (bytes_xmit == 0) {
935 acct_info.dup_pages++;
938 } else {
939 /* When starting the process of a new block, the first page of
940 * the block should be sent out before other pages in the same
941 * block, and all the pages in last block should have been sent
942 * out, keeping this order is important, because the 'cont' flag
943 * is used to avoid resending the block name.
945 if (block != last_sent_block) {
946 flush_compressed_data(f);
947 pages = save_zero_page(f, block, offset, p, bytes_transferred);
948 if (pages == -1) {
949 set_compress_params(&comp_param[0], block, offset);
950 /* Use the qemu thread to compress the data to make sure the
951 * first page is sent out before other pages
953 bytes_xmit = do_compress_ram_page(&comp_param[0]);
954 acct_info.norm_pages++;
955 qemu_put_qemu_file(f, comp_param[0].file);
956 *bytes_transferred += bytes_xmit;
957 pages = 1;
959 } else {
960 pages = save_zero_page(f, block, offset, p, bytes_transferred);
961 if (pages == -1) {
962 pages = compress_page_with_multi_thread(f, block, offset,
963 bytes_transferred);
968 return pages;
972 * Find the next dirty page and update any state associated with
973 * the search process.
975 * Returns: True if a page is found
977 * @f: Current migration stream.
978 * @pss: Data about the state of the current dirty page scan.
979 * @*again: Set to false if the search has scanned the whole of RAM
980 * *ram_addr_abs: Pointer into which to store the address of the dirty page
981 * within the global ram_addr space
983 static bool find_dirty_block(QEMUFile *f, PageSearchStatus *pss,
984 bool *again, ram_addr_t *ram_addr_abs)
986 pss->offset = migration_bitmap_find_dirty(pss->block, pss->offset,
987 ram_addr_abs);
988 if (pss->complete_round && pss->block == last_seen_block &&
989 pss->offset >= last_offset) {
991 * We've been once around the RAM and haven't found anything.
992 * Give up.
994 *again = false;
995 return false;
997 if (pss->offset >= pss->block->used_length) {
998 /* Didn't find anything in this RAM Block */
999 pss->offset = 0;
1000 pss->block = QLIST_NEXT_RCU(pss->block, next);
1001 if (!pss->block) {
1002 /* Hit the end of the list */
1003 pss->block = QLIST_FIRST_RCU(&ram_list.blocks);
1004 /* Flag that we've looped */
1005 pss->complete_round = true;
1006 ram_bulk_stage = false;
1007 if (migrate_use_xbzrle()) {
1008 /* If xbzrle is on, stop using the data compression at this
1009 * point. In theory, xbzrle can do better than compression.
1011 flush_compressed_data(f);
1012 compression_switch = false;
1015 /* Didn't find anything this time, but try again on the new block */
1016 *again = true;
1017 return false;
1018 } else {
1019 /* Can go around again, but... */
1020 *again = true;
1021 /* We've found something so probably don't need to */
1022 return true;
1027 * Helper for 'get_queued_page' - gets a page off the queue
1028 * ms: MigrationState in
1029 * *offset: Used to return the offset within the RAMBlock
1030 * ram_addr_abs: global offset in the dirty/sent bitmaps
1032 * Returns: block (or NULL if none available)
1034 static RAMBlock *unqueue_page(MigrationState *ms, ram_addr_t *offset,
1035 ram_addr_t *ram_addr_abs)
1037 RAMBlock *block = NULL;
1039 qemu_mutex_lock(&ms->src_page_req_mutex);
1040 if (!QSIMPLEQ_EMPTY(&ms->src_page_requests)) {
1041 struct MigrationSrcPageRequest *entry =
1042 QSIMPLEQ_FIRST(&ms->src_page_requests);
1043 block = entry->rb;
1044 *offset = entry->offset;
1045 *ram_addr_abs = (entry->offset + entry->rb->offset) &
1046 TARGET_PAGE_MASK;
1048 if (entry->len > TARGET_PAGE_SIZE) {
1049 entry->len -= TARGET_PAGE_SIZE;
1050 entry->offset += TARGET_PAGE_SIZE;
1051 } else {
1052 memory_region_unref(block->mr);
1053 QSIMPLEQ_REMOVE_HEAD(&ms->src_page_requests, next_req);
1054 g_free(entry);
1057 qemu_mutex_unlock(&ms->src_page_req_mutex);
1059 return block;
1063 * Unqueue a page from the queue fed by postcopy page requests; skips pages
1064 * that are already sent (!dirty)
1066 * ms: MigrationState in
1067 * pss: PageSearchStatus structure updated with found block/offset
1068 * ram_addr_abs: global offset in the dirty/sent bitmaps
1070 * Returns: true if a queued page is found
1072 static bool get_queued_page(MigrationState *ms, PageSearchStatus *pss,
1073 ram_addr_t *ram_addr_abs)
1075 RAMBlock *block;
1076 ram_addr_t offset;
1077 bool dirty;
1079 do {
1080 block = unqueue_page(ms, &offset, ram_addr_abs);
1082 * We're sending this page, and since it's postcopy nothing else
1083 * will dirty it, and we must make sure it doesn't get sent again
1084 * even if this queue request was received after the background
1085 * search already sent it.
1087 if (block) {
1088 unsigned long *bitmap;
1089 bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
1090 dirty = test_bit(*ram_addr_abs >> TARGET_PAGE_BITS, bitmap);
1091 if (!dirty) {
1092 trace_get_queued_page_not_dirty(
1093 block->idstr, (uint64_t)offset,
1094 (uint64_t)*ram_addr_abs,
1095 test_bit(*ram_addr_abs >> TARGET_PAGE_BITS,
1096 atomic_rcu_read(&migration_bitmap_rcu)->unsentmap));
1097 } else {
1098 trace_get_queued_page(block->idstr,
1099 (uint64_t)offset,
1100 (uint64_t)*ram_addr_abs);
1104 } while (block && !dirty);
1106 if (block) {
1108 * As soon as we start servicing pages out of order, then we have
1109 * to kill the bulk stage, since the bulk stage assumes
1110 * in (migration_bitmap_find_and_reset_dirty) that every page is
1111 * dirty, that's no longer true.
1113 ram_bulk_stage = false;
1116 * We want the background search to continue from the queued page
1117 * since the guest is likely to want other pages near to the page
1118 * it just requested.
1120 pss->block = block;
1121 pss->offset = offset;
1124 return !!block;
1128 * flush_page_queue: Flush any remaining pages in the ram request queue
1129 * it should be empty at the end anyway, but in error cases there may be
1130 * some left.
1132 * ms: MigrationState
1134 void flush_page_queue(MigrationState *ms)
1136 struct MigrationSrcPageRequest *mspr, *next_mspr;
1137 /* This queue generally should be empty - but in the case of a failed
1138 * migration might have some droppings in.
1140 rcu_read_lock();
1141 QSIMPLEQ_FOREACH_SAFE(mspr, &ms->src_page_requests, next_req, next_mspr) {
1142 memory_region_unref(mspr->rb->mr);
1143 QSIMPLEQ_REMOVE_HEAD(&ms->src_page_requests, next_req);
1144 g_free(mspr);
1146 rcu_read_unlock();
1150 * Queue the pages for transmission, e.g. a request from postcopy destination
1151 * ms: MigrationStatus in which the queue is held
1152 * rbname: The RAMBlock the request is for - may be NULL (to mean reuse last)
1153 * start: Offset from the start of the RAMBlock
1154 * len: Length (in bytes) to send
1155 * Return: 0 on success
1157 int ram_save_queue_pages(MigrationState *ms, const char *rbname,
1158 ram_addr_t start, ram_addr_t len)
1160 RAMBlock *ramblock;
1162 rcu_read_lock();
1163 if (!rbname) {
1164 /* Reuse last RAMBlock */
1165 ramblock = ms->last_req_rb;
1167 if (!ramblock) {
1169 * Shouldn't happen, we can't reuse the last RAMBlock if
1170 * it's the 1st request.
1172 error_report("ram_save_queue_pages no previous block");
1173 goto err;
1175 } else {
1176 ramblock = qemu_ram_block_by_name(rbname);
1178 if (!ramblock) {
1179 /* We shouldn't be asked for a non-existent RAMBlock */
1180 error_report("ram_save_queue_pages no block '%s'", rbname);
1181 goto err;
1183 ms->last_req_rb = ramblock;
1185 trace_ram_save_queue_pages(ramblock->idstr, start, len);
1186 if (start+len > ramblock->used_length) {
1187 error_report("%s request overrun start=" RAM_ADDR_FMT " len="
1188 RAM_ADDR_FMT " blocklen=" RAM_ADDR_FMT,
1189 __func__, start, len, ramblock->used_length);
1190 goto err;
1193 struct MigrationSrcPageRequest *new_entry =
1194 g_malloc0(sizeof(struct MigrationSrcPageRequest));
1195 new_entry->rb = ramblock;
1196 new_entry->offset = start;
1197 new_entry->len = len;
1199 memory_region_ref(ramblock->mr);
1200 qemu_mutex_lock(&ms->src_page_req_mutex);
1201 QSIMPLEQ_INSERT_TAIL(&ms->src_page_requests, new_entry, next_req);
1202 qemu_mutex_unlock(&ms->src_page_req_mutex);
1203 rcu_read_unlock();
1205 return 0;
1207 err:
1208 rcu_read_unlock();
1209 return -1;
1213 * ram_save_target_page: Save one target page
1216 * @f: QEMUFile where to send the data
1217 * @block: pointer to block that contains the page we want to send
1218 * @offset: offset inside the block for the page;
1219 * @last_stage: if we are at the completion stage
1220 * @bytes_transferred: increase it with the number of transferred bytes
1221 * @dirty_ram_abs: Address of the start of the dirty page in ram_addr_t space
1223 * Returns: Number of pages written.
1225 static int ram_save_target_page(MigrationState *ms, QEMUFile *f,
1226 RAMBlock *block, ram_addr_t offset,
1227 bool last_stage,
1228 uint64_t *bytes_transferred,
1229 ram_addr_t dirty_ram_abs)
1231 int res = 0;
1233 /* Check the pages is dirty and if it is send it */
1234 if (migration_bitmap_clear_dirty(dirty_ram_abs)) {
1235 unsigned long *unsentmap;
1236 if (compression_switch && migrate_use_compression()) {
1237 res = ram_save_compressed_page(f, block, offset,
1238 last_stage,
1239 bytes_transferred);
1240 } else {
1241 res = ram_save_page(f, block, offset, last_stage,
1242 bytes_transferred);
1245 if (res < 0) {
1246 return res;
1248 unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap;
1249 if (unsentmap) {
1250 clear_bit(dirty_ram_abs >> TARGET_PAGE_BITS, unsentmap);
1252 last_sent_block = block;
1255 return res;
1259 * ram_save_host_page: Starting at *offset send pages upto the end
1260 * of the current host page. It's valid for the initial
1261 * offset to point into the middle of a host page
1262 * in which case the remainder of the hostpage is sent.
1263 * Only dirty target pages are sent.
1265 * Returns: Number of pages written.
1267 * @f: QEMUFile where to send the data
1268 * @block: pointer to block that contains the page we want to send
1269 * @offset: offset inside the block for the page; updated to last target page
1270 * sent
1271 * @last_stage: if we are at the completion stage
1272 * @bytes_transferred: increase it with the number of transferred bytes
1273 * @dirty_ram_abs: Address of the start of the dirty page in ram_addr_t space
1275 static int ram_save_host_page(MigrationState *ms, QEMUFile *f, RAMBlock *block,
1276 ram_addr_t *offset, bool last_stage,
1277 uint64_t *bytes_transferred,
1278 ram_addr_t dirty_ram_abs)
1280 int tmppages, pages = 0;
1281 do {
1282 tmppages = ram_save_target_page(ms, f, block, *offset, last_stage,
1283 bytes_transferred, dirty_ram_abs);
1284 if (tmppages < 0) {
1285 return tmppages;
1288 pages += tmppages;
1289 *offset += TARGET_PAGE_SIZE;
1290 dirty_ram_abs += TARGET_PAGE_SIZE;
1291 } while (*offset & (qemu_host_page_size - 1));
1293 /* The offset we leave with is the last one we looked at */
1294 *offset -= TARGET_PAGE_SIZE;
1295 return pages;
1299 * ram_find_and_save_block: Finds a dirty page and sends it to f
1301 * Called within an RCU critical section.
1303 * Returns: The number of pages written
1304 * 0 means no dirty pages
1306 * @f: QEMUFile where to send the data
1307 * @last_stage: if we are at the completion stage
1308 * @bytes_transferred: increase it with the number of transferred bytes
1310 * On systems where host-page-size > target-page-size it will send all the
1311 * pages in a host page that are dirty.
1314 static int ram_find_and_save_block(QEMUFile *f, bool last_stage,
1315 uint64_t *bytes_transferred)
1317 PageSearchStatus pss;
1318 MigrationState *ms = migrate_get_current();
1319 int pages = 0;
1320 bool again, found;
1321 ram_addr_t dirty_ram_abs; /* Address of the start of the dirty page in
1322 ram_addr_t space */
1324 pss.block = last_seen_block;
1325 pss.offset = last_offset;
1326 pss.complete_round = false;
1328 if (!pss.block) {
1329 pss.block = QLIST_FIRST_RCU(&ram_list.blocks);
1332 do {
1333 again = true;
1334 found = get_queued_page(ms, &pss, &dirty_ram_abs);
1336 if (!found) {
1337 /* priority queue empty, so just search for something dirty */
1338 found = find_dirty_block(f, &pss, &again, &dirty_ram_abs);
1341 if (found) {
1342 pages = ram_save_host_page(ms, f, pss.block, &pss.offset,
1343 last_stage, bytes_transferred,
1344 dirty_ram_abs);
1346 } while (!pages && again);
1348 last_seen_block = pss.block;
1349 last_offset = pss.offset;
1351 return pages;
1354 void acct_update_position(QEMUFile *f, size_t size, bool zero)
1356 uint64_t pages = size / TARGET_PAGE_SIZE;
1357 if (zero) {
1358 acct_info.dup_pages += pages;
1359 } else {
1360 acct_info.norm_pages += pages;
1361 bytes_transferred += size;
1362 qemu_update_position(f, size);
1366 static ram_addr_t ram_save_remaining(void)
1368 return migration_dirty_pages;
1371 uint64_t ram_bytes_remaining(void)
1373 return ram_save_remaining() * TARGET_PAGE_SIZE;
1376 uint64_t ram_bytes_transferred(void)
1378 return bytes_transferred;
1381 uint64_t ram_bytes_total(void)
1383 RAMBlock *block;
1384 uint64_t total = 0;
1386 rcu_read_lock();
1387 QLIST_FOREACH_RCU(block, &ram_list.blocks, next)
1388 total += block->used_length;
1389 rcu_read_unlock();
1390 return total;
1393 void free_xbzrle_decoded_buf(void)
1395 g_free(xbzrle_decoded_buf);
1396 xbzrle_decoded_buf = NULL;
1399 static void migration_bitmap_free(struct BitmapRcu *bmap)
1401 g_free(bmap->bmap);
1402 g_free(bmap->unsentmap);
1403 g_free(bmap);
1406 static void ram_migration_cleanup(void *opaque)
1408 /* caller have hold iothread lock or is in a bh, so there is
1409 * no writing race against this migration_bitmap
1411 struct BitmapRcu *bitmap = migration_bitmap_rcu;
1412 atomic_rcu_set(&migration_bitmap_rcu, NULL);
1413 if (bitmap) {
1414 memory_global_dirty_log_stop();
1415 call_rcu(bitmap, migration_bitmap_free, rcu);
1418 XBZRLE_cache_lock();
1419 if (XBZRLE.cache) {
1420 cache_fini(XBZRLE.cache);
1421 g_free(XBZRLE.encoded_buf);
1422 g_free(XBZRLE.current_buf);
1423 XBZRLE.cache = NULL;
1424 XBZRLE.encoded_buf = NULL;
1425 XBZRLE.current_buf = NULL;
1427 XBZRLE_cache_unlock();
1430 static void reset_ram_globals(void)
1432 last_seen_block = NULL;
1433 last_sent_block = NULL;
1434 last_offset = 0;
1435 last_version = ram_list.version;
1436 ram_bulk_stage = true;
1439 #define MAX_WAIT 50 /* ms, half buffered_file limit */
1441 void migration_bitmap_extend(ram_addr_t old, ram_addr_t new)
1443 /* called in qemu main thread, so there is
1444 * no writing race against this migration_bitmap
1446 if (migration_bitmap_rcu) {
1447 struct BitmapRcu *old_bitmap = migration_bitmap_rcu, *bitmap;
1448 bitmap = g_new(struct BitmapRcu, 1);
1449 bitmap->bmap = bitmap_new(new);
1451 /* prevent migration_bitmap content from being set bit
1452 * by migration_bitmap_sync_range() at the same time.
1453 * it is safe to migration if migration_bitmap is cleared bit
1454 * at the same time.
1456 qemu_mutex_lock(&migration_bitmap_mutex);
1457 bitmap_copy(bitmap->bmap, old_bitmap->bmap, old);
1458 bitmap_set(bitmap->bmap, old, new - old);
1460 /* We don't have a way to safely extend the sentmap
1461 * with RCU; so mark it as missing, entry to postcopy
1462 * will fail.
1464 bitmap->unsentmap = NULL;
1466 atomic_rcu_set(&migration_bitmap_rcu, bitmap);
1467 qemu_mutex_unlock(&migration_bitmap_mutex);
1468 migration_dirty_pages += new - old;
1469 call_rcu(old_bitmap, migration_bitmap_free, rcu);
1474 * 'expected' is the value you expect the bitmap mostly to be full
1475 * of; it won't bother printing lines that are all this value.
1476 * If 'todump' is null the migration bitmap is dumped.
1478 void ram_debug_dump_bitmap(unsigned long *todump, bool expected)
1480 int64_t ram_pages = last_ram_offset() >> TARGET_PAGE_BITS;
1482 int64_t cur;
1483 int64_t linelen = 128;
1484 char linebuf[129];
1486 if (!todump) {
1487 todump = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
1490 for (cur = 0; cur < ram_pages; cur += linelen) {
1491 int64_t curb;
1492 bool found = false;
1494 * Last line; catch the case where the line length
1495 * is longer than remaining ram
1497 if (cur + linelen > ram_pages) {
1498 linelen = ram_pages - cur;
1500 for (curb = 0; curb < linelen; curb++) {
1501 bool thisbit = test_bit(cur + curb, todump);
1502 linebuf[curb] = thisbit ? '1' : '.';
1503 found = found || (thisbit != expected);
1505 if (found) {
1506 linebuf[curb] = '\0';
1507 fprintf(stderr, "0x%08" PRIx64 " : %s\n", cur, linebuf);
1512 /* **** functions for postcopy ***** */
1515 * Callback from postcopy_each_ram_send_discard for each RAMBlock
1516 * Note: At this point the 'unsentmap' is the processed bitmap combined
1517 * with the dirtymap; so a '1' means it's either dirty or unsent.
1518 * start,length: Indexes into the bitmap for the first bit
1519 * representing the named block and length in target-pages
1521 static int postcopy_send_discard_bm_ram(MigrationState *ms,
1522 PostcopyDiscardState *pds,
1523 unsigned long start,
1524 unsigned long length)
1526 unsigned long end = start + length; /* one after the end */
1527 unsigned long current;
1528 unsigned long *unsentmap;
1530 unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap;
1531 for (current = start; current < end; ) {
1532 unsigned long one = find_next_bit(unsentmap, end, current);
1534 if (one <= end) {
1535 unsigned long zero = find_next_zero_bit(unsentmap, end, one + 1);
1536 unsigned long discard_length;
1538 if (zero >= end) {
1539 discard_length = end - one;
1540 } else {
1541 discard_length = zero - one;
1543 postcopy_discard_send_range(ms, pds, one, discard_length);
1544 current = one + discard_length;
1545 } else {
1546 current = one;
1550 return 0;
1554 * Utility for the outgoing postcopy code.
1555 * Calls postcopy_send_discard_bm_ram for each RAMBlock
1556 * passing it bitmap indexes and name.
1557 * Returns: 0 on success
1558 * (qemu_ram_foreach_block ends up passing unscaled lengths
1559 * which would mean postcopy code would have to deal with target page)
1561 static int postcopy_each_ram_send_discard(MigrationState *ms)
1563 struct RAMBlock *block;
1564 int ret;
1566 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1567 unsigned long first = block->offset >> TARGET_PAGE_BITS;
1568 PostcopyDiscardState *pds = postcopy_discard_send_init(ms,
1569 first,
1570 block->idstr);
1573 * Postcopy sends chunks of bitmap over the wire, but it
1574 * just needs indexes at this point, avoids it having
1575 * target page specific code.
1577 ret = postcopy_send_discard_bm_ram(ms, pds, first,
1578 block->used_length >> TARGET_PAGE_BITS);
1579 postcopy_discard_send_finish(ms, pds);
1580 if (ret) {
1581 return ret;
1585 return 0;
1589 * Helper for postcopy_chunk_hostpages; it's called twice to cleanup
1590 * the two bitmaps, that are similar, but one is inverted.
1592 * We search for runs of target-pages that don't start or end on a
1593 * host page boundary;
1594 * unsent_pass=true: Cleans up partially unsent host pages by searching
1595 * the unsentmap
1596 * unsent_pass=false: Cleans up partially dirty host pages by searching
1597 * the main migration bitmap
1600 static void postcopy_chunk_hostpages_pass(MigrationState *ms, bool unsent_pass,
1601 RAMBlock *block,
1602 PostcopyDiscardState *pds)
1604 unsigned long *bitmap;
1605 unsigned long *unsentmap;
1606 unsigned int host_ratio = qemu_host_page_size / TARGET_PAGE_SIZE;
1607 unsigned long first = block->offset >> TARGET_PAGE_BITS;
1608 unsigned long len = block->used_length >> TARGET_PAGE_BITS;
1609 unsigned long last = first + (len - 1);
1610 unsigned long run_start;
1612 bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
1613 unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap;
1615 if (unsent_pass) {
1616 /* Find a sent page */
1617 run_start = find_next_zero_bit(unsentmap, last + 1, first);
1618 } else {
1619 /* Find a dirty page */
1620 run_start = find_next_bit(bitmap, last + 1, first);
1623 while (run_start <= last) {
1624 bool do_fixup = false;
1625 unsigned long fixup_start_addr;
1626 unsigned long host_offset;
1629 * If the start of this run of pages is in the middle of a host
1630 * page, then we need to fixup this host page.
1632 host_offset = run_start % host_ratio;
1633 if (host_offset) {
1634 do_fixup = true;
1635 run_start -= host_offset;
1636 fixup_start_addr = run_start;
1637 /* For the next pass */
1638 run_start = run_start + host_ratio;
1639 } else {
1640 /* Find the end of this run */
1641 unsigned long run_end;
1642 if (unsent_pass) {
1643 run_end = find_next_bit(unsentmap, last + 1, run_start + 1);
1644 } else {
1645 run_end = find_next_zero_bit(bitmap, last + 1, run_start + 1);
1648 * If the end isn't at the start of a host page, then the
1649 * run doesn't finish at the end of a host page
1650 * and we need to discard.
1652 host_offset = run_end % host_ratio;
1653 if (host_offset) {
1654 do_fixup = true;
1655 fixup_start_addr = run_end - host_offset;
1657 * This host page has gone, the next loop iteration starts
1658 * from after the fixup
1660 run_start = fixup_start_addr + host_ratio;
1661 } else {
1663 * No discards on this iteration, next loop starts from
1664 * next sent/dirty page
1666 run_start = run_end + 1;
1670 if (do_fixup) {
1671 unsigned long page;
1673 /* Tell the destination to discard this page */
1674 if (unsent_pass || !test_bit(fixup_start_addr, unsentmap)) {
1675 /* For the unsent_pass we:
1676 * discard partially sent pages
1677 * For the !unsent_pass (dirty) we:
1678 * discard partially dirty pages that were sent
1679 * (any partially sent pages were already discarded
1680 * by the previous unsent_pass)
1682 postcopy_discard_send_range(ms, pds, fixup_start_addr,
1683 host_ratio);
1686 /* Clean up the bitmap */
1687 for (page = fixup_start_addr;
1688 page < fixup_start_addr + host_ratio; page++) {
1689 /* All pages in this host page are now not sent */
1690 set_bit(page, unsentmap);
1693 * Remark them as dirty, updating the count for any pages
1694 * that weren't previously dirty.
1696 migration_dirty_pages += !test_and_set_bit(page, bitmap);
1700 if (unsent_pass) {
1701 /* Find the next sent page for the next iteration */
1702 run_start = find_next_zero_bit(unsentmap, last + 1,
1703 run_start);
1704 } else {
1705 /* Find the next dirty page for the next iteration */
1706 run_start = find_next_bit(bitmap, last + 1, run_start);
1712 * Utility for the outgoing postcopy code.
1714 * Discard any partially sent host-page size chunks, mark any partially
1715 * dirty host-page size chunks as all dirty.
1717 * Returns: 0 on success
1719 static int postcopy_chunk_hostpages(MigrationState *ms)
1721 struct RAMBlock *block;
1723 if (qemu_host_page_size == TARGET_PAGE_SIZE) {
1724 /* Easy case - TPS==HPS - nothing to be done */
1725 return 0;
1728 /* Easiest way to make sure we don't resume in the middle of a host-page */
1729 last_seen_block = NULL;
1730 last_sent_block = NULL;
1731 last_offset = 0;
1733 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1734 unsigned long first = block->offset >> TARGET_PAGE_BITS;
1736 PostcopyDiscardState *pds =
1737 postcopy_discard_send_init(ms, first, block->idstr);
1739 /* First pass: Discard all partially sent host pages */
1740 postcopy_chunk_hostpages_pass(ms, true, block, pds);
1742 * Second pass: Ensure that all partially dirty host pages are made
1743 * fully dirty.
1745 postcopy_chunk_hostpages_pass(ms, false, block, pds);
1747 postcopy_discard_send_finish(ms, pds);
1748 } /* ram_list loop */
1750 return 0;
1754 * Transmit the set of pages to be discarded after precopy to the target
1755 * these are pages that:
1756 * a) Have been previously transmitted but are now dirty again
1757 * b) Pages that have never been transmitted, this ensures that
1758 * any pages on the destination that have been mapped by background
1759 * tasks get discarded (transparent huge pages is the specific concern)
1760 * Hopefully this is pretty sparse
1762 int ram_postcopy_send_discard_bitmap(MigrationState *ms)
1764 int ret;
1765 unsigned long *bitmap, *unsentmap;
1767 rcu_read_lock();
1769 /* This should be our last sync, the src is now paused */
1770 migration_bitmap_sync();
1772 unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap;
1773 if (!unsentmap) {
1774 /* We don't have a safe way to resize the sentmap, so
1775 * if the bitmap was resized it will be NULL at this
1776 * point.
1778 error_report("migration ram resized during precopy phase");
1779 rcu_read_unlock();
1780 return -EINVAL;
1783 /* Deal with TPS != HPS */
1784 ret = postcopy_chunk_hostpages(ms);
1785 if (ret) {
1786 rcu_read_unlock();
1787 return ret;
1791 * Update the unsentmap to be unsentmap = unsentmap | dirty
1793 bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
1794 bitmap_or(unsentmap, unsentmap, bitmap,
1795 last_ram_offset() >> TARGET_PAGE_BITS);
1798 trace_ram_postcopy_send_discard_bitmap();
1799 #ifdef DEBUG_POSTCOPY
1800 ram_debug_dump_bitmap(unsentmap, true);
1801 #endif
1803 ret = postcopy_each_ram_send_discard(ms);
1804 rcu_read_unlock();
1806 return ret;
1810 * At the start of the postcopy phase of migration, any now-dirty
1811 * precopied pages are discarded.
1813 * start, length describe a byte address range within the RAMBlock
1815 * Returns 0 on success.
1817 int ram_discard_range(MigrationIncomingState *mis,
1818 const char *block_name,
1819 uint64_t start, size_t length)
1821 int ret = -1;
1823 rcu_read_lock();
1824 RAMBlock *rb = qemu_ram_block_by_name(block_name);
1826 if (!rb) {
1827 error_report("ram_discard_range: Failed to find block '%s'",
1828 block_name);
1829 goto err;
1832 uint8_t *host_startaddr = rb->host + start;
1834 if ((uintptr_t)host_startaddr & (qemu_host_page_size - 1)) {
1835 error_report("ram_discard_range: Unaligned start address: %p",
1836 host_startaddr);
1837 goto err;
1840 if ((start + length) <= rb->used_length) {
1841 uint8_t *host_endaddr = host_startaddr + length;
1842 if ((uintptr_t)host_endaddr & (qemu_host_page_size - 1)) {
1843 error_report("ram_discard_range: Unaligned end address: %p",
1844 host_endaddr);
1845 goto err;
1847 ret = postcopy_ram_discard_range(mis, host_startaddr, length);
1848 } else {
1849 error_report("ram_discard_range: Overrun block '%s' (%" PRIu64
1850 "/%zx/" RAM_ADDR_FMT")",
1851 block_name, start, length, rb->used_length);
1854 err:
1855 rcu_read_unlock();
1857 return ret;
1861 /* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
1862 * long-running RCU critical section. When rcu-reclaims in the code
1863 * start to become numerous it will be necessary to reduce the
1864 * granularity of these critical sections.
1867 static int ram_save_setup(QEMUFile *f, void *opaque)
1869 RAMBlock *block;
1870 int64_t ram_bitmap_pages; /* Size of bitmap in pages, including gaps */
1872 dirty_rate_high_cnt = 0;
1873 bitmap_sync_count = 0;
1874 migration_bitmap_sync_init();
1875 qemu_mutex_init(&migration_bitmap_mutex);
1877 if (migrate_use_xbzrle()) {
1878 XBZRLE_cache_lock();
1879 XBZRLE.cache = cache_init(migrate_xbzrle_cache_size() /
1880 TARGET_PAGE_SIZE,
1881 TARGET_PAGE_SIZE);
1882 if (!XBZRLE.cache) {
1883 XBZRLE_cache_unlock();
1884 error_report("Error creating cache");
1885 return -1;
1887 XBZRLE_cache_unlock();
1889 /* We prefer not to abort if there is no memory */
1890 XBZRLE.encoded_buf = g_try_malloc0(TARGET_PAGE_SIZE);
1891 if (!XBZRLE.encoded_buf) {
1892 error_report("Error allocating encoded_buf");
1893 return -1;
1896 XBZRLE.current_buf = g_try_malloc(TARGET_PAGE_SIZE);
1897 if (!XBZRLE.current_buf) {
1898 error_report("Error allocating current_buf");
1899 g_free(XBZRLE.encoded_buf);
1900 XBZRLE.encoded_buf = NULL;
1901 return -1;
1904 acct_clear();
1907 /* iothread lock needed for ram_list.dirty_memory[] */
1908 qemu_mutex_lock_iothread();
1909 qemu_mutex_lock_ramlist();
1910 rcu_read_lock();
1911 bytes_transferred = 0;
1912 reset_ram_globals();
1914 ram_bitmap_pages = last_ram_offset() >> TARGET_PAGE_BITS;
1915 migration_bitmap_rcu = g_new0(struct BitmapRcu, 1);
1916 migration_bitmap_rcu->bmap = bitmap_new(ram_bitmap_pages);
1917 bitmap_set(migration_bitmap_rcu->bmap, 0, ram_bitmap_pages);
1919 if (migrate_postcopy_ram()) {
1920 migration_bitmap_rcu->unsentmap = bitmap_new(ram_bitmap_pages);
1921 bitmap_set(migration_bitmap_rcu->unsentmap, 0, ram_bitmap_pages);
1925 * Count the total number of pages used by ram blocks not including any
1926 * gaps due to alignment or unplugs.
1928 migration_dirty_pages = ram_bytes_total() >> TARGET_PAGE_BITS;
1930 memory_global_dirty_log_start();
1931 migration_bitmap_sync();
1932 qemu_mutex_unlock_ramlist();
1933 qemu_mutex_unlock_iothread();
1935 qemu_put_be64(f, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE);
1937 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1938 qemu_put_byte(f, strlen(block->idstr));
1939 qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr));
1940 qemu_put_be64(f, block->used_length);
1943 rcu_read_unlock();
1945 ram_control_before_iterate(f, RAM_CONTROL_SETUP);
1946 ram_control_after_iterate(f, RAM_CONTROL_SETUP);
1948 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
1950 return 0;
1953 static int ram_save_iterate(QEMUFile *f, void *opaque)
1955 int ret;
1956 int i;
1957 int64_t t0;
1958 int pages_sent = 0;
1960 rcu_read_lock();
1961 if (ram_list.version != last_version) {
1962 reset_ram_globals();
1965 /* Read version before ram_list.blocks */
1966 smp_rmb();
1968 ram_control_before_iterate(f, RAM_CONTROL_ROUND);
1970 t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
1971 i = 0;
1972 while ((ret = qemu_file_rate_limit(f)) == 0) {
1973 int pages;
1975 pages = ram_find_and_save_block(f, false, &bytes_transferred);
1976 /* no more pages to sent */
1977 if (pages == 0) {
1978 break;
1980 pages_sent += pages;
1981 acct_info.iterations++;
1983 /* we want to check in the 1st loop, just in case it was the 1st time
1984 and we had to sync the dirty bitmap.
1985 qemu_get_clock_ns() is a bit expensive, so we only check each some
1986 iterations
1988 if ((i & 63) == 0) {
1989 uint64_t t1 = (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - t0) / 1000000;
1990 if (t1 > MAX_WAIT) {
1991 DPRINTF("big wait: %" PRIu64 " milliseconds, %d iterations\n",
1992 t1, i);
1993 break;
1996 i++;
1998 flush_compressed_data(f);
1999 rcu_read_unlock();
2002 * Must occur before EOS (or any QEMUFile operation)
2003 * because of RDMA protocol.
2005 ram_control_after_iterate(f, RAM_CONTROL_ROUND);
2007 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
2008 bytes_transferred += 8;
2010 ret = qemu_file_get_error(f);
2011 if (ret < 0) {
2012 return ret;
2015 return pages_sent;
2018 /* Called with iothread lock */
2019 static int ram_save_complete(QEMUFile *f, void *opaque)
2021 rcu_read_lock();
2023 if (!migration_in_postcopy(migrate_get_current())) {
2024 migration_bitmap_sync();
2027 ram_control_before_iterate(f, RAM_CONTROL_FINISH);
2029 /* try transferring iterative blocks of memory */
2031 /* flush all remaining blocks regardless of rate limiting */
2032 while (true) {
2033 int pages;
2035 pages = ram_find_and_save_block(f, true, &bytes_transferred);
2036 /* no more blocks to sent */
2037 if (pages == 0) {
2038 break;
2042 flush_compressed_data(f);
2043 ram_control_after_iterate(f, RAM_CONTROL_FINISH);
2045 rcu_read_unlock();
2047 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
2049 return 0;
2052 static void ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size,
2053 uint64_t *non_postcopiable_pending,
2054 uint64_t *postcopiable_pending)
2056 uint64_t remaining_size;
2058 remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE;
2060 if (!migration_in_postcopy(migrate_get_current()) &&
2061 remaining_size < max_size) {
2062 qemu_mutex_lock_iothread();
2063 rcu_read_lock();
2064 migration_bitmap_sync();
2065 rcu_read_unlock();
2066 qemu_mutex_unlock_iothread();
2067 remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE;
2070 /* We can do postcopy, and all the data is postcopiable */
2071 *postcopiable_pending += remaining_size;
2074 static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
2076 unsigned int xh_len;
2077 int xh_flags;
2079 if (!xbzrle_decoded_buf) {
2080 xbzrle_decoded_buf = g_malloc(TARGET_PAGE_SIZE);
2083 /* extract RLE header */
2084 xh_flags = qemu_get_byte(f);
2085 xh_len = qemu_get_be16(f);
2087 if (xh_flags != ENCODING_FLAG_XBZRLE) {
2088 error_report("Failed to load XBZRLE page - wrong compression!");
2089 return -1;
2092 if (xh_len > TARGET_PAGE_SIZE) {
2093 error_report("Failed to load XBZRLE page - len overflow!");
2094 return -1;
2096 /* load data and decode */
2097 qemu_get_buffer(f, xbzrle_decoded_buf, xh_len);
2099 /* decode RLE */
2100 if (xbzrle_decode_buffer(xbzrle_decoded_buf, xh_len, host,
2101 TARGET_PAGE_SIZE) == -1) {
2102 error_report("Failed to load XBZRLE page - decode error!");
2103 return -1;
2106 return 0;
2109 /* Must be called from within a rcu critical section.
2110 * Returns a pointer from within the RCU-protected ram_list.
2113 * Read a RAMBlock ID from the stream f, find the host address of the
2114 * start of that block and add on 'offset'
2116 * f: Stream to read from
2117 * offset: Offset within the block
2118 * flags: Page flags (mostly to see if it's a continuation of previous block)
2120 static inline void *host_from_stream_offset(QEMUFile *f,
2121 ram_addr_t offset,
2122 int flags)
2124 static RAMBlock *block = NULL;
2125 char id[256];
2126 uint8_t len;
2128 if (flags & RAM_SAVE_FLAG_CONTINUE) {
2129 if (!block || block->max_length <= offset) {
2130 error_report("Ack, bad migration stream!");
2131 return NULL;
2134 return block->host + offset;
2137 len = qemu_get_byte(f);
2138 qemu_get_buffer(f, (uint8_t *)id, len);
2139 id[len] = 0;
2141 block = qemu_ram_block_by_name(id);
2142 if (block && block->max_length > offset) {
2143 return block->host + offset;
2146 error_report("Can't find block %s", id);
2147 return NULL;
2151 * If a page (or a whole RDMA chunk) has been
2152 * determined to be zero, then zap it.
2154 void ram_handle_compressed(void *host, uint8_t ch, uint64_t size)
2156 if (ch != 0 || !is_zero_range(host, size)) {
2157 memset(host, ch, size);
2161 static void *do_data_decompress(void *opaque)
2163 DecompressParam *param = opaque;
2164 unsigned long pagesize;
2166 while (!quit_decomp_thread) {
2167 qemu_mutex_lock(&param->mutex);
2168 while (!param->start && !quit_decomp_thread) {
2169 qemu_cond_wait(&param->cond, &param->mutex);
2170 pagesize = TARGET_PAGE_SIZE;
2171 if (!quit_decomp_thread) {
2172 /* uncompress() will return failed in some case, especially
2173 * when the page is dirted when doing the compression, it's
2174 * not a problem because the dirty page will be retransferred
2175 * and uncompress() won't break the data in other pages.
2177 uncompress((Bytef *)param->des, &pagesize,
2178 (const Bytef *)param->compbuf, param->len);
2180 param->start = false;
2182 qemu_mutex_unlock(&param->mutex);
2185 return NULL;
2188 void migrate_decompress_threads_create(void)
2190 int i, thread_count;
2192 thread_count = migrate_decompress_threads();
2193 decompress_threads = g_new0(QemuThread, thread_count);
2194 decomp_param = g_new0(DecompressParam, thread_count);
2195 compressed_data_buf = g_malloc0(compressBound(TARGET_PAGE_SIZE));
2196 quit_decomp_thread = false;
2197 for (i = 0; i < thread_count; i++) {
2198 qemu_mutex_init(&decomp_param[i].mutex);
2199 qemu_cond_init(&decomp_param[i].cond);
2200 decomp_param[i].compbuf = g_malloc0(compressBound(TARGET_PAGE_SIZE));
2201 qemu_thread_create(decompress_threads + i, "decompress",
2202 do_data_decompress, decomp_param + i,
2203 QEMU_THREAD_JOINABLE);
2207 void migrate_decompress_threads_join(void)
2209 int i, thread_count;
2211 quit_decomp_thread = true;
2212 thread_count = migrate_decompress_threads();
2213 for (i = 0; i < thread_count; i++) {
2214 qemu_mutex_lock(&decomp_param[i].mutex);
2215 qemu_cond_signal(&decomp_param[i].cond);
2216 qemu_mutex_unlock(&decomp_param[i].mutex);
2218 for (i = 0; i < thread_count; i++) {
2219 qemu_thread_join(decompress_threads + i);
2220 qemu_mutex_destroy(&decomp_param[i].mutex);
2221 qemu_cond_destroy(&decomp_param[i].cond);
2222 g_free(decomp_param[i].compbuf);
2224 g_free(decompress_threads);
2225 g_free(decomp_param);
2226 g_free(compressed_data_buf);
2227 decompress_threads = NULL;
2228 decomp_param = NULL;
2229 compressed_data_buf = NULL;
2232 static void decompress_data_with_multi_threads(uint8_t *compbuf,
2233 void *host, int len)
2235 int idx, thread_count;
2237 thread_count = migrate_decompress_threads();
2238 while (true) {
2239 for (idx = 0; idx < thread_count; idx++) {
2240 if (!decomp_param[idx].start) {
2241 memcpy(decomp_param[idx].compbuf, compbuf, len);
2242 decomp_param[idx].des = host;
2243 decomp_param[idx].len = len;
2244 start_decompression(&decomp_param[idx]);
2245 break;
2248 if (idx < thread_count) {
2249 break;
2255 * Allocate data structures etc needed by incoming migration with postcopy-ram
2256 * postcopy-ram's similarly names postcopy_ram_incoming_init does the work
2258 int ram_postcopy_incoming_init(MigrationIncomingState *mis)
2260 size_t ram_pages = last_ram_offset() >> TARGET_PAGE_BITS;
2262 return postcopy_ram_incoming_init(mis, ram_pages);
2266 * Called in postcopy mode by ram_load().
2267 * rcu_read_lock is taken prior to this being called.
2269 static int ram_load_postcopy(QEMUFile *f)
2271 int flags = 0, ret = 0;
2272 bool place_needed = false;
2273 bool matching_page_sizes = qemu_host_page_size == TARGET_PAGE_SIZE;
2274 MigrationIncomingState *mis = migration_incoming_get_current();
2275 /* Temporary page that is later 'placed' */
2276 void *postcopy_host_page = postcopy_get_tmp_page(mis);
2277 void *last_host = NULL;
2278 bool all_zero = false;
2280 while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
2281 ram_addr_t addr;
2282 void *host = NULL;
2283 void *page_buffer = NULL;
2284 void *place_source = NULL;
2285 uint8_t ch;
2287 addr = qemu_get_be64(f);
2288 flags = addr & ~TARGET_PAGE_MASK;
2289 addr &= TARGET_PAGE_MASK;
2291 trace_ram_load_postcopy_loop((uint64_t)addr, flags);
2292 place_needed = false;
2293 if (flags & (RAM_SAVE_FLAG_COMPRESS | RAM_SAVE_FLAG_PAGE)) {
2294 host = host_from_stream_offset(f, addr, flags);
2295 if (!host) {
2296 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
2297 ret = -EINVAL;
2298 break;
2300 page_buffer = host;
2302 * Postcopy requires that we place whole host pages atomically.
2303 * To make it atomic, the data is read into a temporary page
2304 * that's moved into place later.
2305 * The migration protocol uses, possibly smaller, target-pages
2306 * however the source ensures it always sends all the components
2307 * of a host page in order.
2309 page_buffer = postcopy_host_page +
2310 ((uintptr_t)host & ~qemu_host_page_mask);
2311 /* If all TP are zero then we can optimise the place */
2312 if (!((uintptr_t)host & ~qemu_host_page_mask)) {
2313 all_zero = true;
2314 } else {
2315 /* not the 1st TP within the HP */
2316 if (host != (last_host + TARGET_PAGE_SIZE)) {
2317 error_report("Non-sequential target page %p/%p\n",
2318 host, last_host);
2319 ret = -EINVAL;
2320 break;
2326 * If it's the last part of a host page then we place the host
2327 * page
2329 place_needed = (((uintptr_t)host + TARGET_PAGE_SIZE) &
2330 ~qemu_host_page_mask) == 0;
2331 place_source = postcopy_host_page;
2333 last_host = host;
2335 switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
2336 case RAM_SAVE_FLAG_COMPRESS:
2337 ch = qemu_get_byte(f);
2338 memset(page_buffer, ch, TARGET_PAGE_SIZE);
2339 if (ch) {
2340 all_zero = false;
2342 break;
2344 case RAM_SAVE_FLAG_PAGE:
2345 all_zero = false;
2346 if (!place_needed || !matching_page_sizes) {
2347 qemu_get_buffer(f, page_buffer, TARGET_PAGE_SIZE);
2348 } else {
2349 /* Avoids the qemu_file copy during postcopy, which is
2350 * going to do a copy later; can only do it when we
2351 * do this read in one go (matching page sizes)
2353 qemu_get_buffer_in_place(f, (uint8_t **)&place_source,
2354 TARGET_PAGE_SIZE);
2356 break;
2357 case RAM_SAVE_FLAG_EOS:
2358 /* normal exit */
2359 break;
2360 default:
2361 error_report("Unknown combination of migration flags: %#x"
2362 " (postcopy mode)", flags);
2363 ret = -EINVAL;
2366 if (place_needed) {
2367 /* This gets called at the last target page in the host page */
2368 if (all_zero) {
2369 ret = postcopy_place_page_zero(mis,
2370 host + TARGET_PAGE_SIZE -
2371 qemu_host_page_size);
2372 } else {
2373 ret = postcopy_place_page(mis, host + TARGET_PAGE_SIZE -
2374 qemu_host_page_size,
2375 place_source);
2378 if (!ret) {
2379 ret = qemu_file_get_error(f);
2383 return ret;
2386 static int ram_load(QEMUFile *f, void *opaque, int version_id)
2388 int flags = 0, ret = 0;
2389 static uint64_t seq_iter;
2390 int len = 0;
2392 * If system is running in postcopy mode, page inserts to host memory must
2393 * be atomic
2395 bool postcopy_running = postcopy_state_get() >= POSTCOPY_INCOMING_LISTENING;
2397 seq_iter++;
2399 if (version_id != 4) {
2400 ret = -EINVAL;
2403 /* This RCU critical section can be very long running.
2404 * When RCU reclaims in the code start to become numerous,
2405 * it will be necessary to reduce the granularity of this
2406 * critical section.
2408 rcu_read_lock();
2410 if (postcopy_running) {
2411 ret = ram_load_postcopy(f);
2414 while (!postcopy_running && !ret && !(flags & RAM_SAVE_FLAG_EOS)) {
2415 ram_addr_t addr, total_ram_bytes;
2416 void *host = NULL;
2417 uint8_t ch;
2419 addr = qemu_get_be64(f);
2420 flags = addr & ~TARGET_PAGE_MASK;
2421 addr &= TARGET_PAGE_MASK;
2423 if (flags & (RAM_SAVE_FLAG_COMPRESS | RAM_SAVE_FLAG_PAGE |
2424 RAM_SAVE_FLAG_COMPRESS_PAGE | RAM_SAVE_FLAG_XBZRLE)) {
2425 host = host_from_stream_offset(f, addr, flags);
2426 if (!host) {
2427 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
2428 ret = -EINVAL;
2429 break;
2433 switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
2434 case RAM_SAVE_FLAG_MEM_SIZE:
2435 /* Synchronize RAM block list */
2436 total_ram_bytes = addr;
2437 while (!ret && total_ram_bytes) {
2438 RAMBlock *block;
2439 char id[256];
2440 ram_addr_t length;
2442 len = qemu_get_byte(f);
2443 qemu_get_buffer(f, (uint8_t *)id, len);
2444 id[len] = 0;
2445 length = qemu_get_be64(f);
2447 block = qemu_ram_block_by_name(id);
2448 if (block) {
2449 if (length != block->used_length) {
2450 Error *local_err = NULL;
2452 ret = qemu_ram_resize(block->offset, length,
2453 &local_err);
2454 if (local_err) {
2455 error_report_err(local_err);
2458 ram_control_load_hook(f, RAM_CONTROL_BLOCK_REG,
2459 block->idstr);
2460 } else {
2461 error_report("Unknown ramblock \"%s\", cannot "
2462 "accept migration", id);
2463 ret = -EINVAL;
2466 total_ram_bytes -= length;
2468 break;
2470 case RAM_SAVE_FLAG_COMPRESS:
2471 ch = qemu_get_byte(f);
2472 ram_handle_compressed(host, ch, TARGET_PAGE_SIZE);
2473 break;
2475 case RAM_SAVE_FLAG_PAGE:
2476 qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
2477 break;
2479 case RAM_SAVE_FLAG_COMPRESS_PAGE:
2480 len = qemu_get_be32(f);
2481 if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) {
2482 error_report("Invalid compressed data length: %d", len);
2483 ret = -EINVAL;
2484 break;
2486 qemu_get_buffer(f, compressed_data_buf, len);
2487 decompress_data_with_multi_threads(compressed_data_buf, host, len);
2488 break;
2490 case RAM_SAVE_FLAG_XBZRLE:
2491 if (load_xbzrle(f, addr, host) < 0) {
2492 error_report("Failed to decompress XBZRLE page at "
2493 RAM_ADDR_FMT, addr);
2494 ret = -EINVAL;
2495 break;
2497 break;
2498 case RAM_SAVE_FLAG_EOS:
2499 /* normal exit */
2500 break;
2501 default:
2502 if (flags & RAM_SAVE_FLAG_HOOK) {
2503 ram_control_load_hook(f, RAM_CONTROL_HOOK, NULL);
2504 } else {
2505 error_report("Unknown combination of migration flags: %#x",
2506 flags);
2507 ret = -EINVAL;
2510 if (!ret) {
2511 ret = qemu_file_get_error(f);
2515 rcu_read_unlock();
2516 DPRINTF("Completed load of VM with exit code %d seq iteration "
2517 "%" PRIu64 "\n", ret, seq_iter);
2518 return ret;
2521 static SaveVMHandlers savevm_ram_handlers = {
2522 .save_live_setup = ram_save_setup,
2523 .save_live_iterate = ram_save_iterate,
2524 .save_live_complete_postcopy = ram_save_complete,
2525 .save_live_complete_precopy = ram_save_complete,
2526 .save_live_pending = ram_save_pending,
2527 .load_state = ram_load,
2528 .cleanup = ram_migration_cleanup,
2531 void ram_mig_init(void)
2533 qemu_mutex_init(&XBZRLE.lock);
2534 register_savevm_live(NULL, "ram", 0, 4, &savevm_ram_handlers, NULL);