?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Force auto-convegence of live migration
[qemu.git] / exec.c
blob80ee2abbc23191790d72abe7d9944dec02f2257a
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
51
52 #include "exec/memory-internal.h"
53
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 CPUState *first_cpu;
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 typedef PhysPageEntry Node[L2_SIZE];
92
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101 };
102
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
110
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
115
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
124
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
127
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
133
134 static MemoryRegion io_mem_watch;
135 #endif
136
137 #if !defined(CONFIG_USER_ONLY)
138
139 static void phys_map_node_reserve(unsigned nodes)
140 {
141 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
142 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
143 16);
144 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
145 next_map.nodes_nb + nodes);
146 next_map.nodes = g_renew(Node, next_map.nodes,
147 next_map.nodes_nb_alloc);
148 }
149 }
150
151 static uint16_t phys_map_node_alloc(void)
152 {
153 unsigned i;
154 uint16_t ret;
155
156 ret = next_map.nodes_nb++;
157 assert(ret != PHYS_MAP_NODE_NIL);
158 assert(ret != next_map.nodes_nb_alloc);
159 for (i = 0; i < L2_SIZE; ++i) {
160 next_map.nodes[ret][i].is_leaf = 0;
161 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
162 }
163 return ret;
164 }
165
166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
167 hwaddr *nb, uint16_t leaf,
168 int level)
169 {
170 PhysPageEntry *p;
171 int i;
172 hwaddr step = (hwaddr)1 << (level * L2_BITS);
173
174 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
175 lp->ptr = phys_map_node_alloc();
176 p = next_map.nodes[lp->ptr];
177 if (level == 0) {
178 for (i = 0; i < L2_SIZE; i++) {
179 p[i].is_leaf = 1;
180 p[i].ptr = PHYS_SECTION_UNASSIGNED;
181 }
182 }
183 } else {
184 p = next_map.nodes[lp->ptr];
185 }
186 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
187
188 while (*nb && lp < &p[L2_SIZE]) {
189 if ((*index & (step - 1)) == 0 && *nb >= step) {
190 lp->is_leaf = true;
191 lp->ptr = leaf;
192 *index += step;
193 *nb -= step;
194 } else {
195 phys_page_set_level(lp, index, nb, leaf, level - 1);
196 }
197 ++lp;
198 }
199 }
200
201 static void phys_page_set(AddressSpaceDispatch *d,
202 hwaddr index, hwaddr nb,
203 uint16_t leaf)
204 {
205 /* Wildly overreserve - it doesn't matter much. */
206 phys_map_node_reserve(3 * P_L2_LEVELS);
207
208 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
209 }
210
211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
212 Node *nodes, MemoryRegionSection *sections)
213 {
214 PhysPageEntry *p;
215 int i;
216
217 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
218 if (lp.ptr == PHYS_MAP_NODE_NIL) {
219 return &sections[PHYS_SECTION_UNASSIGNED];
220 }
221 p = nodes[lp.ptr];
222 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
223 }
224 return &sections[lp.ptr];
225 }
226
227 bool memory_region_is_unassigned(MemoryRegion *mr)
228 {
229 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
230 && mr != &io_mem_watch;
231 }
232
233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
234 hwaddr addr,
235 bool resolve_subpage)
236 {
237 MemoryRegionSection *section;
238 subpage_t *subpage;
239
240 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
241 d->nodes, d->sections);
242 if (resolve_subpage && section->mr->subpage) {
243 subpage = container_of(section->mr, subpage_t, iomem);
244 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
245 }
246 return section;
247 }
248
249 static MemoryRegionSection *
250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
251 hwaddr *plen, bool resolve_subpage)
252 {
253 MemoryRegionSection *section;
254 Int128 diff;
255
256 section = address_space_lookup_region(d, addr, resolve_subpage);
257 /* Compute offset within MemoryRegionSection */
258 addr -= section->offset_within_address_space;
259
260 /* Compute offset within MemoryRegion */
261 *xlat = addr + section->offset_within_region;
262
263 diff = int128_sub(section->mr->size, int128_make64(addr));
264 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
265 return section;
266 }
267
268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
269 hwaddr *xlat, hwaddr *plen,
270 bool is_write)
271 {
272 IOMMUTLBEntry iotlb;
273 MemoryRegionSection *section;
274 MemoryRegion *mr;
275 hwaddr len = *plen;
276
277 for (;;) {
278 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
279 mr = section->mr;
280
281 if (!mr->iommu_ops) {
282 break;
283 }
284
285 iotlb = mr->iommu_ops->translate(mr, addr);
286 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
287 | (addr & iotlb.addr_mask));
288 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
289 if (!(iotlb.perm & (1 << is_write))) {
290 mr = &io_mem_unassigned;
291 break;
292 }
293
294 as = iotlb.target_as;
295 }
296
297 *plen = len;
298 *xlat = addr;
299 return mr;
300 }
301
302 MemoryRegionSection *
303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
304 hwaddr *plen)
305 {
306 MemoryRegionSection *section;
307 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
308
309 assert(!section->mr->iommu_ops);
310 return section;
311 }
312 #endif
313
314 void cpu_exec_init_all(void)
315 {
316 #if !defined(CONFIG_USER_ONLY)
317 qemu_mutex_init(&ram_list.mutex);
318 memory_map_init();
319 io_mem_init();
320 #endif
321 }
322
323 #if !defined(CONFIG_USER_ONLY)
324
325 static int cpu_common_post_load(void *opaque, int version_id)
326 {
327 CPUState *cpu = opaque;
328
329 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
330 version_id is increased. */
331 cpu->interrupt_request &= ~0x01;
332 tlb_flush(cpu->env_ptr, 1);
333
334 return 0;
335 }
336
337 const VMStateDescription vmstate_cpu_common = {
338 .name = "cpu_common",
339 .version_id = 1,
340 .minimum_version_id = 1,
341 .minimum_version_id_old = 1,
342 .post_load = cpu_common_post_load,
343 .fields = (VMStateField []) {
344 VMSTATE_UINT32(halted, CPUState),
345 VMSTATE_UINT32(interrupt_request, CPUState),
346 VMSTATE_END_OF_LIST()
347 }
348 };
349
350 #endif
351
352 CPUState *qemu_get_cpu(int index)
353 {
354 CPUState *cpu = first_cpu;
355
356 while (cpu) {
357 if (cpu->cpu_index == index) {
358 break;
359 }
360 cpu = cpu->next_cpu;
361 }
362
363 return cpu;
364 }
365
366 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
367 {
368 CPUState *cpu;
369
370 cpu = first_cpu;
371 while (cpu) {
372 func(cpu, data);
373 cpu = cpu->next_cpu;
374 }
375 }
376
377 void cpu_exec_init(CPUArchState *env)
378 {
379 CPUState *cpu = ENV_GET_CPU(env);
380 CPUClass *cc = CPU_GET_CLASS(cpu);
381 CPUState **pcpu;
382 int cpu_index;
383
384 #if defined(CONFIG_USER_ONLY)
385 cpu_list_lock();
386 #endif
387 cpu->next_cpu = NULL;
388 pcpu = &first_cpu;
389 cpu_index = 0;
390 while (*pcpu != NULL) {
391 pcpu = &(*pcpu)->next_cpu;
392 cpu_index++;
393 }
394 cpu->cpu_index = cpu_index;
395 cpu->numa_node = 0;
396 QTAILQ_INIT(&env->breakpoints);
397 QTAILQ_INIT(&env->watchpoints);
398 #ifndef CONFIG_USER_ONLY
399 cpu->thread_id = qemu_get_thread_id();
400 #endif
401 *pcpu = cpu;
402 #if defined(CONFIG_USER_ONLY)
403 cpu_list_unlock();
404 #endif
405 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
406 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
407 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
408 cpu_save, cpu_load, env);
409 assert(cc->vmsd == NULL);
410 #endif
411 if (cc->vmsd != NULL) {
412 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
413 }
414 }
415
416 #if defined(TARGET_HAS_ICE)
417 #if defined(CONFIG_USER_ONLY)
418 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
419 {
420 tb_invalidate_phys_page_range(pc, pc + 1, 0);
421 }
422 #else
423 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
424 {
425 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
426 (pc & ~TARGET_PAGE_MASK));
427 }
428 #endif
429 #endif /* TARGET_HAS_ICE */
430
431 #if defined(CONFIG_USER_ONLY)
432 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
433
434 {
435 }
436
437 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
438 int flags, CPUWatchpoint **watchpoint)
439 {
440 return -ENOSYS;
441 }
442 #else
443 /* Add a watchpoint. */
444 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
445 int flags, CPUWatchpoint **watchpoint)
446 {
447 target_ulong len_mask = ~(len - 1);
448 CPUWatchpoint *wp;
449
450 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
451 if ((len & (len - 1)) || (addr & ~len_mask) ||
452 len == 0 || len > TARGET_PAGE_SIZE) {
453 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
454 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
455 return -EINVAL;
456 }
457 wp = g_malloc(sizeof(*wp));
458
459 wp->vaddr = addr;
460 wp->len_mask = len_mask;
461 wp->flags = flags;
462
463 /* keep all GDB-injected watchpoints in front */
464 if (flags & BP_GDB)
465 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
466 else
467 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
468
469 tlb_flush_page(env, addr);
470
471 if (watchpoint)
472 *watchpoint = wp;
473 return 0;
474 }
475
476 /* Remove a specific watchpoint. */
477 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
478 int flags)
479 {
480 target_ulong len_mask = ~(len - 1);
481 CPUWatchpoint *wp;
482
483 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
484 if (addr == wp->vaddr && len_mask == wp->len_mask
485 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
486 cpu_watchpoint_remove_by_ref(env, wp);
487 return 0;
488 }
489 }
490 return -ENOENT;
491 }
492
493 /* Remove a specific watchpoint by reference. */
494 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
495 {
496 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
497
498 tlb_flush_page(env, watchpoint->vaddr);
499
500 g_free(watchpoint);
501 }
502
503 /* Remove all matching watchpoints. */
504 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
505 {
506 CPUWatchpoint *wp, *next;
507
508 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
509 if (wp->flags & mask)
510 cpu_watchpoint_remove_by_ref(env, wp);
511 }
512 }
513 #endif
514
515 /* Add a breakpoint. */
516 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
517 CPUBreakpoint **breakpoint)
518 {
519 #if defined(TARGET_HAS_ICE)
520 CPUBreakpoint *bp;
521
522 bp = g_malloc(sizeof(*bp));
523
524 bp->pc = pc;
525 bp->flags = flags;
526
527 /* keep all GDB-injected breakpoints in front */
528 if (flags & BP_GDB)
529 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
530 else
531 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
532
533 breakpoint_invalidate(env, pc);
534
535 if (breakpoint)
536 *breakpoint = bp;
537 return 0;
538 #else
539 return -ENOSYS;
540 #endif
541 }
542
543 /* Remove a specific breakpoint. */
544 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
545 {
546 #if defined(TARGET_HAS_ICE)
547 CPUBreakpoint *bp;
548
549 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
550 if (bp->pc == pc && bp->flags == flags) {
551 cpu_breakpoint_remove_by_ref(env, bp);
552 return 0;
553 }
554 }
555 return -ENOENT;
556 #else
557 return -ENOSYS;
558 #endif
559 }
560
561 /* Remove a specific breakpoint by reference. */
562 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
563 {
564 #if defined(TARGET_HAS_ICE)
565 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
566
567 breakpoint_invalidate(env, breakpoint->pc);
568
569 g_free(breakpoint);
570 #endif
571 }
572
573 /* Remove all matching breakpoints. */
574 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
575 {
576 #if defined(TARGET_HAS_ICE)
577 CPUBreakpoint *bp, *next;
578
579 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
580 if (bp->flags & mask)
581 cpu_breakpoint_remove_by_ref(env, bp);
582 }
583 #endif
584 }
585
586 /* enable or disable single step mode. EXCP_DEBUG is returned by the
587 CPU loop after each instruction */
588 void cpu_single_step(CPUArchState *env, int enabled)
589 {
590 #if defined(TARGET_HAS_ICE)
591 if (env->singlestep_enabled != enabled) {
592 env->singlestep_enabled = enabled;
593 if (kvm_enabled())
594 kvm_update_guest_debug(env, 0);
595 else {
596 /* must flush all the translated code to avoid inconsistencies */
597 /* XXX: only flush what is necessary */
598 tb_flush(env);
599 }
600 }
601 #endif
602 }
603
604 void cpu_abort(CPUArchState *env, const char *fmt, ...)
605 {
606 CPUState *cpu = ENV_GET_CPU(env);
607 va_list ap;
608 va_list ap2;
609
610 va_start(ap, fmt);
611 va_copy(ap2, ap);
612 fprintf(stderr, "qemu: fatal: ");
613 vfprintf(stderr, fmt, ap);
614 fprintf(stderr, "\n");
615 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
616 if (qemu_log_enabled()) {
617 qemu_log("qemu: fatal: ");
618 qemu_log_vprintf(fmt, ap2);
619 qemu_log("\n");
620 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
621 qemu_log_flush();
622 qemu_log_close();
623 }
624 va_end(ap2);
625 va_end(ap);
626 #if defined(CONFIG_USER_ONLY)
627 {
628 struct sigaction act;
629 sigfillset(&act.sa_mask);
630 act.sa_handler = SIG_DFL;
631 sigaction(SIGABRT, &act, NULL);
632 }
633 #endif
634 abort();
635 }
636
637 CPUArchState *cpu_copy(CPUArchState *env)
638 {
639 CPUArchState *new_env = cpu_init(env->cpu_model_str);
640 #if defined(TARGET_HAS_ICE)
641 CPUBreakpoint *bp;
642 CPUWatchpoint *wp;
643 #endif
644
645 memcpy(new_env, env, sizeof(CPUArchState));
646
647 /* Clone all break/watchpoints.
648 Note: Once we support ptrace with hw-debug register access, make sure
649 BP_CPU break/watchpoints are handled correctly on clone. */
650 QTAILQ_INIT(&env->breakpoints);
651 QTAILQ_INIT(&env->watchpoints);
652 #if defined(TARGET_HAS_ICE)
653 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
654 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
655 }
656 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
657 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
658 wp->flags, NULL);
659 }
660 #endif
661
662 return new_env;
663 }
664
665 #if !defined(CONFIG_USER_ONLY)
666 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
667 uintptr_t length)
668 {
669 uintptr_t start1;
670
671 /* we modify the TLB cache so that the dirty bit will be set again
672 when accessing the range */
673 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
674 /* Check that we don't span multiple blocks - this breaks the
675 address comparisons below. */
676 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
677 != (end - 1) - start) {
678 abort();
679 }
680 cpu_tlb_reset_dirty_all(start1, length);
681
682 }
683
684 /* Note: start and end must be within the same ram block. */
685 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
686 int dirty_flags)
687 {
688 uintptr_t length;
689
690 start &= TARGET_PAGE_MASK;
691 end = TARGET_PAGE_ALIGN(end);
692
693 length = end - start;
694 if (length == 0)
695 return;
696 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
697
698 if (tcg_enabled()) {
699 tlb_reset_dirty_range_all(start, end, length);
700 }
701 }
702
703 static int cpu_physical_memory_set_dirty_tracking(int enable)
704 {
705 int ret = 0;
706 in_migration = enable;
707 return ret;
708 }
709
710 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
711 MemoryRegionSection *section,
712 target_ulong vaddr,
713 hwaddr paddr, hwaddr xlat,
714 int prot,
715 target_ulong *address)
716 {
717 hwaddr iotlb;
718 CPUWatchpoint *wp;
719
720 if (memory_region_is_ram(section->mr)) {
721 /* Normal RAM. */
722 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
723 + xlat;
724 if (!section->readonly) {
725 iotlb |= PHYS_SECTION_NOTDIRTY;
726 } else {
727 iotlb |= PHYS_SECTION_ROM;
728 }
729 } else {
730 iotlb = section - address_space_memory.dispatch->sections;
731 iotlb += xlat;
732 }
733
734 /* Make accesses to pages with watchpoints go via the
735 watchpoint trap routines. */
736 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
737 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
738 /* Avoid trapping reads of pages with a write breakpoint. */
739 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
740 iotlb = PHYS_SECTION_WATCH + paddr;
741 *address |= TLB_MMIO;
742 break;
743 }
744 }
745 }
746
747 return iotlb;
748 }
749 #endif /* defined(CONFIG_USER_ONLY) */
750
751 #if !defined(CONFIG_USER_ONLY)
752
753 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
754 uint16_t section);
755 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
756
757 static uint16_t phys_section_add(MemoryRegionSection *section)
758 {
759 /* The physical section number is ORed with a page-aligned
760 * pointer to produce the iotlb entries. Thus it should
761 * never overflow into the page-aligned value.
762 */
763 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
764
765 if (next_map.sections_nb == next_map.sections_nb_alloc) {
766 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
767 16);
768 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
769 next_map.sections_nb_alloc);
770 }
771 next_map.sections[next_map.sections_nb] = *section;
772 memory_region_ref(section->mr);
773 return next_map.sections_nb++;
774 }
775
776 static void phys_section_destroy(MemoryRegion *mr)
777 {
778 memory_region_unref(mr);
779
780 if (mr->subpage) {
781 subpage_t *subpage = container_of(mr, subpage_t, iomem);
782 memory_region_destroy(&subpage->iomem);
783 g_free(subpage);
784 }
785 }
786
787 static void phys_sections_free(PhysPageMap *map)
788 {
789 while (map->sections_nb > 0) {
790 MemoryRegionSection *section = &map->sections[--map->sections_nb];
791 phys_section_destroy(section->mr);
792 }
793 g_free(map->sections);
794 g_free(map->nodes);
795 g_free(map);
796 }
797
798 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
799 {
800 subpage_t *subpage;
801 hwaddr base = section->offset_within_address_space
802 & TARGET_PAGE_MASK;
803 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
804 next_map.nodes, next_map.sections);
805 MemoryRegionSection subsection = {
806 .offset_within_address_space = base,
807 .size = int128_make64(TARGET_PAGE_SIZE),
808 };
809 hwaddr start, end;
810
811 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
812
813 if (!(existing->mr->subpage)) {
814 subpage = subpage_init(d->as, base);
815 subsection.mr = &subpage->iomem;
816 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
817 phys_section_add(&subsection));
818 } else {
819 subpage = container_of(existing->mr, subpage_t, iomem);
820 }
821 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
822 end = start + int128_get64(section->size) - 1;
823 subpage_register(subpage, start, end, phys_section_add(section));
824 }
825
826
827 static void register_multipage(AddressSpaceDispatch *d,
828 MemoryRegionSection *section)
829 {
830 hwaddr start_addr = section->offset_within_address_space;
831 uint16_t section_index = phys_section_add(section);
832 uint64_t num_pages = int128_get64(int128_rshift(section->size,
833 TARGET_PAGE_BITS));
834
835 assert(num_pages);
836 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
837 }
838
839 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
840 {
841 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
842 AddressSpaceDispatch *d = as->next_dispatch;
843 MemoryRegionSection now = *section, remain = *section;
844 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
845
846 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
847 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
848 - now.offset_within_address_space;
849
850 now.size = int128_min(int128_make64(left), now.size);
851 register_subpage(d, &now);
852 } else {
853 now.size = int128_zero();
854 }
855 while (int128_ne(remain.size, now.size)) {
856 remain.size = int128_sub(remain.size, now.size);
857 remain.offset_within_address_space += int128_get64(now.size);
858 remain.offset_within_region += int128_get64(now.size);
859 now = remain;
860 if (int128_lt(remain.size, page_size)) {
861 register_subpage(d, &now);
862 } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
863 now.size = page_size;
864 register_subpage(d, &now);
865 } else {
866 now.size = int128_and(now.size, int128_neg(page_size));
867 register_multipage(d, &now);
868 }
869 }
870 }
871
872 void qemu_flush_coalesced_mmio_buffer(void)
873 {
874 if (kvm_enabled())
875 kvm_flush_coalesced_mmio_buffer();
876 }
877
878 void qemu_mutex_lock_ramlist(void)
879 {
880 qemu_mutex_lock(&ram_list.mutex);
881 }
882
883 void qemu_mutex_unlock_ramlist(void)
884 {
885 qemu_mutex_unlock(&ram_list.mutex);
886 }
887
888 #if defined(__linux__) && !defined(TARGET_S390X)
889
890 #include <sys/vfs.h>
891
892 #define HUGETLBFS_MAGIC 0x958458f6
893
894 static long gethugepagesize(const char *path)
895 {
896 struct statfs fs;
897 int ret;
898
899 do {
900 ret = statfs(path, &fs);
901 } while (ret != 0 && errno == EINTR);
902
903 if (ret != 0) {
904 perror(path);
905 return 0;
906 }
907
908 if (fs.f_type != HUGETLBFS_MAGIC)
909 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
910
911 return fs.f_bsize;
912 }
913
914 static void *file_ram_alloc(RAMBlock *block,
915 ram_addr_t memory,
916 const char *path)
917 {
918 char *filename;
919 char *sanitized_name;
920 char *c;
921 void *area;
922 int fd;
923 #ifdef MAP_POPULATE
924 int flags;
925 #endif
926 unsigned long hpagesize;
927
928 hpagesize = gethugepagesize(path);
929 if (!hpagesize) {
930 return NULL;
931 }
932
933 if (memory < hpagesize) {
934 return NULL;
935 }
936
937 if (kvm_enabled() && !kvm_has_sync_mmu()) {
938 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
939 return NULL;
940 }
941
942 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
943 sanitized_name = g_strdup(block->mr->name);
944 for (c = sanitized_name; *c != '\0'; c++) {
945 if (*c == '/')
946 *c = '_';
947 }
948
949 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
950 sanitized_name);
951 g_free(sanitized_name);
952
953 fd = mkstemp(filename);
954 if (fd < 0) {
955 perror("unable to create backing store for hugepages");
956 g_free(filename);
957 return NULL;
958 }
959 unlink(filename);
960 g_free(filename);
961
962 memory = (memory+hpagesize-1) & ~(hpagesize-1);
963
964 /*
965 * ftruncate is not supported by hugetlbfs in older
966 * hosts, so don't bother bailing out on errors.
967 * If anything goes wrong with it under other filesystems,
968 * mmap will fail.
969 */
970 if (ftruncate(fd, memory))
971 perror("ftruncate");
972
973 #ifdef MAP_POPULATE
974 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
975 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
976 * to sidestep this quirk.
977 */
978 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
979 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
980 #else
981 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
982 #endif
983 if (area == MAP_FAILED) {
984 perror("file_ram_alloc: can't mmap RAM pages");
985 close(fd);
986 return (NULL);
987 }
988 block->fd = fd;
989 return area;
990 }
991 #endif
992
993 static ram_addr_t find_ram_offset(ram_addr_t size)
994 {
995 RAMBlock *block, *next_block;
996 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
997
998 assert(size != 0); /* it would hand out same offset multiple times */
999
1000 if (QTAILQ_EMPTY(&ram_list.blocks))
1001 return 0;
1002
1003 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1004 ram_addr_t end, next = RAM_ADDR_MAX;
1005
1006 end = block->offset + block->length;
1007
1008 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1009 if (next_block->offset >= end) {
1010 next = MIN(next, next_block->offset);
1011 }
1012 }
1013 if (next - end >= size && next - end < mingap) {
1014 offset = end;
1015 mingap = next - end;
1016 }
1017 }
1018
1019 if (offset == RAM_ADDR_MAX) {
1020 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1021 (uint64_t)size);
1022 abort();
1023 }
1024
1025 return offset;
1026 }
1027
1028 ram_addr_t last_ram_offset(void)
1029 {
1030 RAMBlock *block;
1031 ram_addr_t last = 0;
1032
1033 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1034 last = MAX(last, block->offset + block->length);
1035
1036 return last;
1037 }
1038
1039 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1040 {
1041 int ret;
1042
1043 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1044 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1045 "dump-guest-core", true)) {
1046 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1047 if (ret) {
1048 perror("qemu_madvise");
1049 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1050 "but dump_guest_core=off specified\n");
1051 }
1052 }
1053 }
1054
1055 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1056 {
1057 RAMBlock *new_block, *block;
1058
1059 new_block = NULL;
1060 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1061 if (block->offset == addr) {
1062 new_block = block;
1063 break;
1064 }
1065 }
1066 assert(new_block);
1067 assert(!new_block->idstr[0]);
1068
1069 if (dev) {
1070 char *id = qdev_get_dev_path(dev);
1071 if (id) {
1072 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1073 g_free(id);
1074 }
1075 }
1076 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1077
1078 /* This assumes the iothread lock is taken here too. */
1079 qemu_mutex_lock_ramlist();
1080 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1081 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1082 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1083 new_block->idstr);
1084 abort();
1085 }
1086 }
1087 qemu_mutex_unlock_ramlist();
1088 }
1089
1090 static int memory_try_enable_merging(void *addr, size_t len)
1091 {
1092 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1093 /* disabled by the user */
1094 return 0;
1095 }
1096
1097 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1098 }
1099
1100 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1101 MemoryRegion *mr)
1102 {
1103 RAMBlock *block, *new_block;
1104
1105 size = TARGET_PAGE_ALIGN(size);
1106 new_block = g_malloc0(sizeof(*new_block));
1107
1108 /* This assumes the iothread lock is taken here too. */
1109 qemu_mutex_lock_ramlist();
1110 new_block->mr = mr;
1111 new_block->offset = find_ram_offset(size);
1112 if (host) {
1113 new_block->host = host;
1114 new_block->flags |= RAM_PREALLOC_MASK;
1115 } else {
1116 if (mem_path) {
1117 #if defined (__linux__) && !defined(TARGET_S390X)
1118 new_block->host = file_ram_alloc(new_block, size, mem_path);
1119 if (!new_block->host) {
1120 new_block->host = qemu_anon_ram_alloc(size);
1121 memory_try_enable_merging(new_block->host, size);
1122 }
1123 #else
1124 fprintf(stderr, "-mem-path option unsupported\n");
1125 exit(1);
1126 #endif
1127 } else {
1128 if (xen_enabled()) {
1129 xen_ram_alloc(new_block->offset, size, mr);
1130 } else if (kvm_enabled()) {
1131 /* some s390/kvm configurations have special constraints */
1132 new_block->host = kvm_ram_alloc(size);
1133 } else {
1134 new_block->host = qemu_anon_ram_alloc(size);
1135 }
1136 memory_try_enable_merging(new_block->host, size);
1137 }
1138 }
1139 new_block->length = size;
1140
1141 /* Keep the list sorted from biggest to smallest block. */
1142 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1143 if (block->length < new_block->length) {
1144 break;
1145 }
1146 }
1147 if (block) {
1148 QTAILQ_INSERT_BEFORE(block, new_block, next);
1149 } else {
1150 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1151 }
1152 ram_list.mru_block = NULL;
1153
1154 ram_list.version++;
1155 qemu_mutex_unlock_ramlist();
1156
1157 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1158 last_ram_offset() >> TARGET_PAGE_BITS);
1159 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1160 0, size >> TARGET_PAGE_BITS);
1161 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1162
1163 qemu_ram_setup_dump(new_block->host, size);
1164 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1165
1166 if (kvm_enabled())
1167 kvm_setup_guest_memory(new_block->host, size);
1168
1169 return new_block->offset;
1170 }
1171
1172 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1173 {
1174 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1175 }
1176
1177 void qemu_ram_free_from_ptr(ram_addr_t addr)
1178 {
1179 RAMBlock *block;
1180
1181 /* This assumes the iothread lock is taken here too. */
1182 qemu_mutex_lock_ramlist();
1183 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1184 if (addr == block->offset) {
1185 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1186 ram_list.mru_block = NULL;
1187 ram_list.version++;
1188 g_free(block);
1189 break;
1190 }
1191 }
1192 qemu_mutex_unlock_ramlist();
1193 }
1194
1195 void qemu_ram_free(ram_addr_t addr)
1196 {
1197 RAMBlock *block;
1198
1199 /* This assumes the iothread lock is taken here too. */
1200 qemu_mutex_lock_ramlist();
1201 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1202 if (addr == block->offset) {
1203 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1204 ram_list.mru_block = NULL;
1205 ram_list.version++;
1206 if (block->flags & RAM_PREALLOC_MASK) {
1207 ;
1208 } else if (mem_path) {
1209 #if defined (__linux__) && !defined(TARGET_S390X)
1210 if (block->fd) {
1211 munmap(block->host, block->length);
1212 close(block->fd);
1213 } else {
1214 qemu_anon_ram_free(block->host, block->length);
1215 }
1216 #else
1217 abort();
1218 #endif
1219 } else {
1220 if (xen_enabled()) {
1221 xen_invalidate_map_cache_entry(block->host);
1222 } else {
1223 qemu_anon_ram_free(block->host, block->length);
1224 }
1225 }
1226 g_free(block);
1227 break;
1228 }
1229 }
1230 qemu_mutex_unlock_ramlist();
1231
1232 }
1233
1234 #ifndef _WIN32
1235 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1236 {
1237 RAMBlock *block;
1238 ram_addr_t offset;
1239 int flags;
1240 void *area, *vaddr;
1241
1242 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1243 offset = addr - block->offset;
1244 if (offset < block->length) {
1245 vaddr = block->host + offset;
1246 if (block->flags & RAM_PREALLOC_MASK) {
1247 ;
1248 } else {
1249 flags = MAP_FIXED;
1250 munmap(vaddr, length);
1251 if (mem_path) {
1252 #if defined(__linux__) && !defined(TARGET_S390X)
1253 if (block->fd) {
1254 #ifdef MAP_POPULATE
1255 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1256 MAP_PRIVATE;
1257 #else
1258 flags |= MAP_PRIVATE;
1259 #endif
1260 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1261 flags, block->fd, offset);
1262 } else {
1263 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1264 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1265 flags, -1, 0);
1266 }
1267 #else
1268 abort();
1269 #endif
1270 } else {
1271 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1272 flags |= MAP_SHARED | MAP_ANONYMOUS;
1273 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1274 flags, -1, 0);
1275 #else
1276 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1277 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1278 flags, -1, 0);
1279 #endif
1280 }
1281 if (area != vaddr) {
1282 fprintf(stderr, "Could not remap addr: "
1283 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1284 length, addr);
1285 exit(1);
1286 }
1287 memory_try_enable_merging(vaddr, length);
1288 qemu_ram_setup_dump(vaddr, length);
1289 }
1290 return;
1291 }
1292 }
1293 }
1294 #endif /* !_WIN32 */
1295
1296 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1297 {
1298 RAMBlock *block;
1299
1300 /* The list is protected by the iothread lock here. */
1301 block = ram_list.mru_block;
1302 if (block && addr - block->offset < block->length) {
1303 goto found;
1304 }
1305 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1306 if (addr - block->offset < block->length) {
1307 goto found;
1308 }
1309 }
1310
1311 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1312 abort();
1313
1314 found:
1315 ram_list.mru_block = block;
1316 return block;
1317 }
1318
1319 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1320 With the exception of the softmmu code in this file, this should
1321 only be used for local memory (e.g. video ram) that the device owns,
1322 and knows it isn't going to access beyond the end of the block.
1323
1324 It should not be used for general purpose DMA.
1325 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1326 */
1327 void *qemu_get_ram_ptr(ram_addr_t addr)
1328 {
1329 RAMBlock *block = qemu_get_ram_block(addr);
1330
1331 if (xen_enabled()) {
1332 /* We need to check if the requested address is in the RAM
1333 * because we don't want to map the entire memory in QEMU.
1334 * In that case just map until the end of the page.
1335 */
1336 if (block->offset == 0) {
1337 return xen_map_cache(addr, 0, 0);
1338 } else if (block->host == NULL) {
1339 block->host =
1340 xen_map_cache(block->offset, block->length, 1);
1341 }
1342 }
1343 return block->host + (addr - block->offset);
1344 }
1345
1346 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1347 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1348 *
1349 * ??? Is this still necessary?
1350 */
1351 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1352 {
1353 RAMBlock *block;
1354
1355 /* The list is protected by the iothread lock here. */
1356 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1357 if (addr - block->offset < block->length) {
1358 if (xen_enabled()) {
1359 /* We need to check if the requested address is in the RAM
1360 * because we don't want to map the entire memory in QEMU.
1361 * In that case just map until the end of the page.
1362 */
1363 if (block->offset == 0) {
1364 return xen_map_cache(addr, 0, 0);
1365 } else if (block->host == NULL) {
1366 block->host =
1367 xen_map_cache(block->offset, block->length, 1);
1368 }
1369 }
1370 return block->host + (addr - block->offset);
1371 }
1372 }
1373
1374 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1375 abort();
1376
1377 return NULL;
1378 }
1379
1380 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1381 * but takes a size argument */
1382 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1383 {
1384 if (*size == 0) {
1385 return NULL;
1386 }
1387 if (xen_enabled()) {
1388 return xen_map_cache(addr, *size, 1);
1389 } else {
1390 RAMBlock *block;
1391
1392 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1393 if (addr - block->offset < block->length) {
1394 if (addr - block->offset + *size > block->length)
1395 *size = block->length - addr + block->offset;
1396 return block->host + (addr - block->offset);
1397 }
1398 }
1399
1400 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1401 abort();
1402 }
1403 }
1404
1405 /* Some of the softmmu routines need to translate from a host pointer
1406 (typically a TLB entry) back to a ram offset. */
1407 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1408 {
1409 RAMBlock *block;
1410 uint8_t *host = ptr;
1411
1412 if (xen_enabled()) {
1413 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1414 return qemu_get_ram_block(*ram_addr)->mr;
1415 }
1416
1417 block = ram_list.mru_block;
1418 if (block && block->host && host - block->host < block->length) {
1419 goto found;
1420 }
1421
1422 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1423 /* This case append when the block is not mapped. */
1424 if (block->host == NULL) {
1425 continue;
1426 }
1427 if (host - block->host < block->length) {
1428 goto found;
1429 }
1430 }
1431
1432 return NULL;
1433
1434 found:
1435 *ram_addr = block->offset + (host - block->host);
1436 return block->mr;
1437 }
1438
1439 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1440 uint64_t val, unsigned size)
1441 {
1442 int dirty_flags;
1443 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1444 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1445 tb_invalidate_phys_page_fast(ram_addr, size);
1446 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1447 }
1448 switch (size) {
1449 case 1:
1450 stb_p(qemu_get_ram_ptr(ram_addr), val);
1451 break;
1452 case 2:
1453 stw_p(qemu_get_ram_ptr(ram_addr), val);
1454 break;
1455 case 4:
1456 stl_p(qemu_get_ram_ptr(ram_addr), val);
1457 break;
1458 default:
1459 abort();
1460 }
1461 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1462 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1463 /* we remove the notdirty callback only if the code has been
1464 flushed */
1465 if (dirty_flags == 0xff) {
1466 CPUArchState *env = current_cpu->env_ptr;
1467 tlb_set_dirty(env, env->mem_io_vaddr);
1468 }
1469 }
1470
1471 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1472 unsigned size, bool is_write)
1473 {
1474 return is_write;
1475 }
1476
1477 static const MemoryRegionOps notdirty_mem_ops = {
1478 .write = notdirty_mem_write,
1479 .valid.accepts = notdirty_mem_accepts,
1480 .endianness = DEVICE_NATIVE_ENDIAN,
1481 };
1482
1483 /* Generate a debug exception if a watchpoint has been hit. */
1484 static void check_watchpoint(int offset, int len_mask, int flags)
1485 {
1486 CPUArchState *env = current_cpu->env_ptr;
1487 target_ulong pc, cs_base;
1488 target_ulong vaddr;
1489 CPUWatchpoint *wp;
1490 int cpu_flags;
1491
1492 if (env->watchpoint_hit) {
1493 /* We re-entered the check after replacing the TB. Now raise
1494 * the debug interrupt so that is will trigger after the
1495 * current instruction. */
1496 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1497 return;
1498 }
1499 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1500 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1501 if ((vaddr == (wp->vaddr & len_mask) ||
1502 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1503 wp->flags |= BP_WATCHPOINT_HIT;
1504 if (!env->watchpoint_hit) {
1505 env->watchpoint_hit = wp;
1506 tb_check_watchpoint(env);
1507 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1508 env->exception_index = EXCP_DEBUG;
1509 cpu_loop_exit(env);
1510 } else {
1511 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1512 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1513 cpu_resume_from_signal(env, NULL);
1514 }
1515 }
1516 } else {
1517 wp->flags &= ~BP_WATCHPOINT_HIT;
1518 }
1519 }
1520 }
1521
1522 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1523 so these check for a hit then pass through to the normal out-of-line
1524 phys routines. */
1525 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1526 unsigned size)
1527 {
1528 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1529 switch (size) {
1530 case 1: return ldub_phys(addr);
1531 case 2: return lduw_phys(addr);
1532 case 4: return ldl_phys(addr);
1533 default: abort();
1534 }
1535 }
1536
1537 static void watch_mem_write(void *opaque, hwaddr addr,
1538 uint64_t val, unsigned size)
1539 {
1540 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1541 switch (size) {
1542 case 1:
1543 stb_phys(addr, val);
1544 break;
1545 case 2:
1546 stw_phys(addr, val);
1547 break;
1548 case 4:
1549 stl_phys(addr, val);
1550 break;
1551 default: abort();
1552 }
1553 }
1554
1555 static const MemoryRegionOps watch_mem_ops = {
1556 .read = watch_mem_read,
1557 .write = watch_mem_write,
1558 .endianness = DEVICE_NATIVE_ENDIAN,
1559 };
1560
1561 static uint64_t subpage_read(void *opaque, hwaddr addr,
1562 unsigned len)
1563 {
1564 subpage_t *subpage = opaque;
1565 uint8_t buf[4];
1566
1567 #if defined(DEBUG_SUBPAGE)
1568 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1569 subpage, len, addr);
1570 #endif
1571 address_space_read(subpage->as, addr + subpage->base, buf, len);
1572 switch (len) {
1573 case 1:
1574 return ldub_p(buf);
1575 case 2:
1576 return lduw_p(buf);
1577 case 4:
1578 return ldl_p(buf);
1579 default:
1580 abort();
1581 }
1582 }
1583
1584 static void subpage_write(void *opaque, hwaddr addr,
1585 uint64_t value, unsigned len)
1586 {
1587 subpage_t *subpage = opaque;
1588 uint8_t buf[4];
1589
1590 #if defined(DEBUG_SUBPAGE)
1591 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1592 " value %"PRIx64"\n",
1593 __func__, subpage, len, addr, value);
1594 #endif
1595 switch (len) {
1596 case 1:
1597 stb_p(buf, value);
1598 break;
1599 case 2:
1600 stw_p(buf, value);
1601 break;
1602 case 4:
1603 stl_p(buf, value);
1604 break;
1605 default:
1606 abort();
1607 }
1608 address_space_write(subpage->as, addr + subpage->base, buf, len);
1609 }
1610
1611 static bool subpage_accepts(void *opaque, hwaddr addr,
1612 unsigned size, bool is_write)
1613 {
1614 subpage_t *subpage = opaque;
1615 #if defined(DEBUG_SUBPAGE)
1616 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1617 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1618 #endif
1619
1620 return address_space_access_valid(subpage->as, addr + subpage->base,
1621 size, is_write);
1622 }
1623
1624 static const MemoryRegionOps subpage_ops = {
1625 .read = subpage_read,
1626 .write = subpage_write,
1627 .valid.accepts = subpage_accepts,
1628 .endianness = DEVICE_NATIVE_ENDIAN,
1629 };
1630
1631 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1632 uint16_t section)
1633 {
1634 int idx, eidx;
1635
1636 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1637 return -1;
1638 idx = SUBPAGE_IDX(start);
1639 eidx = SUBPAGE_IDX(end);
1640 #if defined(DEBUG_SUBPAGE)
1641 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1642 mmio, start, end, idx, eidx, memory);
1643 #endif
1644 for (; idx <= eidx; idx++) {
1645 mmio->sub_section[idx] = section;
1646 }
1647
1648 return 0;
1649 }
1650
1651 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1652 {
1653 subpage_t *mmio;
1654
1655 mmio = g_malloc0(sizeof(subpage_t));
1656
1657 mmio->as = as;
1658 mmio->base = base;
1659 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1660 "subpage", TARGET_PAGE_SIZE);
1661 mmio->iomem.subpage = true;
1662 #if defined(DEBUG_SUBPAGE)
1663 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1664 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1665 #endif
1666 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1667
1668 return mmio;
1669 }
1670
1671 static uint16_t dummy_section(MemoryRegion *mr)
1672 {
1673 MemoryRegionSection section = {
1674 .mr = mr,
1675 .offset_within_address_space = 0,
1676 .offset_within_region = 0,
1677 .size = int128_2_64(),
1678 };
1679
1680 return phys_section_add(&section);
1681 }
1682
1683 MemoryRegion *iotlb_to_region(hwaddr index)
1684 {
1685 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1686 }
1687
1688 static void io_mem_init(void)
1689 {
1690 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1691 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1692 "unassigned", UINT64_MAX);
1693 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1694 "notdirty", UINT64_MAX);
1695 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1696 "watch", UINT64_MAX);
1697 }
1698
1699 static void mem_begin(MemoryListener *listener)
1700 {
1701 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1702 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1703
1704 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1705 d->as = as;
1706 as->next_dispatch = d;
1707 }
1708
1709 static void mem_commit(MemoryListener *listener)
1710 {
1711 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1712 AddressSpaceDispatch *cur = as->dispatch;
1713 AddressSpaceDispatch *next = as->next_dispatch;
1714
1715 next->nodes = next_map.nodes;
1716 next->sections = next_map.sections;
1717
1718 as->dispatch = next;
1719 g_free(cur);
1720 }
1721
1722 static void core_begin(MemoryListener *listener)
1723 {
1724 uint16_t n;
1725
1726 prev_map = g_new(PhysPageMap, 1);
1727 *prev_map = next_map;
1728
1729 memset(&next_map, 0, sizeof(next_map));
1730 n = dummy_section(&io_mem_unassigned);
1731 assert(n == PHYS_SECTION_UNASSIGNED);
1732 n = dummy_section(&io_mem_notdirty);
1733 assert(n == PHYS_SECTION_NOTDIRTY);
1734 n = dummy_section(&io_mem_rom);
1735 assert(n == PHYS_SECTION_ROM);
1736 n = dummy_section(&io_mem_watch);
1737 assert(n == PHYS_SECTION_WATCH);
1738 }
1739
1740 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1741 * All AddressSpaceDispatch instances have switched to the next map.
1742 */
1743 static void core_commit(MemoryListener *listener)
1744 {
1745 phys_sections_free(prev_map);
1746 }
1747
1748 static void tcg_commit(MemoryListener *listener)
1749 {
1750 CPUState *cpu;
1751
1752 /* since each CPU stores ram addresses in its TLB cache, we must
1753 reset the modified entries */
1754 /* XXX: slow ! */
1755 for (cpu = first_cpu; cpu != NULL; cpu = cpu->next_cpu) {
1756 CPUArchState *env = cpu->env_ptr;
1757
1758 tlb_flush(env, 1);
1759 }
1760 }
1761
1762 static void core_log_global_start(MemoryListener *listener)
1763 {
1764 cpu_physical_memory_set_dirty_tracking(1);
1765 }
1766
1767 static void core_log_global_stop(MemoryListener *listener)
1768 {
1769 cpu_physical_memory_set_dirty_tracking(0);
1770 }
1771
1772 static MemoryListener core_memory_listener = {
1773 .begin = core_begin,
1774 .commit = core_commit,
1775 .log_global_start = core_log_global_start,
1776 .log_global_stop = core_log_global_stop,
1777 .priority = 1,
1778 };
1779
1780 static MemoryListener tcg_memory_listener = {
1781 .commit = tcg_commit,
1782 };
1783
1784 void address_space_init_dispatch(AddressSpace *as)
1785 {
1786 as->dispatch = NULL;
1787 as->dispatch_listener = (MemoryListener) {
1788 .begin = mem_begin,
1789 .commit = mem_commit,
1790 .region_add = mem_add,
1791 .region_nop = mem_add,
1792 .priority = 0,
1793 };
1794 memory_listener_register(&as->dispatch_listener, as);
1795 }
1796
1797 void address_space_destroy_dispatch(AddressSpace *as)
1798 {
1799 AddressSpaceDispatch *d = as->dispatch;
1800
1801 memory_listener_unregister(&as->dispatch_listener);
1802 g_free(d);
1803 as->dispatch = NULL;
1804 }
1805
1806 static void memory_map_init(void)
1807 {
1808 system_memory = g_malloc(sizeof(*system_memory));
1809 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1810 address_space_init(&address_space_memory, system_memory, "memory");
1811
1812 system_io = g_malloc(sizeof(*system_io));
1813 memory_region_init(system_io, NULL, "io", 65536);
1814 address_space_init(&address_space_io, system_io, "I/O");
1815
1816 memory_listener_register(&core_memory_listener, &address_space_memory);
1817 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1818 }
1819
1820 MemoryRegion *get_system_memory(void)
1821 {
1822 return system_memory;
1823 }
1824
1825 MemoryRegion *get_system_io(void)
1826 {
1827 return system_io;
1828 }
1829
1830 #endif /* !defined(CONFIG_USER_ONLY) */
1831
1832 /* physical memory access (slow version, mainly for debug) */
1833 #if defined(CONFIG_USER_ONLY)
1834 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1835 uint8_t *buf, int len, int is_write)
1836 {
1837 int l, flags;
1838 target_ulong page;
1839 void * p;
1840
1841 while (len > 0) {
1842 page = addr & TARGET_PAGE_MASK;
1843 l = (page + TARGET_PAGE_SIZE) - addr;
1844 if (l > len)
1845 l = len;
1846 flags = page_get_flags(page);
1847 if (!(flags & PAGE_VALID))
1848 return -1;
1849 if (is_write) {
1850 if (!(flags & PAGE_WRITE))
1851 return -1;
1852 /* XXX: this code should not depend on lock_user */
1853 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1854 return -1;
1855 memcpy(p, buf, l);
1856 unlock_user(p, addr, l);
1857 } else {
1858 if (!(flags & PAGE_READ))
1859 return -1;
1860 /* XXX: this code should not depend on lock_user */
1861 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1862 return -1;
1863 memcpy(buf, p, l);
1864 unlock_user(p, addr, 0);
1865 }
1866 len -= l;
1867 buf += l;
1868 addr += l;
1869 }
1870 return 0;
1871 }
1872
1873 #else
1874
1875 static void invalidate_and_set_dirty(hwaddr addr,
1876 hwaddr length)
1877 {
1878 if (!cpu_physical_memory_is_dirty(addr)) {
1879 /* invalidate code */
1880 tb_invalidate_phys_page_range(addr, addr + length, 0);
1881 /* set dirty bit */
1882 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1883 }
1884 xen_modified_memory(addr, length);
1885 }
1886
1887 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1888 {
1889 if (memory_region_is_ram(mr)) {
1890 return !(is_write && mr->readonly);
1891 }
1892 if (memory_region_is_romd(mr)) {
1893 return !is_write;
1894 }
1895
1896 return false;
1897 }
1898
1899 static inline int memory_access_size(MemoryRegion *mr, int l, hwaddr addr)
1900 {
1901 if (l >= 4 && (((addr & 3) == 0 || mr->ops->impl.unaligned))) {
1902 return 4;
1903 }
1904 if (l >= 2 && (((addr & 1) == 0) || mr->ops->impl.unaligned)) {
1905 return 2;
1906 }
1907 return 1;
1908 }
1909
1910 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1911 int len, bool is_write)
1912 {
1913 hwaddr l;
1914 uint8_t *ptr;
1915 uint64_t val;
1916 hwaddr addr1;
1917 MemoryRegion *mr;
1918 bool error = false;
1919
1920 while (len > 0) {
1921 l = len;
1922 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1923
1924 if (is_write) {
1925 if (!memory_access_is_direct(mr, is_write)) {
1926 l = memory_access_size(mr, l, addr1);
1927 /* XXX: could force current_cpu to NULL to avoid
1928 potential bugs */
1929 if (l == 4) {
1930 /* 32 bit write access */
1931 val = ldl_p(buf);
1932 error |= io_mem_write(mr, addr1, val, 4);
1933 } else if (l == 2) {
1934 /* 16 bit write access */
1935 val = lduw_p(buf);
1936 error |= io_mem_write(mr, addr1, val, 2);
1937 } else {
1938 /* 8 bit write access */
1939 val = ldub_p(buf);
1940 error |= io_mem_write(mr, addr1, val, 1);
1941 }
1942 } else {
1943 addr1 += memory_region_get_ram_addr(mr);
1944 /* RAM case */
1945 ptr = qemu_get_ram_ptr(addr1);
1946 memcpy(ptr, buf, l);
1947 invalidate_and_set_dirty(addr1, l);
1948 }
1949 } else {
1950 if (!memory_access_is_direct(mr, is_write)) {
1951 /* I/O case */
1952 l = memory_access_size(mr, l, addr1);
1953 if (l == 4) {
1954 /* 32 bit read access */
1955 error |= io_mem_read(mr, addr1, &val, 4);
1956 stl_p(buf, val);
1957 } else if (l == 2) {
1958 /* 16 bit read access */
1959 error |= io_mem_read(mr, addr1, &val, 2);
1960 stw_p(buf, val);
1961 } else {
1962 /* 8 bit read access */
1963 error |= io_mem_read(mr, addr1, &val, 1);
1964 stb_p(buf, val);
1965 }
1966 } else {
1967 /* RAM case */
1968 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1969 memcpy(buf, ptr, l);
1970 }
1971 }
1972 len -= l;
1973 buf += l;
1974 addr += l;
1975 }
1976
1977 return error;
1978 }
1979
1980 bool address_space_write(AddressSpace *as, hwaddr addr,
1981 const uint8_t *buf, int len)
1982 {
1983 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
1984 }
1985
1986 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1987 {
1988 return address_space_rw(as, addr, buf, len, false);
1989 }
1990
1991
1992 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1993 int len, int is_write)
1994 {
1995 address_space_rw(&address_space_memory, addr, buf, len, is_write);
1996 }
1997
1998 /* used for ROM loading : can write in RAM and ROM */
1999 void cpu_physical_memory_write_rom(hwaddr addr,
2000 const uint8_t *buf, int len)
2001 {
2002 hwaddr l;
2003 uint8_t *ptr;
2004 hwaddr addr1;
2005 MemoryRegion *mr;
2006
2007 while (len > 0) {
2008 l = len;
2009 mr = address_space_translate(&address_space_memory,
2010 addr, &addr1, &l, true);
2011
2012 if (!(memory_region_is_ram(mr) ||
2013 memory_region_is_romd(mr))) {
2014 /* do nothing */
2015 } else {
2016 addr1 += memory_region_get_ram_addr(mr);
2017 /* ROM/RAM case */
2018 ptr = qemu_get_ram_ptr(addr1);
2019 memcpy(ptr, buf, l);
2020 invalidate_and_set_dirty(addr1, l);
2021 }
2022 len -= l;
2023 buf += l;
2024 addr += l;
2025 }
2026 }
2027
2028 typedef struct {
2029 MemoryRegion *mr;
2030 void *buffer;
2031 hwaddr addr;
2032 hwaddr len;
2033 } BounceBuffer;
2034
2035 static BounceBuffer bounce;
2036
2037 typedef struct MapClient {
2038 void *opaque;
2039 void (*callback)(void *opaque);
2040 QLIST_ENTRY(MapClient) link;
2041 } MapClient;
2042
2043 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2044 = QLIST_HEAD_INITIALIZER(map_client_list);
2045
2046 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2047 {
2048 MapClient *client = g_malloc(sizeof(*client));
2049
2050 client->opaque = opaque;
2051 client->callback = callback;
2052 QLIST_INSERT_HEAD(&map_client_list, client, link);
2053 return client;
2054 }
2055
2056 static void cpu_unregister_map_client(void *_client)
2057 {
2058 MapClient *client = (MapClient *)_client;
2059
2060 QLIST_REMOVE(client, link);
2061 g_free(client);
2062 }
2063
2064 static void cpu_notify_map_clients(void)
2065 {
2066 MapClient *client;
2067
2068 while (!QLIST_EMPTY(&map_client_list)) {
2069 client = QLIST_FIRST(&map_client_list);
2070 client->callback(client->opaque);
2071 cpu_unregister_map_client(client);
2072 }
2073 }
2074
2075 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2076 {
2077 MemoryRegion *mr;
2078 hwaddr l, xlat;
2079
2080 while (len > 0) {
2081 l = len;
2082 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2083 if (!memory_access_is_direct(mr, is_write)) {
2084 l = memory_access_size(mr, l, addr);
2085 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2086 return false;
2087 }
2088 }
2089
2090 len -= l;
2091 addr += l;
2092 }
2093 return true;
2094 }
2095
2096 /* Map a physical memory region into a host virtual address.
2097 * May map a subset of the requested range, given by and returned in *plen.
2098 * May return NULL if resources needed to perform the mapping are exhausted.
2099 * Use only for reads OR writes - not for read-modify-write operations.
2100 * Use cpu_register_map_client() to know when retrying the map operation is
2101 * likely to succeed.
2102 */
2103 void *address_space_map(AddressSpace *as,
2104 hwaddr addr,
2105 hwaddr *plen,
2106 bool is_write)
2107 {
2108 hwaddr len = *plen;
2109 hwaddr done = 0;
2110 hwaddr l, xlat, base;
2111 MemoryRegion *mr, *this_mr;
2112 ram_addr_t raddr;
2113
2114 if (len == 0) {
2115 return NULL;
2116 }
2117
2118 l = len;
2119 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2120 if (!memory_access_is_direct(mr, is_write)) {
2121 if (bounce.buffer) {
2122 return NULL;
2123 }
2124 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2125 bounce.addr = addr;
2126 bounce.len = l;
2127
2128 memory_region_ref(mr);
2129 bounce.mr = mr;
2130 if (!is_write) {
2131 address_space_read(as, addr, bounce.buffer, l);
2132 }
2133
2134 *plen = l;
2135 return bounce.buffer;
2136 }
2137
2138 base = xlat;
2139 raddr = memory_region_get_ram_addr(mr);
2140
2141 for (;;) {
2142 len -= l;
2143 addr += l;
2144 done += l;
2145 if (len == 0) {
2146 break;
2147 }
2148
2149 l = len;
2150 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2151 if (this_mr != mr || xlat != base + done) {
2152 break;
2153 }
2154 }
2155
2156 memory_region_ref(mr);
2157 *plen = done;
2158 return qemu_ram_ptr_length(raddr + base, plen);
2159 }
2160
2161 /* Unmaps a memory region previously mapped by address_space_map().
2162 * Will also mark the memory as dirty if is_write == 1. access_len gives
2163 * the amount of memory that was actually read or written by the caller.
2164 */
2165 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2166 int is_write, hwaddr access_len)
2167 {
2168 if (buffer != bounce.buffer) {
2169 MemoryRegion *mr;
2170 ram_addr_t addr1;
2171
2172 mr = qemu_ram_addr_from_host(buffer, &addr1);
2173 assert(mr != NULL);
2174 if (is_write) {
2175 while (access_len) {
2176 unsigned l;
2177 l = TARGET_PAGE_SIZE;
2178 if (l > access_len)
2179 l = access_len;
2180 invalidate_and_set_dirty(addr1, l);
2181 addr1 += l;
2182 access_len -= l;
2183 }
2184 }
2185 if (xen_enabled()) {
2186 xen_invalidate_map_cache_entry(buffer);
2187 }
2188 memory_region_unref(mr);
2189 return;
2190 }
2191 if (is_write) {
2192 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2193 }
2194 qemu_vfree(bounce.buffer);
2195 bounce.buffer = NULL;
2196 memory_region_unref(bounce.mr);
2197 cpu_notify_map_clients();
2198 }
2199
2200 void *cpu_physical_memory_map(hwaddr addr,
2201 hwaddr *plen,
2202 int is_write)
2203 {
2204 return address_space_map(&address_space_memory, addr, plen, is_write);
2205 }
2206
2207 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2208 int is_write, hwaddr access_len)
2209 {
2210 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2211 }
2212
2213 /* warning: addr must be aligned */
2214 static inline uint32_t ldl_phys_internal(hwaddr addr,
2215 enum device_endian endian)
2216 {
2217 uint8_t *ptr;
2218 uint64_t val;
2219 MemoryRegion *mr;
2220 hwaddr l = 4;
2221 hwaddr addr1;
2222
2223 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2224 false);
2225 if (l < 4 || !memory_access_is_direct(mr, false)) {
2226 /* I/O case */
2227 io_mem_read(mr, addr1, &val, 4);
2228 #if defined(TARGET_WORDS_BIGENDIAN)
2229 if (endian == DEVICE_LITTLE_ENDIAN) {
2230 val = bswap32(val);
2231 }
2232 #else
2233 if (endian == DEVICE_BIG_ENDIAN) {
2234 val = bswap32(val);
2235 }
2236 #endif
2237 } else {
2238 /* RAM case */
2239 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2240 & TARGET_PAGE_MASK)
2241 + addr1);
2242 switch (endian) {
2243 case DEVICE_LITTLE_ENDIAN:
2244 val = ldl_le_p(ptr);
2245 break;
2246 case DEVICE_BIG_ENDIAN:
2247 val = ldl_be_p(ptr);
2248 break;
2249 default:
2250 val = ldl_p(ptr);
2251 break;
2252 }
2253 }
2254 return val;
2255 }
2256
2257 uint32_t ldl_phys(hwaddr addr)
2258 {
2259 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2260 }
2261
2262 uint32_t ldl_le_phys(hwaddr addr)
2263 {
2264 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2265 }
2266
2267 uint32_t ldl_be_phys(hwaddr addr)
2268 {
2269 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2270 }
2271
2272 /* warning: addr must be aligned */
2273 static inline uint64_t ldq_phys_internal(hwaddr addr,
2274 enum device_endian endian)
2275 {
2276 uint8_t *ptr;
2277 uint64_t val;
2278 MemoryRegion *mr;
2279 hwaddr l = 8;
2280 hwaddr addr1;
2281
2282 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2283 false);
2284 if (l < 8 || !memory_access_is_direct(mr, false)) {
2285 /* I/O case */
2286 io_mem_read(mr, addr1, &val, 8);
2287 #if defined(TARGET_WORDS_BIGENDIAN)
2288 if (endian == DEVICE_LITTLE_ENDIAN) {
2289 val = bswap64(val);
2290 }
2291 #else
2292 if (endian == DEVICE_BIG_ENDIAN) {
2293 val = bswap64(val);
2294 }
2295 #endif
2296 } else {
2297 /* RAM case */
2298 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2299 & TARGET_PAGE_MASK)
2300 + addr1);
2301 switch (endian) {
2302 case DEVICE_LITTLE_ENDIAN:
2303 val = ldq_le_p(ptr);
2304 break;
2305 case DEVICE_BIG_ENDIAN:
2306 val = ldq_be_p(ptr);
2307 break;
2308 default:
2309 val = ldq_p(ptr);
2310 break;
2311 }
2312 }
2313 return val;
2314 }
2315
2316 uint64_t ldq_phys(hwaddr addr)
2317 {
2318 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2319 }
2320
2321 uint64_t ldq_le_phys(hwaddr addr)
2322 {
2323 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2324 }
2325
2326 uint64_t ldq_be_phys(hwaddr addr)
2327 {
2328 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2329 }
2330
2331 /* XXX: optimize */
2332 uint32_t ldub_phys(hwaddr addr)
2333 {
2334 uint8_t val;
2335 cpu_physical_memory_read(addr, &val, 1);
2336 return val;
2337 }
2338
2339 /* warning: addr must be aligned */
2340 static inline uint32_t lduw_phys_internal(hwaddr addr,
2341 enum device_endian endian)
2342 {
2343 uint8_t *ptr;
2344 uint64_t val;
2345 MemoryRegion *mr;
2346 hwaddr l = 2;
2347 hwaddr addr1;
2348
2349 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2350 false);
2351 if (l < 2 || !memory_access_is_direct(mr, false)) {
2352 /* I/O case */
2353 io_mem_read(mr, addr1, &val, 2);
2354 #if defined(TARGET_WORDS_BIGENDIAN)
2355 if (endian == DEVICE_LITTLE_ENDIAN) {
2356 val = bswap16(val);
2357 }
2358 #else
2359 if (endian == DEVICE_BIG_ENDIAN) {
2360 val = bswap16(val);
2361 }
2362 #endif
2363 } else {
2364 /* RAM case */
2365 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2366 & TARGET_PAGE_MASK)
2367 + addr1);
2368 switch (endian) {
2369 case DEVICE_LITTLE_ENDIAN:
2370 val = lduw_le_p(ptr);
2371 break;
2372 case DEVICE_BIG_ENDIAN:
2373 val = lduw_be_p(ptr);
2374 break;
2375 default:
2376 val = lduw_p(ptr);
2377 break;
2378 }
2379 }
2380 return val;
2381 }
2382
2383 uint32_t lduw_phys(hwaddr addr)
2384 {
2385 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2386 }
2387
2388 uint32_t lduw_le_phys(hwaddr addr)
2389 {
2390 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2391 }
2392
2393 uint32_t lduw_be_phys(hwaddr addr)
2394 {
2395 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2396 }
2397
2398 /* warning: addr must be aligned. The ram page is not masked as dirty
2399 and the code inside is not invalidated. It is useful if the dirty
2400 bits are used to track modified PTEs */
2401 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2402 {
2403 uint8_t *ptr;
2404 MemoryRegion *mr;
2405 hwaddr l = 4;
2406 hwaddr addr1;
2407
2408 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2409 true);
2410 if (l < 4 || !memory_access_is_direct(mr, true)) {
2411 io_mem_write(mr, addr1, val, 4);
2412 } else {
2413 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2414 ptr = qemu_get_ram_ptr(addr1);
2415 stl_p(ptr, val);
2416
2417 if (unlikely(in_migration)) {
2418 if (!cpu_physical_memory_is_dirty(addr1)) {
2419 /* invalidate code */
2420 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2421 /* set dirty bit */
2422 cpu_physical_memory_set_dirty_flags(
2423 addr1, (0xff & ~CODE_DIRTY_FLAG));
2424 }
2425 }
2426 }
2427 }
2428
2429 /* warning: addr must be aligned */
2430 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2431 enum device_endian endian)
2432 {
2433 uint8_t *ptr;
2434 MemoryRegion *mr;
2435 hwaddr l = 4;
2436 hwaddr addr1;
2437
2438 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2439 true);
2440 if (l < 4 || !memory_access_is_direct(mr, true)) {
2441 #if defined(TARGET_WORDS_BIGENDIAN)
2442 if (endian == DEVICE_LITTLE_ENDIAN) {
2443 val = bswap32(val);
2444 }
2445 #else
2446 if (endian == DEVICE_BIG_ENDIAN) {
2447 val = bswap32(val);
2448 }
2449 #endif
2450 io_mem_write(mr, addr1, val, 4);
2451 } else {
2452 /* RAM case */
2453 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2454 ptr = qemu_get_ram_ptr(addr1);
2455 switch (endian) {
2456 case DEVICE_LITTLE_ENDIAN:
2457 stl_le_p(ptr, val);
2458 break;
2459 case DEVICE_BIG_ENDIAN:
2460 stl_be_p(ptr, val);
2461 break;
2462 default:
2463 stl_p(ptr, val);
2464 break;
2465 }
2466 invalidate_and_set_dirty(addr1, 4);
2467 }
2468 }
2469
2470 void stl_phys(hwaddr addr, uint32_t val)
2471 {
2472 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2473 }
2474
2475 void stl_le_phys(hwaddr addr, uint32_t val)
2476 {
2477 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2478 }
2479
2480 void stl_be_phys(hwaddr addr, uint32_t val)
2481 {
2482 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2483 }
2484
2485 /* XXX: optimize */
2486 void stb_phys(hwaddr addr, uint32_t val)
2487 {
2488 uint8_t v = val;
2489 cpu_physical_memory_write(addr, &v, 1);
2490 }
2491
2492 /* warning: addr must be aligned */
2493 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2494 enum device_endian endian)
2495 {
2496 uint8_t *ptr;
2497 MemoryRegion *mr;
2498 hwaddr l = 2;
2499 hwaddr addr1;
2500
2501 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2502 true);
2503 if (l < 2 || !memory_access_is_direct(mr, true)) {
2504 #if defined(TARGET_WORDS_BIGENDIAN)
2505 if (endian == DEVICE_LITTLE_ENDIAN) {
2506 val = bswap16(val);
2507 }
2508 #else
2509 if (endian == DEVICE_BIG_ENDIAN) {
2510 val = bswap16(val);
2511 }
2512 #endif
2513 io_mem_write(mr, addr1, val, 2);
2514 } else {
2515 /* RAM case */
2516 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2517 ptr = qemu_get_ram_ptr(addr1);
2518 switch (endian) {
2519 case DEVICE_LITTLE_ENDIAN:
2520 stw_le_p(ptr, val);
2521 break;
2522 case DEVICE_BIG_ENDIAN:
2523 stw_be_p(ptr, val);
2524 break;
2525 default:
2526 stw_p(ptr, val);
2527 break;
2528 }
2529 invalidate_and_set_dirty(addr1, 2);
2530 }
2531 }
2532
2533 void stw_phys(hwaddr addr, uint32_t val)
2534 {
2535 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2536 }
2537
2538 void stw_le_phys(hwaddr addr, uint32_t val)
2539 {
2540 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2541 }
2542
2543 void stw_be_phys(hwaddr addr, uint32_t val)
2544 {
2545 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2546 }
2547
2548 /* XXX: optimize */
2549 void stq_phys(hwaddr addr, uint64_t val)
2550 {
2551 val = tswap64(val);
2552 cpu_physical_memory_write(addr, &val, 8);
2553 }
2554
2555 void stq_le_phys(hwaddr addr, uint64_t val)
2556 {
2557 val = cpu_to_le64(val);
2558 cpu_physical_memory_write(addr, &val, 8);
2559 }
2560
2561 void stq_be_phys(hwaddr addr, uint64_t val)
2562 {
2563 val = cpu_to_be64(val);
2564 cpu_physical_memory_write(addr, &val, 8);
2565 }
2566
2567 /* virtual memory access for debug (includes writing to ROM) */
2568 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2569 uint8_t *buf, int len, int is_write)
2570 {
2571 int l;
2572 hwaddr phys_addr;
2573 target_ulong page;
2574
2575 while (len > 0) {
2576 page = addr & TARGET_PAGE_MASK;
2577 phys_addr = cpu_get_phys_page_debug(env, page);
2578 /* if no physical page mapped, return an error */
2579 if (phys_addr == -1)
2580 return -1;
2581 l = (page + TARGET_PAGE_SIZE) - addr;
2582 if (l > len)
2583 l = len;
2584 phys_addr += (addr & ~TARGET_PAGE_MASK);
2585 if (is_write)
2586 cpu_physical_memory_write_rom(phys_addr, buf, l);
2587 else
2588 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2589 len -= l;
2590 buf += l;
2591 addr += l;
2592 }
2593 return 0;
2594 }
2595 #endif
2596
2597 #if !defined(CONFIG_USER_ONLY)
2598
2599 /*
2600 * A helper function for the _utterly broken_ virtio device model to find out if
2601 * it's running on a big endian machine. Don't do this at home kids!
2602 */
2603 bool virtio_is_big_endian(void);
2604 bool virtio_is_big_endian(void)
2605 {
2606 #if defined(TARGET_WORDS_BIGENDIAN)
2607 return true;
2608 #else
2609 return false;
2610 #endif
2611 }
2612
2613 #endif
2614
2615 #ifndef CONFIG_USER_ONLY
2616 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2617 {
2618 MemoryRegion*mr;
2619 hwaddr l = 1;
2620
2621 mr = address_space_translate(&address_space_memory,
2622 phys_addr, &phys_addr, &l, false);
2623
2624 return !(memory_region_is_ram(mr) ||
2625 memory_region_is_romd(mr));
2626 }
2627
2628 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2629 {
2630 RAMBlock *block;
2631
2632 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2633 func(block->host, block->offset, block->length, opaque);
2634 }
2635 }
2636 #endif
QEmu