usb: Also reset max_packet_size on ep_reset
[qemu/ar7.git] / exec.c
blob26469120d949e68473f07cdd9185b02d78594fd0
1 /*
2 * Virtual page mapping
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
52 #include "exec/memory-internal.h"
54 //#define DEBUG_SUBPAGE
56 #if !defined(CONFIG_USER_ONLY)
57 static int in_migration;
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
70 #endif
72 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
81 #if !defined(CONFIG_USER_ONLY)
83 typedef struct PhysPageEntry PhysPageEntry;
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
91 typedef PhysPageEntry Node[L2_SIZE];
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
134 static MemoryRegion io_mem_watch;
135 #endif
137 #if !defined(CONFIG_USER_ONLY)
139 static void phys_map_node_reserve(unsigned nodes)
141 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
142 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
143 16);
144 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
145 next_map.nodes_nb + nodes);
146 next_map.nodes = g_renew(Node, next_map.nodes,
147 next_map.nodes_nb_alloc);
151 static uint16_t phys_map_node_alloc(void)
153 unsigned i;
154 uint16_t ret;
156 ret = next_map.nodes_nb++;
157 assert(ret != PHYS_MAP_NODE_NIL);
158 assert(ret != next_map.nodes_nb_alloc);
159 for (i = 0; i < L2_SIZE; ++i) {
160 next_map.nodes[ret][i].is_leaf = 0;
161 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
163 return ret;
166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
167 hwaddr *nb, uint16_t leaf,
168 int level)
170 PhysPageEntry *p;
171 int i;
172 hwaddr step = (hwaddr)1 << (level * L2_BITS);
174 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
175 lp->ptr = phys_map_node_alloc();
176 p = next_map.nodes[lp->ptr];
177 if (level == 0) {
178 for (i = 0; i < L2_SIZE; i++) {
179 p[i].is_leaf = 1;
180 p[i].ptr = PHYS_SECTION_UNASSIGNED;
183 } else {
184 p = next_map.nodes[lp->ptr];
186 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
188 while (*nb && lp < &p[L2_SIZE]) {
189 if ((*index & (step - 1)) == 0 && *nb >= step) {
190 lp->is_leaf = true;
191 lp->ptr = leaf;
192 *index += step;
193 *nb -= step;
194 } else {
195 phys_page_set_level(lp, index, nb, leaf, level - 1);
197 ++lp;
201 static void phys_page_set(AddressSpaceDispatch *d,
202 hwaddr index, hwaddr nb,
203 uint16_t leaf)
205 /* Wildly overreserve - it doesn't matter much. */
206 phys_map_node_reserve(3 * P_L2_LEVELS);
208 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
212 Node *nodes, MemoryRegionSection *sections)
214 PhysPageEntry *p;
215 int i;
217 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
218 if (lp.ptr == PHYS_MAP_NODE_NIL) {
219 return &sections[PHYS_SECTION_UNASSIGNED];
221 p = nodes[lp.ptr];
222 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
224 return &sections[lp.ptr];
227 bool memory_region_is_unassigned(MemoryRegion *mr)
229 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
230 && mr != &io_mem_watch;
233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
234 hwaddr addr,
235 bool resolve_subpage)
237 MemoryRegionSection *section;
238 subpage_t *subpage;
240 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
241 d->nodes, d->sections);
242 if (resolve_subpage && section->mr->subpage) {
243 subpage = container_of(section->mr, subpage_t, iomem);
244 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
246 return section;
249 static MemoryRegionSection *
250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
251 hwaddr *plen, bool resolve_subpage)
253 MemoryRegionSection *section;
254 Int128 diff;
256 section = address_space_lookup_region(d, addr, resolve_subpage);
257 /* Compute offset within MemoryRegionSection */
258 addr -= section->offset_within_address_space;
260 /* Compute offset within MemoryRegion */
261 *xlat = addr + section->offset_within_region;
263 diff = int128_sub(section->mr->size, int128_make64(addr));
264 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
265 return section;
268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
269 hwaddr *xlat, hwaddr *plen,
270 bool is_write)
272 IOMMUTLBEntry iotlb;
273 MemoryRegionSection *section;
274 MemoryRegion *mr;
275 hwaddr len = *plen;
277 for (;;) {
278 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
279 mr = section->mr;
281 if (!mr->iommu_ops) {
282 break;
285 iotlb = mr->iommu_ops->translate(mr, addr);
286 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
287 | (addr & iotlb.addr_mask));
288 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
289 if (!(iotlb.perm & (1 << is_write))) {
290 mr = &io_mem_unassigned;
291 break;
294 as = iotlb.target_as;
297 *plen = len;
298 *xlat = addr;
299 return mr;
302 MemoryRegionSection *
303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
304 hwaddr *plen)
306 MemoryRegionSection *section;
307 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
309 assert(!section->mr->iommu_ops);
310 return section;
312 #endif
314 void cpu_exec_init_all(void)
316 #if !defined(CONFIG_USER_ONLY)
317 qemu_mutex_init(&ram_list.mutex);
318 memory_map_init();
319 io_mem_init();
320 #endif
323 #if !defined(CONFIG_USER_ONLY)
325 static int cpu_common_post_load(void *opaque, int version_id)
327 CPUState *cpu = opaque;
329 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
330 version_id is increased. */
331 cpu->interrupt_request &= ~0x01;
332 tlb_flush(cpu->env_ptr, 1);
334 return 0;
337 const VMStateDescription vmstate_cpu_common = {
338 .name = "cpu_common",
339 .version_id = 1,
340 .minimum_version_id = 1,
341 .minimum_version_id_old = 1,
342 .post_load = cpu_common_post_load,
343 .fields = (VMStateField []) {
344 VMSTATE_UINT32(halted, CPUState),
345 VMSTATE_UINT32(interrupt_request, CPUState),
346 VMSTATE_END_OF_LIST()
350 #endif
352 CPUState *qemu_get_cpu(int index)
354 CPUState *cpu;
356 CPU_FOREACH(cpu) {
357 if (cpu->cpu_index == index) {
358 return cpu;
362 return NULL;
365 void cpu_exec_init(CPUArchState *env)
367 CPUState *cpu = ENV_GET_CPU(env);
368 CPUClass *cc = CPU_GET_CLASS(cpu);
369 CPUState *some_cpu;
370 int cpu_index;
372 #if defined(CONFIG_USER_ONLY)
373 cpu_list_lock();
374 #endif
375 cpu_index = 0;
376 CPU_FOREACH(some_cpu) {
377 cpu_index++;
379 cpu->cpu_index = cpu_index;
380 cpu->numa_node = 0;
381 QTAILQ_INIT(&env->breakpoints);
382 QTAILQ_INIT(&env->watchpoints);
383 #ifndef CONFIG_USER_ONLY
384 cpu->thread_id = qemu_get_thread_id();
385 #endif
386 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
387 #if defined(CONFIG_USER_ONLY)
388 cpu_list_unlock();
389 #endif
390 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
391 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
393 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
394 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
395 cpu_save, cpu_load, env);
396 assert(cc->vmsd == NULL);
397 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
398 #endif
399 if (cc->vmsd != NULL) {
400 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
404 #if defined(TARGET_HAS_ICE)
405 #if defined(CONFIG_USER_ONLY)
406 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
408 tb_invalidate_phys_page_range(pc, pc + 1, 0);
410 #else
411 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
413 tb_invalidate_phys_addr(cpu_get_phys_page_debug(cpu, pc) |
414 (pc & ~TARGET_PAGE_MASK));
416 #endif
417 #endif /* TARGET_HAS_ICE */
419 #if defined(CONFIG_USER_ONLY)
420 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
425 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
426 int flags, CPUWatchpoint **watchpoint)
428 return -ENOSYS;
430 #else
431 /* Add a watchpoint. */
432 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
433 int flags, CPUWatchpoint **watchpoint)
435 target_ulong len_mask = ~(len - 1);
436 CPUWatchpoint *wp;
438 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
439 if ((len & (len - 1)) || (addr & ~len_mask) ||
440 len == 0 || len > TARGET_PAGE_SIZE) {
441 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
442 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
443 return -EINVAL;
445 wp = g_malloc(sizeof(*wp));
447 wp->vaddr = addr;
448 wp->len_mask = len_mask;
449 wp->flags = flags;
451 /* keep all GDB-injected watchpoints in front */
452 if (flags & BP_GDB)
453 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
454 else
455 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
457 tlb_flush_page(env, addr);
459 if (watchpoint)
460 *watchpoint = wp;
461 return 0;
464 /* Remove a specific watchpoint. */
465 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
466 int flags)
468 target_ulong len_mask = ~(len - 1);
469 CPUWatchpoint *wp;
471 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
472 if (addr == wp->vaddr && len_mask == wp->len_mask
473 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
474 cpu_watchpoint_remove_by_ref(env, wp);
475 return 0;
478 return -ENOENT;
481 /* Remove a specific watchpoint by reference. */
482 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
484 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
486 tlb_flush_page(env, watchpoint->vaddr);
488 g_free(watchpoint);
491 /* Remove all matching watchpoints. */
492 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
494 CPUWatchpoint *wp, *next;
496 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
497 if (wp->flags & mask)
498 cpu_watchpoint_remove_by_ref(env, wp);
501 #endif
503 /* Add a breakpoint. */
504 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
505 CPUBreakpoint **breakpoint)
507 #if defined(TARGET_HAS_ICE)
508 CPUBreakpoint *bp;
510 bp = g_malloc(sizeof(*bp));
512 bp->pc = pc;
513 bp->flags = flags;
515 /* keep all GDB-injected breakpoints in front */
516 if (flags & BP_GDB) {
517 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
518 } else {
519 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
522 breakpoint_invalidate(ENV_GET_CPU(env), pc);
524 if (breakpoint) {
525 *breakpoint = bp;
527 return 0;
528 #else
529 return -ENOSYS;
530 #endif
533 /* Remove a specific breakpoint. */
534 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
536 #if defined(TARGET_HAS_ICE)
537 CPUBreakpoint *bp;
539 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
540 if (bp->pc == pc && bp->flags == flags) {
541 cpu_breakpoint_remove_by_ref(env, bp);
542 return 0;
545 return -ENOENT;
546 #else
547 return -ENOSYS;
548 #endif
551 /* Remove a specific breakpoint by reference. */
552 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
554 #if defined(TARGET_HAS_ICE)
555 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
557 breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
559 g_free(breakpoint);
560 #endif
563 /* Remove all matching breakpoints. */
564 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
566 #if defined(TARGET_HAS_ICE)
567 CPUBreakpoint *bp, *next;
569 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
570 if (bp->flags & mask)
571 cpu_breakpoint_remove_by_ref(env, bp);
573 #endif
576 /* enable or disable single step mode. EXCP_DEBUG is returned by the
577 CPU loop after each instruction */
578 void cpu_single_step(CPUState *cpu, int enabled)
580 #if defined(TARGET_HAS_ICE)
581 if (cpu->singlestep_enabled != enabled) {
582 cpu->singlestep_enabled = enabled;
583 if (kvm_enabled()) {
584 kvm_update_guest_debug(cpu, 0);
585 } else {
586 /* must flush all the translated code to avoid inconsistencies */
587 /* XXX: only flush what is necessary */
588 CPUArchState *env = cpu->env_ptr;
589 tb_flush(env);
592 #endif
595 void cpu_abort(CPUArchState *env, const char *fmt, ...)
597 CPUState *cpu = ENV_GET_CPU(env);
598 va_list ap;
599 va_list ap2;
601 va_start(ap, fmt);
602 va_copy(ap2, ap);
603 fprintf(stderr, "qemu: fatal: ");
604 vfprintf(stderr, fmt, ap);
605 fprintf(stderr, "\n");
606 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
607 if (qemu_log_enabled()) {
608 qemu_log("qemu: fatal: ");
609 qemu_log_vprintf(fmt, ap2);
610 qemu_log("\n");
611 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
612 qemu_log_flush();
613 qemu_log_close();
615 va_end(ap2);
616 va_end(ap);
617 #if defined(CONFIG_USER_ONLY)
619 struct sigaction act;
620 sigfillset(&act.sa_mask);
621 act.sa_handler = SIG_DFL;
622 sigaction(SIGABRT, &act, NULL);
624 #endif
625 abort();
628 CPUArchState *cpu_copy(CPUArchState *env)
630 CPUArchState *new_env = cpu_init(env->cpu_model_str);
631 #if defined(TARGET_HAS_ICE)
632 CPUBreakpoint *bp;
633 CPUWatchpoint *wp;
634 #endif
636 /* Reset non arch specific state */
637 cpu_reset(ENV_GET_CPU(new_env));
639 /* Copy arch specific state into the new CPU */
640 memcpy(new_env, env, sizeof(CPUArchState));
642 /* Clone all break/watchpoints.
643 Note: Once we support ptrace with hw-debug register access, make sure
644 BP_CPU break/watchpoints are handled correctly on clone. */
645 QTAILQ_INIT(&env->breakpoints);
646 QTAILQ_INIT(&env->watchpoints);
647 #if defined(TARGET_HAS_ICE)
648 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
649 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
651 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
652 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
653 wp->flags, NULL);
655 #endif
657 return new_env;
660 #if !defined(CONFIG_USER_ONLY)
661 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
662 uintptr_t length)
664 uintptr_t start1;
666 /* we modify the TLB cache so that the dirty bit will be set again
667 when accessing the range */
668 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
669 /* Check that we don't span multiple blocks - this breaks the
670 address comparisons below. */
671 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
672 != (end - 1) - start) {
673 abort();
675 cpu_tlb_reset_dirty_all(start1, length);
679 /* Note: start and end must be within the same ram block. */
680 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
681 int dirty_flags)
683 uintptr_t length;
685 start &= TARGET_PAGE_MASK;
686 end = TARGET_PAGE_ALIGN(end);
688 length = end - start;
689 if (length == 0)
690 return;
691 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
693 if (tcg_enabled()) {
694 tlb_reset_dirty_range_all(start, end, length);
698 static int cpu_physical_memory_set_dirty_tracking(int enable)
700 int ret = 0;
701 in_migration = enable;
702 return ret;
705 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
706 MemoryRegionSection *section,
707 target_ulong vaddr,
708 hwaddr paddr, hwaddr xlat,
709 int prot,
710 target_ulong *address)
712 hwaddr iotlb;
713 CPUWatchpoint *wp;
715 if (memory_region_is_ram(section->mr)) {
716 /* Normal RAM. */
717 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
718 + xlat;
719 if (!section->readonly) {
720 iotlb |= PHYS_SECTION_NOTDIRTY;
721 } else {
722 iotlb |= PHYS_SECTION_ROM;
724 } else {
725 iotlb = section - address_space_memory.dispatch->sections;
726 iotlb += xlat;
729 /* Make accesses to pages with watchpoints go via the
730 watchpoint trap routines. */
731 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
732 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
733 /* Avoid trapping reads of pages with a write breakpoint. */
734 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
735 iotlb = PHYS_SECTION_WATCH + paddr;
736 *address |= TLB_MMIO;
737 break;
742 return iotlb;
744 #endif /* defined(CONFIG_USER_ONLY) */
746 #if !defined(CONFIG_USER_ONLY)
748 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
749 uint16_t section);
750 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
752 static void *(*phys_mem_alloc)(ram_addr_t size) = qemu_anon_ram_alloc;
755 * Set a custom physical guest memory alloator.
756 * Accelerators with unusual needs may need this. Hopefully, we can
757 * get rid of it eventually.
759 void phys_mem_set_alloc(void *(*alloc)(ram_addr_t))
761 phys_mem_alloc = alloc;
764 static uint16_t phys_section_add(MemoryRegionSection *section)
766 /* The physical section number is ORed with a page-aligned
767 * pointer to produce the iotlb entries. Thus it should
768 * never overflow into the page-aligned value.
770 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
772 if (next_map.sections_nb == next_map.sections_nb_alloc) {
773 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
774 16);
775 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
776 next_map.sections_nb_alloc);
778 next_map.sections[next_map.sections_nb] = *section;
779 memory_region_ref(section->mr);
780 return next_map.sections_nb++;
783 static void phys_section_destroy(MemoryRegion *mr)
785 memory_region_unref(mr);
787 if (mr->subpage) {
788 subpage_t *subpage = container_of(mr, subpage_t, iomem);
789 memory_region_destroy(&subpage->iomem);
790 g_free(subpage);
794 static void phys_sections_free(PhysPageMap *map)
796 while (map->sections_nb > 0) {
797 MemoryRegionSection *section = &map->sections[--map->sections_nb];
798 phys_section_destroy(section->mr);
800 g_free(map->sections);
801 g_free(map->nodes);
802 g_free(map);
805 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
807 subpage_t *subpage;
808 hwaddr base = section->offset_within_address_space
809 & TARGET_PAGE_MASK;
810 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
811 next_map.nodes, next_map.sections);
812 MemoryRegionSection subsection = {
813 .offset_within_address_space = base,
814 .size = int128_make64(TARGET_PAGE_SIZE),
816 hwaddr start, end;
818 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
820 if (!(existing->mr->subpage)) {
821 subpage = subpage_init(d->as, base);
822 subsection.mr = &subpage->iomem;
823 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
824 phys_section_add(&subsection));
825 } else {
826 subpage = container_of(existing->mr, subpage_t, iomem);
828 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
829 end = start + int128_get64(section->size) - 1;
830 subpage_register(subpage, start, end, phys_section_add(section));
834 static void register_multipage(AddressSpaceDispatch *d,
835 MemoryRegionSection *section)
837 hwaddr start_addr = section->offset_within_address_space;
838 uint16_t section_index = phys_section_add(section);
839 uint64_t num_pages = int128_get64(int128_rshift(section->size,
840 TARGET_PAGE_BITS));
842 assert(num_pages);
843 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
846 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
848 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
849 AddressSpaceDispatch *d = as->next_dispatch;
850 MemoryRegionSection now = *section, remain = *section;
851 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
853 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
854 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
855 - now.offset_within_address_space;
857 now.size = int128_min(int128_make64(left), now.size);
858 register_subpage(d, &now);
859 } else {
860 now.size = int128_zero();
862 while (int128_ne(remain.size, now.size)) {
863 remain.size = int128_sub(remain.size, now.size);
864 remain.offset_within_address_space += int128_get64(now.size);
865 remain.offset_within_region += int128_get64(now.size);
866 now = remain;
867 if (int128_lt(remain.size, page_size)) {
868 register_subpage(d, &now);
869 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
870 now.size = page_size;
871 register_subpage(d, &now);
872 } else {
873 now.size = int128_and(now.size, int128_neg(page_size));
874 register_multipage(d, &now);
879 void qemu_flush_coalesced_mmio_buffer(void)
881 if (kvm_enabled())
882 kvm_flush_coalesced_mmio_buffer();
885 void qemu_mutex_lock_ramlist(void)
887 qemu_mutex_lock(&ram_list.mutex);
890 void qemu_mutex_unlock_ramlist(void)
892 qemu_mutex_unlock(&ram_list.mutex);
895 #ifdef __linux__
897 #include <sys/vfs.h>
899 #define HUGETLBFS_MAGIC 0x958458f6
901 static long gethugepagesize(const char *path)
903 struct statfs fs;
904 int ret;
906 do {
907 ret = statfs(path, &fs);
908 } while (ret != 0 && errno == EINTR);
910 if (ret != 0) {
911 perror(path);
912 return 0;
915 if (fs.f_type != HUGETLBFS_MAGIC)
916 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
918 return fs.f_bsize;
921 static void *file_ram_alloc(RAMBlock *block,
922 ram_addr_t memory,
923 const char *path)
925 char *filename;
926 char *sanitized_name;
927 char *c;
928 void *area;
929 int fd;
930 #ifdef MAP_POPULATE
931 int flags;
932 #endif
933 unsigned long hpagesize;
935 hpagesize = gethugepagesize(path);
936 if (!hpagesize) {
937 return NULL;
940 if (memory < hpagesize) {
941 return NULL;
944 if (kvm_enabled() && !kvm_has_sync_mmu()) {
945 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
946 return NULL;
949 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
950 sanitized_name = g_strdup(block->mr->name);
951 for (c = sanitized_name; *c != '\0'; c++) {
952 if (*c == '/')
953 *c = '_';
956 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
957 sanitized_name);
958 g_free(sanitized_name);
960 fd = mkstemp(filename);
961 if (fd < 0) {
962 perror("unable to create backing store for hugepages");
963 g_free(filename);
964 return NULL;
966 unlink(filename);
967 g_free(filename);
969 memory = (memory+hpagesize-1) & ~(hpagesize-1);
972 * ftruncate is not supported by hugetlbfs in older
973 * hosts, so don't bother bailing out on errors.
974 * If anything goes wrong with it under other filesystems,
975 * mmap will fail.
977 if (ftruncate(fd, memory))
978 perror("ftruncate");
980 #ifdef MAP_POPULATE
981 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
982 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
983 * to sidestep this quirk.
985 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
986 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
987 #else
988 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
989 #endif
990 if (area == MAP_FAILED) {
991 perror("file_ram_alloc: can't mmap RAM pages");
992 close(fd);
993 return (NULL);
995 block->fd = fd;
996 return area;
998 #else
999 static void *file_ram_alloc(RAMBlock *block,
1000 ram_addr_t memory,
1001 const char *path)
1003 fprintf(stderr, "-mem-path not supported on this host\n");
1004 exit(1);
1006 #endif
1008 static ram_addr_t find_ram_offset(ram_addr_t size)
1010 RAMBlock *block, *next_block;
1011 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1013 assert(size != 0); /* it would hand out same offset multiple times */
1015 if (QTAILQ_EMPTY(&ram_list.blocks))
1016 return 0;
1018 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1019 ram_addr_t end, next = RAM_ADDR_MAX;
1021 end = block->offset + block->length;
1023 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1024 if (next_block->offset >= end) {
1025 next = MIN(next, next_block->offset);
1028 if (next - end >= size && next - end < mingap) {
1029 offset = end;
1030 mingap = next - end;
1034 if (offset == RAM_ADDR_MAX) {
1035 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1036 (uint64_t)size);
1037 abort();
1040 return offset;
1043 ram_addr_t last_ram_offset(void)
1045 RAMBlock *block;
1046 ram_addr_t last = 0;
1048 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1049 last = MAX(last, block->offset + block->length);
1051 return last;
1054 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1056 int ret;
1058 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1059 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1060 "dump-guest-core", true)) {
1061 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1062 if (ret) {
1063 perror("qemu_madvise");
1064 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1065 "but dump_guest_core=off specified\n");
1070 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1072 RAMBlock *new_block, *block;
1074 new_block = NULL;
1075 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1076 if (block->offset == addr) {
1077 new_block = block;
1078 break;
1081 assert(new_block);
1082 assert(!new_block->idstr[0]);
1084 if (dev) {
1085 char *id = qdev_get_dev_path(dev);
1086 if (id) {
1087 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1088 g_free(id);
1091 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1093 /* This assumes the iothread lock is taken here too. */
1094 qemu_mutex_lock_ramlist();
1095 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1096 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1097 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1098 new_block->idstr);
1099 abort();
1102 qemu_mutex_unlock_ramlist();
1105 static int memory_try_enable_merging(void *addr, size_t len)
1107 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1108 /* disabled by the user */
1109 return 0;
1112 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1115 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1116 MemoryRegion *mr)
1118 RAMBlock *block, *new_block;
1120 size = TARGET_PAGE_ALIGN(size);
1121 new_block = g_malloc0(sizeof(*new_block));
1122 new_block->fd = -1;
1124 /* This assumes the iothread lock is taken here too. */
1125 qemu_mutex_lock_ramlist();
1126 new_block->mr = mr;
1127 new_block->offset = find_ram_offset(size);
1128 if (host) {
1129 new_block->host = host;
1130 new_block->flags |= RAM_PREALLOC_MASK;
1131 } else if (xen_enabled()) {
1132 if (mem_path) {
1133 fprintf(stderr, "-mem-path not supported with Xen\n");
1134 exit(1);
1136 xen_ram_alloc(new_block->offset, size, mr);
1137 } else {
1138 if (mem_path) {
1139 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1141 * file_ram_alloc() needs to allocate just like
1142 * phys_mem_alloc, but we haven't bothered to provide
1143 * a hook there.
1145 fprintf(stderr,
1146 "-mem-path not supported with this accelerator\n");
1147 exit(1);
1149 new_block->host = file_ram_alloc(new_block, size, mem_path);
1151 if (!new_block->host) {
1152 new_block->host = phys_mem_alloc(size);
1153 if (!new_block->host) {
1154 fprintf(stderr, "Cannot set up guest memory '%s': %s\n",
1155 new_block->mr->name, strerror(errno));
1156 exit(1);
1158 memory_try_enable_merging(new_block->host, size);
1161 new_block->length = size;
1163 /* Keep the list sorted from biggest to smallest block. */
1164 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1165 if (block->length < new_block->length) {
1166 break;
1169 if (block) {
1170 QTAILQ_INSERT_BEFORE(block, new_block, next);
1171 } else {
1172 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1174 ram_list.mru_block = NULL;
1176 ram_list.version++;
1177 qemu_mutex_unlock_ramlist();
1179 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1180 last_ram_offset() >> TARGET_PAGE_BITS);
1181 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1182 0, size >> TARGET_PAGE_BITS);
1183 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1185 qemu_ram_setup_dump(new_block->host, size);
1186 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1188 if (kvm_enabled())
1189 kvm_setup_guest_memory(new_block->host, size);
1191 return new_block->offset;
1194 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1196 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1199 void qemu_ram_free_from_ptr(ram_addr_t addr)
1201 RAMBlock *block;
1203 /* This assumes the iothread lock is taken here too. */
1204 qemu_mutex_lock_ramlist();
1205 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1206 if (addr == block->offset) {
1207 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1208 ram_list.mru_block = NULL;
1209 ram_list.version++;
1210 g_free(block);
1211 break;
1214 qemu_mutex_unlock_ramlist();
1217 void qemu_ram_free(ram_addr_t addr)
1219 RAMBlock *block;
1221 /* This assumes the iothread lock is taken here too. */
1222 qemu_mutex_lock_ramlist();
1223 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1224 if (addr == block->offset) {
1225 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1226 ram_list.mru_block = NULL;
1227 ram_list.version++;
1228 if (block->flags & RAM_PREALLOC_MASK) {
1230 } else if (xen_enabled()) {
1231 xen_invalidate_map_cache_entry(block->host);
1232 } else if (block->fd >= 0) {
1233 munmap(block->host, block->length);
1234 close(block->fd);
1235 } else {
1236 qemu_anon_ram_free(block->host, block->length);
1238 g_free(block);
1239 break;
1242 qemu_mutex_unlock_ramlist();
1246 #ifndef _WIN32
1247 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1249 RAMBlock *block;
1250 ram_addr_t offset;
1251 int flags;
1252 void *area, *vaddr;
1254 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1255 offset = addr - block->offset;
1256 if (offset < block->length) {
1257 vaddr = block->host + offset;
1258 if (block->flags & RAM_PREALLOC_MASK) {
1260 } else if (xen_enabled()) {
1261 abort();
1262 } else {
1263 flags = MAP_FIXED;
1264 munmap(vaddr, length);
1265 if (block->fd >= 0) {
1266 #ifdef MAP_POPULATE
1267 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1268 MAP_PRIVATE;
1269 #else
1270 flags |= MAP_PRIVATE;
1271 #endif
1272 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1273 flags, block->fd, offset);
1274 } else {
1276 * Remap needs to match alloc. Accelerators that
1277 * set phys_mem_alloc never remap. If they did,
1278 * we'd need a remap hook here.
1280 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1282 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1283 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1284 flags, -1, 0);
1286 if (area != vaddr) {
1287 fprintf(stderr, "Could not remap addr: "
1288 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1289 length, addr);
1290 exit(1);
1292 memory_try_enable_merging(vaddr, length);
1293 qemu_ram_setup_dump(vaddr, length);
1295 return;
1299 #endif /* !_WIN32 */
1301 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1303 RAMBlock *block;
1305 /* The list is protected by the iothread lock here. */
1306 block = ram_list.mru_block;
1307 if (block && addr - block->offset < block->length) {
1308 goto found;
1310 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1311 if (addr - block->offset < block->length) {
1312 goto found;
1316 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1317 abort();
1319 found:
1320 ram_list.mru_block = block;
1321 return block;
1324 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1325 With the exception of the softmmu code in this file, this should
1326 only be used for local memory (e.g. video ram) that the device owns,
1327 and knows it isn't going to access beyond the end of the block.
1329 It should not be used for general purpose DMA.
1330 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1332 void *qemu_get_ram_ptr(ram_addr_t addr)
1334 RAMBlock *block = qemu_get_ram_block(addr);
1336 if (xen_enabled()) {
1337 /* We need to check if the requested address is in the RAM
1338 * because we don't want to map the entire memory in QEMU.
1339 * In that case just map until the end of the page.
1341 if (block->offset == 0) {
1342 return xen_map_cache(addr, 0, 0);
1343 } else if (block->host == NULL) {
1344 block->host =
1345 xen_map_cache(block->offset, block->length, 1);
1348 return block->host + (addr - block->offset);
1351 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1352 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1354 * ??? Is this still necessary?
1356 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1358 RAMBlock *block;
1360 /* The list is protected by the iothread lock here. */
1361 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1362 if (addr - block->offset < block->length) {
1363 if (xen_enabled()) {
1364 /* We need to check if the requested address is in the RAM
1365 * because we don't want to map the entire memory in QEMU.
1366 * In that case just map until the end of the page.
1368 if (block->offset == 0) {
1369 return xen_map_cache(addr, 0, 0);
1370 } else if (block->host == NULL) {
1371 block->host =
1372 xen_map_cache(block->offset, block->length, 1);
1375 return block->host + (addr - block->offset);
1379 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1380 abort();
1382 return NULL;
1385 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1386 * but takes a size argument */
1387 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1389 if (*size == 0) {
1390 return NULL;
1392 if (xen_enabled()) {
1393 return xen_map_cache(addr, *size, 1);
1394 } else {
1395 RAMBlock *block;
1397 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1398 if (addr - block->offset < block->length) {
1399 if (addr - block->offset + *size > block->length)
1400 *size = block->length - addr + block->offset;
1401 return block->host + (addr - block->offset);
1405 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1406 abort();
1410 /* Some of the softmmu routines need to translate from a host pointer
1411 (typically a TLB entry) back to a ram offset. */
1412 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1414 RAMBlock *block;
1415 uint8_t *host = ptr;
1417 if (xen_enabled()) {
1418 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1419 return qemu_get_ram_block(*ram_addr)->mr;
1422 block = ram_list.mru_block;
1423 if (block && block->host && host - block->host < block->length) {
1424 goto found;
1427 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1428 /* This case append when the block is not mapped. */
1429 if (block->host == NULL) {
1430 continue;
1432 if (host - block->host < block->length) {
1433 goto found;
1437 return NULL;
1439 found:
1440 *ram_addr = block->offset + (host - block->host);
1441 return block->mr;
1444 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1445 uint64_t val, unsigned size)
1447 int dirty_flags;
1448 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1449 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1450 tb_invalidate_phys_page_fast(ram_addr, size);
1451 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1453 switch (size) {
1454 case 1:
1455 stb_p(qemu_get_ram_ptr(ram_addr), val);
1456 break;
1457 case 2:
1458 stw_p(qemu_get_ram_ptr(ram_addr), val);
1459 break;
1460 case 4:
1461 stl_p(qemu_get_ram_ptr(ram_addr), val);
1462 break;
1463 default:
1464 abort();
1466 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1467 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1468 /* we remove the notdirty callback only if the code has been
1469 flushed */
1470 if (dirty_flags == 0xff) {
1471 CPUArchState *env = current_cpu->env_ptr;
1472 tlb_set_dirty(env, env->mem_io_vaddr);
1476 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1477 unsigned size, bool is_write)
1479 return is_write;
1482 static const MemoryRegionOps notdirty_mem_ops = {
1483 .write = notdirty_mem_write,
1484 .valid.accepts = notdirty_mem_accepts,
1485 .endianness = DEVICE_NATIVE_ENDIAN,
1488 /* Generate a debug exception if a watchpoint has been hit. */
1489 static void check_watchpoint(int offset, int len_mask, int flags)
1491 CPUArchState *env = current_cpu->env_ptr;
1492 target_ulong pc, cs_base;
1493 target_ulong vaddr;
1494 CPUWatchpoint *wp;
1495 int cpu_flags;
1497 if (env->watchpoint_hit) {
1498 /* We re-entered the check after replacing the TB. Now raise
1499 * the debug interrupt so that is will trigger after the
1500 * current instruction. */
1501 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1502 return;
1504 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1505 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1506 if ((vaddr == (wp->vaddr & len_mask) ||
1507 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1508 wp->flags |= BP_WATCHPOINT_HIT;
1509 if (!env->watchpoint_hit) {
1510 env->watchpoint_hit = wp;
1511 tb_check_watchpoint(env);
1512 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1513 env->exception_index = EXCP_DEBUG;
1514 cpu_loop_exit(env);
1515 } else {
1516 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1517 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1518 cpu_resume_from_signal(env, NULL);
1521 } else {
1522 wp->flags &= ~BP_WATCHPOINT_HIT;
1527 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1528 so these check for a hit then pass through to the normal out-of-line
1529 phys routines. */
1530 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1531 unsigned size)
1533 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1534 switch (size) {
1535 case 1: return ldub_phys(addr);
1536 case 2: return lduw_phys(addr);
1537 case 4: return ldl_phys(addr);
1538 default: abort();
1542 static void watch_mem_write(void *opaque, hwaddr addr,
1543 uint64_t val, unsigned size)
1545 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1546 switch (size) {
1547 case 1:
1548 stb_phys(addr, val);
1549 break;
1550 case 2:
1551 stw_phys(addr, val);
1552 break;
1553 case 4:
1554 stl_phys(addr, val);
1555 break;
1556 default: abort();
1560 static const MemoryRegionOps watch_mem_ops = {
1561 .read = watch_mem_read,
1562 .write = watch_mem_write,
1563 .endianness = DEVICE_NATIVE_ENDIAN,
1566 static uint64_t subpage_read(void *opaque, hwaddr addr,
1567 unsigned len)
1569 subpage_t *subpage = opaque;
1570 uint8_t buf[4];
1572 #if defined(DEBUG_SUBPAGE)
1573 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1574 subpage, len, addr);
1575 #endif
1576 address_space_read(subpage->as, addr + subpage->base, buf, len);
1577 switch (len) {
1578 case 1:
1579 return ldub_p(buf);
1580 case 2:
1581 return lduw_p(buf);
1582 case 4:
1583 return ldl_p(buf);
1584 default:
1585 abort();
1589 static void subpage_write(void *opaque, hwaddr addr,
1590 uint64_t value, unsigned len)
1592 subpage_t *subpage = opaque;
1593 uint8_t buf[4];
1595 #if defined(DEBUG_SUBPAGE)
1596 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1597 " value %"PRIx64"\n",
1598 __func__, subpage, len, addr, value);
1599 #endif
1600 switch (len) {
1601 case 1:
1602 stb_p(buf, value);
1603 break;
1604 case 2:
1605 stw_p(buf, value);
1606 break;
1607 case 4:
1608 stl_p(buf, value);
1609 break;
1610 default:
1611 abort();
1613 address_space_write(subpage->as, addr + subpage->base, buf, len);
1616 static bool subpage_accepts(void *opaque, hwaddr addr,
1617 unsigned size, bool is_write)
1619 subpage_t *subpage = opaque;
1620 #if defined(DEBUG_SUBPAGE)
1621 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1622 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1623 #endif
1625 return address_space_access_valid(subpage->as, addr + subpage->base,
1626 size, is_write);
1629 static const MemoryRegionOps subpage_ops = {
1630 .read = subpage_read,
1631 .write = subpage_write,
1632 .valid.accepts = subpage_accepts,
1633 .endianness = DEVICE_NATIVE_ENDIAN,
1636 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1637 uint16_t section)
1639 int idx, eidx;
1641 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1642 return -1;
1643 idx = SUBPAGE_IDX(start);
1644 eidx = SUBPAGE_IDX(end);
1645 #if defined(DEBUG_SUBPAGE)
1646 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1647 mmio, start, end, idx, eidx, memory);
1648 #endif
1649 for (; idx <= eidx; idx++) {
1650 mmio->sub_section[idx] = section;
1653 return 0;
1656 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1658 subpage_t *mmio;
1660 mmio = g_malloc0(sizeof(subpage_t));
1662 mmio->as = as;
1663 mmio->base = base;
1664 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1665 "subpage", TARGET_PAGE_SIZE);
1666 mmio->iomem.subpage = true;
1667 #if defined(DEBUG_SUBPAGE)
1668 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1669 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1670 #endif
1671 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1673 return mmio;
1676 static uint16_t dummy_section(MemoryRegion *mr)
1678 MemoryRegionSection section = {
1679 .mr = mr,
1680 .offset_within_address_space = 0,
1681 .offset_within_region = 0,
1682 .size = int128_2_64(),
1685 return phys_section_add(&section);
1688 MemoryRegion *iotlb_to_region(hwaddr index)
1690 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1693 static void io_mem_init(void)
1695 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1696 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1697 "unassigned", UINT64_MAX);
1698 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1699 "notdirty", UINT64_MAX);
1700 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1701 "watch", UINT64_MAX);
1704 static void mem_begin(MemoryListener *listener)
1706 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1707 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1709 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1710 d->as = as;
1711 as->next_dispatch = d;
1714 static void mem_commit(MemoryListener *listener)
1716 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1717 AddressSpaceDispatch *cur = as->dispatch;
1718 AddressSpaceDispatch *next = as->next_dispatch;
1720 next->nodes = next_map.nodes;
1721 next->sections = next_map.sections;
1723 as->dispatch = next;
1724 g_free(cur);
1727 static void core_begin(MemoryListener *listener)
1729 uint16_t n;
1731 prev_map = g_new(PhysPageMap, 1);
1732 *prev_map = next_map;
1734 memset(&next_map, 0, sizeof(next_map));
1735 n = dummy_section(&io_mem_unassigned);
1736 assert(n == PHYS_SECTION_UNASSIGNED);
1737 n = dummy_section(&io_mem_notdirty);
1738 assert(n == PHYS_SECTION_NOTDIRTY);
1739 n = dummy_section(&io_mem_rom);
1740 assert(n == PHYS_SECTION_ROM);
1741 n = dummy_section(&io_mem_watch);
1742 assert(n == PHYS_SECTION_WATCH);
1745 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1746 * All AddressSpaceDispatch instances have switched to the next map.
1748 static void core_commit(MemoryListener *listener)
1750 phys_sections_free(prev_map);
1753 static void tcg_commit(MemoryListener *listener)
1755 CPUState *cpu;
1757 /* since each CPU stores ram addresses in its TLB cache, we must
1758 reset the modified entries */
1759 /* XXX: slow ! */
1760 CPU_FOREACH(cpu) {
1761 CPUArchState *env = cpu->env_ptr;
1763 tlb_flush(env, 1);
1767 static void core_log_global_start(MemoryListener *listener)
1769 cpu_physical_memory_set_dirty_tracking(1);
1772 static void core_log_global_stop(MemoryListener *listener)
1774 cpu_physical_memory_set_dirty_tracking(0);
1777 static MemoryListener core_memory_listener = {
1778 .begin = core_begin,
1779 .commit = core_commit,
1780 .log_global_start = core_log_global_start,
1781 .log_global_stop = core_log_global_stop,
1782 .priority = 1,
1785 static MemoryListener tcg_memory_listener = {
1786 .commit = tcg_commit,
1789 void address_space_init_dispatch(AddressSpace *as)
1791 as->dispatch = NULL;
1792 as->dispatch_listener = (MemoryListener) {
1793 .begin = mem_begin,
1794 .commit = mem_commit,
1795 .region_add = mem_add,
1796 .region_nop = mem_add,
1797 .priority = 0,
1799 memory_listener_register(&as->dispatch_listener, as);
1802 void address_space_destroy_dispatch(AddressSpace *as)
1804 AddressSpaceDispatch *d = as->dispatch;
1806 memory_listener_unregister(&as->dispatch_listener);
1807 g_free(d);
1808 as->dispatch = NULL;
1811 static void memory_map_init(void)
1813 system_memory = g_malloc(sizeof(*system_memory));
1814 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1815 address_space_init(&address_space_memory, system_memory, "memory");
1817 system_io = g_malloc(sizeof(*system_io));
1818 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
1819 65536);
1820 address_space_init(&address_space_io, system_io, "I/O");
1822 memory_listener_register(&core_memory_listener, &address_space_memory);
1823 if (tcg_enabled()) {
1824 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1828 MemoryRegion *get_system_memory(void)
1830 return system_memory;
1833 MemoryRegion *get_system_io(void)
1835 return system_io;
1838 #endif /* !defined(CONFIG_USER_ONLY) */
1840 /* physical memory access (slow version, mainly for debug) */
1841 #if defined(CONFIG_USER_ONLY)
1842 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1843 uint8_t *buf, int len, int is_write)
1845 int l, flags;
1846 target_ulong page;
1847 void * p;
1849 while (len > 0) {
1850 page = addr & TARGET_PAGE_MASK;
1851 l = (page + TARGET_PAGE_SIZE) - addr;
1852 if (l > len)
1853 l = len;
1854 flags = page_get_flags(page);
1855 if (!(flags & PAGE_VALID))
1856 return -1;
1857 if (is_write) {
1858 if (!(flags & PAGE_WRITE))
1859 return -1;
1860 /* XXX: this code should not depend on lock_user */
1861 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1862 return -1;
1863 memcpy(p, buf, l);
1864 unlock_user(p, addr, l);
1865 } else {
1866 if (!(flags & PAGE_READ))
1867 return -1;
1868 /* XXX: this code should not depend on lock_user */
1869 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1870 return -1;
1871 memcpy(buf, p, l);
1872 unlock_user(p, addr, 0);
1874 len -= l;
1875 buf += l;
1876 addr += l;
1878 return 0;
1881 #else
1883 static void invalidate_and_set_dirty(hwaddr addr,
1884 hwaddr length)
1886 if (!cpu_physical_memory_is_dirty(addr)) {
1887 /* invalidate code */
1888 tb_invalidate_phys_page_range(addr, addr + length, 0);
1889 /* set dirty bit */
1890 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1892 xen_modified_memory(addr, length);
1895 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1897 if (memory_region_is_ram(mr)) {
1898 return !(is_write && mr->readonly);
1900 if (memory_region_is_romd(mr)) {
1901 return !is_write;
1904 return false;
1907 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1909 unsigned access_size_max = mr->ops->valid.max_access_size;
1911 /* Regions are assumed to support 1-4 byte accesses unless
1912 otherwise specified. */
1913 if (access_size_max == 0) {
1914 access_size_max = 4;
1917 /* Bound the maximum access by the alignment of the address. */
1918 if (!mr->ops->impl.unaligned) {
1919 unsigned align_size_max = addr & -addr;
1920 if (align_size_max != 0 && align_size_max < access_size_max) {
1921 access_size_max = align_size_max;
1925 /* Don't attempt accesses larger than the maximum. */
1926 if (l > access_size_max) {
1927 l = access_size_max;
1929 if (l & (l - 1)) {
1930 l = 1 << (qemu_fls(l) - 1);
1933 return l;
1936 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1937 int len, bool is_write)
1939 hwaddr l;
1940 uint8_t *ptr;
1941 uint64_t val;
1942 hwaddr addr1;
1943 MemoryRegion *mr;
1944 bool error = false;
1946 while (len > 0) {
1947 l = len;
1948 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1950 if (is_write) {
1951 if (!memory_access_is_direct(mr, is_write)) {
1952 l = memory_access_size(mr, l, addr1);
1953 /* XXX: could force current_cpu to NULL to avoid
1954 potential bugs */
1955 switch (l) {
1956 case 8:
1957 /* 64 bit write access */
1958 val = ldq_p(buf);
1959 error |= io_mem_write(mr, addr1, val, 8);
1960 break;
1961 case 4:
1962 /* 32 bit write access */
1963 val = ldl_p(buf);
1964 error |= io_mem_write(mr, addr1, val, 4);
1965 break;
1966 case 2:
1967 /* 16 bit write access */
1968 val = lduw_p(buf);
1969 error |= io_mem_write(mr, addr1, val, 2);
1970 break;
1971 case 1:
1972 /* 8 bit write access */
1973 val = ldub_p(buf);
1974 error |= io_mem_write(mr, addr1, val, 1);
1975 break;
1976 default:
1977 abort();
1979 } else {
1980 addr1 += memory_region_get_ram_addr(mr);
1981 /* RAM case */
1982 ptr = qemu_get_ram_ptr(addr1);
1983 memcpy(ptr, buf, l);
1984 invalidate_and_set_dirty(addr1, l);
1986 } else {
1987 if (!memory_access_is_direct(mr, is_write)) {
1988 /* I/O case */
1989 l = memory_access_size(mr, l, addr1);
1990 switch (l) {
1991 case 8:
1992 /* 64 bit read access */
1993 error |= io_mem_read(mr, addr1, &val, 8);
1994 stq_p(buf, val);
1995 break;
1996 case 4:
1997 /* 32 bit read access */
1998 error |= io_mem_read(mr, addr1, &val, 4);
1999 stl_p(buf, val);
2000 break;
2001 case 2:
2002 /* 16 bit read access */
2003 error |= io_mem_read(mr, addr1, &val, 2);
2004 stw_p(buf, val);
2005 break;
2006 case 1:
2007 /* 8 bit read access */
2008 error |= io_mem_read(mr, addr1, &val, 1);
2009 stb_p(buf, val);
2010 break;
2011 default:
2012 abort();
2014 } else {
2015 /* RAM case */
2016 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2017 memcpy(buf, ptr, l);
2020 len -= l;
2021 buf += l;
2022 addr += l;
2025 return error;
2028 bool address_space_write(AddressSpace *as, hwaddr addr,
2029 const uint8_t *buf, int len)
2031 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2034 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2036 return address_space_rw(as, addr, buf, len, false);
2040 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2041 int len, int is_write)
2043 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2046 /* used for ROM loading : can write in RAM and ROM */
2047 void cpu_physical_memory_write_rom(hwaddr addr,
2048 const uint8_t *buf, int len)
2050 hwaddr l;
2051 uint8_t *ptr;
2052 hwaddr addr1;
2053 MemoryRegion *mr;
2055 while (len > 0) {
2056 l = len;
2057 mr = address_space_translate(&address_space_memory,
2058 addr, &addr1, &l, true);
2060 if (!(memory_region_is_ram(mr) ||
2061 memory_region_is_romd(mr))) {
2062 /* do nothing */
2063 } else {
2064 addr1 += memory_region_get_ram_addr(mr);
2065 /* ROM/RAM case */
2066 ptr = qemu_get_ram_ptr(addr1);
2067 memcpy(ptr, buf, l);
2068 invalidate_and_set_dirty(addr1, l);
2070 len -= l;
2071 buf += l;
2072 addr += l;
2076 typedef struct {
2077 MemoryRegion *mr;
2078 void *buffer;
2079 hwaddr addr;
2080 hwaddr len;
2081 } BounceBuffer;
2083 static BounceBuffer bounce;
2085 typedef struct MapClient {
2086 void *opaque;
2087 void (*callback)(void *opaque);
2088 QLIST_ENTRY(MapClient) link;
2089 } MapClient;
2091 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2092 = QLIST_HEAD_INITIALIZER(map_client_list);
2094 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2096 MapClient *client = g_malloc(sizeof(*client));
2098 client->opaque = opaque;
2099 client->callback = callback;
2100 QLIST_INSERT_HEAD(&map_client_list, client, link);
2101 return client;
2104 static void cpu_unregister_map_client(void *_client)
2106 MapClient *client = (MapClient *)_client;
2108 QLIST_REMOVE(client, link);
2109 g_free(client);
2112 static void cpu_notify_map_clients(void)
2114 MapClient *client;
2116 while (!QLIST_EMPTY(&map_client_list)) {
2117 client = QLIST_FIRST(&map_client_list);
2118 client->callback(client->opaque);
2119 cpu_unregister_map_client(client);
2123 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2125 MemoryRegion *mr;
2126 hwaddr l, xlat;
2128 while (len > 0) {
2129 l = len;
2130 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2131 if (!memory_access_is_direct(mr, is_write)) {
2132 l = memory_access_size(mr, l, addr);
2133 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2134 return false;
2138 len -= l;
2139 addr += l;
2141 return true;
2144 /* Map a physical memory region into a host virtual address.
2145 * May map a subset of the requested range, given by and returned in *plen.
2146 * May return NULL if resources needed to perform the mapping are exhausted.
2147 * Use only for reads OR writes - not for read-modify-write operations.
2148 * Use cpu_register_map_client() to know when retrying the map operation is
2149 * likely to succeed.
2151 void *address_space_map(AddressSpace *as,
2152 hwaddr addr,
2153 hwaddr *plen,
2154 bool is_write)
2156 hwaddr len = *plen;
2157 hwaddr done = 0;
2158 hwaddr l, xlat, base;
2159 MemoryRegion *mr, *this_mr;
2160 ram_addr_t raddr;
2162 if (len == 0) {
2163 return NULL;
2166 l = len;
2167 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2168 if (!memory_access_is_direct(mr, is_write)) {
2169 if (bounce.buffer) {
2170 return NULL;
2172 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2173 bounce.addr = addr;
2174 bounce.len = l;
2176 memory_region_ref(mr);
2177 bounce.mr = mr;
2178 if (!is_write) {
2179 address_space_read(as, addr, bounce.buffer, l);
2182 *plen = l;
2183 return bounce.buffer;
2186 base = xlat;
2187 raddr = memory_region_get_ram_addr(mr);
2189 for (;;) {
2190 len -= l;
2191 addr += l;
2192 done += l;
2193 if (len == 0) {
2194 break;
2197 l = len;
2198 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2199 if (this_mr != mr || xlat != base + done) {
2200 break;
2204 memory_region_ref(mr);
2205 *plen = done;
2206 return qemu_ram_ptr_length(raddr + base, plen);
2209 /* Unmaps a memory region previously mapped by address_space_map().
2210 * Will also mark the memory as dirty if is_write == 1. access_len gives
2211 * the amount of memory that was actually read or written by the caller.
2213 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2214 int is_write, hwaddr access_len)
2216 if (buffer != bounce.buffer) {
2217 MemoryRegion *mr;
2218 ram_addr_t addr1;
2220 mr = qemu_ram_addr_from_host(buffer, &addr1);
2221 assert(mr != NULL);
2222 if (is_write) {
2223 while (access_len) {
2224 unsigned l;
2225 l = TARGET_PAGE_SIZE;
2226 if (l > access_len)
2227 l = access_len;
2228 invalidate_and_set_dirty(addr1, l);
2229 addr1 += l;
2230 access_len -= l;
2233 if (xen_enabled()) {
2234 xen_invalidate_map_cache_entry(buffer);
2236 memory_region_unref(mr);
2237 return;
2239 if (is_write) {
2240 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2242 qemu_vfree(bounce.buffer);
2243 bounce.buffer = NULL;
2244 memory_region_unref(bounce.mr);
2245 cpu_notify_map_clients();
2248 void *cpu_physical_memory_map(hwaddr addr,
2249 hwaddr *plen,
2250 int is_write)
2252 return address_space_map(&address_space_memory, addr, plen, is_write);
2255 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2256 int is_write, hwaddr access_len)
2258 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2261 /* warning: addr must be aligned */
2262 static inline uint32_t ldl_phys_internal(hwaddr addr,
2263 enum device_endian endian)
2265 uint8_t *ptr;
2266 uint64_t val;
2267 MemoryRegion *mr;
2268 hwaddr l = 4;
2269 hwaddr addr1;
2271 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2272 false);
2273 if (l < 4 || !memory_access_is_direct(mr, false)) {
2274 /* I/O case */
2275 io_mem_read(mr, addr1, &val, 4);
2276 #if defined(TARGET_WORDS_BIGENDIAN)
2277 if (endian == DEVICE_LITTLE_ENDIAN) {
2278 val = bswap32(val);
2280 #else
2281 if (endian == DEVICE_BIG_ENDIAN) {
2282 val = bswap32(val);
2284 #endif
2285 } else {
2286 /* RAM case */
2287 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2288 & TARGET_PAGE_MASK)
2289 + addr1);
2290 switch (endian) {
2291 case DEVICE_LITTLE_ENDIAN:
2292 val = ldl_le_p(ptr);
2293 break;
2294 case DEVICE_BIG_ENDIAN:
2295 val = ldl_be_p(ptr);
2296 break;
2297 default:
2298 val = ldl_p(ptr);
2299 break;
2302 return val;
2305 uint32_t ldl_phys(hwaddr addr)
2307 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2310 uint32_t ldl_le_phys(hwaddr addr)
2312 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2315 uint32_t ldl_be_phys(hwaddr addr)
2317 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2320 /* warning: addr must be aligned */
2321 static inline uint64_t ldq_phys_internal(hwaddr addr,
2322 enum device_endian endian)
2324 uint8_t *ptr;
2325 uint64_t val;
2326 MemoryRegion *mr;
2327 hwaddr l = 8;
2328 hwaddr addr1;
2330 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2331 false);
2332 if (l < 8 || !memory_access_is_direct(mr, false)) {
2333 /* I/O case */
2334 io_mem_read(mr, addr1, &val, 8);
2335 #if defined(TARGET_WORDS_BIGENDIAN)
2336 if (endian == DEVICE_LITTLE_ENDIAN) {
2337 val = bswap64(val);
2339 #else
2340 if (endian == DEVICE_BIG_ENDIAN) {
2341 val = bswap64(val);
2343 #endif
2344 } else {
2345 /* RAM case */
2346 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2347 & TARGET_PAGE_MASK)
2348 + addr1);
2349 switch (endian) {
2350 case DEVICE_LITTLE_ENDIAN:
2351 val = ldq_le_p(ptr);
2352 break;
2353 case DEVICE_BIG_ENDIAN:
2354 val = ldq_be_p(ptr);
2355 break;
2356 default:
2357 val = ldq_p(ptr);
2358 break;
2361 return val;
2364 uint64_t ldq_phys(hwaddr addr)
2366 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2369 uint64_t ldq_le_phys(hwaddr addr)
2371 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2374 uint64_t ldq_be_phys(hwaddr addr)
2376 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2379 /* XXX: optimize */
2380 uint32_t ldub_phys(hwaddr addr)
2382 uint8_t val;
2383 cpu_physical_memory_read(addr, &val, 1);
2384 return val;
2387 /* warning: addr must be aligned */
2388 static inline uint32_t lduw_phys_internal(hwaddr addr,
2389 enum device_endian endian)
2391 uint8_t *ptr;
2392 uint64_t val;
2393 MemoryRegion *mr;
2394 hwaddr l = 2;
2395 hwaddr addr1;
2397 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2398 false);
2399 if (l < 2 || !memory_access_is_direct(mr, false)) {
2400 /* I/O case */
2401 io_mem_read(mr, addr1, &val, 2);
2402 #if defined(TARGET_WORDS_BIGENDIAN)
2403 if (endian == DEVICE_LITTLE_ENDIAN) {
2404 val = bswap16(val);
2406 #else
2407 if (endian == DEVICE_BIG_ENDIAN) {
2408 val = bswap16(val);
2410 #endif
2411 } else {
2412 /* RAM case */
2413 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2414 & TARGET_PAGE_MASK)
2415 + addr1);
2416 switch (endian) {
2417 case DEVICE_LITTLE_ENDIAN:
2418 val = lduw_le_p(ptr);
2419 break;
2420 case DEVICE_BIG_ENDIAN:
2421 val = lduw_be_p(ptr);
2422 break;
2423 default:
2424 val = lduw_p(ptr);
2425 break;
2428 return val;
2431 uint32_t lduw_phys(hwaddr addr)
2433 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2436 uint32_t lduw_le_phys(hwaddr addr)
2438 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2441 uint32_t lduw_be_phys(hwaddr addr)
2443 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2446 /* warning: addr must be aligned. The ram page is not masked as dirty
2447 and the code inside is not invalidated. It is useful if the dirty
2448 bits are used to track modified PTEs */
2449 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2451 uint8_t *ptr;
2452 MemoryRegion *mr;
2453 hwaddr l = 4;
2454 hwaddr addr1;
2456 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2457 true);
2458 if (l < 4 || !memory_access_is_direct(mr, true)) {
2459 io_mem_write(mr, addr1, val, 4);
2460 } else {
2461 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2462 ptr = qemu_get_ram_ptr(addr1);
2463 stl_p(ptr, val);
2465 if (unlikely(in_migration)) {
2466 if (!cpu_physical_memory_is_dirty(addr1)) {
2467 /* invalidate code */
2468 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2469 /* set dirty bit */
2470 cpu_physical_memory_set_dirty_flags(
2471 addr1, (0xff & ~CODE_DIRTY_FLAG));
2477 /* warning: addr must be aligned */
2478 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2479 enum device_endian endian)
2481 uint8_t *ptr;
2482 MemoryRegion *mr;
2483 hwaddr l = 4;
2484 hwaddr addr1;
2486 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2487 true);
2488 if (l < 4 || !memory_access_is_direct(mr, true)) {
2489 #if defined(TARGET_WORDS_BIGENDIAN)
2490 if (endian == DEVICE_LITTLE_ENDIAN) {
2491 val = bswap32(val);
2493 #else
2494 if (endian == DEVICE_BIG_ENDIAN) {
2495 val = bswap32(val);
2497 #endif
2498 io_mem_write(mr, addr1, val, 4);
2499 } else {
2500 /* RAM case */
2501 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2502 ptr = qemu_get_ram_ptr(addr1);
2503 switch (endian) {
2504 case DEVICE_LITTLE_ENDIAN:
2505 stl_le_p(ptr, val);
2506 break;
2507 case DEVICE_BIG_ENDIAN:
2508 stl_be_p(ptr, val);
2509 break;
2510 default:
2511 stl_p(ptr, val);
2512 break;
2514 invalidate_and_set_dirty(addr1, 4);
2518 void stl_phys(hwaddr addr, uint32_t val)
2520 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2523 void stl_le_phys(hwaddr addr, uint32_t val)
2525 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2528 void stl_be_phys(hwaddr addr, uint32_t val)
2530 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2533 /* XXX: optimize */
2534 void stb_phys(hwaddr addr, uint32_t val)
2536 uint8_t v = val;
2537 cpu_physical_memory_write(addr, &v, 1);
2540 /* warning: addr must be aligned */
2541 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2542 enum device_endian endian)
2544 uint8_t *ptr;
2545 MemoryRegion *mr;
2546 hwaddr l = 2;
2547 hwaddr addr1;
2549 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2550 true);
2551 if (l < 2 || !memory_access_is_direct(mr, true)) {
2552 #if defined(TARGET_WORDS_BIGENDIAN)
2553 if (endian == DEVICE_LITTLE_ENDIAN) {
2554 val = bswap16(val);
2556 #else
2557 if (endian == DEVICE_BIG_ENDIAN) {
2558 val = bswap16(val);
2560 #endif
2561 io_mem_write(mr, addr1, val, 2);
2562 } else {
2563 /* RAM case */
2564 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2565 ptr = qemu_get_ram_ptr(addr1);
2566 switch (endian) {
2567 case DEVICE_LITTLE_ENDIAN:
2568 stw_le_p(ptr, val);
2569 break;
2570 case DEVICE_BIG_ENDIAN:
2571 stw_be_p(ptr, val);
2572 break;
2573 default:
2574 stw_p(ptr, val);
2575 break;
2577 invalidate_and_set_dirty(addr1, 2);
2581 void stw_phys(hwaddr addr, uint32_t val)
2583 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2586 void stw_le_phys(hwaddr addr, uint32_t val)
2588 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2591 void stw_be_phys(hwaddr addr, uint32_t val)
2593 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2596 /* XXX: optimize */
2597 void stq_phys(hwaddr addr, uint64_t val)
2599 val = tswap64(val);
2600 cpu_physical_memory_write(addr, &val, 8);
2603 void stq_le_phys(hwaddr addr, uint64_t val)
2605 val = cpu_to_le64(val);
2606 cpu_physical_memory_write(addr, &val, 8);
2609 void stq_be_phys(hwaddr addr, uint64_t val)
2611 val = cpu_to_be64(val);
2612 cpu_physical_memory_write(addr, &val, 8);
2615 /* virtual memory access for debug (includes writing to ROM) */
2616 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2617 uint8_t *buf, int len, int is_write)
2619 int l;
2620 hwaddr phys_addr;
2621 target_ulong page;
2623 while (len > 0) {
2624 page = addr & TARGET_PAGE_MASK;
2625 phys_addr = cpu_get_phys_page_debug(cpu, page);
2626 /* if no physical page mapped, return an error */
2627 if (phys_addr == -1)
2628 return -1;
2629 l = (page + TARGET_PAGE_SIZE) - addr;
2630 if (l > len)
2631 l = len;
2632 phys_addr += (addr & ~TARGET_PAGE_MASK);
2633 if (is_write)
2634 cpu_physical_memory_write_rom(phys_addr, buf, l);
2635 else
2636 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2637 len -= l;
2638 buf += l;
2639 addr += l;
2641 return 0;
2643 #endif
2645 #if !defined(CONFIG_USER_ONLY)
2648 * A helper function for the _utterly broken_ virtio device model to find out if
2649 * it's running on a big endian machine. Don't do this at home kids!
2651 bool virtio_is_big_endian(void);
2652 bool virtio_is_big_endian(void)
2654 #if defined(TARGET_WORDS_BIGENDIAN)
2655 return true;
2656 #else
2657 return false;
2658 #endif
2661 #endif
2663 #ifndef CONFIG_USER_ONLY
2664 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2666 MemoryRegion*mr;
2667 hwaddr l = 1;
2669 mr = address_space_translate(&address_space_memory,
2670 phys_addr, &phys_addr, &l, false);
2672 return !(memory_region_is_ram(mr) ||
2673 memory_region_is_romd(mr));
2676 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2678 RAMBlock *block;
2680 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2681 func(block->host, block->offset, block->length, opaque);
2684 #endif