arm: Fix null pointer crash
[qemu/ar7.git] / exec.c
blobd57e2ae480ca7eb62c843f57d15d68ffc4cd29f7
1 /*
2 * Virtual page mapping
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "config.h"
20 #ifndef _WIN32
21 #include <sys/types.h>
22 #include <sys/mman.h>
23 #endif
25 #include "qemu-common.h"
26 #include "cpu.h"
27 #include "tcg.h"
28 #include "hw/hw.h"
29 #include "hw/qdev.h"
30 #include "qemu/osdep.h"
31 #include "sysemu/kvm.h"
32 #include "sysemu/sysemu.h"
33 #include "hw/xen/xen.h"
34 #include "qemu/timer.h"
35 #include "qemu/config-file.h"
36 #include "qemu/error-report.h"
37 #include "exec/memory.h"
38 #include "sysemu/dma.h"
39 #include "exec/address-spaces.h"
40 #if defined(CONFIG_USER_ONLY)
41 #include <qemu.h>
42 #else /* !CONFIG_USER_ONLY */
43 #include "sysemu/xen-mapcache.h"
44 #include "trace.h"
45 #endif
46 #include "exec/cpu-all.h"
48 #include "exec/cputlb.h"
49 #include "translate-all.h"
51 #include "exec/memory-internal.h"
52 #include "exec/ram_addr.h"
54 #include "qemu/range.h"
56 //#define DEBUG_SUBPAGE
58 #if !defined(CONFIG_USER_ONLY)
59 static bool in_migration;
61 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
63 static MemoryRegion *system_memory;
64 static MemoryRegion *system_io;
66 AddressSpace address_space_io;
67 AddressSpace address_space_memory;
69 MemoryRegion io_mem_rom, io_mem_notdirty;
70 static MemoryRegion io_mem_unassigned;
72 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
73 #define RAM_PREALLOC (1 << 0)
75 /* RAM is mmap-ed with MAP_SHARED */
76 #define RAM_SHARED (1 << 1)
78 #endif
80 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
81 /* current CPU in the current thread. It is only valid inside
82 cpu_exec() */
83 DEFINE_TLS(CPUState *, current_cpu);
84 /* 0 = Do not count executed instructions.
85 1 = Precise instruction counting.
86 2 = Adaptive rate instruction counting. */
87 int use_icount;
89 #if !defined(CONFIG_USER_ONLY)
91 typedef struct PhysPageEntry PhysPageEntry;
93 struct PhysPageEntry {
94 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
95 uint32_t skip : 6;
96 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
97 uint32_t ptr : 26;
100 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
102 /* Size of the L2 (and L3, etc) page tables. */
103 #define ADDR_SPACE_BITS 64
105 #define P_L2_BITS 9
106 #define P_L2_SIZE (1 << P_L2_BITS)
108 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
110 typedef PhysPageEntry Node[P_L2_SIZE];
112 typedef struct PhysPageMap {
113 unsigned sections_nb;
114 unsigned sections_nb_alloc;
115 unsigned nodes_nb;
116 unsigned nodes_nb_alloc;
117 Node *nodes;
118 MemoryRegionSection *sections;
119 } PhysPageMap;
121 struct AddressSpaceDispatch {
122 /* This is a multi-level map on the physical address space.
123 * The bottom level has pointers to MemoryRegionSections.
125 PhysPageEntry phys_map;
126 PhysPageMap map;
127 AddressSpace *as;
130 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
131 typedef struct subpage_t {
132 MemoryRegion iomem;
133 AddressSpace *as;
134 hwaddr base;
135 uint16_t sub_section[TARGET_PAGE_SIZE];
136 } subpage_t;
138 #define PHYS_SECTION_UNASSIGNED 0
139 #define PHYS_SECTION_NOTDIRTY 1
140 #define PHYS_SECTION_ROM 2
141 #define PHYS_SECTION_WATCH 3
143 static void io_mem_init(void);
144 static void memory_map_init(void);
145 static void tcg_commit(MemoryListener *listener);
147 static MemoryRegion io_mem_watch;
148 #endif
150 #if !defined(CONFIG_USER_ONLY)
152 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
154 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
155 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
156 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
157 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
161 static uint32_t phys_map_node_alloc(PhysPageMap *map)
163 unsigned i;
164 uint32_t ret;
166 ret = map->nodes_nb++;
167 assert(ret != PHYS_MAP_NODE_NIL);
168 assert(ret != map->nodes_nb_alloc);
169 for (i = 0; i < P_L2_SIZE; ++i) {
170 map->nodes[ret][i].skip = 1;
171 map->nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
173 return ret;
176 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
177 hwaddr *index, hwaddr *nb, uint16_t leaf,
178 int level)
180 PhysPageEntry *p;
181 int i;
182 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
184 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
185 lp->ptr = phys_map_node_alloc(map);
186 p = map->nodes[lp->ptr];
187 if (level == 0) {
188 for (i = 0; i < P_L2_SIZE; i++) {
189 p[i].skip = 0;
190 p[i].ptr = PHYS_SECTION_UNASSIGNED;
193 } else {
194 p = map->nodes[lp->ptr];
196 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
198 while (*nb && lp < &p[P_L2_SIZE]) {
199 if ((*index & (step - 1)) == 0 && *nb >= step) {
200 lp->skip = 0;
201 lp->ptr = leaf;
202 *index += step;
203 *nb -= step;
204 } else {
205 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
207 ++lp;
211 static void phys_page_set(AddressSpaceDispatch *d,
212 hwaddr index, hwaddr nb,
213 uint16_t leaf)
215 /* Wildly overreserve - it doesn't matter much. */
216 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
218 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
221 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
222 * and update our entry so we can skip it and go directly to the destination.
224 static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)
226 unsigned valid_ptr = P_L2_SIZE;
227 int valid = 0;
228 PhysPageEntry *p;
229 int i;
231 if (lp->ptr == PHYS_MAP_NODE_NIL) {
232 return;
235 p = nodes[lp->ptr];
236 for (i = 0; i < P_L2_SIZE; i++) {
237 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
238 continue;
241 valid_ptr = i;
242 valid++;
243 if (p[i].skip) {
244 phys_page_compact(&p[i], nodes, compacted);
248 /* We can only compress if there's only one child. */
249 if (valid != 1) {
250 return;
253 assert(valid_ptr < P_L2_SIZE);
255 /* Don't compress if it won't fit in the # of bits we have. */
256 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
257 return;
260 lp->ptr = p[valid_ptr].ptr;
261 if (!p[valid_ptr].skip) {
262 /* If our only child is a leaf, make this a leaf. */
263 /* By design, we should have made this node a leaf to begin with so we
264 * should never reach here.
265 * But since it's so simple to handle this, let's do it just in case we
266 * change this rule.
268 lp->skip = 0;
269 } else {
270 lp->skip += p[valid_ptr].skip;
274 static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
276 DECLARE_BITMAP(compacted, nodes_nb);
278 if (d->phys_map.skip) {
279 phys_page_compact(&d->phys_map, d->map.nodes, compacted);
283 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
284 Node *nodes, MemoryRegionSection *sections)
286 PhysPageEntry *p;
287 hwaddr index = addr >> TARGET_PAGE_BITS;
288 int i;
290 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
291 if (lp.ptr == PHYS_MAP_NODE_NIL) {
292 return &sections[PHYS_SECTION_UNASSIGNED];
294 p = nodes[lp.ptr];
295 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
298 if (sections[lp.ptr].size.hi ||
299 range_covers_byte(sections[lp.ptr].offset_within_address_space,
300 sections[lp.ptr].size.lo, addr)) {
301 return &sections[lp.ptr];
302 } else {
303 return &sections[PHYS_SECTION_UNASSIGNED];
307 bool memory_region_is_unassigned(MemoryRegion *mr)
309 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
310 && mr != &io_mem_watch;
313 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
314 hwaddr addr,
315 bool resolve_subpage)
317 MemoryRegionSection *section;
318 subpage_t *subpage;
320 section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);
321 if (resolve_subpage && section->mr->subpage) {
322 subpage = container_of(section->mr, subpage_t, iomem);
323 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
325 return section;
328 static MemoryRegionSection *
329 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
330 hwaddr *plen, bool resolve_subpage)
332 MemoryRegionSection *section;
333 Int128 diff;
335 section = address_space_lookup_region(d, addr, resolve_subpage);
336 /* Compute offset within MemoryRegionSection */
337 addr -= section->offset_within_address_space;
339 /* Compute offset within MemoryRegion */
340 *xlat = addr + section->offset_within_region;
342 diff = int128_sub(section->mr->size, int128_make64(addr));
343 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
344 return section;
347 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
349 if (memory_region_is_ram(mr)) {
350 return !(is_write && mr->readonly);
352 if (memory_region_is_romd(mr)) {
353 return !is_write;
356 return false;
359 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
360 hwaddr *xlat, hwaddr *plen,
361 bool is_write)
363 IOMMUTLBEntry iotlb;
364 MemoryRegionSection *section;
365 MemoryRegion *mr;
366 hwaddr len = *plen;
368 for (;;) {
369 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
370 mr = section->mr;
372 if (!mr->iommu_ops) {
373 break;
376 iotlb = mr->iommu_ops->translate(mr, addr, is_write);
377 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
378 | (addr & iotlb.addr_mask));
379 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
380 if (!(iotlb.perm & (1 << is_write))) {
381 mr = &io_mem_unassigned;
382 break;
385 as = iotlb.target_as;
388 if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
389 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
390 len = MIN(page, len);
393 *plen = len;
394 *xlat = addr;
395 return mr;
398 MemoryRegionSection *
399 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
400 hwaddr *plen)
402 MemoryRegionSection *section;
403 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
405 assert(!section->mr->iommu_ops);
406 return section;
408 #endif
410 void cpu_exec_init_all(void)
412 #if !defined(CONFIG_USER_ONLY)
413 qemu_mutex_init(&ram_list.mutex);
414 memory_map_init();
415 io_mem_init();
416 #endif
419 #if !defined(CONFIG_USER_ONLY)
421 static int cpu_common_post_load(void *opaque, int version_id)
423 CPUState *cpu = opaque;
425 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
426 version_id is increased. */
427 cpu->interrupt_request &= ~0x01;
428 tlb_flush(cpu, 1);
430 return 0;
433 static int cpu_common_pre_load(void *opaque)
435 CPUState *cpu = opaque;
437 cpu->exception_index = 0;
439 return 0;
442 static bool cpu_common_exception_index_needed(void *opaque)
444 CPUState *cpu = opaque;
446 return cpu->exception_index != 0;
449 static const VMStateDescription vmstate_cpu_common_exception_index = {
450 .name = "cpu_common/exception_index",
451 .version_id = 1,
452 .minimum_version_id = 1,
453 .fields = (VMStateField[]) {
454 VMSTATE_INT32(exception_index, CPUState),
455 VMSTATE_END_OF_LIST()
459 const VMStateDescription vmstate_cpu_common = {
460 .name = "cpu_common",
461 .version_id = 1,
462 .minimum_version_id = 1,
463 .pre_load = cpu_common_pre_load,
464 .post_load = cpu_common_post_load,
465 .fields = (VMStateField[]) {
466 VMSTATE_UINT32(halted, CPUState),
467 VMSTATE_UINT32(interrupt_request, CPUState),
468 VMSTATE_END_OF_LIST()
470 .subsections = (VMStateSubsection[]) {
472 .vmsd = &vmstate_cpu_common_exception_index,
473 .needed = cpu_common_exception_index_needed,
474 } , {
475 /* empty */
480 #endif
482 CPUState *qemu_get_cpu(int index)
484 CPUState *cpu;
486 CPU_FOREACH(cpu) {
487 if (cpu->cpu_index == index) {
488 return cpu;
492 return NULL;
495 #if !defined(CONFIG_USER_ONLY)
496 void tcg_cpu_address_space_init(CPUState *cpu, AddressSpace *as)
498 /* We only support one address space per cpu at the moment. */
499 assert(cpu->as == as);
501 if (cpu->tcg_as_listener) {
502 memory_listener_unregister(cpu->tcg_as_listener);
503 } else {
504 cpu->tcg_as_listener = g_new0(MemoryListener, 1);
506 cpu->tcg_as_listener->commit = tcg_commit;
507 memory_listener_register(cpu->tcg_as_listener, as);
509 #endif
511 void cpu_exec_init(CPUArchState *env)
513 CPUState *cpu = ENV_GET_CPU(env);
514 CPUClass *cc = CPU_GET_CLASS(cpu);
515 CPUState *some_cpu;
516 int cpu_index;
518 #ifdef TARGET_WORDS_BIGENDIAN
519 cpu->bigendian = true;
520 #else
521 cpu->bigendian = false;
522 #endif
524 #if defined(CONFIG_USER_ONLY)
525 cpu_list_lock();
526 #endif
527 cpu_index = 0;
528 CPU_FOREACH(some_cpu) {
529 cpu_index++;
531 cpu->cpu_index = cpu_index;
532 cpu->numa_node = 0;
533 QTAILQ_INIT(&cpu->breakpoints);
534 QTAILQ_INIT(&cpu->watchpoints);
535 #ifndef CONFIG_USER_ONLY
536 cpu->as = &address_space_memory;
537 cpu->thread_id = qemu_get_thread_id();
538 #endif
539 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
540 #if defined(CONFIG_USER_ONLY)
541 cpu_list_unlock();
542 #endif
543 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
544 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
546 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
547 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
548 cpu_save, cpu_load, env);
549 assert(cc->vmsd == NULL);
550 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
551 #endif
552 if (cc->vmsd != NULL) {
553 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
557 #if defined(TARGET_HAS_ICE)
558 #if defined(CONFIG_USER_ONLY)
559 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
561 tb_invalidate_phys_page_range(pc, pc + 1, 0);
563 #else
564 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
566 hwaddr phys = cpu_get_phys_page_debug(cpu, pc);
567 if (phys != -1) {
568 tb_invalidate_phys_addr(cpu->as,
569 phys | (pc & ~TARGET_PAGE_MASK));
572 #endif
573 #endif /* TARGET_HAS_ICE */
575 #if defined(CONFIG_USER_ONLY)
576 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
581 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
582 int flags)
584 return -ENOSYS;
587 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
591 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
592 int flags, CPUWatchpoint **watchpoint)
594 return -ENOSYS;
596 #else
597 /* Add a watchpoint. */
598 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
599 int flags, CPUWatchpoint **watchpoint)
601 CPUWatchpoint *wp;
603 /* forbid ranges which are empty or run off the end of the address space */
604 if (len == 0 || (addr + len - 1) < addr) {
605 error_report("tried to set invalid watchpoint at %"
606 VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
607 return -EINVAL;
609 wp = g_malloc(sizeof(*wp));
611 wp->vaddr = addr;
612 wp->len = len;
613 wp->flags = flags;
615 /* keep all GDB-injected watchpoints in front */
616 if (flags & BP_GDB) {
617 QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
618 } else {
619 QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
622 tlb_flush_page(cpu, addr);
624 if (watchpoint)
625 *watchpoint = wp;
626 return 0;
629 /* Remove a specific watchpoint. */
630 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
631 int flags)
633 CPUWatchpoint *wp;
635 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
636 if (addr == wp->vaddr && len == wp->len
637 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
638 cpu_watchpoint_remove_by_ref(cpu, wp);
639 return 0;
642 return -ENOENT;
645 /* Remove a specific watchpoint by reference. */
646 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
648 QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
650 tlb_flush_page(cpu, watchpoint->vaddr);
652 g_free(watchpoint);
655 /* Remove all matching watchpoints. */
656 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
658 CPUWatchpoint *wp, *next;
660 QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
661 if (wp->flags & mask) {
662 cpu_watchpoint_remove_by_ref(cpu, wp);
667 /* Return true if this watchpoint address matches the specified
668 * access (ie the address range covered by the watchpoint overlaps
669 * partially or completely with the address range covered by the
670 * access).
672 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
673 vaddr addr,
674 vaddr len)
676 /* We know the lengths are non-zero, but a little caution is
677 * required to avoid errors in the case where the range ends
678 * exactly at the top of the address space and so addr + len
679 * wraps round to zero.
681 vaddr wpend = wp->vaddr + wp->len - 1;
682 vaddr addrend = addr + len - 1;
684 return !(addr > wpend || wp->vaddr > addrend);
687 #endif
689 /* Add a breakpoint. */
690 int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
691 CPUBreakpoint **breakpoint)
693 #if defined(TARGET_HAS_ICE)
694 CPUBreakpoint *bp;
696 bp = g_malloc(sizeof(*bp));
698 bp->pc = pc;
699 bp->flags = flags;
701 /* keep all GDB-injected breakpoints in front */
702 if (flags & BP_GDB) {
703 QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
704 } else {
705 QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
708 breakpoint_invalidate(cpu, pc);
710 if (breakpoint) {
711 *breakpoint = bp;
713 return 0;
714 #else
715 return -ENOSYS;
716 #endif
719 /* Remove a specific breakpoint. */
720 int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
722 #if defined(TARGET_HAS_ICE)
723 CPUBreakpoint *bp;
725 QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
726 if (bp->pc == pc && bp->flags == flags) {
727 cpu_breakpoint_remove_by_ref(cpu, bp);
728 return 0;
731 return -ENOENT;
732 #else
733 return -ENOSYS;
734 #endif
737 /* Remove a specific breakpoint by reference. */
738 void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
740 #if defined(TARGET_HAS_ICE)
741 QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
743 breakpoint_invalidate(cpu, breakpoint->pc);
745 g_free(breakpoint);
746 #endif
749 /* Remove all matching breakpoints. */
750 void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
752 #if defined(TARGET_HAS_ICE)
753 CPUBreakpoint *bp, *next;
755 QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
756 if (bp->flags & mask) {
757 cpu_breakpoint_remove_by_ref(cpu, bp);
760 #endif
763 /* enable or disable single step mode. EXCP_DEBUG is returned by the
764 CPU loop after each instruction */
765 void cpu_single_step(CPUState *cpu, int enabled)
767 #if defined(TARGET_HAS_ICE)
768 if (cpu->singlestep_enabled != enabled) {
769 cpu->singlestep_enabled = enabled;
770 if (kvm_enabled()) {
771 kvm_update_guest_debug(cpu, 0);
772 } else {
773 /* must flush all the translated code to avoid inconsistencies */
774 /* XXX: only flush what is necessary */
775 CPUArchState *env = cpu->env_ptr;
776 tb_flush(env);
779 #endif
782 void QEMU_NORETURN cpu_abort(CPUState *cpu, const char *fmt, ...)
784 va_list ap;
785 va_list ap2;
787 va_start(ap, fmt);
788 va_copy(ap2, ap);
789 fprintf(stderr, "qemu: fatal: ");
790 vfprintf(stderr, fmt, ap);
791 fprintf(stderr, "\n");
792 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
793 if (qemu_log_enabled()) {
794 qemu_log("qemu: fatal: ");
795 qemu_log_vprintf(fmt, ap2);
796 qemu_log("\n");
797 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
798 qemu_log_flush();
799 qemu_log_close();
801 va_end(ap2);
802 va_end(ap);
803 #if defined(CONFIG_USER_ONLY)
805 struct sigaction act;
806 sigfillset(&act.sa_mask);
807 act.sa_handler = SIG_DFL;
808 sigaction(SIGABRT, &act, NULL);
810 #endif
811 abort();
814 #if !defined(CONFIG_USER_ONLY)
815 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
817 RAMBlock *block;
819 /* The list is protected by the iothread lock here. */
820 block = ram_list.mru_block;
821 if (block && addr - block->offset < block->length) {
822 goto found;
824 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
825 if (addr - block->offset < block->length) {
826 goto found;
830 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
831 abort();
833 found:
834 ram_list.mru_block = block;
835 return block;
838 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
840 ram_addr_t start1;
841 RAMBlock *block;
842 ram_addr_t end;
844 end = TARGET_PAGE_ALIGN(start + length);
845 start &= TARGET_PAGE_MASK;
847 block = qemu_get_ram_block(start);
848 assert(block == qemu_get_ram_block(end - 1));
849 start1 = (uintptr_t)block->host + (start - block->offset);
850 cpu_tlb_reset_dirty_all(start1, length);
853 /* Note: start and end must be within the same ram block. */
854 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length,
855 unsigned client)
857 if (length == 0)
858 return;
859 cpu_physical_memory_clear_dirty_range(start, length, client);
861 if (tcg_enabled()) {
862 tlb_reset_dirty_range_all(start, length);
866 static void cpu_physical_memory_set_dirty_tracking(bool enable)
868 in_migration = enable;
871 hwaddr memory_region_section_get_iotlb(CPUState *cpu,
872 MemoryRegionSection *section,
873 target_ulong vaddr,
874 hwaddr paddr, hwaddr xlat,
875 int prot,
876 target_ulong *address)
878 hwaddr iotlb;
879 CPUWatchpoint *wp;
881 if (memory_region_is_ram(section->mr)) {
882 /* Normal RAM. */
883 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
884 + xlat;
885 if (!section->readonly) {
886 iotlb |= PHYS_SECTION_NOTDIRTY;
887 } else {
888 iotlb |= PHYS_SECTION_ROM;
890 } else {
891 iotlb = section - section->address_space->dispatch->map.sections;
892 iotlb += xlat;
895 /* Make accesses to pages with watchpoints go via the
896 watchpoint trap routines. */
897 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
898 if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
899 /* Avoid trapping reads of pages with a write breakpoint. */
900 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
901 iotlb = PHYS_SECTION_WATCH + paddr;
902 *address |= TLB_MMIO;
903 break;
908 return iotlb;
910 #endif /* defined(CONFIG_USER_ONLY) */
912 #if !defined(CONFIG_USER_ONLY)
914 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
915 uint16_t section);
916 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
918 static void *(*phys_mem_alloc)(size_t size) = qemu_anon_ram_alloc;
921 * Set a custom physical guest memory alloator.
922 * Accelerators with unusual needs may need this. Hopefully, we can
923 * get rid of it eventually.
925 void phys_mem_set_alloc(void *(*alloc)(size_t))
927 phys_mem_alloc = alloc;
930 static uint16_t phys_section_add(PhysPageMap *map,
931 MemoryRegionSection *section)
933 /* The physical section number is ORed with a page-aligned
934 * pointer to produce the iotlb entries. Thus it should
935 * never overflow into the page-aligned value.
937 assert(map->sections_nb < TARGET_PAGE_SIZE);
939 if (map->sections_nb == map->sections_nb_alloc) {
940 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
941 map->sections = g_renew(MemoryRegionSection, map->sections,
942 map->sections_nb_alloc);
944 map->sections[map->sections_nb] = *section;
945 memory_region_ref(section->mr);
946 return map->sections_nb++;
949 static void phys_section_destroy(MemoryRegion *mr)
951 memory_region_unref(mr);
953 if (mr->subpage) {
954 subpage_t *subpage = container_of(mr, subpage_t, iomem);
955 object_unref(OBJECT(&subpage->iomem));
956 g_free(subpage);
960 static void phys_sections_free(PhysPageMap *map)
962 while (map->sections_nb > 0) {
963 MemoryRegionSection *section = &map->sections[--map->sections_nb];
964 phys_section_destroy(section->mr);
966 g_free(map->sections);
967 g_free(map->nodes);
970 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
972 subpage_t *subpage;
973 hwaddr base = section->offset_within_address_space
974 & TARGET_PAGE_MASK;
975 MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
976 d->map.nodes, d->map.sections);
977 MemoryRegionSection subsection = {
978 .offset_within_address_space = base,
979 .size = int128_make64(TARGET_PAGE_SIZE),
981 hwaddr start, end;
983 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
985 if (!(existing->mr->subpage)) {
986 subpage = subpage_init(d->as, base);
987 subsection.address_space = d->as;
988 subsection.mr = &subpage->iomem;
989 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
990 phys_section_add(&d->map, &subsection));
991 } else {
992 subpage = container_of(existing->mr, subpage_t, iomem);
994 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
995 end = start + int128_get64(section->size) - 1;
996 subpage_register(subpage, start, end,
997 phys_section_add(&d->map, section));
1001 static void register_multipage(AddressSpaceDispatch *d,
1002 MemoryRegionSection *section)
1004 hwaddr start_addr = section->offset_within_address_space;
1005 uint16_t section_index = phys_section_add(&d->map, section);
1006 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1007 TARGET_PAGE_BITS));
1009 assert(num_pages);
1010 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
1013 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
1015 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1016 AddressSpaceDispatch *d = as->next_dispatch;
1017 MemoryRegionSection now = *section, remain = *section;
1018 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
1020 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
1021 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
1022 - now.offset_within_address_space;
1024 now.size = int128_min(int128_make64(left), now.size);
1025 register_subpage(d, &now);
1026 } else {
1027 now.size = int128_zero();
1029 while (int128_ne(remain.size, now.size)) {
1030 remain.size = int128_sub(remain.size, now.size);
1031 remain.offset_within_address_space += int128_get64(now.size);
1032 remain.offset_within_region += int128_get64(now.size);
1033 now = remain;
1034 if (int128_lt(remain.size, page_size)) {
1035 register_subpage(d, &now);
1036 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
1037 now.size = page_size;
1038 register_subpage(d, &now);
1039 } else {
1040 now.size = int128_and(now.size, int128_neg(page_size));
1041 register_multipage(d, &now);
1046 void qemu_flush_coalesced_mmio_buffer(void)
1048 if (kvm_enabled())
1049 kvm_flush_coalesced_mmio_buffer();
1052 void qemu_mutex_lock_ramlist(void)
1054 qemu_mutex_lock(&ram_list.mutex);
1057 void qemu_mutex_unlock_ramlist(void)
1059 qemu_mutex_unlock(&ram_list.mutex);
1062 #ifdef __linux__
1064 #include <sys/vfs.h>
1066 #define HUGETLBFS_MAGIC 0x958458f6
1068 static long gethugepagesize(const char *path, Error **errp)
1070 struct statfs fs;
1071 int ret;
1073 do {
1074 ret = statfs(path, &fs);
1075 } while (ret != 0 && errno == EINTR);
1077 if (ret != 0) {
1078 error_setg_errno(errp, errno, "failed to get page size of file %s",
1079 path);
1080 return 0;
1083 if (fs.f_type != HUGETLBFS_MAGIC)
1084 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
1086 return fs.f_bsize;
1089 static void *file_ram_alloc(RAMBlock *block,
1090 ram_addr_t memory,
1091 const char *path,
1092 Error **errp)
1094 char *filename;
1095 char *sanitized_name;
1096 char *c;
1097 void * volatile area = NULL;
1098 int fd;
1099 uint64_t hpagesize;
1100 Error *local_err = NULL;
1102 hpagesize = gethugepagesize(path, &local_err);
1103 if (local_err) {
1104 error_propagate(errp, local_err);
1105 goto error;
1108 if (memory < hpagesize) {
1109 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
1110 "or larger than huge page size 0x%" PRIx64,
1111 memory, hpagesize);
1112 goto error;
1115 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1116 error_setg(errp,
1117 "host lacks kvm mmu notifiers, -mem-path unsupported");
1118 goto error;
1121 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1122 sanitized_name = g_strdup(memory_region_name(block->mr));
1123 for (c = sanitized_name; *c != '\0'; c++) {
1124 if (*c == '/')
1125 *c = '_';
1128 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1129 sanitized_name);
1130 g_free(sanitized_name);
1132 fd = mkstemp(filename);
1133 if (fd < 0) {
1134 error_setg_errno(errp, errno,
1135 "unable to create backing store for hugepages");
1136 g_free(filename);
1137 goto error;
1139 unlink(filename);
1140 g_free(filename);
1142 memory = (memory+hpagesize-1) & ~(hpagesize-1);
1145 * ftruncate is not supported by hugetlbfs in older
1146 * hosts, so don't bother bailing out on errors.
1147 * If anything goes wrong with it under other filesystems,
1148 * mmap will fail.
1150 if (ftruncate(fd, memory)) {
1151 perror("ftruncate");
1154 area = mmap(0, memory, PROT_READ | PROT_WRITE,
1155 (block->flags & RAM_SHARED ? MAP_SHARED : MAP_PRIVATE),
1156 fd, 0);
1157 if (area == MAP_FAILED) {
1158 error_setg_errno(errp, errno,
1159 "unable to map backing store for hugepages");
1160 close(fd);
1161 goto error;
1164 if (mem_prealloc) {
1165 os_mem_prealloc(fd, area, memory);
1168 block->fd = fd;
1169 return area;
1171 error:
1172 if (mem_prealloc) {
1173 error_report("%s\n", error_get_pretty(*errp));
1174 exit(1);
1176 return NULL;
1178 #endif
1180 static ram_addr_t find_ram_offset(ram_addr_t size)
1182 RAMBlock *block, *next_block;
1183 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1185 assert(size != 0); /* it would hand out same offset multiple times */
1187 if (QTAILQ_EMPTY(&ram_list.blocks))
1188 return 0;
1190 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1191 ram_addr_t end, next = RAM_ADDR_MAX;
1193 end = block->offset + block->length;
1195 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1196 if (next_block->offset >= end) {
1197 next = MIN(next, next_block->offset);
1200 if (next - end >= size && next - end < mingap) {
1201 offset = end;
1202 mingap = next - end;
1206 if (offset == RAM_ADDR_MAX) {
1207 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1208 (uint64_t)size);
1209 abort();
1212 return offset;
1215 ram_addr_t last_ram_offset(void)
1217 RAMBlock *block;
1218 ram_addr_t last = 0;
1220 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1221 last = MAX(last, block->offset + block->length);
1223 return last;
1226 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1228 int ret;
1230 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1231 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1232 "dump-guest-core", true)) {
1233 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1234 if (ret) {
1235 perror("qemu_madvise");
1236 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1237 "but dump_guest_core=off specified\n");
1242 static RAMBlock *find_ram_block(ram_addr_t addr)
1244 RAMBlock *block;
1246 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1247 if (block->offset == addr) {
1248 return block;
1252 return NULL;
1255 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1257 RAMBlock *new_block = find_ram_block(addr);
1258 RAMBlock *block;
1260 assert(new_block);
1261 assert(!new_block->idstr[0]);
1263 if (dev) {
1264 char *id = qdev_get_dev_path(dev);
1265 if (id) {
1266 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1267 g_free(id);
1270 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1272 /* This assumes the iothread lock is taken here too. */
1273 qemu_mutex_lock_ramlist();
1274 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1275 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1276 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1277 new_block->idstr);
1278 abort();
1281 qemu_mutex_unlock_ramlist();
1284 void qemu_ram_unset_idstr(ram_addr_t addr)
1286 RAMBlock *block = find_ram_block(addr);
1288 if (block) {
1289 memset(block->idstr, 0, sizeof(block->idstr));
1293 static int memory_try_enable_merging(void *addr, size_t len)
1295 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1296 /* disabled by the user */
1297 return 0;
1300 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1303 static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
1305 RAMBlock *block;
1306 ram_addr_t old_ram_size, new_ram_size;
1308 old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1310 /* This assumes the iothread lock is taken here too. */
1311 qemu_mutex_lock_ramlist();
1312 new_block->offset = find_ram_offset(new_block->length);
1314 if (!new_block->host) {
1315 if (xen_enabled()) {
1316 xen_ram_alloc(new_block->offset, new_block->length, new_block->mr);
1317 } else {
1318 new_block->host = phys_mem_alloc(new_block->length);
1319 if (!new_block->host) {
1320 error_setg_errno(errp, errno,
1321 "cannot set up guest memory '%s'",
1322 memory_region_name(new_block->mr));
1323 qemu_mutex_unlock_ramlist();
1324 return -1;
1326 memory_try_enable_merging(new_block->host, new_block->length);
1330 /* Keep the list sorted from biggest to smallest block. */
1331 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1332 if (block->length < new_block->length) {
1333 break;
1336 if (block) {
1337 QTAILQ_INSERT_BEFORE(block, new_block, next);
1338 } else {
1339 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1341 ram_list.mru_block = NULL;
1343 ram_list.version++;
1344 qemu_mutex_unlock_ramlist();
1346 new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1348 if (new_ram_size > old_ram_size) {
1349 int i;
1350 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1351 ram_list.dirty_memory[i] =
1352 bitmap_zero_extend(ram_list.dirty_memory[i],
1353 old_ram_size, new_ram_size);
1356 cpu_physical_memory_set_dirty_range(new_block->offset, new_block->length);
1358 qemu_ram_setup_dump(new_block->host, new_block->length);
1359 qemu_madvise(new_block->host, new_block->length, QEMU_MADV_HUGEPAGE);
1360 qemu_madvise(new_block->host, new_block->length, QEMU_MADV_DONTFORK);
1362 if (kvm_enabled()) {
1363 kvm_setup_guest_memory(new_block->host, new_block->length);
1366 return new_block->offset;
1369 #ifdef __linux__
1370 ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
1371 bool share, const char *mem_path,
1372 Error **errp)
1374 RAMBlock *new_block;
1375 ram_addr_t addr;
1376 Error *local_err = NULL;
1378 if (xen_enabled()) {
1379 error_setg(errp, "-mem-path not supported with Xen");
1380 return -1;
1383 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1385 * file_ram_alloc() needs to allocate just like
1386 * phys_mem_alloc, but we haven't bothered to provide
1387 * a hook there.
1389 error_setg(errp,
1390 "-mem-path not supported with this accelerator");
1391 return -1;
1394 size = TARGET_PAGE_ALIGN(size);
1395 new_block = g_malloc0(sizeof(*new_block));
1396 new_block->mr = mr;
1397 new_block->length = size;
1398 new_block->flags = share ? RAM_SHARED : 0;
1399 new_block->host = file_ram_alloc(new_block, size,
1400 mem_path, errp);
1401 if (!new_block->host) {
1402 g_free(new_block);
1403 return -1;
1406 addr = ram_block_add(new_block, &local_err);
1407 if (local_err) {
1408 g_free(new_block);
1409 error_propagate(errp, local_err);
1410 return -1;
1412 return addr;
1414 #endif
1416 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1417 MemoryRegion *mr, Error **errp)
1419 RAMBlock *new_block;
1420 ram_addr_t addr;
1421 Error *local_err = NULL;
1423 size = TARGET_PAGE_ALIGN(size);
1424 new_block = g_malloc0(sizeof(*new_block));
1425 new_block->mr = mr;
1426 new_block->length = size;
1427 new_block->fd = -1;
1428 new_block->host = host;
1429 if (host) {
1430 new_block->flags |= RAM_PREALLOC;
1432 addr = ram_block_add(new_block, &local_err);
1433 if (local_err) {
1434 g_free(new_block);
1435 error_propagate(errp, local_err);
1436 return -1;
1438 return addr;
1441 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
1443 return qemu_ram_alloc_from_ptr(size, NULL, mr, errp);
1446 void qemu_ram_free_from_ptr(ram_addr_t addr)
1448 RAMBlock *block;
1450 /* This assumes the iothread lock is taken here too. */
1451 qemu_mutex_lock_ramlist();
1452 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1453 if (addr == block->offset) {
1454 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1455 ram_list.mru_block = NULL;
1456 ram_list.version++;
1457 g_free(block);
1458 break;
1461 qemu_mutex_unlock_ramlist();
1464 void qemu_ram_free(ram_addr_t addr)
1466 RAMBlock *block;
1468 /* This assumes the iothread lock is taken here too. */
1469 qemu_mutex_lock_ramlist();
1470 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1471 if (addr == block->offset) {
1472 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1473 ram_list.mru_block = NULL;
1474 ram_list.version++;
1475 if (block->flags & RAM_PREALLOC) {
1477 } else if (xen_enabled()) {
1478 xen_invalidate_map_cache_entry(block->host);
1479 #ifndef _WIN32
1480 } else if (block->fd >= 0) {
1481 munmap(block->host, block->length);
1482 close(block->fd);
1483 #endif
1484 } else {
1485 qemu_anon_ram_free(block->host, block->length);
1487 g_free(block);
1488 break;
1491 qemu_mutex_unlock_ramlist();
1495 #ifndef _WIN32
1496 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1498 RAMBlock *block;
1499 ram_addr_t offset;
1500 int flags;
1501 void *area, *vaddr;
1503 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1504 offset = addr - block->offset;
1505 if (offset < block->length) {
1506 vaddr = block->host + offset;
1507 if (block->flags & RAM_PREALLOC) {
1509 } else if (xen_enabled()) {
1510 abort();
1511 } else {
1512 flags = MAP_FIXED;
1513 munmap(vaddr, length);
1514 if (block->fd >= 0) {
1515 flags |= (block->flags & RAM_SHARED ?
1516 MAP_SHARED : MAP_PRIVATE);
1517 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1518 flags, block->fd, offset);
1519 } else {
1521 * Remap needs to match alloc. Accelerators that
1522 * set phys_mem_alloc never remap. If they did,
1523 * we'd need a remap hook here.
1525 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1527 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1528 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1529 flags, -1, 0);
1531 if (area != vaddr) {
1532 fprintf(stderr, "Could not remap addr: "
1533 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1534 length, addr);
1535 exit(1);
1537 memory_try_enable_merging(vaddr, length);
1538 qemu_ram_setup_dump(vaddr, length);
1540 return;
1544 #endif /* !_WIN32 */
1546 int qemu_get_ram_fd(ram_addr_t addr)
1548 RAMBlock *block = qemu_get_ram_block(addr);
1550 return block->fd;
1553 void *qemu_get_ram_block_host_ptr(ram_addr_t addr)
1555 RAMBlock *block = qemu_get_ram_block(addr);
1557 return block->host;
1560 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1561 With the exception of the softmmu code in this file, this should
1562 only be used for local memory (e.g. video ram) that the device owns,
1563 and knows it isn't going to access beyond the end of the block.
1565 It should not be used for general purpose DMA.
1566 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1568 void *qemu_get_ram_ptr(ram_addr_t addr)
1570 RAMBlock *block = qemu_get_ram_block(addr);
1572 if (xen_enabled()) {
1573 /* We need to check if the requested address is in the RAM
1574 * because we don't want to map the entire memory in QEMU.
1575 * In that case just map until the end of the page.
1577 if (block->offset == 0) {
1578 return xen_map_cache(addr, 0, 0);
1579 } else if (block->host == NULL) {
1580 block->host =
1581 xen_map_cache(block->offset, block->length, 1);
1584 return block->host + (addr - block->offset);
1587 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1588 * but takes a size argument */
1589 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1591 if (*size == 0) {
1592 return NULL;
1594 if (xen_enabled()) {
1595 return xen_map_cache(addr, *size, 1);
1596 } else {
1597 RAMBlock *block;
1599 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1600 if (addr - block->offset < block->length) {
1601 if (addr - block->offset + *size > block->length)
1602 *size = block->length - addr + block->offset;
1603 return block->host + (addr - block->offset);
1607 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1608 abort();
1612 /* Some of the softmmu routines need to translate from a host pointer
1613 (typically a TLB entry) back to a ram offset. */
1614 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1616 RAMBlock *block;
1617 uint8_t *host = ptr;
1619 if (xen_enabled()) {
1620 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1621 return qemu_get_ram_block(*ram_addr)->mr;
1624 block = ram_list.mru_block;
1625 if (block && block->host && host - block->host < block->length) {
1626 goto found;
1629 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1630 /* This case append when the block is not mapped. */
1631 if (block->host == NULL) {
1632 continue;
1634 if (host - block->host < block->length) {
1635 goto found;
1639 return NULL;
1641 found:
1642 *ram_addr = block->offset + (host - block->host);
1643 return block->mr;
1646 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1647 uint64_t val, unsigned size)
1649 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
1650 tb_invalidate_phys_page_fast(ram_addr, size);
1652 switch (size) {
1653 case 1:
1654 stb_p(qemu_get_ram_ptr(ram_addr), val);
1655 break;
1656 case 2:
1657 stw_p(qemu_get_ram_ptr(ram_addr), val);
1658 break;
1659 case 4:
1660 stl_p(qemu_get_ram_ptr(ram_addr), val);
1661 break;
1662 default:
1663 abort();
1665 cpu_physical_memory_set_dirty_range_nocode(ram_addr, size);
1666 /* we remove the notdirty callback only if the code has been
1667 flushed */
1668 if (!cpu_physical_memory_is_clean(ram_addr)) {
1669 CPUArchState *env = current_cpu->env_ptr;
1670 tlb_set_dirty(env, current_cpu->mem_io_vaddr);
1674 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1675 unsigned size, bool is_write)
1677 return is_write;
1680 static const MemoryRegionOps notdirty_mem_ops = {
1681 .write = notdirty_mem_write,
1682 .valid.accepts = notdirty_mem_accepts,
1683 .endianness = DEVICE_NATIVE_ENDIAN,
1686 /* Generate a debug exception if a watchpoint has been hit. */
1687 static void check_watchpoint(int offset, int len, int flags)
1689 CPUState *cpu = current_cpu;
1690 CPUArchState *env = cpu->env_ptr;
1691 target_ulong pc, cs_base;
1692 target_ulong vaddr;
1693 CPUWatchpoint *wp;
1694 int cpu_flags;
1696 if (cpu->watchpoint_hit) {
1697 /* We re-entered the check after replacing the TB. Now raise
1698 * the debug interrupt so that is will trigger after the
1699 * current instruction. */
1700 cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
1701 return;
1703 vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1704 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
1705 if (cpu_watchpoint_address_matches(wp, vaddr, len)
1706 && (wp->flags & flags)) {
1707 if (flags == BP_MEM_READ) {
1708 wp->flags |= BP_WATCHPOINT_HIT_READ;
1709 } else {
1710 wp->flags |= BP_WATCHPOINT_HIT_WRITE;
1712 wp->hitaddr = vaddr;
1713 if (!cpu->watchpoint_hit) {
1714 cpu->watchpoint_hit = wp;
1715 tb_check_watchpoint(cpu);
1716 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1717 cpu->exception_index = EXCP_DEBUG;
1718 cpu_loop_exit(cpu);
1719 } else {
1720 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1721 tb_gen_code(cpu, pc, cs_base, cpu_flags, 1);
1722 cpu_resume_from_signal(cpu, NULL);
1725 } else {
1726 wp->flags &= ~BP_WATCHPOINT_HIT;
1731 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1732 so these check for a hit then pass through to the normal out-of-line
1733 phys routines. */
1734 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1735 unsigned size)
1737 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, BP_MEM_READ);
1738 switch (size) {
1739 case 1: return ldub_phys(&address_space_memory, addr);
1740 case 2: return lduw_phys(&address_space_memory, addr);
1741 case 4: return ldl_phys(&address_space_memory, addr);
1742 default: abort();
1746 static void watch_mem_write(void *opaque, hwaddr addr,
1747 uint64_t val, unsigned size)
1749 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, BP_MEM_WRITE);
1750 switch (size) {
1751 case 1:
1752 stb_phys(&address_space_memory, addr, val);
1753 break;
1754 case 2:
1755 stw_phys(&address_space_memory, addr, val);
1756 break;
1757 case 4:
1758 stl_phys(&address_space_memory, addr, val);
1759 break;
1760 default: abort();
1764 static const MemoryRegionOps watch_mem_ops = {
1765 .read = watch_mem_read,
1766 .write = watch_mem_write,
1767 .endianness = DEVICE_NATIVE_ENDIAN,
1770 static uint64_t subpage_read(void *opaque, hwaddr addr,
1771 unsigned len)
1773 subpage_t *subpage = opaque;
1774 uint8_t buf[4];
1776 #if defined(DEBUG_SUBPAGE)
1777 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
1778 subpage, len, addr);
1779 #endif
1780 address_space_read(subpage->as, addr + subpage->base, buf, len);
1781 switch (len) {
1782 case 1:
1783 return ldub_p(buf);
1784 case 2:
1785 return lduw_p(buf);
1786 case 4:
1787 return ldl_p(buf);
1788 default:
1789 abort();
1793 static void subpage_write(void *opaque, hwaddr addr,
1794 uint64_t value, unsigned len)
1796 subpage_t *subpage = opaque;
1797 uint8_t buf[4];
1799 #if defined(DEBUG_SUBPAGE)
1800 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
1801 " value %"PRIx64"\n",
1802 __func__, subpage, len, addr, value);
1803 #endif
1804 switch (len) {
1805 case 1:
1806 stb_p(buf, value);
1807 break;
1808 case 2:
1809 stw_p(buf, value);
1810 break;
1811 case 4:
1812 stl_p(buf, value);
1813 break;
1814 default:
1815 abort();
1817 address_space_write(subpage->as, addr + subpage->base, buf, len);
1820 static bool subpage_accepts(void *opaque, hwaddr addr,
1821 unsigned len, bool is_write)
1823 subpage_t *subpage = opaque;
1824 #if defined(DEBUG_SUBPAGE)
1825 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
1826 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1827 #endif
1829 return address_space_access_valid(subpage->as, addr + subpage->base,
1830 len, is_write);
1833 static const MemoryRegionOps subpage_ops = {
1834 .read = subpage_read,
1835 .write = subpage_write,
1836 .valid.accepts = subpage_accepts,
1837 .endianness = DEVICE_NATIVE_ENDIAN,
1840 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1841 uint16_t section)
1843 int idx, eidx;
1845 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1846 return -1;
1847 idx = SUBPAGE_IDX(start);
1848 eidx = SUBPAGE_IDX(end);
1849 #if defined(DEBUG_SUBPAGE)
1850 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
1851 __func__, mmio, start, end, idx, eidx, section);
1852 #endif
1853 for (; idx <= eidx; idx++) {
1854 mmio->sub_section[idx] = section;
1857 return 0;
1860 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1862 subpage_t *mmio;
1864 mmio = g_malloc0(sizeof(subpage_t));
1866 mmio->as = as;
1867 mmio->base = base;
1868 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1869 NULL, TARGET_PAGE_SIZE);
1870 mmio->iomem.subpage = true;
1871 #if defined(DEBUG_SUBPAGE)
1872 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
1873 mmio, base, TARGET_PAGE_SIZE);
1874 #endif
1875 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1877 return mmio;
1880 static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
1881 MemoryRegion *mr)
1883 assert(as);
1884 MemoryRegionSection section = {
1885 .address_space = as,
1886 .mr = mr,
1887 .offset_within_address_space = 0,
1888 .offset_within_region = 0,
1889 .size = int128_2_64(),
1892 return phys_section_add(map, &section);
1895 MemoryRegion *iotlb_to_region(AddressSpace *as, hwaddr index)
1897 return as->dispatch->map.sections[index & ~TARGET_PAGE_MASK].mr;
1900 static void io_mem_init(void)
1902 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, NULL, UINT64_MAX);
1903 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1904 NULL, UINT64_MAX);
1905 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1906 NULL, UINT64_MAX);
1907 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1908 NULL, UINT64_MAX);
1911 static void mem_begin(MemoryListener *listener)
1913 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1914 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
1915 uint16_t n;
1917 n = dummy_section(&d->map, as, &io_mem_unassigned);
1918 assert(n == PHYS_SECTION_UNASSIGNED);
1919 n = dummy_section(&d->map, as, &io_mem_notdirty);
1920 assert(n == PHYS_SECTION_NOTDIRTY);
1921 n = dummy_section(&d->map, as, &io_mem_rom);
1922 assert(n == PHYS_SECTION_ROM);
1923 n = dummy_section(&d->map, as, &io_mem_watch);
1924 assert(n == PHYS_SECTION_WATCH);
1926 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
1927 d->as = as;
1928 as->next_dispatch = d;
1931 static void mem_commit(MemoryListener *listener)
1933 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1934 AddressSpaceDispatch *cur = as->dispatch;
1935 AddressSpaceDispatch *next = as->next_dispatch;
1937 phys_page_compact_all(next, next->map.nodes_nb);
1939 as->dispatch = next;
1941 if (cur) {
1942 phys_sections_free(&cur->map);
1943 g_free(cur);
1947 static void tcg_commit(MemoryListener *listener)
1949 CPUState *cpu;
1951 /* since each CPU stores ram addresses in its TLB cache, we must
1952 reset the modified entries */
1953 /* XXX: slow ! */
1954 CPU_FOREACH(cpu) {
1955 /* FIXME: Disentangle the cpu.h circular files deps so we can
1956 directly get the right CPU from listener. */
1957 if (cpu->tcg_as_listener != listener) {
1958 continue;
1960 tlb_flush(cpu, 1);
1964 static void core_log_global_start(MemoryListener *listener)
1966 cpu_physical_memory_set_dirty_tracking(true);
1969 static void core_log_global_stop(MemoryListener *listener)
1971 cpu_physical_memory_set_dirty_tracking(false);
1974 static MemoryListener core_memory_listener = {
1975 .log_global_start = core_log_global_start,
1976 .log_global_stop = core_log_global_stop,
1977 .priority = 1,
1980 void address_space_init_dispatch(AddressSpace *as)
1982 as->dispatch = NULL;
1983 as->dispatch_listener = (MemoryListener) {
1984 .begin = mem_begin,
1985 .commit = mem_commit,
1986 .region_add = mem_add,
1987 .region_nop = mem_add,
1988 .priority = 0,
1990 memory_listener_register(&as->dispatch_listener, as);
1993 void address_space_destroy_dispatch(AddressSpace *as)
1995 AddressSpaceDispatch *d = as->dispatch;
1997 memory_listener_unregister(&as->dispatch_listener);
1998 g_free(d);
1999 as->dispatch = NULL;
2002 static void memory_map_init(void)
2004 system_memory = g_malloc(sizeof(*system_memory));
2006 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
2007 address_space_init(&address_space_memory, system_memory, "memory");
2009 system_io = g_malloc(sizeof(*system_io));
2010 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2011 65536);
2012 address_space_init(&address_space_io, system_io, "I/O");
2014 memory_listener_register(&core_memory_listener, &address_space_memory);
2017 MemoryRegion *get_system_memory(void)
2019 return system_memory;
2022 MemoryRegion *get_system_io(void)
2024 return system_io;
2027 #endif /* !defined(CONFIG_USER_ONLY) */
2029 /* physical memory access (slow version, mainly for debug) */
2030 #if defined(CONFIG_USER_ONLY)
2031 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2032 uint8_t *buf, int len, int is_write)
2034 int l, flags;
2035 target_ulong page;
2036 void * p;
2038 while (len > 0) {
2039 page = addr & TARGET_PAGE_MASK;
2040 l = (page + TARGET_PAGE_SIZE) - addr;
2041 if (l > len)
2042 l = len;
2043 flags = page_get_flags(page);
2044 if (!(flags & PAGE_VALID))
2045 return -1;
2046 if (is_write) {
2047 if (!(flags & PAGE_WRITE))
2048 return -1;
2049 /* XXX: this code should not depend on lock_user */
2050 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2051 return -1;
2052 memcpy(p, buf, l);
2053 unlock_user(p, addr, l);
2054 } else {
2055 if (!(flags & PAGE_READ))
2056 return -1;
2057 /* XXX: this code should not depend on lock_user */
2058 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2059 return -1;
2060 memcpy(buf, p, l);
2061 unlock_user(p, addr, 0);
2063 len -= l;
2064 buf += l;
2065 addr += l;
2067 return 0;
2070 #else
2072 static void invalidate_and_set_dirty(hwaddr addr,
2073 hwaddr length)
2075 if (cpu_physical_memory_is_clean(addr)) {
2076 /* invalidate code */
2077 tb_invalidate_phys_page_range(addr, addr + length, 0);
2078 /* set dirty bit */
2079 cpu_physical_memory_set_dirty_range_nocode(addr, length);
2081 xen_modified_memory(addr, length);
2084 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
2086 unsigned access_size_max = mr->ops->valid.max_access_size;
2088 /* Regions are assumed to support 1-4 byte accesses unless
2089 otherwise specified. */
2090 if (access_size_max == 0) {
2091 access_size_max = 4;
2094 /* Bound the maximum access by the alignment of the address. */
2095 if (!mr->ops->impl.unaligned) {
2096 unsigned align_size_max = addr & -addr;
2097 if (align_size_max != 0 && align_size_max < access_size_max) {
2098 access_size_max = align_size_max;
2102 /* Don't attempt accesses larger than the maximum. */
2103 if (l > access_size_max) {
2104 l = access_size_max;
2106 if (l & (l - 1)) {
2107 l = 1 << (qemu_fls(l) - 1);
2110 return l;
2113 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
2114 int len, bool is_write)
2116 hwaddr l;
2117 uint8_t *ptr;
2118 uint64_t val;
2119 hwaddr addr1;
2120 MemoryRegion *mr;
2121 bool error = false;
2123 while (len > 0) {
2124 l = len;
2125 mr = address_space_translate(as, addr, &addr1, &l, is_write);
2127 if (is_write) {
2128 if (!memory_access_is_direct(mr, is_write)) {
2129 l = memory_access_size(mr, l, addr1);
2130 /* XXX: could force current_cpu to NULL to avoid
2131 potential bugs */
2132 switch (l) {
2133 case 8:
2134 /* 64 bit write access */
2135 val = ldq_p(buf);
2136 error |= io_mem_write(mr, addr1, val, 8);
2137 break;
2138 case 4:
2139 /* 32 bit write access */
2140 val = ldl_p(buf);
2141 error |= io_mem_write(mr, addr1, val, 4);
2142 break;
2143 case 2:
2144 /* 16 bit write access */
2145 val = lduw_p(buf);
2146 error |= io_mem_write(mr, addr1, val, 2);
2147 break;
2148 case 1:
2149 /* 8 bit write access */
2150 val = ldub_p(buf);
2151 error |= io_mem_write(mr, addr1, val, 1);
2152 break;
2153 default:
2154 abort();
2156 } else {
2157 addr1 += memory_region_get_ram_addr(mr);
2158 /* RAM case */
2159 ptr = qemu_get_ram_ptr(addr1);
2160 memcpy(ptr, buf, l);
2161 invalidate_and_set_dirty(addr1, l);
2163 } else {
2164 if (!memory_access_is_direct(mr, is_write)) {
2165 /* I/O case */
2166 l = memory_access_size(mr, l, addr1);
2167 switch (l) {
2168 case 8:
2169 /* 64 bit read access */
2170 error |= io_mem_read(mr, addr1, &val, 8);
2171 stq_p(buf, val);
2172 break;
2173 case 4:
2174 /* 32 bit read access */
2175 error |= io_mem_read(mr, addr1, &val, 4);
2176 stl_p(buf, val);
2177 break;
2178 case 2:
2179 /* 16 bit read access */
2180 error |= io_mem_read(mr, addr1, &val, 2);
2181 stw_p(buf, val);
2182 break;
2183 case 1:
2184 /* 8 bit read access */
2185 error |= io_mem_read(mr, addr1, &val, 1);
2186 stb_p(buf, val);
2187 break;
2188 default:
2189 abort();
2191 } else {
2192 /* RAM case */
2193 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2194 memcpy(buf, ptr, l);
2197 len -= l;
2198 buf += l;
2199 addr += l;
2202 return error;
2205 bool address_space_write(AddressSpace *as, hwaddr addr,
2206 const uint8_t *buf, int len)
2208 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2211 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2213 return address_space_rw(as, addr, buf, len, false);
2217 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2218 int len, int is_write)
2220 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2223 enum write_rom_type {
2224 WRITE_DATA,
2225 FLUSH_CACHE,
2228 static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
2229 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
2231 hwaddr l;
2232 uint8_t *ptr;
2233 hwaddr addr1;
2234 MemoryRegion *mr;
2236 while (len > 0) {
2237 l = len;
2238 mr = address_space_translate(as, addr, &addr1, &l, true);
2240 if (!(memory_region_is_ram(mr) ||
2241 memory_region_is_romd(mr))) {
2242 /* do nothing */
2243 } else {
2244 addr1 += memory_region_get_ram_addr(mr);
2245 /* ROM/RAM case */
2246 ptr = qemu_get_ram_ptr(addr1);
2247 switch (type) {
2248 case WRITE_DATA:
2249 memcpy(ptr, buf, l);
2250 invalidate_and_set_dirty(addr1, l);
2251 break;
2252 case FLUSH_CACHE:
2253 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
2254 break;
2257 len -= l;
2258 buf += l;
2259 addr += l;
2263 /* used for ROM loading : can write in RAM and ROM */
2264 void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
2265 const uint8_t *buf, int len)
2267 cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
2270 void cpu_flush_icache_range(hwaddr start, int len)
2273 * This function should do the same thing as an icache flush that was
2274 * triggered from within the guest. For TCG we are always cache coherent,
2275 * so there is no need to flush anything. For KVM / Xen we need to flush
2276 * the host's instruction cache at least.
2278 if (tcg_enabled()) {
2279 return;
2282 cpu_physical_memory_write_rom_internal(&address_space_memory,
2283 start, NULL, len, FLUSH_CACHE);
2286 typedef struct {
2287 MemoryRegion *mr;
2288 void *buffer;
2289 hwaddr addr;
2290 hwaddr len;
2291 } BounceBuffer;
2293 static BounceBuffer bounce;
2295 typedef struct MapClient {
2296 void *opaque;
2297 void (*callback)(void *opaque);
2298 QLIST_ENTRY(MapClient) link;
2299 } MapClient;
2301 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2302 = QLIST_HEAD_INITIALIZER(map_client_list);
2304 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2306 MapClient *client = g_malloc(sizeof(*client));
2308 client->opaque = opaque;
2309 client->callback = callback;
2310 QLIST_INSERT_HEAD(&map_client_list, client, link);
2311 return client;
2314 static void cpu_unregister_map_client(void *_client)
2316 MapClient *client = (MapClient *)_client;
2318 QLIST_REMOVE(client, link);
2319 g_free(client);
2322 static void cpu_notify_map_clients(void)
2324 MapClient *client;
2326 while (!QLIST_EMPTY(&map_client_list)) {
2327 client = QLIST_FIRST(&map_client_list);
2328 client->callback(client->opaque);
2329 cpu_unregister_map_client(client);
2333 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2335 MemoryRegion *mr;
2336 hwaddr l, xlat;
2338 while (len > 0) {
2339 l = len;
2340 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2341 if (!memory_access_is_direct(mr, is_write)) {
2342 l = memory_access_size(mr, l, addr);
2343 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2344 return false;
2348 len -= l;
2349 addr += l;
2351 return true;
2354 /* Map a physical memory region into a host virtual address.
2355 * May map a subset of the requested range, given by and returned in *plen.
2356 * May return NULL if resources needed to perform the mapping are exhausted.
2357 * Use only for reads OR writes - not for read-modify-write operations.
2358 * Use cpu_register_map_client() to know when retrying the map operation is
2359 * likely to succeed.
2361 void *address_space_map(AddressSpace *as,
2362 hwaddr addr,
2363 hwaddr *plen,
2364 bool is_write)
2366 hwaddr len = *plen;
2367 hwaddr done = 0;
2368 hwaddr l, xlat, base;
2369 MemoryRegion *mr, *this_mr;
2370 ram_addr_t raddr;
2372 if (len == 0) {
2373 return NULL;
2376 l = len;
2377 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2378 if (!memory_access_is_direct(mr, is_write)) {
2379 if (bounce.buffer) {
2380 return NULL;
2382 /* Avoid unbounded allocations */
2383 l = MIN(l, TARGET_PAGE_SIZE);
2384 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
2385 bounce.addr = addr;
2386 bounce.len = l;
2388 memory_region_ref(mr);
2389 bounce.mr = mr;
2390 if (!is_write) {
2391 address_space_read(as, addr, bounce.buffer, l);
2394 *plen = l;
2395 return bounce.buffer;
2398 base = xlat;
2399 raddr = memory_region_get_ram_addr(mr);
2401 for (;;) {
2402 len -= l;
2403 addr += l;
2404 done += l;
2405 if (len == 0) {
2406 break;
2409 l = len;
2410 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2411 if (this_mr != mr || xlat != base + done) {
2412 break;
2416 memory_region_ref(mr);
2417 *plen = done;
2418 return qemu_ram_ptr_length(raddr + base, plen);
2421 /* Unmaps a memory region previously mapped by address_space_map().
2422 * Will also mark the memory as dirty if is_write == 1. access_len gives
2423 * the amount of memory that was actually read or written by the caller.
2425 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2426 int is_write, hwaddr access_len)
2428 if (buffer != bounce.buffer) {
2429 MemoryRegion *mr;
2430 ram_addr_t addr1;
2432 mr = qemu_ram_addr_from_host(buffer, &addr1);
2433 assert(mr != NULL);
2434 if (is_write) {
2435 invalidate_and_set_dirty(addr1, access_len);
2437 if (xen_enabled()) {
2438 xen_invalidate_map_cache_entry(buffer);
2440 memory_region_unref(mr);
2441 return;
2443 if (is_write) {
2444 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2446 qemu_vfree(bounce.buffer);
2447 bounce.buffer = NULL;
2448 memory_region_unref(bounce.mr);
2449 cpu_notify_map_clients();
2452 void *cpu_physical_memory_map(hwaddr addr,
2453 hwaddr *plen,
2454 int is_write)
2456 return address_space_map(&address_space_memory, addr, plen, is_write);
2459 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2460 int is_write, hwaddr access_len)
2462 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2465 /* warning: addr must be aligned */
2466 static inline uint32_t ldl_phys_internal(AddressSpace *as, hwaddr addr,
2467 enum device_endian endian)
2469 uint8_t *ptr;
2470 uint64_t val;
2471 MemoryRegion *mr;
2472 hwaddr l = 4;
2473 hwaddr addr1;
2475 mr = address_space_translate(as, addr, &addr1, &l, false);
2476 if (l < 4 || !memory_access_is_direct(mr, false)) {
2477 /* I/O case */
2478 io_mem_read(mr, addr1, &val, 4);
2479 #if defined(TARGET_WORDS_BIGENDIAN)
2480 if (endian == DEVICE_LITTLE_ENDIAN) {
2481 val = bswap32(val);
2483 #else
2484 if (endian == DEVICE_BIG_ENDIAN) {
2485 val = bswap32(val);
2487 #endif
2488 } else {
2489 /* RAM case */
2490 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2491 & TARGET_PAGE_MASK)
2492 + addr1);
2493 switch (endian) {
2494 case DEVICE_LITTLE_ENDIAN:
2495 val = ldl_le_p(ptr);
2496 break;
2497 case DEVICE_BIG_ENDIAN:
2498 val = ldl_be_p(ptr);
2499 break;
2500 default:
2501 val = ldl_p(ptr);
2502 break;
2505 return val;
2508 uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
2510 return ldl_phys_internal(as, addr, DEVICE_NATIVE_ENDIAN);
2513 uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
2515 return ldl_phys_internal(as, addr, DEVICE_LITTLE_ENDIAN);
2518 uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
2520 return ldl_phys_internal(as, addr, DEVICE_BIG_ENDIAN);
2523 /* warning: addr must be aligned */
2524 static inline uint64_t ldq_phys_internal(AddressSpace *as, hwaddr addr,
2525 enum device_endian endian)
2527 uint8_t *ptr;
2528 uint64_t val;
2529 MemoryRegion *mr;
2530 hwaddr l = 8;
2531 hwaddr addr1;
2533 mr = address_space_translate(as, addr, &addr1, &l,
2534 false);
2535 if (l < 8 || !memory_access_is_direct(mr, false)) {
2536 /* I/O case */
2537 io_mem_read(mr, addr1, &val, 8);
2538 #if defined(TARGET_WORDS_BIGENDIAN)
2539 if (endian == DEVICE_LITTLE_ENDIAN) {
2540 val = bswap64(val);
2542 #else
2543 if (endian == DEVICE_BIG_ENDIAN) {
2544 val = bswap64(val);
2546 #endif
2547 } else {
2548 /* RAM case */
2549 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2550 & TARGET_PAGE_MASK)
2551 + addr1);
2552 switch (endian) {
2553 case DEVICE_LITTLE_ENDIAN:
2554 val = ldq_le_p(ptr);
2555 break;
2556 case DEVICE_BIG_ENDIAN:
2557 val = ldq_be_p(ptr);
2558 break;
2559 default:
2560 val = ldq_p(ptr);
2561 break;
2564 return val;
2567 uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
2569 return ldq_phys_internal(as, addr, DEVICE_NATIVE_ENDIAN);
2572 uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
2574 return ldq_phys_internal(as, addr, DEVICE_LITTLE_ENDIAN);
2577 uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
2579 return ldq_phys_internal(as, addr, DEVICE_BIG_ENDIAN);
2582 /* XXX: optimize */
2583 uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
2585 uint8_t val;
2586 address_space_rw(as, addr, &val, 1, 0);
2587 return val;
2590 /* warning: addr must be aligned */
2591 static inline uint32_t lduw_phys_internal(AddressSpace *as, hwaddr addr,
2592 enum device_endian endian)
2594 uint8_t *ptr;
2595 uint64_t val;
2596 MemoryRegion *mr;
2597 hwaddr l = 2;
2598 hwaddr addr1;
2600 mr = address_space_translate(as, addr, &addr1, &l,
2601 false);
2602 if (l < 2 || !memory_access_is_direct(mr, false)) {
2603 /* I/O case */
2604 io_mem_read(mr, addr1, &val, 2);
2605 #if defined(TARGET_WORDS_BIGENDIAN)
2606 if (endian == DEVICE_LITTLE_ENDIAN) {
2607 val = bswap16(val);
2609 #else
2610 if (endian == DEVICE_BIG_ENDIAN) {
2611 val = bswap16(val);
2613 #endif
2614 } else {
2615 /* RAM case */
2616 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2617 & TARGET_PAGE_MASK)
2618 + addr1);
2619 switch (endian) {
2620 case DEVICE_LITTLE_ENDIAN:
2621 val = lduw_le_p(ptr);
2622 break;
2623 case DEVICE_BIG_ENDIAN:
2624 val = lduw_be_p(ptr);
2625 break;
2626 default:
2627 val = lduw_p(ptr);
2628 break;
2631 return val;
2634 uint32_t lduw_phys(AddressSpace *as, hwaddr addr)
2636 return lduw_phys_internal(as, addr, DEVICE_NATIVE_ENDIAN);
2639 uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr)
2641 return lduw_phys_internal(as, addr, DEVICE_LITTLE_ENDIAN);
2644 uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr)
2646 return lduw_phys_internal(as, addr, DEVICE_BIG_ENDIAN);
2649 /* warning: addr must be aligned. The ram page is not masked as dirty
2650 and the code inside is not invalidated. It is useful if the dirty
2651 bits are used to track modified PTEs */
2652 void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
2654 uint8_t *ptr;
2655 MemoryRegion *mr;
2656 hwaddr l = 4;
2657 hwaddr addr1;
2659 mr = address_space_translate(as, addr, &addr1, &l,
2660 true);
2661 if (l < 4 || !memory_access_is_direct(mr, true)) {
2662 io_mem_write(mr, addr1, val, 4);
2663 } else {
2664 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2665 ptr = qemu_get_ram_ptr(addr1);
2666 stl_p(ptr, val);
2668 if (unlikely(in_migration)) {
2669 if (cpu_physical_memory_is_clean(addr1)) {
2670 /* invalidate code */
2671 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2672 /* set dirty bit */
2673 cpu_physical_memory_set_dirty_range_nocode(addr1, 4);
2679 /* warning: addr must be aligned */
2680 static inline void stl_phys_internal(AddressSpace *as,
2681 hwaddr addr, uint32_t val,
2682 enum device_endian endian)
2684 uint8_t *ptr;
2685 MemoryRegion *mr;
2686 hwaddr l = 4;
2687 hwaddr addr1;
2689 mr = address_space_translate(as, addr, &addr1, &l,
2690 true);
2691 if (l < 4 || !memory_access_is_direct(mr, true)) {
2692 #if defined(TARGET_WORDS_BIGENDIAN)
2693 if (endian == DEVICE_LITTLE_ENDIAN) {
2694 val = bswap32(val);
2696 #else
2697 if (endian == DEVICE_BIG_ENDIAN) {
2698 val = bswap32(val);
2700 #endif
2701 io_mem_write(mr, addr1, val, 4);
2702 } else {
2703 /* RAM case */
2704 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2705 ptr = qemu_get_ram_ptr(addr1);
2706 switch (endian) {
2707 case DEVICE_LITTLE_ENDIAN:
2708 stl_le_p(ptr, val);
2709 break;
2710 case DEVICE_BIG_ENDIAN:
2711 stl_be_p(ptr, val);
2712 break;
2713 default:
2714 stl_p(ptr, val);
2715 break;
2717 invalidate_and_set_dirty(addr1, 4);
2721 void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2723 stl_phys_internal(as, addr, val, DEVICE_NATIVE_ENDIAN);
2726 void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2728 stl_phys_internal(as, addr, val, DEVICE_LITTLE_ENDIAN);
2731 void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2733 stl_phys_internal(as, addr, val, DEVICE_BIG_ENDIAN);
2736 /* XXX: optimize */
2737 void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2739 uint8_t v = val;
2740 address_space_rw(as, addr, &v, 1, 1);
2743 /* warning: addr must be aligned */
2744 static inline void stw_phys_internal(AddressSpace *as,
2745 hwaddr addr, uint32_t val,
2746 enum device_endian endian)
2748 uint8_t *ptr;
2749 MemoryRegion *mr;
2750 hwaddr l = 2;
2751 hwaddr addr1;
2753 mr = address_space_translate(as, addr, &addr1, &l, true);
2754 if (l < 2 || !memory_access_is_direct(mr, true)) {
2755 #if defined(TARGET_WORDS_BIGENDIAN)
2756 if (endian == DEVICE_LITTLE_ENDIAN) {
2757 val = bswap16(val);
2759 #else
2760 if (endian == DEVICE_BIG_ENDIAN) {
2761 val = bswap16(val);
2763 #endif
2764 io_mem_write(mr, addr1, val, 2);
2765 } else {
2766 /* RAM case */
2767 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2768 ptr = qemu_get_ram_ptr(addr1);
2769 switch (endian) {
2770 case DEVICE_LITTLE_ENDIAN:
2771 stw_le_p(ptr, val);
2772 break;
2773 case DEVICE_BIG_ENDIAN:
2774 stw_be_p(ptr, val);
2775 break;
2776 default:
2777 stw_p(ptr, val);
2778 break;
2780 invalidate_and_set_dirty(addr1, 2);
2784 void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2786 stw_phys_internal(as, addr, val, DEVICE_NATIVE_ENDIAN);
2789 void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2791 stw_phys_internal(as, addr, val, DEVICE_LITTLE_ENDIAN);
2794 void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2796 stw_phys_internal(as, addr, val, DEVICE_BIG_ENDIAN);
2799 /* XXX: optimize */
2800 void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val)
2802 val = tswap64(val);
2803 address_space_rw(as, addr, (void *) &val, 8, 1);
2806 void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val)
2808 val = cpu_to_le64(val);
2809 address_space_rw(as, addr, (void *) &val, 8, 1);
2812 void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val)
2814 val = cpu_to_be64(val);
2815 address_space_rw(as, addr, (void *) &val, 8, 1);
2818 /* virtual memory access for debug (includes writing to ROM) */
2819 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2820 uint8_t *buf, int len, int is_write)
2822 int l;
2823 hwaddr phys_addr;
2824 target_ulong page;
2826 while (len > 0) {
2827 page = addr & TARGET_PAGE_MASK;
2828 phys_addr = cpu_get_phys_page_debug(cpu, page);
2829 /* if no physical page mapped, return an error */
2830 if (phys_addr == -1)
2831 return -1;
2832 l = (page + TARGET_PAGE_SIZE) - addr;
2833 if (l > len)
2834 l = len;
2835 phys_addr += (addr & ~TARGET_PAGE_MASK);
2836 if (is_write) {
2837 cpu_physical_memory_write_rom(cpu->as, phys_addr, buf, l);
2838 } else {
2839 address_space_rw(cpu->as, phys_addr, buf, l, 0);
2841 len -= l;
2842 buf += l;
2843 addr += l;
2845 return 0;
2847 #endif
2850 * A helper function for the _utterly broken_ virtio device model to find out if
2851 * it's running on a big endian machine. Don't do this at home kids!
2853 bool target_words_bigendian(void);
2854 bool target_words_bigendian(void)
2856 #if defined(TARGET_WORDS_BIGENDIAN)
2857 return true;
2858 #else
2859 return false;
2860 #endif
2863 #ifndef CONFIG_USER_ONLY
2864 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2866 MemoryRegion*mr;
2867 hwaddr l = 1;
2869 mr = address_space_translate(&address_space_memory,
2870 phys_addr, &phys_addr, &l, false);
2872 return !(memory_region_is_ram(mr) ||
2873 memory_region_is_romd(mr));
2876 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2878 RAMBlock *block;
2880 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2881 func(block->host, block->offset, block->length, opaque);
2884 #endif