pc: Remove redundant arguments from pc_cmos_init()
[qemu/cris-port.git] / exec.c
blob0a4a0c5af663865df12adb5026a7cd864f350416
1 /*
2 * Virtual page mapping
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "config.h"
20 #ifndef _WIN32
21 #include <sys/types.h>
22 #include <sys/mman.h>
23 #endif
25 #include "qemu-common.h"
26 #include "cpu.h"
27 #include "tcg.h"
28 #include "hw/hw.h"
29 #if !defined(CONFIG_USER_ONLY)
30 #include "hw/boards.h"
31 #endif
32 #include "hw/qdev.h"
33 #include "qemu/osdep.h"
34 #include "sysemu/kvm.h"
35 #include "sysemu/sysemu.h"
36 #include "hw/xen/xen.h"
37 #include "qemu/timer.h"
38 #include "qemu/config-file.h"
39 #include "qemu/error-report.h"
40 #include "exec/memory.h"
41 #include "sysemu/dma.h"
42 #include "exec/address-spaces.h"
43 #if defined(CONFIG_USER_ONLY)
44 #include <qemu.h>
45 #else /* !CONFIG_USER_ONLY */
46 #include "sysemu/xen-mapcache.h"
47 #include "trace.h"
48 #endif
49 #include "exec/cpu-all.h"
50 #include "qemu/rcu_queue.h"
51 #include "qemu/main-loop.h"
52 #include "exec/cputlb.h"
53 #include "translate-all.h"
55 #include "exec/memory-internal.h"
56 #include "exec/ram_addr.h"
58 #include "qemu/range.h"
60 //#define DEBUG_SUBPAGE
62 #if !defined(CONFIG_USER_ONLY)
63 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
64 * are protected by the ramlist lock.
66 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
68 static MemoryRegion *system_memory;
69 static MemoryRegion *system_io;
71 AddressSpace address_space_io;
72 AddressSpace address_space_memory;
74 MemoryRegion io_mem_rom, io_mem_notdirty;
75 static MemoryRegion io_mem_unassigned;
77 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
78 #define RAM_PREALLOC (1 << 0)
80 /* RAM is mmap-ed with MAP_SHARED */
81 #define RAM_SHARED (1 << 1)
83 /* Only a portion of RAM (used_length) is actually used, and migrated.
84 * This used_length size can change across reboots.
86 #define RAM_RESIZEABLE (1 << 2)
88 #endif
90 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
91 /* current CPU in the current thread. It is only valid inside
92 cpu_exec() */
93 DEFINE_TLS(CPUState *, current_cpu);
94 /* 0 = Do not count executed instructions.
95 1 = Precise instruction counting.
96 2 = Adaptive rate instruction counting. */
97 int use_icount;
99 #if !defined(CONFIG_USER_ONLY)
101 typedef struct PhysPageEntry PhysPageEntry;
103 struct PhysPageEntry {
104 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
105 uint32_t skip : 6;
106 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
107 uint32_t ptr : 26;
110 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
112 /* Size of the L2 (and L3, etc) page tables. */
113 #define ADDR_SPACE_BITS 64
115 #define P_L2_BITS 9
116 #define P_L2_SIZE (1 << P_L2_BITS)
118 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
120 typedef PhysPageEntry Node[P_L2_SIZE];
122 typedef struct PhysPageMap {
123 struct rcu_head rcu;
125 unsigned sections_nb;
126 unsigned sections_nb_alloc;
127 unsigned nodes_nb;
128 unsigned nodes_nb_alloc;
129 Node *nodes;
130 MemoryRegionSection *sections;
131 } PhysPageMap;
133 struct AddressSpaceDispatch {
134 struct rcu_head rcu;
136 /* This is a multi-level map on the physical address space.
137 * The bottom level has pointers to MemoryRegionSections.
139 PhysPageEntry phys_map;
140 PhysPageMap map;
141 AddressSpace *as;
144 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
145 typedef struct subpage_t {
146 MemoryRegion iomem;
147 AddressSpace *as;
148 hwaddr base;
149 uint16_t sub_section[TARGET_PAGE_SIZE];
150 } subpage_t;
152 #define PHYS_SECTION_UNASSIGNED 0
153 #define PHYS_SECTION_NOTDIRTY 1
154 #define PHYS_SECTION_ROM 2
155 #define PHYS_SECTION_WATCH 3
157 static void io_mem_init(void);
158 static void memory_map_init(void);
159 static void tcg_commit(MemoryListener *listener);
161 static MemoryRegion io_mem_watch;
162 #endif
164 #if !defined(CONFIG_USER_ONLY)
166 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
168 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
169 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
170 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
171 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
175 static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
177 unsigned i;
178 uint32_t ret;
179 PhysPageEntry e;
180 PhysPageEntry *p;
182 ret = map->nodes_nb++;
183 p = map->nodes[ret];
184 assert(ret != PHYS_MAP_NODE_NIL);
185 assert(ret != map->nodes_nb_alloc);
187 e.skip = leaf ? 0 : 1;
188 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
189 for (i = 0; i < P_L2_SIZE; ++i) {
190 memcpy(&p[i], &e, sizeof(e));
192 return ret;
195 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
196 hwaddr *index, hwaddr *nb, uint16_t leaf,
197 int level)
199 PhysPageEntry *p;
200 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
202 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
203 lp->ptr = phys_map_node_alloc(map, level == 0);
205 p = map->nodes[lp->ptr];
206 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
208 while (*nb && lp < &p[P_L2_SIZE]) {
209 if ((*index & (step - 1)) == 0 && *nb >= step) {
210 lp->skip = 0;
211 lp->ptr = leaf;
212 *index += step;
213 *nb -= step;
214 } else {
215 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
217 ++lp;
221 static void phys_page_set(AddressSpaceDispatch *d,
222 hwaddr index, hwaddr nb,
223 uint16_t leaf)
225 /* Wildly overreserve - it doesn't matter much. */
226 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
228 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
231 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
232 * and update our entry so we can skip it and go directly to the destination.
234 static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)
236 unsigned valid_ptr = P_L2_SIZE;
237 int valid = 0;
238 PhysPageEntry *p;
239 int i;
241 if (lp->ptr == PHYS_MAP_NODE_NIL) {
242 return;
245 p = nodes[lp->ptr];
246 for (i = 0; i < P_L2_SIZE; i++) {
247 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
248 continue;
251 valid_ptr = i;
252 valid++;
253 if (p[i].skip) {
254 phys_page_compact(&p[i], nodes, compacted);
258 /* We can only compress if there's only one child. */
259 if (valid != 1) {
260 return;
263 assert(valid_ptr < P_L2_SIZE);
265 /* Don't compress if it won't fit in the # of bits we have. */
266 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
267 return;
270 lp->ptr = p[valid_ptr].ptr;
271 if (!p[valid_ptr].skip) {
272 /* If our only child is a leaf, make this a leaf. */
273 /* By design, we should have made this node a leaf to begin with so we
274 * should never reach here.
275 * But since it's so simple to handle this, let's do it just in case we
276 * change this rule.
278 lp->skip = 0;
279 } else {
280 lp->skip += p[valid_ptr].skip;
284 static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
286 DECLARE_BITMAP(compacted, nodes_nb);
288 if (d->phys_map.skip) {
289 phys_page_compact(&d->phys_map, d->map.nodes, compacted);
293 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
294 Node *nodes, MemoryRegionSection *sections)
296 PhysPageEntry *p;
297 hwaddr index = addr >> TARGET_PAGE_BITS;
298 int i;
300 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
301 if (lp.ptr == PHYS_MAP_NODE_NIL) {
302 return &sections[PHYS_SECTION_UNASSIGNED];
304 p = nodes[lp.ptr];
305 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
308 if (sections[lp.ptr].size.hi ||
309 range_covers_byte(sections[lp.ptr].offset_within_address_space,
310 sections[lp.ptr].size.lo, addr)) {
311 return &sections[lp.ptr];
312 } else {
313 return &sections[PHYS_SECTION_UNASSIGNED];
317 bool memory_region_is_unassigned(MemoryRegion *mr)
319 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
320 && mr != &io_mem_watch;
323 /* Called from RCU critical section */
324 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
325 hwaddr addr,
326 bool resolve_subpage)
328 MemoryRegionSection *section;
329 subpage_t *subpage;
331 section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);
332 if (resolve_subpage && section->mr->subpage) {
333 subpage = container_of(section->mr, subpage_t, iomem);
334 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
336 return section;
339 /* Called from RCU critical section */
340 static MemoryRegionSection *
341 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
342 hwaddr *plen, bool resolve_subpage)
344 MemoryRegionSection *section;
345 MemoryRegion *mr;
346 Int128 diff;
348 section = address_space_lookup_region(d, addr, resolve_subpage);
349 /* Compute offset within MemoryRegionSection */
350 addr -= section->offset_within_address_space;
352 /* Compute offset within MemoryRegion */
353 *xlat = addr + section->offset_within_region;
355 mr = section->mr;
357 /* MMIO registers can be expected to perform full-width accesses based only
358 * on their address, without considering adjacent registers that could
359 * decode to completely different MemoryRegions. When such registers
360 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
361 * regions overlap wildly. For this reason we cannot clamp the accesses
362 * here.
364 * If the length is small (as is the case for address_space_ldl/stl),
365 * everything works fine. If the incoming length is large, however,
366 * the caller really has to do the clamping through memory_access_size.
368 if (memory_region_is_ram(mr)) {
369 diff = int128_sub(section->size, int128_make64(addr));
370 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
372 return section;
375 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
377 if (memory_region_is_ram(mr)) {
378 return !(is_write && mr->readonly);
380 if (memory_region_is_romd(mr)) {
381 return !is_write;
384 return false;
387 /* Called from RCU critical section */
388 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
389 hwaddr *xlat, hwaddr *plen,
390 bool is_write)
392 IOMMUTLBEntry iotlb;
393 MemoryRegionSection *section;
394 MemoryRegion *mr;
396 for (;;) {
397 AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch);
398 section = address_space_translate_internal(d, addr, &addr, plen, true);
399 mr = section->mr;
401 if (!mr->iommu_ops) {
402 break;
405 iotlb = mr->iommu_ops->translate(mr, addr, is_write);
406 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
407 | (addr & iotlb.addr_mask));
408 *plen = MIN(*plen, (addr | iotlb.addr_mask) - addr + 1);
409 if (!(iotlb.perm & (1 << is_write))) {
410 mr = &io_mem_unassigned;
411 break;
414 as = iotlb.target_as;
417 if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
418 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
419 *plen = MIN(page, *plen);
422 *xlat = addr;
423 return mr;
426 /* Called from RCU critical section */
427 MemoryRegionSection *
428 address_space_translate_for_iotlb(CPUState *cpu, hwaddr addr,
429 hwaddr *xlat, hwaddr *plen)
431 MemoryRegionSection *section;
432 section = address_space_translate_internal(cpu->memory_dispatch,
433 addr, xlat, plen, false);
435 assert(!section->mr->iommu_ops);
436 return section;
438 #endif
440 #if !defined(CONFIG_USER_ONLY)
442 static int cpu_common_post_load(void *opaque, int version_id)
444 CPUState *cpu = opaque;
446 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
447 version_id is increased. */
448 cpu->interrupt_request &= ~0x01;
449 tlb_flush(cpu, 1);
451 return 0;
454 static int cpu_common_pre_load(void *opaque)
456 CPUState *cpu = opaque;
458 cpu->exception_index = -1;
460 return 0;
463 static bool cpu_common_exception_index_needed(void *opaque)
465 CPUState *cpu = opaque;
467 return tcg_enabled() && cpu->exception_index != -1;
470 static const VMStateDescription vmstate_cpu_common_exception_index = {
471 .name = "cpu_common/exception_index",
472 .version_id = 1,
473 .minimum_version_id = 1,
474 .needed = cpu_common_exception_index_needed,
475 .fields = (VMStateField[]) {
476 VMSTATE_INT32(exception_index, CPUState),
477 VMSTATE_END_OF_LIST()
481 const VMStateDescription vmstate_cpu_common = {
482 .name = "cpu_common",
483 .version_id = 1,
484 .minimum_version_id = 1,
485 .pre_load = cpu_common_pre_load,
486 .post_load = cpu_common_post_load,
487 .fields = (VMStateField[]) {
488 VMSTATE_UINT32(halted, CPUState),
489 VMSTATE_UINT32(interrupt_request, CPUState),
490 VMSTATE_END_OF_LIST()
492 .subsections = (const VMStateDescription*[]) {
493 &vmstate_cpu_common_exception_index,
494 NULL
498 #endif
500 CPUState *qemu_get_cpu(int index)
502 CPUState *cpu;
504 CPU_FOREACH(cpu) {
505 if (cpu->cpu_index == index) {
506 return cpu;
510 return NULL;
513 #if !defined(CONFIG_USER_ONLY)
514 void tcg_cpu_address_space_init(CPUState *cpu, AddressSpace *as)
516 /* We only support one address space per cpu at the moment. */
517 assert(cpu->as == as);
519 if (cpu->tcg_as_listener) {
520 memory_listener_unregister(cpu->tcg_as_listener);
521 } else {
522 cpu->tcg_as_listener = g_new0(MemoryListener, 1);
524 cpu->tcg_as_listener->commit = tcg_commit;
525 memory_listener_register(cpu->tcg_as_listener, as);
527 #endif
529 #ifndef CONFIG_USER_ONLY
530 static DECLARE_BITMAP(cpu_index_map, MAX_CPUMASK_BITS);
532 static int cpu_get_free_index(Error **errp)
534 int cpu = find_first_zero_bit(cpu_index_map, MAX_CPUMASK_BITS);
536 if (cpu >= MAX_CPUMASK_BITS) {
537 error_setg(errp, "Trying to use more CPUs than max of %d",
538 MAX_CPUMASK_BITS);
539 return -1;
542 bitmap_set(cpu_index_map, cpu, 1);
543 return cpu;
546 void cpu_exec_exit(CPUState *cpu)
548 if (cpu->cpu_index == -1) {
549 /* cpu_index was never allocated by this @cpu or was already freed. */
550 return;
553 bitmap_clear(cpu_index_map, cpu->cpu_index, 1);
554 cpu->cpu_index = -1;
556 #else
558 static int cpu_get_free_index(Error **errp)
560 CPUState *some_cpu;
561 int cpu_index = 0;
563 CPU_FOREACH(some_cpu) {
564 cpu_index++;
566 return cpu_index;
569 void cpu_exec_exit(CPUState *cpu)
572 #endif
574 void cpu_exec_init(CPUState *cpu, Error **errp)
576 CPUClass *cc = CPU_GET_CLASS(cpu);
577 int cpu_index;
578 Error *local_err = NULL;
580 #ifndef CONFIG_USER_ONLY
581 cpu->as = &address_space_memory;
582 cpu->thread_id = qemu_get_thread_id();
583 cpu_reload_memory_map(cpu);
584 #endif
586 #if defined(CONFIG_USER_ONLY)
587 cpu_list_lock();
588 #endif
589 cpu_index = cpu->cpu_index = cpu_get_free_index(&local_err);
590 if (local_err) {
591 error_propagate(errp, local_err);
592 #if defined(CONFIG_USER_ONLY)
593 cpu_list_unlock();
594 #endif
595 return;
597 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
598 #if defined(CONFIG_USER_ONLY)
599 cpu_list_unlock();
600 #endif
601 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
602 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
604 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
605 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
606 cpu_save, cpu_load, cpu->env_ptr);
607 assert(cc->vmsd == NULL);
608 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
609 #endif
610 if (cc->vmsd != NULL) {
611 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
615 #if defined(CONFIG_USER_ONLY)
616 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
618 tb_invalidate_phys_page_range(pc, pc + 1, 0);
620 #else
621 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
623 hwaddr phys = cpu_get_phys_page_debug(cpu, pc);
624 if (phys != -1) {
625 tb_invalidate_phys_addr(cpu->as,
626 phys | (pc & ~TARGET_PAGE_MASK));
629 #endif
631 #if defined(CONFIG_USER_ONLY)
632 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
637 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
638 int flags)
640 return -ENOSYS;
643 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
647 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
648 int flags, CPUWatchpoint **watchpoint)
650 return -ENOSYS;
652 #else
653 /* Add a watchpoint. */
654 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
655 int flags, CPUWatchpoint **watchpoint)
657 CPUWatchpoint *wp;
659 /* forbid ranges which are empty or run off the end of the address space */
660 if (len == 0 || (addr + len - 1) < addr) {
661 error_report("tried to set invalid watchpoint at %"
662 VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
663 return -EINVAL;
665 wp = g_malloc(sizeof(*wp));
667 wp->vaddr = addr;
668 wp->len = len;
669 wp->flags = flags;
671 /* keep all GDB-injected watchpoints in front */
672 if (flags & BP_GDB) {
673 QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
674 } else {
675 QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
678 tlb_flush_page(cpu, addr);
680 if (watchpoint)
681 *watchpoint = wp;
682 return 0;
685 /* Remove a specific watchpoint. */
686 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
687 int flags)
689 CPUWatchpoint *wp;
691 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
692 if (addr == wp->vaddr && len == wp->len
693 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
694 cpu_watchpoint_remove_by_ref(cpu, wp);
695 return 0;
698 return -ENOENT;
701 /* Remove a specific watchpoint by reference. */
702 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
704 QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
706 tlb_flush_page(cpu, watchpoint->vaddr);
708 g_free(watchpoint);
711 /* Remove all matching watchpoints. */
712 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
714 CPUWatchpoint *wp, *next;
716 QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
717 if (wp->flags & mask) {
718 cpu_watchpoint_remove_by_ref(cpu, wp);
723 /* Return true if this watchpoint address matches the specified
724 * access (ie the address range covered by the watchpoint overlaps
725 * partially or completely with the address range covered by the
726 * access).
728 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
729 vaddr addr,
730 vaddr len)
732 /* We know the lengths are non-zero, but a little caution is
733 * required to avoid errors in the case where the range ends
734 * exactly at the top of the address space and so addr + len
735 * wraps round to zero.
737 vaddr wpend = wp->vaddr + wp->len - 1;
738 vaddr addrend = addr + len - 1;
740 return !(addr > wpend || wp->vaddr > addrend);
743 #endif
745 /* Add a breakpoint. */
746 int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
747 CPUBreakpoint **breakpoint)
749 CPUBreakpoint *bp;
751 bp = g_malloc(sizeof(*bp));
753 bp->pc = pc;
754 bp->flags = flags;
756 /* keep all GDB-injected breakpoints in front */
757 if (flags & BP_GDB) {
758 QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
759 } else {
760 QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
763 breakpoint_invalidate(cpu, pc);
765 if (breakpoint) {
766 *breakpoint = bp;
768 return 0;
771 /* Remove a specific breakpoint. */
772 int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
774 CPUBreakpoint *bp;
776 QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
777 if (bp->pc == pc && bp->flags == flags) {
778 cpu_breakpoint_remove_by_ref(cpu, bp);
779 return 0;
782 return -ENOENT;
785 /* Remove a specific breakpoint by reference. */
786 void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
788 QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
790 breakpoint_invalidate(cpu, breakpoint->pc);
792 g_free(breakpoint);
795 /* Remove all matching breakpoints. */
796 void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
798 CPUBreakpoint *bp, *next;
800 QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
801 if (bp->flags & mask) {
802 cpu_breakpoint_remove_by_ref(cpu, bp);
807 /* enable or disable single step mode. EXCP_DEBUG is returned by the
808 CPU loop after each instruction */
809 void cpu_single_step(CPUState *cpu, int enabled)
811 if (cpu->singlestep_enabled != enabled) {
812 cpu->singlestep_enabled = enabled;
813 if (kvm_enabled()) {
814 kvm_update_guest_debug(cpu, 0);
815 } else {
816 /* must flush all the translated code to avoid inconsistencies */
817 /* XXX: only flush what is necessary */
818 tb_flush(cpu);
823 void cpu_abort(CPUState *cpu, const char *fmt, ...)
825 va_list ap;
826 va_list ap2;
828 va_start(ap, fmt);
829 va_copy(ap2, ap);
830 fprintf(stderr, "qemu: fatal: ");
831 vfprintf(stderr, fmt, ap);
832 fprintf(stderr, "\n");
833 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
834 if (qemu_log_enabled()) {
835 qemu_log("qemu: fatal: ");
836 qemu_log_vprintf(fmt, ap2);
837 qemu_log("\n");
838 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
839 qemu_log_flush();
840 qemu_log_close();
842 va_end(ap2);
843 va_end(ap);
844 #if defined(CONFIG_USER_ONLY)
846 struct sigaction act;
847 sigfillset(&act.sa_mask);
848 act.sa_handler = SIG_DFL;
849 sigaction(SIGABRT, &act, NULL);
851 #endif
852 abort();
855 #if !defined(CONFIG_USER_ONLY)
856 /* Called from RCU critical section */
857 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
859 RAMBlock *block;
861 block = atomic_rcu_read(&ram_list.mru_block);
862 if (block && addr - block->offset < block->max_length) {
863 goto found;
865 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
866 if (addr - block->offset < block->max_length) {
867 goto found;
871 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
872 abort();
874 found:
875 /* It is safe to write mru_block outside the iothread lock. This
876 * is what happens:
878 * mru_block = xxx
879 * rcu_read_unlock()
880 * xxx removed from list
881 * rcu_read_lock()
882 * read mru_block
883 * mru_block = NULL;
884 * call_rcu(reclaim_ramblock, xxx);
885 * rcu_read_unlock()
887 * atomic_rcu_set is not needed here. The block was already published
888 * when it was placed into the list. Here we're just making an extra
889 * copy of the pointer.
891 ram_list.mru_block = block;
892 return block;
895 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
897 ram_addr_t start1;
898 RAMBlock *block;
899 ram_addr_t end;
901 end = TARGET_PAGE_ALIGN(start + length);
902 start &= TARGET_PAGE_MASK;
904 rcu_read_lock();
905 block = qemu_get_ram_block(start);
906 assert(block == qemu_get_ram_block(end - 1));
907 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
908 cpu_tlb_reset_dirty_all(start1, length);
909 rcu_read_unlock();
912 /* Note: start and end must be within the same ram block. */
913 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
914 ram_addr_t length,
915 unsigned client)
917 unsigned long end, page;
918 bool dirty;
920 if (length == 0) {
921 return false;
924 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
925 page = start >> TARGET_PAGE_BITS;
926 dirty = bitmap_test_and_clear_atomic(ram_list.dirty_memory[client],
927 page, end - page);
929 if (dirty && tcg_enabled()) {
930 tlb_reset_dirty_range_all(start, length);
933 return dirty;
936 /* Called from RCU critical section */
937 hwaddr memory_region_section_get_iotlb(CPUState *cpu,
938 MemoryRegionSection *section,
939 target_ulong vaddr,
940 hwaddr paddr, hwaddr xlat,
941 int prot,
942 target_ulong *address)
944 hwaddr iotlb;
945 CPUWatchpoint *wp;
947 if (memory_region_is_ram(section->mr)) {
948 /* Normal RAM. */
949 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
950 + xlat;
951 if (!section->readonly) {
952 iotlb |= PHYS_SECTION_NOTDIRTY;
953 } else {
954 iotlb |= PHYS_SECTION_ROM;
956 } else {
957 AddressSpaceDispatch *d;
959 d = atomic_rcu_read(&section->address_space->dispatch);
960 iotlb = section - d->map.sections;
961 iotlb += xlat;
964 /* Make accesses to pages with watchpoints go via the
965 watchpoint trap routines. */
966 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
967 if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
968 /* Avoid trapping reads of pages with a write breakpoint. */
969 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
970 iotlb = PHYS_SECTION_WATCH + paddr;
971 *address |= TLB_MMIO;
972 break;
977 return iotlb;
979 #endif /* defined(CONFIG_USER_ONLY) */
981 #if !defined(CONFIG_USER_ONLY)
983 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
984 uint16_t section);
985 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
987 static void *(*phys_mem_alloc)(size_t size, uint64_t *align) =
988 qemu_anon_ram_alloc;
991 * Set a custom physical guest memory alloator.
992 * Accelerators with unusual needs may need this. Hopefully, we can
993 * get rid of it eventually.
995 void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align))
997 phys_mem_alloc = alloc;
1000 static uint16_t phys_section_add(PhysPageMap *map,
1001 MemoryRegionSection *section)
1003 /* The physical section number is ORed with a page-aligned
1004 * pointer to produce the iotlb entries. Thus it should
1005 * never overflow into the page-aligned value.
1007 assert(map->sections_nb < TARGET_PAGE_SIZE);
1009 if (map->sections_nb == map->sections_nb_alloc) {
1010 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
1011 map->sections = g_renew(MemoryRegionSection, map->sections,
1012 map->sections_nb_alloc);
1014 map->sections[map->sections_nb] = *section;
1015 memory_region_ref(section->mr);
1016 return map->sections_nb++;
1019 static void phys_section_destroy(MemoryRegion *mr)
1021 memory_region_unref(mr);
1023 if (mr->subpage) {
1024 subpage_t *subpage = container_of(mr, subpage_t, iomem);
1025 object_unref(OBJECT(&subpage->iomem));
1026 g_free(subpage);
1030 static void phys_sections_free(PhysPageMap *map)
1032 while (map->sections_nb > 0) {
1033 MemoryRegionSection *section = &map->sections[--map->sections_nb];
1034 phys_section_destroy(section->mr);
1036 g_free(map->sections);
1037 g_free(map->nodes);
1040 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
1042 subpage_t *subpage;
1043 hwaddr base = section->offset_within_address_space
1044 & TARGET_PAGE_MASK;
1045 MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
1046 d->map.nodes, d->map.sections);
1047 MemoryRegionSection subsection = {
1048 .offset_within_address_space = base,
1049 .size = int128_make64(TARGET_PAGE_SIZE),
1051 hwaddr start, end;
1053 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
1055 if (!(existing->mr->subpage)) {
1056 subpage = subpage_init(d->as, base);
1057 subsection.address_space = d->as;
1058 subsection.mr = &subpage->iomem;
1059 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
1060 phys_section_add(&d->map, &subsection));
1061 } else {
1062 subpage = container_of(existing->mr, subpage_t, iomem);
1064 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
1065 end = start + int128_get64(section->size) - 1;
1066 subpage_register(subpage, start, end,
1067 phys_section_add(&d->map, section));
1071 static void register_multipage(AddressSpaceDispatch *d,
1072 MemoryRegionSection *section)
1074 hwaddr start_addr = section->offset_within_address_space;
1075 uint16_t section_index = phys_section_add(&d->map, section);
1076 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1077 TARGET_PAGE_BITS));
1079 assert(num_pages);
1080 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
1083 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
1085 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1086 AddressSpaceDispatch *d = as->next_dispatch;
1087 MemoryRegionSection now = *section, remain = *section;
1088 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
1090 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
1091 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
1092 - now.offset_within_address_space;
1094 now.size = int128_min(int128_make64(left), now.size);
1095 register_subpage(d, &now);
1096 } else {
1097 now.size = int128_zero();
1099 while (int128_ne(remain.size, now.size)) {
1100 remain.size = int128_sub(remain.size, now.size);
1101 remain.offset_within_address_space += int128_get64(now.size);
1102 remain.offset_within_region += int128_get64(now.size);
1103 now = remain;
1104 if (int128_lt(remain.size, page_size)) {
1105 register_subpage(d, &now);
1106 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
1107 now.size = page_size;
1108 register_subpage(d, &now);
1109 } else {
1110 now.size = int128_and(now.size, int128_neg(page_size));
1111 register_multipage(d, &now);
1116 void qemu_flush_coalesced_mmio_buffer(void)
1118 if (kvm_enabled())
1119 kvm_flush_coalesced_mmio_buffer();
1122 void qemu_mutex_lock_ramlist(void)
1124 qemu_mutex_lock(&ram_list.mutex);
1127 void qemu_mutex_unlock_ramlist(void)
1129 qemu_mutex_unlock(&ram_list.mutex);
1132 #ifdef __linux__
1134 #include <sys/vfs.h>
1136 #define HUGETLBFS_MAGIC 0x958458f6
1138 static long gethugepagesize(const char *path, Error **errp)
1140 struct statfs fs;
1141 int ret;
1143 do {
1144 ret = statfs(path, &fs);
1145 } while (ret != 0 && errno == EINTR);
1147 if (ret != 0) {
1148 error_setg_errno(errp, errno, "failed to get page size of file %s",
1149 path);
1150 return 0;
1153 if (fs.f_type != HUGETLBFS_MAGIC)
1154 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
1156 return fs.f_bsize;
1159 static void *file_ram_alloc(RAMBlock *block,
1160 ram_addr_t memory,
1161 const char *path,
1162 Error **errp)
1164 char *filename;
1165 char *sanitized_name;
1166 char *c;
1167 void *area = NULL;
1168 int fd;
1169 uint64_t hpagesize;
1170 Error *local_err = NULL;
1172 hpagesize = gethugepagesize(path, &local_err);
1173 if (local_err) {
1174 error_propagate(errp, local_err);
1175 goto error;
1177 block->mr->align = hpagesize;
1179 if (memory < hpagesize) {
1180 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
1181 "or larger than huge page size 0x%" PRIx64,
1182 memory, hpagesize);
1183 goto error;
1186 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1187 error_setg(errp,
1188 "host lacks kvm mmu notifiers, -mem-path unsupported");
1189 goto error;
1192 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1193 sanitized_name = g_strdup(memory_region_name(block->mr));
1194 for (c = sanitized_name; *c != '\0'; c++) {
1195 if (*c == '/')
1196 *c = '_';
1199 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1200 sanitized_name);
1201 g_free(sanitized_name);
1203 fd = mkstemp(filename);
1204 if (fd < 0) {
1205 error_setg_errno(errp, errno,
1206 "unable to create backing store for hugepages");
1207 g_free(filename);
1208 goto error;
1210 unlink(filename);
1211 g_free(filename);
1213 memory = (memory+hpagesize-1) & ~(hpagesize-1);
1216 * ftruncate is not supported by hugetlbfs in older
1217 * hosts, so don't bother bailing out on errors.
1218 * If anything goes wrong with it under other filesystems,
1219 * mmap will fail.
1221 if (ftruncate(fd, memory)) {
1222 perror("ftruncate");
1225 area = mmap(0, memory, PROT_READ | PROT_WRITE,
1226 (block->flags & RAM_SHARED ? MAP_SHARED : MAP_PRIVATE),
1227 fd, 0);
1228 if (area == MAP_FAILED) {
1229 error_setg_errno(errp, errno,
1230 "unable to map backing store for hugepages");
1231 close(fd);
1232 goto error;
1235 if (mem_prealloc) {
1236 os_mem_prealloc(fd, area, memory);
1239 block->fd = fd;
1240 return area;
1242 error:
1243 if (mem_prealloc) {
1244 error_report("%s", error_get_pretty(*errp));
1245 exit(1);
1247 return NULL;
1249 #endif
1251 /* Called with the ramlist lock held. */
1252 static ram_addr_t find_ram_offset(ram_addr_t size)
1254 RAMBlock *block, *next_block;
1255 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1257 assert(size != 0); /* it would hand out same offset multiple times */
1259 if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
1260 return 0;
1263 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1264 ram_addr_t end, next = RAM_ADDR_MAX;
1266 end = block->offset + block->max_length;
1268 QLIST_FOREACH_RCU(next_block, &ram_list.blocks, next) {
1269 if (next_block->offset >= end) {
1270 next = MIN(next, next_block->offset);
1273 if (next - end >= size && next - end < mingap) {
1274 offset = end;
1275 mingap = next - end;
1279 if (offset == RAM_ADDR_MAX) {
1280 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1281 (uint64_t)size);
1282 abort();
1285 return offset;
1288 ram_addr_t last_ram_offset(void)
1290 RAMBlock *block;
1291 ram_addr_t last = 0;
1293 rcu_read_lock();
1294 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1295 last = MAX(last, block->offset + block->max_length);
1297 rcu_read_unlock();
1298 return last;
1301 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1303 int ret;
1305 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1306 if (!machine_dump_guest_core(current_machine)) {
1307 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1308 if (ret) {
1309 perror("qemu_madvise");
1310 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1311 "but dump_guest_core=off specified\n");
1316 /* Called within an RCU critical section, or while the ramlist lock
1317 * is held.
1319 static RAMBlock *find_ram_block(ram_addr_t addr)
1321 RAMBlock *block;
1323 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1324 if (block->offset == addr) {
1325 return block;
1329 return NULL;
1332 /* Called with iothread lock held. */
1333 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1335 RAMBlock *new_block, *block;
1337 rcu_read_lock();
1338 new_block = find_ram_block(addr);
1339 assert(new_block);
1340 assert(!new_block->idstr[0]);
1342 if (dev) {
1343 char *id = qdev_get_dev_path(dev);
1344 if (id) {
1345 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1346 g_free(id);
1349 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1351 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1352 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1353 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1354 new_block->idstr);
1355 abort();
1358 rcu_read_unlock();
1361 /* Called with iothread lock held. */
1362 void qemu_ram_unset_idstr(ram_addr_t addr)
1364 RAMBlock *block;
1366 /* FIXME: arch_init.c assumes that this is not called throughout
1367 * migration. Ignore the problem since hot-unplug during migration
1368 * does not work anyway.
1371 rcu_read_lock();
1372 block = find_ram_block(addr);
1373 if (block) {
1374 memset(block->idstr, 0, sizeof(block->idstr));
1376 rcu_read_unlock();
1379 static int memory_try_enable_merging(void *addr, size_t len)
1381 if (!machine_mem_merge(current_machine)) {
1382 /* disabled by the user */
1383 return 0;
1386 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1389 /* Only legal before guest might have detected the memory size: e.g. on
1390 * incoming migration, or right after reset.
1392 * As memory core doesn't know how is memory accessed, it is up to
1393 * resize callback to update device state and/or add assertions to detect
1394 * misuse, if necessary.
1396 int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp)
1398 RAMBlock *block = find_ram_block(base);
1400 assert(block);
1402 newsize = TARGET_PAGE_ALIGN(newsize);
1404 if (block->used_length == newsize) {
1405 return 0;
1408 if (!(block->flags & RAM_RESIZEABLE)) {
1409 error_setg_errno(errp, EINVAL,
1410 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1411 " in != 0x" RAM_ADDR_FMT, block->idstr,
1412 newsize, block->used_length);
1413 return -EINVAL;
1416 if (block->max_length < newsize) {
1417 error_setg_errno(errp, EINVAL,
1418 "Length too large: %s: 0x" RAM_ADDR_FMT
1419 " > 0x" RAM_ADDR_FMT, block->idstr,
1420 newsize, block->max_length);
1421 return -EINVAL;
1424 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
1425 block->used_length = newsize;
1426 cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
1427 DIRTY_CLIENTS_ALL);
1428 memory_region_set_size(block->mr, newsize);
1429 if (block->resized) {
1430 block->resized(block->idstr, newsize, block->host);
1432 return 0;
1435 static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
1437 RAMBlock *block;
1438 RAMBlock *last_block = NULL;
1439 ram_addr_t old_ram_size, new_ram_size;
1441 old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1443 qemu_mutex_lock_ramlist();
1444 new_block->offset = find_ram_offset(new_block->max_length);
1446 if (!new_block->host) {
1447 if (xen_enabled()) {
1448 xen_ram_alloc(new_block->offset, new_block->max_length,
1449 new_block->mr);
1450 } else {
1451 new_block->host = phys_mem_alloc(new_block->max_length,
1452 &new_block->mr->align);
1453 if (!new_block->host) {
1454 error_setg_errno(errp, errno,
1455 "cannot set up guest memory '%s'",
1456 memory_region_name(new_block->mr));
1457 qemu_mutex_unlock_ramlist();
1458 return -1;
1460 memory_try_enable_merging(new_block->host, new_block->max_length);
1464 new_ram_size = MAX(old_ram_size,
1465 (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
1466 if (new_ram_size > old_ram_size) {
1467 migration_bitmap_extend(old_ram_size, new_ram_size);
1469 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1470 * QLIST (which has an RCU-friendly variant) does not have insertion at
1471 * tail, so save the last element in last_block.
1473 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1474 last_block = block;
1475 if (block->max_length < new_block->max_length) {
1476 break;
1479 if (block) {
1480 QLIST_INSERT_BEFORE_RCU(block, new_block, next);
1481 } else if (last_block) {
1482 QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
1483 } else { /* list is empty */
1484 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
1486 ram_list.mru_block = NULL;
1488 /* Write list before version */
1489 smp_wmb();
1490 ram_list.version++;
1491 qemu_mutex_unlock_ramlist();
1493 new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1495 if (new_ram_size > old_ram_size) {
1496 int i;
1498 /* ram_list.dirty_memory[] is protected by the iothread lock. */
1499 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1500 ram_list.dirty_memory[i] =
1501 bitmap_zero_extend(ram_list.dirty_memory[i],
1502 old_ram_size, new_ram_size);
1505 cpu_physical_memory_set_dirty_range(new_block->offset,
1506 new_block->used_length,
1507 DIRTY_CLIENTS_ALL);
1509 if (new_block->host) {
1510 qemu_ram_setup_dump(new_block->host, new_block->max_length);
1511 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
1512 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
1513 if (kvm_enabled()) {
1514 kvm_setup_guest_memory(new_block->host, new_block->max_length);
1518 return new_block->offset;
1521 #ifdef __linux__
1522 ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
1523 bool share, const char *mem_path,
1524 Error **errp)
1526 RAMBlock *new_block;
1527 ram_addr_t addr;
1528 Error *local_err = NULL;
1530 if (xen_enabled()) {
1531 error_setg(errp, "-mem-path not supported with Xen");
1532 return -1;
1535 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1537 * file_ram_alloc() needs to allocate just like
1538 * phys_mem_alloc, but we haven't bothered to provide
1539 * a hook there.
1541 error_setg(errp,
1542 "-mem-path not supported with this accelerator");
1543 return -1;
1546 size = TARGET_PAGE_ALIGN(size);
1547 new_block = g_malloc0(sizeof(*new_block));
1548 new_block->mr = mr;
1549 new_block->used_length = size;
1550 new_block->max_length = size;
1551 new_block->flags = share ? RAM_SHARED : 0;
1552 new_block->host = file_ram_alloc(new_block, size,
1553 mem_path, errp);
1554 if (!new_block->host) {
1555 g_free(new_block);
1556 return -1;
1559 addr = ram_block_add(new_block, &local_err);
1560 if (local_err) {
1561 g_free(new_block);
1562 error_propagate(errp, local_err);
1563 return -1;
1565 return addr;
1567 #endif
1569 static
1570 ram_addr_t qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
1571 void (*resized)(const char*,
1572 uint64_t length,
1573 void *host),
1574 void *host, bool resizeable,
1575 MemoryRegion *mr, Error **errp)
1577 RAMBlock *new_block;
1578 ram_addr_t addr;
1579 Error *local_err = NULL;
1581 size = TARGET_PAGE_ALIGN(size);
1582 max_size = TARGET_PAGE_ALIGN(max_size);
1583 new_block = g_malloc0(sizeof(*new_block));
1584 new_block->mr = mr;
1585 new_block->resized = resized;
1586 new_block->used_length = size;
1587 new_block->max_length = max_size;
1588 assert(max_size >= size);
1589 new_block->fd = -1;
1590 new_block->host = host;
1591 if (host) {
1592 new_block->flags |= RAM_PREALLOC;
1594 if (resizeable) {
1595 new_block->flags |= RAM_RESIZEABLE;
1597 addr = ram_block_add(new_block, &local_err);
1598 if (local_err) {
1599 g_free(new_block);
1600 error_propagate(errp, local_err);
1601 return -1;
1603 return addr;
1606 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1607 MemoryRegion *mr, Error **errp)
1609 return qemu_ram_alloc_internal(size, size, NULL, host, false, mr, errp);
1612 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
1614 return qemu_ram_alloc_internal(size, size, NULL, NULL, false, mr, errp);
1617 ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
1618 void (*resized)(const char*,
1619 uint64_t length,
1620 void *host),
1621 MemoryRegion *mr, Error **errp)
1623 return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true, mr, errp);
1626 void qemu_ram_free_from_ptr(ram_addr_t addr)
1628 RAMBlock *block;
1630 qemu_mutex_lock_ramlist();
1631 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1632 if (addr == block->offset) {
1633 QLIST_REMOVE_RCU(block, next);
1634 ram_list.mru_block = NULL;
1635 /* Write list before version */
1636 smp_wmb();
1637 ram_list.version++;
1638 g_free_rcu(block, rcu);
1639 break;
1642 qemu_mutex_unlock_ramlist();
1645 static void reclaim_ramblock(RAMBlock *block)
1647 if (block->flags & RAM_PREALLOC) {
1649 } else if (xen_enabled()) {
1650 xen_invalidate_map_cache_entry(block->host);
1651 #ifndef _WIN32
1652 } else if (block->fd >= 0) {
1653 munmap(block->host, block->max_length);
1654 close(block->fd);
1655 #endif
1656 } else {
1657 qemu_anon_ram_free(block->host, block->max_length);
1659 g_free(block);
1662 void qemu_ram_free(ram_addr_t addr)
1664 RAMBlock *block;
1666 qemu_mutex_lock_ramlist();
1667 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1668 if (addr == block->offset) {
1669 QLIST_REMOVE_RCU(block, next);
1670 ram_list.mru_block = NULL;
1671 /* Write list before version */
1672 smp_wmb();
1673 ram_list.version++;
1674 call_rcu(block, reclaim_ramblock, rcu);
1675 break;
1678 qemu_mutex_unlock_ramlist();
1681 #ifndef _WIN32
1682 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1684 RAMBlock *block;
1685 ram_addr_t offset;
1686 int flags;
1687 void *area, *vaddr;
1689 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1690 offset = addr - block->offset;
1691 if (offset < block->max_length) {
1692 vaddr = ramblock_ptr(block, offset);
1693 if (block->flags & RAM_PREALLOC) {
1695 } else if (xen_enabled()) {
1696 abort();
1697 } else {
1698 flags = MAP_FIXED;
1699 if (block->fd >= 0) {
1700 flags |= (block->flags & RAM_SHARED ?
1701 MAP_SHARED : MAP_PRIVATE);
1702 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1703 flags, block->fd, offset);
1704 } else {
1706 * Remap needs to match alloc. Accelerators that
1707 * set phys_mem_alloc never remap. If they did,
1708 * we'd need a remap hook here.
1710 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1712 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1713 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1714 flags, -1, 0);
1716 if (area != vaddr) {
1717 fprintf(stderr, "Could not remap addr: "
1718 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1719 length, addr);
1720 exit(1);
1722 memory_try_enable_merging(vaddr, length);
1723 qemu_ram_setup_dump(vaddr, length);
1728 #endif /* !_WIN32 */
1730 int qemu_get_ram_fd(ram_addr_t addr)
1732 RAMBlock *block;
1733 int fd;
1735 rcu_read_lock();
1736 block = qemu_get_ram_block(addr);
1737 fd = block->fd;
1738 rcu_read_unlock();
1739 return fd;
1742 void *qemu_get_ram_block_host_ptr(ram_addr_t addr)
1744 RAMBlock *block;
1745 void *ptr;
1747 rcu_read_lock();
1748 block = qemu_get_ram_block(addr);
1749 ptr = ramblock_ptr(block, 0);
1750 rcu_read_unlock();
1751 return ptr;
1754 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1755 * This should not be used for general purpose DMA. Use address_space_map
1756 * or address_space_rw instead. For local memory (e.g. video ram) that the
1757 * device owns, use memory_region_get_ram_ptr.
1759 * By the time this function returns, the returned pointer is not protected
1760 * by RCU anymore. If the caller is not within an RCU critical section and
1761 * does not hold the iothread lock, it must have other means of protecting the
1762 * pointer, such as a reference to the region that includes the incoming
1763 * ram_addr_t.
1765 void *qemu_get_ram_ptr(ram_addr_t addr)
1767 RAMBlock *block;
1768 void *ptr;
1770 rcu_read_lock();
1771 block = qemu_get_ram_block(addr);
1773 if (xen_enabled() && block->host == NULL) {
1774 /* We need to check if the requested address is in the RAM
1775 * because we don't want to map the entire memory in QEMU.
1776 * In that case just map until the end of the page.
1778 if (block->offset == 0) {
1779 ptr = xen_map_cache(addr, 0, 0);
1780 goto unlock;
1783 block->host = xen_map_cache(block->offset, block->max_length, 1);
1785 ptr = ramblock_ptr(block, addr - block->offset);
1787 unlock:
1788 rcu_read_unlock();
1789 return ptr;
1792 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1793 * but takes a size argument.
1795 * By the time this function returns, the returned pointer is not protected
1796 * by RCU anymore. If the caller is not within an RCU critical section and
1797 * does not hold the iothread lock, it must have other means of protecting the
1798 * pointer, such as a reference to the region that includes the incoming
1799 * ram_addr_t.
1801 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1803 void *ptr;
1804 if (*size == 0) {
1805 return NULL;
1807 if (xen_enabled()) {
1808 return xen_map_cache(addr, *size, 1);
1809 } else {
1810 RAMBlock *block;
1811 rcu_read_lock();
1812 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1813 if (addr - block->offset < block->max_length) {
1814 if (addr - block->offset + *size > block->max_length)
1815 *size = block->max_length - addr + block->offset;
1816 ptr = ramblock_ptr(block, addr - block->offset);
1817 rcu_read_unlock();
1818 return ptr;
1822 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1823 abort();
1827 /* Some of the softmmu routines need to translate from a host pointer
1828 * (typically a TLB entry) back to a ram offset.
1830 * By the time this function returns, the returned pointer is not protected
1831 * by RCU anymore. If the caller is not within an RCU critical section and
1832 * does not hold the iothread lock, it must have other means of protecting the
1833 * pointer, such as a reference to the region that includes the incoming
1834 * ram_addr_t.
1836 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1838 RAMBlock *block;
1839 uint8_t *host = ptr;
1840 MemoryRegion *mr;
1842 if (xen_enabled()) {
1843 rcu_read_lock();
1844 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1845 mr = qemu_get_ram_block(*ram_addr)->mr;
1846 rcu_read_unlock();
1847 return mr;
1850 rcu_read_lock();
1851 block = atomic_rcu_read(&ram_list.mru_block);
1852 if (block && block->host && host - block->host < block->max_length) {
1853 goto found;
1856 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1857 /* This case append when the block is not mapped. */
1858 if (block->host == NULL) {
1859 continue;
1861 if (host - block->host < block->max_length) {
1862 goto found;
1866 rcu_read_unlock();
1867 return NULL;
1869 found:
1870 *ram_addr = block->offset + (host - block->host);
1871 mr = block->mr;
1872 rcu_read_unlock();
1873 return mr;
1876 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1877 uint64_t val, unsigned size)
1879 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
1880 tb_invalidate_phys_page_fast(ram_addr, size);
1882 switch (size) {
1883 case 1:
1884 stb_p(qemu_get_ram_ptr(ram_addr), val);
1885 break;
1886 case 2:
1887 stw_p(qemu_get_ram_ptr(ram_addr), val);
1888 break;
1889 case 4:
1890 stl_p(qemu_get_ram_ptr(ram_addr), val);
1891 break;
1892 default:
1893 abort();
1895 /* Set both VGA and migration bits for simplicity and to remove
1896 * the notdirty callback faster.
1898 cpu_physical_memory_set_dirty_range(ram_addr, size,
1899 DIRTY_CLIENTS_NOCODE);
1900 /* we remove the notdirty callback only if the code has been
1901 flushed */
1902 if (!cpu_physical_memory_is_clean(ram_addr)) {
1903 CPUArchState *env = current_cpu->env_ptr;
1904 tlb_set_dirty(env, current_cpu->mem_io_vaddr);
1908 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1909 unsigned size, bool is_write)
1911 return is_write;
1914 static const MemoryRegionOps notdirty_mem_ops = {
1915 .write = notdirty_mem_write,
1916 .valid.accepts = notdirty_mem_accepts,
1917 .endianness = DEVICE_NATIVE_ENDIAN,
1920 /* Generate a debug exception if a watchpoint has been hit. */
1921 static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
1923 CPUState *cpu = current_cpu;
1924 CPUArchState *env = cpu->env_ptr;
1925 target_ulong pc, cs_base;
1926 target_ulong vaddr;
1927 CPUWatchpoint *wp;
1928 int cpu_flags;
1930 if (cpu->watchpoint_hit) {
1931 /* We re-entered the check after replacing the TB. Now raise
1932 * the debug interrupt so that is will trigger after the
1933 * current instruction. */
1934 cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
1935 return;
1937 vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1938 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
1939 if (cpu_watchpoint_address_matches(wp, vaddr, len)
1940 && (wp->flags & flags)) {
1941 if (flags == BP_MEM_READ) {
1942 wp->flags |= BP_WATCHPOINT_HIT_READ;
1943 } else {
1944 wp->flags |= BP_WATCHPOINT_HIT_WRITE;
1946 wp->hitaddr = vaddr;
1947 wp->hitattrs = attrs;
1948 if (!cpu->watchpoint_hit) {
1949 cpu->watchpoint_hit = wp;
1950 tb_check_watchpoint(cpu);
1951 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1952 cpu->exception_index = EXCP_DEBUG;
1953 cpu_loop_exit(cpu);
1954 } else {
1955 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1956 tb_gen_code(cpu, pc, cs_base, cpu_flags, 1);
1957 cpu_resume_from_signal(cpu, NULL);
1960 } else {
1961 wp->flags &= ~BP_WATCHPOINT_HIT;
1966 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1967 so these check for a hit then pass through to the normal out-of-line
1968 phys routines. */
1969 static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
1970 unsigned size, MemTxAttrs attrs)
1972 MemTxResult res;
1973 uint64_t data;
1975 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
1976 switch (size) {
1977 case 1:
1978 data = address_space_ldub(&address_space_memory, addr, attrs, &res);
1979 break;
1980 case 2:
1981 data = address_space_lduw(&address_space_memory, addr, attrs, &res);
1982 break;
1983 case 4:
1984 data = address_space_ldl(&address_space_memory, addr, attrs, &res);
1985 break;
1986 default: abort();
1988 *pdata = data;
1989 return res;
1992 static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
1993 uint64_t val, unsigned size,
1994 MemTxAttrs attrs)
1996 MemTxResult res;
1998 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
1999 switch (size) {
2000 case 1:
2001 address_space_stb(&address_space_memory, addr, val, attrs, &res);
2002 break;
2003 case 2:
2004 address_space_stw(&address_space_memory, addr, val, attrs, &res);
2005 break;
2006 case 4:
2007 address_space_stl(&address_space_memory, addr, val, attrs, &res);
2008 break;
2009 default: abort();
2011 return res;
2014 static const MemoryRegionOps watch_mem_ops = {
2015 .read_with_attrs = watch_mem_read,
2016 .write_with_attrs = watch_mem_write,
2017 .endianness = DEVICE_NATIVE_ENDIAN,
2020 static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
2021 unsigned len, MemTxAttrs attrs)
2023 subpage_t *subpage = opaque;
2024 uint8_t buf[8];
2025 MemTxResult res;
2027 #if defined(DEBUG_SUBPAGE)
2028 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
2029 subpage, len, addr);
2030 #endif
2031 res = address_space_read(subpage->as, addr + subpage->base,
2032 attrs, buf, len);
2033 if (res) {
2034 return res;
2036 switch (len) {
2037 case 1:
2038 *data = ldub_p(buf);
2039 return MEMTX_OK;
2040 case 2:
2041 *data = lduw_p(buf);
2042 return MEMTX_OK;
2043 case 4:
2044 *data = ldl_p(buf);
2045 return MEMTX_OK;
2046 case 8:
2047 *data = ldq_p(buf);
2048 return MEMTX_OK;
2049 default:
2050 abort();
2054 static MemTxResult subpage_write(void *opaque, hwaddr addr,
2055 uint64_t value, unsigned len, MemTxAttrs attrs)
2057 subpage_t *subpage = opaque;
2058 uint8_t buf[8];
2060 #if defined(DEBUG_SUBPAGE)
2061 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
2062 " value %"PRIx64"\n",
2063 __func__, subpage, len, addr, value);
2064 #endif
2065 switch (len) {
2066 case 1:
2067 stb_p(buf, value);
2068 break;
2069 case 2:
2070 stw_p(buf, value);
2071 break;
2072 case 4:
2073 stl_p(buf, value);
2074 break;
2075 case 8:
2076 stq_p(buf, value);
2077 break;
2078 default:
2079 abort();
2081 return address_space_write(subpage->as, addr + subpage->base,
2082 attrs, buf, len);
2085 static bool subpage_accepts(void *opaque, hwaddr addr,
2086 unsigned len, bool is_write)
2088 subpage_t *subpage = opaque;
2089 #if defined(DEBUG_SUBPAGE)
2090 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
2091 __func__, subpage, is_write ? 'w' : 'r', len, addr);
2092 #endif
2094 return address_space_access_valid(subpage->as, addr + subpage->base,
2095 len, is_write);
2098 static const MemoryRegionOps subpage_ops = {
2099 .read_with_attrs = subpage_read,
2100 .write_with_attrs = subpage_write,
2101 .impl.min_access_size = 1,
2102 .impl.max_access_size = 8,
2103 .valid.min_access_size = 1,
2104 .valid.max_access_size = 8,
2105 .valid.accepts = subpage_accepts,
2106 .endianness = DEVICE_NATIVE_ENDIAN,
2109 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2110 uint16_t section)
2112 int idx, eidx;
2114 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2115 return -1;
2116 idx = SUBPAGE_IDX(start);
2117 eidx = SUBPAGE_IDX(end);
2118 #if defined(DEBUG_SUBPAGE)
2119 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2120 __func__, mmio, start, end, idx, eidx, section);
2121 #endif
2122 for (; idx <= eidx; idx++) {
2123 mmio->sub_section[idx] = section;
2126 return 0;
2129 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
2131 subpage_t *mmio;
2133 mmio = g_malloc0(sizeof(subpage_t));
2135 mmio->as = as;
2136 mmio->base = base;
2137 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
2138 NULL, TARGET_PAGE_SIZE);
2139 mmio->iomem.subpage = true;
2140 #if defined(DEBUG_SUBPAGE)
2141 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
2142 mmio, base, TARGET_PAGE_SIZE);
2143 #endif
2144 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
2146 return mmio;
2149 static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
2150 MemoryRegion *mr)
2152 assert(as);
2153 MemoryRegionSection section = {
2154 .address_space = as,
2155 .mr = mr,
2156 .offset_within_address_space = 0,
2157 .offset_within_region = 0,
2158 .size = int128_2_64(),
2161 return phys_section_add(map, &section);
2164 MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index)
2166 AddressSpaceDispatch *d = atomic_rcu_read(&cpu->memory_dispatch);
2167 MemoryRegionSection *sections = d->map.sections;
2169 return sections[index & ~TARGET_PAGE_MASK].mr;
2172 static void io_mem_init(void)
2174 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, NULL, UINT64_MAX);
2175 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
2176 NULL, UINT64_MAX);
2177 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
2178 NULL, UINT64_MAX);
2179 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
2180 NULL, UINT64_MAX);
2183 static void mem_begin(MemoryListener *listener)
2185 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2186 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
2187 uint16_t n;
2189 n = dummy_section(&d->map, as, &io_mem_unassigned);
2190 assert(n == PHYS_SECTION_UNASSIGNED);
2191 n = dummy_section(&d->map, as, &io_mem_notdirty);
2192 assert(n == PHYS_SECTION_NOTDIRTY);
2193 n = dummy_section(&d->map, as, &io_mem_rom);
2194 assert(n == PHYS_SECTION_ROM);
2195 n = dummy_section(&d->map, as, &io_mem_watch);
2196 assert(n == PHYS_SECTION_WATCH);
2198 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
2199 d->as = as;
2200 as->next_dispatch = d;
2203 static void address_space_dispatch_free(AddressSpaceDispatch *d)
2205 phys_sections_free(&d->map);
2206 g_free(d);
2209 static void mem_commit(MemoryListener *listener)
2211 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2212 AddressSpaceDispatch *cur = as->dispatch;
2213 AddressSpaceDispatch *next = as->next_dispatch;
2215 phys_page_compact_all(next, next->map.nodes_nb);
2217 atomic_rcu_set(&as->dispatch, next);
2218 if (cur) {
2219 call_rcu(cur, address_space_dispatch_free, rcu);
2223 static void tcg_commit(MemoryListener *listener)
2225 CPUState *cpu;
2227 /* since each CPU stores ram addresses in its TLB cache, we must
2228 reset the modified entries */
2229 /* XXX: slow ! */
2230 CPU_FOREACH(cpu) {
2231 /* FIXME: Disentangle the cpu.h circular files deps so we can
2232 directly get the right CPU from listener. */
2233 if (cpu->tcg_as_listener != listener) {
2234 continue;
2236 cpu_reload_memory_map(cpu);
2240 void address_space_init_dispatch(AddressSpace *as)
2242 as->dispatch = NULL;
2243 as->dispatch_listener = (MemoryListener) {
2244 .begin = mem_begin,
2245 .commit = mem_commit,
2246 .region_add = mem_add,
2247 .region_nop = mem_add,
2248 .priority = 0,
2250 memory_listener_register(&as->dispatch_listener, as);
2253 void address_space_unregister(AddressSpace *as)
2255 memory_listener_unregister(&as->dispatch_listener);
2258 void address_space_destroy_dispatch(AddressSpace *as)
2260 AddressSpaceDispatch *d = as->dispatch;
2262 atomic_rcu_set(&as->dispatch, NULL);
2263 if (d) {
2264 call_rcu(d, address_space_dispatch_free, rcu);
2268 static void memory_map_init(void)
2270 system_memory = g_malloc(sizeof(*system_memory));
2272 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
2273 address_space_init(&address_space_memory, system_memory, "memory");
2275 system_io = g_malloc(sizeof(*system_io));
2276 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2277 65536);
2278 address_space_init(&address_space_io, system_io, "I/O");
2281 MemoryRegion *get_system_memory(void)
2283 return system_memory;
2286 MemoryRegion *get_system_io(void)
2288 return system_io;
2291 #endif /* !defined(CONFIG_USER_ONLY) */
2293 /* physical memory access (slow version, mainly for debug) */
2294 #if defined(CONFIG_USER_ONLY)
2295 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2296 uint8_t *buf, int len, int is_write)
2298 int l, flags;
2299 target_ulong page;
2300 void * p;
2302 while (len > 0) {
2303 page = addr & TARGET_PAGE_MASK;
2304 l = (page + TARGET_PAGE_SIZE) - addr;
2305 if (l > len)
2306 l = len;
2307 flags = page_get_flags(page);
2308 if (!(flags & PAGE_VALID))
2309 return -1;
2310 if (is_write) {
2311 if (!(flags & PAGE_WRITE))
2312 return -1;
2313 /* XXX: this code should not depend on lock_user */
2314 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2315 return -1;
2316 memcpy(p, buf, l);
2317 unlock_user(p, addr, l);
2318 } else {
2319 if (!(flags & PAGE_READ))
2320 return -1;
2321 /* XXX: this code should not depend on lock_user */
2322 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2323 return -1;
2324 memcpy(buf, p, l);
2325 unlock_user(p, addr, 0);
2327 len -= l;
2328 buf += l;
2329 addr += l;
2331 return 0;
2334 #else
2336 static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
2337 hwaddr length)
2339 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
2340 /* No early return if dirty_log_mask is or becomes 0, because
2341 * cpu_physical_memory_set_dirty_range will still call
2342 * xen_modified_memory.
2344 if (dirty_log_mask) {
2345 dirty_log_mask =
2346 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
2348 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
2349 tb_invalidate_phys_range(addr, addr + length);
2350 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
2352 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
2355 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
2357 unsigned access_size_max = mr->ops->valid.max_access_size;
2359 /* Regions are assumed to support 1-4 byte accesses unless
2360 otherwise specified. */
2361 if (access_size_max == 0) {
2362 access_size_max = 4;
2365 /* Bound the maximum access by the alignment of the address. */
2366 if (!mr->ops->impl.unaligned) {
2367 unsigned align_size_max = addr & -addr;
2368 if (align_size_max != 0 && align_size_max < access_size_max) {
2369 access_size_max = align_size_max;
2373 /* Don't attempt accesses larger than the maximum. */
2374 if (l > access_size_max) {
2375 l = access_size_max;
2377 if (l & (l - 1)) {
2378 l = 1 << (qemu_fls(l) - 1);
2381 return l;
2384 static bool prepare_mmio_access(MemoryRegion *mr)
2386 bool unlocked = !qemu_mutex_iothread_locked();
2387 bool release_lock = false;
2389 if (unlocked && mr->global_locking) {
2390 qemu_mutex_lock_iothread();
2391 unlocked = false;
2392 release_lock = true;
2394 if (mr->flush_coalesced_mmio) {
2395 if (unlocked) {
2396 qemu_mutex_lock_iothread();
2398 qemu_flush_coalesced_mmio_buffer();
2399 if (unlocked) {
2400 qemu_mutex_unlock_iothread();
2404 return release_lock;
2407 MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2408 uint8_t *buf, int len, bool is_write)
2410 hwaddr l;
2411 uint8_t *ptr;
2412 uint64_t val;
2413 hwaddr addr1;
2414 MemoryRegion *mr;
2415 MemTxResult result = MEMTX_OK;
2416 bool release_lock = false;
2418 rcu_read_lock();
2419 while (len > 0) {
2420 l = len;
2421 mr = address_space_translate(as, addr, &addr1, &l, is_write);
2423 if (is_write) {
2424 if (!memory_access_is_direct(mr, is_write)) {
2425 release_lock |= prepare_mmio_access(mr);
2426 l = memory_access_size(mr, l, addr1);
2427 /* XXX: could force current_cpu to NULL to avoid
2428 potential bugs */
2429 switch (l) {
2430 case 8:
2431 /* 64 bit write access */
2432 val = ldq_p(buf);
2433 result |= memory_region_dispatch_write(mr, addr1, val, 8,
2434 attrs);
2435 break;
2436 case 4:
2437 /* 32 bit write access */
2438 val = ldl_p(buf);
2439 result |= memory_region_dispatch_write(mr, addr1, val, 4,
2440 attrs);
2441 break;
2442 case 2:
2443 /* 16 bit write access */
2444 val = lduw_p(buf);
2445 result |= memory_region_dispatch_write(mr, addr1, val, 2,
2446 attrs);
2447 break;
2448 case 1:
2449 /* 8 bit write access */
2450 val = ldub_p(buf);
2451 result |= memory_region_dispatch_write(mr, addr1, val, 1,
2452 attrs);
2453 break;
2454 default:
2455 abort();
2457 } else {
2458 addr1 += memory_region_get_ram_addr(mr);
2459 /* RAM case */
2460 ptr = qemu_get_ram_ptr(addr1);
2461 memcpy(ptr, buf, l);
2462 invalidate_and_set_dirty(mr, addr1, l);
2464 } else {
2465 if (!memory_access_is_direct(mr, is_write)) {
2466 /* I/O case */
2467 release_lock |= prepare_mmio_access(mr);
2468 l = memory_access_size(mr, l, addr1);
2469 switch (l) {
2470 case 8:
2471 /* 64 bit read access */
2472 result |= memory_region_dispatch_read(mr, addr1, &val, 8,
2473 attrs);
2474 stq_p(buf, val);
2475 break;
2476 case 4:
2477 /* 32 bit read access */
2478 result |= memory_region_dispatch_read(mr, addr1, &val, 4,
2479 attrs);
2480 stl_p(buf, val);
2481 break;
2482 case 2:
2483 /* 16 bit read access */
2484 result |= memory_region_dispatch_read(mr, addr1, &val, 2,
2485 attrs);
2486 stw_p(buf, val);
2487 break;
2488 case 1:
2489 /* 8 bit read access */
2490 result |= memory_region_dispatch_read(mr, addr1, &val, 1,
2491 attrs);
2492 stb_p(buf, val);
2493 break;
2494 default:
2495 abort();
2497 } else {
2498 /* RAM case */
2499 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2500 memcpy(buf, ptr, l);
2504 if (release_lock) {
2505 qemu_mutex_unlock_iothread();
2506 release_lock = false;
2509 len -= l;
2510 buf += l;
2511 addr += l;
2513 rcu_read_unlock();
2515 return result;
2518 MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2519 const uint8_t *buf, int len)
2521 return address_space_rw(as, addr, attrs, (uint8_t *)buf, len, true);
2524 MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2525 uint8_t *buf, int len)
2527 return address_space_rw(as, addr, attrs, buf, len, false);
2531 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2532 int len, int is_write)
2534 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
2535 buf, len, is_write);
2538 enum write_rom_type {
2539 WRITE_DATA,
2540 FLUSH_CACHE,
2543 static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
2544 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
2546 hwaddr l;
2547 uint8_t *ptr;
2548 hwaddr addr1;
2549 MemoryRegion *mr;
2551 rcu_read_lock();
2552 while (len > 0) {
2553 l = len;
2554 mr = address_space_translate(as, addr, &addr1, &l, true);
2556 if (!(memory_region_is_ram(mr) ||
2557 memory_region_is_romd(mr))) {
2558 l = memory_access_size(mr, l, addr1);
2559 } else {
2560 addr1 += memory_region_get_ram_addr(mr);
2561 /* ROM/RAM case */
2562 ptr = qemu_get_ram_ptr(addr1);
2563 switch (type) {
2564 case WRITE_DATA:
2565 memcpy(ptr, buf, l);
2566 invalidate_and_set_dirty(mr, addr1, l);
2567 break;
2568 case FLUSH_CACHE:
2569 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
2570 break;
2573 len -= l;
2574 buf += l;
2575 addr += l;
2577 rcu_read_unlock();
2580 /* used for ROM loading : can write in RAM and ROM */
2581 void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
2582 const uint8_t *buf, int len)
2584 cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
2587 void cpu_flush_icache_range(hwaddr start, int len)
2590 * This function should do the same thing as an icache flush that was
2591 * triggered from within the guest. For TCG we are always cache coherent,
2592 * so there is no need to flush anything. For KVM / Xen we need to flush
2593 * the host's instruction cache at least.
2595 if (tcg_enabled()) {
2596 return;
2599 cpu_physical_memory_write_rom_internal(&address_space_memory,
2600 start, NULL, len, FLUSH_CACHE);
2603 typedef struct {
2604 MemoryRegion *mr;
2605 void *buffer;
2606 hwaddr addr;
2607 hwaddr len;
2608 bool in_use;
2609 } BounceBuffer;
2611 static BounceBuffer bounce;
2613 typedef struct MapClient {
2614 QEMUBH *bh;
2615 QLIST_ENTRY(MapClient) link;
2616 } MapClient;
2618 QemuMutex map_client_list_lock;
2619 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2620 = QLIST_HEAD_INITIALIZER(map_client_list);
2622 static void cpu_unregister_map_client_do(MapClient *client)
2624 QLIST_REMOVE(client, link);
2625 g_free(client);
2628 static void cpu_notify_map_clients_locked(void)
2630 MapClient *client;
2632 while (!QLIST_EMPTY(&map_client_list)) {
2633 client = QLIST_FIRST(&map_client_list);
2634 qemu_bh_schedule(client->bh);
2635 cpu_unregister_map_client_do(client);
2639 void cpu_register_map_client(QEMUBH *bh)
2641 MapClient *client = g_malloc(sizeof(*client));
2643 qemu_mutex_lock(&map_client_list_lock);
2644 client->bh = bh;
2645 QLIST_INSERT_HEAD(&map_client_list, client, link);
2646 if (!atomic_read(&bounce.in_use)) {
2647 cpu_notify_map_clients_locked();
2649 qemu_mutex_unlock(&map_client_list_lock);
2652 void cpu_exec_init_all(void)
2654 qemu_mutex_init(&ram_list.mutex);
2655 memory_map_init();
2656 io_mem_init();
2657 qemu_mutex_init(&map_client_list_lock);
2660 void cpu_unregister_map_client(QEMUBH *bh)
2662 MapClient *client;
2664 qemu_mutex_lock(&map_client_list_lock);
2665 QLIST_FOREACH(client, &map_client_list, link) {
2666 if (client->bh == bh) {
2667 cpu_unregister_map_client_do(client);
2668 break;
2671 qemu_mutex_unlock(&map_client_list_lock);
2674 static void cpu_notify_map_clients(void)
2676 qemu_mutex_lock(&map_client_list_lock);
2677 cpu_notify_map_clients_locked();
2678 qemu_mutex_unlock(&map_client_list_lock);
2681 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2683 MemoryRegion *mr;
2684 hwaddr l, xlat;
2686 rcu_read_lock();
2687 while (len > 0) {
2688 l = len;
2689 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2690 if (!memory_access_is_direct(mr, is_write)) {
2691 l = memory_access_size(mr, l, addr);
2692 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2693 return false;
2697 len -= l;
2698 addr += l;
2700 rcu_read_unlock();
2701 return true;
2704 /* Map a physical memory region into a host virtual address.
2705 * May map a subset of the requested range, given by and returned in *plen.
2706 * May return NULL if resources needed to perform the mapping are exhausted.
2707 * Use only for reads OR writes - not for read-modify-write operations.
2708 * Use cpu_register_map_client() to know when retrying the map operation is
2709 * likely to succeed.
2711 void *address_space_map(AddressSpace *as,
2712 hwaddr addr,
2713 hwaddr *plen,
2714 bool is_write)
2716 hwaddr len = *plen;
2717 hwaddr done = 0;
2718 hwaddr l, xlat, base;
2719 MemoryRegion *mr, *this_mr;
2720 ram_addr_t raddr;
2722 if (len == 0) {
2723 return NULL;
2726 l = len;
2727 rcu_read_lock();
2728 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2730 if (!memory_access_is_direct(mr, is_write)) {
2731 if (atomic_xchg(&bounce.in_use, true)) {
2732 rcu_read_unlock();
2733 return NULL;
2735 /* Avoid unbounded allocations */
2736 l = MIN(l, TARGET_PAGE_SIZE);
2737 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
2738 bounce.addr = addr;
2739 bounce.len = l;
2741 memory_region_ref(mr);
2742 bounce.mr = mr;
2743 if (!is_write) {
2744 address_space_read(as, addr, MEMTXATTRS_UNSPECIFIED,
2745 bounce.buffer, l);
2748 rcu_read_unlock();
2749 *plen = l;
2750 return bounce.buffer;
2753 base = xlat;
2754 raddr = memory_region_get_ram_addr(mr);
2756 for (;;) {
2757 len -= l;
2758 addr += l;
2759 done += l;
2760 if (len == 0) {
2761 break;
2764 l = len;
2765 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2766 if (this_mr != mr || xlat != base + done) {
2767 break;
2771 memory_region_ref(mr);
2772 rcu_read_unlock();
2773 *plen = done;
2774 return qemu_ram_ptr_length(raddr + base, plen);
2777 /* Unmaps a memory region previously mapped by address_space_map().
2778 * Will also mark the memory as dirty if is_write == 1. access_len gives
2779 * the amount of memory that was actually read or written by the caller.
2781 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2782 int is_write, hwaddr access_len)
2784 if (buffer != bounce.buffer) {
2785 MemoryRegion *mr;
2786 ram_addr_t addr1;
2788 mr = qemu_ram_addr_from_host(buffer, &addr1);
2789 assert(mr != NULL);
2790 if (is_write) {
2791 invalidate_and_set_dirty(mr, addr1, access_len);
2793 if (xen_enabled()) {
2794 xen_invalidate_map_cache_entry(buffer);
2796 memory_region_unref(mr);
2797 return;
2799 if (is_write) {
2800 address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
2801 bounce.buffer, access_len);
2803 qemu_vfree(bounce.buffer);
2804 bounce.buffer = NULL;
2805 memory_region_unref(bounce.mr);
2806 atomic_mb_set(&bounce.in_use, false);
2807 cpu_notify_map_clients();
2810 void *cpu_physical_memory_map(hwaddr addr,
2811 hwaddr *plen,
2812 int is_write)
2814 return address_space_map(&address_space_memory, addr, plen, is_write);
2817 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2818 int is_write, hwaddr access_len)
2820 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2823 /* warning: addr must be aligned */
2824 static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
2825 MemTxAttrs attrs,
2826 MemTxResult *result,
2827 enum device_endian endian)
2829 uint8_t *ptr;
2830 uint64_t val;
2831 MemoryRegion *mr;
2832 hwaddr l = 4;
2833 hwaddr addr1;
2834 MemTxResult r;
2835 bool release_lock = false;
2837 rcu_read_lock();
2838 mr = address_space_translate(as, addr, &addr1, &l, false);
2839 if (l < 4 || !memory_access_is_direct(mr, false)) {
2840 release_lock |= prepare_mmio_access(mr);
2842 /* I/O case */
2843 r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
2844 #if defined(TARGET_WORDS_BIGENDIAN)
2845 if (endian == DEVICE_LITTLE_ENDIAN) {
2846 val = bswap32(val);
2848 #else
2849 if (endian == DEVICE_BIG_ENDIAN) {
2850 val = bswap32(val);
2852 #endif
2853 } else {
2854 /* RAM case */
2855 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2856 & TARGET_PAGE_MASK)
2857 + addr1);
2858 switch (endian) {
2859 case DEVICE_LITTLE_ENDIAN:
2860 val = ldl_le_p(ptr);
2861 break;
2862 case DEVICE_BIG_ENDIAN:
2863 val = ldl_be_p(ptr);
2864 break;
2865 default:
2866 val = ldl_p(ptr);
2867 break;
2869 r = MEMTX_OK;
2871 if (result) {
2872 *result = r;
2874 if (release_lock) {
2875 qemu_mutex_unlock_iothread();
2877 rcu_read_unlock();
2878 return val;
2881 uint32_t address_space_ldl(AddressSpace *as, hwaddr addr,
2882 MemTxAttrs attrs, MemTxResult *result)
2884 return address_space_ldl_internal(as, addr, attrs, result,
2885 DEVICE_NATIVE_ENDIAN);
2888 uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr,
2889 MemTxAttrs attrs, MemTxResult *result)
2891 return address_space_ldl_internal(as, addr, attrs, result,
2892 DEVICE_LITTLE_ENDIAN);
2895 uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr,
2896 MemTxAttrs attrs, MemTxResult *result)
2898 return address_space_ldl_internal(as, addr, attrs, result,
2899 DEVICE_BIG_ENDIAN);
2902 uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
2904 return address_space_ldl(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
2907 uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
2909 return address_space_ldl_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
2912 uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
2914 return address_space_ldl_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
2917 /* warning: addr must be aligned */
2918 static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
2919 MemTxAttrs attrs,
2920 MemTxResult *result,
2921 enum device_endian endian)
2923 uint8_t *ptr;
2924 uint64_t val;
2925 MemoryRegion *mr;
2926 hwaddr l = 8;
2927 hwaddr addr1;
2928 MemTxResult r;
2929 bool release_lock = false;
2931 rcu_read_lock();
2932 mr = address_space_translate(as, addr, &addr1, &l,
2933 false);
2934 if (l < 8 || !memory_access_is_direct(mr, false)) {
2935 release_lock |= prepare_mmio_access(mr);
2937 /* I/O case */
2938 r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
2939 #if defined(TARGET_WORDS_BIGENDIAN)
2940 if (endian == DEVICE_LITTLE_ENDIAN) {
2941 val = bswap64(val);
2943 #else
2944 if (endian == DEVICE_BIG_ENDIAN) {
2945 val = bswap64(val);
2947 #endif
2948 } else {
2949 /* RAM case */
2950 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2951 & TARGET_PAGE_MASK)
2952 + addr1);
2953 switch (endian) {
2954 case DEVICE_LITTLE_ENDIAN:
2955 val = ldq_le_p(ptr);
2956 break;
2957 case DEVICE_BIG_ENDIAN:
2958 val = ldq_be_p(ptr);
2959 break;
2960 default:
2961 val = ldq_p(ptr);
2962 break;
2964 r = MEMTX_OK;
2966 if (result) {
2967 *result = r;
2969 if (release_lock) {
2970 qemu_mutex_unlock_iothread();
2972 rcu_read_unlock();
2973 return val;
2976 uint64_t address_space_ldq(AddressSpace *as, hwaddr addr,
2977 MemTxAttrs attrs, MemTxResult *result)
2979 return address_space_ldq_internal(as, addr, attrs, result,
2980 DEVICE_NATIVE_ENDIAN);
2983 uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr,
2984 MemTxAttrs attrs, MemTxResult *result)
2986 return address_space_ldq_internal(as, addr, attrs, result,
2987 DEVICE_LITTLE_ENDIAN);
2990 uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr,
2991 MemTxAttrs attrs, MemTxResult *result)
2993 return address_space_ldq_internal(as, addr, attrs, result,
2994 DEVICE_BIG_ENDIAN);
2997 uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
2999 return address_space_ldq(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3002 uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
3004 return address_space_ldq_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3007 uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
3009 return address_space_ldq_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3012 /* XXX: optimize */
3013 uint32_t address_space_ldub(AddressSpace *as, hwaddr addr,
3014 MemTxAttrs attrs, MemTxResult *result)
3016 uint8_t val;
3017 MemTxResult r;
3019 r = address_space_rw(as, addr, attrs, &val, 1, 0);
3020 if (result) {
3021 *result = r;
3023 return val;
3026 uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
3028 return address_space_ldub(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3031 /* warning: addr must be aligned */
3032 static inline uint32_t address_space_lduw_internal(AddressSpace *as,
3033 hwaddr addr,
3034 MemTxAttrs attrs,
3035 MemTxResult *result,
3036 enum device_endian endian)
3038 uint8_t *ptr;
3039 uint64_t val;
3040 MemoryRegion *mr;
3041 hwaddr l = 2;
3042 hwaddr addr1;
3043 MemTxResult r;
3044 bool release_lock = false;
3046 rcu_read_lock();
3047 mr = address_space_translate(as, addr, &addr1, &l,
3048 false);
3049 if (l < 2 || !memory_access_is_direct(mr, false)) {
3050 release_lock |= prepare_mmio_access(mr);
3052 /* I/O case */
3053 r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
3054 #if defined(TARGET_WORDS_BIGENDIAN)
3055 if (endian == DEVICE_LITTLE_ENDIAN) {
3056 val = bswap16(val);
3058 #else
3059 if (endian == DEVICE_BIG_ENDIAN) {
3060 val = bswap16(val);
3062 #endif
3063 } else {
3064 /* RAM case */
3065 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
3066 & TARGET_PAGE_MASK)
3067 + addr1);
3068 switch (endian) {
3069 case DEVICE_LITTLE_ENDIAN:
3070 val = lduw_le_p(ptr);
3071 break;
3072 case DEVICE_BIG_ENDIAN:
3073 val = lduw_be_p(ptr);
3074 break;
3075 default:
3076 val = lduw_p(ptr);
3077 break;
3079 r = MEMTX_OK;
3081 if (result) {
3082 *result = r;
3084 if (release_lock) {
3085 qemu_mutex_unlock_iothread();
3087 rcu_read_unlock();
3088 return val;
3091 uint32_t address_space_lduw(AddressSpace *as, hwaddr addr,
3092 MemTxAttrs attrs, MemTxResult *result)
3094 return address_space_lduw_internal(as, addr, attrs, result,
3095 DEVICE_NATIVE_ENDIAN);
3098 uint32_t address_space_lduw_le(AddressSpace *as, hwaddr addr,
3099 MemTxAttrs attrs, MemTxResult *result)
3101 return address_space_lduw_internal(as, addr, attrs, result,
3102 DEVICE_LITTLE_ENDIAN);
3105 uint32_t address_space_lduw_be(AddressSpace *as, hwaddr addr,
3106 MemTxAttrs attrs, MemTxResult *result)
3108 return address_space_lduw_internal(as, addr, attrs, result,
3109 DEVICE_BIG_ENDIAN);
3112 uint32_t lduw_phys(AddressSpace *as, hwaddr addr)
3114 return address_space_lduw(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3117 uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr)
3119 return address_space_lduw_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3122 uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr)
3124 return address_space_lduw_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3127 /* warning: addr must be aligned. The ram page is not masked as dirty
3128 and the code inside is not invalidated. It is useful if the dirty
3129 bits are used to track modified PTEs */
3130 void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
3131 MemTxAttrs attrs, MemTxResult *result)
3133 uint8_t *ptr;
3134 MemoryRegion *mr;
3135 hwaddr l = 4;
3136 hwaddr addr1;
3137 MemTxResult r;
3138 uint8_t dirty_log_mask;
3139 bool release_lock = false;
3141 rcu_read_lock();
3142 mr = address_space_translate(as, addr, &addr1, &l,
3143 true);
3144 if (l < 4 || !memory_access_is_direct(mr, true)) {
3145 release_lock |= prepare_mmio_access(mr);
3147 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3148 } else {
3149 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3150 ptr = qemu_get_ram_ptr(addr1);
3151 stl_p(ptr, val);
3153 dirty_log_mask = memory_region_get_dirty_log_mask(mr);
3154 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
3155 cpu_physical_memory_set_dirty_range(addr1, 4, dirty_log_mask);
3156 r = MEMTX_OK;
3158 if (result) {
3159 *result = r;
3161 if (release_lock) {
3162 qemu_mutex_unlock_iothread();
3164 rcu_read_unlock();
3167 void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
3169 address_space_stl_notdirty(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3172 /* warning: addr must be aligned */
3173 static inline void address_space_stl_internal(AddressSpace *as,
3174 hwaddr addr, uint32_t val,
3175 MemTxAttrs attrs,
3176 MemTxResult *result,
3177 enum device_endian endian)
3179 uint8_t *ptr;
3180 MemoryRegion *mr;
3181 hwaddr l = 4;
3182 hwaddr addr1;
3183 MemTxResult r;
3184 bool release_lock = false;
3186 rcu_read_lock();
3187 mr = address_space_translate(as, addr, &addr1, &l,
3188 true);
3189 if (l < 4 || !memory_access_is_direct(mr, true)) {
3190 release_lock |= prepare_mmio_access(mr);
3192 #if defined(TARGET_WORDS_BIGENDIAN)
3193 if (endian == DEVICE_LITTLE_ENDIAN) {
3194 val = bswap32(val);
3196 #else
3197 if (endian == DEVICE_BIG_ENDIAN) {
3198 val = bswap32(val);
3200 #endif
3201 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3202 } else {
3203 /* RAM case */
3204 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3205 ptr = qemu_get_ram_ptr(addr1);
3206 switch (endian) {
3207 case DEVICE_LITTLE_ENDIAN:
3208 stl_le_p(ptr, val);
3209 break;
3210 case DEVICE_BIG_ENDIAN:
3211 stl_be_p(ptr, val);
3212 break;
3213 default:
3214 stl_p(ptr, val);
3215 break;
3217 invalidate_and_set_dirty(mr, addr1, 4);
3218 r = MEMTX_OK;
3220 if (result) {
3221 *result = r;
3223 if (release_lock) {
3224 qemu_mutex_unlock_iothread();
3226 rcu_read_unlock();
3229 void address_space_stl(AddressSpace *as, hwaddr addr, uint32_t val,
3230 MemTxAttrs attrs, MemTxResult *result)
3232 address_space_stl_internal(as, addr, val, attrs, result,
3233 DEVICE_NATIVE_ENDIAN);
3236 void address_space_stl_le(AddressSpace *as, hwaddr addr, uint32_t val,
3237 MemTxAttrs attrs, MemTxResult *result)
3239 address_space_stl_internal(as, addr, val, attrs, result,
3240 DEVICE_LITTLE_ENDIAN);
3243 void address_space_stl_be(AddressSpace *as, hwaddr addr, uint32_t val,
3244 MemTxAttrs attrs, MemTxResult *result)
3246 address_space_stl_internal(as, addr, val, attrs, result,
3247 DEVICE_BIG_ENDIAN);
3250 void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3252 address_space_stl(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3255 void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3257 address_space_stl_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3260 void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3262 address_space_stl_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3265 /* XXX: optimize */
3266 void address_space_stb(AddressSpace *as, hwaddr addr, uint32_t val,
3267 MemTxAttrs attrs, MemTxResult *result)
3269 uint8_t v = val;
3270 MemTxResult r;
3272 r = address_space_rw(as, addr, attrs, &v, 1, 1);
3273 if (result) {
3274 *result = r;
3278 void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3280 address_space_stb(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3283 /* warning: addr must be aligned */
3284 static inline void address_space_stw_internal(AddressSpace *as,
3285 hwaddr addr, uint32_t val,
3286 MemTxAttrs attrs,
3287 MemTxResult *result,
3288 enum device_endian endian)
3290 uint8_t *ptr;
3291 MemoryRegion *mr;
3292 hwaddr l = 2;
3293 hwaddr addr1;
3294 MemTxResult r;
3295 bool release_lock = false;
3297 rcu_read_lock();
3298 mr = address_space_translate(as, addr, &addr1, &l, true);
3299 if (l < 2 || !memory_access_is_direct(mr, true)) {
3300 release_lock |= prepare_mmio_access(mr);
3302 #if defined(TARGET_WORDS_BIGENDIAN)
3303 if (endian == DEVICE_LITTLE_ENDIAN) {
3304 val = bswap16(val);
3306 #else
3307 if (endian == DEVICE_BIG_ENDIAN) {
3308 val = bswap16(val);
3310 #endif
3311 r = memory_region_dispatch_write(mr, addr1, val, 2, attrs);
3312 } else {
3313 /* RAM case */
3314 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3315 ptr = qemu_get_ram_ptr(addr1);
3316 switch (endian) {
3317 case DEVICE_LITTLE_ENDIAN:
3318 stw_le_p(ptr, val);
3319 break;
3320 case DEVICE_BIG_ENDIAN:
3321 stw_be_p(ptr, val);
3322 break;
3323 default:
3324 stw_p(ptr, val);
3325 break;
3327 invalidate_and_set_dirty(mr, addr1, 2);
3328 r = MEMTX_OK;
3330 if (result) {
3331 *result = r;
3333 if (release_lock) {
3334 qemu_mutex_unlock_iothread();
3336 rcu_read_unlock();
3339 void address_space_stw(AddressSpace *as, hwaddr addr, uint32_t val,
3340 MemTxAttrs attrs, MemTxResult *result)
3342 address_space_stw_internal(as, addr, val, attrs, result,
3343 DEVICE_NATIVE_ENDIAN);
3346 void address_space_stw_le(AddressSpace *as, hwaddr addr, uint32_t val,
3347 MemTxAttrs attrs, MemTxResult *result)
3349 address_space_stw_internal(as, addr, val, attrs, result,
3350 DEVICE_LITTLE_ENDIAN);
3353 void address_space_stw_be(AddressSpace *as, hwaddr addr, uint32_t val,
3354 MemTxAttrs attrs, MemTxResult *result)
3356 address_space_stw_internal(as, addr, val, attrs, result,
3357 DEVICE_BIG_ENDIAN);
3360 void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3362 address_space_stw(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3365 void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3367 address_space_stw_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3370 void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3372 address_space_stw_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3375 /* XXX: optimize */
3376 void address_space_stq(AddressSpace *as, hwaddr addr, uint64_t val,
3377 MemTxAttrs attrs, MemTxResult *result)
3379 MemTxResult r;
3380 val = tswap64(val);
3381 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3382 if (result) {
3383 *result = r;
3387 void address_space_stq_le(AddressSpace *as, hwaddr addr, uint64_t val,
3388 MemTxAttrs attrs, MemTxResult *result)
3390 MemTxResult r;
3391 val = cpu_to_le64(val);
3392 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3393 if (result) {
3394 *result = r;
3397 void address_space_stq_be(AddressSpace *as, hwaddr addr, uint64_t val,
3398 MemTxAttrs attrs, MemTxResult *result)
3400 MemTxResult r;
3401 val = cpu_to_be64(val);
3402 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3403 if (result) {
3404 *result = r;
3408 void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3410 address_space_stq(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3413 void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3415 address_space_stq_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3418 void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3420 address_space_stq_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3423 /* virtual memory access for debug (includes writing to ROM) */
3424 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
3425 uint8_t *buf, int len, int is_write)
3427 int l;
3428 hwaddr phys_addr;
3429 target_ulong page;
3431 while (len > 0) {
3432 page = addr & TARGET_PAGE_MASK;
3433 phys_addr = cpu_get_phys_page_debug(cpu, page);
3434 /* if no physical page mapped, return an error */
3435 if (phys_addr == -1)
3436 return -1;
3437 l = (page + TARGET_PAGE_SIZE) - addr;
3438 if (l > len)
3439 l = len;
3440 phys_addr += (addr & ~TARGET_PAGE_MASK);
3441 if (is_write) {
3442 cpu_physical_memory_write_rom(cpu->as, phys_addr, buf, l);
3443 } else {
3444 address_space_rw(cpu->as, phys_addr, MEMTXATTRS_UNSPECIFIED,
3445 buf, l, 0);
3447 len -= l;
3448 buf += l;
3449 addr += l;
3451 return 0;
3453 #endif
3456 * A helper function for the _utterly broken_ virtio device model to find out if
3457 * it's running on a big endian machine. Don't do this at home kids!
3459 bool target_words_bigendian(void);
3460 bool target_words_bigendian(void)
3462 #if defined(TARGET_WORDS_BIGENDIAN)
3463 return true;
3464 #else
3465 return false;
3466 #endif
3469 #ifndef CONFIG_USER_ONLY
3470 bool cpu_physical_memory_is_io(hwaddr phys_addr)
3472 MemoryRegion*mr;
3473 hwaddr l = 1;
3474 bool res;
3476 rcu_read_lock();
3477 mr = address_space_translate(&address_space_memory,
3478 phys_addr, &phys_addr, &l, false);
3480 res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
3481 rcu_read_unlock();
3482 return res;
3485 int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
3487 RAMBlock *block;
3488 int ret = 0;
3490 rcu_read_lock();
3491 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
3492 ret = func(block->idstr, block->host, block->offset,
3493 block->used_length, opaque);
3494 if (ret) {
3495 break;
3498 rcu_read_unlock();
3499 return ret;
3501 #endif