vfio: fix duplicate function call
[qemu.git] / exec.c
blobe63c5a1bd4d5ef4a14aaae5ea5040751407a8f5a
1 /*
2 * Virtual page mapping
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "qemu/osdep.h"
20 #include "qapi/error.h"
21 #ifndef _WIN32
22 #endif
24 #include "qemu/cutils.h"
25 #include "cpu.h"
26 #include "exec/exec-all.h"
27 #include "tcg.h"
28 #include "hw/qdev-core.h"
29 #if !defined(CONFIG_USER_ONLY)
30 #include "hw/boards.h"
31 #include "hw/xen/xen.h"
32 #endif
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "qemu/timer.h"
36 #include "qemu/config-file.h"
37 #include "qemu/error-report.h"
38 #if defined(CONFIG_USER_ONLY)
39 #include "qemu.h"
40 #else /* !CONFIG_USER_ONLY */
41 #include "hw/hw.h"
42 #include "exec/memory.h"
43 #include "exec/ioport.h"
44 #include "sysemu/dma.h"
45 #include "exec/address-spaces.h"
46 #include "sysemu/xen-mapcache.h"
47 #include "trace.h"
48 #endif
49 #include "exec/cpu-all.h"
50 #include "qemu/rcu_queue.h"
51 #include "qemu/main-loop.h"
52 #include "translate-all.h"
53 #include "sysemu/replay.h"
55 #include "exec/memory-internal.h"
56 #include "exec/ram_addr.h"
57 #include "exec/log.h"
59 #include "migration/vmstate.h"
61 #include "qemu/range.h"
62 #ifndef _WIN32
63 #include "qemu/mmap-alloc.h"
64 #endif
66 //#define DEBUG_SUBPAGE
68 #if !defined(CONFIG_USER_ONLY)
69 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
70 * are protected by the ramlist lock.
72 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
74 static MemoryRegion *system_memory;
75 static MemoryRegion *system_io;
77 AddressSpace address_space_io;
78 AddressSpace address_space_memory;
80 MemoryRegion io_mem_rom, io_mem_notdirty;
81 static MemoryRegion io_mem_unassigned;
83 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
84 #define RAM_PREALLOC (1 << 0)
86 /* RAM is mmap-ed with MAP_SHARED */
87 #define RAM_SHARED (1 << 1)
89 /* Only a portion of RAM (used_length) is actually used, and migrated.
90 * This used_length size can change across reboots.
92 #define RAM_RESIZEABLE (1 << 2)
94 #endif
96 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
97 /* current CPU in the current thread. It is only valid inside
98 cpu_exec() */
99 __thread CPUState *current_cpu;
100 /* 0 = Do not count executed instructions.
101 1 = Precise instruction counting.
102 2 = Adaptive rate instruction counting. */
103 int use_icount;
105 #if !defined(CONFIG_USER_ONLY)
107 typedef struct PhysPageEntry PhysPageEntry;
109 struct PhysPageEntry {
110 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
111 uint32_t skip : 6;
112 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
113 uint32_t ptr : 26;
116 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
118 /* Size of the L2 (and L3, etc) page tables. */
119 #define ADDR_SPACE_BITS 64
121 #define P_L2_BITS 9
122 #define P_L2_SIZE (1 << P_L2_BITS)
124 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
126 typedef PhysPageEntry Node[P_L2_SIZE];
128 typedef struct PhysPageMap {
129 struct rcu_head rcu;
131 unsigned sections_nb;
132 unsigned sections_nb_alloc;
133 unsigned nodes_nb;
134 unsigned nodes_nb_alloc;
135 Node *nodes;
136 MemoryRegionSection *sections;
137 } PhysPageMap;
139 struct AddressSpaceDispatch {
140 struct rcu_head rcu;
142 MemoryRegionSection *mru_section;
143 /* This is a multi-level map on the physical address space.
144 * The bottom level has pointers to MemoryRegionSections.
146 PhysPageEntry phys_map;
147 PhysPageMap map;
148 AddressSpace *as;
151 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
152 typedef struct subpage_t {
153 MemoryRegion iomem;
154 AddressSpace *as;
155 hwaddr base;
156 uint16_t sub_section[TARGET_PAGE_SIZE];
157 } subpage_t;
159 #define PHYS_SECTION_UNASSIGNED 0
160 #define PHYS_SECTION_NOTDIRTY 1
161 #define PHYS_SECTION_ROM 2
162 #define PHYS_SECTION_WATCH 3
164 static void io_mem_init(void);
165 static void memory_map_init(void);
166 static void tcg_commit(MemoryListener *listener);
168 static MemoryRegion io_mem_watch;
171 * CPUAddressSpace: all the information a CPU needs about an AddressSpace
172 * @cpu: the CPU whose AddressSpace this is
173 * @as: the AddressSpace itself
174 * @memory_dispatch: its dispatch pointer (cached, RCU protected)
175 * @tcg_as_listener: listener for tracking changes to the AddressSpace
177 struct CPUAddressSpace {
178 CPUState *cpu;
179 AddressSpace *as;
180 struct AddressSpaceDispatch *memory_dispatch;
181 MemoryListener tcg_as_listener;
184 #endif
186 #if !defined(CONFIG_USER_ONLY)
188 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
190 static unsigned alloc_hint = 16;
191 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
192 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, alloc_hint);
193 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
194 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
195 alloc_hint = map->nodes_nb_alloc;
199 static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
201 unsigned i;
202 uint32_t ret;
203 PhysPageEntry e;
204 PhysPageEntry *p;
206 ret = map->nodes_nb++;
207 p = map->nodes[ret];
208 assert(ret != PHYS_MAP_NODE_NIL);
209 assert(ret != map->nodes_nb_alloc);
211 e.skip = leaf ? 0 : 1;
212 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
213 for (i = 0; i < P_L2_SIZE; ++i) {
214 memcpy(&p[i], &e, sizeof(e));
216 return ret;
219 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
220 hwaddr *index, hwaddr *nb, uint16_t leaf,
221 int level)
223 PhysPageEntry *p;
224 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
226 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
227 lp->ptr = phys_map_node_alloc(map, level == 0);
229 p = map->nodes[lp->ptr];
230 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
232 while (*nb && lp < &p[P_L2_SIZE]) {
233 if ((*index & (step - 1)) == 0 && *nb >= step) {
234 lp->skip = 0;
235 lp->ptr = leaf;
236 *index += step;
237 *nb -= step;
238 } else {
239 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
241 ++lp;
245 static void phys_page_set(AddressSpaceDispatch *d,
246 hwaddr index, hwaddr nb,
247 uint16_t leaf)
249 /* Wildly overreserve - it doesn't matter much. */
250 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
252 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
255 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
256 * and update our entry so we can skip it and go directly to the destination.
258 static void phys_page_compact(PhysPageEntry *lp, Node *nodes)
260 unsigned valid_ptr = P_L2_SIZE;
261 int valid = 0;
262 PhysPageEntry *p;
263 int i;
265 if (lp->ptr == PHYS_MAP_NODE_NIL) {
266 return;
269 p = nodes[lp->ptr];
270 for (i = 0; i < P_L2_SIZE; i++) {
271 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
272 continue;
275 valid_ptr = i;
276 valid++;
277 if (p[i].skip) {
278 phys_page_compact(&p[i], nodes);
282 /* We can only compress if there's only one child. */
283 if (valid != 1) {
284 return;
287 assert(valid_ptr < P_L2_SIZE);
289 /* Don't compress if it won't fit in the # of bits we have. */
290 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
291 return;
294 lp->ptr = p[valid_ptr].ptr;
295 if (!p[valid_ptr].skip) {
296 /* If our only child is a leaf, make this a leaf. */
297 /* By design, we should have made this node a leaf to begin with so we
298 * should never reach here.
299 * But since it's so simple to handle this, let's do it just in case we
300 * change this rule.
302 lp->skip = 0;
303 } else {
304 lp->skip += p[valid_ptr].skip;
308 static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
310 if (d->phys_map.skip) {
311 phys_page_compact(&d->phys_map, d->map.nodes);
315 static inline bool section_covers_addr(const MemoryRegionSection *section,
316 hwaddr addr)
318 /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
319 * the section must cover the entire address space.
321 return section->size.hi ||
322 range_covers_byte(section->offset_within_address_space,
323 section->size.lo, addr);
326 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
327 Node *nodes, MemoryRegionSection *sections)
329 PhysPageEntry *p;
330 hwaddr index = addr >> TARGET_PAGE_BITS;
331 int i;
333 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
334 if (lp.ptr == PHYS_MAP_NODE_NIL) {
335 return &sections[PHYS_SECTION_UNASSIGNED];
337 p = nodes[lp.ptr];
338 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
341 if (section_covers_addr(&sections[lp.ptr], addr)) {
342 return &sections[lp.ptr];
343 } else {
344 return &sections[PHYS_SECTION_UNASSIGNED];
348 bool memory_region_is_unassigned(MemoryRegion *mr)
350 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
351 && mr != &io_mem_watch;
354 /* Called from RCU critical section */
355 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
356 hwaddr addr,
357 bool resolve_subpage)
359 MemoryRegionSection *section = atomic_read(&d->mru_section);
360 subpage_t *subpage;
361 bool update;
363 if (section && section != &d->map.sections[PHYS_SECTION_UNASSIGNED] &&
364 section_covers_addr(section, addr)) {
365 update = false;
366 } else {
367 section = phys_page_find(d->phys_map, addr, d->map.nodes,
368 d->map.sections);
369 update = true;
371 if (resolve_subpage && section->mr->subpage) {
372 subpage = container_of(section->mr, subpage_t, iomem);
373 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
375 if (update) {
376 atomic_set(&d->mru_section, section);
378 return section;
381 /* Called from RCU critical section */
382 static MemoryRegionSection *
383 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
384 hwaddr *plen, bool resolve_subpage)
386 MemoryRegionSection *section;
387 MemoryRegion *mr;
388 Int128 diff;
390 section = address_space_lookup_region(d, addr, resolve_subpage);
391 /* Compute offset within MemoryRegionSection */
392 addr -= section->offset_within_address_space;
394 /* Compute offset within MemoryRegion */
395 *xlat = addr + section->offset_within_region;
397 mr = section->mr;
399 /* MMIO registers can be expected to perform full-width accesses based only
400 * on their address, without considering adjacent registers that could
401 * decode to completely different MemoryRegions. When such registers
402 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
403 * regions overlap wildly. For this reason we cannot clamp the accesses
404 * here.
406 * If the length is small (as is the case for address_space_ldl/stl),
407 * everything works fine. If the incoming length is large, however,
408 * the caller really has to do the clamping through memory_access_size.
410 if (memory_region_is_ram(mr)) {
411 diff = int128_sub(section->size, int128_make64(addr));
412 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
414 return section;
417 /* Called from RCU critical section */
418 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
419 hwaddr *xlat, hwaddr *plen,
420 bool is_write)
422 IOMMUTLBEntry iotlb;
423 MemoryRegionSection *section;
424 MemoryRegion *mr;
426 for (;;) {
427 AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch);
428 section = address_space_translate_internal(d, addr, &addr, plen, true);
429 mr = section->mr;
431 if (!mr->iommu_ops) {
432 break;
435 iotlb = mr->iommu_ops->translate(mr, addr, is_write);
436 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
437 | (addr & iotlb.addr_mask));
438 *plen = MIN(*plen, (addr | iotlb.addr_mask) - addr + 1);
439 if (!(iotlb.perm & (1 << is_write))) {
440 mr = &io_mem_unassigned;
441 break;
444 as = iotlb.target_as;
447 if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
448 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
449 *plen = MIN(page, *plen);
452 *xlat = addr;
453 return mr;
456 /* Called from RCU critical section */
457 MemoryRegionSection *
458 address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
459 hwaddr *xlat, hwaddr *plen)
461 MemoryRegionSection *section;
462 AddressSpaceDispatch *d = cpu->cpu_ases[asidx].memory_dispatch;
464 section = address_space_translate_internal(d, addr, xlat, plen, false);
466 assert(!section->mr->iommu_ops);
467 return section;
469 #endif
471 #if !defined(CONFIG_USER_ONLY)
473 static int cpu_common_post_load(void *opaque, int version_id)
475 CPUState *cpu = opaque;
477 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
478 version_id is increased. */
479 cpu->interrupt_request &= ~0x01;
480 tlb_flush(cpu, 1);
482 return 0;
485 static int cpu_common_pre_load(void *opaque)
487 CPUState *cpu = opaque;
489 cpu->exception_index = -1;
491 return 0;
494 static bool cpu_common_exception_index_needed(void *opaque)
496 CPUState *cpu = opaque;
498 return tcg_enabled() && cpu->exception_index != -1;
501 static const VMStateDescription vmstate_cpu_common_exception_index = {
502 .name = "cpu_common/exception_index",
503 .version_id = 1,
504 .minimum_version_id = 1,
505 .needed = cpu_common_exception_index_needed,
506 .fields = (VMStateField[]) {
507 VMSTATE_INT32(exception_index, CPUState),
508 VMSTATE_END_OF_LIST()
512 static bool cpu_common_crash_occurred_needed(void *opaque)
514 CPUState *cpu = opaque;
516 return cpu->crash_occurred;
519 static const VMStateDescription vmstate_cpu_common_crash_occurred = {
520 .name = "cpu_common/crash_occurred",
521 .version_id = 1,
522 .minimum_version_id = 1,
523 .needed = cpu_common_crash_occurred_needed,
524 .fields = (VMStateField[]) {
525 VMSTATE_BOOL(crash_occurred, CPUState),
526 VMSTATE_END_OF_LIST()
530 const VMStateDescription vmstate_cpu_common = {
531 .name = "cpu_common",
532 .version_id = 1,
533 .minimum_version_id = 1,
534 .pre_load = cpu_common_pre_load,
535 .post_load = cpu_common_post_load,
536 .fields = (VMStateField[]) {
537 VMSTATE_UINT32(halted, CPUState),
538 VMSTATE_UINT32(interrupt_request, CPUState),
539 VMSTATE_END_OF_LIST()
541 .subsections = (const VMStateDescription*[]) {
542 &vmstate_cpu_common_exception_index,
543 &vmstate_cpu_common_crash_occurred,
544 NULL
548 #endif
550 CPUState *qemu_get_cpu(int index)
552 CPUState *cpu;
554 CPU_FOREACH(cpu) {
555 if (cpu->cpu_index == index) {
556 return cpu;
560 return NULL;
563 #if !defined(CONFIG_USER_ONLY)
564 void cpu_address_space_init(CPUState *cpu, AddressSpace *as, int asidx)
566 CPUAddressSpace *newas;
568 /* Target code should have set num_ases before calling us */
569 assert(asidx < cpu->num_ases);
571 if (asidx == 0) {
572 /* address space 0 gets the convenience alias */
573 cpu->as = as;
576 /* KVM cannot currently support multiple address spaces. */
577 assert(asidx == 0 || !kvm_enabled());
579 if (!cpu->cpu_ases) {
580 cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
583 newas = &cpu->cpu_ases[asidx];
584 newas->cpu = cpu;
585 newas->as = as;
586 if (tcg_enabled()) {
587 newas->tcg_as_listener.commit = tcg_commit;
588 memory_listener_register(&newas->tcg_as_listener, as);
592 AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
594 /* Return the AddressSpace corresponding to the specified index */
595 return cpu->cpu_ases[asidx].as;
597 #endif
599 void cpu_exec_exit(CPUState *cpu)
601 CPUClass *cc = CPU_GET_CLASS(cpu);
603 cpu_list_remove(cpu);
605 if (cc->vmsd != NULL) {
606 vmstate_unregister(NULL, cc->vmsd, cpu);
608 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
609 vmstate_unregister(NULL, &vmstate_cpu_common, cpu);
613 void cpu_exec_init(CPUState *cpu, Error **errp)
615 CPUClass *cc ATTRIBUTE_UNUSED = CPU_GET_CLASS(cpu);
616 Error *local_err ATTRIBUTE_UNUSED = NULL;
618 cpu->as = NULL;
619 cpu->num_ases = 0;
621 #ifndef CONFIG_USER_ONLY
622 cpu->thread_id = qemu_get_thread_id();
624 /* This is a softmmu CPU object, so create a property for it
625 * so users can wire up its memory. (This can't go in qom/cpu.c
626 * because that file is compiled only once for both user-mode
627 * and system builds.) The default if no link is set up is to use
628 * the system address space.
630 object_property_add_link(OBJECT(cpu), "memory", TYPE_MEMORY_REGION,
631 (Object **)&cpu->memory,
632 qdev_prop_allow_set_link_before_realize,
633 OBJ_PROP_LINK_UNREF_ON_RELEASE,
634 &error_abort);
635 cpu->memory = system_memory;
636 object_ref(OBJECT(cpu->memory));
637 #endif
639 cpu_list_add(cpu);
641 #ifndef CONFIG_USER_ONLY
642 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
643 vmstate_register(NULL, cpu->cpu_index, &vmstate_cpu_common, cpu);
645 if (cc->vmsd != NULL) {
646 vmstate_register(NULL, cpu->cpu_index, cc->vmsd, cpu);
648 #endif
651 #if defined(CONFIG_USER_ONLY)
652 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
654 tb_invalidate_phys_page_range(pc, pc + 1, 0);
656 #else
657 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
659 MemTxAttrs attrs;
660 hwaddr phys = cpu_get_phys_page_attrs_debug(cpu, pc, &attrs);
661 int asidx = cpu_asidx_from_attrs(cpu, attrs);
662 if (phys != -1) {
663 tb_invalidate_phys_addr(cpu->cpu_ases[asidx].as,
664 phys | (pc & ~TARGET_PAGE_MASK));
667 #endif
669 #if defined(CONFIG_USER_ONLY)
670 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
675 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
676 int flags)
678 return -ENOSYS;
681 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
685 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
686 int flags, CPUWatchpoint **watchpoint)
688 return -ENOSYS;
690 #else
691 /* Add a watchpoint. */
692 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
693 int flags, CPUWatchpoint **watchpoint)
695 CPUWatchpoint *wp;
697 /* forbid ranges which are empty or run off the end of the address space */
698 if (len == 0 || (addr + len - 1) < addr) {
699 error_report("tried to set invalid watchpoint at %"
700 VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
701 return -EINVAL;
703 wp = g_malloc(sizeof(*wp));
705 wp->vaddr = addr;
706 wp->len = len;
707 wp->flags = flags;
709 /* keep all GDB-injected watchpoints in front */
710 if (flags & BP_GDB) {
711 QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
712 } else {
713 QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
716 tlb_flush_page(cpu, addr);
718 if (watchpoint)
719 *watchpoint = wp;
720 return 0;
723 /* Remove a specific watchpoint. */
724 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
725 int flags)
727 CPUWatchpoint *wp;
729 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
730 if (addr == wp->vaddr && len == wp->len
731 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
732 cpu_watchpoint_remove_by_ref(cpu, wp);
733 return 0;
736 return -ENOENT;
739 /* Remove a specific watchpoint by reference. */
740 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
742 QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
744 tlb_flush_page(cpu, watchpoint->vaddr);
746 g_free(watchpoint);
749 /* Remove all matching watchpoints. */
750 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
752 CPUWatchpoint *wp, *next;
754 QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
755 if (wp->flags & mask) {
756 cpu_watchpoint_remove_by_ref(cpu, wp);
761 /* Return true if this watchpoint address matches the specified
762 * access (ie the address range covered by the watchpoint overlaps
763 * partially or completely with the address range covered by the
764 * access).
766 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
767 vaddr addr,
768 vaddr len)
770 /* We know the lengths are non-zero, but a little caution is
771 * required to avoid errors in the case where the range ends
772 * exactly at the top of the address space and so addr + len
773 * wraps round to zero.
775 vaddr wpend = wp->vaddr + wp->len - 1;
776 vaddr addrend = addr + len - 1;
778 return !(addr > wpend || wp->vaddr > addrend);
781 #endif
783 /* Add a breakpoint. */
784 int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
785 CPUBreakpoint **breakpoint)
787 CPUBreakpoint *bp;
789 bp = g_malloc(sizeof(*bp));
791 bp->pc = pc;
792 bp->flags = flags;
794 /* keep all GDB-injected breakpoints in front */
795 if (flags & BP_GDB) {
796 QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
797 } else {
798 QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
801 breakpoint_invalidate(cpu, pc);
803 if (breakpoint) {
804 *breakpoint = bp;
806 return 0;
809 /* Remove a specific breakpoint. */
810 int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
812 CPUBreakpoint *bp;
814 QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
815 if (bp->pc == pc && bp->flags == flags) {
816 cpu_breakpoint_remove_by_ref(cpu, bp);
817 return 0;
820 return -ENOENT;
823 /* Remove a specific breakpoint by reference. */
824 void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
826 QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
828 breakpoint_invalidate(cpu, breakpoint->pc);
830 g_free(breakpoint);
833 /* Remove all matching breakpoints. */
834 void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
836 CPUBreakpoint *bp, *next;
838 QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
839 if (bp->flags & mask) {
840 cpu_breakpoint_remove_by_ref(cpu, bp);
845 /* enable or disable single step mode. EXCP_DEBUG is returned by the
846 CPU loop after each instruction */
847 void cpu_single_step(CPUState *cpu, int enabled)
849 if (cpu->singlestep_enabled != enabled) {
850 cpu->singlestep_enabled = enabled;
851 if (kvm_enabled()) {
852 kvm_update_guest_debug(cpu, 0);
853 } else {
854 /* must flush all the translated code to avoid inconsistencies */
855 /* XXX: only flush what is necessary */
856 tb_flush(cpu);
861 void cpu_abort(CPUState *cpu, const char *fmt, ...)
863 va_list ap;
864 va_list ap2;
866 va_start(ap, fmt);
867 va_copy(ap2, ap);
868 fprintf(stderr, "qemu: fatal: ");
869 vfprintf(stderr, fmt, ap);
870 fprintf(stderr, "\n");
871 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
872 if (qemu_log_separate()) {
873 qemu_log("qemu: fatal: ");
874 qemu_log_vprintf(fmt, ap2);
875 qemu_log("\n");
876 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
877 qemu_log_flush();
878 qemu_log_close();
880 va_end(ap2);
881 va_end(ap);
882 replay_finish();
883 #if defined(CONFIG_USER_ONLY)
885 struct sigaction act;
886 sigfillset(&act.sa_mask);
887 act.sa_handler = SIG_DFL;
888 sigaction(SIGABRT, &act, NULL);
890 #endif
891 abort();
894 #if !defined(CONFIG_USER_ONLY)
895 /* Called from RCU critical section */
896 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
898 RAMBlock *block;
900 block = atomic_rcu_read(&ram_list.mru_block);
901 if (block && addr - block->offset < block->max_length) {
902 return block;
904 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
905 if (addr - block->offset < block->max_length) {
906 goto found;
910 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
911 abort();
913 found:
914 /* It is safe to write mru_block outside the iothread lock. This
915 * is what happens:
917 * mru_block = xxx
918 * rcu_read_unlock()
919 * xxx removed from list
920 * rcu_read_lock()
921 * read mru_block
922 * mru_block = NULL;
923 * call_rcu(reclaim_ramblock, xxx);
924 * rcu_read_unlock()
926 * atomic_rcu_set is not needed here. The block was already published
927 * when it was placed into the list. Here we're just making an extra
928 * copy of the pointer.
930 ram_list.mru_block = block;
931 return block;
934 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
936 CPUState *cpu;
937 ram_addr_t start1;
938 RAMBlock *block;
939 ram_addr_t end;
941 end = TARGET_PAGE_ALIGN(start + length);
942 start &= TARGET_PAGE_MASK;
944 rcu_read_lock();
945 block = qemu_get_ram_block(start);
946 assert(block == qemu_get_ram_block(end - 1));
947 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
948 CPU_FOREACH(cpu) {
949 tlb_reset_dirty(cpu, start1, length);
951 rcu_read_unlock();
954 /* Note: start and end must be within the same ram block. */
955 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
956 ram_addr_t length,
957 unsigned client)
959 DirtyMemoryBlocks *blocks;
960 unsigned long end, page;
961 bool dirty = false;
963 if (length == 0) {
964 return false;
967 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
968 page = start >> TARGET_PAGE_BITS;
970 rcu_read_lock();
972 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
974 while (page < end) {
975 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
976 unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
977 unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
979 dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
980 offset, num);
981 page += num;
984 rcu_read_unlock();
986 if (dirty && tcg_enabled()) {
987 tlb_reset_dirty_range_all(start, length);
990 return dirty;
993 /* Called from RCU critical section */
994 hwaddr memory_region_section_get_iotlb(CPUState *cpu,
995 MemoryRegionSection *section,
996 target_ulong vaddr,
997 hwaddr paddr, hwaddr xlat,
998 int prot,
999 target_ulong *address)
1001 hwaddr iotlb;
1002 CPUWatchpoint *wp;
1004 if (memory_region_is_ram(section->mr)) {
1005 /* Normal RAM. */
1006 iotlb = memory_region_get_ram_addr(section->mr) + xlat;
1007 if (!section->readonly) {
1008 iotlb |= PHYS_SECTION_NOTDIRTY;
1009 } else {
1010 iotlb |= PHYS_SECTION_ROM;
1012 } else {
1013 AddressSpaceDispatch *d;
1015 d = atomic_rcu_read(&section->address_space->dispatch);
1016 iotlb = section - d->map.sections;
1017 iotlb += xlat;
1020 /* Make accesses to pages with watchpoints go via the
1021 watchpoint trap routines. */
1022 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
1023 if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
1024 /* Avoid trapping reads of pages with a write breakpoint. */
1025 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
1026 iotlb = PHYS_SECTION_WATCH + paddr;
1027 *address |= TLB_MMIO;
1028 break;
1033 return iotlb;
1035 #endif /* defined(CONFIG_USER_ONLY) */
1037 #if !defined(CONFIG_USER_ONLY)
1039 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1040 uint16_t section);
1041 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
1043 static void *(*phys_mem_alloc)(size_t size, uint64_t *align) =
1044 qemu_anon_ram_alloc;
1047 * Set a custom physical guest memory alloator.
1048 * Accelerators with unusual needs may need this. Hopefully, we can
1049 * get rid of it eventually.
1051 void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align))
1053 phys_mem_alloc = alloc;
1056 static uint16_t phys_section_add(PhysPageMap *map,
1057 MemoryRegionSection *section)
1059 /* The physical section number is ORed with a page-aligned
1060 * pointer to produce the iotlb entries. Thus it should
1061 * never overflow into the page-aligned value.
1063 assert(map->sections_nb < TARGET_PAGE_SIZE);
1065 if (map->sections_nb == map->sections_nb_alloc) {
1066 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
1067 map->sections = g_renew(MemoryRegionSection, map->sections,
1068 map->sections_nb_alloc);
1070 map->sections[map->sections_nb] = *section;
1071 memory_region_ref(section->mr);
1072 return map->sections_nb++;
1075 static void phys_section_destroy(MemoryRegion *mr)
1077 bool have_sub_page = mr->subpage;
1079 memory_region_unref(mr);
1081 if (have_sub_page) {
1082 subpage_t *subpage = container_of(mr, subpage_t, iomem);
1083 object_unref(OBJECT(&subpage->iomem));
1084 g_free(subpage);
1088 static void phys_sections_free(PhysPageMap *map)
1090 while (map->sections_nb > 0) {
1091 MemoryRegionSection *section = &map->sections[--map->sections_nb];
1092 phys_section_destroy(section->mr);
1094 g_free(map->sections);
1095 g_free(map->nodes);
1098 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
1100 subpage_t *subpage;
1101 hwaddr base = section->offset_within_address_space
1102 & TARGET_PAGE_MASK;
1103 MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
1104 d->map.nodes, d->map.sections);
1105 MemoryRegionSection subsection = {
1106 .offset_within_address_space = base,
1107 .size = int128_make64(TARGET_PAGE_SIZE),
1109 hwaddr start, end;
1111 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
1113 if (!(existing->mr->subpage)) {
1114 subpage = subpage_init(d->as, base);
1115 subsection.address_space = d->as;
1116 subsection.mr = &subpage->iomem;
1117 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
1118 phys_section_add(&d->map, &subsection));
1119 } else {
1120 subpage = container_of(existing->mr, subpage_t, iomem);
1122 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
1123 end = start + int128_get64(section->size) - 1;
1124 subpage_register(subpage, start, end,
1125 phys_section_add(&d->map, section));
1129 static void register_multipage(AddressSpaceDispatch *d,
1130 MemoryRegionSection *section)
1132 hwaddr start_addr = section->offset_within_address_space;
1133 uint16_t section_index = phys_section_add(&d->map, section);
1134 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1135 TARGET_PAGE_BITS));
1137 assert(num_pages);
1138 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
1141 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
1143 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1144 AddressSpaceDispatch *d = as->next_dispatch;
1145 MemoryRegionSection now = *section, remain = *section;
1146 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
1148 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
1149 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
1150 - now.offset_within_address_space;
1152 now.size = int128_min(int128_make64(left), now.size);
1153 register_subpage(d, &now);
1154 } else {
1155 now.size = int128_zero();
1157 while (int128_ne(remain.size, now.size)) {
1158 remain.size = int128_sub(remain.size, now.size);
1159 remain.offset_within_address_space += int128_get64(now.size);
1160 remain.offset_within_region += int128_get64(now.size);
1161 now = remain;
1162 if (int128_lt(remain.size, page_size)) {
1163 register_subpage(d, &now);
1164 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
1165 now.size = page_size;
1166 register_subpage(d, &now);
1167 } else {
1168 now.size = int128_and(now.size, int128_neg(page_size));
1169 register_multipage(d, &now);
1174 void qemu_flush_coalesced_mmio_buffer(void)
1176 if (kvm_enabled())
1177 kvm_flush_coalesced_mmio_buffer();
1180 void qemu_mutex_lock_ramlist(void)
1182 qemu_mutex_lock(&ram_list.mutex);
1185 void qemu_mutex_unlock_ramlist(void)
1187 qemu_mutex_unlock(&ram_list.mutex);
1190 #ifdef __linux__
1191 static void *file_ram_alloc(RAMBlock *block,
1192 ram_addr_t memory,
1193 const char *path,
1194 Error **errp)
1196 bool unlink_on_error = false;
1197 char *filename;
1198 char *sanitized_name;
1199 char *c;
1200 void *area = MAP_FAILED;
1201 int fd = -1;
1203 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1204 error_setg(errp,
1205 "host lacks kvm mmu notifiers, -mem-path unsupported");
1206 return NULL;
1209 for (;;) {
1210 fd = open(path, O_RDWR);
1211 if (fd >= 0) {
1212 /* @path names an existing file, use it */
1213 break;
1215 if (errno == ENOENT) {
1216 /* @path names a file that doesn't exist, create it */
1217 fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
1218 if (fd >= 0) {
1219 unlink_on_error = true;
1220 break;
1222 } else if (errno == EISDIR) {
1223 /* @path names a directory, create a file there */
1224 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1225 sanitized_name = g_strdup(memory_region_name(block->mr));
1226 for (c = sanitized_name; *c != '\0'; c++) {
1227 if (*c == '/') {
1228 *c = '_';
1232 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1233 sanitized_name);
1234 g_free(sanitized_name);
1236 fd = mkstemp(filename);
1237 if (fd >= 0) {
1238 unlink(filename);
1239 g_free(filename);
1240 break;
1242 g_free(filename);
1244 if (errno != EEXIST && errno != EINTR) {
1245 error_setg_errno(errp, errno,
1246 "can't open backing store %s for guest RAM",
1247 path);
1248 goto error;
1251 * Try again on EINTR and EEXIST. The latter happens when
1252 * something else creates the file between our two open().
1256 block->page_size = qemu_fd_getpagesize(fd);
1257 block->mr->align = MAX(block->page_size, QEMU_VMALLOC_ALIGN);
1259 if (memory < block->page_size) {
1260 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
1261 "or larger than page size 0x%zx",
1262 memory, block->page_size);
1263 goto error;
1266 memory = ROUND_UP(memory, block->page_size);
1269 * ftruncate is not supported by hugetlbfs in older
1270 * hosts, so don't bother bailing out on errors.
1271 * If anything goes wrong with it under other filesystems,
1272 * mmap will fail.
1274 if (ftruncate(fd, memory)) {
1275 perror("ftruncate");
1278 area = qemu_ram_mmap(fd, memory, block->mr->align,
1279 block->flags & RAM_SHARED);
1280 if (area == MAP_FAILED) {
1281 error_setg_errno(errp, errno,
1282 "unable to map backing store for guest RAM");
1283 goto error;
1286 if (mem_prealloc) {
1287 os_mem_prealloc(fd, area, memory, errp);
1288 if (errp && *errp) {
1289 goto error;
1293 block->fd = fd;
1294 return area;
1296 error:
1297 if (area != MAP_FAILED) {
1298 qemu_ram_munmap(area, memory);
1300 if (unlink_on_error) {
1301 unlink(path);
1303 if (fd != -1) {
1304 close(fd);
1306 return NULL;
1308 #endif
1310 /* Called with the ramlist lock held. */
1311 static ram_addr_t find_ram_offset(ram_addr_t size)
1313 RAMBlock *block, *next_block;
1314 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1316 assert(size != 0); /* it would hand out same offset multiple times */
1318 if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
1319 return 0;
1322 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1323 ram_addr_t end, next = RAM_ADDR_MAX;
1325 end = block->offset + block->max_length;
1327 QLIST_FOREACH_RCU(next_block, &ram_list.blocks, next) {
1328 if (next_block->offset >= end) {
1329 next = MIN(next, next_block->offset);
1332 if (next - end >= size && next - end < mingap) {
1333 offset = end;
1334 mingap = next - end;
1338 if (offset == RAM_ADDR_MAX) {
1339 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1340 (uint64_t)size);
1341 abort();
1344 return offset;
1347 ram_addr_t last_ram_offset(void)
1349 RAMBlock *block;
1350 ram_addr_t last = 0;
1352 rcu_read_lock();
1353 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1354 last = MAX(last, block->offset + block->max_length);
1356 rcu_read_unlock();
1357 return last;
1360 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1362 int ret;
1364 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1365 if (!machine_dump_guest_core(current_machine)) {
1366 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1367 if (ret) {
1368 perror("qemu_madvise");
1369 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1370 "but dump_guest_core=off specified\n");
1375 const char *qemu_ram_get_idstr(RAMBlock *rb)
1377 return rb->idstr;
1380 /* Called with iothread lock held. */
1381 void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev)
1383 RAMBlock *block;
1385 assert(new_block);
1386 assert(!new_block->idstr[0]);
1388 if (dev) {
1389 char *id = qdev_get_dev_path(dev);
1390 if (id) {
1391 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1392 g_free(id);
1395 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1397 rcu_read_lock();
1398 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1399 if (block != new_block &&
1400 !strcmp(block->idstr, new_block->idstr)) {
1401 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1402 new_block->idstr);
1403 abort();
1406 rcu_read_unlock();
1409 /* Called with iothread lock held. */
1410 void qemu_ram_unset_idstr(RAMBlock *block)
1412 /* FIXME: arch_init.c assumes that this is not called throughout
1413 * migration. Ignore the problem since hot-unplug during migration
1414 * does not work anyway.
1416 if (block) {
1417 memset(block->idstr, 0, sizeof(block->idstr));
1421 size_t qemu_ram_pagesize(RAMBlock *rb)
1423 return rb->page_size;
1426 static int memory_try_enable_merging(void *addr, size_t len)
1428 if (!machine_mem_merge(current_machine)) {
1429 /* disabled by the user */
1430 return 0;
1433 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1436 /* Only legal before guest might have detected the memory size: e.g. on
1437 * incoming migration, or right after reset.
1439 * As memory core doesn't know how is memory accessed, it is up to
1440 * resize callback to update device state and/or add assertions to detect
1441 * misuse, if necessary.
1443 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp)
1445 assert(block);
1447 newsize = HOST_PAGE_ALIGN(newsize);
1449 if (block->used_length == newsize) {
1450 return 0;
1453 if (!(block->flags & RAM_RESIZEABLE)) {
1454 error_setg_errno(errp, EINVAL,
1455 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1456 " in != 0x" RAM_ADDR_FMT, block->idstr,
1457 newsize, block->used_length);
1458 return -EINVAL;
1461 if (block->max_length < newsize) {
1462 error_setg_errno(errp, EINVAL,
1463 "Length too large: %s: 0x" RAM_ADDR_FMT
1464 " > 0x" RAM_ADDR_FMT, block->idstr,
1465 newsize, block->max_length);
1466 return -EINVAL;
1469 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
1470 block->used_length = newsize;
1471 cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
1472 DIRTY_CLIENTS_ALL);
1473 memory_region_set_size(block->mr, newsize);
1474 if (block->resized) {
1475 block->resized(block->idstr, newsize, block->host);
1477 return 0;
1480 /* Called with ram_list.mutex held */
1481 static void dirty_memory_extend(ram_addr_t old_ram_size,
1482 ram_addr_t new_ram_size)
1484 ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size,
1485 DIRTY_MEMORY_BLOCK_SIZE);
1486 ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size,
1487 DIRTY_MEMORY_BLOCK_SIZE);
1488 int i;
1490 /* Only need to extend if block count increased */
1491 if (new_num_blocks <= old_num_blocks) {
1492 return;
1495 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1496 DirtyMemoryBlocks *old_blocks;
1497 DirtyMemoryBlocks *new_blocks;
1498 int j;
1500 old_blocks = atomic_rcu_read(&ram_list.dirty_memory[i]);
1501 new_blocks = g_malloc(sizeof(*new_blocks) +
1502 sizeof(new_blocks->blocks[0]) * new_num_blocks);
1504 if (old_num_blocks) {
1505 memcpy(new_blocks->blocks, old_blocks->blocks,
1506 old_num_blocks * sizeof(old_blocks->blocks[0]));
1509 for (j = old_num_blocks; j < new_num_blocks; j++) {
1510 new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
1513 atomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
1515 if (old_blocks) {
1516 g_free_rcu(old_blocks, rcu);
1521 static void ram_block_add(RAMBlock *new_block, Error **errp)
1523 RAMBlock *block;
1524 RAMBlock *last_block = NULL;
1525 ram_addr_t old_ram_size, new_ram_size;
1526 Error *err = NULL;
1528 old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1530 qemu_mutex_lock_ramlist();
1531 new_block->offset = find_ram_offset(new_block->max_length);
1533 if (!new_block->host) {
1534 if (xen_enabled()) {
1535 xen_ram_alloc(new_block->offset, new_block->max_length,
1536 new_block->mr, &err);
1537 if (err) {
1538 error_propagate(errp, err);
1539 qemu_mutex_unlock_ramlist();
1540 return;
1542 } else {
1543 new_block->host = phys_mem_alloc(new_block->max_length,
1544 &new_block->mr->align);
1545 if (!new_block->host) {
1546 error_setg_errno(errp, errno,
1547 "cannot set up guest memory '%s'",
1548 memory_region_name(new_block->mr));
1549 qemu_mutex_unlock_ramlist();
1550 return;
1552 memory_try_enable_merging(new_block->host, new_block->max_length);
1556 new_ram_size = MAX(old_ram_size,
1557 (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
1558 if (new_ram_size > old_ram_size) {
1559 migration_bitmap_extend(old_ram_size, new_ram_size);
1560 dirty_memory_extend(old_ram_size, new_ram_size);
1562 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1563 * QLIST (which has an RCU-friendly variant) does not have insertion at
1564 * tail, so save the last element in last_block.
1566 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1567 last_block = block;
1568 if (block->max_length < new_block->max_length) {
1569 break;
1572 if (block) {
1573 QLIST_INSERT_BEFORE_RCU(block, new_block, next);
1574 } else if (last_block) {
1575 QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
1576 } else { /* list is empty */
1577 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
1579 ram_list.mru_block = NULL;
1581 /* Write list before version */
1582 smp_wmb();
1583 ram_list.version++;
1584 qemu_mutex_unlock_ramlist();
1586 cpu_physical_memory_set_dirty_range(new_block->offset,
1587 new_block->used_length,
1588 DIRTY_CLIENTS_ALL);
1590 if (new_block->host) {
1591 qemu_ram_setup_dump(new_block->host, new_block->max_length);
1592 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
1593 /* MADV_DONTFORK is also needed by KVM in absence of synchronous MMU */
1594 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
1598 #ifdef __linux__
1599 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
1600 bool share, const char *mem_path,
1601 Error **errp)
1603 RAMBlock *new_block;
1604 Error *local_err = NULL;
1606 if (xen_enabled()) {
1607 error_setg(errp, "-mem-path not supported with Xen");
1608 return NULL;
1611 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1613 * file_ram_alloc() needs to allocate just like
1614 * phys_mem_alloc, but we haven't bothered to provide
1615 * a hook there.
1617 error_setg(errp,
1618 "-mem-path not supported with this accelerator");
1619 return NULL;
1622 size = HOST_PAGE_ALIGN(size);
1623 new_block = g_malloc0(sizeof(*new_block));
1624 new_block->mr = mr;
1625 new_block->used_length = size;
1626 new_block->max_length = size;
1627 new_block->flags = share ? RAM_SHARED : 0;
1628 new_block->host = file_ram_alloc(new_block, size,
1629 mem_path, errp);
1630 if (!new_block->host) {
1631 g_free(new_block);
1632 return NULL;
1635 ram_block_add(new_block, &local_err);
1636 if (local_err) {
1637 g_free(new_block);
1638 error_propagate(errp, local_err);
1639 return NULL;
1641 return new_block;
1643 #endif
1645 static
1646 RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
1647 void (*resized)(const char*,
1648 uint64_t length,
1649 void *host),
1650 void *host, bool resizeable,
1651 MemoryRegion *mr, Error **errp)
1653 RAMBlock *new_block;
1654 Error *local_err = NULL;
1656 size = HOST_PAGE_ALIGN(size);
1657 max_size = HOST_PAGE_ALIGN(max_size);
1658 new_block = g_malloc0(sizeof(*new_block));
1659 new_block->mr = mr;
1660 new_block->resized = resized;
1661 new_block->used_length = size;
1662 new_block->max_length = max_size;
1663 assert(max_size >= size);
1664 new_block->fd = -1;
1665 new_block->page_size = getpagesize();
1666 new_block->host = host;
1667 if (host) {
1668 new_block->flags |= RAM_PREALLOC;
1670 if (resizeable) {
1671 new_block->flags |= RAM_RESIZEABLE;
1673 ram_block_add(new_block, &local_err);
1674 if (local_err) {
1675 g_free(new_block);
1676 error_propagate(errp, local_err);
1677 return NULL;
1679 return new_block;
1682 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1683 MemoryRegion *mr, Error **errp)
1685 return qemu_ram_alloc_internal(size, size, NULL, host, false, mr, errp);
1688 RAMBlock *qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
1690 return qemu_ram_alloc_internal(size, size, NULL, NULL, false, mr, errp);
1693 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
1694 void (*resized)(const char*,
1695 uint64_t length,
1696 void *host),
1697 MemoryRegion *mr, Error **errp)
1699 return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true, mr, errp);
1702 static void reclaim_ramblock(RAMBlock *block)
1704 if (block->flags & RAM_PREALLOC) {
1706 } else if (xen_enabled()) {
1707 xen_invalidate_map_cache_entry(block->host);
1708 #ifndef _WIN32
1709 } else if (block->fd >= 0) {
1710 qemu_ram_munmap(block->host, block->max_length);
1711 close(block->fd);
1712 #endif
1713 } else {
1714 qemu_anon_ram_free(block->host, block->max_length);
1716 g_free(block);
1719 void qemu_ram_free(RAMBlock *block)
1721 if (!block) {
1722 return;
1725 qemu_mutex_lock_ramlist();
1726 QLIST_REMOVE_RCU(block, next);
1727 ram_list.mru_block = NULL;
1728 /* Write list before version */
1729 smp_wmb();
1730 ram_list.version++;
1731 call_rcu(block, reclaim_ramblock, rcu);
1732 qemu_mutex_unlock_ramlist();
1735 #ifndef _WIN32
1736 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1738 RAMBlock *block;
1739 ram_addr_t offset;
1740 int flags;
1741 void *area, *vaddr;
1743 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1744 offset = addr - block->offset;
1745 if (offset < block->max_length) {
1746 vaddr = ramblock_ptr(block, offset);
1747 if (block->flags & RAM_PREALLOC) {
1749 } else if (xen_enabled()) {
1750 abort();
1751 } else {
1752 flags = MAP_FIXED;
1753 if (block->fd >= 0) {
1754 flags |= (block->flags & RAM_SHARED ?
1755 MAP_SHARED : MAP_PRIVATE);
1756 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1757 flags, block->fd, offset);
1758 } else {
1760 * Remap needs to match alloc. Accelerators that
1761 * set phys_mem_alloc never remap. If they did,
1762 * we'd need a remap hook here.
1764 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1766 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1767 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1768 flags, -1, 0);
1770 if (area != vaddr) {
1771 fprintf(stderr, "Could not remap addr: "
1772 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1773 length, addr);
1774 exit(1);
1776 memory_try_enable_merging(vaddr, length);
1777 qemu_ram_setup_dump(vaddr, length);
1782 #endif /* !_WIN32 */
1784 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1785 * This should not be used for general purpose DMA. Use address_space_map
1786 * or address_space_rw instead. For local memory (e.g. video ram) that the
1787 * device owns, use memory_region_get_ram_ptr.
1789 * Called within RCU critical section.
1791 void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
1793 RAMBlock *block = ram_block;
1795 if (block == NULL) {
1796 block = qemu_get_ram_block(addr);
1797 addr -= block->offset;
1800 if (xen_enabled() && block->host == NULL) {
1801 /* We need to check if the requested address is in the RAM
1802 * because we don't want to map the entire memory in QEMU.
1803 * In that case just map until the end of the page.
1805 if (block->offset == 0) {
1806 return xen_map_cache(addr, 0, 0);
1809 block->host = xen_map_cache(block->offset, block->max_length, 1);
1811 return ramblock_ptr(block, addr);
1814 /* Return a host pointer to guest's ram. Similar to qemu_map_ram_ptr
1815 * but takes a size argument.
1817 * Called within RCU critical section.
1819 static void *qemu_ram_ptr_length(RAMBlock *ram_block, ram_addr_t addr,
1820 hwaddr *size)
1822 RAMBlock *block = ram_block;
1823 if (*size == 0) {
1824 return NULL;
1827 if (block == NULL) {
1828 block = qemu_get_ram_block(addr);
1829 addr -= block->offset;
1831 *size = MIN(*size, block->max_length - addr);
1833 if (xen_enabled() && block->host == NULL) {
1834 /* We need to check if the requested address is in the RAM
1835 * because we don't want to map the entire memory in QEMU.
1836 * In that case just map the requested area.
1838 if (block->offset == 0) {
1839 return xen_map_cache(addr, *size, 1);
1842 block->host = xen_map_cache(block->offset, block->max_length, 1);
1845 return ramblock_ptr(block, addr);
1849 * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
1850 * in that RAMBlock.
1852 * ptr: Host pointer to look up
1853 * round_offset: If true round the result offset down to a page boundary
1854 * *ram_addr: set to result ram_addr
1855 * *offset: set to result offset within the RAMBlock
1857 * Returns: RAMBlock (or NULL if not found)
1859 * By the time this function returns, the returned pointer is not protected
1860 * by RCU anymore. If the caller is not within an RCU critical section and
1861 * does not hold the iothread lock, it must have other means of protecting the
1862 * pointer, such as a reference to the region that includes the incoming
1863 * ram_addr_t.
1865 RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
1866 ram_addr_t *offset)
1868 RAMBlock *block;
1869 uint8_t *host = ptr;
1871 if (xen_enabled()) {
1872 ram_addr_t ram_addr;
1873 rcu_read_lock();
1874 ram_addr = xen_ram_addr_from_mapcache(ptr);
1875 block = qemu_get_ram_block(ram_addr);
1876 if (block) {
1877 *offset = ram_addr - block->offset;
1879 rcu_read_unlock();
1880 return block;
1883 rcu_read_lock();
1884 block = atomic_rcu_read(&ram_list.mru_block);
1885 if (block && block->host && host - block->host < block->max_length) {
1886 goto found;
1889 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1890 /* This case append when the block is not mapped. */
1891 if (block->host == NULL) {
1892 continue;
1894 if (host - block->host < block->max_length) {
1895 goto found;
1899 rcu_read_unlock();
1900 return NULL;
1902 found:
1903 *offset = (host - block->host);
1904 if (round_offset) {
1905 *offset &= TARGET_PAGE_MASK;
1907 rcu_read_unlock();
1908 return block;
1912 * Finds the named RAMBlock
1914 * name: The name of RAMBlock to find
1916 * Returns: RAMBlock (or NULL if not found)
1918 RAMBlock *qemu_ram_block_by_name(const char *name)
1920 RAMBlock *block;
1922 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1923 if (!strcmp(name, block->idstr)) {
1924 return block;
1928 return NULL;
1931 /* Some of the softmmu routines need to translate from a host pointer
1932 (typically a TLB entry) back to a ram offset. */
1933 ram_addr_t qemu_ram_addr_from_host(void *ptr)
1935 RAMBlock *block;
1936 ram_addr_t offset;
1938 block = qemu_ram_block_from_host(ptr, false, &offset);
1939 if (!block) {
1940 return RAM_ADDR_INVALID;
1943 return block->offset + offset;
1946 /* Called within RCU critical section. */
1947 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1948 uint64_t val, unsigned size)
1950 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
1951 tb_invalidate_phys_page_fast(ram_addr, size);
1953 switch (size) {
1954 case 1:
1955 stb_p(qemu_map_ram_ptr(NULL, ram_addr), val);
1956 break;
1957 case 2:
1958 stw_p(qemu_map_ram_ptr(NULL, ram_addr), val);
1959 break;
1960 case 4:
1961 stl_p(qemu_map_ram_ptr(NULL, ram_addr), val);
1962 break;
1963 default:
1964 abort();
1966 /* Set both VGA and migration bits for simplicity and to remove
1967 * the notdirty callback faster.
1969 cpu_physical_memory_set_dirty_range(ram_addr, size,
1970 DIRTY_CLIENTS_NOCODE);
1971 /* we remove the notdirty callback only if the code has been
1972 flushed */
1973 if (!cpu_physical_memory_is_clean(ram_addr)) {
1974 tlb_set_dirty(current_cpu, current_cpu->mem_io_vaddr);
1978 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1979 unsigned size, bool is_write)
1981 return is_write;
1984 static const MemoryRegionOps notdirty_mem_ops = {
1985 .write = notdirty_mem_write,
1986 .valid.accepts = notdirty_mem_accepts,
1987 .endianness = DEVICE_NATIVE_ENDIAN,
1990 /* Generate a debug exception if a watchpoint has been hit. */
1991 static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
1993 CPUState *cpu = current_cpu;
1994 CPUClass *cc = CPU_GET_CLASS(cpu);
1995 CPUArchState *env = cpu->env_ptr;
1996 target_ulong pc, cs_base;
1997 target_ulong vaddr;
1998 CPUWatchpoint *wp;
1999 uint32_t cpu_flags;
2001 if (cpu->watchpoint_hit) {
2002 /* We re-entered the check after replacing the TB. Now raise
2003 * the debug interrupt so that is will trigger after the
2004 * current instruction. */
2005 cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
2006 return;
2008 vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2009 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
2010 if (cpu_watchpoint_address_matches(wp, vaddr, len)
2011 && (wp->flags & flags)) {
2012 if (flags == BP_MEM_READ) {
2013 wp->flags |= BP_WATCHPOINT_HIT_READ;
2014 } else {
2015 wp->flags |= BP_WATCHPOINT_HIT_WRITE;
2017 wp->hitaddr = vaddr;
2018 wp->hitattrs = attrs;
2019 if (!cpu->watchpoint_hit) {
2020 if (wp->flags & BP_CPU &&
2021 !cc->debug_check_watchpoint(cpu, wp)) {
2022 wp->flags &= ~BP_WATCHPOINT_HIT;
2023 continue;
2025 cpu->watchpoint_hit = wp;
2026 tb_check_watchpoint(cpu);
2027 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2028 cpu->exception_index = EXCP_DEBUG;
2029 cpu_loop_exit(cpu);
2030 } else {
2031 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2032 tb_gen_code(cpu, pc, cs_base, cpu_flags, 1);
2033 cpu_loop_exit_noexc(cpu);
2036 } else {
2037 wp->flags &= ~BP_WATCHPOINT_HIT;
2042 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2043 so these check for a hit then pass through to the normal out-of-line
2044 phys routines. */
2045 static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
2046 unsigned size, MemTxAttrs attrs)
2048 MemTxResult res;
2049 uint64_t data;
2050 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2051 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
2053 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
2054 switch (size) {
2055 case 1:
2056 data = address_space_ldub(as, addr, attrs, &res);
2057 break;
2058 case 2:
2059 data = address_space_lduw(as, addr, attrs, &res);
2060 break;
2061 case 4:
2062 data = address_space_ldl(as, addr, attrs, &res);
2063 break;
2064 default: abort();
2066 *pdata = data;
2067 return res;
2070 static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
2071 uint64_t val, unsigned size,
2072 MemTxAttrs attrs)
2074 MemTxResult res;
2075 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2076 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
2078 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
2079 switch (size) {
2080 case 1:
2081 address_space_stb(as, addr, val, attrs, &res);
2082 break;
2083 case 2:
2084 address_space_stw(as, addr, val, attrs, &res);
2085 break;
2086 case 4:
2087 address_space_stl(as, addr, val, attrs, &res);
2088 break;
2089 default: abort();
2091 return res;
2094 static const MemoryRegionOps watch_mem_ops = {
2095 .read_with_attrs = watch_mem_read,
2096 .write_with_attrs = watch_mem_write,
2097 .endianness = DEVICE_NATIVE_ENDIAN,
2100 static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
2101 unsigned len, MemTxAttrs attrs)
2103 subpage_t *subpage = opaque;
2104 uint8_t buf[8];
2105 MemTxResult res;
2107 #if defined(DEBUG_SUBPAGE)
2108 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
2109 subpage, len, addr);
2110 #endif
2111 res = address_space_read(subpage->as, addr + subpage->base,
2112 attrs, buf, len);
2113 if (res) {
2114 return res;
2116 switch (len) {
2117 case 1:
2118 *data = ldub_p(buf);
2119 return MEMTX_OK;
2120 case 2:
2121 *data = lduw_p(buf);
2122 return MEMTX_OK;
2123 case 4:
2124 *data = ldl_p(buf);
2125 return MEMTX_OK;
2126 case 8:
2127 *data = ldq_p(buf);
2128 return MEMTX_OK;
2129 default:
2130 abort();
2134 static MemTxResult subpage_write(void *opaque, hwaddr addr,
2135 uint64_t value, unsigned len, MemTxAttrs attrs)
2137 subpage_t *subpage = opaque;
2138 uint8_t buf[8];
2140 #if defined(DEBUG_SUBPAGE)
2141 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
2142 " value %"PRIx64"\n",
2143 __func__, subpage, len, addr, value);
2144 #endif
2145 switch (len) {
2146 case 1:
2147 stb_p(buf, value);
2148 break;
2149 case 2:
2150 stw_p(buf, value);
2151 break;
2152 case 4:
2153 stl_p(buf, value);
2154 break;
2155 case 8:
2156 stq_p(buf, value);
2157 break;
2158 default:
2159 abort();
2161 return address_space_write(subpage->as, addr + subpage->base,
2162 attrs, buf, len);
2165 static bool subpage_accepts(void *opaque, hwaddr addr,
2166 unsigned len, bool is_write)
2168 subpage_t *subpage = opaque;
2169 #if defined(DEBUG_SUBPAGE)
2170 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
2171 __func__, subpage, is_write ? 'w' : 'r', len, addr);
2172 #endif
2174 return address_space_access_valid(subpage->as, addr + subpage->base,
2175 len, is_write);
2178 static const MemoryRegionOps subpage_ops = {
2179 .read_with_attrs = subpage_read,
2180 .write_with_attrs = subpage_write,
2181 .impl.min_access_size = 1,
2182 .impl.max_access_size = 8,
2183 .valid.min_access_size = 1,
2184 .valid.max_access_size = 8,
2185 .valid.accepts = subpage_accepts,
2186 .endianness = DEVICE_NATIVE_ENDIAN,
2189 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2190 uint16_t section)
2192 int idx, eidx;
2194 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2195 return -1;
2196 idx = SUBPAGE_IDX(start);
2197 eidx = SUBPAGE_IDX(end);
2198 #if defined(DEBUG_SUBPAGE)
2199 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2200 __func__, mmio, start, end, idx, eidx, section);
2201 #endif
2202 for (; idx <= eidx; idx++) {
2203 mmio->sub_section[idx] = section;
2206 return 0;
2209 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
2211 subpage_t *mmio;
2213 mmio = g_malloc0(sizeof(subpage_t));
2215 mmio->as = as;
2216 mmio->base = base;
2217 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
2218 NULL, TARGET_PAGE_SIZE);
2219 mmio->iomem.subpage = true;
2220 #if defined(DEBUG_SUBPAGE)
2221 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
2222 mmio, base, TARGET_PAGE_SIZE);
2223 #endif
2224 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
2226 return mmio;
2229 static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
2230 MemoryRegion *mr)
2232 assert(as);
2233 MemoryRegionSection section = {
2234 .address_space = as,
2235 .mr = mr,
2236 .offset_within_address_space = 0,
2237 .offset_within_region = 0,
2238 .size = int128_2_64(),
2241 return phys_section_add(map, &section);
2244 MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index, MemTxAttrs attrs)
2246 int asidx = cpu_asidx_from_attrs(cpu, attrs);
2247 CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
2248 AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
2249 MemoryRegionSection *sections = d->map.sections;
2251 return sections[index & ~TARGET_PAGE_MASK].mr;
2254 static void io_mem_init(void)
2256 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, NULL, UINT64_MAX);
2257 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
2258 NULL, UINT64_MAX);
2259 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
2260 NULL, UINT64_MAX);
2261 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
2262 NULL, UINT64_MAX);
2265 static void mem_begin(MemoryListener *listener)
2267 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2268 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
2269 uint16_t n;
2271 n = dummy_section(&d->map, as, &io_mem_unassigned);
2272 assert(n == PHYS_SECTION_UNASSIGNED);
2273 n = dummy_section(&d->map, as, &io_mem_notdirty);
2274 assert(n == PHYS_SECTION_NOTDIRTY);
2275 n = dummy_section(&d->map, as, &io_mem_rom);
2276 assert(n == PHYS_SECTION_ROM);
2277 n = dummy_section(&d->map, as, &io_mem_watch);
2278 assert(n == PHYS_SECTION_WATCH);
2280 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
2281 d->as = as;
2282 as->next_dispatch = d;
2285 static void address_space_dispatch_free(AddressSpaceDispatch *d)
2287 phys_sections_free(&d->map);
2288 g_free(d);
2291 static void mem_commit(MemoryListener *listener)
2293 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2294 AddressSpaceDispatch *cur = as->dispatch;
2295 AddressSpaceDispatch *next = as->next_dispatch;
2297 phys_page_compact_all(next, next->map.nodes_nb);
2299 atomic_rcu_set(&as->dispatch, next);
2300 if (cur) {
2301 call_rcu(cur, address_space_dispatch_free, rcu);
2305 static void tcg_commit(MemoryListener *listener)
2307 CPUAddressSpace *cpuas;
2308 AddressSpaceDispatch *d;
2310 /* since each CPU stores ram addresses in its TLB cache, we must
2311 reset the modified entries */
2312 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
2313 cpu_reloading_memory_map();
2314 /* The CPU and TLB are protected by the iothread lock.
2315 * We reload the dispatch pointer now because cpu_reloading_memory_map()
2316 * may have split the RCU critical section.
2318 d = atomic_rcu_read(&cpuas->as->dispatch);
2319 cpuas->memory_dispatch = d;
2320 tlb_flush(cpuas->cpu, 1);
2323 void address_space_init_dispatch(AddressSpace *as)
2325 as->dispatch = NULL;
2326 as->dispatch_listener = (MemoryListener) {
2327 .begin = mem_begin,
2328 .commit = mem_commit,
2329 .region_add = mem_add,
2330 .region_nop = mem_add,
2331 .priority = 0,
2333 memory_listener_register(&as->dispatch_listener, as);
2336 void address_space_unregister(AddressSpace *as)
2338 memory_listener_unregister(&as->dispatch_listener);
2341 void address_space_destroy_dispatch(AddressSpace *as)
2343 AddressSpaceDispatch *d = as->dispatch;
2345 atomic_rcu_set(&as->dispatch, NULL);
2346 if (d) {
2347 call_rcu(d, address_space_dispatch_free, rcu);
2351 static void memory_map_init(void)
2353 system_memory = g_malloc(sizeof(*system_memory));
2355 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
2356 address_space_init(&address_space_memory, system_memory, "memory");
2358 system_io = g_malloc(sizeof(*system_io));
2359 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2360 65536);
2361 address_space_init(&address_space_io, system_io, "I/O");
2364 MemoryRegion *get_system_memory(void)
2366 return system_memory;
2369 MemoryRegion *get_system_io(void)
2371 return system_io;
2374 #endif /* !defined(CONFIG_USER_ONLY) */
2376 /* physical memory access (slow version, mainly for debug) */
2377 #if defined(CONFIG_USER_ONLY)
2378 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2379 uint8_t *buf, int len, int is_write)
2381 int l, flags;
2382 target_ulong page;
2383 void * p;
2385 while (len > 0) {
2386 page = addr & TARGET_PAGE_MASK;
2387 l = (page + TARGET_PAGE_SIZE) - addr;
2388 if (l > len)
2389 l = len;
2390 flags = page_get_flags(page);
2391 if (!(flags & PAGE_VALID))
2392 return -1;
2393 if (is_write) {
2394 if (!(flags & PAGE_WRITE))
2395 return -1;
2396 /* XXX: this code should not depend on lock_user */
2397 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2398 return -1;
2399 memcpy(p, buf, l);
2400 unlock_user(p, addr, l);
2401 } else {
2402 if (!(flags & PAGE_READ))
2403 return -1;
2404 /* XXX: this code should not depend on lock_user */
2405 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2406 return -1;
2407 memcpy(buf, p, l);
2408 unlock_user(p, addr, 0);
2410 len -= l;
2411 buf += l;
2412 addr += l;
2414 return 0;
2417 #else
2419 static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
2420 hwaddr length)
2422 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
2423 addr += memory_region_get_ram_addr(mr);
2425 /* No early return if dirty_log_mask is or becomes 0, because
2426 * cpu_physical_memory_set_dirty_range will still call
2427 * xen_modified_memory.
2429 if (dirty_log_mask) {
2430 dirty_log_mask =
2431 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
2433 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
2434 tb_invalidate_phys_range(addr, addr + length);
2435 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
2437 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
2440 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
2442 unsigned access_size_max = mr->ops->valid.max_access_size;
2444 /* Regions are assumed to support 1-4 byte accesses unless
2445 otherwise specified. */
2446 if (access_size_max == 0) {
2447 access_size_max = 4;
2450 /* Bound the maximum access by the alignment of the address. */
2451 if (!mr->ops->impl.unaligned) {
2452 unsigned align_size_max = addr & -addr;
2453 if (align_size_max != 0 && align_size_max < access_size_max) {
2454 access_size_max = align_size_max;
2458 /* Don't attempt accesses larger than the maximum. */
2459 if (l > access_size_max) {
2460 l = access_size_max;
2462 l = pow2floor(l);
2464 return l;
2467 static bool prepare_mmio_access(MemoryRegion *mr)
2469 bool unlocked = !qemu_mutex_iothread_locked();
2470 bool release_lock = false;
2472 if (unlocked && mr->global_locking) {
2473 qemu_mutex_lock_iothread();
2474 unlocked = false;
2475 release_lock = true;
2477 if (mr->flush_coalesced_mmio) {
2478 if (unlocked) {
2479 qemu_mutex_lock_iothread();
2481 qemu_flush_coalesced_mmio_buffer();
2482 if (unlocked) {
2483 qemu_mutex_unlock_iothread();
2487 return release_lock;
2490 /* Called within RCU critical section. */
2491 static MemTxResult address_space_write_continue(AddressSpace *as, hwaddr addr,
2492 MemTxAttrs attrs,
2493 const uint8_t *buf,
2494 int len, hwaddr addr1,
2495 hwaddr l, MemoryRegion *mr)
2497 uint8_t *ptr;
2498 uint64_t val;
2499 MemTxResult result = MEMTX_OK;
2500 bool release_lock = false;
2502 for (;;) {
2503 if (!memory_access_is_direct(mr, true)) {
2504 release_lock |= prepare_mmio_access(mr);
2505 l = memory_access_size(mr, l, addr1);
2506 /* XXX: could force current_cpu to NULL to avoid
2507 potential bugs */
2508 switch (l) {
2509 case 8:
2510 /* 64 bit write access */
2511 val = ldq_p(buf);
2512 result |= memory_region_dispatch_write(mr, addr1, val, 8,
2513 attrs);
2514 break;
2515 case 4:
2516 /* 32 bit write access */
2517 val = ldl_p(buf);
2518 result |= memory_region_dispatch_write(mr, addr1, val, 4,
2519 attrs);
2520 break;
2521 case 2:
2522 /* 16 bit write access */
2523 val = lduw_p(buf);
2524 result |= memory_region_dispatch_write(mr, addr1, val, 2,
2525 attrs);
2526 break;
2527 case 1:
2528 /* 8 bit write access */
2529 val = ldub_p(buf);
2530 result |= memory_region_dispatch_write(mr, addr1, val, 1,
2531 attrs);
2532 break;
2533 default:
2534 abort();
2536 } else {
2537 /* RAM case */
2538 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
2539 memcpy(ptr, buf, l);
2540 invalidate_and_set_dirty(mr, addr1, l);
2543 if (release_lock) {
2544 qemu_mutex_unlock_iothread();
2545 release_lock = false;
2548 len -= l;
2549 buf += l;
2550 addr += l;
2552 if (!len) {
2553 break;
2556 l = len;
2557 mr = address_space_translate(as, addr, &addr1, &l, true);
2560 return result;
2563 MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2564 const uint8_t *buf, int len)
2566 hwaddr l;
2567 hwaddr addr1;
2568 MemoryRegion *mr;
2569 MemTxResult result = MEMTX_OK;
2571 if (len > 0) {
2572 rcu_read_lock();
2573 l = len;
2574 mr = address_space_translate(as, addr, &addr1, &l, true);
2575 result = address_space_write_continue(as, addr, attrs, buf, len,
2576 addr1, l, mr);
2577 rcu_read_unlock();
2580 return result;
2583 /* Called within RCU critical section. */
2584 MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr,
2585 MemTxAttrs attrs, uint8_t *buf,
2586 int len, hwaddr addr1, hwaddr l,
2587 MemoryRegion *mr)
2589 uint8_t *ptr;
2590 uint64_t val;
2591 MemTxResult result = MEMTX_OK;
2592 bool release_lock = false;
2594 for (;;) {
2595 if (!memory_access_is_direct(mr, false)) {
2596 /* I/O case */
2597 release_lock |= prepare_mmio_access(mr);
2598 l = memory_access_size(mr, l, addr1);
2599 switch (l) {
2600 case 8:
2601 /* 64 bit read access */
2602 result |= memory_region_dispatch_read(mr, addr1, &val, 8,
2603 attrs);
2604 stq_p(buf, val);
2605 break;
2606 case 4:
2607 /* 32 bit read access */
2608 result |= memory_region_dispatch_read(mr, addr1, &val, 4,
2609 attrs);
2610 stl_p(buf, val);
2611 break;
2612 case 2:
2613 /* 16 bit read access */
2614 result |= memory_region_dispatch_read(mr, addr1, &val, 2,
2615 attrs);
2616 stw_p(buf, val);
2617 break;
2618 case 1:
2619 /* 8 bit read access */
2620 result |= memory_region_dispatch_read(mr, addr1, &val, 1,
2621 attrs);
2622 stb_p(buf, val);
2623 break;
2624 default:
2625 abort();
2627 } else {
2628 /* RAM case */
2629 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
2630 memcpy(buf, ptr, l);
2633 if (release_lock) {
2634 qemu_mutex_unlock_iothread();
2635 release_lock = false;
2638 len -= l;
2639 buf += l;
2640 addr += l;
2642 if (!len) {
2643 break;
2646 l = len;
2647 mr = address_space_translate(as, addr, &addr1, &l, false);
2650 return result;
2653 MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
2654 MemTxAttrs attrs, uint8_t *buf, int len)
2656 hwaddr l;
2657 hwaddr addr1;
2658 MemoryRegion *mr;
2659 MemTxResult result = MEMTX_OK;
2661 if (len > 0) {
2662 rcu_read_lock();
2663 l = len;
2664 mr = address_space_translate(as, addr, &addr1, &l, false);
2665 result = address_space_read_continue(as, addr, attrs, buf, len,
2666 addr1, l, mr);
2667 rcu_read_unlock();
2670 return result;
2673 MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2674 uint8_t *buf, int len, bool is_write)
2676 if (is_write) {
2677 return address_space_write(as, addr, attrs, (uint8_t *)buf, len);
2678 } else {
2679 return address_space_read(as, addr, attrs, (uint8_t *)buf, len);
2683 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2684 int len, int is_write)
2686 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
2687 buf, len, is_write);
2690 enum write_rom_type {
2691 WRITE_DATA,
2692 FLUSH_CACHE,
2695 static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
2696 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
2698 hwaddr l;
2699 uint8_t *ptr;
2700 hwaddr addr1;
2701 MemoryRegion *mr;
2703 rcu_read_lock();
2704 while (len > 0) {
2705 l = len;
2706 mr = address_space_translate(as, addr, &addr1, &l, true);
2708 if (!(memory_region_is_ram(mr) ||
2709 memory_region_is_romd(mr))) {
2710 l = memory_access_size(mr, l, addr1);
2711 } else {
2712 /* ROM/RAM case */
2713 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
2714 switch (type) {
2715 case WRITE_DATA:
2716 memcpy(ptr, buf, l);
2717 invalidate_and_set_dirty(mr, addr1, l);
2718 break;
2719 case FLUSH_CACHE:
2720 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
2721 break;
2724 len -= l;
2725 buf += l;
2726 addr += l;
2728 rcu_read_unlock();
2731 /* used for ROM loading : can write in RAM and ROM */
2732 void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
2733 const uint8_t *buf, int len)
2735 cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
2738 void cpu_flush_icache_range(hwaddr start, int len)
2741 * This function should do the same thing as an icache flush that was
2742 * triggered from within the guest. For TCG we are always cache coherent,
2743 * so there is no need to flush anything. For KVM / Xen we need to flush
2744 * the host's instruction cache at least.
2746 if (tcg_enabled()) {
2747 return;
2750 cpu_physical_memory_write_rom_internal(&address_space_memory,
2751 start, NULL, len, FLUSH_CACHE);
2754 typedef struct {
2755 MemoryRegion *mr;
2756 void *buffer;
2757 hwaddr addr;
2758 hwaddr len;
2759 bool in_use;
2760 } BounceBuffer;
2762 static BounceBuffer bounce;
2764 typedef struct MapClient {
2765 QEMUBH *bh;
2766 QLIST_ENTRY(MapClient) link;
2767 } MapClient;
2769 QemuMutex map_client_list_lock;
2770 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2771 = QLIST_HEAD_INITIALIZER(map_client_list);
2773 static void cpu_unregister_map_client_do(MapClient *client)
2775 QLIST_REMOVE(client, link);
2776 g_free(client);
2779 static void cpu_notify_map_clients_locked(void)
2781 MapClient *client;
2783 while (!QLIST_EMPTY(&map_client_list)) {
2784 client = QLIST_FIRST(&map_client_list);
2785 qemu_bh_schedule(client->bh);
2786 cpu_unregister_map_client_do(client);
2790 void cpu_register_map_client(QEMUBH *bh)
2792 MapClient *client = g_malloc(sizeof(*client));
2794 qemu_mutex_lock(&map_client_list_lock);
2795 client->bh = bh;
2796 QLIST_INSERT_HEAD(&map_client_list, client, link);
2797 if (!atomic_read(&bounce.in_use)) {
2798 cpu_notify_map_clients_locked();
2800 qemu_mutex_unlock(&map_client_list_lock);
2803 void cpu_exec_init_all(void)
2805 qemu_mutex_init(&ram_list.mutex);
2806 io_mem_init();
2807 memory_map_init();
2808 qemu_mutex_init(&map_client_list_lock);
2811 void cpu_unregister_map_client(QEMUBH *bh)
2813 MapClient *client;
2815 qemu_mutex_lock(&map_client_list_lock);
2816 QLIST_FOREACH(client, &map_client_list, link) {
2817 if (client->bh == bh) {
2818 cpu_unregister_map_client_do(client);
2819 break;
2822 qemu_mutex_unlock(&map_client_list_lock);
2825 static void cpu_notify_map_clients(void)
2827 qemu_mutex_lock(&map_client_list_lock);
2828 cpu_notify_map_clients_locked();
2829 qemu_mutex_unlock(&map_client_list_lock);
2832 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2834 MemoryRegion *mr;
2835 hwaddr l, xlat;
2837 rcu_read_lock();
2838 while (len > 0) {
2839 l = len;
2840 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2841 if (!memory_access_is_direct(mr, is_write)) {
2842 l = memory_access_size(mr, l, addr);
2843 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2844 return false;
2848 len -= l;
2849 addr += l;
2851 rcu_read_unlock();
2852 return true;
2855 /* Map a physical memory region into a host virtual address.
2856 * May map a subset of the requested range, given by and returned in *plen.
2857 * May return NULL if resources needed to perform the mapping are exhausted.
2858 * Use only for reads OR writes - not for read-modify-write operations.
2859 * Use cpu_register_map_client() to know when retrying the map operation is
2860 * likely to succeed.
2862 void *address_space_map(AddressSpace *as,
2863 hwaddr addr,
2864 hwaddr *plen,
2865 bool is_write)
2867 hwaddr len = *plen;
2868 hwaddr done = 0;
2869 hwaddr l, xlat, base;
2870 MemoryRegion *mr, *this_mr;
2871 void *ptr;
2873 if (len == 0) {
2874 return NULL;
2877 l = len;
2878 rcu_read_lock();
2879 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2881 if (!memory_access_is_direct(mr, is_write)) {
2882 if (atomic_xchg(&bounce.in_use, true)) {
2883 rcu_read_unlock();
2884 return NULL;
2886 /* Avoid unbounded allocations */
2887 l = MIN(l, TARGET_PAGE_SIZE);
2888 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
2889 bounce.addr = addr;
2890 bounce.len = l;
2892 memory_region_ref(mr);
2893 bounce.mr = mr;
2894 if (!is_write) {
2895 address_space_read(as, addr, MEMTXATTRS_UNSPECIFIED,
2896 bounce.buffer, l);
2899 rcu_read_unlock();
2900 *plen = l;
2901 return bounce.buffer;
2904 base = xlat;
2906 for (;;) {
2907 len -= l;
2908 addr += l;
2909 done += l;
2910 if (len == 0) {
2911 break;
2914 l = len;
2915 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2916 if (this_mr != mr || xlat != base + done) {
2917 break;
2921 memory_region_ref(mr);
2922 *plen = done;
2923 ptr = qemu_ram_ptr_length(mr->ram_block, base, plen);
2924 rcu_read_unlock();
2926 return ptr;
2929 /* Unmaps a memory region previously mapped by address_space_map().
2930 * Will also mark the memory as dirty if is_write == 1. access_len gives
2931 * the amount of memory that was actually read or written by the caller.
2933 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2934 int is_write, hwaddr access_len)
2936 if (buffer != bounce.buffer) {
2937 MemoryRegion *mr;
2938 ram_addr_t addr1;
2940 mr = memory_region_from_host(buffer, &addr1);
2941 assert(mr != NULL);
2942 if (is_write) {
2943 invalidate_and_set_dirty(mr, addr1, access_len);
2945 if (xen_enabled()) {
2946 xen_invalidate_map_cache_entry(buffer);
2948 memory_region_unref(mr);
2949 return;
2951 if (is_write) {
2952 address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
2953 bounce.buffer, access_len);
2955 qemu_vfree(bounce.buffer);
2956 bounce.buffer = NULL;
2957 memory_region_unref(bounce.mr);
2958 atomic_mb_set(&bounce.in_use, false);
2959 cpu_notify_map_clients();
2962 void *cpu_physical_memory_map(hwaddr addr,
2963 hwaddr *plen,
2964 int is_write)
2966 return address_space_map(&address_space_memory, addr, plen, is_write);
2969 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2970 int is_write, hwaddr access_len)
2972 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2975 /* warning: addr must be aligned */
2976 static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
2977 MemTxAttrs attrs,
2978 MemTxResult *result,
2979 enum device_endian endian)
2981 uint8_t *ptr;
2982 uint64_t val;
2983 MemoryRegion *mr;
2984 hwaddr l = 4;
2985 hwaddr addr1;
2986 MemTxResult r;
2987 bool release_lock = false;
2989 rcu_read_lock();
2990 mr = address_space_translate(as, addr, &addr1, &l, false);
2991 if (l < 4 || !memory_access_is_direct(mr, false)) {
2992 release_lock |= prepare_mmio_access(mr);
2994 /* I/O case */
2995 r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
2996 #if defined(TARGET_WORDS_BIGENDIAN)
2997 if (endian == DEVICE_LITTLE_ENDIAN) {
2998 val = bswap32(val);
3000 #else
3001 if (endian == DEVICE_BIG_ENDIAN) {
3002 val = bswap32(val);
3004 #endif
3005 } else {
3006 /* RAM case */
3007 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3008 switch (endian) {
3009 case DEVICE_LITTLE_ENDIAN:
3010 val = ldl_le_p(ptr);
3011 break;
3012 case DEVICE_BIG_ENDIAN:
3013 val = ldl_be_p(ptr);
3014 break;
3015 default:
3016 val = ldl_p(ptr);
3017 break;
3019 r = MEMTX_OK;
3021 if (result) {
3022 *result = r;
3024 if (release_lock) {
3025 qemu_mutex_unlock_iothread();
3027 rcu_read_unlock();
3028 return val;
3031 uint32_t address_space_ldl(AddressSpace *as, hwaddr addr,
3032 MemTxAttrs attrs, MemTxResult *result)
3034 return address_space_ldl_internal(as, addr, attrs, result,
3035 DEVICE_NATIVE_ENDIAN);
3038 uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr,
3039 MemTxAttrs attrs, MemTxResult *result)
3041 return address_space_ldl_internal(as, addr, attrs, result,
3042 DEVICE_LITTLE_ENDIAN);
3045 uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr,
3046 MemTxAttrs attrs, MemTxResult *result)
3048 return address_space_ldl_internal(as, addr, attrs, result,
3049 DEVICE_BIG_ENDIAN);
3052 uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
3054 return address_space_ldl(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3057 uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
3059 return address_space_ldl_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3062 uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
3064 return address_space_ldl_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3067 /* warning: addr must be aligned */
3068 static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
3069 MemTxAttrs attrs,
3070 MemTxResult *result,
3071 enum device_endian endian)
3073 uint8_t *ptr;
3074 uint64_t val;
3075 MemoryRegion *mr;
3076 hwaddr l = 8;
3077 hwaddr addr1;
3078 MemTxResult r;
3079 bool release_lock = false;
3081 rcu_read_lock();
3082 mr = address_space_translate(as, addr, &addr1, &l,
3083 false);
3084 if (l < 8 || !memory_access_is_direct(mr, false)) {
3085 release_lock |= prepare_mmio_access(mr);
3087 /* I/O case */
3088 r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
3089 #if defined(TARGET_WORDS_BIGENDIAN)
3090 if (endian == DEVICE_LITTLE_ENDIAN) {
3091 val = bswap64(val);
3093 #else
3094 if (endian == DEVICE_BIG_ENDIAN) {
3095 val = bswap64(val);
3097 #endif
3098 } else {
3099 /* RAM case */
3100 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3101 switch (endian) {
3102 case DEVICE_LITTLE_ENDIAN:
3103 val = ldq_le_p(ptr);
3104 break;
3105 case DEVICE_BIG_ENDIAN:
3106 val = ldq_be_p(ptr);
3107 break;
3108 default:
3109 val = ldq_p(ptr);
3110 break;
3112 r = MEMTX_OK;
3114 if (result) {
3115 *result = r;
3117 if (release_lock) {
3118 qemu_mutex_unlock_iothread();
3120 rcu_read_unlock();
3121 return val;
3124 uint64_t address_space_ldq(AddressSpace *as, hwaddr addr,
3125 MemTxAttrs attrs, MemTxResult *result)
3127 return address_space_ldq_internal(as, addr, attrs, result,
3128 DEVICE_NATIVE_ENDIAN);
3131 uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr,
3132 MemTxAttrs attrs, MemTxResult *result)
3134 return address_space_ldq_internal(as, addr, attrs, result,
3135 DEVICE_LITTLE_ENDIAN);
3138 uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr,
3139 MemTxAttrs attrs, MemTxResult *result)
3141 return address_space_ldq_internal(as, addr, attrs, result,
3142 DEVICE_BIG_ENDIAN);
3145 uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
3147 return address_space_ldq(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3150 uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
3152 return address_space_ldq_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3155 uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
3157 return address_space_ldq_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3160 /* XXX: optimize */
3161 uint32_t address_space_ldub(AddressSpace *as, hwaddr addr,
3162 MemTxAttrs attrs, MemTxResult *result)
3164 uint8_t val;
3165 MemTxResult r;
3167 r = address_space_rw(as, addr, attrs, &val, 1, 0);
3168 if (result) {
3169 *result = r;
3171 return val;
3174 uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
3176 return address_space_ldub(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3179 /* warning: addr must be aligned */
3180 static inline uint32_t address_space_lduw_internal(AddressSpace *as,
3181 hwaddr addr,
3182 MemTxAttrs attrs,
3183 MemTxResult *result,
3184 enum device_endian endian)
3186 uint8_t *ptr;
3187 uint64_t val;
3188 MemoryRegion *mr;
3189 hwaddr l = 2;
3190 hwaddr addr1;
3191 MemTxResult r;
3192 bool release_lock = false;
3194 rcu_read_lock();
3195 mr = address_space_translate(as, addr, &addr1, &l,
3196 false);
3197 if (l < 2 || !memory_access_is_direct(mr, false)) {
3198 release_lock |= prepare_mmio_access(mr);
3200 /* I/O case */
3201 r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
3202 #if defined(TARGET_WORDS_BIGENDIAN)
3203 if (endian == DEVICE_LITTLE_ENDIAN) {
3204 val = bswap16(val);
3206 #else
3207 if (endian == DEVICE_BIG_ENDIAN) {
3208 val = bswap16(val);
3210 #endif
3211 } else {
3212 /* RAM case */
3213 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3214 switch (endian) {
3215 case DEVICE_LITTLE_ENDIAN:
3216 val = lduw_le_p(ptr);
3217 break;
3218 case DEVICE_BIG_ENDIAN:
3219 val = lduw_be_p(ptr);
3220 break;
3221 default:
3222 val = lduw_p(ptr);
3223 break;
3225 r = MEMTX_OK;
3227 if (result) {
3228 *result = r;
3230 if (release_lock) {
3231 qemu_mutex_unlock_iothread();
3233 rcu_read_unlock();
3234 return val;
3237 uint32_t address_space_lduw(AddressSpace *as, hwaddr addr,
3238 MemTxAttrs attrs, MemTxResult *result)
3240 return address_space_lduw_internal(as, addr, attrs, result,
3241 DEVICE_NATIVE_ENDIAN);
3244 uint32_t address_space_lduw_le(AddressSpace *as, hwaddr addr,
3245 MemTxAttrs attrs, MemTxResult *result)
3247 return address_space_lduw_internal(as, addr, attrs, result,
3248 DEVICE_LITTLE_ENDIAN);
3251 uint32_t address_space_lduw_be(AddressSpace *as, hwaddr addr,
3252 MemTxAttrs attrs, MemTxResult *result)
3254 return address_space_lduw_internal(as, addr, attrs, result,
3255 DEVICE_BIG_ENDIAN);
3258 uint32_t lduw_phys(AddressSpace *as, hwaddr addr)
3260 return address_space_lduw(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3263 uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr)
3265 return address_space_lduw_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3268 uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr)
3270 return address_space_lduw_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3273 /* warning: addr must be aligned. The ram page is not masked as dirty
3274 and the code inside is not invalidated. It is useful if the dirty
3275 bits are used to track modified PTEs */
3276 void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
3277 MemTxAttrs attrs, MemTxResult *result)
3279 uint8_t *ptr;
3280 MemoryRegion *mr;
3281 hwaddr l = 4;
3282 hwaddr addr1;
3283 MemTxResult r;
3284 uint8_t dirty_log_mask;
3285 bool release_lock = false;
3287 rcu_read_lock();
3288 mr = address_space_translate(as, addr, &addr1, &l,
3289 true);
3290 if (l < 4 || !memory_access_is_direct(mr, true)) {
3291 release_lock |= prepare_mmio_access(mr);
3293 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3294 } else {
3295 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3296 stl_p(ptr, val);
3298 dirty_log_mask = memory_region_get_dirty_log_mask(mr);
3299 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
3300 cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr,
3301 4, dirty_log_mask);
3302 r = MEMTX_OK;
3304 if (result) {
3305 *result = r;
3307 if (release_lock) {
3308 qemu_mutex_unlock_iothread();
3310 rcu_read_unlock();
3313 void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
3315 address_space_stl_notdirty(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3318 /* warning: addr must be aligned */
3319 static inline void address_space_stl_internal(AddressSpace *as,
3320 hwaddr addr, uint32_t val,
3321 MemTxAttrs attrs,
3322 MemTxResult *result,
3323 enum device_endian endian)
3325 uint8_t *ptr;
3326 MemoryRegion *mr;
3327 hwaddr l = 4;
3328 hwaddr addr1;
3329 MemTxResult r;
3330 bool release_lock = false;
3332 rcu_read_lock();
3333 mr = address_space_translate(as, addr, &addr1, &l,
3334 true);
3335 if (l < 4 || !memory_access_is_direct(mr, true)) {
3336 release_lock |= prepare_mmio_access(mr);
3338 #if defined(TARGET_WORDS_BIGENDIAN)
3339 if (endian == DEVICE_LITTLE_ENDIAN) {
3340 val = bswap32(val);
3342 #else
3343 if (endian == DEVICE_BIG_ENDIAN) {
3344 val = bswap32(val);
3346 #endif
3347 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3348 } else {
3349 /* RAM case */
3350 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3351 switch (endian) {
3352 case DEVICE_LITTLE_ENDIAN:
3353 stl_le_p(ptr, val);
3354 break;
3355 case DEVICE_BIG_ENDIAN:
3356 stl_be_p(ptr, val);
3357 break;
3358 default:
3359 stl_p(ptr, val);
3360 break;
3362 invalidate_and_set_dirty(mr, addr1, 4);
3363 r = MEMTX_OK;
3365 if (result) {
3366 *result = r;
3368 if (release_lock) {
3369 qemu_mutex_unlock_iothread();
3371 rcu_read_unlock();
3374 void address_space_stl(AddressSpace *as, hwaddr addr, uint32_t val,
3375 MemTxAttrs attrs, MemTxResult *result)
3377 address_space_stl_internal(as, addr, val, attrs, result,
3378 DEVICE_NATIVE_ENDIAN);
3381 void address_space_stl_le(AddressSpace *as, hwaddr addr, uint32_t val,
3382 MemTxAttrs attrs, MemTxResult *result)
3384 address_space_stl_internal(as, addr, val, attrs, result,
3385 DEVICE_LITTLE_ENDIAN);
3388 void address_space_stl_be(AddressSpace *as, hwaddr addr, uint32_t val,
3389 MemTxAttrs attrs, MemTxResult *result)
3391 address_space_stl_internal(as, addr, val, attrs, result,
3392 DEVICE_BIG_ENDIAN);
3395 void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3397 address_space_stl(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3400 void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3402 address_space_stl_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3405 void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3407 address_space_stl_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3410 /* XXX: optimize */
3411 void address_space_stb(AddressSpace *as, hwaddr addr, uint32_t val,
3412 MemTxAttrs attrs, MemTxResult *result)
3414 uint8_t v = val;
3415 MemTxResult r;
3417 r = address_space_rw(as, addr, attrs, &v, 1, 1);
3418 if (result) {
3419 *result = r;
3423 void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3425 address_space_stb(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3428 /* warning: addr must be aligned */
3429 static inline void address_space_stw_internal(AddressSpace *as,
3430 hwaddr addr, uint32_t val,
3431 MemTxAttrs attrs,
3432 MemTxResult *result,
3433 enum device_endian endian)
3435 uint8_t *ptr;
3436 MemoryRegion *mr;
3437 hwaddr l = 2;
3438 hwaddr addr1;
3439 MemTxResult r;
3440 bool release_lock = false;
3442 rcu_read_lock();
3443 mr = address_space_translate(as, addr, &addr1, &l, true);
3444 if (l < 2 || !memory_access_is_direct(mr, true)) {
3445 release_lock |= prepare_mmio_access(mr);
3447 #if defined(TARGET_WORDS_BIGENDIAN)
3448 if (endian == DEVICE_LITTLE_ENDIAN) {
3449 val = bswap16(val);
3451 #else
3452 if (endian == DEVICE_BIG_ENDIAN) {
3453 val = bswap16(val);
3455 #endif
3456 r = memory_region_dispatch_write(mr, addr1, val, 2, attrs);
3457 } else {
3458 /* RAM case */
3459 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3460 switch (endian) {
3461 case DEVICE_LITTLE_ENDIAN:
3462 stw_le_p(ptr, val);
3463 break;
3464 case DEVICE_BIG_ENDIAN:
3465 stw_be_p(ptr, val);
3466 break;
3467 default:
3468 stw_p(ptr, val);
3469 break;
3471 invalidate_and_set_dirty(mr, addr1, 2);
3472 r = MEMTX_OK;
3474 if (result) {
3475 *result = r;
3477 if (release_lock) {
3478 qemu_mutex_unlock_iothread();
3480 rcu_read_unlock();
3483 void address_space_stw(AddressSpace *as, hwaddr addr, uint32_t val,
3484 MemTxAttrs attrs, MemTxResult *result)
3486 address_space_stw_internal(as, addr, val, attrs, result,
3487 DEVICE_NATIVE_ENDIAN);
3490 void address_space_stw_le(AddressSpace *as, hwaddr addr, uint32_t val,
3491 MemTxAttrs attrs, MemTxResult *result)
3493 address_space_stw_internal(as, addr, val, attrs, result,
3494 DEVICE_LITTLE_ENDIAN);
3497 void address_space_stw_be(AddressSpace *as, hwaddr addr, uint32_t val,
3498 MemTxAttrs attrs, MemTxResult *result)
3500 address_space_stw_internal(as, addr, val, attrs, result,
3501 DEVICE_BIG_ENDIAN);
3504 void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3506 address_space_stw(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3509 void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3511 address_space_stw_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3514 void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3516 address_space_stw_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3519 /* XXX: optimize */
3520 void address_space_stq(AddressSpace *as, hwaddr addr, uint64_t val,
3521 MemTxAttrs attrs, MemTxResult *result)
3523 MemTxResult r;
3524 val = tswap64(val);
3525 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3526 if (result) {
3527 *result = r;
3531 void address_space_stq_le(AddressSpace *as, hwaddr addr, uint64_t val,
3532 MemTxAttrs attrs, MemTxResult *result)
3534 MemTxResult r;
3535 val = cpu_to_le64(val);
3536 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3537 if (result) {
3538 *result = r;
3541 void address_space_stq_be(AddressSpace *as, hwaddr addr, uint64_t val,
3542 MemTxAttrs attrs, MemTxResult *result)
3544 MemTxResult r;
3545 val = cpu_to_be64(val);
3546 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3547 if (result) {
3548 *result = r;
3552 void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3554 address_space_stq(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3557 void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3559 address_space_stq_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3562 void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3564 address_space_stq_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3567 /* virtual memory access for debug (includes writing to ROM) */
3568 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
3569 uint8_t *buf, int len, int is_write)
3571 int l;
3572 hwaddr phys_addr;
3573 target_ulong page;
3575 while (len > 0) {
3576 int asidx;
3577 MemTxAttrs attrs;
3579 page = addr & TARGET_PAGE_MASK;
3580 phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
3581 asidx = cpu_asidx_from_attrs(cpu, attrs);
3582 /* if no physical page mapped, return an error */
3583 if (phys_addr == -1)
3584 return -1;
3585 l = (page + TARGET_PAGE_SIZE) - addr;
3586 if (l > len)
3587 l = len;
3588 phys_addr += (addr & ~TARGET_PAGE_MASK);
3589 if (is_write) {
3590 cpu_physical_memory_write_rom(cpu->cpu_ases[asidx].as,
3591 phys_addr, buf, l);
3592 } else {
3593 address_space_rw(cpu->cpu_ases[asidx].as, phys_addr,
3594 MEMTXATTRS_UNSPECIFIED,
3595 buf, l, 0);
3597 len -= l;
3598 buf += l;
3599 addr += l;
3601 return 0;
3605 * Allows code that needs to deal with migration bitmaps etc to still be built
3606 * target independent.
3608 size_t qemu_target_page_bits(void)
3610 return TARGET_PAGE_BITS;
3613 #endif
3616 * A helper function for the _utterly broken_ virtio device model to find out if
3617 * it's running on a big endian machine. Don't do this at home kids!
3619 bool target_words_bigendian(void);
3620 bool target_words_bigendian(void)
3622 #if defined(TARGET_WORDS_BIGENDIAN)
3623 return true;
3624 #else
3625 return false;
3626 #endif
3629 #ifndef CONFIG_USER_ONLY
3630 bool cpu_physical_memory_is_io(hwaddr phys_addr)
3632 MemoryRegion*mr;
3633 hwaddr l = 1;
3634 bool res;
3636 rcu_read_lock();
3637 mr = address_space_translate(&address_space_memory,
3638 phys_addr, &phys_addr, &l, false);
3640 res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
3641 rcu_read_unlock();
3642 return res;
3645 int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
3647 RAMBlock *block;
3648 int ret = 0;
3650 rcu_read_lock();
3651 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
3652 ret = func(block->idstr, block->host, block->offset,
3653 block->used_length, opaque);
3654 if (ret) {
3655 break;
3658 rcu_read_unlock();
3659 return ret;
3661 #endif