hw/input/tsc210x: Remove superfluous memset
[qemu/ar7.git] / exec.c
blobed88e72f14e711750522a308a659ce84d1292e61
1 /*
2 * Virtual page mapping
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "config.h"
20 #ifndef _WIN32
21 #include <sys/types.h>
22 #include <sys/mman.h>
23 #endif
25 #include "qemu-common.h"
26 #include "cpu.h"
27 #include "tcg.h"
28 #include "hw/hw.h"
29 #if !defined(CONFIG_USER_ONLY)
30 #include "hw/boards.h"
31 #endif
32 #include "hw/qdev.h"
33 #include "qemu/osdep.h"
34 #include "sysemu/kvm.h"
35 #include "sysemu/sysemu.h"
36 #include "hw/xen/xen.h"
37 #include "qemu/timer.h"
38 #include "qemu/config-file.h"
39 #include "qemu/error-report.h"
40 #include "exec/memory.h"
41 #include "sysemu/dma.h"
42 #include "exec/address-spaces.h"
43 #if defined(CONFIG_USER_ONLY)
44 #include <qemu.h>
45 #else /* !CONFIG_USER_ONLY */
46 #include "sysemu/xen-mapcache.h"
47 #include "trace.h"
48 #endif
49 #include "exec/cpu-all.h"
50 #include "qemu/rcu_queue.h"
51 #include "qemu/main-loop.h"
52 #include "translate-all.h"
53 #include "sysemu/replay.h"
55 #include "exec/memory-internal.h"
56 #include "exec/ram_addr.h"
58 #include "qemu/range.h"
59 #ifndef _WIN32
60 #include "qemu/mmap-alloc.h"
61 #endif
63 //#define DEBUG_SUBPAGE
65 #if !defined(CONFIG_USER_ONLY)
66 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
67 * are protected by the ramlist lock.
69 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
71 static MemoryRegion *system_memory;
72 static MemoryRegion *system_io;
74 AddressSpace address_space_io;
75 AddressSpace address_space_memory;
77 MemoryRegion io_mem_rom, io_mem_notdirty;
78 static MemoryRegion io_mem_unassigned;
80 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
81 #define RAM_PREALLOC (1 << 0)
83 /* RAM is mmap-ed with MAP_SHARED */
84 #define RAM_SHARED (1 << 1)
86 /* Only a portion of RAM (used_length) is actually used, and migrated.
87 * This used_length size can change across reboots.
89 #define RAM_RESIZEABLE (1 << 2)
91 /* RAM is backed by an mmapped file.
93 #define RAM_FILE (1 << 3)
94 #endif
96 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
97 /* current CPU in the current thread. It is only valid inside
98 cpu_exec() */
99 __thread CPUState *current_cpu;
100 /* 0 = Do not count executed instructions.
101 1 = Precise instruction counting.
102 2 = Adaptive rate instruction counting. */
103 int use_icount;
105 #if !defined(CONFIG_USER_ONLY)
107 typedef struct PhysPageEntry PhysPageEntry;
109 struct PhysPageEntry {
110 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
111 uint32_t skip : 6;
112 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
113 uint32_t ptr : 26;
116 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
118 /* Size of the L2 (and L3, etc) page tables. */
119 #define ADDR_SPACE_BITS 64
121 #define P_L2_BITS 9
122 #define P_L2_SIZE (1 << P_L2_BITS)
124 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
126 typedef PhysPageEntry Node[P_L2_SIZE];
128 typedef struct PhysPageMap {
129 struct rcu_head rcu;
131 unsigned sections_nb;
132 unsigned sections_nb_alloc;
133 unsigned nodes_nb;
134 unsigned nodes_nb_alloc;
135 Node *nodes;
136 MemoryRegionSection *sections;
137 } PhysPageMap;
139 struct AddressSpaceDispatch {
140 struct rcu_head rcu;
142 /* This is a multi-level map on the physical address space.
143 * The bottom level has pointers to MemoryRegionSections.
145 PhysPageEntry phys_map;
146 PhysPageMap map;
147 AddressSpace *as;
150 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
151 typedef struct subpage_t {
152 MemoryRegion iomem;
153 AddressSpace *as;
154 hwaddr base;
155 uint16_t sub_section[TARGET_PAGE_SIZE];
156 } subpage_t;
158 #define PHYS_SECTION_UNASSIGNED 0
159 #define PHYS_SECTION_NOTDIRTY 1
160 #define PHYS_SECTION_ROM 2
161 #define PHYS_SECTION_WATCH 3
163 static void io_mem_init(void);
164 static void memory_map_init(void);
165 static void tcg_commit(MemoryListener *listener);
167 static MemoryRegion io_mem_watch;
170 * CPUAddressSpace: all the information a CPU needs about an AddressSpace
171 * @cpu: the CPU whose AddressSpace this is
172 * @as: the AddressSpace itself
173 * @memory_dispatch: its dispatch pointer (cached, RCU protected)
174 * @tcg_as_listener: listener for tracking changes to the AddressSpace
176 struct CPUAddressSpace {
177 CPUState *cpu;
178 AddressSpace *as;
179 struct AddressSpaceDispatch *memory_dispatch;
180 MemoryListener tcg_as_listener;
183 #endif
185 #if !defined(CONFIG_USER_ONLY)
187 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
189 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
190 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
191 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
192 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
196 static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
198 unsigned i;
199 uint32_t ret;
200 PhysPageEntry e;
201 PhysPageEntry *p;
203 ret = map->nodes_nb++;
204 p = map->nodes[ret];
205 assert(ret != PHYS_MAP_NODE_NIL);
206 assert(ret != map->nodes_nb_alloc);
208 e.skip = leaf ? 0 : 1;
209 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
210 for (i = 0; i < P_L2_SIZE; ++i) {
211 memcpy(&p[i], &e, sizeof(e));
213 return ret;
216 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
217 hwaddr *index, hwaddr *nb, uint16_t leaf,
218 int level)
220 PhysPageEntry *p;
221 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
223 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
224 lp->ptr = phys_map_node_alloc(map, level == 0);
226 p = map->nodes[lp->ptr];
227 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
229 while (*nb && lp < &p[P_L2_SIZE]) {
230 if ((*index & (step - 1)) == 0 && *nb >= step) {
231 lp->skip = 0;
232 lp->ptr = leaf;
233 *index += step;
234 *nb -= step;
235 } else {
236 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
238 ++lp;
242 static void phys_page_set(AddressSpaceDispatch *d,
243 hwaddr index, hwaddr nb,
244 uint16_t leaf)
246 /* Wildly overreserve - it doesn't matter much. */
247 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
249 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
252 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
253 * and update our entry so we can skip it and go directly to the destination.
255 static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)
257 unsigned valid_ptr = P_L2_SIZE;
258 int valid = 0;
259 PhysPageEntry *p;
260 int i;
262 if (lp->ptr == PHYS_MAP_NODE_NIL) {
263 return;
266 p = nodes[lp->ptr];
267 for (i = 0; i < P_L2_SIZE; i++) {
268 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
269 continue;
272 valid_ptr = i;
273 valid++;
274 if (p[i].skip) {
275 phys_page_compact(&p[i], nodes, compacted);
279 /* We can only compress if there's only one child. */
280 if (valid != 1) {
281 return;
284 assert(valid_ptr < P_L2_SIZE);
286 /* Don't compress if it won't fit in the # of bits we have. */
287 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
288 return;
291 lp->ptr = p[valid_ptr].ptr;
292 if (!p[valid_ptr].skip) {
293 /* If our only child is a leaf, make this a leaf. */
294 /* By design, we should have made this node a leaf to begin with so we
295 * should never reach here.
296 * But since it's so simple to handle this, let's do it just in case we
297 * change this rule.
299 lp->skip = 0;
300 } else {
301 lp->skip += p[valid_ptr].skip;
305 static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
307 DECLARE_BITMAP(compacted, nodes_nb);
309 if (d->phys_map.skip) {
310 phys_page_compact(&d->phys_map, d->map.nodes, compacted);
314 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
315 Node *nodes, MemoryRegionSection *sections)
317 PhysPageEntry *p;
318 hwaddr index = addr >> TARGET_PAGE_BITS;
319 int i;
321 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
322 if (lp.ptr == PHYS_MAP_NODE_NIL) {
323 return &sections[PHYS_SECTION_UNASSIGNED];
325 p = nodes[lp.ptr];
326 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
329 if (sections[lp.ptr].size.hi ||
330 range_covers_byte(sections[lp.ptr].offset_within_address_space,
331 sections[lp.ptr].size.lo, addr)) {
332 return &sections[lp.ptr];
333 } else {
334 return &sections[PHYS_SECTION_UNASSIGNED];
338 bool memory_region_is_unassigned(MemoryRegion *mr)
340 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
341 && mr != &io_mem_watch;
344 /* Called from RCU critical section */
345 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
346 hwaddr addr,
347 bool resolve_subpage)
349 MemoryRegionSection *section;
350 subpage_t *subpage;
352 section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);
353 if (resolve_subpage && section->mr->subpage) {
354 subpage = container_of(section->mr, subpage_t, iomem);
355 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
357 return section;
360 /* Called from RCU critical section */
361 static MemoryRegionSection *
362 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
363 hwaddr *plen, bool resolve_subpage)
365 MemoryRegionSection *section;
366 MemoryRegion *mr;
367 Int128 diff;
369 section = address_space_lookup_region(d, addr, resolve_subpage);
370 /* Compute offset within MemoryRegionSection */
371 addr -= section->offset_within_address_space;
373 /* Compute offset within MemoryRegion */
374 *xlat = addr + section->offset_within_region;
376 mr = section->mr;
378 /* MMIO registers can be expected to perform full-width accesses based only
379 * on their address, without considering adjacent registers that could
380 * decode to completely different MemoryRegions. When such registers
381 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
382 * regions overlap wildly. For this reason we cannot clamp the accesses
383 * here.
385 * If the length is small (as is the case for address_space_ldl/stl),
386 * everything works fine. If the incoming length is large, however,
387 * the caller really has to do the clamping through memory_access_size.
389 if (memory_region_is_ram(mr)) {
390 diff = int128_sub(section->size, int128_make64(addr));
391 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
393 return section;
396 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
398 if (memory_region_is_ram(mr)) {
399 return !(is_write && mr->readonly);
401 if (memory_region_is_romd(mr)) {
402 return !is_write;
405 return false;
408 /* Called from RCU critical section */
409 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
410 hwaddr *xlat, hwaddr *plen,
411 bool is_write)
413 IOMMUTLBEntry iotlb;
414 MemoryRegionSection *section;
415 MemoryRegion *mr;
417 for (;;) {
418 AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch);
419 section = address_space_translate_internal(d, addr, &addr, plen, true);
420 mr = section->mr;
422 if (!mr->iommu_ops) {
423 break;
426 iotlb = mr->iommu_ops->translate(mr, addr, is_write);
427 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
428 | (addr & iotlb.addr_mask));
429 *plen = MIN(*plen, (addr | iotlb.addr_mask) - addr + 1);
430 if (!(iotlb.perm & (1 << is_write))) {
431 mr = &io_mem_unassigned;
432 break;
435 as = iotlb.target_as;
438 if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
439 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
440 *plen = MIN(page, *plen);
443 *xlat = addr;
444 return mr;
447 /* Called from RCU critical section */
448 MemoryRegionSection *
449 address_space_translate_for_iotlb(CPUState *cpu, hwaddr addr,
450 hwaddr *xlat, hwaddr *plen)
452 MemoryRegionSection *section;
453 section = address_space_translate_internal(cpu->cpu_ases[0].memory_dispatch,
454 addr, xlat, plen, false);
456 assert(!section->mr->iommu_ops);
457 return section;
459 #endif
461 #if !defined(CONFIG_USER_ONLY)
463 static int cpu_common_post_load(void *opaque, int version_id)
465 CPUState *cpu = opaque;
467 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
468 version_id is increased. */
469 cpu->interrupt_request &= ~0x01;
470 tlb_flush(cpu, 1);
472 return 0;
475 static int cpu_common_pre_load(void *opaque)
477 CPUState *cpu = opaque;
479 cpu->exception_index = -1;
481 return 0;
484 static bool cpu_common_exception_index_needed(void *opaque)
486 CPUState *cpu = opaque;
488 return tcg_enabled() && cpu->exception_index != -1;
491 static const VMStateDescription vmstate_cpu_common_exception_index = {
492 .name = "cpu_common/exception_index",
493 .version_id = 1,
494 .minimum_version_id = 1,
495 .needed = cpu_common_exception_index_needed,
496 .fields = (VMStateField[]) {
497 VMSTATE_INT32(exception_index, CPUState),
498 VMSTATE_END_OF_LIST()
502 static bool cpu_common_crash_occurred_needed(void *opaque)
504 CPUState *cpu = opaque;
506 return cpu->crash_occurred;
509 static const VMStateDescription vmstate_cpu_common_crash_occurred = {
510 .name = "cpu_common/crash_occurred",
511 .version_id = 1,
512 .minimum_version_id = 1,
513 .needed = cpu_common_crash_occurred_needed,
514 .fields = (VMStateField[]) {
515 VMSTATE_BOOL(crash_occurred, CPUState),
516 VMSTATE_END_OF_LIST()
520 const VMStateDescription vmstate_cpu_common = {
521 .name = "cpu_common",
522 .version_id = 1,
523 .minimum_version_id = 1,
524 .pre_load = cpu_common_pre_load,
525 .post_load = cpu_common_post_load,
526 .fields = (VMStateField[]) {
527 VMSTATE_UINT32(halted, CPUState),
528 VMSTATE_UINT32(interrupt_request, CPUState),
529 VMSTATE_END_OF_LIST()
531 .subsections = (const VMStateDescription*[]) {
532 &vmstate_cpu_common_exception_index,
533 &vmstate_cpu_common_crash_occurred,
534 NULL
538 #endif
540 CPUState *qemu_get_cpu(int index)
542 CPUState *cpu;
544 CPU_FOREACH(cpu) {
545 if (cpu->cpu_index == index) {
546 return cpu;
550 return NULL;
553 #if !defined(CONFIG_USER_ONLY)
554 void tcg_cpu_address_space_init(CPUState *cpu, AddressSpace *as)
556 /* We only support one address space per cpu at the moment. */
557 assert(cpu->as == as);
559 if (cpu->cpu_ases) {
560 /* We've already registered the listener for our only AS */
561 return;
564 cpu->cpu_ases = g_new0(CPUAddressSpace, 1);
565 cpu->cpu_ases[0].cpu = cpu;
566 cpu->cpu_ases[0].as = as;
567 cpu->cpu_ases[0].tcg_as_listener.commit = tcg_commit;
568 memory_listener_register(&cpu->cpu_ases[0].tcg_as_listener, as);
570 #endif
572 #ifndef CONFIG_USER_ONLY
573 static DECLARE_BITMAP(cpu_index_map, MAX_CPUMASK_BITS);
575 static int cpu_get_free_index(Error **errp)
577 int cpu = find_first_zero_bit(cpu_index_map, MAX_CPUMASK_BITS);
579 if (cpu >= MAX_CPUMASK_BITS) {
580 error_setg(errp, "Trying to use more CPUs than max of %d",
581 MAX_CPUMASK_BITS);
582 return -1;
585 bitmap_set(cpu_index_map, cpu, 1);
586 return cpu;
589 void cpu_exec_exit(CPUState *cpu)
591 if (cpu->cpu_index == -1) {
592 /* cpu_index was never allocated by this @cpu or was already freed. */
593 return;
596 bitmap_clear(cpu_index_map, cpu->cpu_index, 1);
597 cpu->cpu_index = -1;
599 #else
601 static int cpu_get_free_index(Error **errp)
603 CPUState *some_cpu;
604 int cpu_index = 0;
606 CPU_FOREACH(some_cpu) {
607 cpu_index++;
609 return cpu_index;
612 void cpu_exec_exit(CPUState *cpu)
615 #endif
617 void cpu_exec_init(CPUState *cpu, Error **errp)
619 CPUClass *cc = CPU_GET_CLASS(cpu);
620 int cpu_index;
621 Error *local_err = NULL;
623 #ifndef CONFIG_USER_ONLY
624 cpu->as = &address_space_memory;
625 cpu->thread_id = qemu_get_thread_id();
626 #endif
628 #if defined(CONFIG_USER_ONLY)
629 cpu_list_lock();
630 #endif
631 cpu_index = cpu->cpu_index = cpu_get_free_index(&local_err);
632 if (local_err) {
633 error_propagate(errp, local_err);
634 #if defined(CONFIG_USER_ONLY)
635 cpu_list_unlock();
636 #endif
637 return;
639 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
640 #if defined(CONFIG_USER_ONLY)
641 cpu_list_unlock();
642 #endif
643 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
644 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
646 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
647 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
648 cpu_save, cpu_load, cpu->env_ptr);
649 assert(cc->vmsd == NULL);
650 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
651 #endif
652 if (cc->vmsd != NULL) {
653 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
657 #if defined(CONFIG_USER_ONLY)
658 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
660 tb_invalidate_phys_page_range(pc, pc + 1, 0);
662 #else
663 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
665 hwaddr phys = cpu_get_phys_page_debug(cpu, pc);
666 if (phys != -1) {
667 tb_invalidate_phys_addr(cpu->as,
668 phys | (pc & ~TARGET_PAGE_MASK));
671 #endif
673 #if defined(CONFIG_USER_ONLY)
674 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
679 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
680 int flags)
682 return -ENOSYS;
685 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
689 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
690 int flags, CPUWatchpoint **watchpoint)
692 return -ENOSYS;
694 #else
695 /* Add a watchpoint. */
696 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
697 int flags, CPUWatchpoint **watchpoint)
699 CPUWatchpoint *wp;
701 /* forbid ranges which are empty or run off the end of the address space */
702 if (len == 0 || (addr + len - 1) < addr) {
703 error_report("tried to set invalid watchpoint at %"
704 VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
705 return -EINVAL;
707 wp = g_malloc(sizeof(*wp));
709 wp->vaddr = addr;
710 wp->len = len;
711 wp->flags = flags;
713 /* keep all GDB-injected watchpoints in front */
714 if (flags & BP_GDB) {
715 QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
716 } else {
717 QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
720 tlb_flush_page(cpu, addr);
722 if (watchpoint)
723 *watchpoint = wp;
724 return 0;
727 /* Remove a specific watchpoint. */
728 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
729 int flags)
731 CPUWatchpoint *wp;
733 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
734 if (addr == wp->vaddr && len == wp->len
735 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
736 cpu_watchpoint_remove_by_ref(cpu, wp);
737 return 0;
740 return -ENOENT;
743 /* Remove a specific watchpoint by reference. */
744 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
746 QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
748 tlb_flush_page(cpu, watchpoint->vaddr);
750 g_free(watchpoint);
753 /* Remove all matching watchpoints. */
754 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
756 CPUWatchpoint *wp, *next;
758 QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
759 if (wp->flags & mask) {
760 cpu_watchpoint_remove_by_ref(cpu, wp);
765 /* Return true if this watchpoint address matches the specified
766 * access (ie the address range covered by the watchpoint overlaps
767 * partially or completely with the address range covered by the
768 * access).
770 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
771 vaddr addr,
772 vaddr len)
774 /* We know the lengths are non-zero, but a little caution is
775 * required to avoid errors in the case where the range ends
776 * exactly at the top of the address space and so addr + len
777 * wraps round to zero.
779 vaddr wpend = wp->vaddr + wp->len - 1;
780 vaddr addrend = addr + len - 1;
782 return !(addr > wpend || wp->vaddr > addrend);
785 #endif
787 /* Add a breakpoint. */
788 int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
789 CPUBreakpoint **breakpoint)
791 CPUBreakpoint *bp;
793 bp = g_malloc(sizeof(*bp));
795 bp->pc = pc;
796 bp->flags = flags;
798 /* keep all GDB-injected breakpoints in front */
799 if (flags & BP_GDB) {
800 QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
801 } else {
802 QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
805 breakpoint_invalidate(cpu, pc);
807 if (breakpoint) {
808 *breakpoint = bp;
810 return 0;
813 /* Remove a specific breakpoint. */
814 int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
816 CPUBreakpoint *bp;
818 QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
819 if (bp->pc == pc && bp->flags == flags) {
820 cpu_breakpoint_remove_by_ref(cpu, bp);
821 return 0;
824 return -ENOENT;
827 /* Remove a specific breakpoint by reference. */
828 void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
830 QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
832 breakpoint_invalidate(cpu, breakpoint->pc);
834 g_free(breakpoint);
837 /* Remove all matching breakpoints. */
838 void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
840 CPUBreakpoint *bp, *next;
842 QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
843 if (bp->flags & mask) {
844 cpu_breakpoint_remove_by_ref(cpu, bp);
849 /* enable or disable single step mode. EXCP_DEBUG is returned by the
850 CPU loop after each instruction */
851 void cpu_single_step(CPUState *cpu, int enabled)
853 if (cpu->singlestep_enabled != enabled) {
854 cpu->singlestep_enabled = enabled;
855 if (kvm_enabled()) {
856 kvm_update_guest_debug(cpu, 0);
857 } else {
858 /* must flush all the translated code to avoid inconsistencies */
859 /* XXX: only flush what is necessary */
860 tb_flush(cpu);
865 void cpu_abort(CPUState *cpu, const char *fmt, ...)
867 va_list ap;
868 va_list ap2;
870 va_start(ap, fmt);
871 va_copy(ap2, ap);
872 fprintf(stderr, "qemu: fatal: ");
873 vfprintf(stderr, fmt, ap);
874 fprintf(stderr, "\n");
875 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
876 if (qemu_log_enabled()) {
877 qemu_log("qemu: fatal: ");
878 qemu_log_vprintf(fmt, ap2);
879 qemu_log("\n");
880 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
881 qemu_log_flush();
882 qemu_log_close();
884 va_end(ap2);
885 va_end(ap);
886 replay_finish();
887 #if defined(CONFIG_USER_ONLY)
889 struct sigaction act;
890 sigfillset(&act.sa_mask);
891 act.sa_handler = SIG_DFL;
892 sigaction(SIGABRT, &act, NULL);
894 #endif
895 abort();
898 #if !defined(CONFIG_USER_ONLY)
899 /* Called from RCU critical section */
900 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
902 RAMBlock *block;
904 block = atomic_rcu_read(&ram_list.mru_block);
905 if (block && addr - block->offset < block->max_length) {
906 goto found;
908 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
909 if (addr - block->offset < block->max_length) {
910 goto found;
914 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
915 abort();
917 found:
918 /* It is safe to write mru_block outside the iothread lock. This
919 * is what happens:
921 * mru_block = xxx
922 * rcu_read_unlock()
923 * xxx removed from list
924 * rcu_read_lock()
925 * read mru_block
926 * mru_block = NULL;
927 * call_rcu(reclaim_ramblock, xxx);
928 * rcu_read_unlock()
930 * atomic_rcu_set is not needed here. The block was already published
931 * when it was placed into the list. Here we're just making an extra
932 * copy of the pointer.
934 ram_list.mru_block = block;
935 return block;
938 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
940 CPUState *cpu;
941 ram_addr_t start1;
942 RAMBlock *block;
943 ram_addr_t end;
945 end = TARGET_PAGE_ALIGN(start + length);
946 start &= TARGET_PAGE_MASK;
948 rcu_read_lock();
949 block = qemu_get_ram_block(start);
950 assert(block == qemu_get_ram_block(end - 1));
951 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
952 CPU_FOREACH(cpu) {
953 tlb_reset_dirty(cpu, start1, length);
955 rcu_read_unlock();
958 /* Note: start and end must be within the same ram block. */
959 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
960 ram_addr_t length,
961 unsigned client)
963 unsigned long end, page;
964 bool dirty;
966 if (length == 0) {
967 return false;
970 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
971 page = start >> TARGET_PAGE_BITS;
972 dirty = bitmap_test_and_clear_atomic(ram_list.dirty_memory[client],
973 page, end - page);
975 if (dirty && tcg_enabled()) {
976 tlb_reset_dirty_range_all(start, length);
979 return dirty;
982 /* Called from RCU critical section */
983 hwaddr memory_region_section_get_iotlb(CPUState *cpu,
984 MemoryRegionSection *section,
985 target_ulong vaddr,
986 hwaddr paddr, hwaddr xlat,
987 int prot,
988 target_ulong *address)
990 hwaddr iotlb;
991 CPUWatchpoint *wp;
993 if (memory_region_is_ram(section->mr)) {
994 /* Normal RAM. */
995 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
996 + xlat;
997 if (!section->readonly) {
998 iotlb |= PHYS_SECTION_NOTDIRTY;
999 } else {
1000 iotlb |= PHYS_SECTION_ROM;
1002 } else {
1003 AddressSpaceDispatch *d;
1005 d = atomic_rcu_read(&section->address_space->dispatch);
1006 iotlb = section - d->map.sections;
1007 iotlb += xlat;
1010 /* Make accesses to pages with watchpoints go via the
1011 watchpoint trap routines. */
1012 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
1013 if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
1014 /* Avoid trapping reads of pages with a write breakpoint. */
1015 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
1016 iotlb = PHYS_SECTION_WATCH + paddr;
1017 *address |= TLB_MMIO;
1018 break;
1023 return iotlb;
1025 #endif /* defined(CONFIG_USER_ONLY) */
1027 #if !defined(CONFIG_USER_ONLY)
1029 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1030 uint16_t section);
1031 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
1033 static void *(*phys_mem_alloc)(size_t size, uint64_t *align) =
1034 qemu_anon_ram_alloc;
1037 * Set a custom physical guest memory alloator.
1038 * Accelerators with unusual needs may need this. Hopefully, we can
1039 * get rid of it eventually.
1041 void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align))
1043 phys_mem_alloc = alloc;
1046 static uint16_t phys_section_add(PhysPageMap *map,
1047 MemoryRegionSection *section)
1049 /* The physical section number is ORed with a page-aligned
1050 * pointer to produce the iotlb entries. Thus it should
1051 * never overflow into the page-aligned value.
1053 assert(map->sections_nb < TARGET_PAGE_SIZE);
1055 if (map->sections_nb == map->sections_nb_alloc) {
1056 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
1057 map->sections = g_renew(MemoryRegionSection, map->sections,
1058 map->sections_nb_alloc);
1060 map->sections[map->sections_nb] = *section;
1061 memory_region_ref(section->mr);
1062 return map->sections_nb++;
1065 static void phys_section_destroy(MemoryRegion *mr)
1067 memory_region_unref(mr);
1069 if (mr->subpage) {
1070 subpage_t *subpage = container_of(mr, subpage_t, iomem);
1071 object_unref(OBJECT(&subpage->iomem));
1072 g_free(subpage);
1076 static void phys_sections_free(PhysPageMap *map)
1078 while (map->sections_nb > 0) {
1079 MemoryRegionSection *section = &map->sections[--map->sections_nb];
1080 phys_section_destroy(section->mr);
1082 g_free(map->sections);
1083 g_free(map->nodes);
1086 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
1088 subpage_t *subpage;
1089 hwaddr base = section->offset_within_address_space
1090 & TARGET_PAGE_MASK;
1091 MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
1092 d->map.nodes, d->map.sections);
1093 MemoryRegionSection subsection = {
1094 .offset_within_address_space = base,
1095 .size = int128_make64(TARGET_PAGE_SIZE),
1097 hwaddr start, end;
1099 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
1101 if (!(existing->mr->subpage)) {
1102 subpage = subpage_init(d->as, base);
1103 subsection.address_space = d->as;
1104 subsection.mr = &subpage->iomem;
1105 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
1106 phys_section_add(&d->map, &subsection));
1107 } else {
1108 subpage = container_of(existing->mr, subpage_t, iomem);
1110 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
1111 end = start + int128_get64(section->size) - 1;
1112 subpage_register(subpage, start, end,
1113 phys_section_add(&d->map, section));
1117 static void register_multipage(AddressSpaceDispatch *d,
1118 MemoryRegionSection *section)
1120 hwaddr start_addr = section->offset_within_address_space;
1121 uint16_t section_index = phys_section_add(&d->map, section);
1122 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1123 TARGET_PAGE_BITS));
1125 assert(num_pages);
1126 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
1129 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
1131 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1132 AddressSpaceDispatch *d = as->next_dispatch;
1133 MemoryRegionSection now = *section, remain = *section;
1134 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
1136 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
1137 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
1138 - now.offset_within_address_space;
1140 now.size = int128_min(int128_make64(left), now.size);
1141 register_subpage(d, &now);
1142 } else {
1143 now.size = int128_zero();
1145 while (int128_ne(remain.size, now.size)) {
1146 remain.size = int128_sub(remain.size, now.size);
1147 remain.offset_within_address_space += int128_get64(now.size);
1148 remain.offset_within_region += int128_get64(now.size);
1149 now = remain;
1150 if (int128_lt(remain.size, page_size)) {
1151 register_subpage(d, &now);
1152 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
1153 now.size = page_size;
1154 register_subpage(d, &now);
1155 } else {
1156 now.size = int128_and(now.size, int128_neg(page_size));
1157 register_multipage(d, &now);
1162 void qemu_flush_coalesced_mmio_buffer(void)
1164 if (kvm_enabled())
1165 kvm_flush_coalesced_mmio_buffer();
1168 void qemu_mutex_lock_ramlist(void)
1170 qemu_mutex_lock(&ram_list.mutex);
1173 void qemu_mutex_unlock_ramlist(void)
1175 qemu_mutex_unlock(&ram_list.mutex);
1178 #ifdef __linux__
1180 #include <sys/vfs.h>
1182 #define HUGETLBFS_MAGIC 0x958458f6
1184 static long gethugepagesize(const char *path, Error **errp)
1186 struct statfs fs;
1187 int ret;
1189 do {
1190 ret = statfs(path, &fs);
1191 } while (ret != 0 && errno == EINTR);
1193 if (ret != 0) {
1194 error_setg_errno(errp, errno, "failed to get page size of file %s",
1195 path);
1196 return 0;
1199 if (fs.f_type != HUGETLBFS_MAGIC)
1200 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
1202 return fs.f_bsize;
1205 static void *file_ram_alloc(RAMBlock *block,
1206 ram_addr_t memory,
1207 const char *path,
1208 Error **errp)
1210 struct stat st;
1211 char *filename;
1212 char *sanitized_name;
1213 char *c;
1214 void *area;
1215 int fd;
1216 uint64_t hpagesize;
1217 Error *local_err = NULL;
1219 hpagesize = gethugepagesize(path, &local_err);
1220 if (local_err) {
1221 error_propagate(errp, local_err);
1222 goto error;
1224 block->mr->align = hpagesize;
1226 if (memory < hpagesize) {
1227 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
1228 "or larger than huge page size 0x%" PRIx64,
1229 memory, hpagesize);
1230 goto error;
1233 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1234 error_setg(errp,
1235 "host lacks kvm mmu notifiers, -mem-path unsupported");
1236 goto error;
1239 if (!stat(path, &st) && S_ISDIR(st.st_mode)) {
1240 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1241 sanitized_name = g_strdup(memory_region_name(block->mr));
1242 for (c = sanitized_name; *c != '\0'; c++) {
1243 if (*c == '/') {
1244 *c = '_';
1248 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1249 sanitized_name);
1250 g_free(sanitized_name);
1252 fd = mkstemp(filename);
1253 if (fd >= 0) {
1254 unlink(filename);
1256 g_free(filename);
1257 } else {
1258 fd = open(path, O_RDWR | O_CREAT, 0644);
1261 if (fd < 0) {
1262 error_setg_errno(errp, errno,
1263 "unable to create backing store for hugepages");
1264 goto error;
1267 memory = ROUND_UP(memory, hpagesize);
1270 * ftruncate is not supported by hugetlbfs in older
1271 * hosts, so don't bother bailing out on errors.
1272 * If anything goes wrong with it under other filesystems,
1273 * mmap will fail.
1275 if (ftruncate(fd, memory)) {
1276 perror("ftruncate");
1279 area = qemu_ram_mmap(fd, memory, hpagesize, block->flags & RAM_SHARED);
1280 if (area == MAP_FAILED) {
1281 error_setg_errno(errp, errno,
1282 "unable to map backing store for hugepages");
1283 close(fd);
1284 goto error;
1287 if (mem_prealloc) {
1288 os_mem_prealloc(fd, area, memory);
1291 block->fd = fd;
1292 return area;
1294 error:
1295 return NULL;
1297 #endif
1299 /* Called with the ramlist lock held. */
1300 static ram_addr_t find_ram_offset(ram_addr_t size)
1302 RAMBlock *block, *next_block;
1303 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1305 assert(size != 0); /* it would hand out same offset multiple times */
1307 if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
1308 return 0;
1311 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1312 ram_addr_t end, next = RAM_ADDR_MAX;
1314 end = block->offset + block->max_length;
1316 QLIST_FOREACH_RCU(next_block, &ram_list.blocks, next) {
1317 if (next_block->offset >= end) {
1318 next = MIN(next, next_block->offset);
1321 if (next - end >= size && next - end < mingap) {
1322 offset = end;
1323 mingap = next - end;
1327 if (offset == RAM_ADDR_MAX) {
1328 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1329 (uint64_t)size);
1330 abort();
1333 return offset;
1336 ram_addr_t last_ram_offset(void)
1338 RAMBlock *block;
1339 ram_addr_t last = 0;
1341 rcu_read_lock();
1342 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1343 last = MAX(last, block->offset + block->max_length);
1345 rcu_read_unlock();
1346 return last;
1349 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1351 int ret;
1353 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1354 if (!machine_dump_guest_core(current_machine)) {
1355 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1356 if (ret) {
1357 perror("qemu_madvise");
1358 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1359 "but dump_guest_core=off specified\n");
1364 /* Called within an RCU critical section, or while the ramlist lock
1365 * is held.
1367 static RAMBlock *find_ram_block(ram_addr_t addr)
1369 RAMBlock *block;
1371 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1372 if (block->offset == addr) {
1373 return block;
1377 return NULL;
1380 /* Called with iothread lock held. */
1381 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1383 RAMBlock *new_block, *block;
1385 rcu_read_lock();
1386 new_block = find_ram_block(addr);
1387 assert(new_block);
1388 assert(!new_block->idstr[0]);
1390 if (dev) {
1391 char *id = qdev_get_dev_path(dev);
1392 if (id) {
1393 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1394 g_free(id);
1397 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1399 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1400 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1401 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1402 new_block->idstr);
1403 abort();
1406 rcu_read_unlock();
1409 /* Called with iothread lock held. */
1410 void qemu_ram_unset_idstr(ram_addr_t addr)
1412 RAMBlock *block;
1414 /* FIXME: arch_init.c assumes that this is not called throughout
1415 * migration. Ignore the problem since hot-unplug during migration
1416 * does not work anyway.
1419 rcu_read_lock();
1420 block = find_ram_block(addr);
1421 if (block) {
1422 memset(block->idstr, 0, sizeof(block->idstr));
1424 rcu_read_unlock();
1427 static int memory_try_enable_merging(void *addr, size_t len)
1429 if (!machine_mem_merge(current_machine)) {
1430 /* disabled by the user */
1431 return 0;
1434 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1437 /* Only legal before guest might have detected the memory size: e.g. on
1438 * incoming migration, or right after reset.
1440 * As memory core doesn't know how is memory accessed, it is up to
1441 * resize callback to update device state and/or add assertions to detect
1442 * misuse, if necessary.
1444 int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp)
1446 RAMBlock *block = find_ram_block(base);
1448 assert(block);
1450 newsize = TARGET_PAGE_ALIGN(newsize);
1452 if (block->used_length == newsize) {
1453 return 0;
1456 if (!(block->flags & RAM_RESIZEABLE)) {
1457 error_setg_errno(errp, EINVAL,
1458 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1459 " in != 0x" RAM_ADDR_FMT, block->idstr,
1460 newsize, block->used_length);
1461 return -EINVAL;
1464 if (block->max_length < newsize) {
1465 error_setg_errno(errp, EINVAL,
1466 "Length too large: %s: 0x" RAM_ADDR_FMT
1467 " > 0x" RAM_ADDR_FMT, block->idstr,
1468 newsize, block->max_length);
1469 return -EINVAL;
1472 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
1473 block->used_length = newsize;
1474 cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
1475 DIRTY_CLIENTS_ALL);
1476 memory_region_set_size(block->mr, newsize);
1477 if (block->resized) {
1478 block->resized(block->idstr, newsize, block->host);
1480 return 0;
1483 static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
1485 RAMBlock *block;
1486 RAMBlock *last_block = NULL;
1487 ram_addr_t old_ram_size, new_ram_size;
1489 old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1491 qemu_mutex_lock_ramlist();
1492 new_block->offset = find_ram_offset(new_block->max_length);
1494 if (!new_block->host) {
1495 if (xen_enabled()) {
1496 xen_ram_alloc(new_block->offset, new_block->max_length,
1497 new_block->mr);
1498 } else {
1499 new_block->host = phys_mem_alloc(new_block->max_length,
1500 &new_block->mr->align);
1501 if (!new_block->host) {
1502 error_setg_errno(errp, errno,
1503 "cannot set up guest memory '%s'",
1504 memory_region_name(new_block->mr));
1505 qemu_mutex_unlock_ramlist();
1506 return -1;
1508 memory_try_enable_merging(new_block->host, new_block->max_length);
1512 new_ram_size = MAX(old_ram_size,
1513 (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
1514 if (new_ram_size > old_ram_size) {
1515 migration_bitmap_extend(old_ram_size, new_ram_size);
1517 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1518 * QLIST (which has an RCU-friendly variant) does not have insertion at
1519 * tail, so save the last element in last_block.
1521 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1522 last_block = block;
1523 if (block->max_length < new_block->max_length) {
1524 break;
1527 if (block) {
1528 QLIST_INSERT_BEFORE_RCU(block, new_block, next);
1529 } else if (last_block) {
1530 QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
1531 } else { /* list is empty */
1532 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
1534 ram_list.mru_block = NULL;
1536 /* Write list before version */
1537 smp_wmb();
1538 ram_list.version++;
1539 qemu_mutex_unlock_ramlist();
1541 new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1543 if (new_ram_size > old_ram_size) {
1544 int i;
1546 /* ram_list.dirty_memory[] is protected by the iothread lock. */
1547 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1548 ram_list.dirty_memory[i] =
1549 bitmap_zero_extend(ram_list.dirty_memory[i],
1550 old_ram_size, new_ram_size);
1553 cpu_physical_memory_set_dirty_range(new_block->offset,
1554 new_block->used_length,
1555 DIRTY_CLIENTS_ALL);
1557 if (new_block->host) {
1558 qemu_ram_setup_dump(new_block->host, new_block->max_length);
1559 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
1560 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
1561 if (kvm_enabled()) {
1562 kvm_setup_guest_memory(new_block->host, new_block->max_length);
1566 return new_block->offset;
1569 #ifdef __linux__
1570 ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
1571 bool share, const char *mem_path,
1572 Error **errp)
1574 RAMBlock *new_block;
1575 ram_addr_t addr;
1576 Error *local_err = NULL;
1578 if (xen_enabled()) {
1579 error_setg(errp, "-mem-path not supported with Xen");
1580 return -1;
1583 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1585 * file_ram_alloc() needs to allocate just like
1586 * phys_mem_alloc, but we haven't bothered to provide
1587 * a hook there.
1589 error_setg(errp,
1590 "-mem-path not supported with this accelerator");
1591 return -1;
1594 size = TARGET_PAGE_ALIGN(size);
1595 new_block = g_malloc0(sizeof(*new_block));
1596 new_block->mr = mr;
1597 new_block->used_length = size;
1598 new_block->max_length = size;
1599 new_block->flags = share ? RAM_SHARED : 0;
1600 new_block->flags |= RAM_FILE;
1601 new_block->host = file_ram_alloc(new_block, size,
1602 mem_path, errp);
1603 if (!new_block->host) {
1604 g_free(new_block);
1605 return -1;
1608 addr = ram_block_add(new_block, &local_err);
1609 if (local_err) {
1610 g_free(new_block);
1611 error_propagate(errp, local_err);
1612 return -1;
1614 return addr;
1616 #endif
1618 static
1619 ram_addr_t qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
1620 void (*resized)(const char*,
1621 uint64_t length,
1622 void *host),
1623 void *host, bool resizeable,
1624 MemoryRegion *mr, Error **errp)
1626 RAMBlock *new_block;
1627 ram_addr_t addr;
1628 Error *local_err = NULL;
1630 size = TARGET_PAGE_ALIGN(size);
1631 max_size = TARGET_PAGE_ALIGN(max_size);
1632 new_block = g_malloc0(sizeof(*new_block));
1633 new_block->mr = mr;
1634 new_block->resized = resized;
1635 new_block->used_length = size;
1636 new_block->max_length = max_size;
1637 assert(max_size >= size);
1638 new_block->fd = -1;
1639 new_block->host = host;
1640 if (host) {
1641 new_block->flags |= RAM_PREALLOC;
1643 if (resizeable) {
1644 new_block->flags |= RAM_RESIZEABLE;
1646 addr = ram_block_add(new_block, &local_err);
1647 if (local_err) {
1648 g_free(new_block);
1649 error_propagate(errp, local_err);
1650 return -1;
1652 return addr;
1655 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1656 MemoryRegion *mr, Error **errp)
1658 return qemu_ram_alloc_internal(size, size, NULL, host, false, mr, errp);
1661 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
1663 return qemu_ram_alloc_internal(size, size, NULL, NULL, false, mr, errp);
1666 ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
1667 void (*resized)(const char*,
1668 uint64_t length,
1669 void *host),
1670 MemoryRegion *mr, Error **errp)
1672 return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true, mr, errp);
1675 void qemu_ram_free_from_ptr(ram_addr_t addr)
1677 RAMBlock *block;
1679 qemu_mutex_lock_ramlist();
1680 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1681 if (addr == block->offset) {
1682 QLIST_REMOVE_RCU(block, next);
1683 ram_list.mru_block = NULL;
1684 /* Write list before version */
1685 smp_wmb();
1686 ram_list.version++;
1687 g_free_rcu(block, rcu);
1688 break;
1691 qemu_mutex_unlock_ramlist();
1694 static void reclaim_ramblock(RAMBlock *block)
1696 if (block->flags & RAM_PREALLOC) {
1698 } else if (xen_enabled()) {
1699 xen_invalidate_map_cache_entry(block->host);
1700 #ifndef _WIN32
1701 } else if (block->fd >= 0) {
1702 if (block->flags & RAM_FILE) {
1703 qemu_ram_munmap(block->host, block->max_length);
1704 } else {
1705 munmap(block->host, block->max_length);
1707 close(block->fd);
1708 #endif
1709 } else {
1710 qemu_anon_ram_free(block->host, block->max_length);
1712 g_free(block);
1715 void qemu_ram_free(ram_addr_t addr)
1717 RAMBlock *block;
1719 qemu_mutex_lock_ramlist();
1720 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1721 if (addr == block->offset) {
1722 QLIST_REMOVE_RCU(block, next);
1723 ram_list.mru_block = NULL;
1724 /* Write list before version */
1725 smp_wmb();
1726 ram_list.version++;
1727 call_rcu(block, reclaim_ramblock, rcu);
1728 break;
1731 qemu_mutex_unlock_ramlist();
1734 #ifndef _WIN32
1735 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1737 RAMBlock *block;
1738 ram_addr_t offset;
1739 int flags;
1740 void *area, *vaddr;
1742 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1743 offset = addr - block->offset;
1744 if (offset < block->max_length) {
1745 vaddr = ramblock_ptr(block, offset);
1746 if (block->flags & RAM_PREALLOC) {
1748 } else if (xen_enabled()) {
1749 abort();
1750 } else {
1751 flags = MAP_FIXED;
1752 if (block->fd >= 0) {
1753 flags |= (block->flags & RAM_SHARED ?
1754 MAP_SHARED : MAP_PRIVATE);
1755 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1756 flags, block->fd, offset);
1757 } else {
1759 * Remap needs to match alloc. Accelerators that
1760 * set phys_mem_alloc never remap. If they did,
1761 * we'd need a remap hook here.
1763 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1765 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1766 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1767 flags, -1, 0);
1769 if (area != vaddr) {
1770 fprintf(stderr, "Could not remap addr: "
1771 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1772 length, addr);
1773 exit(1);
1775 memory_try_enable_merging(vaddr, length);
1776 qemu_ram_setup_dump(vaddr, length);
1781 #endif /* !_WIN32 */
1783 int qemu_get_ram_fd(ram_addr_t addr)
1785 RAMBlock *block;
1786 int fd;
1788 rcu_read_lock();
1789 block = qemu_get_ram_block(addr);
1790 fd = block->fd;
1791 rcu_read_unlock();
1792 return fd;
1795 void *qemu_get_ram_block_host_ptr(ram_addr_t addr)
1797 RAMBlock *block;
1798 void *ptr;
1800 rcu_read_lock();
1801 block = qemu_get_ram_block(addr);
1802 ptr = ramblock_ptr(block, 0);
1803 rcu_read_unlock();
1804 return ptr;
1807 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1808 * This should not be used for general purpose DMA. Use address_space_map
1809 * or address_space_rw instead. For local memory (e.g. video ram) that the
1810 * device owns, use memory_region_get_ram_ptr.
1812 * By the time this function returns, the returned pointer is not protected
1813 * by RCU anymore. If the caller is not within an RCU critical section and
1814 * does not hold the iothread lock, it must have other means of protecting the
1815 * pointer, such as a reference to the region that includes the incoming
1816 * ram_addr_t.
1818 void *qemu_get_ram_ptr(ram_addr_t addr)
1820 RAMBlock *block;
1821 void *ptr;
1823 rcu_read_lock();
1824 block = qemu_get_ram_block(addr);
1826 if (xen_enabled() && block->host == NULL) {
1827 /* We need to check if the requested address is in the RAM
1828 * because we don't want to map the entire memory in QEMU.
1829 * In that case just map until the end of the page.
1831 if (block->offset == 0) {
1832 ptr = xen_map_cache(addr, 0, 0);
1833 goto unlock;
1836 block->host = xen_map_cache(block->offset, block->max_length, 1);
1838 ptr = ramblock_ptr(block, addr - block->offset);
1840 unlock:
1841 rcu_read_unlock();
1842 return ptr;
1845 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1846 * but takes a size argument.
1848 * By the time this function returns, the returned pointer is not protected
1849 * by RCU anymore. If the caller is not within an RCU critical section and
1850 * does not hold the iothread lock, it must have other means of protecting the
1851 * pointer, such as a reference to the region that includes the incoming
1852 * ram_addr_t.
1854 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1856 void *ptr;
1857 if (*size == 0) {
1858 return NULL;
1860 if (xen_enabled()) {
1861 return xen_map_cache(addr, *size, 1);
1862 } else {
1863 RAMBlock *block;
1864 rcu_read_lock();
1865 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1866 if (addr - block->offset < block->max_length) {
1867 if (addr - block->offset + *size > block->max_length)
1868 *size = block->max_length - addr + block->offset;
1869 ptr = ramblock_ptr(block, addr - block->offset);
1870 rcu_read_unlock();
1871 return ptr;
1875 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1876 abort();
1880 /* Some of the softmmu routines need to translate from a host pointer
1881 * (typically a TLB entry) back to a ram offset.
1883 * By the time this function returns, the returned pointer is not protected
1884 * by RCU anymore. If the caller is not within an RCU critical section and
1885 * does not hold the iothread lock, it must have other means of protecting the
1886 * pointer, such as a reference to the region that includes the incoming
1887 * ram_addr_t.
1889 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1891 RAMBlock *block;
1892 uint8_t *host = ptr;
1893 MemoryRegion *mr;
1895 if (xen_enabled()) {
1896 rcu_read_lock();
1897 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1898 mr = qemu_get_ram_block(*ram_addr)->mr;
1899 rcu_read_unlock();
1900 return mr;
1903 rcu_read_lock();
1904 block = atomic_rcu_read(&ram_list.mru_block);
1905 if (block && block->host && host - block->host < block->max_length) {
1906 goto found;
1909 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1910 /* This case append when the block is not mapped. */
1911 if (block->host == NULL) {
1912 continue;
1914 if (host - block->host < block->max_length) {
1915 goto found;
1919 rcu_read_unlock();
1920 return NULL;
1922 found:
1923 *ram_addr = block->offset + (host - block->host);
1924 mr = block->mr;
1925 rcu_read_unlock();
1926 return mr;
1929 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1930 uint64_t val, unsigned size)
1932 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
1933 tb_invalidate_phys_page_fast(ram_addr, size);
1935 switch (size) {
1936 case 1:
1937 stb_p(qemu_get_ram_ptr(ram_addr), val);
1938 break;
1939 case 2:
1940 stw_p(qemu_get_ram_ptr(ram_addr), val);
1941 break;
1942 case 4:
1943 stl_p(qemu_get_ram_ptr(ram_addr), val);
1944 break;
1945 default:
1946 abort();
1948 /* Set both VGA and migration bits for simplicity and to remove
1949 * the notdirty callback faster.
1951 cpu_physical_memory_set_dirty_range(ram_addr, size,
1952 DIRTY_CLIENTS_NOCODE);
1953 /* we remove the notdirty callback only if the code has been
1954 flushed */
1955 if (!cpu_physical_memory_is_clean(ram_addr)) {
1956 tlb_set_dirty(current_cpu, current_cpu->mem_io_vaddr);
1960 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1961 unsigned size, bool is_write)
1963 return is_write;
1966 static const MemoryRegionOps notdirty_mem_ops = {
1967 .write = notdirty_mem_write,
1968 .valid.accepts = notdirty_mem_accepts,
1969 .endianness = DEVICE_NATIVE_ENDIAN,
1972 /* Generate a debug exception if a watchpoint has been hit. */
1973 static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
1975 CPUState *cpu = current_cpu;
1976 CPUArchState *env = cpu->env_ptr;
1977 target_ulong pc, cs_base;
1978 target_ulong vaddr;
1979 CPUWatchpoint *wp;
1980 int cpu_flags;
1982 if (cpu->watchpoint_hit) {
1983 /* We re-entered the check after replacing the TB. Now raise
1984 * the debug interrupt so that is will trigger after the
1985 * current instruction. */
1986 cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
1987 return;
1989 vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1990 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
1991 if (cpu_watchpoint_address_matches(wp, vaddr, len)
1992 && (wp->flags & flags)) {
1993 if (flags == BP_MEM_READ) {
1994 wp->flags |= BP_WATCHPOINT_HIT_READ;
1995 } else {
1996 wp->flags |= BP_WATCHPOINT_HIT_WRITE;
1998 wp->hitaddr = vaddr;
1999 wp->hitattrs = attrs;
2000 if (!cpu->watchpoint_hit) {
2001 cpu->watchpoint_hit = wp;
2002 tb_check_watchpoint(cpu);
2003 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2004 cpu->exception_index = EXCP_DEBUG;
2005 cpu_loop_exit(cpu);
2006 } else {
2007 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2008 tb_gen_code(cpu, pc, cs_base, cpu_flags, 1);
2009 cpu_resume_from_signal(cpu, NULL);
2012 } else {
2013 wp->flags &= ~BP_WATCHPOINT_HIT;
2018 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2019 so these check for a hit then pass through to the normal out-of-line
2020 phys routines. */
2021 static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
2022 unsigned size, MemTxAttrs attrs)
2024 MemTxResult res;
2025 uint64_t data;
2027 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
2028 switch (size) {
2029 case 1:
2030 data = address_space_ldub(&address_space_memory, addr, attrs, &res);
2031 break;
2032 case 2:
2033 data = address_space_lduw(&address_space_memory, addr, attrs, &res);
2034 break;
2035 case 4:
2036 data = address_space_ldl(&address_space_memory, addr, attrs, &res);
2037 break;
2038 default: abort();
2040 *pdata = data;
2041 return res;
2044 static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
2045 uint64_t val, unsigned size,
2046 MemTxAttrs attrs)
2048 MemTxResult res;
2050 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
2051 switch (size) {
2052 case 1:
2053 address_space_stb(&address_space_memory, addr, val, attrs, &res);
2054 break;
2055 case 2:
2056 address_space_stw(&address_space_memory, addr, val, attrs, &res);
2057 break;
2058 case 4:
2059 address_space_stl(&address_space_memory, addr, val, attrs, &res);
2060 break;
2061 default: abort();
2063 return res;
2066 static const MemoryRegionOps watch_mem_ops = {
2067 .read_with_attrs = watch_mem_read,
2068 .write_with_attrs = watch_mem_write,
2069 .endianness = DEVICE_NATIVE_ENDIAN,
2072 static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
2073 unsigned len, MemTxAttrs attrs)
2075 subpage_t *subpage = opaque;
2076 uint8_t buf[8];
2077 MemTxResult res;
2079 #if defined(DEBUG_SUBPAGE)
2080 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
2081 subpage, len, addr);
2082 #endif
2083 res = address_space_read(subpage->as, addr + subpage->base,
2084 attrs, buf, len);
2085 if (res) {
2086 return res;
2088 switch (len) {
2089 case 1:
2090 *data = ldub_p(buf);
2091 return MEMTX_OK;
2092 case 2:
2093 *data = lduw_p(buf);
2094 return MEMTX_OK;
2095 case 4:
2096 *data = ldl_p(buf);
2097 return MEMTX_OK;
2098 case 8:
2099 *data = ldq_p(buf);
2100 return MEMTX_OK;
2101 default:
2102 abort();
2106 static MemTxResult subpage_write(void *opaque, hwaddr addr,
2107 uint64_t value, unsigned len, MemTxAttrs attrs)
2109 subpage_t *subpage = opaque;
2110 uint8_t buf[8];
2112 #if defined(DEBUG_SUBPAGE)
2113 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
2114 " value %"PRIx64"\n",
2115 __func__, subpage, len, addr, value);
2116 #endif
2117 switch (len) {
2118 case 1:
2119 stb_p(buf, value);
2120 break;
2121 case 2:
2122 stw_p(buf, value);
2123 break;
2124 case 4:
2125 stl_p(buf, value);
2126 break;
2127 case 8:
2128 stq_p(buf, value);
2129 break;
2130 default:
2131 abort();
2133 return address_space_write(subpage->as, addr + subpage->base,
2134 attrs, buf, len);
2137 static bool subpage_accepts(void *opaque, hwaddr addr,
2138 unsigned len, bool is_write)
2140 subpage_t *subpage = opaque;
2141 #if defined(DEBUG_SUBPAGE)
2142 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
2143 __func__, subpage, is_write ? 'w' : 'r', len, addr);
2144 #endif
2146 return address_space_access_valid(subpage->as, addr + subpage->base,
2147 len, is_write);
2150 static const MemoryRegionOps subpage_ops = {
2151 .read_with_attrs = subpage_read,
2152 .write_with_attrs = subpage_write,
2153 .impl.min_access_size = 1,
2154 .impl.max_access_size = 8,
2155 .valid.min_access_size = 1,
2156 .valid.max_access_size = 8,
2157 .valid.accepts = subpage_accepts,
2158 .endianness = DEVICE_NATIVE_ENDIAN,
2161 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2162 uint16_t section)
2164 int idx, eidx;
2166 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2167 return -1;
2168 idx = SUBPAGE_IDX(start);
2169 eidx = SUBPAGE_IDX(end);
2170 #if defined(DEBUG_SUBPAGE)
2171 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2172 __func__, mmio, start, end, idx, eidx, section);
2173 #endif
2174 for (; idx <= eidx; idx++) {
2175 mmio->sub_section[idx] = section;
2178 return 0;
2181 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
2183 subpage_t *mmio;
2185 mmio = g_malloc0(sizeof(subpage_t));
2187 mmio->as = as;
2188 mmio->base = base;
2189 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
2190 NULL, TARGET_PAGE_SIZE);
2191 mmio->iomem.subpage = true;
2192 #if defined(DEBUG_SUBPAGE)
2193 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
2194 mmio, base, TARGET_PAGE_SIZE);
2195 #endif
2196 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
2198 return mmio;
2201 static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
2202 MemoryRegion *mr)
2204 assert(as);
2205 MemoryRegionSection section = {
2206 .address_space = as,
2207 .mr = mr,
2208 .offset_within_address_space = 0,
2209 .offset_within_region = 0,
2210 .size = int128_2_64(),
2213 return phys_section_add(map, &section);
2216 MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index)
2218 CPUAddressSpace *cpuas = &cpu->cpu_ases[0];
2219 AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
2220 MemoryRegionSection *sections = d->map.sections;
2222 return sections[index & ~TARGET_PAGE_MASK].mr;
2225 static void io_mem_init(void)
2227 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, NULL, UINT64_MAX);
2228 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
2229 NULL, UINT64_MAX);
2230 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
2231 NULL, UINT64_MAX);
2232 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
2233 NULL, UINT64_MAX);
2236 static void mem_begin(MemoryListener *listener)
2238 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2239 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
2240 uint16_t n;
2242 n = dummy_section(&d->map, as, &io_mem_unassigned);
2243 assert(n == PHYS_SECTION_UNASSIGNED);
2244 n = dummy_section(&d->map, as, &io_mem_notdirty);
2245 assert(n == PHYS_SECTION_NOTDIRTY);
2246 n = dummy_section(&d->map, as, &io_mem_rom);
2247 assert(n == PHYS_SECTION_ROM);
2248 n = dummy_section(&d->map, as, &io_mem_watch);
2249 assert(n == PHYS_SECTION_WATCH);
2251 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
2252 d->as = as;
2253 as->next_dispatch = d;
2256 static void address_space_dispatch_free(AddressSpaceDispatch *d)
2258 phys_sections_free(&d->map);
2259 g_free(d);
2262 static void mem_commit(MemoryListener *listener)
2264 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2265 AddressSpaceDispatch *cur = as->dispatch;
2266 AddressSpaceDispatch *next = as->next_dispatch;
2268 phys_page_compact_all(next, next->map.nodes_nb);
2270 atomic_rcu_set(&as->dispatch, next);
2271 if (cur) {
2272 call_rcu(cur, address_space_dispatch_free, rcu);
2276 static void tcg_commit(MemoryListener *listener)
2278 CPUAddressSpace *cpuas;
2279 AddressSpaceDispatch *d;
2281 /* since each CPU stores ram addresses in its TLB cache, we must
2282 reset the modified entries */
2283 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
2284 cpu_reloading_memory_map();
2285 /* The CPU and TLB are protected by the iothread lock.
2286 * We reload the dispatch pointer now because cpu_reloading_memory_map()
2287 * may have split the RCU critical section.
2289 d = atomic_rcu_read(&cpuas->as->dispatch);
2290 cpuas->memory_dispatch = d;
2291 tlb_flush(cpuas->cpu, 1);
2294 void address_space_init_dispatch(AddressSpace *as)
2296 as->dispatch = NULL;
2297 as->dispatch_listener = (MemoryListener) {
2298 .begin = mem_begin,
2299 .commit = mem_commit,
2300 .region_add = mem_add,
2301 .region_nop = mem_add,
2302 .priority = 0,
2304 memory_listener_register(&as->dispatch_listener, as);
2307 void address_space_unregister(AddressSpace *as)
2309 memory_listener_unregister(&as->dispatch_listener);
2312 void address_space_destroy_dispatch(AddressSpace *as)
2314 AddressSpaceDispatch *d = as->dispatch;
2316 atomic_rcu_set(&as->dispatch, NULL);
2317 if (d) {
2318 call_rcu(d, address_space_dispatch_free, rcu);
2322 static void memory_map_init(void)
2324 system_memory = g_malloc(sizeof(*system_memory));
2326 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
2327 address_space_init(&address_space_memory, system_memory, "memory");
2329 system_io = g_malloc(sizeof(*system_io));
2330 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2331 65536);
2332 address_space_init(&address_space_io, system_io, "I/O");
2335 MemoryRegion *get_system_memory(void)
2337 return system_memory;
2340 MemoryRegion *get_system_io(void)
2342 return system_io;
2345 #endif /* !defined(CONFIG_USER_ONLY) */
2347 /* physical memory access (slow version, mainly for debug) */
2348 #if defined(CONFIG_USER_ONLY)
2349 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2350 uint8_t *buf, int len, int is_write)
2352 int l, flags;
2353 target_ulong page;
2354 void * p;
2356 while (len > 0) {
2357 page = addr & TARGET_PAGE_MASK;
2358 l = (page + TARGET_PAGE_SIZE) - addr;
2359 if (l > len)
2360 l = len;
2361 flags = page_get_flags(page);
2362 if (!(flags & PAGE_VALID))
2363 return -1;
2364 if (is_write) {
2365 if (!(flags & PAGE_WRITE))
2366 return -1;
2367 /* XXX: this code should not depend on lock_user */
2368 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2369 return -1;
2370 memcpy(p, buf, l);
2371 unlock_user(p, addr, l);
2372 } else {
2373 if (!(flags & PAGE_READ))
2374 return -1;
2375 /* XXX: this code should not depend on lock_user */
2376 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2377 return -1;
2378 memcpy(buf, p, l);
2379 unlock_user(p, addr, 0);
2381 len -= l;
2382 buf += l;
2383 addr += l;
2385 return 0;
2388 #else
2390 static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
2391 hwaddr length)
2393 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
2394 /* No early return if dirty_log_mask is or becomes 0, because
2395 * cpu_physical_memory_set_dirty_range will still call
2396 * xen_modified_memory.
2398 if (dirty_log_mask) {
2399 dirty_log_mask =
2400 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
2402 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
2403 tb_invalidate_phys_range(addr, addr + length);
2404 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
2406 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
2409 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
2411 unsigned access_size_max = mr->ops->valid.max_access_size;
2413 /* Regions are assumed to support 1-4 byte accesses unless
2414 otherwise specified. */
2415 if (access_size_max == 0) {
2416 access_size_max = 4;
2419 /* Bound the maximum access by the alignment of the address. */
2420 if (!mr->ops->impl.unaligned) {
2421 unsigned align_size_max = addr & -addr;
2422 if (align_size_max != 0 && align_size_max < access_size_max) {
2423 access_size_max = align_size_max;
2427 /* Don't attempt accesses larger than the maximum. */
2428 if (l > access_size_max) {
2429 l = access_size_max;
2431 l = pow2floor(l);
2433 return l;
2436 static bool prepare_mmio_access(MemoryRegion *mr)
2438 bool unlocked = !qemu_mutex_iothread_locked();
2439 bool release_lock = false;
2441 if (unlocked && mr->global_locking) {
2442 qemu_mutex_lock_iothread();
2443 unlocked = false;
2444 release_lock = true;
2446 if (mr->flush_coalesced_mmio) {
2447 if (unlocked) {
2448 qemu_mutex_lock_iothread();
2450 qemu_flush_coalesced_mmio_buffer();
2451 if (unlocked) {
2452 qemu_mutex_unlock_iothread();
2456 return release_lock;
2459 MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2460 uint8_t *buf, int len, bool is_write)
2462 hwaddr l;
2463 uint8_t *ptr;
2464 uint64_t val;
2465 hwaddr addr1;
2466 MemoryRegion *mr;
2467 MemTxResult result = MEMTX_OK;
2468 bool release_lock = false;
2470 rcu_read_lock();
2471 while (len > 0) {
2472 l = len;
2473 mr = address_space_translate(as, addr, &addr1, &l, is_write);
2475 if (is_write) {
2476 if (!memory_access_is_direct(mr, is_write)) {
2477 release_lock |= prepare_mmio_access(mr);
2478 l = memory_access_size(mr, l, addr1);
2479 /* XXX: could force current_cpu to NULL to avoid
2480 potential bugs */
2481 switch (l) {
2482 case 8:
2483 /* 64 bit write access */
2484 val = ldq_p(buf);
2485 result |= memory_region_dispatch_write(mr, addr1, val, 8,
2486 attrs);
2487 break;
2488 case 4:
2489 /* 32 bit write access */
2490 val = ldl_p(buf);
2491 result |= memory_region_dispatch_write(mr, addr1, val, 4,
2492 attrs);
2493 break;
2494 case 2:
2495 /* 16 bit write access */
2496 val = lduw_p(buf);
2497 result |= memory_region_dispatch_write(mr, addr1, val, 2,
2498 attrs);
2499 break;
2500 case 1:
2501 /* 8 bit write access */
2502 val = ldub_p(buf);
2503 result |= memory_region_dispatch_write(mr, addr1, val, 1,
2504 attrs);
2505 break;
2506 default:
2507 abort();
2509 } else {
2510 addr1 += memory_region_get_ram_addr(mr);
2511 /* RAM case */
2512 ptr = qemu_get_ram_ptr(addr1);
2513 memcpy(ptr, buf, l);
2514 invalidate_and_set_dirty(mr, addr1, l);
2516 } else {
2517 if (!memory_access_is_direct(mr, is_write)) {
2518 /* I/O case */
2519 release_lock |= prepare_mmio_access(mr);
2520 l = memory_access_size(mr, l, addr1);
2521 switch (l) {
2522 case 8:
2523 /* 64 bit read access */
2524 result |= memory_region_dispatch_read(mr, addr1, &val, 8,
2525 attrs);
2526 stq_p(buf, val);
2527 break;
2528 case 4:
2529 /* 32 bit read access */
2530 result |= memory_region_dispatch_read(mr, addr1, &val, 4,
2531 attrs);
2532 stl_p(buf, val);
2533 break;
2534 case 2:
2535 /* 16 bit read access */
2536 result |= memory_region_dispatch_read(mr, addr1, &val, 2,
2537 attrs);
2538 stw_p(buf, val);
2539 break;
2540 case 1:
2541 /* 8 bit read access */
2542 result |= memory_region_dispatch_read(mr, addr1, &val, 1,
2543 attrs);
2544 stb_p(buf, val);
2545 break;
2546 default:
2547 abort();
2549 } else {
2550 /* RAM case */
2551 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2552 memcpy(buf, ptr, l);
2556 if (release_lock) {
2557 qemu_mutex_unlock_iothread();
2558 release_lock = false;
2561 len -= l;
2562 buf += l;
2563 addr += l;
2565 rcu_read_unlock();
2567 return result;
2570 MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2571 const uint8_t *buf, int len)
2573 return address_space_rw(as, addr, attrs, (uint8_t *)buf, len, true);
2576 MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2577 uint8_t *buf, int len)
2579 return address_space_rw(as, addr, attrs, buf, len, false);
2583 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2584 int len, int is_write)
2586 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
2587 buf, len, is_write);
2590 enum write_rom_type {
2591 WRITE_DATA,
2592 FLUSH_CACHE,
2595 static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
2596 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
2598 hwaddr l;
2599 uint8_t *ptr;
2600 hwaddr addr1;
2601 MemoryRegion *mr;
2603 rcu_read_lock();
2604 while (len > 0) {
2605 l = len;
2606 mr = address_space_translate(as, addr, &addr1, &l, true);
2608 if (!(memory_region_is_ram(mr) ||
2609 memory_region_is_romd(mr))) {
2610 l = memory_access_size(mr, l, addr1);
2611 } else {
2612 addr1 += memory_region_get_ram_addr(mr);
2613 /* ROM/RAM case */
2614 ptr = qemu_get_ram_ptr(addr1);
2615 switch (type) {
2616 case WRITE_DATA:
2617 memcpy(ptr, buf, l);
2618 invalidate_and_set_dirty(mr, addr1, l);
2619 break;
2620 case FLUSH_CACHE:
2621 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
2622 break;
2625 len -= l;
2626 buf += l;
2627 addr += l;
2629 rcu_read_unlock();
2632 /* used for ROM loading : can write in RAM and ROM */
2633 void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
2634 const uint8_t *buf, int len)
2636 cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
2639 void cpu_flush_icache_range(hwaddr start, int len)
2642 * This function should do the same thing as an icache flush that was
2643 * triggered from within the guest. For TCG we are always cache coherent,
2644 * so there is no need to flush anything. For KVM / Xen we need to flush
2645 * the host's instruction cache at least.
2647 if (tcg_enabled()) {
2648 return;
2651 cpu_physical_memory_write_rom_internal(&address_space_memory,
2652 start, NULL, len, FLUSH_CACHE);
2655 typedef struct {
2656 MemoryRegion *mr;
2657 void *buffer;
2658 hwaddr addr;
2659 hwaddr len;
2660 bool in_use;
2661 } BounceBuffer;
2663 static BounceBuffer bounce;
2665 typedef struct MapClient {
2666 QEMUBH *bh;
2667 QLIST_ENTRY(MapClient) link;
2668 } MapClient;
2670 QemuMutex map_client_list_lock;
2671 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2672 = QLIST_HEAD_INITIALIZER(map_client_list);
2674 static void cpu_unregister_map_client_do(MapClient *client)
2676 QLIST_REMOVE(client, link);
2677 g_free(client);
2680 static void cpu_notify_map_clients_locked(void)
2682 MapClient *client;
2684 while (!QLIST_EMPTY(&map_client_list)) {
2685 client = QLIST_FIRST(&map_client_list);
2686 qemu_bh_schedule(client->bh);
2687 cpu_unregister_map_client_do(client);
2691 void cpu_register_map_client(QEMUBH *bh)
2693 MapClient *client = g_malloc(sizeof(*client));
2695 qemu_mutex_lock(&map_client_list_lock);
2696 client->bh = bh;
2697 QLIST_INSERT_HEAD(&map_client_list, client, link);
2698 if (!atomic_read(&bounce.in_use)) {
2699 cpu_notify_map_clients_locked();
2701 qemu_mutex_unlock(&map_client_list_lock);
2704 void cpu_exec_init_all(void)
2706 qemu_mutex_init(&ram_list.mutex);
2707 io_mem_init();
2708 memory_map_init();
2709 qemu_mutex_init(&map_client_list_lock);
2712 void cpu_unregister_map_client(QEMUBH *bh)
2714 MapClient *client;
2716 qemu_mutex_lock(&map_client_list_lock);
2717 QLIST_FOREACH(client, &map_client_list, link) {
2718 if (client->bh == bh) {
2719 cpu_unregister_map_client_do(client);
2720 break;
2723 qemu_mutex_unlock(&map_client_list_lock);
2726 static void cpu_notify_map_clients(void)
2728 qemu_mutex_lock(&map_client_list_lock);
2729 cpu_notify_map_clients_locked();
2730 qemu_mutex_unlock(&map_client_list_lock);
2733 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2735 MemoryRegion *mr;
2736 hwaddr l, xlat;
2738 rcu_read_lock();
2739 while (len > 0) {
2740 l = len;
2741 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2742 if (!memory_access_is_direct(mr, is_write)) {
2743 l = memory_access_size(mr, l, addr);
2744 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2745 return false;
2749 len -= l;
2750 addr += l;
2752 rcu_read_unlock();
2753 return true;
2756 /* Map a physical memory region into a host virtual address.
2757 * May map a subset of the requested range, given by and returned in *plen.
2758 * May return NULL if resources needed to perform the mapping are exhausted.
2759 * Use only for reads OR writes - not for read-modify-write operations.
2760 * Use cpu_register_map_client() to know when retrying the map operation is
2761 * likely to succeed.
2763 void *address_space_map(AddressSpace *as,
2764 hwaddr addr,
2765 hwaddr *plen,
2766 bool is_write)
2768 hwaddr len = *plen;
2769 hwaddr done = 0;
2770 hwaddr l, xlat, base;
2771 MemoryRegion *mr, *this_mr;
2772 ram_addr_t raddr;
2774 if (len == 0) {
2775 return NULL;
2778 l = len;
2779 rcu_read_lock();
2780 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2782 if (!memory_access_is_direct(mr, is_write)) {
2783 if (atomic_xchg(&bounce.in_use, true)) {
2784 rcu_read_unlock();
2785 return NULL;
2787 /* Avoid unbounded allocations */
2788 l = MIN(l, TARGET_PAGE_SIZE);
2789 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
2790 bounce.addr = addr;
2791 bounce.len = l;
2793 memory_region_ref(mr);
2794 bounce.mr = mr;
2795 if (!is_write) {
2796 address_space_read(as, addr, MEMTXATTRS_UNSPECIFIED,
2797 bounce.buffer, l);
2800 rcu_read_unlock();
2801 *plen = l;
2802 return bounce.buffer;
2805 base = xlat;
2806 raddr = memory_region_get_ram_addr(mr);
2808 for (;;) {
2809 len -= l;
2810 addr += l;
2811 done += l;
2812 if (len == 0) {
2813 break;
2816 l = len;
2817 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2818 if (this_mr != mr || xlat != base + done) {
2819 break;
2823 memory_region_ref(mr);
2824 rcu_read_unlock();
2825 *plen = done;
2826 return qemu_ram_ptr_length(raddr + base, plen);
2829 /* Unmaps a memory region previously mapped by address_space_map().
2830 * Will also mark the memory as dirty if is_write == 1. access_len gives
2831 * the amount of memory that was actually read or written by the caller.
2833 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2834 int is_write, hwaddr access_len)
2836 if (buffer != bounce.buffer) {
2837 MemoryRegion *mr;
2838 ram_addr_t addr1;
2840 mr = qemu_ram_addr_from_host(buffer, &addr1);
2841 assert(mr != NULL);
2842 if (is_write) {
2843 invalidate_and_set_dirty(mr, addr1, access_len);
2845 if (xen_enabled()) {
2846 xen_invalidate_map_cache_entry(buffer);
2848 memory_region_unref(mr);
2849 return;
2851 if (is_write) {
2852 address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
2853 bounce.buffer, access_len);
2855 qemu_vfree(bounce.buffer);
2856 bounce.buffer = NULL;
2857 memory_region_unref(bounce.mr);
2858 atomic_mb_set(&bounce.in_use, false);
2859 cpu_notify_map_clients();
2862 void *cpu_physical_memory_map(hwaddr addr,
2863 hwaddr *plen,
2864 int is_write)
2866 return address_space_map(&address_space_memory, addr, plen, is_write);
2869 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2870 int is_write, hwaddr access_len)
2872 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2875 /* warning: addr must be aligned */
2876 static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
2877 MemTxAttrs attrs,
2878 MemTxResult *result,
2879 enum device_endian endian)
2881 uint8_t *ptr;
2882 uint64_t val;
2883 MemoryRegion *mr;
2884 hwaddr l = 4;
2885 hwaddr addr1;
2886 MemTxResult r;
2887 bool release_lock = false;
2889 rcu_read_lock();
2890 mr = address_space_translate(as, addr, &addr1, &l, false);
2891 if (l < 4 || !memory_access_is_direct(mr, false)) {
2892 release_lock |= prepare_mmio_access(mr);
2894 /* I/O case */
2895 r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
2896 #if defined(TARGET_WORDS_BIGENDIAN)
2897 if (endian == DEVICE_LITTLE_ENDIAN) {
2898 val = bswap32(val);
2900 #else
2901 if (endian == DEVICE_BIG_ENDIAN) {
2902 val = bswap32(val);
2904 #endif
2905 } else {
2906 /* RAM case */
2907 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2908 & TARGET_PAGE_MASK)
2909 + addr1);
2910 switch (endian) {
2911 case DEVICE_LITTLE_ENDIAN:
2912 val = ldl_le_p(ptr);
2913 break;
2914 case DEVICE_BIG_ENDIAN:
2915 val = ldl_be_p(ptr);
2916 break;
2917 default:
2918 val = ldl_p(ptr);
2919 break;
2921 r = MEMTX_OK;
2923 if (result) {
2924 *result = r;
2926 if (release_lock) {
2927 qemu_mutex_unlock_iothread();
2929 rcu_read_unlock();
2930 return val;
2933 uint32_t address_space_ldl(AddressSpace *as, hwaddr addr,
2934 MemTxAttrs attrs, MemTxResult *result)
2936 return address_space_ldl_internal(as, addr, attrs, result,
2937 DEVICE_NATIVE_ENDIAN);
2940 uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr,
2941 MemTxAttrs attrs, MemTxResult *result)
2943 return address_space_ldl_internal(as, addr, attrs, result,
2944 DEVICE_LITTLE_ENDIAN);
2947 uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr,
2948 MemTxAttrs attrs, MemTxResult *result)
2950 return address_space_ldl_internal(as, addr, attrs, result,
2951 DEVICE_BIG_ENDIAN);
2954 uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
2956 return address_space_ldl(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
2959 uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
2961 return address_space_ldl_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
2964 uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
2966 return address_space_ldl_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
2969 /* warning: addr must be aligned */
2970 static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
2971 MemTxAttrs attrs,
2972 MemTxResult *result,
2973 enum device_endian endian)
2975 uint8_t *ptr;
2976 uint64_t val;
2977 MemoryRegion *mr;
2978 hwaddr l = 8;
2979 hwaddr addr1;
2980 MemTxResult r;
2981 bool release_lock = false;
2983 rcu_read_lock();
2984 mr = address_space_translate(as, addr, &addr1, &l,
2985 false);
2986 if (l < 8 || !memory_access_is_direct(mr, false)) {
2987 release_lock |= prepare_mmio_access(mr);
2989 /* I/O case */
2990 r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
2991 #if defined(TARGET_WORDS_BIGENDIAN)
2992 if (endian == DEVICE_LITTLE_ENDIAN) {
2993 val = bswap64(val);
2995 #else
2996 if (endian == DEVICE_BIG_ENDIAN) {
2997 val = bswap64(val);
2999 #endif
3000 } else {
3001 /* RAM case */
3002 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
3003 & TARGET_PAGE_MASK)
3004 + addr1);
3005 switch (endian) {
3006 case DEVICE_LITTLE_ENDIAN:
3007 val = ldq_le_p(ptr);
3008 break;
3009 case DEVICE_BIG_ENDIAN:
3010 val = ldq_be_p(ptr);
3011 break;
3012 default:
3013 val = ldq_p(ptr);
3014 break;
3016 r = MEMTX_OK;
3018 if (result) {
3019 *result = r;
3021 if (release_lock) {
3022 qemu_mutex_unlock_iothread();
3024 rcu_read_unlock();
3025 return val;
3028 uint64_t address_space_ldq(AddressSpace *as, hwaddr addr,
3029 MemTxAttrs attrs, MemTxResult *result)
3031 return address_space_ldq_internal(as, addr, attrs, result,
3032 DEVICE_NATIVE_ENDIAN);
3035 uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr,
3036 MemTxAttrs attrs, MemTxResult *result)
3038 return address_space_ldq_internal(as, addr, attrs, result,
3039 DEVICE_LITTLE_ENDIAN);
3042 uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr,
3043 MemTxAttrs attrs, MemTxResult *result)
3045 return address_space_ldq_internal(as, addr, attrs, result,
3046 DEVICE_BIG_ENDIAN);
3049 uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
3051 return address_space_ldq(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3054 uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
3056 return address_space_ldq_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3059 uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
3061 return address_space_ldq_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3064 /* XXX: optimize */
3065 uint32_t address_space_ldub(AddressSpace *as, hwaddr addr,
3066 MemTxAttrs attrs, MemTxResult *result)
3068 uint8_t val;
3069 MemTxResult r;
3071 r = address_space_rw(as, addr, attrs, &val, 1, 0);
3072 if (result) {
3073 *result = r;
3075 return val;
3078 uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
3080 return address_space_ldub(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3083 /* warning: addr must be aligned */
3084 static inline uint32_t address_space_lduw_internal(AddressSpace *as,
3085 hwaddr addr,
3086 MemTxAttrs attrs,
3087 MemTxResult *result,
3088 enum device_endian endian)
3090 uint8_t *ptr;
3091 uint64_t val;
3092 MemoryRegion *mr;
3093 hwaddr l = 2;
3094 hwaddr addr1;
3095 MemTxResult r;
3096 bool release_lock = false;
3098 rcu_read_lock();
3099 mr = address_space_translate(as, addr, &addr1, &l,
3100 false);
3101 if (l < 2 || !memory_access_is_direct(mr, false)) {
3102 release_lock |= prepare_mmio_access(mr);
3104 /* I/O case */
3105 r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
3106 #if defined(TARGET_WORDS_BIGENDIAN)
3107 if (endian == DEVICE_LITTLE_ENDIAN) {
3108 val = bswap16(val);
3110 #else
3111 if (endian == DEVICE_BIG_ENDIAN) {
3112 val = bswap16(val);
3114 #endif
3115 } else {
3116 /* RAM case */
3117 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
3118 & TARGET_PAGE_MASK)
3119 + addr1);
3120 switch (endian) {
3121 case DEVICE_LITTLE_ENDIAN:
3122 val = lduw_le_p(ptr);
3123 break;
3124 case DEVICE_BIG_ENDIAN:
3125 val = lduw_be_p(ptr);
3126 break;
3127 default:
3128 val = lduw_p(ptr);
3129 break;
3131 r = MEMTX_OK;
3133 if (result) {
3134 *result = r;
3136 if (release_lock) {
3137 qemu_mutex_unlock_iothread();
3139 rcu_read_unlock();
3140 return val;
3143 uint32_t address_space_lduw(AddressSpace *as, hwaddr addr,
3144 MemTxAttrs attrs, MemTxResult *result)
3146 return address_space_lduw_internal(as, addr, attrs, result,
3147 DEVICE_NATIVE_ENDIAN);
3150 uint32_t address_space_lduw_le(AddressSpace *as, hwaddr addr,
3151 MemTxAttrs attrs, MemTxResult *result)
3153 return address_space_lduw_internal(as, addr, attrs, result,
3154 DEVICE_LITTLE_ENDIAN);
3157 uint32_t address_space_lduw_be(AddressSpace *as, hwaddr addr,
3158 MemTxAttrs attrs, MemTxResult *result)
3160 return address_space_lduw_internal(as, addr, attrs, result,
3161 DEVICE_BIG_ENDIAN);
3164 uint32_t lduw_phys(AddressSpace *as, hwaddr addr)
3166 return address_space_lduw(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3169 uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr)
3171 return address_space_lduw_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3174 uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr)
3176 return address_space_lduw_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3179 /* warning: addr must be aligned. The ram page is not masked as dirty
3180 and the code inside is not invalidated. It is useful if the dirty
3181 bits are used to track modified PTEs */
3182 void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
3183 MemTxAttrs attrs, MemTxResult *result)
3185 uint8_t *ptr;
3186 MemoryRegion *mr;
3187 hwaddr l = 4;
3188 hwaddr addr1;
3189 MemTxResult r;
3190 uint8_t dirty_log_mask;
3191 bool release_lock = false;
3193 rcu_read_lock();
3194 mr = address_space_translate(as, addr, &addr1, &l,
3195 true);
3196 if (l < 4 || !memory_access_is_direct(mr, true)) {
3197 release_lock |= prepare_mmio_access(mr);
3199 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3200 } else {
3201 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3202 ptr = qemu_get_ram_ptr(addr1);
3203 stl_p(ptr, val);
3205 dirty_log_mask = memory_region_get_dirty_log_mask(mr);
3206 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
3207 cpu_physical_memory_set_dirty_range(addr1, 4, dirty_log_mask);
3208 r = MEMTX_OK;
3210 if (result) {
3211 *result = r;
3213 if (release_lock) {
3214 qemu_mutex_unlock_iothread();
3216 rcu_read_unlock();
3219 void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
3221 address_space_stl_notdirty(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3224 /* warning: addr must be aligned */
3225 static inline void address_space_stl_internal(AddressSpace *as,
3226 hwaddr addr, uint32_t val,
3227 MemTxAttrs attrs,
3228 MemTxResult *result,
3229 enum device_endian endian)
3231 uint8_t *ptr;
3232 MemoryRegion *mr;
3233 hwaddr l = 4;
3234 hwaddr addr1;
3235 MemTxResult r;
3236 bool release_lock = false;
3238 rcu_read_lock();
3239 mr = address_space_translate(as, addr, &addr1, &l,
3240 true);
3241 if (l < 4 || !memory_access_is_direct(mr, true)) {
3242 release_lock |= prepare_mmio_access(mr);
3244 #if defined(TARGET_WORDS_BIGENDIAN)
3245 if (endian == DEVICE_LITTLE_ENDIAN) {
3246 val = bswap32(val);
3248 #else
3249 if (endian == DEVICE_BIG_ENDIAN) {
3250 val = bswap32(val);
3252 #endif
3253 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3254 } else {
3255 /* RAM case */
3256 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3257 ptr = qemu_get_ram_ptr(addr1);
3258 switch (endian) {
3259 case DEVICE_LITTLE_ENDIAN:
3260 stl_le_p(ptr, val);
3261 break;
3262 case DEVICE_BIG_ENDIAN:
3263 stl_be_p(ptr, val);
3264 break;
3265 default:
3266 stl_p(ptr, val);
3267 break;
3269 invalidate_and_set_dirty(mr, addr1, 4);
3270 r = MEMTX_OK;
3272 if (result) {
3273 *result = r;
3275 if (release_lock) {
3276 qemu_mutex_unlock_iothread();
3278 rcu_read_unlock();
3281 void address_space_stl(AddressSpace *as, hwaddr addr, uint32_t val,
3282 MemTxAttrs attrs, MemTxResult *result)
3284 address_space_stl_internal(as, addr, val, attrs, result,
3285 DEVICE_NATIVE_ENDIAN);
3288 void address_space_stl_le(AddressSpace *as, hwaddr addr, uint32_t val,
3289 MemTxAttrs attrs, MemTxResult *result)
3291 address_space_stl_internal(as, addr, val, attrs, result,
3292 DEVICE_LITTLE_ENDIAN);
3295 void address_space_stl_be(AddressSpace *as, hwaddr addr, uint32_t val,
3296 MemTxAttrs attrs, MemTxResult *result)
3298 address_space_stl_internal(as, addr, val, attrs, result,
3299 DEVICE_BIG_ENDIAN);
3302 void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3304 address_space_stl(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3307 void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3309 address_space_stl_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3312 void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3314 address_space_stl_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3317 /* XXX: optimize */
3318 void address_space_stb(AddressSpace *as, hwaddr addr, uint32_t val,
3319 MemTxAttrs attrs, MemTxResult *result)
3321 uint8_t v = val;
3322 MemTxResult r;
3324 r = address_space_rw(as, addr, attrs, &v, 1, 1);
3325 if (result) {
3326 *result = r;
3330 void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3332 address_space_stb(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3335 /* warning: addr must be aligned */
3336 static inline void address_space_stw_internal(AddressSpace *as,
3337 hwaddr addr, uint32_t val,
3338 MemTxAttrs attrs,
3339 MemTxResult *result,
3340 enum device_endian endian)
3342 uint8_t *ptr;
3343 MemoryRegion *mr;
3344 hwaddr l = 2;
3345 hwaddr addr1;
3346 MemTxResult r;
3347 bool release_lock = false;
3349 rcu_read_lock();
3350 mr = address_space_translate(as, addr, &addr1, &l, true);
3351 if (l < 2 || !memory_access_is_direct(mr, true)) {
3352 release_lock |= prepare_mmio_access(mr);
3354 #if defined(TARGET_WORDS_BIGENDIAN)
3355 if (endian == DEVICE_LITTLE_ENDIAN) {
3356 val = bswap16(val);
3358 #else
3359 if (endian == DEVICE_BIG_ENDIAN) {
3360 val = bswap16(val);
3362 #endif
3363 r = memory_region_dispatch_write(mr, addr1, val, 2, attrs);
3364 } else {
3365 /* RAM case */
3366 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3367 ptr = qemu_get_ram_ptr(addr1);
3368 switch (endian) {
3369 case DEVICE_LITTLE_ENDIAN:
3370 stw_le_p(ptr, val);
3371 break;
3372 case DEVICE_BIG_ENDIAN:
3373 stw_be_p(ptr, val);
3374 break;
3375 default:
3376 stw_p(ptr, val);
3377 break;
3379 invalidate_and_set_dirty(mr, addr1, 2);
3380 r = MEMTX_OK;
3382 if (result) {
3383 *result = r;
3385 if (release_lock) {
3386 qemu_mutex_unlock_iothread();
3388 rcu_read_unlock();
3391 void address_space_stw(AddressSpace *as, hwaddr addr, uint32_t val,
3392 MemTxAttrs attrs, MemTxResult *result)
3394 address_space_stw_internal(as, addr, val, attrs, result,
3395 DEVICE_NATIVE_ENDIAN);
3398 void address_space_stw_le(AddressSpace *as, hwaddr addr, uint32_t val,
3399 MemTxAttrs attrs, MemTxResult *result)
3401 address_space_stw_internal(as, addr, val, attrs, result,
3402 DEVICE_LITTLE_ENDIAN);
3405 void address_space_stw_be(AddressSpace *as, hwaddr addr, uint32_t val,
3406 MemTxAttrs attrs, MemTxResult *result)
3408 address_space_stw_internal(as, addr, val, attrs, result,
3409 DEVICE_BIG_ENDIAN);
3412 void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3414 address_space_stw(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3417 void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3419 address_space_stw_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3422 void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3424 address_space_stw_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3427 /* XXX: optimize */
3428 void address_space_stq(AddressSpace *as, hwaddr addr, uint64_t val,
3429 MemTxAttrs attrs, MemTxResult *result)
3431 MemTxResult r;
3432 val = tswap64(val);
3433 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3434 if (result) {
3435 *result = r;
3439 void address_space_stq_le(AddressSpace *as, hwaddr addr, uint64_t val,
3440 MemTxAttrs attrs, MemTxResult *result)
3442 MemTxResult r;
3443 val = cpu_to_le64(val);
3444 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3445 if (result) {
3446 *result = r;
3449 void address_space_stq_be(AddressSpace *as, hwaddr addr, uint64_t val,
3450 MemTxAttrs attrs, MemTxResult *result)
3452 MemTxResult r;
3453 val = cpu_to_be64(val);
3454 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3455 if (result) {
3456 *result = r;
3460 void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3462 address_space_stq(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3465 void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3467 address_space_stq_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3470 void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3472 address_space_stq_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3475 /* virtual memory access for debug (includes writing to ROM) */
3476 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
3477 uint8_t *buf, int len, int is_write)
3479 int l;
3480 hwaddr phys_addr;
3481 target_ulong page;
3483 while (len > 0) {
3484 page = addr & TARGET_PAGE_MASK;
3485 phys_addr = cpu_get_phys_page_debug(cpu, page);
3486 /* if no physical page mapped, return an error */
3487 if (phys_addr == -1)
3488 return -1;
3489 l = (page + TARGET_PAGE_SIZE) - addr;
3490 if (l > len)
3491 l = len;
3492 phys_addr += (addr & ~TARGET_PAGE_MASK);
3493 if (is_write) {
3494 cpu_physical_memory_write_rom(cpu->as, phys_addr, buf, l);
3495 } else {
3496 address_space_rw(cpu->as, phys_addr, MEMTXATTRS_UNSPECIFIED,
3497 buf, l, 0);
3499 len -= l;
3500 buf += l;
3501 addr += l;
3503 return 0;
3505 #endif
3508 * A helper function for the _utterly broken_ virtio device model to find out if
3509 * it's running on a big endian machine. Don't do this at home kids!
3511 bool target_words_bigendian(void);
3512 bool target_words_bigendian(void)
3514 #if defined(TARGET_WORDS_BIGENDIAN)
3515 return true;
3516 #else
3517 return false;
3518 #endif
3521 #ifndef CONFIG_USER_ONLY
3522 bool cpu_physical_memory_is_io(hwaddr phys_addr)
3524 MemoryRegion*mr;
3525 hwaddr l = 1;
3526 bool res;
3528 rcu_read_lock();
3529 mr = address_space_translate(&address_space_memory,
3530 phys_addr, &phys_addr, &l, false);
3532 res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
3533 rcu_read_unlock();
3534 return res;
3537 int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
3539 RAMBlock *block;
3540 int ret = 0;
3542 rcu_read_lock();
3543 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
3544 ret = func(block->idstr, block->host, block->offset,
3545 block->used_length, opaque);
3546 if (ret) {
3547 break;
3550 rcu_read_unlock();
3551 return ret;
3553 #endif