include/qemu/osdep.h: Don't include qapi/error.h
[qemu/ar7.git] / exec.c
blob151908ebdfe3c89a8327dc786b94b340cd7f0fd0
1 /*
2 * Virtual page mapping
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "qemu/osdep.h"
20 #include "qapi/error.h"
21 #ifndef _WIN32
22 #include <sys/mman.h>
23 #endif
25 #include "qemu-common.h"
26 #include "cpu.h"
27 #include "tcg.h"
28 #include "hw/hw.h"
29 #if !defined(CONFIG_USER_ONLY)
30 #include "hw/boards.h"
31 #endif
32 #include "hw/qdev.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "qemu/error-report.h"
39 #include "exec/memory.h"
40 #include "sysemu/dma.h"
41 #include "exec/address-spaces.h"
42 #if defined(CONFIG_USER_ONLY)
43 #include <qemu.h>
44 #else /* !CONFIG_USER_ONLY */
45 #include "sysemu/xen-mapcache.h"
46 #include "trace.h"
47 #endif
48 #include "exec/cpu-all.h"
49 #include "qemu/rcu_queue.h"
50 #include "qemu/main-loop.h"
51 #include "translate-all.h"
52 #include "sysemu/replay.h"
54 #include "exec/memory-internal.h"
55 #include "exec/ram_addr.h"
56 #include "exec/log.h"
58 #include "qemu/range.h"
59 #ifndef _WIN32
60 #include "qemu/mmap-alloc.h"
61 #endif
63 //#define DEBUG_SUBPAGE
65 #if !defined(CONFIG_USER_ONLY)
66 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
67 * are protected by the ramlist lock.
69 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
71 static MemoryRegion *system_memory;
72 static MemoryRegion *system_io;
74 AddressSpace address_space_io;
75 AddressSpace address_space_memory;
77 MemoryRegion io_mem_rom, io_mem_notdirty;
78 static MemoryRegion io_mem_unassigned;
80 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
81 #define RAM_PREALLOC (1 << 0)
83 /* RAM is mmap-ed with MAP_SHARED */
84 #define RAM_SHARED (1 << 1)
86 /* Only a portion of RAM (used_length) is actually used, and migrated.
87 * This used_length size can change across reboots.
89 #define RAM_RESIZEABLE (1 << 2)
91 #endif
93 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
94 /* current CPU in the current thread. It is only valid inside
95 cpu_exec() */
96 __thread CPUState *current_cpu;
97 /* 0 = Do not count executed instructions.
98 1 = Precise instruction counting.
99 2 = Adaptive rate instruction counting. */
100 int use_icount;
102 #if !defined(CONFIG_USER_ONLY)
104 typedef struct PhysPageEntry PhysPageEntry;
106 struct PhysPageEntry {
107 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
108 uint32_t skip : 6;
109 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
110 uint32_t ptr : 26;
113 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
115 /* Size of the L2 (and L3, etc) page tables. */
116 #define ADDR_SPACE_BITS 64
118 #define P_L2_BITS 9
119 #define P_L2_SIZE (1 << P_L2_BITS)
121 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
123 typedef PhysPageEntry Node[P_L2_SIZE];
125 typedef struct PhysPageMap {
126 struct rcu_head rcu;
128 unsigned sections_nb;
129 unsigned sections_nb_alloc;
130 unsigned nodes_nb;
131 unsigned nodes_nb_alloc;
132 Node *nodes;
133 MemoryRegionSection *sections;
134 } PhysPageMap;
136 struct AddressSpaceDispatch {
137 struct rcu_head rcu;
139 MemoryRegionSection *mru_section;
140 /* This is a multi-level map on the physical address space.
141 * The bottom level has pointers to MemoryRegionSections.
143 PhysPageEntry phys_map;
144 PhysPageMap map;
145 AddressSpace *as;
148 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
149 typedef struct subpage_t {
150 MemoryRegion iomem;
151 AddressSpace *as;
152 hwaddr base;
153 uint16_t sub_section[TARGET_PAGE_SIZE];
154 } subpage_t;
156 #define PHYS_SECTION_UNASSIGNED 0
157 #define PHYS_SECTION_NOTDIRTY 1
158 #define PHYS_SECTION_ROM 2
159 #define PHYS_SECTION_WATCH 3
161 static void io_mem_init(void);
162 static void memory_map_init(void);
163 static void tcg_commit(MemoryListener *listener);
165 static MemoryRegion io_mem_watch;
168 * CPUAddressSpace: all the information a CPU needs about an AddressSpace
169 * @cpu: the CPU whose AddressSpace this is
170 * @as: the AddressSpace itself
171 * @memory_dispatch: its dispatch pointer (cached, RCU protected)
172 * @tcg_as_listener: listener for tracking changes to the AddressSpace
174 struct CPUAddressSpace {
175 CPUState *cpu;
176 AddressSpace *as;
177 struct AddressSpaceDispatch *memory_dispatch;
178 MemoryListener tcg_as_listener;
181 #endif
183 #if !defined(CONFIG_USER_ONLY)
185 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
187 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
188 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
189 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
190 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
194 static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
196 unsigned i;
197 uint32_t ret;
198 PhysPageEntry e;
199 PhysPageEntry *p;
201 ret = map->nodes_nb++;
202 p = map->nodes[ret];
203 assert(ret != PHYS_MAP_NODE_NIL);
204 assert(ret != map->nodes_nb_alloc);
206 e.skip = leaf ? 0 : 1;
207 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
208 for (i = 0; i < P_L2_SIZE; ++i) {
209 memcpy(&p[i], &e, sizeof(e));
211 return ret;
214 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
215 hwaddr *index, hwaddr *nb, uint16_t leaf,
216 int level)
218 PhysPageEntry *p;
219 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
221 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
222 lp->ptr = phys_map_node_alloc(map, level == 0);
224 p = map->nodes[lp->ptr];
225 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
227 while (*nb && lp < &p[P_L2_SIZE]) {
228 if ((*index & (step - 1)) == 0 && *nb >= step) {
229 lp->skip = 0;
230 lp->ptr = leaf;
231 *index += step;
232 *nb -= step;
233 } else {
234 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
236 ++lp;
240 static void phys_page_set(AddressSpaceDispatch *d,
241 hwaddr index, hwaddr nb,
242 uint16_t leaf)
244 /* Wildly overreserve - it doesn't matter much. */
245 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
247 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
250 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
251 * and update our entry so we can skip it and go directly to the destination.
253 static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)
255 unsigned valid_ptr = P_L2_SIZE;
256 int valid = 0;
257 PhysPageEntry *p;
258 int i;
260 if (lp->ptr == PHYS_MAP_NODE_NIL) {
261 return;
264 p = nodes[lp->ptr];
265 for (i = 0; i < P_L2_SIZE; i++) {
266 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
267 continue;
270 valid_ptr = i;
271 valid++;
272 if (p[i].skip) {
273 phys_page_compact(&p[i], nodes, compacted);
277 /* We can only compress if there's only one child. */
278 if (valid != 1) {
279 return;
282 assert(valid_ptr < P_L2_SIZE);
284 /* Don't compress if it won't fit in the # of bits we have. */
285 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
286 return;
289 lp->ptr = p[valid_ptr].ptr;
290 if (!p[valid_ptr].skip) {
291 /* If our only child is a leaf, make this a leaf. */
292 /* By design, we should have made this node a leaf to begin with so we
293 * should never reach here.
294 * But since it's so simple to handle this, let's do it just in case we
295 * change this rule.
297 lp->skip = 0;
298 } else {
299 lp->skip += p[valid_ptr].skip;
303 static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
305 DECLARE_BITMAP(compacted, nodes_nb);
307 if (d->phys_map.skip) {
308 phys_page_compact(&d->phys_map, d->map.nodes, compacted);
312 static inline bool section_covers_addr(const MemoryRegionSection *section,
313 hwaddr addr)
315 /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
316 * the section must cover the entire address space.
318 return section->size.hi ||
319 range_covers_byte(section->offset_within_address_space,
320 section->size.lo, addr);
323 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
324 Node *nodes, MemoryRegionSection *sections)
326 PhysPageEntry *p;
327 hwaddr index = addr >> TARGET_PAGE_BITS;
328 int i;
330 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
331 if (lp.ptr == PHYS_MAP_NODE_NIL) {
332 return &sections[PHYS_SECTION_UNASSIGNED];
334 p = nodes[lp.ptr];
335 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
338 if (section_covers_addr(&sections[lp.ptr], addr)) {
339 return &sections[lp.ptr];
340 } else {
341 return &sections[PHYS_SECTION_UNASSIGNED];
345 bool memory_region_is_unassigned(MemoryRegion *mr)
347 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
348 && mr != &io_mem_watch;
351 /* Called from RCU critical section */
352 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
353 hwaddr addr,
354 bool resolve_subpage)
356 MemoryRegionSection *section = atomic_read(&d->mru_section);
357 subpage_t *subpage;
358 bool update;
360 if (section && section != &d->map.sections[PHYS_SECTION_UNASSIGNED] &&
361 section_covers_addr(section, addr)) {
362 update = false;
363 } else {
364 section = phys_page_find(d->phys_map, addr, d->map.nodes,
365 d->map.sections);
366 update = true;
368 if (resolve_subpage && section->mr->subpage) {
369 subpage = container_of(section->mr, subpage_t, iomem);
370 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
372 if (update) {
373 atomic_set(&d->mru_section, section);
375 return section;
378 /* Called from RCU critical section */
379 static MemoryRegionSection *
380 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
381 hwaddr *plen, bool resolve_subpage)
383 MemoryRegionSection *section;
384 MemoryRegion *mr;
385 Int128 diff;
387 section = address_space_lookup_region(d, addr, resolve_subpage);
388 /* Compute offset within MemoryRegionSection */
389 addr -= section->offset_within_address_space;
391 /* Compute offset within MemoryRegion */
392 *xlat = addr + section->offset_within_region;
394 mr = section->mr;
396 /* MMIO registers can be expected to perform full-width accesses based only
397 * on their address, without considering adjacent registers that could
398 * decode to completely different MemoryRegions. When such registers
399 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
400 * regions overlap wildly. For this reason we cannot clamp the accesses
401 * here.
403 * If the length is small (as is the case for address_space_ldl/stl),
404 * everything works fine. If the incoming length is large, however,
405 * the caller really has to do the clamping through memory_access_size.
407 if (memory_region_is_ram(mr)) {
408 diff = int128_sub(section->size, int128_make64(addr));
409 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
411 return section;
414 /* Called from RCU critical section */
415 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
416 hwaddr *xlat, hwaddr *plen,
417 bool is_write)
419 IOMMUTLBEntry iotlb;
420 MemoryRegionSection *section;
421 MemoryRegion *mr;
423 for (;;) {
424 AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch);
425 section = address_space_translate_internal(d, addr, &addr, plen, true);
426 mr = section->mr;
428 if (!mr->iommu_ops) {
429 break;
432 iotlb = mr->iommu_ops->translate(mr, addr, is_write);
433 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
434 | (addr & iotlb.addr_mask));
435 *plen = MIN(*plen, (addr | iotlb.addr_mask) - addr + 1);
436 if (!(iotlb.perm & (1 << is_write))) {
437 mr = &io_mem_unassigned;
438 break;
441 as = iotlb.target_as;
444 if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
445 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
446 *plen = MIN(page, *plen);
449 *xlat = addr;
450 return mr;
453 /* Called from RCU critical section */
454 MemoryRegionSection *
455 address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
456 hwaddr *xlat, hwaddr *plen)
458 MemoryRegionSection *section;
459 AddressSpaceDispatch *d = cpu->cpu_ases[asidx].memory_dispatch;
461 section = address_space_translate_internal(d, addr, xlat, plen, false);
463 assert(!section->mr->iommu_ops);
464 return section;
466 #endif
468 #if !defined(CONFIG_USER_ONLY)
470 static int cpu_common_post_load(void *opaque, int version_id)
472 CPUState *cpu = opaque;
474 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
475 version_id is increased. */
476 cpu->interrupt_request &= ~0x01;
477 tlb_flush(cpu, 1);
479 return 0;
482 static int cpu_common_pre_load(void *opaque)
484 CPUState *cpu = opaque;
486 cpu->exception_index = -1;
488 return 0;
491 static bool cpu_common_exception_index_needed(void *opaque)
493 CPUState *cpu = opaque;
495 return tcg_enabled() && cpu->exception_index != -1;
498 static const VMStateDescription vmstate_cpu_common_exception_index = {
499 .name = "cpu_common/exception_index",
500 .version_id = 1,
501 .minimum_version_id = 1,
502 .needed = cpu_common_exception_index_needed,
503 .fields = (VMStateField[]) {
504 VMSTATE_INT32(exception_index, CPUState),
505 VMSTATE_END_OF_LIST()
509 static bool cpu_common_crash_occurred_needed(void *opaque)
511 CPUState *cpu = opaque;
513 return cpu->crash_occurred;
516 static const VMStateDescription vmstate_cpu_common_crash_occurred = {
517 .name = "cpu_common/crash_occurred",
518 .version_id = 1,
519 .minimum_version_id = 1,
520 .needed = cpu_common_crash_occurred_needed,
521 .fields = (VMStateField[]) {
522 VMSTATE_BOOL(crash_occurred, CPUState),
523 VMSTATE_END_OF_LIST()
527 const VMStateDescription vmstate_cpu_common = {
528 .name = "cpu_common",
529 .version_id = 1,
530 .minimum_version_id = 1,
531 .pre_load = cpu_common_pre_load,
532 .post_load = cpu_common_post_load,
533 .fields = (VMStateField[]) {
534 VMSTATE_UINT32(halted, CPUState),
535 VMSTATE_UINT32(interrupt_request, CPUState),
536 VMSTATE_END_OF_LIST()
538 .subsections = (const VMStateDescription*[]) {
539 &vmstate_cpu_common_exception_index,
540 &vmstate_cpu_common_crash_occurred,
541 NULL
545 #endif
547 CPUState *qemu_get_cpu(int index)
549 CPUState *cpu;
551 CPU_FOREACH(cpu) {
552 if (cpu->cpu_index == index) {
553 return cpu;
557 return NULL;
560 #if !defined(CONFIG_USER_ONLY)
561 void cpu_address_space_init(CPUState *cpu, AddressSpace *as, int asidx)
563 CPUAddressSpace *newas;
565 /* Target code should have set num_ases before calling us */
566 assert(asidx < cpu->num_ases);
568 if (asidx == 0) {
569 /* address space 0 gets the convenience alias */
570 cpu->as = as;
573 /* KVM cannot currently support multiple address spaces. */
574 assert(asidx == 0 || !kvm_enabled());
576 if (!cpu->cpu_ases) {
577 cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
580 newas = &cpu->cpu_ases[asidx];
581 newas->cpu = cpu;
582 newas->as = as;
583 if (tcg_enabled()) {
584 newas->tcg_as_listener.commit = tcg_commit;
585 memory_listener_register(&newas->tcg_as_listener, as);
589 AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
591 /* Return the AddressSpace corresponding to the specified index */
592 return cpu->cpu_ases[asidx].as;
594 #endif
596 #ifndef CONFIG_USER_ONLY
597 static DECLARE_BITMAP(cpu_index_map, MAX_CPUMASK_BITS);
599 static int cpu_get_free_index(Error **errp)
601 int cpu = find_first_zero_bit(cpu_index_map, MAX_CPUMASK_BITS);
603 if (cpu >= MAX_CPUMASK_BITS) {
604 error_setg(errp, "Trying to use more CPUs than max of %d",
605 MAX_CPUMASK_BITS);
606 return -1;
609 bitmap_set(cpu_index_map, cpu, 1);
610 return cpu;
613 void cpu_exec_exit(CPUState *cpu)
615 if (cpu->cpu_index == -1) {
616 /* cpu_index was never allocated by this @cpu or was already freed. */
617 return;
620 bitmap_clear(cpu_index_map, cpu->cpu_index, 1);
621 cpu->cpu_index = -1;
623 #else
625 static int cpu_get_free_index(Error **errp)
627 CPUState *some_cpu;
628 int cpu_index = 0;
630 CPU_FOREACH(some_cpu) {
631 cpu_index++;
633 return cpu_index;
636 void cpu_exec_exit(CPUState *cpu)
639 #endif
641 void cpu_exec_init(CPUState *cpu, Error **errp)
643 CPUClass *cc = CPU_GET_CLASS(cpu);
644 int cpu_index;
645 Error *local_err = NULL;
647 cpu->as = NULL;
648 cpu->num_ases = 0;
650 #ifndef CONFIG_USER_ONLY
651 cpu->thread_id = qemu_get_thread_id();
653 /* This is a softmmu CPU object, so create a property for it
654 * so users can wire up its memory. (This can't go in qom/cpu.c
655 * because that file is compiled only once for both user-mode
656 * and system builds.) The default if no link is set up is to use
657 * the system address space.
659 object_property_add_link(OBJECT(cpu), "memory", TYPE_MEMORY_REGION,
660 (Object **)&cpu->memory,
661 qdev_prop_allow_set_link_before_realize,
662 OBJ_PROP_LINK_UNREF_ON_RELEASE,
663 &error_abort);
664 cpu->memory = system_memory;
665 object_ref(OBJECT(cpu->memory));
666 #endif
668 #if defined(CONFIG_USER_ONLY)
669 cpu_list_lock();
670 #endif
671 cpu_index = cpu->cpu_index = cpu_get_free_index(&local_err);
672 if (local_err) {
673 error_propagate(errp, local_err);
674 #if defined(CONFIG_USER_ONLY)
675 cpu_list_unlock();
676 #endif
677 return;
679 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
680 #if defined(CONFIG_USER_ONLY)
681 cpu_list_unlock();
682 #endif
683 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
684 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
686 if (cc->vmsd != NULL) {
687 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
691 #if defined(CONFIG_USER_ONLY)
692 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
694 tb_invalidate_phys_page_range(pc, pc + 1, 0);
696 #else
697 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
699 MemTxAttrs attrs;
700 hwaddr phys = cpu_get_phys_page_attrs_debug(cpu, pc, &attrs);
701 int asidx = cpu_asidx_from_attrs(cpu, attrs);
702 if (phys != -1) {
703 tb_invalidate_phys_addr(cpu->cpu_ases[asidx].as,
704 phys | (pc & ~TARGET_PAGE_MASK));
707 #endif
709 #if defined(CONFIG_USER_ONLY)
710 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
715 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
716 int flags)
718 return -ENOSYS;
721 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
725 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
726 int flags, CPUWatchpoint **watchpoint)
728 return -ENOSYS;
730 #else
731 /* Add a watchpoint. */
732 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
733 int flags, CPUWatchpoint **watchpoint)
735 CPUWatchpoint *wp;
737 /* forbid ranges which are empty or run off the end of the address space */
738 if (len == 0 || (addr + len - 1) < addr) {
739 error_report("tried to set invalid watchpoint at %"
740 VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
741 return -EINVAL;
743 wp = g_malloc(sizeof(*wp));
745 wp->vaddr = addr;
746 wp->len = len;
747 wp->flags = flags;
749 /* keep all GDB-injected watchpoints in front */
750 if (flags & BP_GDB) {
751 QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
752 } else {
753 QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
756 tlb_flush_page(cpu, addr);
758 if (watchpoint)
759 *watchpoint = wp;
760 return 0;
763 /* Remove a specific watchpoint. */
764 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
765 int flags)
767 CPUWatchpoint *wp;
769 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
770 if (addr == wp->vaddr && len == wp->len
771 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
772 cpu_watchpoint_remove_by_ref(cpu, wp);
773 return 0;
776 return -ENOENT;
779 /* Remove a specific watchpoint by reference. */
780 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
782 QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
784 tlb_flush_page(cpu, watchpoint->vaddr);
786 g_free(watchpoint);
789 /* Remove all matching watchpoints. */
790 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
792 CPUWatchpoint *wp, *next;
794 QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
795 if (wp->flags & mask) {
796 cpu_watchpoint_remove_by_ref(cpu, wp);
801 /* Return true if this watchpoint address matches the specified
802 * access (ie the address range covered by the watchpoint overlaps
803 * partially or completely with the address range covered by the
804 * access).
806 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
807 vaddr addr,
808 vaddr len)
810 /* We know the lengths are non-zero, but a little caution is
811 * required to avoid errors in the case where the range ends
812 * exactly at the top of the address space and so addr + len
813 * wraps round to zero.
815 vaddr wpend = wp->vaddr + wp->len - 1;
816 vaddr addrend = addr + len - 1;
818 return !(addr > wpend || wp->vaddr > addrend);
821 #endif
823 /* Add a breakpoint. */
824 int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
825 CPUBreakpoint **breakpoint)
827 CPUBreakpoint *bp;
829 bp = g_malloc(sizeof(*bp));
831 bp->pc = pc;
832 bp->flags = flags;
834 /* keep all GDB-injected breakpoints in front */
835 if (flags & BP_GDB) {
836 QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
837 } else {
838 QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
841 breakpoint_invalidate(cpu, pc);
843 if (breakpoint) {
844 *breakpoint = bp;
846 return 0;
849 /* Remove a specific breakpoint. */
850 int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
852 CPUBreakpoint *bp;
854 QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
855 if (bp->pc == pc && bp->flags == flags) {
856 cpu_breakpoint_remove_by_ref(cpu, bp);
857 return 0;
860 return -ENOENT;
863 /* Remove a specific breakpoint by reference. */
864 void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
866 QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
868 breakpoint_invalidate(cpu, breakpoint->pc);
870 g_free(breakpoint);
873 /* Remove all matching breakpoints. */
874 void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
876 CPUBreakpoint *bp, *next;
878 QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
879 if (bp->flags & mask) {
880 cpu_breakpoint_remove_by_ref(cpu, bp);
885 /* enable or disable single step mode. EXCP_DEBUG is returned by the
886 CPU loop after each instruction */
887 void cpu_single_step(CPUState *cpu, int enabled)
889 if (cpu->singlestep_enabled != enabled) {
890 cpu->singlestep_enabled = enabled;
891 if (kvm_enabled()) {
892 kvm_update_guest_debug(cpu, 0);
893 } else {
894 /* must flush all the translated code to avoid inconsistencies */
895 /* XXX: only flush what is necessary */
896 tb_flush(cpu);
901 void cpu_abort(CPUState *cpu, const char *fmt, ...)
903 va_list ap;
904 va_list ap2;
906 va_start(ap, fmt);
907 va_copy(ap2, ap);
908 fprintf(stderr, "qemu: fatal: ");
909 vfprintf(stderr, fmt, ap);
910 fprintf(stderr, "\n");
911 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
912 if (qemu_log_separate()) {
913 qemu_log("qemu: fatal: ");
914 qemu_log_vprintf(fmt, ap2);
915 qemu_log("\n");
916 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
917 qemu_log_flush();
918 qemu_log_close();
920 va_end(ap2);
921 va_end(ap);
922 replay_finish();
923 #if defined(CONFIG_USER_ONLY)
925 struct sigaction act;
926 sigfillset(&act.sa_mask);
927 act.sa_handler = SIG_DFL;
928 sigaction(SIGABRT, &act, NULL);
930 #endif
931 abort();
934 #if !defined(CONFIG_USER_ONLY)
935 /* Called from RCU critical section */
936 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
938 RAMBlock *block;
940 block = atomic_rcu_read(&ram_list.mru_block);
941 if (block && addr - block->offset < block->max_length) {
942 return block;
944 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
945 if (addr - block->offset < block->max_length) {
946 goto found;
950 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
951 abort();
953 found:
954 /* It is safe to write mru_block outside the iothread lock. This
955 * is what happens:
957 * mru_block = xxx
958 * rcu_read_unlock()
959 * xxx removed from list
960 * rcu_read_lock()
961 * read mru_block
962 * mru_block = NULL;
963 * call_rcu(reclaim_ramblock, xxx);
964 * rcu_read_unlock()
966 * atomic_rcu_set is not needed here. The block was already published
967 * when it was placed into the list. Here we're just making an extra
968 * copy of the pointer.
970 ram_list.mru_block = block;
971 return block;
974 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
976 CPUState *cpu;
977 ram_addr_t start1;
978 RAMBlock *block;
979 ram_addr_t end;
981 end = TARGET_PAGE_ALIGN(start + length);
982 start &= TARGET_PAGE_MASK;
984 rcu_read_lock();
985 block = qemu_get_ram_block(start);
986 assert(block == qemu_get_ram_block(end - 1));
987 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
988 CPU_FOREACH(cpu) {
989 tlb_reset_dirty(cpu, start1, length);
991 rcu_read_unlock();
994 /* Note: start and end must be within the same ram block. */
995 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
996 ram_addr_t length,
997 unsigned client)
999 DirtyMemoryBlocks *blocks;
1000 unsigned long end, page;
1001 bool dirty = false;
1003 if (length == 0) {
1004 return false;
1007 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
1008 page = start >> TARGET_PAGE_BITS;
1010 rcu_read_lock();
1012 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
1014 while (page < end) {
1015 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
1016 unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
1017 unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
1019 dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
1020 offset, num);
1021 page += num;
1024 rcu_read_unlock();
1026 if (dirty && tcg_enabled()) {
1027 tlb_reset_dirty_range_all(start, length);
1030 return dirty;
1033 /* Called from RCU critical section */
1034 hwaddr memory_region_section_get_iotlb(CPUState *cpu,
1035 MemoryRegionSection *section,
1036 target_ulong vaddr,
1037 hwaddr paddr, hwaddr xlat,
1038 int prot,
1039 target_ulong *address)
1041 hwaddr iotlb;
1042 CPUWatchpoint *wp;
1044 if (memory_region_is_ram(section->mr)) {
1045 /* Normal RAM. */
1046 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
1047 + xlat;
1048 if (!section->readonly) {
1049 iotlb |= PHYS_SECTION_NOTDIRTY;
1050 } else {
1051 iotlb |= PHYS_SECTION_ROM;
1053 } else {
1054 AddressSpaceDispatch *d;
1056 d = atomic_rcu_read(&section->address_space->dispatch);
1057 iotlb = section - d->map.sections;
1058 iotlb += xlat;
1061 /* Make accesses to pages with watchpoints go via the
1062 watchpoint trap routines. */
1063 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
1064 if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
1065 /* Avoid trapping reads of pages with a write breakpoint. */
1066 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
1067 iotlb = PHYS_SECTION_WATCH + paddr;
1068 *address |= TLB_MMIO;
1069 break;
1074 return iotlb;
1076 #endif /* defined(CONFIG_USER_ONLY) */
1078 #if !defined(CONFIG_USER_ONLY)
1080 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1081 uint16_t section);
1082 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
1084 static void *(*phys_mem_alloc)(size_t size, uint64_t *align) =
1085 qemu_anon_ram_alloc;
1088 * Set a custom physical guest memory alloator.
1089 * Accelerators with unusual needs may need this. Hopefully, we can
1090 * get rid of it eventually.
1092 void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align))
1094 phys_mem_alloc = alloc;
1097 static uint16_t phys_section_add(PhysPageMap *map,
1098 MemoryRegionSection *section)
1100 /* The physical section number is ORed with a page-aligned
1101 * pointer to produce the iotlb entries. Thus it should
1102 * never overflow into the page-aligned value.
1104 assert(map->sections_nb < TARGET_PAGE_SIZE);
1106 if (map->sections_nb == map->sections_nb_alloc) {
1107 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
1108 map->sections = g_renew(MemoryRegionSection, map->sections,
1109 map->sections_nb_alloc);
1111 map->sections[map->sections_nb] = *section;
1112 memory_region_ref(section->mr);
1113 return map->sections_nb++;
1116 static void phys_section_destroy(MemoryRegion *mr)
1118 bool have_sub_page = mr->subpage;
1120 memory_region_unref(mr);
1122 if (have_sub_page) {
1123 subpage_t *subpage = container_of(mr, subpage_t, iomem);
1124 object_unref(OBJECT(&subpage->iomem));
1125 g_free(subpage);
1129 static void phys_sections_free(PhysPageMap *map)
1131 while (map->sections_nb > 0) {
1132 MemoryRegionSection *section = &map->sections[--map->sections_nb];
1133 phys_section_destroy(section->mr);
1135 g_free(map->sections);
1136 g_free(map->nodes);
1139 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
1141 subpage_t *subpage;
1142 hwaddr base = section->offset_within_address_space
1143 & TARGET_PAGE_MASK;
1144 MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
1145 d->map.nodes, d->map.sections);
1146 MemoryRegionSection subsection = {
1147 .offset_within_address_space = base,
1148 .size = int128_make64(TARGET_PAGE_SIZE),
1150 hwaddr start, end;
1152 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
1154 if (!(existing->mr->subpage)) {
1155 subpage = subpage_init(d->as, base);
1156 subsection.address_space = d->as;
1157 subsection.mr = &subpage->iomem;
1158 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
1159 phys_section_add(&d->map, &subsection));
1160 } else {
1161 subpage = container_of(existing->mr, subpage_t, iomem);
1163 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
1164 end = start + int128_get64(section->size) - 1;
1165 subpage_register(subpage, start, end,
1166 phys_section_add(&d->map, section));
1170 static void register_multipage(AddressSpaceDispatch *d,
1171 MemoryRegionSection *section)
1173 hwaddr start_addr = section->offset_within_address_space;
1174 uint16_t section_index = phys_section_add(&d->map, section);
1175 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1176 TARGET_PAGE_BITS));
1178 assert(num_pages);
1179 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
1182 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
1184 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1185 AddressSpaceDispatch *d = as->next_dispatch;
1186 MemoryRegionSection now = *section, remain = *section;
1187 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
1189 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
1190 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
1191 - now.offset_within_address_space;
1193 now.size = int128_min(int128_make64(left), now.size);
1194 register_subpage(d, &now);
1195 } else {
1196 now.size = int128_zero();
1198 while (int128_ne(remain.size, now.size)) {
1199 remain.size = int128_sub(remain.size, now.size);
1200 remain.offset_within_address_space += int128_get64(now.size);
1201 remain.offset_within_region += int128_get64(now.size);
1202 now = remain;
1203 if (int128_lt(remain.size, page_size)) {
1204 register_subpage(d, &now);
1205 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
1206 now.size = page_size;
1207 register_subpage(d, &now);
1208 } else {
1209 now.size = int128_and(now.size, int128_neg(page_size));
1210 register_multipage(d, &now);
1215 void qemu_flush_coalesced_mmio_buffer(void)
1217 if (kvm_enabled())
1218 kvm_flush_coalesced_mmio_buffer();
1221 void qemu_mutex_lock_ramlist(void)
1223 qemu_mutex_lock(&ram_list.mutex);
1226 void qemu_mutex_unlock_ramlist(void)
1228 qemu_mutex_unlock(&ram_list.mutex);
1231 #ifdef __linux__
1232 static void *file_ram_alloc(RAMBlock *block,
1233 ram_addr_t memory,
1234 const char *path,
1235 Error **errp)
1237 bool unlink_on_error = false;
1238 char *filename;
1239 char *sanitized_name;
1240 char *c;
1241 void *area;
1242 int fd;
1243 int64_t page_size;
1245 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1246 error_setg(errp,
1247 "host lacks kvm mmu notifiers, -mem-path unsupported");
1248 return NULL;
1251 for (;;) {
1252 fd = open(path, O_RDWR);
1253 if (fd >= 0) {
1254 /* @path names an existing file, use it */
1255 break;
1257 if (errno == ENOENT) {
1258 /* @path names a file that doesn't exist, create it */
1259 fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
1260 if (fd >= 0) {
1261 unlink_on_error = true;
1262 break;
1264 } else if (errno == EISDIR) {
1265 /* @path names a directory, create a file there */
1266 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1267 sanitized_name = g_strdup(memory_region_name(block->mr));
1268 for (c = sanitized_name; *c != '\0'; c++) {
1269 if (*c == '/') {
1270 *c = '_';
1274 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1275 sanitized_name);
1276 g_free(sanitized_name);
1278 fd = mkstemp(filename);
1279 if (fd >= 0) {
1280 unlink(filename);
1281 g_free(filename);
1282 break;
1284 g_free(filename);
1286 if (errno != EEXIST && errno != EINTR) {
1287 error_setg_errno(errp, errno,
1288 "can't open backing store %s for guest RAM",
1289 path);
1290 goto error;
1293 * Try again on EINTR and EEXIST. The latter happens when
1294 * something else creates the file between our two open().
1298 page_size = qemu_fd_getpagesize(fd);
1299 block->mr->align = page_size;
1301 if (memory < page_size) {
1302 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
1303 "or larger than page size 0x%" PRIx64,
1304 memory, page_size);
1305 goto error;
1308 memory = ROUND_UP(memory, page_size);
1311 * ftruncate is not supported by hugetlbfs in older
1312 * hosts, so don't bother bailing out on errors.
1313 * If anything goes wrong with it under other filesystems,
1314 * mmap will fail.
1316 if (ftruncate(fd, memory)) {
1317 perror("ftruncate");
1320 area = qemu_ram_mmap(fd, memory, page_size, block->flags & RAM_SHARED);
1321 if (area == MAP_FAILED) {
1322 error_setg_errno(errp, errno,
1323 "unable to map backing store for guest RAM");
1324 close(fd);
1325 goto error;
1328 if (mem_prealloc) {
1329 os_mem_prealloc(fd, area, memory);
1332 block->fd = fd;
1333 return area;
1335 error:
1336 if (unlink_on_error) {
1337 unlink(path);
1339 close(fd);
1340 return NULL;
1342 #endif
1344 /* Called with the ramlist lock held. */
1345 static ram_addr_t find_ram_offset(ram_addr_t size)
1347 RAMBlock *block, *next_block;
1348 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1350 assert(size != 0); /* it would hand out same offset multiple times */
1352 if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
1353 return 0;
1356 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1357 ram_addr_t end, next = RAM_ADDR_MAX;
1359 end = block->offset + block->max_length;
1361 QLIST_FOREACH_RCU(next_block, &ram_list.blocks, next) {
1362 if (next_block->offset >= end) {
1363 next = MIN(next, next_block->offset);
1366 if (next - end >= size && next - end < mingap) {
1367 offset = end;
1368 mingap = next - end;
1372 if (offset == RAM_ADDR_MAX) {
1373 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1374 (uint64_t)size);
1375 abort();
1378 return offset;
1381 ram_addr_t last_ram_offset(void)
1383 RAMBlock *block;
1384 ram_addr_t last = 0;
1386 rcu_read_lock();
1387 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1388 last = MAX(last, block->offset + block->max_length);
1390 rcu_read_unlock();
1391 return last;
1394 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1396 int ret;
1398 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1399 if (!machine_dump_guest_core(current_machine)) {
1400 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1401 if (ret) {
1402 perror("qemu_madvise");
1403 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1404 "but dump_guest_core=off specified\n");
1409 /* Called within an RCU critical section, or while the ramlist lock
1410 * is held.
1412 static RAMBlock *find_ram_block(ram_addr_t addr)
1414 RAMBlock *block;
1416 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1417 if (block->offset == addr) {
1418 return block;
1422 return NULL;
1425 const char *qemu_ram_get_idstr(RAMBlock *rb)
1427 return rb->idstr;
1430 /* Called with iothread lock held. */
1431 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1433 RAMBlock *new_block, *block;
1435 rcu_read_lock();
1436 new_block = find_ram_block(addr);
1437 assert(new_block);
1438 assert(!new_block->idstr[0]);
1440 if (dev) {
1441 char *id = qdev_get_dev_path(dev);
1442 if (id) {
1443 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1444 g_free(id);
1447 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1449 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1450 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1451 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1452 new_block->idstr);
1453 abort();
1456 rcu_read_unlock();
1459 /* Called with iothread lock held. */
1460 void qemu_ram_unset_idstr(ram_addr_t addr)
1462 RAMBlock *block;
1464 /* FIXME: arch_init.c assumes that this is not called throughout
1465 * migration. Ignore the problem since hot-unplug during migration
1466 * does not work anyway.
1469 rcu_read_lock();
1470 block = find_ram_block(addr);
1471 if (block) {
1472 memset(block->idstr, 0, sizeof(block->idstr));
1474 rcu_read_unlock();
1477 static int memory_try_enable_merging(void *addr, size_t len)
1479 if (!machine_mem_merge(current_machine)) {
1480 /* disabled by the user */
1481 return 0;
1484 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1487 /* Only legal before guest might have detected the memory size: e.g. on
1488 * incoming migration, or right after reset.
1490 * As memory core doesn't know how is memory accessed, it is up to
1491 * resize callback to update device state and/or add assertions to detect
1492 * misuse, if necessary.
1494 int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp)
1496 RAMBlock *block = find_ram_block(base);
1498 assert(block);
1500 newsize = HOST_PAGE_ALIGN(newsize);
1502 if (block->used_length == newsize) {
1503 return 0;
1506 if (!(block->flags & RAM_RESIZEABLE)) {
1507 error_setg_errno(errp, EINVAL,
1508 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1509 " in != 0x" RAM_ADDR_FMT, block->idstr,
1510 newsize, block->used_length);
1511 return -EINVAL;
1514 if (block->max_length < newsize) {
1515 error_setg_errno(errp, EINVAL,
1516 "Length too large: %s: 0x" RAM_ADDR_FMT
1517 " > 0x" RAM_ADDR_FMT, block->idstr,
1518 newsize, block->max_length);
1519 return -EINVAL;
1522 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
1523 block->used_length = newsize;
1524 cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
1525 DIRTY_CLIENTS_ALL);
1526 memory_region_set_size(block->mr, newsize);
1527 if (block->resized) {
1528 block->resized(block->idstr, newsize, block->host);
1530 return 0;
1533 /* Called with ram_list.mutex held */
1534 static void dirty_memory_extend(ram_addr_t old_ram_size,
1535 ram_addr_t new_ram_size)
1537 ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size,
1538 DIRTY_MEMORY_BLOCK_SIZE);
1539 ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size,
1540 DIRTY_MEMORY_BLOCK_SIZE);
1541 int i;
1543 /* Only need to extend if block count increased */
1544 if (new_num_blocks <= old_num_blocks) {
1545 return;
1548 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1549 DirtyMemoryBlocks *old_blocks;
1550 DirtyMemoryBlocks *new_blocks;
1551 int j;
1553 old_blocks = atomic_rcu_read(&ram_list.dirty_memory[i]);
1554 new_blocks = g_malloc(sizeof(*new_blocks) +
1555 sizeof(new_blocks->blocks[0]) * new_num_blocks);
1557 if (old_num_blocks) {
1558 memcpy(new_blocks->blocks, old_blocks->blocks,
1559 old_num_blocks * sizeof(old_blocks->blocks[0]));
1562 for (j = old_num_blocks; j < new_num_blocks; j++) {
1563 new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
1566 atomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
1568 if (old_blocks) {
1569 g_free_rcu(old_blocks, rcu);
1574 static void ram_block_add(RAMBlock *new_block, Error **errp)
1576 RAMBlock *block;
1577 RAMBlock *last_block = NULL;
1578 ram_addr_t old_ram_size, new_ram_size;
1579 Error *err = NULL;
1581 old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1583 qemu_mutex_lock_ramlist();
1584 new_block->offset = find_ram_offset(new_block->max_length);
1586 if (!new_block->host) {
1587 if (xen_enabled()) {
1588 xen_ram_alloc(new_block->offset, new_block->max_length,
1589 new_block->mr, &err);
1590 if (err) {
1591 error_propagate(errp, err);
1592 qemu_mutex_unlock_ramlist();
1593 return;
1595 } else {
1596 new_block->host = phys_mem_alloc(new_block->max_length,
1597 &new_block->mr->align);
1598 if (!new_block->host) {
1599 error_setg_errno(errp, errno,
1600 "cannot set up guest memory '%s'",
1601 memory_region_name(new_block->mr));
1602 qemu_mutex_unlock_ramlist();
1603 return;
1605 memory_try_enable_merging(new_block->host, new_block->max_length);
1609 new_ram_size = MAX(old_ram_size,
1610 (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
1611 if (new_ram_size > old_ram_size) {
1612 migration_bitmap_extend(old_ram_size, new_ram_size);
1613 dirty_memory_extend(old_ram_size, new_ram_size);
1615 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1616 * QLIST (which has an RCU-friendly variant) does not have insertion at
1617 * tail, so save the last element in last_block.
1619 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1620 last_block = block;
1621 if (block->max_length < new_block->max_length) {
1622 break;
1625 if (block) {
1626 QLIST_INSERT_BEFORE_RCU(block, new_block, next);
1627 } else if (last_block) {
1628 QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
1629 } else { /* list is empty */
1630 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
1632 ram_list.mru_block = NULL;
1634 /* Write list before version */
1635 smp_wmb();
1636 ram_list.version++;
1637 qemu_mutex_unlock_ramlist();
1639 cpu_physical_memory_set_dirty_range(new_block->offset,
1640 new_block->used_length,
1641 DIRTY_CLIENTS_ALL);
1643 if (new_block->host) {
1644 qemu_ram_setup_dump(new_block->host, new_block->max_length);
1645 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
1646 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
1647 if (kvm_enabled()) {
1648 kvm_setup_guest_memory(new_block->host, new_block->max_length);
1653 #ifdef __linux__
1654 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
1655 bool share, const char *mem_path,
1656 Error **errp)
1658 RAMBlock *new_block;
1659 Error *local_err = NULL;
1661 if (xen_enabled()) {
1662 error_setg(errp, "-mem-path not supported with Xen");
1663 return NULL;
1666 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1668 * file_ram_alloc() needs to allocate just like
1669 * phys_mem_alloc, but we haven't bothered to provide
1670 * a hook there.
1672 error_setg(errp,
1673 "-mem-path not supported with this accelerator");
1674 return NULL;
1677 size = HOST_PAGE_ALIGN(size);
1678 new_block = g_malloc0(sizeof(*new_block));
1679 new_block->mr = mr;
1680 new_block->used_length = size;
1681 new_block->max_length = size;
1682 new_block->flags = share ? RAM_SHARED : 0;
1683 new_block->host = file_ram_alloc(new_block, size,
1684 mem_path, errp);
1685 if (!new_block->host) {
1686 g_free(new_block);
1687 return NULL;
1690 ram_block_add(new_block, &local_err);
1691 if (local_err) {
1692 g_free(new_block);
1693 error_propagate(errp, local_err);
1694 return NULL;
1696 return new_block;
1698 #endif
1700 static
1701 RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
1702 void (*resized)(const char*,
1703 uint64_t length,
1704 void *host),
1705 void *host, bool resizeable,
1706 MemoryRegion *mr, Error **errp)
1708 RAMBlock *new_block;
1709 Error *local_err = NULL;
1711 size = HOST_PAGE_ALIGN(size);
1712 max_size = HOST_PAGE_ALIGN(max_size);
1713 new_block = g_malloc0(sizeof(*new_block));
1714 new_block->mr = mr;
1715 new_block->resized = resized;
1716 new_block->used_length = size;
1717 new_block->max_length = max_size;
1718 assert(max_size >= size);
1719 new_block->fd = -1;
1720 new_block->host = host;
1721 if (host) {
1722 new_block->flags |= RAM_PREALLOC;
1724 if (resizeable) {
1725 new_block->flags |= RAM_RESIZEABLE;
1727 ram_block_add(new_block, &local_err);
1728 if (local_err) {
1729 g_free(new_block);
1730 error_propagate(errp, local_err);
1731 return NULL;
1733 return new_block;
1736 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1737 MemoryRegion *mr, Error **errp)
1739 return qemu_ram_alloc_internal(size, size, NULL, host, false, mr, errp);
1742 RAMBlock *qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
1744 return qemu_ram_alloc_internal(size, size, NULL, NULL, false, mr, errp);
1747 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
1748 void (*resized)(const char*,
1749 uint64_t length,
1750 void *host),
1751 MemoryRegion *mr, Error **errp)
1753 return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true, mr, errp);
1756 static void reclaim_ramblock(RAMBlock *block)
1758 if (block->flags & RAM_PREALLOC) {
1760 } else if (xen_enabled()) {
1761 xen_invalidate_map_cache_entry(block->host);
1762 #ifndef _WIN32
1763 } else if (block->fd >= 0) {
1764 qemu_ram_munmap(block->host, block->max_length);
1765 close(block->fd);
1766 #endif
1767 } else {
1768 qemu_anon_ram_free(block->host, block->max_length);
1770 g_free(block);
1773 void qemu_ram_free(RAMBlock *block)
1775 qemu_mutex_lock_ramlist();
1776 QLIST_REMOVE_RCU(block, next);
1777 ram_list.mru_block = NULL;
1778 /* Write list before version */
1779 smp_wmb();
1780 ram_list.version++;
1781 call_rcu(block, reclaim_ramblock, rcu);
1782 qemu_mutex_unlock_ramlist();
1785 #ifndef _WIN32
1786 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1788 RAMBlock *block;
1789 ram_addr_t offset;
1790 int flags;
1791 void *area, *vaddr;
1793 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1794 offset = addr - block->offset;
1795 if (offset < block->max_length) {
1796 vaddr = ramblock_ptr(block, offset);
1797 if (block->flags & RAM_PREALLOC) {
1799 } else if (xen_enabled()) {
1800 abort();
1801 } else {
1802 flags = MAP_FIXED;
1803 if (block->fd >= 0) {
1804 flags |= (block->flags & RAM_SHARED ?
1805 MAP_SHARED : MAP_PRIVATE);
1806 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1807 flags, block->fd, offset);
1808 } else {
1810 * Remap needs to match alloc. Accelerators that
1811 * set phys_mem_alloc never remap. If they did,
1812 * we'd need a remap hook here.
1814 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1816 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1817 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1818 flags, -1, 0);
1820 if (area != vaddr) {
1821 fprintf(stderr, "Could not remap addr: "
1822 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1823 length, addr);
1824 exit(1);
1826 memory_try_enable_merging(vaddr, length);
1827 qemu_ram_setup_dump(vaddr, length);
1832 #endif /* !_WIN32 */
1834 int qemu_get_ram_fd(ram_addr_t addr)
1836 RAMBlock *block;
1837 int fd;
1839 rcu_read_lock();
1840 block = qemu_get_ram_block(addr);
1841 fd = block->fd;
1842 rcu_read_unlock();
1843 return fd;
1846 void qemu_set_ram_fd(ram_addr_t addr, int fd)
1848 RAMBlock *block;
1850 rcu_read_lock();
1851 block = qemu_get_ram_block(addr);
1852 block->fd = fd;
1853 rcu_read_unlock();
1856 void *qemu_get_ram_block_host_ptr(ram_addr_t addr)
1858 RAMBlock *block;
1859 void *ptr;
1861 rcu_read_lock();
1862 block = qemu_get_ram_block(addr);
1863 ptr = ramblock_ptr(block, 0);
1864 rcu_read_unlock();
1865 return ptr;
1868 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1869 * This should not be used for general purpose DMA. Use address_space_map
1870 * or address_space_rw instead. For local memory (e.g. video ram) that the
1871 * device owns, use memory_region_get_ram_ptr.
1873 * Called within RCU critical section.
1875 void *qemu_get_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
1877 RAMBlock *block = ram_block;
1879 if (block == NULL) {
1880 block = qemu_get_ram_block(addr);
1883 if (xen_enabled() && block->host == NULL) {
1884 /* We need to check if the requested address is in the RAM
1885 * because we don't want to map the entire memory in QEMU.
1886 * In that case just map until the end of the page.
1888 if (block->offset == 0) {
1889 return xen_map_cache(addr, 0, 0);
1892 block->host = xen_map_cache(block->offset, block->max_length, 1);
1894 return ramblock_ptr(block, addr - block->offset);
1897 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1898 * but takes a size argument.
1900 * Called within RCU critical section.
1902 static void *qemu_ram_ptr_length(RAMBlock *ram_block, ram_addr_t addr,
1903 hwaddr *size)
1905 RAMBlock *block = ram_block;
1906 ram_addr_t offset_inside_block;
1907 if (*size == 0) {
1908 return NULL;
1911 if (block == NULL) {
1912 block = qemu_get_ram_block(addr);
1914 offset_inside_block = addr - block->offset;
1915 *size = MIN(*size, block->max_length - offset_inside_block);
1917 if (xen_enabled() && block->host == NULL) {
1918 /* We need to check if the requested address is in the RAM
1919 * because we don't want to map the entire memory in QEMU.
1920 * In that case just map the requested area.
1922 if (block->offset == 0) {
1923 return xen_map_cache(addr, *size, 1);
1926 block->host = xen_map_cache(block->offset, block->max_length, 1);
1929 return ramblock_ptr(block, offset_inside_block);
1933 * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
1934 * in that RAMBlock.
1936 * ptr: Host pointer to look up
1937 * round_offset: If true round the result offset down to a page boundary
1938 * *ram_addr: set to result ram_addr
1939 * *offset: set to result offset within the RAMBlock
1941 * Returns: RAMBlock (or NULL if not found)
1943 * By the time this function returns, the returned pointer is not protected
1944 * by RCU anymore. If the caller is not within an RCU critical section and
1945 * does not hold the iothread lock, it must have other means of protecting the
1946 * pointer, such as a reference to the region that includes the incoming
1947 * ram_addr_t.
1949 RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
1950 ram_addr_t *ram_addr,
1951 ram_addr_t *offset)
1953 RAMBlock *block;
1954 uint8_t *host = ptr;
1956 if (xen_enabled()) {
1957 rcu_read_lock();
1958 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1959 block = qemu_get_ram_block(*ram_addr);
1960 if (block) {
1961 *offset = (host - block->host);
1963 rcu_read_unlock();
1964 return block;
1967 rcu_read_lock();
1968 block = atomic_rcu_read(&ram_list.mru_block);
1969 if (block && block->host && host - block->host < block->max_length) {
1970 goto found;
1973 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1974 /* This case append when the block is not mapped. */
1975 if (block->host == NULL) {
1976 continue;
1978 if (host - block->host < block->max_length) {
1979 goto found;
1983 rcu_read_unlock();
1984 return NULL;
1986 found:
1987 *offset = (host - block->host);
1988 if (round_offset) {
1989 *offset &= TARGET_PAGE_MASK;
1991 *ram_addr = block->offset + *offset;
1992 rcu_read_unlock();
1993 return block;
1997 * Finds the named RAMBlock
1999 * name: The name of RAMBlock to find
2001 * Returns: RAMBlock (or NULL if not found)
2003 RAMBlock *qemu_ram_block_by_name(const char *name)
2005 RAMBlock *block;
2007 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
2008 if (!strcmp(name, block->idstr)) {
2009 return block;
2013 return NULL;
2016 /* Some of the softmmu routines need to translate from a host pointer
2017 (typically a TLB entry) back to a ram offset. */
2018 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
2020 RAMBlock *block;
2021 ram_addr_t offset; /* Not used */
2023 block = qemu_ram_block_from_host(ptr, false, ram_addr, &offset);
2025 if (!block) {
2026 return NULL;
2029 return block->mr;
2032 /* Called within RCU critical section. */
2033 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
2034 uint64_t val, unsigned size)
2036 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
2037 tb_invalidate_phys_page_fast(ram_addr, size);
2039 switch (size) {
2040 case 1:
2041 stb_p(qemu_get_ram_ptr(NULL, ram_addr), val);
2042 break;
2043 case 2:
2044 stw_p(qemu_get_ram_ptr(NULL, ram_addr), val);
2045 break;
2046 case 4:
2047 stl_p(qemu_get_ram_ptr(NULL, ram_addr), val);
2048 break;
2049 default:
2050 abort();
2052 /* Set both VGA and migration bits for simplicity and to remove
2053 * the notdirty callback faster.
2055 cpu_physical_memory_set_dirty_range(ram_addr, size,
2056 DIRTY_CLIENTS_NOCODE);
2057 /* we remove the notdirty callback only if the code has been
2058 flushed */
2059 if (!cpu_physical_memory_is_clean(ram_addr)) {
2060 tlb_set_dirty(current_cpu, current_cpu->mem_io_vaddr);
2064 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
2065 unsigned size, bool is_write)
2067 return is_write;
2070 static const MemoryRegionOps notdirty_mem_ops = {
2071 .write = notdirty_mem_write,
2072 .valid.accepts = notdirty_mem_accepts,
2073 .endianness = DEVICE_NATIVE_ENDIAN,
2076 /* Generate a debug exception if a watchpoint has been hit. */
2077 static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
2079 CPUState *cpu = current_cpu;
2080 CPUClass *cc = CPU_GET_CLASS(cpu);
2081 CPUArchState *env = cpu->env_ptr;
2082 target_ulong pc, cs_base;
2083 target_ulong vaddr;
2084 CPUWatchpoint *wp;
2085 int cpu_flags;
2087 if (cpu->watchpoint_hit) {
2088 /* We re-entered the check after replacing the TB. Now raise
2089 * the debug interrupt so that is will trigger after the
2090 * current instruction. */
2091 cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
2092 return;
2094 vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2095 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
2096 if (cpu_watchpoint_address_matches(wp, vaddr, len)
2097 && (wp->flags & flags)) {
2098 if (flags == BP_MEM_READ) {
2099 wp->flags |= BP_WATCHPOINT_HIT_READ;
2100 } else {
2101 wp->flags |= BP_WATCHPOINT_HIT_WRITE;
2103 wp->hitaddr = vaddr;
2104 wp->hitattrs = attrs;
2105 if (!cpu->watchpoint_hit) {
2106 if (wp->flags & BP_CPU &&
2107 !cc->debug_check_watchpoint(cpu, wp)) {
2108 wp->flags &= ~BP_WATCHPOINT_HIT;
2109 continue;
2111 cpu->watchpoint_hit = wp;
2112 tb_check_watchpoint(cpu);
2113 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2114 cpu->exception_index = EXCP_DEBUG;
2115 cpu_loop_exit(cpu);
2116 } else {
2117 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2118 tb_gen_code(cpu, pc, cs_base, cpu_flags, 1);
2119 cpu_resume_from_signal(cpu, NULL);
2122 } else {
2123 wp->flags &= ~BP_WATCHPOINT_HIT;
2128 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2129 so these check for a hit then pass through to the normal out-of-line
2130 phys routines. */
2131 static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
2132 unsigned size, MemTxAttrs attrs)
2134 MemTxResult res;
2135 uint64_t data;
2136 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2137 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
2139 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
2140 switch (size) {
2141 case 1:
2142 data = address_space_ldub(as, addr, attrs, &res);
2143 break;
2144 case 2:
2145 data = address_space_lduw(as, addr, attrs, &res);
2146 break;
2147 case 4:
2148 data = address_space_ldl(as, addr, attrs, &res);
2149 break;
2150 default: abort();
2152 *pdata = data;
2153 return res;
2156 static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
2157 uint64_t val, unsigned size,
2158 MemTxAttrs attrs)
2160 MemTxResult res;
2161 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2162 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
2164 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
2165 switch (size) {
2166 case 1:
2167 address_space_stb(as, addr, val, attrs, &res);
2168 break;
2169 case 2:
2170 address_space_stw(as, addr, val, attrs, &res);
2171 break;
2172 case 4:
2173 address_space_stl(as, addr, val, attrs, &res);
2174 break;
2175 default: abort();
2177 return res;
2180 static const MemoryRegionOps watch_mem_ops = {
2181 .read_with_attrs = watch_mem_read,
2182 .write_with_attrs = watch_mem_write,
2183 .endianness = DEVICE_NATIVE_ENDIAN,
2186 static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
2187 unsigned len, MemTxAttrs attrs)
2189 subpage_t *subpage = opaque;
2190 uint8_t buf[8];
2191 MemTxResult res;
2193 #if defined(DEBUG_SUBPAGE)
2194 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
2195 subpage, len, addr);
2196 #endif
2197 res = address_space_read(subpage->as, addr + subpage->base,
2198 attrs, buf, len);
2199 if (res) {
2200 return res;
2202 switch (len) {
2203 case 1:
2204 *data = ldub_p(buf);
2205 return MEMTX_OK;
2206 case 2:
2207 *data = lduw_p(buf);
2208 return MEMTX_OK;
2209 case 4:
2210 *data = ldl_p(buf);
2211 return MEMTX_OK;
2212 case 8:
2213 *data = ldq_p(buf);
2214 return MEMTX_OK;
2215 default:
2216 abort();
2220 static MemTxResult subpage_write(void *opaque, hwaddr addr,
2221 uint64_t value, unsigned len, MemTxAttrs attrs)
2223 subpage_t *subpage = opaque;
2224 uint8_t buf[8];
2226 #if defined(DEBUG_SUBPAGE)
2227 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
2228 " value %"PRIx64"\n",
2229 __func__, subpage, len, addr, value);
2230 #endif
2231 switch (len) {
2232 case 1:
2233 stb_p(buf, value);
2234 break;
2235 case 2:
2236 stw_p(buf, value);
2237 break;
2238 case 4:
2239 stl_p(buf, value);
2240 break;
2241 case 8:
2242 stq_p(buf, value);
2243 break;
2244 default:
2245 abort();
2247 return address_space_write(subpage->as, addr + subpage->base,
2248 attrs, buf, len);
2251 static bool subpage_accepts(void *opaque, hwaddr addr,
2252 unsigned len, bool is_write)
2254 subpage_t *subpage = opaque;
2255 #if defined(DEBUG_SUBPAGE)
2256 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
2257 __func__, subpage, is_write ? 'w' : 'r', len, addr);
2258 #endif
2260 return address_space_access_valid(subpage->as, addr + subpage->base,
2261 len, is_write);
2264 static const MemoryRegionOps subpage_ops = {
2265 .read_with_attrs = subpage_read,
2266 .write_with_attrs = subpage_write,
2267 .impl.min_access_size = 1,
2268 .impl.max_access_size = 8,
2269 .valid.min_access_size = 1,
2270 .valid.max_access_size = 8,
2271 .valid.accepts = subpage_accepts,
2272 .endianness = DEVICE_NATIVE_ENDIAN,
2275 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2276 uint16_t section)
2278 int idx, eidx;
2280 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2281 return -1;
2282 idx = SUBPAGE_IDX(start);
2283 eidx = SUBPAGE_IDX(end);
2284 #if defined(DEBUG_SUBPAGE)
2285 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2286 __func__, mmio, start, end, idx, eidx, section);
2287 #endif
2288 for (; idx <= eidx; idx++) {
2289 mmio->sub_section[idx] = section;
2292 return 0;
2295 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
2297 subpage_t *mmio;
2299 mmio = g_malloc0(sizeof(subpage_t));
2301 mmio->as = as;
2302 mmio->base = base;
2303 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
2304 NULL, TARGET_PAGE_SIZE);
2305 mmio->iomem.subpage = true;
2306 #if defined(DEBUG_SUBPAGE)
2307 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
2308 mmio, base, TARGET_PAGE_SIZE);
2309 #endif
2310 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
2312 return mmio;
2315 static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
2316 MemoryRegion *mr)
2318 assert(as);
2319 MemoryRegionSection section = {
2320 .address_space = as,
2321 .mr = mr,
2322 .offset_within_address_space = 0,
2323 .offset_within_region = 0,
2324 .size = int128_2_64(),
2327 return phys_section_add(map, &section);
2330 MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index, MemTxAttrs attrs)
2332 int asidx = cpu_asidx_from_attrs(cpu, attrs);
2333 CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
2334 AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
2335 MemoryRegionSection *sections = d->map.sections;
2337 return sections[index & ~TARGET_PAGE_MASK].mr;
2340 static void io_mem_init(void)
2342 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, NULL, UINT64_MAX);
2343 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
2344 NULL, UINT64_MAX);
2345 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
2346 NULL, UINT64_MAX);
2347 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
2348 NULL, UINT64_MAX);
2351 static void mem_begin(MemoryListener *listener)
2353 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2354 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
2355 uint16_t n;
2357 n = dummy_section(&d->map, as, &io_mem_unassigned);
2358 assert(n == PHYS_SECTION_UNASSIGNED);
2359 n = dummy_section(&d->map, as, &io_mem_notdirty);
2360 assert(n == PHYS_SECTION_NOTDIRTY);
2361 n = dummy_section(&d->map, as, &io_mem_rom);
2362 assert(n == PHYS_SECTION_ROM);
2363 n = dummy_section(&d->map, as, &io_mem_watch);
2364 assert(n == PHYS_SECTION_WATCH);
2366 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
2367 d->as = as;
2368 as->next_dispatch = d;
2371 static void address_space_dispatch_free(AddressSpaceDispatch *d)
2373 phys_sections_free(&d->map);
2374 g_free(d);
2377 static void mem_commit(MemoryListener *listener)
2379 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2380 AddressSpaceDispatch *cur = as->dispatch;
2381 AddressSpaceDispatch *next = as->next_dispatch;
2383 phys_page_compact_all(next, next->map.nodes_nb);
2385 atomic_rcu_set(&as->dispatch, next);
2386 if (cur) {
2387 call_rcu(cur, address_space_dispatch_free, rcu);
2391 static void tcg_commit(MemoryListener *listener)
2393 CPUAddressSpace *cpuas;
2394 AddressSpaceDispatch *d;
2396 /* since each CPU stores ram addresses in its TLB cache, we must
2397 reset the modified entries */
2398 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
2399 cpu_reloading_memory_map();
2400 /* The CPU and TLB are protected by the iothread lock.
2401 * We reload the dispatch pointer now because cpu_reloading_memory_map()
2402 * may have split the RCU critical section.
2404 d = atomic_rcu_read(&cpuas->as->dispatch);
2405 cpuas->memory_dispatch = d;
2406 tlb_flush(cpuas->cpu, 1);
2409 void address_space_init_dispatch(AddressSpace *as)
2411 as->dispatch = NULL;
2412 as->dispatch_listener = (MemoryListener) {
2413 .begin = mem_begin,
2414 .commit = mem_commit,
2415 .region_add = mem_add,
2416 .region_nop = mem_add,
2417 .priority = 0,
2419 memory_listener_register(&as->dispatch_listener, as);
2422 void address_space_unregister(AddressSpace *as)
2424 memory_listener_unregister(&as->dispatch_listener);
2427 void address_space_destroy_dispatch(AddressSpace *as)
2429 AddressSpaceDispatch *d = as->dispatch;
2431 atomic_rcu_set(&as->dispatch, NULL);
2432 if (d) {
2433 call_rcu(d, address_space_dispatch_free, rcu);
2437 static void memory_map_init(void)
2439 system_memory = g_malloc(sizeof(*system_memory));
2441 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
2442 address_space_init(&address_space_memory, system_memory, "memory");
2444 system_io = g_malloc(sizeof(*system_io));
2445 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2446 65536);
2447 address_space_init(&address_space_io, system_io, "I/O");
2450 MemoryRegion *get_system_memory(void)
2452 return system_memory;
2455 MemoryRegion *get_system_io(void)
2457 return system_io;
2460 #endif /* !defined(CONFIG_USER_ONLY) */
2462 /* physical memory access (slow version, mainly for debug) */
2463 #if defined(CONFIG_USER_ONLY)
2464 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2465 uint8_t *buf, int len, int is_write)
2467 int l, flags;
2468 target_ulong page;
2469 void * p;
2471 while (len > 0) {
2472 page = addr & TARGET_PAGE_MASK;
2473 l = (page + TARGET_PAGE_SIZE) - addr;
2474 if (l > len)
2475 l = len;
2476 flags = page_get_flags(page);
2477 if (!(flags & PAGE_VALID))
2478 return -1;
2479 if (is_write) {
2480 if (!(flags & PAGE_WRITE))
2481 return -1;
2482 /* XXX: this code should not depend on lock_user */
2483 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2484 return -1;
2485 memcpy(p, buf, l);
2486 unlock_user(p, addr, l);
2487 } else {
2488 if (!(flags & PAGE_READ))
2489 return -1;
2490 /* XXX: this code should not depend on lock_user */
2491 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2492 return -1;
2493 memcpy(buf, p, l);
2494 unlock_user(p, addr, 0);
2496 len -= l;
2497 buf += l;
2498 addr += l;
2500 return 0;
2503 #else
2505 static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
2506 hwaddr length)
2508 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
2509 /* No early return if dirty_log_mask is or becomes 0, because
2510 * cpu_physical_memory_set_dirty_range will still call
2511 * xen_modified_memory.
2513 if (dirty_log_mask) {
2514 dirty_log_mask =
2515 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
2517 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
2518 tb_invalidate_phys_range(addr, addr + length);
2519 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
2521 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
2524 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
2526 unsigned access_size_max = mr->ops->valid.max_access_size;
2528 /* Regions are assumed to support 1-4 byte accesses unless
2529 otherwise specified. */
2530 if (access_size_max == 0) {
2531 access_size_max = 4;
2534 /* Bound the maximum access by the alignment of the address. */
2535 if (!mr->ops->impl.unaligned) {
2536 unsigned align_size_max = addr & -addr;
2537 if (align_size_max != 0 && align_size_max < access_size_max) {
2538 access_size_max = align_size_max;
2542 /* Don't attempt accesses larger than the maximum. */
2543 if (l > access_size_max) {
2544 l = access_size_max;
2546 l = pow2floor(l);
2548 return l;
2551 static bool prepare_mmio_access(MemoryRegion *mr)
2553 bool unlocked = !qemu_mutex_iothread_locked();
2554 bool release_lock = false;
2556 if (unlocked && mr->global_locking) {
2557 qemu_mutex_lock_iothread();
2558 unlocked = false;
2559 release_lock = true;
2561 if (mr->flush_coalesced_mmio) {
2562 if (unlocked) {
2563 qemu_mutex_lock_iothread();
2565 qemu_flush_coalesced_mmio_buffer();
2566 if (unlocked) {
2567 qemu_mutex_unlock_iothread();
2571 return release_lock;
2574 /* Called within RCU critical section. */
2575 static MemTxResult address_space_write_continue(AddressSpace *as, hwaddr addr,
2576 MemTxAttrs attrs,
2577 const uint8_t *buf,
2578 int len, hwaddr addr1,
2579 hwaddr l, MemoryRegion *mr)
2581 uint8_t *ptr;
2582 uint64_t val;
2583 MemTxResult result = MEMTX_OK;
2584 bool release_lock = false;
2586 for (;;) {
2587 if (!memory_access_is_direct(mr, true)) {
2588 release_lock |= prepare_mmio_access(mr);
2589 l = memory_access_size(mr, l, addr1);
2590 /* XXX: could force current_cpu to NULL to avoid
2591 potential bugs */
2592 switch (l) {
2593 case 8:
2594 /* 64 bit write access */
2595 val = ldq_p(buf);
2596 result |= memory_region_dispatch_write(mr, addr1, val, 8,
2597 attrs);
2598 break;
2599 case 4:
2600 /* 32 bit write access */
2601 val = ldl_p(buf);
2602 result |= memory_region_dispatch_write(mr, addr1, val, 4,
2603 attrs);
2604 break;
2605 case 2:
2606 /* 16 bit write access */
2607 val = lduw_p(buf);
2608 result |= memory_region_dispatch_write(mr, addr1, val, 2,
2609 attrs);
2610 break;
2611 case 1:
2612 /* 8 bit write access */
2613 val = ldub_p(buf);
2614 result |= memory_region_dispatch_write(mr, addr1, val, 1,
2615 attrs);
2616 break;
2617 default:
2618 abort();
2620 } else {
2621 addr1 += memory_region_get_ram_addr(mr);
2622 /* RAM case */
2623 ptr = qemu_get_ram_ptr(mr->ram_block, addr1);
2624 memcpy(ptr, buf, l);
2625 invalidate_and_set_dirty(mr, addr1, l);
2628 if (release_lock) {
2629 qemu_mutex_unlock_iothread();
2630 release_lock = false;
2633 len -= l;
2634 buf += l;
2635 addr += l;
2637 if (!len) {
2638 break;
2641 l = len;
2642 mr = address_space_translate(as, addr, &addr1, &l, true);
2645 return result;
2648 MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2649 const uint8_t *buf, int len)
2651 hwaddr l;
2652 hwaddr addr1;
2653 MemoryRegion *mr;
2654 MemTxResult result = MEMTX_OK;
2656 if (len > 0) {
2657 rcu_read_lock();
2658 l = len;
2659 mr = address_space_translate(as, addr, &addr1, &l, true);
2660 result = address_space_write_continue(as, addr, attrs, buf, len,
2661 addr1, l, mr);
2662 rcu_read_unlock();
2665 return result;
2668 /* Called within RCU critical section. */
2669 MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr,
2670 MemTxAttrs attrs, uint8_t *buf,
2671 int len, hwaddr addr1, hwaddr l,
2672 MemoryRegion *mr)
2674 uint8_t *ptr;
2675 uint64_t val;
2676 MemTxResult result = MEMTX_OK;
2677 bool release_lock = false;
2679 for (;;) {
2680 if (!memory_access_is_direct(mr, false)) {
2681 /* I/O case */
2682 release_lock |= prepare_mmio_access(mr);
2683 l = memory_access_size(mr, l, addr1);
2684 switch (l) {
2685 case 8:
2686 /* 64 bit read access */
2687 result |= memory_region_dispatch_read(mr, addr1, &val, 8,
2688 attrs);
2689 stq_p(buf, val);
2690 break;
2691 case 4:
2692 /* 32 bit read access */
2693 result |= memory_region_dispatch_read(mr, addr1, &val, 4,
2694 attrs);
2695 stl_p(buf, val);
2696 break;
2697 case 2:
2698 /* 16 bit read access */
2699 result |= memory_region_dispatch_read(mr, addr1, &val, 2,
2700 attrs);
2701 stw_p(buf, val);
2702 break;
2703 case 1:
2704 /* 8 bit read access */
2705 result |= memory_region_dispatch_read(mr, addr1, &val, 1,
2706 attrs);
2707 stb_p(buf, val);
2708 break;
2709 default:
2710 abort();
2712 } else {
2713 /* RAM case */
2714 ptr = qemu_get_ram_ptr(mr->ram_block,
2715 memory_region_get_ram_addr(mr) + addr1);
2716 memcpy(buf, ptr, l);
2719 if (release_lock) {
2720 qemu_mutex_unlock_iothread();
2721 release_lock = false;
2724 len -= l;
2725 buf += l;
2726 addr += l;
2728 if (!len) {
2729 break;
2732 l = len;
2733 mr = address_space_translate(as, addr, &addr1, &l, false);
2736 return result;
2739 MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
2740 MemTxAttrs attrs, uint8_t *buf, int len)
2742 hwaddr l;
2743 hwaddr addr1;
2744 MemoryRegion *mr;
2745 MemTxResult result = MEMTX_OK;
2747 if (len > 0) {
2748 rcu_read_lock();
2749 l = len;
2750 mr = address_space_translate(as, addr, &addr1, &l, false);
2751 result = address_space_read_continue(as, addr, attrs, buf, len,
2752 addr1, l, mr);
2753 rcu_read_unlock();
2756 return result;
2759 MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2760 uint8_t *buf, int len, bool is_write)
2762 if (is_write) {
2763 return address_space_write(as, addr, attrs, (uint8_t *)buf, len);
2764 } else {
2765 return address_space_read(as, addr, attrs, (uint8_t *)buf, len);
2769 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2770 int len, int is_write)
2772 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
2773 buf, len, is_write);
2776 enum write_rom_type {
2777 WRITE_DATA,
2778 FLUSH_CACHE,
2781 static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
2782 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
2784 hwaddr l;
2785 uint8_t *ptr;
2786 hwaddr addr1;
2787 MemoryRegion *mr;
2789 rcu_read_lock();
2790 while (len > 0) {
2791 l = len;
2792 mr = address_space_translate(as, addr, &addr1, &l, true);
2794 if (!(memory_region_is_ram(mr) ||
2795 memory_region_is_romd(mr))) {
2796 l = memory_access_size(mr, l, addr1);
2797 } else {
2798 addr1 += memory_region_get_ram_addr(mr);
2799 /* ROM/RAM case */
2800 ptr = qemu_get_ram_ptr(mr->ram_block, addr1);
2801 switch (type) {
2802 case WRITE_DATA:
2803 memcpy(ptr, buf, l);
2804 invalidate_and_set_dirty(mr, addr1, l);
2805 break;
2806 case FLUSH_CACHE:
2807 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
2808 break;
2811 len -= l;
2812 buf += l;
2813 addr += l;
2815 rcu_read_unlock();
2818 /* used for ROM loading : can write in RAM and ROM */
2819 void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
2820 const uint8_t *buf, int len)
2822 cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
2825 void cpu_flush_icache_range(hwaddr start, int len)
2828 * This function should do the same thing as an icache flush that was
2829 * triggered from within the guest. For TCG we are always cache coherent,
2830 * so there is no need to flush anything. For KVM / Xen we need to flush
2831 * the host's instruction cache at least.
2833 if (tcg_enabled()) {
2834 return;
2837 cpu_physical_memory_write_rom_internal(&address_space_memory,
2838 start, NULL, len, FLUSH_CACHE);
2841 typedef struct {
2842 MemoryRegion *mr;
2843 void *buffer;
2844 hwaddr addr;
2845 hwaddr len;
2846 bool in_use;
2847 } BounceBuffer;
2849 static BounceBuffer bounce;
2851 typedef struct MapClient {
2852 QEMUBH *bh;
2853 QLIST_ENTRY(MapClient) link;
2854 } MapClient;
2856 QemuMutex map_client_list_lock;
2857 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2858 = QLIST_HEAD_INITIALIZER(map_client_list);
2860 static void cpu_unregister_map_client_do(MapClient *client)
2862 QLIST_REMOVE(client, link);
2863 g_free(client);
2866 static void cpu_notify_map_clients_locked(void)
2868 MapClient *client;
2870 while (!QLIST_EMPTY(&map_client_list)) {
2871 client = QLIST_FIRST(&map_client_list);
2872 qemu_bh_schedule(client->bh);
2873 cpu_unregister_map_client_do(client);
2877 void cpu_register_map_client(QEMUBH *bh)
2879 MapClient *client = g_malloc(sizeof(*client));
2881 qemu_mutex_lock(&map_client_list_lock);
2882 client->bh = bh;
2883 QLIST_INSERT_HEAD(&map_client_list, client, link);
2884 if (!atomic_read(&bounce.in_use)) {
2885 cpu_notify_map_clients_locked();
2887 qemu_mutex_unlock(&map_client_list_lock);
2890 void cpu_exec_init_all(void)
2892 qemu_mutex_init(&ram_list.mutex);
2893 io_mem_init();
2894 memory_map_init();
2895 qemu_mutex_init(&map_client_list_lock);
2898 void cpu_unregister_map_client(QEMUBH *bh)
2900 MapClient *client;
2902 qemu_mutex_lock(&map_client_list_lock);
2903 QLIST_FOREACH(client, &map_client_list, link) {
2904 if (client->bh == bh) {
2905 cpu_unregister_map_client_do(client);
2906 break;
2909 qemu_mutex_unlock(&map_client_list_lock);
2912 static void cpu_notify_map_clients(void)
2914 qemu_mutex_lock(&map_client_list_lock);
2915 cpu_notify_map_clients_locked();
2916 qemu_mutex_unlock(&map_client_list_lock);
2919 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2921 MemoryRegion *mr;
2922 hwaddr l, xlat;
2924 rcu_read_lock();
2925 while (len > 0) {
2926 l = len;
2927 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2928 if (!memory_access_is_direct(mr, is_write)) {
2929 l = memory_access_size(mr, l, addr);
2930 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2931 return false;
2935 len -= l;
2936 addr += l;
2938 rcu_read_unlock();
2939 return true;
2942 /* Map a physical memory region into a host virtual address.
2943 * May map a subset of the requested range, given by and returned in *plen.
2944 * May return NULL if resources needed to perform the mapping are exhausted.
2945 * Use only for reads OR writes - not for read-modify-write operations.
2946 * Use cpu_register_map_client() to know when retrying the map operation is
2947 * likely to succeed.
2949 void *address_space_map(AddressSpace *as,
2950 hwaddr addr,
2951 hwaddr *plen,
2952 bool is_write)
2954 hwaddr len = *plen;
2955 hwaddr done = 0;
2956 hwaddr l, xlat, base;
2957 MemoryRegion *mr, *this_mr;
2958 ram_addr_t raddr;
2959 void *ptr;
2961 if (len == 0) {
2962 return NULL;
2965 l = len;
2966 rcu_read_lock();
2967 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2969 if (!memory_access_is_direct(mr, is_write)) {
2970 if (atomic_xchg(&bounce.in_use, true)) {
2971 rcu_read_unlock();
2972 return NULL;
2974 /* Avoid unbounded allocations */
2975 l = MIN(l, TARGET_PAGE_SIZE);
2976 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
2977 bounce.addr = addr;
2978 bounce.len = l;
2980 memory_region_ref(mr);
2981 bounce.mr = mr;
2982 if (!is_write) {
2983 address_space_read(as, addr, MEMTXATTRS_UNSPECIFIED,
2984 bounce.buffer, l);
2987 rcu_read_unlock();
2988 *plen = l;
2989 return bounce.buffer;
2992 base = xlat;
2993 raddr = memory_region_get_ram_addr(mr);
2995 for (;;) {
2996 len -= l;
2997 addr += l;
2998 done += l;
2999 if (len == 0) {
3000 break;
3003 l = len;
3004 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
3005 if (this_mr != mr || xlat != base + done) {
3006 break;
3010 memory_region_ref(mr);
3011 *plen = done;
3012 ptr = qemu_ram_ptr_length(mr->ram_block, raddr + base, plen);
3013 rcu_read_unlock();
3015 return ptr;
3018 /* Unmaps a memory region previously mapped by address_space_map().
3019 * Will also mark the memory as dirty if is_write == 1. access_len gives
3020 * the amount of memory that was actually read or written by the caller.
3022 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
3023 int is_write, hwaddr access_len)
3025 if (buffer != bounce.buffer) {
3026 MemoryRegion *mr;
3027 ram_addr_t addr1;
3029 mr = qemu_ram_addr_from_host(buffer, &addr1);
3030 assert(mr != NULL);
3031 if (is_write) {
3032 invalidate_and_set_dirty(mr, addr1, access_len);
3034 if (xen_enabled()) {
3035 xen_invalidate_map_cache_entry(buffer);
3037 memory_region_unref(mr);
3038 return;
3040 if (is_write) {
3041 address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
3042 bounce.buffer, access_len);
3044 qemu_vfree(bounce.buffer);
3045 bounce.buffer = NULL;
3046 memory_region_unref(bounce.mr);
3047 atomic_mb_set(&bounce.in_use, false);
3048 cpu_notify_map_clients();
3051 void *cpu_physical_memory_map(hwaddr addr,
3052 hwaddr *plen,
3053 int is_write)
3055 return address_space_map(&address_space_memory, addr, plen, is_write);
3058 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
3059 int is_write, hwaddr access_len)
3061 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
3064 /* warning: addr must be aligned */
3065 static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
3066 MemTxAttrs attrs,
3067 MemTxResult *result,
3068 enum device_endian endian)
3070 uint8_t *ptr;
3071 uint64_t val;
3072 MemoryRegion *mr;
3073 hwaddr l = 4;
3074 hwaddr addr1;
3075 MemTxResult r;
3076 bool release_lock = false;
3078 rcu_read_lock();
3079 mr = address_space_translate(as, addr, &addr1, &l, false);
3080 if (l < 4 || !memory_access_is_direct(mr, false)) {
3081 release_lock |= prepare_mmio_access(mr);
3083 /* I/O case */
3084 r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
3085 #if defined(TARGET_WORDS_BIGENDIAN)
3086 if (endian == DEVICE_LITTLE_ENDIAN) {
3087 val = bswap32(val);
3089 #else
3090 if (endian == DEVICE_BIG_ENDIAN) {
3091 val = bswap32(val);
3093 #endif
3094 } else {
3095 /* RAM case */
3096 ptr = qemu_get_ram_ptr(mr->ram_block,
3097 (memory_region_get_ram_addr(mr)
3098 & TARGET_PAGE_MASK)
3099 + addr1);
3100 switch (endian) {
3101 case DEVICE_LITTLE_ENDIAN:
3102 val = ldl_le_p(ptr);
3103 break;
3104 case DEVICE_BIG_ENDIAN:
3105 val = ldl_be_p(ptr);
3106 break;
3107 default:
3108 val = ldl_p(ptr);
3109 break;
3111 r = MEMTX_OK;
3113 if (result) {
3114 *result = r;
3116 if (release_lock) {
3117 qemu_mutex_unlock_iothread();
3119 rcu_read_unlock();
3120 return val;
3123 uint32_t address_space_ldl(AddressSpace *as, hwaddr addr,
3124 MemTxAttrs attrs, MemTxResult *result)
3126 return address_space_ldl_internal(as, addr, attrs, result,
3127 DEVICE_NATIVE_ENDIAN);
3130 uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr,
3131 MemTxAttrs attrs, MemTxResult *result)
3133 return address_space_ldl_internal(as, addr, attrs, result,
3134 DEVICE_LITTLE_ENDIAN);
3137 uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr,
3138 MemTxAttrs attrs, MemTxResult *result)
3140 return address_space_ldl_internal(as, addr, attrs, result,
3141 DEVICE_BIG_ENDIAN);
3144 uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
3146 return address_space_ldl(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3149 uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
3151 return address_space_ldl_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3154 uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
3156 return address_space_ldl_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3159 /* warning: addr must be aligned */
3160 static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
3161 MemTxAttrs attrs,
3162 MemTxResult *result,
3163 enum device_endian endian)
3165 uint8_t *ptr;
3166 uint64_t val;
3167 MemoryRegion *mr;
3168 hwaddr l = 8;
3169 hwaddr addr1;
3170 MemTxResult r;
3171 bool release_lock = false;
3173 rcu_read_lock();
3174 mr = address_space_translate(as, addr, &addr1, &l,
3175 false);
3176 if (l < 8 || !memory_access_is_direct(mr, false)) {
3177 release_lock |= prepare_mmio_access(mr);
3179 /* I/O case */
3180 r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
3181 #if defined(TARGET_WORDS_BIGENDIAN)
3182 if (endian == DEVICE_LITTLE_ENDIAN) {
3183 val = bswap64(val);
3185 #else
3186 if (endian == DEVICE_BIG_ENDIAN) {
3187 val = bswap64(val);
3189 #endif
3190 } else {
3191 /* RAM case */
3192 ptr = qemu_get_ram_ptr(mr->ram_block,
3193 (memory_region_get_ram_addr(mr)
3194 & TARGET_PAGE_MASK)
3195 + addr1);
3196 switch (endian) {
3197 case DEVICE_LITTLE_ENDIAN:
3198 val = ldq_le_p(ptr);
3199 break;
3200 case DEVICE_BIG_ENDIAN:
3201 val = ldq_be_p(ptr);
3202 break;
3203 default:
3204 val = ldq_p(ptr);
3205 break;
3207 r = MEMTX_OK;
3209 if (result) {
3210 *result = r;
3212 if (release_lock) {
3213 qemu_mutex_unlock_iothread();
3215 rcu_read_unlock();
3216 return val;
3219 uint64_t address_space_ldq(AddressSpace *as, hwaddr addr,
3220 MemTxAttrs attrs, MemTxResult *result)
3222 return address_space_ldq_internal(as, addr, attrs, result,
3223 DEVICE_NATIVE_ENDIAN);
3226 uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr,
3227 MemTxAttrs attrs, MemTxResult *result)
3229 return address_space_ldq_internal(as, addr, attrs, result,
3230 DEVICE_LITTLE_ENDIAN);
3233 uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr,
3234 MemTxAttrs attrs, MemTxResult *result)
3236 return address_space_ldq_internal(as, addr, attrs, result,
3237 DEVICE_BIG_ENDIAN);
3240 uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
3242 return address_space_ldq(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3245 uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
3247 return address_space_ldq_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3250 uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
3252 return address_space_ldq_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3255 /* XXX: optimize */
3256 uint32_t address_space_ldub(AddressSpace *as, hwaddr addr,
3257 MemTxAttrs attrs, MemTxResult *result)
3259 uint8_t val;
3260 MemTxResult r;
3262 r = address_space_rw(as, addr, attrs, &val, 1, 0);
3263 if (result) {
3264 *result = r;
3266 return val;
3269 uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
3271 return address_space_ldub(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3274 /* warning: addr must be aligned */
3275 static inline uint32_t address_space_lduw_internal(AddressSpace *as,
3276 hwaddr addr,
3277 MemTxAttrs attrs,
3278 MemTxResult *result,
3279 enum device_endian endian)
3281 uint8_t *ptr;
3282 uint64_t val;
3283 MemoryRegion *mr;
3284 hwaddr l = 2;
3285 hwaddr addr1;
3286 MemTxResult r;
3287 bool release_lock = false;
3289 rcu_read_lock();
3290 mr = address_space_translate(as, addr, &addr1, &l,
3291 false);
3292 if (l < 2 || !memory_access_is_direct(mr, false)) {
3293 release_lock |= prepare_mmio_access(mr);
3295 /* I/O case */
3296 r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
3297 #if defined(TARGET_WORDS_BIGENDIAN)
3298 if (endian == DEVICE_LITTLE_ENDIAN) {
3299 val = bswap16(val);
3301 #else
3302 if (endian == DEVICE_BIG_ENDIAN) {
3303 val = bswap16(val);
3305 #endif
3306 } else {
3307 /* RAM case */
3308 ptr = qemu_get_ram_ptr(mr->ram_block,
3309 (memory_region_get_ram_addr(mr)
3310 & TARGET_PAGE_MASK)
3311 + addr1);
3312 switch (endian) {
3313 case DEVICE_LITTLE_ENDIAN:
3314 val = lduw_le_p(ptr);
3315 break;
3316 case DEVICE_BIG_ENDIAN:
3317 val = lduw_be_p(ptr);
3318 break;
3319 default:
3320 val = lduw_p(ptr);
3321 break;
3323 r = MEMTX_OK;
3325 if (result) {
3326 *result = r;
3328 if (release_lock) {
3329 qemu_mutex_unlock_iothread();
3331 rcu_read_unlock();
3332 return val;
3335 uint32_t address_space_lduw(AddressSpace *as, hwaddr addr,
3336 MemTxAttrs attrs, MemTxResult *result)
3338 return address_space_lduw_internal(as, addr, attrs, result,
3339 DEVICE_NATIVE_ENDIAN);
3342 uint32_t address_space_lduw_le(AddressSpace *as, hwaddr addr,
3343 MemTxAttrs attrs, MemTxResult *result)
3345 return address_space_lduw_internal(as, addr, attrs, result,
3346 DEVICE_LITTLE_ENDIAN);
3349 uint32_t address_space_lduw_be(AddressSpace *as, hwaddr addr,
3350 MemTxAttrs attrs, MemTxResult *result)
3352 return address_space_lduw_internal(as, addr, attrs, result,
3353 DEVICE_BIG_ENDIAN);
3356 uint32_t lduw_phys(AddressSpace *as, hwaddr addr)
3358 return address_space_lduw(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3361 uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr)
3363 return address_space_lduw_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3366 uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr)
3368 return address_space_lduw_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3371 /* warning: addr must be aligned. The ram page is not masked as dirty
3372 and the code inside is not invalidated. It is useful if the dirty
3373 bits are used to track modified PTEs */
3374 void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
3375 MemTxAttrs attrs, MemTxResult *result)
3377 uint8_t *ptr;
3378 MemoryRegion *mr;
3379 hwaddr l = 4;
3380 hwaddr addr1;
3381 MemTxResult r;
3382 uint8_t dirty_log_mask;
3383 bool release_lock = false;
3385 rcu_read_lock();
3386 mr = address_space_translate(as, addr, &addr1, &l,
3387 true);
3388 if (l < 4 || !memory_access_is_direct(mr, true)) {
3389 release_lock |= prepare_mmio_access(mr);
3391 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3392 } else {
3393 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3394 ptr = qemu_get_ram_ptr(mr->ram_block, addr1);
3395 stl_p(ptr, val);
3397 dirty_log_mask = memory_region_get_dirty_log_mask(mr);
3398 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
3399 cpu_physical_memory_set_dirty_range(addr1, 4, dirty_log_mask);
3400 r = MEMTX_OK;
3402 if (result) {
3403 *result = r;
3405 if (release_lock) {
3406 qemu_mutex_unlock_iothread();
3408 rcu_read_unlock();
3411 void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
3413 address_space_stl_notdirty(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3416 /* warning: addr must be aligned */
3417 static inline void address_space_stl_internal(AddressSpace *as,
3418 hwaddr addr, uint32_t val,
3419 MemTxAttrs attrs,
3420 MemTxResult *result,
3421 enum device_endian endian)
3423 uint8_t *ptr;
3424 MemoryRegion *mr;
3425 hwaddr l = 4;
3426 hwaddr addr1;
3427 MemTxResult r;
3428 bool release_lock = false;
3430 rcu_read_lock();
3431 mr = address_space_translate(as, addr, &addr1, &l,
3432 true);
3433 if (l < 4 || !memory_access_is_direct(mr, true)) {
3434 release_lock |= prepare_mmio_access(mr);
3436 #if defined(TARGET_WORDS_BIGENDIAN)
3437 if (endian == DEVICE_LITTLE_ENDIAN) {
3438 val = bswap32(val);
3440 #else
3441 if (endian == DEVICE_BIG_ENDIAN) {
3442 val = bswap32(val);
3444 #endif
3445 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3446 } else {
3447 /* RAM case */
3448 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3449 ptr = qemu_get_ram_ptr(mr->ram_block, addr1);
3450 switch (endian) {
3451 case DEVICE_LITTLE_ENDIAN:
3452 stl_le_p(ptr, val);
3453 break;
3454 case DEVICE_BIG_ENDIAN:
3455 stl_be_p(ptr, val);
3456 break;
3457 default:
3458 stl_p(ptr, val);
3459 break;
3461 invalidate_and_set_dirty(mr, addr1, 4);
3462 r = MEMTX_OK;
3464 if (result) {
3465 *result = r;
3467 if (release_lock) {
3468 qemu_mutex_unlock_iothread();
3470 rcu_read_unlock();
3473 void address_space_stl(AddressSpace *as, hwaddr addr, uint32_t val,
3474 MemTxAttrs attrs, MemTxResult *result)
3476 address_space_stl_internal(as, addr, val, attrs, result,
3477 DEVICE_NATIVE_ENDIAN);
3480 void address_space_stl_le(AddressSpace *as, hwaddr addr, uint32_t val,
3481 MemTxAttrs attrs, MemTxResult *result)
3483 address_space_stl_internal(as, addr, val, attrs, result,
3484 DEVICE_LITTLE_ENDIAN);
3487 void address_space_stl_be(AddressSpace *as, hwaddr addr, uint32_t val,
3488 MemTxAttrs attrs, MemTxResult *result)
3490 address_space_stl_internal(as, addr, val, attrs, result,
3491 DEVICE_BIG_ENDIAN);
3494 void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3496 address_space_stl(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3499 void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3501 address_space_stl_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3504 void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3506 address_space_stl_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3509 /* XXX: optimize */
3510 void address_space_stb(AddressSpace *as, hwaddr addr, uint32_t val,
3511 MemTxAttrs attrs, MemTxResult *result)
3513 uint8_t v = val;
3514 MemTxResult r;
3516 r = address_space_rw(as, addr, attrs, &v, 1, 1);
3517 if (result) {
3518 *result = r;
3522 void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3524 address_space_stb(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3527 /* warning: addr must be aligned */
3528 static inline void address_space_stw_internal(AddressSpace *as,
3529 hwaddr addr, uint32_t val,
3530 MemTxAttrs attrs,
3531 MemTxResult *result,
3532 enum device_endian endian)
3534 uint8_t *ptr;
3535 MemoryRegion *mr;
3536 hwaddr l = 2;
3537 hwaddr addr1;
3538 MemTxResult r;
3539 bool release_lock = false;
3541 rcu_read_lock();
3542 mr = address_space_translate(as, addr, &addr1, &l, true);
3543 if (l < 2 || !memory_access_is_direct(mr, true)) {
3544 release_lock |= prepare_mmio_access(mr);
3546 #if defined(TARGET_WORDS_BIGENDIAN)
3547 if (endian == DEVICE_LITTLE_ENDIAN) {
3548 val = bswap16(val);
3550 #else
3551 if (endian == DEVICE_BIG_ENDIAN) {
3552 val = bswap16(val);
3554 #endif
3555 r = memory_region_dispatch_write(mr, addr1, val, 2, attrs);
3556 } else {
3557 /* RAM case */
3558 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3559 ptr = qemu_get_ram_ptr(mr->ram_block, addr1);
3560 switch (endian) {
3561 case DEVICE_LITTLE_ENDIAN:
3562 stw_le_p(ptr, val);
3563 break;
3564 case DEVICE_BIG_ENDIAN:
3565 stw_be_p(ptr, val);
3566 break;
3567 default:
3568 stw_p(ptr, val);
3569 break;
3571 invalidate_and_set_dirty(mr, addr1, 2);
3572 r = MEMTX_OK;
3574 if (result) {
3575 *result = r;
3577 if (release_lock) {
3578 qemu_mutex_unlock_iothread();
3580 rcu_read_unlock();
3583 void address_space_stw(AddressSpace *as, hwaddr addr, uint32_t val,
3584 MemTxAttrs attrs, MemTxResult *result)
3586 address_space_stw_internal(as, addr, val, attrs, result,
3587 DEVICE_NATIVE_ENDIAN);
3590 void address_space_stw_le(AddressSpace *as, hwaddr addr, uint32_t val,
3591 MemTxAttrs attrs, MemTxResult *result)
3593 address_space_stw_internal(as, addr, val, attrs, result,
3594 DEVICE_LITTLE_ENDIAN);
3597 void address_space_stw_be(AddressSpace *as, hwaddr addr, uint32_t val,
3598 MemTxAttrs attrs, MemTxResult *result)
3600 address_space_stw_internal(as, addr, val, attrs, result,
3601 DEVICE_BIG_ENDIAN);
3604 void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3606 address_space_stw(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3609 void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3611 address_space_stw_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3614 void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3616 address_space_stw_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3619 /* XXX: optimize */
3620 void address_space_stq(AddressSpace *as, hwaddr addr, uint64_t val,
3621 MemTxAttrs attrs, MemTxResult *result)
3623 MemTxResult r;
3624 val = tswap64(val);
3625 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3626 if (result) {
3627 *result = r;
3631 void address_space_stq_le(AddressSpace *as, hwaddr addr, uint64_t val,
3632 MemTxAttrs attrs, MemTxResult *result)
3634 MemTxResult r;
3635 val = cpu_to_le64(val);
3636 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3637 if (result) {
3638 *result = r;
3641 void address_space_stq_be(AddressSpace *as, hwaddr addr, uint64_t val,
3642 MemTxAttrs attrs, MemTxResult *result)
3644 MemTxResult r;
3645 val = cpu_to_be64(val);
3646 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3647 if (result) {
3648 *result = r;
3652 void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3654 address_space_stq(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3657 void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3659 address_space_stq_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3662 void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3664 address_space_stq_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3667 /* virtual memory access for debug (includes writing to ROM) */
3668 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
3669 uint8_t *buf, int len, int is_write)
3671 int l;
3672 hwaddr phys_addr;
3673 target_ulong page;
3675 while (len > 0) {
3676 int asidx;
3677 MemTxAttrs attrs;
3679 page = addr & TARGET_PAGE_MASK;
3680 phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
3681 asidx = cpu_asidx_from_attrs(cpu, attrs);
3682 /* if no physical page mapped, return an error */
3683 if (phys_addr == -1)
3684 return -1;
3685 l = (page + TARGET_PAGE_SIZE) - addr;
3686 if (l > len)
3687 l = len;
3688 phys_addr += (addr & ~TARGET_PAGE_MASK);
3689 if (is_write) {
3690 cpu_physical_memory_write_rom(cpu->cpu_ases[asidx].as,
3691 phys_addr, buf, l);
3692 } else {
3693 address_space_rw(cpu->cpu_ases[asidx].as, phys_addr,
3694 MEMTXATTRS_UNSPECIFIED,
3695 buf, l, 0);
3697 len -= l;
3698 buf += l;
3699 addr += l;
3701 return 0;
3705 * Allows code that needs to deal with migration bitmaps etc to still be built
3706 * target independent.
3708 size_t qemu_target_page_bits(void)
3710 return TARGET_PAGE_BITS;
3713 #endif
3716 * A helper function for the _utterly broken_ virtio device model to find out if
3717 * it's running on a big endian machine. Don't do this at home kids!
3719 bool target_words_bigendian(void);
3720 bool target_words_bigendian(void)
3722 #if defined(TARGET_WORDS_BIGENDIAN)
3723 return true;
3724 #else
3725 return false;
3726 #endif
3729 #ifndef CONFIG_USER_ONLY
3730 bool cpu_physical_memory_is_io(hwaddr phys_addr)
3732 MemoryRegion*mr;
3733 hwaddr l = 1;
3734 bool res;
3736 rcu_read_lock();
3737 mr = address_space_translate(&address_space_memory,
3738 phys_addr, &phys_addr, &l, false);
3740 res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
3741 rcu_read_unlock();
3742 return res;
3745 int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
3747 RAMBlock *block;
3748 int ret = 0;
3750 rcu_read_lock();
3751 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
3752 ret = func(block->idstr, block->host, block->offset,
3753 block->used_length, opaque);
3754 if (ret) {
3755 break;
3758 rcu_read_unlock();
3759 return ret;
3761 #endif