trace: convert stderr backend to log
[qemu.git] / exec.c
blobab373604d77aedbbfcb94afd674541a35d183302
1 /*
2 * Virtual page mapping
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "qemu/osdep.h"
20 #ifndef _WIN32
21 #include <sys/mman.h>
22 #endif
24 #include "qemu-common.h"
25 #include "cpu.h"
26 #include "tcg.h"
27 #include "hw/hw.h"
28 #if !defined(CONFIG_USER_ONLY)
29 #include "hw/boards.h"
30 #endif
31 #include "hw/qdev.h"
32 #include "sysemu/kvm.h"
33 #include "sysemu/sysemu.h"
34 #include "hw/xen/xen.h"
35 #include "qemu/timer.h"
36 #include "qemu/config-file.h"
37 #include "qemu/error-report.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48 #include "qemu/rcu_queue.h"
49 #include "qemu/main-loop.h"
50 #include "translate-all.h"
51 #include "sysemu/replay.h"
53 #include "exec/memory-internal.h"
54 #include "exec/ram_addr.h"
55 #include "exec/log.h"
57 #include "qemu/range.h"
58 #ifndef _WIN32
59 #include "qemu/mmap-alloc.h"
60 #endif
62 //#define DEBUG_SUBPAGE
64 #if !defined(CONFIG_USER_ONLY)
65 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
66 * are protected by the ramlist lock.
68 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
70 static MemoryRegion *system_memory;
71 static MemoryRegion *system_io;
73 AddressSpace address_space_io;
74 AddressSpace address_space_memory;
76 MemoryRegion io_mem_rom, io_mem_notdirty;
77 static MemoryRegion io_mem_unassigned;
79 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
80 #define RAM_PREALLOC (1 << 0)
82 /* RAM is mmap-ed with MAP_SHARED */
83 #define RAM_SHARED (1 << 1)
85 /* Only a portion of RAM (used_length) is actually used, and migrated.
86 * This used_length size can change across reboots.
88 #define RAM_RESIZEABLE (1 << 2)
90 #endif
92 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
93 /* current CPU in the current thread. It is only valid inside
94 cpu_exec() */
95 __thread CPUState *current_cpu;
96 /* 0 = Do not count executed instructions.
97 1 = Precise instruction counting.
98 2 = Adaptive rate instruction counting. */
99 int use_icount;
101 #if !defined(CONFIG_USER_ONLY)
103 typedef struct PhysPageEntry PhysPageEntry;
105 struct PhysPageEntry {
106 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
107 uint32_t skip : 6;
108 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
109 uint32_t ptr : 26;
112 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
114 /* Size of the L2 (and L3, etc) page tables. */
115 #define ADDR_SPACE_BITS 64
117 #define P_L2_BITS 9
118 #define P_L2_SIZE (1 << P_L2_BITS)
120 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
122 typedef PhysPageEntry Node[P_L2_SIZE];
124 typedef struct PhysPageMap {
125 struct rcu_head rcu;
127 unsigned sections_nb;
128 unsigned sections_nb_alloc;
129 unsigned nodes_nb;
130 unsigned nodes_nb_alloc;
131 Node *nodes;
132 MemoryRegionSection *sections;
133 } PhysPageMap;
135 struct AddressSpaceDispatch {
136 struct rcu_head rcu;
138 /* This is a multi-level map on the physical address space.
139 * The bottom level has pointers to MemoryRegionSections.
141 PhysPageEntry phys_map;
142 PhysPageMap map;
143 AddressSpace *as;
146 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
147 typedef struct subpage_t {
148 MemoryRegion iomem;
149 AddressSpace *as;
150 hwaddr base;
151 uint16_t sub_section[TARGET_PAGE_SIZE];
152 } subpage_t;
154 #define PHYS_SECTION_UNASSIGNED 0
155 #define PHYS_SECTION_NOTDIRTY 1
156 #define PHYS_SECTION_ROM 2
157 #define PHYS_SECTION_WATCH 3
159 static void io_mem_init(void);
160 static void memory_map_init(void);
161 static void tcg_commit(MemoryListener *listener);
163 static MemoryRegion io_mem_watch;
166 * CPUAddressSpace: all the information a CPU needs about an AddressSpace
167 * @cpu: the CPU whose AddressSpace this is
168 * @as: the AddressSpace itself
169 * @memory_dispatch: its dispatch pointer (cached, RCU protected)
170 * @tcg_as_listener: listener for tracking changes to the AddressSpace
172 struct CPUAddressSpace {
173 CPUState *cpu;
174 AddressSpace *as;
175 struct AddressSpaceDispatch *memory_dispatch;
176 MemoryListener tcg_as_listener;
179 #endif
181 #if !defined(CONFIG_USER_ONLY)
183 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
185 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
186 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
187 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
188 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
192 static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
194 unsigned i;
195 uint32_t ret;
196 PhysPageEntry e;
197 PhysPageEntry *p;
199 ret = map->nodes_nb++;
200 p = map->nodes[ret];
201 assert(ret != PHYS_MAP_NODE_NIL);
202 assert(ret != map->nodes_nb_alloc);
204 e.skip = leaf ? 0 : 1;
205 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
206 for (i = 0; i < P_L2_SIZE; ++i) {
207 memcpy(&p[i], &e, sizeof(e));
209 return ret;
212 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
213 hwaddr *index, hwaddr *nb, uint16_t leaf,
214 int level)
216 PhysPageEntry *p;
217 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
219 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
220 lp->ptr = phys_map_node_alloc(map, level == 0);
222 p = map->nodes[lp->ptr];
223 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
225 while (*nb && lp < &p[P_L2_SIZE]) {
226 if ((*index & (step - 1)) == 0 && *nb >= step) {
227 lp->skip = 0;
228 lp->ptr = leaf;
229 *index += step;
230 *nb -= step;
231 } else {
232 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
234 ++lp;
238 static void phys_page_set(AddressSpaceDispatch *d,
239 hwaddr index, hwaddr nb,
240 uint16_t leaf)
242 /* Wildly overreserve - it doesn't matter much. */
243 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
245 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
248 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
249 * and update our entry so we can skip it and go directly to the destination.
251 static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)
253 unsigned valid_ptr = P_L2_SIZE;
254 int valid = 0;
255 PhysPageEntry *p;
256 int i;
258 if (lp->ptr == PHYS_MAP_NODE_NIL) {
259 return;
262 p = nodes[lp->ptr];
263 for (i = 0; i < P_L2_SIZE; i++) {
264 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
265 continue;
268 valid_ptr = i;
269 valid++;
270 if (p[i].skip) {
271 phys_page_compact(&p[i], nodes, compacted);
275 /* We can only compress if there's only one child. */
276 if (valid != 1) {
277 return;
280 assert(valid_ptr < P_L2_SIZE);
282 /* Don't compress if it won't fit in the # of bits we have. */
283 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
284 return;
287 lp->ptr = p[valid_ptr].ptr;
288 if (!p[valid_ptr].skip) {
289 /* If our only child is a leaf, make this a leaf. */
290 /* By design, we should have made this node a leaf to begin with so we
291 * should never reach here.
292 * But since it's so simple to handle this, let's do it just in case we
293 * change this rule.
295 lp->skip = 0;
296 } else {
297 lp->skip += p[valid_ptr].skip;
301 static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
303 DECLARE_BITMAP(compacted, nodes_nb);
305 if (d->phys_map.skip) {
306 phys_page_compact(&d->phys_map, d->map.nodes, compacted);
310 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
311 Node *nodes, MemoryRegionSection *sections)
313 PhysPageEntry *p;
314 hwaddr index = addr >> TARGET_PAGE_BITS;
315 int i;
317 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
318 if (lp.ptr == PHYS_MAP_NODE_NIL) {
319 return &sections[PHYS_SECTION_UNASSIGNED];
321 p = nodes[lp.ptr];
322 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
325 if (sections[lp.ptr].size.hi ||
326 range_covers_byte(sections[lp.ptr].offset_within_address_space,
327 sections[lp.ptr].size.lo, addr)) {
328 return &sections[lp.ptr];
329 } else {
330 return &sections[PHYS_SECTION_UNASSIGNED];
334 bool memory_region_is_unassigned(MemoryRegion *mr)
336 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
337 && mr != &io_mem_watch;
340 /* Called from RCU critical section */
341 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
342 hwaddr addr,
343 bool resolve_subpage)
345 MemoryRegionSection *section;
346 subpage_t *subpage;
348 section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);
349 if (resolve_subpage && section->mr->subpage) {
350 subpage = container_of(section->mr, subpage_t, iomem);
351 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
353 return section;
356 /* Called from RCU critical section */
357 static MemoryRegionSection *
358 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
359 hwaddr *plen, bool resolve_subpage)
361 MemoryRegionSection *section;
362 MemoryRegion *mr;
363 Int128 diff;
365 section = address_space_lookup_region(d, addr, resolve_subpage);
366 /* Compute offset within MemoryRegionSection */
367 addr -= section->offset_within_address_space;
369 /* Compute offset within MemoryRegion */
370 *xlat = addr + section->offset_within_region;
372 mr = section->mr;
374 /* MMIO registers can be expected to perform full-width accesses based only
375 * on their address, without considering adjacent registers that could
376 * decode to completely different MemoryRegions. When such registers
377 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
378 * regions overlap wildly. For this reason we cannot clamp the accesses
379 * here.
381 * If the length is small (as is the case for address_space_ldl/stl),
382 * everything works fine. If the incoming length is large, however,
383 * the caller really has to do the clamping through memory_access_size.
385 if (memory_region_is_ram(mr)) {
386 diff = int128_sub(section->size, int128_make64(addr));
387 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
389 return section;
392 /* Called from RCU critical section */
393 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
394 hwaddr *xlat, hwaddr *plen,
395 bool is_write)
397 IOMMUTLBEntry iotlb;
398 MemoryRegionSection *section;
399 MemoryRegion *mr;
401 for (;;) {
402 AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch);
403 section = address_space_translate_internal(d, addr, &addr, plen, true);
404 mr = section->mr;
406 if (!mr->iommu_ops) {
407 break;
410 iotlb = mr->iommu_ops->translate(mr, addr, is_write);
411 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
412 | (addr & iotlb.addr_mask));
413 *plen = MIN(*plen, (addr | iotlb.addr_mask) - addr + 1);
414 if (!(iotlb.perm & (1 << is_write))) {
415 mr = &io_mem_unassigned;
416 break;
419 as = iotlb.target_as;
422 if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
423 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
424 *plen = MIN(page, *plen);
427 *xlat = addr;
428 return mr;
431 /* Called from RCU critical section */
432 MemoryRegionSection *
433 address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
434 hwaddr *xlat, hwaddr *plen)
436 MemoryRegionSection *section;
437 AddressSpaceDispatch *d = cpu->cpu_ases[asidx].memory_dispatch;
439 section = address_space_translate_internal(d, addr, xlat, plen, false);
441 assert(!section->mr->iommu_ops);
442 return section;
444 #endif
446 #if !defined(CONFIG_USER_ONLY)
448 static int cpu_common_post_load(void *opaque, int version_id)
450 CPUState *cpu = opaque;
452 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
453 version_id is increased. */
454 cpu->interrupt_request &= ~0x01;
455 tlb_flush(cpu, 1);
457 return 0;
460 static int cpu_common_pre_load(void *opaque)
462 CPUState *cpu = opaque;
464 cpu->exception_index = -1;
466 return 0;
469 static bool cpu_common_exception_index_needed(void *opaque)
471 CPUState *cpu = opaque;
473 return tcg_enabled() && cpu->exception_index != -1;
476 static const VMStateDescription vmstate_cpu_common_exception_index = {
477 .name = "cpu_common/exception_index",
478 .version_id = 1,
479 .minimum_version_id = 1,
480 .needed = cpu_common_exception_index_needed,
481 .fields = (VMStateField[]) {
482 VMSTATE_INT32(exception_index, CPUState),
483 VMSTATE_END_OF_LIST()
487 static bool cpu_common_crash_occurred_needed(void *opaque)
489 CPUState *cpu = opaque;
491 return cpu->crash_occurred;
494 static const VMStateDescription vmstate_cpu_common_crash_occurred = {
495 .name = "cpu_common/crash_occurred",
496 .version_id = 1,
497 .minimum_version_id = 1,
498 .needed = cpu_common_crash_occurred_needed,
499 .fields = (VMStateField[]) {
500 VMSTATE_BOOL(crash_occurred, CPUState),
501 VMSTATE_END_OF_LIST()
505 const VMStateDescription vmstate_cpu_common = {
506 .name = "cpu_common",
507 .version_id = 1,
508 .minimum_version_id = 1,
509 .pre_load = cpu_common_pre_load,
510 .post_load = cpu_common_post_load,
511 .fields = (VMStateField[]) {
512 VMSTATE_UINT32(halted, CPUState),
513 VMSTATE_UINT32(interrupt_request, CPUState),
514 VMSTATE_END_OF_LIST()
516 .subsections = (const VMStateDescription*[]) {
517 &vmstate_cpu_common_exception_index,
518 &vmstate_cpu_common_crash_occurred,
519 NULL
523 #endif
525 CPUState *qemu_get_cpu(int index)
527 CPUState *cpu;
529 CPU_FOREACH(cpu) {
530 if (cpu->cpu_index == index) {
531 return cpu;
535 return NULL;
538 #if !defined(CONFIG_USER_ONLY)
539 void cpu_address_space_init(CPUState *cpu, AddressSpace *as, int asidx)
541 CPUAddressSpace *newas;
543 /* Target code should have set num_ases before calling us */
544 assert(asidx < cpu->num_ases);
546 if (asidx == 0) {
547 /* address space 0 gets the convenience alias */
548 cpu->as = as;
551 /* KVM cannot currently support multiple address spaces. */
552 assert(asidx == 0 || !kvm_enabled());
554 if (!cpu->cpu_ases) {
555 cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
558 newas = &cpu->cpu_ases[asidx];
559 newas->cpu = cpu;
560 newas->as = as;
561 if (tcg_enabled()) {
562 newas->tcg_as_listener.commit = tcg_commit;
563 memory_listener_register(&newas->tcg_as_listener, as);
567 AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
569 /* Return the AddressSpace corresponding to the specified index */
570 return cpu->cpu_ases[asidx].as;
572 #endif
574 #ifndef CONFIG_USER_ONLY
575 static DECLARE_BITMAP(cpu_index_map, MAX_CPUMASK_BITS);
577 static int cpu_get_free_index(Error **errp)
579 int cpu = find_first_zero_bit(cpu_index_map, MAX_CPUMASK_BITS);
581 if (cpu >= MAX_CPUMASK_BITS) {
582 error_setg(errp, "Trying to use more CPUs than max of %d",
583 MAX_CPUMASK_BITS);
584 return -1;
587 bitmap_set(cpu_index_map, cpu, 1);
588 return cpu;
591 void cpu_exec_exit(CPUState *cpu)
593 if (cpu->cpu_index == -1) {
594 /* cpu_index was never allocated by this @cpu or was already freed. */
595 return;
598 bitmap_clear(cpu_index_map, cpu->cpu_index, 1);
599 cpu->cpu_index = -1;
601 #else
603 static int cpu_get_free_index(Error **errp)
605 CPUState *some_cpu;
606 int cpu_index = 0;
608 CPU_FOREACH(some_cpu) {
609 cpu_index++;
611 return cpu_index;
614 void cpu_exec_exit(CPUState *cpu)
617 #endif
619 void cpu_exec_init(CPUState *cpu, Error **errp)
621 CPUClass *cc = CPU_GET_CLASS(cpu);
622 int cpu_index;
623 Error *local_err = NULL;
625 cpu->as = NULL;
626 cpu->num_ases = 0;
628 #ifndef CONFIG_USER_ONLY
629 cpu->thread_id = qemu_get_thread_id();
631 /* This is a softmmu CPU object, so create a property for it
632 * so users can wire up its memory. (This can't go in qom/cpu.c
633 * because that file is compiled only once for both user-mode
634 * and system builds.) The default if no link is set up is to use
635 * the system address space.
637 object_property_add_link(OBJECT(cpu), "memory", TYPE_MEMORY_REGION,
638 (Object **)&cpu->memory,
639 qdev_prop_allow_set_link_before_realize,
640 OBJ_PROP_LINK_UNREF_ON_RELEASE,
641 &error_abort);
642 cpu->memory = system_memory;
643 object_ref(OBJECT(cpu->memory));
644 #endif
646 #if defined(CONFIG_USER_ONLY)
647 cpu_list_lock();
648 #endif
649 cpu_index = cpu->cpu_index = cpu_get_free_index(&local_err);
650 if (local_err) {
651 error_propagate(errp, local_err);
652 #if defined(CONFIG_USER_ONLY)
653 cpu_list_unlock();
654 #endif
655 return;
657 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
658 #if defined(CONFIG_USER_ONLY)
659 cpu_list_unlock();
660 #endif
661 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
662 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
664 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
665 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
666 cpu_save, cpu_load, cpu->env_ptr);
667 assert(cc->vmsd == NULL);
668 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
669 #endif
670 if (cc->vmsd != NULL) {
671 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
675 #if defined(CONFIG_USER_ONLY)
676 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
678 tb_invalidate_phys_page_range(pc, pc + 1, 0);
680 #else
681 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
683 MemTxAttrs attrs;
684 hwaddr phys = cpu_get_phys_page_attrs_debug(cpu, pc, &attrs);
685 int asidx = cpu_asidx_from_attrs(cpu, attrs);
686 if (phys != -1) {
687 tb_invalidate_phys_addr(cpu->cpu_ases[asidx].as,
688 phys | (pc & ~TARGET_PAGE_MASK));
691 #endif
693 #if defined(CONFIG_USER_ONLY)
694 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
699 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
700 int flags)
702 return -ENOSYS;
705 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
709 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
710 int flags, CPUWatchpoint **watchpoint)
712 return -ENOSYS;
714 #else
715 /* Add a watchpoint. */
716 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
717 int flags, CPUWatchpoint **watchpoint)
719 CPUWatchpoint *wp;
721 /* forbid ranges which are empty or run off the end of the address space */
722 if (len == 0 || (addr + len - 1) < addr) {
723 error_report("tried to set invalid watchpoint at %"
724 VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
725 return -EINVAL;
727 wp = g_malloc(sizeof(*wp));
729 wp->vaddr = addr;
730 wp->len = len;
731 wp->flags = flags;
733 /* keep all GDB-injected watchpoints in front */
734 if (flags & BP_GDB) {
735 QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
736 } else {
737 QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
740 tlb_flush_page(cpu, addr);
742 if (watchpoint)
743 *watchpoint = wp;
744 return 0;
747 /* Remove a specific watchpoint. */
748 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
749 int flags)
751 CPUWatchpoint *wp;
753 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
754 if (addr == wp->vaddr && len == wp->len
755 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
756 cpu_watchpoint_remove_by_ref(cpu, wp);
757 return 0;
760 return -ENOENT;
763 /* Remove a specific watchpoint by reference. */
764 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
766 QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
768 tlb_flush_page(cpu, watchpoint->vaddr);
770 g_free(watchpoint);
773 /* Remove all matching watchpoints. */
774 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
776 CPUWatchpoint *wp, *next;
778 QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
779 if (wp->flags & mask) {
780 cpu_watchpoint_remove_by_ref(cpu, wp);
785 /* Return true if this watchpoint address matches the specified
786 * access (ie the address range covered by the watchpoint overlaps
787 * partially or completely with the address range covered by the
788 * access).
790 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
791 vaddr addr,
792 vaddr len)
794 /* We know the lengths are non-zero, but a little caution is
795 * required to avoid errors in the case where the range ends
796 * exactly at the top of the address space and so addr + len
797 * wraps round to zero.
799 vaddr wpend = wp->vaddr + wp->len - 1;
800 vaddr addrend = addr + len - 1;
802 return !(addr > wpend || wp->vaddr > addrend);
805 #endif
807 /* Add a breakpoint. */
808 int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
809 CPUBreakpoint **breakpoint)
811 CPUBreakpoint *bp;
813 bp = g_malloc(sizeof(*bp));
815 bp->pc = pc;
816 bp->flags = flags;
818 /* keep all GDB-injected breakpoints in front */
819 if (flags & BP_GDB) {
820 QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
821 } else {
822 QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
825 breakpoint_invalidate(cpu, pc);
827 if (breakpoint) {
828 *breakpoint = bp;
830 return 0;
833 /* Remove a specific breakpoint. */
834 int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
836 CPUBreakpoint *bp;
838 QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
839 if (bp->pc == pc && bp->flags == flags) {
840 cpu_breakpoint_remove_by_ref(cpu, bp);
841 return 0;
844 return -ENOENT;
847 /* Remove a specific breakpoint by reference. */
848 void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
850 QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
852 breakpoint_invalidate(cpu, breakpoint->pc);
854 g_free(breakpoint);
857 /* Remove all matching breakpoints. */
858 void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
860 CPUBreakpoint *bp, *next;
862 QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
863 if (bp->flags & mask) {
864 cpu_breakpoint_remove_by_ref(cpu, bp);
869 /* enable or disable single step mode. EXCP_DEBUG is returned by the
870 CPU loop after each instruction */
871 void cpu_single_step(CPUState *cpu, int enabled)
873 if (cpu->singlestep_enabled != enabled) {
874 cpu->singlestep_enabled = enabled;
875 if (kvm_enabled()) {
876 kvm_update_guest_debug(cpu, 0);
877 } else {
878 /* must flush all the translated code to avoid inconsistencies */
879 /* XXX: only flush what is necessary */
880 tb_flush(cpu);
885 void cpu_abort(CPUState *cpu, const char *fmt, ...)
887 va_list ap;
888 va_list ap2;
890 va_start(ap, fmt);
891 va_copy(ap2, ap);
892 fprintf(stderr, "qemu: fatal: ");
893 vfprintf(stderr, fmt, ap);
894 fprintf(stderr, "\n");
895 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
896 if (qemu_log_separate()) {
897 qemu_log("qemu: fatal: ");
898 qemu_log_vprintf(fmt, ap2);
899 qemu_log("\n");
900 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
901 qemu_log_flush();
902 qemu_log_close();
904 va_end(ap2);
905 va_end(ap);
906 replay_finish();
907 #if defined(CONFIG_USER_ONLY)
909 struct sigaction act;
910 sigfillset(&act.sa_mask);
911 act.sa_handler = SIG_DFL;
912 sigaction(SIGABRT, &act, NULL);
914 #endif
915 abort();
918 #if !defined(CONFIG_USER_ONLY)
919 /* Called from RCU critical section */
920 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
922 RAMBlock *block;
924 block = atomic_rcu_read(&ram_list.mru_block);
925 if (block && addr - block->offset < block->max_length) {
926 return block;
928 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
929 if (addr - block->offset < block->max_length) {
930 goto found;
934 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
935 abort();
937 found:
938 /* It is safe to write mru_block outside the iothread lock. This
939 * is what happens:
941 * mru_block = xxx
942 * rcu_read_unlock()
943 * xxx removed from list
944 * rcu_read_lock()
945 * read mru_block
946 * mru_block = NULL;
947 * call_rcu(reclaim_ramblock, xxx);
948 * rcu_read_unlock()
950 * atomic_rcu_set is not needed here. The block was already published
951 * when it was placed into the list. Here we're just making an extra
952 * copy of the pointer.
954 ram_list.mru_block = block;
955 return block;
958 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
960 CPUState *cpu;
961 ram_addr_t start1;
962 RAMBlock *block;
963 ram_addr_t end;
965 end = TARGET_PAGE_ALIGN(start + length);
966 start &= TARGET_PAGE_MASK;
968 rcu_read_lock();
969 block = qemu_get_ram_block(start);
970 assert(block == qemu_get_ram_block(end - 1));
971 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
972 CPU_FOREACH(cpu) {
973 tlb_reset_dirty(cpu, start1, length);
975 rcu_read_unlock();
978 /* Note: start and end must be within the same ram block. */
979 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
980 ram_addr_t length,
981 unsigned client)
983 unsigned long end, page;
984 bool dirty;
986 if (length == 0) {
987 return false;
990 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
991 page = start >> TARGET_PAGE_BITS;
992 dirty = bitmap_test_and_clear_atomic(ram_list.dirty_memory[client],
993 page, end - page);
995 if (dirty && tcg_enabled()) {
996 tlb_reset_dirty_range_all(start, length);
999 return dirty;
1002 /* Called from RCU critical section */
1003 hwaddr memory_region_section_get_iotlb(CPUState *cpu,
1004 MemoryRegionSection *section,
1005 target_ulong vaddr,
1006 hwaddr paddr, hwaddr xlat,
1007 int prot,
1008 target_ulong *address)
1010 hwaddr iotlb;
1011 CPUWatchpoint *wp;
1013 if (memory_region_is_ram(section->mr)) {
1014 /* Normal RAM. */
1015 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
1016 + xlat;
1017 if (!section->readonly) {
1018 iotlb |= PHYS_SECTION_NOTDIRTY;
1019 } else {
1020 iotlb |= PHYS_SECTION_ROM;
1022 } else {
1023 AddressSpaceDispatch *d;
1025 d = atomic_rcu_read(&section->address_space->dispatch);
1026 iotlb = section - d->map.sections;
1027 iotlb += xlat;
1030 /* Make accesses to pages with watchpoints go via the
1031 watchpoint trap routines. */
1032 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
1033 if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
1034 /* Avoid trapping reads of pages with a write breakpoint. */
1035 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
1036 iotlb = PHYS_SECTION_WATCH + paddr;
1037 *address |= TLB_MMIO;
1038 break;
1043 return iotlb;
1045 #endif /* defined(CONFIG_USER_ONLY) */
1047 #if !defined(CONFIG_USER_ONLY)
1049 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1050 uint16_t section);
1051 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
1053 static void *(*phys_mem_alloc)(size_t size, uint64_t *align) =
1054 qemu_anon_ram_alloc;
1057 * Set a custom physical guest memory alloator.
1058 * Accelerators with unusual needs may need this. Hopefully, we can
1059 * get rid of it eventually.
1061 void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align))
1063 phys_mem_alloc = alloc;
1066 static uint16_t phys_section_add(PhysPageMap *map,
1067 MemoryRegionSection *section)
1069 /* The physical section number is ORed with a page-aligned
1070 * pointer to produce the iotlb entries. Thus it should
1071 * never overflow into the page-aligned value.
1073 assert(map->sections_nb < TARGET_PAGE_SIZE);
1075 if (map->sections_nb == map->sections_nb_alloc) {
1076 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
1077 map->sections = g_renew(MemoryRegionSection, map->sections,
1078 map->sections_nb_alloc);
1080 map->sections[map->sections_nb] = *section;
1081 memory_region_ref(section->mr);
1082 return map->sections_nb++;
1085 static void phys_section_destroy(MemoryRegion *mr)
1087 bool have_sub_page = mr->subpage;
1089 memory_region_unref(mr);
1091 if (have_sub_page) {
1092 subpage_t *subpage = container_of(mr, subpage_t, iomem);
1093 object_unref(OBJECT(&subpage->iomem));
1094 g_free(subpage);
1098 static void phys_sections_free(PhysPageMap *map)
1100 while (map->sections_nb > 0) {
1101 MemoryRegionSection *section = &map->sections[--map->sections_nb];
1102 phys_section_destroy(section->mr);
1104 g_free(map->sections);
1105 g_free(map->nodes);
1108 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
1110 subpage_t *subpage;
1111 hwaddr base = section->offset_within_address_space
1112 & TARGET_PAGE_MASK;
1113 MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
1114 d->map.nodes, d->map.sections);
1115 MemoryRegionSection subsection = {
1116 .offset_within_address_space = base,
1117 .size = int128_make64(TARGET_PAGE_SIZE),
1119 hwaddr start, end;
1121 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
1123 if (!(existing->mr->subpage)) {
1124 subpage = subpage_init(d->as, base);
1125 subsection.address_space = d->as;
1126 subsection.mr = &subpage->iomem;
1127 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
1128 phys_section_add(&d->map, &subsection));
1129 } else {
1130 subpage = container_of(existing->mr, subpage_t, iomem);
1132 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
1133 end = start + int128_get64(section->size) - 1;
1134 subpage_register(subpage, start, end,
1135 phys_section_add(&d->map, section));
1139 static void register_multipage(AddressSpaceDispatch *d,
1140 MemoryRegionSection *section)
1142 hwaddr start_addr = section->offset_within_address_space;
1143 uint16_t section_index = phys_section_add(&d->map, section);
1144 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1145 TARGET_PAGE_BITS));
1147 assert(num_pages);
1148 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
1151 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
1153 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1154 AddressSpaceDispatch *d = as->next_dispatch;
1155 MemoryRegionSection now = *section, remain = *section;
1156 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
1158 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
1159 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
1160 - now.offset_within_address_space;
1162 now.size = int128_min(int128_make64(left), now.size);
1163 register_subpage(d, &now);
1164 } else {
1165 now.size = int128_zero();
1167 while (int128_ne(remain.size, now.size)) {
1168 remain.size = int128_sub(remain.size, now.size);
1169 remain.offset_within_address_space += int128_get64(now.size);
1170 remain.offset_within_region += int128_get64(now.size);
1171 now = remain;
1172 if (int128_lt(remain.size, page_size)) {
1173 register_subpage(d, &now);
1174 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
1175 now.size = page_size;
1176 register_subpage(d, &now);
1177 } else {
1178 now.size = int128_and(now.size, int128_neg(page_size));
1179 register_multipage(d, &now);
1184 void qemu_flush_coalesced_mmio_buffer(void)
1186 if (kvm_enabled())
1187 kvm_flush_coalesced_mmio_buffer();
1190 void qemu_mutex_lock_ramlist(void)
1192 qemu_mutex_lock(&ram_list.mutex);
1195 void qemu_mutex_unlock_ramlist(void)
1197 qemu_mutex_unlock(&ram_list.mutex);
1200 #ifdef __linux__
1202 #include <sys/vfs.h>
1204 #define HUGETLBFS_MAGIC 0x958458f6
1206 static long gethugepagesize(const char *path, Error **errp)
1208 struct statfs fs;
1209 int ret;
1211 do {
1212 ret = statfs(path, &fs);
1213 } while (ret != 0 && errno == EINTR);
1215 if (ret != 0) {
1216 error_setg_errno(errp, errno, "failed to get page size of file %s",
1217 path);
1218 return 0;
1221 return fs.f_bsize;
1224 static void *file_ram_alloc(RAMBlock *block,
1225 ram_addr_t memory,
1226 const char *path,
1227 Error **errp)
1229 struct stat st;
1230 char *filename;
1231 char *sanitized_name;
1232 char *c;
1233 void *area;
1234 int fd;
1235 uint64_t hpagesize;
1236 Error *local_err = NULL;
1238 hpagesize = gethugepagesize(path, &local_err);
1239 if (local_err) {
1240 error_propagate(errp, local_err);
1241 goto error;
1243 block->mr->align = hpagesize;
1245 if (memory < hpagesize) {
1246 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
1247 "or larger than huge page size 0x%" PRIx64,
1248 memory, hpagesize);
1249 goto error;
1252 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1253 error_setg(errp,
1254 "host lacks kvm mmu notifiers, -mem-path unsupported");
1255 goto error;
1258 if (!stat(path, &st) && S_ISDIR(st.st_mode)) {
1259 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1260 sanitized_name = g_strdup(memory_region_name(block->mr));
1261 for (c = sanitized_name; *c != '\0'; c++) {
1262 if (*c == '/') {
1263 *c = '_';
1267 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1268 sanitized_name);
1269 g_free(sanitized_name);
1271 fd = mkstemp(filename);
1272 if (fd >= 0) {
1273 unlink(filename);
1275 g_free(filename);
1276 } else {
1277 fd = open(path, O_RDWR | O_CREAT, 0644);
1280 if (fd < 0) {
1281 error_setg_errno(errp, errno,
1282 "unable to create backing store for hugepages");
1283 goto error;
1286 memory = ROUND_UP(memory, hpagesize);
1289 * ftruncate is not supported by hugetlbfs in older
1290 * hosts, so don't bother bailing out on errors.
1291 * If anything goes wrong with it under other filesystems,
1292 * mmap will fail.
1294 if (ftruncate(fd, memory)) {
1295 perror("ftruncate");
1298 area = qemu_ram_mmap(fd, memory, hpagesize, block->flags & RAM_SHARED);
1299 if (area == MAP_FAILED) {
1300 error_setg_errno(errp, errno,
1301 "unable to map backing store for hugepages");
1302 close(fd);
1303 goto error;
1306 if (mem_prealloc) {
1307 os_mem_prealloc(fd, area, memory);
1310 block->fd = fd;
1311 return area;
1313 error:
1314 return NULL;
1316 #endif
1318 /* Called with the ramlist lock held. */
1319 static ram_addr_t find_ram_offset(ram_addr_t size)
1321 RAMBlock *block, *next_block;
1322 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1324 assert(size != 0); /* it would hand out same offset multiple times */
1326 if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
1327 return 0;
1330 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1331 ram_addr_t end, next = RAM_ADDR_MAX;
1333 end = block->offset + block->max_length;
1335 QLIST_FOREACH_RCU(next_block, &ram_list.blocks, next) {
1336 if (next_block->offset >= end) {
1337 next = MIN(next, next_block->offset);
1340 if (next - end >= size && next - end < mingap) {
1341 offset = end;
1342 mingap = next - end;
1346 if (offset == RAM_ADDR_MAX) {
1347 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1348 (uint64_t)size);
1349 abort();
1352 return offset;
1355 ram_addr_t last_ram_offset(void)
1357 RAMBlock *block;
1358 ram_addr_t last = 0;
1360 rcu_read_lock();
1361 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1362 last = MAX(last, block->offset + block->max_length);
1364 rcu_read_unlock();
1365 return last;
1368 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1370 int ret;
1372 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1373 if (!machine_dump_guest_core(current_machine)) {
1374 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1375 if (ret) {
1376 perror("qemu_madvise");
1377 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1378 "but dump_guest_core=off specified\n");
1383 /* Called within an RCU critical section, or while the ramlist lock
1384 * is held.
1386 static RAMBlock *find_ram_block(ram_addr_t addr)
1388 RAMBlock *block;
1390 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1391 if (block->offset == addr) {
1392 return block;
1396 return NULL;
1399 const char *qemu_ram_get_idstr(RAMBlock *rb)
1401 return rb->idstr;
1404 /* Called with iothread lock held. */
1405 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1407 RAMBlock *new_block, *block;
1409 rcu_read_lock();
1410 new_block = find_ram_block(addr);
1411 assert(new_block);
1412 assert(!new_block->idstr[0]);
1414 if (dev) {
1415 char *id = qdev_get_dev_path(dev);
1416 if (id) {
1417 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1418 g_free(id);
1421 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1423 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1424 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1425 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1426 new_block->idstr);
1427 abort();
1430 rcu_read_unlock();
1433 /* Called with iothread lock held. */
1434 void qemu_ram_unset_idstr(ram_addr_t addr)
1436 RAMBlock *block;
1438 /* FIXME: arch_init.c assumes that this is not called throughout
1439 * migration. Ignore the problem since hot-unplug during migration
1440 * does not work anyway.
1443 rcu_read_lock();
1444 block = find_ram_block(addr);
1445 if (block) {
1446 memset(block->idstr, 0, sizeof(block->idstr));
1448 rcu_read_unlock();
1451 static int memory_try_enable_merging(void *addr, size_t len)
1453 if (!machine_mem_merge(current_machine)) {
1454 /* disabled by the user */
1455 return 0;
1458 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1461 /* Only legal before guest might have detected the memory size: e.g. on
1462 * incoming migration, or right after reset.
1464 * As memory core doesn't know how is memory accessed, it is up to
1465 * resize callback to update device state and/or add assertions to detect
1466 * misuse, if necessary.
1468 int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp)
1470 RAMBlock *block = find_ram_block(base);
1472 assert(block);
1474 newsize = HOST_PAGE_ALIGN(newsize);
1476 if (block->used_length == newsize) {
1477 return 0;
1480 if (!(block->flags & RAM_RESIZEABLE)) {
1481 error_setg_errno(errp, EINVAL,
1482 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1483 " in != 0x" RAM_ADDR_FMT, block->idstr,
1484 newsize, block->used_length);
1485 return -EINVAL;
1488 if (block->max_length < newsize) {
1489 error_setg_errno(errp, EINVAL,
1490 "Length too large: %s: 0x" RAM_ADDR_FMT
1491 " > 0x" RAM_ADDR_FMT, block->idstr,
1492 newsize, block->max_length);
1493 return -EINVAL;
1496 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
1497 block->used_length = newsize;
1498 cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
1499 DIRTY_CLIENTS_ALL);
1500 memory_region_set_size(block->mr, newsize);
1501 if (block->resized) {
1502 block->resized(block->idstr, newsize, block->host);
1504 return 0;
1507 static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
1509 RAMBlock *block;
1510 RAMBlock *last_block = NULL;
1511 ram_addr_t old_ram_size, new_ram_size;
1512 Error *err = NULL;
1514 old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1516 qemu_mutex_lock_ramlist();
1517 new_block->offset = find_ram_offset(new_block->max_length);
1519 if (!new_block->host) {
1520 if (xen_enabled()) {
1521 xen_ram_alloc(new_block->offset, new_block->max_length,
1522 new_block->mr, &err);
1523 if (err) {
1524 error_propagate(errp, err);
1525 qemu_mutex_unlock_ramlist();
1526 return -1;
1528 } else {
1529 new_block->host = phys_mem_alloc(new_block->max_length,
1530 &new_block->mr->align);
1531 if (!new_block->host) {
1532 error_setg_errno(errp, errno,
1533 "cannot set up guest memory '%s'",
1534 memory_region_name(new_block->mr));
1535 qemu_mutex_unlock_ramlist();
1536 return -1;
1538 memory_try_enable_merging(new_block->host, new_block->max_length);
1542 new_ram_size = MAX(old_ram_size,
1543 (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
1544 if (new_ram_size > old_ram_size) {
1545 migration_bitmap_extend(old_ram_size, new_ram_size);
1547 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1548 * QLIST (which has an RCU-friendly variant) does not have insertion at
1549 * tail, so save the last element in last_block.
1551 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1552 last_block = block;
1553 if (block->max_length < new_block->max_length) {
1554 break;
1557 if (block) {
1558 QLIST_INSERT_BEFORE_RCU(block, new_block, next);
1559 } else if (last_block) {
1560 QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
1561 } else { /* list is empty */
1562 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
1564 ram_list.mru_block = NULL;
1566 /* Write list before version */
1567 smp_wmb();
1568 ram_list.version++;
1569 qemu_mutex_unlock_ramlist();
1571 new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1573 if (new_ram_size > old_ram_size) {
1574 int i;
1576 /* ram_list.dirty_memory[] is protected by the iothread lock. */
1577 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1578 ram_list.dirty_memory[i] =
1579 bitmap_zero_extend(ram_list.dirty_memory[i],
1580 old_ram_size, new_ram_size);
1583 cpu_physical_memory_set_dirty_range(new_block->offset,
1584 new_block->used_length,
1585 DIRTY_CLIENTS_ALL);
1587 if (new_block->host) {
1588 qemu_ram_setup_dump(new_block->host, new_block->max_length);
1589 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
1590 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
1591 if (kvm_enabled()) {
1592 kvm_setup_guest_memory(new_block->host, new_block->max_length);
1596 return new_block->offset;
1599 #ifdef __linux__
1600 ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
1601 bool share, const char *mem_path,
1602 Error **errp)
1604 RAMBlock *new_block;
1605 ram_addr_t addr;
1606 Error *local_err = NULL;
1608 if (xen_enabled()) {
1609 error_setg(errp, "-mem-path not supported with Xen");
1610 return -1;
1613 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1615 * file_ram_alloc() needs to allocate just like
1616 * phys_mem_alloc, but we haven't bothered to provide
1617 * a hook there.
1619 error_setg(errp,
1620 "-mem-path not supported with this accelerator");
1621 return -1;
1624 size = HOST_PAGE_ALIGN(size);
1625 new_block = g_malloc0(sizeof(*new_block));
1626 new_block->mr = mr;
1627 new_block->used_length = size;
1628 new_block->max_length = size;
1629 new_block->flags = share ? RAM_SHARED : 0;
1630 new_block->host = file_ram_alloc(new_block, size,
1631 mem_path, errp);
1632 if (!new_block->host) {
1633 g_free(new_block);
1634 return -1;
1637 addr = ram_block_add(new_block, &local_err);
1638 if (local_err) {
1639 g_free(new_block);
1640 error_propagate(errp, local_err);
1641 return -1;
1643 return addr;
1645 #endif
1647 static
1648 ram_addr_t qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
1649 void (*resized)(const char*,
1650 uint64_t length,
1651 void *host),
1652 void *host, bool resizeable,
1653 MemoryRegion *mr, Error **errp)
1655 RAMBlock *new_block;
1656 ram_addr_t addr;
1657 Error *local_err = NULL;
1659 size = HOST_PAGE_ALIGN(size);
1660 max_size = HOST_PAGE_ALIGN(max_size);
1661 new_block = g_malloc0(sizeof(*new_block));
1662 new_block->mr = mr;
1663 new_block->resized = resized;
1664 new_block->used_length = size;
1665 new_block->max_length = max_size;
1666 assert(max_size >= size);
1667 new_block->fd = -1;
1668 new_block->host = host;
1669 if (host) {
1670 new_block->flags |= RAM_PREALLOC;
1672 if (resizeable) {
1673 new_block->flags |= RAM_RESIZEABLE;
1675 addr = ram_block_add(new_block, &local_err);
1676 if (local_err) {
1677 g_free(new_block);
1678 error_propagate(errp, local_err);
1679 return -1;
1681 return addr;
1684 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1685 MemoryRegion *mr, Error **errp)
1687 return qemu_ram_alloc_internal(size, size, NULL, host, false, mr, errp);
1690 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
1692 return qemu_ram_alloc_internal(size, size, NULL, NULL, false, mr, errp);
1695 ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
1696 void (*resized)(const char*,
1697 uint64_t length,
1698 void *host),
1699 MemoryRegion *mr, Error **errp)
1701 return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true, mr, errp);
1704 static void reclaim_ramblock(RAMBlock *block)
1706 if (block->flags & RAM_PREALLOC) {
1708 } else if (xen_enabled()) {
1709 xen_invalidate_map_cache_entry(block->host);
1710 #ifndef _WIN32
1711 } else if (block->fd >= 0) {
1712 qemu_ram_munmap(block->host, block->max_length);
1713 close(block->fd);
1714 #endif
1715 } else {
1716 qemu_anon_ram_free(block->host, block->max_length);
1718 g_free(block);
1721 void qemu_ram_free(ram_addr_t addr)
1723 RAMBlock *block;
1725 qemu_mutex_lock_ramlist();
1726 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1727 if (addr == block->offset) {
1728 QLIST_REMOVE_RCU(block, next);
1729 ram_list.mru_block = NULL;
1730 /* Write list before version */
1731 smp_wmb();
1732 ram_list.version++;
1733 call_rcu(block, reclaim_ramblock, rcu);
1734 break;
1737 qemu_mutex_unlock_ramlist();
1740 #ifndef _WIN32
1741 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1743 RAMBlock *block;
1744 ram_addr_t offset;
1745 int flags;
1746 void *area, *vaddr;
1748 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1749 offset = addr - block->offset;
1750 if (offset < block->max_length) {
1751 vaddr = ramblock_ptr(block, offset);
1752 if (block->flags & RAM_PREALLOC) {
1754 } else if (xen_enabled()) {
1755 abort();
1756 } else {
1757 flags = MAP_FIXED;
1758 if (block->fd >= 0) {
1759 flags |= (block->flags & RAM_SHARED ?
1760 MAP_SHARED : MAP_PRIVATE);
1761 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1762 flags, block->fd, offset);
1763 } else {
1765 * Remap needs to match alloc. Accelerators that
1766 * set phys_mem_alloc never remap. If they did,
1767 * we'd need a remap hook here.
1769 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1771 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1772 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1773 flags, -1, 0);
1775 if (area != vaddr) {
1776 fprintf(stderr, "Could not remap addr: "
1777 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1778 length, addr);
1779 exit(1);
1781 memory_try_enable_merging(vaddr, length);
1782 qemu_ram_setup_dump(vaddr, length);
1787 #endif /* !_WIN32 */
1789 int qemu_get_ram_fd(ram_addr_t addr)
1791 RAMBlock *block;
1792 int fd;
1794 rcu_read_lock();
1795 block = qemu_get_ram_block(addr);
1796 fd = block->fd;
1797 rcu_read_unlock();
1798 return fd;
1801 void qemu_set_ram_fd(ram_addr_t addr, int fd)
1803 RAMBlock *block;
1805 rcu_read_lock();
1806 block = qemu_get_ram_block(addr);
1807 block->fd = fd;
1808 rcu_read_unlock();
1811 void *qemu_get_ram_block_host_ptr(ram_addr_t addr)
1813 RAMBlock *block;
1814 void *ptr;
1816 rcu_read_lock();
1817 block = qemu_get_ram_block(addr);
1818 ptr = ramblock_ptr(block, 0);
1819 rcu_read_unlock();
1820 return ptr;
1823 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1824 * This should not be used for general purpose DMA. Use address_space_map
1825 * or address_space_rw instead. For local memory (e.g. video ram) that the
1826 * device owns, use memory_region_get_ram_ptr.
1828 * Called within RCU critical section.
1830 void *qemu_get_ram_ptr(ram_addr_t addr)
1832 RAMBlock *block = qemu_get_ram_block(addr);
1834 if (xen_enabled() && block->host == NULL) {
1835 /* We need to check if the requested address is in the RAM
1836 * because we don't want to map the entire memory in QEMU.
1837 * In that case just map until the end of the page.
1839 if (block->offset == 0) {
1840 return xen_map_cache(addr, 0, 0);
1843 block->host = xen_map_cache(block->offset, block->max_length, 1);
1845 return ramblock_ptr(block, addr - block->offset);
1848 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1849 * but takes a size argument.
1851 * Called within RCU critical section.
1853 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1855 RAMBlock *block;
1856 ram_addr_t offset_inside_block;
1857 if (*size == 0) {
1858 return NULL;
1861 block = qemu_get_ram_block(addr);
1862 offset_inside_block = addr - block->offset;
1863 *size = MIN(*size, block->max_length - offset_inside_block);
1865 if (xen_enabled() && block->host == NULL) {
1866 /* We need to check if the requested address is in the RAM
1867 * because we don't want to map the entire memory in QEMU.
1868 * In that case just map the requested area.
1870 if (block->offset == 0) {
1871 return xen_map_cache(addr, *size, 1);
1874 block->host = xen_map_cache(block->offset, block->max_length, 1);
1877 return ramblock_ptr(block, offset_inside_block);
1881 * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
1882 * in that RAMBlock.
1884 * ptr: Host pointer to look up
1885 * round_offset: If true round the result offset down to a page boundary
1886 * *ram_addr: set to result ram_addr
1887 * *offset: set to result offset within the RAMBlock
1889 * Returns: RAMBlock (or NULL if not found)
1891 * By the time this function returns, the returned pointer is not protected
1892 * by RCU anymore. If the caller is not within an RCU critical section and
1893 * does not hold the iothread lock, it must have other means of protecting the
1894 * pointer, such as a reference to the region that includes the incoming
1895 * ram_addr_t.
1897 RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
1898 ram_addr_t *ram_addr,
1899 ram_addr_t *offset)
1901 RAMBlock *block;
1902 uint8_t *host = ptr;
1904 if (xen_enabled()) {
1905 rcu_read_lock();
1906 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1907 block = qemu_get_ram_block(*ram_addr);
1908 if (block) {
1909 *offset = (host - block->host);
1911 rcu_read_unlock();
1912 return block;
1915 rcu_read_lock();
1916 block = atomic_rcu_read(&ram_list.mru_block);
1917 if (block && block->host && host - block->host < block->max_length) {
1918 goto found;
1921 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1922 /* This case append when the block is not mapped. */
1923 if (block->host == NULL) {
1924 continue;
1926 if (host - block->host < block->max_length) {
1927 goto found;
1931 rcu_read_unlock();
1932 return NULL;
1934 found:
1935 *offset = (host - block->host);
1936 if (round_offset) {
1937 *offset &= TARGET_PAGE_MASK;
1939 *ram_addr = block->offset + *offset;
1940 rcu_read_unlock();
1941 return block;
1945 * Finds the named RAMBlock
1947 * name: The name of RAMBlock to find
1949 * Returns: RAMBlock (or NULL if not found)
1951 RAMBlock *qemu_ram_block_by_name(const char *name)
1953 RAMBlock *block;
1955 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1956 if (!strcmp(name, block->idstr)) {
1957 return block;
1961 return NULL;
1964 /* Some of the softmmu routines need to translate from a host pointer
1965 (typically a TLB entry) back to a ram offset. */
1966 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1968 RAMBlock *block;
1969 ram_addr_t offset; /* Not used */
1971 block = qemu_ram_block_from_host(ptr, false, ram_addr, &offset);
1973 if (!block) {
1974 return NULL;
1977 return block->mr;
1980 /* Called within RCU critical section. */
1981 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1982 uint64_t val, unsigned size)
1984 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
1985 tb_invalidate_phys_page_fast(ram_addr, size);
1987 switch (size) {
1988 case 1:
1989 stb_p(qemu_get_ram_ptr(ram_addr), val);
1990 break;
1991 case 2:
1992 stw_p(qemu_get_ram_ptr(ram_addr), val);
1993 break;
1994 case 4:
1995 stl_p(qemu_get_ram_ptr(ram_addr), val);
1996 break;
1997 default:
1998 abort();
2000 /* Set both VGA and migration bits for simplicity and to remove
2001 * the notdirty callback faster.
2003 cpu_physical_memory_set_dirty_range(ram_addr, size,
2004 DIRTY_CLIENTS_NOCODE);
2005 /* we remove the notdirty callback only if the code has been
2006 flushed */
2007 if (!cpu_physical_memory_is_clean(ram_addr)) {
2008 tlb_set_dirty(current_cpu, current_cpu->mem_io_vaddr);
2012 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
2013 unsigned size, bool is_write)
2015 return is_write;
2018 static const MemoryRegionOps notdirty_mem_ops = {
2019 .write = notdirty_mem_write,
2020 .valid.accepts = notdirty_mem_accepts,
2021 .endianness = DEVICE_NATIVE_ENDIAN,
2024 /* Generate a debug exception if a watchpoint has been hit. */
2025 static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
2027 CPUState *cpu = current_cpu;
2028 CPUArchState *env = cpu->env_ptr;
2029 target_ulong pc, cs_base;
2030 target_ulong vaddr;
2031 CPUWatchpoint *wp;
2032 int cpu_flags;
2034 if (cpu->watchpoint_hit) {
2035 /* We re-entered the check after replacing the TB. Now raise
2036 * the debug interrupt so that is will trigger after the
2037 * current instruction. */
2038 cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
2039 return;
2041 vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2042 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
2043 if (cpu_watchpoint_address_matches(wp, vaddr, len)
2044 && (wp->flags & flags)) {
2045 if (flags == BP_MEM_READ) {
2046 wp->flags |= BP_WATCHPOINT_HIT_READ;
2047 } else {
2048 wp->flags |= BP_WATCHPOINT_HIT_WRITE;
2050 wp->hitaddr = vaddr;
2051 wp->hitattrs = attrs;
2052 if (!cpu->watchpoint_hit) {
2053 cpu->watchpoint_hit = wp;
2054 tb_check_watchpoint(cpu);
2055 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2056 cpu->exception_index = EXCP_DEBUG;
2057 cpu_loop_exit(cpu);
2058 } else {
2059 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2060 tb_gen_code(cpu, pc, cs_base, cpu_flags, 1);
2061 cpu_resume_from_signal(cpu, NULL);
2064 } else {
2065 wp->flags &= ~BP_WATCHPOINT_HIT;
2070 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2071 so these check for a hit then pass through to the normal out-of-line
2072 phys routines. */
2073 static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
2074 unsigned size, MemTxAttrs attrs)
2076 MemTxResult res;
2077 uint64_t data;
2078 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2079 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
2081 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
2082 switch (size) {
2083 case 1:
2084 data = address_space_ldub(as, addr, attrs, &res);
2085 break;
2086 case 2:
2087 data = address_space_lduw(as, addr, attrs, &res);
2088 break;
2089 case 4:
2090 data = address_space_ldl(as, addr, attrs, &res);
2091 break;
2092 default: abort();
2094 *pdata = data;
2095 return res;
2098 static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
2099 uint64_t val, unsigned size,
2100 MemTxAttrs attrs)
2102 MemTxResult res;
2103 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2104 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
2106 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
2107 switch (size) {
2108 case 1:
2109 address_space_stb(as, addr, val, attrs, &res);
2110 break;
2111 case 2:
2112 address_space_stw(as, addr, val, attrs, &res);
2113 break;
2114 case 4:
2115 address_space_stl(as, addr, val, attrs, &res);
2116 break;
2117 default: abort();
2119 return res;
2122 static const MemoryRegionOps watch_mem_ops = {
2123 .read_with_attrs = watch_mem_read,
2124 .write_with_attrs = watch_mem_write,
2125 .endianness = DEVICE_NATIVE_ENDIAN,
2128 static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
2129 unsigned len, MemTxAttrs attrs)
2131 subpage_t *subpage = opaque;
2132 uint8_t buf[8];
2133 MemTxResult res;
2135 #if defined(DEBUG_SUBPAGE)
2136 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
2137 subpage, len, addr);
2138 #endif
2139 res = address_space_read(subpage->as, addr + subpage->base,
2140 attrs, buf, len);
2141 if (res) {
2142 return res;
2144 switch (len) {
2145 case 1:
2146 *data = ldub_p(buf);
2147 return MEMTX_OK;
2148 case 2:
2149 *data = lduw_p(buf);
2150 return MEMTX_OK;
2151 case 4:
2152 *data = ldl_p(buf);
2153 return MEMTX_OK;
2154 case 8:
2155 *data = ldq_p(buf);
2156 return MEMTX_OK;
2157 default:
2158 abort();
2162 static MemTxResult subpage_write(void *opaque, hwaddr addr,
2163 uint64_t value, unsigned len, MemTxAttrs attrs)
2165 subpage_t *subpage = opaque;
2166 uint8_t buf[8];
2168 #if defined(DEBUG_SUBPAGE)
2169 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
2170 " value %"PRIx64"\n",
2171 __func__, subpage, len, addr, value);
2172 #endif
2173 switch (len) {
2174 case 1:
2175 stb_p(buf, value);
2176 break;
2177 case 2:
2178 stw_p(buf, value);
2179 break;
2180 case 4:
2181 stl_p(buf, value);
2182 break;
2183 case 8:
2184 stq_p(buf, value);
2185 break;
2186 default:
2187 abort();
2189 return address_space_write(subpage->as, addr + subpage->base,
2190 attrs, buf, len);
2193 static bool subpage_accepts(void *opaque, hwaddr addr,
2194 unsigned len, bool is_write)
2196 subpage_t *subpage = opaque;
2197 #if defined(DEBUG_SUBPAGE)
2198 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
2199 __func__, subpage, is_write ? 'w' : 'r', len, addr);
2200 #endif
2202 return address_space_access_valid(subpage->as, addr + subpage->base,
2203 len, is_write);
2206 static const MemoryRegionOps subpage_ops = {
2207 .read_with_attrs = subpage_read,
2208 .write_with_attrs = subpage_write,
2209 .impl.min_access_size = 1,
2210 .impl.max_access_size = 8,
2211 .valid.min_access_size = 1,
2212 .valid.max_access_size = 8,
2213 .valid.accepts = subpage_accepts,
2214 .endianness = DEVICE_NATIVE_ENDIAN,
2217 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2218 uint16_t section)
2220 int idx, eidx;
2222 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2223 return -1;
2224 idx = SUBPAGE_IDX(start);
2225 eidx = SUBPAGE_IDX(end);
2226 #if defined(DEBUG_SUBPAGE)
2227 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2228 __func__, mmio, start, end, idx, eidx, section);
2229 #endif
2230 for (; idx <= eidx; idx++) {
2231 mmio->sub_section[idx] = section;
2234 return 0;
2237 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
2239 subpage_t *mmio;
2241 mmio = g_malloc0(sizeof(subpage_t));
2243 mmio->as = as;
2244 mmio->base = base;
2245 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
2246 NULL, TARGET_PAGE_SIZE);
2247 mmio->iomem.subpage = true;
2248 #if defined(DEBUG_SUBPAGE)
2249 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
2250 mmio, base, TARGET_PAGE_SIZE);
2251 #endif
2252 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
2254 return mmio;
2257 static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
2258 MemoryRegion *mr)
2260 assert(as);
2261 MemoryRegionSection section = {
2262 .address_space = as,
2263 .mr = mr,
2264 .offset_within_address_space = 0,
2265 .offset_within_region = 0,
2266 .size = int128_2_64(),
2269 return phys_section_add(map, &section);
2272 MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index, MemTxAttrs attrs)
2274 int asidx = cpu_asidx_from_attrs(cpu, attrs);
2275 CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
2276 AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
2277 MemoryRegionSection *sections = d->map.sections;
2279 return sections[index & ~TARGET_PAGE_MASK].mr;
2282 static void io_mem_init(void)
2284 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, NULL, UINT64_MAX);
2285 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
2286 NULL, UINT64_MAX);
2287 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
2288 NULL, UINT64_MAX);
2289 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
2290 NULL, UINT64_MAX);
2293 static void mem_begin(MemoryListener *listener)
2295 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2296 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
2297 uint16_t n;
2299 n = dummy_section(&d->map, as, &io_mem_unassigned);
2300 assert(n == PHYS_SECTION_UNASSIGNED);
2301 n = dummy_section(&d->map, as, &io_mem_notdirty);
2302 assert(n == PHYS_SECTION_NOTDIRTY);
2303 n = dummy_section(&d->map, as, &io_mem_rom);
2304 assert(n == PHYS_SECTION_ROM);
2305 n = dummy_section(&d->map, as, &io_mem_watch);
2306 assert(n == PHYS_SECTION_WATCH);
2308 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
2309 d->as = as;
2310 as->next_dispatch = d;
2313 static void address_space_dispatch_free(AddressSpaceDispatch *d)
2315 phys_sections_free(&d->map);
2316 g_free(d);
2319 static void mem_commit(MemoryListener *listener)
2321 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2322 AddressSpaceDispatch *cur = as->dispatch;
2323 AddressSpaceDispatch *next = as->next_dispatch;
2325 phys_page_compact_all(next, next->map.nodes_nb);
2327 atomic_rcu_set(&as->dispatch, next);
2328 if (cur) {
2329 call_rcu(cur, address_space_dispatch_free, rcu);
2333 static void tcg_commit(MemoryListener *listener)
2335 CPUAddressSpace *cpuas;
2336 AddressSpaceDispatch *d;
2338 /* since each CPU stores ram addresses in its TLB cache, we must
2339 reset the modified entries */
2340 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
2341 cpu_reloading_memory_map();
2342 /* The CPU and TLB are protected by the iothread lock.
2343 * We reload the dispatch pointer now because cpu_reloading_memory_map()
2344 * may have split the RCU critical section.
2346 d = atomic_rcu_read(&cpuas->as->dispatch);
2347 cpuas->memory_dispatch = d;
2348 tlb_flush(cpuas->cpu, 1);
2351 void address_space_init_dispatch(AddressSpace *as)
2353 as->dispatch = NULL;
2354 as->dispatch_listener = (MemoryListener) {
2355 .begin = mem_begin,
2356 .commit = mem_commit,
2357 .region_add = mem_add,
2358 .region_nop = mem_add,
2359 .priority = 0,
2361 memory_listener_register(&as->dispatch_listener, as);
2364 void address_space_unregister(AddressSpace *as)
2366 memory_listener_unregister(&as->dispatch_listener);
2369 void address_space_destroy_dispatch(AddressSpace *as)
2371 AddressSpaceDispatch *d = as->dispatch;
2373 atomic_rcu_set(&as->dispatch, NULL);
2374 if (d) {
2375 call_rcu(d, address_space_dispatch_free, rcu);
2379 static void memory_map_init(void)
2381 system_memory = g_malloc(sizeof(*system_memory));
2383 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
2384 address_space_init(&address_space_memory, system_memory, "memory");
2386 system_io = g_malloc(sizeof(*system_io));
2387 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2388 65536);
2389 address_space_init(&address_space_io, system_io, "I/O");
2392 MemoryRegion *get_system_memory(void)
2394 return system_memory;
2397 MemoryRegion *get_system_io(void)
2399 return system_io;
2402 #endif /* !defined(CONFIG_USER_ONLY) */
2404 /* physical memory access (slow version, mainly for debug) */
2405 #if defined(CONFIG_USER_ONLY)
2406 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2407 uint8_t *buf, int len, int is_write)
2409 int l, flags;
2410 target_ulong page;
2411 void * p;
2413 while (len > 0) {
2414 page = addr & TARGET_PAGE_MASK;
2415 l = (page + TARGET_PAGE_SIZE) - addr;
2416 if (l > len)
2417 l = len;
2418 flags = page_get_flags(page);
2419 if (!(flags & PAGE_VALID))
2420 return -1;
2421 if (is_write) {
2422 if (!(flags & PAGE_WRITE))
2423 return -1;
2424 /* XXX: this code should not depend on lock_user */
2425 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2426 return -1;
2427 memcpy(p, buf, l);
2428 unlock_user(p, addr, l);
2429 } else {
2430 if (!(flags & PAGE_READ))
2431 return -1;
2432 /* XXX: this code should not depend on lock_user */
2433 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2434 return -1;
2435 memcpy(buf, p, l);
2436 unlock_user(p, addr, 0);
2438 len -= l;
2439 buf += l;
2440 addr += l;
2442 return 0;
2445 #else
2447 static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
2448 hwaddr length)
2450 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
2451 /* No early return if dirty_log_mask is or becomes 0, because
2452 * cpu_physical_memory_set_dirty_range will still call
2453 * xen_modified_memory.
2455 if (dirty_log_mask) {
2456 dirty_log_mask =
2457 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
2459 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
2460 tb_invalidate_phys_range(addr, addr + length);
2461 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
2463 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
2466 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
2468 unsigned access_size_max = mr->ops->valid.max_access_size;
2470 /* Regions are assumed to support 1-4 byte accesses unless
2471 otherwise specified. */
2472 if (access_size_max == 0) {
2473 access_size_max = 4;
2476 /* Bound the maximum access by the alignment of the address. */
2477 if (!mr->ops->impl.unaligned) {
2478 unsigned align_size_max = addr & -addr;
2479 if (align_size_max != 0 && align_size_max < access_size_max) {
2480 access_size_max = align_size_max;
2484 /* Don't attempt accesses larger than the maximum. */
2485 if (l > access_size_max) {
2486 l = access_size_max;
2488 l = pow2floor(l);
2490 return l;
2493 static bool prepare_mmio_access(MemoryRegion *mr)
2495 bool unlocked = !qemu_mutex_iothread_locked();
2496 bool release_lock = false;
2498 if (unlocked && mr->global_locking) {
2499 qemu_mutex_lock_iothread();
2500 unlocked = false;
2501 release_lock = true;
2503 if (mr->flush_coalesced_mmio) {
2504 if (unlocked) {
2505 qemu_mutex_lock_iothread();
2507 qemu_flush_coalesced_mmio_buffer();
2508 if (unlocked) {
2509 qemu_mutex_unlock_iothread();
2513 return release_lock;
2516 /* Called within RCU critical section. */
2517 static MemTxResult address_space_write_continue(AddressSpace *as, hwaddr addr,
2518 MemTxAttrs attrs,
2519 const uint8_t *buf,
2520 int len, hwaddr addr1,
2521 hwaddr l, MemoryRegion *mr)
2523 uint8_t *ptr;
2524 uint64_t val;
2525 MemTxResult result = MEMTX_OK;
2526 bool release_lock = false;
2528 for (;;) {
2529 if (!memory_access_is_direct(mr, true)) {
2530 release_lock |= prepare_mmio_access(mr);
2531 l = memory_access_size(mr, l, addr1);
2532 /* XXX: could force current_cpu to NULL to avoid
2533 potential bugs */
2534 switch (l) {
2535 case 8:
2536 /* 64 bit write access */
2537 val = ldq_p(buf);
2538 result |= memory_region_dispatch_write(mr, addr1, val, 8,
2539 attrs);
2540 break;
2541 case 4:
2542 /* 32 bit write access */
2543 val = ldl_p(buf);
2544 result |= memory_region_dispatch_write(mr, addr1, val, 4,
2545 attrs);
2546 break;
2547 case 2:
2548 /* 16 bit write access */
2549 val = lduw_p(buf);
2550 result |= memory_region_dispatch_write(mr, addr1, val, 2,
2551 attrs);
2552 break;
2553 case 1:
2554 /* 8 bit write access */
2555 val = ldub_p(buf);
2556 result |= memory_region_dispatch_write(mr, addr1, val, 1,
2557 attrs);
2558 break;
2559 default:
2560 abort();
2562 } else {
2563 addr1 += memory_region_get_ram_addr(mr);
2564 /* RAM case */
2565 ptr = qemu_get_ram_ptr(addr1);
2566 memcpy(ptr, buf, l);
2567 invalidate_and_set_dirty(mr, addr1, l);
2570 if (release_lock) {
2571 qemu_mutex_unlock_iothread();
2572 release_lock = false;
2575 len -= l;
2576 buf += l;
2577 addr += l;
2579 if (!len) {
2580 break;
2583 l = len;
2584 mr = address_space_translate(as, addr, &addr1, &l, true);
2587 return result;
2590 MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2591 const uint8_t *buf, int len)
2593 hwaddr l;
2594 hwaddr addr1;
2595 MemoryRegion *mr;
2596 MemTxResult result = MEMTX_OK;
2598 if (len > 0) {
2599 rcu_read_lock();
2600 l = len;
2601 mr = address_space_translate(as, addr, &addr1, &l, true);
2602 result = address_space_write_continue(as, addr, attrs, buf, len,
2603 addr1, l, mr);
2604 rcu_read_unlock();
2607 return result;
2610 /* Called within RCU critical section. */
2611 MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr,
2612 MemTxAttrs attrs, uint8_t *buf,
2613 int len, hwaddr addr1, hwaddr l,
2614 MemoryRegion *mr)
2616 uint8_t *ptr;
2617 uint64_t val;
2618 MemTxResult result = MEMTX_OK;
2619 bool release_lock = false;
2621 for (;;) {
2622 if (!memory_access_is_direct(mr, false)) {
2623 /* I/O case */
2624 release_lock |= prepare_mmio_access(mr);
2625 l = memory_access_size(mr, l, addr1);
2626 switch (l) {
2627 case 8:
2628 /* 64 bit read access */
2629 result |= memory_region_dispatch_read(mr, addr1, &val, 8,
2630 attrs);
2631 stq_p(buf, val);
2632 break;
2633 case 4:
2634 /* 32 bit read access */
2635 result |= memory_region_dispatch_read(mr, addr1, &val, 4,
2636 attrs);
2637 stl_p(buf, val);
2638 break;
2639 case 2:
2640 /* 16 bit read access */
2641 result |= memory_region_dispatch_read(mr, addr1, &val, 2,
2642 attrs);
2643 stw_p(buf, val);
2644 break;
2645 case 1:
2646 /* 8 bit read access */
2647 result |= memory_region_dispatch_read(mr, addr1, &val, 1,
2648 attrs);
2649 stb_p(buf, val);
2650 break;
2651 default:
2652 abort();
2654 } else {
2655 /* RAM case */
2656 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2657 memcpy(buf, ptr, l);
2660 if (release_lock) {
2661 qemu_mutex_unlock_iothread();
2662 release_lock = false;
2665 len -= l;
2666 buf += l;
2667 addr += l;
2669 if (!len) {
2670 break;
2673 l = len;
2674 mr = address_space_translate(as, addr, &addr1, &l, false);
2677 return result;
2680 MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
2681 MemTxAttrs attrs, uint8_t *buf, int len)
2683 hwaddr l;
2684 hwaddr addr1;
2685 MemoryRegion *mr;
2686 MemTxResult result = MEMTX_OK;
2688 if (len > 0) {
2689 rcu_read_lock();
2690 l = len;
2691 mr = address_space_translate(as, addr, &addr1, &l, false);
2692 result = address_space_read_continue(as, addr, attrs, buf, len,
2693 addr1, l, mr);
2694 rcu_read_unlock();
2697 return result;
2700 MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2701 uint8_t *buf, int len, bool is_write)
2703 if (is_write) {
2704 return address_space_write(as, addr, attrs, (uint8_t *)buf, len);
2705 } else {
2706 return address_space_read(as, addr, attrs, (uint8_t *)buf, len);
2710 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2711 int len, int is_write)
2713 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
2714 buf, len, is_write);
2717 enum write_rom_type {
2718 WRITE_DATA,
2719 FLUSH_CACHE,
2722 static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
2723 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
2725 hwaddr l;
2726 uint8_t *ptr;
2727 hwaddr addr1;
2728 MemoryRegion *mr;
2730 rcu_read_lock();
2731 while (len > 0) {
2732 l = len;
2733 mr = address_space_translate(as, addr, &addr1, &l, true);
2735 if (!(memory_region_is_ram(mr) ||
2736 memory_region_is_romd(mr))) {
2737 l = memory_access_size(mr, l, addr1);
2738 } else {
2739 addr1 += memory_region_get_ram_addr(mr);
2740 /* ROM/RAM case */
2741 ptr = qemu_get_ram_ptr(addr1);
2742 switch (type) {
2743 case WRITE_DATA:
2744 memcpy(ptr, buf, l);
2745 invalidate_and_set_dirty(mr, addr1, l);
2746 break;
2747 case FLUSH_CACHE:
2748 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
2749 break;
2752 len -= l;
2753 buf += l;
2754 addr += l;
2756 rcu_read_unlock();
2759 /* used for ROM loading : can write in RAM and ROM */
2760 void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
2761 const uint8_t *buf, int len)
2763 cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
2766 void cpu_flush_icache_range(hwaddr start, int len)
2769 * This function should do the same thing as an icache flush that was
2770 * triggered from within the guest. For TCG we are always cache coherent,
2771 * so there is no need to flush anything. For KVM / Xen we need to flush
2772 * the host's instruction cache at least.
2774 if (tcg_enabled()) {
2775 return;
2778 cpu_physical_memory_write_rom_internal(&address_space_memory,
2779 start, NULL, len, FLUSH_CACHE);
2782 typedef struct {
2783 MemoryRegion *mr;
2784 void *buffer;
2785 hwaddr addr;
2786 hwaddr len;
2787 bool in_use;
2788 } BounceBuffer;
2790 static BounceBuffer bounce;
2792 typedef struct MapClient {
2793 QEMUBH *bh;
2794 QLIST_ENTRY(MapClient) link;
2795 } MapClient;
2797 QemuMutex map_client_list_lock;
2798 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2799 = QLIST_HEAD_INITIALIZER(map_client_list);
2801 static void cpu_unregister_map_client_do(MapClient *client)
2803 QLIST_REMOVE(client, link);
2804 g_free(client);
2807 static void cpu_notify_map_clients_locked(void)
2809 MapClient *client;
2811 while (!QLIST_EMPTY(&map_client_list)) {
2812 client = QLIST_FIRST(&map_client_list);
2813 qemu_bh_schedule(client->bh);
2814 cpu_unregister_map_client_do(client);
2818 void cpu_register_map_client(QEMUBH *bh)
2820 MapClient *client = g_malloc(sizeof(*client));
2822 qemu_mutex_lock(&map_client_list_lock);
2823 client->bh = bh;
2824 QLIST_INSERT_HEAD(&map_client_list, client, link);
2825 if (!atomic_read(&bounce.in_use)) {
2826 cpu_notify_map_clients_locked();
2828 qemu_mutex_unlock(&map_client_list_lock);
2831 void cpu_exec_init_all(void)
2833 qemu_mutex_init(&ram_list.mutex);
2834 io_mem_init();
2835 memory_map_init();
2836 qemu_mutex_init(&map_client_list_lock);
2839 void cpu_unregister_map_client(QEMUBH *bh)
2841 MapClient *client;
2843 qemu_mutex_lock(&map_client_list_lock);
2844 QLIST_FOREACH(client, &map_client_list, link) {
2845 if (client->bh == bh) {
2846 cpu_unregister_map_client_do(client);
2847 break;
2850 qemu_mutex_unlock(&map_client_list_lock);
2853 static void cpu_notify_map_clients(void)
2855 qemu_mutex_lock(&map_client_list_lock);
2856 cpu_notify_map_clients_locked();
2857 qemu_mutex_unlock(&map_client_list_lock);
2860 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2862 MemoryRegion *mr;
2863 hwaddr l, xlat;
2865 rcu_read_lock();
2866 while (len > 0) {
2867 l = len;
2868 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2869 if (!memory_access_is_direct(mr, is_write)) {
2870 l = memory_access_size(mr, l, addr);
2871 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2872 return false;
2876 len -= l;
2877 addr += l;
2879 rcu_read_unlock();
2880 return true;
2883 /* Map a physical memory region into a host virtual address.
2884 * May map a subset of the requested range, given by and returned in *plen.
2885 * May return NULL if resources needed to perform the mapping are exhausted.
2886 * Use only for reads OR writes - not for read-modify-write operations.
2887 * Use cpu_register_map_client() to know when retrying the map operation is
2888 * likely to succeed.
2890 void *address_space_map(AddressSpace *as,
2891 hwaddr addr,
2892 hwaddr *plen,
2893 bool is_write)
2895 hwaddr len = *plen;
2896 hwaddr done = 0;
2897 hwaddr l, xlat, base;
2898 MemoryRegion *mr, *this_mr;
2899 ram_addr_t raddr;
2900 void *ptr;
2902 if (len == 0) {
2903 return NULL;
2906 l = len;
2907 rcu_read_lock();
2908 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2910 if (!memory_access_is_direct(mr, is_write)) {
2911 if (atomic_xchg(&bounce.in_use, true)) {
2912 rcu_read_unlock();
2913 return NULL;
2915 /* Avoid unbounded allocations */
2916 l = MIN(l, TARGET_PAGE_SIZE);
2917 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
2918 bounce.addr = addr;
2919 bounce.len = l;
2921 memory_region_ref(mr);
2922 bounce.mr = mr;
2923 if (!is_write) {
2924 address_space_read(as, addr, MEMTXATTRS_UNSPECIFIED,
2925 bounce.buffer, l);
2928 rcu_read_unlock();
2929 *plen = l;
2930 return bounce.buffer;
2933 base = xlat;
2934 raddr = memory_region_get_ram_addr(mr);
2936 for (;;) {
2937 len -= l;
2938 addr += l;
2939 done += l;
2940 if (len == 0) {
2941 break;
2944 l = len;
2945 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2946 if (this_mr != mr || xlat != base + done) {
2947 break;
2951 memory_region_ref(mr);
2952 *plen = done;
2953 ptr = qemu_ram_ptr_length(raddr + base, plen);
2954 rcu_read_unlock();
2956 return ptr;
2959 /* Unmaps a memory region previously mapped by address_space_map().
2960 * Will also mark the memory as dirty if is_write == 1. access_len gives
2961 * the amount of memory that was actually read or written by the caller.
2963 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2964 int is_write, hwaddr access_len)
2966 if (buffer != bounce.buffer) {
2967 MemoryRegion *mr;
2968 ram_addr_t addr1;
2970 mr = qemu_ram_addr_from_host(buffer, &addr1);
2971 assert(mr != NULL);
2972 if (is_write) {
2973 invalidate_and_set_dirty(mr, addr1, access_len);
2975 if (xen_enabled()) {
2976 xen_invalidate_map_cache_entry(buffer);
2978 memory_region_unref(mr);
2979 return;
2981 if (is_write) {
2982 address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
2983 bounce.buffer, access_len);
2985 qemu_vfree(bounce.buffer);
2986 bounce.buffer = NULL;
2987 memory_region_unref(bounce.mr);
2988 atomic_mb_set(&bounce.in_use, false);
2989 cpu_notify_map_clients();
2992 void *cpu_physical_memory_map(hwaddr addr,
2993 hwaddr *plen,
2994 int is_write)
2996 return address_space_map(&address_space_memory, addr, plen, is_write);
2999 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
3000 int is_write, hwaddr access_len)
3002 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
3005 /* warning: addr must be aligned */
3006 static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
3007 MemTxAttrs attrs,
3008 MemTxResult *result,
3009 enum device_endian endian)
3011 uint8_t *ptr;
3012 uint64_t val;
3013 MemoryRegion *mr;
3014 hwaddr l = 4;
3015 hwaddr addr1;
3016 MemTxResult r;
3017 bool release_lock = false;
3019 rcu_read_lock();
3020 mr = address_space_translate(as, addr, &addr1, &l, false);
3021 if (l < 4 || !memory_access_is_direct(mr, false)) {
3022 release_lock |= prepare_mmio_access(mr);
3024 /* I/O case */
3025 r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
3026 #if defined(TARGET_WORDS_BIGENDIAN)
3027 if (endian == DEVICE_LITTLE_ENDIAN) {
3028 val = bswap32(val);
3030 #else
3031 if (endian == DEVICE_BIG_ENDIAN) {
3032 val = bswap32(val);
3034 #endif
3035 } else {
3036 /* RAM case */
3037 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
3038 & TARGET_PAGE_MASK)
3039 + addr1);
3040 switch (endian) {
3041 case DEVICE_LITTLE_ENDIAN:
3042 val = ldl_le_p(ptr);
3043 break;
3044 case DEVICE_BIG_ENDIAN:
3045 val = ldl_be_p(ptr);
3046 break;
3047 default:
3048 val = ldl_p(ptr);
3049 break;
3051 r = MEMTX_OK;
3053 if (result) {
3054 *result = r;
3056 if (release_lock) {
3057 qemu_mutex_unlock_iothread();
3059 rcu_read_unlock();
3060 return val;
3063 uint32_t address_space_ldl(AddressSpace *as, hwaddr addr,
3064 MemTxAttrs attrs, MemTxResult *result)
3066 return address_space_ldl_internal(as, addr, attrs, result,
3067 DEVICE_NATIVE_ENDIAN);
3070 uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr,
3071 MemTxAttrs attrs, MemTxResult *result)
3073 return address_space_ldl_internal(as, addr, attrs, result,
3074 DEVICE_LITTLE_ENDIAN);
3077 uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr,
3078 MemTxAttrs attrs, MemTxResult *result)
3080 return address_space_ldl_internal(as, addr, attrs, result,
3081 DEVICE_BIG_ENDIAN);
3084 uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
3086 return address_space_ldl(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3089 uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
3091 return address_space_ldl_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3094 uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
3096 return address_space_ldl_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3099 /* warning: addr must be aligned */
3100 static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
3101 MemTxAttrs attrs,
3102 MemTxResult *result,
3103 enum device_endian endian)
3105 uint8_t *ptr;
3106 uint64_t val;
3107 MemoryRegion *mr;
3108 hwaddr l = 8;
3109 hwaddr addr1;
3110 MemTxResult r;
3111 bool release_lock = false;
3113 rcu_read_lock();
3114 mr = address_space_translate(as, addr, &addr1, &l,
3115 false);
3116 if (l < 8 || !memory_access_is_direct(mr, false)) {
3117 release_lock |= prepare_mmio_access(mr);
3119 /* I/O case */
3120 r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
3121 #if defined(TARGET_WORDS_BIGENDIAN)
3122 if (endian == DEVICE_LITTLE_ENDIAN) {
3123 val = bswap64(val);
3125 #else
3126 if (endian == DEVICE_BIG_ENDIAN) {
3127 val = bswap64(val);
3129 #endif
3130 } else {
3131 /* RAM case */
3132 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
3133 & TARGET_PAGE_MASK)
3134 + addr1);
3135 switch (endian) {
3136 case DEVICE_LITTLE_ENDIAN:
3137 val = ldq_le_p(ptr);
3138 break;
3139 case DEVICE_BIG_ENDIAN:
3140 val = ldq_be_p(ptr);
3141 break;
3142 default:
3143 val = ldq_p(ptr);
3144 break;
3146 r = MEMTX_OK;
3148 if (result) {
3149 *result = r;
3151 if (release_lock) {
3152 qemu_mutex_unlock_iothread();
3154 rcu_read_unlock();
3155 return val;
3158 uint64_t address_space_ldq(AddressSpace *as, hwaddr addr,
3159 MemTxAttrs attrs, MemTxResult *result)
3161 return address_space_ldq_internal(as, addr, attrs, result,
3162 DEVICE_NATIVE_ENDIAN);
3165 uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr,
3166 MemTxAttrs attrs, MemTxResult *result)
3168 return address_space_ldq_internal(as, addr, attrs, result,
3169 DEVICE_LITTLE_ENDIAN);
3172 uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr,
3173 MemTxAttrs attrs, MemTxResult *result)
3175 return address_space_ldq_internal(as, addr, attrs, result,
3176 DEVICE_BIG_ENDIAN);
3179 uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
3181 return address_space_ldq(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3184 uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
3186 return address_space_ldq_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3189 uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
3191 return address_space_ldq_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3194 /* XXX: optimize */
3195 uint32_t address_space_ldub(AddressSpace *as, hwaddr addr,
3196 MemTxAttrs attrs, MemTxResult *result)
3198 uint8_t val;
3199 MemTxResult r;
3201 r = address_space_rw(as, addr, attrs, &val, 1, 0);
3202 if (result) {
3203 *result = r;
3205 return val;
3208 uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
3210 return address_space_ldub(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3213 /* warning: addr must be aligned */
3214 static inline uint32_t address_space_lduw_internal(AddressSpace *as,
3215 hwaddr addr,
3216 MemTxAttrs attrs,
3217 MemTxResult *result,
3218 enum device_endian endian)
3220 uint8_t *ptr;
3221 uint64_t val;
3222 MemoryRegion *mr;
3223 hwaddr l = 2;
3224 hwaddr addr1;
3225 MemTxResult r;
3226 bool release_lock = false;
3228 rcu_read_lock();
3229 mr = address_space_translate(as, addr, &addr1, &l,
3230 false);
3231 if (l < 2 || !memory_access_is_direct(mr, false)) {
3232 release_lock |= prepare_mmio_access(mr);
3234 /* I/O case */
3235 r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
3236 #if defined(TARGET_WORDS_BIGENDIAN)
3237 if (endian == DEVICE_LITTLE_ENDIAN) {
3238 val = bswap16(val);
3240 #else
3241 if (endian == DEVICE_BIG_ENDIAN) {
3242 val = bswap16(val);
3244 #endif
3245 } else {
3246 /* RAM case */
3247 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
3248 & TARGET_PAGE_MASK)
3249 + addr1);
3250 switch (endian) {
3251 case DEVICE_LITTLE_ENDIAN:
3252 val = lduw_le_p(ptr);
3253 break;
3254 case DEVICE_BIG_ENDIAN:
3255 val = lduw_be_p(ptr);
3256 break;
3257 default:
3258 val = lduw_p(ptr);
3259 break;
3261 r = MEMTX_OK;
3263 if (result) {
3264 *result = r;
3266 if (release_lock) {
3267 qemu_mutex_unlock_iothread();
3269 rcu_read_unlock();
3270 return val;
3273 uint32_t address_space_lduw(AddressSpace *as, hwaddr addr,
3274 MemTxAttrs attrs, MemTxResult *result)
3276 return address_space_lduw_internal(as, addr, attrs, result,
3277 DEVICE_NATIVE_ENDIAN);
3280 uint32_t address_space_lduw_le(AddressSpace *as, hwaddr addr,
3281 MemTxAttrs attrs, MemTxResult *result)
3283 return address_space_lduw_internal(as, addr, attrs, result,
3284 DEVICE_LITTLE_ENDIAN);
3287 uint32_t address_space_lduw_be(AddressSpace *as, hwaddr addr,
3288 MemTxAttrs attrs, MemTxResult *result)
3290 return address_space_lduw_internal(as, addr, attrs, result,
3291 DEVICE_BIG_ENDIAN);
3294 uint32_t lduw_phys(AddressSpace *as, hwaddr addr)
3296 return address_space_lduw(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3299 uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr)
3301 return address_space_lduw_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3304 uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr)
3306 return address_space_lduw_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3309 /* warning: addr must be aligned. The ram page is not masked as dirty
3310 and the code inside is not invalidated. It is useful if the dirty
3311 bits are used to track modified PTEs */
3312 void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
3313 MemTxAttrs attrs, MemTxResult *result)
3315 uint8_t *ptr;
3316 MemoryRegion *mr;
3317 hwaddr l = 4;
3318 hwaddr addr1;
3319 MemTxResult r;
3320 uint8_t dirty_log_mask;
3321 bool release_lock = false;
3323 rcu_read_lock();
3324 mr = address_space_translate(as, addr, &addr1, &l,
3325 true);
3326 if (l < 4 || !memory_access_is_direct(mr, true)) {
3327 release_lock |= prepare_mmio_access(mr);
3329 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3330 } else {
3331 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3332 ptr = qemu_get_ram_ptr(addr1);
3333 stl_p(ptr, val);
3335 dirty_log_mask = memory_region_get_dirty_log_mask(mr);
3336 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
3337 cpu_physical_memory_set_dirty_range(addr1, 4, dirty_log_mask);
3338 r = MEMTX_OK;
3340 if (result) {
3341 *result = r;
3343 if (release_lock) {
3344 qemu_mutex_unlock_iothread();
3346 rcu_read_unlock();
3349 void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
3351 address_space_stl_notdirty(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3354 /* warning: addr must be aligned */
3355 static inline void address_space_stl_internal(AddressSpace *as,
3356 hwaddr addr, uint32_t val,
3357 MemTxAttrs attrs,
3358 MemTxResult *result,
3359 enum device_endian endian)
3361 uint8_t *ptr;
3362 MemoryRegion *mr;
3363 hwaddr l = 4;
3364 hwaddr addr1;
3365 MemTxResult r;
3366 bool release_lock = false;
3368 rcu_read_lock();
3369 mr = address_space_translate(as, addr, &addr1, &l,
3370 true);
3371 if (l < 4 || !memory_access_is_direct(mr, true)) {
3372 release_lock |= prepare_mmio_access(mr);
3374 #if defined(TARGET_WORDS_BIGENDIAN)
3375 if (endian == DEVICE_LITTLE_ENDIAN) {
3376 val = bswap32(val);
3378 #else
3379 if (endian == DEVICE_BIG_ENDIAN) {
3380 val = bswap32(val);
3382 #endif
3383 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3384 } else {
3385 /* RAM case */
3386 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3387 ptr = qemu_get_ram_ptr(addr1);
3388 switch (endian) {
3389 case DEVICE_LITTLE_ENDIAN:
3390 stl_le_p(ptr, val);
3391 break;
3392 case DEVICE_BIG_ENDIAN:
3393 stl_be_p(ptr, val);
3394 break;
3395 default:
3396 stl_p(ptr, val);
3397 break;
3399 invalidate_and_set_dirty(mr, addr1, 4);
3400 r = MEMTX_OK;
3402 if (result) {
3403 *result = r;
3405 if (release_lock) {
3406 qemu_mutex_unlock_iothread();
3408 rcu_read_unlock();
3411 void address_space_stl(AddressSpace *as, hwaddr addr, uint32_t val,
3412 MemTxAttrs attrs, MemTxResult *result)
3414 address_space_stl_internal(as, addr, val, attrs, result,
3415 DEVICE_NATIVE_ENDIAN);
3418 void address_space_stl_le(AddressSpace *as, hwaddr addr, uint32_t val,
3419 MemTxAttrs attrs, MemTxResult *result)
3421 address_space_stl_internal(as, addr, val, attrs, result,
3422 DEVICE_LITTLE_ENDIAN);
3425 void address_space_stl_be(AddressSpace *as, hwaddr addr, uint32_t val,
3426 MemTxAttrs attrs, MemTxResult *result)
3428 address_space_stl_internal(as, addr, val, attrs, result,
3429 DEVICE_BIG_ENDIAN);
3432 void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3434 address_space_stl(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3437 void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3439 address_space_stl_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3442 void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3444 address_space_stl_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3447 /* XXX: optimize */
3448 void address_space_stb(AddressSpace *as, hwaddr addr, uint32_t val,
3449 MemTxAttrs attrs, MemTxResult *result)
3451 uint8_t v = val;
3452 MemTxResult r;
3454 r = address_space_rw(as, addr, attrs, &v, 1, 1);
3455 if (result) {
3456 *result = r;
3460 void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3462 address_space_stb(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3465 /* warning: addr must be aligned */
3466 static inline void address_space_stw_internal(AddressSpace *as,
3467 hwaddr addr, uint32_t val,
3468 MemTxAttrs attrs,
3469 MemTxResult *result,
3470 enum device_endian endian)
3472 uint8_t *ptr;
3473 MemoryRegion *mr;
3474 hwaddr l = 2;
3475 hwaddr addr1;
3476 MemTxResult r;
3477 bool release_lock = false;
3479 rcu_read_lock();
3480 mr = address_space_translate(as, addr, &addr1, &l, true);
3481 if (l < 2 || !memory_access_is_direct(mr, true)) {
3482 release_lock |= prepare_mmio_access(mr);
3484 #if defined(TARGET_WORDS_BIGENDIAN)
3485 if (endian == DEVICE_LITTLE_ENDIAN) {
3486 val = bswap16(val);
3488 #else
3489 if (endian == DEVICE_BIG_ENDIAN) {
3490 val = bswap16(val);
3492 #endif
3493 r = memory_region_dispatch_write(mr, addr1, val, 2, attrs);
3494 } else {
3495 /* RAM case */
3496 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3497 ptr = qemu_get_ram_ptr(addr1);
3498 switch (endian) {
3499 case DEVICE_LITTLE_ENDIAN:
3500 stw_le_p(ptr, val);
3501 break;
3502 case DEVICE_BIG_ENDIAN:
3503 stw_be_p(ptr, val);
3504 break;
3505 default:
3506 stw_p(ptr, val);
3507 break;
3509 invalidate_and_set_dirty(mr, addr1, 2);
3510 r = MEMTX_OK;
3512 if (result) {
3513 *result = r;
3515 if (release_lock) {
3516 qemu_mutex_unlock_iothread();
3518 rcu_read_unlock();
3521 void address_space_stw(AddressSpace *as, hwaddr addr, uint32_t val,
3522 MemTxAttrs attrs, MemTxResult *result)
3524 address_space_stw_internal(as, addr, val, attrs, result,
3525 DEVICE_NATIVE_ENDIAN);
3528 void address_space_stw_le(AddressSpace *as, hwaddr addr, uint32_t val,
3529 MemTxAttrs attrs, MemTxResult *result)
3531 address_space_stw_internal(as, addr, val, attrs, result,
3532 DEVICE_LITTLE_ENDIAN);
3535 void address_space_stw_be(AddressSpace *as, hwaddr addr, uint32_t val,
3536 MemTxAttrs attrs, MemTxResult *result)
3538 address_space_stw_internal(as, addr, val, attrs, result,
3539 DEVICE_BIG_ENDIAN);
3542 void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3544 address_space_stw(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3547 void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3549 address_space_stw_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3552 void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3554 address_space_stw_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3557 /* XXX: optimize */
3558 void address_space_stq(AddressSpace *as, hwaddr addr, uint64_t val,
3559 MemTxAttrs attrs, MemTxResult *result)
3561 MemTxResult r;
3562 val = tswap64(val);
3563 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3564 if (result) {
3565 *result = r;
3569 void address_space_stq_le(AddressSpace *as, hwaddr addr, uint64_t val,
3570 MemTxAttrs attrs, MemTxResult *result)
3572 MemTxResult r;
3573 val = cpu_to_le64(val);
3574 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3575 if (result) {
3576 *result = r;
3579 void address_space_stq_be(AddressSpace *as, hwaddr addr, uint64_t val,
3580 MemTxAttrs attrs, MemTxResult *result)
3582 MemTxResult r;
3583 val = cpu_to_be64(val);
3584 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3585 if (result) {
3586 *result = r;
3590 void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3592 address_space_stq(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3595 void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3597 address_space_stq_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3600 void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3602 address_space_stq_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3605 /* virtual memory access for debug (includes writing to ROM) */
3606 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
3607 uint8_t *buf, int len, int is_write)
3609 int l;
3610 hwaddr phys_addr;
3611 target_ulong page;
3613 while (len > 0) {
3614 int asidx;
3615 MemTxAttrs attrs;
3617 page = addr & TARGET_PAGE_MASK;
3618 phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
3619 asidx = cpu_asidx_from_attrs(cpu, attrs);
3620 /* if no physical page mapped, return an error */
3621 if (phys_addr == -1)
3622 return -1;
3623 l = (page + TARGET_PAGE_SIZE) - addr;
3624 if (l > len)
3625 l = len;
3626 phys_addr += (addr & ~TARGET_PAGE_MASK);
3627 if (is_write) {
3628 cpu_physical_memory_write_rom(cpu->cpu_ases[asidx].as,
3629 phys_addr, buf, l);
3630 } else {
3631 address_space_rw(cpu->cpu_ases[asidx].as, phys_addr,
3632 MEMTXATTRS_UNSPECIFIED,
3633 buf, l, 0);
3635 len -= l;
3636 buf += l;
3637 addr += l;
3639 return 0;
3643 * Allows code that needs to deal with migration bitmaps etc to still be built
3644 * target independent.
3646 size_t qemu_target_page_bits(void)
3648 return TARGET_PAGE_BITS;
3651 #endif
3654 * A helper function for the _utterly broken_ virtio device model to find out if
3655 * it's running on a big endian machine. Don't do this at home kids!
3657 bool target_words_bigendian(void);
3658 bool target_words_bigendian(void)
3660 #if defined(TARGET_WORDS_BIGENDIAN)
3661 return true;
3662 #else
3663 return false;
3664 #endif
3667 #ifndef CONFIG_USER_ONLY
3668 bool cpu_physical_memory_is_io(hwaddr phys_addr)
3670 MemoryRegion*mr;
3671 hwaddr l = 1;
3672 bool res;
3674 rcu_read_lock();
3675 mr = address_space_translate(&address_space_memory,
3676 phys_addr, &phys_addr, &l, false);
3678 res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
3679 rcu_read_unlock();
3680 return res;
3683 int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
3685 RAMBlock *block;
3686 int ret = 0;
3688 rcu_read_lock();
3689 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
3690 ret = func(block->idstr, block->host, block->offset,
3691 block->used_length, opaque);
3692 if (ret) {
3693 break;
3696 rcu_read_unlock();
3697 return ret;
3699 #endif