qemu-sockets: simplify error handling
[qemu/ar7.git] / exec.c
blobca7f8df9093804da47e4cd6c6644d9fa1d3531ff
1 /*
2 * Virtual page mapping
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "qemu/osdep.h"
20 #ifndef _WIN32
21 #include <sys/mman.h>
22 #endif
24 #include "qemu-common.h"
25 #include "cpu.h"
26 #include "tcg.h"
27 #include "hw/hw.h"
28 #if !defined(CONFIG_USER_ONLY)
29 #include "hw/boards.h"
30 #endif
31 #include "hw/qdev.h"
32 #include "sysemu/kvm.h"
33 #include "sysemu/sysemu.h"
34 #include "hw/xen/xen.h"
35 #include "qemu/timer.h"
36 #include "qemu/config-file.h"
37 #include "qemu/error-report.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48 #include "qemu/rcu_queue.h"
49 #include "qemu/main-loop.h"
50 #include "translate-all.h"
51 #include "sysemu/replay.h"
53 #include "exec/memory-internal.h"
54 #include "exec/ram_addr.h"
55 #include "exec/log.h"
57 #include "qemu/range.h"
58 #ifndef _WIN32
59 #include "qemu/mmap-alloc.h"
60 #endif
62 //#define DEBUG_SUBPAGE
64 #if !defined(CONFIG_USER_ONLY)
65 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
66 * are protected by the ramlist lock.
68 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
70 static MemoryRegion *system_memory;
71 static MemoryRegion *system_io;
73 AddressSpace address_space_io;
74 AddressSpace address_space_memory;
76 MemoryRegion io_mem_rom, io_mem_notdirty;
77 static MemoryRegion io_mem_unassigned;
79 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
80 #define RAM_PREALLOC (1 << 0)
82 /* RAM is mmap-ed with MAP_SHARED */
83 #define RAM_SHARED (1 << 1)
85 /* Only a portion of RAM (used_length) is actually used, and migrated.
86 * This used_length size can change across reboots.
88 #define RAM_RESIZEABLE (1 << 2)
90 #endif
92 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
93 /* current CPU in the current thread. It is only valid inside
94 cpu_exec() */
95 __thread CPUState *current_cpu;
96 /* 0 = Do not count executed instructions.
97 1 = Precise instruction counting.
98 2 = Adaptive rate instruction counting. */
99 int use_icount;
101 #if !defined(CONFIG_USER_ONLY)
103 typedef struct PhysPageEntry PhysPageEntry;
105 struct PhysPageEntry {
106 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
107 uint32_t skip : 6;
108 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
109 uint32_t ptr : 26;
112 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
114 /* Size of the L2 (and L3, etc) page tables. */
115 #define ADDR_SPACE_BITS 64
117 #define P_L2_BITS 9
118 #define P_L2_SIZE (1 << P_L2_BITS)
120 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
122 typedef PhysPageEntry Node[P_L2_SIZE];
124 typedef struct PhysPageMap {
125 struct rcu_head rcu;
127 unsigned sections_nb;
128 unsigned sections_nb_alloc;
129 unsigned nodes_nb;
130 unsigned nodes_nb_alloc;
131 Node *nodes;
132 MemoryRegionSection *sections;
133 } PhysPageMap;
135 struct AddressSpaceDispatch {
136 struct rcu_head rcu;
138 /* This is a multi-level map on the physical address space.
139 * The bottom level has pointers to MemoryRegionSections.
141 PhysPageEntry phys_map;
142 PhysPageMap map;
143 AddressSpace *as;
146 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
147 typedef struct subpage_t {
148 MemoryRegion iomem;
149 AddressSpace *as;
150 hwaddr base;
151 uint16_t sub_section[TARGET_PAGE_SIZE];
152 } subpage_t;
154 #define PHYS_SECTION_UNASSIGNED 0
155 #define PHYS_SECTION_NOTDIRTY 1
156 #define PHYS_SECTION_ROM 2
157 #define PHYS_SECTION_WATCH 3
159 static void io_mem_init(void);
160 static void memory_map_init(void);
161 static void tcg_commit(MemoryListener *listener);
163 static MemoryRegion io_mem_watch;
166 * CPUAddressSpace: all the information a CPU needs about an AddressSpace
167 * @cpu: the CPU whose AddressSpace this is
168 * @as: the AddressSpace itself
169 * @memory_dispatch: its dispatch pointer (cached, RCU protected)
170 * @tcg_as_listener: listener for tracking changes to the AddressSpace
172 struct CPUAddressSpace {
173 CPUState *cpu;
174 AddressSpace *as;
175 struct AddressSpaceDispatch *memory_dispatch;
176 MemoryListener tcg_as_listener;
179 #endif
181 #if !defined(CONFIG_USER_ONLY)
183 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
185 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
186 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
187 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
188 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
192 static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
194 unsigned i;
195 uint32_t ret;
196 PhysPageEntry e;
197 PhysPageEntry *p;
199 ret = map->nodes_nb++;
200 p = map->nodes[ret];
201 assert(ret != PHYS_MAP_NODE_NIL);
202 assert(ret != map->nodes_nb_alloc);
204 e.skip = leaf ? 0 : 1;
205 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
206 for (i = 0; i < P_L2_SIZE; ++i) {
207 memcpy(&p[i], &e, sizeof(e));
209 return ret;
212 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
213 hwaddr *index, hwaddr *nb, uint16_t leaf,
214 int level)
216 PhysPageEntry *p;
217 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
219 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
220 lp->ptr = phys_map_node_alloc(map, level == 0);
222 p = map->nodes[lp->ptr];
223 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
225 while (*nb && lp < &p[P_L2_SIZE]) {
226 if ((*index & (step - 1)) == 0 && *nb >= step) {
227 lp->skip = 0;
228 lp->ptr = leaf;
229 *index += step;
230 *nb -= step;
231 } else {
232 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
234 ++lp;
238 static void phys_page_set(AddressSpaceDispatch *d,
239 hwaddr index, hwaddr nb,
240 uint16_t leaf)
242 /* Wildly overreserve - it doesn't matter much. */
243 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
245 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
248 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
249 * and update our entry so we can skip it and go directly to the destination.
251 static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)
253 unsigned valid_ptr = P_L2_SIZE;
254 int valid = 0;
255 PhysPageEntry *p;
256 int i;
258 if (lp->ptr == PHYS_MAP_NODE_NIL) {
259 return;
262 p = nodes[lp->ptr];
263 for (i = 0; i < P_L2_SIZE; i++) {
264 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
265 continue;
268 valid_ptr = i;
269 valid++;
270 if (p[i].skip) {
271 phys_page_compact(&p[i], nodes, compacted);
275 /* We can only compress if there's only one child. */
276 if (valid != 1) {
277 return;
280 assert(valid_ptr < P_L2_SIZE);
282 /* Don't compress if it won't fit in the # of bits we have. */
283 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
284 return;
287 lp->ptr = p[valid_ptr].ptr;
288 if (!p[valid_ptr].skip) {
289 /* If our only child is a leaf, make this a leaf. */
290 /* By design, we should have made this node a leaf to begin with so we
291 * should never reach here.
292 * But since it's so simple to handle this, let's do it just in case we
293 * change this rule.
295 lp->skip = 0;
296 } else {
297 lp->skip += p[valid_ptr].skip;
301 static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
303 DECLARE_BITMAP(compacted, nodes_nb);
305 if (d->phys_map.skip) {
306 phys_page_compact(&d->phys_map, d->map.nodes, compacted);
310 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
311 Node *nodes, MemoryRegionSection *sections)
313 PhysPageEntry *p;
314 hwaddr index = addr >> TARGET_PAGE_BITS;
315 int i;
317 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
318 if (lp.ptr == PHYS_MAP_NODE_NIL) {
319 return &sections[PHYS_SECTION_UNASSIGNED];
321 p = nodes[lp.ptr];
322 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
325 if (sections[lp.ptr].size.hi ||
326 range_covers_byte(sections[lp.ptr].offset_within_address_space,
327 sections[lp.ptr].size.lo, addr)) {
328 return &sections[lp.ptr];
329 } else {
330 return &sections[PHYS_SECTION_UNASSIGNED];
334 bool memory_region_is_unassigned(MemoryRegion *mr)
336 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
337 && mr != &io_mem_watch;
340 /* Called from RCU critical section */
341 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
342 hwaddr addr,
343 bool resolve_subpage)
345 MemoryRegionSection *section;
346 subpage_t *subpage;
348 section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);
349 if (resolve_subpage && section->mr->subpage) {
350 subpage = container_of(section->mr, subpage_t, iomem);
351 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
353 return section;
356 /* Called from RCU critical section */
357 static MemoryRegionSection *
358 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
359 hwaddr *plen, bool resolve_subpage)
361 MemoryRegionSection *section;
362 MemoryRegion *mr;
363 Int128 diff;
365 section = address_space_lookup_region(d, addr, resolve_subpage);
366 /* Compute offset within MemoryRegionSection */
367 addr -= section->offset_within_address_space;
369 /* Compute offset within MemoryRegion */
370 *xlat = addr + section->offset_within_region;
372 mr = section->mr;
374 /* MMIO registers can be expected to perform full-width accesses based only
375 * on their address, without considering adjacent registers that could
376 * decode to completely different MemoryRegions. When such registers
377 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
378 * regions overlap wildly. For this reason we cannot clamp the accesses
379 * here.
381 * If the length is small (as is the case for address_space_ldl/stl),
382 * everything works fine. If the incoming length is large, however,
383 * the caller really has to do the clamping through memory_access_size.
385 if (memory_region_is_ram(mr)) {
386 diff = int128_sub(section->size, int128_make64(addr));
387 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
389 return section;
392 /* Called from RCU critical section */
393 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
394 hwaddr *xlat, hwaddr *plen,
395 bool is_write)
397 IOMMUTLBEntry iotlb;
398 MemoryRegionSection *section;
399 MemoryRegion *mr;
401 for (;;) {
402 AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch);
403 section = address_space_translate_internal(d, addr, &addr, plen, true);
404 mr = section->mr;
406 if (!mr->iommu_ops) {
407 break;
410 iotlb = mr->iommu_ops->translate(mr, addr, is_write);
411 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
412 | (addr & iotlb.addr_mask));
413 *plen = MIN(*plen, (addr | iotlb.addr_mask) - addr + 1);
414 if (!(iotlb.perm & (1 << is_write))) {
415 mr = &io_mem_unassigned;
416 break;
419 as = iotlb.target_as;
422 if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
423 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
424 *plen = MIN(page, *plen);
427 *xlat = addr;
428 return mr;
431 /* Called from RCU critical section */
432 MemoryRegionSection *
433 address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
434 hwaddr *xlat, hwaddr *plen)
436 MemoryRegionSection *section;
437 AddressSpaceDispatch *d = cpu->cpu_ases[asidx].memory_dispatch;
439 section = address_space_translate_internal(d, addr, xlat, plen, false);
441 assert(!section->mr->iommu_ops);
442 return section;
444 #endif
446 #if !defined(CONFIG_USER_ONLY)
448 static int cpu_common_post_load(void *opaque, int version_id)
450 CPUState *cpu = opaque;
452 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
453 version_id is increased. */
454 cpu->interrupt_request &= ~0x01;
455 tlb_flush(cpu, 1);
457 return 0;
460 static int cpu_common_pre_load(void *opaque)
462 CPUState *cpu = opaque;
464 cpu->exception_index = -1;
466 return 0;
469 static bool cpu_common_exception_index_needed(void *opaque)
471 CPUState *cpu = opaque;
473 return tcg_enabled() && cpu->exception_index != -1;
476 static const VMStateDescription vmstate_cpu_common_exception_index = {
477 .name = "cpu_common/exception_index",
478 .version_id = 1,
479 .minimum_version_id = 1,
480 .needed = cpu_common_exception_index_needed,
481 .fields = (VMStateField[]) {
482 VMSTATE_INT32(exception_index, CPUState),
483 VMSTATE_END_OF_LIST()
487 static bool cpu_common_crash_occurred_needed(void *opaque)
489 CPUState *cpu = opaque;
491 return cpu->crash_occurred;
494 static const VMStateDescription vmstate_cpu_common_crash_occurred = {
495 .name = "cpu_common/crash_occurred",
496 .version_id = 1,
497 .minimum_version_id = 1,
498 .needed = cpu_common_crash_occurred_needed,
499 .fields = (VMStateField[]) {
500 VMSTATE_BOOL(crash_occurred, CPUState),
501 VMSTATE_END_OF_LIST()
505 const VMStateDescription vmstate_cpu_common = {
506 .name = "cpu_common",
507 .version_id = 1,
508 .minimum_version_id = 1,
509 .pre_load = cpu_common_pre_load,
510 .post_load = cpu_common_post_load,
511 .fields = (VMStateField[]) {
512 VMSTATE_UINT32(halted, CPUState),
513 VMSTATE_UINT32(interrupt_request, CPUState),
514 VMSTATE_END_OF_LIST()
516 .subsections = (const VMStateDescription*[]) {
517 &vmstate_cpu_common_exception_index,
518 &vmstate_cpu_common_crash_occurred,
519 NULL
523 #endif
525 CPUState *qemu_get_cpu(int index)
527 CPUState *cpu;
529 CPU_FOREACH(cpu) {
530 if (cpu->cpu_index == index) {
531 return cpu;
535 return NULL;
538 #if !defined(CONFIG_USER_ONLY)
539 void cpu_address_space_init(CPUState *cpu, AddressSpace *as, int asidx)
541 CPUAddressSpace *newas;
543 /* Target code should have set num_ases before calling us */
544 assert(asidx < cpu->num_ases);
546 if (asidx == 0) {
547 /* address space 0 gets the convenience alias */
548 cpu->as = as;
551 /* KVM cannot currently support multiple address spaces. */
552 assert(asidx == 0 || !kvm_enabled());
554 if (!cpu->cpu_ases) {
555 cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
558 newas = &cpu->cpu_ases[asidx];
559 newas->cpu = cpu;
560 newas->as = as;
561 if (tcg_enabled()) {
562 newas->tcg_as_listener.commit = tcg_commit;
563 memory_listener_register(&newas->tcg_as_listener, as);
567 AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
569 /* Return the AddressSpace corresponding to the specified index */
570 return cpu->cpu_ases[asidx].as;
572 #endif
574 #ifndef CONFIG_USER_ONLY
575 static DECLARE_BITMAP(cpu_index_map, MAX_CPUMASK_BITS);
577 static int cpu_get_free_index(Error **errp)
579 int cpu = find_first_zero_bit(cpu_index_map, MAX_CPUMASK_BITS);
581 if (cpu >= MAX_CPUMASK_BITS) {
582 error_setg(errp, "Trying to use more CPUs than max of %d",
583 MAX_CPUMASK_BITS);
584 return -1;
587 bitmap_set(cpu_index_map, cpu, 1);
588 return cpu;
591 void cpu_exec_exit(CPUState *cpu)
593 if (cpu->cpu_index == -1) {
594 /* cpu_index was never allocated by this @cpu or was already freed. */
595 return;
598 bitmap_clear(cpu_index_map, cpu->cpu_index, 1);
599 cpu->cpu_index = -1;
601 #else
603 static int cpu_get_free_index(Error **errp)
605 CPUState *some_cpu;
606 int cpu_index = 0;
608 CPU_FOREACH(some_cpu) {
609 cpu_index++;
611 return cpu_index;
614 void cpu_exec_exit(CPUState *cpu)
617 #endif
619 void cpu_exec_init(CPUState *cpu, Error **errp)
621 CPUClass *cc = CPU_GET_CLASS(cpu);
622 int cpu_index;
623 Error *local_err = NULL;
625 cpu->as = NULL;
626 cpu->num_ases = 0;
628 #ifndef CONFIG_USER_ONLY
629 cpu->thread_id = qemu_get_thread_id();
631 /* This is a softmmu CPU object, so create a property for it
632 * so users can wire up its memory. (This can't go in qom/cpu.c
633 * because that file is compiled only once for both user-mode
634 * and system builds.) The default if no link is set up is to use
635 * the system address space.
637 object_property_add_link(OBJECT(cpu), "memory", TYPE_MEMORY_REGION,
638 (Object **)&cpu->memory,
639 qdev_prop_allow_set_link_before_realize,
640 OBJ_PROP_LINK_UNREF_ON_RELEASE,
641 &error_abort);
642 cpu->memory = system_memory;
643 object_ref(OBJECT(cpu->memory));
644 #endif
646 #if defined(CONFIG_USER_ONLY)
647 cpu_list_lock();
648 #endif
649 cpu_index = cpu->cpu_index = cpu_get_free_index(&local_err);
650 if (local_err) {
651 error_propagate(errp, local_err);
652 #if defined(CONFIG_USER_ONLY)
653 cpu_list_unlock();
654 #endif
655 return;
657 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
658 #if defined(CONFIG_USER_ONLY)
659 cpu_list_unlock();
660 #endif
661 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
662 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
664 if (cc->vmsd != NULL) {
665 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
669 #if defined(CONFIG_USER_ONLY)
670 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
672 tb_invalidate_phys_page_range(pc, pc + 1, 0);
674 #else
675 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
677 MemTxAttrs attrs;
678 hwaddr phys = cpu_get_phys_page_attrs_debug(cpu, pc, &attrs);
679 int asidx = cpu_asidx_from_attrs(cpu, attrs);
680 if (phys != -1) {
681 tb_invalidate_phys_addr(cpu->cpu_ases[asidx].as,
682 phys | (pc & ~TARGET_PAGE_MASK));
685 #endif
687 #if defined(CONFIG_USER_ONLY)
688 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
693 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
694 int flags)
696 return -ENOSYS;
699 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
703 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
704 int flags, CPUWatchpoint **watchpoint)
706 return -ENOSYS;
708 #else
709 /* Add a watchpoint. */
710 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
711 int flags, CPUWatchpoint **watchpoint)
713 CPUWatchpoint *wp;
715 /* forbid ranges which are empty or run off the end of the address space */
716 if (len == 0 || (addr + len - 1) < addr) {
717 error_report("tried to set invalid watchpoint at %"
718 VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
719 return -EINVAL;
721 wp = g_malloc(sizeof(*wp));
723 wp->vaddr = addr;
724 wp->len = len;
725 wp->flags = flags;
727 /* keep all GDB-injected watchpoints in front */
728 if (flags & BP_GDB) {
729 QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
730 } else {
731 QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
734 tlb_flush_page(cpu, addr);
736 if (watchpoint)
737 *watchpoint = wp;
738 return 0;
741 /* Remove a specific watchpoint. */
742 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
743 int flags)
745 CPUWatchpoint *wp;
747 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
748 if (addr == wp->vaddr && len == wp->len
749 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
750 cpu_watchpoint_remove_by_ref(cpu, wp);
751 return 0;
754 return -ENOENT;
757 /* Remove a specific watchpoint by reference. */
758 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
760 QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
762 tlb_flush_page(cpu, watchpoint->vaddr);
764 g_free(watchpoint);
767 /* Remove all matching watchpoints. */
768 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
770 CPUWatchpoint *wp, *next;
772 QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
773 if (wp->flags & mask) {
774 cpu_watchpoint_remove_by_ref(cpu, wp);
779 /* Return true if this watchpoint address matches the specified
780 * access (ie the address range covered by the watchpoint overlaps
781 * partially or completely with the address range covered by the
782 * access).
784 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
785 vaddr addr,
786 vaddr len)
788 /* We know the lengths are non-zero, but a little caution is
789 * required to avoid errors in the case where the range ends
790 * exactly at the top of the address space and so addr + len
791 * wraps round to zero.
793 vaddr wpend = wp->vaddr + wp->len - 1;
794 vaddr addrend = addr + len - 1;
796 return !(addr > wpend || wp->vaddr > addrend);
799 #endif
801 /* Add a breakpoint. */
802 int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
803 CPUBreakpoint **breakpoint)
805 CPUBreakpoint *bp;
807 bp = g_malloc(sizeof(*bp));
809 bp->pc = pc;
810 bp->flags = flags;
812 /* keep all GDB-injected breakpoints in front */
813 if (flags & BP_GDB) {
814 QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
815 } else {
816 QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
819 breakpoint_invalidate(cpu, pc);
821 if (breakpoint) {
822 *breakpoint = bp;
824 return 0;
827 /* Remove a specific breakpoint. */
828 int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
830 CPUBreakpoint *bp;
832 QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
833 if (bp->pc == pc && bp->flags == flags) {
834 cpu_breakpoint_remove_by_ref(cpu, bp);
835 return 0;
838 return -ENOENT;
841 /* Remove a specific breakpoint by reference. */
842 void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
844 QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
846 breakpoint_invalidate(cpu, breakpoint->pc);
848 g_free(breakpoint);
851 /* Remove all matching breakpoints. */
852 void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
854 CPUBreakpoint *bp, *next;
856 QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
857 if (bp->flags & mask) {
858 cpu_breakpoint_remove_by_ref(cpu, bp);
863 /* enable or disable single step mode. EXCP_DEBUG is returned by the
864 CPU loop after each instruction */
865 void cpu_single_step(CPUState *cpu, int enabled)
867 if (cpu->singlestep_enabled != enabled) {
868 cpu->singlestep_enabled = enabled;
869 if (kvm_enabled()) {
870 kvm_update_guest_debug(cpu, 0);
871 } else {
872 /* must flush all the translated code to avoid inconsistencies */
873 /* XXX: only flush what is necessary */
874 tb_flush(cpu);
879 void cpu_abort(CPUState *cpu, const char *fmt, ...)
881 va_list ap;
882 va_list ap2;
884 va_start(ap, fmt);
885 va_copy(ap2, ap);
886 fprintf(stderr, "qemu: fatal: ");
887 vfprintf(stderr, fmt, ap);
888 fprintf(stderr, "\n");
889 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
890 if (qemu_log_separate()) {
891 qemu_log("qemu: fatal: ");
892 qemu_log_vprintf(fmt, ap2);
893 qemu_log("\n");
894 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
895 qemu_log_flush();
896 qemu_log_close();
898 va_end(ap2);
899 va_end(ap);
900 replay_finish();
901 #if defined(CONFIG_USER_ONLY)
903 struct sigaction act;
904 sigfillset(&act.sa_mask);
905 act.sa_handler = SIG_DFL;
906 sigaction(SIGABRT, &act, NULL);
908 #endif
909 abort();
912 #if !defined(CONFIG_USER_ONLY)
913 /* Called from RCU critical section */
914 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
916 RAMBlock *block;
918 block = atomic_rcu_read(&ram_list.mru_block);
919 if (block && addr - block->offset < block->max_length) {
920 return block;
922 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
923 if (addr - block->offset < block->max_length) {
924 goto found;
928 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
929 abort();
931 found:
932 /* It is safe to write mru_block outside the iothread lock. This
933 * is what happens:
935 * mru_block = xxx
936 * rcu_read_unlock()
937 * xxx removed from list
938 * rcu_read_lock()
939 * read mru_block
940 * mru_block = NULL;
941 * call_rcu(reclaim_ramblock, xxx);
942 * rcu_read_unlock()
944 * atomic_rcu_set is not needed here. The block was already published
945 * when it was placed into the list. Here we're just making an extra
946 * copy of the pointer.
948 ram_list.mru_block = block;
949 return block;
952 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
954 CPUState *cpu;
955 ram_addr_t start1;
956 RAMBlock *block;
957 ram_addr_t end;
959 end = TARGET_PAGE_ALIGN(start + length);
960 start &= TARGET_PAGE_MASK;
962 rcu_read_lock();
963 block = qemu_get_ram_block(start);
964 assert(block == qemu_get_ram_block(end - 1));
965 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
966 CPU_FOREACH(cpu) {
967 tlb_reset_dirty(cpu, start1, length);
969 rcu_read_unlock();
972 /* Note: start and end must be within the same ram block. */
973 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
974 ram_addr_t length,
975 unsigned client)
977 DirtyMemoryBlocks *blocks;
978 unsigned long end, page;
979 bool dirty = false;
981 if (length == 0) {
982 return false;
985 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
986 page = start >> TARGET_PAGE_BITS;
988 rcu_read_lock();
990 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
992 while (page < end) {
993 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
994 unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
995 unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
997 dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
998 offset, num);
999 page += num;
1002 rcu_read_unlock();
1004 if (dirty && tcg_enabled()) {
1005 tlb_reset_dirty_range_all(start, length);
1008 return dirty;
1011 /* Called from RCU critical section */
1012 hwaddr memory_region_section_get_iotlb(CPUState *cpu,
1013 MemoryRegionSection *section,
1014 target_ulong vaddr,
1015 hwaddr paddr, hwaddr xlat,
1016 int prot,
1017 target_ulong *address)
1019 hwaddr iotlb;
1020 CPUWatchpoint *wp;
1022 if (memory_region_is_ram(section->mr)) {
1023 /* Normal RAM. */
1024 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
1025 + xlat;
1026 if (!section->readonly) {
1027 iotlb |= PHYS_SECTION_NOTDIRTY;
1028 } else {
1029 iotlb |= PHYS_SECTION_ROM;
1031 } else {
1032 AddressSpaceDispatch *d;
1034 d = atomic_rcu_read(&section->address_space->dispatch);
1035 iotlb = section - d->map.sections;
1036 iotlb += xlat;
1039 /* Make accesses to pages with watchpoints go via the
1040 watchpoint trap routines. */
1041 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
1042 if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
1043 /* Avoid trapping reads of pages with a write breakpoint. */
1044 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
1045 iotlb = PHYS_SECTION_WATCH + paddr;
1046 *address |= TLB_MMIO;
1047 break;
1052 return iotlb;
1054 #endif /* defined(CONFIG_USER_ONLY) */
1056 #if !defined(CONFIG_USER_ONLY)
1058 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1059 uint16_t section);
1060 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
1062 static void *(*phys_mem_alloc)(size_t size, uint64_t *align) =
1063 qemu_anon_ram_alloc;
1066 * Set a custom physical guest memory alloator.
1067 * Accelerators with unusual needs may need this. Hopefully, we can
1068 * get rid of it eventually.
1070 void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align))
1072 phys_mem_alloc = alloc;
1075 static uint16_t phys_section_add(PhysPageMap *map,
1076 MemoryRegionSection *section)
1078 /* The physical section number is ORed with a page-aligned
1079 * pointer to produce the iotlb entries. Thus it should
1080 * never overflow into the page-aligned value.
1082 assert(map->sections_nb < TARGET_PAGE_SIZE);
1084 if (map->sections_nb == map->sections_nb_alloc) {
1085 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
1086 map->sections = g_renew(MemoryRegionSection, map->sections,
1087 map->sections_nb_alloc);
1089 map->sections[map->sections_nb] = *section;
1090 memory_region_ref(section->mr);
1091 return map->sections_nb++;
1094 static void phys_section_destroy(MemoryRegion *mr)
1096 bool have_sub_page = mr->subpage;
1098 memory_region_unref(mr);
1100 if (have_sub_page) {
1101 subpage_t *subpage = container_of(mr, subpage_t, iomem);
1102 object_unref(OBJECT(&subpage->iomem));
1103 g_free(subpage);
1107 static void phys_sections_free(PhysPageMap *map)
1109 while (map->sections_nb > 0) {
1110 MemoryRegionSection *section = &map->sections[--map->sections_nb];
1111 phys_section_destroy(section->mr);
1113 g_free(map->sections);
1114 g_free(map->nodes);
1117 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
1119 subpage_t *subpage;
1120 hwaddr base = section->offset_within_address_space
1121 & TARGET_PAGE_MASK;
1122 MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
1123 d->map.nodes, d->map.sections);
1124 MemoryRegionSection subsection = {
1125 .offset_within_address_space = base,
1126 .size = int128_make64(TARGET_PAGE_SIZE),
1128 hwaddr start, end;
1130 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
1132 if (!(existing->mr->subpage)) {
1133 subpage = subpage_init(d->as, base);
1134 subsection.address_space = d->as;
1135 subsection.mr = &subpage->iomem;
1136 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
1137 phys_section_add(&d->map, &subsection));
1138 } else {
1139 subpage = container_of(existing->mr, subpage_t, iomem);
1141 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
1142 end = start + int128_get64(section->size) - 1;
1143 subpage_register(subpage, start, end,
1144 phys_section_add(&d->map, section));
1148 static void register_multipage(AddressSpaceDispatch *d,
1149 MemoryRegionSection *section)
1151 hwaddr start_addr = section->offset_within_address_space;
1152 uint16_t section_index = phys_section_add(&d->map, section);
1153 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1154 TARGET_PAGE_BITS));
1156 assert(num_pages);
1157 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
1160 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
1162 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1163 AddressSpaceDispatch *d = as->next_dispatch;
1164 MemoryRegionSection now = *section, remain = *section;
1165 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
1167 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
1168 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
1169 - now.offset_within_address_space;
1171 now.size = int128_min(int128_make64(left), now.size);
1172 register_subpage(d, &now);
1173 } else {
1174 now.size = int128_zero();
1176 while (int128_ne(remain.size, now.size)) {
1177 remain.size = int128_sub(remain.size, now.size);
1178 remain.offset_within_address_space += int128_get64(now.size);
1179 remain.offset_within_region += int128_get64(now.size);
1180 now = remain;
1181 if (int128_lt(remain.size, page_size)) {
1182 register_subpage(d, &now);
1183 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
1184 now.size = page_size;
1185 register_subpage(d, &now);
1186 } else {
1187 now.size = int128_and(now.size, int128_neg(page_size));
1188 register_multipage(d, &now);
1193 void qemu_flush_coalesced_mmio_buffer(void)
1195 if (kvm_enabled())
1196 kvm_flush_coalesced_mmio_buffer();
1199 void qemu_mutex_lock_ramlist(void)
1201 qemu_mutex_lock(&ram_list.mutex);
1204 void qemu_mutex_unlock_ramlist(void)
1206 qemu_mutex_unlock(&ram_list.mutex);
1209 #ifdef __linux__
1211 #include <sys/vfs.h>
1213 #define HUGETLBFS_MAGIC 0x958458f6
1215 static long gethugepagesize(const char *path, Error **errp)
1217 struct statfs fs;
1218 int ret;
1220 do {
1221 ret = statfs(path, &fs);
1222 } while (ret != 0 && errno == EINTR);
1224 if (ret != 0) {
1225 error_setg_errno(errp, errno, "failed to get page size of file %s",
1226 path);
1227 return 0;
1230 return fs.f_bsize;
1233 static void *file_ram_alloc(RAMBlock *block,
1234 ram_addr_t memory,
1235 const char *path,
1236 Error **errp)
1238 struct stat st;
1239 char *filename;
1240 char *sanitized_name;
1241 char *c;
1242 void *area;
1243 int fd;
1244 uint64_t hpagesize;
1245 Error *local_err = NULL;
1247 hpagesize = gethugepagesize(path, &local_err);
1248 if (local_err) {
1249 error_propagate(errp, local_err);
1250 goto error;
1252 block->mr->align = hpagesize;
1254 if (memory < hpagesize) {
1255 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
1256 "or larger than huge page size 0x%" PRIx64,
1257 memory, hpagesize);
1258 goto error;
1261 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1262 error_setg(errp,
1263 "host lacks kvm mmu notifiers, -mem-path unsupported");
1264 goto error;
1267 if (!stat(path, &st) && S_ISDIR(st.st_mode)) {
1268 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1269 sanitized_name = g_strdup(memory_region_name(block->mr));
1270 for (c = sanitized_name; *c != '\0'; c++) {
1271 if (*c == '/') {
1272 *c = '_';
1276 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1277 sanitized_name);
1278 g_free(sanitized_name);
1280 fd = mkstemp(filename);
1281 if (fd >= 0) {
1282 unlink(filename);
1284 g_free(filename);
1285 } else {
1286 fd = open(path, O_RDWR | O_CREAT, 0644);
1289 if (fd < 0) {
1290 error_setg_errno(errp, errno,
1291 "unable to create backing store for hugepages");
1292 goto error;
1295 memory = ROUND_UP(memory, hpagesize);
1298 * ftruncate is not supported by hugetlbfs in older
1299 * hosts, so don't bother bailing out on errors.
1300 * If anything goes wrong with it under other filesystems,
1301 * mmap will fail.
1303 if (ftruncate(fd, memory)) {
1304 perror("ftruncate");
1307 area = qemu_ram_mmap(fd, memory, hpagesize, block->flags & RAM_SHARED);
1308 if (area == MAP_FAILED) {
1309 error_setg_errno(errp, errno,
1310 "unable to map backing store for hugepages");
1311 close(fd);
1312 goto error;
1315 if (mem_prealloc) {
1316 os_mem_prealloc(fd, area, memory);
1319 block->fd = fd;
1320 return area;
1322 error:
1323 return NULL;
1325 #endif
1327 /* Called with the ramlist lock held. */
1328 static ram_addr_t find_ram_offset(ram_addr_t size)
1330 RAMBlock *block, *next_block;
1331 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1333 assert(size != 0); /* it would hand out same offset multiple times */
1335 if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
1336 return 0;
1339 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1340 ram_addr_t end, next = RAM_ADDR_MAX;
1342 end = block->offset + block->max_length;
1344 QLIST_FOREACH_RCU(next_block, &ram_list.blocks, next) {
1345 if (next_block->offset >= end) {
1346 next = MIN(next, next_block->offset);
1349 if (next - end >= size && next - end < mingap) {
1350 offset = end;
1351 mingap = next - end;
1355 if (offset == RAM_ADDR_MAX) {
1356 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1357 (uint64_t)size);
1358 abort();
1361 return offset;
1364 ram_addr_t last_ram_offset(void)
1366 RAMBlock *block;
1367 ram_addr_t last = 0;
1369 rcu_read_lock();
1370 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1371 last = MAX(last, block->offset + block->max_length);
1373 rcu_read_unlock();
1374 return last;
1377 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1379 int ret;
1381 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1382 if (!machine_dump_guest_core(current_machine)) {
1383 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1384 if (ret) {
1385 perror("qemu_madvise");
1386 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1387 "but dump_guest_core=off specified\n");
1392 /* Called within an RCU critical section, or while the ramlist lock
1393 * is held.
1395 static RAMBlock *find_ram_block(ram_addr_t addr)
1397 RAMBlock *block;
1399 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1400 if (block->offset == addr) {
1401 return block;
1405 return NULL;
1408 const char *qemu_ram_get_idstr(RAMBlock *rb)
1410 return rb->idstr;
1413 /* Called with iothread lock held. */
1414 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1416 RAMBlock *new_block, *block;
1418 rcu_read_lock();
1419 new_block = find_ram_block(addr);
1420 assert(new_block);
1421 assert(!new_block->idstr[0]);
1423 if (dev) {
1424 char *id = qdev_get_dev_path(dev);
1425 if (id) {
1426 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1427 g_free(id);
1430 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1432 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1433 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1434 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1435 new_block->idstr);
1436 abort();
1439 rcu_read_unlock();
1442 /* Called with iothread lock held. */
1443 void qemu_ram_unset_idstr(ram_addr_t addr)
1445 RAMBlock *block;
1447 /* FIXME: arch_init.c assumes that this is not called throughout
1448 * migration. Ignore the problem since hot-unplug during migration
1449 * does not work anyway.
1452 rcu_read_lock();
1453 block = find_ram_block(addr);
1454 if (block) {
1455 memset(block->idstr, 0, sizeof(block->idstr));
1457 rcu_read_unlock();
1460 static int memory_try_enable_merging(void *addr, size_t len)
1462 if (!machine_mem_merge(current_machine)) {
1463 /* disabled by the user */
1464 return 0;
1467 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1470 /* Only legal before guest might have detected the memory size: e.g. on
1471 * incoming migration, or right after reset.
1473 * As memory core doesn't know how is memory accessed, it is up to
1474 * resize callback to update device state and/or add assertions to detect
1475 * misuse, if necessary.
1477 int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp)
1479 RAMBlock *block = find_ram_block(base);
1481 assert(block);
1483 newsize = HOST_PAGE_ALIGN(newsize);
1485 if (block->used_length == newsize) {
1486 return 0;
1489 if (!(block->flags & RAM_RESIZEABLE)) {
1490 error_setg_errno(errp, EINVAL,
1491 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1492 " in != 0x" RAM_ADDR_FMT, block->idstr,
1493 newsize, block->used_length);
1494 return -EINVAL;
1497 if (block->max_length < newsize) {
1498 error_setg_errno(errp, EINVAL,
1499 "Length too large: %s: 0x" RAM_ADDR_FMT
1500 " > 0x" RAM_ADDR_FMT, block->idstr,
1501 newsize, block->max_length);
1502 return -EINVAL;
1505 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
1506 block->used_length = newsize;
1507 cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
1508 DIRTY_CLIENTS_ALL);
1509 memory_region_set_size(block->mr, newsize);
1510 if (block->resized) {
1511 block->resized(block->idstr, newsize, block->host);
1513 return 0;
1516 /* Called with ram_list.mutex held */
1517 static void dirty_memory_extend(ram_addr_t old_ram_size,
1518 ram_addr_t new_ram_size)
1520 ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size,
1521 DIRTY_MEMORY_BLOCK_SIZE);
1522 ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size,
1523 DIRTY_MEMORY_BLOCK_SIZE);
1524 int i;
1526 /* Only need to extend if block count increased */
1527 if (new_num_blocks <= old_num_blocks) {
1528 return;
1531 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1532 DirtyMemoryBlocks *old_blocks;
1533 DirtyMemoryBlocks *new_blocks;
1534 int j;
1536 old_blocks = atomic_rcu_read(&ram_list.dirty_memory[i]);
1537 new_blocks = g_malloc(sizeof(*new_blocks) +
1538 sizeof(new_blocks->blocks[0]) * new_num_blocks);
1540 if (old_num_blocks) {
1541 memcpy(new_blocks->blocks, old_blocks->blocks,
1542 old_num_blocks * sizeof(old_blocks->blocks[0]));
1545 for (j = old_num_blocks; j < new_num_blocks; j++) {
1546 new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
1549 atomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
1551 if (old_blocks) {
1552 g_free_rcu(old_blocks, rcu);
1557 static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
1559 RAMBlock *block;
1560 RAMBlock *last_block = NULL;
1561 ram_addr_t old_ram_size, new_ram_size;
1562 Error *err = NULL;
1564 old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1566 qemu_mutex_lock_ramlist();
1567 new_block->offset = find_ram_offset(new_block->max_length);
1569 if (!new_block->host) {
1570 if (xen_enabled()) {
1571 xen_ram_alloc(new_block->offset, new_block->max_length,
1572 new_block->mr, &err);
1573 if (err) {
1574 error_propagate(errp, err);
1575 qemu_mutex_unlock_ramlist();
1576 return -1;
1578 } else {
1579 new_block->host = phys_mem_alloc(new_block->max_length,
1580 &new_block->mr->align);
1581 if (!new_block->host) {
1582 error_setg_errno(errp, errno,
1583 "cannot set up guest memory '%s'",
1584 memory_region_name(new_block->mr));
1585 qemu_mutex_unlock_ramlist();
1586 return -1;
1588 memory_try_enable_merging(new_block->host, new_block->max_length);
1592 new_ram_size = MAX(old_ram_size,
1593 (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
1594 if (new_ram_size > old_ram_size) {
1595 migration_bitmap_extend(old_ram_size, new_ram_size);
1596 dirty_memory_extend(old_ram_size, new_ram_size);
1598 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1599 * QLIST (which has an RCU-friendly variant) does not have insertion at
1600 * tail, so save the last element in last_block.
1602 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1603 last_block = block;
1604 if (block->max_length < new_block->max_length) {
1605 break;
1608 if (block) {
1609 QLIST_INSERT_BEFORE_RCU(block, new_block, next);
1610 } else if (last_block) {
1611 QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
1612 } else { /* list is empty */
1613 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
1615 ram_list.mru_block = NULL;
1617 /* Write list before version */
1618 smp_wmb();
1619 ram_list.version++;
1620 qemu_mutex_unlock_ramlist();
1622 cpu_physical_memory_set_dirty_range(new_block->offset,
1623 new_block->used_length,
1624 DIRTY_CLIENTS_ALL);
1626 if (new_block->host) {
1627 qemu_ram_setup_dump(new_block->host, new_block->max_length);
1628 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
1629 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
1630 if (kvm_enabled()) {
1631 kvm_setup_guest_memory(new_block->host, new_block->max_length);
1635 return new_block->offset;
1638 #ifdef __linux__
1639 ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
1640 bool share, const char *mem_path,
1641 Error **errp)
1643 RAMBlock *new_block;
1644 ram_addr_t addr;
1645 Error *local_err = NULL;
1647 if (xen_enabled()) {
1648 error_setg(errp, "-mem-path not supported with Xen");
1649 return -1;
1652 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1654 * file_ram_alloc() needs to allocate just like
1655 * phys_mem_alloc, but we haven't bothered to provide
1656 * a hook there.
1658 error_setg(errp,
1659 "-mem-path not supported with this accelerator");
1660 return -1;
1663 size = HOST_PAGE_ALIGN(size);
1664 new_block = g_malloc0(sizeof(*new_block));
1665 new_block->mr = mr;
1666 new_block->used_length = size;
1667 new_block->max_length = size;
1668 new_block->flags = share ? RAM_SHARED : 0;
1669 new_block->host = file_ram_alloc(new_block, size,
1670 mem_path, errp);
1671 if (!new_block->host) {
1672 g_free(new_block);
1673 return -1;
1676 addr = ram_block_add(new_block, &local_err);
1677 if (local_err) {
1678 g_free(new_block);
1679 error_propagate(errp, local_err);
1680 return -1;
1682 return addr;
1684 #endif
1686 static
1687 ram_addr_t qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
1688 void (*resized)(const char*,
1689 uint64_t length,
1690 void *host),
1691 void *host, bool resizeable,
1692 MemoryRegion *mr, Error **errp)
1694 RAMBlock *new_block;
1695 ram_addr_t addr;
1696 Error *local_err = NULL;
1698 size = HOST_PAGE_ALIGN(size);
1699 max_size = HOST_PAGE_ALIGN(max_size);
1700 new_block = g_malloc0(sizeof(*new_block));
1701 new_block->mr = mr;
1702 new_block->resized = resized;
1703 new_block->used_length = size;
1704 new_block->max_length = max_size;
1705 assert(max_size >= size);
1706 new_block->fd = -1;
1707 new_block->host = host;
1708 if (host) {
1709 new_block->flags |= RAM_PREALLOC;
1711 if (resizeable) {
1712 new_block->flags |= RAM_RESIZEABLE;
1714 addr = ram_block_add(new_block, &local_err);
1715 if (local_err) {
1716 g_free(new_block);
1717 error_propagate(errp, local_err);
1718 return -1;
1720 return addr;
1723 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1724 MemoryRegion *mr, Error **errp)
1726 return qemu_ram_alloc_internal(size, size, NULL, host, false, mr, errp);
1729 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
1731 return qemu_ram_alloc_internal(size, size, NULL, NULL, false, mr, errp);
1734 ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
1735 void (*resized)(const char*,
1736 uint64_t length,
1737 void *host),
1738 MemoryRegion *mr, Error **errp)
1740 return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true, mr, errp);
1743 static void reclaim_ramblock(RAMBlock *block)
1745 if (block->flags & RAM_PREALLOC) {
1747 } else if (xen_enabled()) {
1748 xen_invalidate_map_cache_entry(block->host);
1749 #ifndef _WIN32
1750 } else if (block->fd >= 0) {
1751 qemu_ram_munmap(block->host, block->max_length);
1752 close(block->fd);
1753 #endif
1754 } else {
1755 qemu_anon_ram_free(block->host, block->max_length);
1757 g_free(block);
1760 void qemu_ram_free(ram_addr_t addr)
1762 RAMBlock *block;
1764 qemu_mutex_lock_ramlist();
1765 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1766 if (addr == block->offset) {
1767 QLIST_REMOVE_RCU(block, next);
1768 ram_list.mru_block = NULL;
1769 /* Write list before version */
1770 smp_wmb();
1771 ram_list.version++;
1772 call_rcu(block, reclaim_ramblock, rcu);
1773 break;
1776 qemu_mutex_unlock_ramlist();
1779 #ifndef _WIN32
1780 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1782 RAMBlock *block;
1783 ram_addr_t offset;
1784 int flags;
1785 void *area, *vaddr;
1787 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1788 offset = addr - block->offset;
1789 if (offset < block->max_length) {
1790 vaddr = ramblock_ptr(block, offset);
1791 if (block->flags & RAM_PREALLOC) {
1793 } else if (xen_enabled()) {
1794 abort();
1795 } else {
1796 flags = MAP_FIXED;
1797 if (block->fd >= 0) {
1798 flags |= (block->flags & RAM_SHARED ?
1799 MAP_SHARED : MAP_PRIVATE);
1800 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1801 flags, block->fd, offset);
1802 } else {
1804 * Remap needs to match alloc. Accelerators that
1805 * set phys_mem_alloc never remap. If they did,
1806 * we'd need a remap hook here.
1808 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1810 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1811 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1812 flags, -1, 0);
1814 if (area != vaddr) {
1815 fprintf(stderr, "Could not remap addr: "
1816 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1817 length, addr);
1818 exit(1);
1820 memory_try_enable_merging(vaddr, length);
1821 qemu_ram_setup_dump(vaddr, length);
1826 #endif /* !_WIN32 */
1828 int qemu_get_ram_fd(ram_addr_t addr)
1830 RAMBlock *block;
1831 int fd;
1833 rcu_read_lock();
1834 block = qemu_get_ram_block(addr);
1835 fd = block->fd;
1836 rcu_read_unlock();
1837 return fd;
1840 void qemu_set_ram_fd(ram_addr_t addr, int fd)
1842 RAMBlock *block;
1844 rcu_read_lock();
1845 block = qemu_get_ram_block(addr);
1846 block->fd = fd;
1847 rcu_read_unlock();
1850 void *qemu_get_ram_block_host_ptr(ram_addr_t addr)
1852 RAMBlock *block;
1853 void *ptr;
1855 rcu_read_lock();
1856 block = qemu_get_ram_block(addr);
1857 ptr = ramblock_ptr(block, 0);
1858 rcu_read_unlock();
1859 return ptr;
1862 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1863 * This should not be used for general purpose DMA. Use address_space_map
1864 * or address_space_rw instead. For local memory (e.g. video ram) that the
1865 * device owns, use memory_region_get_ram_ptr.
1867 * Called within RCU critical section.
1869 void *qemu_get_ram_ptr(ram_addr_t addr)
1871 RAMBlock *block = qemu_get_ram_block(addr);
1873 if (xen_enabled() && block->host == NULL) {
1874 /* We need to check if the requested address is in the RAM
1875 * because we don't want to map the entire memory in QEMU.
1876 * In that case just map until the end of the page.
1878 if (block->offset == 0) {
1879 return xen_map_cache(addr, 0, 0);
1882 block->host = xen_map_cache(block->offset, block->max_length, 1);
1884 return ramblock_ptr(block, addr - block->offset);
1887 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1888 * but takes a size argument.
1890 * Called within RCU critical section.
1892 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1894 RAMBlock *block;
1895 ram_addr_t offset_inside_block;
1896 if (*size == 0) {
1897 return NULL;
1900 block = qemu_get_ram_block(addr);
1901 offset_inside_block = addr - block->offset;
1902 *size = MIN(*size, block->max_length - offset_inside_block);
1904 if (xen_enabled() && block->host == NULL) {
1905 /* We need to check if the requested address is in the RAM
1906 * because we don't want to map the entire memory in QEMU.
1907 * In that case just map the requested area.
1909 if (block->offset == 0) {
1910 return xen_map_cache(addr, *size, 1);
1913 block->host = xen_map_cache(block->offset, block->max_length, 1);
1916 return ramblock_ptr(block, offset_inside_block);
1920 * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
1921 * in that RAMBlock.
1923 * ptr: Host pointer to look up
1924 * round_offset: If true round the result offset down to a page boundary
1925 * *ram_addr: set to result ram_addr
1926 * *offset: set to result offset within the RAMBlock
1928 * Returns: RAMBlock (or NULL if not found)
1930 * By the time this function returns, the returned pointer is not protected
1931 * by RCU anymore. If the caller is not within an RCU critical section and
1932 * does not hold the iothread lock, it must have other means of protecting the
1933 * pointer, such as a reference to the region that includes the incoming
1934 * ram_addr_t.
1936 RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
1937 ram_addr_t *ram_addr,
1938 ram_addr_t *offset)
1940 RAMBlock *block;
1941 uint8_t *host = ptr;
1943 if (xen_enabled()) {
1944 rcu_read_lock();
1945 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1946 block = qemu_get_ram_block(*ram_addr);
1947 if (block) {
1948 *offset = (host - block->host);
1950 rcu_read_unlock();
1951 return block;
1954 rcu_read_lock();
1955 block = atomic_rcu_read(&ram_list.mru_block);
1956 if (block && block->host && host - block->host < block->max_length) {
1957 goto found;
1960 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1961 /* This case append when the block is not mapped. */
1962 if (block->host == NULL) {
1963 continue;
1965 if (host - block->host < block->max_length) {
1966 goto found;
1970 rcu_read_unlock();
1971 return NULL;
1973 found:
1974 *offset = (host - block->host);
1975 if (round_offset) {
1976 *offset &= TARGET_PAGE_MASK;
1978 *ram_addr = block->offset + *offset;
1979 rcu_read_unlock();
1980 return block;
1984 * Finds the named RAMBlock
1986 * name: The name of RAMBlock to find
1988 * Returns: RAMBlock (or NULL if not found)
1990 RAMBlock *qemu_ram_block_by_name(const char *name)
1992 RAMBlock *block;
1994 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1995 if (!strcmp(name, block->idstr)) {
1996 return block;
2000 return NULL;
2003 /* Some of the softmmu routines need to translate from a host pointer
2004 (typically a TLB entry) back to a ram offset. */
2005 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
2007 RAMBlock *block;
2008 ram_addr_t offset; /* Not used */
2010 block = qemu_ram_block_from_host(ptr, false, ram_addr, &offset);
2012 if (!block) {
2013 return NULL;
2016 return block->mr;
2019 /* Called within RCU critical section. */
2020 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
2021 uint64_t val, unsigned size)
2023 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
2024 tb_invalidate_phys_page_fast(ram_addr, size);
2026 switch (size) {
2027 case 1:
2028 stb_p(qemu_get_ram_ptr(ram_addr), val);
2029 break;
2030 case 2:
2031 stw_p(qemu_get_ram_ptr(ram_addr), val);
2032 break;
2033 case 4:
2034 stl_p(qemu_get_ram_ptr(ram_addr), val);
2035 break;
2036 default:
2037 abort();
2039 /* Set both VGA and migration bits for simplicity and to remove
2040 * the notdirty callback faster.
2042 cpu_physical_memory_set_dirty_range(ram_addr, size,
2043 DIRTY_CLIENTS_NOCODE);
2044 /* we remove the notdirty callback only if the code has been
2045 flushed */
2046 if (!cpu_physical_memory_is_clean(ram_addr)) {
2047 tlb_set_dirty(current_cpu, current_cpu->mem_io_vaddr);
2051 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
2052 unsigned size, bool is_write)
2054 return is_write;
2057 static const MemoryRegionOps notdirty_mem_ops = {
2058 .write = notdirty_mem_write,
2059 .valid.accepts = notdirty_mem_accepts,
2060 .endianness = DEVICE_NATIVE_ENDIAN,
2063 /* Generate a debug exception if a watchpoint has been hit. */
2064 static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
2066 CPUState *cpu = current_cpu;
2067 CPUArchState *env = cpu->env_ptr;
2068 target_ulong pc, cs_base;
2069 target_ulong vaddr;
2070 CPUWatchpoint *wp;
2071 int cpu_flags;
2073 if (cpu->watchpoint_hit) {
2074 /* We re-entered the check after replacing the TB. Now raise
2075 * the debug interrupt so that is will trigger after the
2076 * current instruction. */
2077 cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
2078 return;
2080 vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2081 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
2082 if (cpu_watchpoint_address_matches(wp, vaddr, len)
2083 && (wp->flags & flags)) {
2084 if (flags == BP_MEM_READ) {
2085 wp->flags |= BP_WATCHPOINT_HIT_READ;
2086 } else {
2087 wp->flags |= BP_WATCHPOINT_HIT_WRITE;
2089 wp->hitaddr = vaddr;
2090 wp->hitattrs = attrs;
2091 if (!cpu->watchpoint_hit) {
2092 cpu->watchpoint_hit = wp;
2093 tb_check_watchpoint(cpu);
2094 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2095 cpu->exception_index = EXCP_DEBUG;
2096 cpu_loop_exit(cpu);
2097 } else {
2098 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2099 tb_gen_code(cpu, pc, cs_base, cpu_flags, 1);
2100 cpu_resume_from_signal(cpu, NULL);
2103 } else {
2104 wp->flags &= ~BP_WATCHPOINT_HIT;
2109 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2110 so these check for a hit then pass through to the normal out-of-line
2111 phys routines. */
2112 static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
2113 unsigned size, MemTxAttrs attrs)
2115 MemTxResult res;
2116 uint64_t data;
2117 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2118 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
2120 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
2121 switch (size) {
2122 case 1:
2123 data = address_space_ldub(as, addr, attrs, &res);
2124 break;
2125 case 2:
2126 data = address_space_lduw(as, addr, attrs, &res);
2127 break;
2128 case 4:
2129 data = address_space_ldl(as, addr, attrs, &res);
2130 break;
2131 default: abort();
2133 *pdata = data;
2134 return res;
2137 static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
2138 uint64_t val, unsigned size,
2139 MemTxAttrs attrs)
2141 MemTxResult res;
2142 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2143 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
2145 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
2146 switch (size) {
2147 case 1:
2148 address_space_stb(as, addr, val, attrs, &res);
2149 break;
2150 case 2:
2151 address_space_stw(as, addr, val, attrs, &res);
2152 break;
2153 case 4:
2154 address_space_stl(as, addr, val, attrs, &res);
2155 break;
2156 default: abort();
2158 return res;
2161 static const MemoryRegionOps watch_mem_ops = {
2162 .read_with_attrs = watch_mem_read,
2163 .write_with_attrs = watch_mem_write,
2164 .endianness = DEVICE_NATIVE_ENDIAN,
2167 static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
2168 unsigned len, MemTxAttrs attrs)
2170 subpage_t *subpage = opaque;
2171 uint8_t buf[8];
2172 MemTxResult res;
2174 #if defined(DEBUG_SUBPAGE)
2175 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
2176 subpage, len, addr);
2177 #endif
2178 res = address_space_read(subpage->as, addr + subpage->base,
2179 attrs, buf, len);
2180 if (res) {
2181 return res;
2183 switch (len) {
2184 case 1:
2185 *data = ldub_p(buf);
2186 return MEMTX_OK;
2187 case 2:
2188 *data = lduw_p(buf);
2189 return MEMTX_OK;
2190 case 4:
2191 *data = ldl_p(buf);
2192 return MEMTX_OK;
2193 case 8:
2194 *data = ldq_p(buf);
2195 return MEMTX_OK;
2196 default:
2197 abort();
2201 static MemTxResult subpage_write(void *opaque, hwaddr addr,
2202 uint64_t value, unsigned len, MemTxAttrs attrs)
2204 subpage_t *subpage = opaque;
2205 uint8_t buf[8];
2207 #if defined(DEBUG_SUBPAGE)
2208 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
2209 " value %"PRIx64"\n",
2210 __func__, subpage, len, addr, value);
2211 #endif
2212 switch (len) {
2213 case 1:
2214 stb_p(buf, value);
2215 break;
2216 case 2:
2217 stw_p(buf, value);
2218 break;
2219 case 4:
2220 stl_p(buf, value);
2221 break;
2222 case 8:
2223 stq_p(buf, value);
2224 break;
2225 default:
2226 abort();
2228 return address_space_write(subpage->as, addr + subpage->base,
2229 attrs, buf, len);
2232 static bool subpage_accepts(void *opaque, hwaddr addr,
2233 unsigned len, bool is_write)
2235 subpage_t *subpage = opaque;
2236 #if defined(DEBUG_SUBPAGE)
2237 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
2238 __func__, subpage, is_write ? 'w' : 'r', len, addr);
2239 #endif
2241 return address_space_access_valid(subpage->as, addr + subpage->base,
2242 len, is_write);
2245 static const MemoryRegionOps subpage_ops = {
2246 .read_with_attrs = subpage_read,
2247 .write_with_attrs = subpage_write,
2248 .impl.min_access_size = 1,
2249 .impl.max_access_size = 8,
2250 .valid.min_access_size = 1,
2251 .valid.max_access_size = 8,
2252 .valid.accepts = subpage_accepts,
2253 .endianness = DEVICE_NATIVE_ENDIAN,
2256 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2257 uint16_t section)
2259 int idx, eidx;
2261 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2262 return -1;
2263 idx = SUBPAGE_IDX(start);
2264 eidx = SUBPAGE_IDX(end);
2265 #if defined(DEBUG_SUBPAGE)
2266 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2267 __func__, mmio, start, end, idx, eidx, section);
2268 #endif
2269 for (; idx <= eidx; idx++) {
2270 mmio->sub_section[idx] = section;
2273 return 0;
2276 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
2278 subpage_t *mmio;
2280 mmio = g_malloc0(sizeof(subpage_t));
2282 mmio->as = as;
2283 mmio->base = base;
2284 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
2285 NULL, TARGET_PAGE_SIZE);
2286 mmio->iomem.subpage = true;
2287 #if defined(DEBUG_SUBPAGE)
2288 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
2289 mmio, base, TARGET_PAGE_SIZE);
2290 #endif
2291 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
2293 return mmio;
2296 static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
2297 MemoryRegion *mr)
2299 assert(as);
2300 MemoryRegionSection section = {
2301 .address_space = as,
2302 .mr = mr,
2303 .offset_within_address_space = 0,
2304 .offset_within_region = 0,
2305 .size = int128_2_64(),
2308 return phys_section_add(map, &section);
2311 MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index, MemTxAttrs attrs)
2313 int asidx = cpu_asidx_from_attrs(cpu, attrs);
2314 CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
2315 AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
2316 MemoryRegionSection *sections = d->map.sections;
2318 return sections[index & ~TARGET_PAGE_MASK].mr;
2321 static void io_mem_init(void)
2323 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, NULL, UINT64_MAX);
2324 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
2325 NULL, UINT64_MAX);
2326 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
2327 NULL, UINT64_MAX);
2328 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
2329 NULL, UINT64_MAX);
2332 static void mem_begin(MemoryListener *listener)
2334 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2335 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
2336 uint16_t n;
2338 n = dummy_section(&d->map, as, &io_mem_unassigned);
2339 assert(n == PHYS_SECTION_UNASSIGNED);
2340 n = dummy_section(&d->map, as, &io_mem_notdirty);
2341 assert(n == PHYS_SECTION_NOTDIRTY);
2342 n = dummy_section(&d->map, as, &io_mem_rom);
2343 assert(n == PHYS_SECTION_ROM);
2344 n = dummy_section(&d->map, as, &io_mem_watch);
2345 assert(n == PHYS_SECTION_WATCH);
2347 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
2348 d->as = as;
2349 as->next_dispatch = d;
2352 static void address_space_dispatch_free(AddressSpaceDispatch *d)
2354 phys_sections_free(&d->map);
2355 g_free(d);
2358 static void mem_commit(MemoryListener *listener)
2360 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2361 AddressSpaceDispatch *cur = as->dispatch;
2362 AddressSpaceDispatch *next = as->next_dispatch;
2364 phys_page_compact_all(next, next->map.nodes_nb);
2366 atomic_rcu_set(&as->dispatch, next);
2367 if (cur) {
2368 call_rcu(cur, address_space_dispatch_free, rcu);
2372 static void tcg_commit(MemoryListener *listener)
2374 CPUAddressSpace *cpuas;
2375 AddressSpaceDispatch *d;
2377 /* since each CPU stores ram addresses in its TLB cache, we must
2378 reset the modified entries */
2379 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
2380 cpu_reloading_memory_map();
2381 /* The CPU and TLB are protected by the iothread lock.
2382 * We reload the dispatch pointer now because cpu_reloading_memory_map()
2383 * may have split the RCU critical section.
2385 d = atomic_rcu_read(&cpuas->as->dispatch);
2386 cpuas->memory_dispatch = d;
2387 tlb_flush(cpuas->cpu, 1);
2390 void address_space_init_dispatch(AddressSpace *as)
2392 as->dispatch = NULL;
2393 as->dispatch_listener = (MemoryListener) {
2394 .begin = mem_begin,
2395 .commit = mem_commit,
2396 .region_add = mem_add,
2397 .region_nop = mem_add,
2398 .priority = 0,
2400 memory_listener_register(&as->dispatch_listener, as);
2403 void address_space_unregister(AddressSpace *as)
2405 memory_listener_unregister(&as->dispatch_listener);
2408 void address_space_destroy_dispatch(AddressSpace *as)
2410 AddressSpaceDispatch *d = as->dispatch;
2412 atomic_rcu_set(&as->dispatch, NULL);
2413 if (d) {
2414 call_rcu(d, address_space_dispatch_free, rcu);
2418 static void memory_map_init(void)
2420 system_memory = g_malloc(sizeof(*system_memory));
2422 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
2423 address_space_init(&address_space_memory, system_memory, "memory");
2425 system_io = g_malloc(sizeof(*system_io));
2426 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2427 65536);
2428 address_space_init(&address_space_io, system_io, "I/O");
2431 MemoryRegion *get_system_memory(void)
2433 return system_memory;
2436 MemoryRegion *get_system_io(void)
2438 return system_io;
2441 #endif /* !defined(CONFIG_USER_ONLY) */
2443 /* physical memory access (slow version, mainly for debug) */
2444 #if defined(CONFIG_USER_ONLY)
2445 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2446 uint8_t *buf, int len, int is_write)
2448 int l, flags;
2449 target_ulong page;
2450 void * p;
2452 while (len > 0) {
2453 page = addr & TARGET_PAGE_MASK;
2454 l = (page + TARGET_PAGE_SIZE) - addr;
2455 if (l > len)
2456 l = len;
2457 flags = page_get_flags(page);
2458 if (!(flags & PAGE_VALID))
2459 return -1;
2460 if (is_write) {
2461 if (!(flags & PAGE_WRITE))
2462 return -1;
2463 /* XXX: this code should not depend on lock_user */
2464 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2465 return -1;
2466 memcpy(p, buf, l);
2467 unlock_user(p, addr, l);
2468 } else {
2469 if (!(flags & PAGE_READ))
2470 return -1;
2471 /* XXX: this code should not depend on lock_user */
2472 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2473 return -1;
2474 memcpy(buf, p, l);
2475 unlock_user(p, addr, 0);
2477 len -= l;
2478 buf += l;
2479 addr += l;
2481 return 0;
2484 #else
2486 static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
2487 hwaddr length)
2489 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
2490 /* No early return if dirty_log_mask is or becomes 0, because
2491 * cpu_physical_memory_set_dirty_range will still call
2492 * xen_modified_memory.
2494 if (dirty_log_mask) {
2495 dirty_log_mask =
2496 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
2498 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
2499 tb_invalidate_phys_range(addr, addr + length);
2500 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
2502 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
2505 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
2507 unsigned access_size_max = mr->ops->valid.max_access_size;
2509 /* Regions are assumed to support 1-4 byte accesses unless
2510 otherwise specified. */
2511 if (access_size_max == 0) {
2512 access_size_max = 4;
2515 /* Bound the maximum access by the alignment of the address. */
2516 if (!mr->ops->impl.unaligned) {
2517 unsigned align_size_max = addr & -addr;
2518 if (align_size_max != 0 && align_size_max < access_size_max) {
2519 access_size_max = align_size_max;
2523 /* Don't attempt accesses larger than the maximum. */
2524 if (l > access_size_max) {
2525 l = access_size_max;
2527 l = pow2floor(l);
2529 return l;
2532 static bool prepare_mmio_access(MemoryRegion *mr)
2534 bool unlocked = !qemu_mutex_iothread_locked();
2535 bool release_lock = false;
2537 if (unlocked && mr->global_locking) {
2538 qemu_mutex_lock_iothread();
2539 unlocked = false;
2540 release_lock = true;
2542 if (mr->flush_coalesced_mmio) {
2543 if (unlocked) {
2544 qemu_mutex_lock_iothread();
2546 qemu_flush_coalesced_mmio_buffer();
2547 if (unlocked) {
2548 qemu_mutex_unlock_iothread();
2552 return release_lock;
2555 /* Called within RCU critical section. */
2556 static MemTxResult address_space_write_continue(AddressSpace *as, hwaddr addr,
2557 MemTxAttrs attrs,
2558 const uint8_t *buf,
2559 int len, hwaddr addr1,
2560 hwaddr l, MemoryRegion *mr)
2562 uint8_t *ptr;
2563 uint64_t val;
2564 MemTxResult result = MEMTX_OK;
2565 bool release_lock = false;
2567 for (;;) {
2568 if (!memory_access_is_direct(mr, true)) {
2569 release_lock |= prepare_mmio_access(mr);
2570 l = memory_access_size(mr, l, addr1);
2571 /* XXX: could force current_cpu to NULL to avoid
2572 potential bugs */
2573 switch (l) {
2574 case 8:
2575 /* 64 bit write access */
2576 val = ldq_p(buf);
2577 result |= memory_region_dispatch_write(mr, addr1, val, 8,
2578 attrs);
2579 break;
2580 case 4:
2581 /* 32 bit write access */
2582 val = ldl_p(buf);
2583 result |= memory_region_dispatch_write(mr, addr1, val, 4,
2584 attrs);
2585 break;
2586 case 2:
2587 /* 16 bit write access */
2588 val = lduw_p(buf);
2589 result |= memory_region_dispatch_write(mr, addr1, val, 2,
2590 attrs);
2591 break;
2592 case 1:
2593 /* 8 bit write access */
2594 val = ldub_p(buf);
2595 result |= memory_region_dispatch_write(mr, addr1, val, 1,
2596 attrs);
2597 break;
2598 default:
2599 abort();
2601 } else {
2602 addr1 += memory_region_get_ram_addr(mr);
2603 /* RAM case */
2604 ptr = qemu_get_ram_ptr(addr1);
2605 memcpy(ptr, buf, l);
2606 invalidate_and_set_dirty(mr, addr1, l);
2609 if (release_lock) {
2610 qemu_mutex_unlock_iothread();
2611 release_lock = false;
2614 len -= l;
2615 buf += l;
2616 addr += l;
2618 if (!len) {
2619 break;
2622 l = len;
2623 mr = address_space_translate(as, addr, &addr1, &l, true);
2626 return result;
2629 MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2630 const uint8_t *buf, int len)
2632 hwaddr l;
2633 hwaddr addr1;
2634 MemoryRegion *mr;
2635 MemTxResult result = MEMTX_OK;
2637 if (len > 0) {
2638 rcu_read_lock();
2639 l = len;
2640 mr = address_space_translate(as, addr, &addr1, &l, true);
2641 result = address_space_write_continue(as, addr, attrs, buf, len,
2642 addr1, l, mr);
2643 rcu_read_unlock();
2646 return result;
2649 /* Called within RCU critical section. */
2650 MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr,
2651 MemTxAttrs attrs, uint8_t *buf,
2652 int len, hwaddr addr1, hwaddr l,
2653 MemoryRegion *mr)
2655 uint8_t *ptr;
2656 uint64_t val;
2657 MemTxResult result = MEMTX_OK;
2658 bool release_lock = false;
2660 for (;;) {
2661 if (!memory_access_is_direct(mr, false)) {
2662 /* I/O case */
2663 release_lock |= prepare_mmio_access(mr);
2664 l = memory_access_size(mr, l, addr1);
2665 switch (l) {
2666 case 8:
2667 /* 64 bit read access */
2668 result |= memory_region_dispatch_read(mr, addr1, &val, 8,
2669 attrs);
2670 stq_p(buf, val);
2671 break;
2672 case 4:
2673 /* 32 bit read access */
2674 result |= memory_region_dispatch_read(mr, addr1, &val, 4,
2675 attrs);
2676 stl_p(buf, val);
2677 break;
2678 case 2:
2679 /* 16 bit read access */
2680 result |= memory_region_dispatch_read(mr, addr1, &val, 2,
2681 attrs);
2682 stw_p(buf, val);
2683 break;
2684 case 1:
2685 /* 8 bit read access */
2686 result |= memory_region_dispatch_read(mr, addr1, &val, 1,
2687 attrs);
2688 stb_p(buf, val);
2689 break;
2690 default:
2691 abort();
2693 } else {
2694 /* RAM case */
2695 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2696 memcpy(buf, ptr, l);
2699 if (release_lock) {
2700 qemu_mutex_unlock_iothread();
2701 release_lock = false;
2704 len -= l;
2705 buf += l;
2706 addr += l;
2708 if (!len) {
2709 break;
2712 l = len;
2713 mr = address_space_translate(as, addr, &addr1, &l, false);
2716 return result;
2719 MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
2720 MemTxAttrs attrs, uint8_t *buf, int len)
2722 hwaddr l;
2723 hwaddr addr1;
2724 MemoryRegion *mr;
2725 MemTxResult result = MEMTX_OK;
2727 if (len > 0) {
2728 rcu_read_lock();
2729 l = len;
2730 mr = address_space_translate(as, addr, &addr1, &l, false);
2731 result = address_space_read_continue(as, addr, attrs, buf, len,
2732 addr1, l, mr);
2733 rcu_read_unlock();
2736 return result;
2739 MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2740 uint8_t *buf, int len, bool is_write)
2742 if (is_write) {
2743 return address_space_write(as, addr, attrs, (uint8_t *)buf, len);
2744 } else {
2745 return address_space_read(as, addr, attrs, (uint8_t *)buf, len);
2749 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2750 int len, int is_write)
2752 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
2753 buf, len, is_write);
2756 enum write_rom_type {
2757 WRITE_DATA,
2758 FLUSH_CACHE,
2761 static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
2762 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
2764 hwaddr l;
2765 uint8_t *ptr;
2766 hwaddr addr1;
2767 MemoryRegion *mr;
2769 rcu_read_lock();
2770 while (len > 0) {
2771 l = len;
2772 mr = address_space_translate(as, addr, &addr1, &l, true);
2774 if (!(memory_region_is_ram(mr) ||
2775 memory_region_is_romd(mr))) {
2776 l = memory_access_size(mr, l, addr1);
2777 } else {
2778 addr1 += memory_region_get_ram_addr(mr);
2779 /* ROM/RAM case */
2780 ptr = qemu_get_ram_ptr(addr1);
2781 switch (type) {
2782 case WRITE_DATA:
2783 memcpy(ptr, buf, l);
2784 invalidate_and_set_dirty(mr, addr1, l);
2785 break;
2786 case FLUSH_CACHE:
2787 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
2788 break;
2791 len -= l;
2792 buf += l;
2793 addr += l;
2795 rcu_read_unlock();
2798 /* used for ROM loading : can write in RAM and ROM */
2799 void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
2800 const uint8_t *buf, int len)
2802 cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
2805 void cpu_flush_icache_range(hwaddr start, int len)
2808 * This function should do the same thing as an icache flush that was
2809 * triggered from within the guest. For TCG we are always cache coherent,
2810 * so there is no need to flush anything. For KVM / Xen we need to flush
2811 * the host's instruction cache at least.
2813 if (tcg_enabled()) {
2814 return;
2817 cpu_physical_memory_write_rom_internal(&address_space_memory,
2818 start, NULL, len, FLUSH_CACHE);
2821 typedef struct {
2822 MemoryRegion *mr;
2823 void *buffer;
2824 hwaddr addr;
2825 hwaddr len;
2826 bool in_use;
2827 } BounceBuffer;
2829 static BounceBuffer bounce;
2831 typedef struct MapClient {
2832 QEMUBH *bh;
2833 QLIST_ENTRY(MapClient) link;
2834 } MapClient;
2836 QemuMutex map_client_list_lock;
2837 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2838 = QLIST_HEAD_INITIALIZER(map_client_list);
2840 static void cpu_unregister_map_client_do(MapClient *client)
2842 QLIST_REMOVE(client, link);
2843 g_free(client);
2846 static void cpu_notify_map_clients_locked(void)
2848 MapClient *client;
2850 while (!QLIST_EMPTY(&map_client_list)) {
2851 client = QLIST_FIRST(&map_client_list);
2852 qemu_bh_schedule(client->bh);
2853 cpu_unregister_map_client_do(client);
2857 void cpu_register_map_client(QEMUBH *bh)
2859 MapClient *client = g_malloc(sizeof(*client));
2861 qemu_mutex_lock(&map_client_list_lock);
2862 client->bh = bh;
2863 QLIST_INSERT_HEAD(&map_client_list, client, link);
2864 if (!atomic_read(&bounce.in_use)) {
2865 cpu_notify_map_clients_locked();
2867 qemu_mutex_unlock(&map_client_list_lock);
2870 void cpu_exec_init_all(void)
2872 qemu_mutex_init(&ram_list.mutex);
2873 io_mem_init();
2874 memory_map_init();
2875 qemu_mutex_init(&map_client_list_lock);
2878 void cpu_unregister_map_client(QEMUBH *bh)
2880 MapClient *client;
2882 qemu_mutex_lock(&map_client_list_lock);
2883 QLIST_FOREACH(client, &map_client_list, link) {
2884 if (client->bh == bh) {
2885 cpu_unregister_map_client_do(client);
2886 break;
2889 qemu_mutex_unlock(&map_client_list_lock);
2892 static void cpu_notify_map_clients(void)
2894 qemu_mutex_lock(&map_client_list_lock);
2895 cpu_notify_map_clients_locked();
2896 qemu_mutex_unlock(&map_client_list_lock);
2899 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2901 MemoryRegion *mr;
2902 hwaddr l, xlat;
2904 rcu_read_lock();
2905 while (len > 0) {
2906 l = len;
2907 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2908 if (!memory_access_is_direct(mr, is_write)) {
2909 l = memory_access_size(mr, l, addr);
2910 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2911 return false;
2915 len -= l;
2916 addr += l;
2918 rcu_read_unlock();
2919 return true;
2922 /* Map a physical memory region into a host virtual address.
2923 * May map a subset of the requested range, given by and returned in *plen.
2924 * May return NULL if resources needed to perform the mapping are exhausted.
2925 * Use only for reads OR writes - not for read-modify-write operations.
2926 * Use cpu_register_map_client() to know when retrying the map operation is
2927 * likely to succeed.
2929 void *address_space_map(AddressSpace *as,
2930 hwaddr addr,
2931 hwaddr *plen,
2932 bool is_write)
2934 hwaddr len = *plen;
2935 hwaddr done = 0;
2936 hwaddr l, xlat, base;
2937 MemoryRegion *mr, *this_mr;
2938 ram_addr_t raddr;
2939 void *ptr;
2941 if (len == 0) {
2942 return NULL;
2945 l = len;
2946 rcu_read_lock();
2947 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2949 if (!memory_access_is_direct(mr, is_write)) {
2950 if (atomic_xchg(&bounce.in_use, true)) {
2951 rcu_read_unlock();
2952 return NULL;
2954 /* Avoid unbounded allocations */
2955 l = MIN(l, TARGET_PAGE_SIZE);
2956 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
2957 bounce.addr = addr;
2958 bounce.len = l;
2960 memory_region_ref(mr);
2961 bounce.mr = mr;
2962 if (!is_write) {
2963 address_space_read(as, addr, MEMTXATTRS_UNSPECIFIED,
2964 bounce.buffer, l);
2967 rcu_read_unlock();
2968 *plen = l;
2969 return bounce.buffer;
2972 base = xlat;
2973 raddr = memory_region_get_ram_addr(mr);
2975 for (;;) {
2976 len -= l;
2977 addr += l;
2978 done += l;
2979 if (len == 0) {
2980 break;
2983 l = len;
2984 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2985 if (this_mr != mr || xlat != base + done) {
2986 break;
2990 memory_region_ref(mr);
2991 *plen = done;
2992 ptr = qemu_ram_ptr_length(raddr + base, plen);
2993 rcu_read_unlock();
2995 return ptr;
2998 /* Unmaps a memory region previously mapped by address_space_map().
2999 * Will also mark the memory as dirty if is_write == 1. access_len gives
3000 * the amount of memory that was actually read or written by the caller.
3002 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
3003 int is_write, hwaddr access_len)
3005 if (buffer != bounce.buffer) {
3006 MemoryRegion *mr;
3007 ram_addr_t addr1;
3009 mr = qemu_ram_addr_from_host(buffer, &addr1);
3010 assert(mr != NULL);
3011 if (is_write) {
3012 invalidate_and_set_dirty(mr, addr1, access_len);
3014 if (xen_enabled()) {
3015 xen_invalidate_map_cache_entry(buffer);
3017 memory_region_unref(mr);
3018 return;
3020 if (is_write) {
3021 address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
3022 bounce.buffer, access_len);
3024 qemu_vfree(bounce.buffer);
3025 bounce.buffer = NULL;
3026 memory_region_unref(bounce.mr);
3027 atomic_mb_set(&bounce.in_use, false);
3028 cpu_notify_map_clients();
3031 void *cpu_physical_memory_map(hwaddr addr,
3032 hwaddr *plen,
3033 int is_write)
3035 return address_space_map(&address_space_memory, addr, plen, is_write);
3038 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
3039 int is_write, hwaddr access_len)
3041 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
3044 /* warning: addr must be aligned */
3045 static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
3046 MemTxAttrs attrs,
3047 MemTxResult *result,
3048 enum device_endian endian)
3050 uint8_t *ptr;
3051 uint64_t val;
3052 MemoryRegion *mr;
3053 hwaddr l = 4;
3054 hwaddr addr1;
3055 MemTxResult r;
3056 bool release_lock = false;
3058 rcu_read_lock();
3059 mr = address_space_translate(as, addr, &addr1, &l, false);
3060 if (l < 4 || !memory_access_is_direct(mr, false)) {
3061 release_lock |= prepare_mmio_access(mr);
3063 /* I/O case */
3064 r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
3065 #if defined(TARGET_WORDS_BIGENDIAN)
3066 if (endian == DEVICE_LITTLE_ENDIAN) {
3067 val = bswap32(val);
3069 #else
3070 if (endian == DEVICE_BIG_ENDIAN) {
3071 val = bswap32(val);
3073 #endif
3074 } else {
3075 /* RAM case */
3076 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
3077 & TARGET_PAGE_MASK)
3078 + addr1);
3079 switch (endian) {
3080 case DEVICE_LITTLE_ENDIAN:
3081 val = ldl_le_p(ptr);
3082 break;
3083 case DEVICE_BIG_ENDIAN:
3084 val = ldl_be_p(ptr);
3085 break;
3086 default:
3087 val = ldl_p(ptr);
3088 break;
3090 r = MEMTX_OK;
3092 if (result) {
3093 *result = r;
3095 if (release_lock) {
3096 qemu_mutex_unlock_iothread();
3098 rcu_read_unlock();
3099 return val;
3102 uint32_t address_space_ldl(AddressSpace *as, hwaddr addr,
3103 MemTxAttrs attrs, MemTxResult *result)
3105 return address_space_ldl_internal(as, addr, attrs, result,
3106 DEVICE_NATIVE_ENDIAN);
3109 uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr,
3110 MemTxAttrs attrs, MemTxResult *result)
3112 return address_space_ldl_internal(as, addr, attrs, result,
3113 DEVICE_LITTLE_ENDIAN);
3116 uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr,
3117 MemTxAttrs attrs, MemTxResult *result)
3119 return address_space_ldl_internal(as, addr, attrs, result,
3120 DEVICE_BIG_ENDIAN);
3123 uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
3125 return address_space_ldl(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3128 uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
3130 return address_space_ldl_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3133 uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
3135 return address_space_ldl_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3138 /* warning: addr must be aligned */
3139 static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
3140 MemTxAttrs attrs,
3141 MemTxResult *result,
3142 enum device_endian endian)
3144 uint8_t *ptr;
3145 uint64_t val;
3146 MemoryRegion *mr;
3147 hwaddr l = 8;
3148 hwaddr addr1;
3149 MemTxResult r;
3150 bool release_lock = false;
3152 rcu_read_lock();
3153 mr = address_space_translate(as, addr, &addr1, &l,
3154 false);
3155 if (l < 8 || !memory_access_is_direct(mr, false)) {
3156 release_lock |= prepare_mmio_access(mr);
3158 /* I/O case */
3159 r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
3160 #if defined(TARGET_WORDS_BIGENDIAN)
3161 if (endian == DEVICE_LITTLE_ENDIAN) {
3162 val = bswap64(val);
3164 #else
3165 if (endian == DEVICE_BIG_ENDIAN) {
3166 val = bswap64(val);
3168 #endif
3169 } else {
3170 /* RAM case */
3171 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
3172 & TARGET_PAGE_MASK)
3173 + addr1);
3174 switch (endian) {
3175 case DEVICE_LITTLE_ENDIAN:
3176 val = ldq_le_p(ptr);
3177 break;
3178 case DEVICE_BIG_ENDIAN:
3179 val = ldq_be_p(ptr);
3180 break;
3181 default:
3182 val = ldq_p(ptr);
3183 break;
3185 r = MEMTX_OK;
3187 if (result) {
3188 *result = r;
3190 if (release_lock) {
3191 qemu_mutex_unlock_iothread();
3193 rcu_read_unlock();
3194 return val;
3197 uint64_t address_space_ldq(AddressSpace *as, hwaddr addr,
3198 MemTxAttrs attrs, MemTxResult *result)
3200 return address_space_ldq_internal(as, addr, attrs, result,
3201 DEVICE_NATIVE_ENDIAN);
3204 uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr,
3205 MemTxAttrs attrs, MemTxResult *result)
3207 return address_space_ldq_internal(as, addr, attrs, result,
3208 DEVICE_LITTLE_ENDIAN);
3211 uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr,
3212 MemTxAttrs attrs, MemTxResult *result)
3214 return address_space_ldq_internal(as, addr, attrs, result,
3215 DEVICE_BIG_ENDIAN);
3218 uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
3220 return address_space_ldq(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3223 uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
3225 return address_space_ldq_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3228 uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
3230 return address_space_ldq_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3233 /* XXX: optimize */
3234 uint32_t address_space_ldub(AddressSpace *as, hwaddr addr,
3235 MemTxAttrs attrs, MemTxResult *result)
3237 uint8_t val;
3238 MemTxResult r;
3240 r = address_space_rw(as, addr, attrs, &val, 1, 0);
3241 if (result) {
3242 *result = r;
3244 return val;
3247 uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
3249 return address_space_ldub(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3252 /* warning: addr must be aligned */
3253 static inline uint32_t address_space_lduw_internal(AddressSpace *as,
3254 hwaddr addr,
3255 MemTxAttrs attrs,
3256 MemTxResult *result,
3257 enum device_endian endian)
3259 uint8_t *ptr;
3260 uint64_t val;
3261 MemoryRegion *mr;
3262 hwaddr l = 2;
3263 hwaddr addr1;
3264 MemTxResult r;
3265 bool release_lock = false;
3267 rcu_read_lock();
3268 mr = address_space_translate(as, addr, &addr1, &l,
3269 false);
3270 if (l < 2 || !memory_access_is_direct(mr, false)) {
3271 release_lock |= prepare_mmio_access(mr);
3273 /* I/O case */
3274 r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
3275 #if defined(TARGET_WORDS_BIGENDIAN)
3276 if (endian == DEVICE_LITTLE_ENDIAN) {
3277 val = bswap16(val);
3279 #else
3280 if (endian == DEVICE_BIG_ENDIAN) {
3281 val = bswap16(val);
3283 #endif
3284 } else {
3285 /* RAM case */
3286 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
3287 & TARGET_PAGE_MASK)
3288 + addr1);
3289 switch (endian) {
3290 case DEVICE_LITTLE_ENDIAN:
3291 val = lduw_le_p(ptr);
3292 break;
3293 case DEVICE_BIG_ENDIAN:
3294 val = lduw_be_p(ptr);
3295 break;
3296 default:
3297 val = lduw_p(ptr);
3298 break;
3300 r = MEMTX_OK;
3302 if (result) {
3303 *result = r;
3305 if (release_lock) {
3306 qemu_mutex_unlock_iothread();
3308 rcu_read_unlock();
3309 return val;
3312 uint32_t address_space_lduw(AddressSpace *as, hwaddr addr,
3313 MemTxAttrs attrs, MemTxResult *result)
3315 return address_space_lduw_internal(as, addr, attrs, result,
3316 DEVICE_NATIVE_ENDIAN);
3319 uint32_t address_space_lduw_le(AddressSpace *as, hwaddr addr,
3320 MemTxAttrs attrs, MemTxResult *result)
3322 return address_space_lduw_internal(as, addr, attrs, result,
3323 DEVICE_LITTLE_ENDIAN);
3326 uint32_t address_space_lduw_be(AddressSpace *as, hwaddr addr,
3327 MemTxAttrs attrs, MemTxResult *result)
3329 return address_space_lduw_internal(as, addr, attrs, result,
3330 DEVICE_BIG_ENDIAN);
3333 uint32_t lduw_phys(AddressSpace *as, hwaddr addr)
3335 return address_space_lduw(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3338 uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr)
3340 return address_space_lduw_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3343 uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr)
3345 return address_space_lduw_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3348 /* warning: addr must be aligned. The ram page is not masked as dirty
3349 and the code inside is not invalidated. It is useful if the dirty
3350 bits are used to track modified PTEs */
3351 void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
3352 MemTxAttrs attrs, MemTxResult *result)
3354 uint8_t *ptr;
3355 MemoryRegion *mr;
3356 hwaddr l = 4;
3357 hwaddr addr1;
3358 MemTxResult r;
3359 uint8_t dirty_log_mask;
3360 bool release_lock = false;
3362 rcu_read_lock();
3363 mr = address_space_translate(as, addr, &addr1, &l,
3364 true);
3365 if (l < 4 || !memory_access_is_direct(mr, true)) {
3366 release_lock |= prepare_mmio_access(mr);
3368 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3369 } else {
3370 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3371 ptr = qemu_get_ram_ptr(addr1);
3372 stl_p(ptr, val);
3374 dirty_log_mask = memory_region_get_dirty_log_mask(mr);
3375 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
3376 cpu_physical_memory_set_dirty_range(addr1, 4, dirty_log_mask);
3377 r = MEMTX_OK;
3379 if (result) {
3380 *result = r;
3382 if (release_lock) {
3383 qemu_mutex_unlock_iothread();
3385 rcu_read_unlock();
3388 void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
3390 address_space_stl_notdirty(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3393 /* warning: addr must be aligned */
3394 static inline void address_space_stl_internal(AddressSpace *as,
3395 hwaddr addr, uint32_t val,
3396 MemTxAttrs attrs,
3397 MemTxResult *result,
3398 enum device_endian endian)
3400 uint8_t *ptr;
3401 MemoryRegion *mr;
3402 hwaddr l = 4;
3403 hwaddr addr1;
3404 MemTxResult r;
3405 bool release_lock = false;
3407 rcu_read_lock();
3408 mr = address_space_translate(as, addr, &addr1, &l,
3409 true);
3410 if (l < 4 || !memory_access_is_direct(mr, true)) {
3411 release_lock |= prepare_mmio_access(mr);
3413 #if defined(TARGET_WORDS_BIGENDIAN)
3414 if (endian == DEVICE_LITTLE_ENDIAN) {
3415 val = bswap32(val);
3417 #else
3418 if (endian == DEVICE_BIG_ENDIAN) {
3419 val = bswap32(val);
3421 #endif
3422 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3423 } else {
3424 /* RAM case */
3425 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3426 ptr = qemu_get_ram_ptr(addr1);
3427 switch (endian) {
3428 case DEVICE_LITTLE_ENDIAN:
3429 stl_le_p(ptr, val);
3430 break;
3431 case DEVICE_BIG_ENDIAN:
3432 stl_be_p(ptr, val);
3433 break;
3434 default:
3435 stl_p(ptr, val);
3436 break;
3438 invalidate_and_set_dirty(mr, addr1, 4);
3439 r = MEMTX_OK;
3441 if (result) {
3442 *result = r;
3444 if (release_lock) {
3445 qemu_mutex_unlock_iothread();
3447 rcu_read_unlock();
3450 void address_space_stl(AddressSpace *as, hwaddr addr, uint32_t val,
3451 MemTxAttrs attrs, MemTxResult *result)
3453 address_space_stl_internal(as, addr, val, attrs, result,
3454 DEVICE_NATIVE_ENDIAN);
3457 void address_space_stl_le(AddressSpace *as, hwaddr addr, uint32_t val,
3458 MemTxAttrs attrs, MemTxResult *result)
3460 address_space_stl_internal(as, addr, val, attrs, result,
3461 DEVICE_LITTLE_ENDIAN);
3464 void address_space_stl_be(AddressSpace *as, hwaddr addr, uint32_t val,
3465 MemTxAttrs attrs, MemTxResult *result)
3467 address_space_stl_internal(as, addr, val, attrs, result,
3468 DEVICE_BIG_ENDIAN);
3471 void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3473 address_space_stl(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3476 void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3478 address_space_stl_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3481 void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3483 address_space_stl_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3486 /* XXX: optimize */
3487 void address_space_stb(AddressSpace *as, hwaddr addr, uint32_t val,
3488 MemTxAttrs attrs, MemTxResult *result)
3490 uint8_t v = val;
3491 MemTxResult r;
3493 r = address_space_rw(as, addr, attrs, &v, 1, 1);
3494 if (result) {
3495 *result = r;
3499 void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3501 address_space_stb(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3504 /* warning: addr must be aligned */
3505 static inline void address_space_stw_internal(AddressSpace *as,
3506 hwaddr addr, uint32_t val,
3507 MemTxAttrs attrs,
3508 MemTxResult *result,
3509 enum device_endian endian)
3511 uint8_t *ptr;
3512 MemoryRegion *mr;
3513 hwaddr l = 2;
3514 hwaddr addr1;
3515 MemTxResult r;
3516 bool release_lock = false;
3518 rcu_read_lock();
3519 mr = address_space_translate(as, addr, &addr1, &l, true);
3520 if (l < 2 || !memory_access_is_direct(mr, true)) {
3521 release_lock |= prepare_mmio_access(mr);
3523 #if defined(TARGET_WORDS_BIGENDIAN)
3524 if (endian == DEVICE_LITTLE_ENDIAN) {
3525 val = bswap16(val);
3527 #else
3528 if (endian == DEVICE_BIG_ENDIAN) {
3529 val = bswap16(val);
3531 #endif
3532 r = memory_region_dispatch_write(mr, addr1, val, 2, attrs);
3533 } else {
3534 /* RAM case */
3535 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
3536 ptr = qemu_get_ram_ptr(addr1);
3537 switch (endian) {
3538 case DEVICE_LITTLE_ENDIAN:
3539 stw_le_p(ptr, val);
3540 break;
3541 case DEVICE_BIG_ENDIAN:
3542 stw_be_p(ptr, val);
3543 break;
3544 default:
3545 stw_p(ptr, val);
3546 break;
3548 invalidate_and_set_dirty(mr, addr1, 2);
3549 r = MEMTX_OK;
3551 if (result) {
3552 *result = r;
3554 if (release_lock) {
3555 qemu_mutex_unlock_iothread();
3557 rcu_read_unlock();
3560 void address_space_stw(AddressSpace *as, hwaddr addr, uint32_t val,
3561 MemTxAttrs attrs, MemTxResult *result)
3563 address_space_stw_internal(as, addr, val, attrs, result,
3564 DEVICE_NATIVE_ENDIAN);
3567 void address_space_stw_le(AddressSpace *as, hwaddr addr, uint32_t val,
3568 MemTxAttrs attrs, MemTxResult *result)
3570 address_space_stw_internal(as, addr, val, attrs, result,
3571 DEVICE_LITTLE_ENDIAN);
3574 void address_space_stw_be(AddressSpace *as, hwaddr addr, uint32_t val,
3575 MemTxAttrs attrs, MemTxResult *result)
3577 address_space_stw_internal(as, addr, val, attrs, result,
3578 DEVICE_BIG_ENDIAN);
3581 void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3583 address_space_stw(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3586 void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3588 address_space_stw_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3591 void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3593 address_space_stw_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3596 /* XXX: optimize */
3597 void address_space_stq(AddressSpace *as, hwaddr addr, uint64_t val,
3598 MemTxAttrs attrs, MemTxResult *result)
3600 MemTxResult r;
3601 val = tswap64(val);
3602 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3603 if (result) {
3604 *result = r;
3608 void address_space_stq_le(AddressSpace *as, hwaddr addr, uint64_t val,
3609 MemTxAttrs attrs, MemTxResult *result)
3611 MemTxResult r;
3612 val = cpu_to_le64(val);
3613 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3614 if (result) {
3615 *result = r;
3618 void address_space_stq_be(AddressSpace *as, hwaddr addr, uint64_t val,
3619 MemTxAttrs attrs, MemTxResult *result)
3621 MemTxResult r;
3622 val = cpu_to_be64(val);
3623 r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
3624 if (result) {
3625 *result = r;
3629 void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3631 address_space_stq(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3634 void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3636 address_space_stq_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3639 void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val)
3641 address_space_stq_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3644 /* virtual memory access for debug (includes writing to ROM) */
3645 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
3646 uint8_t *buf, int len, int is_write)
3648 int l;
3649 hwaddr phys_addr;
3650 target_ulong page;
3652 while (len > 0) {
3653 int asidx;
3654 MemTxAttrs attrs;
3656 page = addr & TARGET_PAGE_MASK;
3657 phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
3658 asidx = cpu_asidx_from_attrs(cpu, attrs);
3659 /* if no physical page mapped, return an error */
3660 if (phys_addr == -1)
3661 return -1;
3662 l = (page + TARGET_PAGE_SIZE) - addr;
3663 if (l > len)
3664 l = len;
3665 phys_addr += (addr & ~TARGET_PAGE_MASK);
3666 if (is_write) {
3667 cpu_physical_memory_write_rom(cpu->cpu_ases[asidx].as,
3668 phys_addr, buf, l);
3669 } else {
3670 address_space_rw(cpu->cpu_ases[asidx].as, phys_addr,
3671 MEMTXATTRS_UNSPECIFIED,
3672 buf, l, 0);
3674 len -= l;
3675 buf += l;
3676 addr += l;
3678 return 0;
3682 * Allows code that needs to deal with migration bitmaps etc to still be built
3683 * target independent.
3685 size_t qemu_target_page_bits(void)
3687 return TARGET_PAGE_BITS;
3690 #endif
3693 * A helper function for the _utterly broken_ virtio device model to find out if
3694 * it's running on a big endian machine. Don't do this at home kids!
3696 bool target_words_bigendian(void);
3697 bool target_words_bigendian(void)
3699 #if defined(TARGET_WORDS_BIGENDIAN)
3700 return true;
3701 #else
3702 return false;
3703 #endif
3706 #ifndef CONFIG_USER_ONLY
3707 bool cpu_physical_memory_is_io(hwaddr phys_addr)
3709 MemoryRegion*mr;
3710 hwaddr l = 1;
3711 bool res;
3713 rcu_read_lock();
3714 mr = address_space_translate(&address_space_memory,
3715 phys_addr, &phys_addr, &l, false);
3717 res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
3718 rcu_read_unlock();
3719 return res;
3722 int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
3724 RAMBlock *block;
3725 int ret = 0;
3727 rcu_read_lock();
3728 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
3729 ret = func(block->idstr, block->host, block->offset,
3730 block->used_length, opaque);
3731 if (ret) {
3732 break;
3735 rcu_read_unlock();
3736 return ret;
3738 #endif