4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "qemu/osdep.h"
24 #include "qemu-common.h"
28 #if !defined(CONFIG_USER_ONLY)
29 #include "hw/boards.h"
32 #include "sysemu/kvm.h"
33 #include "sysemu/sysemu.h"
34 #include "hw/xen/xen.h"
35 #include "qemu/timer.h"
36 #include "qemu/config-file.h"
37 #include "qemu/error-report.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
47 #include "exec/cpu-all.h"
48 #include "qemu/rcu_queue.h"
49 #include "qemu/main-loop.h"
50 #include "translate-all.h"
51 #include "sysemu/replay.h"
53 #include "exec/memory-internal.h"
54 #include "exec/ram_addr.h"
57 #include "qemu/range.h"
59 #include "qemu/mmap-alloc.h"
62 //#define DEBUG_SUBPAGE
64 #if !defined(CONFIG_USER_ONLY)
65 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
66 * are protected by the ramlist lock.
68 RAMList ram_list
= { .blocks
= QLIST_HEAD_INITIALIZER(ram_list
.blocks
) };
70 static MemoryRegion
*system_memory
;
71 static MemoryRegion
*system_io
;
73 AddressSpace address_space_io
;
74 AddressSpace address_space_memory
;
76 MemoryRegion io_mem_rom
, io_mem_notdirty
;
77 static MemoryRegion io_mem_unassigned
;
79 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
80 #define RAM_PREALLOC (1 << 0)
82 /* RAM is mmap-ed with MAP_SHARED */
83 #define RAM_SHARED (1 << 1)
85 /* Only a portion of RAM (used_length) is actually used, and migrated.
86 * This used_length size can change across reboots.
88 #define RAM_RESIZEABLE (1 << 2)
92 struct CPUTailQ cpus
= QTAILQ_HEAD_INITIALIZER(cpus
);
93 /* current CPU in the current thread. It is only valid inside
95 __thread CPUState
*current_cpu
;
96 /* 0 = Do not count executed instructions.
97 1 = Precise instruction counting.
98 2 = Adaptive rate instruction counting. */
101 #if !defined(CONFIG_USER_ONLY)
103 typedef struct PhysPageEntry PhysPageEntry
;
105 struct PhysPageEntry
{
106 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
108 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
112 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
114 /* Size of the L2 (and L3, etc) page tables. */
115 #define ADDR_SPACE_BITS 64
118 #define P_L2_SIZE (1 << P_L2_BITS)
120 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
122 typedef PhysPageEntry Node
[P_L2_SIZE
];
124 typedef struct PhysPageMap
{
127 unsigned sections_nb
;
128 unsigned sections_nb_alloc
;
130 unsigned nodes_nb_alloc
;
132 MemoryRegionSection
*sections
;
135 struct AddressSpaceDispatch
{
138 /* This is a multi-level map on the physical address space.
139 * The bottom level has pointers to MemoryRegionSections.
141 PhysPageEntry phys_map
;
146 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
147 typedef struct subpage_t
{
151 uint16_t sub_section
[TARGET_PAGE_SIZE
];
154 #define PHYS_SECTION_UNASSIGNED 0
155 #define PHYS_SECTION_NOTDIRTY 1
156 #define PHYS_SECTION_ROM 2
157 #define PHYS_SECTION_WATCH 3
159 static void io_mem_init(void);
160 static void memory_map_init(void);
161 static void tcg_commit(MemoryListener
*listener
);
163 static MemoryRegion io_mem_watch
;
166 * CPUAddressSpace: all the information a CPU needs about an AddressSpace
167 * @cpu: the CPU whose AddressSpace this is
168 * @as: the AddressSpace itself
169 * @memory_dispatch: its dispatch pointer (cached, RCU protected)
170 * @tcg_as_listener: listener for tracking changes to the AddressSpace
172 struct CPUAddressSpace
{
175 struct AddressSpaceDispatch
*memory_dispatch
;
176 MemoryListener tcg_as_listener
;
181 #if !defined(CONFIG_USER_ONLY)
183 static void phys_map_node_reserve(PhysPageMap
*map
, unsigned nodes
)
185 if (map
->nodes_nb
+ nodes
> map
->nodes_nb_alloc
) {
186 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
* 2, 16);
187 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
, map
->nodes_nb
+ nodes
);
188 map
->nodes
= g_renew(Node
, map
->nodes
, map
->nodes_nb_alloc
);
192 static uint32_t phys_map_node_alloc(PhysPageMap
*map
, bool leaf
)
199 ret
= map
->nodes_nb
++;
201 assert(ret
!= PHYS_MAP_NODE_NIL
);
202 assert(ret
!= map
->nodes_nb_alloc
);
204 e
.skip
= leaf
? 0 : 1;
205 e
.ptr
= leaf
? PHYS_SECTION_UNASSIGNED
: PHYS_MAP_NODE_NIL
;
206 for (i
= 0; i
< P_L2_SIZE
; ++i
) {
207 memcpy(&p
[i
], &e
, sizeof(e
));
212 static void phys_page_set_level(PhysPageMap
*map
, PhysPageEntry
*lp
,
213 hwaddr
*index
, hwaddr
*nb
, uint16_t leaf
,
217 hwaddr step
= (hwaddr
)1 << (level
* P_L2_BITS
);
219 if (lp
->skip
&& lp
->ptr
== PHYS_MAP_NODE_NIL
) {
220 lp
->ptr
= phys_map_node_alloc(map
, level
== 0);
222 p
= map
->nodes
[lp
->ptr
];
223 lp
= &p
[(*index
>> (level
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
225 while (*nb
&& lp
< &p
[P_L2_SIZE
]) {
226 if ((*index
& (step
- 1)) == 0 && *nb
>= step
) {
232 phys_page_set_level(map
, lp
, index
, nb
, leaf
, level
- 1);
238 static void phys_page_set(AddressSpaceDispatch
*d
,
239 hwaddr index
, hwaddr nb
,
242 /* Wildly overreserve - it doesn't matter much. */
243 phys_map_node_reserve(&d
->map
, 3 * P_L2_LEVELS
);
245 phys_page_set_level(&d
->map
, &d
->phys_map
, &index
, &nb
, leaf
, P_L2_LEVELS
- 1);
248 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
249 * and update our entry so we can skip it and go directly to the destination.
251 static void phys_page_compact(PhysPageEntry
*lp
, Node
*nodes
, unsigned long *compacted
)
253 unsigned valid_ptr
= P_L2_SIZE
;
258 if (lp
->ptr
== PHYS_MAP_NODE_NIL
) {
263 for (i
= 0; i
< P_L2_SIZE
; i
++) {
264 if (p
[i
].ptr
== PHYS_MAP_NODE_NIL
) {
271 phys_page_compact(&p
[i
], nodes
, compacted
);
275 /* We can only compress if there's only one child. */
280 assert(valid_ptr
< P_L2_SIZE
);
282 /* Don't compress if it won't fit in the # of bits we have. */
283 if (lp
->skip
+ p
[valid_ptr
].skip
>= (1 << 3)) {
287 lp
->ptr
= p
[valid_ptr
].ptr
;
288 if (!p
[valid_ptr
].skip
) {
289 /* If our only child is a leaf, make this a leaf. */
290 /* By design, we should have made this node a leaf to begin with so we
291 * should never reach here.
292 * But since it's so simple to handle this, let's do it just in case we
297 lp
->skip
+= p
[valid_ptr
].skip
;
301 static void phys_page_compact_all(AddressSpaceDispatch
*d
, int nodes_nb
)
303 DECLARE_BITMAP(compacted
, nodes_nb
);
305 if (d
->phys_map
.skip
) {
306 phys_page_compact(&d
->phys_map
, d
->map
.nodes
, compacted
);
310 static MemoryRegionSection
*phys_page_find(PhysPageEntry lp
, hwaddr addr
,
311 Node
*nodes
, MemoryRegionSection
*sections
)
314 hwaddr index
= addr
>> TARGET_PAGE_BITS
;
317 for (i
= P_L2_LEVELS
; lp
.skip
&& (i
-= lp
.skip
) >= 0;) {
318 if (lp
.ptr
== PHYS_MAP_NODE_NIL
) {
319 return §ions
[PHYS_SECTION_UNASSIGNED
];
322 lp
= p
[(index
>> (i
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
325 if (sections
[lp
.ptr
].size
.hi
||
326 range_covers_byte(sections
[lp
.ptr
].offset_within_address_space
,
327 sections
[lp
.ptr
].size
.lo
, addr
)) {
328 return §ions
[lp
.ptr
];
330 return §ions
[PHYS_SECTION_UNASSIGNED
];
334 bool memory_region_is_unassigned(MemoryRegion
*mr
)
336 return mr
!= &io_mem_rom
&& mr
!= &io_mem_notdirty
&& !mr
->rom_device
337 && mr
!= &io_mem_watch
;
340 /* Called from RCU critical section */
341 static MemoryRegionSection
*address_space_lookup_region(AddressSpaceDispatch
*d
,
343 bool resolve_subpage
)
345 MemoryRegionSection
*section
;
348 section
= phys_page_find(d
->phys_map
, addr
, d
->map
.nodes
, d
->map
.sections
);
349 if (resolve_subpage
&& section
->mr
->subpage
) {
350 subpage
= container_of(section
->mr
, subpage_t
, iomem
);
351 section
= &d
->map
.sections
[subpage
->sub_section
[SUBPAGE_IDX(addr
)]];
356 /* Called from RCU critical section */
357 static MemoryRegionSection
*
358 address_space_translate_internal(AddressSpaceDispatch
*d
, hwaddr addr
, hwaddr
*xlat
,
359 hwaddr
*plen
, bool resolve_subpage
)
361 MemoryRegionSection
*section
;
365 section
= address_space_lookup_region(d
, addr
, resolve_subpage
);
366 /* Compute offset within MemoryRegionSection */
367 addr
-= section
->offset_within_address_space
;
369 /* Compute offset within MemoryRegion */
370 *xlat
= addr
+ section
->offset_within_region
;
374 /* MMIO registers can be expected to perform full-width accesses based only
375 * on their address, without considering adjacent registers that could
376 * decode to completely different MemoryRegions. When such registers
377 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
378 * regions overlap wildly. For this reason we cannot clamp the accesses
381 * If the length is small (as is the case for address_space_ldl/stl),
382 * everything works fine. If the incoming length is large, however,
383 * the caller really has to do the clamping through memory_access_size.
385 if (memory_region_is_ram(mr
)) {
386 diff
= int128_sub(section
->size
, int128_make64(addr
));
387 *plen
= int128_get64(int128_min(diff
, int128_make64(*plen
)));
392 /* Called from RCU critical section */
393 MemoryRegion
*address_space_translate(AddressSpace
*as
, hwaddr addr
,
394 hwaddr
*xlat
, hwaddr
*plen
,
398 MemoryRegionSection
*section
;
402 AddressSpaceDispatch
*d
= atomic_rcu_read(&as
->dispatch
);
403 section
= address_space_translate_internal(d
, addr
, &addr
, plen
, true);
406 if (!mr
->iommu_ops
) {
410 iotlb
= mr
->iommu_ops
->translate(mr
, addr
, is_write
);
411 addr
= ((iotlb
.translated_addr
& ~iotlb
.addr_mask
)
412 | (addr
& iotlb
.addr_mask
));
413 *plen
= MIN(*plen
, (addr
| iotlb
.addr_mask
) - addr
+ 1);
414 if (!(iotlb
.perm
& (1 << is_write
))) {
415 mr
= &io_mem_unassigned
;
419 as
= iotlb
.target_as
;
422 if (xen_enabled() && memory_access_is_direct(mr
, is_write
)) {
423 hwaddr page
= ((addr
& TARGET_PAGE_MASK
) + TARGET_PAGE_SIZE
) - addr
;
424 *plen
= MIN(page
, *plen
);
431 /* Called from RCU critical section */
432 MemoryRegionSection
*
433 address_space_translate_for_iotlb(CPUState
*cpu
, int asidx
, hwaddr addr
,
434 hwaddr
*xlat
, hwaddr
*plen
)
436 MemoryRegionSection
*section
;
437 AddressSpaceDispatch
*d
= cpu
->cpu_ases
[asidx
].memory_dispatch
;
439 section
= address_space_translate_internal(d
, addr
, xlat
, plen
, false);
441 assert(!section
->mr
->iommu_ops
);
446 #if !defined(CONFIG_USER_ONLY)
448 static int cpu_common_post_load(void *opaque
, int version_id
)
450 CPUState
*cpu
= opaque
;
452 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
453 version_id is increased. */
454 cpu
->interrupt_request
&= ~0x01;
460 static int cpu_common_pre_load(void *opaque
)
462 CPUState
*cpu
= opaque
;
464 cpu
->exception_index
= -1;
469 static bool cpu_common_exception_index_needed(void *opaque
)
471 CPUState
*cpu
= opaque
;
473 return tcg_enabled() && cpu
->exception_index
!= -1;
476 static const VMStateDescription vmstate_cpu_common_exception_index
= {
477 .name
= "cpu_common/exception_index",
479 .minimum_version_id
= 1,
480 .needed
= cpu_common_exception_index_needed
,
481 .fields
= (VMStateField
[]) {
482 VMSTATE_INT32(exception_index
, CPUState
),
483 VMSTATE_END_OF_LIST()
487 static bool cpu_common_crash_occurred_needed(void *opaque
)
489 CPUState
*cpu
= opaque
;
491 return cpu
->crash_occurred
;
494 static const VMStateDescription vmstate_cpu_common_crash_occurred
= {
495 .name
= "cpu_common/crash_occurred",
497 .minimum_version_id
= 1,
498 .needed
= cpu_common_crash_occurred_needed
,
499 .fields
= (VMStateField
[]) {
500 VMSTATE_BOOL(crash_occurred
, CPUState
),
501 VMSTATE_END_OF_LIST()
505 const VMStateDescription vmstate_cpu_common
= {
506 .name
= "cpu_common",
508 .minimum_version_id
= 1,
509 .pre_load
= cpu_common_pre_load
,
510 .post_load
= cpu_common_post_load
,
511 .fields
= (VMStateField
[]) {
512 VMSTATE_UINT32(halted
, CPUState
),
513 VMSTATE_UINT32(interrupt_request
, CPUState
),
514 VMSTATE_END_OF_LIST()
516 .subsections
= (const VMStateDescription
*[]) {
517 &vmstate_cpu_common_exception_index
,
518 &vmstate_cpu_common_crash_occurred
,
525 CPUState
*qemu_get_cpu(int index
)
530 if (cpu
->cpu_index
== index
) {
538 #if !defined(CONFIG_USER_ONLY)
539 void cpu_address_space_init(CPUState
*cpu
, AddressSpace
*as
, int asidx
)
541 CPUAddressSpace
*newas
;
543 /* Target code should have set num_ases before calling us */
544 assert(asidx
< cpu
->num_ases
);
547 /* address space 0 gets the convenience alias */
551 /* KVM cannot currently support multiple address spaces. */
552 assert(asidx
== 0 || !kvm_enabled());
554 if (!cpu
->cpu_ases
) {
555 cpu
->cpu_ases
= g_new0(CPUAddressSpace
, cpu
->num_ases
);
558 newas
= &cpu
->cpu_ases
[asidx
];
562 newas
->tcg_as_listener
.commit
= tcg_commit
;
563 memory_listener_register(&newas
->tcg_as_listener
, as
);
567 AddressSpace
*cpu_get_address_space(CPUState
*cpu
, int asidx
)
569 /* Return the AddressSpace corresponding to the specified index */
570 return cpu
->cpu_ases
[asidx
].as
;
574 #ifndef CONFIG_USER_ONLY
575 static DECLARE_BITMAP(cpu_index_map
, MAX_CPUMASK_BITS
);
577 static int cpu_get_free_index(Error
**errp
)
579 int cpu
= find_first_zero_bit(cpu_index_map
, MAX_CPUMASK_BITS
);
581 if (cpu
>= MAX_CPUMASK_BITS
) {
582 error_setg(errp
, "Trying to use more CPUs than max of %d",
587 bitmap_set(cpu_index_map
, cpu
, 1);
591 void cpu_exec_exit(CPUState
*cpu
)
593 if (cpu
->cpu_index
== -1) {
594 /* cpu_index was never allocated by this @cpu or was already freed. */
598 bitmap_clear(cpu_index_map
, cpu
->cpu_index
, 1);
603 static int cpu_get_free_index(Error
**errp
)
608 CPU_FOREACH(some_cpu
) {
614 void cpu_exec_exit(CPUState
*cpu
)
619 void cpu_exec_init(CPUState
*cpu
, Error
**errp
)
621 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
623 Error
*local_err
= NULL
;
628 #ifndef CONFIG_USER_ONLY
629 cpu
->thread_id
= qemu_get_thread_id();
631 /* This is a softmmu CPU object, so create a property for it
632 * so users can wire up its memory. (This can't go in qom/cpu.c
633 * because that file is compiled only once for both user-mode
634 * and system builds.) The default if no link is set up is to use
635 * the system address space.
637 object_property_add_link(OBJECT(cpu
), "memory", TYPE_MEMORY_REGION
,
638 (Object
**)&cpu
->memory
,
639 qdev_prop_allow_set_link_before_realize
,
640 OBJ_PROP_LINK_UNREF_ON_RELEASE
,
642 cpu
->memory
= system_memory
;
643 object_ref(OBJECT(cpu
->memory
));
646 #if defined(CONFIG_USER_ONLY)
649 cpu_index
= cpu
->cpu_index
= cpu_get_free_index(&local_err
);
651 error_propagate(errp
, local_err
);
652 #if defined(CONFIG_USER_ONLY)
657 QTAILQ_INSERT_TAIL(&cpus
, cpu
, node
);
658 #if defined(CONFIG_USER_ONLY)
661 if (qdev_get_vmsd(DEVICE(cpu
)) == NULL
) {
662 vmstate_register(NULL
, cpu_index
, &vmstate_cpu_common
, cpu
);
664 if (cc
->vmsd
!= NULL
) {
665 vmstate_register(NULL
, cpu_index
, cc
->vmsd
, cpu
);
669 #if defined(CONFIG_USER_ONLY)
670 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
672 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
675 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
678 hwaddr phys
= cpu_get_phys_page_attrs_debug(cpu
, pc
, &attrs
);
679 int asidx
= cpu_asidx_from_attrs(cpu
, attrs
);
681 tb_invalidate_phys_addr(cpu
->cpu_ases
[asidx
].as
,
682 phys
| (pc
& ~TARGET_PAGE_MASK
));
687 #if defined(CONFIG_USER_ONLY)
688 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
693 int cpu_watchpoint_remove(CPUState
*cpu
, vaddr addr
, vaddr len
,
699 void cpu_watchpoint_remove_by_ref(CPUState
*cpu
, CPUWatchpoint
*watchpoint
)
703 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
704 int flags
, CPUWatchpoint
**watchpoint
)
709 /* Add a watchpoint. */
710 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
711 int flags
, CPUWatchpoint
**watchpoint
)
715 /* forbid ranges which are empty or run off the end of the address space */
716 if (len
== 0 || (addr
+ len
- 1) < addr
) {
717 error_report("tried to set invalid watchpoint at %"
718 VADDR_PRIx
", len=%" VADDR_PRIu
, addr
, len
);
721 wp
= g_malloc(sizeof(*wp
));
727 /* keep all GDB-injected watchpoints in front */
728 if (flags
& BP_GDB
) {
729 QTAILQ_INSERT_HEAD(&cpu
->watchpoints
, wp
, entry
);
731 QTAILQ_INSERT_TAIL(&cpu
->watchpoints
, wp
, entry
);
734 tlb_flush_page(cpu
, addr
);
741 /* Remove a specific watchpoint. */
742 int cpu_watchpoint_remove(CPUState
*cpu
, vaddr addr
, vaddr len
,
747 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
748 if (addr
== wp
->vaddr
&& len
== wp
->len
749 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
750 cpu_watchpoint_remove_by_ref(cpu
, wp
);
757 /* Remove a specific watchpoint by reference. */
758 void cpu_watchpoint_remove_by_ref(CPUState
*cpu
, CPUWatchpoint
*watchpoint
)
760 QTAILQ_REMOVE(&cpu
->watchpoints
, watchpoint
, entry
);
762 tlb_flush_page(cpu
, watchpoint
->vaddr
);
767 /* Remove all matching watchpoints. */
768 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
770 CPUWatchpoint
*wp
, *next
;
772 QTAILQ_FOREACH_SAFE(wp
, &cpu
->watchpoints
, entry
, next
) {
773 if (wp
->flags
& mask
) {
774 cpu_watchpoint_remove_by_ref(cpu
, wp
);
779 /* Return true if this watchpoint address matches the specified
780 * access (ie the address range covered by the watchpoint overlaps
781 * partially or completely with the address range covered by the
784 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint
*wp
,
788 /* We know the lengths are non-zero, but a little caution is
789 * required to avoid errors in the case where the range ends
790 * exactly at the top of the address space and so addr + len
791 * wraps round to zero.
793 vaddr wpend
= wp
->vaddr
+ wp
->len
- 1;
794 vaddr addrend
= addr
+ len
- 1;
796 return !(addr
> wpend
|| wp
->vaddr
> addrend
);
801 /* Add a breakpoint. */
802 int cpu_breakpoint_insert(CPUState
*cpu
, vaddr pc
, int flags
,
803 CPUBreakpoint
**breakpoint
)
807 bp
= g_malloc(sizeof(*bp
));
812 /* keep all GDB-injected breakpoints in front */
813 if (flags
& BP_GDB
) {
814 QTAILQ_INSERT_HEAD(&cpu
->breakpoints
, bp
, entry
);
816 QTAILQ_INSERT_TAIL(&cpu
->breakpoints
, bp
, entry
);
819 breakpoint_invalidate(cpu
, pc
);
827 /* Remove a specific breakpoint. */
828 int cpu_breakpoint_remove(CPUState
*cpu
, vaddr pc
, int flags
)
832 QTAILQ_FOREACH(bp
, &cpu
->breakpoints
, entry
) {
833 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
834 cpu_breakpoint_remove_by_ref(cpu
, bp
);
841 /* Remove a specific breakpoint by reference. */
842 void cpu_breakpoint_remove_by_ref(CPUState
*cpu
, CPUBreakpoint
*breakpoint
)
844 QTAILQ_REMOVE(&cpu
->breakpoints
, breakpoint
, entry
);
846 breakpoint_invalidate(cpu
, breakpoint
->pc
);
851 /* Remove all matching breakpoints. */
852 void cpu_breakpoint_remove_all(CPUState
*cpu
, int mask
)
854 CPUBreakpoint
*bp
, *next
;
856 QTAILQ_FOREACH_SAFE(bp
, &cpu
->breakpoints
, entry
, next
) {
857 if (bp
->flags
& mask
) {
858 cpu_breakpoint_remove_by_ref(cpu
, bp
);
863 /* enable or disable single step mode. EXCP_DEBUG is returned by the
864 CPU loop after each instruction */
865 void cpu_single_step(CPUState
*cpu
, int enabled
)
867 if (cpu
->singlestep_enabled
!= enabled
) {
868 cpu
->singlestep_enabled
= enabled
;
870 kvm_update_guest_debug(cpu
, 0);
872 /* must flush all the translated code to avoid inconsistencies */
873 /* XXX: only flush what is necessary */
879 void cpu_abort(CPUState
*cpu
, const char *fmt
, ...)
886 fprintf(stderr
, "qemu: fatal: ");
887 vfprintf(stderr
, fmt
, ap
);
888 fprintf(stderr
, "\n");
889 cpu_dump_state(cpu
, stderr
, fprintf
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
890 if (qemu_log_separate()) {
891 qemu_log("qemu: fatal: ");
892 qemu_log_vprintf(fmt
, ap2
);
894 log_cpu_state(cpu
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
901 #if defined(CONFIG_USER_ONLY)
903 struct sigaction act
;
904 sigfillset(&act
.sa_mask
);
905 act
.sa_handler
= SIG_DFL
;
906 sigaction(SIGABRT
, &act
, NULL
);
912 #if !defined(CONFIG_USER_ONLY)
913 /* Called from RCU critical section */
914 static RAMBlock
*qemu_get_ram_block(ram_addr_t addr
)
918 block
= atomic_rcu_read(&ram_list
.mru_block
);
919 if (block
&& addr
- block
->offset
< block
->max_length
) {
922 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
923 if (addr
- block
->offset
< block
->max_length
) {
928 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
932 /* It is safe to write mru_block outside the iothread lock. This
937 * xxx removed from list
941 * call_rcu(reclaim_ramblock, xxx);
944 * atomic_rcu_set is not needed here. The block was already published
945 * when it was placed into the list. Here we're just making an extra
946 * copy of the pointer.
948 ram_list
.mru_block
= block
;
952 static void tlb_reset_dirty_range_all(ram_addr_t start
, ram_addr_t length
)
959 end
= TARGET_PAGE_ALIGN(start
+ length
);
960 start
&= TARGET_PAGE_MASK
;
963 block
= qemu_get_ram_block(start
);
964 assert(block
== qemu_get_ram_block(end
- 1));
965 start1
= (uintptr_t)ramblock_ptr(block
, start
- block
->offset
);
967 tlb_reset_dirty(cpu
, start1
, length
);
972 /* Note: start and end must be within the same ram block. */
973 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start
,
977 DirtyMemoryBlocks
*blocks
;
978 unsigned long end
, page
;
985 end
= TARGET_PAGE_ALIGN(start
+ length
) >> TARGET_PAGE_BITS
;
986 page
= start
>> TARGET_PAGE_BITS
;
990 blocks
= atomic_rcu_read(&ram_list
.dirty_memory
[client
]);
993 unsigned long idx
= page
/ DIRTY_MEMORY_BLOCK_SIZE
;
994 unsigned long offset
= page
% DIRTY_MEMORY_BLOCK_SIZE
;
995 unsigned long num
= MIN(end
- page
, DIRTY_MEMORY_BLOCK_SIZE
- offset
);
997 dirty
|= bitmap_test_and_clear_atomic(blocks
->blocks
[idx
],
1004 if (dirty
&& tcg_enabled()) {
1005 tlb_reset_dirty_range_all(start
, length
);
1011 /* Called from RCU critical section */
1012 hwaddr
memory_region_section_get_iotlb(CPUState
*cpu
,
1013 MemoryRegionSection
*section
,
1015 hwaddr paddr
, hwaddr xlat
,
1017 target_ulong
*address
)
1022 if (memory_region_is_ram(section
->mr
)) {
1024 iotlb
= (memory_region_get_ram_addr(section
->mr
) & TARGET_PAGE_MASK
)
1026 if (!section
->readonly
) {
1027 iotlb
|= PHYS_SECTION_NOTDIRTY
;
1029 iotlb
|= PHYS_SECTION_ROM
;
1032 AddressSpaceDispatch
*d
;
1034 d
= atomic_rcu_read(§ion
->address_space
->dispatch
);
1035 iotlb
= section
- d
->map
.sections
;
1039 /* Make accesses to pages with watchpoints go via the
1040 watchpoint trap routines. */
1041 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
1042 if (cpu_watchpoint_address_matches(wp
, vaddr
, TARGET_PAGE_SIZE
)) {
1043 /* Avoid trapping reads of pages with a write breakpoint. */
1044 if ((prot
& PAGE_WRITE
) || (wp
->flags
& BP_MEM_READ
)) {
1045 iotlb
= PHYS_SECTION_WATCH
+ paddr
;
1046 *address
|= TLB_MMIO
;
1054 #endif /* defined(CONFIG_USER_ONLY) */
1056 #if !defined(CONFIG_USER_ONLY)
1058 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
1060 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
);
1062 static void *(*phys_mem_alloc
)(size_t size
, uint64_t *align
) =
1063 qemu_anon_ram_alloc
;
1066 * Set a custom physical guest memory alloator.
1067 * Accelerators with unusual needs may need this. Hopefully, we can
1068 * get rid of it eventually.
1070 void phys_mem_set_alloc(void *(*alloc
)(size_t, uint64_t *align
))
1072 phys_mem_alloc
= alloc
;
1075 static uint16_t phys_section_add(PhysPageMap
*map
,
1076 MemoryRegionSection
*section
)
1078 /* The physical section number is ORed with a page-aligned
1079 * pointer to produce the iotlb entries. Thus it should
1080 * never overflow into the page-aligned value.
1082 assert(map
->sections_nb
< TARGET_PAGE_SIZE
);
1084 if (map
->sections_nb
== map
->sections_nb_alloc
) {
1085 map
->sections_nb_alloc
= MAX(map
->sections_nb_alloc
* 2, 16);
1086 map
->sections
= g_renew(MemoryRegionSection
, map
->sections
,
1087 map
->sections_nb_alloc
);
1089 map
->sections
[map
->sections_nb
] = *section
;
1090 memory_region_ref(section
->mr
);
1091 return map
->sections_nb
++;
1094 static void phys_section_destroy(MemoryRegion
*mr
)
1096 bool have_sub_page
= mr
->subpage
;
1098 memory_region_unref(mr
);
1100 if (have_sub_page
) {
1101 subpage_t
*subpage
= container_of(mr
, subpage_t
, iomem
);
1102 object_unref(OBJECT(&subpage
->iomem
));
1107 static void phys_sections_free(PhysPageMap
*map
)
1109 while (map
->sections_nb
> 0) {
1110 MemoryRegionSection
*section
= &map
->sections
[--map
->sections_nb
];
1111 phys_section_destroy(section
->mr
);
1113 g_free(map
->sections
);
1117 static void register_subpage(AddressSpaceDispatch
*d
, MemoryRegionSection
*section
)
1120 hwaddr base
= section
->offset_within_address_space
1122 MemoryRegionSection
*existing
= phys_page_find(d
->phys_map
, base
,
1123 d
->map
.nodes
, d
->map
.sections
);
1124 MemoryRegionSection subsection
= {
1125 .offset_within_address_space
= base
,
1126 .size
= int128_make64(TARGET_PAGE_SIZE
),
1130 assert(existing
->mr
->subpage
|| existing
->mr
== &io_mem_unassigned
);
1132 if (!(existing
->mr
->subpage
)) {
1133 subpage
= subpage_init(d
->as
, base
);
1134 subsection
.address_space
= d
->as
;
1135 subsection
.mr
= &subpage
->iomem
;
1136 phys_page_set(d
, base
>> TARGET_PAGE_BITS
, 1,
1137 phys_section_add(&d
->map
, &subsection
));
1139 subpage
= container_of(existing
->mr
, subpage_t
, iomem
);
1141 start
= section
->offset_within_address_space
& ~TARGET_PAGE_MASK
;
1142 end
= start
+ int128_get64(section
->size
) - 1;
1143 subpage_register(subpage
, start
, end
,
1144 phys_section_add(&d
->map
, section
));
1148 static void register_multipage(AddressSpaceDispatch
*d
,
1149 MemoryRegionSection
*section
)
1151 hwaddr start_addr
= section
->offset_within_address_space
;
1152 uint16_t section_index
= phys_section_add(&d
->map
, section
);
1153 uint64_t num_pages
= int128_get64(int128_rshift(section
->size
,
1157 phys_page_set(d
, start_addr
>> TARGET_PAGE_BITS
, num_pages
, section_index
);
1160 static void mem_add(MemoryListener
*listener
, MemoryRegionSection
*section
)
1162 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
1163 AddressSpaceDispatch
*d
= as
->next_dispatch
;
1164 MemoryRegionSection now
= *section
, remain
= *section
;
1165 Int128 page_size
= int128_make64(TARGET_PAGE_SIZE
);
1167 if (now
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
1168 uint64_t left
= TARGET_PAGE_ALIGN(now
.offset_within_address_space
)
1169 - now
.offset_within_address_space
;
1171 now
.size
= int128_min(int128_make64(left
), now
.size
);
1172 register_subpage(d
, &now
);
1174 now
.size
= int128_zero();
1176 while (int128_ne(remain
.size
, now
.size
)) {
1177 remain
.size
= int128_sub(remain
.size
, now
.size
);
1178 remain
.offset_within_address_space
+= int128_get64(now
.size
);
1179 remain
.offset_within_region
+= int128_get64(now
.size
);
1181 if (int128_lt(remain
.size
, page_size
)) {
1182 register_subpage(d
, &now
);
1183 } else if (remain
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
1184 now
.size
= page_size
;
1185 register_subpage(d
, &now
);
1187 now
.size
= int128_and(now
.size
, int128_neg(page_size
));
1188 register_multipage(d
, &now
);
1193 void qemu_flush_coalesced_mmio_buffer(void)
1196 kvm_flush_coalesced_mmio_buffer();
1199 void qemu_mutex_lock_ramlist(void)
1201 qemu_mutex_lock(&ram_list
.mutex
);
1204 void qemu_mutex_unlock_ramlist(void)
1206 qemu_mutex_unlock(&ram_list
.mutex
);
1211 #include <sys/vfs.h>
1213 #define HUGETLBFS_MAGIC 0x958458f6
1215 static long gethugepagesize(const char *path
, Error
**errp
)
1221 ret
= statfs(path
, &fs
);
1222 } while (ret
!= 0 && errno
== EINTR
);
1225 error_setg_errno(errp
, errno
, "failed to get page size of file %s",
1233 static void *file_ram_alloc(RAMBlock
*block
,
1240 char *sanitized_name
;
1245 Error
*local_err
= NULL
;
1247 hpagesize
= gethugepagesize(path
, &local_err
);
1249 error_propagate(errp
, local_err
);
1252 block
->mr
->align
= hpagesize
;
1254 if (memory
< hpagesize
) {
1255 error_setg(errp
, "memory size 0x" RAM_ADDR_FMT
" must be equal to "
1256 "or larger than huge page size 0x%" PRIx64
,
1261 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1263 "host lacks kvm mmu notifiers, -mem-path unsupported");
1267 if (!stat(path
, &st
) && S_ISDIR(st
.st_mode
)) {
1268 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1269 sanitized_name
= g_strdup(memory_region_name(block
->mr
));
1270 for (c
= sanitized_name
; *c
!= '\0'; c
++) {
1276 filename
= g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path
,
1278 g_free(sanitized_name
);
1280 fd
= mkstemp(filename
);
1286 fd
= open(path
, O_RDWR
| O_CREAT
, 0644);
1290 error_setg_errno(errp
, errno
,
1291 "unable to create backing store for hugepages");
1295 memory
= ROUND_UP(memory
, hpagesize
);
1298 * ftruncate is not supported by hugetlbfs in older
1299 * hosts, so don't bother bailing out on errors.
1300 * If anything goes wrong with it under other filesystems,
1303 if (ftruncate(fd
, memory
)) {
1304 perror("ftruncate");
1307 area
= qemu_ram_mmap(fd
, memory
, hpagesize
, block
->flags
& RAM_SHARED
);
1308 if (area
== MAP_FAILED
) {
1309 error_setg_errno(errp
, errno
,
1310 "unable to map backing store for hugepages");
1316 os_mem_prealloc(fd
, area
, memory
);
1327 /* Called with the ramlist lock held. */
1328 static ram_addr_t
find_ram_offset(ram_addr_t size
)
1330 RAMBlock
*block
, *next_block
;
1331 ram_addr_t offset
= RAM_ADDR_MAX
, mingap
= RAM_ADDR_MAX
;
1333 assert(size
!= 0); /* it would hand out same offset multiple times */
1335 if (QLIST_EMPTY_RCU(&ram_list
.blocks
)) {
1339 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1340 ram_addr_t end
, next
= RAM_ADDR_MAX
;
1342 end
= block
->offset
+ block
->max_length
;
1344 QLIST_FOREACH_RCU(next_block
, &ram_list
.blocks
, next
) {
1345 if (next_block
->offset
>= end
) {
1346 next
= MIN(next
, next_block
->offset
);
1349 if (next
- end
>= size
&& next
- end
< mingap
) {
1351 mingap
= next
- end
;
1355 if (offset
== RAM_ADDR_MAX
) {
1356 fprintf(stderr
, "Failed to find gap of requested size: %" PRIu64
"\n",
1364 ram_addr_t
last_ram_offset(void)
1367 ram_addr_t last
= 0;
1370 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1371 last
= MAX(last
, block
->offset
+ block
->max_length
);
1377 static void qemu_ram_setup_dump(void *addr
, ram_addr_t size
)
1381 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1382 if (!machine_dump_guest_core(current_machine
)) {
1383 ret
= qemu_madvise(addr
, size
, QEMU_MADV_DONTDUMP
);
1385 perror("qemu_madvise");
1386 fprintf(stderr
, "madvise doesn't support MADV_DONTDUMP, "
1387 "but dump_guest_core=off specified\n");
1392 /* Called within an RCU critical section, or while the ramlist lock
1395 static RAMBlock
*find_ram_block(ram_addr_t addr
)
1399 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1400 if (block
->offset
== addr
) {
1408 const char *qemu_ram_get_idstr(RAMBlock
*rb
)
1413 /* Called with iothread lock held. */
1414 void qemu_ram_set_idstr(ram_addr_t addr
, const char *name
, DeviceState
*dev
)
1416 RAMBlock
*new_block
, *block
;
1419 new_block
= find_ram_block(addr
);
1421 assert(!new_block
->idstr
[0]);
1424 char *id
= qdev_get_dev_path(dev
);
1426 snprintf(new_block
->idstr
, sizeof(new_block
->idstr
), "%s/", id
);
1430 pstrcat(new_block
->idstr
, sizeof(new_block
->idstr
), name
);
1432 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1433 if (block
!= new_block
&& !strcmp(block
->idstr
, new_block
->idstr
)) {
1434 fprintf(stderr
, "RAMBlock \"%s\" already registered, abort!\n",
1442 /* Called with iothread lock held. */
1443 void qemu_ram_unset_idstr(ram_addr_t addr
)
1447 /* FIXME: arch_init.c assumes that this is not called throughout
1448 * migration. Ignore the problem since hot-unplug during migration
1449 * does not work anyway.
1453 block
= find_ram_block(addr
);
1455 memset(block
->idstr
, 0, sizeof(block
->idstr
));
1460 static int memory_try_enable_merging(void *addr
, size_t len
)
1462 if (!machine_mem_merge(current_machine
)) {
1463 /* disabled by the user */
1467 return qemu_madvise(addr
, len
, QEMU_MADV_MERGEABLE
);
1470 /* Only legal before guest might have detected the memory size: e.g. on
1471 * incoming migration, or right after reset.
1473 * As memory core doesn't know how is memory accessed, it is up to
1474 * resize callback to update device state and/or add assertions to detect
1475 * misuse, if necessary.
1477 int qemu_ram_resize(ram_addr_t base
, ram_addr_t newsize
, Error
**errp
)
1479 RAMBlock
*block
= find_ram_block(base
);
1483 newsize
= HOST_PAGE_ALIGN(newsize
);
1485 if (block
->used_length
== newsize
) {
1489 if (!(block
->flags
& RAM_RESIZEABLE
)) {
1490 error_setg_errno(errp
, EINVAL
,
1491 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1492 " in != 0x" RAM_ADDR_FMT
, block
->idstr
,
1493 newsize
, block
->used_length
);
1497 if (block
->max_length
< newsize
) {
1498 error_setg_errno(errp
, EINVAL
,
1499 "Length too large: %s: 0x" RAM_ADDR_FMT
1500 " > 0x" RAM_ADDR_FMT
, block
->idstr
,
1501 newsize
, block
->max_length
);
1505 cpu_physical_memory_clear_dirty_range(block
->offset
, block
->used_length
);
1506 block
->used_length
= newsize
;
1507 cpu_physical_memory_set_dirty_range(block
->offset
, block
->used_length
,
1509 memory_region_set_size(block
->mr
, newsize
);
1510 if (block
->resized
) {
1511 block
->resized(block
->idstr
, newsize
, block
->host
);
1516 /* Called with ram_list.mutex held */
1517 static void dirty_memory_extend(ram_addr_t old_ram_size
,
1518 ram_addr_t new_ram_size
)
1520 ram_addr_t old_num_blocks
= DIV_ROUND_UP(old_ram_size
,
1521 DIRTY_MEMORY_BLOCK_SIZE
);
1522 ram_addr_t new_num_blocks
= DIV_ROUND_UP(new_ram_size
,
1523 DIRTY_MEMORY_BLOCK_SIZE
);
1526 /* Only need to extend if block count increased */
1527 if (new_num_blocks
<= old_num_blocks
) {
1531 for (i
= 0; i
< DIRTY_MEMORY_NUM
; i
++) {
1532 DirtyMemoryBlocks
*old_blocks
;
1533 DirtyMemoryBlocks
*new_blocks
;
1536 old_blocks
= atomic_rcu_read(&ram_list
.dirty_memory
[i
]);
1537 new_blocks
= g_malloc(sizeof(*new_blocks
) +
1538 sizeof(new_blocks
->blocks
[0]) * new_num_blocks
);
1540 if (old_num_blocks
) {
1541 memcpy(new_blocks
->blocks
, old_blocks
->blocks
,
1542 old_num_blocks
* sizeof(old_blocks
->blocks
[0]));
1545 for (j
= old_num_blocks
; j
< new_num_blocks
; j
++) {
1546 new_blocks
->blocks
[j
] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE
);
1549 atomic_rcu_set(&ram_list
.dirty_memory
[i
], new_blocks
);
1552 g_free_rcu(old_blocks
, rcu
);
1557 static ram_addr_t
ram_block_add(RAMBlock
*new_block
, Error
**errp
)
1560 RAMBlock
*last_block
= NULL
;
1561 ram_addr_t old_ram_size
, new_ram_size
;
1564 old_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1566 qemu_mutex_lock_ramlist();
1567 new_block
->offset
= find_ram_offset(new_block
->max_length
);
1569 if (!new_block
->host
) {
1570 if (xen_enabled()) {
1571 xen_ram_alloc(new_block
->offset
, new_block
->max_length
,
1572 new_block
->mr
, &err
);
1574 error_propagate(errp
, err
);
1575 qemu_mutex_unlock_ramlist();
1579 new_block
->host
= phys_mem_alloc(new_block
->max_length
,
1580 &new_block
->mr
->align
);
1581 if (!new_block
->host
) {
1582 error_setg_errno(errp
, errno
,
1583 "cannot set up guest memory '%s'",
1584 memory_region_name(new_block
->mr
));
1585 qemu_mutex_unlock_ramlist();
1588 memory_try_enable_merging(new_block
->host
, new_block
->max_length
);
1592 new_ram_size
= MAX(old_ram_size
,
1593 (new_block
->offset
+ new_block
->max_length
) >> TARGET_PAGE_BITS
);
1594 if (new_ram_size
> old_ram_size
) {
1595 migration_bitmap_extend(old_ram_size
, new_ram_size
);
1596 dirty_memory_extend(old_ram_size
, new_ram_size
);
1598 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1599 * QLIST (which has an RCU-friendly variant) does not have insertion at
1600 * tail, so save the last element in last_block.
1602 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1604 if (block
->max_length
< new_block
->max_length
) {
1609 QLIST_INSERT_BEFORE_RCU(block
, new_block
, next
);
1610 } else if (last_block
) {
1611 QLIST_INSERT_AFTER_RCU(last_block
, new_block
, next
);
1612 } else { /* list is empty */
1613 QLIST_INSERT_HEAD_RCU(&ram_list
.blocks
, new_block
, next
);
1615 ram_list
.mru_block
= NULL
;
1617 /* Write list before version */
1620 qemu_mutex_unlock_ramlist();
1622 cpu_physical_memory_set_dirty_range(new_block
->offset
,
1623 new_block
->used_length
,
1626 if (new_block
->host
) {
1627 qemu_ram_setup_dump(new_block
->host
, new_block
->max_length
);
1628 qemu_madvise(new_block
->host
, new_block
->max_length
, QEMU_MADV_HUGEPAGE
);
1629 qemu_madvise(new_block
->host
, new_block
->max_length
, QEMU_MADV_DONTFORK
);
1630 if (kvm_enabled()) {
1631 kvm_setup_guest_memory(new_block
->host
, new_block
->max_length
);
1635 return new_block
->offset
;
1639 ram_addr_t
qemu_ram_alloc_from_file(ram_addr_t size
, MemoryRegion
*mr
,
1640 bool share
, const char *mem_path
,
1643 RAMBlock
*new_block
;
1645 Error
*local_err
= NULL
;
1647 if (xen_enabled()) {
1648 error_setg(errp
, "-mem-path not supported with Xen");
1652 if (phys_mem_alloc
!= qemu_anon_ram_alloc
) {
1654 * file_ram_alloc() needs to allocate just like
1655 * phys_mem_alloc, but we haven't bothered to provide
1659 "-mem-path not supported with this accelerator");
1663 size
= HOST_PAGE_ALIGN(size
);
1664 new_block
= g_malloc0(sizeof(*new_block
));
1666 new_block
->used_length
= size
;
1667 new_block
->max_length
= size
;
1668 new_block
->flags
= share
? RAM_SHARED
: 0;
1669 new_block
->host
= file_ram_alloc(new_block
, size
,
1671 if (!new_block
->host
) {
1676 addr
= ram_block_add(new_block
, &local_err
);
1679 error_propagate(errp
, local_err
);
1687 ram_addr_t
qemu_ram_alloc_internal(ram_addr_t size
, ram_addr_t max_size
,
1688 void (*resized
)(const char*,
1691 void *host
, bool resizeable
,
1692 MemoryRegion
*mr
, Error
**errp
)
1694 RAMBlock
*new_block
;
1696 Error
*local_err
= NULL
;
1698 size
= HOST_PAGE_ALIGN(size
);
1699 max_size
= HOST_PAGE_ALIGN(max_size
);
1700 new_block
= g_malloc0(sizeof(*new_block
));
1702 new_block
->resized
= resized
;
1703 new_block
->used_length
= size
;
1704 new_block
->max_length
= max_size
;
1705 assert(max_size
>= size
);
1707 new_block
->host
= host
;
1709 new_block
->flags
|= RAM_PREALLOC
;
1712 new_block
->flags
|= RAM_RESIZEABLE
;
1714 addr
= ram_block_add(new_block
, &local_err
);
1717 error_propagate(errp
, local_err
);
1721 mr
->ram_block
= new_block
;
1725 ram_addr_t
qemu_ram_alloc_from_ptr(ram_addr_t size
, void *host
,
1726 MemoryRegion
*mr
, Error
**errp
)
1728 return qemu_ram_alloc_internal(size
, size
, NULL
, host
, false, mr
, errp
);
1731 ram_addr_t
qemu_ram_alloc(ram_addr_t size
, MemoryRegion
*mr
, Error
**errp
)
1733 return qemu_ram_alloc_internal(size
, size
, NULL
, NULL
, false, mr
, errp
);
1736 ram_addr_t
qemu_ram_alloc_resizeable(ram_addr_t size
, ram_addr_t maxsz
,
1737 void (*resized
)(const char*,
1740 MemoryRegion
*mr
, Error
**errp
)
1742 return qemu_ram_alloc_internal(size
, maxsz
, resized
, NULL
, true, mr
, errp
);
1745 static void reclaim_ramblock(RAMBlock
*block
)
1747 if (block
->flags
& RAM_PREALLOC
) {
1749 } else if (xen_enabled()) {
1750 xen_invalidate_map_cache_entry(block
->host
);
1752 } else if (block
->fd
>= 0) {
1753 qemu_ram_munmap(block
->host
, block
->max_length
);
1757 qemu_anon_ram_free(block
->host
, block
->max_length
);
1762 void qemu_ram_free(ram_addr_t addr
)
1766 qemu_mutex_lock_ramlist();
1767 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1768 if (addr
== block
->offset
) {
1769 QLIST_REMOVE_RCU(block
, next
);
1770 ram_list
.mru_block
= NULL
;
1771 /* Write list before version */
1774 call_rcu(block
, reclaim_ramblock
, rcu
);
1778 qemu_mutex_unlock_ramlist();
1782 void qemu_ram_remap(ram_addr_t addr
, ram_addr_t length
)
1789 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1790 offset
= addr
- block
->offset
;
1791 if (offset
< block
->max_length
) {
1792 vaddr
= ramblock_ptr(block
, offset
);
1793 if (block
->flags
& RAM_PREALLOC
) {
1795 } else if (xen_enabled()) {
1799 if (block
->fd
>= 0) {
1800 flags
|= (block
->flags
& RAM_SHARED
?
1801 MAP_SHARED
: MAP_PRIVATE
);
1802 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1803 flags
, block
->fd
, offset
);
1806 * Remap needs to match alloc. Accelerators that
1807 * set phys_mem_alloc never remap. If they did,
1808 * we'd need a remap hook here.
1810 assert(phys_mem_alloc
== qemu_anon_ram_alloc
);
1812 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
1813 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1816 if (area
!= vaddr
) {
1817 fprintf(stderr
, "Could not remap addr: "
1818 RAM_ADDR_FMT
"@" RAM_ADDR_FMT
"\n",
1822 memory_try_enable_merging(vaddr
, length
);
1823 qemu_ram_setup_dump(vaddr
, length
);
1828 #endif /* !_WIN32 */
1830 int qemu_get_ram_fd(ram_addr_t addr
)
1836 block
= qemu_get_ram_block(addr
);
1842 void qemu_set_ram_fd(ram_addr_t addr
, int fd
)
1847 block
= qemu_get_ram_block(addr
);
1852 void *qemu_get_ram_block_host_ptr(ram_addr_t addr
)
1858 block
= qemu_get_ram_block(addr
);
1859 ptr
= ramblock_ptr(block
, 0);
1864 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1865 * This should not be used for general purpose DMA. Use address_space_map
1866 * or address_space_rw instead. For local memory (e.g. video ram) that the
1867 * device owns, use memory_region_get_ram_ptr.
1869 * Called within RCU critical section.
1871 void *qemu_get_ram_ptr(ram_addr_t addr
)
1873 RAMBlock
*block
= qemu_get_ram_block(addr
);
1875 if (xen_enabled() && block
->host
== NULL
) {
1876 /* We need to check if the requested address is in the RAM
1877 * because we don't want to map the entire memory in QEMU.
1878 * In that case just map until the end of the page.
1880 if (block
->offset
== 0) {
1881 return xen_map_cache(addr
, 0, 0);
1884 block
->host
= xen_map_cache(block
->offset
, block
->max_length
, 1);
1886 return ramblock_ptr(block
, addr
- block
->offset
);
1889 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1890 * but takes a size argument.
1892 * Called within RCU critical section.
1894 static void *qemu_ram_ptr_length(ram_addr_t addr
, hwaddr
*size
)
1897 ram_addr_t offset_inside_block
;
1902 block
= qemu_get_ram_block(addr
);
1903 offset_inside_block
= addr
- block
->offset
;
1904 *size
= MIN(*size
, block
->max_length
- offset_inside_block
);
1906 if (xen_enabled() && block
->host
== NULL
) {
1907 /* We need to check if the requested address is in the RAM
1908 * because we don't want to map the entire memory in QEMU.
1909 * In that case just map the requested area.
1911 if (block
->offset
== 0) {
1912 return xen_map_cache(addr
, *size
, 1);
1915 block
->host
= xen_map_cache(block
->offset
, block
->max_length
, 1);
1918 return ramblock_ptr(block
, offset_inside_block
);
1922 * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
1925 * ptr: Host pointer to look up
1926 * round_offset: If true round the result offset down to a page boundary
1927 * *ram_addr: set to result ram_addr
1928 * *offset: set to result offset within the RAMBlock
1930 * Returns: RAMBlock (or NULL if not found)
1932 * By the time this function returns, the returned pointer is not protected
1933 * by RCU anymore. If the caller is not within an RCU critical section and
1934 * does not hold the iothread lock, it must have other means of protecting the
1935 * pointer, such as a reference to the region that includes the incoming
1938 RAMBlock
*qemu_ram_block_from_host(void *ptr
, bool round_offset
,
1939 ram_addr_t
*ram_addr
,
1943 uint8_t *host
= ptr
;
1945 if (xen_enabled()) {
1947 *ram_addr
= xen_ram_addr_from_mapcache(ptr
);
1948 block
= qemu_get_ram_block(*ram_addr
);
1950 *offset
= (host
- block
->host
);
1957 block
= atomic_rcu_read(&ram_list
.mru_block
);
1958 if (block
&& block
->host
&& host
- block
->host
< block
->max_length
) {
1962 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1963 /* This case append when the block is not mapped. */
1964 if (block
->host
== NULL
) {
1967 if (host
- block
->host
< block
->max_length
) {
1976 *offset
= (host
- block
->host
);
1978 *offset
&= TARGET_PAGE_MASK
;
1980 *ram_addr
= block
->offset
+ *offset
;
1986 * Finds the named RAMBlock
1988 * name: The name of RAMBlock to find
1990 * Returns: RAMBlock (or NULL if not found)
1992 RAMBlock
*qemu_ram_block_by_name(const char *name
)
1996 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1997 if (!strcmp(name
, block
->idstr
)) {
2005 /* Some of the softmmu routines need to translate from a host pointer
2006 (typically a TLB entry) back to a ram offset. */
2007 MemoryRegion
*qemu_ram_addr_from_host(void *ptr
, ram_addr_t
*ram_addr
)
2010 ram_addr_t offset
; /* Not used */
2012 block
= qemu_ram_block_from_host(ptr
, false, ram_addr
, &offset
);
2021 /* Called within RCU critical section. */
2022 static void notdirty_mem_write(void *opaque
, hwaddr ram_addr
,
2023 uint64_t val
, unsigned size
)
2025 if (!cpu_physical_memory_get_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
)) {
2026 tb_invalidate_phys_page_fast(ram_addr
, size
);
2030 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
2033 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
2036 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
2041 /* Set both VGA and migration bits for simplicity and to remove
2042 * the notdirty callback faster.
2044 cpu_physical_memory_set_dirty_range(ram_addr
, size
,
2045 DIRTY_CLIENTS_NOCODE
);
2046 /* we remove the notdirty callback only if the code has been
2048 if (!cpu_physical_memory_is_clean(ram_addr
)) {
2049 tlb_set_dirty(current_cpu
, current_cpu
->mem_io_vaddr
);
2053 static bool notdirty_mem_accepts(void *opaque
, hwaddr addr
,
2054 unsigned size
, bool is_write
)
2059 static const MemoryRegionOps notdirty_mem_ops
= {
2060 .write
= notdirty_mem_write
,
2061 .valid
.accepts
= notdirty_mem_accepts
,
2062 .endianness
= DEVICE_NATIVE_ENDIAN
,
2065 /* Generate a debug exception if a watchpoint has been hit. */
2066 static void check_watchpoint(int offset
, int len
, MemTxAttrs attrs
, int flags
)
2068 CPUState
*cpu
= current_cpu
;
2069 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
2070 CPUArchState
*env
= cpu
->env_ptr
;
2071 target_ulong pc
, cs_base
;
2076 if (cpu
->watchpoint_hit
) {
2077 /* We re-entered the check after replacing the TB. Now raise
2078 * the debug interrupt so that is will trigger after the
2079 * current instruction. */
2080 cpu_interrupt(cpu
, CPU_INTERRUPT_DEBUG
);
2083 vaddr
= (cpu
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
2084 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
2085 if (cpu_watchpoint_address_matches(wp
, vaddr
, len
)
2086 && (wp
->flags
& flags
)) {
2087 if (flags
== BP_MEM_READ
) {
2088 wp
->flags
|= BP_WATCHPOINT_HIT_READ
;
2090 wp
->flags
|= BP_WATCHPOINT_HIT_WRITE
;
2092 wp
->hitaddr
= vaddr
;
2093 wp
->hitattrs
= attrs
;
2094 if (!cpu
->watchpoint_hit
) {
2095 if (wp
->flags
& BP_CPU
&&
2096 !cc
->debug_check_watchpoint(cpu
, wp
)) {
2097 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
2100 cpu
->watchpoint_hit
= wp
;
2101 tb_check_watchpoint(cpu
);
2102 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
2103 cpu
->exception_index
= EXCP_DEBUG
;
2106 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
2107 tb_gen_code(cpu
, pc
, cs_base
, cpu_flags
, 1);
2108 cpu_resume_from_signal(cpu
, NULL
);
2112 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
2117 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2118 so these check for a hit then pass through to the normal out-of-line
2120 static MemTxResult
watch_mem_read(void *opaque
, hwaddr addr
, uint64_t *pdata
,
2121 unsigned size
, MemTxAttrs attrs
)
2125 int asidx
= cpu_asidx_from_attrs(current_cpu
, attrs
);
2126 AddressSpace
*as
= current_cpu
->cpu_ases
[asidx
].as
;
2128 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, size
, attrs
, BP_MEM_READ
);
2131 data
= address_space_ldub(as
, addr
, attrs
, &res
);
2134 data
= address_space_lduw(as
, addr
, attrs
, &res
);
2137 data
= address_space_ldl(as
, addr
, attrs
, &res
);
2145 static MemTxResult
watch_mem_write(void *opaque
, hwaddr addr
,
2146 uint64_t val
, unsigned size
,
2150 int asidx
= cpu_asidx_from_attrs(current_cpu
, attrs
);
2151 AddressSpace
*as
= current_cpu
->cpu_ases
[asidx
].as
;
2153 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, size
, attrs
, BP_MEM_WRITE
);
2156 address_space_stb(as
, addr
, val
, attrs
, &res
);
2159 address_space_stw(as
, addr
, val
, attrs
, &res
);
2162 address_space_stl(as
, addr
, val
, attrs
, &res
);
2169 static const MemoryRegionOps watch_mem_ops
= {
2170 .read_with_attrs
= watch_mem_read
,
2171 .write_with_attrs
= watch_mem_write
,
2172 .endianness
= DEVICE_NATIVE_ENDIAN
,
2175 static MemTxResult
subpage_read(void *opaque
, hwaddr addr
, uint64_t *data
,
2176 unsigned len
, MemTxAttrs attrs
)
2178 subpage_t
*subpage
= opaque
;
2182 #if defined(DEBUG_SUBPAGE)
2183 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
"\n", __func__
,
2184 subpage
, len
, addr
);
2186 res
= address_space_read(subpage
->as
, addr
+ subpage
->base
,
2193 *data
= ldub_p(buf
);
2196 *data
= lduw_p(buf
);
2209 static MemTxResult
subpage_write(void *opaque
, hwaddr addr
,
2210 uint64_t value
, unsigned len
, MemTxAttrs attrs
)
2212 subpage_t
*subpage
= opaque
;
2215 #if defined(DEBUG_SUBPAGE)
2216 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
2217 " value %"PRIx64
"\n",
2218 __func__
, subpage
, len
, addr
, value
);
2236 return address_space_write(subpage
->as
, addr
+ subpage
->base
,
2240 static bool subpage_accepts(void *opaque
, hwaddr addr
,
2241 unsigned len
, bool is_write
)
2243 subpage_t
*subpage
= opaque
;
2244 #if defined(DEBUG_SUBPAGE)
2245 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx
"\n",
2246 __func__
, subpage
, is_write
? 'w' : 'r', len
, addr
);
2249 return address_space_access_valid(subpage
->as
, addr
+ subpage
->base
,
2253 static const MemoryRegionOps subpage_ops
= {
2254 .read_with_attrs
= subpage_read
,
2255 .write_with_attrs
= subpage_write
,
2256 .impl
.min_access_size
= 1,
2257 .impl
.max_access_size
= 8,
2258 .valid
.min_access_size
= 1,
2259 .valid
.max_access_size
= 8,
2260 .valid
.accepts
= subpage_accepts
,
2261 .endianness
= DEVICE_NATIVE_ENDIAN
,
2264 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2269 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
2271 idx
= SUBPAGE_IDX(start
);
2272 eidx
= SUBPAGE_IDX(end
);
2273 #if defined(DEBUG_SUBPAGE)
2274 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2275 __func__
, mmio
, start
, end
, idx
, eidx
, section
);
2277 for (; idx
<= eidx
; idx
++) {
2278 mmio
->sub_section
[idx
] = section
;
2284 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
)
2288 mmio
= g_malloc0(sizeof(subpage_t
));
2292 memory_region_init_io(&mmio
->iomem
, NULL
, &subpage_ops
, mmio
,
2293 NULL
, TARGET_PAGE_SIZE
);
2294 mmio
->iomem
.subpage
= true;
2295 #if defined(DEBUG_SUBPAGE)
2296 printf("%s: %p base " TARGET_FMT_plx
" len %08x\n", __func__
,
2297 mmio
, base
, TARGET_PAGE_SIZE
);
2299 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
-1, PHYS_SECTION_UNASSIGNED
);
2304 static uint16_t dummy_section(PhysPageMap
*map
, AddressSpace
*as
,
2308 MemoryRegionSection section
= {
2309 .address_space
= as
,
2311 .offset_within_address_space
= 0,
2312 .offset_within_region
= 0,
2313 .size
= int128_2_64(),
2316 return phys_section_add(map
, §ion
);
2319 MemoryRegion
*iotlb_to_region(CPUState
*cpu
, hwaddr index
, MemTxAttrs attrs
)
2321 int asidx
= cpu_asidx_from_attrs(cpu
, attrs
);
2322 CPUAddressSpace
*cpuas
= &cpu
->cpu_ases
[asidx
];
2323 AddressSpaceDispatch
*d
= atomic_rcu_read(&cpuas
->memory_dispatch
);
2324 MemoryRegionSection
*sections
= d
->map
.sections
;
2326 return sections
[index
& ~TARGET_PAGE_MASK
].mr
;
2329 static void io_mem_init(void)
2331 memory_region_init_io(&io_mem_rom
, NULL
, &unassigned_mem_ops
, NULL
, NULL
, UINT64_MAX
);
2332 memory_region_init_io(&io_mem_unassigned
, NULL
, &unassigned_mem_ops
, NULL
,
2334 memory_region_init_io(&io_mem_notdirty
, NULL
, ¬dirty_mem_ops
, NULL
,
2336 memory_region_init_io(&io_mem_watch
, NULL
, &watch_mem_ops
, NULL
,
2340 static void mem_begin(MemoryListener
*listener
)
2342 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
2343 AddressSpaceDispatch
*d
= g_new0(AddressSpaceDispatch
, 1);
2346 n
= dummy_section(&d
->map
, as
, &io_mem_unassigned
);
2347 assert(n
== PHYS_SECTION_UNASSIGNED
);
2348 n
= dummy_section(&d
->map
, as
, &io_mem_notdirty
);
2349 assert(n
== PHYS_SECTION_NOTDIRTY
);
2350 n
= dummy_section(&d
->map
, as
, &io_mem_rom
);
2351 assert(n
== PHYS_SECTION_ROM
);
2352 n
= dummy_section(&d
->map
, as
, &io_mem_watch
);
2353 assert(n
== PHYS_SECTION_WATCH
);
2355 d
->phys_map
= (PhysPageEntry
) { .ptr
= PHYS_MAP_NODE_NIL
, .skip
= 1 };
2357 as
->next_dispatch
= d
;
2360 static void address_space_dispatch_free(AddressSpaceDispatch
*d
)
2362 phys_sections_free(&d
->map
);
2366 static void mem_commit(MemoryListener
*listener
)
2368 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
2369 AddressSpaceDispatch
*cur
= as
->dispatch
;
2370 AddressSpaceDispatch
*next
= as
->next_dispatch
;
2372 phys_page_compact_all(next
, next
->map
.nodes_nb
);
2374 atomic_rcu_set(&as
->dispatch
, next
);
2376 call_rcu(cur
, address_space_dispatch_free
, rcu
);
2380 static void tcg_commit(MemoryListener
*listener
)
2382 CPUAddressSpace
*cpuas
;
2383 AddressSpaceDispatch
*d
;
2385 /* since each CPU stores ram addresses in its TLB cache, we must
2386 reset the modified entries */
2387 cpuas
= container_of(listener
, CPUAddressSpace
, tcg_as_listener
);
2388 cpu_reloading_memory_map();
2389 /* The CPU and TLB are protected by the iothread lock.
2390 * We reload the dispatch pointer now because cpu_reloading_memory_map()
2391 * may have split the RCU critical section.
2393 d
= atomic_rcu_read(&cpuas
->as
->dispatch
);
2394 cpuas
->memory_dispatch
= d
;
2395 tlb_flush(cpuas
->cpu
, 1);
2398 void address_space_init_dispatch(AddressSpace
*as
)
2400 as
->dispatch
= NULL
;
2401 as
->dispatch_listener
= (MemoryListener
) {
2403 .commit
= mem_commit
,
2404 .region_add
= mem_add
,
2405 .region_nop
= mem_add
,
2408 memory_listener_register(&as
->dispatch_listener
, as
);
2411 void address_space_unregister(AddressSpace
*as
)
2413 memory_listener_unregister(&as
->dispatch_listener
);
2416 void address_space_destroy_dispatch(AddressSpace
*as
)
2418 AddressSpaceDispatch
*d
= as
->dispatch
;
2420 atomic_rcu_set(&as
->dispatch
, NULL
);
2422 call_rcu(d
, address_space_dispatch_free
, rcu
);
2426 static void memory_map_init(void)
2428 system_memory
= g_malloc(sizeof(*system_memory
));
2430 memory_region_init(system_memory
, NULL
, "system", UINT64_MAX
);
2431 address_space_init(&address_space_memory
, system_memory
, "memory");
2433 system_io
= g_malloc(sizeof(*system_io
));
2434 memory_region_init_io(system_io
, NULL
, &unassigned_io_ops
, NULL
, "io",
2436 address_space_init(&address_space_io
, system_io
, "I/O");
2439 MemoryRegion
*get_system_memory(void)
2441 return system_memory
;
2444 MemoryRegion
*get_system_io(void)
2449 #endif /* !defined(CONFIG_USER_ONLY) */
2451 /* physical memory access (slow version, mainly for debug) */
2452 #if defined(CONFIG_USER_ONLY)
2453 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
2454 uint8_t *buf
, int len
, int is_write
)
2461 page
= addr
& TARGET_PAGE_MASK
;
2462 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2465 flags
= page_get_flags(page
);
2466 if (!(flags
& PAGE_VALID
))
2469 if (!(flags
& PAGE_WRITE
))
2471 /* XXX: this code should not depend on lock_user */
2472 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
2475 unlock_user(p
, addr
, l
);
2477 if (!(flags
& PAGE_READ
))
2479 /* XXX: this code should not depend on lock_user */
2480 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
2483 unlock_user(p
, addr
, 0);
2494 static void invalidate_and_set_dirty(MemoryRegion
*mr
, hwaddr addr
,
2497 uint8_t dirty_log_mask
= memory_region_get_dirty_log_mask(mr
);
2498 /* No early return if dirty_log_mask is or becomes 0, because
2499 * cpu_physical_memory_set_dirty_range will still call
2500 * xen_modified_memory.
2502 if (dirty_log_mask
) {
2504 cpu_physical_memory_range_includes_clean(addr
, length
, dirty_log_mask
);
2506 if (dirty_log_mask
& (1 << DIRTY_MEMORY_CODE
)) {
2507 tb_invalidate_phys_range(addr
, addr
+ length
);
2508 dirty_log_mask
&= ~(1 << DIRTY_MEMORY_CODE
);
2510 cpu_physical_memory_set_dirty_range(addr
, length
, dirty_log_mask
);
2513 static int memory_access_size(MemoryRegion
*mr
, unsigned l
, hwaddr addr
)
2515 unsigned access_size_max
= mr
->ops
->valid
.max_access_size
;
2517 /* Regions are assumed to support 1-4 byte accesses unless
2518 otherwise specified. */
2519 if (access_size_max
== 0) {
2520 access_size_max
= 4;
2523 /* Bound the maximum access by the alignment of the address. */
2524 if (!mr
->ops
->impl
.unaligned
) {
2525 unsigned align_size_max
= addr
& -addr
;
2526 if (align_size_max
!= 0 && align_size_max
< access_size_max
) {
2527 access_size_max
= align_size_max
;
2531 /* Don't attempt accesses larger than the maximum. */
2532 if (l
> access_size_max
) {
2533 l
= access_size_max
;
2540 static bool prepare_mmio_access(MemoryRegion
*mr
)
2542 bool unlocked
= !qemu_mutex_iothread_locked();
2543 bool release_lock
= false;
2545 if (unlocked
&& mr
->global_locking
) {
2546 qemu_mutex_lock_iothread();
2548 release_lock
= true;
2550 if (mr
->flush_coalesced_mmio
) {
2552 qemu_mutex_lock_iothread();
2554 qemu_flush_coalesced_mmio_buffer();
2556 qemu_mutex_unlock_iothread();
2560 return release_lock
;
2563 /* Called within RCU critical section. */
2564 static MemTxResult
address_space_write_continue(AddressSpace
*as
, hwaddr addr
,
2567 int len
, hwaddr addr1
,
2568 hwaddr l
, MemoryRegion
*mr
)
2572 MemTxResult result
= MEMTX_OK
;
2573 bool release_lock
= false;
2576 if (!memory_access_is_direct(mr
, true)) {
2577 release_lock
|= prepare_mmio_access(mr
);
2578 l
= memory_access_size(mr
, l
, addr1
);
2579 /* XXX: could force current_cpu to NULL to avoid
2583 /* 64 bit write access */
2585 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 8,
2589 /* 32 bit write access */
2591 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 4,
2595 /* 16 bit write access */
2597 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 2,
2601 /* 8 bit write access */
2603 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 1,
2610 addr1
+= memory_region_get_ram_addr(mr
);
2612 ptr
= qemu_get_ram_ptr(addr1
);
2613 memcpy(ptr
, buf
, l
);
2614 invalidate_and_set_dirty(mr
, addr1
, l
);
2618 qemu_mutex_unlock_iothread();
2619 release_lock
= false;
2631 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
2637 MemTxResult
address_space_write(AddressSpace
*as
, hwaddr addr
, MemTxAttrs attrs
,
2638 const uint8_t *buf
, int len
)
2643 MemTxResult result
= MEMTX_OK
;
2648 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
2649 result
= address_space_write_continue(as
, addr
, attrs
, buf
, len
,
2657 /* Called within RCU critical section. */
2658 MemTxResult
address_space_read_continue(AddressSpace
*as
, hwaddr addr
,
2659 MemTxAttrs attrs
, uint8_t *buf
,
2660 int len
, hwaddr addr1
, hwaddr l
,
2665 MemTxResult result
= MEMTX_OK
;
2666 bool release_lock
= false;
2669 if (!memory_access_is_direct(mr
, false)) {
2671 release_lock
|= prepare_mmio_access(mr
);
2672 l
= memory_access_size(mr
, l
, addr1
);
2675 /* 64 bit read access */
2676 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 8,
2681 /* 32 bit read access */
2682 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 4,
2687 /* 16 bit read access */
2688 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 2,
2693 /* 8 bit read access */
2694 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 1,
2703 ptr
= qemu_get_ram_ptr(mr
->ram_addr
+ addr1
);
2704 memcpy(buf
, ptr
, l
);
2708 qemu_mutex_unlock_iothread();
2709 release_lock
= false;
2721 mr
= address_space_translate(as
, addr
, &addr1
, &l
, false);
2727 MemTxResult
address_space_read_full(AddressSpace
*as
, hwaddr addr
,
2728 MemTxAttrs attrs
, uint8_t *buf
, int len
)
2733 MemTxResult result
= MEMTX_OK
;
2738 mr
= address_space_translate(as
, addr
, &addr1
, &l
, false);
2739 result
= address_space_read_continue(as
, addr
, attrs
, buf
, len
,
2747 MemTxResult
address_space_rw(AddressSpace
*as
, hwaddr addr
, MemTxAttrs attrs
,
2748 uint8_t *buf
, int len
, bool is_write
)
2751 return address_space_write(as
, addr
, attrs
, (uint8_t *)buf
, len
);
2753 return address_space_read(as
, addr
, attrs
, (uint8_t *)buf
, len
);
2757 void cpu_physical_memory_rw(hwaddr addr
, uint8_t *buf
,
2758 int len
, int is_write
)
2760 address_space_rw(&address_space_memory
, addr
, MEMTXATTRS_UNSPECIFIED
,
2761 buf
, len
, is_write
);
2764 enum write_rom_type
{
2769 static inline void cpu_physical_memory_write_rom_internal(AddressSpace
*as
,
2770 hwaddr addr
, const uint8_t *buf
, int len
, enum write_rom_type type
)
2780 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
2782 if (!(memory_region_is_ram(mr
) ||
2783 memory_region_is_romd(mr
))) {
2784 l
= memory_access_size(mr
, l
, addr1
);
2786 addr1
+= memory_region_get_ram_addr(mr
);
2788 ptr
= qemu_get_ram_ptr(addr1
);
2791 memcpy(ptr
, buf
, l
);
2792 invalidate_and_set_dirty(mr
, addr1
, l
);
2795 flush_icache_range((uintptr_t)ptr
, (uintptr_t)ptr
+ l
);
2806 /* used for ROM loading : can write in RAM and ROM */
2807 void cpu_physical_memory_write_rom(AddressSpace
*as
, hwaddr addr
,
2808 const uint8_t *buf
, int len
)
2810 cpu_physical_memory_write_rom_internal(as
, addr
, buf
, len
, WRITE_DATA
);
2813 void cpu_flush_icache_range(hwaddr start
, int len
)
2816 * This function should do the same thing as an icache flush that was
2817 * triggered from within the guest. For TCG we are always cache coherent,
2818 * so there is no need to flush anything. For KVM / Xen we need to flush
2819 * the host's instruction cache at least.
2821 if (tcg_enabled()) {
2825 cpu_physical_memory_write_rom_internal(&address_space_memory
,
2826 start
, NULL
, len
, FLUSH_CACHE
);
2837 static BounceBuffer bounce
;
2839 typedef struct MapClient
{
2841 QLIST_ENTRY(MapClient
) link
;
2844 QemuMutex map_client_list_lock
;
2845 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
2846 = QLIST_HEAD_INITIALIZER(map_client_list
);
2848 static void cpu_unregister_map_client_do(MapClient
*client
)
2850 QLIST_REMOVE(client
, link
);
2854 static void cpu_notify_map_clients_locked(void)
2858 while (!QLIST_EMPTY(&map_client_list
)) {
2859 client
= QLIST_FIRST(&map_client_list
);
2860 qemu_bh_schedule(client
->bh
);
2861 cpu_unregister_map_client_do(client
);
2865 void cpu_register_map_client(QEMUBH
*bh
)
2867 MapClient
*client
= g_malloc(sizeof(*client
));
2869 qemu_mutex_lock(&map_client_list_lock
);
2871 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
2872 if (!atomic_read(&bounce
.in_use
)) {
2873 cpu_notify_map_clients_locked();
2875 qemu_mutex_unlock(&map_client_list_lock
);
2878 void cpu_exec_init_all(void)
2880 qemu_mutex_init(&ram_list
.mutex
);
2883 qemu_mutex_init(&map_client_list_lock
);
2886 void cpu_unregister_map_client(QEMUBH
*bh
)
2890 qemu_mutex_lock(&map_client_list_lock
);
2891 QLIST_FOREACH(client
, &map_client_list
, link
) {
2892 if (client
->bh
== bh
) {
2893 cpu_unregister_map_client_do(client
);
2897 qemu_mutex_unlock(&map_client_list_lock
);
2900 static void cpu_notify_map_clients(void)
2902 qemu_mutex_lock(&map_client_list_lock
);
2903 cpu_notify_map_clients_locked();
2904 qemu_mutex_unlock(&map_client_list_lock
);
2907 bool address_space_access_valid(AddressSpace
*as
, hwaddr addr
, int len
, bool is_write
)
2915 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2916 if (!memory_access_is_direct(mr
, is_write
)) {
2917 l
= memory_access_size(mr
, l
, addr
);
2918 if (!memory_region_access_valid(mr
, xlat
, l
, is_write
)) {
2930 /* Map a physical memory region into a host virtual address.
2931 * May map a subset of the requested range, given by and returned in *plen.
2932 * May return NULL if resources needed to perform the mapping are exhausted.
2933 * Use only for reads OR writes - not for read-modify-write operations.
2934 * Use cpu_register_map_client() to know when retrying the map operation is
2935 * likely to succeed.
2937 void *address_space_map(AddressSpace
*as
,
2944 hwaddr l
, xlat
, base
;
2945 MemoryRegion
*mr
, *this_mr
;
2955 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2957 if (!memory_access_is_direct(mr
, is_write
)) {
2958 if (atomic_xchg(&bounce
.in_use
, true)) {
2962 /* Avoid unbounded allocations */
2963 l
= MIN(l
, TARGET_PAGE_SIZE
);
2964 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, l
);
2968 memory_region_ref(mr
);
2971 address_space_read(as
, addr
, MEMTXATTRS_UNSPECIFIED
,
2977 return bounce
.buffer
;
2981 raddr
= memory_region_get_ram_addr(mr
);
2992 this_mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2993 if (this_mr
!= mr
|| xlat
!= base
+ done
) {
2998 memory_region_ref(mr
);
3000 ptr
= qemu_ram_ptr_length(raddr
+ base
, plen
);
3006 /* Unmaps a memory region previously mapped by address_space_map().
3007 * Will also mark the memory as dirty if is_write == 1. access_len gives
3008 * the amount of memory that was actually read or written by the caller.
3010 void address_space_unmap(AddressSpace
*as
, void *buffer
, hwaddr len
,
3011 int is_write
, hwaddr access_len
)
3013 if (buffer
!= bounce
.buffer
) {
3017 mr
= qemu_ram_addr_from_host(buffer
, &addr1
);
3020 invalidate_and_set_dirty(mr
, addr1
, access_len
);
3022 if (xen_enabled()) {
3023 xen_invalidate_map_cache_entry(buffer
);
3025 memory_region_unref(mr
);
3029 address_space_write(as
, bounce
.addr
, MEMTXATTRS_UNSPECIFIED
,
3030 bounce
.buffer
, access_len
);
3032 qemu_vfree(bounce
.buffer
);
3033 bounce
.buffer
= NULL
;
3034 memory_region_unref(bounce
.mr
);
3035 atomic_mb_set(&bounce
.in_use
, false);
3036 cpu_notify_map_clients();
3039 void *cpu_physical_memory_map(hwaddr addr
,
3043 return address_space_map(&address_space_memory
, addr
, plen
, is_write
);
3046 void cpu_physical_memory_unmap(void *buffer
, hwaddr len
,
3047 int is_write
, hwaddr access_len
)
3049 return address_space_unmap(&address_space_memory
, buffer
, len
, is_write
, access_len
);
3052 /* warning: addr must be aligned */
3053 static inline uint32_t address_space_ldl_internal(AddressSpace
*as
, hwaddr addr
,
3055 MemTxResult
*result
,
3056 enum device_endian endian
)
3064 bool release_lock
= false;
3067 mr
= address_space_translate(as
, addr
, &addr1
, &l
, false);
3068 if (l
< 4 || !memory_access_is_direct(mr
, false)) {
3069 release_lock
|= prepare_mmio_access(mr
);
3072 r
= memory_region_dispatch_read(mr
, addr1
, &val
, 4, attrs
);
3073 #if defined(TARGET_WORDS_BIGENDIAN)
3074 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3078 if (endian
== DEVICE_BIG_ENDIAN
) {
3084 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
3088 case DEVICE_LITTLE_ENDIAN
:
3089 val
= ldl_le_p(ptr
);
3091 case DEVICE_BIG_ENDIAN
:
3092 val
= ldl_be_p(ptr
);
3104 qemu_mutex_unlock_iothread();
3110 uint32_t address_space_ldl(AddressSpace
*as
, hwaddr addr
,
3111 MemTxAttrs attrs
, MemTxResult
*result
)
3113 return address_space_ldl_internal(as
, addr
, attrs
, result
,
3114 DEVICE_NATIVE_ENDIAN
);
3117 uint32_t address_space_ldl_le(AddressSpace
*as
, hwaddr addr
,
3118 MemTxAttrs attrs
, MemTxResult
*result
)
3120 return address_space_ldl_internal(as
, addr
, attrs
, result
,
3121 DEVICE_LITTLE_ENDIAN
);
3124 uint32_t address_space_ldl_be(AddressSpace
*as
, hwaddr addr
,
3125 MemTxAttrs attrs
, MemTxResult
*result
)
3127 return address_space_ldl_internal(as
, addr
, attrs
, result
,
3131 uint32_t ldl_phys(AddressSpace
*as
, hwaddr addr
)
3133 return address_space_ldl(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3136 uint32_t ldl_le_phys(AddressSpace
*as
, hwaddr addr
)
3138 return address_space_ldl_le(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3141 uint32_t ldl_be_phys(AddressSpace
*as
, hwaddr addr
)
3143 return address_space_ldl_be(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3146 /* warning: addr must be aligned */
3147 static inline uint64_t address_space_ldq_internal(AddressSpace
*as
, hwaddr addr
,
3149 MemTxResult
*result
,
3150 enum device_endian endian
)
3158 bool release_lock
= false;
3161 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
3163 if (l
< 8 || !memory_access_is_direct(mr
, false)) {
3164 release_lock
|= prepare_mmio_access(mr
);
3167 r
= memory_region_dispatch_read(mr
, addr1
, &val
, 8, attrs
);
3168 #if defined(TARGET_WORDS_BIGENDIAN)
3169 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3173 if (endian
== DEVICE_BIG_ENDIAN
) {
3179 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
3183 case DEVICE_LITTLE_ENDIAN
:
3184 val
= ldq_le_p(ptr
);
3186 case DEVICE_BIG_ENDIAN
:
3187 val
= ldq_be_p(ptr
);
3199 qemu_mutex_unlock_iothread();
3205 uint64_t address_space_ldq(AddressSpace
*as
, hwaddr addr
,
3206 MemTxAttrs attrs
, MemTxResult
*result
)
3208 return address_space_ldq_internal(as
, addr
, attrs
, result
,
3209 DEVICE_NATIVE_ENDIAN
);
3212 uint64_t address_space_ldq_le(AddressSpace
*as
, hwaddr addr
,
3213 MemTxAttrs attrs
, MemTxResult
*result
)
3215 return address_space_ldq_internal(as
, addr
, attrs
, result
,
3216 DEVICE_LITTLE_ENDIAN
);
3219 uint64_t address_space_ldq_be(AddressSpace
*as
, hwaddr addr
,
3220 MemTxAttrs attrs
, MemTxResult
*result
)
3222 return address_space_ldq_internal(as
, addr
, attrs
, result
,
3226 uint64_t ldq_phys(AddressSpace
*as
, hwaddr addr
)
3228 return address_space_ldq(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3231 uint64_t ldq_le_phys(AddressSpace
*as
, hwaddr addr
)
3233 return address_space_ldq_le(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3236 uint64_t ldq_be_phys(AddressSpace
*as
, hwaddr addr
)
3238 return address_space_ldq_be(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3242 uint32_t address_space_ldub(AddressSpace
*as
, hwaddr addr
,
3243 MemTxAttrs attrs
, MemTxResult
*result
)
3248 r
= address_space_rw(as
, addr
, attrs
, &val
, 1, 0);
3255 uint32_t ldub_phys(AddressSpace
*as
, hwaddr addr
)
3257 return address_space_ldub(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3260 /* warning: addr must be aligned */
3261 static inline uint32_t address_space_lduw_internal(AddressSpace
*as
,
3264 MemTxResult
*result
,
3265 enum device_endian endian
)
3273 bool release_lock
= false;
3276 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
3278 if (l
< 2 || !memory_access_is_direct(mr
, false)) {
3279 release_lock
|= prepare_mmio_access(mr
);
3282 r
= memory_region_dispatch_read(mr
, addr1
, &val
, 2, attrs
);
3283 #if defined(TARGET_WORDS_BIGENDIAN)
3284 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3288 if (endian
== DEVICE_BIG_ENDIAN
) {
3294 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
3298 case DEVICE_LITTLE_ENDIAN
:
3299 val
= lduw_le_p(ptr
);
3301 case DEVICE_BIG_ENDIAN
:
3302 val
= lduw_be_p(ptr
);
3314 qemu_mutex_unlock_iothread();
3320 uint32_t address_space_lduw(AddressSpace
*as
, hwaddr addr
,
3321 MemTxAttrs attrs
, MemTxResult
*result
)
3323 return address_space_lduw_internal(as
, addr
, attrs
, result
,
3324 DEVICE_NATIVE_ENDIAN
);
3327 uint32_t address_space_lduw_le(AddressSpace
*as
, hwaddr addr
,
3328 MemTxAttrs attrs
, MemTxResult
*result
)
3330 return address_space_lduw_internal(as
, addr
, attrs
, result
,
3331 DEVICE_LITTLE_ENDIAN
);
3334 uint32_t address_space_lduw_be(AddressSpace
*as
, hwaddr addr
,
3335 MemTxAttrs attrs
, MemTxResult
*result
)
3337 return address_space_lduw_internal(as
, addr
, attrs
, result
,
3341 uint32_t lduw_phys(AddressSpace
*as
, hwaddr addr
)
3343 return address_space_lduw(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3346 uint32_t lduw_le_phys(AddressSpace
*as
, hwaddr addr
)
3348 return address_space_lduw_le(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3351 uint32_t lduw_be_phys(AddressSpace
*as
, hwaddr addr
)
3353 return address_space_lduw_be(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3356 /* warning: addr must be aligned. The ram page is not masked as dirty
3357 and the code inside is not invalidated. It is useful if the dirty
3358 bits are used to track modified PTEs */
3359 void address_space_stl_notdirty(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3360 MemTxAttrs attrs
, MemTxResult
*result
)
3367 uint8_t dirty_log_mask
;
3368 bool release_lock
= false;
3371 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
3373 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
3374 release_lock
|= prepare_mmio_access(mr
);
3376 r
= memory_region_dispatch_write(mr
, addr1
, val
, 4, attrs
);
3378 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
3379 ptr
= qemu_get_ram_ptr(addr1
);
3382 dirty_log_mask
= memory_region_get_dirty_log_mask(mr
);
3383 dirty_log_mask
&= ~(1 << DIRTY_MEMORY_CODE
);
3384 cpu_physical_memory_set_dirty_range(addr1
, 4, dirty_log_mask
);
3391 qemu_mutex_unlock_iothread();
3396 void stl_phys_notdirty(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3398 address_space_stl_notdirty(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3401 /* warning: addr must be aligned */
3402 static inline void address_space_stl_internal(AddressSpace
*as
,
3403 hwaddr addr
, uint32_t val
,
3405 MemTxResult
*result
,
3406 enum device_endian endian
)
3413 bool release_lock
= false;
3416 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
3418 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
3419 release_lock
|= prepare_mmio_access(mr
);
3421 #if defined(TARGET_WORDS_BIGENDIAN)
3422 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3426 if (endian
== DEVICE_BIG_ENDIAN
) {
3430 r
= memory_region_dispatch_write(mr
, addr1
, val
, 4, attrs
);
3433 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
3434 ptr
= qemu_get_ram_ptr(addr1
);
3436 case DEVICE_LITTLE_ENDIAN
:
3439 case DEVICE_BIG_ENDIAN
:
3446 invalidate_and_set_dirty(mr
, addr1
, 4);
3453 qemu_mutex_unlock_iothread();
3458 void address_space_stl(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3459 MemTxAttrs attrs
, MemTxResult
*result
)
3461 address_space_stl_internal(as
, addr
, val
, attrs
, result
,
3462 DEVICE_NATIVE_ENDIAN
);
3465 void address_space_stl_le(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3466 MemTxAttrs attrs
, MemTxResult
*result
)
3468 address_space_stl_internal(as
, addr
, val
, attrs
, result
,
3469 DEVICE_LITTLE_ENDIAN
);
3472 void address_space_stl_be(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3473 MemTxAttrs attrs
, MemTxResult
*result
)
3475 address_space_stl_internal(as
, addr
, val
, attrs
, result
,
3479 void stl_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3481 address_space_stl(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3484 void stl_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3486 address_space_stl_le(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3489 void stl_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3491 address_space_stl_be(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3495 void address_space_stb(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3496 MemTxAttrs attrs
, MemTxResult
*result
)
3501 r
= address_space_rw(as
, addr
, attrs
, &v
, 1, 1);
3507 void stb_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3509 address_space_stb(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3512 /* warning: addr must be aligned */
3513 static inline void address_space_stw_internal(AddressSpace
*as
,
3514 hwaddr addr
, uint32_t val
,
3516 MemTxResult
*result
,
3517 enum device_endian endian
)
3524 bool release_lock
= false;
3527 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
3528 if (l
< 2 || !memory_access_is_direct(mr
, true)) {
3529 release_lock
|= prepare_mmio_access(mr
);
3531 #if defined(TARGET_WORDS_BIGENDIAN)
3532 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3536 if (endian
== DEVICE_BIG_ENDIAN
) {
3540 r
= memory_region_dispatch_write(mr
, addr1
, val
, 2, attrs
);
3543 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
3544 ptr
= qemu_get_ram_ptr(addr1
);
3546 case DEVICE_LITTLE_ENDIAN
:
3549 case DEVICE_BIG_ENDIAN
:
3556 invalidate_and_set_dirty(mr
, addr1
, 2);
3563 qemu_mutex_unlock_iothread();
3568 void address_space_stw(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3569 MemTxAttrs attrs
, MemTxResult
*result
)
3571 address_space_stw_internal(as
, addr
, val
, attrs
, result
,
3572 DEVICE_NATIVE_ENDIAN
);
3575 void address_space_stw_le(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3576 MemTxAttrs attrs
, MemTxResult
*result
)
3578 address_space_stw_internal(as
, addr
, val
, attrs
, result
,
3579 DEVICE_LITTLE_ENDIAN
);
3582 void address_space_stw_be(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3583 MemTxAttrs attrs
, MemTxResult
*result
)
3585 address_space_stw_internal(as
, addr
, val
, attrs
, result
,
3589 void stw_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3591 address_space_stw(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3594 void stw_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3596 address_space_stw_le(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3599 void stw_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3601 address_space_stw_be(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3605 void address_space_stq(AddressSpace
*as
, hwaddr addr
, uint64_t val
,
3606 MemTxAttrs attrs
, MemTxResult
*result
)
3610 r
= address_space_rw(as
, addr
, attrs
, (void *) &val
, 8, 1);
3616 void address_space_stq_le(AddressSpace
*as
, hwaddr addr
, uint64_t val
,
3617 MemTxAttrs attrs
, MemTxResult
*result
)
3620 val
= cpu_to_le64(val
);
3621 r
= address_space_rw(as
, addr
, attrs
, (void *) &val
, 8, 1);
3626 void address_space_stq_be(AddressSpace
*as
, hwaddr addr
, uint64_t val
,
3627 MemTxAttrs attrs
, MemTxResult
*result
)
3630 val
= cpu_to_be64(val
);
3631 r
= address_space_rw(as
, addr
, attrs
, (void *) &val
, 8, 1);
3637 void stq_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3639 address_space_stq(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3642 void stq_le_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3644 address_space_stq_le(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3647 void stq_be_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3649 address_space_stq_be(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3652 /* virtual memory access for debug (includes writing to ROM) */
3653 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
3654 uint8_t *buf
, int len
, int is_write
)
3664 page
= addr
& TARGET_PAGE_MASK
;
3665 phys_addr
= cpu_get_phys_page_attrs_debug(cpu
, page
, &attrs
);
3666 asidx
= cpu_asidx_from_attrs(cpu
, attrs
);
3667 /* if no physical page mapped, return an error */
3668 if (phys_addr
== -1)
3670 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3673 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
3675 cpu_physical_memory_write_rom(cpu
->cpu_ases
[asidx
].as
,
3678 address_space_rw(cpu
->cpu_ases
[asidx
].as
, phys_addr
,
3679 MEMTXATTRS_UNSPECIFIED
,
3690 * Allows code that needs to deal with migration bitmaps etc to still be built
3691 * target independent.
3693 size_t qemu_target_page_bits(void)
3695 return TARGET_PAGE_BITS
;
3701 * A helper function for the _utterly broken_ virtio device model to find out if
3702 * it's running on a big endian machine. Don't do this at home kids!
3704 bool target_words_bigendian(void);
3705 bool target_words_bigendian(void)
3707 #if defined(TARGET_WORDS_BIGENDIAN)
3714 #ifndef CONFIG_USER_ONLY
3715 bool cpu_physical_memory_is_io(hwaddr phys_addr
)
3722 mr
= address_space_translate(&address_space_memory
,
3723 phys_addr
, &phys_addr
, &l
, false);
3725 res
= !(memory_region_is_ram(mr
) || memory_region_is_romd(mr
));
3730 int qemu_ram_foreach_block(RAMBlockIterFunc func
, void *opaque
)
3736 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
3737 ret
= func(block
->idstr
, block
->host
, block
->offset
,
3738 block
->used_length
, opaque
);