4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "qemu/osdep.h"
24 #include "qemu-common.h"
28 #if !defined(CONFIG_USER_ONLY)
29 #include "hw/boards.h"
32 #include "sysemu/kvm.h"
33 #include "sysemu/sysemu.h"
34 #include "hw/xen/xen.h"
35 #include "qemu/timer.h"
36 #include "qemu/config-file.h"
37 #include "qemu/error-report.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
47 #include "exec/cpu-all.h"
48 #include "qemu/rcu_queue.h"
49 #include "qemu/main-loop.h"
50 #include "translate-all.h"
51 #include "sysemu/replay.h"
53 #include "exec/memory-internal.h"
54 #include "exec/ram_addr.h"
57 #include "qemu/range.h"
59 #include "qemu/mmap-alloc.h"
62 //#define DEBUG_SUBPAGE
64 #if !defined(CONFIG_USER_ONLY)
65 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
66 * are protected by the ramlist lock.
68 RAMList ram_list
= { .blocks
= QLIST_HEAD_INITIALIZER(ram_list
.blocks
) };
70 static MemoryRegion
*system_memory
;
71 static MemoryRegion
*system_io
;
73 AddressSpace address_space_io
;
74 AddressSpace address_space_memory
;
76 MemoryRegion io_mem_rom
, io_mem_notdirty
;
77 static MemoryRegion io_mem_unassigned
;
79 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
80 #define RAM_PREALLOC (1 << 0)
82 /* RAM is mmap-ed with MAP_SHARED */
83 #define RAM_SHARED (1 << 1)
85 /* Only a portion of RAM (used_length) is actually used, and migrated.
86 * This used_length size can change across reboots.
88 #define RAM_RESIZEABLE (1 << 2)
92 struct CPUTailQ cpus
= QTAILQ_HEAD_INITIALIZER(cpus
);
93 /* current CPU in the current thread. It is only valid inside
95 __thread CPUState
*current_cpu
;
96 /* 0 = Do not count executed instructions.
97 1 = Precise instruction counting.
98 2 = Adaptive rate instruction counting. */
101 #if !defined(CONFIG_USER_ONLY)
103 typedef struct PhysPageEntry PhysPageEntry
;
105 struct PhysPageEntry
{
106 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
108 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
112 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
114 /* Size of the L2 (and L3, etc) page tables. */
115 #define ADDR_SPACE_BITS 64
118 #define P_L2_SIZE (1 << P_L2_BITS)
120 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
122 typedef PhysPageEntry Node
[P_L2_SIZE
];
124 typedef struct PhysPageMap
{
127 unsigned sections_nb
;
128 unsigned sections_nb_alloc
;
130 unsigned nodes_nb_alloc
;
132 MemoryRegionSection
*sections
;
135 struct AddressSpaceDispatch
{
138 /* This is a multi-level map on the physical address space.
139 * The bottom level has pointers to MemoryRegionSections.
141 PhysPageEntry phys_map
;
146 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
147 typedef struct subpage_t
{
151 uint16_t sub_section
[TARGET_PAGE_SIZE
];
154 #define PHYS_SECTION_UNASSIGNED 0
155 #define PHYS_SECTION_NOTDIRTY 1
156 #define PHYS_SECTION_ROM 2
157 #define PHYS_SECTION_WATCH 3
159 static void io_mem_init(void);
160 static void memory_map_init(void);
161 static void tcg_commit(MemoryListener
*listener
);
163 static MemoryRegion io_mem_watch
;
166 * CPUAddressSpace: all the information a CPU needs about an AddressSpace
167 * @cpu: the CPU whose AddressSpace this is
168 * @as: the AddressSpace itself
169 * @memory_dispatch: its dispatch pointer (cached, RCU protected)
170 * @tcg_as_listener: listener for tracking changes to the AddressSpace
172 struct CPUAddressSpace
{
175 struct AddressSpaceDispatch
*memory_dispatch
;
176 MemoryListener tcg_as_listener
;
181 #if !defined(CONFIG_USER_ONLY)
183 static void phys_map_node_reserve(PhysPageMap
*map
, unsigned nodes
)
185 if (map
->nodes_nb
+ nodes
> map
->nodes_nb_alloc
) {
186 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
* 2, 16);
187 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
, map
->nodes_nb
+ nodes
);
188 map
->nodes
= g_renew(Node
, map
->nodes
, map
->nodes_nb_alloc
);
192 static uint32_t phys_map_node_alloc(PhysPageMap
*map
, bool leaf
)
199 ret
= map
->nodes_nb
++;
201 assert(ret
!= PHYS_MAP_NODE_NIL
);
202 assert(ret
!= map
->nodes_nb_alloc
);
204 e
.skip
= leaf
? 0 : 1;
205 e
.ptr
= leaf
? PHYS_SECTION_UNASSIGNED
: PHYS_MAP_NODE_NIL
;
206 for (i
= 0; i
< P_L2_SIZE
; ++i
) {
207 memcpy(&p
[i
], &e
, sizeof(e
));
212 static void phys_page_set_level(PhysPageMap
*map
, PhysPageEntry
*lp
,
213 hwaddr
*index
, hwaddr
*nb
, uint16_t leaf
,
217 hwaddr step
= (hwaddr
)1 << (level
* P_L2_BITS
);
219 if (lp
->skip
&& lp
->ptr
== PHYS_MAP_NODE_NIL
) {
220 lp
->ptr
= phys_map_node_alloc(map
, level
== 0);
222 p
= map
->nodes
[lp
->ptr
];
223 lp
= &p
[(*index
>> (level
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
225 while (*nb
&& lp
< &p
[P_L2_SIZE
]) {
226 if ((*index
& (step
- 1)) == 0 && *nb
>= step
) {
232 phys_page_set_level(map
, lp
, index
, nb
, leaf
, level
- 1);
238 static void phys_page_set(AddressSpaceDispatch
*d
,
239 hwaddr index
, hwaddr nb
,
242 /* Wildly overreserve - it doesn't matter much. */
243 phys_map_node_reserve(&d
->map
, 3 * P_L2_LEVELS
);
245 phys_page_set_level(&d
->map
, &d
->phys_map
, &index
, &nb
, leaf
, P_L2_LEVELS
- 1);
248 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
249 * and update our entry so we can skip it and go directly to the destination.
251 static void phys_page_compact(PhysPageEntry
*lp
, Node
*nodes
, unsigned long *compacted
)
253 unsigned valid_ptr
= P_L2_SIZE
;
258 if (lp
->ptr
== PHYS_MAP_NODE_NIL
) {
263 for (i
= 0; i
< P_L2_SIZE
; i
++) {
264 if (p
[i
].ptr
== PHYS_MAP_NODE_NIL
) {
271 phys_page_compact(&p
[i
], nodes
, compacted
);
275 /* We can only compress if there's only one child. */
280 assert(valid_ptr
< P_L2_SIZE
);
282 /* Don't compress if it won't fit in the # of bits we have. */
283 if (lp
->skip
+ p
[valid_ptr
].skip
>= (1 << 3)) {
287 lp
->ptr
= p
[valid_ptr
].ptr
;
288 if (!p
[valid_ptr
].skip
) {
289 /* If our only child is a leaf, make this a leaf. */
290 /* By design, we should have made this node a leaf to begin with so we
291 * should never reach here.
292 * But since it's so simple to handle this, let's do it just in case we
297 lp
->skip
+= p
[valid_ptr
].skip
;
301 static void phys_page_compact_all(AddressSpaceDispatch
*d
, int nodes_nb
)
303 DECLARE_BITMAP(compacted
, nodes_nb
);
305 if (d
->phys_map
.skip
) {
306 phys_page_compact(&d
->phys_map
, d
->map
.nodes
, compacted
);
310 static MemoryRegionSection
*phys_page_find(PhysPageEntry lp
, hwaddr addr
,
311 Node
*nodes
, MemoryRegionSection
*sections
)
314 hwaddr index
= addr
>> TARGET_PAGE_BITS
;
317 for (i
= P_L2_LEVELS
; lp
.skip
&& (i
-= lp
.skip
) >= 0;) {
318 if (lp
.ptr
== PHYS_MAP_NODE_NIL
) {
319 return §ions
[PHYS_SECTION_UNASSIGNED
];
322 lp
= p
[(index
>> (i
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
325 if (sections
[lp
.ptr
].size
.hi
||
326 range_covers_byte(sections
[lp
.ptr
].offset_within_address_space
,
327 sections
[lp
.ptr
].size
.lo
, addr
)) {
328 return §ions
[lp
.ptr
];
330 return §ions
[PHYS_SECTION_UNASSIGNED
];
334 bool memory_region_is_unassigned(MemoryRegion
*mr
)
336 return mr
!= &io_mem_rom
&& mr
!= &io_mem_notdirty
&& !mr
->rom_device
337 && mr
!= &io_mem_watch
;
340 /* Called from RCU critical section */
341 static MemoryRegionSection
*address_space_lookup_region(AddressSpaceDispatch
*d
,
343 bool resolve_subpage
)
345 MemoryRegionSection
*section
;
348 section
= phys_page_find(d
->phys_map
, addr
, d
->map
.nodes
, d
->map
.sections
);
349 if (resolve_subpage
&& section
->mr
->subpage
) {
350 subpage
= container_of(section
->mr
, subpage_t
, iomem
);
351 section
= &d
->map
.sections
[subpage
->sub_section
[SUBPAGE_IDX(addr
)]];
356 /* Called from RCU critical section */
357 static MemoryRegionSection
*
358 address_space_translate_internal(AddressSpaceDispatch
*d
, hwaddr addr
, hwaddr
*xlat
,
359 hwaddr
*plen
, bool resolve_subpage
)
361 MemoryRegionSection
*section
;
365 section
= address_space_lookup_region(d
, addr
, resolve_subpage
);
366 /* Compute offset within MemoryRegionSection */
367 addr
-= section
->offset_within_address_space
;
369 /* Compute offset within MemoryRegion */
370 *xlat
= addr
+ section
->offset_within_region
;
374 /* MMIO registers can be expected to perform full-width accesses based only
375 * on their address, without considering adjacent registers that could
376 * decode to completely different MemoryRegions. When such registers
377 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
378 * regions overlap wildly. For this reason we cannot clamp the accesses
381 * If the length is small (as is the case for address_space_ldl/stl),
382 * everything works fine. If the incoming length is large, however,
383 * the caller really has to do the clamping through memory_access_size.
385 if (memory_region_is_ram(mr
)) {
386 diff
= int128_sub(section
->size
, int128_make64(addr
));
387 *plen
= int128_get64(int128_min(diff
, int128_make64(*plen
)));
392 /* Called from RCU critical section */
393 MemoryRegion
*address_space_translate(AddressSpace
*as
, hwaddr addr
,
394 hwaddr
*xlat
, hwaddr
*plen
,
398 MemoryRegionSection
*section
;
402 AddressSpaceDispatch
*d
= atomic_rcu_read(&as
->dispatch
);
403 section
= address_space_translate_internal(d
, addr
, &addr
, plen
, true);
406 if (!mr
->iommu_ops
) {
410 iotlb
= mr
->iommu_ops
->translate(mr
, addr
, is_write
);
411 addr
= ((iotlb
.translated_addr
& ~iotlb
.addr_mask
)
412 | (addr
& iotlb
.addr_mask
));
413 *plen
= MIN(*plen
, (addr
| iotlb
.addr_mask
) - addr
+ 1);
414 if (!(iotlb
.perm
& (1 << is_write
))) {
415 mr
= &io_mem_unassigned
;
419 as
= iotlb
.target_as
;
422 if (xen_enabled() && memory_access_is_direct(mr
, is_write
)) {
423 hwaddr page
= ((addr
& TARGET_PAGE_MASK
) + TARGET_PAGE_SIZE
) - addr
;
424 *plen
= MIN(page
, *plen
);
431 /* Called from RCU critical section */
432 MemoryRegionSection
*
433 address_space_translate_for_iotlb(CPUState
*cpu
, int asidx
, hwaddr addr
,
434 hwaddr
*xlat
, hwaddr
*plen
)
436 MemoryRegionSection
*section
;
437 AddressSpaceDispatch
*d
= cpu
->cpu_ases
[asidx
].memory_dispatch
;
439 section
= address_space_translate_internal(d
, addr
, xlat
, plen
, false);
441 assert(!section
->mr
->iommu_ops
);
446 #if !defined(CONFIG_USER_ONLY)
448 static int cpu_common_post_load(void *opaque
, int version_id
)
450 CPUState
*cpu
= opaque
;
452 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
453 version_id is increased. */
454 cpu
->interrupt_request
&= ~0x01;
460 static int cpu_common_pre_load(void *opaque
)
462 CPUState
*cpu
= opaque
;
464 cpu
->exception_index
= -1;
469 static bool cpu_common_exception_index_needed(void *opaque
)
471 CPUState
*cpu
= opaque
;
473 return tcg_enabled() && cpu
->exception_index
!= -1;
476 static const VMStateDescription vmstate_cpu_common_exception_index
= {
477 .name
= "cpu_common/exception_index",
479 .minimum_version_id
= 1,
480 .needed
= cpu_common_exception_index_needed
,
481 .fields
= (VMStateField
[]) {
482 VMSTATE_INT32(exception_index
, CPUState
),
483 VMSTATE_END_OF_LIST()
487 static bool cpu_common_crash_occurred_needed(void *opaque
)
489 CPUState
*cpu
= opaque
;
491 return cpu
->crash_occurred
;
494 static const VMStateDescription vmstate_cpu_common_crash_occurred
= {
495 .name
= "cpu_common/crash_occurred",
497 .minimum_version_id
= 1,
498 .needed
= cpu_common_crash_occurred_needed
,
499 .fields
= (VMStateField
[]) {
500 VMSTATE_BOOL(crash_occurred
, CPUState
),
501 VMSTATE_END_OF_LIST()
505 const VMStateDescription vmstate_cpu_common
= {
506 .name
= "cpu_common",
508 .minimum_version_id
= 1,
509 .pre_load
= cpu_common_pre_load
,
510 .post_load
= cpu_common_post_load
,
511 .fields
= (VMStateField
[]) {
512 VMSTATE_UINT32(halted
, CPUState
),
513 VMSTATE_UINT32(interrupt_request
, CPUState
),
514 VMSTATE_END_OF_LIST()
516 .subsections
= (const VMStateDescription
*[]) {
517 &vmstate_cpu_common_exception_index
,
518 &vmstate_cpu_common_crash_occurred
,
525 CPUState
*qemu_get_cpu(int index
)
530 if (cpu
->cpu_index
== index
) {
538 #if !defined(CONFIG_USER_ONLY)
539 void cpu_address_space_init(CPUState
*cpu
, AddressSpace
*as
, int asidx
)
541 CPUAddressSpace
*newas
;
543 /* Target code should have set num_ases before calling us */
544 assert(asidx
< cpu
->num_ases
);
547 /* address space 0 gets the convenience alias */
551 /* KVM cannot currently support multiple address spaces. */
552 assert(asidx
== 0 || !kvm_enabled());
554 if (!cpu
->cpu_ases
) {
555 cpu
->cpu_ases
= g_new0(CPUAddressSpace
, cpu
->num_ases
);
558 newas
= &cpu
->cpu_ases
[asidx
];
562 newas
->tcg_as_listener
.commit
= tcg_commit
;
563 memory_listener_register(&newas
->tcg_as_listener
, as
);
567 AddressSpace
*cpu_get_address_space(CPUState
*cpu
, int asidx
)
569 /* Return the AddressSpace corresponding to the specified index */
570 return cpu
->cpu_ases
[asidx
].as
;
574 #ifndef CONFIG_USER_ONLY
575 static DECLARE_BITMAP(cpu_index_map
, MAX_CPUMASK_BITS
);
577 static int cpu_get_free_index(Error
**errp
)
579 int cpu
= find_first_zero_bit(cpu_index_map
, MAX_CPUMASK_BITS
);
581 if (cpu
>= MAX_CPUMASK_BITS
) {
582 error_setg(errp
, "Trying to use more CPUs than max of %d",
587 bitmap_set(cpu_index_map
, cpu
, 1);
591 void cpu_exec_exit(CPUState
*cpu
)
593 if (cpu
->cpu_index
== -1) {
594 /* cpu_index was never allocated by this @cpu or was already freed. */
598 bitmap_clear(cpu_index_map
, cpu
->cpu_index
, 1);
603 static int cpu_get_free_index(Error
**errp
)
608 CPU_FOREACH(some_cpu
) {
614 void cpu_exec_exit(CPUState
*cpu
)
619 void cpu_exec_init(CPUState
*cpu
, Error
**errp
)
621 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
623 Error
*local_err
= NULL
;
628 #ifndef CONFIG_USER_ONLY
629 cpu
->thread_id
= qemu_get_thread_id();
631 /* This is a softmmu CPU object, so create a property for it
632 * so users can wire up its memory. (This can't go in qom/cpu.c
633 * because that file is compiled only once for both user-mode
634 * and system builds.) The default if no link is set up is to use
635 * the system address space.
637 object_property_add_link(OBJECT(cpu
), "memory", TYPE_MEMORY_REGION
,
638 (Object
**)&cpu
->memory
,
639 qdev_prop_allow_set_link_before_realize
,
640 OBJ_PROP_LINK_UNREF_ON_RELEASE
,
642 cpu
->memory
= system_memory
;
643 object_ref(OBJECT(cpu
->memory
));
646 #if defined(CONFIG_USER_ONLY)
649 cpu_index
= cpu
->cpu_index
= cpu_get_free_index(&local_err
);
651 error_propagate(errp
, local_err
);
652 #if defined(CONFIG_USER_ONLY)
657 QTAILQ_INSERT_TAIL(&cpus
, cpu
, node
);
658 #if defined(CONFIG_USER_ONLY)
661 if (qdev_get_vmsd(DEVICE(cpu
)) == NULL
) {
662 vmstate_register(NULL
, cpu_index
, &vmstate_cpu_common
, cpu
);
664 if (cc
->vmsd
!= NULL
) {
665 vmstate_register(NULL
, cpu_index
, cc
->vmsd
, cpu
);
669 #if defined(CONFIG_USER_ONLY)
670 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
672 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
675 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
678 hwaddr phys
= cpu_get_phys_page_attrs_debug(cpu
, pc
, &attrs
);
679 int asidx
= cpu_asidx_from_attrs(cpu
, attrs
);
681 tb_invalidate_phys_addr(cpu
->cpu_ases
[asidx
].as
,
682 phys
| (pc
& ~TARGET_PAGE_MASK
));
687 #if defined(CONFIG_USER_ONLY)
688 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
693 int cpu_watchpoint_remove(CPUState
*cpu
, vaddr addr
, vaddr len
,
699 void cpu_watchpoint_remove_by_ref(CPUState
*cpu
, CPUWatchpoint
*watchpoint
)
703 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
704 int flags
, CPUWatchpoint
**watchpoint
)
709 /* Add a watchpoint. */
710 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
711 int flags
, CPUWatchpoint
**watchpoint
)
715 /* forbid ranges which are empty or run off the end of the address space */
716 if (len
== 0 || (addr
+ len
- 1) < addr
) {
717 error_report("tried to set invalid watchpoint at %"
718 VADDR_PRIx
", len=%" VADDR_PRIu
, addr
, len
);
721 wp
= g_malloc(sizeof(*wp
));
727 /* keep all GDB-injected watchpoints in front */
728 if (flags
& BP_GDB
) {
729 QTAILQ_INSERT_HEAD(&cpu
->watchpoints
, wp
, entry
);
731 QTAILQ_INSERT_TAIL(&cpu
->watchpoints
, wp
, entry
);
734 tlb_flush_page(cpu
, addr
);
741 /* Remove a specific watchpoint. */
742 int cpu_watchpoint_remove(CPUState
*cpu
, vaddr addr
, vaddr len
,
747 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
748 if (addr
== wp
->vaddr
&& len
== wp
->len
749 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
750 cpu_watchpoint_remove_by_ref(cpu
, wp
);
757 /* Remove a specific watchpoint by reference. */
758 void cpu_watchpoint_remove_by_ref(CPUState
*cpu
, CPUWatchpoint
*watchpoint
)
760 QTAILQ_REMOVE(&cpu
->watchpoints
, watchpoint
, entry
);
762 tlb_flush_page(cpu
, watchpoint
->vaddr
);
767 /* Remove all matching watchpoints. */
768 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
770 CPUWatchpoint
*wp
, *next
;
772 QTAILQ_FOREACH_SAFE(wp
, &cpu
->watchpoints
, entry
, next
) {
773 if (wp
->flags
& mask
) {
774 cpu_watchpoint_remove_by_ref(cpu
, wp
);
779 /* Return true if this watchpoint address matches the specified
780 * access (ie the address range covered by the watchpoint overlaps
781 * partially or completely with the address range covered by the
784 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint
*wp
,
788 /* We know the lengths are non-zero, but a little caution is
789 * required to avoid errors in the case where the range ends
790 * exactly at the top of the address space and so addr + len
791 * wraps round to zero.
793 vaddr wpend
= wp
->vaddr
+ wp
->len
- 1;
794 vaddr addrend
= addr
+ len
- 1;
796 return !(addr
> wpend
|| wp
->vaddr
> addrend
);
801 /* Add a breakpoint. */
802 int cpu_breakpoint_insert(CPUState
*cpu
, vaddr pc
, int flags
,
803 CPUBreakpoint
**breakpoint
)
807 bp
= g_malloc(sizeof(*bp
));
812 /* keep all GDB-injected breakpoints in front */
813 if (flags
& BP_GDB
) {
814 QTAILQ_INSERT_HEAD(&cpu
->breakpoints
, bp
, entry
);
816 QTAILQ_INSERT_TAIL(&cpu
->breakpoints
, bp
, entry
);
819 breakpoint_invalidate(cpu
, pc
);
827 /* Remove a specific breakpoint. */
828 int cpu_breakpoint_remove(CPUState
*cpu
, vaddr pc
, int flags
)
832 QTAILQ_FOREACH(bp
, &cpu
->breakpoints
, entry
) {
833 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
834 cpu_breakpoint_remove_by_ref(cpu
, bp
);
841 /* Remove a specific breakpoint by reference. */
842 void cpu_breakpoint_remove_by_ref(CPUState
*cpu
, CPUBreakpoint
*breakpoint
)
844 QTAILQ_REMOVE(&cpu
->breakpoints
, breakpoint
, entry
);
846 breakpoint_invalidate(cpu
, breakpoint
->pc
);
851 /* Remove all matching breakpoints. */
852 void cpu_breakpoint_remove_all(CPUState
*cpu
, int mask
)
854 CPUBreakpoint
*bp
, *next
;
856 QTAILQ_FOREACH_SAFE(bp
, &cpu
->breakpoints
, entry
, next
) {
857 if (bp
->flags
& mask
) {
858 cpu_breakpoint_remove_by_ref(cpu
, bp
);
863 /* enable or disable single step mode. EXCP_DEBUG is returned by the
864 CPU loop after each instruction */
865 void cpu_single_step(CPUState
*cpu
, int enabled
)
867 if (cpu
->singlestep_enabled
!= enabled
) {
868 cpu
->singlestep_enabled
= enabled
;
870 kvm_update_guest_debug(cpu
, 0);
872 /* must flush all the translated code to avoid inconsistencies */
873 /* XXX: only flush what is necessary */
879 void cpu_abort(CPUState
*cpu
, const char *fmt
, ...)
886 fprintf(stderr
, "qemu: fatal: ");
887 vfprintf(stderr
, fmt
, ap
);
888 fprintf(stderr
, "\n");
889 cpu_dump_state(cpu
, stderr
, fprintf
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
890 if (qemu_log_separate()) {
891 qemu_log("qemu: fatal: ");
892 qemu_log_vprintf(fmt
, ap2
);
894 log_cpu_state(cpu
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
901 #if defined(CONFIG_USER_ONLY)
903 struct sigaction act
;
904 sigfillset(&act
.sa_mask
);
905 act
.sa_handler
= SIG_DFL
;
906 sigaction(SIGABRT
, &act
, NULL
);
912 #if !defined(CONFIG_USER_ONLY)
913 /* Called from RCU critical section */
914 static RAMBlock
*qemu_get_ram_block(ram_addr_t addr
)
918 block
= atomic_rcu_read(&ram_list
.mru_block
);
919 if (block
&& addr
- block
->offset
< block
->max_length
) {
922 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
923 if (addr
- block
->offset
< block
->max_length
) {
928 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
932 /* It is safe to write mru_block outside the iothread lock. This
937 * xxx removed from list
941 * call_rcu(reclaim_ramblock, xxx);
944 * atomic_rcu_set is not needed here. The block was already published
945 * when it was placed into the list. Here we're just making an extra
946 * copy of the pointer.
948 ram_list
.mru_block
= block
;
952 static void tlb_reset_dirty_range_all(ram_addr_t start
, ram_addr_t length
)
959 end
= TARGET_PAGE_ALIGN(start
+ length
);
960 start
&= TARGET_PAGE_MASK
;
963 block
= qemu_get_ram_block(start
);
964 assert(block
== qemu_get_ram_block(end
- 1));
965 start1
= (uintptr_t)ramblock_ptr(block
, start
- block
->offset
);
967 tlb_reset_dirty(cpu
, start1
, length
);
972 /* Note: start and end must be within the same ram block. */
973 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start
,
977 DirtyMemoryBlocks
*blocks
;
978 unsigned long end
, page
;
985 end
= TARGET_PAGE_ALIGN(start
+ length
) >> TARGET_PAGE_BITS
;
986 page
= start
>> TARGET_PAGE_BITS
;
990 blocks
= atomic_rcu_read(&ram_list
.dirty_memory
[client
]);
993 unsigned long idx
= page
/ DIRTY_MEMORY_BLOCK_SIZE
;
994 unsigned long offset
= page
% DIRTY_MEMORY_BLOCK_SIZE
;
995 unsigned long num
= MIN(end
- page
, DIRTY_MEMORY_BLOCK_SIZE
- offset
);
997 dirty
|= bitmap_test_and_clear_atomic(blocks
->blocks
[idx
],
1004 if (dirty
&& tcg_enabled()) {
1005 tlb_reset_dirty_range_all(start
, length
);
1011 /* Called from RCU critical section */
1012 hwaddr
memory_region_section_get_iotlb(CPUState
*cpu
,
1013 MemoryRegionSection
*section
,
1015 hwaddr paddr
, hwaddr xlat
,
1017 target_ulong
*address
)
1022 if (memory_region_is_ram(section
->mr
)) {
1024 iotlb
= (memory_region_get_ram_addr(section
->mr
) & TARGET_PAGE_MASK
)
1026 if (!section
->readonly
) {
1027 iotlb
|= PHYS_SECTION_NOTDIRTY
;
1029 iotlb
|= PHYS_SECTION_ROM
;
1032 AddressSpaceDispatch
*d
;
1034 d
= atomic_rcu_read(§ion
->address_space
->dispatch
);
1035 iotlb
= section
- d
->map
.sections
;
1039 /* Make accesses to pages with watchpoints go via the
1040 watchpoint trap routines. */
1041 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
1042 if (cpu_watchpoint_address_matches(wp
, vaddr
, TARGET_PAGE_SIZE
)) {
1043 /* Avoid trapping reads of pages with a write breakpoint. */
1044 if ((prot
& PAGE_WRITE
) || (wp
->flags
& BP_MEM_READ
)) {
1045 iotlb
= PHYS_SECTION_WATCH
+ paddr
;
1046 *address
|= TLB_MMIO
;
1054 #endif /* defined(CONFIG_USER_ONLY) */
1056 #if !defined(CONFIG_USER_ONLY)
1058 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
1060 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
);
1062 static void *(*phys_mem_alloc
)(size_t size
, uint64_t *align
) =
1063 qemu_anon_ram_alloc
;
1066 * Set a custom physical guest memory alloator.
1067 * Accelerators with unusual needs may need this. Hopefully, we can
1068 * get rid of it eventually.
1070 void phys_mem_set_alloc(void *(*alloc
)(size_t, uint64_t *align
))
1072 phys_mem_alloc
= alloc
;
1075 static uint16_t phys_section_add(PhysPageMap
*map
,
1076 MemoryRegionSection
*section
)
1078 /* The physical section number is ORed with a page-aligned
1079 * pointer to produce the iotlb entries. Thus it should
1080 * never overflow into the page-aligned value.
1082 assert(map
->sections_nb
< TARGET_PAGE_SIZE
);
1084 if (map
->sections_nb
== map
->sections_nb_alloc
) {
1085 map
->sections_nb_alloc
= MAX(map
->sections_nb_alloc
* 2, 16);
1086 map
->sections
= g_renew(MemoryRegionSection
, map
->sections
,
1087 map
->sections_nb_alloc
);
1089 map
->sections
[map
->sections_nb
] = *section
;
1090 memory_region_ref(section
->mr
);
1091 return map
->sections_nb
++;
1094 static void phys_section_destroy(MemoryRegion
*mr
)
1096 bool have_sub_page
= mr
->subpage
;
1098 memory_region_unref(mr
);
1100 if (have_sub_page
) {
1101 subpage_t
*subpage
= container_of(mr
, subpage_t
, iomem
);
1102 object_unref(OBJECT(&subpage
->iomem
));
1107 static void phys_sections_free(PhysPageMap
*map
)
1109 while (map
->sections_nb
> 0) {
1110 MemoryRegionSection
*section
= &map
->sections
[--map
->sections_nb
];
1111 phys_section_destroy(section
->mr
);
1113 g_free(map
->sections
);
1117 static void register_subpage(AddressSpaceDispatch
*d
, MemoryRegionSection
*section
)
1120 hwaddr base
= section
->offset_within_address_space
1122 MemoryRegionSection
*existing
= phys_page_find(d
->phys_map
, base
,
1123 d
->map
.nodes
, d
->map
.sections
);
1124 MemoryRegionSection subsection
= {
1125 .offset_within_address_space
= base
,
1126 .size
= int128_make64(TARGET_PAGE_SIZE
),
1130 assert(existing
->mr
->subpage
|| existing
->mr
== &io_mem_unassigned
);
1132 if (!(existing
->mr
->subpage
)) {
1133 subpage
= subpage_init(d
->as
, base
);
1134 subsection
.address_space
= d
->as
;
1135 subsection
.mr
= &subpage
->iomem
;
1136 phys_page_set(d
, base
>> TARGET_PAGE_BITS
, 1,
1137 phys_section_add(&d
->map
, &subsection
));
1139 subpage
= container_of(existing
->mr
, subpage_t
, iomem
);
1141 start
= section
->offset_within_address_space
& ~TARGET_PAGE_MASK
;
1142 end
= start
+ int128_get64(section
->size
) - 1;
1143 subpage_register(subpage
, start
, end
,
1144 phys_section_add(&d
->map
, section
));
1148 static void register_multipage(AddressSpaceDispatch
*d
,
1149 MemoryRegionSection
*section
)
1151 hwaddr start_addr
= section
->offset_within_address_space
;
1152 uint16_t section_index
= phys_section_add(&d
->map
, section
);
1153 uint64_t num_pages
= int128_get64(int128_rshift(section
->size
,
1157 phys_page_set(d
, start_addr
>> TARGET_PAGE_BITS
, num_pages
, section_index
);
1160 static void mem_add(MemoryListener
*listener
, MemoryRegionSection
*section
)
1162 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
1163 AddressSpaceDispatch
*d
= as
->next_dispatch
;
1164 MemoryRegionSection now
= *section
, remain
= *section
;
1165 Int128 page_size
= int128_make64(TARGET_PAGE_SIZE
);
1167 if (now
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
1168 uint64_t left
= TARGET_PAGE_ALIGN(now
.offset_within_address_space
)
1169 - now
.offset_within_address_space
;
1171 now
.size
= int128_min(int128_make64(left
), now
.size
);
1172 register_subpage(d
, &now
);
1174 now
.size
= int128_zero();
1176 while (int128_ne(remain
.size
, now
.size
)) {
1177 remain
.size
= int128_sub(remain
.size
, now
.size
);
1178 remain
.offset_within_address_space
+= int128_get64(now
.size
);
1179 remain
.offset_within_region
+= int128_get64(now
.size
);
1181 if (int128_lt(remain
.size
, page_size
)) {
1182 register_subpage(d
, &now
);
1183 } else if (remain
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
1184 now
.size
= page_size
;
1185 register_subpage(d
, &now
);
1187 now
.size
= int128_and(now
.size
, int128_neg(page_size
));
1188 register_multipage(d
, &now
);
1193 void qemu_flush_coalesced_mmio_buffer(void)
1196 kvm_flush_coalesced_mmio_buffer();
1199 void qemu_mutex_lock_ramlist(void)
1201 qemu_mutex_lock(&ram_list
.mutex
);
1204 void qemu_mutex_unlock_ramlist(void)
1206 qemu_mutex_unlock(&ram_list
.mutex
);
1211 #include <sys/vfs.h>
1213 #define HUGETLBFS_MAGIC 0x958458f6
1215 static long gethugepagesize(const char *path
, Error
**errp
)
1221 ret
= statfs(path
, &fs
);
1222 } while (ret
!= 0 && errno
== EINTR
);
1225 error_setg_errno(errp
, errno
, "failed to get page size of file %s",
1233 static void *file_ram_alloc(RAMBlock
*block
,
1240 char *sanitized_name
;
1245 Error
*local_err
= NULL
;
1247 hpagesize
= gethugepagesize(path
, &local_err
);
1249 error_propagate(errp
, local_err
);
1252 block
->mr
->align
= hpagesize
;
1254 if (memory
< hpagesize
) {
1255 error_setg(errp
, "memory size 0x" RAM_ADDR_FMT
" must be equal to "
1256 "or larger than huge page size 0x%" PRIx64
,
1261 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1263 "host lacks kvm mmu notifiers, -mem-path unsupported");
1267 if (!stat(path
, &st
) && S_ISDIR(st
.st_mode
)) {
1268 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1269 sanitized_name
= g_strdup(memory_region_name(block
->mr
));
1270 for (c
= sanitized_name
; *c
!= '\0'; c
++) {
1276 filename
= g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path
,
1278 g_free(sanitized_name
);
1280 fd
= mkstemp(filename
);
1286 fd
= open(path
, O_RDWR
| O_CREAT
, 0644);
1290 error_setg_errno(errp
, errno
,
1291 "unable to create backing store for hugepages");
1295 memory
= ROUND_UP(memory
, hpagesize
);
1298 * ftruncate is not supported by hugetlbfs in older
1299 * hosts, so don't bother bailing out on errors.
1300 * If anything goes wrong with it under other filesystems,
1303 if (ftruncate(fd
, memory
)) {
1304 perror("ftruncate");
1307 area
= qemu_ram_mmap(fd
, memory
, hpagesize
, block
->flags
& RAM_SHARED
);
1308 if (area
== MAP_FAILED
) {
1309 error_setg_errno(errp
, errno
,
1310 "unable to map backing store for hugepages");
1316 os_mem_prealloc(fd
, area
, memory
);
1327 /* Called with the ramlist lock held. */
1328 static ram_addr_t
find_ram_offset(ram_addr_t size
)
1330 RAMBlock
*block
, *next_block
;
1331 ram_addr_t offset
= RAM_ADDR_MAX
, mingap
= RAM_ADDR_MAX
;
1333 assert(size
!= 0); /* it would hand out same offset multiple times */
1335 if (QLIST_EMPTY_RCU(&ram_list
.blocks
)) {
1339 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1340 ram_addr_t end
, next
= RAM_ADDR_MAX
;
1342 end
= block
->offset
+ block
->max_length
;
1344 QLIST_FOREACH_RCU(next_block
, &ram_list
.blocks
, next
) {
1345 if (next_block
->offset
>= end
) {
1346 next
= MIN(next
, next_block
->offset
);
1349 if (next
- end
>= size
&& next
- end
< mingap
) {
1351 mingap
= next
- end
;
1355 if (offset
== RAM_ADDR_MAX
) {
1356 fprintf(stderr
, "Failed to find gap of requested size: %" PRIu64
"\n",
1364 ram_addr_t
last_ram_offset(void)
1367 ram_addr_t last
= 0;
1370 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1371 last
= MAX(last
, block
->offset
+ block
->max_length
);
1377 static void qemu_ram_setup_dump(void *addr
, ram_addr_t size
)
1381 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1382 if (!machine_dump_guest_core(current_machine
)) {
1383 ret
= qemu_madvise(addr
, size
, QEMU_MADV_DONTDUMP
);
1385 perror("qemu_madvise");
1386 fprintf(stderr
, "madvise doesn't support MADV_DONTDUMP, "
1387 "but dump_guest_core=off specified\n");
1392 /* Called within an RCU critical section, or while the ramlist lock
1395 static RAMBlock
*find_ram_block(ram_addr_t addr
)
1399 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1400 if (block
->offset
== addr
) {
1408 const char *qemu_ram_get_idstr(RAMBlock
*rb
)
1413 /* Called with iothread lock held. */
1414 void qemu_ram_set_idstr(ram_addr_t addr
, const char *name
, DeviceState
*dev
)
1416 RAMBlock
*new_block
, *block
;
1419 new_block
= find_ram_block(addr
);
1421 assert(!new_block
->idstr
[0]);
1424 char *id
= qdev_get_dev_path(dev
);
1426 snprintf(new_block
->idstr
, sizeof(new_block
->idstr
), "%s/", id
);
1430 pstrcat(new_block
->idstr
, sizeof(new_block
->idstr
), name
);
1432 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1433 if (block
!= new_block
&& !strcmp(block
->idstr
, new_block
->idstr
)) {
1434 fprintf(stderr
, "RAMBlock \"%s\" already registered, abort!\n",
1442 /* Called with iothread lock held. */
1443 void qemu_ram_unset_idstr(ram_addr_t addr
)
1447 /* FIXME: arch_init.c assumes that this is not called throughout
1448 * migration. Ignore the problem since hot-unplug during migration
1449 * does not work anyway.
1453 block
= find_ram_block(addr
);
1455 memset(block
->idstr
, 0, sizeof(block
->idstr
));
1460 static int memory_try_enable_merging(void *addr
, size_t len
)
1462 if (!machine_mem_merge(current_machine
)) {
1463 /* disabled by the user */
1467 return qemu_madvise(addr
, len
, QEMU_MADV_MERGEABLE
);
1470 /* Only legal before guest might have detected the memory size: e.g. on
1471 * incoming migration, or right after reset.
1473 * As memory core doesn't know how is memory accessed, it is up to
1474 * resize callback to update device state and/or add assertions to detect
1475 * misuse, if necessary.
1477 int qemu_ram_resize(ram_addr_t base
, ram_addr_t newsize
, Error
**errp
)
1479 RAMBlock
*block
= find_ram_block(base
);
1483 newsize
= HOST_PAGE_ALIGN(newsize
);
1485 if (block
->used_length
== newsize
) {
1489 if (!(block
->flags
& RAM_RESIZEABLE
)) {
1490 error_setg_errno(errp
, EINVAL
,
1491 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1492 " in != 0x" RAM_ADDR_FMT
, block
->idstr
,
1493 newsize
, block
->used_length
);
1497 if (block
->max_length
< newsize
) {
1498 error_setg_errno(errp
, EINVAL
,
1499 "Length too large: %s: 0x" RAM_ADDR_FMT
1500 " > 0x" RAM_ADDR_FMT
, block
->idstr
,
1501 newsize
, block
->max_length
);
1505 cpu_physical_memory_clear_dirty_range(block
->offset
, block
->used_length
);
1506 block
->used_length
= newsize
;
1507 cpu_physical_memory_set_dirty_range(block
->offset
, block
->used_length
,
1509 memory_region_set_size(block
->mr
, newsize
);
1510 if (block
->resized
) {
1511 block
->resized(block
->idstr
, newsize
, block
->host
);
1516 /* Called with ram_list.mutex held */
1517 static void dirty_memory_extend(ram_addr_t old_ram_size
,
1518 ram_addr_t new_ram_size
)
1520 ram_addr_t old_num_blocks
= DIV_ROUND_UP(old_ram_size
,
1521 DIRTY_MEMORY_BLOCK_SIZE
);
1522 ram_addr_t new_num_blocks
= DIV_ROUND_UP(new_ram_size
,
1523 DIRTY_MEMORY_BLOCK_SIZE
);
1526 /* Only need to extend if block count increased */
1527 if (new_num_blocks
<= old_num_blocks
) {
1531 for (i
= 0; i
< DIRTY_MEMORY_NUM
; i
++) {
1532 DirtyMemoryBlocks
*old_blocks
;
1533 DirtyMemoryBlocks
*new_blocks
;
1536 old_blocks
= atomic_rcu_read(&ram_list
.dirty_memory
[i
]);
1537 new_blocks
= g_malloc(sizeof(*new_blocks
) +
1538 sizeof(new_blocks
->blocks
[0]) * new_num_blocks
);
1540 if (old_num_blocks
) {
1541 memcpy(new_blocks
->blocks
, old_blocks
->blocks
,
1542 old_num_blocks
* sizeof(old_blocks
->blocks
[0]));
1545 for (j
= old_num_blocks
; j
< new_num_blocks
; j
++) {
1546 new_blocks
->blocks
[j
] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE
);
1549 atomic_rcu_set(&ram_list
.dirty_memory
[i
], new_blocks
);
1552 g_free_rcu(old_blocks
, rcu
);
1557 static ram_addr_t
ram_block_add(RAMBlock
*new_block
, Error
**errp
)
1560 RAMBlock
*last_block
= NULL
;
1561 ram_addr_t old_ram_size
, new_ram_size
;
1564 old_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1566 qemu_mutex_lock_ramlist();
1567 new_block
->offset
= find_ram_offset(new_block
->max_length
);
1569 if (!new_block
->host
) {
1570 if (xen_enabled()) {
1571 xen_ram_alloc(new_block
->offset
, new_block
->max_length
,
1572 new_block
->mr
, &err
);
1574 error_propagate(errp
, err
);
1575 qemu_mutex_unlock_ramlist();
1579 new_block
->host
= phys_mem_alloc(new_block
->max_length
,
1580 &new_block
->mr
->align
);
1581 if (!new_block
->host
) {
1582 error_setg_errno(errp
, errno
,
1583 "cannot set up guest memory '%s'",
1584 memory_region_name(new_block
->mr
));
1585 qemu_mutex_unlock_ramlist();
1588 memory_try_enable_merging(new_block
->host
, new_block
->max_length
);
1592 new_ram_size
= MAX(old_ram_size
,
1593 (new_block
->offset
+ new_block
->max_length
) >> TARGET_PAGE_BITS
);
1594 if (new_ram_size
> old_ram_size
) {
1595 migration_bitmap_extend(old_ram_size
, new_ram_size
);
1596 dirty_memory_extend(old_ram_size
, new_ram_size
);
1598 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1599 * QLIST (which has an RCU-friendly variant) does not have insertion at
1600 * tail, so save the last element in last_block.
1602 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1604 if (block
->max_length
< new_block
->max_length
) {
1609 QLIST_INSERT_BEFORE_RCU(block
, new_block
, next
);
1610 } else if (last_block
) {
1611 QLIST_INSERT_AFTER_RCU(last_block
, new_block
, next
);
1612 } else { /* list is empty */
1613 QLIST_INSERT_HEAD_RCU(&ram_list
.blocks
, new_block
, next
);
1615 ram_list
.mru_block
= NULL
;
1617 /* Write list before version */
1620 qemu_mutex_unlock_ramlist();
1622 cpu_physical_memory_set_dirty_range(new_block
->offset
,
1623 new_block
->used_length
,
1626 if (new_block
->host
) {
1627 qemu_ram_setup_dump(new_block
->host
, new_block
->max_length
);
1628 qemu_madvise(new_block
->host
, new_block
->max_length
, QEMU_MADV_HUGEPAGE
);
1629 qemu_madvise(new_block
->host
, new_block
->max_length
, QEMU_MADV_DONTFORK
);
1630 if (kvm_enabled()) {
1631 kvm_setup_guest_memory(new_block
->host
, new_block
->max_length
);
1635 return new_block
->offset
;
1639 ram_addr_t
qemu_ram_alloc_from_file(ram_addr_t size
, MemoryRegion
*mr
,
1640 bool share
, const char *mem_path
,
1643 RAMBlock
*new_block
;
1645 Error
*local_err
= NULL
;
1647 if (xen_enabled()) {
1648 error_setg(errp
, "-mem-path not supported with Xen");
1652 if (phys_mem_alloc
!= qemu_anon_ram_alloc
) {
1654 * file_ram_alloc() needs to allocate just like
1655 * phys_mem_alloc, but we haven't bothered to provide
1659 "-mem-path not supported with this accelerator");
1663 size
= HOST_PAGE_ALIGN(size
);
1664 new_block
= g_malloc0(sizeof(*new_block
));
1666 new_block
->used_length
= size
;
1667 new_block
->max_length
= size
;
1668 new_block
->flags
= share
? RAM_SHARED
: 0;
1669 new_block
->host
= file_ram_alloc(new_block
, size
,
1671 if (!new_block
->host
) {
1676 addr
= ram_block_add(new_block
, &local_err
);
1679 error_propagate(errp
, local_err
);
1687 ram_addr_t
qemu_ram_alloc_internal(ram_addr_t size
, ram_addr_t max_size
,
1688 void (*resized
)(const char*,
1691 void *host
, bool resizeable
,
1692 MemoryRegion
*mr
, Error
**errp
)
1694 RAMBlock
*new_block
;
1696 Error
*local_err
= NULL
;
1698 size
= HOST_PAGE_ALIGN(size
);
1699 max_size
= HOST_PAGE_ALIGN(max_size
);
1700 new_block
= g_malloc0(sizeof(*new_block
));
1702 new_block
->resized
= resized
;
1703 new_block
->used_length
= size
;
1704 new_block
->max_length
= max_size
;
1705 assert(max_size
>= size
);
1707 new_block
->host
= host
;
1709 new_block
->flags
|= RAM_PREALLOC
;
1712 new_block
->flags
|= RAM_RESIZEABLE
;
1714 addr
= ram_block_add(new_block
, &local_err
);
1717 error_propagate(errp
, local_err
);
1723 ram_addr_t
qemu_ram_alloc_from_ptr(ram_addr_t size
, void *host
,
1724 MemoryRegion
*mr
, Error
**errp
)
1726 return qemu_ram_alloc_internal(size
, size
, NULL
, host
, false, mr
, errp
);
1729 ram_addr_t
qemu_ram_alloc(ram_addr_t size
, MemoryRegion
*mr
, Error
**errp
)
1731 return qemu_ram_alloc_internal(size
, size
, NULL
, NULL
, false, mr
, errp
);
1734 ram_addr_t
qemu_ram_alloc_resizeable(ram_addr_t size
, ram_addr_t maxsz
,
1735 void (*resized
)(const char*,
1738 MemoryRegion
*mr
, Error
**errp
)
1740 return qemu_ram_alloc_internal(size
, maxsz
, resized
, NULL
, true, mr
, errp
);
1743 static void reclaim_ramblock(RAMBlock
*block
)
1745 if (block
->flags
& RAM_PREALLOC
) {
1747 } else if (xen_enabled()) {
1748 xen_invalidate_map_cache_entry(block
->host
);
1750 } else if (block
->fd
>= 0) {
1751 qemu_ram_munmap(block
->host
, block
->max_length
);
1755 qemu_anon_ram_free(block
->host
, block
->max_length
);
1760 void qemu_ram_free(ram_addr_t addr
)
1764 qemu_mutex_lock_ramlist();
1765 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1766 if (addr
== block
->offset
) {
1767 QLIST_REMOVE_RCU(block
, next
);
1768 ram_list
.mru_block
= NULL
;
1769 /* Write list before version */
1772 call_rcu(block
, reclaim_ramblock
, rcu
);
1776 qemu_mutex_unlock_ramlist();
1780 void qemu_ram_remap(ram_addr_t addr
, ram_addr_t length
)
1787 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1788 offset
= addr
- block
->offset
;
1789 if (offset
< block
->max_length
) {
1790 vaddr
= ramblock_ptr(block
, offset
);
1791 if (block
->flags
& RAM_PREALLOC
) {
1793 } else if (xen_enabled()) {
1797 if (block
->fd
>= 0) {
1798 flags
|= (block
->flags
& RAM_SHARED
?
1799 MAP_SHARED
: MAP_PRIVATE
);
1800 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1801 flags
, block
->fd
, offset
);
1804 * Remap needs to match alloc. Accelerators that
1805 * set phys_mem_alloc never remap. If they did,
1806 * we'd need a remap hook here.
1808 assert(phys_mem_alloc
== qemu_anon_ram_alloc
);
1810 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
1811 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1814 if (area
!= vaddr
) {
1815 fprintf(stderr
, "Could not remap addr: "
1816 RAM_ADDR_FMT
"@" RAM_ADDR_FMT
"\n",
1820 memory_try_enable_merging(vaddr
, length
);
1821 qemu_ram_setup_dump(vaddr
, length
);
1826 #endif /* !_WIN32 */
1828 int qemu_get_ram_fd(ram_addr_t addr
)
1834 block
= qemu_get_ram_block(addr
);
1840 void qemu_set_ram_fd(ram_addr_t addr
, int fd
)
1845 block
= qemu_get_ram_block(addr
);
1850 void *qemu_get_ram_block_host_ptr(ram_addr_t addr
)
1856 block
= qemu_get_ram_block(addr
);
1857 ptr
= ramblock_ptr(block
, 0);
1862 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1863 * This should not be used for general purpose DMA. Use address_space_map
1864 * or address_space_rw instead. For local memory (e.g. video ram) that the
1865 * device owns, use memory_region_get_ram_ptr.
1867 * Called within RCU critical section.
1869 void *qemu_get_ram_ptr(ram_addr_t addr
)
1871 RAMBlock
*block
= qemu_get_ram_block(addr
);
1873 if (xen_enabled() && block
->host
== NULL
) {
1874 /* We need to check if the requested address is in the RAM
1875 * because we don't want to map the entire memory in QEMU.
1876 * In that case just map until the end of the page.
1878 if (block
->offset
== 0) {
1879 return xen_map_cache(addr
, 0, 0);
1882 block
->host
= xen_map_cache(block
->offset
, block
->max_length
, 1);
1884 return ramblock_ptr(block
, addr
- block
->offset
);
1887 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1888 * but takes a size argument.
1890 * Called within RCU critical section.
1892 static void *qemu_ram_ptr_length(ram_addr_t addr
, hwaddr
*size
)
1895 ram_addr_t offset_inside_block
;
1900 block
= qemu_get_ram_block(addr
);
1901 offset_inside_block
= addr
- block
->offset
;
1902 *size
= MIN(*size
, block
->max_length
- offset_inside_block
);
1904 if (xen_enabled() && block
->host
== NULL
) {
1905 /* We need to check if the requested address is in the RAM
1906 * because we don't want to map the entire memory in QEMU.
1907 * In that case just map the requested area.
1909 if (block
->offset
== 0) {
1910 return xen_map_cache(addr
, *size
, 1);
1913 block
->host
= xen_map_cache(block
->offset
, block
->max_length
, 1);
1916 return ramblock_ptr(block
, offset_inside_block
);
1920 * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
1923 * ptr: Host pointer to look up
1924 * round_offset: If true round the result offset down to a page boundary
1925 * *ram_addr: set to result ram_addr
1926 * *offset: set to result offset within the RAMBlock
1928 * Returns: RAMBlock (or NULL if not found)
1930 * By the time this function returns, the returned pointer is not protected
1931 * by RCU anymore. If the caller is not within an RCU critical section and
1932 * does not hold the iothread lock, it must have other means of protecting the
1933 * pointer, such as a reference to the region that includes the incoming
1936 RAMBlock
*qemu_ram_block_from_host(void *ptr
, bool round_offset
,
1937 ram_addr_t
*ram_addr
,
1941 uint8_t *host
= ptr
;
1943 if (xen_enabled()) {
1945 *ram_addr
= xen_ram_addr_from_mapcache(ptr
);
1946 block
= qemu_get_ram_block(*ram_addr
);
1948 *offset
= (host
- block
->host
);
1955 block
= atomic_rcu_read(&ram_list
.mru_block
);
1956 if (block
&& block
->host
&& host
- block
->host
< block
->max_length
) {
1960 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1961 /* This case append when the block is not mapped. */
1962 if (block
->host
== NULL
) {
1965 if (host
- block
->host
< block
->max_length
) {
1974 *offset
= (host
- block
->host
);
1976 *offset
&= TARGET_PAGE_MASK
;
1978 *ram_addr
= block
->offset
+ *offset
;
1984 * Finds the named RAMBlock
1986 * name: The name of RAMBlock to find
1988 * Returns: RAMBlock (or NULL if not found)
1990 RAMBlock
*qemu_ram_block_by_name(const char *name
)
1994 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1995 if (!strcmp(name
, block
->idstr
)) {
2003 /* Some of the softmmu routines need to translate from a host pointer
2004 (typically a TLB entry) back to a ram offset. */
2005 MemoryRegion
*qemu_ram_addr_from_host(void *ptr
, ram_addr_t
*ram_addr
)
2008 ram_addr_t offset
; /* Not used */
2010 block
= qemu_ram_block_from_host(ptr
, false, ram_addr
, &offset
);
2019 /* Called within RCU critical section. */
2020 static void notdirty_mem_write(void *opaque
, hwaddr ram_addr
,
2021 uint64_t val
, unsigned size
)
2023 if (!cpu_physical_memory_get_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
)) {
2024 tb_invalidate_phys_page_fast(ram_addr
, size
);
2028 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
2031 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
2034 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
2039 /* Set both VGA and migration bits for simplicity and to remove
2040 * the notdirty callback faster.
2042 cpu_physical_memory_set_dirty_range(ram_addr
, size
,
2043 DIRTY_CLIENTS_NOCODE
);
2044 /* we remove the notdirty callback only if the code has been
2046 if (!cpu_physical_memory_is_clean(ram_addr
)) {
2047 tlb_set_dirty(current_cpu
, current_cpu
->mem_io_vaddr
);
2051 static bool notdirty_mem_accepts(void *opaque
, hwaddr addr
,
2052 unsigned size
, bool is_write
)
2057 static const MemoryRegionOps notdirty_mem_ops
= {
2058 .write
= notdirty_mem_write
,
2059 .valid
.accepts
= notdirty_mem_accepts
,
2060 .endianness
= DEVICE_NATIVE_ENDIAN
,
2063 /* Generate a debug exception if a watchpoint has been hit. */
2064 static void check_watchpoint(int offset
, int len
, MemTxAttrs attrs
, int flags
)
2066 CPUState
*cpu
= current_cpu
;
2067 CPUArchState
*env
= cpu
->env_ptr
;
2068 target_ulong pc
, cs_base
;
2073 if (cpu
->watchpoint_hit
) {
2074 /* We re-entered the check after replacing the TB. Now raise
2075 * the debug interrupt so that is will trigger after the
2076 * current instruction. */
2077 cpu_interrupt(cpu
, CPU_INTERRUPT_DEBUG
);
2080 vaddr
= (cpu
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
2081 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
2082 if (cpu_watchpoint_address_matches(wp
, vaddr
, len
)
2083 && (wp
->flags
& flags
)) {
2084 if (flags
== BP_MEM_READ
) {
2085 wp
->flags
|= BP_WATCHPOINT_HIT_READ
;
2087 wp
->flags
|= BP_WATCHPOINT_HIT_WRITE
;
2089 wp
->hitaddr
= vaddr
;
2090 wp
->hitattrs
= attrs
;
2091 if (!cpu
->watchpoint_hit
) {
2092 cpu
->watchpoint_hit
= wp
;
2093 tb_check_watchpoint(cpu
);
2094 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
2095 cpu
->exception_index
= EXCP_DEBUG
;
2098 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
2099 tb_gen_code(cpu
, pc
, cs_base
, cpu_flags
, 1);
2100 cpu_resume_from_signal(cpu
, NULL
);
2104 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
2109 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2110 so these check for a hit then pass through to the normal out-of-line
2112 static MemTxResult
watch_mem_read(void *opaque
, hwaddr addr
, uint64_t *pdata
,
2113 unsigned size
, MemTxAttrs attrs
)
2117 int asidx
= cpu_asidx_from_attrs(current_cpu
, attrs
);
2118 AddressSpace
*as
= current_cpu
->cpu_ases
[asidx
].as
;
2120 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, size
, attrs
, BP_MEM_READ
);
2123 data
= address_space_ldub(as
, addr
, attrs
, &res
);
2126 data
= address_space_lduw(as
, addr
, attrs
, &res
);
2129 data
= address_space_ldl(as
, addr
, attrs
, &res
);
2137 static MemTxResult
watch_mem_write(void *opaque
, hwaddr addr
,
2138 uint64_t val
, unsigned size
,
2142 int asidx
= cpu_asidx_from_attrs(current_cpu
, attrs
);
2143 AddressSpace
*as
= current_cpu
->cpu_ases
[asidx
].as
;
2145 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, size
, attrs
, BP_MEM_WRITE
);
2148 address_space_stb(as
, addr
, val
, attrs
, &res
);
2151 address_space_stw(as
, addr
, val
, attrs
, &res
);
2154 address_space_stl(as
, addr
, val
, attrs
, &res
);
2161 static const MemoryRegionOps watch_mem_ops
= {
2162 .read_with_attrs
= watch_mem_read
,
2163 .write_with_attrs
= watch_mem_write
,
2164 .endianness
= DEVICE_NATIVE_ENDIAN
,
2167 static MemTxResult
subpage_read(void *opaque
, hwaddr addr
, uint64_t *data
,
2168 unsigned len
, MemTxAttrs attrs
)
2170 subpage_t
*subpage
= opaque
;
2174 #if defined(DEBUG_SUBPAGE)
2175 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
"\n", __func__
,
2176 subpage
, len
, addr
);
2178 res
= address_space_read(subpage
->as
, addr
+ subpage
->base
,
2185 *data
= ldub_p(buf
);
2188 *data
= lduw_p(buf
);
2201 static MemTxResult
subpage_write(void *opaque
, hwaddr addr
,
2202 uint64_t value
, unsigned len
, MemTxAttrs attrs
)
2204 subpage_t
*subpage
= opaque
;
2207 #if defined(DEBUG_SUBPAGE)
2208 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
2209 " value %"PRIx64
"\n",
2210 __func__
, subpage
, len
, addr
, value
);
2228 return address_space_write(subpage
->as
, addr
+ subpage
->base
,
2232 static bool subpage_accepts(void *opaque
, hwaddr addr
,
2233 unsigned len
, bool is_write
)
2235 subpage_t
*subpage
= opaque
;
2236 #if defined(DEBUG_SUBPAGE)
2237 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx
"\n",
2238 __func__
, subpage
, is_write
? 'w' : 'r', len
, addr
);
2241 return address_space_access_valid(subpage
->as
, addr
+ subpage
->base
,
2245 static const MemoryRegionOps subpage_ops
= {
2246 .read_with_attrs
= subpage_read
,
2247 .write_with_attrs
= subpage_write
,
2248 .impl
.min_access_size
= 1,
2249 .impl
.max_access_size
= 8,
2250 .valid
.min_access_size
= 1,
2251 .valid
.max_access_size
= 8,
2252 .valid
.accepts
= subpage_accepts
,
2253 .endianness
= DEVICE_NATIVE_ENDIAN
,
2256 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2261 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
2263 idx
= SUBPAGE_IDX(start
);
2264 eidx
= SUBPAGE_IDX(end
);
2265 #if defined(DEBUG_SUBPAGE)
2266 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2267 __func__
, mmio
, start
, end
, idx
, eidx
, section
);
2269 for (; idx
<= eidx
; idx
++) {
2270 mmio
->sub_section
[idx
] = section
;
2276 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
)
2280 mmio
= g_malloc0(sizeof(subpage_t
));
2284 memory_region_init_io(&mmio
->iomem
, NULL
, &subpage_ops
, mmio
,
2285 NULL
, TARGET_PAGE_SIZE
);
2286 mmio
->iomem
.subpage
= true;
2287 #if defined(DEBUG_SUBPAGE)
2288 printf("%s: %p base " TARGET_FMT_plx
" len %08x\n", __func__
,
2289 mmio
, base
, TARGET_PAGE_SIZE
);
2291 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
-1, PHYS_SECTION_UNASSIGNED
);
2296 static uint16_t dummy_section(PhysPageMap
*map
, AddressSpace
*as
,
2300 MemoryRegionSection section
= {
2301 .address_space
= as
,
2303 .offset_within_address_space
= 0,
2304 .offset_within_region
= 0,
2305 .size
= int128_2_64(),
2308 return phys_section_add(map
, §ion
);
2311 MemoryRegion
*iotlb_to_region(CPUState
*cpu
, hwaddr index
, MemTxAttrs attrs
)
2313 int asidx
= cpu_asidx_from_attrs(cpu
, attrs
);
2314 CPUAddressSpace
*cpuas
= &cpu
->cpu_ases
[asidx
];
2315 AddressSpaceDispatch
*d
= atomic_rcu_read(&cpuas
->memory_dispatch
);
2316 MemoryRegionSection
*sections
= d
->map
.sections
;
2318 return sections
[index
& ~TARGET_PAGE_MASK
].mr
;
2321 static void io_mem_init(void)
2323 memory_region_init_io(&io_mem_rom
, NULL
, &unassigned_mem_ops
, NULL
, NULL
, UINT64_MAX
);
2324 memory_region_init_io(&io_mem_unassigned
, NULL
, &unassigned_mem_ops
, NULL
,
2326 memory_region_init_io(&io_mem_notdirty
, NULL
, ¬dirty_mem_ops
, NULL
,
2328 memory_region_init_io(&io_mem_watch
, NULL
, &watch_mem_ops
, NULL
,
2332 static void mem_begin(MemoryListener
*listener
)
2334 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
2335 AddressSpaceDispatch
*d
= g_new0(AddressSpaceDispatch
, 1);
2338 n
= dummy_section(&d
->map
, as
, &io_mem_unassigned
);
2339 assert(n
== PHYS_SECTION_UNASSIGNED
);
2340 n
= dummy_section(&d
->map
, as
, &io_mem_notdirty
);
2341 assert(n
== PHYS_SECTION_NOTDIRTY
);
2342 n
= dummy_section(&d
->map
, as
, &io_mem_rom
);
2343 assert(n
== PHYS_SECTION_ROM
);
2344 n
= dummy_section(&d
->map
, as
, &io_mem_watch
);
2345 assert(n
== PHYS_SECTION_WATCH
);
2347 d
->phys_map
= (PhysPageEntry
) { .ptr
= PHYS_MAP_NODE_NIL
, .skip
= 1 };
2349 as
->next_dispatch
= d
;
2352 static void address_space_dispatch_free(AddressSpaceDispatch
*d
)
2354 phys_sections_free(&d
->map
);
2358 static void mem_commit(MemoryListener
*listener
)
2360 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
2361 AddressSpaceDispatch
*cur
= as
->dispatch
;
2362 AddressSpaceDispatch
*next
= as
->next_dispatch
;
2364 phys_page_compact_all(next
, next
->map
.nodes_nb
);
2366 atomic_rcu_set(&as
->dispatch
, next
);
2368 call_rcu(cur
, address_space_dispatch_free
, rcu
);
2372 static void tcg_commit(MemoryListener
*listener
)
2374 CPUAddressSpace
*cpuas
;
2375 AddressSpaceDispatch
*d
;
2377 /* since each CPU stores ram addresses in its TLB cache, we must
2378 reset the modified entries */
2379 cpuas
= container_of(listener
, CPUAddressSpace
, tcg_as_listener
);
2380 cpu_reloading_memory_map();
2381 /* The CPU and TLB are protected by the iothread lock.
2382 * We reload the dispatch pointer now because cpu_reloading_memory_map()
2383 * may have split the RCU critical section.
2385 d
= atomic_rcu_read(&cpuas
->as
->dispatch
);
2386 cpuas
->memory_dispatch
= d
;
2387 tlb_flush(cpuas
->cpu
, 1);
2390 void address_space_init_dispatch(AddressSpace
*as
)
2392 as
->dispatch
= NULL
;
2393 as
->dispatch_listener
= (MemoryListener
) {
2395 .commit
= mem_commit
,
2396 .region_add
= mem_add
,
2397 .region_nop
= mem_add
,
2400 memory_listener_register(&as
->dispatch_listener
, as
);
2403 void address_space_unregister(AddressSpace
*as
)
2405 memory_listener_unregister(&as
->dispatch_listener
);
2408 void address_space_destroy_dispatch(AddressSpace
*as
)
2410 AddressSpaceDispatch
*d
= as
->dispatch
;
2412 atomic_rcu_set(&as
->dispatch
, NULL
);
2414 call_rcu(d
, address_space_dispatch_free
, rcu
);
2418 static void memory_map_init(void)
2420 system_memory
= g_malloc(sizeof(*system_memory
));
2422 memory_region_init(system_memory
, NULL
, "system", UINT64_MAX
);
2423 address_space_init(&address_space_memory
, system_memory
, "memory");
2425 system_io
= g_malloc(sizeof(*system_io
));
2426 memory_region_init_io(system_io
, NULL
, &unassigned_io_ops
, NULL
, "io",
2428 address_space_init(&address_space_io
, system_io
, "I/O");
2431 MemoryRegion
*get_system_memory(void)
2433 return system_memory
;
2436 MemoryRegion
*get_system_io(void)
2441 #endif /* !defined(CONFIG_USER_ONLY) */
2443 /* physical memory access (slow version, mainly for debug) */
2444 #if defined(CONFIG_USER_ONLY)
2445 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
2446 uint8_t *buf
, int len
, int is_write
)
2453 page
= addr
& TARGET_PAGE_MASK
;
2454 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2457 flags
= page_get_flags(page
);
2458 if (!(flags
& PAGE_VALID
))
2461 if (!(flags
& PAGE_WRITE
))
2463 /* XXX: this code should not depend on lock_user */
2464 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
2467 unlock_user(p
, addr
, l
);
2469 if (!(flags
& PAGE_READ
))
2471 /* XXX: this code should not depend on lock_user */
2472 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
2475 unlock_user(p
, addr
, 0);
2486 static void invalidate_and_set_dirty(MemoryRegion
*mr
, hwaddr addr
,
2489 uint8_t dirty_log_mask
= memory_region_get_dirty_log_mask(mr
);
2490 /* No early return if dirty_log_mask is or becomes 0, because
2491 * cpu_physical_memory_set_dirty_range will still call
2492 * xen_modified_memory.
2494 if (dirty_log_mask
) {
2496 cpu_physical_memory_range_includes_clean(addr
, length
, dirty_log_mask
);
2498 if (dirty_log_mask
& (1 << DIRTY_MEMORY_CODE
)) {
2499 tb_invalidate_phys_range(addr
, addr
+ length
);
2500 dirty_log_mask
&= ~(1 << DIRTY_MEMORY_CODE
);
2502 cpu_physical_memory_set_dirty_range(addr
, length
, dirty_log_mask
);
2505 static int memory_access_size(MemoryRegion
*mr
, unsigned l
, hwaddr addr
)
2507 unsigned access_size_max
= mr
->ops
->valid
.max_access_size
;
2509 /* Regions are assumed to support 1-4 byte accesses unless
2510 otherwise specified. */
2511 if (access_size_max
== 0) {
2512 access_size_max
= 4;
2515 /* Bound the maximum access by the alignment of the address. */
2516 if (!mr
->ops
->impl
.unaligned
) {
2517 unsigned align_size_max
= addr
& -addr
;
2518 if (align_size_max
!= 0 && align_size_max
< access_size_max
) {
2519 access_size_max
= align_size_max
;
2523 /* Don't attempt accesses larger than the maximum. */
2524 if (l
> access_size_max
) {
2525 l
= access_size_max
;
2532 static bool prepare_mmio_access(MemoryRegion
*mr
)
2534 bool unlocked
= !qemu_mutex_iothread_locked();
2535 bool release_lock
= false;
2537 if (unlocked
&& mr
->global_locking
) {
2538 qemu_mutex_lock_iothread();
2540 release_lock
= true;
2542 if (mr
->flush_coalesced_mmio
) {
2544 qemu_mutex_lock_iothread();
2546 qemu_flush_coalesced_mmio_buffer();
2548 qemu_mutex_unlock_iothread();
2552 return release_lock
;
2555 /* Called within RCU critical section. */
2556 static MemTxResult
address_space_write_continue(AddressSpace
*as
, hwaddr addr
,
2559 int len
, hwaddr addr1
,
2560 hwaddr l
, MemoryRegion
*mr
)
2564 MemTxResult result
= MEMTX_OK
;
2565 bool release_lock
= false;
2568 if (!memory_access_is_direct(mr
, true)) {
2569 release_lock
|= prepare_mmio_access(mr
);
2570 l
= memory_access_size(mr
, l
, addr1
);
2571 /* XXX: could force current_cpu to NULL to avoid
2575 /* 64 bit write access */
2577 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 8,
2581 /* 32 bit write access */
2583 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 4,
2587 /* 16 bit write access */
2589 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 2,
2593 /* 8 bit write access */
2595 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 1,
2602 addr1
+= memory_region_get_ram_addr(mr
);
2604 ptr
= qemu_get_ram_ptr(addr1
);
2605 memcpy(ptr
, buf
, l
);
2606 invalidate_and_set_dirty(mr
, addr1
, l
);
2610 qemu_mutex_unlock_iothread();
2611 release_lock
= false;
2623 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
2629 MemTxResult
address_space_write(AddressSpace
*as
, hwaddr addr
, MemTxAttrs attrs
,
2630 const uint8_t *buf
, int len
)
2635 MemTxResult result
= MEMTX_OK
;
2640 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
2641 result
= address_space_write_continue(as
, addr
, attrs
, buf
, len
,
2649 /* Called within RCU critical section. */
2650 MemTxResult
address_space_read_continue(AddressSpace
*as
, hwaddr addr
,
2651 MemTxAttrs attrs
, uint8_t *buf
,
2652 int len
, hwaddr addr1
, hwaddr l
,
2657 MemTxResult result
= MEMTX_OK
;
2658 bool release_lock
= false;
2661 if (!memory_access_is_direct(mr
, false)) {
2663 release_lock
|= prepare_mmio_access(mr
);
2664 l
= memory_access_size(mr
, l
, addr1
);
2667 /* 64 bit read access */
2668 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 8,
2673 /* 32 bit read access */
2674 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 4,
2679 /* 16 bit read access */
2680 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 2,
2685 /* 8 bit read access */
2686 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 1,
2695 ptr
= qemu_get_ram_ptr(mr
->ram_addr
+ addr1
);
2696 memcpy(buf
, ptr
, l
);
2700 qemu_mutex_unlock_iothread();
2701 release_lock
= false;
2713 mr
= address_space_translate(as
, addr
, &addr1
, &l
, false);
2719 MemTxResult
address_space_read_full(AddressSpace
*as
, hwaddr addr
,
2720 MemTxAttrs attrs
, uint8_t *buf
, int len
)
2725 MemTxResult result
= MEMTX_OK
;
2730 mr
= address_space_translate(as
, addr
, &addr1
, &l
, false);
2731 result
= address_space_read_continue(as
, addr
, attrs
, buf
, len
,
2739 MemTxResult
address_space_rw(AddressSpace
*as
, hwaddr addr
, MemTxAttrs attrs
,
2740 uint8_t *buf
, int len
, bool is_write
)
2743 return address_space_write(as
, addr
, attrs
, (uint8_t *)buf
, len
);
2745 return address_space_read(as
, addr
, attrs
, (uint8_t *)buf
, len
);
2749 void cpu_physical_memory_rw(hwaddr addr
, uint8_t *buf
,
2750 int len
, int is_write
)
2752 address_space_rw(&address_space_memory
, addr
, MEMTXATTRS_UNSPECIFIED
,
2753 buf
, len
, is_write
);
2756 enum write_rom_type
{
2761 static inline void cpu_physical_memory_write_rom_internal(AddressSpace
*as
,
2762 hwaddr addr
, const uint8_t *buf
, int len
, enum write_rom_type type
)
2772 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
2774 if (!(memory_region_is_ram(mr
) ||
2775 memory_region_is_romd(mr
))) {
2776 l
= memory_access_size(mr
, l
, addr1
);
2778 addr1
+= memory_region_get_ram_addr(mr
);
2780 ptr
= qemu_get_ram_ptr(addr1
);
2783 memcpy(ptr
, buf
, l
);
2784 invalidate_and_set_dirty(mr
, addr1
, l
);
2787 flush_icache_range((uintptr_t)ptr
, (uintptr_t)ptr
+ l
);
2798 /* used for ROM loading : can write in RAM and ROM */
2799 void cpu_physical_memory_write_rom(AddressSpace
*as
, hwaddr addr
,
2800 const uint8_t *buf
, int len
)
2802 cpu_physical_memory_write_rom_internal(as
, addr
, buf
, len
, WRITE_DATA
);
2805 void cpu_flush_icache_range(hwaddr start
, int len
)
2808 * This function should do the same thing as an icache flush that was
2809 * triggered from within the guest. For TCG we are always cache coherent,
2810 * so there is no need to flush anything. For KVM / Xen we need to flush
2811 * the host's instruction cache at least.
2813 if (tcg_enabled()) {
2817 cpu_physical_memory_write_rom_internal(&address_space_memory
,
2818 start
, NULL
, len
, FLUSH_CACHE
);
2829 static BounceBuffer bounce
;
2831 typedef struct MapClient
{
2833 QLIST_ENTRY(MapClient
) link
;
2836 QemuMutex map_client_list_lock
;
2837 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
2838 = QLIST_HEAD_INITIALIZER(map_client_list
);
2840 static void cpu_unregister_map_client_do(MapClient
*client
)
2842 QLIST_REMOVE(client
, link
);
2846 static void cpu_notify_map_clients_locked(void)
2850 while (!QLIST_EMPTY(&map_client_list
)) {
2851 client
= QLIST_FIRST(&map_client_list
);
2852 qemu_bh_schedule(client
->bh
);
2853 cpu_unregister_map_client_do(client
);
2857 void cpu_register_map_client(QEMUBH
*bh
)
2859 MapClient
*client
= g_malloc(sizeof(*client
));
2861 qemu_mutex_lock(&map_client_list_lock
);
2863 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
2864 if (!atomic_read(&bounce
.in_use
)) {
2865 cpu_notify_map_clients_locked();
2867 qemu_mutex_unlock(&map_client_list_lock
);
2870 void cpu_exec_init_all(void)
2872 qemu_mutex_init(&ram_list
.mutex
);
2875 qemu_mutex_init(&map_client_list_lock
);
2878 void cpu_unregister_map_client(QEMUBH
*bh
)
2882 qemu_mutex_lock(&map_client_list_lock
);
2883 QLIST_FOREACH(client
, &map_client_list
, link
) {
2884 if (client
->bh
== bh
) {
2885 cpu_unregister_map_client_do(client
);
2889 qemu_mutex_unlock(&map_client_list_lock
);
2892 static void cpu_notify_map_clients(void)
2894 qemu_mutex_lock(&map_client_list_lock
);
2895 cpu_notify_map_clients_locked();
2896 qemu_mutex_unlock(&map_client_list_lock
);
2899 bool address_space_access_valid(AddressSpace
*as
, hwaddr addr
, int len
, bool is_write
)
2907 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2908 if (!memory_access_is_direct(mr
, is_write
)) {
2909 l
= memory_access_size(mr
, l
, addr
);
2910 if (!memory_region_access_valid(mr
, xlat
, l
, is_write
)) {
2922 /* Map a physical memory region into a host virtual address.
2923 * May map a subset of the requested range, given by and returned in *plen.
2924 * May return NULL if resources needed to perform the mapping are exhausted.
2925 * Use only for reads OR writes - not for read-modify-write operations.
2926 * Use cpu_register_map_client() to know when retrying the map operation is
2927 * likely to succeed.
2929 void *address_space_map(AddressSpace
*as
,
2936 hwaddr l
, xlat
, base
;
2937 MemoryRegion
*mr
, *this_mr
;
2947 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2949 if (!memory_access_is_direct(mr
, is_write
)) {
2950 if (atomic_xchg(&bounce
.in_use
, true)) {
2954 /* Avoid unbounded allocations */
2955 l
= MIN(l
, TARGET_PAGE_SIZE
);
2956 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, l
);
2960 memory_region_ref(mr
);
2963 address_space_read(as
, addr
, MEMTXATTRS_UNSPECIFIED
,
2969 return bounce
.buffer
;
2973 raddr
= memory_region_get_ram_addr(mr
);
2984 this_mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2985 if (this_mr
!= mr
|| xlat
!= base
+ done
) {
2990 memory_region_ref(mr
);
2992 ptr
= qemu_ram_ptr_length(raddr
+ base
, plen
);
2998 /* Unmaps a memory region previously mapped by address_space_map().
2999 * Will also mark the memory as dirty if is_write == 1. access_len gives
3000 * the amount of memory that was actually read or written by the caller.
3002 void address_space_unmap(AddressSpace
*as
, void *buffer
, hwaddr len
,
3003 int is_write
, hwaddr access_len
)
3005 if (buffer
!= bounce
.buffer
) {
3009 mr
= qemu_ram_addr_from_host(buffer
, &addr1
);
3012 invalidate_and_set_dirty(mr
, addr1
, access_len
);
3014 if (xen_enabled()) {
3015 xen_invalidate_map_cache_entry(buffer
);
3017 memory_region_unref(mr
);
3021 address_space_write(as
, bounce
.addr
, MEMTXATTRS_UNSPECIFIED
,
3022 bounce
.buffer
, access_len
);
3024 qemu_vfree(bounce
.buffer
);
3025 bounce
.buffer
= NULL
;
3026 memory_region_unref(bounce
.mr
);
3027 atomic_mb_set(&bounce
.in_use
, false);
3028 cpu_notify_map_clients();
3031 void *cpu_physical_memory_map(hwaddr addr
,
3035 return address_space_map(&address_space_memory
, addr
, plen
, is_write
);
3038 void cpu_physical_memory_unmap(void *buffer
, hwaddr len
,
3039 int is_write
, hwaddr access_len
)
3041 return address_space_unmap(&address_space_memory
, buffer
, len
, is_write
, access_len
);
3044 /* warning: addr must be aligned */
3045 static inline uint32_t address_space_ldl_internal(AddressSpace
*as
, hwaddr addr
,
3047 MemTxResult
*result
,
3048 enum device_endian endian
)
3056 bool release_lock
= false;
3059 mr
= address_space_translate(as
, addr
, &addr1
, &l
, false);
3060 if (l
< 4 || !memory_access_is_direct(mr
, false)) {
3061 release_lock
|= prepare_mmio_access(mr
);
3064 r
= memory_region_dispatch_read(mr
, addr1
, &val
, 4, attrs
);
3065 #if defined(TARGET_WORDS_BIGENDIAN)
3066 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3070 if (endian
== DEVICE_BIG_ENDIAN
) {
3076 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
3080 case DEVICE_LITTLE_ENDIAN
:
3081 val
= ldl_le_p(ptr
);
3083 case DEVICE_BIG_ENDIAN
:
3084 val
= ldl_be_p(ptr
);
3096 qemu_mutex_unlock_iothread();
3102 uint32_t address_space_ldl(AddressSpace
*as
, hwaddr addr
,
3103 MemTxAttrs attrs
, MemTxResult
*result
)
3105 return address_space_ldl_internal(as
, addr
, attrs
, result
,
3106 DEVICE_NATIVE_ENDIAN
);
3109 uint32_t address_space_ldl_le(AddressSpace
*as
, hwaddr addr
,
3110 MemTxAttrs attrs
, MemTxResult
*result
)
3112 return address_space_ldl_internal(as
, addr
, attrs
, result
,
3113 DEVICE_LITTLE_ENDIAN
);
3116 uint32_t address_space_ldl_be(AddressSpace
*as
, hwaddr addr
,
3117 MemTxAttrs attrs
, MemTxResult
*result
)
3119 return address_space_ldl_internal(as
, addr
, attrs
, result
,
3123 uint32_t ldl_phys(AddressSpace
*as
, hwaddr addr
)
3125 return address_space_ldl(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3128 uint32_t ldl_le_phys(AddressSpace
*as
, hwaddr addr
)
3130 return address_space_ldl_le(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3133 uint32_t ldl_be_phys(AddressSpace
*as
, hwaddr addr
)
3135 return address_space_ldl_be(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3138 /* warning: addr must be aligned */
3139 static inline uint64_t address_space_ldq_internal(AddressSpace
*as
, hwaddr addr
,
3141 MemTxResult
*result
,
3142 enum device_endian endian
)
3150 bool release_lock
= false;
3153 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
3155 if (l
< 8 || !memory_access_is_direct(mr
, false)) {
3156 release_lock
|= prepare_mmio_access(mr
);
3159 r
= memory_region_dispatch_read(mr
, addr1
, &val
, 8, attrs
);
3160 #if defined(TARGET_WORDS_BIGENDIAN)
3161 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3165 if (endian
== DEVICE_BIG_ENDIAN
) {
3171 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
3175 case DEVICE_LITTLE_ENDIAN
:
3176 val
= ldq_le_p(ptr
);
3178 case DEVICE_BIG_ENDIAN
:
3179 val
= ldq_be_p(ptr
);
3191 qemu_mutex_unlock_iothread();
3197 uint64_t address_space_ldq(AddressSpace
*as
, hwaddr addr
,
3198 MemTxAttrs attrs
, MemTxResult
*result
)
3200 return address_space_ldq_internal(as
, addr
, attrs
, result
,
3201 DEVICE_NATIVE_ENDIAN
);
3204 uint64_t address_space_ldq_le(AddressSpace
*as
, hwaddr addr
,
3205 MemTxAttrs attrs
, MemTxResult
*result
)
3207 return address_space_ldq_internal(as
, addr
, attrs
, result
,
3208 DEVICE_LITTLE_ENDIAN
);
3211 uint64_t address_space_ldq_be(AddressSpace
*as
, hwaddr addr
,
3212 MemTxAttrs attrs
, MemTxResult
*result
)
3214 return address_space_ldq_internal(as
, addr
, attrs
, result
,
3218 uint64_t ldq_phys(AddressSpace
*as
, hwaddr addr
)
3220 return address_space_ldq(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3223 uint64_t ldq_le_phys(AddressSpace
*as
, hwaddr addr
)
3225 return address_space_ldq_le(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3228 uint64_t ldq_be_phys(AddressSpace
*as
, hwaddr addr
)
3230 return address_space_ldq_be(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3234 uint32_t address_space_ldub(AddressSpace
*as
, hwaddr addr
,
3235 MemTxAttrs attrs
, MemTxResult
*result
)
3240 r
= address_space_rw(as
, addr
, attrs
, &val
, 1, 0);
3247 uint32_t ldub_phys(AddressSpace
*as
, hwaddr addr
)
3249 return address_space_ldub(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3252 /* warning: addr must be aligned */
3253 static inline uint32_t address_space_lduw_internal(AddressSpace
*as
,
3256 MemTxResult
*result
,
3257 enum device_endian endian
)
3265 bool release_lock
= false;
3268 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
3270 if (l
< 2 || !memory_access_is_direct(mr
, false)) {
3271 release_lock
|= prepare_mmio_access(mr
);
3274 r
= memory_region_dispatch_read(mr
, addr1
, &val
, 2, attrs
);
3275 #if defined(TARGET_WORDS_BIGENDIAN)
3276 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3280 if (endian
== DEVICE_BIG_ENDIAN
) {
3286 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
3290 case DEVICE_LITTLE_ENDIAN
:
3291 val
= lduw_le_p(ptr
);
3293 case DEVICE_BIG_ENDIAN
:
3294 val
= lduw_be_p(ptr
);
3306 qemu_mutex_unlock_iothread();
3312 uint32_t address_space_lduw(AddressSpace
*as
, hwaddr addr
,
3313 MemTxAttrs attrs
, MemTxResult
*result
)
3315 return address_space_lduw_internal(as
, addr
, attrs
, result
,
3316 DEVICE_NATIVE_ENDIAN
);
3319 uint32_t address_space_lduw_le(AddressSpace
*as
, hwaddr addr
,
3320 MemTxAttrs attrs
, MemTxResult
*result
)
3322 return address_space_lduw_internal(as
, addr
, attrs
, result
,
3323 DEVICE_LITTLE_ENDIAN
);
3326 uint32_t address_space_lduw_be(AddressSpace
*as
, hwaddr addr
,
3327 MemTxAttrs attrs
, MemTxResult
*result
)
3329 return address_space_lduw_internal(as
, addr
, attrs
, result
,
3333 uint32_t lduw_phys(AddressSpace
*as
, hwaddr addr
)
3335 return address_space_lduw(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3338 uint32_t lduw_le_phys(AddressSpace
*as
, hwaddr addr
)
3340 return address_space_lduw_le(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3343 uint32_t lduw_be_phys(AddressSpace
*as
, hwaddr addr
)
3345 return address_space_lduw_be(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3348 /* warning: addr must be aligned. The ram page is not masked as dirty
3349 and the code inside is not invalidated. It is useful if the dirty
3350 bits are used to track modified PTEs */
3351 void address_space_stl_notdirty(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3352 MemTxAttrs attrs
, MemTxResult
*result
)
3359 uint8_t dirty_log_mask
;
3360 bool release_lock
= false;
3363 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
3365 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
3366 release_lock
|= prepare_mmio_access(mr
);
3368 r
= memory_region_dispatch_write(mr
, addr1
, val
, 4, attrs
);
3370 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
3371 ptr
= qemu_get_ram_ptr(addr1
);
3374 dirty_log_mask
= memory_region_get_dirty_log_mask(mr
);
3375 dirty_log_mask
&= ~(1 << DIRTY_MEMORY_CODE
);
3376 cpu_physical_memory_set_dirty_range(addr1
, 4, dirty_log_mask
);
3383 qemu_mutex_unlock_iothread();
3388 void stl_phys_notdirty(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3390 address_space_stl_notdirty(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3393 /* warning: addr must be aligned */
3394 static inline void address_space_stl_internal(AddressSpace
*as
,
3395 hwaddr addr
, uint32_t val
,
3397 MemTxResult
*result
,
3398 enum device_endian endian
)
3405 bool release_lock
= false;
3408 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
3410 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
3411 release_lock
|= prepare_mmio_access(mr
);
3413 #if defined(TARGET_WORDS_BIGENDIAN)
3414 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3418 if (endian
== DEVICE_BIG_ENDIAN
) {
3422 r
= memory_region_dispatch_write(mr
, addr1
, val
, 4, attrs
);
3425 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
3426 ptr
= qemu_get_ram_ptr(addr1
);
3428 case DEVICE_LITTLE_ENDIAN
:
3431 case DEVICE_BIG_ENDIAN
:
3438 invalidate_and_set_dirty(mr
, addr1
, 4);
3445 qemu_mutex_unlock_iothread();
3450 void address_space_stl(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3451 MemTxAttrs attrs
, MemTxResult
*result
)
3453 address_space_stl_internal(as
, addr
, val
, attrs
, result
,
3454 DEVICE_NATIVE_ENDIAN
);
3457 void address_space_stl_le(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3458 MemTxAttrs attrs
, MemTxResult
*result
)
3460 address_space_stl_internal(as
, addr
, val
, attrs
, result
,
3461 DEVICE_LITTLE_ENDIAN
);
3464 void address_space_stl_be(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3465 MemTxAttrs attrs
, MemTxResult
*result
)
3467 address_space_stl_internal(as
, addr
, val
, attrs
, result
,
3471 void stl_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3473 address_space_stl(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3476 void stl_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3478 address_space_stl_le(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3481 void stl_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3483 address_space_stl_be(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3487 void address_space_stb(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3488 MemTxAttrs attrs
, MemTxResult
*result
)
3493 r
= address_space_rw(as
, addr
, attrs
, &v
, 1, 1);
3499 void stb_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3501 address_space_stb(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3504 /* warning: addr must be aligned */
3505 static inline void address_space_stw_internal(AddressSpace
*as
,
3506 hwaddr addr
, uint32_t val
,
3508 MemTxResult
*result
,
3509 enum device_endian endian
)
3516 bool release_lock
= false;
3519 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
3520 if (l
< 2 || !memory_access_is_direct(mr
, true)) {
3521 release_lock
|= prepare_mmio_access(mr
);
3523 #if defined(TARGET_WORDS_BIGENDIAN)
3524 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3528 if (endian
== DEVICE_BIG_ENDIAN
) {
3532 r
= memory_region_dispatch_write(mr
, addr1
, val
, 2, attrs
);
3535 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
3536 ptr
= qemu_get_ram_ptr(addr1
);
3538 case DEVICE_LITTLE_ENDIAN
:
3541 case DEVICE_BIG_ENDIAN
:
3548 invalidate_and_set_dirty(mr
, addr1
, 2);
3555 qemu_mutex_unlock_iothread();
3560 void address_space_stw(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3561 MemTxAttrs attrs
, MemTxResult
*result
)
3563 address_space_stw_internal(as
, addr
, val
, attrs
, result
,
3564 DEVICE_NATIVE_ENDIAN
);
3567 void address_space_stw_le(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3568 MemTxAttrs attrs
, MemTxResult
*result
)
3570 address_space_stw_internal(as
, addr
, val
, attrs
, result
,
3571 DEVICE_LITTLE_ENDIAN
);
3574 void address_space_stw_be(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3575 MemTxAttrs attrs
, MemTxResult
*result
)
3577 address_space_stw_internal(as
, addr
, val
, attrs
, result
,
3581 void stw_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3583 address_space_stw(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3586 void stw_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3588 address_space_stw_le(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3591 void stw_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3593 address_space_stw_be(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3597 void address_space_stq(AddressSpace
*as
, hwaddr addr
, uint64_t val
,
3598 MemTxAttrs attrs
, MemTxResult
*result
)
3602 r
= address_space_rw(as
, addr
, attrs
, (void *) &val
, 8, 1);
3608 void address_space_stq_le(AddressSpace
*as
, hwaddr addr
, uint64_t val
,
3609 MemTxAttrs attrs
, MemTxResult
*result
)
3612 val
= cpu_to_le64(val
);
3613 r
= address_space_rw(as
, addr
, attrs
, (void *) &val
, 8, 1);
3618 void address_space_stq_be(AddressSpace
*as
, hwaddr addr
, uint64_t val
,
3619 MemTxAttrs attrs
, MemTxResult
*result
)
3622 val
= cpu_to_be64(val
);
3623 r
= address_space_rw(as
, addr
, attrs
, (void *) &val
, 8, 1);
3629 void stq_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3631 address_space_stq(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3634 void stq_le_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3636 address_space_stq_le(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3639 void stq_be_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3641 address_space_stq_be(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3644 /* virtual memory access for debug (includes writing to ROM) */
3645 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
3646 uint8_t *buf
, int len
, int is_write
)
3656 page
= addr
& TARGET_PAGE_MASK
;
3657 phys_addr
= cpu_get_phys_page_attrs_debug(cpu
, page
, &attrs
);
3658 asidx
= cpu_asidx_from_attrs(cpu
, attrs
);
3659 /* if no physical page mapped, return an error */
3660 if (phys_addr
== -1)
3662 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3665 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
3667 cpu_physical_memory_write_rom(cpu
->cpu_ases
[asidx
].as
,
3670 address_space_rw(cpu
->cpu_ases
[asidx
].as
, phys_addr
,
3671 MEMTXATTRS_UNSPECIFIED
,
3682 * Allows code that needs to deal with migration bitmaps etc to still be built
3683 * target independent.
3685 size_t qemu_target_page_bits(void)
3687 return TARGET_PAGE_BITS
;
3693 * A helper function for the _utterly broken_ virtio device model to find out if
3694 * it's running on a big endian machine. Don't do this at home kids!
3696 bool target_words_bigendian(void);
3697 bool target_words_bigendian(void)
3699 #if defined(TARGET_WORDS_BIGENDIAN)
3706 #ifndef CONFIG_USER_ONLY
3707 bool cpu_physical_memory_is_io(hwaddr phys_addr
)
3714 mr
= address_space_translate(&address_space_memory
,
3715 phys_addr
, &phys_addr
, &l
, false);
3717 res
= !(memory_region_is_ram(mr
) || memory_region_is_romd(mr
));
3722 int qemu_ram_foreach_block(RAMBlockIterFunc func
, void *opaque
)
3728 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
3729 ret
= func(block
->idstr
, block
->host
, block
->offset
,
3730 block
->used_length
, opaque
);