4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
21 #include <sys/types.h>
25 #include "qemu-common.h"
29 #include "hw/boards.h"
31 #include "qemu/osdep.h"
32 #include "sysemu/kvm.h"
33 #include "sysemu/sysemu.h"
34 #include "hw/xen/xen.h"
35 #include "qemu/timer.h"
36 #include "qemu/config-file.h"
37 #include "qemu/error-report.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
47 #include "exec/cpu-all.h"
48 #include "qemu/rcu_queue.h"
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
52 #include "exec/memory-internal.h"
53 #include "exec/ram_addr.h"
55 #include "qemu/range.h"
57 //#define DEBUG_SUBPAGE
59 #if !defined(CONFIG_USER_ONLY)
60 static bool in_migration
;
62 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
63 * are protected by the ramlist lock.
65 RAMList ram_list
= { .blocks
= QLIST_HEAD_INITIALIZER(ram_list
.blocks
) };
67 static MemoryRegion
*system_memory
;
68 static MemoryRegion
*system_io
;
70 AddressSpace address_space_io
;
71 AddressSpace address_space_memory
;
73 MemoryRegion io_mem_rom
, io_mem_notdirty
;
74 static MemoryRegion io_mem_unassigned
;
76 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
77 #define RAM_PREALLOC (1 << 0)
79 /* RAM is mmap-ed with MAP_SHARED */
80 #define RAM_SHARED (1 << 1)
82 /* Only a portion of RAM (used_length) is actually used, and migrated.
83 * This used_length size can change across reboots.
85 #define RAM_RESIZEABLE (1 << 2)
89 struct CPUTailQ cpus
= QTAILQ_HEAD_INITIALIZER(cpus
);
90 /* current CPU in the current thread. It is only valid inside
92 DEFINE_TLS(CPUState
*, current_cpu
);
93 /* 0 = Do not count executed instructions.
94 1 = Precise instruction counting.
95 2 = Adaptive rate instruction counting. */
98 #if !defined(CONFIG_USER_ONLY)
100 typedef struct PhysPageEntry PhysPageEntry
;
102 struct PhysPageEntry
{
103 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
105 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
109 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
111 /* Size of the L2 (and L3, etc) page tables. */
112 #define ADDR_SPACE_BITS 64
115 #define P_L2_SIZE (1 << P_L2_BITS)
117 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
119 typedef PhysPageEntry Node
[P_L2_SIZE
];
121 typedef struct PhysPageMap
{
124 unsigned sections_nb
;
125 unsigned sections_nb_alloc
;
127 unsigned nodes_nb_alloc
;
129 MemoryRegionSection
*sections
;
132 struct AddressSpaceDispatch
{
135 /* This is a multi-level map on the physical address space.
136 * The bottom level has pointers to MemoryRegionSections.
138 PhysPageEntry phys_map
;
143 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
144 typedef struct subpage_t
{
148 uint16_t sub_section
[TARGET_PAGE_SIZE
];
151 #define PHYS_SECTION_UNASSIGNED 0
152 #define PHYS_SECTION_NOTDIRTY 1
153 #define PHYS_SECTION_ROM 2
154 #define PHYS_SECTION_WATCH 3
156 static void io_mem_init(void);
157 static void memory_map_init(void);
158 static void tcg_commit(MemoryListener
*listener
);
160 static MemoryRegion io_mem_watch
;
163 #if !defined(CONFIG_USER_ONLY)
165 static void phys_map_node_reserve(PhysPageMap
*map
, unsigned nodes
)
167 if (map
->nodes_nb
+ nodes
> map
->nodes_nb_alloc
) {
168 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
* 2, 16);
169 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
, map
->nodes_nb
+ nodes
);
170 map
->nodes
= g_renew(Node
, map
->nodes
, map
->nodes_nb_alloc
);
174 static uint32_t phys_map_node_alloc(PhysPageMap
*map
)
179 ret
= map
->nodes_nb
++;
180 assert(ret
!= PHYS_MAP_NODE_NIL
);
181 assert(ret
!= map
->nodes_nb_alloc
);
182 for (i
= 0; i
< P_L2_SIZE
; ++i
) {
183 map
->nodes
[ret
][i
].skip
= 1;
184 map
->nodes
[ret
][i
].ptr
= PHYS_MAP_NODE_NIL
;
189 static void phys_page_set_level(PhysPageMap
*map
, PhysPageEntry
*lp
,
190 hwaddr
*index
, hwaddr
*nb
, uint16_t leaf
,
195 hwaddr step
= (hwaddr
)1 << (level
* P_L2_BITS
);
197 if (lp
->skip
&& lp
->ptr
== PHYS_MAP_NODE_NIL
) {
198 lp
->ptr
= phys_map_node_alloc(map
);
199 p
= map
->nodes
[lp
->ptr
];
201 for (i
= 0; i
< P_L2_SIZE
; i
++) {
203 p
[i
].ptr
= PHYS_SECTION_UNASSIGNED
;
207 p
= map
->nodes
[lp
->ptr
];
209 lp
= &p
[(*index
>> (level
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
211 while (*nb
&& lp
< &p
[P_L2_SIZE
]) {
212 if ((*index
& (step
- 1)) == 0 && *nb
>= step
) {
218 phys_page_set_level(map
, lp
, index
, nb
, leaf
, level
- 1);
224 static void phys_page_set(AddressSpaceDispatch
*d
,
225 hwaddr index
, hwaddr nb
,
228 /* Wildly overreserve - it doesn't matter much. */
229 phys_map_node_reserve(&d
->map
, 3 * P_L2_LEVELS
);
231 phys_page_set_level(&d
->map
, &d
->phys_map
, &index
, &nb
, leaf
, P_L2_LEVELS
- 1);
234 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
235 * and update our entry so we can skip it and go directly to the destination.
237 static void phys_page_compact(PhysPageEntry
*lp
, Node
*nodes
, unsigned long *compacted
)
239 unsigned valid_ptr
= P_L2_SIZE
;
244 if (lp
->ptr
== PHYS_MAP_NODE_NIL
) {
249 for (i
= 0; i
< P_L2_SIZE
; i
++) {
250 if (p
[i
].ptr
== PHYS_MAP_NODE_NIL
) {
257 phys_page_compact(&p
[i
], nodes
, compacted
);
261 /* We can only compress if there's only one child. */
266 assert(valid_ptr
< P_L2_SIZE
);
268 /* Don't compress if it won't fit in the # of bits we have. */
269 if (lp
->skip
+ p
[valid_ptr
].skip
>= (1 << 3)) {
273 lp
->ptr
= p
[valid_ptr
].ptr
;
274 if (!p
[valid_ptr
].skip
) {
275 /* If our only child is a leaf, make this a leaf. */
276 /* By design, we should have made this node a leaf to begin with so we
277 * should never reach here.
278 * But since it's so simple to handle this, let's do it just in case we
283 lp
->skip
+= p
[valid_ptr
].skip
;
287 static void phys_page_compact_all(AddressSpaceDispatch
*d
, int nodes_nb
)
289 DECLARE_BITMAP(compacted
, nodes_nb
);
291 if (d
->phys_map
.skip
) {
292 phys_page_compact(&d
->phys_map
, d
->map
.nodes
, compacted
);
296 static MemoryRegionSection
*phys_page_find(PhysPageEntry lp
, hwaddr addr
,
297 Node
*nodes
, MemoryRegionSection
*sections
)
300 hwaddr index
= addr
>> TARGET_PAGE_BITS
;
303 for (i
= P_L2_LEVELS
; lp
.skip
&& (i
-= lp
.skip
) >= 0;) {
304 if (lp
.ptr
== PHYS_MAP_NODE_NIL
) {
305 return §ions
[PHYS_SECTION_UNASSIGNED
];
308 lp
= p
[(index
>> (i
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
311 if (sections
[lp
.ptr
].size
.hi
||
312 range_covers_byte(sections
[lp
.ptr
].offset_within_address_space
,
313 sections
[lp
.ptr
].size
.lo
, addr
)) {
314 return §ions
[lp
.ptr
];
316 return §ions
[PHYS_SECTION_UNASSIGNED
];
320 bool memory_region_is_unassigned(MemoryRegion
*mr
)
322 return mr
!= &io_mem_rom
&& mr
!= &io_mem_notdirty
&& !mr
->rom_device
323 && mr
!= &io_mem_watch
;
326 /* Called from RCU critical section */
327 static MemoryRegionSection
*address_space_lookup_region(AddressSpaceDispatch
*d
,
329 bool resolve_subpage
)
331 MemoryRegionSection
*section
;
334 section
= phys_page_find(d
->phys_map
, addr
, d
->map
.nodes
, d
->map
.sections
);
335 if (resolve_subpage
&& section
->mr
->subpage
) {
336 subpage
= container_of(section
->mr
, subpage_t
, iomem
);
337 section
= &d
->map
.sections
[subpage
->sub_section
[SUBPAGE_IDX(addr
)]];
342 /* Called from RCU critical section */
343 static MemoryRegionSection
*
344 address_space_translate_internal(AddressSpaceDispatch
*d
, hwaddr addr
, hwaddr
*xlat
,
345 hwaddr
*plen
, bool resolve_subpage
)
347 MemoryRegionSection
*section
;
350 section
= address_space_lookup_region(d
, addr
, resolve_subpage
);
351 /* Compute offset within MemoryRegionSection */
352 addr
-= section
->offset_within_address_space
;
354 /* Compute offset within MemoryRegion */
355 *xlat
= addr
+ section
->offset_within_region
;
357 diff
= int128_sub(section
->mr
->size
, int128_make64(addr
));
358 *plen
= int128_get64(int128_min(diff
, int128_make64(*plen
)));
362 static inline bool memory_access_is_direct(MemoryRegion
*mr
, bool is_write
)
364 if (memory_region_is_ram(mr
)) {
365 return !(is_write
&& mr
->readonly
);
367 if (memory_region_is_romd(mr
)) {
374 MemoryRegion
*address_space_translate(AddressSpace
*as
, hwaddr addr
,
375 hwaddr
*xlat
, hwaddr
*plen
,
379 MemoryRegionSection
*section
;
385 AddressSpaceDispatch
*d
= atomic_rcu_read(&as
->dispatch
);
386 section
= address_space_translate_internal(d
, addr
, &addr
, plen
, true);
389 if (!mr
->iommu_ops
) {
393 iotlb
= mr
->iommu_ops
->translate(mr
, addr
, is_write
);
394 addr
= ((iotlb
.translated_addr
& ~iotlb
.addr_mask
)
395 | (addr
& iotlb
.addr_mask
));
396 len
= MIN(len
, (addr
| iotlb
.addr_mask
) - addr
+ 1);
397 if (!(iotlb
.perm
& (1 << is_write
))) {
398 mr
= &io_mem_unassigned
;
402 as
= iotlb
.target_as
;
405 if (xen_enabled() && memory_access_is_direct(mr
, is_write
)) {
406 hwaddr page
= ((addr
& TARGET_PAGE_MASK
) + TARGET_PAGE_SIZE
) - addr
;
407 len
= MIN(page
, len
);
416 /* Called from RCU critical section */
417 MemoryRegionSection
*
418 address_space_translate_for_iotlb(CPUState
*cpu
, hwaddr addr
,
419 hwaddr
*xlat
, hwaddr
*plen
)
421 MemoryRegionSection
*section
;
422 section
= address_space_translate_internal(cpu
->memory_dispatch
,
423 addr
, xlat
, plen
, false);
425 assert(!section
->mr
->iommu_ops
);
430 void cpu_exec_init_all(void)
432 #if !defined(CONFIG_USER_ONLY)
433 qemu_mutex_init(&ram_list
.mutex
);
439 #if !defined(CONFIG_USER_ONLY)
441 static int cpu_common_post_load(void *opaque
, int version_id
)
443 CPUState
*cpu
= opaque
;
445 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
446 version_id is increased. */
447 cpu
->interrupt_request
&= ~0x01;
453 static int cpu_common_pre_load(void *opaque
)
455 CPUState
*cpu
= opaque
;
457 cpu
->exception_index
= -1;
462 static bool cpu_common_exception_index_needed(void *opaque
)
464 CPUState
*cpu
= opaque
;
466 return tcg_enabled() && cpu
->exception_index
!= -1;
469 static const VMStateDescription vmstate_cpu_common_exception_index
= {
470 .name
= "cpu_common/exception_index",
472 .minimum_version_id
= 1,
473 .fields
= (VMStateField
[]) {
474 VMSTATE_INT32(exception_index
, CPUState
),
475 VMSTATE_END_OF_LIST()
479 const VMStateDescription vmstate_cpu_common
= {
480 .name
= "cpu_common",
482 .minimum_version_id
= 1,
483 .pre_load
= cpu_common_pre_load
,
484 .post_load
= cpu_common_post_load
,
485 .fields
= (VMStateField
[]) {
486 VMSTATE_UINT32(halted
, CPUState
),
487 VMSTATE_UINT32(interrupt_request
, CPUState
),
488 VMSTATE_END_OF_LIST()
490 .subsections
= (VMStateSubsection
[]) {
492 .vmsd
= &vmstate_cpu_common_exception_index
,
493 .needed
= cpu_common_exception_index_needed
,
502 CPUState
*qemu_get_cpu(int index
)
507 if (cpu
->cpu_index
== index
) {
515 #if !defined(CONFIG_USER_ONLY)
516 void tcg_cpu_address_space_init(CPUState
*cpu
, AddressSpace
*as
)
518 /* We only support one address space per cpu at the moment. */
519 assert(cpu
->as
== as
);
521 if (cpu
->tcg_as_listener
) {
522 memory_listener_unregister(cpu
->tcg_as_listener
);
524 cpu
->tcg_as_listener
= g_new0(MemoryListener
, 1);
526 cpu
->tcg_as_listener
->commit
= tcg_commit
;
527 memory_listener_register(cpu
->tcg_as_listener
, as
);
531 void cpu_exec_init(CPUArchState
*env
)
533 CPUState
*cpu
= ENV_GET_CPU(env
);
534 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
538 #if defined(CONFIG_USER_ONLY)
542 CPU_FOREACH(some_cpu
) {
545 cpu
->cpu_index
= cpu_index
;
547 QTAILQ_INIT(&cpu
->breakpoints
);
548 QTAILQ_INIT(&cpu
->watchpoints
);
549 #ifndef CONFIG_USER_ONLY
550 cpu
->as
= &address_space_memory
;
551 cpu
->thread_id
= qemu_get_thread_id();
553 QTAILQ_INSERT_TAIL(&cpus
, cpu
, node
);
554 #if defined(CONFIG_USER_ONLY)
557 if (qdev_get_vmsd(DEVICE(cpu
)) == NULL
) {
558 vmstate_register(NULL
, cpu_index
, &vmstate_cpu_common
, cpu
);
560 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
561 register_savevm(NULL
, "cpu", cpu_index
, CPU_SAVE_VERSION
,
562 cpu_save
, cpu_load
, env
);
563 assert(cc
->vmsd
== NULL
);
564 assert(qdev_get_vmsd(DEVICE(cpu
)) == NULL
);
566 if (cc
->vmsd
!= NULL
) {
567 vmstate_register(NULL
, cpu_index
, cc
->vmsd
, cpu
);
571 #if defined(CONFIG_USER_ONLY)
572 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
574 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
577 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
579 hwaddr phys
= cpu_get_phys_page_debug(cpu
, pc
);
581 tb_invalidate_phys_addr(cpu
->as
,
582 phys
| (pc
& ~TARGET_PAGE_MASK
));
587 #if defined(CONFIG_USER_ONLY)
588 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
593 int cpu_watchpoint_remove(CPUState
*cpu
, vaddr addr
, vaddr len
,
599 void cpu_watchpoint_remove_by_ref(CPUState
*cpu
, CPUWatchpoint
*watchpoint
)
603 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
604 int flags
, CPUWatchpoint
**watchpoint
)
609 /* Add a watchpoint. */
610 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
611 int flags
, CPUWatchpoint
**watchpoint
)
615 /* forbid ranges which are empty or run off the end of the address space */
616 if (len
== 0 || (addr
+ len
- 1) < addr
) {
617 error_report("tried to set invalid watchpoint at %"
618 VADDR_PRIx
", len=%" VADDR_PRIu
, addr
, len
);
621 wp
= g_malloc(sizeof(*wp
));
627 /* keep all GDB-injected watchpoints in front */
628 if (flags
& BP_GDB
) {
629 QTAILQ_INSERT_HEAD(&cpu
->watchpoints
, wp
, entry
);
631 QTAILQ_INSERT_TAIL(&cpu
->watchpoints
, wp
, entry
);
634 tlb_flush_page(cpu
, addr
);
641 /* Remove a specific watchpoint. */
642 int cpu_watchpoint_remove(CPUState
*cpu
, vaddr addr
, vaddr len
,
647 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
648 if (addr
== wp
->vaddr
&& len
== wp
->len
649 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
650 cpu_watchpoint_remove_by_ref(cpu
, wp
);
657 /* Remove a specific watchpoint by reference. */
658 void cpu_watchpoint_remove_by_ref(CPUState
*cpu
, CPUWatchpoint
*watchpoint
)
660 QTAILQ_REMOVE(&cpu
->watchpoints
, watchpoint
, entry
);
662 tlb_flush_page(cpu
, watchpoint
->vaddr
);
667 /* Remove all matching watchpoints. */
668 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
670 CPUWatchpoint
*wp
, *next
;
672 QTAILQ_FOREACH_SAFE(wp
, &cpu
->watchpoints
, entry
, next
) {
673 if (wp
->flags
& mask
) {
674 cpu_watchpoint_remove_by_ref(cpu
, wp
);
679 /* Return true if this watchpoint address matches the specified
680 * access (ie the address range covered by the watchpoint overlaps
681 * partially or completely with the address range covered by the
684 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint
*wp
,
688 /* We know the lengths are non-zero, but a little caution is
689 * required to avoid errors in the case where the range ends
690 * exactly at the top of the address space and so addr + len
691 * wraps round to zero.
693 vaddr wpend
= wp
->vaddr
+ wp
->len
- 1;
694 vaddr addrend
= addr
+ len
- 1;
696 return !(addr
> wpend
|| wp
->vaddr
> addrend
);
701 /* Add a breakpoint. */
702 int cpu_breakpoint_insert(CPUState
*cpu
, vaddr pc
, int flags
,
703 CPUBreakpoint
**breakpoint
)
707 bp
= g_malloc(sizeof(*bp
));
712 /* keep all GDB-injected breakpoints in front */
713 if (flags
& BP_GDB
) {
714 QTAILQ_INSERT_HEAD(&cpu
->breakpoints
, bp
, entry
);
716 QTAILQ_INSERT_TAIL(&cpu
->breakpoints
, bp
, entry
);
719 breakpoint_invalidate(cpu
, pc
);
727 /* Remove a specific breakpoint. */
728 int cpu_breakpoint_remove(CPUState
*cpu
, vaddr pc
, int flags
)
732 QTAILQ_FOREACH(bp
, &cpu
->breakpoints
, entry
) {
733 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
734 cpu_breakpoint_remove_by_ref(cpu
, bp
);
741 /* Remove a specific breakpoint by reference. */
742 void cpu_breakpoint_remove_by_ref(CPUState
*cpu
, CPUBreakpoint
*breakpoint
)
744 QTAILQ_REMOVE(&cpu
->breakpoints
, breakpoint
, entry
);
746 breakpoint_invalidate(cpu
, breakpoint
->pc
);
751 /* Remove all matching breakpoints. */
752 void cpu_breakpoint_remove_all(CPUState
*cpu
, int mask
)
754 CPUBreakpoint
*bp
, *next
;
756 QTAILQ_FOREACH_SAFE(bp
, &cpu
->breakpoints
, entry
, next
) {
757 if (bp
->flags
& mask
) {
758 cpu_breakpoint_remove_by_ref(cpu
, bp
);
763 /* enable or disable single step mode. EXCP_DEBUG is returned by the
764 CPU loop after each instruction */
765 void cpu_single_step(CPUState
*cpu
, int enabled
)
767 if (cpu
->singlestep_enabled
!= enabled
) {
768 cpu
->singlestep_enabled
= enabled
;
770 kvm_update_guest_debug(cpu
, 0);
772 /* must flush all the translated code to avoid inconsistencies */
773 /* XXX: only flush what is necessary */
774 CPUArchState
*env
= cpu
->env_ptr
;
780 void cpu_abort(CPUState
*cpu
, const char *fmt
, ...)
787 fprintf(stderr
, "qemu: fatal: ");
788 vfprintf(stderr
, fmt
, ap
);
789 fprintf(stderr
, "\n");
790 cpu_dump_state(cpu
, stderr
, fprintf
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
791 if (qemu_log_enabled()) {
792 qemu_log("qemu: fatal: ");
793 qemu_log_vprintf(fmt
, ap2
);
795 log_cpu_state(cpu
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
801 #if defined(CONFIG_USER_ONLY)
803 struct sigaction act
;
804 sigfillset(&act
.sa_mask
);
805 act
.sa_handler
= SIG_DFL
;
806 sigaction(SIGABRT
, &act
, NULL
);
812 #if !defined(CONFIG_USER_ONLY)
813 /* Called from RCU critical section */
814 static RAMBlock
*qemu_get_ram_block(ram_addr_t addr
)
818 block
= atomic_rcu_read(&ram_list
.mru_block
);
819 if (block
&& addr
- block
->offset
< block
->max_length
) {
822 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
823 if (addr
- block
->offset
< block
->max_length
) {
828 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
832 /* It is safe to write mru_block outside the iothread lock. This
837 * xxx removed from list
841 * call_rcu(reclaim_ramblock, xxx);
844 * atomic_rcu_set is not needed here. The block was already published
845 * when it was placed into the list. Here we're just making an extra
846 * copy of the pointer.
848 ram_list
.mru_block
= block
;
852 static void tlb_reset_dirty_range_all(ram_addr_t start
, ram_addr_t length
)
858 end
= TARGET_PAGE_ALIGN(start
+ length
);
859 start
&= TARGET_PAGE_MASK
;
862 block
= qemu_get_ram_block(start
);
863 assert(block
== qemu_get_ram_block(end
- 1));
864 start1
= (uintptr_t)ramblock_ptr(block
, start
- block
->offset
);
865 cpu_tlb_reset_dirty_all(start1
, length
);
869 /* Note: start and end must be within the same ram block. */
870 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t length
,
875 cpu_physical_memory_clear_dirty_range_type(start
, length
, client
);
878 tlb_reset_dirty_range_all(start
, length
);
882 static void cpu_physical_memory_set_dirty_tracking(bool enable
)
884 in_migration
= enable
;
887 /* Called from RCU critical section */
888 hwaddr
memory_region_section_get_iotlb(CPUState
*cpu
,
889 MemoryRegionSection
*section
,
891 hwaddr paddr
, hwaddr xlat
,
893 target_ulong
*address
)
898 if (memory_region_is_ram(section
->mr
)) {
900 iotlb
= (memory_region_get_ram_addr(section
->mr
) & TARGET_PAGE_MASK
)
902 if (!section
->readonly
) {
903 iotlb
|= PHYS_SECTION_NOTDIRTY
;
905 iotlb
|= PHYS_SECTION_ROM
;
908 iotlb
= section
- section
->address_space
->dispatch
->map
.sections
;
912 /* Make accesses to pages with watchpoints go via the
913 watchpoint trap routines. */
914 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
915 if (cpu_watchpoint_address_matches(wp
, vaddr
, TARGET_PAGE_SIZE
)) {
916 /* Avoid trapping reads of pages with a write breakpoint. */
917 if ((prot
& PAGE_WRITE
) || (wp
->flags
& BP_MEM_READ
)) {
918 iotlb
= PHYS_SECTION_WATCH
+ paddr
;
919 *address
|= TLB_MMIO
;
927 #endif /* defined(CONFIG_USER_ONLY) */
929 #if !defined(CONFIG_USER_ONLY)
931 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
933 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
);
935 static void *(*phys_mem_alloc
)(size_t size
, uint64_t *align
) =
939 * Set a custom physical guest memory alloator.
940 * Accelerators with unusual needs may need this. Hopefully, we can
941 * get rid of it eventually.
943 void phys_mem_set_alloc(void *(*alloc
)(size_t, uint64_t *align
))
945 phys_mem_alloc
= alloc
;
948 static uint16_t phys_section_add(PhysPageMap
*map
,
949 MemoryRegionSection
*section
)
951 /* The physical section number is ORed with a page-aligned
952 * pointer to produce the iotlb entries. Thus it should
953 * never overflow into the page-aligned value.
955 assert(map
->sections_nb
< TARGET_PAGE_SIZE
);
957 if (map
->sections_nb
== map
->sections_nb_alloc
) {
958 map
->sections_nb_alloc
= MAX(map
->sections_nb_alloc
* 2, 16);
959 map
->sections
= g_renew(MemoryRegionSection
, map
->sections
,
960 map
->sections_nb_alloc
);
962 map
->sections
[map
->sections_nb
] = *section
;
963 memory_region_ref(section
->mr
);
964 return map
->sections_nb
++;
967 static void phys_section_destroy(MemoryRegion
*mr
)
969 memory_region_unref(mr
);
972 subpage_t
*subpage
= container_of(mr
, subpage_t
, iomem
);
973 object_unref(OBJECT(&subpage
->iomem
));
978 static void phys_sections_free(PhysPageMap
*map
)
980 while (map
->sections_nb
> 0) {
981 MemoryRegionSection
*section
= &map
->sections
[--map
->sections_nb
];
982 phys_section_destroy(section
->mr
);
984 g_free(map
->sections
);
988 static void register_subpage(AddressSpaceDispatch
*d
, MemoryRegionSection
*section
)
991 hwaddr base
= section
->offset_within_address_space
993 MemoryRegionSection
*existing
= phys_page_find(d
->phys_map
, base
,
994 d
->map
.nodes
, d
->map
.sections
);
995 MemoryRegionSection subsection
= {
996 .offset_within_address_space
= base
,
997 .size
= int128_make64(TARGET_PAGE_SIZE
),
1001 assert(existing
->mr
->subpage
|| existing
->mr
== &io_mem_unassigned
);
1003 if (!(existing
->mr
->subpage
)) {
1004 subpage
= subpage_init(d
->as
, base
);
1005 subsection
.address_space
= d
->as
;
1006 subsection
.mr
= &subpage
->iomem
;
1007 phys_page_set(d
, base
>> TARGET_PAGE_BITS
, 1,
1008 phys_section_add(&d
->map
, &subsection
));
1010 subpage
= container_of(existing
->mr
, subpage_t
, iomem
);
1012 start
= section
->offset_within_address_space
& ~TARGET_PAGE_MASK
;
1013 end
= start
+ int128_get64(section
->size
) - 1;
1014 subpage_register(subpage
, start
, end
,
1015 phys_section_add(&d
->map
, section
));
1019 static void register_multipage(AddressSpaceDispatch
*d
,
1020 MemoryRegionSection
*section
)
1022 hwaddr start_addr
= section
->offset_within_address_space
;
1023 uint16_t section_index
= phys_section_add(&d
->map
, section
);
1024 uint64_t num_pages
= int128_get64(int128_rshift(section
->size
,
1028 phys_page_set(d
, start_addr
>> TARGET_PAGE_BITS
, num_pages
, section_index
);
1031 static void mem_add(MemoryListener
*listener
, MemoryRegionSection
*section
)
1033 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
1034 AddressSpaceDispatch
*d
= as
->next_dispatch
;
1035 MemoryRegionSection now
= *section
, remain
= *section
;
1036 Int128 page_size
= int128_make64(TARGET_PAGE_SIZE
);
1038 if (now
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
1039 uint64_t left
= TARGET_PAGE_ALIGN(now
.offset_within_address_space
)
1040 - now
.offset_within_address_space
;
1042 now
.size
= int128_min(int128_make64(left
), now
.size
);
1043 register_subpage(d
, &now
);
1045 now
.size
= int128_zero();
1047 while (int128_ne(remain
.size
, now
.size
)) {
1048 remain
.size
= int128_sub(remain
.size
, now
.size
);
1049 remain
.offset_within_address_space
+= int128_get64(now
.size
);
1050 remain
.offset_within_region
+= int128_get64(now
.size
);
1052 if (int128_lt(remain
.size
, page_size
)) {
1053 register_subpage(d
, &now
);
1054 } else if (remain
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
1055 now
.size
= page_size
;
1056 register_subpage(d
, &now
);
1058 now
.size
= int128_and(now
.size
, int128_neg(page_size
));
1059 register_multipage(d
, &now
);
1064 void qemu_flush_coalesced_mmio_buffer(void)
1067 kvm_flush_coalesced_mmio_buffer();
1070 void qemu_mutex_lock_ramlist(void)
1072 qemu_mutex_lock(&ram_list
.mutex
);
1075 void qemu_mutex_unlock_ramlist(void)
1077 qemu_mutex_unlock(&ram_list
.mutex
);
1082 #include <sys/vfs.h>
1084 #define HUGETLBFS_MAGIC 0x958458f6
1086 static long gethugepagesize(const char *path
, Error
**errp
)
1092 ret
= statfs(path
, &fs
);
1093 } while (ret
!= 0 && errno
== EINTR
);
1096 error_setg_errno(errp
, errno
, "failed to get page size of file %s",
1101 if (fs
.f_type
!= HUGETLBFS_MAGIC
)
1102 fprintf(stderr
, "Warning: path not on HugeTLBFS: %s\n", path
);
1107 static void *file_ram_alloc(RAMBlock
*block
,
1113 char *sanitized_name
;
1118 Error
*local_err
= NULL
;
1120 hpagesize
= gethugepagesize(path
, &local_err
);
1122 error_propagate(errp
, local_err
);
1125 block
->mr
->align
= hpagesize
;
1127 if (memory
< hpagesize
) {
1128 error_setg(errp
, "memory size 0x" RAM_ADDR_FMT
" must be equal to "
1129 "or larger than huge page size 0x%" PRIx64
,
1134 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1136 "host lacks kvm mmu notifiers, -mem-path unsupported");
1140 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1141 sanitized_name
= g_strdup(memory_region_name(block
->mr
));
1142 for (c
= sanitized_name
; *c
!= '\0'; c
++) {
1147 filename
= g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path
,
1149 g_free(sanitized_name
);
1151 fd
= mkstemp(filename
);
1153 error_setg_errno(errp
, errno
,
1154 "unable to create backing store for hugepages");
1161 memory
= (memory
+hpagesize
-1) & ~(hpagesize
-1);
1164 * ftruncate is not supported by hugetlbfs in older
1165 * hosts, so don't bother bailing out on errors.
1166 * If anything goes wrong with it under other filesystems,
1169 if (ftruncate(fd
, memory
)) {
1170 perror("ftruncate");
1173 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
,
1174 (block
->flags
& RAM_SHARED
? MAP_SHARED
: MAP_PRIVATE
),
1176 if (area
== MAP_FAILED
) {
1177 error_setg_errno(errp
, errno
,
1178 "unable to map backing store for hugepages");
1184 os_mem_prealloc(fd
, area
, memory
);
1192 error_report("%s", error_get_pretty(*errp
));
1199 /* Called with the ramlist lock held. */
1200 static ram_addr_t
find_ram_offset(ram_addr_t size
)
1202 RAMBlock
*block
, *next_block
;
1203 ram_addr_t offset
= RAM_ADDR_MAX
, mingap
= RAM_ADDR_MAX
;
1205 assert(size
!= 0); /* it would hand out same offset multiple times */
1207 if (QLIST_EMPTY_RCU(&ram_list
.blocks
)) {
1211 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1212 ram_addr_t end
, next
= RAM_ADDR_MAX
;
1214 end
= block
->offset
+ block
->max_length
;
1216 QLIST_FOREACH_RCU(next_block
, &ram_list
.blocks
, next
) {
1217 if (next_block
->offset
>= end
) {
1218 next
= MIN(next
, next_block
->offset
);
1221 if (next
- end
>= size
&& next
- end
< mingap
) {
1223 mingap
= next
- end
;
1227 if (offset
== RAM_ADDR_MAX
) {
1228 fprintf(stderr
, "Failed to find gap of requested size: %" PRIu64
"\n",
1236 ram_addr_t
last_ram_offset(void)
1239 ram_addr_t last
= 0;
1242 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1243 last
= MAX(last
, block
->offset
+ block
->max_length
);
1249 static void qemu_ram_setup_dump(void *addr
, ram_addr_t size
)
1253 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1254 if (!machine_dump_guest_core(current_machine
)) {
1255 ret
= qemu_madvise(addr
, size
, QEMU_MADV_DONTDUMP
);
1257 perror("qemu_madvise");
1258 fprintf(stderr
, "madvise doesn't support MADV_DONTDUMP, "
1259 "but dump_guest_core=off specified\n");
1264 /* Called within an RCU critical section, or while the ramlist lock
1267 static RAMBlock
*find_ram_block(ram_addr_t addr
)
1271 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1272 if (block
->offset
== addr
) {
1280 /* Called with iothread lock held. */
1281 void qemu_ram_set_idstr(ram_addr_t addr
, const char *name
, DeviceState
*dev
)
1283 RAMBlock
*new_block
, *block
;
1286 new_block
= find_ram_block(addr
);
1288 assert(!new_block
->idstr
[0]);
1291 char *id
= qdev_get_dev_path(dev
);
1293 snprintf(new_block
->idstr
, sizeof(new_block
->idstr
), "%s/", id
);
1297 pstrcat(new_block
->idstr
, sizeof(new_block
->idstr
), name
);
1299 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1300 if (block
!= new_block
&& !strcmp(block
->idstr
, new_block
->idstr
)) {
1301 fprintf(stderr
, "RAMBlock \"%s\" already registered, abort!\n",
1309 /* Called with iothread lock held. */
1310 void qemu_ram_unset_idstr(ram_addr_t addr
)
1314 /* FIXME: arch_init.c assumes that this is not called throughout
1315 * migration. Ignore the problem since hot-unplug during migration
1316 * does not work anyway.
1320 block
= find_ram_block(addr
);
1322 memset(block
->idstr
, 0, sizeof(block
->idstr
));
1327 static int memory_try_enable_merging(void *addr
, size_t len
)
1329 if (!machine_mem_merge(current_machine
)) {
1330 /* disabled by the user */
1334 return qemu_madvise(addr
, len
, QEMU_MADV_MERGEABLE
);
1337 /* Only legal before guest might have detected the memory size: e.g. on
1338 * incoming migration, or right after reset.
1340 * As memory core doesn't know how is memory accessed, it is up to
1341 * resize callback to update device state and/or add assertions to detect
1342 * misuse, if necessary.
1344 int qemu_ram_resize(ram_addr_t base
, ram_addr_t newsize
, Error
**errp
)
1346 RAMBlock
*block
= find_ram_block(base
);
1350 newsize
= TARGET_PAGE_ALIGN(newsize
);
1352 if (block
->used_length
== newsize
) {
1356 if (!(block
->flags
& RAM_RESIZEABLE
)) {
1357 error_setg_errno(errp
, EINVAL
,
1358 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1359 " in != 0x" RAM_ADDR_FMT
, block
->idstr
,
1360 newsize
, block
->used_length
);
1364 if (block
->max_length
< newsize
) {
1365 error_setg_errno(errp
, EINVAL
,
1366 "Length too large: %s: 0x" RAM_ADDR_FMT
1367 " > 0x" RAM_ADDR_FMT
, block
->idstr
,
1368 newsize
, block
->max_length
);
1372 cpu_physical_memory_clear_dirty_range(block
->offset
, block
->used_length
);
1373 block
->used_length
= newsize
;
1374 cpu_physical_memory_set_dirty_range(block
->offset
, block
->used_length
);
1375 memory_region_set_size(block
->mr
, newsize
);
1376 if (block
->resized
) {
1377 block
->resized(block
->idstr
, newsize
, block
->host
);
1382 static ram_addr_t
ram_block_add(RAMBlock
*new_block
, Error
**errp
)
1385 RAMBlock
*last_block
= NULL
;
1386 ram_addr_t old_ram_size
, new_ram_size
;
1388 old_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1390 qemu_mutex_lock_ramlist();
1391 new_block
->offset
= find_ram_offset(new_block
->max_length
);
1393 if (!new_block
->host
) {
1394 if (xen_enabled()) {
1395 xen_ram_alloc(new_block
->offset
, new_block
->max_length
,
1398 new_block
->host
= phys_mem_alloc(new_block
->max_length
,
1399 &new_block
->mr
->align
);
1400 if (!new_block
->host
) {
1401 error_setg_errno(errp
, errno
,
1402 "cannot set up guest memory '%s'",
1403 memory_region_name(new_block
->mr
));
1404 qemu_mutex_unlock_ramlist();
1407 memory_try_enable_merging(new_block
->host
, new_block
->max_length
);
1411 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1412 * QLIST (which has an RCU-friendly variant) does not have insertion at
1413 * tail, so save the last element in last_block.
1415 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1417 if (block
->max_length
< new_block
->max_length
) {
1422 QLIST_INSERT_BEFORE_RCU(block
, new_block
, next
);
1423 } else if (last_block
) {
1424 QLIST_INSERT_AFTER_RCU(last_block
, new_block
, next
);
1425 } else { /* list is empty */
1426 QLIST_INSERT_HEAD_RCU(&ram_list
.blocks
, new_block
, next
);
1428 ram_list
.mru_block
= NULL
;
1430 /* Write list before version */
1433 qemu_mutex_unlock_ramlist();
1435 new_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1437 if (new_ram_size
> old_ram_size
) {
1440 /* ram_list.dirty_memory[] is protected by the iothread lock. */
1441 for (i
= 0; i
< DIRTY_MEMORY_NUM
; i
++) {
1442 ram_list
.dirty_memory
[i
] =
1443 bitmap_zero_extend(ram_list
.dirty_memory
[i
],
1444 old_ram_size
, new_ram_size
);
1447 cpu_physical_memory_set_dirty_range(new_block
->offset
,
1448 new_block
->used_length
);
1450 if (new_block
->host
) {
1451 qemu_ram_setup_dump(new_block
->host
, new_block
->max_length
);
1452 qemu_madvise(new_block
->host
, new_block
->max_length
, QEMU_MADV_HUGEPAGE
);
1453 qemu_madvise(new_block
->host
, new_block
->max_length
, QEMU_MADV_DONTFORK
);
1454 if (kvm_enabled()) {
1455 kvm_setup_guest_memory(new_block
->host
, new_block
->max_length
);
1459 return new_block
->offset
;
1463 ram_addr_t
qemu_ram_alloc_from_file(ram_addr_t size
, MemoryRegion
*mr
,
1464 bool share
, const char *mem_path
,
1467 RAMBlock
*new_block
;
1469 Error
*local_err
= NULL
;
1471 if (xen_enabled()) {
1472 error_setg(errp
, "-mem-path not supported with Xen");
1476 if (phys_mem_alloc
!= qemu_anon_ram_alloc
) {
1478 * file_ram_alloc() needs to allocate just like
1479 * phys_mem_alloc, but we haven't bothered to provide
1483 "-mem-path not supported with this accelerator");
1487 size
= TARGET_PAGE_ALIGN(size
);
1488 new_block
= g_malloc0(sizeof(*new_block
));
1490 new_block
->used_length
= size
;
1491 new_block
->max_length
= size
;
1492 new_block
->flags
= share
? RAM_SHARED
: 0;
1493 new_block
->host
= file_ram_alloc(new_block
, size
,
1495 if (!new_block
->host
) {
1500 addr
= ram_block_add(new_block
, &local_err
);
1503 error_propagate(errp
, local_err
);
1511 ram_addr_t
qemu_ram_alloc_internal(ram_addr_t size
, ram_addr_t max_size
,
1512 void (*resized
)(const char*,
1515 void *host
, bool resizeable
,
1516 MemoryRegion
*mr
, Error
**errp
)
1518 RAMBlock
*new_block
;
1520 Error
*local_err
= NULL
;
1522 size
= TARGET_PAGE_ALIGN(size
);
1523 max_size
= TARGET_PAGE_ALIGN(max_size
);
1524 new_block
= g_malloc0(sizeof(*new_block
));
1526 new_block
->resized
= resized
;
1527 new_block
->used_length
= size
;
1528 new_block
->max_length
= max_size
;
1529 assert(max_size
>= size
);
1531 new_block
->host
= host
;
1533 new_block
->flags
|= RAM_PREALLOC
;
1536 new_block
->flags
|= RAM_RESIZEABLE
;
1538 addr
= ram_block_add(new_block
, &local_err
);
1541 error_propagate(errp
, local_err
);
1547 ram_addr_t
qemu_ram_alloc_from_ptr(ram_addr_t size
, void *host
,
1548 MemoryRegion
*mr
, Error
**errp
)
1550 return qemu_ram_alloc_internal(size
, size
, NULL
, host
, false, mr
, errp
);
1553 ram_addr_t
qemu_ram_alloc(ram_addr_t size
, MemoryRegion
*mr
, Error
**errp
)
1555 return qemu_ram_alloc_internal(size
, size
, NULL
, NULL
, false, mr
, errp
);
1558 ram_addr_t
qemu_ram_alloc_resizeable(ram_addr_t size
, ram_addr_t maxsz
,
1559 void (*resized
)(const char*,
1562 MemoryRegion
*mr
, Error
**errp
)
1564 return qemu_ram_alloc_internal(size
, maxsz
, resized
, NULL
, true, mr
, errp
);
1567 void qemu_ram_free_from_ptr(ram_addr_t addr
)
1571 qemu_mutex_lock_ramlist();
1572 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1573 if (addr
== block
->offset
) {
1574 QLIST_REMOVE_RCU(block
, next
);
1575 ram_list
.mru_block
= NULL
;
1576 /* Write list before version */
1579 g_free_rcu(block
, rcu
);
1583 qemu_mutex_unlock_ramlist();
1586 static void reclaim_ramblock(RAMBlock
*block
)
1588 if (block
->flags
& RAM_PREALLOC
) {
1590 } else if (xen_enabled()) {
1591 xen_invalidate_map_cache_entry(block
->host
);
1593 } else if (block
->fd
>= 0) {
1594 munmap(block
->host
, block
->max_length
);
1598 qemu_anon_ram_free(block
->host
, block
->max_length
);
1603 void qemu_ram_free(ram_addr_t addr
)
1607 qemu_mutex_lock_ramlist();
1608 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1609 if (addr
== block
->offset
) {
1610 QLIST_REMOVE_RCU(block
, next
);
1611 ram_list
.mru_block
= NULL
;
1612 /* Write list before version */
1615 call_rcu(block
, reclaim_ramblock
, rcu
);
1619 qemu_mutex_unlock_ramlist();
1623 void qemu_ram_remap(ram_addr_t addr
, ram_addr_t length
)
1630 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1631 offset
= addr
- block
->offset
;
1632 if (offset
< block
->max_length
) {
1633 vaddr
= ramblock_ptr(block
, offset
);
1634 if (block
->flags
& RAM_PREALLOC
) {
1636 } else if (xen_enabled()) {
1640 munmap(vaddr
, length
);
1641 if (block
->fd
>= 0) {
1642 flags
|= (block
->flags
& RAM_SHARED
?
1643 MAP_SHARED
: MAP_PRIVATE
);
1644 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1645 flags
, block
->fd
, offset
);
1648 * Remap needs to match alloc. Accelerators that
1649 * set phys_mem_alloc never remap. If they did,
1650 * we'd need a remap hook here.
1652 assert(phys_mem_alloc
== qemu_anon_ram_alloc
);
1654 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
1655 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1658 if (area
!= vaddr
) {
1659 fprintf(stderr
, "Could not remap addr: "
1660 RAM_ADDR_FMT
"@" RAM_ADDR_FMT
"\n",
1664 memory_try_enable_merging(vaddr
, length
);
1665 qemu_ram_setup_dump(vaddr
, length
);
1670 #endif /* !_WIN32 */
1672 int qemu_get_ram_fd(ram_addr_t addr
)
1678 block
= qemu_get_ram_block(addr
);
1684 void *qemu_get_ram_block_host_ptr(ram_addr_t addr
)
1690 block
= qemu_get_ram_block(addr
);
1691 ptr
= ramblock_ptr(block
, 0);
1696 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1697 * This should not be used for general purpose DMA. Use address_space_map
1698 * or address_space_rw instead. For local memory (e.g. video ram) that the
1699 * device owns, use memory_region_get_ram_ptr.
1701 * By the time this function returns, the returned pointer is not protected
1702 * by RCU anymore. If the caller is not within an RCU critical section and
1703 * does not hold the iothread lock, it must have other means of protecting the
1704 * pointer, such as a reference to the region that includes the incoming
1707 void *qemu_get_ram_ptr(ram_addr_t addr
)
1713 block
= qemu_get_ram_block(addr
);
1715 if (xen_enabled() && block
->host
== NULL
) {
1716 /* We need to check if the requested address is in the RAM
1717 * because we don't want to map the entire memory in QEMU.
1718 * In that case just map until the end of the page.
1720 if (block
->offset
== 0) {
1721 ptr
= xen_map_cache(addr
, 0, 0);
1725 block
->host
= xen_map_cache(block
->offset
, block
->max_length
, 1);
1727 ptr
= ramblock_ptr(block
, addr
- block
->offset
);
1734 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1735 * but takes a size argument.
1737 * By the time this function returns, the returned pointer is not protected
1738 * by RCU anymore. If the caller is not within an RCU critical section and
1739 * does not hold the iothread lock, it must have other means of protecting the
1740 * pointer, such as a reference to the region that includes the incoming
1743 static void *qemu_ram_ptr_length(ram_addr_t addr
, hwaddr
*size
)
1749 if (xen_enabled()) {
1750 return xen_map_cache(addr
, *size
, 1);
1754 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1755 if (addr
- block
->offset
< block
->max_length
) {
1756 if (addr
- block
->offset
+ *size
> block
->max_length
)
1757 *size
= block
->max_length
- addr
+ block
->offset
;
1758 ptr
= ramblock_ptr(block
, addr
- block
->offset
);
1764 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
1769 /* Some of the softmmu routines need to translate from a host pointer
1770 * (typically a TLB entry) back to a ram offset.
1772 * By the time this function returns, the returned pointer is not protected
1773 * by RCU anymore. If the caller is not within an RCU critical section and
1774 * does not hold the iothread lock, it must have other means of protecting the
1775 * pointer, such as a reference to the region that includes the incoming
1778 MemoryRegion
*qemu_ram_addr_from_host(void *ptr
, ram_addr_t
*ram_addr
)
1781 uint8_t *host
= ptr
;
1784 if (xen_enabled()) {
1786 *ram_addr
= xen_ram_addr_from_mapcache(ptr
);
1787 mr
= qemu_get_ram_block(*ram_addr
)->mr
;
1793 block
= atomic_rcu_read(&ram_list
.mru_block
);
1794 if (block
&& block
->host
&& host
- block
->host
< block
->max_length
) {
1798 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1799 /* This case append when the block is not mapped. */
1800 if (block
->host
== NULL
) {
1803 if (host
- block
->host
< block
->max_length
) {
1812 *ram_addr
= block
->offset
+ (host
- block
->host
);
1818 static void notdirty_mem_write(void *opaque
, hwaddr ram_addr
,
1819 uint64_t val
, unsigned size
)
1821 if (!cpu_physical_memory_get_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
)) {
1822 tb_invalidate_phys_page_fast(ram_addr
, size
);
1826 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
1829 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
1832 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
1837 cpu_physical_memory_set_dirty_range_nocode(ram_addr
, size
);
1838 /* we remove the notdirty callback only if the code has been
1840 if (!cpu_physical_memory_is_clean(ram_addr
)) {
1841 CPUArchState
*env
= current_cpu
->env_ptr
;
1842 tlb_set_dirty(env
, current_cpu
->mem_io_vaddr
);
1846 static bool notdirty_mem_accepts(void *opaque
, hwaddr addr
,
1847 unsigned size
, bool is_write
)
1852 static const MemoryRegionOps notdirty_mem_ops
= {
1853 .write
= notdirty_mem_write
,
1854 .valid
.accepts
= notdirty_mem_accepts
,
1855 .endianness
= DEVICE_NATIVE_ENDIAN
,
1858 /* Generate a debug exception if a watchpoint has been hit. */
1859 static void check_watchpoint(int offset
, int len
, int flags
)
1861 CPUState
*cpu
= current_cpu
;
1862 CPUArchState
*env
= cpu
->env_ptr
;
1863 target_ulong pc
, cs_base
;
1868 if (cpu
->watchpoint_hit
) {
1869 /* We re-entered the check after replacing the TB. Now raise
1870 * the debug interrupt so that is will trigger after the
1871 * current instruction. */
1872 cpu_interrupt(cpu
, CPU_INTERRUPT_DEBUG
);
1875 vaddr
= (cpu
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
1876 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
1877 if (cpu_watchpoint_address_matches(wp
, vaddr
, len
)
1878 && (wp
->flags
& flags
)) {
1879 if (flags
== BP_MEM_READ
) {
1880 wp
->flags
|= BP_WATCHPOINT_HIT_READ
;
1882 wp
->flags
|= BP_WATCHPOINT_HIT_WRITE
;
1884 wp
->hitaddr
= vaddr
;
1885 if (!cpu
->watchpoint_hit
) {
1886 cpu
->watchpoint_hit
= wp
;
1887 tb_check_watchpoint(cpu
);
1888 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
1889 cpu
->exception_index
= EXCP_DEBUG
;
1892 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
1893 tb_gen_code(cpu
, pc
, cs_base
, cpu_flags
, 1);
1894 cpu_resume_from_signal(cpu
, NULL
);
1898 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
1903 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1904 so these check for a hit then pass through to the normal out-of-line
1906 static uint64_t watch_mem_read(void *opaque
, hwaddr addr
,
1909 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, size
, BP_MEM_READ
);
1911 case 1: return ldub_phys(&address_space_memory
, addr
);
1912 case 2: return lduw_phys(&address_space_memory
, addr
);
1913 case 4: return ldl_phys(&address_space_memory
, addr
);
1918 static void watch_mem_write(void *opaque
, hwaddr addr
,
1919 uint64_t val
, unsigned size
)
1921 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, size
, BP_MEM_WRITE
);
1924 stb_phys(&address_space_memory
, addr
, val
);
1927 stw_phys(&address_space_memory
, addr
, val
);
1930 stl_phys(&address_space_memory
, addr
, val
);
1936 static const MemoryRegionOps watch_mem_ops
= {
1937 .read
= watch_mem_read
,
1938 .write
= watch_mem_write
,
1939 .endianness
= DEVICE_NATIVE_ENDIAN
,
1942 static uint64_t subpage_read(void *opaque
, hwaddr addr
,
1945 subpage_t
*subpage
= opaque
;
1948 #if defined(DEBUG_SUBPAGE)
1949 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
"\n", __func__
,
1950 subpage
, len
, addr
);
1952 address_space_read(subpage
->as
, addr
+ subpage
->base
, buf
, len
);
1967 static void subpage_write(void *opaque
, hwaddr addr
,
1968 uint64_t value
, unsigned len
)
1970 subpage_t
*subpage
= opaque
;
1973 #if defined(DEBUG_SUBPAGE)
1974 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
1975 " value %"PRIx64
"\n",
1976 __func__
, subpage
, len
, addr
, value
);
1994 address_space_write(subpage
->as
, addr
+ subpage
->base
, buf
, len
);
1997 static bool subpage_accepts(void *opaque
, hwaddr addr
,
1998 unsigned len
, bool is_write
)
2000 subpage_t
*subpage
= opaque
;
2001 #if defined(DEBUG_SUBPAGE)
2002 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx
"\n",
2003 __func__
, subpage
, is_write
? 'w' : 'r', len
, addr
);
2006 return address_space_access_valid(subpage
->as
, addr
+ subpage
->base
,
2010 static const MemoryRegionOps subpage_ops
= {
2011 .read
= subpage_read
,
2012 .write
= subpage_write
,
2013 .impl
.min_access_size
= 1,
2014 .impl
.max_access_size
= 8,
2015 .valid
.min_access_size
= 1,
2016 .valid
.max_access_size
= 8,
2017 .valid
.accepts
= subpage_accepts
,
2018 .endianness
= DEVICE_NATIVE_ENDIAN
,
2021 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2026 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
2028 idx
= SUBPAGE_IDX(start
);
2029 eidx
= SUBPAGE_IDX(end
);
2030 #if defined(DEBUG_SUBPAGE)
2031 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2032 __func__
, mmio
, start
, end
, idx
, eidx
, section
);
2034 for (; idx
<= eidx
; idx
++) {
2035 mmio
->sub_section
[idx
] = section
;
2041 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
)
2045 mmio
= g_malloc0(sizeof(subpage_t
));
2049 memory_region_init_io(&mmio
->iomem
, NULL
, &subpage_ops
, mmio
,
2050 NULL
, TARGET_PAGE_SIZE
);
2051 mmio
->iomem
.subpage
= true;
2052 #if defined(DEBUG_SUBPAGE)
2053 printf("%s: %p base " TARGET_FMT_plx
" len %08x\n", __func__
,
2054 mmio
, base
, TARGET_PAGE_SIZE
);
2056 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
-1, PHYS_SECTION_UNASSIGNED
);
2061 static uint16_t dummy_section(PhysPageMap
*map
, AddressSpace
*as
,
2065 MemoryRegionSection section
= {
2066 .address_space
= as
,
2068 .offset_within_address_space
= 0,
2069 .offset_within_region
= 0,
2070 .size
= int128_2_64(),
2073 return phys_section_add(map
, §ion
);
2076 MemoryRegion
*iotlb_to_region(CPUState
*cpu
, hwaddr index
)
2078 AddressSpaceDispatch
*d
= atomic_rcu_read(&cpu
->memory_dispatch
);
2079 MemoryRegionSection
*sections
= d
->map
.sections
;
2081 return sections
[index
& ~TARGET_PAGE_MASK
].mr
;
2084 static void io_mem_init(void)
2086 memory_region_init_io(&io_mem_rom
, NULL
, &unassigned_mem_ops
, NULL
, NULL
, UINT64_MAX
);
2087 memory_region_init_io(&io_mem_unassigned
, NULL
, &unassigned_mem_ops
, NULL
,
2089 memory_region_init_io(&io_mem_notdirty
, NULL
, ¬dirty_mem_ops
, NULL
,
2091 memory_region_init_io(&io_mem_watch
, NULL
, &watch_mem_ops
, NULL
,
2095 static void mem_begin(MemoryListener
*listener
)
2097 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
2098 AddressSpaceDispatch
*d
= g_new0(AddressSpaceDispatch
, 1);
2101 n
= dummy_section(&d
->map
, as
, &io_mem_unassigned
);
2102 assert(n
== PHYS_SECTION_UNASSIGNED
);
2103 n
= dummy_section(&d
->map
, as
, &io_mem_notdirty
);
2104 assert(n
== PHYS_SECTION_NOTDIRTY
);
2105 n
= dummy_section(&d
->map
, as
, &io_mem_rom
);
2106 assert(n
== PHYS_SECTION_ROM
);
2107 n
= dummy_section(&d
->map
, as
, &io_mem_watch
);
2108 assert(n
== PHYS_SECTION_WATCH
);
2110 d
->phys_map
= (PhysPageEntry
) { .ptr
= PHYS_MAP_NODE_NIL
, .skip
= 1 };
2112 as
->next_dispatch
= d
;
2115 static void address_space_dispatch_free(AddressSpaceDispatch
*d
)
2117 phys_sections_free(&d
->map
);
2121 static void mem_commit(MemoryListener
*listener
)
2123 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
2124 AddressSpaceDispatch
*cur
= as
->dispatch
;
2125 AddressSpaceDispatch
*next
= as
->next_dispatch
;
2127 phys_page_compact_all(next
, next
->map
.nodes_nb
);
2129 atomic_rcu_set(&as
->dispatch
, next
);
2131 call_rcu(cur
, address_space_dispatch_free
, rcu
);
2135 static void tcg_commit(MemoryListener
*listener
)
2139 /* since each CPU stores ram addresses in its TLB cache, we must
2140 reset the modified entries */
2143 /* FIXME: Disentangle the cpu.h circular files deps so we can
2144 directly get the right CPU from listener. */
2145 if (cpu
->tcg_as_listener
!= listener
) {
2148 cpu_reload_memory_map(cpu
);
2152 static void core_log_global_start(MemoryListener
*listener
)
2154 cpu_physical_memory_set_dirty_tracking(true);
2157 static void core_log_global_stop(MemoryListener
*listener
)
2159 cpu_physical_memory_set_dirty_tracking(false);
2162 static MemoryListener core_memory_listener
= {
2163 .log_global_start
= core_log_global_start
,
2164 .log_global_stop
= core_log_global_stop
,
2168 void address_space_init_dispatch(AddressSpace
*as
)
2170 as
->dispatch
= NULL
;
2171 as
->dispatch_listener
= (MemoryListener
) {
2173 .commit
= mem_commit
,
2174 .region_add
= mem_add
,
2175 .region_nop
= mem_add
,
2178 memory_listener_register(&as
->dispatch_listener
, as
);
2181 void address_space_unregister(AddressSpace
*as
)
2183 memory_listener_unregister(&as
->dispatch_listener
);
2186 void address_space_destroy_dispatch(AddressSpace
*as
)
2188 AddressSpaceDispatch
*d
= as
->dispatch
;
2190 atomic_rcu_set(&as
->dispatch
, NULL
);
2192 call_rcu(d
, address_space_dispatch_free
, rcu
);
2196 static void memory_map_init(void)
2198 system_memory
= g_malloc(sizeof(*system_memory
));
2200 memory_region_init(system_memory
, NULL
, "system", UINT64_MAX
);
2201 address_space_init(&address_space_memory
, system_memory
, "memory");
2203 system_io
= g_malloc(sizeof(*system_io
));
2204 memory_region_init_io(system_io
, NULL
, &unassigned_io_ops
, NULL
, "io",
2206 address_space_init(&address_space_io
, system_io
, "I/O");
2208 memory_listener_register(&core_memory_listener
, &address_space_memory
);
2211 MemoryRegion
*get_system_memory(void)
2213 return system_memory
;
2216 MemoryRegion
*get_system_io(void)
2221 #endif /* !defined(CONFIG_USER_ONLY) */
2223 /* physical memory access (slow version, mainly for debug) */
2224 #if defined(CONFIG_USER_ONLY)
2225 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
2226 uint8_t *buf
, int len
, int is_write
)
2233 page
= addr
& TARGET_PAGE_MASK
;
2234 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2237 flags
= page_get_flags(page
);
2238 if (!(flags
& PAGE_VALID
))
2241 if (!(flags
& PAGE_WRITE
))
2243 /* XXX: this code should not depend on lock_user */
2244 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
2247 unlock_user(p
, addr
, l
);
2249 if (!(flags
& PAGE_READ
))
2251 /* XXX: this code should not depend on lock_user */
2252 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
2255 unlock_user(p
, addr
, 0);
2266 static void invalidate_and_set_dirty(hwaddr addr
,
2269 if (cpu_physical_memory_range_includes_clean(addr
, length
)) {
2270 tb_invalidate_phys_range(addr
, addr
+ length
, 0);
2271 cpu_physical_memory_set_dirty_range_nocode(addr
, length
);
2273 xen_modified_memory(addr
, length
);
2276 static int memory_access_size(MemoryRegion
*mr
, unsigned l
, hwaddr addr
)
2278 unsigned access_size_max
= mr
->ops
->valid
.max_access_size
;
2280 /* Regions are assumed to support 1-4 byte accesses unless
2281 otherwise specified. */
2282 if (access_size_max
== 0) {
2283 access_size_max
= 4;
2286 /* Bound the maximum access by the alignment of the address. */
2287 if (!mr
->ops
->impl
.unaligned
) {
2288 unsigned align_size_max
= addr
& -addr
;
2289 if (align_size_max
!= 0 && align_size_max
< access_size_max
) {
2290 access_size_max
= align_size_max
;
2294 /* Don't attempt accesses larger than the maximum. */
2295 if (l
> access_size_max
) {
2296 l
= access_size_max
;
2299 l
= 1 << (qemu_fls(l
) - 1);
2305 bool address_space_rw(AddressSpace
*as
, hwaddr addr
, uint8_t *buf
,
2306 int len
, bool is_write
)
2317 mr
= address_space_translate(as
, addr
, &addr1
, &l
, is_write
);
2320 if (!memory_access_is_direct(mr
, is_write
)) {
2321 l
= memory_access_size(mr
, l
, addr1
);
2322 /* XXX: could force current_cpu to NULL to avoid
2326 /* 64 bit write access */
2328 error
|= io_mem_write(mr
, addr1
, val
, 8);
2331 /* 32 bit write access */
2333 error
|= io_mem_write(mr
, addr1
, val
, 4);
2336 /* 16 bit write access */
2338 error
|= io_mem_write(mr
, addr1
, val
, 2);
2341 /* 8 bit write access */
2343 error
|= io_mem_write(mr
, addr1
, val
, 1);
2349 addr1
+= memory_region_get_ram_addr(mr
);
2351 ptr
= qemu_get_ram_ptr(addr1
);
2352 memcpy(ptr
, buf
, l
);
2353 invalidate_and_set_dirty(addr1
, l
);
2356 if (!memory_access_is_direct(mr
, is_write
)) {
2358 l
= memory_access_size(mr
, l
, addr1
);
2361 /* 64 bit read access */
2362 error
|= io_mem_read(mr
, addr1
, &val
, 8);
2366 /* 32 bit read access */
2367 error
|= io_mem_read(mr
, addr1
, &val
, 4);
2371 /* 16 bit read access */
2372 error
|= io_mem_read(mr
, addr1
, &val
, 2);
2376 /* 8 bit read access */
2377 error
|= io_mem_read(mr
, addr1
, &val
, 1);
2385 ptr
= qemu_get_ram_ptr(mr
->ram_addr
+ addr1
);
2386 memcpy(buf
, ptr
, l
);
2397 bool address_space_write(AddressSpace
*as
, hwaddr addr
,
2398 const uint8_t *buf
, int len
)
2400 return address_space_rw(as
, addr
, (uint8_t *)buf
, len
, true);
2403 bool address_space_read(AddressSpace
*as
, hwaddr addr
, uint8_t *buf
, int len
)
2405 return address_space_rw(as
, addr
, buf
, len
, false);
2409 void cpu_physical_memory_rw(hwaddr addr
, uint8_t *buf
,
2410 int len
, int is_write
)
2412 address_space_rw(&address_space_memory
, addr
, buf
, len
, is_write
);
2415 enum write_rom_type
{
2420 static inline void cpu_physical_memory_write_rom_internal(AddressSpace
*as
,
2421 hwaddr addr
, const uint8_t *buf
, int len
, enum write_rom_type type
)
2430 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
2432 if (!(memory_region_is_ram(mr
) ||
2433 memory_region_is_romd(mr
))) {
2436 addr1
+= memory_region_get_ram_addr(mr
);
2438 ptr
= qemu_get_ram_ptr(addr1
);
2441 memcpy(ptr
, buf
, l
);
2442 invalidate_and_set_dirty(addr1
, l
);
2445 flush_icache_range((uintptr_t)ptr
, (uintptr_t)ptr
+ l
);
2455 /* used for ROM loading : can write in RAM and ROM */
2456 void cpu_physical_memory_write_rom(AddressSpace
*as
, hwaddr addr
,
2457 const uint8_t *buf
, int len
)
2459 cpu_physical_memory_write_rom_internal(as
, addr
, buf
, len
, WRITE_DATA
);
2462 void cpu_flush_icache_range(hwaddr start
, int len
)
2465 * This function should do the same thing as an icache flush that was
2466 * triggered from within the guest. For TCG we are always cache coherent,
2467 * so there is no need to flush anything. For KVM / Xen we need to flush
2468 * the host's instruction cache at least.
2470 if (tcg_enabled()) {
2474 cpu_physical_memory_write_rom_internal(&address_space_memory
,
2475 start
, NULL
, len
, FLUSH_CACHE
);
2485 static BounceBuffer bounce
;
2487 typedef struct MapClient
{
2489 void (*callback
)(void *opaque
);
2490 QLIST_ENTRY(MapClient
) link
;
2493 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
2494 = QLIST_HEAD_INITIALIZER(map_client_list
);
2496 void *cpu_register_map_client(void *opaque
, void (*callback
)(void *opaque
))
2498 MapClient
*client
= g_malloc(sizeof(*client
));
2500 client
->opaque
= opaque
;
2501 client
->callback
= callback
;
2502 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
2506 static void cpu_unregister_map_client(void *_client
)
2508 MapClient
*client
= (MapClient
*)_client
;
2510 QLIST_REMOVE(client
, link
);
2514 static void cpu_notify_map_clients(void)
2518 while (!QLIST_EMPTY(&map_client_list
)) {
2519 client
= QLIST_FIRST(&map_client_list
);
2520 client
->callback(client
->opaque
);
2521 cpu_unregister_map_client(client
);
2525 bool address_space_access_valid(AddressSpace
*as
, hwaddr addr
, int len
, bool is_write
)
2532 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2533 if (!memory_access_is_direct(mr
, is_write
)) {
2534 l
= memory_access_size(mr
, l
, addr
);
2535 if (!memory_region_access_valid(mr
, xlat
, l
, is_write
)) {
2546 /* Map a physical memory region into a host virtual address.
2547 * May map a subset of the requested range, given by and returned in *plen.
2548 * May return NULL if resources needed to perform the mapping are exhausted.
2549 * Use only for reads OR writes - not for read-modify-write operations.
2550 * Use cpu_register_map_client() to know when retrying the map operation is
2551 * likely to succeed.
2553 void *address_space_map(AddressSpace
*as
,
2560 hwaddr l
, xlat
, base
;
2561 MemoryRegion
*mr
, *this_mr
;
2569 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2570 if (!memory_access_is_direct(mr
, is_write
)) {
2571 if (bounce
.buffer
) {
2574 /* Avoid unbounded allocations */
2575 l
= MIN(l
, TARGET_PAGE_SIZE
);
2576 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, l
);
2580 memory_region_ref(mr
);
2583 address_space_read(as
, addr
, bounce
.buffer
, l
);
2587 return bounce
.buffer
;
2591 raddr
= memory_region_get_ram_addr(mr
);
2602 this_mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2603 if (this_mr
!= mr
|| xlat
!= base
+ done
) {
2608 memory_region_ref(mr
);
2610 return qemu_ram_ptr_length(raddr
+ base
, plen
);
2613 /* Unmaps a memory region previously mapped by address_space_map().
2614 * Will also mark the memory as dirty if is_write == 1. access_len gives
2615 * the amount of memory that was actually read or written by the caller.
2617 void address_space_unmap(AddressSpace
*as
, void *buffer
, hwaddr len
,
2618 int is_write
, hwaddr access_len
)
2620 if (buffer
!= bounce
.buffer
) {
2624 mr
= qemu_ram_addr_from_host(buffer
, &addr1
);
2627 invalidate_and_set_dirty(addr1
, access_len
);
2629 if (xen_enabled()) {
2630 xen_invalidate_map_cache_entry(buffer
);
2632 memory_region_unref(mr
);
2636 address_space_write(as
, bounce
.addr
, bounce
.buffer
, access_len
);
2638 qemu_vfree(bounce
.buffer
);
2639 bounce
.buffer
= NULL
;
2640 memory_region_unref(bounce
.mr
);
2641 cpu_notify_map_clients();
2644 void *cpu_physical_memory_map(hwaddr addr
,
2648 return address_space_map(&address_space_memory
, addr
, plen
, is_write
);
2651 void cpu_physical_memory_unmap(void *buffer
, hwaddr len
,
2652 int is_write
, hwaddr access_len
)
2654 return address_space_unmap(&address_space_memory
, buffer
, len
, is_write
, access_len
);
2657 /* warning: addr must be aligned */
2658 static inline uint32_t ldl_phys_internal(AddressSpace
*as
, hwaddr addr
,
2659 enum device_endian endian
)
2667 mr
= address_space_translate(as
, addr
, &addr1
, &l
, false);
2668 if (l
< 4 || !memory_access_is_direct(mr
, false)) {
2670 io_mem_read(mr
, addr1
, &val
, 4);
2671 #if defined(TARGET_WORDS_BIGENDIAN)
2672 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2676 if (endian
== DEVICE_BIG_ENDIAN
) {
2682 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2686 case DEVICE_LITTLE_ENDIAN
:
2687 val
= ldl_le_p(ptr
);
2689 case DEVICE_BIG_ENDIAN
:
2690 val
= ldl_be_p(ptr
);
2700 uint32_t ldl_phys(AddressSpace
*as
, hwaddr addr
)
2702 return ldl_phys_internal(as
, addr
, DEVICE_NATIVE_ENDIAN
);
2705 uint32_t ldl_le_phys(AddressSpace
*as
, hwaddr addr
)
2707 return ldl_phys_internal(as
, addr
, DEVICE_LITTLE_ENDIAN
);
2710 uint32_t ldl_be_phys(AddressSpace
*as
, hwaddr addr
)
2712 return ldl_phys_internal(as
, addr
, DEVICE_BIG_ENDIAN
);
2715 /* warning: addr must be aligned */
2716 static inline uint64_t ldq_phys_internal(AddressSpace
*as
, hwaddr addr
,
2717 enum device_endian endian
)
2725 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2727 if (l
< 8 || !memory_access_is_direct(mr
, false)) {
2729 io_mem_read(mr
, addr1
, &val
, 8);
2730 #if defined(TARGET_WORDS_BIGENDIAN)
2731 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2735 if (endian
== DEVICE_BIG_ENDIAN
) {
2741 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2745 case DEVICE_LITTLE_ENDIAN
:
2746 val
= ldq_le_p(ptr
);
2748 case DEVICE_BIG_ENDIAN
:
2749 val
= ldq_be_p(ptr
);
2759 uint64_t ldq_phys(AddressSpace
*as
, hwaddr addr
)
2761 return ldq_phys_internal(as
, addr
, DEVICE_NATIVE_ENDIAN
);
2764 uint64_t ldq_le_phys(AddressSpace
*as
, hwaddr addr
)
2766 return ldq_phys_internal(as
, addr
, DEVICE_LITTLE_ENDIAN
);
2769 uint64_t ldq_be_phys(AddressSpace
*as
, hwaddr addr
)
2771 return ldq_phys_internal(as
, addr
, DEVICE_BIG_ENDIAN
);
2775 uint32_t ldub_phys(AddressSpace
*as
, hwaddr addr
)
2778 address_space_rw(as
, addr
, &val
, 1, 0);
2782 /* warning: addr must be aligned */
2783 static inline uint32_t lduw_phys_internal(AddressSpace
*as
, hwaddr addr
,
2784 enum device_endian endian
)
2792 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2794 if (l
< 2 || !memory_access_is_direct(mr
, false)) {
2796 io_mem_read(mr
, addr1
, &val
, 2);
2797 #if defined(TARGET_WORDS_BIGENDIAN)
2798 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2802 if (endian
== DEVICE_BIG_ENDIAN
) {
2808 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2812 case DEVICE_LITTLE_ENDIAN
:
2813 val
= lduw_le_p(ptr
);
2815 case DEVICE_BIG_ENDIAN
:
2816 val
= lduw_be_p(ptr
);
2826 uint32_t lduw_phys(AddressSpace
*as
, hwaddr addr
)
2828 return lduw_phys_internal(as
, addr
, DEVICE_NATIVE_ENDIAN
);
2831 uint32_t lduw_le_phys(AddressSpace
*as
, hwaddr addr
)
2833 return lduw_phys_internal(as
, addr
, DEVICE_LITTLE_ENDIAN
);
2836 uint32_t lduw_be_phys(AddressSpace
*as
, hwaddr addr
)
2838 return lduw_phys_internal(as
, addr
, DEVICE_BIG_ENDIAN
);
2841 /* warning: addr must be aligned. The ram page is not masked as dirty
2842 and the code inside is not invalidated. It is useful if the dirty
2843 bits are used to track modified PTEs */
2844 void stl_phys_notdirty(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2851 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2853 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
2854 io_mem_write(mr
, addr1
, val
, 4);
2856 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
2857 ptr
= qemu_get_ram_ptr(addr1
);
2860 if (unlikely(in_migration
)) {
2861 if (cpu_physical_memory_is_clean(addr1
)) {
2862 /* invalidate code */
2863 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
2865 cpu_physical_memory_set_dirty_range_nocode(addr1
, 4);
2871 /* warning: addr must be aligned */
2872 static inline void stl_phys_internal(AddressSpace
*as
,
2873 hwaddr addr
, uint32_t val
,
2874 enum device_endian endian
)
2881 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2883 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
2884 #if defined(TARGET_WORDS_BIGENDIAN)
2885 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2889 if (endian
== DEVICE_BIG_ENDIAN
) {
2893 io_mem_write(mr
, addr1
, val
, 4);
2896 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
2897 ptr
= qemu_get_ram_ptr(addr1
);
2899 case DEVICE_LITTLE_ENDIAN
:
2902 case DEVICE_BIG_ENDIAN
:
2909 invalidate_and_set_dirty(addr1
, 4);
2913 void stl_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2915 stl_phys_internal(as
, addr
, val
, DEVICE_NATIVE_ENDIAN
);
2918 void stl_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2920 stl_phys_internal(as
, addr
, val
, DEVICE_LITTLE_ENDIAN
);
2923 void stl_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2925 stl_phys_internal(as
, addr
, val
, DEVICE_BIG_ENDIAN
);
2929 void stb_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2932 address_space_rw(as
, addr
, &v
, 1, 1);
2935 /* warning: addr must be aligned */
2936 static inline void stw_phys_internal(AddressSpace
*as
,
2937 hwaddr addr
, uint32_t val
,
2938 enum device_endian endian
)
2945 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
2946 if (l
< 2 || !memory_access_is_direct(mr
, true)) {
2947 #if defined(TARGET_WORDS_BIGENDIAN)
2948 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2952 if (endian
== DEVICE_BIG_ENDIAN
) {
2956 io_mem_write(mr
, addr1
, val
, 2);
2959 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
2960 ptr
= qemu_get_ram_ptr(addr1
);
2962 case DEVICE_LITTLE_ENDIAN
:
2965 case DEVICE_BIG_ENDIAN
:
2972 invalidate_and_set_dirty(addr1
, 2);
2976 void stw_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2978 stw_phys_internal(as
, addr
, val
, DEVICE_NATIVE_ENDIAN
);
2981 void stw_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2983 stw_phys_internal(as
, addr
, val
, DEVICE_LITTLE_ENDIAN
);
2986 void stw_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2988 stw_phys_internal(as
, addr
, val
, DEVICE_BIG_ENDIAN
);
2992 void stq_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
2995 address_space_rw(as
, addr
, (void *) &val
, 8, 1);
2998 void stq_le_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3000 val
= cpu_to_le64(val
);
3001 address_space_rw(as
, addr
, (void *) &val
, 8, 1);
3004 void stq_be_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3006 val
= cpu_to_be64(val
);
3007 address_space_rw(as
, addr
, (void *) &val
, 8, 1);
3010 /* virtual memory access for debug (includes writing to ROM) */
3011 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
3012 uint8_t *buf
, int len
, int is_write
)
3019 page
= addr
& TARGET_PAGE_MASK
;
3020 phys_addr
= cpu_get_phys_page_debug(cpu
, page
);
3021 /* if no physical page mapped, return an error */
3022 if (phys_addr
== -1)
3024 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3027 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
3029 cpu_physical_memory_write_rom(cpu
->as
, phys_addr
, buf
, l
);
3031 address_space_rw(cpu
->as
, phys_addr
, buf
, l
, 0);
3042 * A helper function for the _utterly broken_ virtio device model to find out if
3043 * it's running on a big endian machine. Don't do this at home kids!
3045 bool target_words_bigendian(void);
3046 bool target_words_bigendian(void)
3048 #if defined(TARGET_WORDS_BIGENDIAN)
3055 #ifndef CONFIG_USER_ONLY
3056 bool cpu_physical_memory_is_io(hwaddr phys_addr
)
3061 mr
= address_space_translate(&address_space_memory
,
3062 phys_addr
, &phys_addr
, &l
, false);
3064 return !(memory_region_is_ram(mr
) ||
3065 memory_region_is_romd(mr
));
3068 void qemu_ram_foreach_block(RAMBlockIterFunc func
, void *opaque
)
3073 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
3074 func(block
->host
, block
->offset
, block
->used_length
, opaque
);