4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
21 #include <sys/types.h>
25 #include "qemu-common.h"
29 #if !defined(CONFIG_USER_ONLY)
30 #include "hw/boards.h"
33 #include "qemu/osdep.h"
34 #include "sysemu/kvm.h"
35 #include "sysemu/sysemu.h"
36 #include "hw/xen/xen.h"
37 #include "qemu/timer.h"
38 #include "qemu/config-file.h"
39 #include "qemu/error-report.h"
40 #include "exec/memory.h"
41 #include "sysemu/dma.h"
42 #include "exec/address-spaces.h"
43 #if defined(CONFIG_USER_ONLY)
45 #else /* !CONFIG_USER_ONLY */
46 #include "sysemu/xen-mapcache.h"
49 #include "exec/cpu-all.h"
50 #include "qemu/rcu_queue.h"
51 #include "exec/cputlb.h"
52 #include "translate-all.h"
54 #include "exec/memory-internal.h"
55 #include "exec/ram_addr.h"
57 #include "qemu/range.h"
59 //#define DEBUG_SUBPAGE
61 #if !defined(CONFIG_USER_ONLY)
62 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
63 * are protected by the ramlist lock.
65 RAMList ram_list
= { .blocks
= QLIST_HEAD_INITIALIZER(ram_list
.blocks
) };
67 static MemoryRegion
*system_memory
;
68 static MemoryRegion
*system_io
;
70 AddressSpace address_space_io
;
71 AddressSpace address_space_memory
;
73 MemoryRegion io_mem_rom
, io_mem_notdirty
;
74 static MemoryRegion io_mem_unassigned
;
76 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
77 #define RAM_PREALLOC (1 << 0)
79 /* RAM is mmap-ed with MAP_SHARED */
80 #define RAM_SHARED (1 << 1)
82 /* Only a portion of RAM (used_length) is actually used, and migrated.
83 * This used_length size can change across reboots.
85 #define RAM_RESIZEABLE (1 << 2)
89 struct CPUTailQ cpus
= QTAILQ_HEAD_INITIALIZER(cpus
);
90 /* current CPU in the current thread. It is only valid inside
92 DEFINE_TLS(CPUState
*, current_cpu
);
93 /* 0 = Do not count executed instructions.
94 1 = Precise instruction counting.
95 2 = Adaptive rate instruction counting. */
98 #if !defined(CONFIG_USER_ONLY)
100 typedef struct PhysPageEntry PhysPageEntry
;
102 struct PhysPageEntry
{
103 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
105 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
109 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
111 /* Size of the L2 (and L3, etc) page tables. */
112 #define ADDR_SPACE_BITS 64
115 #define P_L2_SIZE (1 << P_L2_BITS)
117 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
119 typedef PhysPageEntry Node
[P_L2_SIZE
];
121 typedef struct PhysPageMap
{
124 unsigned sections_nb
;
125 unsigned sections_nb_alloc
;
127 unsigned nodes_nb_alloc
;
129 MemoryRegionSection
*sections
;
132 struct AddressSpaceDispatch
{
135 /* This is a multi-level map on the physical address space.
136 * The bottom level has pointers to MemoryRegionSections.
138 PhysPageEntry phys_map
;
143 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
144 typedef struct subpage_t
{
148 uint16_t sub_section
[TARGET_PAGE_SIZE
];
151 #define PHYS_SECTION_UNASSIGNED 0
152 #define PHYS_SECTION_NOTDIRTY 1
153 #define PHYS_SECTION_ROM 2
154 #define PHYS_SECTION_WATCH 3
156 static void io_mem_init(void);
157 static void memory_map_init(void);
158 static void tcg_commit(MemoryListener
*listener
);
160 static MemoryRegion io_mem_watch
;
163 #if !defined(CONFIG_USER_ONLY)
165 static void phys_map_node_reserve(PhysPageMap
*map
, unsigned nodes
)
167 if (map
->nodes_nb
+ nodes
> map
->nodes_nb_alloc
) {
168 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
* 2, 16);
169 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
, map
->nodes_nb
+ nodes
);
170 map
->nodes
= g_renew(Node
, map
->nodes
, map
->nodes_nb_alloc
);
174 static uint32_t phys_map_node_alloc(PhysPageMap
*map
, bool leaf
)
181 ret
= map
->nodes_nb
++;
183 assert(ret
!= PHYS_MAP_NODE_NIL
);
184 assert(ret
!= map
->nodes_nb_alloc
);
186 e
.skip
= leaf
? 0 : 1;
187 e
.ptr
= leaf
? PHYS_SECTION_UNASSIGNED
: PHYS_MAP_NODE_NIL
;
188 for (i
= 0; i
< P_L2_SIZE
; ++i
) {
189 memcpy(&p
[i
], &e
, sizeof(e
));
194 static void phys_page_set_level(PhysPageMap
*map
, PhysPageEntry
*lp
,
195 hwaddr
*index
, hwaddr
*nb
, uint16_t leaf
,
199 hwaddr step
= (hwaddr
)1 << (level
* P_L2_BITS
);
201 if (lp
->skip
&& lp
->ptr
== PHYS_MAP_NODE_NIL
) {
202 lp
->ptr
= phys_map_node_alloc(map
, level
== 0);
204 p
= map
->nodes
[lp
->ptr
];
205 lp
= &p
[(*index
>> (level
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
207 while (*nb
&& lp
< &p
[P_L2_SIZE
]) {
208 if ((*index
& (step
- 1)) == 0 && *nb
>= step
) {
214 phys_page_set_level(map
, lp
, index
, nb
, leaf
, level
- 1);
220 static void phys_page_set(AddressSpaceDispatch
*d
,
221 hwaddr index
, hwaddr nb
,
224 /* Wildly overreserve - it doesn't matter much. */
225 phys_map_node_reserve(&d
->map
, 3 * P_L2_LEVELS
);
227 phys_page_set_level(&d
->map
, &d
->phys_map
, &index
, &nb
, leaf
, P_L2_LEVELS
- 1);
230 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
231 * and update our entry so we can skip it and go directly to the destination.
233 static void phys_page_compact(PhysPageEntry
*lp
, Node
*nodes
, unsigned long *compacted
)
235 unsigned valid_ptr
= P_L2_SIZE
;
240 if (lp
->ptr
== PHYS_MAP_NODE_NIL
) {
245 for (i
= 0; i
< P_L2_SIZE
; i
++) {
246 if (p
[i
].ptr
== PHYS_MAP_NODE_NIL
) {
253 phys_page_compact(&p
[i
], nodes
, compacted
);
257 /* We can only compress if there's only one child. */
262 assert(valid_ptr
< P_L2_SIZE
);
264 /* Don't compress if it won't fit in the # of bits we have. */
265 if (lp
->skip
+ p
[valid_ptr
].skip
>= (1 << 3)) {
269 lp
->ptr
= p
[valid_ptr
].ptr
;
270 if (!p
[valid_ptr
].skip
) {
271 /* If our only child is a leaf, make this a leaf. */
272 /* By design, we should have made this node a leaf to begin with so we
273 * should never reach here.
274 * But since it's so simple to handle this, let's do it just in case we
279 lp
->skip
+= p
[valid_ptr
].skip
;
283 static void phys_page_compact_all(AddressSpaceDispatch
*d
, int nodes_nb
)
285 DECLARE_BITMAP(compacted
, nodes_nb
);
287 if (d
->phys_map
.skip
) {
288 phys_page_compact(&d
->phys_map
, d
->map
.nodes
, compacted
);
292 static MemoryRegionSection
*phys_page_find(PhysPageEntry lp
, hwaddr addr
,
293 Node
*nodes
, MemoryRegionSection
*sections
)
296 hwaddr index
= addr
>> TARGET_PAGE_BITS
;
299 for (i
= P_L2_LEVELS
; lp
.skip
&& (i
-= lp
.skip
) >= 0;) {
300 if (lp
.ptr
== PHYS_MAP_NODE_NIL
) {
301 return §ions
[PHYS_SECTION_UNASSIGNED
];
304 lp
= p
[(index
>> (i
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
307 if (sections
[lp
.ptr
].size
.hi
||
308 range_covers_byte(sections
[lp
.ptr
].offset_within_address_space
,
309 sections
[lp
.ptr
].size
.lo
, addr
)) {
310 return §ions
[lp
.ptr
];
312 return §ions
[PHYS_SECTION_UNASSIGNED
];
316 bool memory_region_is_unassigned(MemoryRegion
*mr
)
318 return mr
!= &io_mem_rom
&& mr
!= &io_mem_notdirty
&& !mr
->rom_device
319 && mr
!= &io_mem_watch
;
322 /* Called from RCU critical section */
323 static MemoryRegionSection
*address_space_lookup_region(AddressSpaceDispatch
*d
,
325 bool resolve_subpage
)
327 MemoryRegionSection
*section
;
330 section
= phys_page_find(d
->phys_map
, addr
, d
->map
.nodes
, d
->map
.sections
);
331 if (resolve_subpage
&& section
->mr
->subpage
) {
332 subpage
= container_of(section
->mr
, subpage_t
, iomem
);
333 section
= &d
->map
.sections
[subpage
->sub_section
[SUBPAGE_IDX(addr
)]];
338 /* Called from RCU critical section */
339 static MemoryRegionSection
*
340 address_space_translate_internal(AddressSpaceDispatch
*d
, hwaddr addr
, hwaddr
*xlat
,
341 hwaddr
*plen
, bool resolve_subpage
)
343 MemoryRegionSection
*section
;
346 section
= address_space_lookup_region(d
, addr
, resolve_subpage
);
347 /* Compute offset within MemoryRegionSection */
348 addr
-= section
->offset_within_address_space
;
350 /* Compute offset within MemoryRegion */
351 *xlat
= addr
+ section
->offset_within_region
;
353 diff
= int128_sub(section
->mr
->size
, int128_make64(addr
));
354 *plen
= int128_get64(int128_min(diff
, int128_make64(*plen
)));
358 static inline bool memory_access_is_direct(MemoryRegion
*mr
, bool is_write
)
360 if (memory_region_is_ram(mr
)) {
361 return !(is_write
&& mr
->readonly
);
363 if (memory_region_is_romd(mr
)) {
370 /* Called from RCU critical section */
371 MemoryRegion
*address_space_translate(AddressSpace
*as
, hwaddr addr
,
372 hwaddr
*xlat
, hwaddr
*plen
,
376 MemoryRegionSection
*section
;
380 AddressSpaceDispatch
*d
= atomic_rcu_read(&as
->dispatch
);
381 section
= address_space_translate_internal(d
, addr
, &addr
, plen
, true);
384 if (!mr
->iommu_ops
) {
388 iotlb
= mr
->iommu_ops
->translate(mr
, addr
, is_write
);
389 addr
= ((iotlb
.translated_addr
& ~iotlb
.addr_mask
)
390 | (addr
& iotlb
.addr_mask
));
391 *plen
= MIN(*plen
, (addr
| iotlb
.addr_mask
) - addr
+ 1);
392 if (!(iotlb
.perm
& (1 << is_write
))) {
393 mr
= &io_mem_unassigned
;
397 as
= iotlb
.target_as
;
400 if (xen_enabled() && memory_access_is_direct(mr
, is_write
)) {
401 hwaddr page
= ((addr
& TARGET_PAGE_MASK
) + TARGET_PAGE_SIZE
) - addr
;
402 *plen
= MIN(page
, *plen
);
409 /* Called from RCU critical section */
410 MemoryRegionSection
*
411 address_space_translate_for_iotlb(CPUState
*cpu
, hwaddr addr
,
412 hwaddr
*xlat
, hwaddr
*plen
)
414 MemoryRegionSection
*section
;
415 section
= address_space_translate_internal(cpu
->memory_dispatch
,
416 addr
, xlat
, plen
, false);
418 assert(!section
->mr
->iommu_ops
);
423 #if !defined(CONFIG_USER_ONLY)
425 static int cpu_common_post_load(void *opaque
, int version_id
)
427 CPUState
*cpu
= opaque
;
429 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
430 version_id is increased. */
431 cpu
->interrupt_request
&= ~0x01;
437 static int cpu_common_pre_load(void *opaque
)
439 CPUState
*cpu
= opaque
;
441 cpu
->exception_index
= -1;
446 static bool cpu_common_exception_index_needed(void *opaque
)
448 CPUState
*cpu
= opaque
;
450 return tcg_enabled() && cpu
->exception_index
!= -1;
453 static const VMStateDescription vmstate_cpu_common_exception_index
= {
454 .name
= "cpu_common/exception_index",
456 .minimum_version_id
= 1,
457 .fields
= (VMStateField
[]) {
458 VMSTATE_INT32(exception_index
, CPUState
),
459 VMSTATE_END_OF_LIST()
463 const VMStateDescription vmstate_cpu_common
= {
464 .name
= "cpu_common",
466 .minimum_version_id
= 1,
467 .pre_load
= cpu_common_pre_load
,
468 .post_load
= cpu_common_post_load
,
469 .fields
= (VMStateField
[]) {
470 VMSTATE_UINT32(halted
, CPUState
),
471 VMSTATE_UINT32(interrupt_request
, CPUState
),
472 VMSTATE_END_OF_LIST()
474 .subsections
= (VMStateSubsection
[]) {
476 .vmsd
= &vmstate_cpu_common_exception_index
,
477 .needed
= cpu_common_exception_index_needed
,
486 CPUState
*qemu_get_cpu(int index
)
491 if (cpu
->cpu_index
== index
) {
499 #if !defined(CONFIG_USER_ONLY)
500 void tcg_cpu_address_space_init(CPUState
*cpu
, AddressSpace
*as
)
502 /* We only support one address space per cpu at the moment. */
503 assert(cpu
->as
== as
);
505 if (cpu
->tcg_as_listener
) {
506 memory_listener_unregister(cpu
->tcg_as_listener
);
508 cpu
->tcg_as_listener
= g_new0(MemoryListener
, 1);
510 cpu
->tcg_as_listener
->commit
= tcg_commit
;
511 memory_listener_register(cpu
->tcg_as_listener
, as
);
515 void cpu_exec_init(CPUArchState
*env
)
517 CPUState
*cpu
= ENV_GET_CPU(env
);
518 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
522 #if defined(CONFIG_USER_ONLY)
526 CPU_FOREACH(some_cpu
) {
529 cpu
->cpu_index
= cpu_index
;
531 QTAILQ_INIT(&cpu
->breakpoints
);
532 QTAILQ_INIT(&cpu
->watchpoints
);
533 #ifndef CONFIG_USER_ONLY
534 cpu
->as
= &address_space_memory
;
535 cpu
->thread_id
= qemu_get_thread_id();
536 cpu_reload_memory_map(cpu
);
538 QTAILQ_INSERT_TAIL(&cpus
, cpu
, node
);
539 #if defined(CONFIG_USER_ONLY)
542 if (qdev_get_vmsd(DEVICE(cpu
)) == NULL
) {
543 vmstate_register(NULL
, cpu_index
, &vmstate_cpu_common
, cpu
);
545 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
546 register_savevm(NULL
, "cpu", cpu_index
, CPU_SAVE_VERSION
,
547 cpu_save
, cpu_load
, env
);
548 assert(cc
->vmsd
== NULL
);
549 assert(qdev_get_vmsd(DEVICE(cpu
)) == NULL
);
551 if (cc
->vmsd
!= NULL
) {
552 vmstate_register(NULL
, cpu_index
, cc
->vmsd
, cpu
);
556 #if defined(CONFIG_USER_ONLY)
557 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
559 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
562 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
564 hwaddr phys
= cpu_get_phys_page_debug(cpu
, pc
);
566 tb_invalidate_phys_addr(cpu
->as
,
567 phys
| (pc
& ~TARGET_PAGE_MASK
));
572 #if defined(CONFIG_USER_ONLY)
573 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
578 int cpu_watchpoint_remove(CPUState
*cpu
, vaddr addr
, vaddr len
,
584 void cpu_watchpoint_remove_by_ref(CPUState
*cpu
, CPUWatchpoint
*watchpoint
)
588 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
589 int flags
, CPUWatchpoint
**watchpoint
)
594 /* Add a watchpoint. */
595 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
596 int flags
, CPUWatchpoint
**watchpoint
)
600 /* forbid ranges which are empty or run off the end of the address space */
601 if (len
== 0 || (addr
+ len
- 1) < addr
) {
602 error_report("tried to set invalid watchpoint at %"
603 VADDR_PRIx
", len=%" VADDR_PRIu
, addr
, len
);
606 wp
= g_malloc(sizeof(*wp
));
612 /* keep all GDB-injected watchpoints in front */
613 if (flags
& BP_GDB
) {
614 QTAILQ_INSERT_HEAD(&cpu
->watchpoints
, wp
, entry
);
616 QTAILQ_INSERT_TAIL(&cpu
->watchpoints
, wp
, entry
);
619 tlb_flush_page(cpu
, addr
);
626 /* Remove a specific watchpoint. */
627 int cpu_watchpoint_remove(CPUState
*cpu
, vaddr addr
, vaddr len
,
632 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
633 if (addr
== wp
->vaddr
&& len
== wp
->len
634 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
635 cpu_watchpoint_remove_by_ref(cpu
, wp
);
642 /* Remove a specific watchpoint by reference. */
643 void cpu_watchpoint_remove_by_ref(CPUState
*cpu
, CPUWatchpoint
*watchpoint
)
645 QTAILQ_REMOVE(&cpu
->watchpoints
, watchpoint
, entry
);
647 tlb_flush_page(cpu
, watchpoint
->vaddr
);
652 /* Remove all matching watchpoints. */
653 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
655 CPUWatchpoint
*wp
, *next
;
657 QTAILQ_FOREACH_SAFE(wp
, &cpu
->watchpoints
, entry
, next
) {
658 if (wp
->flags
& mask
) {
659 cpu_watchpoint_remove_by_ref(cpu
, wp
);
664 /* Return true if this watchpoint address matches the specified
665 * access (ie the address range covered by the watchpoint overlaps
666 * partially or completely with the address range covered by the
669 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint
*wp
,
673 /* We know the lengths are non-zero, but a little caution is
674 * required to avoid errors in the case where the range ends
675 * exactly at the top of the address space and so addr + len
676 * wraps round to zero.
678 vaddr wpend
= wp
->vaddr
+ wp
->len
- 1;
679 vaddr addrend
= addr
+ len
- 1;
681 return !(addr
> wpend
|| wp
->vaddr
> addrend
);
686 /* Add a breakpoint. */
687 int cpu_breakpoint_insert(CPUState
*cpu
, vaddr pc
, int flags
,
688 CPUBreakpoint
**breakpoint
)
692 bp
= g_malloc(sizeof(*bp
));
697 /* keep all GDB-injected breakpoints in front */
698 if (flags
& BP_GDB
) {
699 QTAILQ_INSERT_HEAD(&cpu
->breakpoints
, bp
, entry
);
701 QTAILQ_INSERT_TAIL(&cpu
->breakpoints
, bp
, entry
);
704 breakpoint_invalidate(cpu
, pc
);
712 /* Remove a specific breakpoint. */
713 int cpu_breakpoint_remove(CPUState
*cpu
, vaddr pc
, int flags
)
717 QTAILQ_FOREACH(bp
, &cpu
->breakpoints
, entry
) {
718 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
719 cpu_breakpoint_remove_by_ref(cpu
, bp
);
726 /* Remove a specific breakpoint by reference. */
727 void cpu_breakpoint_remove_by_ref(CPUState
*cpu
, CPUBreakpoint
*breakpoint
)
729 QTAILQ_REMOVE(&cpu
->breakpoints
, breakpoint
, entry
);
731 breakpoint_invalidate(cpu
, breakpoint
->pc
);
736 /* Remove all matching breakpoints. */
737 void cpu_breakpoint_remove_all(CPUState
*cpu
, int mask
)
739 CPUBreakpoint
*bp
, *next
;
741 QTAILQ_FOREACH_SAFE(bp
, &cpu
->breakpoints
, entry
, next
) {
742 if (bp
->flags
& mask
) {
743 cpu_breakpoint_remove_by_ref(cpu
, bp
);
748 /* enable or disable single step mode. EXCP_DEBUG is returned by the
749 CPU loop after each instruction */
750 void cpu_single_step(CPUState
*cpu
, int enabled
)
752 if (cpu
->singlestep_enabled
!= enabled
) {
753 cpu
->singlestep_enabled
= enabled
;
755 kvm_update_guest_debug(cpu
, 0);
757 /* must flush all the translated code to avoid inconsistencies */
758 /* XXX: only flush what is necessary */
759 CPUArchState
*env
= cpu
->env_ptr
;
765 void cpu_abort(CPUState
*cpu
, const char *fmt
, ...)
772 fprintf(stderr
, "qemu: fatal: ");
773 vfprintf(stderr
, fmt
, ap
);
774 fprintf(stderr
, "\n");
775 cpu_dump_state(cpu
, stderr
, fprintf
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
776 if (qemu_log_enabled()) {
777 qemu_log("qemu: fatal: ");
778 qemu_log_vprintf(fmt
, ap2
);
780 log_cpu_state(cpu
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
786 #if defined(CONFIG_USER_ONLY)
788 struct sigaction act
;
789 sigfillset(&act
.sa_mask
);
790 act
.sa_handler
= SIG_DFL
;
791 sigaction(SIGABRT
, &act
, NULL
);
797 #if !defined(CONFIG_USER_ONLY)
798 /* Called from RCU critical section */
799 static RAMBlock
*qemu_get_ram_block(ram_addr_t addr
)
803 block
= atomic_rcu_read(&ram_list
.mru_block
);
804 if (block
&& addr
- block
->offset
< block
->max_length
) {
807 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
808 if (addr
- block
->offset
< block
->max_length
) {
813 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
817 /* It is safe to write mru_block outside the iothread lock. This
822 * xxx removed from list
826 * call_rcu(reclaim_ramblock, xxx);
829 * atomic_rcu_set is not needed here. The block was already published
830 * when it was placed into the list. Here we're just making an extra
831 * copy of the pointer.
833 ram_list
.mru_block
= block
;
837 static void tlb_reset_dirty_range_all(ram_addr_t start
, ram_addr_t length
)
843 end
= TARGET_PAGE_ALIGN(start
+ length
);
844 start
&= TARGET_PAGE_MASK
;
847 block
= qemu_get_ram_block(start
);
848 assert(block
== qemu_get_ram_block(end
- 1));
849 start1
= (uintptr_t)ramblock_ptr(block
, start
- block
->offset
);
850 cpu_tlb_reset_dirty_all(start1
, length
);
854 /* Note: start and end must be within the same ram block. */
855 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t length
,
860 cpu_physical_memory_clear_dirty_range_type(start
, length
, client
);
863 tlb_reset_dirty_range_all(start
, length
);
867 /* Called from RCU critical section */
868 hwaddr
memory_region_section_get_iotlb(CPUState
*cpu
,
869 MemoryRegionSection
*section
,
871 hwaddr paddr
, hwaddr xlat
,
873 target_ulong
*address
)
878 if (memory_region_is_ram(section
->mr
)) {
880 iotlb
= (memory_region_get_ram_addr(section
->mr
) & TARGET_PAGE_MASK
)
882 if (!section
->readonly
) {
883 iotlb
|= PHYS_SECTION_NOTDIRTY
;
885 iotlb
|= PHYS_SECTION_ROM
;
888 iotlb
= section
- section
->address_space
->dispatch
->map
.sections
;
892 /* Make accesses to pages with watchpoints go via the
893 watchpoint trap routines. */
894 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
895 if (cpu_watchpoint_address_matches(wp
, vaddr
, TARGET_PAGE_SIZE
)) {
896 /* Avoid trapping reads of pages with a write breakpoint. */
897 if ((prot
& PAGE_WRITE
) || (wp
->flags
& BP_MEM_READ
)) {
898 iotlb
= PHYS_SECTION_WATCH
+ paddr
;
899 *address
|= TLB_MMIO
;
907 #endif /* defined(CONFIG_USER_ONLY) */
909 #if !defined(CONFIG_USER_ONLY)
911 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
913 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
);
915 static void *(*phys_mem_alloc
)(size_t size
, uint64_t *align
) =
919 * Set a custom physical guest memory alloator.
920 * Accelerators with unusual needs may need this. Hopefully, we can
921 * get rid of it eventually.
923 void phys_mem_set_alloc(void *(*alloc
)(size_t, uint64_t *align
))
925 phys_mem_alloc
= alloc
;
928 static uint16_t phys_section_add(PhysPageMap
*map
,
929 MemoryRegionSection
*section
)
931 /* The physical section number is ORed with a page-aligned
932 * pointer to produce the iotlb entries. Thus it should
933 * never overflow into the page-aligned value.
935 assert(map
->sections_nb
< TARGET_PAGE_SIZE
);
937 if (map
->sections_nb
== map
->sections_nb_alloc
) {
938 map
->sections_nb_alloc
= MAX(map
->sections_nb_alloc
* 2, 16);
939 map
->sections
= g_renew(MemoryRegionSection
, map
->sections
,
940 map
->sections_nb_alloc
);
942 map
->sections
[map
->sections_nb
] = *section
;
943 memory_region_ref(section
->mr
);
944 return map
->sections_nb
++;
947 static void phys_section_destroy(MemoryRegion
*mr
)
949 memory_region_unref(mr
);
952 subpage_t
*subpage
= container_of(mr
, subpage_t
, iomem
);
953 object_unref(OBJECT(&subpage
->iomem
));
958 static void phys_sections_free(PhysPageMap
*map
)
960 while (map
->sections_nb
> 0) {
961 MemoryRegionSection
*section
= &map
->sections
[--map
->sections_nb
];
962 phys_section_destroy(section
->mr
);
964 g_free(map
->sections
);
968 static void register_subpage(AddressSpaceDispatch
*d
, MemoryRegionSection
*section
)
971 hwaddr base
= section
->offset_within_address_space
973 MemoryRegionSection
*existing
= phys_page_find(d
->phys_map
, base
,
974 d
->map
.nodes
, d
->map
.sections
);
975 MemoryRegionSection subsection
= {
976 .offset_within_address_space
= base
,
977 .size
= int128_make64(TARGET_PAGE_SIZE
),
981 assert(existing
->mr
->subpage
|| existing
->mr
== &io_mem_unassigned
);
983 if (!(existing
->mr
->subpage
)) {
984 subpage
= subpage_init(d
->as
, base
);
985 subsection
.address_space
= d
->as
;
986 subsection
.mr
= &subpage
->iomem
;
987 phys_page_set(d
, base
>> TARGET_PAGE_BITS
, 1,
988 phys_section_add(&d
->map
, &subsection
));
990 subpage
= container_of(existing
->mr
, subpage_t
, iomem
);
992 start
= section
->offset_within_address_space
& ~TARGET_PAGE_MASK
;
993 end
= start
+ int128_get64(section
->size
) - 1;
994 subpage_register(subpage
, start
, end
,
995 phys_section_add(&d
->map
, section
));
999 static void register_multipage(AddressSpaceDispatch
*d
,
1000 MemoryRegionSection
*section
)
1002 hwaddr start_addr
= section
->offset_within_address_space
;
1003 uint16_t section_index
= phys_section_add(&d
->map
, section
);
1004 uint64_t num_pages
= int128_get64(int128_rshift(section
->size
,
1008 phys_page_set(d
, start_addr
>> TARGET_PAGE_BITS
, num_pages
, section_index
);
1011 static void mem_add(MemoryListener
*listener
, MemoryRegionSection
*section
)
1013 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
1014 AddressSpaceDispatch
*d
= as
->next_dispatch
;
1015 MemoryRegionSection now
= *section
, remain
= *section
;
1016 Int128 page_size
= int128_make64(TARGET_PAGE_SIZE
);
1018 if (now
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
1019 uint64_t left
= TARGET_PAGE_ALIGN(now
.offset_within_address_space
)
1020 - now
.offset_within_address_space
;
1022 now
.size
= int128_min(int128_make64(left
), now
.size
);
1023 register_subpage(d
, &now
);
1025 now
.size
= int128_zero();
1027 while (int128_ne(remain
.size
, now
.size
)) {
1028 remain
.size
= int128_sub(remain
.size
, now
.size
);
1029 remain
.offset_within_address_space
+= int128_get64(now
.size
);
1030 remain
.offset_within_region
+= int128_get64(now
.size
);
1032 if (int128_lt(remain
.size
, page_size
)) {
1033 register_subpage(d
, &now
);
1034 } else if (remain
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
1035 now
.size
= page_size
;
1036 register_subpage(d
, &now
);
1038 now
.size
= int128_and(now
.size
, int128_neg(page_size
));
1039 register_multipage(d
, &now
);
1044 void qemu_flush_coalesced_mmio_buffer(void)
1047 kvm_flush_coalesced_mmio_buffer();
1050 void qemu_mutex_lock_ramlist(void)
1052 qemu_mutex_lock(&ram_list
.mutex
);
1055 void qemu_mutex_unlock_ramlist(void)
1057 qemu_mutex_unlock(&ram_list
.mutex
);
1062 #include <sys/vfs.h>
1064 #define HUGETLBFS_MAGIC 0x958458f6
1066 static long gethugepagesize(const char *path
, Error
**errp
)
1072 ret
= statfs(path
, &fs
);
1073 } while (ret
!= 0 && errno
== EINTR
);
1076 error_setg_errno(errp
, errno
, "failed to get page size of file %s",
1081 if (fs
.f_type
!= HUGETLBFS_MAGIC
)
1082 fprintf(stderr
, "Warning: path not on HugeTLBFS: %s\n", path
);
1087 static void *file_ram_alloc(RAMBlock
*block
,
1093 char *sanitized_name
;
1098 Error
*local_err
= NULL
;
1100 hpagesize
= gethugepagesize(path
, &local_err
);
1102 error_propagate(errp
, local_err
);
1105 block
->mr
->align
= hpagesize
;
1107 if (memory
< hpagesize
) {
1108 error_setg(errp
, "memory size 0x" RAM_ADDR_FMT
" must be equal to "
1109 "or larger than huge page size 0x%" PRIx64
,
1114 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1116 "host lacks kvm mmu notifiers, -mem-path unsupported");
1120 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1121 sanitized_name
= g_strdup(memory_region_name(block
->mr
));
1122 for (c
= sanitized_name
; *c
!= '\0'; c
++) {
1127 filename
= g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path
,
1129 g_free(sanitized_name
);
1131 fd
= mkstemp(filename
);
1133 error_setg_errno(errp
, errno
,
1134 "unable to create backing store for hugepages");
1141 memory
= (memory
+hpagesize
-1) & ~(hpagesize
-1);
1144 * ftruncate is not supported by hugetlbfs in older
1145 * hosts, so don't bother bailing out on errors.
1146 * If anything goes wrong with it under other filesystems,
1149 if (ftruncate(fd
, memory
)) {
1150 perror("ftruncate");
1153 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
,
1154 (block
->flags
& RAM_SHARED
? MAP_SHARED
: MAP_PRIVATE
),
1156 if (area
== MAP_FAILED
) {
1157 error_setg_errno(errp
, errno
,
1158 "unable to map backing store for hugepages");
1164 os_mem_prealloc(fd
, area
, memory
);
1172 error_report("%s", error_get_pretty(*errp
));
1179 /* Called with the ramlist lock held. */
1180 static ram_addr_t
find_ram_offset(ram_addr_t size
)
1182 RAMBlock
*block
, *next_block
;
1183 ram_addr_t offset
= RAM_ADDR_MAX
, mingap
= RAM_ADDR_MAX
;
1185 assert(size
!= 0); /* it would hand out same offset multiple times */
1187 if (QLIST_EMPTY_RCU(&ram_list
.blocks
)) {
1191 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1192 ram_addr_t end
, next
= RAM_ADDR_MAX
;
1194 end
= block
->offset
+ block
->max_length
;
1196 QLIST_FOREACH_RCU(next_block
, &ram_list
.blocks
, next
) {
1197 if (next_block
->offset
>= end
) {
1198 next
= MIN(next
, next_block
->offset
);
1201 if (next
- end
>= size
&& next
- end
< mingap
) {
1203 mingap
= next
- end
;
1207 if (offset
== RAM_ADDR_MAX
) {
1208 fprintf(stderr
, "Failed to find gap of requested size: %" PRIu64
"\n",
1216 ram_addr_t
last_ram_offset(void)
1219 ram_addr_t last
= 0;
1222 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1223 last
= MAX(last
, block
->offset
+ block
->max_length
);
1229 static void qemu_ram_setup_dump(void *addr
, ram_addr_t size
)
1233 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1234 if (!machine_dump_guest_core(current_machine
)) {
1235 ret
= qemu_madvise(addr
, size
, QEMU_MADV_DONTDUMP
);
1237 perror("qemu_madvise");
1238 fprintf(stderr
, "madvise doesn't support MADV_DONTDUMP, "
1239 "but dump_guest_core=off specified\n");
1244 /* Called within an RCU critical section, or while the ramlist lock
1247 static RAMBlock
*find_ram_block(ram_addr_t addr
)
1251 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1252 if (block
->offset
== addr
) {
1260 /* Called with iothread lock held. */
1261 void qemu_ram_set_idstr(ram_addr_t addr
, const char *name
, DeviceState
*dev
)
1263 RAMBlock
*new_block
, *block
;
1266 new_block
= find_ram_block(addr
);
1268 assert(!new_block
->idstr
[0]);
1271 char *id
= qdev_get_dev_path(dev
);
1273 snprintf(new_block
->idstr
, sizeof(new_block
->idstr
), "%s/", id
);
1277 pstrcat(new_block
->idstr
, sizeof(new_block
->idstr
), name
);
1279 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1280 if (block
!= new_block
&& !strcmp(block
->idstr
, new_block
->idstr
)) {
1281 fprintf(stderr
, "RAMBlock \"%s\" already registered, abort!\n",
1289 /* Called with iothread lock held. */
1290 void qemu_ram_unset_idstr(ram_addr_t addr
)
1294 /* FIXME: arch_init.c assumes that this is not called throughout
1295 * migration. Ignore the problem since hot-unplug during migration
1296 * does not work anyway.
1300 block
= find_ram_block(addr
);
1302 memset(block
->idstr
, 0, sizeof(block
->idstr
));
1307 static int memory_try_enable_merging(void *addr
, size_t len
)
1309 if (!machine_mem_merge(current_machine
)) {
1310 /* disabled by the user */
1314 return qemu_madvise(addr
, len
, QEMU_MADV_MERGEABLE
);
1317 /* Only legal before guest might have detected the memory size: e.g. on
1318 * incoming migration, or right after reset.
1320 * As memory core doesn't know how is memory accessed, it is up to
1321 * resize callback to update device state and/or add assertions to detect
1322 * misuse, if necessary.
1324 int qemu_ram_resize(ram_addr_t base
, ram_addr_t newsize
, Error
**errp
)
1326 RAMBlock
*block
= find_ram_block(base
);
1330 newsize
= TARGET_PAGE_ALIGN(newsize
);
1332 if (block
->used_length
== newsize
) {
1336 if (!(block
->flags
& RAM_RESIZEABLE
)) {
1337 error_setg_errno(errp
, EINVAL
,
1338 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1339 " in != 0x" RAM_ADDR_FMT
, block
->idstr
,
1340 newsize
, block
->used_length
);
1344 if (block
->max_length
< newsize
) {
1345 error_setg_errno(errp
, EINVAL
,
1346 "Length too large: %s: 0x" RAM_ADDR_FMT
1347 " > 0x" RAM_ADDR_FMT
, block
->idstr
,
1348 newsize
, block
->max_length
);
1352 cpu_physical_memory_clear_dirty_range(block
->offset
, block
->used_length
);
1353 block
->used_length
= newsize
;
1354 cpu_physical_memory_set_dirty_range(block
->offset
, block
->used_length
,
1356 memory_region_set_size(block
->mr
, newsize
);
1357 if (block
->resized
) {
1358 block
->resized(block
->idstr
, newsize
, block
->host
);
1363 static ram_addr_t
ram_block_add(RAMBlock
*new_block
, Error
**errp
)
1366 RAMBlock
*last_block
= NULL
;
1367 ram_addr_t old_ram_size
, new_ram_size
;
1369 old_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1371 qemu_mutex_lock_ramlist();
1372 new_block
->offset
= find_ram_offset(new_block
->max_length
);
1374 if (!new_block
->host
) {
1375 if (xen_enabled()) {
1376 xen_ram_alloc(new_block
->offset
, new_block
->max_length
,
1379 new_block
->host
= phys_mem_alloc(new_block
->max_length
,
1380 &new_block
->mr
->align
);
1381 if (!new_block
->host
) {
1382 error_setg_errno(errp
, errno
,
1383 "cannot set up guest memory '%s'",
1384 memory_region_name(new_block
->mr
));
1385 qemu_mutex_unlock_ramlist();
1388 memory_try_enable_merging(new_block
->host
, new_block
->max_length
);
1392 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1393 * QLIST (which has an RCU-friendly variant) does not have insertion at
1394 * tail, so save the last element in last_block.
1396 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1398 if (block
->max_length
< new_block
->max_length
) {
1403 QLIST_INSERT_BEFORE_RCU(block
, new_block
, next
);
1404 } else if (last_block
) {
1405 QLIST_INSERT_AFTER_RCU(last_block
, new_block
, next
);
1406 } else { /* list is empty */
1407 QLIST_INSERT_HEAD_RCU(&ram_list
.blocks
, new_block
, next
);
1409 ram_list
.mru_block
= NULL
;
1411 /* Write list before version */
1414 qemu_mutex_unlock_ramlist();
1416 new_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1418 if (new_ram_size
> old_ram_size
) {
1421 /* ram_list.dirty_memory[] is protected by the iothread lock. */
1422 for (i
= 0; i
< DIRTY_MEMORY_NUM
; i
++) {
1423 ram_list
.dirty_memory
[i
] =
1424 bitmap_zero_extend(ram_list
.dirty_memory
[i
],
1425 old_ram_size
, new_ram_size
);
1428 cpu_physical_memory_set_dirty_range(new_block
->offset
,
1429 new_block
->used_length
,
1432 if (new_block
->host
) {
1433 qemu_ram_setup_dump(new_block
->host
, new_block
->max_length
);
1434 qemu_madvise(new_block
->host
, new_block
->max_length
, QEMU_MADV_HUGEPAGE
);
1435 qemu_madvise(new_block
->host
, new_block
->max_length
, QEMU_MADV_DONTFORK
);
1436 if (kvm_enabled()) {
1437 kvm_setup_guest_memory(new_block
->host
, new_block
->max_length
);
1441 return new_block
->offset
;
1445 ram_addr_t
qemu_ram_alloc_from_file(ram_addr_t size
, MemoryRegion
*mr
,
1446 bool share
, const char *mem_path
,
1449 RAMBlock
*new_block
;
1451 Error
*local_err
= NULL
;
1453 if (xen_enabled()) {
1454 error_setg(errp
, "-mem-path not supported with Xen");
1458 if (phys_mem_alloc
!= qemu_anon_ram_alloc
) {
1460 * file_ram_alloc() needs to allocate just like
1461 * phys_mem_alloc, but we haven't bothered to provide
1465 "-mem-path not supported with this accelerator");
1469 size
= TARGET_PAGE_ALIGN(size
);
1470 new_block
= g_malloc0(sizeof(*new_block
));
1472 new_block
->used_length
= size
;
1473 new_block
->max_length
= size
;
1474 new_block
->flags
= share
? RAM_SHARED
: 0;
1475 new_block
->host
= file_ram_alloc(new_block
, size
,
1477 if (!new_block
->host
) {
1482 addr
= ram_block_add(new_block
, &local_err
);
1485 error_propagate(errp
, local_err
);
1493 ram_addr_t
qemu_ram_alloc_internal(ram_addr_t size
, ram_addr_t max_size
,
1494 void (*resized
)(const char*,
1497 void *host
, bool resizeable
,
1498 MemoryRegion
*mr
, Error
**errp
)
1500 RAMBlock
*new_block
;
1502 Error
*local_err
= NULL
;
1504 size
= TARGET_PAGE_ALIGN(size
);
1505 max_size
= TARGET_PAGE_ALIGN(max_size
);
1506 new_block
= g_malloc0(sizeof(*new_block
));
1508 new_block
->resized
= resized
;
1509 new_block
->used_length
= size
;
1510 new_block
->max_length
= max_size
;
1511 assert(max_size
>= size
);
1513 new_block
->host
= host
;
1515 new_block
->flags
|= RAM_PREALLOC
;
1518 new_block
->flags
|= RAM_RESIZEABLE
;
1520 addr
= ram_block_add(new_block
, &local_err
);
1523 error_propagate(errp
, local_err
);
1529 ram_addr_t
qemu_ram_alloc_from_ptr(ram_addr_t size
, void *host
,
1530 MemoryRegion
*mr
, Error
**errp
)
1532 return qemu_ram_alloc_internal(size
, size
, NULL
, host
, false, mr
, errp
);
1535 ram_addr_t
qemu_ram_alloc(ram_addr_t size
, MemoryRegion
*mr
, Error
**errp
)
1537 return qemu_ram_alloc_internal(size
, size
, NULL
, NULL
, false, mr
, errp
);
1540 ram_addr_t
qemu_ram_alloc_resizeable(ram_addr_t size
, ram_addr_t maxsz
,
1541 void (*resized
)(const char*,
1544 MemoryRegion
*mr
, Error
**errp
)
1546 return qemu_ram_alloc_internal(size
, maxsz
, resized
, NULL
, true, mr
, errp
);
1549 void qemu_ram_free_from_ptr(ram_addr_t addr
)
1553 qemu_mutex_lock_ramlist();
1554 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1555 if (addr
== block
->offset
) {
1556 QLIST_REMOVE_RCU(block
, next
);
1557 ram_list
.mru_block
= NULL
;
1558 /* Write list before version */
1561 g_free_rcu(block
, rcu
);
1565 qemu_mutex_unlock_ramlist();
1568 static void reclaim_ramblock(RAMBlock
*block
)
1570 if (block
->flags
& RAM_PREALLOC
) {
1572 } else if (xen_enabled()) {
1573 xen_invalidate_map_cache_entry(block
->host
);
1575 } else if (block
->fd
>= 0) {
1576 munmap(block
->host
, block
->max_length
);
1580 qemu_anon_ram_free(block
->host
, block
->max_length
);
1585 void qemu_ram_free(ram_addr_t addr
)
1589 qemu_mutex_lock_ramlist();
1590 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1591 if (addr
== block
->offset
) {
1592 QLIST_REMOVE_RCU(block
, next
);
1593 ram_list
.mru_block
= NULL
;
1594 /* Write list before version */
1597 call_rcu(block
, reclaim_ramblock
, rcu
);
1601 qemu_mutex_unlock_ramlist();
1605 void qemu_ram_remap(ram_addr_t addr
, ram_addr_t length
)
1612 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1613 offset
= addr
- block
->offset
;
1614 if (offset
< block
->max_length
) {
1615 vaddr
= ramblock_ptr(block
, offset
);
1616 if (block
->flags
& RAM_PREALLOC
) {
1618 } else if (xen_enabled()) {
1622 if (block
->fd
>= 0) {
1623 flags
|= (block
->flags
& RAM_SHARED
?
1624 MAP_SHARED
: MAP_PRIVATE
);
1625 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1626 flags
, block
->fd
, offset
);
1629 * Remap needs to match alloc. Accelerators that
1630 * set phys_mem_alloc never remap. If they did,
1631 * we'd need a remap hook here.
1633 assert(phys_mem_alloc
== qemu_anon_ram_alloc
);
1635 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
1636 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1639 if (area
!= vaddr
) {
1640 fprintf(stderr
, "Could not remap addr: "
1641 RAM_ADDR_FMT
"@" RAM_ADDR_FMT
"\n",
1645 memory_try_enable_merging(vaddr
, length
);
1646 qemu_ram_setup_dump(vaddr
, length
);
1651 #endif /* !_WIN32 */
1653 int qemu_get_ram_fd(ram_addr_t addr
)
1659 block
= qemu_get_ram_block(addr
);
1665 void *qemu_get_ram_block_host_ptr(ram_addr_t addr
)
1671 block
= qemu_get_ram_block(addr
);
1672 ptr
= ramblock_ptr(block
, 0);
1677 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1678 * This should not be used for general purpose DMA. Use address_space_map
1679 * or address_space_rw instead. For local memory (e.g. video ram) that the
1680 * device owns, use memory_region_get_ram_ptr.
1682 * By the time this function returns, the returned pointer is not protected
1683 * by RCU anymore. If the caller is not within an RCU critical section and
1684 * does not hold the iothread lock, it must have other means of protecting the
1685 * pointer, such as a reference to the region that includes the incoming
1688 void *qemu_get_ram_ptr(ram_addr_t addr
)
1694 block
= qemu_get_ram_block(addr
);
1696 if (xen_enabled() && block
->host
== NULL
) {
1697 /* We need to check if the requested address is in the RAM
1698 * because we don't want to map the entire memory in QEMU.
1699 * In that case just map until the end of the page.
1701 if (block
->offset
== 0) {
1702 ptr
= xen_map_cache(addr
, 0, 0);
1706 block
->host
= xen_map_cache(block
->offset
, block
->max_length
, 1);
1708 ptr
= ramblock_ptr(block
, addr
- block
->offset
);
1715 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1716 * but takes a size argument.
1718 * By the time this function returns, the returned pointer is not protected
1719 * by RCU anymore. If the caller is not within an RCU critical section and
1720 * does not hold the iothread lock, it must have other means of protecting the
1721 * pointer, such as a reference to the region that includes the incoming
1724 static void *qemu_ram_ptr_length(ram_addr_t addr
, hwaddr
*size
)
1730 if (xen_enabled()) {
1731 return xen_map_cache(addr
, *size
, 1);
1735 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1736 if (addr
- block
->offset
< block
->max_length
) {
1737 if (addr
- block
->offset
+ *size
> block
->max_length
)
1738 *size
= block
->max_length
- addr
+ block
->offset
;
1739 ptr
= ramblock_ptr(block
, addr
- block
->offset
);
1745 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
1750 /* Some of the softmmu routines need to translate from a host pointer
1751 * (typically a TLB entry) back to a ram offset.
1753 * By the time this function returns, the returned pointer is not protected
1754 * by RCU anymore. If the caller is not within an RCU critical section and
1755 * does not hold the iothread lock, it must have other means of protecting the
1756 * pointer, such as a reference to the region that includes the incoming
1759 MemoryRegion
*qemu_ram_addr_from_host(void *ptr
, ram_addr_t
*ram_addr
)
1762 uint8_t *host
= ptr
;
1765 if (xen_enabled()) {
1767 *ram_addr
= xen_ram_addr_from_mapcache(ptr
);
1768 mr
= qemu_get_ram_block(*ram_addr
)->mr
;
1774 block
= atomic_rcu_read(&ram_list
.mru_block
);
1775 if (block
&& block
->host
&& host
- block
->host
< block
->max_length
) {
1779 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1780 /* This case append when the block is not mapped. */
1781 if (block
->host
== NULL
) {
1784 if (host
- block
->host
< block
->max_length
) {
1793 *ram_addr
= block
->offset
+ (host
- block
->host
);
1799 static void notdirty_mem_write(void *opaque
, hwaddr ram_addr
,
1800 uint64_t val
, unsigned size
)
1802 if (!cpu_physical_memory_get_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
)) {
1803 tb_invalidate_phys_page_fast(ram_addr
, size
);
1807 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
1810 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
1813 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
1818 /* Set both VGA and migration bits for simplicity and to remove
1819 * the notdirty callback faster.
1821 cpu_physical_memory_set_dirty_range(ram_addr
, size
,
1822 DIRTY_CLIENTS_NOCODE
);
1823 /* we remove the notdirty callback only if the code has been
1825 if (!cpu_physical_memory_is_clean(ram_addr
)) {
1826 CPUArchState
*env
= current_cpu
->env_ptr
;
1827 tlb_set_dirty(env
, current_cpu
->mem_io_vaddr
);
1831 static bool notdirty_mem_accepts(void *opaque
, hwaddr addr
,
1832 unsigned size
, bool is_write
)
1837 static const MemoryRegionOps notdirty_mem_ops
= {
1838 .write
= notdirty_mem_write
,
1839 .valid
.accepts
= notdirty_mem_accepts
,
1840 .endianness
= DEVICE_NATIVE_ENDIAN
,
1843 /* Generate a debug exception if a watchpoint has been hit. */
1844 static void check_watchpoint(int offset
, int len
, MemTxAttrs attrs
, int flags
)
1846 CPUState
*cpu
= current_cpu
;
1847 CPUArchState
*env
= cpu
->env_ptr
;
1848 target_ulong pc
, cs_base
;
1853 if (cpu
->watchpoint_hit
) {
1854 /* We re-entered the check after replacing the TB. Now raise
1855 * the debug interrupt so that is will trigger after the
1856 * current instruction. */
1857 cpu_interrupt(cpu
, CPU_INTERRUPT_DEBUG
);
1860 vaddr
= (cpu
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
1861 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
1862 if (cpu_watchpoint_address_matches(wp
, vaddr
, len
)
1863 && (wp
->flags
& flags
)) {
1864 if (flags
== BP_MEM_READ
) {
1865 wp
->flags
|= BP_WATCHPOINT_HIT_READ
;
1867 wp
->flags
|= BP_WATCHPOINT_HIT_WRITE
;
1869 wp
->hitaddr
= vaddr
;
1870 wp
->hitattrs
= attrs
;
1871 if (!cpu
->watchpoint_hit
) {
1872 cpu
->watchpoint_hit
= wp
;
1873 tb_check_watchpoint(cpu
);
1874 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
1875 cpu
->exception_index
= EXCP_DEBUG
;
1878 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
1879 tb_gen_code(cpu
, pc
, cs_base
, cpu_flags
, 1);
1880 cpu_resume_from_signal(cpu
, NULL
);
1884 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
1889 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1890 so these check for a hit then pass through to the normal out-of-line
1892 static MemTxResult
watch_mem_read(void *opaque
, hwaddr addr
, uint64_t *pdata
,
1893 unsigned size
, MemTxAttrs attrs
)
1898 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, size
, attrs
, BP_MEM_READ
);
1901 data
= address_space_ldub(&address_space_memory
, addr
, attrs
, &res
);
1904 data
= address_space_lduw(&address_space_memory
, addr
, attrs
, &res
);
1907 data
= address_space_ldl(&address_space_memory
, addr
, attrs
, &res
);
1915 static MemTxResult
watch_mem_write(void *opaque
, hwaddr addr
,
1916 uint64_t val
, unsigned size
,
1921 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, size
, attrs
, BP_MEM_WRITE
);
1924 address_space_stb(&address_space_memory
, addr
, val
, attrs
, &res
);
1927 address_space_stw(&address_space_memory
, addr
, val
, attrs
, &res
);
1930 address_space_stl(&address_space_memory
, addr
, val
, attrs
, &res
);
1937 static const MemoryRegionOps watch_mem_ops
= {
1938 .read_with_attrs
= watch_mem_read
,
1939 .write_with_attrs
= watch_mem_write
,
1940 .endianness
= DEVICE_NATIVE_ENDIAN
,
1943 static MemTxResult
subpage_read(void *opaque
, hwaddr addr
, uint64_t *data
,
1944 unsigned len
, MemTxAttrs attrs
)
1946 subpage_t
*subpage
= opaque
;
1950 #if defined(DEBUG_SUBPAGE)
1951 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
"\n", __func__
,
1952 subpage
, len
, addr
);
1954 res
= address_space_read(subpage
->as
, addr
+ subpage
->base
,
1961 *data
= ldub_p(buf
);
1964 *data
= lduw_p(buf
);
1977 static MemTxResult
subpage_write(void *opaque
, hwaddr addr
,
1978 uint64_t value
, unsigned len
, MemTxAttrs attrs
)
1980 subpage_t
*subpage
= opaque
;
1983 #if defined(DEBUG_SUBPAGE)
1984 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
1985 " value %"PRIx64
"\n",
1986 __func__
, subpage
, len
, addr
, value
);
2004 return address_space_write(subpage
->as
, addr
+ subpage
->base
,
2008 static bool subpage_accepts(void *opaque
, hwaddr addr
,
2009 unsigned len
, bool is_write
)
2011 subpage_t
*subpage
= opaque
;
2012 #if defined(DEBUG_SUBPAGE)
2013 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx
"\n",
2014 __func__
, subpage
, is_write
? 'w' : 'r', len
, addr
);
2017 return address_space_access_valid(subpage
->as
, addr
+ subpage
->base
,
2021 static const MemoryRegionOps subpage_ops
= {
2022 .read_with_attrs
= subpage_read
,
2023 .write_with_attrs
= subpage_write
,
2024 .impl
.min_access_size
= 1,
2025 .impl
.max_access_size
= 8,
2026 .valid
.min_access_size
= 1,
2027 .valid
.max_access_size
= 8,
2028 .valid
.accepts
= subpage_accepts
,
2029 .endianness
= DEVICE_NATIVE_ENDIAN
,
2032 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2037 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
2039 idx
= SUBPAGE_IDX(start
);
2040 eidx
= SUBPAGE_IDX(end
);
2041 #if defined(DEBUG_SUBPAGE)
2042 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2043 __func__
, mmio
, start
, end
, idx
, eidx
, section
);
2045 for (; idx
<= eidx
; idx
++) {
2046 mmio
->sub_section
[idx
] = section
;
2052 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
)
2056 mmio
= g_malloc0(sizeof(subpage_t
));
2060 memory_region_init_io(&mmio
->iomem
, NULL
, &subpage_ops
, mmio
,
2061 NULL
, TARGET_PAGE_SIZE
);
2062 mmio
->iomem
.subpage
= true;
2063 #if defined(DEBUG_SUBPAGE)
2064 printf("%s: %p base " TARGET_FMT_plx
" len %08x\n", __func__
,
2065 mmio
, base
, TARGET_PAGE_SIZE
);
2067 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
-1, PHYS_SECTION_UNASSIGNED
);
2072 static uint16_t dummy_section(PhysPageMap
*map
, AddressSpace
*as
,
2076 MemoryRegionSection section
= {
2077 .address_space
= as
,
2079 .offset_within_address_space
= 0,
2080 .offset_within_region
= 0,
2081 .size
= int128_2_64(),
2084 return phys_section_add(map
, §ion
);
2087 MemoryRegion
*iotlb_to_region(CPUState
*cpu
, hwaddr index
)
2089 AddressSpaceDispatch
*d
= atomic_rcu_read(&cpu
->memory_dispatch
);
2090 MemoryRegionSection
*sections
= d
->map
.sections
;
2092 return sections
[index
& ~TARGET_PAGE_MASK
].mr
;
2095 static void io_mem_init(void)
2097 memory_region_init_io(&io_mem_rom
, NULL
, &unassigned_mem_ops
, NULL
, NULL
, UINT64_MAX
);
2098 memory_region_init_io(&io_mem_unassigned
, NULL
, &unassigned_mem_ops
, NULL
,
2100 memory_region_init_io(&io_mem_notdirty
, NULL
, ¬dirty_mem_ops
, NULL
,
2102 memory_region_init_io(&io_mem_watch
, NULL
, &watch_mem_ops
, NULL
,
2106 static void mem_begin(MemoryListener
*listener
)
2108 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
2109 AddressSpaceDispatch
*d
= g_new0(AddressSpaceDispatch
, 1);
2112 n
= dummy_section(&d
->map
, as
, &io_mem_unassigned
);
2113 assert(n
== PHYS_SECTION_UNASSIGNED
);
2114 n
= dummy_section(&d
->map
, as
, &io_mem_notdirty
);
2115 assert(n
== PHYS_SECTION_NOTDIRTY
);
2116 n
= dummy_section(&d
->map
, as
, &io_mem_rom
);
2117 assert(n
== PHYS_SECTION_ROM
);
2118 n
= dummy_section(&d
->map
, as
, &io_mem_watch
);
2119 assert(n
== PHYS_SECTION_WATCH
);
2121 d
->phys_map
= (PhysPageEntry
) { .ptr
= PHYS_MAP_NODE_NIL
, .skip
= 1 };
2123 as
->next_dispatch
= d
;
2126 static void address_space_dispatch_free(AddressSpaceDispatch
*d
)
2128 phys_sections_free(&d
->map
);
2132 static void mem_commit(MemoryListener
*listener
)
2134 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
2135 AddressSpaceDispatch
*cur
= as
->dispatch
;
2136 AddressSpaceDispatch
*next
= as
->next_dispatch
;
2138 phys_page_compact_all(next
, next
->map
.nodes_nb
);
2140 atomic_rcu_set(&as
->dispatch
, next
);
2142 call_rcu(cur
, address_space_dispatch_free
, rcu
);
2146 static void tcg_commit(MemoryListener
*listener
)
2150 /* since each CPU stores ram addresses in its TLB cache, we must
2151 reset the modified entries */
2154 /* FIXME: Disentangle the cpu.h circular files deps so we can
2155 directly get the right CPU from listener. */
2156 if (cpu
->tcg_as_listener
!= listener
) {
2159 cpu_reload_memory_map(cpu
);
2163 void address_space_init_dispatch(AddressSpace
*as
)
2165 as
->dispatch
= NULL
;
2166 as
->dispatch_listener
= (MemoryListener
) {
2168 .commit
= mem_commit
,
2169 .region_add
= mem_add
,
2170 .region_nop
= mem_add
,
2173 memory_listener_register(&as
->dispatch_listener
, as
);
2176 void address_space_unregister(AddressSpace
*as
)
2178 memory_listener_unregister(&as
->dispatch_listener
);
2181 void address_space_destroy_dispatch(AddressSpace
*as
)
2183 AddressSpaceDispatch
*d
= as
->dispatch
;
2185 atomic_rcu_set(&as
->dispatch
, NULL
);
2187 call_rcu(d
, address_space_dispatch_free
, rcu
);
2191 static void memory_map_init(void)
2193 system_memory
= g_malloc(sizeof(*system_memory
));
2195 memory_region_init(system_memory
, NULL
, "system", UINT64_MAX
);
2196 address_space_init(&address_space_memory
, system_memory
, "memory");
2198 system_io
= g_malloc(sizeof(*system_io
));
2199 memory_region_init_io(system_io
, NULL
, &unassigned_io_ops
, NULL
, "io",
2201 address_space_init(&address_space_io
, system_io
, "I/O");
2204 MemoryRegion
*get_system_memory(void)
2206 return system_memory
;
2209 MemoryRegion
*get_system_io(void)
2214 #endif /* !defined(CONFIG_USER_ONLY) */
2216 /* physical memory access (slow version, mainly for debug) */
2217 #if defined(CONFIG_USER_ONLY)
2218 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
2219 uint8_t *buf
, int len
, int is_write
)
2226 page
= addr
& TARGET_PAGE_MASK
;
2227 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2230 flags
= page_get_flags(page
);
2231 if (!(flags
& PAGE_VALID
))
2234 if (!(flags
& PAGE_WRITE
))
2236 /* XXX: this code should not depend on lock_user */
2237 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
2240 unlock_user(p
, addr
, l
);
2242 if (!(flags
& PAGE_READ
))
2244 /* XXX: this code should not depend on lock_user */
2245 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
2248 unlock_user(p
, addr
, 0);
2259 static void invalidate_and_set_dirty(MemoryRegion
*mr
, hwaddr addr
,
2262 uint8_t dirty_log_mask
= memory_region_get_dirty_log_mask(mr
);
2263 /* No early return if dirty_log_mask is or becomes 0, because
2264 * cpu_physical_memory_set_dirty_range will still call
2265 * xen_modified_memory.
2267 if (dirty_log_mask
) {
2269 cpu_physical_memory_range_includes_clean(addr
, length
, dirty_log_mask
);
2271 if (dirty_log_mask
& (1 << DIRTY_MEMORY_CODE
)) {
2272 tb_invalidate_phys_range(addr
, addr
+ length
);
2273 dirty_log_mask
&= ~(1 << DIRTY_MEMORY_CODE
);
2275 cpu_physical_memory_set_dirty_range(addr
, length
, dirty_log_mask
);
2278 static int memory_access_size(MemoryRegion
*mr
, unsigned l
, hwaddr addr
)
2280 unsigned access_size_max
= mr
->ops
->valid
.max_access_size
;
2282 /* Regions are assumed to support 1-4 byte accesses unless
2283 otherwise specified. */
2284 if (access_size_max
== 0) {
2285 access_size_max
= 4;
2288 /* Bound the maximum access by the alignment of the address. */
2289 if (!mr
->ops
->impl
.unaligned
) {
2290 unsigned align_size_max
= addr
& -addr
;
2291 if (align_size_max
!= 0 && align_size_max
< access_size_max
) {
2292 access_size_max
= align_size_max
;
2296 /* Don't attempt accesses larger than the maximum. */
2297 if (l
> access_size_max
) {
2298 l
= access_size_max
;
2301 l
= 1 << (qemu_fls(l
) - 1);
2307 MemTxResult
address_space_rw(AddressSpace
*as
, hwaddr addr
, MemTxAttrs attrs
,
2308 uint8_t *buf
, int len
, bool is_write
)
2315 MemTxResult result
= MEMTX_OK
;
2320 mr
= address_space_translate(as
, addr
, &addr1
, &l
, is_write
);
2323 if (!memory_access_is_direct(mr
, is_write
)) {
2324 l
= memory_access_size(mr
, l
, addr1
);
2325 /* XXX: could force current_cpu to NULL to avoid
2329 /* 64 bit write access */
2331 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 8,
2335 /* 32 bit write access */
2337 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 4,
2341 /* 16 bit write access */
2343 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 2,
2347 /* 8 bit write access */
2349 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 1,
2356 addr1
+= memory_region_get_ram_addr(mr
);
2358 ptr
= qemu_get_ram_ptr(addr1
);
2359 memcpy(ptr
, buf
, l
);
2360 invalidate_and_set_dirty(mr
, addr1
, l
);
2363 if (!memory_access_is_direct(mr
, is_write
)) {
2365 l
= memory_access_size(mr
, l
, addr1
);
2368 /* 64 bit read access */
2369 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 8,
2374 /* 32 bit read access */
2375 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 4,
2380 /* 16 bit read access */
2381 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 2,
2386 /* 8 bit read access */
2387 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 1,
2396 ptr
= qemu_get_ram_ptr(mr
->ram_addr
+ addr1
);
2397 memcpy(buf
, ptr
, l
);
2409 MemTxResult
address_space_write(AddressSpace
*as
, hwaddr addr
, MemTxAttrs attrs
,
2410 const uint8_t *buf
, int len
)
2412 return address_space_rw(as
, addr
, attrs
, (uint8_t *)buf
, len
, true);
2415 MemTxResult
address_space_read(AddressSpace
*as
, hwaddr addr
, MemTxAttrs attrs
,
2416 uint8_t *buf
, int len
)
2418 return address_space_rw(as
, addr
, attrs
, buf
, len
, false);
2422 void cpu_physical_memory_rw(hwaddr addr
, uint8_t *buf
,
2423 int len
, int is_write
)
2425 address_space_rw(&address_space_memory
, addr
, MEMTXATTRS_UNSPECIFIED
,
2426 buf
, len
, is_write
);
2429 enum write_rom_type
{
2434 static inline void cpu_physical_memory_write_rom_internal(AddressSpace
*as
,
2435 hwaddr addr
, const uint8_t *buf
, int len
, enum write_rom_type type
)
2445 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
2447 if (!(memory_region_is_ram(mr
) ||
2448 memory_region_is_romd(mr
))) {
2451 addr1
+= memory_region_get_ram_addr(mr
);
2453 ptr
= qemu_get_ram_ptr(addr1
);
2456 memcpy(ptr
, buf
, l
);
2457 invalidate_and_set_dirty(mr
, addr1
, l
);
2460 flush_icache_range((uintptr_t)ptr
, (uintptr_t)ptr
+ l
);
2471 /* used for ROM loading : can write in RAM and ROM */
2472 void cpu_physical_memory_write_rom(AddressSpace
*as
, hwaddr addr
,
2473 const uint8_t *buf
, int len
)
2475 cpu_physical_memory_write_rom_internal(as
, addr
, buf
, len
, WRITE_DATA
);
2478 void cpu_flush_icache_range(hwaddr start
, int len
)
2481 * This function should do the same thing as an icache flush that was
2482 * triggered from within the guest. For TCG we are always cache coherent,
2483 * so there is no need to flush anything. For KVM / Xen we need to flush
2484 * the host's instruction cache at least.
2486 if (tcg_enabled()) {
2490 cpu_physical_memory_write_rom_internal(&address_space_memory
,
2491 start
, NULL
, len
, FLUSH_CACHE
);
2502 static BounceBuffer bounce
;
2504 typedef struct MapClient
{
2506 QLIST_ENTRY(MapClient
) link
;
2509 QemuMutex map_client_list_lock
;
2510 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
2511 = QLIST_HEAD_INITIALIZER(map_client_list
);
2513 static void cpu_unregister_map_client_do(MapClient
*client
)
2515 QLIST_REMOVE(client
, link
);
2519 static void cpu_notify_map_clients_locked(void)
2523 while (!QLIST_EMPTY(&map_client_list
)) {
2524 client
= QLIST_FIRST(&map_client_list
);
2525 qemu_bh_schedule(client
->bh
);
2526 cpu_unregister_map_client_do(client
);
2530 void cpu_register_map_client(QEMUBH
*bh
)
2532 MapClient
*client
= g_malloc(sizeof(*client
));
2534 qemu_mutex_lock(&map_client_list_lock
);
2536 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
2537 if (!atomic_read(&bounce
.in_use
)) {
2538 cpu_notify_map_clients_locked();
2540 qemu_mutex_unlock(&map_client_list_lock
);
2543 void cpu_exec_init_all(void)
2545 qemu_mutex_init(&ram_list
.mutex
);
2548 qemu_mutex_init(&map_client_list_lock
);
2551 void cpu_unregister_map_client(QEMUBH
*bh
)
2555 qemu_mutex_lock(&map_client_list_lock
);
2556 QLIST_FOREACH(client
, &map_client_list
, link
) {
2557 if (client
->bh
== bh
) {
2558 cpu_unregister_map_client_do(client
);
2562 qemu_mutex_unlock(&map_client_list_lock
);
2565 static void cpu_notify_map_clients(void)
2567 qemu_mutex_lock(&map_client_list_lock
);
2568 cpu_notify_map_clients_locked();
2569 qemu_mutex_unlock(&map_client_list_lock
);
2572 bool address_space_access_valid(AddressSpace
*as
, hwaddr addr
, int len
, bool is_write
)
2580 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2581 if (!memory_access_is_direct(mr
, is_write
)) {
2582 l
= memory_access_size(mr
, l
, addr
);
2583 if (!memory_region_access_valid(mr
, xlat
, l
, is_write
)) {
2595 /* Map a physical memory region into a host virtual address.
2596 * May map a subset of the requested range, given by and returned in *plen.
2597 * May return NULL if resources needed to perform the mapping are exhausted.
2598 * Use only for reads OR writes - not for read-modify-write operations.
2599 * Use cpu_register_map_client() to know when retrying the map operation is
2600 * likely to succeed.
2602 void *address_space_map(AddressSpace
*as
,
2609 hwaddr l
, xlat
, base
;
2610 MemoryRegion
*mr
, *this_mr
;
2619 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2621 if (!memory_access_is_direct(mr
, is_write
)) {
2622 if (atomic_xchg(&bounce
.in_use
, true)) {
2626 /* Avoid unbounded allocations */
2627 l
= MIN(l
, TARGET_PAGE_SIZE
);
2628 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, l
);
2632 memory_region_ref(mr
);
2635 address_space_read(as
, addr
, MEMTXATTRS_UNSPECIFIED
,
2641 return bounce
.buffer
;
2645 raddr
= memory_region_get_ram_addr(mr
);
2656 this_mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2657 if (this_mr
!= mr
|| xlat
!= base
+ done
) {
2662 memory_region_ref(mr
);
2665 return qemu_ram_ptr_length(raddr
+ base
, plen
);
2668 /* Unmaps a memory region previously mapped by address_space_map().
2669 * Will also mark the memory as dirty if is_write == 1. access_len gives
2670 * the amount of memory that was actually read or written by the caller.
2672 void address_space_unmap(AddressSpace
*as
, void *buffer
, hwaddr len
,
2673 int is_write
, hwaddr access_len
)
2675 if (buffer
!= bounce
.buffer
) {
2679 mr
= qemu_ram_addr_from_host(buffer
, &addr1
);
2682 invalidate_and_set_dirty(mr
, addr1
, access_len
);
2684 if (xen_enabled()) {
2685 xen_invalidate_map_cache_entry(buffer
);
2687 memory_region_unref(mr
);
2691 address_space_write(as
, bounce
.addr
, MEMTXATTRS_UNSPECIFIED
,
2692 bounce
.buffer
, access_len
);
2694 qemu_vfree(bounce
.buffer
);
2695 bounce
.buffer
= NULL
;
2696 memory_region_unref(bounce
.mr
);
2697 atomic_mb_set(&bounce
.in_use
, false);
2698 cpu_notify_map_clients();
2701 void *cpu_physical_memory_map(hwaddr addr
,
2705 return address_space_map(&address_space_memory
, addr
, plen
, is_write
);
2708 void cpu_physical_memory_unmap(void *buffer
, hwaddr len
,
2709 int is_write
, hwaddr access_len
)
2711 return address_space_unmap(&address_space_memory
, buffer
, len
, is_write
, access_len
);
2714 /* warning: addr must be aligned */
2715 static inline uint32_t address_space_ldl_internal(AddressSpace
*as
, hwaddr addr
,
2717 MemTxResult
*result
,
2718 enum device_endian endian
)
2728 mr
= address_space_translate(as
, addr
, &addr1
, &l
, false);
2729 if (l
< 4 || !memory_access_is_direct(mr
, false)) {
2731 r
= memory_region_dispatch_read(mr
, addr1
, &val
, 4, attrs
);
2732 #if defined(TARGET_WORDS_BIGENDIAN)
2733 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2737 if (endian
== DEVICE_BIG_ENDIAN
) {
2743 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2747 case DEVICE_LITTLE_ENDIAN
:
2748 val
= ldl_le_p(ptr
);
2750 case DEVICE_BIG_ENDIAN
:
2751 val
= ldl_be_p(ptr
);
2766 uint32_t address_space_ldl(AddressSpace
*as
, hwaddr addr
,
2767 MemTxAttrs attrs
, MemTxResult
*result
)
2769 return address_space_ldl_internal(as
, addr
, attrs
, result
,
2770 DEVICE_NATIVE_ENDIAN
);
2773 uint32_t address_space_ldl_le(AddressSpace
*as
, hwaddr addr
,
2774 MemTxAttrs attrs
, MemTxResult
*result
)
2776 return address_space_ldl_internal(as
, addr
, attrs
, result
,
2777 DEVICE_LITTLE_ENDIAN
);
2780 uint32_t address_space_ldl_be(AddressSpace
*as
, hwaddr addr
,
2781 MemTxAttrs attrs
, MemTxResult
*result
)
2783 return address_space_ldl_internal(as
, addr
, attrs
, result
,
2787 uint32_t ldl_phys(AddressSpace
*as
, hwaddr addr
)
2789 return address_space_ldl(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2792 uint32_t ldl_le_phys(AddressSpace
*as
, hwaddr addr
)
2794 return address_space_ldl_le(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2797 uint32_t ldl_be_phys(AddressSpace
*as
, hwaddr addr
)
2799 return address_space_ldl_be(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2802 /* warning: addr must be aligned */
2803 static inline uint64_t address_space_ldq_internal(AddressSpace
*as
, hwaddr addr
,
2805 MemTxResult
*result
,
2806 enum device_endian endian
)
2816 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2818 if (l
< 8 || !memory_access_is_direct(mr
, false)) {
2820 r
= memory_region_dispatch_read(mr
, addr1
, &val
, 8, attrs
);
2821 #if defined(TARGET_WORDS_BIGENDIAN)
2822 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2826 if (endian
== DEVICE_BIG_ENDIAN
) {
2832 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2836 case DEVICE_LITTLE_ENDIAN
:
2837 val
= ldq_le_p(ptr
);
2839 case DEVICE_BIG_ENDIAN
:
2840 val
= ldq_be_p(ptr
);
2855 uint64_t address_space_ldq(AddressSpace
*as
, hwaddr addr
,
2856 MemTxAttrs attrs
, MemTxResult
*result
)
2858 return address_space_ldq_internal(as
, addr
, attrs
, result
,
2859 DEVICE_NATIVE_ENDIAN
);
2862 uint64_t address_space_ldq_le(AddressSpace
*as
, hwaddr addr
,
2863 MemTxAttrs attrs
, MemTxResult
*result
)
2865 return address_space_ldq_internal(as
, addr
, attrs
, result
,
2866 DEVICE_LITTLE_ENDIAN
);
2869 uint64_t address_space_ldq_be(AddressSpace
*as
, hwaddr addr
,
2870 MemTxAttrs attrs
, MemTxResult
*result
)
2872 return address_space_ldq_internal(as
, addr
, attrs
, result
,
2876 uint64_t ldq_phys(AddressSpace
*as
, hwaddr addr
)
2878 return address_space_ldq(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2881 uint64_t ldq_le_phys(AddressSpace
*as
, hwaddr addr
)
2883 return address_space_ldq_le(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2886 uint64_t ldq_be_phys(AddressSpace
*as
, hwaddr addr
)
2888 return address_space_ldq_be(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2892 uint32_t address_space_ldub(AddressSpace
*as
, hwaddr addr
,
2893 MemTxAttrs attrs
, MemTxResult
*result
)
2898 r
= address_space_rw(as
, addr
, attrs
, &val
, 1, 0);
2905 uint32_t ldub_phys(AddressSpace
*as
, hwaddr addr
)
2907 return address_space_ldub(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2910 /* warning: addr must be aligned */
2911 static inline uint32_t address_space_lduw_internal(AddressSpace
*as
,
2914 MemTxResult
*result
,
2915 enum device_endian endian
)
2925 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2927 if (l
< 2 || !memory_access_is_direct(mr
, false)) {
2929 r
= memory_region_dispatch_read(mr
, addr1
, &val
, 2, attrs
);
2930 #if defined(TARGET_WORDS_BIGENDIAN)
2931 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2935 if (endian
== DEVICE_BIG_ENDIAN
) {
2941 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2945 case DEVICE_LITTLE_ENDIAN
:
2946 val
= lduw_le_p(ptr
);
2948 case DEVICE_BIG_ENDIAN
:
2949 val
= lduw_be_p(ptr
);
2964 uint32_t address_space_lduw(AddressSpace
*as
, hwaddr addr
,
2965 MemTxAttrs attrs
, MemTxResult
*result
)
2967 return address_space_lduw_internal(as
, addr
, attrs
, result
,
2968 DEVICE_NATIVE_ENDIAN
);
2971 uint32_t address_space_lduw_le(AddressSpace
*as
, hwaddr addr
,
2972 MemTxAttrs attrs
, MemTxResult
*result
)
2974 return address_space_lduw_internal(as
, addr
, attrs
, result
,
2975 DEVICE_LITTLE_ENDIAN
);
2978 uint32_t address_space_lduw_be(AddressSpace
*as
, hwaddr addr
,
2979 MemTxAttrs attrs
, MemTxResult
*result
)
2981 return address_space_lduw_internal(as
, addr
, attrs
, result
,
2985 uint32_t lduw_phys(AddressSpace
*as
, hwaddr addr
)
2987 return address_space_lduw(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2990 uint32_t lduw_le_phys(AddressSpace
*as
, hwaddr addr
)
2992 return address_space_lduw_le(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2995 uint32_t lduw_be_phys(AddressSpace
*as
, hwaddr addr
)
2997 return address_space_lduw_be(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3000 /* warning: addr must be aligned. The ram page is not masked as dirty
3001 and the code inside is not invalidated. It is useful if the dirty
3002 bits are used to track modified PTEs */
3003 void address_space_stl_notdirty(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3004 MemTxAttrs attrs
, MemTxResult
*result
)
3011 uint8_t dirty_log_mask
;
3014 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
3016 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
3017 r
= memory_region_dispatch_write(mr
, addr1
, val
, 4, attrs
);
3019 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
3020 ptr
= qemu_get_ram_ptr(addr1
);
3023 dirty_log_mask
= memory_region_get_dirty_log_mask(mr
);
3024 dirty_log_mask
&= ~(1 << DIRTY_MEMORY_CODE
);
3025 cpu_physical_memory_set_dirty_range(addr1
, 4, dirty_log_mask
);
3034 void stl_phys_notdirty(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3036 address_space_stl_notdirty(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3039 /* warning: addr must be aligned */
3040 static inline void address_space_stl_internal(AddressSpace
*as
,
3041 hwaddr addr
, uint32_t val
,
3043 MemTxResult
*result
,
3044 enum device_endian endian
)
3053 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
3055 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
3056 #if defined(TARGET_WORDS_BIGENDIAN)
3057 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3061 if (endian
== DEVICE_BIG_ENDIAN
) {
3065 r
= memory_region_dispatch_write(mr
, addr1
, val
, 4, attrs
);
3068 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
3069 ptr
= qemu_get_ram_ptr(addr1
);
3071 case DEVICE_LITTLE_ENDIAN
:
3074 case DEVICE_BIG_ENDIAN
:
3081 invalidate_and_set_dirty(mr
, addr1
, 4);
3090 void address_space_stl(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3091 MemTxAttrs attrs
, MemTxResult
*result
)
3093 address_space_stl_internal(as
, addr
, val
, attrs
, result
,
3094 DEVICE_NATIVE_ENDIAN
);
3097 void address_space_stl_le(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3098 MemTxAttrs attrs
, MemTxResult
*result
)
3100 address_space_stl_internal(as
, addr
, val
, attrs
, result
,
3101 DEVICE_LITTLE_ENDIAN
);
3104 void address_space_stl_be(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3105 MemTxAttrs attrs
, MemTxResult
*result
)
3107 address_space_stl_internal(as
, addr
, val
, attrs
, result
,
3111 void stl_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3113 address_space_stl(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3116 void stl_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3118 address_space_stl_le(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3121 void stl_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3123 address_space_stl_be(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3127 void address_space_stb(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3128 MemTxAttrs attrs
, MemTxResult
*result
)
3133 r
= address_space_rw(as
, addr
, attrs
, &v
, 1, 1);
3139 void stb_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3141 address_space_stb(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3144 /* warning: addr must be aligned */
3145 static inline void address_space_stw_internal(AddressSpace
*as
,
3146 hwaddr addr
, uint32_t val
,
3148 MemTxResult
*result
,
3149 enum device_endian endian
)
3158 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
3159 if (l
< 2 || !memory_access_is_direct(mr
, true)) {
3160 #if defined(TARGET_WORDS_BIGENDIAN)
3161 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3165 if (endian
== DEVICE_BIG_ENDIAN
) {
3169 r
= memory_region_dispatch_write(mr
, addr1
, val
, 2, attrs
);
3172 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
3173 ptr
= qemu_get_ram_ptr(addr1
);
3175 case DEVICE_LITTLE_ENDIAN
:
3178 case DEVICE_BIG_ENDIAN
:
3185 invalidate_and_set_dirty(mr
, addr1
, 2);
3194 void address_space_stw(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3195 MemTxAttrs attrs
, MemTxResult
*result
)
3197 address_space_stw_internal(as
, addr
, val
, attrs
, result
,
3198 DEVICE_NATIVE_ENDIAN
);
3201 void address_space_stw_le(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3202 MemTxAttrs attrs
, MemTxResult
*result
)
3204 address_space_stw_internal(as
, addr
, val
, attrs
, result
,
3205 DEVICE_LITTLE_ENDIAN
);
3208 void address_space_stw_be(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3209 MemTxAttrs attrs
, MemTxResult
*result
)
3211 address_space_stw_internal(as
, addr
, val
, attrs
, result
,
3215 void stw_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3217 address_space_stw(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3220 void stw_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3222 address_space_stw_le(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3225 void stw_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3227 address_space_stw_be(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3231 void address_space_stq(AddressSpace
*as
, hwaddr addr
, uint64_t val
,
3232 MemTxAttrs attrs
, MemTxResult
*result
)
3236 r
= address_space_rw(as
, addr
, attrs
, (void *) &val
, 8, 1);
3242 void address_space_stq_le(AddressSpace
*as
, hwaddr addr
, uint64_t val
,
3243 MemTxAttrs attrs
, MemTxResult
*result
)
3246 val
= cpu_to_le64(val
);
3247 r
= address_space_rw(as
, addr
, attrs
, (void *) &val
, 8, 1);
3252 void address_space_stq_be(AddressSpace
*as
, hwaddr addr
, uint64_t val
,
3253 MemTxAttrs attrs
, MemTxResult
*result
)
3256 val
= cpu_to_be64(val
);
3257 r
= address_space_rw(as
, addr
, attrs
, (void *) &val
, 8, 1);
3263 void stq_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3265 address_space_stq(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3268 void stq_le_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3270 address_space_stq_le(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3273 void stq_be_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3275 address_space_stq_be(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3278 /* virtual memory access for debug (includes writing to ROM) */
3279 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
3280 uint8_t *buf
, int len
, int is_write
)
3287 page
= addr
& TARGET_PAGE_MASK
;
3288 phys_addr
= cpu_get_phys_page_debug(cpu
, page
);
3289 /* if no physical page mapped, return an error */
3290 if (phys_addr
== -1)
3292 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3295 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
3297 cpu_physical_memory_write_rom(cpu
->as
, phys_addr
, buf
, l
);
3299 address_space_rw(cpu
->as
, phys_addr
, MEMTXATTRS_UNSPECIFIED
,
3311 * A helper function for the _utterly broken_ virtio device model to find out if
3312 * it's running on a big endian machine. Don't do this at home kids!
3314 bool target_words_bigendian(void);
3315 bool target_words_bigendian(void)
3317 #if defined(TARGET_WORDS_BIGENDIAN)
3324 #ifndef CONFIG_USER_ONLY
3325 bool cpu_physical_memory_is_io(hwaddr phys_addr
)
3332 mr
= address_space_translate(&address_space_memory
,
3333 phys_addr
, &phys_addr
, &l
, false);
3335 res
= !(memory_region_is_ram(mr
) || memory_region_is_romd(mr
));
3340 void qemu_ram_foreach_block(RAMBlockIterFunc func
, void *opaque
)
3345 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
3346 func(block
->host
, block
->offset
, block
->used_length
, opaque
);