4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
21 #include <sys/types.h>
25 #include "qemu-common.h"
29 #if !defined(CONFIG_USER_ONLY)
30 #include "hw/boards.h"
33 #include "qemu/osdep.h"
34 #include "sysemu/kvm.h"
35 #include "sysemu/sysemu.h"
36 #include "hw/xen/xen.h"
37 #include "qemu/timer.h"
38 #include "qemu/config-file.h"
39 #include "qemu/error-report.h"
40 #include "exec/memory.h"
41 #include "sysemu/dma.h"
42 #include "exec/address-spaces.h"
43 #if defined(CONFIG_USER_ONLY)
45 #else /* !CONFIG_USER_ONLY */
46 #include "sysemu/xen-mapcache.h"
49 #include "exec/cpu-all.h"
50 #include "qemu/rcu_queue.h"
51 #include "exec/cputlb.h"
52 #include "translate-all.h"
54 #include "exec/memory-internal.h"
55 #include "exec/ram_addr.h"
57 #include "qemu/range.h"
59 //#define DEBUG_SUBPAGE
61 #if !defined(CONFIG_USER_ONLY)
62 static bool in_migration
;
64 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
65 * are protected by the ramlist lock.
67 RAMList ram_list
= { .blocks
= QLIST_HEAD_INITIALIZER(ram_list
.blocks
) };
69 static MemoryRegion
*system_memory
;
70 static MemoryRegion
*system_io
;
72 AddressSpace address_space_io
;
73 AddressSpace address_space_memory
;
75 MemoryRegion io_mem_rom
, io_mem_notdirty
;
76 static MemoryRegion io_mem_unassigned
;
78 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
79 #define RAM_PREALLOC (1 << 0)
81 /* RAM is mmap-ed with MAP_SHARED */
82 #define RAM_SHARED (1 << 1)
84 /* Only a portion of RAM (used_length) is actually used, and migrated.
85 * This used_length size can change across reboots.
87 #define RAM_RESIZEABLE (1 << 2)
91 struct CPUTailQ cpus
= QTAILQ_HEAD_INITIALIZER(cpus
);
92 /* current CPU in the current thread. It is only valid inside
94 DEFINE_TLS(CPUState
*, current_cpu
);
95 /* 0 = Do not count executed instructions.
96 1 = Precise instruction counting.
97 2 = Adaptive rate instruction counting. */
100 #if !defined(CONFIG_USER_ONLY)
102 typedef struct PhysPageEntry PhysPageEntry
;
104 struct PhysPageEntry
{
105 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
107 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
111 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
113 /* Size of the L2 (and L3, etc) page tables. */
114 #define ADDR_SPACE_BITS 64
117 #define P_L2_SIZE (1 << P_L2_BITS)
119 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
121 typedef PhysPageEntry Node
[P_L2_SIZE
];
123 typedef struct PhysPageMap
{
126 unsigned sections_nb
;
127 unsigned sections_nb_alloc
;
129 unsigned nodes_nb_alloc
;
131 MemoryRegionSection
*sections
;
134 struct AddressSpaceDispatch
{
137 /* This is a multi-level map on the physical address space.
138 * The bottom level has pointers to MemoryRegionSections.
140 PhysPageEntry phys_map
;
145 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
146 typedef struct subpage_t
{
150 uint16_t sub_section
[TARGET_PAGE_SIZE
];
153 #define PHYS_SECTION_UNASSIGNED 0
154 #define PHYS_SECTION_NOTDIRTY 1
155 #define PHYS_SECTION_ROM 2
156 #define PHYS_SECTION_WATCH 3
158 static void io_mem_init(void);
159 static void memory_map_init(void);
160 static void tcg_commit(MemoryListener
*listener
);
162 static MemoryRegion io_mem_watch
;
165 #if !defined(CONFIG_USER_ONLY)
167 static void phys_map_node_reserve(PhysPageMap
*map
, unsigned nodes
)
169 if (map
->nodes_nb
+ nodes
> map
->nodes_nb_alloc
) {
170 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
* 2, 16);
171 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
, map
->nodes_nb
+ nodes
);
172 map
->nodes
= g_renew(Node
, map
->nodes
, map
->nodes_nb_alloc
);
176 static uint32_t phys_map_node_alloc(PhysPageMap
*map
)
181 ret
= map
->nodes_nb
++;
182 assert(ret
!= PHYS_MAP_NODE_NIL
);
183 assert(ret
!= map
->nodes_nb_alloc
);
184 for (i
= 0; i
< P_L2_SIZE
; ++i
) {
185 map
->nodes
[ret
][i
].skip
= 1;
186 map
->nodes
[ret
][i
].ptr
= PHYS_MAP_NODE_NIL
;
191 static void phys_page_set_level(PhysPageMap
*map
, PhysPageEntry
*lp
,
192 hwaddr
*index
, hwaddr
*nb
, uint16_t leaf
,
197 hwaddr step
= (hwaddr
)1 << (level
* P_L2_BITS
);
199 if (lp
->skip
&& lp
->ptr
== PHYS_MAP_NODE_NIL
) {
200 lp
->ptr
= phys_map_node_alloc(map
);
201 p
= map
->nodes
[lp
->ptr
];
203 for (i
= 0; i
< P_L2_SIZE
; i
++) {
205 p
[i
].ptr
= PHYS_SECTION_UNASSIGNED
;
209 p
= map
->nodes
[lp
->ptr
];
211 lp
= &p
[(*index
>> (level
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
213 while (*nb
&& lp
< &p
[P_L2_SIZE
]) {
214 if ((*index
& (step
- 1)) == 0 && *nb
>= step
) {
220 phys_page_set_level(map
, lp
, index
, nb
, leaf
, level
- 1);
226 static void phys_page_set(AddressSpaceDispatch
*d
,
227 hwaddr index
, hwaddr nb
,
230 /* Wildly overreserve - it doesn't matter much. */
231 phys_map_node_reserve(&d
->map
, 3 * P_L2_LEVELS
);
233 phys_page_set_level(&d
->map
, &d
->phys_map
, &index
, &nb
, leaf
, P_L2_LEVELS
- 1);
236 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
237 * and update our entry so we can skip it and go directly to the destination.
239 static void phys_page_compact(PhysPageEntry
*lp
, Node
*nodes
, unsigned long *compacted
)
241 unsigned valid_ptr
= P_L2_SIZE
;
246 if (lp
->ptr
== PHYS_MAP_NODE_NIL
) {
251 for (i
= 0; i
< P_L2_SIZE
; i
++) {
252 if (p
[i
].ptr
== PHYS_MAP_NODE_NIL
) {
259 phys_page_compact(&p
[i
], nodes
, compacted
);
263 /* We can only compress if there's only one child. */
268 assert(valid_ptr
< P_L2_SIZE
);
270 /* Don't compress if it won't fit in the # of bits we have. */
271 if (lp
->skip
+ p
[valid_ptr
].skip
>= (1 << 3)) {
275 lp
->ptr
= p
[valid_ptr
].ptr
;
276 if (!p
[valid_ptr
].skip
) {
277 /* If our only child is a leaf, make this a leaf. */
278 /* By design, we should have made this node a leaf to begin with so we
279 * should never reach here.
280 * But since it's so simple to handle this, let's do it just in case we
285 lp
->skip
+= p
[valid_ptr
].skip
;
289 static void phys_page_compact_all(AddressSpaceDispatch
*d
, int nodes_nb
)
291 DECLARE_BITMAP(compacted
, nodes_nb
);
293 if (d
->phys_map
.skip
) {
294 phys_page_compact(&d
->phys_map
, d
->map
.nodes
, compacted
);
298 static MemoryRegionSection
*phys_page_find(PhysPageEntry lp
, hwaddr addr
,
299 Node
*nodes
, MemoryRegionSection
*sections
)
302 hwaddr index
= addr
>> TARGET_PAGE_BITS
;
305 for (i
= P_L2_LEVELS
; lp
.skip
&& (i
-= lp
.skip
) >= 0;) {
306 if (lp
.ptr
== PHYS_MAP_NODE_NIL
) {
307 return §ions
[PHYS_SECTION_UNASSIGNED
];
310 lp
= p
[(index
>> (i
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
313 if (sections
[lp
.ptr
].size
.hi
||
314 range_covers_byte(sections
[lp
.ptr
].offset_within_address_space
,
315 sections
[lp
.ptr
].size
.lo
, addr
)) {
316 return §ions
[lp
.ptr
];
318 return §ions
[PHYS_SECTION_UNASSIGNED
];
322 bool memory_region_is_unassigned(MemoryRegion
*mr
)
324 return mr
!= &io_mem_rom
&& mr
!= &io_mem_notdirty
&& !mr
->rom_device
325 && mr
!= &io_mem_watch
;
328 /* Called from RCU critical section */
329 static MemoryRegionSection
*address_space_lookup_region(AddressSpaceDispatch
*d
,
331 bool resolve_subpage
)
333 MemoryRegionSection
*section
;
336 section
= phys_page_find(d
->phys_map
, addr
, d
->map
.nodes
, d
->map
.sections
);
337 if (resolve_subpage
&& section
->mr
->subpage
) {
338 subpage
= container_of(section
->mr
, subpage_t
, iomem
);
339 section
= &d
->map
.sections
[subpage
->sub_section
[SUBPAGE_IDX(addr
)]];
344 /* Called from RCU critical section */
345 static MemoryRegionSection
*
346 address_space_translate_internal(AddressSpaceDispatch
*d
, hwaddr addr
, hwaddr
*xlat
,
347 hwaddr
*plen
, bool resolve_subpage
)
349 MemoryRegionSection
*section
;
352 section
= address_space_lookup_region(d
, addr
, resolve_subpage
);
353 /* Compute offset within MemoryRegionSection */
354 addr
-= section
->offset_within_address_space
;
356 /* Compute offset within MemoryRegion */
357 *xlat
= addr
+ section
->offset_within_region
;
359 diff
= int128_sub(section
->mr
->size
, int128_make64(addr
));
360 *plen
= int128_get64(int128_min(diff
, int128_make64(*plen
)));
364 static inline bool memory_access_is_direct(MemoryRegion
*mr
, bool is_write
)
366 if (memory_region_is_ram(mr
)) {
367 return !(is_write
&& mr
->readonly
);
369 if (memory_region_is_romd(mr
)) {
376 /* Called from RCU critical section */
377 MemoryRegion
*address_space_translate(AddressSpace
*as
, hwaddr addr
,
378 hwaddr
*xlat
, hwaddr
*plen
,
382 MemoryRegionSection
*section
;
386 AddressSpaceDispatch
*d
= atomic_rcu_read(&as
->dispatch
);
387 section
= address_space_translate_internal(d
, addr
, &addr
, plen
, true);
390 if (!mr
->iommu_ops
) {
394 iotlb
= mr
->iommu_ops
->translate(mr
, addr
, is_write
);
395 addr
= ((iotlb
.translated_addr
& ~iotlb
.addr_mask
)
396 | (addr
& iotlb
.addr_mask
));
397 *plen
= MIN(*plen
, (addr
| iotlb
.addr_mask
) - addr
+ 1);
398 if (!(iotlb
.perm
& (1 << is_write
))) {
399 mr
= &io_mem_unassigned
;
403 as
= iotlb
.target_as
;
406 if (xen_enabled() && memory_access_is_direct(mr
, is_write
)) {
407 hwaddr page
= ((addr
& TARGET_PAGE_MASK
) + TARGET_PAGE_SIZE
) - addr
;
408 *plen
= MIN(page
, *plen
);
415 /* Called from RCU critical section */
416 MemoryRegionSection
*
417 address_space_translate_for_iotlb(CPUState
*cpu
, hwaddr addr
,
418 hwaddr
*xlat
, hwaddr
*plen
)
420 MemoryRegionSection
*section
;
421 section
= address_space_translate_internal(cpu
->memory_dispatch
,
422 addr
, xlat
, plen
, false);
424 assert(!section
->mr
->iommu_ops
);
429 #if !defined(CONFIG_USER_ONLY)
431 static int cpu_common_post_load(void *opaque
, int version_id
)
433 CPUState
*cpu
= opaque
;
435 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
436 version_id is increased. */
437 cpu
->interrupt_request
&= ~0x01;
443 static int cpu_common_pre_load(void *opaque
)
445 CPUState
*cpu
= opaque
;
447 cpu
->exception_index
= -1;
452 static bool cpu_common_exception_index_needed(void *opaque
)
454 CPUState
*cpu
= opaque
;
456 return tcg_enabled() && cpu
->exception_index
!= -1;
459 static const VMStateDescription vmstate_cpu_common_exception_index
= {
460 .name
= "cpu_common/exception_index",
462 .minimum_version_id
= 1,
463 .fields
= (VMStateField
[]) {
464 VMSTATE_INT32(exception_index
, CPUState
),
465 VMSTATE_END_OF_LIST()
469 const VMStateDescription vmstate_cpu_common
= {
470 .name
= "cpu_common",
472 .minimum_version_id
= 1,
473 .pre_load
= cpu_common_pre_load
,
474 .post_load
= cpu_common_post_load
,
475 .fields
= (VMStateField
[]) {
476 VMSTATE_UINT32(halted
, CPUState
),
477 VMSTATE_UINT32(interrupt_request
, CPUState
),
478 VMSTATE_END_OF_LIST()
480 .subsections
= (VMStateSubsection
[]) {
482 .vmsd
= &vmstate_cpu_common_exception_index
,
483 .needed
= cpu_common_exception_index_needed
,
492 CPUState
*qemu_get_cpu(int index
)
497 if (cpu
->cpu_index
== index
) {
505 #if !defined(CONFIG_USER_ONLY)
506 void tcg_cpu_address_space_init(CPUState
*cpu
, AddressSpace
*as
)
508 /* We only support one address space per cpu at the moment. */
509 assert(cpu
->as
== as
);
511 if (cpu
->tcg_as_listener
) {
512 memory_listener_unregister(cpu
->tcg_as_listener
);
514 cpu
->tcg_as_listener
= g_new0(MemoryListener
, 1);
516 cpu
->tcg_as_listener
->commit
= tcg_commit
;
517 memory_listener_register(cpu
->tcg_as_listener
, as
);
521 void cpu_exec_init(CPUArchState
*env
)
523 CPUState
*cpu
= ENV_GET_CPU(env
);
524 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
528 #ifdef TARGET_WORDS_BIGENDIAN
529 cpu
->bigendian
= true;
531 cpu
->bigendian
= false;
534 #if defined(CONFIG_USER_ONLY)
538 CPU_FOREACH(some_cpu
) {
541 cpu
->cpu_index
= cpu_index
;
543 QTAILQ_INIT(&cpu
->breakpoints
);
544 QTAILQ_INIT(&cpu
->watchpoints
);
545 #ifndef CONFIG_USER_ONLY
546 cpu
->as
= &address_space_memory
;
547 cpu
->thread_id
= qemu_get_thread_id();
548 cpu_reload_memory_map(cpu
);
550 QTAILQ_INSERT_TAIL(&cpus
, cpu
, node
);
551 #if defined(CONFIG_USER_ONLY)
554 if (qdev_get_vmsd(DEVICE(cpu
)) == NULL
) {
555 vmstate_register(NULL
, cpu_index
, &vmstate_cpu_common
, cpu
);
557 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
558 register_savevm(NULL
, "cpu", cpu_index
, CPU_SAVE_VERSION
,
559 cpu_save
, cpu_load
, env
);
560 assert(cc
->vmsd
== NULL
);
561 assert(qdev_get_vmsd(DEVICE(cpu
)) == NULL
);
563 if (cc
->vmsd
!= NULL
) {
564 vmstate_register(NULL
, cpu_index
, cc
->vmsd
, cpu
);
568 #if defined(CONFIG_USER_ONLY)
569 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
571 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
574 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
576 hwaddr phys
= cpu_get_phys_page_debug(cpu
, pc
);
578 tb_invalidate_phys_addr(cpu
->as
,
579 phys
| (pc
& ~TARGET_PAGE_MASK
));
584 #if defined(CONFIG_USER_ONLY)
585 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
590 int cpu_watchpoint_remove(CPUState
*cpu
, vaddr addr
, vaddr len
,
596 void cpu_watchpoint_remove_by_ref(CPUState
*cpu
, CPUWatchpoint
*watchpoint
)
600 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
601 int flags
, CPUWatchpoint
**watchpoint
)
606 /* Add a watchpoint. */
607 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
608 int flags
, CPUWatchpoint
**watchpoint
)
612 /* forbid ranges which are empty or run off the end of the address space */
613 if (len
== 0 || (addr
+ len
- 1) < addr
) {
614 error_report("tried to set invalid watchpoint at %"
615 VADDR_PRIx
", len=%" VADDR_PRIu
, addr
, len
);
618 wp
= g_malloc(sizeof(*wp
));
624 /* keep all GDB-injected watchpoints in front */
625 if (flags
& BP_GDB
) {
626 QTAILQ_INSERT_HEAD(&cpu
->watchpoints
, wp
, entry
);
628 QTAILQ_INSERT_TAIL(&cpu
->watchpoints
, wp
, entry
);
631 tlb_flush_page(cpu
, addr
);
638 /* Remove a specific watchpoint. */
639 int cpu_watchpoint_remove(CPUState
*cpu
, vaddr addr
, vaddr len
,
644 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
645 if (addr
== wp
->vaddr
&& len
== wp
->len
646 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
647 cpu_watchpoint_remove_by_ref(cpu
, wp
);
654 /* Remove a specific watchpoint by reference. */
655 void cpu_watchpoint_remove_by_ref(CPUState
*cpu
, CPUWatchpoint
*watchpoint
)
657 QTAILQ_REMOVE(&cpu
->watchpoints
, watchpoint
, entry
);
659 tlb_flush_page(cpu
, watchpoint
->vaddr
);
664 /* Remove all matching watchpoints. */
665 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
667 CPUWatchpoint
*wp
, *next
;
669 QTAILQ_FOREACH_SAFE(wp
, &cpu
->watchpoints
, entry
, next
) {
670 if (wp
->flags
& mask
) {
671 cpu_watchpoint_remove_by_ref(cpu
, wp
);
676 /* Return true if this watchpoint address matches the specified
677 * access (ie the address range covered by the watchpoint overlaps
678 * partially or completely with the address range covered by the
681 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint
*wp
,
685 /* We know the lengths are non-zero, but a little caution is
686 * required to avoid errors in the case where the range ends
687 * exactly at the top of the address space and so addr + len
688 * wraps round to zero.
690 vaddr wpend
= wp
->vaddr
+ wp
->len
- 1;
691 vaddr addrend
= addr
+ len
- 1;
693 return !(addr
> wpend
|| wp
->vaddr
> addrend
);
698 /* Add a breakpoint. */
699 int cpu_breakpoint_insert(CPUState
*cpu
, vaddr pc
, int flags
,
700 CPUBreakpoint
**breakpoint
)
704 bp
= g_malloc(sizeof(*bp
));
709 /* keep all GDB-injected breakpoints in front */
710 if (flags
& BP_GDB
) {
711 QTAILQ_INSERT_HEAD(&cpu
->breakpoints
, bp
, entry
);
713 QTAILQ_INSERT_TAIL(&cpu
->breakpoints
, bp
, entry
);
716 breakpoint_invalidate(cpu
, pc
);
724 /* Remove a specific breakpoint. */
725 int cpu_breakpoint_remove(CPUState
*cpu
, vaddr pc
, int flags
)
729 QTAILQ_FOREACH(bp
, &cpu
->breakpoints
, entry
) {
730 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
731 cpu_breakpoint_remove_by_ref(cpu
, bp
);
738 /* Remove a specific breakpoint by reference. */
739 void cpu_breakpoint_remove_by_ref(CPUState
*cpu
, CPUBreakpoint
*breakpoint
)
741 QTAILQ_REMOVE(&cpu
->breakpoints
, breakpoint
, entry
);
743 breakpoint_invalidate(cpu
, breakpoint
->pc
);
748 /* Remove all matching breakpoints. */
749 void cpu_breakpoint_remove_all(CPUState
*cpu
, int mask
)
751 CPUBreakpoint
*bp
, *next
;
753 QTAILQ_FOREACH_SAFE(bp
, &cpu
->breakpoints
, entry
, next
) {
754 if (bp
->flags
& mask
) {
755 cpu_breakpoint_remove_by_ref(cpu
, bp
);
760 /* enable or disable single step mode. EXCP_DEBUG is returned by the
761 CPU loop after each instruction */
762 void cpu_single_step(CPUState
*cpu
, int enabled
)
764 if (cpu
->singlestep_enabled
!= enabled
) {
765 cpu
->singlestep_enabled
= enabled
;
767 kvm_update_guest_debug(cpu
, 0);
769 /* must flush all the translated code to avoid inconsistencies */
770 /* XXX: only flush what is necessary */
771 CPUArchState
*env
= cpu
->env_ptr
;
777 void QEMU_NORETURN
cpu_abort(CPUState
*cpu
, const char *fmt
, ...)
784 fprintf(stderr
, "qemu: fatal: ");
785 vfprintf(stderr
, fmt
, ap
);
786 fprintf(stderr
, "\n");
787 cpu_dump_state(cpu
, stderr
, fprintf
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
788 if (qemu_log_enabled()) {
789 qemu_log("qemu: fatal: ");
790 qemu_log_vprintf(fmt
, ap2
);
792 log_cpu_state(cpu
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
798 #if defined(CONFIG_USER_ONLY)
800 struct sigaction act
;
801 sigfillset(&act
.sa_mask
);
802 act
.sa_handler
= SIG_DFL
;
803 sigaction(SIGABRT
, &act
, NULL
);
809 #if !defined(CONFIG_USER_ONLY)
810 /* Called from RCU critical section */
811 static RAMBlock
*qemu_get_ram_block(ram_addr_t addr
)
815 block
= atomic_rcu_read(&ram_list
.mru_block
);
816 if (block
&& addr
- block
->offset
< block
->max_length
) {
819 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
820 if (addr
- block
->offset
< block
->max_length
) {
825 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
829 /* It is safe to write mru_block outside the iothread lock. This
834 * xxx removed from list
838 * call_rcu(reclaim_ramblock, xxx);
841 * atomic_rcu_set is not needed here. The block was already published
842 * when it was placed into the list. Here we're just making an extra
843 * copy of the pointer.
845 ram_list
.mru_block
= block
;
849 static void tlb_reset_dirty_range_all(ram_addr_t start
, ram_addr_t length
)
855 end
= TARGET_PAGE_ALIGN(start
+ length
);
856 start
&= TARGET_PAGE_MASK
;
859 block
= qemu_get_ram_block(start
);
860 assert(block
== qemu_get_ram_block(end
- 1));
861 start1
= (uintptr_t)ramblock_ptr(block
, start
- block
->offset
);
862 cpu_tlb_reset_dirty_all(start1
, length
);
866 /* Note: start and end must be within the same ram block. */
867 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t length
,
872 cpu_physical_memory_clear_dirty_range_type(start
, length
, client
);
875 tlb_reset_dirty_range_all(start
, length
);
879 static void cpu_physical_memory_set_dirty_tracking(bool enable
)
881 in_migration
= enable
;
884 /* Called from RCU critical section */
885 hwaddr
memory_region_section_get_iotlb(CPUState
*cpu
,
886 MemoryRegionSection
*section
,
888 hwaddr paddr
, hwaddr xlat
,
890 target_ulong
*address
)
895 if (memory_region_is_ram(section
->mr
)) {
897 iotlb
= (memory_region_get_ram_addr(section
->mr
) & TARGET_PAGE_MASK
)
899 if (!section
->readonly
) {
900 iotlb
|= PHYS_SECTION_NOTDIRTY
;
902 iotlb
|= PHYS_SECTION_ROM
;
905 iotlb
= section
- section
->address_space
->dispatch
->map
.sections
;
909 /* Make accesses to pages with watchpoints go via the
910 watchpoint trap routines. */
911 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
912 if (cpu_watchpoint_address_matches(wp
, vaddr
, TARGET_PAGE_SIZE
)) {
913 /* Avoid trapping reads of pages with a write breakpoint. */
914 if ((prot
& PAGE_WRITE
) || (wp
->flags
& BP_MEM_READ
)) {
915 iotlb
= PHYS_SECTION_WATCH
+ paddr
;
916 *address
|= TLB_MMIO
;
924 #endif /* defined(CONFIG_USER_ONLY) */
926 #if !defined(CONFIG_USER_ONLY)
928 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
930 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
);
932 static void *(*phys_mem_alloc
)(size_t size
, uint64_t *align
) =
936 * Set a custom physical guest memory alloator.
937 * Accelerators with unusual needs may need this. Hopefully, we can
938 * get rid of it eventually.
940 void phys_mem_set_alloc(void *(*alloc
)(size_t, uint64_t *align
))
942 phys_mem_alloc
= alloc
;
945 static uint16_t phys_section_add(PhysPageMap
*map
,
946 MemoryRegionSection
*section
)
948 /* The physical section number is ORed with a page-aligned
949 * pointer to produce the iotlb entries. Thus it should
950 * never overflow into the page-aligned value.
952 assert(map
->sections_nb
< TARGET_PAGE_SIZE
);
954 if (map
->sections_nb
== map
->sections_nb_alloc
) {
955 map
->sections_nb_alloc
= MAX(map
->sections_nb_alloc
* 2, 16);
956 map
->sections
= g_renew(MemoryRegionSection
, map
->sections
,
957 map
->sections_nb_alloc
);
959 map
->sections
[map
->sections_nb
] = *section
;
960 memory_region_ref(section
->mr
);
961 return map
->sections_nb
++;
964 static void phys_section_destroy(MemoryRegion
*mr
)
966 memory_region_unref(mr
);
969 subpage_t
*subpage
= container_of(mr
, subpage_t
, iomem
);
970 object_unref(OBJECT(&subpage
->iomem
));
975 static void phys_sections_free(PhysPageMap
*map
)
977 while (map
->sections_nb
> 0) {
978 MemoryRegionSection
*section
= &map
->sections
[--map
->sections_nb
];
979 phys_section_destroy(section
->mr
);
981 g_free(map
->sections
);
985 static void register_subpage(AddressSpaceDispatch
*d
, MemoryRegionSection
*section
)
988 hwaddr base
= section
->offset_within_address_space
990 MemoryRegionSection
*existing
= phys_page_find(d
->phys_map
, base
,
991 d
->map
.nodes
, d
->map
.sections
);
992 MemoryRegionSection subsection
= {
993 .offset_within_address_space
= base
,
994 .size
= int128_make64(TARGET_PAGE_SIZE
),
998 assert(existing
->mr
->subpage
|| existing
->mr
== &io_mem_unassigned
);
1000 if (!(existing
->mr
->subpage
)) {
1001 subpage
= subpage_init(d
->as
, base
);
1002 subsection
.address_space
= d
->as
;
1003 subsection
.mr
= &subpage
->iomem
;
1004 phys_page_set(d
, base
>> TARGET_PAGE_BITS
, 1,
1005 phys_section_add(&d
->map
, &subsection
));
1007 subpage
= container_of(existing
->mr
, subpage_t
, iomem
);
1009 start
= section
->offset_within_address_space
& ~TARGET_PAGE_MASK
;
1010 end
= start
+ int128_get64(section
->size
) - 1;
1011 subpage_register(subpage
, start
, end
,
1012 phys_section_add(&d
->map
, section
));
1016 static void register_multipage(AddressSpaceDispatch
*d
,
1017 MemoryRegionSection
*section
)
1019 hwaddr start_addr
= section
->offset_within_address_space
;
1020 uint16_t section_index
= phys_section_add(&d
->map
, section
);
1021 uint64_t num_pages
= int128_get64(int128_rshift(section
->size
,
1025 phys_page_set(d
, start_addr
>> TARGET_PAGE_BITS
, num_pages
, section_index
);
1028 static void mem_add(MemoryListener
*listener
, MemoryRegionSection
*section
)
1030 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
1031 AddressSpaceDispatch
*d
= as
->next_dispatch
;
1032 MemoryRegionSection now
= *section
, remain
= *section
;
1033 Int128 page_size
= int128_make64(TARGET_PAGE_SIZE
);
1035 if (now
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
1036 uint64_t left
= TARGET_PAGE_ALIGN(now
.offset_within_address_space
)
1037 - now
.offset_within_address_space
;
1039 now
.size
= int128_min(int128_make64(left
), now
.size
);
1040 register_subpage(d
, &now
);
1042 now
.size
= int128_zero();
1044 while (int128_ne(remain
.size
, now
.size
)) {
1045 remain
.size
= int128_sub(remain
.size
, now
.size
);
1046 remain
.offset_within_address_space
+= int128_get64(now
.size
);
1047 remain
.offset_within_region
+= int128_get64(now
.size
);
1049 if (int128_lt(remain
.size
, page_size
)) {
1050 register_subpage(d
, &now
);
1051 } else if (remain
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
1052 now
.size
= page_size
;
1053 register_subpage(d
, &now
);
1055 now
.size
= int128_and(now
.size
, int128_neg(page_size
));
1056 register_multipage(d
, &now
);
1061 void qemu_flush_coalesced_mmio_buffer(void)
1064 kvm_flush_coalesced_mmio_buffer();
1067 void qemu_mutex_lock_ramlist(void)
1069 qemu_mutex_lock(&ram_list
.mutex
);
1072 void qemu_mutex_unlock_ramlist(void)
1074 qemu_mutex_unlock(&ram_list
.mutex
);
1079 #include <sys/vfs.h>
1081 #define HUGETLBFS_MAGIC 0x958458f6
1083 static long gethugepagesize(const char *path
, Error
**errp
)
1089 ret
= statfs(path
, &fs
);
1090 } while (ret
!= 0 && errno
== EINTR
);
1093 error_setg_errno(errp
, errno
, "failed to get page size of file %s",
1098 if (fs
.f_type
!= HUGETLBFS_MAGIC
)
1099 fprintf(stderr
, "Warning: path not on HugeTLBFS: %s\n", path
);
1104 static void *file_ram_alloc(RAMBlock
*block
,
1110 char *sanitized_name
;
1112 void * volatile area
= NULL
;
1115 Error
*local_err
= NULL
;
1117 hpagesize
= gethugepagesize(path
, &local_err
);
1119 error_propagate(errp
, local_err
);
1122 block
->mr
->align
= hpagesize
;
1124 if (memory
< hpagesize
) {
1125 error_setg(errp
, "memory size 0x" RAM_ADDR_FMT
" must be equal to "
1126 "or larger than huge page size 0x%" PRIx64
,
1131 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1133 "host lacks kvm mmu notifiers, -mem-path unsupported");
1137 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1138 sanitized_name
= g_strdup(memory_region_name(block
->mr
));
1139 for (c
= sanitized_name
; *c
!= '\0'; c
++) {
1144 filename
= g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path
,
1146 g_free(sanitized_name
);
1148 fd
= mkstemp(filename
);
1150 error_setg_errno(errp
, errno
,
1151 "unable to create backing store for hugepages");
1158 memory
= (memory
+hpagesize
-1) & ~(hpagesize
-1);
1161 * ftruncate is not supported by hugetlbfs in older
1162 * hosts, so don't bother bailing out on errors.
1163 * If anything goes wrong with it under other filesystems,
1166 if (ftruncate(fd
, memory
)) {
1167 perror("ftruncate");
1170 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
,
1171 (block
->flags
& RAM_SHARED
? MAP_SHARED
: MAP_PRIVATE
),
1173 if (area
== MAP_FAILED
) {
1174 error_setg_errno(errp
, errno
,
1175 "unable to map backing store for hugepages");
1181 os_mem_prealloc(fd
, area
, memory
);
1189 error_report("%s", error_get_pretty(*errp
));
1196 /* Called with the ramlist lock held. */
1197 static ram_addr_t
find_ram_offset(ram_addr_t size
)
1199 RAMBlock
*block
, *next_block
;
1200 ram_addr_t offset
= RAM_ADDR_MAX
, mingap
= RAM_ADDR_MAX
;
1202 assert(size
!= 0); /* it would hand out same offset multiple times */
1204 if (QLIST_EMPTY_RCU(&ram_list
.blocks
)) {
1208 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1209 ram_addr_t end
, next
= RAM_ADDR_MAX
;
1211 end
= block
->offset
+ block
->max_length
;
1213 QLIST_FOREACH_RCU(next_block
, &ram_list
.blocks
, next
) {
1214 if (next_block
->offset
>= end
) {
1215 next
= MIN(next
, next_block
->offset
);
1218 if (next
- end
>= size
&& next
- end
< mingap
) {
1220 mingap
= next
- end
;
1224 if (offset
== RAM_ADDR_MAX
) {
1225 fprintf(stderr
, "Failed to find gap of requested size: %" PRIu64
"\n",
1233 ram_addr_t
last_ram_offset(void)
1236 ram_addr_t last
= 0;
1239 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1240 last
= MAX(last
, block
->offset
+ block
->max_length
);
1246 static void qemu_ram_setup_dump(void *addr
, ram_addr_t size
)
1250 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1251 if (!machine_dump_guest_core(current_machine
)) {
1252 ret
= qemu_madvise(addr
, size
, QEMU_MADV_DONTDUMP
);
1254 perror("qemu_madvise");
1255 fprintf(stderr
, "madvise doesn't support MADV_DONTDUMP, "
1256 "but dump_guest_core=off specified\n");
1261 /* Called within an RCU critical section, or while the ramlist lock
1264 static RAMBlock
*find_ram_block(ram_addr_t addr
)
1268 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1269 if (block
->offset
== addr
) {
1277 /* Called with iothread lock held. */
1278 void qemu_ram_set_idstr(ram_addr_t addr
, const char *name
, DeviceState
*dev
)
1280 RAMBlock
*new_block
, *block
;
1283 new_block
= find_ram_block(addr
);
1285 assert(!new_block
->idstr
[0]);
1288 char *id
= qdev_get_dev_path(dev
);
1290 snprintf(new_block
->idstr
, sizeof(new_block
->idstr
), "%s/", id
);
1294 pstrcat(new_block
->idstr
, sizeof(new_block
->idstr
), name
);
1296 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1297 if (block
!= new_block
&& !strcmp(block
->idstr
, new_block
->idstr
)) {
1298 fprintf(stderr
, "RAMBlock \"%s\" already registered, abort!\n",
1306 /* Called with iothread lock held. */
1307 void qemu_ram_unset_idstr(ram_addr_t addr
)
1311 /* FIXME: arch_init.c assumes that this is not called throughout
1312 * migration. Ignore the problem since hot-unplug during migration
1313 * does not work anyway.
1317 block
= find_ram_block(addr
);
1319 memset(block
->idstr
, 0, sizeof(block
->idstr
));
1324 static int memory_try_enable_merging(void *addr
, size_t len
)
1326 if (!machine_mem_merge(current_machine
)) {
1327 /* disabled by the user */
1331 return qemu_madvise(addr
, len
, QEMU_MADV_MERGEABLE
);
1334 /* Only legal before guest might have detected the memory size: e.g. on
1335 * incoming migration, or right after reset.
1337 * As memory core doesn't know how is memory accessed, it is up to
1338 * resize callback to update device state and/or add assertions to detect
1339 * misuse, if necessary.
1341 int qemu_ram_resize(ram_addr_t base
, ram_addr_t newsize
, Error
**errp
)
1343 RAMBlock
*block
= find_ram_block(base
);
1347 newsize
= TARGET_PAGE_ALIGN(newsize
);
1349 if (block
->used_length
== newsize
) {
1353 if (!(block
->flags
& RAM_RESIZEABLE
)) {
1354 error_setg_errno(errp
, EINVAL
,
1355 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1356 " in != 0x" RAM_ADDR_FMT
, block
->idstr
,
1357 newsize
, block
->used_length
);
1361 if (block
->max_length
< newsize
) {
1362 error_setg_errno(errp
, EINVAL
,
1363 "Length too large: %s: 0x" RAM_ADDR_FMT
1364 " > 0x" RAM_ADDR_FMT
, block
->idstr
,
1365 newsize
, block
->max_length
);
1369 cpu_physical_memory_clear_dirty_range(block
->offset
, block
->used_length
);
1370 block
->used_length
= newsize
;
1371 cpu_physical_memory_set_dirty_range(block
->offset
, block
->used_length
);
1372 memory_region_set_size(block
->mr
, newsize
);
1373 if (block
->resized
) {
1374 block
->resized(block
->idstr
, newsize
, block
->host
);
1379 static ram_addr_t
ram_block_add(RAMBlock
*new_block
, Error
**errp
)
1382 RAMBlock
*last_block
= NULL
;
1383 ram_addr_t old_ram_size
, new_ram_size
;
1385 old_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1387 qemu_mutex_lock_ramlist();
1388 new_block
->offset
= find_ram_offset(new_block
->max_length
);
1390 if (!new_block
->host
) {
1391 if (xen_enabled()) {
1392 xen_ram_alloc(new_block
->offset
, new_block
->max_length
,
1395 new_block
->host
= phys_mem_alloc(new_block
->max_length
,
1396 &new_block
->mr
->align
);
1397 if (!new_block
->host
) {
1398 error_setg_errno(errp
, errno
,
1399 "cannot set up guest memory '%s'",
1400 memory_region_name(new_block
->mr
));
1401 qemu_mutex_unlock_ramlist();
1404 memory_try_enable_merging(new_block
->host
, new_block
->max_length
);
1408 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1409 * QLIST (which has an RCU-friendly variant) does not have insertion at
1410 * tail, so save the last element in last_block.
1412 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1414 if (block
->max_length
< new_block
->max_length
) {
1419 QLIST_INSERT_BEFORE_RCU(block
, new_block
, next
);
1420 } else if (last_block
) {
1421 QLIST_INSERT_AFTER_RCU(last_block
, new_block
, next
);
1422 } else { /* list is empty */
1423 QLIST_INSERT_HEAD_RCU(&ram_list
.blocks
, new_block
, next
);
1425 ram_list
.mru_block
= NULL
;
1427 /* Write list before version */
1430 qemu_mutex_unlock_ramlist();
1432 new_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1434 if (new_ram_size
> old_ram_size
) {
1437 /* ram_list.dirty_memory[] is protected by the iothread lock. */
1438 for (i
= 0; i
< DIRTY_MEMORY_NUM
; i
++) {
1439 ram_list
.dirty_memory
[i
] =
1440 bitmap_zero_extend(ram_list
.dirty_memory
[i
],
1441 old_ram_size
, new_ram_size
);
1444 cpu_physical_memory_set_dirty_range(new_block
->offset
,
1445 new_block
->used_length
);
1447 if (new_block
->host
) {
1448 qemu_ram_setup_dump(new_block
->host
, new_block
->max_length
);
1449 qemu_madvise(new_block
->host
, new_block
->max_length
, QEMU_MADV_HUGEPAGE
);
1450 qemu_madvise(new_block
->host
, new_block
->max_length
, QEMU_MADV_DONTFORK
);
1451 if (kvm_enabled()) {
1452 kvm_setup_guest_memory(new_block
->host
, new_block
->max_length
);
1456 return new_block
->offset
;
1460 ram_addr_t
qemu_ram_alloc_from_file(ram_addr_t size
, MemoryRegion
*mr
,
1461 bool share
, const char *mem_path
,
1464 RAMBlock
*new_block
;
1466 Error
*local_err
= NULL
;
1468 if (xen_enabled()) {
1469 error_setg(errp
, "-mem-path not supported with Xen");
1473 if (phys_mem_alloc
!= qemu_anon_ram_alloc
) {
1475 * file_ram_alloc() needs to allocate just like
1476 * phys_mem_alloc, but we haven't bothered to provide
1480 "-mem-path not supported with this accelerator");
1484 size
= TARGET_PAGE_ALIGN(size
);
1485 new_block
= g_malloc0(sizeof(*new_block
));
1487 new_block
->used_length
= size
;
1488 new_block
->max_length
= size
;
1489 new_block
->flags
= share
? RAM_SHARED
: 0;
1490 new_block
->host
= file_ram_alloc(new_block
, size
,
1492 if (!new_block
->host
) {
1497 addr
= ram_block_add(new_block
, &local_err
);
1500 error_propagate(errp
, local_err
);
1508 ram_addr_t
qemu_ram_alloc_internal(ram_addr_t size
, ram_addr_t max_size
,
1509 void (*resized
)(const char*,
1512 void *host
, bool resizeable
,
1513 MemoryRegion
*mr
, Error
**errp
)
1515 RAMBlock
*new_block
;
1517 Error
*local_err
= NULL
;
1519 size
= TARGET_PAGE_ALIGN(size
);
1520 max_size
= TARGET_PAGE_ALIGN(max_size
);
1521 new_block
= g_malloc0(sizeof(*new_block
));
1523 new_block
->resized
= resized
;
1524 new_block
->used_length
= size
;
1525 new_block
->max_length
= max_size
;
1526 assert(max_size
>= size
);
1528 new_block
->host
= host
;
1530 new_block
->flags
|= RAM_PREALLOC
;
1533 new_block
->flags
|= RAM_RESIZEABLE
;
1535 addr
= ram_block_add(new_block
, &local_err
);
1538 error_propagate(errp
, local_err
);
1544 ram_addr_t
qemu_ram_alloc_from_ptr(ram_addr_t size
, void *host
,
1545 MemoryRegion
*mr
, Error
**errp
)
1547 return qemu_ram_alloc_internal(size
, size
, NULL
, host
, false, mr
, errp
);
1550 ram_addr_t
qemu_ram_alloc(ram_addr_t size
, MemoryRegion
*mr
, Error
**errp
)
1552 return qemu_ram_alloc_internal(size
, size
, NULL
, NULL
, false, mr
, errp
);
1555 ram_addr_t
qemu_ram_alloc_resizeable(ram_addr_t size
, ram_addr_t maxsz
,
1556 void (*resized
)(const char*,
1559 MemoryRegion
*mr
, Error
**errp
)
1561 return qemu_ram_alloc_internal(size
, maxsz
, resized
, NULL
, true, mr
, errp
);
1564 void qemu_ram_free_from_ptr(ram_addr_t addr
)
1568 qemu_mutex_lock_ramlist();
1569 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1570 if (addr
== block
->offset
) {
1571 QLIST_REMOVE_RCU(block
, next
);
1572 ram_list
.mru_block
= NULL
;
1573 /* Write list before version */
1576 g_free_rcu(block
, rcu
);
1580 qemu_mutex_unlock_ramlist();
1583 static void reclaim_ramblock(RAMBlock
*block
)
1585 if (block
->flags
& RAM_PREALLOC
) {
1587 } else if (xen_enabled()) {
1588 xen_invalidate_map_cache_entry(block
->host
);
1590 } else if (block
->fd
>= 0) {
1591 munmap(block
->host
, block
->max_length
);
1595 qemu_anon_ram_free(block
->host
, block
->max_length
);
1600 void qemu_ram_free(ram_addr_t addr
)
1604 qemu_mutex_lock_ramlist();
1605 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1606 if (addr
== block
->offset
) {
1607 QLIST_REMOVE_RCU(block
, next
);
1608 ram_list
.mru_block
= NULL
;
1609 /* Write list before version */
1612 call_rcu(block
, reclaim_ramblock
, rcu
);
1616 qemu_mutex_unlock_ramlist();
1620 void qemu_ram_remap(ram_addr_t addr
, ram_addr_t length
)
1627 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1628 offset
= addr
- block
->offset
;
1629 if (offset
< block
->max_length
) {
1630 vaddr
= ramblock_ptr(block
, offset
);
1631 if (block
->flags
& RAM_PREALLOC
) {
1633 } else if (xen_enabled()) {
1637 if (block
->fd
>= 0) {
1638 flags
|= (block
->flags
& RAM_SHARED
?
1639 MAP_SHARED
: MAP_PRIVATE
);
1640 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1641 flags
, block
->fd
, offset
);
1644 * Remap needs to match alloc. Accelerators that
1645 * set phys_mem_alloc never remap. If they did,
1646 * we'd need a remap hook here.
1648 assert(phys_mem_alloc
== qemu_anon_ram_alloc
);
1650 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
1651 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1654 if (area
!= vaddr
) {
1655 fprintf(stderr
, "Could not remap addr: "
1656 RAM_ADDR_FMT
"@" RAM_ADDR_FMT
"\n",
1660 memory_try_enable_merging(vaddr
, length
);
1661 qemu_ram_setup_dump(vaddr
, length
);
1666 #endif /* !_WIN32 */
1668 int qemu_get_ram_fd(ram_addr_t addr
)
1674 block
= qemu_get_ram_block(addr
);
1680 void *qemu_get_ram_block_host_ptr(ram_addr_t addr
)
1686 block
= qemu_get_ram_block(addr
);
1687 ptr
= ramblock_ptr(block
, 0);
1692 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1693 * This should not be used for general purpose DMA. Use address_space_map
1694 * or address_space_rw instead. For local memory (e.g. video ram) that the
1695 * device owns, use memory_region_get_ram_ptr.
1697 * By the time this function returns, the returned pointer is not protected
1698 * by RCU anymore. If the caller is not within an RCU critical section and
1699 * does not hold the iothread lock, it must have other means of protecting the
1700 * pointer, such as a reference to the region that includes the incoming
1703 void *qemu_get_ram_ptr(ram_addr_t addr
)
1709 block
= qemu_get_ram_block(addr
);
1711 if (xen_enabled() && block
->host
== NULL
) {
1712 /* We need to check if the requested address is in the RAM
1713 * because we don't want to map the entire memory in QEMU.
1714 * In that case just map until the end of the page.
1716 if (block
->offset
== 0) {
1717 ptr
= xen_map_cache(addr
, 0, 0);
1721 block
->host
= xen_map_cache(block
->offset
, block
->max_length
, 1);
1723 ptr
= ramblock_ptr(block
, addr
- block
->offset
);
1730 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1731 * but takes a size argument.
1733 * By the time this function returns, the returned pointer is not protected
1734 * by RCU anymore. If the caller is not within an RCU critical section and
1735 * does not hold the iothread lock, it must have other means of protecting the
1736 * pointer, such as a reference to the region that includes the incoming
1739 static void *qemu_ram_ptr_length(ram_addr_t addr
, hwaddr
*size
)
1745 if (xen_enabled()) {
1746 return xen_map_cache(addr
, *size
, 1);
1750 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1751 if (addr
- block
->offset
< block
->max_length
) {
1752 if (addr
- block
->offset
+ *size
> block
->max_length
)
1753 *size
= block
->max_length
- addr
+ block
->offset
;
1754 ptr
= ramblock_ptr(block
, addr
- block
->offset
);
1760 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
1765 /* Some of the softmmu routines need to translate from a host pointer
1766 * (typically a TLB entry) back to a ram offset.
1768 * By the time this function returns, the returned pointer is not protected
1769 * by RCU anymore. If the caller is not within an RCU critical section and
1770 * does not hold the iothread lock, it must have other means of protecting the
1771 * pointer, such as a reference to the region that includes the incoming
1774 MemoryRegion
*qemu_ram_addr_from_host(void *ptr
, ram_addr_t
*ram_addr
)
1777 uint8_t *host
= ptr
;
1780 if (xen_enabled()) {
1782 *ram_addr
= xen_ram_addr_from_mapcache(ptr
);
1783 mr
= qemu_get_ram_block(*ram_addr
)->mr
;
1789 block
= atomic_rcu_read(&ram_list
.mru_block
);
1790 if (block
&& block
->host
&& host
- block
->host
< block
->max_length
) {
1794 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1795 /* This case append when the block is not mapped. */
1796 if (block
->host
== NULL
) {
1799 if (host
- block
->host
< block
->max_length
) {
1808 *ram_addr
= block
->offset
+ (host
- block
->host
);
1814 static void notdirty_mem_write(void *opaque
, hwaddr ram_addr
,
1815 uint64_t val
, unsigned size
)
1817 if (!cpu_physical_memory_get_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
)) {
1818 tb_invalidate_phys_page_fast(ram_addr
, size
);
1822 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
1825 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
1828 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
1833 cpu_physical_memory_set_dirty_range_nocode(ram_addr
, size
);
1834 /* we remove the notdirty callback only if the code has been
1836 if (!cpu_physical_memory_is_clean(ram_addr
)) {
1837 CPUArchState
*env
= current_cpu
->env_ptr
;
1838 tlb_set_dirty(env
, current_cpu
->mem_io_vaddr
);
1842 static bool notdirty_mem_accepts(void *opaque
, hwaddr addr
,
1843 unsigned size
, bool is_write
)
1848 static const MemoryRegionOps notdirty_mem_ops
= {
1849 .write
= notdirty_mem_write
,
1850 .valid
.accepts
= notdirty_mem_accepts
,
1851 .endianness
= DEVICE_NATIVE_ENDIAN
,
1854 /* Generate a debug exception if a watchpoint has been hit. */
1855 static void check_watchpoint(int offset
, int len
, MemTxAttrs attrs
, int flags
)
1857 CPUState
*cpu
= current_cpu
;
1858 CPUArchState
*env
= cpu
->env_ptr
;
1859 target_ulong pc
, cs_base
;
1864 if (cpu
->watchpoint_hit
) {
1865 /* We re-entered the check after replacing the TB. Now raise
1866 * the debug interrupt so that is will trigger after the
1867 * current instruction. */
1868 cpu_interrupt(cpu
, CPU_INTERRUPT_DEBUG
);
1871 vaddr
= (cpu
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
1872 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
1873 if (cpu_watchpoint_address_matches(wp
, vaddr
, len
)
1874 && (wp
->flags
& flags
)) {
1875 if (flags
== BP_MEM_READ
) {
1876 wp
->flags
|= BP_WATCHPOINT_HIT_READ
;
1878 wp
->flags
|= BP_WATCHPOINT_HIT_WRITE
;
1880 wp
->hitaddr
= vaddr
;
1881 wp
->hitattrs
= attrs
;
1882 if (!cpu
->watchpoint_hit
) {
1883 cpu
->watchpoint_hit
= wp
;
1884 tb_check_watchpoint(cpu
);
1885 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
1886 cpu
->exception_index
= EXCP_DEBUG
;
1889 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
1890 tb_gen_code(cpu
, pc
, cs_base
, cpu_flags
, 1);
1891 cpu_resume_from_signal(cpu
, NULL
);
1895 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
1900 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1901 so these check for a hit then pass through to the normal out-of-line
1903 static MemTxResult
watch_mem_read(void *opaque
, hwaddr addr
, uint64_t *pdata
,
1904 unsigned size
, MemTxAttrs attrs
)
1909 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, size
, attrs
, BP_MEM_READ
);
1912 data
= address_space_ldub(&address_space_memory
, addr
, attrs
, &res
);
1915 data
= address_space_lduw(&address_space_memory
, addr
, attrs
, &res
);
1918 data
= address_space_ldl(&address_space_memory
, addr
, attrs
, &res
);
1926 static MemTxResult
watch_mem_write(void *opaque
, hwaddr addr
,
1927 uint64_t val
, unsigned size
,
1932 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, size
, attrs
, BP_MEM_WRITE
);
1935 address_space_stb(&address_space_memory
, addr
, val
, attrs
, &res
);
1938 address_space_stw(&address_space_memory
, addr
, val
, attrs
, &res
);
1941 address_space_stl(&address_space_memory
, addr
, val
, attrs
, &res
);
1948 static const MemoryRegionOps watch_mem_ops
= {
1949 .read_with_attrs
= watch_mem_read
,
1950 .write_with_attrs
= watch_mem_write
,
1951 .endianness
= DEVICE_NATIVE_ENDIAN
,
1954 static MemTxResult
subpage_read(void *opaque
, hwaddr addr
, uint64_t *data
,
1955 unsigned len
, MemTxAttrs attrs
)
1957 subpage_t
*subpage
= opaque
;
1961 #if defined(DEBUG_SUBPAGE)
1962 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
"\n", __func__
,
1963 subpage
, len
, addr
);
1965 res
= address_space_read(subpage
->as
, addr
+ subpage
->base
,
1972 *data
= ldub_p(buf
);
1975 *data
= lduw_p(buf
);
1988 static MemTxResult
subpage_write(void *opaque
, hwaddr addr
,
1989 uint64_t value
, unsigned len
, MemTxAttrs attrs
)
1991 subpage_t
*subpage
= opaque
;
1994 #if defined(DEBUG_SUBPAGE)
1995 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
1996 " value %"PRIx64
"\n",
1997 __func__
, subpage
, len
, addr
, value
);
2015 return address_space_write(subpage
->as
, addr
+ subpage
->base
,
2019 static bool subpage_accepts(void *opaque
, hwaddr addr
,
2020 unsigned len
, bool is_write
)
2022 subpage_t
*subpage
= opaque
;
2023 #if defined(DEBUG_SUBPAGE)
2024 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx
"\n",
2025 __func__
, subpage
, is_write
? 'w' : 'r', len
, addr
);
2028 return address_space_access_valid(subpage
->as
, addr
+ subpage
->base
,
2032 static const MemoryRegionOps subpage_ops
= {
2033 .read_with_attrs
= subpage_read
,
2034 .write_with_attrs
= subpage_write
,
2035 .impl
.min_access_size
= 1,
2036 .impl
.max_access_size
= 8,
2037 .valid
.min_access_size
= 1,
2038 .valid
.max_access_size
= 8,
2039 .valid
.accepts
= subpage_accepts
,
2040 .endianness
= DEVICE_NATIVE_ENDIAN
,
2043 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2048 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
2050 idx
= SUBPAGE_IDX(start
);
2051 eidx
= SUBPAGE_IDX(end
);
2052 #if defined(DEBUG_SUBPAGE)
2053 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2054 __func__
, mmio
, start
, end
, idx
, eidx
, section
);
2056 for (; idx
<= eidx
; idx
++) {
2057 mmio
->sub_section
[idx
] = section
;
2063 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
)
2067 mmio
= g_malloc0(sizeof(subpage_t
));
2071 memory_region_init_io(&mmio
->iomem
, NULL
, &subpage_ops
, mmio
,
2072 NULL
, TARGET_PAGE_SIZE
);
2073 mmio
->iomem
.subpage
= true;
2074 #if defined(DEBUG_SUBPAGE)
2075 printf("%s: %p base " TARGET_FMT_plx
" len %08x\n", __func__
,
2076 mmio
, base
, TARGET_PAGE_SIZE
);
2078 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
-1, PHYS_SECTION_UNASSIGNED
);
2083 static uint16_t dummy_section(PhysPageMap
*map
, AddressSpace
*as
,
2087 MemoryRegionSection section
= {
2088 .address_space
= as
,
2090 .offset_within_address_space
= 0,
2091 .offset_within_region
= 0,
2092 .size
= int128_2_64(),
2095 return phys_section_add(map
, §ion
);
2098 MemoryRegion
*iotlb_to_region(CPUState
*cpu
, hwaddr index
)
2100 AddressSpaceDispatch
*d
= atomic_rcu_read(&cpu
->memory_dispatch
);
2101 MemoryRegionSection
*sections
= d
->map
.sections
;
2103 return sections
[index
& ~TARGET_PAGE_MASK
].mr
;
2106 static void io_mem_init(void)
2108 memory_region_init_io(&io_mem_rom
, NULL
, &unassigned_mem_ops
, NULL
, NULL
, UINT64_MAX
);
2109 memory_region_init_io(&io_mem_unassigned
, NULL
, &unassigned_mem_ops
, NULL
,
2111 memory_region_init_io(&io_mem_notdirty
, NULL
, ¬dirty_mem_ops
, NULL
,
2113 memory_region_init_io(&io_mem_watch
, NULL
, &watch_mem_ops
, NULL
,
2117 static void mem_begin(MemoryListener
*listener
)
2119 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
2120 AddressSpaceDispatch
*d
= g_new0(AddressSpaceDispatch
, 1);
2123 n
= dummy_section(&d
->map
, as
, &io_mem_unassigned
);
2124 assert(n
== PHYS_SECTION_UNASSIGNED
);
2125 n
= dummy_section(&d
->map
, as
, &io_mem_notdirty
);
2126 assert(n
== PHYS_SECTION_NOTDIRTY
);
2127 n
= dummy_section(&d
->map
, as
, &io_mem_rom
);
2128 assert(n
== PHYS_SECTION_ROM
);
2129 n
= dummy_section(&d
->map
, as
, &io_mem_watch
);
2130 assert(n
== PHYS_SECTION_WATCH
);
2132 d
->phys_map
= (PhysPageEntry
) { .ptr
= PHYS_MAP_NODE_NIL
, .skip
= 1 };
2134 as
->next_dispatch
= d
;
2137 static void address_space_dispatch_free(AddressSpaceDispatch
*d
)
2139 phys_sections_free(&d
->map
);
2143 static void mem_commit(MemoryListener
*listener
)
2145 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
2146 AddressSpaceDispatch
*cur
= as
->dispatch
;
2147 AddressSpaceDispatch
*next
= as
->next_dispatch
;
2149 phys_page_compact_all(next
, next
->map
.nodes_nb
);
2151 atomic_rcu_set(&as
->dispatch
, next
);
2153 call_rcu(cur
, address_space_dispatch_free
, rcu
);
2157 static void tcg_commit(MemoryListener
*listener
)
2161 /* since each CPU stores ram addresses in its TLB cache, we must
2162 reset the modified entries */
2165 /* FIXME: Disentangle the cpu.h circular files deps so we can
2166 directly get the right CPU from listener. */
2167 if (cpu
->tcg_as_listener
!= listener
) {
2170 cpu_reload_memory_map(cpu
);
2174 static void core_log_global_start(MemoryListener
*listener
)
2176 cpu_physical_memory_set_dirty_tracking(true);
2179 static void core_log_global_stop(MemoryListener
*listener
)
2181 cpu_physical_memory_set_dirty_tracking(false);
2184 static MemoryListener core_memory_listener
= {
2185 .log_global_start
= core_log_global_start
,
2186 .log_global_stop
= core_log_global_stop
,
2190 void address_space_init_dispatch(AddressSpace
*as
)
2192 as
->dispatch
= NULL
;
2193 as
->dispatch_listener
= (MemoryListener
) {
2195 .commit
= mem_commit
,
2196 .region_add
= mem_add
,
2197 .region_nop
= mem_add
,
2200 memory_listener_register(&as
->dispatch_listener
, as
);
2203 void address_space_unregister(AddressSpace
*as
)
2205 memory_listener_unregister(&as
->dispatch_listener
);
2208 void address_space_destroy_dispatch(AddressSpace
*as
)
2210 AddressSpaceDispatch
*d
= as
->dispatch
;
2212 atomic_rcu_set(&as
->dispatch
, NULL
);
2214 call_rcu(d
, address_space_dispatch_free
, rcu
);
2218 static void memory_map_init(void)
2220 system_memory
= g_malloc(sizeof(*system_memory
));
2222 memory_region_init(system_memory
, NULL
, "system", UINT64_MAX
);
2223 address_space_init(&address_space_memory
, system_memory
, "memory");
2225 system_io
= g_malloc(sizeof(*system_io
));
2226 memory_region_init_io(system_io
, NULL
, &unassigned_io_ops
, NULL
, "io",
2228 address_space_init(&address_space_io
, system_io
, "I/O");
2230 memory_listener_register(&core_memory_listener
, &address_space_memory
);
2233 MemoryRegion
*get_system_memory(void)
2235 return system_memory
;
2238 MemoryRegion
*get_system_io(void)
2243 #endif /* !defined(CONFIG_USER_ONLY) */
2245 /* physical memory access (slow version, mainly for debug) */
2246 #if defined(CONFIG_USER_ONLY)
2247 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
2248 uint8_t *buf
, int len
, int is_write
)
2255 page
= addr
& TARGET_PAGE_MASK
;
2256 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2259 flags
= page_get_flags(page
);
2260 if (!(flags
& PAGE_VALID
))
2263 if (!(flags
& PAGE_WRITE
))
2265 /* XXX: this code should not depend on lock_user */
2266 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
2269 unlock_user(p
, addr
, l
);
2271 if (!(flags
& PAGE_READ
))
2273 /* XXX: this code should not depend on lock_user */
2274 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
2277 unlock_user(p
, addr
, 0);
2288 static void invalidate_and_set_dirty(hwaddr addr
,
2291 if (cpu_physical_memory_range_includes_clean(addr
, length
)) {
2292 tb_invalidate_phys_range(addr
, addr
+ length
, 0);
2293 cpu_physical_memory_set_dirty_range_nocode(addr
, length
);
2295 xen_modified_memory(addr
, length
);
2298 static int memory_access_size(MemoryRegion
*mr
, unsigned l
, hwaddr addr
)
2300 unsigned access_size_max
= mr
->ops
->valid
.max_access_size
;
2302 /* Regions are assumed to support 1-4 byte accesses unless
2303 otherwise specified. */
2304 if (access_size_max
== 0) {
2305 access_size_max
= 4;
2308 /* Bound the maximum access by the alignment of the address. */
2309 if (!mr
->ops
->impl
.unaligned
) {
2310 unsigned align_size_max
= addr
& -addr
;
2311 if (align_size_max
!= 0 && align_size_max
< access_size_max
) {
2312 access_size_max
= align_size_max
;
2316 /* Don't attempt accesses larger than the maximum. */
2317 if (l
> access_size_max
) {
2318 l
= access_size_max
;
2321 l
= 1 << (qemu_fls(l
) - 1);
2327 MemTxResult
address_space_rw(AddressSpace
*as
, hwaddr addr
, MemTxAttrs attrs
,
2328 uint8_t *buf
, int len
, bool is_write
)
2335 MemTxResult result
= MEMTX_OK
;
2340 mr
= address_space_translate(as
, addr
, &addr1
, &l
, is_write
);
2343 if (!memory_access_is_direct(mr
, is_write
)) {
2344 l
= memory_access_size(mr
, l
, addr1
);
2345 /* XXX: could force current_cpu to NULL to avoid
2349 /* 64 bit write access */
2351 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 8,
2355 /* 32 bit write access */
2357 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 4,
2361 /* 16 bit write access */
2363 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 2,
2367 /* 8 bit write access */
2369 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 1,
2376 addr1
+= memory_region_get_ram_addr(mr
);
2378 ptr
= qemu_get_ram_ptr(addr1
);
2379 memcpy(ptr
, buf
, l
);
2380 invalidate_and_set_dirty(addr1
, l
);
2383 if (!memory_access_is_direct(mr
, is_write
)) {
2385 l
= memory_access_size(mr
, l
, addr1
);
2388 /* 64 bit read access */
2389 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 8,
2394 /* 32 bit read access */
2395 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 4,
2400 /* 16 bit read access */
2401 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 2,
2406 /* 8 bit read access */
2407 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 1,
2416 ptr
= qemu_get_ram_ptr(mr
->ram_addr
+ addr1
);
2417 memcpy(buf
, ptr
, l
);
2429 MemTxResult
address_space_write(AddressSpace
*as
, hwaddr addr
, MemTxAttrs attrs
,
2430 const uint8_t *buf
, int len
)
2432 return address_space_rw(as
, addr
, attrs
, (uint8_t *)buf
, len
, true);
2435 MemTxResult
address_space_read(AddressSpace
*as
, hwaddr addr
, MemTxAttrs attrs
,
2436 uint8_t *buf
, int len
)
2438 return address_space_rw(as
, addr
, attrs
, buf
, len
, false);
2442 void cpu_physical_memory_rw(hwaddr addr
, uint8_t *buf
,
2443 int len
, int is_write
)
2445 address_space_rw(&address_space_memory
, addr
, MEMTXATTRS_UNSPECIFIED
,
2446 buf
, len
, is_write
);
2449 enum write_rom_type
{
2454 static inline void cpu_physical_memory_write_rom_internal(AddressSpace
*as
,
2455 hwaddr addr
, const uint8_t *buf
, int len
, enum write_rom_type type
)
2465 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
2467 if (!(memory_region_is_ram(mr
) ||
2468 memory_region_is_romd(mr
))) {
2471 addr1
+= memory_region_get_ram_addr(mr
);
2473 ptr
= qemu_get_ram_ptr(addr1
);
2476 memcpy(ptr
, buf
, l
);
2477 invalidate_and_set_dirty(addr1
, l
);
2480 flush_icache_range((uintptr_t)ptr
, (uintptr_t)ptr
+ l
);
2491 /* used for ROM loading : can write in RAM and ROM */
2492 void cpu_physical_memory_write_rom(AddressSpace
*as
, hwaddr addr
,
2493 const uint8_t *buf
, int len
)
2495 cpu_physical_memory_write_rom_internal(as
, addr
, buf
, len
, WRITE_DATA
);
2498 void cpu_flush_icache_range(hwaddr start
, int len
)
2501 * This function should do the same thing as an icache flush that was
2502 * triggered from within the guest. For TCG we are always cache coherent,
2503 * so there is no need to flush anything. For KVM / Xen we need to flush
2504 * the host's instruction cache at least.
2506 if (tcg_enabled()) {
2510 cpu_physical_memory_write_rom_internal(&address_space_memory
,
2511 start
, NULL
, len
, FLUSH_CACHE
);
2522 static BounceBuffer bounce
;
2524 typedef struct MapClient
{
2526 QLIST_ENTRY(MapClient
) link
;
2529 QemuMutex map_client_list_lock
;
2530 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
2531 = QLIST_HEAD_INITIALIZER(map_client_list
);
2533 static void cpu_unregister_map_client_do(MapClient
*client
)
2535 QLIST_REMOVE(client
, link
);
2539 static void cpu_notify_map_clients_locked(void)
2543 while (!QLIST_EMPTY(&map_client_list
)) {
2544 client
= QLIST_FIRST(&map_client_list
);
2545 qemu_bh_schedule(client
->bh
);
2546 cpu_unregister_map_client_do(client
);
2550 void cpu_register_map_client(QEMUBH
*bh
)
2552 MapClient
*client
= g_malloc(sizeof(*client
));
2554 qemu_mutex_lock(&map_client_list_lock
);
2556 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
2557 if (!atomic_read(&bounce
.in_use
)) {
2558 cpu_notify_map_clients_locked();
2560 qemu_mutex_unlock(&map_client_list_lock
);
2563 void cpu_exec_init_all(void)
2565 qemu_mutex_init(&ram_list
.mutex
);
2568 qemu_mutex_init(&map_client_list_lock
);
2571 void cpu_unregister_map_client(QEMUBH
*bh
)
2575 qemu_mutex_lock(&map_client_list_lock
);
2576 QLIST_FOREACH(client
, &map_client_list
, link
) {
2577 if (client
->bh
== bh
) {
2578 cpu_unregister_map_client_do(client
);
2582 qemu_mutex_unlock(&map_client_list_lock
);
2585 static void cpu_notify_map_clients(void)
2587 qemu_mutex_lock(&map_client_list_lock
);
2588 cpu_notify_map_clients_locked();
2589 qemu_mutex_unlock(&map_client_list_lock
);
2592 bool address_space_access_valid(AddressSpace
*as
, hwaddr addr
, int len
, bool is_write
)
2600 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2601 if (!memory_access_is_direct(mr
, is_write
)) {
2602 l
= memory_access_size(mr
, l
, addr
);
2603 if (!memory_region_access_valid(mr
, xlat
, l
, is_write
)) {
2615 /* Map a physical memory region into a host virtual address.
2616 * May map a subset of the requested range, given by and returned in *plen.
2617 * May return NULL if resources needed to perform the mapping are exhausted.
2618 * Use only for reads OR writes - not for read-modify-write operations.
2619 * Use cpu_register_map_client() to know when retrying the map operation is
2620 * likely to succeed.
2622 void *address_space_map(AddressSpace
*as
,
2629 hwaddr l
, xlat
, base
;
2630 MemoryRegion
*mr
, *this_mr
;
2639 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2641 if (!memory_access_is_direct(mr
, is_write
)) {
2642 if (atomic_xchg(&bounce
.in_use
, true)) {
2646 /* Avoid unbounded allocations */
2647 l
= MIN(l
, TARGET_PAGE_SIZE
);
2648 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, l
);
2652 memory_region_ref(mr
);
2655 address_space_read(as
, addr
, MEMTXATTRS_UNSPECIFIED
,
2661 return bounce
.buffer
;
2665 raddr
= memory_region_get_ram_addr(mr
);
2676 this_mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2677 if (this_mr
!= mr
|| xlat
!= base
+ done
) {
2682 memory_region_ref(mr
);
2685 return qemu_ram_ptr_length(raddr
+ base
, plen
);
2688 /* Unmaps a memory region previously mapped by address_space_map().
2689 * Will also mark the memory as dirty if is_write == 1. access_len gives
2690 * the amount of memory that was actually read or written by the caller.
2692 void address_space_unmap(AddressSpace
*as
, void *buffer
, hwaddr len
,
2693 int is_write
, hwaddr access_len
)
2695 if (buffer
!= bounce
.buffer
) {
2699 mr
= qemu_ram_addr_from_host(buffer
, &addr1
);
2702 invalidate_and_set_dirty(addr1
, access_len
);
2704 if (xen_enabled()) {
2705 xen_invalidate_map_cache_entry(buffer
);
2707 memory_region_unref(mr
);
2711 address_space_write(as
, bounce
.addr
, MEMTXATTRS_UNSPECIFIED
,
2712 bounce
.buffer
, access_len
);
2714 qemu_vfree(bounce
.buffer
);
2715 bounce
.buffer
= NULL
;
2716 memory_region_unref(bounce
.mr
);
2717 atomic_mb_set(&bounce
.in_use
, false);
2718 cpu_notify_map_clients();
2721 void *cpu_physical_memory_map(hwaddr addr
,
2725 return address_space_map(&address_space_memory
, addr
, plen
, is_write
);
2728 void cpu_physical_memory_unmap(void *buffer
, hwaddr len
,
2729 int is_write
, hwaddr access_len
)
2731 return address_space_unmap(&address_space_memory
, buffer
, len
, is_write
, access_len
);
2734 /* warning: addr must be aligned */
2735 static inline uint32_t address_space_ldl_internal(AddressSpace
*as
, hwaddr addr
,
2737 MemTxResult
*result
,
2738 enum device_endian endian
)
2748 mr
= address_space_translate(as
, addr
, &addr1
, &l
, false);
2749 if (l
< 4 || !memory_access_is_direct(mr
, false)) {
2751 r
= memory_region_dispatch_read(mr
, addr1
, &val
, 4, attrs
);
2752 #if defined(TARGET_WORDS_BIGENDIAN)
2753 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2757 if (endian
== DEVICE_BIG_ENDIAN
) {
2763 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2767 case DEVICE_LITTLE_ENDIAN
:
2768 val
= ldl_le_p(ptr
);
2770 case DEVICE_BIG_ENDIAN
:
2771 val
= ldl_be_p(ptr
);
2786 uint32_t address_space_ldl(AddressSpace
*as
, hwaddr addr
,
2787 MemTxAttrs attrs
, MemTxResult
*result
)
2789 return address_space_ldl_internal(as
, addr
, attrs
, result
,
2790 DEVICE_NATIVE_ENDIAN
);
2793 uint32_t address_space_ldl_le(AddressSpace
*as
, hwaddr addr
,
2794 MemTxAttrs attrs
, MemTxResult
*result
)
2796 return address_space_ldl_internal(as
, addr
, attrs
, result
,
2797 DEVICE_LITTLE_ENDIAN
);
2800 uint32_t address_space_ldl_be(AddressSpace
*as
, hwaddr addr
,
2801 MemTxAttrs attrs
, MemTxResult
*result
)
2803 return address_space_ldl_internal(as
, addr
, attrs
, result
,
2807 uint32_t ldl_phys(AddressSpace
*as
, hwaddr addr
)
2809 return address_space_ldl(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2812 uint32_t ldl_le_phys(AddressSpace
*as
, hwaddr addr
)
2814 return address_space_ldl_le(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2817 uint32_t ldl_be_phys(AddressSpace
*as
, hwaddr addr
)
2819 return address_space_ldl_be(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2822 /* warning: addr must be aligned */
2823 static inline uint64_t address_space_ldq_internal(AddressSpace
*as
, hwaddr addr
,
2825 MemTxResult
*result
,
2826 enum device_endian endian
)
2836 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2838 if (l
< 8 || !memory_access_is_direct(mr
, false)) {
2840 r
= memory_region_dispatch_read(mr
, addr1
, &val
, 8, attrs
);
2841 #if defined(TARGET_WORDS_BIGENDIAN)
2842 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2846 if (endian
== DEVICE_BIG_ENDIAN
) {
2852 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2856 case DEVICE_LITTLE_ENDIAN
:
2857 val
= ldq_le_p(ptr
);
2859 case DEVICE_BIG_ENDIAN
:
2860 val
= ldq_be_p(ptr
);
2875 uint64_t address_space_ldq(AddressSpace
*as
, hwaddr addr
,
2876 MemTxAttrs attrs
, MemTxResult
*result
)
2878 return address_space_ldq_internal(as
, addr
, attrs
, result
,
2879 DEVICE_NATIVE_ENDIAN
);
2882 uint64_t address_space_ldq_le(AddressSpace
*as
, hwaddr addr
,
2883 MemTxAttrs attrs
, MemTxResult
*result
)
2885 return address_space_ldq_internal(as
, addr
, attrs
, result
,
2886 DEVICE_LITTLE_ENDIAN
);
2889 uint64_t address_space_ldq_be(AddressSpace
*as
, hwaddr addr
,
2890 MemTxAttrs attrs
, MemTxResult
*result
)
2892 return address_space_ldq_internal(as
, addr
, attrs
, result
,
2896 uint64_t ldq_phys(AddressSpace
*as
, hwaddr addr
)
2898 return address_space_ldq(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2901 uint64_t ldq_le_phys(AddressSpace
*as
, hwaddr addr
)
2903 return address_space_ldq_le(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2906 uint64_t ldq_be_phys(AddressSpace
*as
, hwaddr addr
)
2908 return address_space_ldq_be(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2912 uint32_t address_space_ldub(AddressSpace
*as
, hwaddr addr
,
2913 MemTxAttrs attrs
, MemTxResult
*result
)
2918 r
= address_space_rw(as
, addr
, attrs
, &val
, 1, 0);
2925 uint32_t ldub_phys(AddressSpace
*as
, hwaddr addr
)
2927 return address_space_ldub(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2930 /* warning: addr must be aligned */
2931 static inline uint32_t address_space_lduw_internal(AddressSpace
*as
,
2934 MemTxResult
*result
,
2935 enum device_endian endian
)
2945 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2947 if (l
< 2 || !memory_access_is_direct(mr
, false)) {
2949 r
= memory_region_dispatch_read(mr
, addr1
, &val
, 2, attrs
);
2950 #if defined(TARGET_WORDS_BIGENDIAN)
2951 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2955 if (endian
== DEVICE_BIG_ENDIAN
) {
2961 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2965 case DEVICE_LITTLE_ENDIAN
:
2966 val
= lduw_le_p(ptr
);
2968 case DEVICE_BIG_ENDIAN
:
2969 val
= lduw_be_p(ptr
);
2984 uint32_t address_space_lduw(AddressSpace
*as
, hwaddr addr
,
2985 MemTxAttrs attrs
, MemTxResult
*result
)
2987 return address_space_lduw_internal(as
, addr
, attrs
, result
,
2988 DEVICE_NATIVE_ENDIAN
);
2991 uint32_t address_space_lduw_le(AddressSpace
*as
, hwaddr addr
,
2992 MemTxAttrs attrs
, MemTxResult
*result
)
2994 return address_space_lduw_internal(as
, addr
, attrs
, result
,
2995 DEVICE_LITTLE_ENDIAN
);
2998 uint32_t address_space_lduw_be(AddressSpace
*as
, hwaddr addr
,
2999 MemTxAttrs attrs
, MemTxResult
*result
)
3001 return address_space_lduw_internal(as
, addr
, attrs
, result
,
3005 uint32_t lduw_phys(AddressSpace
*as
, hwaddr addr
)
3007 return address_space_lduw(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3010 uint32_t lduw_le_phys(AddressSpace
*as
, hwaddr addr
)
3012 return address_space_lduw_le(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3015 uint32_t lduw_be_phys(AddressSpace
*as
, hwaddr addr
)
3017 return address_space_lduw_be(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3020 /* warning: addr must be aligned. The ram page is not masked as dirty
3021 and the code inside is not invalidated. It is useful if the dirty
3022 bits are used to track modified PTEs */
3023 void address_space_stl_notdirty(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3024 MemTxAttrs attrs
, MemTxResult
*result
)
3033 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
3035 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
3036 r
= memory_region_dispatch_write(mr
, addr1
, val
, 4, attrs
);
3038 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
3039 ptr
= qemu_get_ram_ptr(addr1
);
3042 if (unlikely(in_migration
)) {
3043 if (cpu_physical_memory_is_clean(addr1
)) {
3044 /* invalidate code */
3045 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3047 cpu_physical_memory_set_dirty_range_nocode(addr1
, 4);
3058 void stl_phys_notdirty(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3060 address_space_stl_notdirty(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3063 /* warning: addr must be aligned */
3064 static inline void address_space_stl_internal(AddressSpace
*as
,
3065 hwaddr addr
, uint32_t val
,
3067 MemTxResult
*result
,
3068 enum device_endian endian
)
3077 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
3079 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
3080 #if defined(TARGET_WORDS_BIGENDIAN)
3081 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3085 if (endian
== DEVICE_BIG_ENDIAN
) {
3089 r
= memory_region_dispatch_write(mr
, addr1
, val
, 4, attrs
);
3092 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
3093 ptr
= qemu_get_ram_ptr(addr1
);
3095 case DEVICE_LITTLE_ENDIAN
:
3098 case DEVICE_BIG_ENDIAN
:
3105 invalidate_and_set_dirty(addr1
, 4);
3114 void address_space_stl(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3115 MemTxAttrs attrs
, MemTxResult
*result
)
3117 address_space_stl_internal(as
, addr
, val
, attrs
, result
,
3118 DEVICE_NATIVE_ENDIAN
);
3121 void address_space_stl_le(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3122 MemTxAttrs attrs
, MemTxResult
*result
)
3124 address_space_stl_internal(as
, addr
, val
, attrs
, result
,
3125 DEVICE_LITTLE_ENDIAN
);
3128 void address_space_stl_be(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3129 MemTxAttrs attrs
, MemTxResult
*result
)
3131 address_space_stl_internal(as
, addr
, val
, attrs
, result
,
3135 void stl_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3137 address_space_stl(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3140 void stl_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3142 address_space_stl_le(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3145 void stl_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3147 address_space_stl_be(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3151 void address_space_stb(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3152 MemTxAttrs attrs
, MemTxResult
*result
)
3157 r
= address_space_rw(as
, addr
, attrs
, &v
, 1, 1);
3163 void stb_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3165 address_space_stb(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3168 /* warning: addr must be aligned */
3169 static inline void address_space_stw_internal(AddressSpace
*as
,
3170 hwaddr addr
, uint32_t val
,
3172 MemTxResult
*result
,
3173 enum device_endian endian
)
3182 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
3183 if (l
< 2 || !memory_access_is_direct(mr
, true)) {
3184 #if defined(TARGET_WORDS_BIGENDIAN)
3185 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3189 if (endian
== DEVICE_BIG_ENDIAN
) {
3193 r
= memory_region_dispatch_write(mr
, addr1
, val
, 2, attrs
);
3196 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
3197 ptr
= qemu_get_ram_ptr(addr1
);
3199 case DEVICE_LITTLE_ENDIAN
:
3202 case DEVICE_BIG_ENDIAN
:
3209 invalidate_and_set_dirty(addr1
, 2);
3218 void address_space_stw(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3219 MemTxAttrs attrs
, MemTxResult
*result
)
3221 address_space_stw_internal(as
, addr
, val
, attrs
, result
,
3222 DEVICE_NATIVE_ENDIAN
);
3225 void address_space_stw_le(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3226 MemTxAttrs attrs
, MemTxResult
*result
)
3228 address_space_stw_internal(as
, addr
, val
, attrs
, result
,
3229 DEVICE_LITTLE_ENDIAN
);
3232 void address_space_stw_be(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3233 MemTxAttrs attrs
, MemTxResult
*result
)
3235 address_space_stw_internal(as
, addr
, val
, attrs
, result
,
3239 void stw_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3241 address_space_stw(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3244 void stw_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3246 address_space_stw_le(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3249 void stw_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3251 address_space_stw_be(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3255 void address_space_stq(AddressSpace
*as
, hwaddr addr
, uint64_t val
,
3256 MemTxAttrs attrs
, MemTxResult
*result
)
3260 r
= address_space_rw(as
, addr
, attrs
, (void *) &val
, 8, 1);
3266 void address_space_stq_le(AddressSpace
*as
, hwaddr addr
, uint64_t val
,
3267 MemTxAttrs attrs
, MemTxResult
*result
)
3270 val
= cpu_to_le64(val
);
3271 r
= address_space_rw(as
, addr
, attrs
, (void *) &val
, 8, 1);
3276 void address_space_stq_be(AddressSpace
*as
, hwaddr addr
, uint64_t val
,
3277 MemTxAttrs attrs
, MemTxResult
*result
)
3280 val
= cpu_to_be64(val
);
3281 r
= address_space_rw(as
, addr
, attrs
, (void *) &val
, 8, 1);
3287 void stq_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3289 address_space_stq(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3292 void stq_le_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3294 address_space_stq_le(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3297 void stq_be_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3299 address_space_stq_be(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3302 /* virtual memory access for debug (includes writing to ROM) */
3303 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
3304 uint8_t *buf
, int len
, int is_write
)
3311 page
= addr
& TARGET_PAGE_MASK
;
3312 phys_addr
= cpu_get_phys_page_debug(cpu
, page
);
3313 /* if no physical page mapped, return an error */
3314 if (phys_addr
== -1)
3316 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3319 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
3321 cpu_physical_memory_write_rom(cpu
->as
, phys_addr
, buf
, l
);
3323 address_space_rw(cpu
->as
, phys_addr
, MEMTXATTRS_UNSPECIFIED
,
3335 * A helper function for the _utterly broken_ virtio device model to find out if
3336 * it's running on a big endian machine. Don't do this at home kids!
3338 bool target_words_bigendian(void);
3339 bool target_words_bigendian(void)
3341 #if defined(TARGET_WORDS_BIGENDIAN)
3348 #ifndef CONFIG_USER_ONLY
3349 bool cpu_physical_memory_is_io(hwaddr phys_addr
)
3356 mr
= address_space_translate(&address_space_memory
,
3357 phys_addr
, &phys_addr
, &l
, false);
3359 res
= !(memory_region_is_ram(mr
) || memory_region_is_romd(mr
));
3364 void qemu_ram_foreach_block(RAMBlockIterFunc func
, void *opaque
)
3369 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
3370 func(block
->host
, block
->offset
, block
->used_length
, opaque
);