4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
21 #include <sys/types.h>
25 #include "qemu-common.h"
29 #if !defined(CONFIG_USER_ONLY)
30 #include "hw/boards.h"
33 #include "qemu/osdep.h"
34 #include "sysemu/kvm.h"
35 #include "sysemu/sysemu.h"
36 #include "hw/xen/xen.h"
37 #include "qemu/timer.h"
38 #include "qemu/config-file.h"
39 #include "qemu/error-report.h"
40 #include "exec/memory.h"
41 #include "sysemu/dma.h"
42 #include "exec/address-spaces.h"
43 #if defined(CONFIG_USER_ONLY)
45 #else /* !CONFIG_USER_ONLY */
46 #include "sysemu/xen-mapcache.h"
49 #include "exec/cpu-all.h"
50 #include "qemu/rcu_queue.h"
51 #include "exec/cputlb.h"
52 #include "translate-all.h"
54 #include "exec/memory-internal.h"
55 #include "exec/ram_addr.h"
57 #include "qemu/range.h"
59 //#define DEBUG_SUBPAGE
61 #if !defined(CONFIG_USER_ONLY)
62 static bool in_migration
;
64 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
65 * are protected by the ramlist lock.
67 RAMList ram_list
= { .blocks
= QLIST_HEAD_INITIALIZER(ram_list
.blocks
) };
69 static MemoryRegion
*system_memory
;
70 static MemoryRegion
*system_io
;
72 AddressSpace address_space_io
;
73 AddressSpace address_space_memory
;
75 MemoryRegion io_mem_rom
, io_mem_notdirty
;
76 static MemoryRegion io_mem_unassigned
;
78 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
79 #define RAM_PREALLOC (1 << 0)
81 /* RAM is mmap-ed with MAP_SHARED */
82 #define RAM_SHARED (1 << 1)
84 /* Only a portion of RAM (used_length) is actually used, and migrated.
85 * This used_length size can change across reboots.
87 #define RAM_RESIZEABLE (1 << 2)
91 struct CPUTailQ cpus
= QTAILQ_HEAD_INITIALIZER(cpus
);
92 /* current CPU in the current thread. It is only valid inside
94 DEFINE_TLS(CPUState
*, current_cpu
);
95 /* 0 = Do not count executed instructions.
96 1 = Precise instruction counting.
97 2 = Adaptive rate instruction counting. */
100 #if !defined(CONFIG_USER_ONLY)
102 typedef struct PhysPageEntry PhysPageEntry
;
104 struct PhysPageEntry
{
105 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
107 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
111 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
113 /* Size of the L2 (and L3, etc) page tables. */
114 #define ADDR_SPACE_BITS 64
117 #define P_L2_SIZE (1 << P_L2_BITS)
119 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
121 typedef PhysPageEntry Node
[P_L2_SIZE
];
123 typedef struct PhysPageMap
{
126 unsigned sections_nb
;
127 unsigned sections_nb_alloc
;
129 unsigned nodes_nb_alloc
;
131 MemoryRegionSection
*sections
;
134 struct AddressSpaceDispatch
{
137 /* This is a multi-level map on the physical address space.
138 * The bottom level has pointers to MemoryRegionSections.
140 PhysPageEntry phys_map
;
145 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
146 typedef struct subpage_t
{
150 uint16_t sub_section
[TARGET_PAGE_SIZE
];
153 #define PHYS_SECTION_UNASSIGNED 0
154 #define PHYS_SECTION_NOTDIRTY 1
155 #define PHYS_SECTION_ROM 2
156 #define PHYS_SECTION_WATCH 3
158 static void io_mem_init(void);
159 static void memory_map_init(void);
160 static void tcg_commit(MemoryListener
*listener
);
162 static MemoryRegion io_mem_watch
;
165 #if !defined(CONFIG_USER_ONLY)
167 static void phys_map_node_reserve(PhysPageMap
*map
, unsigned nodes
)
169 if (map
->nodes_nb
+ nodes
> map
->nodes_nb_alloc
) {
170 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
* 2, 16);
171 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
, map
->nodes_nb
+ nodes
);
172 map
->nodes
= g_renew(Node
, map
->nodes
, map
->nodes_nb_alloc
);
176 static uint32_t phys_map_node_alloc(PhysPageMap
*map
)
181 ret
= map
->nodes_nb
++;
182 assert(ret
!= PHYS_MAP_NODE_NIL
);
183 assert(ret
!= map
->nodes_nb_alloc
);
184 for (i
= 0; i
< P_L2_SIZE
; ++i
) {
185 map
->nodes
[ret
][i
].skip
= 1;
186 map
->nodes
[ret
][i
].ptr
= PHYS_MAP_NODE_NIL
;
191 static void phys_page_set_level(PhysPageMap
*map
, PhysPageEntry
*lp
,
192 hwaddr
*index
, hwaddr
*nb
, uint16_t leaf
,
197 hwaddr step
= (hwaddr
)1 << (level
* P_L2_BITS
);
199 if (lp
->skip
&& lp
->ptr
== PHYS_MAP_NODE_NIL
) {
200 lp
->ptr
= phys_map_node_alloc(map
);
201 p
= map
->nodes
[lp
->ptr
];
203 for (i
= 0; i
< P_L2_SIZE
; i
++) {
205 p
[i
].ptr
= PHYS_SECTION_UNASSIGNED
;
209 p
= map
->nodes
[lp
->ptr
];
211 lp
= &p
[(*index
>> (level
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
213 while (*nb
&& lp
< &p
[P_L2_SIZE
]) {
214 if ((*index
& (step
- 1)) == 0 && *nb
>= step
) {
220 phys_page_set_level(map
, lp
, index
, nb
, leaf
, level
- 1);
226 static void phys_page_set(AddressSpaceDispatch
*d
,
227 hwaddr index
, hwaddr nb
,
230 /* Wildly overreserve - it doesn't matter much. */
231 phys_map_node_reserve(&d
->map
, 3 * P_L2_LEVELS
);
233 phys_page_set_level(&d
->map
, &d
->phys_map
, &index
, &nb
, leaf
, P_L2_LEVELS
- 1);
236 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
237 * and update our entry so we can skip it and go directly to the destination.
239 static void phys_page_compact(PhysPageEntry
*lp
, Node
*nodes
, unsigned long *compacted
)
241 unsigned valid_ptr
= P_L2_SIZE
;
246 if (lp
->ptr
== PHYS_MAP_NODE_NIL
) {
251 for (i
= 0; i
< P_L2_SIZE
; i
++) {
252 if (p
[i
].ptr
== PHYS_MAP_NODE_NIL
) {
259 phys_page_compact(&p
[i
], nodes
, compacted
);
263 /* We can only compress if there's only one child. */
268 assert(valid_ptr
< P_L2_SIZE
);
270 /* Don't compress if it won't fit in the # of bits we have. */
271 if (lp
->skip
+ p
[valid_ptr
].skip
>= (1 << 3)) {
275 lp
->ptr
= p
[valid_ptr
].ptr
;
276 if (!p
[valid_ptr
].skip
) {
277 /* If our only child is a leaf, make this a leaf. */
278 /* By design, we should have made this node a leaf to begin with so we
279 * should never reach here.
280 * But since it's so simple to handle this, let's do it just in case we
285 lp
->skip
+= p
[valid_ptr
].skip
;
289 static void phys_page_compact_all(AddressSpaceDispatch
*d
, int nodes_nb
)
291 DECLARE_BITMAP(compacted
, nodes_nb
);
293 if (d
->phys_map
.skip
) {
294 phys_page_compact(&d
->phys_map
, d
->map
.nodes
, compacted
);
298 static MemoryRegionSection
*phys_page_find(PhysPageEntry lp
, hwaddr addr
,
299 Node
*nodes
, MemoryRegionSection
*sections
)
302 hwaddr index
= addr
>> TARGET_PAGE_BITS
;
305 for (i
= P_L2_LEVELS
; lp
.skip
&& (i
-= lp
.skip
) >= 0;) {
306 if (lp
.ptr
== PHYS_MAP_NODE_NIL
) {
307 return §ions
[PHYS_SECTION_UNASSIGNED
];
310 lp
= p
[(index
>> (i
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
313 if (sections
[lp
.ptr
].size
.hi
||
314 range_covers_byte(sections
[lp
.ptr
].offset_within_address_space
,
315 sections
[lp
.ptr
].size
.lo
, addr
)) {
316 return §ions
[lp
.ptr
];
318 return §ions
[PHYS_SECTION_UNASSIGNED
];
322 bool memory_region_is_unassigned(MemoryRegion
*mr
)
324 return mr
!= &io_mem_rom
&& mr
!= &io_mem_notdirty
&& !mr
->rom_device
325 && mr
!= &io_mem_watch
;
328 /* Called from RCU critical section */
329 static MemoryRegionSection
*address_space_lookup_region(AddressSpaceDispatch
*d
,
331 bool resolve_subpage
)
333 MemoryRegionSection
*section
;
336 section
= phys_page_find(d
->phys_map
, addr
, d
->map
.nodes
, d
->map
.sections
);
337 if (resolve_subpage
&& section
->mr
->subpage
) {
338 subpage
= container_of(section
->mr
, subpage_t
, iomem
);
339 section
= &d
->map
.sections
[subpage
->sub_section
[SUBPAGE_IDX(addr
)]];
344 /* Called from RCU critical section */
345 static MemoryRegionSection
*
346 address_space_translate_internal(AddressSpaceDispatch
*d
, hwaddr addr
, hwaddr
*xlat
,
347 hwaddr
*plen
, bool resolve_subpage
)
349 MemoryRegionSection
*section
;
352 section
= address_space_lookup_region(d
, addr
, resolve_subpage
);
353 /* Compute offset within MemoryRegionSection */
354 addr
-= section
->offset_within_address_space
;
356 /* Compute offset within MemoryRegion */
357 *xlat
= addr
+ section
->offset_within_region
;
359 diff
= int128_sub(section
->mr
->size
, int128_make64(addr
));
360 *plen
= int128_get64(int128_min(diff
, int128_make64(*plen
)));
364 static inline bool memory_access_is_direct(MemoryRegion
*mr
, bool is_write
)
366 if (memory_region_is_ram(mr
)) {
367 return !(is_write
&& mr
->readonly
);
369 if (memory_region_is_romd(mr
)) {
376 MemoryRegion
*address_space_translate(AddressSpace
*as
, hwaddr addr
,
377 hwaddr
*xlat
, hwaddr
*plen
,
381 MemoryRegionSection
*section
;
387 AddressSpaceDispatch
*d
= atomic_rcu_read(&as
->dispatch
);
388 section
= address_space_translate_internal(d
, addr
, &addr
, plen
, true);
391 if (!mr
->iommu_ops
) {
395 iotlb
= mr
->iommu_ops
->translate(mr
, addr
, is_write
);
396 addr
= ((iotlb
.translated_addr
& ~iotlb
.addr_mask
)
397 | (addr
& iotlb
.addr_mask
));
398 len
= MIN(len
, (addr
| iotlb
.addr_mask
) - addr
+ 1);
399 if (!(iotlb
.perm
& (1 << is_write
))) {
400 mr
= &io_mem_unassigned
;
404 as
= iotlb
.target_as
;
407 if (xen_enabled() && memory_access_is_direct(mr
, is_write
)) {
408 hwaddr page
= ((addr
& TARGET_PAGE_MASK
) + TARGET_PAGE_SIZE
) - addr
;
409 len
= MIN(page
, len
);
418 /* Called from RCU critical section */
419 MemoryRegionSection
*
420 address_space_translate_for_iotlb(CPUState
*cpu
, hwaddr addr
,
421 hwaddr
*xlat
, hwaddr
*plen
)
423 MemoryRegionSection
*section
;
424 section
= address_space_translate_internal(cpu
->memory_dispatch
,
425 addr
, xlat
, plen
, false);
427 assert(!section
->mr
->iommu_ops
);
432 void cpu_exec_init_all(void)
434 #if !defined(CONFIG_USER_ONLY)
435 qemu_mutex_init(&ram_list
.mutex
);
441 #if !defined(CONFIG_USER_ONLY)
443 static int cpu_common_post_load(void *opaque
, int version_id
)
445 CPUState
*cpu
= opaque
;
447 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
448 version_id is increased. */
449 cpu
->interrupt_request
&= ~0x01;
455 static int cpu_common_pre_load(void *opaque
)
457 CPUState
*cpu
= opaque
;
459 cpu
->exception_index
= -1;
464 static bool cpu_common_exception_index_needed(void *opaque
)
466 CPUState
*cpu
= opaque
;
468 return tcg_enabled() && cpu
->exception_index
!= -1;
471 static const VMStateDescription vmstate_cpu_common_exception_index
= {
472 .name
= "cpu_common/exception_index",
474 .minimum_version_id
= 1,
475 .fields
= (VMStateField
[]) {
476 VMSTATE_INT32(exception_index
, CPUState
),
477 VMSTATE_END_OF_LIST()
481 const VMStateDescription vmstate_cpu_common
= {
482 .name
= "cpu_common",
484 .minimum_version_id
= 1,
485 .pre_load
= cpu_common_pre_load
,
486 .post_load
= cpu_common_post_load
,
487 .fields
= (VMStateField
[]) {
488 VMSTATE_UINT32(halted
, CPUState
),
489 VMSTATE_UINT32(interrupt_request
, CPUState
),
490 VMSTATE_END_OF_LIST()
492 .subsections
= (VMStateSubsection
[]) {
494 .vmsd
= &vmstate_cpu_common_exception_index
,
495 .needed
= cpu_common_exception_index_needed
,
504 CPUState
*qemu_get_cpu(int index
)
509 if (cpu
->cpu_index
== index
) {
517 #if !defined(CONFIG_USER_ONLY)
518 void tcg_cpu_address_space_init(CPUState
*cpu
, AddressSpace
*as
)
520 /* We only support one address space per cpu at the moment. */
521 assert(cpu
->as
== as
);
523 if (cpu
->tcg_as_listener
) {
524 memory_listener_unregister(cpu
->tcg_as_listener
);
526 cpu
->tcg_as_listener
= g_new0(MemoryListener
, 1);
528 cpu
->tcg_as_listener
->commit
= tcg_commit
;
529 memory_listener_register(cpu
->tcg_as_listener
, as
);
533 void cpu_exec_init(CPUArchState
*env
)
535 CPUState
*cpu
= ENV_GET_CPU(env
);
536 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
540 #if defined(CONFIG_USER_ONLY)
544 CPU_FOREACH(some_cpu
) {
547 cpu
->cpu_index
= cpu_index
;
549 QTAILQ_INIT(&cpu
->breakpoints
);
550 QTAILQ_INIT(&cpu
->watchpoints
);
551 #ifndef CONFIG_USER_ONLY
552 cpu
->as
= &address_space_memory
;
553 cpu
->thread_id
= qemu_get_thread_id();
555 QTAILQ_INSERT_TAIL(&cpus
, cpu
, node
);
556 #if defined(CONFIG_USER_ONLY)
559 if (qdev_get_vmsd(DEVICE(cpu
)) == NULL
) {
560 vmstate_register(NULL
, cpu_index
, &vmstate_cpu_common
, cpu
);
562 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
563 register_savevm(NULL
, "cpu", cpu_index
, CPU_SAVE_VERSION
,
564 cpu_save
, cpu_load
, env
);
565 assert(cc
->vmsd
== NULL
);
566 assert(qdev_get_vmsd(DEVICE(cpu
)) == NULL
);
568 if (cc
->vmsd
!= NULL
) {
569 vmstate_register(NULL
, cpu_index
, cc
->vmsd
, cpu
);
573 #if defined(CONFIG_USER_ONLY)
574 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
576 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
579 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
581 hwaddr phys
= cpu_get_phys_page_debug(cpu
, pc
);
583 tb_invalidate_phys_addr(cpu
->as
,
584 phys
| (pc
& ~TARGET_PAGE_MASK
));
589 #if defined(CONFIG_USER_ONLY)
590 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
595 int cpu_watchpoint_remove(CPUState
*cpu
, vaddr addr
, vaddr len
,
601 void cpu_watchpoint_remove_by_ref(CPUState
*cpu
, CPUWatchpoint
*watchpoint
)
605 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
606 int flags
, CPUWatchpoint
**watchpoint
)
611 /* Add a watchpoint. */
612 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
613 int flags
, CPUWatchpoint
**watchpoint
)
617 /* forbid ranges which are empty or run off the end of the address space */
618 if (len
== 0 || (addr
+ len
- 1) < addr
) {
619 error_report("tried to set invalid watchpoint at %"
620 VADDR_PRIx
", len=%" VADDR_PRIu
, addr
, len
);
623 wp
= g_malloc(sizeof(*wp
));
629 /* keep all GDB-injected watchpoints in front */
630 if (flags
& BP_GDB
) {
631 QTAILQ_INSERT_HEAD(&cpu
->watchpoints
, wp
, entry
);
633 QTAILQ_INSERT_TAIL(&cpu
->watchpoints
, wp
, entry
);
636 tlb_flush_page(cpu
, addr
);
643 /* Remove a specific watchpoint. */
644 int cpu_watchpoint_remove(CPUState
*cpu
, vaddr addr
, vaddr len
,
649 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
650 if (addr
== wp
->vaddr
&& len
== wp
->len
651 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
652 cpu_watchpoint_remove_by_ref(cpu
, wp
);
659 /* Remove a specific watchpoint by reference. */
660 void cpu_watchpoint_remove_by_ref(CPUState
*cpu
, CPUWatchpoint
*watchpoint
)
662 QTAILQ_REMOVE(&cpu
->watchpoints
, watchpoint
, entry
);
664 tlb_flush_page(cpu
, watchpoint
->vaddr
);
669 /* Remove all matching watchpoints. */
670 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
672 CPUWatchpoint
*wp
, *next
;
674 QTAILQ_FOREACH_SAFE(wp
, &cpu
->watchpoints
, entry
, next
) {
675 if (wp
->flags
& mask
) {
676 cpu_watchpoint_remove_by_ref(cpu
, wp
);
681 /* Return true if this watchpoint address matches the specified
682 * access (ie the address range covered by the watchpoint overlaps
683 * partially or completely with the address range covered by the
686 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint
*wp
,
690 /* We know the lengths are non-zero, but a little caution is
691 * required to avoid errors in the case where the range ends
692 * exactly at the top of the address space and so addr + len
693 * wraps round to zero.
695 vaddr wpend
= wp
->vaddr
+ wp
->len
- 1;
696 vaddr addrend
= addr
+ len
- 1;
698 return !(addr
> wpend
|| wp
->vaddr
> addrend
);
703 /* Add a breakpoint. */
704 int cpu_breakpoint_insert(CPUState
*cpu
, vaddr pc
, int flags
,
705 CPUBreakpoint
**breakpoint
)
709 bp
= g_malloc(sizeof(*bp
));
714 /* keep all GDB-injected breakpoints in front */
715 if (flags
& BP_GDB
) {
716 QTAILQ_INSERT_HEAD(&cpu
->breakpoints
, bp
, entry
);
718 QTAILQ_INSERT_TAIL(&cpu
->breakpoints
, bp
, entry
);
721 breakpoint_invalidate(cpu
, pc
);
729 /* Remove a specific breakpoint. */
730 int cpu_breakpoint_remove(CPUState
*cpu
, vaddr pc
, int flags
)
734 QTAILQ_FOREACH(bp
, &cpu
->breakpoints
, entry
) {
735 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
736 cpu_breakpoint_remove_by_ref(cpu
, bp
);
743 /* Remove a specific breakpoint by reference. */
744 void cpu_breakpoint_remove_by_ref(CPUState
*cpu
, CPUBreakpoint
*breakpoint
)
746 QTAILQ_REMOVE(&cpu
->breakpoints
, breakpoint
, entry
);
748 breakpoint_invalidate(cpu
, breakpoint
->pc
);
753 /* Remove all matching breakpoints. */
754 void cpu_breakpoint_remove_all(CPUState
*cpu
, int mask
)
756 CPUBreakpoint
*bp
, *next
;
758 QTAILQ_FOREACH_SAFE(bp
, &cpu
->breakpoints
, entry
, next
) {
759 if (bp
->flags
& mask
) {
760 cpu_breakpoint_remove_by_ref(cpu
, bp
);
765 /* enable or disable single step mode. EXCP_DEBUG is returned by the
766 CPU loop after each instruction */
767 void cpu_single_step(CPUState
*cpu
, int enabled
)
769 if (cpu
->singlestep_enabled
!= enabled
) {
770 cpu
->singlestep_enabled
= enabled
;
772 kvm_update_guest_debug(cpu
, 0);
774 /* must flush all the translated code to avoid inconsistencies */
775 /* XXX: only flush what is necessary */
776 CPUArchState
*env
= cpu
->env_ptr
;
782 void cpu_abort(CPUState
*cpu
, const char *fmt
, ...)
789 fprintf(stderr
, "qemu: fatal: ");
790 vfprintf(stderr
, fmt
, ap
);
791 fprintf(stderr
, "\n");
792 cpu_dump_state(cpu
, stderr
, fprintf
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
793 if (qemu_log_enabled()) {
794 qemu_log("qemu: fatal: ");
795 qemu_log_vprintf(fmt
, ap2
);
797 log_cpu_state(cpu
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
803 #if defined(CONFIG_USER_ONLY)
805 struct sigaction act
;
806 sigfillset(&act
.sa_mask
);
807 act
.sa_handler
= SIG_DFL
;
808 sigaction(SIGABRT
, &act
, NULL
);
814 #if !defined(CONFIG_USER_ONLY)
815 /* Called from RCU critical section */
816 static RAMBlock
*qemu_get_ram_block(ram_addr_t addr
)
820 block
= atomic_rcu_read(&ram_list
.mru_block
);
821 if (block
&& addr
- block
->offset
< block
->max_length
) {
824 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
825 if (addr
- block
->offset
< block
->max_length
) {
830 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
834 /* It is safe to write mru_block outside the iothread lock. This
839 * xxx removed from list
843 * call_rcu(reclaim_ramblock, xxx);
846 * atomic_rcu_set is not needed here. The block was already published
847 * when it was placed into the list. Here we're just making an extra
848 * copy of the pointer.
850 ram_list
.mru_block
= block
;
854 static void tlb_reset_dirty_range_all(ram_addr_t start
, ram_addr_t length
)
860 end
= TARGET_PAGE_ALIGN(start
+ length
);
861 start
&= TARGET_PAGE_MASK
;
864 block
= qemu_get_ram_block(start
);
865 assert(block
== qemu_get_ram_block(end
- 1));
866 start1
= (uintptr_t)ramblock_ptr(block
, start
- block
->offset
);
867 cpu_tlb_reset_dirty_all(start1
, length
);
871 /* Note: start and end must be within the same ram block. */
872 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t length
,
877 cpu_physical_memory_clear_dirty_range_type(start
, length
, client
);
880 tlb_reset_dirty_range_all(start
, length
);
884 static void cpu_physical_memory_set_dirty_tracking(bool enable
)
886 in_migration
= enable
;
889 /* Called from RCU critical section */
890 hwaddr
memory_region_section_get_iotlb(CPUState
*cpu
,
891 MemoryRegionSection
*section
,
893 hwaddr paddr
, hwaddr xlat
,
895 target_ulong
*address
)
900 if (memory_region_is_ram(section
->mr
)) {
902 iotlb
= (memory_region_get_ram_addr(section
->mr
) & TARGET_PAGE_MASK
)
904 if (!section
->readonly
) {
905 iotlb
|= PHYS_SECTION_NOTDIRTY
;
907 iotlb
|= PHYS_SECTION_ROM
;
910 iotlb
= section
- section
->address_space
->dispatch
->map
.sections
;
914 /* Make accesses to pages with watchpoints go via the
915 watchpoint trap routines. */
916 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
917 if (cpu_watchpoint_address_matches(wp
, vaddr
, TARGET_PAGE_SIZE
)) {
918 /* Avoid trapping reads of pages with a write breakpoint. */
919 if ((prot
& PAGE_WRITE
) || (wp
->flags
& BP_MEM_READ
)) {
920 iotlb
= PHYS_SECTION_WATCH
+ paddr
;
921 *address
|= TLB_MMIO
;
929 #endif /* defined(CONFIG_USER_ONLY) */
931 #if !defined(CONFIG_USER_ONLY)
933 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
935 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
);
937 static void *(*phys_mem_alloc
)(size_t size
, uint64_t *align
) =
941 * Set a custom physical guest memory alloator.
942 * Accelerators with unusual needs may need this. Hopefully, we can
943 * get rid of it eventually.
945 void phys_mem_set_alloc(void *(*alloc
)(size_t, uint64_t *align
))
947 phys_mem_alloc
= alloc
;
950 static uint16_t phys_section_add(PhysPageMap
*map
,
951 MemoryRegionSection
*section
)
953 /* The physical section number is ORed with a page-aligned
954 * pointer to produce the iotlb entries. Thus it should
955 * never overflow into the page-aligned value.
957 assert(map
->sections_nb
< TARGET_PAGE_SIZE
);
959 if (map
->sections_nb
== map
->sections_nb_alloc
) {
960 map
->sections_nb_alloc
= MAX(map
->sections_nb_alloc
* 2, 16);
961 map
->sections
= g_renew(MemoryRegionSection
, map
->sections
,
962 map
->sections_nb_alloc
);
964 map
->sections
[map
->sections_nb
] = *section
;
965 memory_region_ref(section
->mr
);
966 return map
->sections_nb
++;
969 static void phys_section_destroy(MemoryRegion
*mr
)
971 memory_region_unref(mr
);
974 subpage_t
*subpage
= container_of(mr
, subpage_t
, iomem
);
975 object_unref(OBJECT(&subpage
->iomem
));
980 static void phys_sections_free(PhysPageMap
*map
)
982 while (map
->sections_nb
> 0) {
983 MemoryRegionSection
*section
= &map
->sections
[--map
->sections_nb
];
984 phys_section_destroy(section
->mr
);
986 g_free(map
->sections
);
990 static void register_subpage(AddressSpaceDispatch
*d
, MemoryRegionSection
*section
)
993 hwaddr base
= section
->offset_within_address_space
995 MemoryRegionSection
*existing
= phys_page_find(d
->phys_map
, base
,
996 d
->map
.nodes
, d
->map
.sections
);
997 MemoryRegionSection subsection
= {
998 .offset_within_address_space
= base
,
999 .size
= int128_make64(TARGET_PAGE_SIZE
),
1003 assert(existing
->mr
->subpage
|| existing
->mr
== &io_mem_unassigned
);
1005 if (!(existing
->mr
->subpage
)) {
1006 subpage
= subpage_init(d
->as
, base
);
1007 subsection
.address_space
= d
->as
;
1008 subsection
.mr
= &subpage
->iomem
;
1009 phys_page_set(d
, base
>> TARGET_PAGE_BITS
, 1,
1010 phys_section_add(&d
->map
, &subsection
));
1012 subpage
= container_of(existing
->mr
, subpage_t
, iomem
);
1014 start
= section
->offset_within_address_space
& ~TARGET_PAGE_MASK
;
1015 end
= start
+ int128_get64(section
->size
) - 1;
1016 subpage_register(subpage
, start
, end
,
1017 phys_section_add(&d
->map
, section
));
1021 static void register_multipage(AddressSpaceDispatch
*d
,
1022 MemoryRegionSection
*section
)
1024 hwaddr start_addr
= section
->offset_within_address_space
;
1025 uint16_t section_index
= phys_section_add(&d
->map
, section
);
1026 uint64_t num_pages
= int128_get64(int128_rshift(section
->size
,
1030 phys_page_set(d
, start_addr
>> TARGET_PAGE_BITS
, num_pages
, section_index
);
1033 static void mem_add(MemoryListener
*listener
, MemoryRegionSection
*section
)
1035 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
1036 AddressSpaceDispatch
*d
= as
->next_dispatch
;
1037 MemoryRegionSection now
= *section
, remain
= *section
;
1038 Int128 page_size
= int128_make64(TARGET_PAGE_SIZE
);
1040 if (now
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
1041 uint64_t left
= TARGET_PAGE_ALIGN(now
.offset_within_address_space
)
1042 - now
.offset_within_address_space
;
1044 now
.size
= int128_min(int128_make64(left
), now
.size
);
1045 register_subpage(d
, &now
);
1047 now
.size
= int128_zero();
1049 while (int128_ne(remain
.size
, now
.size
)) {
1050 remain
.size
= int128_sub(remain
.size
, now
.size
);
1051 remain
.offset_within_address_space
+= int128_get64(now
.size
);
1052 remain
.offset_within_region
+= int128_get64(now
.size
);
1054 if (int128_lt(remain
.size
, page_size
)) {
1055 register_subpage(d
, &now
);
1056 } else if (remain
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
1057 now
.size
= page_size
;
1058 register_subpage(d
, &now
);
1060 now
.size
= int128_and(now
.size
, int128_neg(page_size
));
1061 register_multipage(d
, &now
);
1066 void qemu_flush_coalesced_mmio_buffer(void)
1069 kvm_flush_coalesced_mmio_buffer();
1072 void qemu_mutex_lock_ramlist(void)
1074 qemu_mutex_lock(&ram_list
.mutex
);
1077 void qemu_mutex_unlock_ramlist(void)
1079 qemu_mutex_unlock(&ram_list
.mutex
);
1084 #include <sys/vfs.h>
1086 #define HUGETLBFS_MAGIC 0x958458f6
1088 static long gethugepagesize(const char *path
, Error
**errp
)
1094 ret
= statfs(path
, &fs
);
1095 } while (ret
!= 0 && errno
== EINTR
);
1098 error_setg_errno(errp
, errno
, "failed to get page size of file %s",
1103 if (fs
.f_type
!= HUGETLBFS_MAGIC
)
1104 fprintf(stderr
, "Warning: path not on HugeTLBFS: %s\n", path
);
1109 static void *file_ram_alloc(RAMBlock
*block
,
1115 char *sanitized_name
;
1120 Error
*local_err
= NULL
;
1122 hpagesize
= gethugepagesize(path
, &local_err
);
1124 error_propagate(errp
, local_err
);
1127 block
->mr
->align
= hpagesize
;
1129 if (memory
< hpagesize
) {
1130 error_setg(errp
, "memory size 0x" RAM_ADDR_FMT
" must be equal to "
1131 "or larger than huge page size 0x%" PRIx64
,
1136 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1138 "host lacks kvm mmu notifiers, -mem-path unsupported");
1142 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1143 sanitized_name
= g_strdup(memory_region_name(block
->mr
));
1144 for (c
= sanitized_name
; *c
!= '\0'; c
++) {
1149 filename
= g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path
,
1151 g_free(sanitized_name
);
1153 fd
= mkstemp(filename
);
1155 error_setg_errno(errp
, errno
,
1156 "unable to create backing store for hugepages");
1163 memory
= (memory
+hpagesize
-1) & ~(hpagesize
-1);
1166 * ftruncate is not supported by hugetlbfs in older
1167 * hosts, so don't bother bailing out on errors.
1168 * If anything goes wrong with it under other filesystems,
1171 if (ftruncate(fd
, memory
)) {
1172 perror("ftruncate");
1175 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
,
1176 (block
->flags
& RAM_SHARED
? MAP_SHARED
: MAP_PRIVATE
),
1178 if (area
== MAP_FAILED
) {
1179 error_setg_errno(errp
, errno
,
1180 "unable to map backing store for hugepages");
1186 os_mem_prealloc(fd
, area
, memory
);
1194 error_report("%s", error_get_pretty(*errp
));
1201 /* Called with the ramlist lock held. */
1202 static ram_addr_t
find_ram_offset(ram_addr_t size
)
1204 RAMBlock
*block
, *next_block
;
1205 ram_addr_t offset
= RAM_ADDR_MAX
, mingap
= RAM_ADDR_MAX
;
1207 assert(size
!= 0); /* it would hand out same offset multiple times */
1209 if (QLIST_EMPTY_RCU(&ram_list
.blocks
)) {
1213 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1214 ram_addr_t end
, next
= RAM_ADDR_MAX
;
1216 end
= block
->offset
+ block
->max_length
;
1218 QLIST_FOREACH_RCU(next_block
, &ram_list
.blocks
, next
) {
1219 if (next_block
->offset
>= end
) {
1220 next
= MIN(next
, next_block
->offset
);
1223 if (next
- end
>= size
&& next
- end
< mingap
) {
1225 mingap
= next
- end
;
1229 if (offset
== RAM_ADDR_MAX
) {
1230 fprintf(stderr
, "Failed to find gap of requested size: %" PRIu64
"\n",
1238 ram_addr_t
last_ram_offset(void)
1241 ram_addr_t last
= 0;
1244 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1245 last
= MAX(last
, block
->offset
+ block
->max_length
);
1251 static void qemu_ram_setup_dump(void *addr
, ram_addr_t size
)
1255 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1256 if (!machine_dump_guest_core(current_machine
)) {
1257 ret
= qemu_madvise(addr
, size
, QEMU_MADV_DONTDUMP
);
1259 perror("qemu_madvise");
1260 fprintf(stderr
, "madvise doesn't support MADV_DONTDUMP, "
1261 "but dump_guest_core=off specified\n");
1266 /* Called within an RCU critical section, or while the ramlist lock
1269 static RAMBlock
*find_ram_block(ram_addr_t addr
)
1273 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1274 if (block
->offset
== addr
) {
1282 /* Called with iothread lock held. */
1283 void qemu_ram_set_idstr(ram_addr_t addr
, const char *name
, DeviceState
*dev
)
1285 RAMBlock
*new_block
, *block
;
1288 new_block
= find_ram_block(addr
);
1290 assert(!new_block
->idstr
[0]);
1293 char *id
= qdev_get_dev_path(dev
);
1295 snprintf(new_block
->idstr
, sizeof(new_block
->idstr
), "%s/", id
);
1299 pstrcat(new_block
->idstr
, sizeof(new_block
->idstr
), name
);
1301 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1302 if (block
!= new_block
&& !strcmp(block
->idstr
, new_block
->idstr
)) {
1303 fprintf(stderr
, "RAMBlock \"%s\" already registered, abort!\n",
1311 /* Called with iothread lock held. */
1312 void qemu_ram_unset_idstr(ram_addr_t addr
)
1316 /* FIXME: arch_init.c assumes that this is not called throughout
1317 * migration. Ignore the problem since hot-unplug during migration
1318 * does not work anyway.
1322 block
= find_ram_block(addr
);
1324 memset(block
->idstr
, 0, sizeof(block
->idstr
));
1329 static int memory_try_enable_merging(void *addr
, size_t len
)
1331 if (!machine_mem_merge(current_machine
)) {
1332 /* disabled by the user */
1336 return qemu_madvise(addr
, len
, QEMU_MADV_MERGEABLE
);
1339 /* Only legal before guest might have detected the memory size: e.g. on
1340 * incoming migration, or right after reset.
1342 * As memory core doesn't know how is memory accessed, it is up to
1343 * resize callback to update device state and/or add assertions to detect
1344 * misuse, if necessary.
1346 int qemu_ram_resize(ram_addr_t base
, ram_addr_t newsize
, Error
**errp
)
1348 RAMBlock
*block
= find_ram_block(base
);
1352 newsize
= TARGET_PAGE_ALIGN(newsize
);
1354 if (block
->used_length
== newsize
) {
1358 if (!(block
->flags
& RAM_RESIZEABLE
)) {
1359 error_setg_errno(errp
, EINVAL
,
1360 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1361 " in != 0x" RAM_ADDR_FMT
, block
->idstr
,
1362 newsize
, block
->used_length
);
1366 if (block
->max_length
< newsize
) {
1367 error_setg_errno(errp
, EINVAL
,
1368 "Length too large: %s: 0x" RAM_ADDR_FMT
1369 " > 0x" RAM_ADDR_FMT
, block
->idstr
,
1370 newsize
, block
->max_length
);
1374 cpu_physical_memory_clear_dirty_range(block
->offset
, block
->used_length
);
1375 block
->used_length
= newsize
;
1376 cpu_physical_memory_set_dirty_range(block
->offset
, block
->used_length
);
1377 memory_region_set_size(block
->mr
, newsize
);
1378 if (block
->resized
) {
1379 block
->resized(block
->idstr
, newsize
, block
->host
);
1384 static ram_addr_t
ram_block_add(RAMBlock
*new_block
, Error
**errp
)
1387 RAMBlock
*last_block
= NULL
;
1388 ram_addr_t old_ram_size
, new_ram_size
;
1390 old_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1392 qemu_mutex_lock_ramlist();
1393 new_block
->offset
= find_ram_offset(new_block
->max_length
);
1395 if (!new_block
->host
) {
1396 if (xen_enabled()) {
1397 xen_ram_alloc(new_block
->offset
, new_block
->max_length
,
1400 new_block
->host
= phys_mem_alloc(new_block
->max_length
,
1401 &new_block
->mr
->align
);
1402 if (!new_block
->host
) {
1403 error_setg_errno(errp
, errno
,
1404 "cannot set up guest memory '%s'",
1405 memory_region_name(new_block
->mr
));
1406 qemu_mutex_unlock_ramlist();
1409 memory_try_enable_merging(new_block
->host
, new_block
->max_length
);
1413 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1414 * QLIST (which has an RCU-friendly variant) does not have insertion at
1415 * tail, so save the last element in last_block.
1417 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1419 if (block
->max_length
< new_block
->max_length
) {
1424 QLIST_INSERT_BEFORE_RCU(block
, new_block
, next
);
1425 } else if (last_block
) {
1426 QLIST_INSERT_AFTER_RCU(last_block
, new_block
, next
);
1427 } else { /* list is empty */
1428 QLIST_INSERT_HEAD_RCU(&ram_list
.blocks
, new_block
, next
);
1430 ram_list
.mru_block
= NULL
;
1432 /* Write list before version */
1435 qemu_mutex_unlock_ramlist();
1437 new_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1439 if (new_ram_size
> old_ram_size
) {
1442 /* ram_list.dirty_memory[] is protected by the iothread lock. */
1443 for (i
= 0; i
< DIRTY_MEMORY_NUM
; i
++) {
1444 ram_list
.dirty_memory
[i
] =
1445 bitmap_zero_extend(ram_list
.dirty_memory
[i
],
1446 old_ram_size
, new_ram_size
);
1449 cpu_physical_memory_set_dirty_range(new_block
->offset
,
1450 new_block
->used_length
);
1452 if (new_block
->host
) {
1453 qemu_ram_setup_dump(new_block
->host
, new_block
->max_length
);
1454 qemu_madvise(new_block
->host
, new_block
->max_length
, QEMU_MADV_HUGEPAGE
);
1455 qemu_madvise(new_block
->host
, new_block
->max_length
, QEMU_MADV_DONTFORK
);
1456 if (kvm_enabled()) {
1457 kvm_setup_guest_memory(new_block
->host
, new_block
->max_length
);
1461 return new_block
->offset
;
1465 ram_addr_t
qemu_ram_alloc_from_file(ram_addr_t size
, MemoryRegion
*mr
,
1466 bool share
, const char *mem_path
,
1469 RAMBlock
*new_block
;
1471 Error
*local_err
= NULL
;
1473 if (xen_enabled()) {
1474 error_setg(errp
, "-mem-path not supported with Xen");
1478 if (phys_mem_alloc
!= qemu_anon_ram_alloc
) {
1480 * file_ram_alloc() needs to allocate just like
1481 * phys_mem_alloc, but we haven't bothered to provide
1485 "-mem-path not supported with this accelerator");
1489 size
= TARGET_PAGE_ALIGN(size
);
1490 new_block
= g_malloc0(sizeof(*new_block
));
1492 new_block
->used_length
= size
;
1493 new_block
->max_length
= size
;
1494 new_block
->flags
= share
? RAM_SHARED
: 0;
1495 new_block
->host
= file_ram_alloc(new_block
, size
,
1497 if (!new_block
->host
) {
1502 addr
= ram_block_add(new_block
, &local_err
);
1505 error_propagate(errp
, local_err
);
1513 ram_addr_t
qemu_ram_alloc_internal(ram_addr_t size
, ram_addr_t max_size
,
1514 void (*resized
)(const char*,
1517 void *host
, bool resizeable
,
1518 MemoryRegion
*mr
, Error
**errp
)
1520 RAMBlock
*new_block
;
1522 Error
*local_err
= NULL
;
1524 size
= TARGET_PAGE_ALIGN(size
);
1525 max_size
= TARGET_PAGE_ALIGN(max_size
);
1526 new_block
= g_malloc0(sizeof(*new_block
));
1528 new_block
->resized
= resized
;
1529 new_block
->used_length
= size
;
1530 new_block
->max_length
= max_size
;
1531 assert(max_size
>= size
);
1533 new_block
->host
= host
;
1535 new_block
->flags
|= RAM_PREALLOC
;
1538 new_block
->flags
|= RAM_RESIZEABLE
;
1540 addr
= ram_block_add(new_block
, &local_err
);
1543 error_propagate(errp
, local_err
);
1549 ram_addr_t
qemu_ram_alloc_from_ptr(ram_addr_t size
, void *host
,
1550 MemoryRegion
*mr
, Error
**errp
)
1552 return qemu_ram_alloc_internal(size
, size
, NULL
, host
, false, mr
, errp
);
1555 ram_addr_t
qemu_ram_alloc(ram_addr_t size
, MemoryRegion
*mr
, Error
**errp
)
1557 return qemu_ram_alloc_internal(size
, size
, NULL
, NULL
, false, mr
, errp
);
1560 ram_addr_t
qemu_ram_alloc_resizeable(ram_addr_t size
, ram_addr_t maxsz
,
1561 void (*resized
)(const char*,
1564 MemoryRegion
*mr
, Error
**errp
)
1566 return qemu_ram_alloc_internal(size
, maxsz
, resized
, NULL
, true, mr
, errp
);
1569 void qemu_ram_free_from_ptr(ram_addr_t addr
)
1573 qemu_mutex_lock_ramlist();
1574 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1575 if (addr
== block
->offset
) {
1576 QLIST_REMOVE_RCU(block
, next
);
1577 ram_list
.mru_block
= NULL
;
1578 /* Write list before version */
1581 g_free_rcu(block
, rcu
);
1585 qemu_mutex_unlock_ramlist();
1588 static void reclaim_ramblock(RAMBlock
*block
)
1590 if (block
->flags
& RAM_PREALLOC
) {
1592 } else if (xen_enabled()) {
1593 xen_invalidate_map_cache_entry(block
->host
);
1595 } else if (block
->fd
>= 0) {
1596 munmap(block
->host
, block
->max_length
);
1600 qemu_anon_ram_free(block
->host
, block
->max_length
);
1605 void qemu_ram_free(ram_addr_t addr
)
1609 qemu_mutex_lock_ramlist();
1610 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1611 if (addr
== block
->offset
) {
1612 QLIST_REMOVE_RCU(block
, next
);
1613 ram_list
.mru_block
= NULL
;
1614 /* Write list before version */
1617 call_rcu(block
, reclaim_ramblock
, rcu
);
1621 qemu_mutex_unlock_ramlist();
1625 void qemu_ram_remap(ram_addr_t addr
, ram_addr_t length
)
1632 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1633 offset
= addr
- block
->offset
;
1634 if (offset
< block
->max_length
) {
1635 vaddr
= ramblock_ptr(block
, offset
);
1636 if (block
->flags
& RAM_PREALLOC
) {
1638 } else if (xen_enabled()) {
1642 munmap(vaddr
, length
);
1643 if (block
->fd
>= 0) {
1644 flags
|= (block
->flags
& RAM_SHARED
?
1645 MAP_SHARED
: MAP_PRIVATE
);
1646 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1647 flags
, block
->fd
, offset
);
1650 * Remap needs to match alloc. Accelerators that
1651 * set phys_mem_alloc never remap. If they did,
1652 * we'd need a remap hook here.
1654 assert(phys_mem_alloc
== qemu_anon_ram_alloc
);
1656 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
1657 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1660 if (area
!= vaddr
) {
1661 fprintf(stderr
, "Could not remap addr: "
1662 RAM_ADDR_FMT
"@" RAM_ADDR_FMT
"\n",
1666 memory_try_enable_merging(vaddr
, length
);
1667 qemu_ram_setup_dump(vaddr
, length
);
1672 #endif /* !_WIN32 */
1674 int qemu_get_ram_fd(ram_addr_t addr
)
1680 block
= qemu_get_ram_block(addr
);
1686 void *qemu_get_ram_block_host_ptr(ram_addr_t addr
)
1692 block
= qemu_get_ram_block(addr
);
1693 ptr
= ramblock_ptr(block
, 0);
1698 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1699 * This should not be used for general purpose DMA. Use address_space_map
1700 * or address_space_rw instead. For local memory (e.g. video ram) that the
1701 * device owns, use memory_region_get_ram_ptr.
1703 * By the time this function returns, the returned pointer is not protected
1704 * by RCU anymore. If the caller is not within an RCU critical section and
1705 * does not hold the iothread lock, it must have other means of protecting the
1706 * pointer, such as a reference to the region that includes the incoming
1709 void *qemu_get_ram_ptr(ram_addr_t addr
)
1715 block
= qemu_get_ram_block(addr
);
1717 if (xen_enabled() && block
->host
== NULL
) {
1718 /* We need to check if the requested address is in the RAM
1719 * because we don't want to map the entire memory in QEMU.
1720 * In that case just map until the end of the page.
1722 if (block
->offset
== 0) {
1723 ptr
= xen_map_cache(addr
, 0, 0);
1727 block
->host
= xen_map_cache(block
->offset
, block
->max_length
, 1);
1729 ptr
= ramblock_ptr(block
, addr
- block
->offset
);
1736 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1737 * but takes a size argument.
1739 * By the time this function returns, the returned pointer is not protected
1740 * by RCU anymore. If the caller is not within an RCU critical section and
1741 * does not hold the iothread lock, it must have other means of protecting the
1742 * pointer, such as a reference to the region that includes the incoming
1745 static void *qemu_ram_ptr_length(ram_addr_t addr
, hwaddr
*size
)
1751 if (xen_enabled()) {
1752 return xen_map_cache(addr
, *size
, 1);
1756 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1757 if (addr
- block
->offset
< block
->max_length
) {
1758 if (addr
- block
->offset
+ *size
> block
->max_length
)
1759 *size
= block
->max_length
- addr
+ block
->offset
;
1760 ptr
= ramblock_ptr(block
, addr
- block
->offset
);
1766 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
1771 /* Some of the softmmu routines need to translate from a host pointer
1772 * (typically a TLB entry) back to a ram offset.
1774 * By the time this function returns, the returned pointer is not protected
1775 * by RCU anymore. If the caller is not within an RCU critical section and
1776 * does not hold the iothread lock, it must have other means of protecting the
1777 * pointer, such as a reference to the region that includes the incoming
1780 MemoryRegion
*qemu_ram_addr_from_host(void *ptr
, ram_addr_t
*ram_addr
)
1783 uint8_t *host
= ptr
;
1786 if (xen_enabled()) {
1788 *ram_addr
= xen_ram_addr_from_mapcache(ptr
);
1789 mr
= qemu_get_ram_block(*ram_addr
)->mr
;
1795 block
= atomic_rcu_read(&ram_list
.mru_block
);
1796 if (block
&& block
->host
&& host
- block
->host
< block
->max_length
) {
1800 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1801 /* This case append when the block is not mapped. */
1802 if (block
->host
== NULL
) {
1805 if (host
- block
->host
< block
->max_length
) {
1814 *ram_addr
= block
->offset
+ (host
- block
->host
);
1820 static void notdirty_mem_write(void *opaque
, hwaddr ram_addr
,
1821 uint64_t val
, unsigned size
)
1823 if (!cpu_physical_memory_get_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
)) {
1824 tb_invalidate_phys_page_fast(ram_addr
, size
);
1828 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
1831 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
1834 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
1839 cpu_physical_memory_set_dirty_range_nocode(ram_addr
, size
);
1840 /* we remove the notdirty callback only if the code has been
1842 if (!cpu_physical_memory_is_clean(ram_addr
)) {
1843 CPUArchState
*env
= current_cpu
->env_ptr
;
1844 tlb_set_dirty(env
, current_cpu
->mem_io_vaddr
);
1848 static bool notdirty_mem_accepts(void *opaque
, hwaddr addr
,
1849 unsigned size
, bool is_write
)
1854 static const MemoryRegionOps notdirty_mem_ops
= {
1855 .write
= notdirty_mem_write
,
1856 .valid
.accepts
= notdirty_mem_accepts
,
1857 .endianness
= DEVICE_NATIVE_ENDIAN
,
1860 /* Generate a debug exception if a watchpoint has been hit. */
1861 static void check_watchpoint(int offset
, int len
, int flags
)
1863 CPUState
*cpu
= current_cpu
;
1864 CPUArchState
*env
= cpu
->env_ptr
;
1865 target_ulong pc
, cs_base
;
1870 if (cpu
->watchpoint_hit
) {
1871 /* We re-entered the check after replacing the TB. Now raise
1872 * the debug interrupt so that is will trigger after the
1873 * current instruction. */
1874 cpu_interrupt(cpu
, CPU_INTERRUPT_DEBUG
);
1877 vaddr
= (cpu
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
1878 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
1879 if (cpu_watchpoint_address_matches(wp
, vaddr
, len
)
1880 && (wp
->flags
& flags
)) {
1881 if (flags
== BP_MEM_READ
) {
1882 wp
->flags
|= BP_WATCHPOINT_HIT_READ
;
1884 wp
->flags
|= BP_WATCHPOINT_HIT_WRITE
;
1886 wp
->hitaddr
= vaddr
;
1887 if (!cpu
->watchpoint_hit
) {
1888 cpu
->watchpoint_hit
= wp
;
1889 tb_check_watchpoint(cpu
);
1890 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
1891 cpu
->exception_index
= EXCP_DEBUG
;
1894 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
1895 tb_gen_code(cpu
, pc
, cs_base
, cpu_flags
, 1);
1896 cpu_resume_from_signal(cpu
, NULL
);
1900 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
1905 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1906 so these check for a hit then pass through to the normal out-of-line
1908 static uint64_t watch_mem_read(void *opaque
, hwaddr addr
,
1911 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, size
, BP_MEM_READ
);
1913 case 1: return ldub_phys(&address_space_memory
, addr
);
1914 case 2: return lduw_phys(&address_space_memory
, addr
);
1915 case 4: return ldl_phys(&address_space_memory
, addr
);
1920 static void watch_mem_write(void *opaque
, hwaddr addr
,
1921 uint64_t val
, unsigned size
)
1923 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, size
, BP_MEM_WRITE
);
1926 stb_phys(&address_space_memory
, addr
, val
);
1929 stw_phys(&address_space_memory
, addr
, val
);
1932 stl_phys(&address_space_memory
, addr
, val
);
1938 static const MemoryRegionOps watch_mem_ops
= {
1939 .read
= watch_mem_read
,
1940 .write
= watch_mem_write
,
1941 .endianness
= DEVICE_NATIVE_ENDIAN
,
1944 static uint64_t subpage_read(void *opaque
, hwaddr addr
,
1947 subpage_t
*subpage
= opaque
;
1950 #if defined(DEBUG_SUBPAGE)
1951 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
"\n", __func__
,
1952 subpage
, len
, addr
);
1954 address_space_read(subpage
->as
, addr
+ subpage
->base
, buf
, len
);
1969 static void subpage_write(void *opaque
, hwaddr addr
,
1970 uint64_t value
, unsigned len
)
1972 subpage_t
*subpage
= opaque
;
1975 #if defined(DEBUG_SUBPAGE)
1976 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
1977 " value %"PRIx64
"\n",
1978 __func__
, subpage
, len
, addr
, value
);
1996 address_space_write(subpage
->as
, addr
+ subpage
->base
, buf
, len
);
1999 static bool subpage_accepts(void *opaque
, hwaddr addr
,
2000 unsigned len
, bool is_write
)
2002 subpage_t
*subpage
= opaque
;
2003 #if defined(DEBUG_SUBPAGE)
2004 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx
"\n",
2005 __func__
, subpage
, is_write
? 'w' : 'r', len
, addr
);
2008 return address_space_access_valid(subpage
->as
, addr
+ subpage
->base
,
2012 static const MemoryRegionOps subpage_ops
= {
2013 .read
= subpage_read
,
2014 .write
= subpage_write
,
2015 .impl
.min_access_size
= 1,
2016 .impl
.max_access_size
= 8,
2017 .valid
.min_access_size
= 1,
2018 .valid
.max_access_size
= 8,
2019 .valid
.accepts
= subpage_accepts
,
2020 .endianness
= DEVICE_NATIVE_ENDIAN
,
2023 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2028 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
2030 idx
= SUBPAGE_IDX(start
);
2031 eidx
= SUBPAGE_IDX(end
);
2032 #if defined(DEBUG_SUBPAGE)
2033 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2034 __func__
, mmio
, start
, end
, idx
, eidx
, section
);
2036 for (; idx
<= eidx
; idx
++) {
2037 mmio
->sub_section
[idx
] = section
;
2043 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
)
2047 mmio
= g_malloc0(sizeof(subpage_t
));
2051 memory_region_init_io(&mmio
->iomem
, NULL
, &subpage_ops
, mmio
,
2052 NULL
, TARGET_PAGE_SIZE
);
2053 mmio
->iomem
.subpage
= true;
2054 #if defined(DEBUG_SUBPAGE)
2055 printf("%s: %p base " TARGET_FMT_plx
" len %08x\n", __func__
,
2056 mmio
, base
, TARGET_PAGE_SIZE
);
2058 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
-1, PHYS_SECTION_UNASSIGNED
);
2063 static uint16_t dummy_section(PhysPageMap
*map
, AddressSpace
*as
,
2067 MemoryRegionSection section
= {
2068 .address_space
= as
,
2070 .offset_within_address_space
= 0,
2071 .offset_within_region
= 0,
2072 .size
= int128_2_64(),
2075 return phys_section_add(map
, §ion
);
2078 MemoryRegion
*iotlb_to_region(CPUState
*cpu
, hwaddr index
)
2080 AddressSpaceDispatch
*d
= atomic_rcu_read(&cpu
->memory_dispatch
);
2081 MemoryRegionSection
*sections
= d
->map
.sections
;
2083 return sections
[index
& ~TARGET_PAGE_MASK
].mr
;
2086 static void io_mem_init(void)
2088 memory_region_init_io(&io_mem_rom
, NULL
, &unassigned_mem_ops
, NULL
, NULL
, UINT64_MAX
);
2089 memory_region_init_io(&io_mem_unassigned
, NULL
, &unassigned_mem_ops
, NULL
,
2091 memory_region_init_io(&io_mem_notdirty
, NULL
, ¬dirty_mem_ops
, NULL
,
2093 memory_region_init_io(&io_mem_watch
, NULL
, &watch_mem_ops
, NULL
,
2097 static void mem_begin(MemoryListener
*listener
)
2099 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
2100 AddressSpaceDispatch
*d
= g_new0(AddressSpaceDispatch
, 1);
2103 n
= dummy_section(&d
->map
, as
, &io_mem_unassigned
);
2104 assert(n
== PHYS_SECTION_UNASSIGNED
);
2105 n
= dummy_section(&d
->map
, as
, &io_mem_notdirty
);
2106 assert(n
== PHYS_SECTION_NOTDIRTY
);
2107 n
= dummy_section(&d
->map
, as
, &io_mem_rom
);
2108 assert(n
== PHYS_SECTION_ROM
);
2109 n
= dummy_section(&d
->map
, as
, &io_mem_watch
);
2110 assert(n
== PHYS_SECTION_WATCH
);
2112 d
->phys_map
= (PhysPageEntry
) { .ptr
= PHYS_MAP_NODE_NIL
, .skip
= 1 };
2114 as
->next_dispatch
= d
;
2117 static void address_space_dispatch_free(AddressSpaceDispatch
*d
)
2119 phys_sections_free(&d
->map
);
2123 static void mem_commit(MemoryListener
*listener
)
2125 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
2126 AddressSpaceDispatch
*cur
= as
->dispatch
;
2127 AddressSpaceDispatch
*next
= as
->next_dispatch
;
2129 phys_page_compact_all(next
, next
->map
.nodes_nb
);
2131 atomic_rcu_set(&as
->dispatch
, next
);
2133 call_rcu(cur
, address_space_dispatch_free
, rcu
);
2137 static void tcg_commit(MemoryListener
*listener
)
2141 /* since each CPU stores ram addresses in its TLB cache, we must
2142 reset the modified entries */
2145 /* FIXME: Disentangle the cpu.h circular files deps so we can
2146 directly get the right CPU from listener. */
2147 if (cpu
->tcg_as_listener
!= listener
) {
2150 cpu_reload_memory_map(cpu
);
2154 static void core_log_global_start(MemoryListener
*listener
)
2156 cpu_physical_memory_set_dirty_tracking(true);
2159 static void core_log_global_stop(MemoryListener
*listener
)
2161 cpu_physical_memory_set_dirty_tracking(false);
2164 static MemoryListener core_memory_listener
= {
2165 .log_global_start
= core_log_global_start
,
2166 .log_global_stop
= core_log_global_stop
,
2170 void address_space_init_dispatch(AddressSpace
*as
)
2172 as
->dispatch
= NULL
;
2173 as
->dispatch_listener
= (MemoryListener
) {
2175 .commit
= mem_commit
,
2176 .region_add
= mem_add
,
2177 .region_nop
= mem_add
,
2180 memory_listener_register(&as
->dispatch_listener
, as
);
2183 void address_space_unregister(AddressSpace
*as
)
2185 memory_listener_unregister(&as
->dispatch_listener
);
2188 void address_space_destroy_dispatch(AddressSpace
*as
)
2190 AddressSpaceDispatch
*d
= as
->dispatch
;
2192 atomic_rcu_set(&as
->dispatch
, NULL
);
2194 call_rcu(d
, address_space_dispatch_free
, rcu
);
2198 static void memory_map_init(void)
2200 system_memory
= g_malloc(sizeof(*system_memory
));
2202 memory_region_init(system_memory
, NULL
, "system", UINT64_MAX
);
2203 address_space_init(&address_space_memory
, system_memory
, "memory");
2205 system_io
= g_malloc(sizeof(*system_io
));
2206 memory_region_init_io(system_io
, NULL
, &unassigned_io_ops
, NULL
, "io",
2208 address_space_init(&address_space_io
, system_io
, "I/O");
2210 memory_listener_register(&core_memory_listener
, &address_space_memory
);
2213 MemoryRegion
*get_system_memory(void)
2215 return system_memory
;
2218 MemoryRegion
*get_system_io(void)
2223 #endif /* !defined(CONFIG_USER_ONLY) */
2225 /* physical memory access (slow version, mainly for debug) */
2226 #if defined(CONFIG_USER_ONLY)
2227 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
2228 uint8_t *buf
, int len
, int is_write
)
2235 page
= addr
& TARGET_PAGE_MASK
;
2236 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2239 flags
= page_get_flags(page
);
2240 if (!(flags
& PAGE_VALID
))
2243 if (!(flags
& PAGE_WRITE
))
2245 /* XXX: this code should not depend on lock_user */
2246 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
2249 unlock_user(p
, addr
, l
);
2251 if (!(flags
& PAGE_READ
))
2253 /* XXX: this code should not depend on lock_user */
2254 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
2257 unlock_user(p
, addr
, 0);
2268 static void invalidate_and_set_dirty(hwaddr addr
,
2271 if (cpu_physical_memory_range_includes_clean(addr
, length
)) {
2272 tb_invalidate_phys_range(addr
, addr
+ length
, 0);
2273 cpu_physical_memory_set_dirty_range_nocode(addr
, length
);
2275 xen_modified_memory(addr
, length
);
2278 static int memory_access_size(MemoryRegion
*mr
, unsigned l
, hwaddr addr
)
2280 unsigned access_size_max
= mr
->ops
->valid
.max_access_size
;
2282 /* Regions are assumed to support 1-4 byte accesses unless
2283 otherwise specified. */
2284 if (access_size_max
== 0) {
2285 access_size_max
= 4;
2288 /* Bound the maximum access by the alignment of the address. */
2289 if (!mr
->ops
->impl
.unaligned
) {
2290 unsigned align_size_max
= addr
& -addr
;
2291 if (align_size_max
!= 0 && align_size_max
< access_size_max
) {
2292 access_size_max
= align_size_max
;
2296 /* Don't attempt accesses larger than the maximum. */
2297 if (l
> access_size_max
) {
2298 l
= access_size_max
;
2301 l
= 1 << (qemu_fls(l
) - 1);
2307 bool address_space_rw(AddressSpace
*as
, hwaddr addr
, uint8_t *buf
,
2308 int len
, bool is_write
)
2319 mr
= address_space_translate(as
, addr
, &addr1
, &l
, is_write
);
2322 if (!memory_access_is_direct(mr
, is_write
)) {
2323 l
= memory_access_size(mr
, l
, addr1
);
2324 /* XXX: could force current_cpu to NULL to avoid
2328 /* 64 bit write access */
2330 error
|= io_mem_write(mr
, addr1
, val
, 8);
2333 /* 32 bit write access */
2335 error
|= io_mem_write(mr
, addr1
, val
, 4);
2338 /* 16 bit write access */
2340 error
|= io_mem_write(mr
, addr1
, val
, 2);
2343 /* 8 bit write access */
2345 error
|= io_mem_write(mr
, addr1
, val
, 1);
2351 addr1
+= memory_region_get_ram_addr(mr
);
2353 ptr
= qemu_get_ram_ptr(addr1
);
2354 memcpy(ptr
, buf
, l
);
2355 invalidate_and_set_dirty(addr1
, l
);
2358 if (!memory_access_is_direct(mr
, is_write
)) {
2360 l
= memory_access_size(mr
, l
, addr1
);
2363 /* 64 bit read access */
2364 error
|= io_mem_read(mr
, addr1
, &val
, 8);
2368 /* 32 bit read access */
2369 error
|= io_mem_read(mr
, addr1
, &val
, 4);
2373 /* 16 bit read access */
2374 error
|= io_mem_read(mr
, addr1
, &val
, 2);
2378 /* 8 bit read access */
2379 error
|= io_mem_read(mr
, addr1
, &val
, 1);
2387 ptr
= qemu_get_ram_ptr(mr
->ram_addr
+ addr1
);
2388 memcpy(buf
, ptr
, l
);
2399 bool address_space_write(AddressSpace
*as
, hwaddr addr
,
2400 const uint8_t *buf
, int len
)
2402 return address_space_rw(as
, addr
, (uint8_t *)buf
, len
, true);
2405 bool address_space_read(AddressSpace
*as
, hwaddr addr
, uint8_t *buf
, int len
)
2407 return address_space_rw(as
, addr
, buf
, len
, false);
2411 void cpu_physical_memory_rw(hwaddr addr
, uint8_t *buf
,
2412 int len
, int is_write
)
2414 address_space_rw(&address_space_memory
, addr
, buf
, len
, is_write
);
2417 enum write_rom_type
{
2422 static inline void cpu_physical_memory_write_rom_internal(AddressSpace
*as
,
2423 hwaddr addr
, const uint8_t *buf
, int len
, enum write_rom_type type
)
2432 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
2434 if (!(memory_region_is_ram(mr
) ||
2435 memory_region_is_romd(mr
))) {
2438 addr1
+= memory_region_get_ram_addr(mr
);
2440 ptr
= qemu_get_ram_ptr(addr1
);
2443 memcpy(ptr
, buf
, l
);
2444 invalidate_and_set_dirty(addr1
, l
);
2447 flush_icache_range((uintptr_t)ptr
, (uintptr_t)ptr
+ l
);
2457 /* used for ROM loading : can write in RAM and ROM */
2458 void cpu_physical_memory_write_rom(AddressSpace
*as
, hwaddr addr
,
2459 const uint8_t *buf
, int len
)
2461 cpu_physical_memory_write_rom_internal(as
, addr
, buf
, len
, WRITE_DATA
);
2464 void cpu_flush_icache_range(hwaddr start
, int len
)
2467 * This function should do the same thing as an icache flush that was
2468 * triggered from within the guest. For TCG we are always cache coherent,
2469 * so there is no need to flush anything. For KVM / Xen we need to flush
2470 * the host's instruction cache at least.
2472 if (tcg_enabled()) {
2476 cpu_physical_memory_write_rom_internal(&address_space_memory
,
2477 start
, NULL
, len
, FLUSH_CACHE
);
2487 static BounceBuffer bounce
;
2489 typedef struct MapClient
{
2491 void (*callback
)(void *opaque
);
2492 QLIST_ENTRY(MapClient
) link
;
2495 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
2496 = QLIST_HEAD_INITIALIZER(map_client_list
);
2498 void *cpu_register_map_client(void *opaque
, void (*callback
)(void *opaque
))
2500 MapClient
*client
= g_malloc(sizeof(*client
));
2502 client
->opaque
= opaque
;
2503 client
->callback
= callback
;
2504 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
2508 static void cpu_unregister_map_client(void *_client
)
2510 MapClient
*client
= (MapClient
*)_client
;
2512 QLIST_REMOVE(client
, link
);
2516 static void cpu_notify_map_clients(void)
2520 while (!QLIST_EMPTY(&map_client_list
)) {
2521 client
= QLIST_FIRST(&map_client_list
);
2522 client
->callback(client
->opaque
);
2523 cpu_unregister_map_client(client
);
2527 bool address_space_access_valid(AddressSpace
*as
, hwaddr addr
, int len
, bool is_write
)
2534 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2535 if (!memory_access_is_direct(mr
, is_write
)) {
2536 l
= memory_access_size(mr
, l
, addr
);
2537 if (!memory_region_access_valid(mr
, xlat
, l
, is_write
)) {
2548 /* Map a physical memory region into a host virtual address.
2549 * May map a subset of the requested range, given by and returned in *plen.
2550 * May return NULL if resources needed to perform the mapping are exhausted.
2551 * Use only for reads OR writes - not for read-modify-write operations.
2552 * Use cpu_register_map_client() to know when retrying the map operation is
2553 * likely to succeed.
2555 void *address_space_map(AddressSpace
*as
,
2562 hwaddr l
, xlat
, base
;
2563 MemoryRegion
*mr
, *this_mr
;
2571 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2572 if (!memory_access_is_direct(mr
, is_write
)) {
2573 if (bounce
.buffer
) {
2576 /* Avoid unbounded allocations */
2577 l
= MIN(l
, TARGET_PAGE_SIZE
);
2578 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, l
);
2582 memory_region_ref(mr
);
2585 address_space_read(as
, addr
, bounce
.buffer
, l
);
2589 return bounce
.buffer
;
2593 raddr
= memory_region_get_ram_addr(mr
);
2604 this_mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2605 if (this_mr
!= mr
|| xlat
!= base
+ done
) {
2610 memory_region_ref(mr
);
2612 return qemu_ram_ptr_length(raddr
+ base
, plen
);
2615 /* Unmaps a memory region previously mapped by address_space_map().
2616 * Will also mark the memory as dirty if is_write == 1. access_len gives
2617 * the amount of memory that was actually read or written by the caller.
2619 void address_space_unmap(AddressSpace
*as
, void *buffer
, hwaddr len
,
2620 int is_write
, hwaddr access_len
)
2622 if (buffer
!= bounce
.buffer
) {
2626 mr
= qemu_ram_addr_from_host(buffer
, &addr1
);
2629 invalidate_and_set_dirty(addr1
, access_len
);
2631 if (xen_enabled()) {
2632 xen_invalidate_map_cache_entry(buffer
);
2634 memory_region_unref(mr
);
2638 address_space_write(as
, bounce
.addr
, bounce
.buffer
, access_len
);
2640 qemu_vfree(bounce
.buffer
);
2641 bounce
.buffer
= NULL
;
2642 memory_region_unref(bounce
.mr
);
2643 cpu_notify_map_clients();
2646 void *cpu_physical_memory_map(hwaddr addr
,
2650 return address_space_map(&address_space_memory
, addr
, plen
, is_write
);
2653 void cpu_physical_memory_unmap(void *buffer
, hwaddr len
,
2654 int is_write
, hwaddr access_len
)
2656 return address_space_unmap(&address_space_memory
, buffer
, len
, is_write
, access_len
);
2659 /* warning: addr must be aligned */
2660 static inline uint32_t ldl_phys_internal(AddressSpace
*as
, hwaddr addr
,
2661 enum device_endian endian
)
2669 mr
= address_space_translate(as
, addr
, &addr1
, &l
, false);
2670 if (l
< 4 || !memory_access_is_direct(mr
, false)) {
2672 io_mem_read(mr
, addr1
, &val
, 4);
2673 #if defined(TARGET_WORDS_BIGENDIAN)
2674 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2678 if (endian
== DEVICE_BIG_ENDIAN
) {
2684 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2688 case DEVICE_LITTLE_ENDIAN
:
2689 val
= ldl_le_p(ptr
);
2691 case DEVICE_BIG_ENDIAN
:
2692 val
= ldl_be_p(ptr
);
2702 uint32_t ldl_phys(AddressSpace
*as
, hwaddr addr
)
2704 return ldl_phys_internal(as
, addr
, DEVICE_NATIVE_ENDIAN
);
2707 uint32_t ldl_le_phys(AddressSpace
*as
, hwaddr addr
)
2709 return ldl_phys_internal(as
, addr
, DEVICE_LITTLE_ENDIAN
);
2712 uint32_t ldl_be_phys(AddressSpace
*as
, hwaddr addr
)
2714 return ldl_phys_internal(as
, addr
, DEVICE_BIG_ENDIAN
);
2717 /* warning: addr must be aligned */
2718 static inline uint64_t ldq_phys_internal(AddressSpace
*as
, hwaddr addr
,
2719 enum device_endian endian
)
2727 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2729 if (l
< 8 || !memory_access_is_direct(mr
, false)) {
2731 io_mem_read(mr
, addr1
, &val
, 8);
2732 #if defined(TARGET_WORDS_BIGENDIAN)
2733 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2737 if (endian
== DEVICE_BIG_ENDIAN
) {
2743 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2747 case DEVICE_LITTLE_ENDIAN
:
2748 val
= ldq_le_p(ptr
);
2750 case DEVICE_BIG_ENDIAN
:
2751 val
= ldq_be_p(ptr
);
2761 uint64_t ldq_phys(AddressSpace
*as
, hwaddr addr
)
2763 return ldq_phys_internal(as
, addr
, DEVICE_NATIVE_ENDIAN
);
2766 uint64_t ldq_le_phys(AddressSpace
*as
, hwaddr addr
)
2768 return ldq_phys_internal(as
, addr
, DEVICE_LITTLE_ENDIAN
);
2771 uint64_t ldq_be_phys(AddressSpace
*as
, hwaddr addr
)
2773 return ldq_phys_internal(as
, addr
, DEVICE_BIG_ENDIAN
);
2777 uint32_t ldub_phys(AddressSpace
*as
, hwaddr addr
)
2780 address_space_rw(as
, addr
, &val
, 1, 0);
2784 /* warning: addr must be aligned */
2785 static inline uint32_t lduw_phys_internal(AddressSpace
*as
, hwaddr addr
,
2786 enum device_endian endian
)
2794 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2796 if (l
< 2 || !memory_access_is_direct(mr
, false)) {
2798 io_mem_read(mr
, addr1
, &val
, 2);
2799 #if defined(TARGET_WORDS_BIGENDIAN)
2800 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2804 if (endian
== DEVICE_BIG_ENDIAN
) {
2810 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2814 case DEVICE_LITTLE_ENDIAN
:
2815 val
= lduw_le_p(ptr
);
2817 case DEVICE_BIG_ENDIAN
:
2818 val
= lduw_be_p(ptr
);
2828 uint32_t lduw_phys(AddressSpace
*as
, hwaddr addr
)
2830 return lduw_phys_internal(as
, addr
, DEVICE_NATIVE_ENDIAN
);
2833 uint32_t lduw_le_phys(AddressSpace
*as
, hwaddr addr
)
2835 return lduw_phys_internal(as
, addr
, DEVICE_LITTLE_ENDIAN
);
2838 uint32_t lduw_be_phys(AddressSpace
*as
, hwaddr addr
)
2840 return lduw_phys_internal(as
, addr
, DEVICE_BIG_ENDIAN
);
2843 /* warning: addr must be aligned. The ram page is not masked as dirty
2844 and the code inside is not invalidated. It is useful if the dirty
2845 bits are used to track modified PTEs */
2846 void stl_phys_notdirty(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2853 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2855 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
2856 io_mem_write(mr
, addr1
, val
, 4);
2858 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
2859 ptr
= qemu_get_ram_ptr(addr1
);
2862 if (unlikely(in_migration
)) {
2863 if (cpu_physical_memory_is_clean(addr1
)) {
2864 /* invalidate code */
2865 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
2867 cpu_physical_memory_set_dirty_range_nocode(addr1
, 4);
2873 /* warning: addr must be aligned */
2874 static inline void stl_phys_internal(AddressSpace
*as
,
2875 hwaddr addr
, uint32_t val
,
2876 enum device_endian endian
)
2883 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2885 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
2886 #if defined(TARGET_WORDS_BIGENDIAN)
2887 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2891 if (endian
== DEVICE_BIG_ENDIAN
) {
2895 io_mem_write(mr
, addr1
, val
, 4);
2898 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
2899 ptr
= qemu_get_ram_ptr(addr1
);
2901 case DEVICE_LITTLE_ENDIAN
:
2904 case DEVICE_BIG_ENDIAN
:
2911 invalidate_and_set_dirty(addr1
, 4);
2915 void stl_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2917 stl_phys_internal(as
, addr
, val
, DEVICE_NATIVE_ENDIAN
);
2920 void stl_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2922 stl_phys_internal(as
, addr
, val
, DEVICE_LITTLE_ENDIAN
);
2925 void stl_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2927 stl_phys_internal(as
, addr
, val
, DEVICE_BIG_ENDIAN
);
2931 void stb_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2934 address_space_rw(as
, addr
, &v
, 1, 1);
2937 /* warning: addr must be aligned */
2938 static inline void stw_phys_internal(AddressSpace
*as
,
2939 hwaddr addr
, uint32_t val
,
2940 enum device_endian endian
)
2947 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
2948 if (l
< 2 || !memory_access_is_direct(mr
, true)) {
2949 #if defined(TARGET_WORDS_BIGENDIAN)
2950 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2954 if (endian
== DEVICE_BIG_ENDIAN
) {
2958 io_mem_write(mr
, addr1
, val
, 2);
2961 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
2962 ptr
= qemu_get_ram_ptr(addr1
);
2964 case DEVICE_LITTLE_ENDIAN
:
2967 case DEVICE_BIG_ENDIAN
:
2974 invalidate_and_set_dirty(addr1
, 2);
2978 void stw_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2980 stw_phys_internal(as
, addr
, val
, DEVICE_NATIVE_ENDIAN
);
2983 void stw_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2985 stw_phys_internal(as
, addr
, val
, DEVICE_LITTLE_ENDIAN
);
2988 void stw_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2990 stw_phys_internal(as
, addr
, val
, DEVICE_BIG_ENDIAN
);
2994 void stq_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
2997 address_space_rw(as
, addr
, (void *) &val
, 8, 1);
3000 void stq_le_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3002 val
= cpu_to_le64(val
);
3003 address_space_rw(as
, addr
, (void *) &val
, 8, 1);
3006 void stq_be_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3008 val
= cpu_to_be64(val
);
3009 address_space_rw(as
, addr
, (void *) &val
, 8, 1);
3012 /* virtual memory access for debug (includes writing to ROM) */
3013 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
3014 uint8_t *buf
, int len
, int is_write
)
3021 page
= addr
& TARGET_PAGE_MASK
;
3022 phys_addr
= cpu_get_phys_page_debug(cpu
, page
);
3023 /* if no physical page mapped, return an error */
3024 if (phys_addr
== -1)
3026 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3029 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
3031 cpu_physical_memory_write_rom(cpu
->as
, phys_addr
, buf
, l
);
3033 address_space_rw(cpu
->as
, phys_addr
, buf
, l
, 0);
3044 * A helper function for the _utterly broken_ virtio device model to find out if
3045 * it's running on a big endian machine. Don't do this at home kids!
3047 bool target_words_bigendian(void);
3048 bool target_words_bigendian(void)
3050 #if defined(TARGET_WORDS_BIGENDIAN)
3057 #ifndef CONFIG_USER_ONLY
3058 bool cpu_physical_memory_is_io(hwaddr phys_addr
)
3063 mr
= address_space_translate(&address_space_memory
,
3064 phys_addr
, &phys_addr
, &l
, false);
3066 return !(memory_region_is_ram(mr
) ||
3067 memory_region_is_romd(mr
));
3070 void qemu_ram_foreach_block(RAMBlockIterFunc func
, void *opaque
)
3075 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
3076 func(block
->host
, block
->offset
, block
->used_length
, opaque
);