4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
21 #include <sys/types.h>
25 #include "qemu-common.h"
30 #include "qemu/osdep.h"
31 #include "sysemu/kvm.h"
32 #include "sysemu/sysemu.h"
33 #include "hw/xen/xen.h"
34 #include "qemu/timer.h"
35 #include "qemu/config-file.h"
36 #include "qemu/error-report.h"
37 #include "exec/memory.h"
38 #include "sysemu/dma.h"
39 #include "exec/address-spaces.h"
40 #if defined(CONFIG_USER_ONLY)
42 #else /* !CONFIG_USER_ONLY */
43 #include "sysemu/xen-mapcache.h"
46 #include "exec/cpu-all.h"
48 #include "exec/cputlb.h"
49 #include "translate-all.h"
51 #include "exec/memory-internal.h"
52 #include "exec/ram_addr.h"
53 #include "qemu/cache-utils.h"
55 #include "qemu/range.h"
57 //#define DEBUG_SUBPAGE
59 #if !defined(CONFIG_USER_ONLY)
60 static bool in_migration
;
62 RAMList ram_list
= { .blocks
= QTAILQ_HEAD_INITIALIZER(ram_list
.blocks
) };
64 static MemoryRegion
*system_memory
;
65 static MemoryRegion
*system_io
;
67 AddressSpace address_space_io
;
68 AddressSpace address_space_memory
;
70 MemoryRegion io_mem_rom
, io_mem_notdirty
;
71 static MemoryRegion io_mem_unassigned
;
75 struct CPUTailQ cpus
= QTAILQ_HEAD_INITIALIZER(cpus
);
76 /* current CPU in the current thread. It is only valid inside
78 DEFINE_TLS(CPUState
*, current_cpu
);
79 /* 0 = Do not count executed instructions.
80 1 = Precise instruction counting.
81 2 = Adaptive rate instruction counting. */
84 #if !defined(CONFIG_USER_ONLY)
86 typedef struct PhysPageEntry PhysPageEntry
;
88 struct PhysPageEntry
{
89 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
91 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
95 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
97 /* Size of the L2 (and L3, etc) page tables. */
98 #define ADDR_SPACE_BITS 64
101 #define P_L2_SIZE (1 << P_L2_BITS)
103 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
105 typedef PhysPageEntry Node
[P_L2_SIZE
];
107 typedef struct PhysPageMap
{
108 unsigned sections_nb
;
109 unsigned sections_nb_alloc
;
111 unsigned nodes_nb_alloc
;
113 MemoryRegionSection
*sections
;
116 struct AddressSpaceDispatch
{
117 /* This is a multi-level map on the physical address space.
118 * The bottom level has pointers to MemoryRegionSections.
120 PhysPageEntry phys_map
;
125 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
126 typedef struct subpage_t
{
130 uint16_t sub_section
[TARGET_PAGE_SIZE
];
133 #define PHYS_SECTION_UNASSIGNED 0
134 #define PHYS_SECTION_NOTDIRTY 1
135 #define PHYS_SECTION_ROM 2
136 #define PHYS_SECTION_WATCH 3
138 static void io_mem_init(void);
139 static void memory_map_init(void);
140 static void tcg_commit(MemoryListener
*listener
);
142 static MemoryRegion io_mem_watch
;
145 #if !defined(CONFIG_USER_ONLY)
147 static void phys_map_node_reserve(PhysPageMap
*map
, unsigned nodes
)
149 if (map
->nodes_nb
+ nodes
> map
->nodes_nb_alloc
) {
150 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
* 2, 16);
151 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
, map
->nodes_nb
+ nodes
);
152 map
->nodes
= g_renew(Node
, map
->nodes
, map
->nodes_nb_alloc
);
156 static uint32_t phys_map_node_alloc(PhysPageMap
*map
)
161 ret
= map
->nodes_nb
++;
162 assert(ret
!= PHYS_MAP_NODE_NIL
);
163 assert(ret
!= map
->nodes_nb_alloc
);
164 for (i
= 0; i
< P_L2_SIZE
; ++i
) {
165 map
->nodes
[ret
][i
].skip
= 1;
166 map
->nodes
[ret
][i
].ptr
= PHYS_MAP_NODE_NIL
;
171 static void phys_page_set_level(PhysPageMap
*map
, PhysPageEntry
*lp
,
172 hwaddr
*index
, hwaddr
*nb
, uint16_t leaf
,
177 hwaddr step
= (hwaddr
)1 << (level
* P_L2_BITS
);
179 if (lp
->skip
&& lp
->ptr
== PHYS_MAP_NODE_NIL
) {
180 lp
->ptr
= phys_map_node_alloc(map
);
181 p
= map
->nodes
[lp
->ptr
];
183 for (i
= 0; i
< P_L2_SIZE
; i
++) {
185 p
[i
].ptr
= PHYS_SECTION_UNASSIGNED
;
189 p
= map
->nodes
[lp
->ptr
];
191 lp
= &p
[(*index
>> (level
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
193 while (*nb
&& lp
< &p
[P_L2_SIZE
]) {
194 if ((*index
& (step
- 1)) == 0 && *nb
>= step
) {
200 phys_page_set_level(map
, lp
, index
, nb
, leaf
, level
- 1);
206 static void phys_page_set(AddressSpaceDispatch
*d
,
207 hwaddr index
, hwaddr nb
,
210 /* Wildly overreserve - it doesn't matter much. */
211 phys_map_node_reserve(&d
->map
, 3 * P_L2_LEVELS
);
213 phys_page_set_level(&d
->map
, &d
->phys_map
, &index
, &nb
, leaf
, P_L2_LEVELS
- 1);
216 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
217 * and update our entry so we can skip it and go directly to the destination.
219 static void phys_page_compact(PhysPageEntry
*lp
, Node
*nodes
, unsigned long *compacted
)
221 unsigned valid_ptr
= P_L2_SIZE
;
226 if (lp
->ptr
== PHYS_MAP_NODE_NIL
) {
231 for (i
= 0; i
< P_L2_SIZE
; i
++) {
232 if (p
[i
].ptr
== PHYS_MAP_NODE_NIL
) {
239 phys_page_compact(&p
[i
], nodes
, compacted
);
243 /* We can only compress if there's only one child. */
248 assert(valid_ptr
< P_L2_SIZE
);
250 /* Don't compress if it won't fit in the # of bits we have. */
251 if (lp
->skip
+ p
[valid_ptr
].skip
>= (1 << 3)) {
255 lp
->ptr
= p
[valid_ptr
].ptr
;
256 if (!p
[valid_ptr
].skip
) {
257 /* If our only child is a leaf, make this a leaf. */
258 /* By design, we should have made this node a leaf to begin with so we
259 * should never reach here.
260 * But since it's so simple to handle this, let's do it just in case we
265 lp
->skip
+= p
[valid_ptr
].skip
;
269 static void phys_page_compact_all(AddressSpaceDispatch
*d
, int nodes_nb
)
271 DECLARE_BITMAP(compacted
, nodes_nb
);
273 if (d
->phys_map
.skip
) {
274 phys_page_compact(&d
->phys_map
, d
->map
.nodes
, compacted
);
278 static MemoryRegionSection
*phys_page_find(PhysPageEntry lp
, hwaddr addr
,
279 Node
*nodes
, MemoryRegionSection
*sections
)
282 hwaddr index
= addr
>> TARGET_PAGE_BITS
;
285 for (i
= P_L2_LEVELS
; lp
.skip
&& (i
-= lp
.skip
) >= 0;) {
286 if (lp
.ptr
== PHYS_MAP_NODE_NIL
) {
287 return §ions
[PHYS_SECTION_UNASSIGNED
];
290 lp
= p
[(index
>> (i
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
293 if (sections
[lp
.ptr
].size
.hi
||
294 range_covers_byte(sections
[lp
.ptr
].offset_within_address_space
,
295 sections
[lp
.ptr
].size
.lo
, addr
)) {
296 return §ions
[lp
.ptr
];
298 return §ions
[PHYS_SECTION_UNASSIGNED
];
302 bool memory_region_is_unassigned(MemoryRegion
*mr
)
304 return mr
!= &io_mem_rom
&& mr
!= &io_mem_notdirty
&& !mr
->rom_device
305 && mr
!= &io_mem_watch
;
308 static MemoryRegionSection
*address_space_lookup_region(AddressSpaceDispatch
*d
,
310 bool resolve_subpage
)
312 MemoryRegionSection
*section
;
315 section
= phys_page_find(d
->phys_map
, addr
, d
->map
.nodes
, d
->map
.sections
);
316 if (resolve_subpage
&& section
->mr
->subpage
) {
317 subpage
= container_of(section
->mr
, subpage_t
, iomem
);
318 section
= &d
->map
.sections
[subpage
->sub_section
[SUBPAGE_IDX(addr
)]];
323 static MemoryRegionSection
*
324 address_space_translate_internal(AddressSpaceDispatch
*d
, hwaddr addr
, hwaddr
*xlat
,
325 hwaddr
*plen
, bool resolve_subpage
)
327 MemoryRegionSection
*section
;
330 section
= address_space_lookup_region(d
, addr
, resolve_subpage
);
331 /* Compute offset within MemoryRegionSection */
332 addr
-= section
->offset_within_address_space
;
334 /* Compute offset within MemoryRegion */
335 *xlat
= addr
+ section
->offset_within_region
;
337 diff
= int128_sub(section
->mr
->size
, int128_make64(addr
));
338 *plen
= int128_get64(int128_min(diff
, int128_make64(*plen
)));
342 static inline bool memory_access_is_direct(MemoryRegion
*mr
, bool is_write
)
344 if (memory_region_is_ram(mr
)) {
345 return !(is_write
&& mr
->readonly
);
347 if (memory_region_is_romd(mr
)) {
354 MemoryRegion
*address_space_translate(AddressSpace
*as
, hwaddr addr
,
355 hwaddr
*xlat
, hwaddr
*plen
,
359 MemoryRegionSection
*section
;
364 section
= address_space_translate_internal(as
->dispatch
, addr
, &addr
, plen
, true);
367 if (!mr
->iommu_ops
) {
371 iotlb
= mr
->iommu_ops
->translate(mr
, addr
);
372 addr
= ((iotlb
.translated_addr
& ~iotlb
.addr_mask
)
373 | (addr
& iotlb
.addr_mask
));
374 len
= MIN(len
, (addr
| iotlb
.addr_mask
) - addr
+ 1);
375 if (!(iotlb
.perm
& (1 << is_write
))) {
376 mr
= &io_mem_unassigned
;
380 as
= iotlb
.target_as
;
383 if (xen_enabled() && memory_access_is_direct(mr
, is_write
)) {
384 hwaddr page
= ((addr
& TARGET_PAGE_MASK
) + TARGET_PAGE_SIZE
) - addr
;
385 len
= MIN(page
, len
);
393 MemoryRegionSection
*
394 address_space_translate_for_iotlb(AddressSpace
*as
, hwaddr addr
, hwaddr
*xlat
,
397 MemoryRegionSection
*section
;
398 section
= address_space_translate_internal(as
->dispatch
, addr
, xlat
, plen
, false);
400 assert(!section
->mr
->iommu_ops
);
405 void cpu_exec_init_all(void)
407 #if !defined(CONFIG_USER_ONLY)
408 qemu_mutex_init(&ram_list
.mutex
);
414 #if !defined(CONFIG_USER_ONLY)
416 static int cpu_common_post_load(void *opaque
, int version_id
)
418 CPUState
*cpu
= opaque
;
420 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
421 version_id is increased. */
422 cpu
->interrupt_request
&= ~0x01;
428 const VMStateDescription vmstate_cpu_common
= {
429 .name
= "cpu_common",
431 .minimum_version_id
= 1,
432 .post_load
= cpu_common_post_load
,
433 .fields
= (VMStateField
[]) {
434 VMSTATE_UINT32(halted
, CPUState
),
435 VMSTATE_UINT32(interrupt_request
, CPUState
),
436 VMSTATE_END_OF_LIST()
442 CPUState
*qemu_get_cpu(int index
)
447 if (cpu
->cpu_index
== index
) {
455 #if !defined(CONFIG_USER_ONLY)
456 void tcg_cpu_address_space_init(CPUState
*cpu
, AddressSpace
*as
)
458 /* We only support one address space per cpu at the moment. */
459 assert(cpu
->as
== as
);
461 if (cpu
->tcg_as_listener
) {
462 memory_listener_unregister(cpu
->tcg_as_listener
);
464 cpu
->tcg_as_listener
= g_new0(MemoryListener
, 1);
466 cpu
->tcg_as_listener
->commit
= tcg_commit
;
467 memory_listener_register(cpu
->tcg_as_listener
, as
);
471 void cpu_exec_init(CPUArchState
*env
)
473 CPUState
*cpu
= ENV_GET_CPU(env
);
474 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
478 #if defined(CONFIG_USER_ONLY)
482 CPU_FOREACH(some_cpu
) {
485 cpu
->cpu_index
= cpu_index
;
487 QTAILQ_INIT(&cpu
->breakpoints
);
488 QTAILQ_INIT(&cpu
->watchpoints
);
489 #ifndef CONFIG_USER_ONLY
490 cpu
->as
= &address_space_memory
;
491 cpu
->thread_id
= qemu_get_thread_id();
493 QTAILQ_INSERT_TAIL(&cpus
, cpu
, node
);
494 #if defined(CONFIG_USER_ONLY)
497 if (qdev_get_vmsd(DEVICE(cpu
)) == NULL
) {
498 vmstate_register(NULL
, cpu_index
, &vmstate_cpu_common
, cpu
);
500 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
501 register_savevm(NULL
, "cpu", cpu_index
, CPU_SAVE_VERSION
,
502 cpu_save
, cpu_load
, env
);
503 assert(cc
->vmsd
== NULL
);
504 assert(qdev_get_vmsd(DEVICE(cpu
)) == NULL
);
506 if (cc
->vmsd
!= NULL
) {
507 vmstate_register(NULL
, cpu_index
, cc
->vmsd
, cpu
);
511 #if defined(TARGET_HAS_ICE)
512 #if defined(CONFIG_USER_ONLY)
513 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
515 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
518 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
520 hwaddr phys
= cpu_get_phys_page_debug(cpu
, pc
);
522 tb_invalidate_phys_addr(cpu
->as
,
523 phys
| (pc
& ~TARGET_PAGE_MASK
));
527 #endif /* TARGET_HAS_ICE */
529 #if defined(CONFIG_USER_ONLY)
530 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
535 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
536 int flags
, CPUWatchpoint
**watchpoint
)
541 /* Add a watchpoint. */
542 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
543 int flags
, CPUWatchpoint
**watchpoint
)
545 vaddr len_mask
= ~(len
- 1);
548 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
549 if ((len
& (len
- 1)) || (addr
& ~len_mask
) ||
550 len
== 0 || len
> TARGET_PAGE_SIZE
) {
551 error_report("tried to set invalid watchpoint at %"
552 VADDR_PRIx
", len=%" VADDR_PRIu
, addr
, len
);
555 wp
= g_malloc(sizeof(*wp
));
558 wp
->len_mask
= len_mask
;
561 /* keep all GDB-injected watchpoints in front */
562 if (flags
& BP_GDB
) {
563 QTAILQ_INSERT_HEAD(&cpu
->watchpoints
, wp
, entry
);
565 QTAILQ_INSERT_TAIL(&cpu
->watchpoints
, wp
, entry
);
568 tlb_flush_page(cpu
, addr
);
575 /* Remove a specific watchpoint. */
576 int cpu_watchpoint_remove(CPUState
*cpu
, vaddr addr
, vaddr len
,
579 vaddr len_mask
= ~(len
- 1);
582 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
583 if (addr
== wp
->vaddr
&& len_mask
== wp
->len_mask
584 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
585 cpu_watchpoint_remove_by_ref(cpu
, wp
);
592 /* Remove a specific watchpoint by reference. */
593 void cpu_watchpoint_remove_by_ref(CPUState
*cpu
, CPUWatchpoint
*watchpoint
)
595 QTAILQ_REMOVE(&cpu
->watchpoints
, watchpoint
, entry
);
597 tlb_flush_page(cpu
, watchpoint
->vaddr
);
602 /* Remove all matching watchpoints. */
603 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
605 CPUWatchpoint
*wp
, *next
;
607 QTAILQ_FOREACH_SAFE(wp
, &cpu
->watchpoints
, entry
, next
) {
608 if (wp
->flags
& mask
) {
609 cpu_watchpoint_remove_by_ref(cpu
, wp
);
615 /* Add a breakpoint. */
616 int cpu_breakpoint_insert(CPUState
*cpu
, vaddr pc
, int flags
,
617 CPUBreakpoint
**breakpoint
)
619 #if defined(TARGET_HAS_ICE)
622 bp
= g_malloc(sizeof(*bp
));
627 /* keep all GDB-injected breakpoints in front */
628 if (flags
& BP_GDB
) {
629 QTAILQ_INSERT_HEAD(&cpu
->breakpoints
, bp
, entry
);
631 QTAILQ_INSERT_TAIL(&cpu
->breakpoints
, bp
, entry
);
634 breakpoint_invalidate(cpu
, pc
);
645 /* Remove a specific breakpoint. */
646 int cpu_breakpoint_remove(CPUState
*cpu
, vaddr pc
, int flags
)
648 #if defined(TARGET_HAS_ICE)
651 QTAILQ_FOREACH(bp
, &cpu
->breakpoints
, entry
) {
652 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
653 cpu_breakpoint_remove_by_ref(cpu
, bp
);
663 /* Remove a specific breakpoint by reference. */
664 void cpu_breakpoint_remove_by_ref(CPUState
*cpu
, CPUBreakpoint
*breakpoint
)
666 #if defined(TARGET_HAS_ICE)
667 QTAILQ_REMOVE(&cpu
->breakpoints
, breakpoint
, entry
);
669 breakpoint_invalidate(cpu
, breakpoint
->pc
);
675 /* Remove all matching breakpoints. */
676 void cpu_breakpoint_remove_all(CPUState
*cpu
, int mask
)
678 #if defined(TARGET_HAS_ICE)
679 CPUBreakpoint
*bp
, *next
;
681 QTAILQ_FOREACH_SAFE(bp
, &cpu
->breakpoints
, entry
, next
) {
682 if (bp
->flags
& mask
) {
683 cpu_breakpoint_remove_by_ref(cpu
, bp
);
689 /* enable or disable single step mode. EXCP_DEBUG is returned by the
690 CPU loop after each instruction */
691 void cpu_single_step(CPUState
*cpu
, int enabled
)
693 #if defined(TARGET_HAS_ICE)
694 if (cpu
->singlestep_enabled
!= enabled
) {
695 cpu
->singlestep_enabled
= enabled
;
697 kvm_update_guest_debug(cpu
, 0);
699 /* must flush all the translated code to avoid inconsistencies */
700 /* XXX: only flush what is necessary */
701 CPUArchState
*env
= cpu
->env_ptr
;
708 void cpu_abort(CPUState
*cpu
, const char *fmt
, ...)
715 fprintf(stderr
, "qemu: fatal: ");
716 vfprintf(stderr
, fmt
, ap
);
717 fprintf(stderr
, "\n");
718 cpu_dump_state(cpu
, stderr
, fprintf
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
719 if (qemu_log_enabled()) {
720 qemu_log("qemu: fatal: ");
721 qemu_log_vprintf(fmt
, ap2
);
723 log_cpu_state(cpu
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
729 #if defined(CONFIG_USER_ONLY)
731 struct sigaction act
;
732 sigfillset(&act
.sa_mask
);
733 act
.sa_handler
= SIG_DFL
;
734 sigaction(SIGABRT
, &act
, NULL
);
740 #if !defined(CONFIG_USER_ONLY)
741 static RAMBlock
*qemu_get_ram_block(ram_addr_t addr
)
745 /* The list is protected by the iothread lock here. */
746 block
= ram_list
.mru_block
;
747 if (block
&& addr
- block
->offset
< block
->length
) {
750 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
751 if (addr
- block
->offset
< block
->length
) {
756 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
760 ram_list
.mru_block
= block
;
764 static void tlb_reset_dirty_range_all(ram_addr_t start
, ram_addr_t length
)
770 end
= TARGET_PAGE_ALIGN(start
+ length
);
771 start
&= TARGET_PAGE_MASK
;
773 block
= qemu_get_ram_block(start
);
774 assert(block
== qemu_get_ram_block(end
- 1));
775 start1
= (uintptr_t)block
->host
+ (start
- block
->offset
);
776 cpu_tlb_reset_dirty_all(start1
, length
);
779 /* Note: start and end must be within the same ram block. */
780 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t length
,
785 cpu_physical_memory_clear_dirty_range(start
, length
, client
);
788 tlb_reset_dirty_range_all(start
, length
);
792 static void cpu_physical_memory_set_dirty_tracking(bool enable
)
794 in_migration
= enable
;
797 hwaddr
memory_region_section_get_iotlb(CPUState
*cpu
,
798 MemoryRegionSection
*section
,
800 hwaddr paddr
, hwaddr xlat
,
802 target_ulong
*address
)
807 if (memory_region_is_ram(section
->mr
)) {
809 iotlb
= (memory_region_get_ram_addr(section
->mr
) & TARGET_PAGE_MASK
)
811 if (!section
->readonly
) {
812 iotlb
|= PHYS_SECTION_NOTDIRTY
;
814 iotlb
|= PHYS_SECTION_ROM
;
817 iotlb
= section
- section
->address_space
->dispatch
->map
.sections
;
821 /* Make accesses to pages with watchpoints go via the
822 watchpoint trap routines. */
823 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
824 if (vaddr
== (wp
->vaddr
& TARGET_PAGE_MASK
)) {
825 /* Avoid trapping reads of pages with a write breakpoint. */
826 if ((prot
& PAGE_WRITE
) || (wp
->flags
& BP_MEM_READ
)) {
827 iotlb
= PHYS_SECTION_WATCH
+ paddr
;
828 *address
|= TLB_MMIO
;
836 #endif /* defined(CONFIG_USER_ONLY) */
838 #if !defined(CONFIG_USER_ONLY)
840 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
842 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
);
844 static void *(*phys_mem_alloc
)(size_t size
) = qemu_anon_ram_alloc
;
847 * Set a custom physical guest memory alloator.
848 * Accelerators with unusual needs may need this. Hopefully, we can
849 * get rid of it eventually.
851 void phys_mem_set_alloc(void *(*alloc
)(size_t))
853 phys_mem_alloc
= alloc
;
856 static uint16_t phys_section_add(PhysPageMap
*map
,
857 MemoryRegionSection
*section
)
859 /* The physical section number is ORed with a page-aligned
860 * pointer to produce the iotlb entries. Thus it should
861 * never overflow into the page-aligned value.
863 assert(map
->sections_nb
< TARGET_PAGE_SIZE
);
865 if (map
->sections_nb
== map
->sections_nb_alloc
) {
866 map
->sections_nb_alloc
= MAX(map
->sections_nb_alloc
* 2, 16);
867 map
->sections
= g_renew(MemoryRegionSection
, map
->sections
,
868 map
->sections_nb_alloc
);
870 map
->sections
[map
->sections_nb
] = *section
;
871 memory_region_ref(section
->mr
);
872 return map
->sections_nb
++;
875 static void phys_section_destroy(MemoryRegion
*mr
)
877 memory_region_unref(mr
);
880 subpage_t
*subpage
= container_of(mr
, subpage_t
, iomem
);
881 memory_region_destroy(&subpage
->iomem
);
886 static void phys_sections_free(PhysPageMap
*map
)
888 while (map
->sections_nb
> 0) {
889 MemoryRegionSection
*section
= &map
->sections
[--map
->sections_nb
];
890 phys_section_destroy(section
->mr
);
892 g_free(map
->sections
);
896 static void register_subpage(AddressSpaceDispatch
*d
, MemoryRegionSection
*section
)
899 hwaddr base
= section
->offset_within_address_space
901 MemoryRegionSection
*existing
= phys_page_find(d
->phys_map
, base
,
902 d
->map
.nodes
, d
->map
.sections
);
903 MemoryRegionSection subsection
= {
904 .offset_within_address_space
= base
,
905 .size
= int128_make64(TARGET_PAGE_SIZE
),
909 assert(existing
->mr
->subpage
|| existing
->mr
== &io_mem_unassigned
);
911 if (!(existing
->mr
->subpage
)) {
912 subpage
= subpage_init(d
->as
, base
);
913 subsection
.address_space
= d
->as
;
914 subsection
.mr
= &subpage
->iomem
;
915 phys_page_set(d
, base
>> TARGET_PAGE_BITS
, 1,
916 phys_section_add(&d
->map
, &subsection
));
918 subpage
= container_of(existing
->mr
, subpage_t
, iomem
);
920 start
= section
->offset_within_address_space
& ~TARGET_PAGE_MASK
;
921 end
= start
+ int128_get64(section
->size
) - 1;
922 subpage_register(subpage
, start
, end
,
923 phys_section_add(&d
->map
, section
));
927 static void register_multipage(AddressSpaceDispatch
*d
,
928 MemoryRegionSection
*section
)
930 hwaddr start_addr
= section
->offset_within_address_space
;
931 uint16_t section_index
= phys_section_add(&d
->map
, section
);
932 uint64_t num_pages
= int128_get64(int128_rshift(section
->size
,
936 phys_page_set(d
, start_addr
>> TARGET_PAGE_BITS
, num_pages
, section_index
);
939 static void mem_add(MemoryListener
*listener
, MemoryRegionSection
*section
)
941 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
942 AddressSpaceDispatch
*d
= as
->next_dispatch
;
943 MemoryRegionSection now
= *section
, remain
= *section
;
944 Int128 page_size
= int128_make64(TARGET_PAGE_SIZE
);
946 if (now
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
947 uint64_t left
= TARGET_PAGE_ALIGN(now
.offset_within_address_space
)
948 - now
.offset_within_address_space
;
950 now
.size
= int128_min(int128_make64(left
), now
.size
);
951 register_subpage(d
, &now
);
953 now
.size
= int128_zero();
955 while (int128_ne(remain
.size
, now
.size
)) {
956 remain
.size
= int128_sub(remain
.size
, now
.size
);
957 remain
.offset_within_address_space
+= int128_get64(now
.size
);
958 remain
.offset_within_region
+= int128_get64(now
.size
);
960 if (int128_lt(remain
.size
, page_size
)) {
961 register_subpage(d
, &now
);
962 } else if (remain
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
963 now
.size
= page_size
;
964 register_subpage(d
, &now
);
966 now
.size
= int128_and(now
.size
, int128_neg(page_size
));
967 register_multipage(d
, &now
);
972 void qemu_flush_coalesced_mmio_buffer(void)
975 kvm_flush_coalesced_mmio_buffer();
978 void qemu_mutex_lock_ramlist(void)
980 qemu_mutex_lock(&ram_list
.mutex
);
983 void qemu_mutex_unlock_ramlist(void)
985 qemu_mutex_unlock(&ram_list
.mutex
);
992 #define HUGETLBFS_MAGIC 0x958458f6
994 static long gethugepagesize(const char *path
)
1000 ret
= statfs(path
, &fs
);
1001 } while (ret
!= 0 && errno
== EINTR
);
1008 if (fs
.f_type
!= HUGETLBFS_MAGIC
)
1009 fprintf(stderr
, "Warning: path not on HugeTLBFS: %s\n", path
);
1014 static void *file_ram_alloc(RAMBlock
*block
,
1019 char *sanitized_name
;
1023 unsigned long hpagesize
;
1025 hpagesize
= gethugepagesize(path
);
1030 if (memory
< hpagesize
) {
1034 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1035 fprintf(stderr
, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
1039 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1040 sanitized_name
= g_strdup(block
->mr
->name
);
1041 for (c
= sanitized_name
; *c
!= '\0'; c
++) {
1046 filename
= g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path
,
1048 g_free(sanitized_name
);
1050 fd
= mkstemp(filename
);
1052 perror("unable to create backing store for hugepages");
1059 memory
= (memory
+hpagesize
-1) & ~(hpagesize
-1);
1062 * ftruncate is not supported by hugetlbfs in older
1063 * hosts, so don't bother bailing out on errors.
1064 * If anything goes wrong with it under other filesystems,
1067 if (ftruncate(fd
, memory
))
1068 perror("ftruncate");
1070 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, MAP_PRIVATE
, fd
, 0);
1071 if (area
== MAP_FAILED
) {
1072 perror("file_ram_alloc: can't mmap RAM pages");
1078 os_mem_prealloc(fd
, area
, memory
);
1091 static void *file_ram_alloc(RAMBlock
*block
,
1095 fprintf(stderr
, "-mem-path not supported on this host\n");
1100 static ram_addr_t
find_ram_offset(ram_addr_t size
)
1102 RAMBlock
*block
, *next_block
;
1103 ram_addr_t offset
= RAM_ADDR_MAX
, mingap
= RAM_ADDR_MAX
;
1105 assert(size
!= 0); /* it would hand out same offset multiple times */
1107 if (QTAILQ_EMPTY(&ram_list
.blocks
))
1110 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1111 ram_addr_t end
, next
= RAM_ADDR_MAX
;
1113 end
= block
->offset
+ block
->length
;
1115 QTAILQ_FOREACH(next_block
, &ram_list
.blocks
, next
) {
1116 if (next_block
->offset
>= end
) {
1117 next
= MIN(next
, next_block
->offset
);
1120 if (next
- end
>= size
&& next
- end
< mingap
) {
1122 mingap
= next
- end
;
1126 if (offset
== RAM_ADDR_MAX
) {
1127 fprintf(stderr
, "Failed to find gap of requested size: %" PRIu64
"\n",
1135 ram_addr_t
last_ram_offset(void)
1138 ram_addr_t last
= 0;
1140 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
)
1141 last
= MAX(last
, block
->offset
+ block
->length
);
1146 static void qemu_ram_setup_dump(void *addr
, ram_addr_t size
)
1150 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1151 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1152 "dump-guest-core", true)) {
1153 ret
= qemu_madvise(addr
, size
, QEMU_MADV_DONTDUMP
);
1155 perror("qemu_madvise");
1156 fprintf(stderr
, "madvise doesn't support MADV_DONTDUMP, "
1157 "but dump_guest_core=off specified\n");
1162 static RAMBlock
*find_ram_block(ram_addr_t addr
)
1166 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1167 if (block
->offset
== addr
) {
1175 void qemu_ram_set_idstr(ram_addr_t addr
, const char *name
, DeviceState
*dev
)
1177 RAMBlock
*new_block
= find_ram_block(addr
);
1181 assert(!new_block
->idstr
[0]);
1184 char *id
= qdev_get_dev_path(dev
);
1186 snprintf(new_block
->idstr
, sizeof(new_block
->idstr
), "%s/", id
);
1190 pstrcat(new_block
->idstr
, sizeof(new_block
->idstr
), name
);
1192 /* This assumes the iothread lock is taken here too. */
1193 qemu_mutex_lock_ramlist();
1194 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1195 if (block
!= new_block
&& !strcmp(block
->idstr
, new_block
->idstr
)) {
1196 fprintf(stderr
, "RAMBlock \"%s\" already registered, abort!\n",
1201 qemu_mutex_unlock_ramlist();
1204 void qemu_ram_unset_idstr(ram_addr_t addr
)
1206 RAMBlock
*block
= find_ram_block(addr
);
1209 memset(block
->idstr
, 0, sizeof(block
->idstr
));
1213 static int memory_try_enable_merging(void *addr
, size_t len
)
1215 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1216 /* disabled by the user */
1220 return qemu_madvise(addr
, len
, QEMU_MADV_MERGEABLE
);
1223 static ram_addr_t
ram_block_add(RAMBlock
*new_block
)
1226 ram_addr_t old_ram_size
, new_ram_size
;
1228 old_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1230 /* This assumes the iothread lock is taken here too. */
1231 qemu_mutex_lock_ramlist();
1232 new_block
->offset
= find_ram_offset(new_block
->length
);
1234 if (!new_block
->host
) {
1235 if (xen_enabled()) {
1236 xen_ram_alloc(new_block
->offset
, new_block
->length
, new_block
->mr
);
1238 new_block
->host
= phys_mem_alloc(new_block
->length
);
1239 if (!new_block
->host
) {
1240 fprintf(stderr
, "Cannot set up guest memory '%s': %s\n",
1241 new_block
->mr
->name
, strerror(errno
));
1244 memory_try_enable_merging(new_block
->host
, new_block
->length
);
1248 /* Keep the list sorted from biggest to smallest block. */
1249 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1250 if (block
->length
< new_block
->length
) {
1255 QTAILQ_INSERT_BEFORE(block
, new_block
, next
);
1257 QTAILQ_INSERT_TAIL(&ram_list
.blocks
, new_block
, next
);
1259 ram_list
.mru_block
= NULL
;
1262 qemu_mutex_unlock_ramlist();
1264 new_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1266 if (new_ram_size
> old_ram_size
) {
1268 for (i
= 0; i
< DIRTY_MEMORY_NUM
; i
++) {
1269 ram_list
.dirty_memory
[i
] =
1270 bitmap_zero_extend(ram_list
.dirty_memory
[i
],
1271 old_ram_size
, new_ram_size
);
1274 cpu_physical_memory_set_dirty_range(new_block
->offset
, new_block
->length
);
1276 qemu_ram_setup_dump(new_block
->host
, new_block
->length
);
1277 qemu_madvise(new_block
->host
, new_block
->length
, QEMU_MADV_HUGEPAGE
);
1278 qemu_madvise(new_block
->host
, new_block
->length
, QEMU_MADV_DONTFORK
);
1280 if (kvm_enabled()) {
1281 kvm_setup_guest_memory(new_block
->host
, new_block
->length
);
1284 return new_block
->offset
;
1287 ram_addr_t
qemu_ram_alloc_from_file(ram_addr_t size
, MemoryRegion
*mr
,
1288 const char *mem_path
)
1290 RAMBlock
*new_block
;
1292 if (xen_enabled()) {
1293 fprintf(stderr
, "-mem-path not supported with Xen\n");
1297 if (phys_mem_alloc
!= qemu_anon_ram_alloc
) {
1299 * file_ram_alloc() needs to allocate just like
1300 * phys_mem_alloc, but we haven't bothered to provide
1304 "-mem-path not supported with this accelerator\n");
1308 size
= TARGET_PAGE_ALIGN(size
);
1309 new_block
= g_malloc0(sizeof(*new_block
));
1311 new_block
->length
= size
;
1312 new_block
->host
= file_ram_alloc(new_block
, size
, mem_path
);
1313 return ram_block_add(new_block
);
1316 ram_addr_t
qemu_ram_alloc_from_ptr(ram_addr_t size
, void *host
,
1319 RAMBlock
*new_block
;
1321 size
= TARGET_PAGE_ALIGN(size
);
1322 new_block
= g_malloc0(sizeof(*new_block
));
1324 new_block
->length
= size
;
1326 new_block
->host
= host
;
1328 new_block
->flags
|= RAM_PREALLOC_MASK
;
1330 return ram_block_add(new_block
);
1333 ram_addr_t
qemu_ram_alloc(ram_addr_t size
, MemoryRegion
*mr
)
1335 return qemu_ram_alloc_from_ptr(size
, NULL
, mr
);
1338 void qemu_ram_free_from_ptr(ram_addr_t addr
)
1342 /* This assumes the iothread lock is taken here too. */
1343 qemu_mutex_lock_ramlist();
1344 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1345 if (addr
== block
->offset
) {
1346 QTAILQ_REMOVE(&ram_list
.blocks
, block
, next
);
1347 ram_list
.mru_block
= NULL
;
1353 qemu_mutex_unlock_ramlist();
1356 void qemu_ram_free(ram_addr_t addr
)
1360 /* This assumes the iothread lock is taken here too. */
1361 qemu_mutex_lock_ramlist();
1362 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1363 if (addr
== block
->offset
) {
1364 QTAILQ_REMOVE(&ram_list
.blocks
, block
, next
);
1365 ram_list
.mru_block
= NULL
;
1367 if (block
->flags
& RAM_PREALLOC_MASK
) {
1369 } else if (xen_enabled()) {
1370 xen_invalidate_map_cache_entry(block
->host
);
1372 } else if (block
->fd
>= 0) {
1373 munmap(block
->host
, block
->length
);
1377 qemu_anon_ram_free(block
->host
, block
->length
);
1383 qemu_mutex_unlock_ramlist();
1388 void qemu_ram_remap(ram_addr_t addr
, ram_addr_t length
)
1395 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1396 offset
= addr
- block
->offset
;
1397 if (offset
< block
->length
) {
1398 vaddr
= block
->host
+ offset
;
1399 if (block
->flags
& RAM_PREALLOC_MASK
) {
1401 } else if (xen_enabled()) {
1405 munmap(vaddr
, length
);
1406 if (block
->fd
>= 0) {
1408 flags
|= mem_prealloc
? MAP_POPULATE
| MAP_SHARED
:
1411 flags
|= MAP_PRIVATE
;
1413 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1414 flags
, block
->fd
, offset
);
1417 * Remap needs to match alloc. Accelerators that
1418 * set phys_mem_alloc never remap. If they did,
1419 * we'd need a remap hook here.
1421 assert(phys_mem_alloc
== qemu_anon_ram_alloc
);
1423 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
1424 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1427 if (area
!= vaddr
) {
1428 fprintf(stderr
, "Could not remap addr: "
1429 RAM_ADDR_FMT
"@" RAM_ADDR_FMT
"\n",
1433 memory_try_enable_merging(vaddr
, length
);
1434 qemu_ram_setup_dump(vaddr
, length
);
1440 #endif /* !_WIN32 */
1442 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1443 With the exception of the softmmu code in this file, this should
1444 only be used for local memory (e.g. video ram) that the device owns,
1445 and knows it isn't going to access beyond the end of the block.
1447 It should not be used for general purpose DMA.
1448 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1450 void *qemu_get_ram_ptr(ram_addr_t addr
)
1452 RAMBlock
*block
= qemu_get_ram_block(addr
);
1454 if (xen_enabled()) {
1455 /* We need to check if the requested address is in the RAM
1456 * because we don't want to map the entire memory in QEMU.
1457 * In that case just map until the end of the page.
1459 if (block
->offset
== 0) {
1460 return xen_map_cache(addr
, 0, 0);
1461 } else if (block
->host
== NULL
) {
1463 xen_map_cache(block
->offset
, block
->length
, 1);
1466 return block
->host
+ (addr
- block
->offset
);
1469 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1470 * but takes a size argument */
1471 static void *qemu_ram_ptr_length(ram_addr_t addr
, hwaddr
*size
)
1476 if (xen_enabled()) {
1477 return xen_map_cache(addr
, *size
, 1);
1481 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1482 if (addr
- block
->offset
< block
->length
) {
1483 if (addr
- block
->offset
+ *size
> block
->length
)
1484 *size
= block
->length
- addr
+ block
->offset
;
1485 return block
->host
+ (addr
- block
->offset
);
1489 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
1494 /* Some of the softmmu routines need to translate from a host pointer
1495 (typically a TLB entry) back to a ram offset. */
1496 MemoryRegion
*qemu_ram_addr_from_host(void *ptr
, ram_addr_t
*ram_addr
)
1499 uint8_t *host
= ptr
;
1501 if (xen_enabled()) {
1502 *ram_addr
= xen_ram_addr_from_mapcache(ptr
);
1503 return qemu_get_ram_block(*ram_addr
)->mr
;
1506 block
= ram_list
.mru_block
;
1507 if (block
&& block
->host
&& host
- block
->host
< block
->length
) {
1511 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1512 /* This case append when the block is not mapped. */
1513 if (block
->host
== NULL
) {
1516 if (host
- block
->host
< block
->length
) {
1524 *ram_addr
= block
->offset
+ (host
- block
->host
);
1528 static void notdirty_mem_write(void *opaque
, hwaddr ram_addr
,
1529 uint64_t val
, unsigned size
)
1531 if (!cpu_physical_memory_get_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
)) {
1532 tb_invalidate_phys_page_fast(ram_addr
, size
);
1536 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
1539 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
1542 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
1547 cpu_physical_memory_set_dirty_flag(ram_addr
, DIRTY_MEMORY_MIGRATION
);
1548 cpu_physical_memory_set_dirty_flag(ram_addr
, DIRTY_MEMORY_VGA
);
1549 /* we remove the notdirty callback only if the code has been
1551 if (!cpu_physical_memory_is_clean(ram_addr
)) {
1552 CPUArchState
*env
= current_cpu
->env_ptr
;
1553 tlb_set_dirty(env
, current_cpu
->mem_io_vaddr
);
1557 static bool notdirty_mem_accepts(void *opaque
, hwaddr addr
,
1558 unsigned size
, bool is_write
)
1563 static const MemoryRegionOps notdirty_mem_ops
= {
1564 .write
= notdirty_mem_write
,
1565 .valid
.accepts
= notdirty_mem_accepts
,
1566 .endianness
= DEVICE_NATIVE_ENDIAN
,
1569 /* Generate a debug exception if a watchpoint has been hit. */
1570 static void check_watchpoint(int offset
, int len_mask
, int flags
)
1572 CPUState
*cpu
= current_cpu
;
1573 CPUArchState
*env
= cpu
->env_ptr
;
1574 target_ulong pc
, cs_base
;
1579 if (cpu
->watchpoint_hit
) {
1580 /* We re-entered the check after replacing the TB. Now raise
1581 * the debug interrupt so that is will trigger after the
1582 * current instruction. */
1583 cpu_interrupt(cpu
, CPU_INTERRUPT_DEBUG
);
1586 vaddr
= (cpu
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
1587 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
1588 if ((vaddr
== (wp
->vaddr
& len_mask
) ||
1589 (vaddr
& wp
->len_mask
) == wp
->vaddr
) && (wp
->flags
& flags
)) {
1590 wp
->flags
|= BP_WATCHPOINT_HIT
;
1591 if (!cpu
->watchpoint_hit
) {
1592 cpu
->watchpoint_hit
= wp
;
1593 tb_check_watchpoint(cpu
);
1594 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
1595 cpu
->exception_index
= EXCP_DEBUG
;
1598 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
1599 tb_gen_code(cpu
, pc
, cs_base
, cpu_flags
, 1);
1600 cpu_resume_from_signal(cpu
, NULL
);
1604 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
1609 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1610 so these check for a hit then pass through to the normal out-of-line
1612 static uint64_t watch_mem_read(void *opaque
, hwaddr addr
,
1615 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~(size
- 1), BP_MEM_READ
);
1617 case 1: return ldub_phys(&address_space_memory
, addr
);
1618 case 2: return lduw_phys(&address_space_memory
, addr
);
1619 case 4: return ldl_phys(&address_space_memory
, addr
);
1624 static void watch_mem_write(void *opaque
, hwaddr addr
,
1625 uint64_t val
, unsigned size
)
1627 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~(size
- 1), BP_MEM_WRITE
);
1630 stb_phys(&address_space_memory
, addr
, val
);
1633 stw_phys(&address_space_memory
, addr
, val
);
1636 stl_phys(&address_space_memory
, addr
, val
);
1642 static const MemoryRegionOps watch_mem_ops
= {
1643 .read
= watch_mem_read
,
1644 .write
= watch_mem_write
,
1645 .endianness
= DEVICE_NATIVE_ENDIAN
,
1648 static uint64_t subpage_read(void *opaque
, hwaddr addr
,
1651 subpage_t
*subpage
= opaque
;
1654 #if defined(DEBUG_SUBPAGE)
1655 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
"\n", __func__
,
1656 subpage
, len
, addr
);
1658 address_space_read(subpage
->as
, addr
+ subpage
->base
, buf
, len
);
1671 static void subpage_write(void *opaque
, hwaddr addr
,
1672 uint64_t value
, unsigned len
)
1674 subpage_t
*subpage
= opaque
;
1677 #if defined(DEBUG_SUBPAGE)
1678 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
1679 " value %"PRIx64
"\n",
1680 __func__
, subpage
, len
, addr
, value
);
1695 address_space_write(subpage
->as
, addr
+ subpage
->base
, buf
, len
);
1698 static bool subpage_accepts(void *opaque
, hwaddr addr
,
1699 unsigned len
, bool is_write
)
1701 subpage_t
*subpage
= opaque
;
1702 #if defined(DEBUG_SUBPAGE)
1703 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx
"\n",
1704 __func__
, subpage
, is_write
? 'w' : 'r', len
, addr
);
1707 return address_space_access_valid(subpage
->as
, addr
+ subpage
->base
,
1711 static const MemoryRegionOps subpage_ops
= {
1712 .read
= subpage_read
,
1713 .write
= subpage_write
,
1714 .valid
.accepts
= subpage_accepts
,
1715 .endianness
= DEVICE_NATIVE_ENDIAN
,
1718 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
1723 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
1725 idx
= SUBPAGE_IDX(start
);
1726 eidx
= SUBPAGE_IDX(end
);
1727 #if defined(DEBUG_SUBPAGE)
1728 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
1729 __func__
, mmio
, start
, end
, idx
, eidx
, section
);
1731 for (; idx
<= eidx
; idx
++) {
1732 mmio
->sub_section
[idx
] = section
;
1738 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
)
1742 mmio
= g_malloc0(sizeof(subpage_t
));
1746 memory_region_init_io(&mmio
->iomem
, NULL
, &subpage_ops
, mmio
,
1747 "subpage", TARGET_PAGE_SIZE
);
1748 mmio
->iomem
.subpage
= true;
1749 #if defined(DEBUG_SUBPAGE)
1750 printf("%s: %p base " TARGET_FMT_plx
" len %08x\n", __func__
,
1751 mmio
, base
, TARGET_PAGE_SIZE
);
1753 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
-1, PHYS_SECTION_UNASSIGNED
);
1758 static uint16_t dummy_section(PhysPageMap
*map
, AddressSpace
*as
,
1762 MemoryRegionSection section
= {
1763 .address_space
= as
,
1765 .offset_within_address_space
= 0,
1766 .offset_within_region
= 0,
1767 .size
= int128_2_64(),
1770 return phys_section_add(map
, §ion
);
1773 MemoryRegion
*iotlb_to_region(AddressSpace
*as
, hwaddr index
)
1775 return as
->dispatch
->map
.sections
[index
& ~TARGET_PAGE_MASK
].mr
;
1778 static void io_mem_init(void)
1780 memory_region_init_io(&io_mem_rom
, NULL
, &unassigned_mem_ops
, NULL
, "rom", UINT64_MAX
);
1781 memory_region_init_io(&io_mem_unassigned
, NULL
, &unassigned_mem_ops
, NULL
,
1782 "unassigned", UINT64_MAX
);
1783 memory_region_init_io(&io_mem_notdirty
, NULL
, ¬dirty_mem_ops
, NULL
,
1784 "notdirty", UINT64_MAX
);
1785 memory_region_init_io(&io_mem_watch
, NULL
, &watch_mem_ops
, NULL
,
1786 "watch", UINT64_MAX
);
1789 static void mem_begin(MemoryListener
*listener
)
1791 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
1792 AddressSpaceDispatch
*d
= g_new0(AddressSpaceDispatch
, 1);
1795 n
= dummy_section(&d
->map
, as
, &io_mem_unassigned
);
1796 assert(n
== PHYS_SECTION_UNASSIGNED
);
1797 n
= dummy_section(&d
->map
, as
, &io_mem_notdirty
);
1798 assert(n
== PHYS_SECTION_NOTDIRTY
);
1799 n
= dummy_section(&d
->map
, as
, &io_mem_rom
);
1800 assert(n
== PHYS_SECTION_ROM
);
1801 n
= dummy_section(&d
->map
, as
, &io_mem_watch
);
1802 assert(n
== PHYS_SECTION_WATCH
);
1804 d
->phys_map
= (PhysPageEntry
) { .ptr
= PHYS_MAP_NODE_NIL
, .skip
= 1 };
1806 as
->next_dispatch
= d
;
1809 static void mem_commit(MemoryListener
*listener
)
1811 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
1812 AddressSpaceDispatch
*cur
= as
->dispatch
;
1813 AddressSpaceDispatch
*next
= as
->next_dispatch
;
1815 phys_page_compact_all(next
, next
->map
.nodes_nb
);
1817 as
->dispatch
= next
;
1820 phys_sections_free(&cur
->map
);
1825 static void tcg_commit(MemoryListener
*listener
)
1829 /* since each CPU stores ram addresses in its TLB cache, we must
1830 reset the modified entries */
1833 /* FIXME: Disentangle the cpu.h circular files deps so we can
1834 directly get the right CPU from listener. */
1835 if (cpu
->tcg_as_listener
!= listener
) {
1842 static void core_log_global_start(MemoryListener
*listener
)
1844 cpu_physical_memory_set_dirty_tracking(true);
1847 static void core_log_global_stop(MemoryListener
*listener
)
1849 cpu_physical_memory_set_dirty_tracking(false);
1852 static MemoryListener core_memory_listener
= {
1853 .log_global_start
= core_log_global_start
,
1854 .log_global_stop
= core_log_global_stop
,
1858 void address_space_init_dispatch(AddressSpace
*as
)
1860 as
->dispatch
= NULL
;
1861 as
->dispatch_listener
= (MemoryListener
) {
1863 .commit
= mem_commit
,
1864 .region_add
= mem_add
,
1865 .region_nop
= mem_add
,
1868 memory_listener_register(&as
->dispatch_listener
, as
);
1871 void address_space_destroy_dispatch(AddressSpace
*as
)
1873 AddressSpaceDispatch
*d
= as
->dispatch
;
1875 memory_listener_unregister(&as
->dispatch_listener
);
1877 as
->dispatch
= NULL
;
1880 static void memory_map_init(void)
1882 system_memory
= g_malloc(sizeof(*system_memory
));
1884 memory_region_init(system_memory
, NULL
, "system", UINT64_MAX
);
1885 address_space_init(&address_space_memory
, system_memory
, "memory");
1887 system_io
= g_malloc(sizeof(*system_io
));
1888 memory_region_init_io(system_io
, NULL
, &unassigned_io_ops
, NULL
, "io",
1890 address_space_init(&address_space_io
, system_io
, "I/O");
1892 memory_listener_register(&core_memory_listener
, &address_space_memory
);
1895 MemoryRegion
*get_system_memory(void)
1897 return system_memory
;
1900 MemoryRegion
*get_system_io(void)
1905 #endif /* !defined(CONFIG_USER_ONLY) */
1907 /* physical memory access (slow version, mainly for debug) */
1908 #if defined(CONFIG_USER_ONLY)
1909 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
1910 uint8_t *buf
, int len
, int is_write
)
1917 page
= addr
& TARGET_PAGE_MASK
;
1918 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
1921 flags
= page_get_flags(page
);
1922 if (!(flags
& PAGE_VALID
))
1925 if (!(flags
& PAGE_WRITE
))
1927 /* XXX: this code should not depend on lock_user */
1928 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
1931 unlock_user(p
, addr
, l
);
1933 if (!(flags
& PAGE_READ
))
1935 /* XXX: this code should not depend on lock_user */
1936 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
1939 unlock_user(p
, addr
, 0);
1950 static void invalidate_and_set_dirty(hwaddr addr
,
1953 if (cpu_physical_memory_is_clean(addr
)) {
1954 /* invalidate code */
1955 tb_invalidate_phys_page_range(addr
, addr
+ length
, 0);
1957 cpu_physical_memory_set_dirty_flag(addr
, DIRTY_MEMORY_VGA
);
1958 cpu_physical_memory_set_dirty_flag(addr
, DIRTY_MEMORY_MIGRATION
);
1960 xen_modified_memory(addr
, length
);
1963 static int memory_access_size(MemoryRegion
*mr
, unsigned l
, hwaddr addr
)
1965 unsigned access_size_max
= mr
->ops
->valid
.max_access_size
;
1967 /* Regions are assumed to support 1-4 byte accesses unless
1968 otherwise specified. */
1969 if (access_size_max
== 0) {
1970 access_size_max
= 4;
1973 /* Bound the maximum access by the alignment of the address. */
1974 if (!mr
->ops
->impl
.unaligned
) {
1975 unsigned align_size_max
= addr
& -addr
;
1976 if (align_size_max
!= 0 && align_size_max
< access_size_max
) {
1977 access_size_max
= align_size_max
;
1981 /* Don't attempt accesses larger than the maximum. */
1982 if (l
> access_size_max
) {
1983 l
= access_size_max
;
1986 l
= 1 << (qemu_fls(l
) - 1);
1992 bool address_space_rw(AddressSpace
*as
, hwaddr addr
, uint8_t *buf
,
1993 int len
, bool is_write
)
2004 mr
= address_space_translate(as
, addr
, &addr1
, &l
, is_write
);
2007 if (!memory_access_is_direct(mr
, is_write
)) {
2008 l
= memory_access_size(mr
, l
, addr1
);
2009 /* XXX: could force current_cpu to NULL to avoid
2013 /* 64 bit write access */
2015 error
|= io_mem_write(mr
, addr1
, val
, 8);
2018 /* 32 bit write access */
2020 error
|= io_mem_write(mr
, addr1
, val
, 4);
2023 /* 16 bit write access */
2025 error
|= io_mem_write(mr
, addr1
, val
, 2);
2028 /* 8 bit write access */
2030 error
|= io_mem_write(mr
, addr1
, val
, 1);
2036 addr1
+= memory_region_get_ram_addr(mr
);
2038 ptr
= qemu_get_ram_ptr(addr1
);
2039 memcpy(ptr
, buf
, l
);
2040 invalidate_and_set_dirty(addr1
, l
);
2043 if (!memory_access_is_direct(mr
, is_write
)) {
2045 l
= memory_access_size(mr
, l
, addr1
);
2048 /* 64 bit read access */
2049 error
|= io_mem_read(mr
, addr1
, &val
, 8);
2053 /* 32 bit read access */
2054 error
|= io_mem_read(mr
, addr1
, &val
, 4);
2058 /* 16 bit read access */
2059 error
|= io_mem_read(mr
, addr1
, &val
, 2);
2063 /* 8 bit read access */
2064 error
|= io_mem_read(mr
, addr1
, &val
, 1);
2072 ptr
= qemu_get_ram_ptr(mr
->ram_addr
+ addr1
);
2073 memcpy(buf
, ptr
, l
);
2084 bool address_space_write(AddressSpace
*as
, hwaddr addr
,
2085 const uint8_t *buf
, int len
)
2087 return address_space_rw(as
, addr
, (uint8_t *)buf
, len
, true);
2090 bool address_space_read(AddressSpace
*as
, hwaddr addr
, uint8_t *buf
, int len
)
2092 return address_space_rw(as
, addr
, buf
, len
, false);
2096 void cpu_physical_memory_rw(hwaddr addr
, uint8_t *buf
,
2097 int len
, int is_write
)
2099 address_space_rw(&address_space_memory
, addr
, buf
, len
, is_write
);
2102 enum write_rom_type
{
2107 static inline void cpu_physical_memory_write_rom_internal(AddressSpace
*as
,
2108 hwaddr addr
, const uint8_t *buf
, int len
, enum write_rom_type type
)
2117 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
2119 if (!(memory_region_is_ram(mr
) ||
2120 memory_region_is_romd(mr
))) {
2123 addr1
+= memory_region_get_ram_addr(mr
);
2125 ptr
= qemu_get_ram_ptr(addr1
);
2128 memcpy(ptr
, buf
, l
);
2129 invalidate_and_set_dirty(addr1
, l
);
2132 flush_icache_range((uintptr_t)ptr
, (uintptr_t)ptr
+ l
);
2142 /* used for ROM loading : can write in RAM and ROM */
2143 void cpu_physical_memory_write_rom(AddressSpace
*as
, hwaddr addr
,
2144 const uint8_t *buf
, int len
)
2146 cpu_physical_memory_write_rom_internal(as
, addr
, buf
, len
, WRITE_DATA
);
2149 void cpu_flush_icache_range(hwaddr start
, int len
)
2152 * This function should do the same thing as an icache flush that was
2153 * triggered from within the guest. For TCG we are always cache coherent,
2154 * so there is no need to flush anything. For KVM / Xen we need to flush
2155 * the host's instruction cache at least.
2157 if (tcg_enabled()) {
2161 cpu_physical_memory_write_rom_internal(&address_space_memory
,
2162 start
, NULL
, len
, FLUSH_CACHE
);
2172 static BounceBuffer bounce
;
2174 typedef struct MapClient
{
2176 void (*callback
)(void *opaque
);
2177 QLIST_ENTRY(MapClient
) link
;
2180 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
2181 = QLIST_HEAD_INITIALIZER(map_client_list
);
2183 void *cpu_register_map_client(void *opaque
, void (*callback
)(void *opaque
))
2185 MapClient
*client
= g_malloc(sizeof(*client
));
2187 client
->opaque
= opaque
;
2188 client
->callback
= callback
;
2189 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
2193 static void cpu_unregister_map_client(void *_client
)
2195 MapClient
*client
= (MapClient
*)_client
;
2197 QLIST_REMOVE(client
, link
);
2201 static void cpu_notify_map_clients(void)
2205 while (!QLIST_EMPTY(&map_client_list
)) {
2206 client
= QLIST_FIRST(&map_client_list
);
2207 client
->callback(client
->opaque
);
2208 cpu_unregister_map_client(client
);
2212 bool address_space_access_valid(AddressSpace
*as
, hwaddr addr
, int len
, bool is_write
)
2219 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2220 if (!memory_access_is_direct(mr
, is_write
)) {
2221 l
= memory_access_size(mr
, l
, addr
);
2222 if (!memory_region_access_valid(mr
, xlat
, l
, is_write
)) {
2233 /* Map a physical memory region into a host virtual address.
2234 * May map a subset of the requested range, given by and returned in *plen.
2235 * May return NULL if resources needed to perform the mapping are exhausted.
2236 * Use only for reads OR writes - not for read-modify-write operations.
2237 * Use cpu_register_map_client() to know when retrying the map operation is
2238 * likely to succeed.
2240 void *address_space_map(AddressSpace
*as
,
2247 hwaddr l
, xlat
, base
;
2248 MemoryRegion
*mr
, *this_mr
;
2256 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2257 if (!memory_access_is_direct(mr
, is_write
)) {
2258 if (bounce
.buffer
) {
2261 /* Avoid unbounded allocations */
2262 l
= MIN(l
, TARGET_PAGE_SIZE
);
2263 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, l
);
2267 memory_region_ref(mr
);
2270 address_space_read(as
, addr
, bounce
.buffer
, l
);
2274 return bounce
.buffer
;
2278 raddr
= memory_region_get_ram_addr(mr
);
2289 this_mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2290 if (this_mr
!= mr
|| xlat
!= base
+ done
) {
2295 memory_region_ref(mr
);
2297 return qemu_ram_ptr_length(raddr
+ base
, plen
);
2300 /* Unmaps a memory region previously mapped by address_space_map().
2301 * Will also mark the memory as dirty if is_write == 1. access_len gives
2302 * the amount of memory that was actually read or written by the caller.
2304 void address_space_unmap(AddressSpace
*as
, void *buffer
, hwaddr len
,
2305 int is_write
, hwaddr access_len
)
2307 if (buffer
!= bounce
.buffer
) {
2311 mr
= qemu_ram_addr_from_host(buffer
, &addr1
);
2314 while (access_len
) {
2316 l
= TARGET_PAGE_SIZE
;
2319 invalidate_and_set_dirty(addr1
, l
);
2324 if (xen_enabled()) {
2325 xen_invalidate_map_cache_entry(buffer
);
2327 memory_region_unref(mr
);
2331 address_space_write(as
, bounce
.addr
, bounce
.buffer
, access_len
);
2333 qemu_vfree(bounce
.buffer
);
2334 bounce
.buffer
= NULL
;
2335 memory_region_unref(bounce
.mr
);
2336 cpu_notify_map_clients();
2339 void *cpu_physical_memory_map(hwaddr addr
,
2343 return address_space_map(&address_space_memory
, addr
, plen
, is_write
);
2346 void cpu_physical_memory_unmap(void *buffer
, hwaddr len
,
2347 int is_write
, hwaddr access_len
)
2349 return address_space_unmap(&address_space_memory
, buffer
, len
, is_write
, access_len
);
2352 /* warning: addr must be aligned */
2353 static inline uint32_t ldl_phys_internal(AddressSpace
*as
, hwaddr addr
,
2354 enum device_endian endian
)
2362 mr
= address_space_translate(as
, addr
, &addr1
, &l
, false);
2363 if (l
< 4 || !memory_access_is_direct(mr
, false)) {
2365 io_mem_read(mr
, addr1
, &val
, 4);
2366 #if defined(TARGET_WORDS_BIGENDIAN)
2367 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2371 if (endian
== DEVICE_BIG_ENDIAN
) {
2377 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2381 case DEVICE_LITTLE_ENDIAN
:
2382 val
= ldl_le_p(ptr
);
2384 case DEVICE_BIG_ENDIAN
:
2385 val
= ldl_be_p(ptr
);
2395 uint32_t ldl_phys(AddressSpace
*as
, hwaddr addr
)
2397 return ldl_phys_internal(as
, addr
, DEVICE_NATIVE_ENDIAN
);
2400 uint32_t ldl_le_phys(AddressSpace
*as
, hwaddr addr
)
2402 return ldl_phys_internal(as
, addr
, DEVICE_LITTLE_ENDIAN
);
2405 uint32_t ldl_be_phys(AddressSpace
*as
, hwaddr addr
)
2407 return ldl_phys_internal(as
, addr
, DEVICE_BIG_ENDIAN
);
2410 /* warning: addr must be aligned */
2411 static inline uint64_t ldq_phys_internal(AddressSpace
*as
, hwaddr addr
,
2412 enum device_endian endian
)
2420 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2422 if (l
< 8 || !memory_access_is_direct(mr
, false)) {
2424 io_mem_read(mr
, addr1
, &val
, 8);
2425 #if defined(TARGET_WORDS_BIGENDIAN)
2426 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2430 if (endian
== DEVICE_BIG_ENDIAN
) {
2436 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2440 case DEVICE_LITTLE_ENDIAN
:
2441 val
= ldq_le_p(ptr
);
2443 case DEVICE_BIG_ENDIAN
:
2444 val
= ldq_be_p(ptr
);
2454 uint64_t ldq_phys(AddressSpace
*as
, hwaddr addr
)
2456 return ldq_phys_internal(as
, addr
, DEVICE_NATIVE_ENDIAN
);
2459 uint64_t ldq_le_phys(AddressSpace
*as
, hwaddr addr
)
2461 return ldq_phys_internal(as
, addr
, DEVICE_LITTLE_ENDIAN
);
2464 uint64_t ldq_be_phys(AddressSpace
*as
, hwaddr addr
)
2466 return ldq_phys_internal(as
, addr
, DEVICE_BIG_ENDIAN
);
2470 uint32_t ldub_phys(AddressSpace
*as
, hwaddr addr
)
2473 address_space_rw(as
, addr
, &val
, 1, 0);
2477 /* warning: addr must be aligned */
2478 static inline uint32_t lduw_phys_internal(AddressSpace
*as
, hwaddr addr
,
2479 enum device_endian endian
)
2487 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2489 if (l
< 2 || !memory_access_is_direct(mr
, false)) {
2491 io_mem_read(mr
, addr1
, &val
, 2);
2492 #if defined(TARGET_WORDS_BIGENDIAN)
2493 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2497 if (endian
== DEVICE_BIG_ENDIAN
) {
2503 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2507 case DEVICE_LITTLE_ENDIAN
:
2508 val
= lduw_le_p(ptr
);
2510 case DEVICE_BIG_ENDIAN
:
2511 val
= lduw_be_p(ptr
);
2521 uint32_t lduw_phys(AddressSpace
*as
, hwaddr addr
)
2523 return lduw_phys_internal(as
, addr
, DEVICE_NATIVE_ENDIAN
);
2526 uint32_t lduw_le_phys(AddressSpace
*as
, hwaddr addr
)
2528 return lduw_phys_internal(as
, addr
, DEVICE_LITTLE_ENDIAN
);
2531 uint32_t lduw_be_phys(AddressSpace
*as
, hwaddr addr
)
2533 return lduw_phys_internal(as
, addr
, DEVICE_BIG_ENDIAN
);
2536 /* warning: addr must be aligned. The ram page is not masked as dirty
2537 and the code inside is not invalidated. It is useful if the dirty
2538 bits are used to track modified PTEs */
2539 void stl_phys_notdirty(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2546 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2548 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
2549 io_mem_write(mr
, addr1
, val
, 4);
2551 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
2552 ptr
= qemu_get_ram_ptr(addr1
);
2555 if (unlikely(in_migration
)) {
2556 if (cpu_physical_memory_is_clean(addr1
)) {
2557 /* invalidate code */
2558 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
2560 cpu_physical_memory_set_dirty_flag(addr1
,
2561 DIRTY_MEMORY_MIGRATION
);
2562 cpu_physical_memory_set_dirty_flag(addr1
, DIRTY_MEMORY_VGA
);
2568 /* warning: addr must be aligned */
2569 static inline void stl_phys_internal(AddressSpace
*as
,
2570 hwaddr addr
, uint32_t val
,
2571 enum device_endian endian
)
2578 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2580 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
2581 #if defined(TARGET_WORDS_BIGENDIAN)
2582 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2586 if (endian
== DEVICE_BIG_ENDIAN
) {
2590 io_mem_write(mr
, addr1
, val
, 4);
2593 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
2594 ptr
= qemu_get_ram_ptr(addr1
);
2596 case DEVICE_LITTLE_ENDIAN
:
2599 case DEVICE_BIG_ENDIAN
:
2606 invalidate_and_set_dirty(addr1
, 4);
2610 void stl_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2612 stl_phys_internal(as
, addr
, val
, DEVICE_NATIVE_ENDIAN
);
2615 void stl_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2617 stl_phys_internal(as
, addr
, val
, DEVICE_LITTLE_ENDIAN
);
2620 void stl_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2622 stl_phys_internal(as
, addr
, val
, DEVICE_BIG_ENDIAN
);
2626 void stb_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2629 address_space_rw(as
, addr
, &v
, 1, 1);
2632 /* warning: addr must be aligned */
2633 static inline void stw_phys_internal(AddressSpace
*as
,
2634 hwaddr addr
, uint32_t val
,
2635 enum device_endian endian
)
2642 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
2643 if (l
< 2 || !memory_access_is_direct(mr
, true)) {
2644 #if defined(TARGET_WORDS_BIGENDIAN)
2645 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2649 if (endian
== DEVICE_BIG_ENDIAN
) {
2653 io_mem_write(mr
, addr1
, val
, 2);
2656 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
2657 ptr
= qemu_get_ram_ptr(addr1
);
2659 case DEVICE_LITTLE_ENDIAN
:
2662 case DEVICE_BIG_ENDIAN
:
2669 invalidate_and_set_dirty(addr1
, 2);
2673 void stw_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2675 stw_phys_internal(as
, addr
, val
, DEVICE_NATIVE_ENDIAN
);
2678 void stw_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2680 stw_phys_internal(as
, addr
, val
, DEVICE_LITTLE_ENDIAN
);
2683 void stw_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2685 stw_phys_internal(as
, addr
, val
, DEVICE_BIG_ENDIAN
);
2689 void stq_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
2692 address_space_rw(as
, addr
, (void *) &val
, 8, 1);
2695 void stq_le_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
2697 val
= cpu_to_le64(val
);
2698 address_space_rw(as
, addr
, (void *) &val
, 8, 1);
2701 void stq_be_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
2703 val
= cpu_to_be64(val
);
2704 address_space_rw(as
, addr
, (void *) &val
, 8, 1);
2707 /* virtual memory access for debug (includes writing to ROM) */
2708 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
2709 uint8_t *buf
, int len
, int is_write
)
2716 page
= addr
& TARGET_PAGE_MASK
;
2717 phys_addr
= cpu_get_phys_page_debug(cpu
, page
);
2718 /* if no physical page mapped, return an error */
2719 if (phys_addr
== -1)
2721 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2724 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
2726 cpu_physical_memory_write_rom(cpu
->as
, phys_addr
, buf
, l
);
2728 address_space_rw(cpu
->as
, phys_addr
, buf
, l
, 0);
2738 #if !defined(CONFIG_USER_ONLY)
2741 * A helper function for the _utterly broken_ virtio device model to find out if
2742 * it's running on a big endian machine. Don't do this at home kids!
2744 bool virtio_is_big_endian(void);
2745 bool virtio_is_big_endian(void)
2747 #if defined(TARGET_WORDS_BIGENDIAN)
2756 #ifndef CONFIG_USER_ONLY
2757 bool cpu_physical_memory_is_io(hwaddr phys_addr
)
2762 mr
= address_space_translate(&address_space_memory
,
2763 phys_addr
, &phys_addr
, &l
, false);
2765 return !(memory_region_is_ram(mr
) ||
2766 memory_region_is_romd(mr
));
2769 void qemu_ram_foreach_block(RAMBlockIterFunc func
, void *opaque
)
2773 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
2774 func(block
->host
, block
->offset
, block
->length
, opaque
);