4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
23 #include <sys/types.h>
27 #include "qemu-common.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
47 #include "exec/cpu-all.h"
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
52 #include "exec/memory-internal.h"
53 #include "exec/ram_addr.h"
54 #include "qemu/cache-utils.h"
56 #include "qemu/range.h"
58 //#define DEBUG_SUBPAGE
60 #if !defined(CONFIG_USER_ONLY)
61 static bool in_migration
;
63 RAMList ram_list
= { .blocks
= QTAILQ_HEAD_INITIALIZER(ram_list
.blocks
) };
65 static MemoryRegion
*system_memory
;
66 static MemoryRegion
*system_io
;
68 AddressSpace address_space_io
;
69 AddressSpace address_space_memory
;
71 MemoryRegion io_mem_rom
, io_mem_notdirty
;
72 static MemoryRegion io_mem_unassigned
;
76 struct CPUTailQ cpus
= QTAILQ_HEAD_INITIALIZER(cpus
);
77 /* current CPU in the current thread. It is only valid inside
79 DEFINE_TLS(CPUState
*, current_cpu
);
80 /* 0 = Do not count executed instructions.
81 1 = Precise instruction counting.
82 2 = Adaptive rate instruction counting. */
85 #if !defined(CONFIG_USER_ONLY)
87 typedef struct PhysPageEntry PhysPageEntry
;
89 struct PhysPageEntry
{
90 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
92 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
96 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
98 /* Size of the L2 (and L3, etc) page tables. */
99 #define ADDR_SPACE_BITS 64
102 #define P_L2_SIZE (1 << P_L2_BITS)
104 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
106 typedef PhysPageEntry Node
[P_L2_SIZE
];
108 typedef struct PhysPageMap
{
109 unsigned sections_nb
;
110 unsigned sections_nb_alloc
;
112 unsigned nodes_nb_alloc
;
114 MemoryRegionSection
*sections
;
117 struct AddressSpaceDispatch
{
118 /* This is a multi-level map on the physical address space.
119 * The bottom level has pointers to MemoryRegionSections.
121 PhysPageEntry phys_map
;
126 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
127 typedef struct subpage_t
{
131 uint16_t sub_section
[TARGET_PAGE_SIZE
];
134 #define PHYS_SECTION_UNASSIGNED 0
135 #define PHYS_SECTION_NOTDIRTY 1
136 #define PHYS_SECTION_ROM 2
137 #define PHYS_SECTION_WATCH 3
139 static void io_mem_init(void);
140 static void memory_map_init(void);
142 static MemoryRegion io_mem_watch
;
145 #if !defined(CONFIG_USER_ONLY)
147 static void phys_map_node_reserve(PhysPageMap
*map
, unsigned nodes
)
149 if (map
->nodes_nb
+ nodes
> map
->nodes_nb_alloc
) {
150 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
* 2, 16);
151 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
, map
->nodes_nb
+ nodes
);
152 map
->nodes
= g_renew(Node
, map
->nodes
, map
->nodes_nb_alloc
);
156 static uint32_t phys_map_node_alloc(PhysPageMap
*map
)
161 ret
= map
->nodes_nb
++;
162 assert(ret
!= PHYS_MAP_NODE_NIL
);
163 assert(ret
!= map
->nodes_nb_alloc
);
164 for (i
= 0; i
< P_L2_SIZE
; ++i
) {
165 map
->nodes
[ret
][i
].skip
= 1;
166 map
->nodes
[ret
][i
].ptr
= PHYS_MAP_NODE_NIL
;
171 static void phys_page_set_level(PhysPageMap
*map
, PhysPageEntry
*lp
,
172 hwaddr
*index
, hwaddr
*nb
, uint16_t leaf
,
177 hwaddr step
= (hwaddr
)1 << (level
* P_L2_BITS
);
179 if (lp
->skip
&& lp
->ptr
== PHYS_MAP_NODE_NIL
) {
180 lp
->ptr
= phys_map_node_alloc(map
);
181 p
= map
->nodes
[lp
->ptr
];
183 for (i
= 0; i
< P_L2_SIZE
; i
++) {
185 p
[i
].ptr
= PHYS_SECTION_UNASSIGNED
;
189 p
= map
->nodes
[lp
->ptr
];
191 lp
= &p
[(*index
>> (level
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
193 while (*nb
&& lp
< &p
[P_L2_SIZE
]) {
194 if ((*index
& (step
- 1)) == 0 && *nb
>= step
) {
200 phys_page_set_level(map
, lp
, index
, nb
, leaf
, level
- 1);
206 static void phys_page_set(AddressSpaceDispatch
*d
,
207 hwaddr index
, hwaddr nb
,
210 /* Wildly overreserve - it doesn't matter much. */
211 phys_map_node_reserve(&d
->map
, 3 * P_L2_LEVELS
);
213 phys_page_set_level(&d
->map
, &d
->phys_map
, &index
, &nb
, leaf
, P_L2_LEVELS
- 1);
216 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
217 * and update our entry so we can skip it and go directly to the destination.
219 static void phys_page_compact(PhysPageEntry
*lp
, Node
*nodes
, unsigned long *compacted
)
221 unsigned valid_ptr
= P_L2_SIZE
;
226 if (lp
->ptr
== PHYS_MAP_NODE_NIL
) {
231 for (i
= 0; i
< P_L2_SIZE
; i
++) {
232 if (p
[i
].ptr
== PHYS_MAP_NODE_NIL
) {
239 phys_page_compact(&p
[i
], nodes
, compacted
);
243 /* We can only compress if there's only one child. */
248 assert(valid_ptr
< P_L2_SIZE
);
250 /* Don't compress if it won't fit in the # of bits we have. */
251 if (lp
->skip
+ p
[valid_ptr
].skip
>= (1 << 3)) {
255 lp
->ptr
= p
[valid_ptr
].ptr
;
256 if (!p
[valid_ptr
].skip
) {
257 /* If our only child is a leaf, make this a leaf. */
258 /* By design, we should have made this node a leaf to begin with so we
259 * should never reach here.
260 * But since it's so simple to handle this, let's do it just in case we
265 lp
->skip
+= p
[valid_ptr
].skip
;
269 static void phys_page_compact_all(AddressSpaceDispatch
*d
, int nodes_nb
)
271 DECLARE_BITMAP(compacted
, nodes_nb
);
273 if (d
->phys_map
.skip
) {
274 phys_page_compact(&d
->phys_map
, d
->map
.nodes
, compacted
);
278 static MemoryRegionSection
*phys_page_find(PhysPageEntry lp
, hwaddr addr
,
279 Node
*nodes
, MemoryRegionSection
*sections
)
282 hwaddr index
= addr
>> TARGET_PAGE_BITS
;
285 for (i
= P_L2_LEVELS
; lp
.skip
&& (i
-= lp
.skip
) >= 0;) {
286 if (lp
.ptr
== PHYS_MAP_NODE_NIL
) {
287 return §ions
[PHYS_SECTION_UNASSIGNED
];
290 lp
= p
[(index
>> (i
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
293 if (sections
[lp
.ptr
].size
.hi
||
294 range_covers_byte(sections
[lp
.ptr
].offset_within_address_space
,
295 sections
[lp
.ptr
].size
.lo
, addr
)) {
296 return §ions
[lp
.ptr
];
298 return §ions
[PHYS_SECTION_UNASSIGNED
];
302 bool memory_region_is_unassigned(MemoryRegion
*mr
)
304 return mr
!= &io_mem_rom
&& mr
!= &io_mem_notdirty
&& !mr
->rom_device
305 && mr
!= &io_mem_watch
;
308 static MemoryRegionSection
*address_space_lookup_region(AddressSpaceDispatch
*d
,
310 bool resolve_subpage
)
312 MemoryRegionSection
*section
;
315 section
= phys_page_find(d
->phys_map
, addr
, d
->map
.nodes
, d
->map
.sections
);
316 if (resolve_subpage
&& section
->mr
->subpage
) {
317 subpage
= container_of(section
->mr
, subpage_t
, iomem
);
318 section
= &d
->map
.sections
[subpage
->sub_section
[SUBPAGE_IDX(addr
)]];
323 static MemoryRegionSection
*
324 address_space_translate_internal(AddressSpaceDispatch
*d
, hwaddr addr
, hwaddr
*xlat
,
325 hwaddr
*plen
, bool resolve_subpage
)
327 MemoryRegionSection
*section
;
330 section
= address_space_lookup_region(d
, addr
, resolve_subpage
);
331 /* Compute offset within MemoryRegionSection */
332 addr
-= section
->offset_within_address_space
;
334 /* Compute offset within MemoryRegion */
335 *xlat
= addr
+ section
->offset_within_region
;
337 diff
= int128_sub(section
->mr
->size
, int128_make64(addr
));
338 *plen
= int128_get64(int128_min(diff
, int128_make64(*plen
)));
342 MemoryRegion
*address_space_translate(AddressSpace
*as
, hwaddr addr
,
343 hwaddr
*xlat
, hwaddr
*plen
,
347 MemoryRegionSection
*section
;
352 section
= address_space_translate_internal(as
->dispatch
, addr
, &addr
, plen
, true);
355 if (!mr
->iommu_ops
) {
359 iotlb
= mr
->iommu_ops
->translate(mr
, addr
);
360 addr
= ((iotlb
.translated_addr
& ~iotlb
.addr_mask
)
361 | (addr
& iotlb
.addr_mask
));
362 len
= MIN(len
, (addr
| iotlb
.addr_mask
) - addr
+ 1);
363 if (!(iotlb
.perm
& (1 << is_write
))) {
364 mr
= &io_mem_unassigned
;
368 as
= iotlb
.target_as
;
376 MemoryRegionSection
*
377 address_space_translate_for_iotlb(AddressSpace
*as
, hwaddr addr
, hwaddr
*xlat
,
380 MemoryRegionSection
*section
;
381 section
= address_space_translate_internal(as
->dispatch
, addr
, xlat
, plen
, false);
383 assert(!section
->mr
->iommu_ops
);
388 void cpu_exec_init_all(void)
390 #if !defined(CONFIG_USER_ONLY)
391 qemu_mutex_init(&ram_list
.mutex
);
397 #if !defined(CONFIG_USER_ONLY)
399 static int cpu_common_post_load(void *opaque
, int version_id
)
401 CPUState
*cpu
= opaque
;
403 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
404 version_id is increased. */
405 cpu
->interrupt_request
&= ~0x01;
406 tlb_flush(cpu
->env_ptr
, 1);
411 const VMStateDescription vmstate_cpu_common
= {
412 .name
= "cpu_common",
414 .minimum_version_id
= 1,
415 .minimum_version_id_old
= 1,
416 .post_load
= cpu_common_post_load
,
417 .fields
= (VMStateField
[]) {
418 VMSTATE_UINT32(halted
, CPUState
),
419 VMSTATE_UINT32(interrupt_request
, CPUState
),
420 VMSTATE_END_OF_LIST()
426 CPUState
*qemu_get_cpu(int index
)
431 if (cpu
->cpu_index
== index
) {
439 void cpu_exec_init(CPUArchState
*env
)
441 CPUState
*cpu
= ENV_GET_CPU(env
);
442 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
446 #ifdef TARGET_WORDS_BIGENDIAN
452 #if defined(CONFIG_USER_ONLY)
456 CPU_FOREACH(some_cpu
) {
459 cpu
->cpu_index
= cpu_index
;
461 QTAILQ_INIT(&env
->breakpoints
);
462 QTAILQ_INIT(&env
->watchpoints
);
463 #ifndef CONFIG_USER_ONLY
464 cpu
->thread_id
= qemu_get_thread_id();
466 QTAILQ_INSERT_TAIL(&cpus
, cpu
, node
);
467 #if defined(CONFIG_USER_ONLY)
470 if (qdev_get_vmsd(DEVICE(cpu
)) == NULL
) {
471 vmstate_register(NULL
, cpu_index
, &vmstate_cpu_common
, cpu
);
473 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
474 register_savevm(NULL
, "cpu", cpu_index
, CPU_SAVE_VERSION
,
475 cpu_save
, cpu_load
, env
);
476 assert(cc
->vmsd
== NULL
);
477 assert(qdev_get_vmsd(DEVICE(cpu
)) == NULL
);
479 if (cc
->vmsd
!= NULL
) {
480 vmstate_register(NULL
, cpu_index
, cc
->vmsd
, cpu
);
484 #if defined(TARGET_HAS_ICE)
485 #if defined(CONFIG_USER_ONLY)
486 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
488 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
491 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
493 hwaddr phys
= cpu_get_phys_page_debug(cpu
, pc
);
495 tb_invalidate_phys_addr(phys
| (pc
& ~TARGET_PAGE_MASK
));
499 #endif /* TARGET_HAS_ICE */
501 #if defined(CONFIG_USER_ONLY)
502 void cpu_watchpoint_remove_all(CPUArchState
*env
, int mask
)
507 int cpu_watchpoint_insert(CPUArchState
*env
, target_ulong addr
, target_ulong len
,
508 int flags
, CPUWatchpoint
**watchpoint
)
513 /* Add a watchpoint. */
514 int cpu_watchpoint_insert(CPUArchState
*env
, target_ulong addr
, target_ulong len
,
515 int flags
, CPUWatchpoint
**watchpoint
)
517 target_ulong len_mask
= ~(len
- 1);
520 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
521 if ((len
& (len
- 1)) || (addr
& ~len_mask
) ||
522 len
== 0 || len
> TARGET_PAGE_SIZE
) {
523 fprintf(stderr
, "qemu: tried to set invalid watchpoint at "
524 TARGET_FMT_lx
", len=" TARGET_FMT_lu
"\n", addr
, len
);
527 wp
= g_malloc(sizeof(*wp
));
530 wp
->len_mask
= len_mask
;
533 /* keep all GDB-injected watchpoints in front */
535 QTAILQ_INSERT_HEAD(&env
->watchpoints
, wp
, entry
);
537 QTAILQ_INSERT_TAIL(&env
->watchpoints
, wp
, entry
);
539 tlb_flush_page(env
, addr
);
546 /* Remove a specific watchpoint. */
547 int cpu_watchpoint_remove(CPUArchState
*env
, target_ulong addr
, target_ulong len
,
550 target_ulong len_mask
= ~(len
- 1);
553 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
554 if (addr
== wp
->vaddr
&& len_mask
== wp
->len_mask
555 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
556 cpu_watchpoint_remove_by_ref(env
, wp
);
563 /* Remove a specific watchpoint by reference. */
564 void cpu_watchpoint_remove_by_ref(CPUArchState
*env
, CPUWatchpoint
*watchpoint
)
566 QTAILQ_REMOVE(&env
->watchpoints
, watchpoint
, entry
);
568 tlb_flush_page(env
, watchpoint
->vaddr
);
573 /* Remove all matching watchpoints. */
574 void cpu_watchpoint_remove_all(CPUArchState
*env
, int mask
)
576 CPUWatchpoint
*wp
, *next
;
578 QTAILQ_FOREACH_SAFE(wp
, &env
->watchpoints
, entry
, next
) {
579 if (wp
->flags
& mask
)
580 cpu_watchpoint_remove_by_ref(env
, wp
);
585 /* Add a breakpoint. */
586 int cpu_breakpoint_insert(CPUArchState
*env
, target_ulong pc
, int flags
,
587 CPUBreakpoint
**breakpoint
)
589 #if defined(TARGET_HAS_ICE)
592 bp
= g_malloc(sizeof(*bp
));
597 /* keep all GDB-injected breakpoints in front */
598 if (flags
& BP_GDB
) {
599 QTAILQ_INSERT_HEAD(&env
->breakpoints
, bp
, entry
);
601 QTAILQ_INSERT_TAIL(&env
->breakpoints
, bp
, entry
);
604 breakpoint_invalidate(ENV_GET_CPU(env
), pc
);
615 /* Remove a specific breakpoint. */
616 int cpu_breakpoint_remove(CPUArchState
*env
, target_ulong pc
, int flags
)
618 #if defined(TARGET_HAS_ICE)
621 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
622 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
623 cpu_breakpoint_remove_by_ref(env
, bp
);
633 /* Remove a specific breakpoint by reference. */
634 void cpu_breakpoint_remove_by_ref(CPUArchState
*env
, CPUBreakpoint
*breakpoint
)
636 #if defined(TARGET_HAS_ICE)
637 QTAILQ_REMOVE(&env
->breakpoints
, breakpoint
, entry
);
639 breakpoint_invalidate(ENV_GET_CPU(env
), breakpoint
->pc
);
645 /* Remove all matching breakpoints. */
646 void cpu_breakpoint_remove_all(CPUArchState
*env
, int mask
)
648 #if defined(TARGET_HAS_ICE)
649 CPUBreakpoint
*bp
, *next
;
651 QTAILQ_FOREACH_SAFE(bp
, &env
->breakpoints
, entry
, next
) {
652 if (bp
->flags
& mask
)
653 cpu_breakpoint_remove_by_ref(env
, bp
);
658 /* enable or disable single step mode. EXCP_DEBUG is returned by the
659 CPU loop after each instruction */
660 void cpu_single_step(CPUState
*cpu
, int enabled
)
662 #if defined(TARGET_HAS_ICE)
663 if (cpu
->singlestep_enabled
!= enabled
) {
664 cpu
->singlestep_enabled
= enabled
;
666 kvm_update_guest_debug(cpu
, 0);
668 /* must flush all the translated code to avoid inconsistencies */
669 /* XXX: only flush what is necessary */
670 CPUArchState
*env
= cpu
->env_ptr
;
677 void QEMU_NORETURN
cpu_abort(CPUArchState
*env
, const char *fmt
, ...)
679 CPUState
*cpu
= ENV_GET_CPU(env
);
685 fprintf(stderr
, "qemu: fatal: ");
686 vfprintf(stderr
, fmt
, ap
);
687 fprintf(stderr
, "\n");
688 cpu_dump_state(cpu
, stderr
, fprintf
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
689 if (qemu_log_enabled()) {
690 qemu_log("qemu: fatal: ");
691 qemu_log_vprintf(fmt
, ap2
);
693 log_cpu_state(cpu
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
699 #if defined(CONFIG_USER_ONLY)
701 struct sigaction act
;
702 sigfillset(&act
.sa_mask
);
703 act
.sa_handler
= SIG_DFL
;
704 sigaction(SIGABRT
, &act
, NULL
);
710 #if !defined(CONFIG_USER_ONLY)
711 static RAMBlock
*qemu_get_ram_block(ram_addr_t addr
)
715 /* The list is protected by the iothread lock here. */
716 block
= ram_list
.mru_block
;
717 if (block
&& addr
- block
->offset
< block
->length
) {
720 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
721 if (addr
- block
->offset
< block
->length
) {
726 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
730 ram_list
.mru_block
= block
;
734 static void tlb_reset_dirty_range_all(ram_addr_t start
, ram_addr_t length
)
740 end
= TARGET_PAGE_ALIGN(start
+ length
);
741 start
&= TARGET_PAGE_MASK
;
743 block
= qemu_get_ram_block(start
);
744 assert(block
== qemu_get_ram_block(end
- 1));
745 start1
= (uintptr_t)block
->host
+ (start
- block
->offset
);
746 cpu_tlb_reset_dirty_all(start1
, length
);
749 /* Note: start and end must be within the same ram block. */
750 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t length
,
755 cpu_physical_memory_clear_dirty_range(start
, length
, client
);
758 tlb_reset_dirty_range_all(start
, length
);
762 static void cpu_physical_memory_set_dirty_tracking(bool enable
)
764 in_migration
= enable
;
767 hwaddr
memory_region_section_get_iotlb(CPUArchState
*env
,
768 MemoryRegionSection
*section
,
770 hwaddr paddr
, hwaddr xlat
,
772 target_ulong
*address
)
777 if (memory_region_is_ram(section
->mr
)) {
779 iotlb
= (memory_region_get_ram_addr(section
->mr
) & TARGET_PAGE_MASK
)
781 if (!section
->readonly
) {
782 iotlb
|= PHYS_SECTION_NOTDIRTY
;
784 iotlb
|= PHYS_SECTION_ROM
;
787 iotlb
= section
- address_space_memory
.dispatch
->map
.sections
;
791 /* Make accesses to pages with watchpoints go via the
792 watchpoint trap routines. */
793 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
794 if (vaddr
== (wp
->vaddr
& TARGET_PAGE_MASK
)) {
795 /* Avoid trapping reads of pages with a write breakpoint. */
796 if ((prot
& PAGE_WRITE
) || (wp
->flags
& BP_MEM_READ
)) {
797 iotlb
= PHYS_SECTION_WATCH
+ paddr
;
798 *address
|= TLB_MMIO
;
806 #endif /* defined(CONFIG_USER_ONLY) */
808 #if !defined(CONFIG_USER_ONLY)
810 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
812 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
);
814 static void *(*phys_mem_alloc
)(size_t size
) = qemu_anon_ram_alloc
;
817 * Set a custom physical guest memory alloator.
818 * Accelerators with unusual needs may need this. Hopefully, we can
819 * get rid of it eventually.
821 void phys_mem_set_alloc(void *(*alloc
)(size_t))
823 phys_mem_alloc
= alloc
;
826 static uint16_t phys_section_add(PhysPageMap
*map
,
827 MemoryRegionSection
*section
)
829 /* The physical section number is ORed with a page-aligned
830 * pointer to produce the iotlb entries. Thus it should
831 * never overflow into the page-aligned value.
833 assert(map
->sections_nb
< TARGET_PAGE_SIZE
);
835 if (map
->sections_nb
== map
->sections_nb_alloc
) {
836 map
->sections_nb_alloc
= MAX(map
->sections_nb_alloc
* 2, 16);
837 map
->sections
= g_renew(MemoryRegionSection
, map
->sections
,
838 map
->sections_nb_alloc
);
840 map
->sections
[map
->sections_nb
] = *section
;
841 memory_region_ref(section
->mr
);
842 return map
->sections_nb
++;
845 static void phys_section_destroy(MemoryRegion
*mr
)
847 memory_region_unref(mr
);
850 subpage_t
*subpage
= container_of(mr
, subpage_t
, iomem
);
851 memory_region_destroy(&subpage
->iomem
);
856 static void phys_sections_free(PhysPageMap
*map
)
858 while (map
->sections_nb
> 0) {
859 MemoryRegionSection
*section
= &map
->sections
[--map
->sections_nb
];
860 phys_section_destroy(section
->mr
);
862 g_free(map
->sections
);
866 static void register_subpage(AddressSpaceDispatch
*d
, MemoryRegionSection
*section
)
869 hwaddr base
= section
->offset_within_address_space
871 MemoryRegionSection
*existing
= phys_page_find(d
->phys_map
, base
,
872 d
->map
.nodes
, d
->map
.sections
);
873 MemoryRegionSection subsection
= {
874 .offset_within_address_space
= base
,
875 .size
= int128_make64(TARGET_PAGE_SIZE
),
879 assert(existing
->mr
->subpage
|| existing
->mr
== &io_mem_unassigned
);
881 if (!(existing
->mr
->subpage
)) {
882 subpage
= subpage_init(d
->as
, base
);
883 subsection
.mr
= &subpage
->iomem
;
884 phys_page_set(d
, base
>> TARGET_PAGE_BITS
, 1,
885 phys_section_add(&d
->map
, &subsection
));
887 subpage
= container_of(existing
->mr
, subpage_t
, iomem
);
889 start
= section
->offset_within_address_space
& ~TARGET_PAGE_MASK
;
890 end
= start
+ int128_get64(section
->size
) - 1;
891 subpage_register(subpage
, start
, end
,
892 phys_section_add(&d
->map
, section
));
896 static void register_multipage(AddressSpaceDispatch
*d
,
897 MemoryRegionSection
*section
)
899 hwaddr start_addr
= section
->offset_within_address_space
;
900 uint16_t section_index
= phys_section_add(&d
->map
, section
);
901 uint64_t num_pages
= int128_get64(int128_rshift(section
->size
,
905 phys_page_set(d
, start_addr
>> TARGET_PAGE_BITS
, num_pages
, section_index
);
908 static void mem_add(MemoryListener
*listener
, MemoryRegionSection
*section
)
910 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
911 AddressSpaceDispatch
*d
= as
->next_dispatch
;
912 MemoryRegionSection now
= *section
, remain
= *section
;
913 Int128 page_size
= int128_make64(TARGET_PAGE_SIZE
);
915 if (now
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
916 uint64_t left
= TARGET_PAGE_ALIGN(now
.offset_within_address_space
)
917 - now
.offset_within_address_space
;
919 now
.size
= int128_min(int128_make64(left
), now
.size
);
920 register_subpage(d
, &now
);
922 now
.size
= int128_zero();
924 while (int128_ne(remain
.size
, now
.size
)) {
925 remain
.size
= int128_sub(remain
.size
, now
.size
);
926 remain
.offset_within_address_space
+= int128_get64(now
.size
);
927 remain
.offset_within_region
+= int128_get64(now
.size
);
929 if (int128_lt(remain
.size
, page_size
)) {
930 register_subpage(d
, &now
);
931 } else if (remain
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
932 now
.size
= page_size
;
933 register_subpage(d
, &now
);
935 now
.size
= int128_and(now
.size
, int128_neg(page_size
));
936 register_multipage(d
, &now
);
941 void qemu_flush_coalesced_mmio_buffer(void)
944 kvm_flush_coalesced_mmio_buffer();
947 void qemu_mutex_lock_ramlist(void)
949 qemu_mutex_lock(&ram_list
.mutex
);
952 void qemu_mutex_unlock_ramlist(void)
954 qemu_mutex_unlock(&ram_list
.mutex
);
961 #define HUGETLBFS_MAGIC 0x958458f6
963 static long gethugepagesize(const char *path
)
969 ret
= statfs(path
, &fs
);
970 } while (ret
!= 0 && errno
== EINTR
);
977 if (fs
.f_type
!= HUGETLBFS_MAGIC
)
978 fprintf(stderr
, "Warning: path not on HugeTLBFS: %s\n", path
);
983 static sigjmp_buf sigjump
;
985 static void sigbus_handler(int signal
)
987 siglongjmp(sigjump
, 1);
990 static void *file_ram_alloc(RAMBlock
*block
,
995 char *sanitized_name
;
997 void * volatile area
;
1001 hpagesize
= gethugepagesize(path
);
1006 if (memory
< hpagesize
) {
1010 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1011 fprintf(stderr
, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
1015 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1016 sanitized_name
= g_strdup(block
->mr
->name
);
1017 for (c
= sanitized_name
; *c
!= '\0'; c
++) {
1022 filename
= g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path
,
1024 g_free(sanitized_name
);
1026 fd
= mkstemp(filename
);
1028 perror("unable to create backing store for hugepages");
1035 memory
= (memory
+hpagesize
-1) & ~(hpagesize
-1);
1038 * ftruncate is not supported by hugetlbfs in older
1039 * hosts, so don't bother bailing out on errors.
1040 * If anything goes wrong with it under other filesystems,
1043 if (ftruncate(fd
, memory
))
1044 perror("ftruncate");
1046 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, MAP_PRIVATE
, fd
, 0);
1047 if (area
== MAP_FAILED
) {
1048 perror("file_ram_alloc: can't mmap RAM pages");
1055 struct sigaction act
, oldact
;
1056 sigset_t set
, oldset
;
1058 memset(&act
, 0, sizeof(act
));
1059 act
.sa_handler
= &sigbus_handler
;
1062 ret
= sigaction(SIGBUS
, &act
, &oldact
);
1064 perror("file_ram_alloc: failed to install signal handler");
1068 /* unblock SIGBUS */
1070 sigaddset(&set
, SIGBUS
);
1071 pthread_sigmask(SIG_UNBLOCK
, &set
, &oldset
);
1073 if (sigsetjmp(sigjump
, 1)) {
1074 fprintf(stderr
, "file_ram_alloc: failed to preallocate pages\n");
1078 /* MAP_POPULATE silently ignores failures */
1079 for (i
= 0; i
< (memory
/hpagesize
); i
++) {
1080 memset(area
+ (hpagesize
*i
), 0, 1);
1083 ret
= sigaction(SIGBUS
, &oldact
, NULL
);
1085 perror("file_ram_alloc: failed to reinstall signal handler");
1089 pthread_sigmask(SIG_SETMASK
, &oldset
, NULL
);
1096 static void *file_ram_alloc(RAMBlock
*block
,
1100 fprintf(stderr
, "-mem-path not supported on this host\n");
1105 static ram_addr_t
find_ram_offset(ram_addr_t size
)
1107 RAMBlock
*block
, *next_block
;
1108 ram_addr_t offset
= RAM_ADDR_MAX
, mingap
= RAM_ADDR_MAX
;
1110 assert(size
!= 0); /* it would hand out same offset multiple times */
1112 if (QTAILQ_EMPTY(&ram_list
.blocks
))
1115 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1116 ram_addr_t end
, next
= RAM_ADDR_MAX
;
1118 end
= block
->offset
+ block
->length
;
1120 QTAILQ_FOREACH(next_block
, &ram_list
.blocks
, next
) {
1121 if (next_block
->offset
>= end
) {
1122 next
= MIN(next
, next_block
->offset
);
1125 if (next
- end
>= size
&& next
- end
< mingap
) {
1127 mingap
= next
- end
;
1131 if (offset
== RAM_ADDR_MAX
) {
1132 fprintf(stderr
, "Failed to find gap of requested size: %" PRIu64
"\n",
1140 ram_addr_t
last_ram_offset(void)
1143 ram_addr_t last
= 0;
1145 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
)
1146 last
= MAX(last
, block
->offset
+ block
->length
);
1151 static void qemu_ram_setup_dump(void *addr
, ram_addr_t size
)
1155 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1156 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1157 "dump-guest-core", true)) {
1158 ret
= qemu_madvise(addr
, size
, QEMU_MADV_DONTDUMP
);
1160 perror("qemu_madvise");
1161 fprintf(stderr
, "madvise doesn't support MADV_DONTDUMP, "
1162 "but dump_guest_core=off specified\n");
1167 void qemu_ram_set_idstr(ram_addr_t addr
, const char *name
, DeviceState
*dev
)
1169 RAMBlock
*new_block
, *block
;
1172 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1173 if (block
->offset
== addr
) {
1179 assert(!new_block
->idstr
[0]);
1182 char *id
= qdev_get_dev_path(dev
);
1184 snprintf(new_block
->idstr
, sizeof(new_block
->idstr
), "%s/", id
);
1188 pstrcat(new_block
->idstr
, sizeof(new_block
->idstr
), name
);
1190 /* This assumes the iothread lock is taken here too. */
1191 qemu_mutex_lock_ramlist();
1192 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1193 if (block
!= new_block
&& !strcmp(block
->idstr
, new_block
->idstr
)) {
1194 fprintf(stderr
, "RAMBlock \"%s\" already registered, abort!\n",
1199 qemu_mutex_unlock_ramlist();
1202 static int memory_try_enable_merging(void *addr
, size_t len
)
1204 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1205 /* disabled by the user */
1209 return qemu_madvise(addr
, len
, QEMU_MADV_MERGEABLE
);
1212 ram_addr_t
qemu_ram_alloc_from_ptr(ram_addr_t size
, void *host
,
1215 RAMBlock
*block
, *new_block
;
1216 ram_addr_t old_ram_size
, new_ram_size
;
1218 old_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1220 size
= TARGET_PAGE_ALIGN(size
);
1221 new_block
= g_malloc0(sizeof(*new_block
));
1224 /* This assumes the iothread lock is taken here too. */
1225 qemu_mutex_lock_ramlist();
1227 new_block
->offset
= find_ram_offset(size
);
1229 new_block
->host
= host
;
1230 new_block
->flags
|= RAM_PREALLOC_MASK
;
1231 } else if (xen_enabled()) {
1233 fprintf(stderr
, "-mem-path not supported with Xen\n");
1236 xen_ram_alloc(new_block
->offset
, size
, mr
);
1239 if (phys_mem_alloc
!= qemu_anon_ram_alloc
) {
1241 * file_ram_alloc() needs to allocate just like
1242 * phys_mem_alloc, but we haven't bothered to provide
1246 "-mem-path not supported with this accelerator\n");
1249 new_block
->host
= file_ram_alloc(new_block
, size
, mem_path
);
1251 if (!new_block
->host
) {
1252 new_block
->host
= phys_mem_alloc(size
);
1253 if (!new_block
->host
) {
1254 fprintf(stderr
, "Cannot set up guest memory '%s': %s\n",
1255 new_block
->mr
->name
, strerror(errno
));
1258 memory_try_enable_merging(new_block
->host
, size
);
1261 new_block
->length
= size
;
1263 /* Keep the list sorted from biggest to smallest block. */
1264 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1265 if (block
->length
< new_block
->length
) {
1270 QTAILQ_INSERT_BEFORE(block
, new_block
, next
);
1272 QTAILQ_INSERT_TAIL(&ram_list
.blocks
, new_block
, next
);
1274 ram_list
.mru_block
= NULL
;
1277 qemu_mutex_unlock_ramlist();
1279 new_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1281 if (new_ram_size
> old_ram_size
) {
1283 for (i
= 0; i
< DIRTY_MEMORY_NUM
; i
++) {
1284 ram_list
.dirty_memory
[i
] =
1285 bitmap_zero_extend(ram_list
.dirty_memory
[i
],
1286 old_ram_size
, new_ram_size
);
1289 cpu_physical_memory_set_dirty_range(new_block
->offset
, size
);
1291 qemu_ram_setup_dump(new_block
->host
, size
);
1292 qemu_madvise(new_block
->host
, size
, QEMU_MADV_HUGEPAGE
);
1293 qemu_madvise(new_block
->host
, size
, QEMU_MADV_DONTFORK
);
1296 kvm_setup_guest_memory(new_block
->host
, size
);
1298 return new_block
->offset
;
1301 ram_addr_t
qemu_ram_alloc(ram_addr_t size
, MemoryRegion
*mr
)
1303 return qemu_ram_alloc_from_ptr(size
, NULL
, mr
);
1306 void qemu_ram_free_from_ptr(ram_addr_t addr
)
1310 /* This assumes the iothread lock is taken here too. */
1311 qemu_mutex_lock_ramlist();
1312 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1313 if (addr
== block
->offset
) {
1314 QTAILQ_REMOVE(&ram_list
.blocks
, block
, next
);
1315 ram_list
.mru_block
= NULL
;
1321 qemu_mutex_unlock_ramlist();
1324 void qemu_ram_free(ram_addr_t addr
)
1328 /* This assumes the iothread lock is taken here too. */
1329 qemu_mutex_lock_ramlist();
1330 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1331 if (addr
== block
->offset
) {
1332 QTAILQ_REMOVE(&ram_list
.blocks
, block
, next
);
1333 ram_list
.mru_block
= NULL
;
1335 if (block
->flags
& RAM_PREALLOC_MASK
) {
1337 } else if (xen_enabled()) {
1338 xen_invalidate_map_cache_entry(block
->host
);
1340 } else if (block
->fd
>= 0) {
1341 munmap(block
->host
, block
->length
);
1345 qemu_anon_ram_free(block
->host
, block
->length
);
1351 qemu_mutex_unlock_ramlist();
1356 void qemu_ram_remap(ram_addr_t addr
, ram_addr_t length
)
1363 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1364 offset
= addr
- block
->offset
;
1365 if (offset
< block
->length
) {
1366 vaddr
= block
->host
+ offset
;
1367 if (block
->flags
& RAM_PREALLOC_MASK
) {
1369 } else if (xen_enabled()) {
1373 munmap(vaddr
, length
);
1374 if (block
->fd
>= 0) {
1376 flags
|= mem_prealloc
? MAP_POPULATE
| MAP_SHARED
:
1379 flags
|= MAP_PRIVATE
;
1381 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1382 flags
, block
->fd
, offset
);
1385 * Remap needs to match alloc. Accelerators that
1386 * set phys_mem_alloc never remap. If they did,
1387 * we'd need a remap hook here.
1389 assert(phys_mem_alloc
== qemu_anon_ram_alloc
);
1391 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
1392 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1395 if (area
!= vaddr
) {
1396 fprintf(stderr
, "Could not remap addr: "
1397 RAM_ADDR_FMT
"@" RAM_ADDR_FMT
"\n",
1401 memory_try_enable_merging(vaddr
, length
);
1402 qemu_ram_setup_dump(vaddr
, length
);
1408 #endif /* !_WIN32 */
1410 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1411 With the exception of the softmmu code in this file, this should
1412 only be used for local memory (e.g. video ram) that the device owns,
1413 and knows it isn't going to access beyond the end of the block.
1415 It should not be used for general purpose DMA.
1416 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1418 void *qemu_get_ram_ptr(ram_addr_t addr
)
1420 RAMBlock
*block
= qemu_get_ram_block(addr
);
1422 if (xen_enabled()) {
1423 /* We need to check if the requested address is in the RAM
1424 * because we don't want to map the entire memory in QEMU.
1425 * In that case just map until the end of the page.
1427 if (block
->offset
== 0) {
1428 return xen_map_cache(addr
, 0, 0);
1429 } else if (block
->host
== NULL
) {
1431 xen_map_cache(block
->offset
, block
->length
, 1);
1434 return block
->host
+ (addr
- block
->offset
);
1437 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1438 * but takes a size argument */
1439 static void *qemu_ram_ptr_length(ram_addr_t addr
, hwaddr
*size
)
1444 if (xen_enabled()) {
1445 return xen_map_cache(addr
, *size
, 1);
1449 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1450 if (addr
- block
->offset
< block
->length
) {
1451 if (addr
- block
->offset
+ *size
> block
->length
)
1452 *size
= block
->length
- addr
+ block
->offset
;
1453 return block
->host
+ (addr
- block
->offset
);
1457 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
1462 /* Some of the softmmu routines need to translate from a host pointer
1463 (typically a TLB entry) back to a ram offset. */
1464 MemoryRegion
*qemu_ram_addr_from_host(void *ptr
, ram_addr_t
*ram_addr
)
1467 uint8_t *host
= ptr
;
1469 if (xen_enabled()) {
1470 *ram_addr
= xen_ram_addr_from_mapcache(ptr
);
1471 return qemu_get_ram_block(*ram_addr
)->mr
;
1474 block
= ram_list
.mru_block
;
1475 if (block
&& block
->host
&& host
- block
->host
< block
->length
) {
1479 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1480 /* This case append when the block is not mapped. */
1481 if (block
->host
== NULL
) {
1484 if (host
- block
->host
< block
->length
) {
1492 *ram_addr
= block
->offset
+ (host
- block
->host
);
1496 static void notdirty_mem_write(void *opaque
, hwaddr ram_addr
,
1497 uint64_t val
, unsigned size
)
1499 if (!cpu_physical_memory_get_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
)) {
1500 tb_invalidate_phys_page_fast(ram_addr
, size
);
1504 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
1507 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
1510 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
1515 cpu_physical_memory_set_dirty_flag(ram_addr
, DIRTY_MEMORY_MIGRATION
);
1516 cpu_physical_memory_set_dirty_flag(ram_addr
, DIRTY_MEMORY_VGA
);
1517 /* we remove the notdirty callback only if the code has been
1519 if (!cpu_physical_memory_is_clean(ram_addr
)) {
1520 CPUArchState
*env
= current_cpu
->env_ptr
;
1521 tlb_set_dirty(env
, env
->mem_io_vaddr
);
1525 static bool notdirty_mem_accepts(void *opaque
, hwaddr addr
,
1526 unsigned size
, bool is_write
)
1531 static const MemoryRegionOps notdirty_mem_ops
= {
1532 .write
= notdirty_mem_write
,
1533 .valid
.accepts
= notdirty_mem_accepts
,
1534 .endianness
= DEVICE_NATIVE_ENDIAN
,
1537 /* Generate a debug exception if a watchpoint has been hit. */
1538 static void check_watchpoint(int offset
, int len_mask
, int flags
)
1540 CPUArchState
*env
= current_cpu
->env_ptr
;
1541 target_ulong pc
, cs_base
;
1546 if (env
->watchpoint_hit
) {
1547 /* We re-entered the check after replacing the TB. Now raise
1548 * the debug interrupt so that is will trigger after the
1549 * current instruction. */
1550 cpu_interrupt(ENV_GET_CPU(env
), CPU_INTERRUPT_DEBUG
);
1553 vaddr
= (env
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
1554 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1555 if ((vaddr
== (wp
->vaddr
& len_mask
) ||
1556 (vaddr
& wp
->len_mask
) == wp
->vaddr
) && (wp
->flags
& flags
)) {
1557 wp
->flags
|= BP_WATCHPOINT_HIT
;
1558 if (!env
->watchpoint_hit
) {
1559 env
->watchpoint_hit
= wp
;
1560 tb_check_watchpoint(env
);
1561 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
1562 env
->exception_index
= EXCP_DEBUG
;
1565 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
1566 tb_gen_code(env
, pc
, cs_base
, cpu_flags
, 1);
1567 cpu_resume_from_signal(env
, NULL
);
1571 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
1576 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1577 so these check for a hit then pass through to the normal out-of-line
1579 static uint64_t watch_mem_read(void *opaque
, hwaddr addr
,
1582 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~(size
- 1), BP_MEM_READ
);
1584 case 1: return ldub_phys(addr
);
1585 case 2: return lduw_phys(addr
);
1586 case 4: return ldl_phys(addr
);
1591 static void watch_mem_write(void *opaque
, hwaddr addr
,
1592 uint64_t val
, unsigned size
)
1594 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~(size
- 1), BP_MEM_WRITE
);
1597 stb_phys(addr
, val
);
1600 stw_phys(addr
, val
);
1603 stl_phys(addr
, val
);
1609 static const MemoryRegionOps watch_mem_ops
= {
1610 .read
= watch_mem_read
,
1611 .write
= watch_mem_write
,
1612 .endianness
= DEVICE_NATIVE_ENDIAN
,
1615 static uint64_t subpage_read(void *opaque
, hwaddr addr
,
1618 subpage_t
*subpage
= opaque
;
1621 #if defined(DEBUG_SUBPAGE)
1622 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
"\n", __func__
,
1623 subpage
, len
, addr
);
1625 address_space_read(subpage
->as
, addr
+ subpage
->base
, buf
, len
);
1638 static void subpage_write(void *opaque
, hwaddr addr
,
1639 uint64_t value
, unsigned len
)
1641 subpage_t
*subpage
= opaque
;
1644 #if defined(DEBUG_SUBPAGE)
1645 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
1646 " value %"PRIx64
"\n",
1647 __func__
, subpage
, len
, addr
, value
);
1662 address_space_write(subpage
->as
, addr
+ subpage
->base
, buf
, len
);
1665 static bool subpage_accepts(void *opaque
, hwaddr addr
,
1666 unsigned len
, bool is_write
)
1668 subpage_t
*subpage
= opaque
;
1669 #if defined(DEBUG_SUBPAGE)
1670 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx
"\n",
1671 __func__
, subpage
, is_write
? 'w' : 'r', len
, addr
);
1674 return address_space_access_valid(subpage
->as
, addr
+ subpage
->base
,
1678 static const MemoryRegionOps subpage_ops
= {
1679 .read
= subpage_read
,
1680 .write
= subpage_write
,
1681 .valid
.accepts
= subpage_accepts
,
1682 .endianness
= DEVICE_NATIVE_ENDIAN
,
1685 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
1690 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
1692 idx
= SUBPAGE_IDX(start
);
1693 eidx
= SUBPAGE_IDX(end
);
1694 #if defined(DEBUG_SUBPAGE)
1695 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
1696 __func__
, mmio
, start
, end
, idx
, eidx
, section
);
1698 for (; idx
<= eidx
; idx
++) {
1699 mmio
->sub_section
[idx
] = section
;
1705 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
)
1709 mmio
= g_malloc0(sizeof(subpage_t
));
1713 memory_region_init_io(&mmio
->iomem
, NULL
, &subpage_ops
, mmio
,
1714 "subpage", TARGET_PAGE_SIZE
);
1715 mmio
->iomem
.subpage
= true;
1716 #if defined(DEBUG_SUBPAGE)
1717 printf("%s: %p base " TARGET_FMT_plx
" len %08x\n", __func__
,
1718 mmio
, base
, TARGET_PAGE_SIZE
);
1720 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
-1, PHYS_SECTION_UNASSIGNED
);
1725 static uint16_t dummy_section(PhysPageMap
*map
, MemoryRegion
*mr
)
1727 MemoryRegionSection section
= {
1729 .offset_within_address_space
= 0,
1730 .offset_within_region
= 0,
1731 .size
= int128_2_64(),
1734 return phys_section_add(map
, §ion
);
1737 MemoryRegion
*iotlb_to_region(hwaddr index
)
1739 return address_space_memory
.dispatch
->map
.sections
[
1740 index
& ~TARGET_PAGE_MASK
].mr
;
1743 static void io_mem_init(void)
1745 memory_region_init_io(&io_mem_rom
, NULL
, &unassigned_mem_ops
, NULL
, "rom", UINT64_MAX
);
1746 memory_region_init_io(&io_mem_unassigned
, NULL
, &unassigned_mem_ops
, NULL
,
1747 "unassigned", UINT64_MAX
);
1748 memory_region_init_io(&io_mem_notdirty
, NULL
, ¬dirty_mem_ops
, NULL
,
1749 "notdirty", UINT64_MAX
);
1750 memory_region_init_io(&io_mem_watch
, NULL
, &watch_mem_ops
, NULL
,
1751 "watch", UINT64_MAX
);
1754 static void mem_begin(MemoryListener
*listener
)
1756 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
1757 AddressSpaceDispatch
*d
= g_new0(AddressSpaceDispatch
, 1);
1760 n
= dummy_section(&d
->map
, &io_mem_unassigned
);
1761 assert(n
== PHYS_SECTION_UNASSIGNED
);
1762 n
= dummy_section(&d
->map
, &io_mem_notdirty
);
1763 assert(n
== PHYS_SECTION_NOTDIRTY
);
1764 n
= dummy_section(&d
->map
, &io_mem_rom
);
1765 assert(n
== PHYS_SECTION_ROM
);
1766 n
= dummy_section(&d
->map
, &io_mem_watch
);
1767 assert(n
== PHYS_SECTION_WATCH
);
1769 d
->phys_map
= (PhysPageEntry
) { .ptr
= PHYS_MAP_NODE_NIL
, .skip
= 1 };
1771 as
->next_dispatch
= d
;
1774 static void mem_commit(MemoryListener
*listener
)
1776 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
1777 AddressSpaceDispatch
*cur
= as
->dispatch
;
1778 AddressSpaceDispatch
*next
= as
->next_dispatch
;
1780 phys_page_compact_all(next
, next
->map
.nodes_nb
);
1782 as
->dispatch
= next
;
1785 phys_sections_free(&cur
->map
);
1790 static void tcg_commit(MemoryListener
*listener
)
1794 /* since each CPU stores ram addresses in its TLB cache, we must
1795 reset the modified entries */
1798 CPUArchState
*env
= cpu
->env_ptr
;
1804 static void core_log_global_start(MemoryListener
*listener
)
1806 cpu_physical_memory_set_dirty_tracking(true);
1809 static void core_log_global_stop(MemoryListener
*listener
)
1811 cpu_physical_memory_set_dirty_tracking(false);
1814 static MemoryListener core_memory_listener
= {
1815 .log_global_start
= core_log_global_start
,
1816 .log_global_stop
= core_log_global_stop
,
1820 static MemoryListener tcg_memory_listener
= {
1821 .commit
= tcg_commit
,
1824 void address_space_init_dispatch(AddressSpace
*as
)
1826 as
->dispatch
= NULL
;
1827 as
->dispatch_listener
= (MemoryListener
) {
1829 .commit
= mem_commit
,
1830 .region_add
= mem_add
,
1831 .region_nop
= mem_add
,
1834 memory_listener_register(&as
->dispatch_listener
, as
);
1837 void address_space_destroy_dispatch(AddressSpace
*as
)
1839 AddressSpaceDispatch
*d
= as
->dispatch
;
1841 memory_listener_unregister(&as
->dispatch_listener
);
1843 as
->dispatch
= NULL
;
1846 static void memory_map_init(void)
1848 system_memory
= g_malloc(sizeof(*system_memory
));
1850 memory_region_init(system_memory
, NULL
, "system", UINT64_MAX
);
1851 address_space_init(&address_space_memory
, system_memory
, "memory");
1853 system_io
= g_malloc(sizeof(*system_io
));
1854 memory_region_init_io(system_io
, NULL
, &unassigned_io_ops
, NULL
, "io",
1856 address_space_init(&address_space_io
, system_io
, "I/O");
1858 memory_listener_register(&core_memory_listener
, &address_space_memory
);
1859 if (tcg_enabled()) {
1860 memory_listener_register(&tcg_memory_listener
, &address_space_memory
);
1864 MemoryRegion
*get_system_memory(void)
1866 return system_memory
;
1869 MemoryRegion
*get_system_io(void)
1874 #endif /* !defined(CONFIG_USER_ONLY) */
1876 /* physical memory access (slow version, mainly for debug) */
1877 #if defined(CONFIG_USER_ONLY)
1878 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
1879 uint8_t *buf
, int len
, int is_write
)
1886 page
= addr
& TARGET_PAGE_MASK
;
1887 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
1890 flags
= page_get_flags(page
);
1891 if (!(flags
& PAGE_VALID
))
1894 if (!(flags
& PAGE_WRITE
))
1896 /* XXX: this code should not depend on lock_user */
1897 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
1900 unlock_user(p
, addr
, l
);
1902 if (!(flags
& PAGE_READ
))
1904 /* XXX: this code should not depend on lock_user */
1905 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
1908 unlock_user(p
, addr
, 0);
1919 static void invalidate_and_set_dirty(hwaddr addr
,
1922 if (cpu_physical_memory_is_clean(addr
)) {
1923 /* invalidate code */
1924 tb_invalidate_phys_page_range(addr
, addr
+ length
, 0);
1926 cpu_physical_memory_set_dirty_flag(addr
, DIRTY_MEMORY_VGA
);
1927 cpu_physical_memory_set_dirty_flag(addr
, DIRTY_MEMORY_MIGRATION
);
1929 xen_modified_memory(addr
, length
);
1932 static inline bool memory_access_is_direct(MemoryRegion
*mr
, bool is_write
)
1934 if (memory_region_is_ram(mr
)) {
1935 return !(is_write
&& mr
->readonly
);
1937 if (memory_region_is_romd(mr
)) {
1944 static int memory_access_size(MemoryRegion
*mr
, unsigned l
, hwaddr addr
)
1946 unsigned access_size_max
= mr
->ops
->valid
.max_access_size
;
1948 /* Regions are assumed to support 1-4 byte accesses unless
1949 otherwise specified. */
1950 if (access_size_max
== 0) {
1951 access_size_max
= 4;
1954 /* Bound the maximum access by the alignment of the address. */
1955 if (!mr
->ops
->impl
.unaligned
) {
1956 unsigned align_size_max
= addr
& -addr
;
1957 if (align_size_max
!= 0 && align_size_max
< access_size_max
) {
1958 access_size_max
= align_size_max
;
1962 /* Don't attempt accesses larger than the maximum. */
1963 if (l
> access_size_max
) {
1964 l
= access_size_max
;
1967 l
= 1 << (qemu_fls(l
) - 1);
1973 bool address_space_rw(AddressSpace
*as
, hwaddr addr
, uint8_t *buf
,
1974 int len
, bool is_write
)
1985 mr
= address_space_translate(as
, addr
, &addr1
, &l
, is_write
);
1988 if (!memory_access_is_direct(mr
, is_write
)) {
1989 l
= memory_access_size(mr
, l
, addr1
);
1990 /* XXX: could force current_cpu to NULL to avoid
1994 /* 64 bit write access */
1996 error
|= io_mem_write(mr
, addr1
, val
, 8);
1999 /* 32 bit write access */
2001 error
|= io_mem_write(mr
, addr1
, val
, 4);
2004 /* 16 bit write access */
2006 error
|= io_mem_write(mr
, addr1
, val
, 2);
2009 /* 8 bit write access */
2011 error
|= io_mem_write(mr
, addr1
, val
, 1);
2017 addr1
+= memory_region_get_ram_addr(mr
);
2019 ptr
= qemu_get_ram_ptr(addr1
);
2020 memcpy(ptr
, buf
, l
);
2021 invalidate_and_set_dirty(addr1
, l
);
2024 if (!memory_access_is_direct(mr
, is_write
)) {
2026 l
= memory_access_size(mr
, l
, addr1
);
2029 /* 64 bit read access */
2030 error
|= io_mem_read(mr
, addr1
, &val
, 8);
2034 /* 32 bit read access */
2035 error
|= io_mem_read(mr
, addr1
, &val
, 4);
2039 /* 16 bit read access */
2040 error
|= io_mem_read(mr
, addr1
, &val
, 2);
2044 /* 8 bit read access */
2045 error
|= io_mem_read(mr
, addr1
, &val
, 1);
2053 ptr
= qemu_get_ram_ptr(mr
->ram_addr
+ addr1
);
2054 memcpy(buf
, ptr
, l
);
2065 bool address_space_write(AddressSpace
*as
, hwaddr addr
,
2066 const uint8_t *buf
, int len
)
2068 return address_space_rw(as
, addr
, (uint8_t *)buf
, len
, true);
2071 bool address_space_read(AddressSpace
*as
, hwaddr addr
, uint8_t *buf
, int len
)
2073 return address_space_rw(as
, addr
, buf
, len
, false);
2077 void cpu_physical_memory_rw(hwaddr addr
, uint8_t *buf
,
2078 int len
, int is_write
)
2080 address_space_rw(&address_space_memory
, addr
, buf
, len
, is_write
);
2083 enum write_rom_type
{
2088 static inline void cpu_physical_memory_write_rom_internal(
2089 hwaddr addr
, const uint8_t *buf
, int len
, enum write_rom_type type
)
2098 mr
= address_space_translate(&address_space_memory
,
2099 addr
, &addr1
, &l
, true);
2101 if (!(memory_region_is_ram(mr
) ||
2102 memory_region_is_romd(mr
))) {
2105 addr1
+= memory_region_get_ram_addr(mr
);
2107 ptr
= qemu_get_ram_ptr(addr1
);
2110 memcpy(ptr
, buf
, l
);
2111 invalidate_and_set_dirty(addr1
, l
);
2114 flush_icache_range((uintptr_t)ptr
, (uintptr_t)ptr
+ l
);
2124 /* used for ROM loading : can write in RAM and ROM */
2125 void cpu_physical_memory_write_rom(hwaddr addr
,
2126 const uint8_t *buf
, int len
)
2128 cpu_physical_memory_write_rom_internal(addr
, buf
, len
, WRITE_DATA
);
2131 void cpu_flush_icache_range(hwaddr start
, int len
)
2134 * This function should do the same thing as an icache flush that was
2135 * triggered from within the guest. For TCG we are always cache coherent,
2136 * so there is no need to flush anything. For KVM / Xen we need to flush
2137 * the host's instruction cache at least.
2139 if (tcg_enabled()) {
2143 cpu_physical_memory_write_rom_internal(start
, NULL
, len
, FLUSH_CACHE
);
2153 static BounceBuffer bounce
;
2155 typedef struct MapClient
{
2157 void (*callback
)(void *opaque
);
2158 QLIST_ENTRY(MapClient
) link
;
2161 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
2162 = QLIST_HEAD_INITIALIZER(map_client_list
);
2164 void *cpu_register_map_client(void *opaque
, void (*callback
)(void *opaque
))
2166 MapClient
*client
= g_malloc(sizeof(*client
));
2168 client
->opaque
= opaque
;
2169 client
->callback
= callback
;
2170 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
2174 static void cpu_unregister_map_client(void *_client
)
2176 MapClient
*client
= (MapClient
*)_client
;
2178 QLIST_REMOVE(client
, link
);
2182 static void cpu_notify_map_clients(void)
2186 while (!QLIST_EMPTY(&map_client_list
)) {
2187 client
= QLIST_FIRST(&map_client_list
);
2188 client
->callback(client
->opaque
);
2189 cpu_unregister_map_client(client
);
2193 bool address_space_access_valid(AddressSpace
*as
, hwaddr addr
, int len
, bool is_write
)
2200 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2201 if (!memory_access_is_direct(mr
, is_write
)) {
2202 l
= memory_access_size(mr
, l
, addr
);
2203 if (!memory_region_access_valid(mr
, xlat
, l
, is_write
)) {
2214 /* Map a physical memory region into a host virtual address.
2215 * May map a subset of the requested range, given by and returned in *plen.
2216 * May return NULL if resources needed to perform the mapping are exhausted.
2217 * Use only for reads OR writes - not for read-modify-write operations.
2218 * Use cpu_register_map_client() to know when retrying the map operation is
2219 * likely to succeed.
2221 void *address_space_map(AddressSpace
*as
,
2228 hwaddr l
, xlat
, base
;
2229 MemoryRegion
*mr
, *this_mr
;
2237 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2238 if (!memory_access_is_direct(mr
, is_write
)) {
2239 if (bounce
.buffer
) {
2242 /* Avoid unbounded allocations */
2243 l
= MIN(l
, TARGET_PAGE_SIZE
);
2244 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, l
);
2248 memory_region_ref(mr
);
2251 address_space_read(as
, addr
, bounce
.buffer
, l
);
2255 return bounce
.buffer
;
2259 raddr
= memory_region_get_ram_addr(mr
);
2270 this_mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2271 if (this_mr
!= mr
|| xlat
!= base
+ done
) {
2276 memory_region_ref(mr
);
2278 return qemu_ram_ptr_length(raddr
+ base
, plen
);
2281 /* Unmaps a memory region previously mapped by address_space_map().
2282 * Will also mark the memory as dirty if is_write == 1. access_len gives
2283 * the amount of memory that was actually read or written by the caller.
2285 void address_space_unmap(AddressSpace
*as
, void *buffer
, hwaddr len
,
2286 int is_write
, hwaddr access_len
)
2288 if (buffer
!= bounce
.buffer
) {
2292 mr
= qemu_ram_addr_from_host(buffer
, &addr1
);
2295 while (access_len
) {
2297 l
= TARGET_PAGE_SIZE
;
2300 invalidate_and_set_dirty(addr1
, l
);
2305 if (xen_enabled()) {
2306 xen_invalidate_map_cache_entry(buffer
);
2308 memory_region_unref(mr
);
2312 address_space_write(as
, bounce
.addr
, bounce
.buffer
, access_len
);
2314 qemu_vfree(bounce
.buffer
);
2315 bounce
.buffer
= NULL
;
2316 memory_region_unref(bounce
.mr
);
2317 cpu_notify_map_clients();
2320 void *cpu_physical_memory_map(hwaddr addr
,
2324 return address_space_map(&address_space_memory
, addr
, plen
, is_write
);
2327 void cpu_physical_memory_unmap(void *buffer
, hwaddr len
,
2328 int is_write
, hwaddr access_len
)
2330 return address_space_unmap(&address_space_memory
, buffer
, len
, is_write
, access_len
);
2333 /* warning: addr must be aligned */
2334 static inline uint32_t ldl_phys_internal(hwaddr addr
,
2335 enum device_endian endian
)
2343 mr
= address_space_translate(&address_space_memory
, addr
, &addr1
, &l
,
2345 if (l
< 4 || !memory_access_is_direct(mr
, false)) {
2347 io_mem_read(mr
, addr1
, &val
, 4);
2348 #if defined(TARGET_WORDS_BIGENDIAN)
2349 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2353 if (endian
== DEVICE_BIG_ENDIAN
) {
2359 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2363 case DEVICE_LITTLE_ENDIAN
:
2364 val
= ldl_le_p(ptr
);
2366 case DEVICE_BIG_ENDIAN
:
2367 val
= ldl_be_p(ptr
);
2377 uint32_t ldl_phys(hwaddr addr
)
2379 return ldl_phys_internal(addr
, DEVICE_NATIVE_ENDIAN
);
2382 uint32_t ldl_le_phys(hwaddr addr
)
2384 return ldl_phys_internal(addr
, DEVICE_LITTLE_ENDIAN
);
2387 uint32_t ldl_be_phys(hwaddr addr
)
2389 return ldl_phys_internal(addr
, DEVICE_BIG_ENDIAN
);
2392 /* warning: addr must be aligned */
2393 static inline uint64_t ldq_phys_internal(hwaddr addr
,
2394 enum device_endian endian
)
2402 mr
= address_space_translate(&address_space_memory
, addr
, &addr1
, &l
,
2404 if (l
< 8 || !memory_access_is_direct(mr
, false)) {
2406 io_mem_read(mr
, addr1
, &val
, 8);
2407 #if defined(TARGET_WORDS_BIGENDIAN)
2408 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2412 if (endian
== DEVICE_BIG_ENDIAN
) {
2418 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2422 case DEVICE_LITTLE_ENDIAN
:
2423 val
= ldq_le_p(ptr
);
2425 case DEVICE_BIG_ENDIAN
:
2426 val
= ldq_be_p(ptr
);
2436 uint64_t ldq_phys(hwaddr addr
)
2438 return ldq_phys_internal(addr
, DEVICE_NATIVE_ENDIAN
);
2441 uint64_t ldq_le_phys(hwaddr addr
)
2443 return ldq_phys_internal(addr
, DEVICE_LITTLE_ENDIAN
);
2446 uint64_t ldq_be_phys(hwaddr addr
)
2448 return ldq_phys_internal(addr
, DEVICE_BIG_ENDIAN
);
2452 uint32_t ldub_phys(hwaddr addr
)
2455 cpu_physical_memory_read(addr
, &val
, 1);
2459 /* warning: addr must be aligned */
2460 static inline uint32_t lduw_phys_internal(hwaddr addr
,
2461 enum device_endian endian
)
2469 mr
= address_space_translate(&address_space_memory
, addr
, &addr1
, &l
,
2471 if (l
< 2 || !memory_access_is_direct(mr
, false)) {
2473 io_mem_read(mr
, addr1
, &val
, 2);
2474 #if defined(TARGET_WORDS_BIGENDIAN)
2475 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2479 if (endian
== DEVICE_BIG_ENDIAN
) {
2485 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2489 case DEVICE_LITTLE_ENDIAN
:
2490 val
= lduw_le_p(ptr
);
2492 case DEVICE_BIG_ENDIAN
:
2493 val
= lduw_be_p(ptr
);
2503 uint32_t lduw_phys(hwaddr addr
)
2505 return lduw_phys_internal(addr
, DEVICE_NATIVE_ENDIAN
);
2508 uint32_t lduw_le_phys(hwaddr addr
)
2510 return lduw_phys_internal(addr
, DEVICE_LITTLE_ENDIAN
);
2513 uint32_t lduw_be_phys(hwaddr addr
)
2515 return lduw_phys_internal(addr
, DEVICE_BIG_ENDIAN
);
2518 /* warning: addr must be aligned. The ram page is not masked as dirty
2519 and the code inside is not invalidated. It is useful if the dirty
2520 bits are used to track modified PTEs */
2521 void stl_phys_notdirty(hwaddr addr
, uint32_t val
)
2528 mr
= address_space_translate(&address_space_memory
, addr
, &addr1
, &l
,
2530 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
2531 io_mem_write(mr
, addr1
, val
, 4);
2533 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
2534 ptr
= qemu_get_ram_ptr(addr1
);
2537 if (unlikely(in_migration
)) {
2538 if (cpu_physical_memory_is_clean(addr1
)) {
2539 /* invalidate code */
2540 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
2542 cpu_physical_memory_set_dirty_flag(addr1
,
2543 DIRTY_MEMORY_MIGRATION
);
2544 cpu_physical_memory_set_dirty_flag(addr1
, DIRTY_MEMORY_VGA
);
2550 /* warning: addr must be aligned */
2551 static inline void stl_phys_internal(hwaddr addr
, uint32_t val
,
2552 enum device_endian endian
)
2559 mr
= address_space_translate(&address_space_memory
, addr
, &addr1
, &l
,
2561 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
2562 #if defined(TARGET_WORDS_BIGENDIAN)
2563 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2567 if (endian
== DEVICE_BIG_ENDIAN
) {
2571 io_mem_write(mr
, addr1
, val
, 4);
2574 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
2575 ptr
= qemu_get_ram_ptr(addr1
);
2577 case DEVICE_LITTLE_ENDIAN
:
2580 case DEVICE_BIG_ENDIAN
:
2587 invalidate_and_set_dirty(addr1
, 4);
2591 void stl_phys(hwaddr addr
, uint32_t val
)
2593 stl_phys_internal(addr
, val
, DEVICE_NATIVE_ENDIAN
);
2596 void stl_le_phys(hwaddr addr
, uint32_t val
)
2598 stl_phys_internal(addr
, val
, DEVICE_LITTLE_ENDIAN
);
2601 void stl_be_phys(hwaddr addr
, uint32_t val
)
2603 stl_phys_internal(addr
, val
, DEVICE_BIG_ENDIAN
);
2607 void stb_phys(hwaddr addr
, uint32_t val
)
2610 cpu_physical_memory_write(addr
, &v
, 1);
2613 /* warning: addr must be aligned */
2614 static inline void stw_phys_internal(hwaddr addr
, uint32_t val
,
2615 enum device_endian endian
)
2622 mr
= address_space_translate(&address_space_memory
, addr
, &addr1
, &l
,
2624 if (l
< 2 || !memory_access_is_direct(mr
, true)) {
2625 #if defined(TARGET_WORDS_BIGENDIAN)
2626 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2630 if (endian
== DEVICE_BIG_ENDIAN
) {
2634 io_mem_write(mr
, addr1
, val
, 2);
2637 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
2638 ptr
= qemu_get_ram_ptr(addr1
);
2640 case DEVICE_LITTLE_ENDIAN
:
2643 case DEVICE_BIG_ENDIAN
:
2650 invalidate_and_set_dirty(addr1
, 2);
2654 void stw_phys(hwaddr addr
, uint32_t val
)
2656 stw_phys_internal(addr
, val
, DEVICE_NATIVE_ENDIAN
);
2659 void stw_le_phys(hwaddr addr
, uint32_t val
)
2661 stw_phys_internal(addr
, val
, DEVICE_LITTLE_ENDIAN
);
2664 void stw_be_phys(hwaddr addr
, uint32_t val
)
2666 stw_phys_internal(addr
, val
, DEVICE_BIG_ENDIAN
);
2670 void stq_phys(hwaddr addr
, uint64_t val
)
2673 cpu_physical_memory_write(addr
, &val
, 8);
2676 void stq_le_phys(hwaddr addr
, uint64_t val
)
2678 val
= cpu_to_le64(val
);
2679 cpu_physical_memory_write(addr
, &val
, 8);
2682 void stq_be_phys(hwaddr addr
, uint64_t val
)
2684 val
= cpu_to_be64(val
);
2685 cpu_physical_memory_write(addr
, &val
, 8);
2688 /* virtual memory access for debug (includes writing to ROM) */
2689 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
2690 uint8_t *buf
, int len
, int is_write
)
2697 page
= addr
& TARGET_PAGE_MASK
;
2698 phys_addr
= cpu_get_phys_page_debug(cpu
, page
);
2699 /* if no physical page mapped, return an error */
2700 if (phys_addr
== -1)
2702 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2705 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
2707 cpu_physical_memory_write_rom(phys_addr
, buf
, l
);
2709 cpu_physical_memory_rw(phys_addr
, buf
, l
, is_write
);
2718 #if !defined(CONFIG_USER_ONLY)
2721 * A helper function for the _utterly broken_ virtio device model to find out if
2722 * it's running on a big endian machine. Don't do this at home kids!
2724 bool virtio_is_big_endian(void);
2725 bool virtio_is_big_endian(void)
2727 #if defined(TARGET_WORDS_BIGENDIAN)
2736 #ifndef CONFIG_USER_ONLY
2737 bool cpu_physical_memory_is_io(hwaddr phys_addr
)
2742 mr
= address_space_translate(&address_space_memory
,
2743 phys_addr
, &phys_addr
, &l
, false);
2745 return !(memory_region_is_ram(mr
) ||
2746 memory_region_is_romd(mr
));
2749 void qemu_ram_foreach_block(RAMBlockIterFunc func
, void *opaque
)
2753 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
2754 func(block
->host
, block
->offset
, block
->length
, opaque
);