2 * Physical memory management
4 * Copyright 2011 Red Hat, Inc. and/or its affiliates
7 * Avi Kivity <avi@redhat.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
15 #include "exec-memory.h"
21 unsigned memory_region_transaction_depth
= 0;
23 typedef struct AddrRange AddrRange
;
26 * Note using signed integers limits us to physical addresses at most
27 * 63 bits wide. They are needed for negative offsetting in aliases
28 * (large MemoryRegion::alias_offset).
35 static AddrRange
addrrange_make(Int128 start
, Int128 size
)
37 return (AddrRange
) { start
, size
};
40 static bool addrrange_equal(AddrRange r1
, AddrRange r2
)
42 return int128_eq(r1
.start
, r2
.start
) && int128_eq(r1
.size
, r2
.size
);
45 static Int128
addrrange_end(AddrRange r
)
47 return int128_add(r
.start
, r
.size
);
50 static AddrRange
addrrange_shift(AddrRange range
, Int128 delta
)
52 int128_addto(&range
.start
, delta
);
56 static bool addrrange_contains(AddrRange range
, Int128 addr
)
58 return int128_ge(addr
, range
.start
)
59 && int128_lt(addr
, addrrange_end(range
));
62 static bool addrrange_intersects(AddrRange r1
, AddrRange r2
)
64 return addrrange_contains(r1
, r2
.start
)
65 || addrrange_contains(r2
, r1
.start
);
68 static AddrRange
addrrange_intersection(AddrRange r1
, AddrRange r2
)
70 Int128 start
= int128_max(r1
.start
, r2
.start
);
71 Int128 end
= int128_min(addrrange_end(r1
), addrrange_end(r2
));
72 return addrrange_make(start
, int128_sub(end
, start
));
75 struct CoalescedMemoryRange
{
77 QTAILQ_ENTRY(CoalescedMemoryRange
) link
;
80 struct MemoryRegionIoeventfd
{
87 static bool memory_region_ioeventfd_before(MemoryRegionIoeventfd a
,
88 MemoryRegionIoeventfd b
)
90 if (int128_lt(a
.addr
.start
, b
.addr
.start
)) {
92 } else if (int128_gt(a
.addr
.start
, b
.addr
.start
)) {
94 } else if (int128_lt(a
.addr
.size
, b
.addr
.size
)) {
96 } else if (int128_gt(a
.addr
.size
, b
.addr
.size
)) {
98 } else if (a
.match_data
< b
.match_data
) {
100 } else if (a
.match_data
> b
.match_data
) {
102 } else if (a
.match_data
) {
103 if (a
.data
< b
.data
) {
105 } else if (a
.data
> b
.data
) {
111 } else if (a
.fd
> b
.fd
) {
117 static bool memory_region_ioeventfd_equal(MemoryRegionIoeventfd a
,
118 MemoryRegionIoeventfd b
)
120 return !memory_region_ioeventfd_before(a
, b
)
121 && !memory_region_ioeventfd_before(b
, a
);
124 typedef struct FlatRange FlatRange
;
125 typedef struct FlatView FlatView
;
127 /* Range of memory in the global map. Addresses are absolute. */
130 target_phys_addr_t offset_in_region
;
132 uint8_t dirty_log_mask
;
137 /* Flattened global view of current active memory hierarchy. Kept in sorted
143 unsigned nr_allocated
;
146 typedef struct AddressSpace AddressSpace
;
147 typedef struct AddressSpaceOps AddressSpaceOps
;
149 /* A system address space - I/O, memory, etc. */
150 struct AddressSpace
{
151 const AddressSpaceOps
*ops
;
153 FlatView current_map
;
155 MemoryRegionIoeventfd
*ioeventfds
;
158 struct AddressSpaceOps
{
159 void (*range_add
)(AddressSpace
*as
, FlatRange
*fr
);
160 void (*range_del
)(AddressSpace
*as
, FlatRange
*fr
);
161 void (*log_start
)(AddressSpace
*as
, FlatRange
*fr
);
162 void (*log_stop
)(AddressSpace
*as
, FlatRange
*fr
);
163 void (*ioeventfd_add
)(AddressSpace
*as
, MemoryRegionIoeventfd
*fd
);
164 void (*ioeventfd_del
)(AddressSpace
*as
, MemoryRegionIoeventfd
*fd
);
167 #define FOR_EACH_FLAT_RANGE(var, view) \
168 for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var)
170 static bool flatrange_equal(FlatRange
*a
, FlatRange
*b
)
172 return a
->mr
== b
->mr
173 && addrrange_equal(a
->addr
, b
->addr
)
174 && a
->offset_in_region
== b
->offset_in_region
175 && a
->readable
== b
->readable
176 && a
->readonly
== b
->readonly
;
179 static void flatview_init(FlatView
*view
)
183 view
->nr_allocated
= 0;
186 /* Insert a range into a given position. Caller is responsible for maintaining
189 static void flatview_insert(FlatView
*view
, unsigned pos
, FlatRange
*range
)
191 if (view
->nr
== view
->nr_allocated
) {
192 view
->nr_allocated
= MAX(2 * view
->nr
, 10);
193 view
->ranges
= g_realloc(view
->ranges
,
194 view
->nr_allocated
* sizeof(*view
->ranges
));
196 memmove(view
->ranges
+ pos
+ 1, view
->ranges
+ pos
,
197 (view
->nr
- pos
) * sizeof(FlatRange
));
198 view
->ranges
[pos
] = *range
;
202 static void flatview_destroy(FlatView
*view
)
204 g_free(view
->ranges
);
207 static bool can_merge(FlatRange
*r1
, FlatRange
*r2
)
209 return int128_eq(addrrange_end(r1
->addr
), r2
->addr
.start
)
211 && int128_eq(int128_add(int128_make64(r1
->offset_in_region
),
213 int128_make64(r2
->offset_in_region
))
214 && r1
->dirty_log_mask
== r2
->dirty_log_mask
215 && r1
->readable
== r2
->readable
216 && r1
->readonly
== r2
->readonly
;
219 /* Attempt to simplify a view by merging ajacent ranges */
220 static void flatview_simplify(FlatView
*view
)
225 while (i
< view
->nr
) {
228 && can_merge(&view
->ranges
[j
-1], &view
->ranges
[j
])) {
229 int128_addto(&view
->ranges
[i
].addr
.size
, view
->ranges
[j
].addr
.size
);
233 memmove(&view
->ranges
[i
], &view
->ranges
[j
],
234 (view
->nr
- j
) * sizeof(view
->ranges
[j
]));
239 static void memory_region_read_accessor(void *opaque
,
240 target_phys_addr_t addr
,
246 MemoryRegion
*mr
= opaque
;
249 tmp
= mr
->ops
->read(mr
->opaque
, addr
, size
);
250 *value
|= (tmp
& mask
) << shift
;
253 static void memory_region_write_accessor(void *opaque
,
254 target_phys_addr_t addr
,
260 MemoryRegion
*mr
= opaque
;
263 tmp
= (*value
>> shift
) & mask
;
264 mr
->ops
->write(mr
->opaque
, addr
, tmp
, size
);
267 static void access_with_adjusted_size(target_phys_addr_t addr
,
270 unsigned access_size_min
,
271 unsigned access_size_max
,
272 void (*access
)(void *opaque
,
273 target_phys_addr_t addr
,
280 uint64_t access_mask
;
281 unsigned access_size
;
284 if (!access_size_min
) {
287 if (!access_size_max
) {
290 access_size
= MAX(MIN(size
, access_size_max
), access_size_min
);
291 access_mask
= -1ULL >> (64 - access_size
* 8);
292 for (i
= 0; i
< size
; i
+= access_size
) {
293 /* FIXME: big-endian support */
294 access(opaque
, addr
+ i
, value
, access_size
, i
* 8, access_mask
);
298 static void memory_region_prepare_ram_addr(MemoryRegion
*mr
);
300 static void as_memory_range_add(AddressSpace
*as
, FlatRange
*fr
)
302 ram_addr_t phys_offset
, region_offset
;
304 memory_region_prepare_ram_addr(fr
->mr
);
306 phys_offset
= fr
->mr
->ram_addr
;
307 region_offset
= fr
->offset_in_region
;
308 /* cpu_register_physical_memory_log() wants region_offset for
309 * mmio, but prefers offseting phys_offset for RAM. Humour it.
311 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
) {
312 phys_offset
+= region_offset
;
317 phys_offset
&= ~TARGET_PAGE_MASK
& ~IO_MEM_ROMD
;
321 phys_offset
|= IO_MEM_ROM
;
324 cpu_register_physical_memory_log(int128_get64(fr
->addr
.start
),
325 int128_get64(fr
->addr
.size
),
331 static void as_memory_range_del(AddressSpace
*as
, FlatRange
*fr
)
333 if (fr
->dirty_log_mask
) {
334 Int128 end
= addrrange_end(fr
->addr
);
335 cpu_physical_sync_dirty_bitmap(int128_get64(fr
->addr
.start
),
338 cpu_register_physical_memory(int128_get64(fr
->addr
.start
),
339 int128_get64(fr
->addr
.size
),
343 static void as_memory_log_start(AddressSpace
*as
, FlatRange
*fr
)
345 cpu_physical_log_start(int128_get64(fr
->addr
.start
),
346 int128_get64(fr
->addr
.size
));
349 static void as_memory_log_stop(AddressSpace
*as
, FlatRange
*fr
)
351 cpu_physical_log_stop(int128_get64(fr
->addr
.start
),
352 int128_get64(fr
->addr
.size
));
355 static void as_memory_ioeventfd_add(AddressSpace
*as
, MemoryRegionIoeventfd
*fd
)
359 assert(fd
->match_data
&& int128_get64(fd
->addr
.size
) == 4);
361 r
= kvm_set_ioeventfd_mmio_long(fd
->fd
, int128_get64(fd
->addr
.start
),
368 static void as_memory_ioeventfd_del(AddressSpace
*as
, MemoryRegionIoeventfd
*fd
)
372 r
= kvm_set_ioeventfd_mmio_long(fd
->fd
, int128_get64(fd
->addr
.start
),
379 static const AddressSpaceOps address_space_ops_memory
= {
380 .range_add
= as_memory_range_add
,
381 .range_del
= as_memory_range_del
,
382 .log_start
= as_memory_log_start
,
383 .log_stop
= as_memory_log_stop
,
384 .ioeventfd_add
= as_memory_ioeventfd_add
,
385 .ioeventfd_del
= as_memory_ioeventfd_del
,
388 static AddressSpace address_space_memory
= {
389 .ops
= &address_space_ops_memory
,
392 static const MemoryRegionPortio
*find_portio(MemoryRegion
*mr
, uint64_t offset
,
393 unsigned width
, bool write
)
395 const MemoryRegionPortio
*mrp
;
397 for (mrp
= mr
->ops
->old_portio
; mrp
->size
; ++mrp
) {
398 if (offset
>= mrp
->offset
&& offset
< mrp
->offset
+ mrp
->len
399 && width
== mrp
->size
400 && (write
? (bool)mrp
->write
: (bool)mrp
->read
)) {
407 static void memory_region_iorange_read(IORange
*iorange
,
412 MemoryRegion
*mr
= container_of(iorange
, MemoryRegion
, iorange
);
414 if (mr
->ops
->old_portio
) {
415 const MemoryRegionPortio
*mrp
= find_portio(mr
, offset
, width
, false);
417 *data
= ((uint64_t)1 << (width
* 8)) - 1;
419 *data
= mrp
->read(mr
->opaque
, offset
+ mr
->offset
);
420 } else if (width
== 2) {
421 mrp
= find_portio(mr
, offset
, 1, false);
423 *data
= mrp
->read(mr
->opaque
, offset
+ mr
->offset
) |
424 (mrp
->read(mr
->opaque
, offset
+ mr
->offset
+ 1) << 8);
429 access_with_adjusted_size(offset
+ mr
->offset
, data
, width
,
430 mr
->ops
->impl
.min_access_size
,
431 mr
->ops
->impl
.max_access_size
,
432 memory_region_read_accessor
, mr
);
435 static void memory_region_iorange_write(IORange
*iorange
,
440 MemoryRegion
*mr
= container_of(iorange
, MemoryRegion
, iorange
);
442 if (mr
->ops
->old_portio
) {
443 const MemoryRegionPortio
*mrp
= find_portio(mr
, offset
, width
, true);
446 mrp
->write(mr
->opaque
, offset
+ mr
->offset
, data
);
447 } else if (width
== 2) {
448 mrp
= find_portio(mr
, offset
, 1, false);
450 mrp
->write(mr
->opaque
, offset
+ mr
->offset
, data
& 0xff);
451 mrp
->write(mr
->opaque
, offset
+ mr
->offset
+ 1, data
>> 8);
455 access_with_adjusted_size(offset
+ mr
->offset
, &data
, width
,
456 mr
->ops
->impl
.min_access_size
,
457 mr
->ops
->impl
.max_access_size
,
458 memory_region_write_accessor
, mr
);
461 static const IORangeOps memory_region_iorange_ops
= {
462 .read
= memory_region_iorange_read
,
463 .write
= memory_region_iorange_write
,
466 static void as_io_range_add(AddressSpace
*as
, FlatRange
*fr
)
468 iorange_init(&fr
->mr
->iorange
, &memory_region_iorange_ops
,
469 int128_get64(fr
->addr
.start
), int128_get64(fr
->addr
.size
));
470 ioport_register(&fr
->mr
->iorange
);
473 static void as_io_range_del(AddressSpace
*as
, FlatRange
*fr
)
475 isa_unassign_ioport(int128_get64(fr
->addr
.start
),
476 int128_get64(fr
->addr
.size
));
479 static void as_io_ioeventfd_add(AddressSpace
*as
, MemoryRegionIoeventfd
*fd
)
483 assert(fd
->match_data
&& int128_get64(fd
->addr
.size
) == 2);
485 r
= kvm_set_ioeventfd_pio_word(fd
->fd
, int128_get64(fd
->addr
.start
),
492 static void as_io_ioeventfd_del(AddressSpace
*as
, MemoryRegionIoeventfd
*fd
)
496 r
= kvm_set_ioeventfd_pio_word(fd
->fd
, int128_get64(fd
->addr
.start
),
503 static const AddressSpaceOps address_space_ops_io
= {
504 .range_add
= as_io_range_add
,
505 .range_del
= as_io_range_del
,
506 .ioeventfd_add
= as_io_ioeventfd_add
,
507 .ioeventfd_del
= as_io_ioeventfd_del
,
510 static AddressSpace address_space_io
= {
511 .ops
= &address_space_ops_io
,
514 /* Render a memory region into the global view. Ranges in @view obscure
517 static void render_memory_region(FlatView
*view
,
523 MemoryRegion
*subregion
;
525 target_phys_addr_t offset_in_region
;
535 int128_addto(&base
, int128_make64(mr
->addr
));
536 readonly
|= mr
->readonly
;
538 tmp
= addrrange_make(base
, mr
->size
);
540 if (!addrrange_intersects(tmp
, clip
)) {
544 clip
= addrrange_intersection(tmp
, clip
);
547 int128_subfrom(&base
, int128_make64(mr
->alias
->addr
));
548 int128_subfrom(&base
, int128_make64(mr
->alias_offset
));
549 render_memory_region(view
, mr
->alias
, base
, clip
, readonly
);
553 /* Render subregions in priority order. */
554 QTAILQ_FOREACH(subregion
, &mr
->subregions
, subregions_link
) {
555 render_memory_region(view
, subregion
, base
, clip
, readonly
);
558 if (!mr
->terminates
) {
562 offset_in_region
= int128_get64(int128_sub(clip
.start
, base
));
566 /* Render the region itself into any gaps left by the current view. */
567 for (i
= 0; i
< view
->nr
&& int128_nz(remain
); ++i
) {
568 if (int128_ge(base
, addrrange_end(view
->ranges
[i
].addr
))) {
571 if (int128_lt(base
, view
->ranges
[i
].addr
.start
)) {
572 now
= int128_min(remain
,
573 int128_sub(view
->ranges
[i
].addr
.start
, base
));
575 fr
.offset_in_region
= offset_in_region
;
576 fr
.addr
= addrrange_make(base
, now
);
577 fr
.dirty_log_mask
= mr
->dirty_log_mask
;
578 fr
.readable
= mr
->readable
;
579 fr
.readonly
= readonly
;
580 flatview_insert(view
, i
, &fr
);
582 int128_addto(&base
, now
);
583 offset_in_region
+= int128_get64(now
);
584 int128_subfrom(&remain
, now
);
586 if (int128_eq(base
, view
->ranges
[i
].addr
.start
)) {
587 now
= int128_min(remain
, view
->ranges
[i
].addr
.size
);
588 int128_addto(&base
, now
);
589 offset_in_region
+= int128_get64(now
);
590 int128_subfrom(&remain
, now
);
593 if (int128_nz(remain
)) {
595 fr
.offset_in_region
= offset_in_region
;
596 fr
.addr
= addrrange_make(base
, remain
);
597 fr
.dirty_log_mask
= mr
->dirty_log_mask
;
598 fr
.readable
= mr
->readable
;
599 fr
.readonly
= readonly
;
600 flatview_insert(view
, i
, &fr
);
604 /* Render a memory topology into a list of disjoint absolute ranges. */
605 static FlatView
generate_memory_topology(MemoryRegion
*mr
)
609 flatview_init(&view
);
611 render_memory_region(&view
, mr
, int128_zero(),
612 addrrange_make(int128_zero(), int128_2_64()), false);
613 flatview_simplify(&view
);
618 static void address_space_add_del_ioeventfds(AddressSpace
*as
,
619 MemoryRegionIoeventfd
*fds_new
,
621 MemoryRegionIoeventfd
*fds_old
,
626 /* Generate a symmetric difference of the old and new fd sets, adding
627 * and deleting as necessary.
631 while (iold
< fds_old_nb
|| inew
< fds_new_nb
) {
632 if (iold
< fds_old_nb
633 && (inew
== fds_new_nb
634 || memory_region_ioeventfd_before(fds_old
[iold
],
636 as
->ops
->ioeventfd_del(as
, &fds_old
[iold
]);
638 } else if (inew
< fds_new_nb
639 && (iold
== fds_old_nb
640 || memory_region_ioeventfd_before(fds_new
[inew
],
642 as
->ops
->ioeventfd_add(as
, &fds_new
[inew
]);
651 static void address_space_update_ioeventfds(AddressSpace
*as
)
654 unsigned ioeventfd_nb
= 0;
655 MemoryRegionIoeventfd
*ioeventfds
= NULL
;
659 FOR_EACH_FLAT_RANGE(fr
, &as
->current_map
) {
660 for (i
= 0; i
< fr
->mr
->ioeventfd_nb
; ++i
) {
661 tmp
= addrrange_shift(fr
->mr
->ioeventfds
[i
].addr
,
662 int128_sub(fr
->addr
.start
,
663 int128_make64(fr
->offset_in_region
)));
664 if (addrrange_intersects(fr
->addr
, tmp
)) {
666 ioeventfds
= g_realloc(ioeventfds
,
667 ioeventfd_nb
* sizeof(*ioeventfds
));
668 ioeventfds
[ioeventfd_nb
-1] = fr
->mr
->ioeventfds
[i
];
669 ioeventfds
[ioeventfd_nb
-1].addr
= tmp
;
674 address_space_add_del_ioeventfds(as
, ioeventfds
, ioeventfd_nb
,
675 as
->ioeventfds
, as
->ioeventfd_nb
);
677 g_free(as
->ioeventfds
);
678 as
->ioeventfds
= ioeventfds
;
679 as
->ioeventfd_nb
= ioeventfd_nb
;
682 static void address_space_update_topology_pass(AddressSpace
*as
,
688 FlatRange
*frold
, *frnew
;
690 /* Generate a symmetric difference of the old and new memory maps.
691 * Kill ranges in the old map, and instantiate ranges in the new map.
694 while (iold
< old_view
.nr
|| inew
< new_view
.nr
) {
695 if (iold
< old_view
.nr
) {
696 frold
= &old_view
.ranges
[iold
];
700 if (inew
< new_view
.nr
) {
701 frnew
= &new_view
.ranges
[inew
];
708 || int128_lt(frold
->addr
.start
, frnew
->addr
.start
)
709 || (int128_eq(frold
->addr
.start
, frnew
->addr
.start
)
710 && !flatrange_equal(frold
, frnew
)))) {
711 /* In old, but (not in new, or in new but attributes changed). */
714 as
->ops
->range_del(as
, frold
);
718 } else if (frold
&& frnew
&& flatrange_equal(frold
, frnew
)) {
719 /* In both (logging may have changed) */
722 if (frold
->dirty_log_mask
&& !frnew
->dirty_log_mask
) {
723 as
->ops
->log_stop(as
, frnew
);
724 } else if (frnew
->dirty_log_mask
&& !frold
->dirty_log_mask
) {
725 as
->ops
->log_start(as
, frnew
);
735 as
->ops
->range_add(as
, frnew
);
744 static void address_space_update_topology(AddressSpace
*as
)
746 FlatView old_view
= as
->current_map
;
747 FlatView new_view
= generate_memory_topology(as
->root
);
749 address_space_update_topology_pass(as
, old_view
, new_view
, false);
750 address_space_update_topology_pass(as
, old_view
, new_view
, true);
752 as
->current_map
= new_view
;
753 flatview_destroy(&old_view
);
754 address_space_update_ioeventfds(as
);
757 static void memory_region_update_topology(MemoryRegion
*mr
)
759 if (memory_region_transaction_depth
) {
763 if (mr
&& !mr
->enabled
) {
767 if (address_space_memory
.root
) {
768 address_space_update_topology(&address_space_memory
);
770 if (address_space_io
.root
) {
771 address_space_update_topology(&address_space_io
);
775 void memory_region_transaction_begin(void)
777 ++memory_region_transaction_depth
;
780 void memory_region_transaction_commit(void)
782 assert(memory_region_transaction_depth
);
783 --memory_region_transaction_depth
;
784 memory_region_update_topology(NULL
);
787 static void memory_region_destructor_none(MemoryRegion
*mr
)
791 static void memory_region_destructor_ram(MemoryRegion
*mr
)
793 qemu_ram_free(mr
->ram_addr
);
796 static void memory_region_destructor_ram_from_ptr(MemoryRegion
*mr
)
798 qemu_ram_free_from_ptr(mr
->ram_addr
);
801 static void memory_region_destructor_iomem(MemoryRegion
*mr
)
803 cpu_unregister_io_memory(mr
->ram_addr
);
806 static void memory_region_destructor_rom_device(MemoryRegion
*mr
)
808 qemu_ram_free(mr
->ram_addr
& TARGET_PAGE_MASK
);
809 cpu_unregister_io_memory(mr
->ram_addr
& ~(TARGET_PAGE_MASK
| IO_MEM_ROMD
));
812 void memory_region_init(MemoryRegion
*mr
,
818 mr
->size
= int128_make64(size
);
819 if (size
== UINT64_MAX
) {
820 mr
->size
= int128_2_64();
825 mr
->terminates
= false;
827 mr
->readonly
= false;
828 mr
->destructor
= memory_region_destructor_none
;
830 mr
->may_overlap
= false;
832 QTAILQ_INIT(&mr
->subregions
);
833 memset(&mr
->subregions_link
, 0, sizeof mr
->subregions_link
);
834 QTAILQ_INIT(&mr
->coalesced
);
835 mr
->name
= g_strdup(name
);
836 mr
->dirty_log_mask
= 0;
837 mr
->ioeventfd_nb
= 0;
838 mr
->ioeventfds
= NULL
;
841 static bool memory_region_access_valid(MemoryRegion
*mr
,
842 target_phys_addr_t addr
,
846 if (mr
->ops
->valid
.accepts
847 && !mr
->ops
->valid
.accepts(mr
->opaque
, addr
, size
, is_write
)) {
851 if (!mr
->ops
->valid
.unaligned
&& (addr
& (size
- 1))) {
855 /* Treat zero as compatibility all valid */
856 if (!mr
->ops
->valid
.max_access_size
) {
860 if (size
> mr
->ops
->valid
.max_access_size
861 || size
< mr
->ops
->valid
.min_access_size
) {
867 static uint32_t memory_region_read_thunk_n(void *_mr
,
868 target_phys_addr_t addr
,
871 MemoryRegion
*mr
= _mr
;
874 if (!memory_region_access_valid(mr
, addr
, size
, false)) {
875 return -1U; /* FIXME: better signalling */
878 if (!mr
->ops
->read
) {
879 return mr
->ops
->old_mmio
.read
[bitops_ffsl(size
)](mr
->opaque
, addr
);
882 /* FIXME: support unaligned access */
883 access_with_adjusted_size(addr
+ mr
->offset
, &data
, size
,
884 mr
->ops
->impl
.min_access_size
,
885 mr
->ops
->impl
.max_access_size
,
886 memory_region_read_accessor
, mr
);
891 static void memory_region_write_thunk_n(void *_mr
,
892 target_phys_addr_t addr
,
896 MemoryRegion
*mr
= _mr
;
898 if (!memory_region_access_valid(mr
, addr
, size
, true)) {
899 return; /* FIXME: better signalling */
902 if (!mr
->ops
->write
) {
903 mr
->ops
->old_mmio
.write
[bitops_ffsl(size
)](mr
->opaque
, addr
, data
);
907 /* FIXME: support unaligned access */
908 access_with_adjusted_size(addr
+ mr
->offset
, &data
, size
,
909 mr
->ops
->impl
.min_access_size
,
910 mr
->ops
->impl
.max_access_size
,
911 memory_region_write_accessor
, mr
);
914 static uint32_t memory_region_read_thunk_b(void *mr
, target_phys_addr_t addr
)
916 return memory_region_read_thunk_n(mr
, addr
, 1);
919 static uint32_t memory_region_read_thunk_w(void *mr
, target_phys_addr_t addr
)
921 return memory_region_read_thunk_n(mr
, addr
, 2);
924 static uint32_t memory_region_read_thunk_l(void *mr
, target_phys_addr_t addr
)
926 return memory_region_read_thunk_n(mr
, addr
, 4);
929 static void memory_region_write_thunk_b(void *mr
, target_phys_addr_t addr
,
932 memory_region_write_thunk_n(mr
, addr
, 1, data
);
935 static void memory_region_write_thunk_w(void *mr
, target_phys_addr_t addr
,
938 memory_region_write_thunk_n(mr
, addr
, 2, data
);
941 static void memory_region_write_thunk_l(void *mr
, target_phys_addr_t addr
,
944 memory_region_write_thunk_n(mr
, addr
, 4, data
);
947 static CPUReadMemoryFunc
* const memory_region_read_thunk
[] = {
948 memory_region_read_thunk_b
,
949 memory_region_read_thunk_w
,
950 memory_region_read_thunk_l
,
953 static CPUWriteMemoryFunc
* const memory_region_write_thunk
[] = {
954 memory_region_write_thunk_b
,
955 memory_region_write_thunk_w
,
956 memory_region_write_thunk_l
,
959 static void memory_region_prepare_ram_addr(MemoryRegion
*mr
)
961 if (mr
->backend_registered
) {
965 mr
->destructor
= memory_region_destructor_iomem
;
966 mr
->ram_addr
= cpu_register_io_memory(memory_region_read_thunk
,
967 memory_region_write_thunk
,
969 mr
->ops
->endianness
);
970 mr
->backend_registered
= true;
973 void memory_region_init_io(MemoryRegion
*mr
,
974 const MemoryRegionOps
*ops
,
979 memory_region_init(mr
, name
, size
);
982 mr
->terminates
= true;
983 mr
->backend_registered
= false;
986 void memory_region_init_ram(MemoryRegion
*mr
,
991 memory_region_init(mr
, name
, size
);
992 mr
->terminates
= true;
993 mr
->destructor
= memory_region_destructor_ram
;
994 mr
->ram_addr
= qemu_ram_alloc(dev
, name
, size
);
995 mr
->backend_registered
= true;
998 void memory_region_init_ram_ptr(MemoryRegion
*mr
,
1004 memory_region_init(mr
, name
, size
);
1005 mr
->terminates
= true;
1006 mr
->destructor
= memory_region_destructor_ram_from_ptr
;
1007 mr
->ram_addr
= qemu_ram_alloc_from_ptr(dev
, name
, size
, ptr
);
1008 mr
->backend_registered
= true;
1011 void memory_region_init_alias(MemoryRegion
*mr
,
1014 target_phys_addr_t offset
,
1017 memory_region_init(mr
, name
, size
);
1019 mr
->alias_offset
= offset
;
1022 void memory_region_init_rom_device(MemoryRegion
*mr
,
1023 const MemoryRegionOps
*ops
,
1029 memory_region_init(mr
, name
, size
);
1031 mr
->opaque
= opaque
;
1032 mr
->terminates
= true;
1033 mr
->destructor
= memory_region_destructor_rom_device
;
1034 mr
->ram_addr
= qemu_ram_alloc(dev
, name
, size
);
1035 mr
->ram_addr
|= cpu_register_io_memory(memory_region_read_thunk
,
1036 memory_region_write_thunk
,
1038 mr
->ops
->endianness
);
1039 mr
->ram_addr
|= IO_MEM_ROMD
;
1040 mr
->backend_registered
= true;
1043 void memory_region_destroy(MemoryRegion
*mr
)
1045 assert(QTAILQ_EMPTY(&mr
->subregions
));
1047 memory_region_clear_coalescing(mr
);
1048 g_free((char *)mr
->name
);
1049 g_free(mr
->ioeventfds
);
1052 uint64_t memory_region_size(MemoryRegion
*mr
)
1054 if (int128_eq(mr
->size
, int128_2_64())) {
1057 return int128_get64(mr
->size
);
1060 void memory_region_set_offset(MemoryRegion
*mr
, target_phys_addr_t offset
)
1062 mr
->offset
= offset
;
1065 void memory_region_set_log(MemoryRegion
*mr
, bool log
, unsigned client
)
1067 uint8_t mask
= 1 << client
;
1069 mr
->dirty_log_mask
= (mr
->dirty_log_mask
& ~mask
) | (log
* mask
);
1070 memory_region_update_topology(mr
);
1073 bool memory_region_get_dirty(MemoryRegion
*mr
, target_phys_addr_t addr
,
1076 assert(mr
->terminates
);
1077 return cpu_physical_memory_get_dirty(mr
->ram_addr
+ addr
, 1 << client
);
1080 void memory_region_set_dirty(MemoryRegion
*mr
, target_phys_addr_t addr
)
1082 assert(mr
->terminates
);
1083 return cpu_physical_memory_set_dirty(mr
->ram_addr
+ addr
);
1086 void memory_region_sync_dirty_bitmap(MemoryRegion
*mr
)
1090 FOR_EACH_FLAT_RANGE(fr
, &address_space_memory
.current_map
) {
1092 cpu_physical_sync_dirty_bitmap(int128_get64(fr
->addr
.start
),
1093 int128_get64(addrrange_end(fr
->addr
)));
1098 void memory_region_set_readonly(MemoryRegion
*mr
, bool readonly
)
1100 if (mr
->readonly
!= readonly
) {
1101 mr
->readonly
= readonly
;
1102 memory_region_update_topology(mr
);
1106 void memory_region_rom_device_set_readable(MemoryRegion
*mr
, bool readable
)
1108 if (mr
->readable
!= readable
) {
1109 mr
->readable
= readable
;
1110 memory_region_update_topology(mr
);
1114 void memory_region_reset_dirty(MemoryRegion
*mr
, target_phys_addr_t addr
,
1115 target_phys_addr_t size
, unsigned client
)
1117 assert(mr
->terminates
);
1118 cpu_physical_memory_reset_dirty(mr
->ram_addr
+ addr
,
1119 mr
->ram_addr
+ addr
+ size
,
1123 void *memory_region_get_ram_ptr(MemoryRegion
*mr
)
1126 return memory_region_get_ram_ptr(mr
->alias
) + mr
->alias_offset
;
1129 assert(mr
->terminates
);
1131 return qemu_get_ram_ptr(mr
->ram_addr
& TARGET_PAGE_MASK
);
1134 static void memory_region_update_coalesced_range(MemoryRegion
*mr
)
1137 CoalescedMemoryRange
*cmr
;
1140 FOR_EACH_FLAT_RANGE(fr
, &address_space_memory
.current_map
) {
1142 qemu_unregister_coalesced_mmio(int128_get64(fr
->addr
.start
),
1143 int128_get64(fr
->addr
.size
));
1144 QTAILQ_FOREACH(cmr
, &mr
->coalesced
, link
) {
1145 tmp
= addrrange_shift(cmr
->addr
,
1146 int128_sub(fr
->addr
.start
,
1147 int128_make64(fr
->offset_in_region
)));
1148 if (!addrrange_intersects(tmp
, fr
->addr
)) {
1151 tmp
= addrrange_intersection(tmp
, fr
->addr
);
1152 qemu_register_coalesced_mmio(int128_get64(tmp
.start
),
1153 int128_get64(tmp
.size
));
1159 void memory_region_set_coalescing(MemoryRegion
*mr
)
1161 memory_region_clear_coalescing(mr
);
1162 memory_region_add_coalescing(mr
, 0, int128_get64(mr
->size
));
1165 void memory_region_add_coalescing(MemoryRegion
*mr
,
1166 target_phys_addr_t offset
,
1169 CoalescedMemoryRange
*cmr
= g_malloc(sizeof(*cmr
));
1171 cmr
->addr
= addrrange_make(int128_make64(offset
), int128_make64(size
));
1172 QTAILQ_INSERT_TAIL(&mr
->coalesced
, cmr
, link
);
1173 memory_region_update_coalesced_range(mr
);
1176 void memory_region_clear_coalescing(MemoryRegion
*mr
)
1178 CoalescedMemoryRange
*cmr
;
1180 while (!QTAILQ_EMPTY(&mr
->coalesced
)) {
1181 cmr
= QTAILQ_FIRST(&mr
->coalesced
);
1182 QTAILQ_REMOVE(&mr
->coalesced
, cmr
, link
);
1185 memory_region_update_coalesced_range(mr
);
1188 void memory_region_add_eventfd(MemoryRegion
*mr
,
1189 target_phys_addr_t addr
,
1195 MemoryRegionIoeventfd mrfd
= {
1196 .addr
.start
= int128_make64(addr
),
1197 .addr
.size
= int128_make64(size
),
1198 .match_data
= match_data
,
1204 for (i
= 0; i
< mr
->ioeventfd_nb
; ++i
) {
1205 if (memory_region_ioeventfd_before(mrfd
, mr
->ioeventfds
[i
])) {
1210 mr
->ioeventfds
= g_realloc(mr
->ioeventfds
,
1211 sizeof(*mr
->ioeventfds
) * mr
->ioeventfd_nb
);
1212 memmove(&mr
->ioeventfds
[i
+1], &mr
->ioeventfds
[i
],
1213 sizeof(*mr
->ioeventfds
) * (mr
->ioeventfd_nb
-1 - i
));
1214 mr
->ioeventfds
[i
] = mrfd
;
1215 memory_region_update_topology(mr
);
1218 void memory_region_del_eventfd(MemoryRegion
*mr
,
1219 target_phys_addr_t addr
,
1225 MemoryRegionIoeventfd mrfd
= {
1226 .addr
.start
= int128_make64(addr
),
1227 .addr
.size
= int128_make64(size
),
1228 .match_data
= match_data
,
1234 for (i
= 0; i
< mr
->ioeventfd_nb
; ++i
) {
1235 if (memory_region_ioeventfd_equal(mrfd
, mr
->ioeventfds
[i
])) {
1239 assert(i
!= mr
->ioeventfd_nb
);
1240 memmove(&mr
->ioeventfds
[i
], &mr
->ioeventfds
[i
+1],
1241 sizeof(*mr
->ioeventfds
) * (mr
->ioeventfd_nb
- (i
+1)));
1243 mr
->ioeventfds
= g_realloc(mr
->ioeventfds
,
1244 sizeof(*mr
->ioeventfds
)*mr
->ioeventfd_nb
+ 1);
1245 memory_region_update_topology(mr
);
1248 static void memory_region_add_subregion_common(MemoryRegion
*mr
,
1249 target_phys_addr_t offset
,
1250 MemoryRegion
*subregion
)
1252 MemoryRegion
*other
;
1254 assert(!subregion
->parent
);
1255 subregion
->parent
= mr
;
1256 subregion
->addr
= offset
;
1257 QTAILQ_FOREACH(other
, &mr
->subregions
, subregions_link
) {
1258 if (subregion
->may_overlap
|| other
->may_overlap
) {
1261 if (int128_gt(int128_make64(offset
),
1262 int128_add(int128_make64(other
->addr
), other
->size
))
1263 || int128_le(int128_add(int128_make64(offset
), subregion
->size
),
1264 int128_make64(other
->addr
))) {
1268 printf("warning: subregion collision %llx/%llx (%s) "
1269 "vs %llx/%llx (%s)\n",
1270 (unsigned long long)offset
,
1271 (unsigned long long)int128_get64(subregion
->size
),
1273 (unsigned long long)other
->addr
,
1274 (unsigned long long)int128_get64(other
->size
),
1278 QTAILQ_FOREACH(other
, &mr
->subregions
, subregions_link
) {
1279 if (subregion
->priority
>= other
->priority
) {
1280 QTAILQ_INSERT_BEFORE(other
, subregion
, subregions_link
);
1284 QTAILQ_INSERT_TAIL(&mr
->subregions
, subregion
, subregions_link
);
1286 memory_region_update_topology(mr
);
1290 void memory_region_add_subregion(MemoryRegion
*mr
,
1291 target_phys_addr_t offset
,
1292 MemoryRegion
*subregion
)
1294 subregion
->may_overlap
= false;
1295 subregion
->priority
= 0;
1296 memory_region_add_subregion_common(mr
, offset
, subregion
);
1299 void memory_region_add_subregion_overlap(MemoryRegion
*mr
,
1300 target_phys_addr_t offset
,
1301 MemoryRegion
*subregion
,
1304 subregion
->may_overlap
= true;
1305 subregion
->priority
= priority
;
1306 memory_region_add_subregion_common(mr
, offset
, subregion
);
1309 void memory_region_del_subregion(MemoryRegion
*mr
,
1310 MemoryRegion
*subregion
)
1312 assert(subregion
->parent
== mr
);
1313 subregion
->parent
= NULL
;
1314 QTAILQ_REMOVE(&mr
->subregions
, subregion
, subregions_link
);
1315 memory_region_update_topology(mr
);
1318 void memory_region_set_enabled(MemoryRegion
*mr
, bool enabled
)
1320 if (enabled
== mr
->enabled
) {
1323 mr
->enabled
= enabled
;
1324 memory_region_update_topology(NULL
);
1327 void memory_region_set_address(MemoryRegion
*mr
, target_phys_addr_t addr
)
1329 MemoryRegion
*parent
= mr
->parent
;
1330 unsigned priority
= mr
->priority
;
1331 bool may_overlap
= mr
->may_overlap
;
1333 if (addr
== mr
->addr
|| !parent
) {
1338 memory_region_transaction_begin();
1339 memory_region_del_subregion(parent
, mr
);
1341 memory_region_add_subregion_overlap(parent
, addr
, mr
, priority
);
1343 memory_region_add_subregion(parent
, addr
, mr
);
1345 memory_region_transaction_commit();
1348 void memory_region_set_alias_offset(MemoryRegion
*mr
, target_phys_addr_t offset
)
1350 target_phys_addr_t old_offset
= mr
->alias_offset
;
1353 mr
->alias_offset
= offset
;
1355 if (offset
== old_offset
|| !mr
->parent
) {
1359 memory_region_update_topology(mr
);
1362 void set_system_memory_map(MemoryRegion
*mr
)
1364 address_space_memory
.root
= mr
;
1365 memory_region_update_topology(NULL
);
1368 void set_system_io_map(MemoryRegion
*mr
)
1370 address_space_io
.root
= mr
;
1371 memory_region_update_topology(NULL
);
1374 typedef struct MemoryRegionList MemoryRegionList
;
1376 struct MemoryRegionList
{
1377 const MemoryRegion
*mr
;
1379 QTAILQ_ENTRY(MemoryRegionList
) queue
;
1382 typedef QTAILQ_HEAD(queue
, MemoryRegionList
) MemoryRegionListHead
;
1384 static void mtree_print_mr(fprintf_function mon_printf
, void *f
,
1385 const MemoryRegion
*mr
, unsigned int level
,
1386 target_phys_addr_t base
,
1387 MemoryRegionListHead
*alias_print_queue
)
1389 MemoryRegionList
*new_ml
, *ml
, *next_ml
;
1390 MemoryRegionListHead submr_print_queue
;
1391 const MemoryRegion
*submr
;
1398 for (i
= 0; i
< level
; i
++) {
1403 MemoryRegionList
*ml
;
1406 /* check if the alias is already in the queue */
1407 QTAILQ_FOREACH(ml
, alias_print_queue
, queue
) {
1408 if (ml
->mr
== mr
->alias
&& !ml
->printed
) {
1414 ml
= g_new(MemoryRegionList
, 1);
1416 ml
->printed
= false;
1417 QTAILQ_INSERT_TAIL(alias_print_queue
, ml
, queue
);
1419 mon_printf(f
, TARGET_FMT_plx
"-" TARGET_FMT_plx
" (prio %d): alias %s @%s "
1420 TARGET_FMT_plx
"-" TARGET_FMT_plx
"\n",
1423 + (target_phys_addr_t
)int128_get64(mr
->size
) - 1,
1429 + (target_phys_addr_t
)int128_get64(mr
->size
) - 1);
1431 mon_printf(f
, TARGET_FMT_plx
"-" TARGET_FMT_plx
" (prio %d): %s\n",
1434 + (target_phys_addr_t
)int128_get64(mr
->size
) - 1,
1439 QTAILQ_INIT(&submr_print_queue
);
1441 QTAILQ_FOREACH(submr
, &mr
->subregions
, subregions_link
) {
1442 new_ml
= g_new(MemoryRegionList
, 1);
1444 QTAILQ_FOREACH(ml
, &submr_print_queue
, queue
) {
1445 if (new_ml
->mr
->addr
< ml
->mr
->addr
||
1446 (new_ml
->mr
->addr
== ml
->mr
->addr
&&
1447 new_ml
->mr
->priority
> ml
->mr
->priority
)) {
1448 QTAILQ_INSERT_BEFORE(ml
, new_ml
, queue
);
1454 QTAILQ_INSERT_TAIL(&submr_print_queue
, new_ml
, queue
);
1458 QTAILQ_FOREACH(ml
, &submr_print_queue
, queue
) {
1459 mtree_print_mr(mon_printf
, f
, ml
->mr
, level
+ 1, base
+ mr
->addr
,
1463 QTAILQ_FOREACH_SAFE(ml
, &submr_print_queue
, queue
, next_ml
) {
1468 void mtree_info(fprintf_function mon_printf
, void *f
)
1470 MemoryRegionListHead ml_head
;
1471 MemoryRegionList
*ml
, *ml2
;
1473 QTAILQ_INIT(&ml_head
);
1475 mon_printf(f
, "memory\n");
1476 mtree_print_mr(mon_printf
, f
, address_space_memory
.root
, 0, 0, &ml_head
);
1478 /* print aliased regions */
1479 QTAILQ_FOREACH(ml
, &ml_head
, queue
) {
1481 mon_printf(f
, "%s\n", ml
->mr
->name
);
1482 mtree_print_mr(mon_printf
, f
, ml
->mr
, 0, 0, &ml_head
);
1486 QTAILQ_FOREACH_SAFE(ml
, &ml_head
, queue
, ml2
) {
1490 if (address_space_io
.root
&&
1491 !QTAILQ_EMPTY(&address_space_io
.root
->subregions
)) {
1492 QTAILQ_INIT(&ml_head
);
1493 mon_printf(f
, "I/O\n");
1494 mtree_print_mr(mon_printf
, f
, address_space_io
.root
, 0, 0, &ml_head
);