?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
intel_iommu: redo configuraton check in realize
[qemu.git] / memory.c
blob58f92693e29a99107dbe9d3c2dd529e49ae3aac0
1 /*
2 * Physical memory management
3 *
4 * Copyright 2011 Red Hat, Inc. and/or its affiliates
5 *
6 * Authors:
7 * Avi Kivity <avi@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 * Contributions after 2012-01-13 are licensed under the terms of the
13 * GNU GPL, version 2 or (at your option) any later version.
14 */
15
16 #include "qemu/osdep.h"
17 #include "qapi/error.h"
18 #include "qemu-common.h"
19 #include "cpu.h"
20 #include "exec/memory.h"
21 #include "exec/address-spaces.h"
22 #include "exec/ioport.h"
23 #include "qapi/visitor.h"
24 #include "qemu/bitops.h"
25 #include "qemu/error-report.h"
26 #include "qom/object.h"
27 #include "trace.h"
28
29 #include "exec/memory-internal.h"
30 #include "exec/ram_addr.h"
31 #include "sysemu/kvm.h"
32 #include "sysemu/sysemu.h"
33
34 //#define DEBUG_UNASSIGNED
35
36 static unsigned memory_region_transaction_depth;
37 static bool memory_region_update_pending;
38 static bool ioeventfd_update_pending;
39 static bool global_dirty_log = false;
40
41 static QTAILQ_HEAD(memory_listeners, MemoryListener) memory_listeners
42 = QTAILQ_HEAD_INITIALIZER(memory_listeners);
43
44 static QTAILQ_HEAD(, AddressSpace) address_spaces
45 = QTAILQ_HEAD_INITIALIZER(address_spaces);
46
47 typedef struct AddrRange AddrRange;
48
49 /*
50 * Note that signed integers are needed for negative offsetting in aliases
51 * (large MemoryRegion::alias_offset).
52 */
53 struct AddrRange {
54 Int128 start;
55 Int128 size;
56 };
57
58 static AddrRange addrrange_make(Int128 start, Int128 size)
59 {
60 return (AddrRange) { start, size };
61 }
62
63 static bool addrrange_equal(AddrRange r1, AddrRange r2)
64 {
65 return int128_eq(r1.start, r2.start) && int128_eq(r1.size, r2.size);
66 }
67
68 static Int128 addrrange_end(AddrRange r)
69 {
70 return int128_add(r.start, r.size);
71 }
72
73 static AddrRange addrrange_shift(AddrRange range, Int128 delta)
74 {
75 int128_addto(&range.start, delta);
76 return range;
77 }
78
79 static bool addrrange_contains(AddrRange range, Int128 addr)
80 {
81 return int128_ge(addr, range.start)
82 && int128_lt(addr, addrrange_end(range));
83 }
84
85 static bool addrrange_intersects(AddrRange r1, AddrRange r2)
86 {
87 return addrrange_contains(r1, r2.start)
88 || addrrange_contains(r2, r1.start);
89 }
90
91 static AddrRange addrrange_intersection(AddrRange r1, AddrRange r2)
92 {
93 Int128 start = int128_max(r1.start, r2.start);
94 Int128 end = int128_min(addrrange_end(r1), addrrange_end(r2));
95 return addrrange_make(start, int128_sub(end, start));
96 }
97
98 enum ListenerDirection { Forward, Reverse };
99
100 static bool memory_listener_match(MemoryListener *listener,
101 MemoryRegionSection *section)
102 {
103 return !listener->address_space_filter
104 || listener->address_space_filter == section->address_space;
105 }
106
107 #define MEMORY_LISTENER_CALL_GLOBAL(_callback, _direction, _args...) \
108 do { \
109 MemoryListener *_listener; \
110 \
111 switch (_direction) { \
112 case Forward: \
113 QTAILQ_FOREACH(_listener, &memory_listeners, link) { \
114 if (_listener->_callback) { \
115 _listener->_callback(_listener, ##_args); \
116 } \
117 } \
118 break; \
119 case Reverse: \
120 QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \
121 memory_listeners, link) { \
122 if (_listener->_callback) { \
123 _listener->_callback(_listener, ##_args); \
124 } \
125 } \
126 break; \
127 default: \
128 abort(); \
129 } \
130 } while (0)
131
132 #define MEMORY_LISTENER_CALL(_callback, _direction, _section, _args...) \
133 do { \
134 MemoryListener *_listener; \
135 \
136 switch (_direction) { \
137 case Forward: \
138 QTAILQ_FOREACH(_listener, &memory_listeners, link) { \
139 if (_listener->_callback \
140 && memory_listener_match(_listener, _section)) { \
141 _listener->_callback(_listener, _section, ##_args); \
142 } \
143 } \
144 break; \
145 case Reverse: \
146 QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \
147 memory_listeners, link) { \
148 if (_listener->_callback \
149 && memory_listener_match(_listener, _section)) { \
150 _listener->_callback(_listener, _section, ##_args); \
151 } \
152 } \
153 break; \
154 default: \
155 abort(); \
156 } \
157 } while (0)
158
159 /* No need to ref/unref .mr, the FlatRange keeps it alive. */
160 #define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback, _args...) \
161 do { \
162 MemoryRegionSection mrs = section_from_flat_range(fr, as); \
163 MEMORY_LISTENER_CALL(callback, dir, &mrs, ##_args); \
164 } while(0)
165
166 struct CoalescedMemoryRange {
167 AddrRange addr;
168 QTAILQ_ENTRY(CoalescedMemoryRange) link;
169 };
170
171 struct MemoryRegionIoeventfd {
172 AddrRange addr;
173 bool match_data;
174 uint64_t data;
175 EventNotifier *e;
176 };
177
178 static bool memory_region_ioeventfd_before(MemoryRegionIoeventfd a,
179 MemoryRegionIoeventfd b)
180 {
181 if (int128_lt(a.addr.start, b.addr.start)) {
182 return true;
183 } else if (int128_gt(a.addr.start, b.addr.start)) {
184 return false;
185 } else if (int128_lt(a.addr.size, b.addr.size)) {
186 return true;
187 } else if (int128_gt(a.addr.size, b.addr.size)) {
188 return false;
189 } else if (a.match_data < b.match_data) {
190 return true;
191 } else if (a.match_data > b.match_data) {
192 return false;
193 } else if (a.match_data) {
194 if (a.data < b.data) {
195 return true;
196 } else if (a.data > b.data) {
197 return false;
198 }
199 }
200 if (a.e < b.e) {
201 return true;
202 } else if (a.e > b.e) {
203 return false;
204 }
205 return false;
206 }
207
208 static bool memory_region_ioeventfd_equal(MemoryRegionIoeventfd a,
209 MemoryRegionIoeventfd b)
210 {
211 return !memory_region_ioeventfd_before(a, b)
212 && !memory_region_ioeventfd_before(b, a);
213 }
214
215 typedef struct FlatRange FlatRange;
216 typedef struct FlatView FlatView;
217
218 /* Range of memory in the global map. Addresses are absolute. */
219 struct FlatRange {
220 MemoryRegion *mr;
221 hwaddr offset_in_region;
222 AddrRange addr;
223 uint8_t dirty_log_mask;
224 bool romd_mode;
225 bool readonly;
226 };
227
228 /* Flattened global view of current active memory hierarchy. Kept in sorted
229 * order.
230 */
231 struct FlatView {
232 struct rcu_head rcu;
233 unsigned ref;
234 FlatRange *ranges;
235 unsigned nr;
236 unsigned nr_allocated;
237 };
238
239 typedef struct AddressSpaceOps AddressSpaceOps;
240
241 #define FOR_EACH_FLAT_RANGE(var, view) \
242 for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var)
243
244 static inline MemoryRegionSection
245 section_from_flat_range(FlatRange *fr, AddressSpace *as)
246 {
247 return (MemoryRegionSection) {
248 .mr = fr->mr,
249 .address_space = as,
250 .offset_within_region = fr->offset_in_region,
251 .size = fr->addr.size,
252 .offset_within_address_space = int128_get64(fr->addr.start),
253 .readonly = fr->readonly,
254 };
255 }
256
257 static bool flatrange_equal(FlatRange *a, FlatRange *b)
258 {
259 return a->mr == b->mr
260 && addrrange_equal(a->addr, b->addr)
261 && a->offset_in_region == b->offset_in_region
262 && a->romd_mode == b->romd_mode
263 && a->readonly == b->readonly;
264 }
265
266 static void flatview_init(FlatView *view)
267 {
268 view->ref = 1;
269 view->ranges = NULL;
270 view->nr = 0;
271 view->nr_allocated = 0;
272 }
273
274 /* Insert a range into a given position. Caller is responsible for maintaining
275 * sorting order.
276 */
277 static void flatview_insert(FlatView *view, unsigned pos, FlatRange *range)
278 {
279 if (view->nr == view->nr_allocated) {
280 view->nr_allocated = MAX(2 * view->nr, 10);
281 view->ranges = g_realloc(view->ranges,
282 view->nr_allocated * sizeof(*view->ranges));
283 }
284 memmove(view->ranges + pos + 1, view->ranges + pos,
285 (view->nr - pos) * sizeof(FlatRange));
286 view->ranges[pos] = *range;
287 memory_region_ref(range->mr);
288 ++view->nr;
289 }
290
291 static void flatview_destroy(FlatView *view)
292 {
293 int i;
294
295 for (i = 0; i < view->nr; i++) {
296 memory_region_unref(view->ranges[i].mr);
297 }
298 g_free(view->ranges);
299 g_free(view);
300 }
301
302 static void flatview_ref(FlatView *view)
303 {
304 atomic_inc(&view->ref);
305 }
306
307 static void flatview_unref(FlatView *view)
308 {
309 if (atomic_fetch_dec(&view->ref) == 1) {
310 flatview_destroy(view);
311 }
312 }
313
314 static bool can_merge(FlatRange *r1, FlatRange *r2)
315 {
316 return int128_eq(addrrange_end(r1->addr), r2->addr.start)
317 && r1->mr == r2->mr
318 && int128_eq(int128_add(int128_make64(r1->offset_in_region),
319 r1->addr.size),
320 int128_make64(r2->offset_in_region))
321 && r1->dirty_log_mask == r2->dirty_log_mask
322 && r1->romd_mode == r2->romd_mode
323 && r1->readonly == r2->readonly;
324 }
325
326 /* Attempt to simplify a view by merging adjacent ranges */
327 static void flatview_simplify(FlatView *view)
328 {
329 unsigned i, j;
330
331 i = 0;
332 while (i < view->nr) {
333 j = i + 1;
334 while (j < view->nr
335 && can_merge(&view->ranges[j-1], &view->ranges[j])) {
336 int128_addto(&view->ranges[i].addr.size, view->ranges[j].addr.size);
337 ++j;
338 }
339 ++i;
340 memmove(&view->ranges[i], &view->ranges[j],
341 (view->nr - j) * sizeof(view->ranges[j]));
342 view->nr -= j - i;
343 }
344 }
345
346 static bool memory_region_big_endian(MemoryRegion *mr)
347 {
348 #ifdef TARGET_WORDS_BIGENDIAN
349 return mr->ops->endianness != DEVICE_LITTLE_ENDIAN;
350 #else
351 return mr->ops->endianness == DEVICE_BIG_ENDIAN;
352 #endif
353 }
354
355 static bool memory_region_wrong_endianness(MemoryRegion *mr)
356 {
357 #ifdef TARGET_WORDS_BIGENDIAN
358 return mr->ops->endianness == DEVICE_LITTLE_ENDIAN;
359 #else
360 return mr->ops->endianness == DEVICE_BIG_ENDIAN;
361 #endif
362 }
363
364 static void adjust_endianness(MemoryRegion *mr, uint64_t *data, unsigned size)
365 {
366 if (memory_region_wrong_endianness(mr)) {
367 switch (size) {
368 case 1:
369 break;
370 case 2:
371 *data = bswap16(*data);
372 break;
373 case 4:
374 *data = bswap32(*data);
375 break;
376 case 8:
377 *data = bswap64(*data);
378 break;
379 default:
380 abort();
381 }
382 }
383 }
384
385 static hwaddr memory_region_to_absolute_addr(MemoryRegion *mr, hwaddr offset)
386 {
387 MemoryRegion *root;
388 hwaddr abs_addr = offset;
389
390 abs_addr += mr->addr;
391 for (root = mr; root->container; ) {
392 root = root->container;
393 abs_addr += root->addr;
394 }
395
396 return abs_addr;
397 }
398
399 static int get_cpu_index(void)
400 {
401 if (current_cpu) {
402 return current_cpu->cpu_index;
403 }
404 return -1;
405 }
406
407 static MemTxResult memory_region_oldmmio_read_accessor(MemoryRegion *mr,
408 hwaddr addr,
409 uint64_t *value,
410 unsigned size,
411 unsigned shift,
412 uint64_t mask,
413 MemTxAttrs attrs)
414 {
415 uint64_t tmp;
416
417 tmp = mr->ops->old_mmio.read[ctz32(size)](mr->opaque, addr);
418 if (mr->subpage) {
419 trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);
420 } else if (mr == &io_mem_notdirty) {
421 /* Accesses to code which has previously been translated into a TB show
422 * up in the MMIO path, as accesses to the io_mem_notdirty
423 * MemoryRegion. */
424 trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);
425 } else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {
426 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
427 trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);
428 }
429 *value |= (tmp & mask) << shift;
430 return MEMTX_OK;
431 }
432
433 static MemTxResult memory_region_read_accessor(MemoryRegion *mr,
434 hwaddr addr,
435 uint64_t *value,
436 unsigned size,
437 unsigned shift,
438 uint64_t mask,
439 MemTxAttrs attrs)
440 {
441 uint64_t tmp;
442
443 tmp = mr->ops->read(mr->opaque, addr, size);
444 if (mr->subpage) {
445 trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);
446 } else if (mr == &io_mem_notdirty) {
447 /* Accesses to code which has previously been translated into a TB show
448 * up in the MMIO path, as accesses to the io_mem_notdirty
449 * MemoryRegion. */
450 trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);
451 } else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {
452 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
453 trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);
454 }
455 *value |= (tmp & mask) << shift;
456 return MEMTX_OK;
457 }
458
459 static MemTxResult memory_region_read_with_attrs_accessor(MemoryRegion *mr,
460 hwaddr addr,
461 uint64_t *value,
462 unsigned size,
463 unsigned shift,
464 uint64_t mask,
465 MemTxAttrs attrs)
466 {
467 uint64_t tmp = 0;
468 MemTxResult r;
469
470 r = mr->ops->read_with_attrs(mr->opaque, addr, &tmp, size, attrs);
471 if (mr->subpage) {
472 trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);
473 } else if (mr == &io_mem_notdirty) {
474 /* Accesses to code which has previously been translated into a TB show
475 * up in the MMIO path, as accesses to the io_mem_notdirty
476 * MemoryRegion. */
477 trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);
478 } else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {
479 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
480 trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);
481 }
482 *value |= (tmp & mask) << shift;
483 return r;
484 }
485
486 static MemTxResult memory_region_oldmmio_write_accessor(MemoryRegion *mr,
487 hwaddr addr,
488 uint64_t *value,
489 unsigned size,
490 unsigned shift,
491 uint64_t mask,
492 MemTxAttrs attrs)
493 {
494 uint64_t tmp;
495
496 tmp = (*value >> shift) & mask;
497 if (mr->subpage) {
498 trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);
499 } else if (mr == &io_mem_notdirty) {
500 /* Accesses to code which has previously been translated into a TB show
501 * up in the MMIO path, as accesses to the io_mem_notdirty
502 * MemoryRegion. */
503 trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);
504 } else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {
505 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
506 trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);
507 }
508 mr->ops->old_mmio.write[ctz32(size)](mr->opaque, addr, tmp);
509 return MEMTX_OK;
510 }
511
512 static MemTxResult memory_region_write_accessor(MemoryRegion *mr,
513 hwaddr addr,
514 uint64_t *value,
515 unsigned size,
516 unsigned shift,
517 uint64_t mask,
518 MemTxAttrs attrs)
519 {
520 uint64_t tmp;
521
522 tmp = (*value >> shift) & mask;
523 if (mr->subpage) {
524 trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);
525 } else if (mr == &io_mem_notdirty) {
526 /* Accesses to code which has previously been translated into a TB show
527 * up in the MMIO path, as accesses to the io_mem_notdirty
528 * MemoryRegion. */
529 trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);
530 } else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {
531 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
532 trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);
533 }
534 mr->ops->write(mr->opaque, addr, tmp, size);
535 return MEMTX_OK;
536 }
537
538 static MemTxResult memory_region_write_with_attrs_accessor(MemoryRegion *mr,
539 hwaddr addr,
540 uint64_t *value,
541 unsigned size,
542 unsigned shift,
543 uint64_t mask,
544 MemTxAttrs attrs)
545 {
546 uint64_t tmp;
547
548 tmp = (*value >> shift) & mask;
549 if (mr->subpage) {
550 trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);
551 } else if (mr == &io_mem_notdirty) {
552 /* Accesses to code which has previously been translated into a TB show
553 * up in the MMIO path, as accesses to the io_mem_notdirty
554 * MemoryRegion. */
555 trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);
556 } else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {
557 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
558 trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);
559 }
560 return mr->ops->write_with_attrs(mr->opaque, addr, tmp, size, attrs);
561 }
562
563 static MemTxResult access_with_adjusted_size(hwaddr addr,
564 uint64_t *value,
565 unsigned size,
566 unsigned access_size_min,
567 unsigned access_size_max,
568 MemTxResult (*access)(MemoryRegion *mr,
569 hwaddr addr,
570 uint64_t *value,
571 unsigned size,
572 unsigned shift,
573 uint64_t mask,
574 MemTxAttrs attrs),
575 MemoryRegion *mr,
576 MemTxAttrs attrs)
577 {
578 uint64_t access_mask;
579 unsigned access_size;
580 unsigned i;
581 MemTxResult r = MEMTX_OK;
582
583 if (!access_size_min) {
584 access_size_min = 1;
585 }
586 if (!access_size_max) {
587 access_size_max = 4;
588 }
589
590 /* FIXME: support unaligned access? */
591 access_size = MAX(MIN(size, access_size_max), access_size_min);
592 access_mask = -1ULL >> (64 - access_size * 8);
593 if (memory_region_big_endian(mr)) {
594 for (i = 0; i < size; i += access_size) {
595 r |= access(mr, addr + i, value, access_size,
596 (size - access_size - i) * 8, access_mask, attrs);
597 }
598 } else {
599 for (i = 0; i < size; i += access_size) {
600 r |= access(mr, addr + i, value, access_size, i * 8,
601 access_mask, attrs);
602 }
603 }
604 return r;
605 }
606
607 static AddressSpace *memory_region_to_address_space(MemoryRegion *mr)
608 {
609 AddressSpace *as;
610
611 while (mr->container) {
612 mr = mr->container;
613 }
614 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
615 if (mr == as->root) {
616 return as;
617 }
618 }
619 return NULL;
620 }
621
622 /* Render a memory region into the global view. Ranges in @view obscure
623 * ranges in @mr.
624 */
625 static void render_memory_region(FlatView *view,
626 MemoryRegion *mr,
627 Int128 base,
628 AddrRange clip,
629 bool readonly)
630 {
631 MemoryRegion *subregion;
632 unsigned i;
633 hwaddr offset_in_region;
634 Int128 remain;
635 Int128 now;
636 FlatRange fr;
637 AddrRange tmp;
638
639 if (!mr->enabled) {
640 return;
641 }
642
643 int128_addto(&base, int128_make64(mr->addr));
644 readonly |= mr->readonly;
645
646 tmp = addrrange_make(base, mr->size);
647
648 if (!addrrange_intersects(tmp, clip)) {
649 return;
650 }
651
652 clip = addrrange_intersection(tmp, clip);
653
654 if (mr->alias) {
655 int128_subfrom(&base, int128_make64(mr->alias->addr));
656 int128_subfrom(&base, int128_make64(mr->alias_offset));
657 render_memory_region(view, mr->alias, base, clip, readonly);
658 return;
659 }
660
661 /* Render subregions in priority order. */
662 QTAILQ_FOREACH(subregion, &mr->subregions, subregions_link) {
663 render_memory_region(view, subregion, base, clip, readonly);
664 }
665
666 if (!mr->terminates) {
667 return;
668 }
669
670 offset_in_region = int128_get64(int128_sub(clip.start, base));
671 base = clip.start;
672 remain = clip.size;
673
674 fr.mr = mr;
675 fr.dirty_log_mask = memory_region_get_dirty_log_mask(mr);
676 fr.romd_mode = mr->romd_mode;
677 fr.readonly = readonly;
678
679 /* Render the region itself into any gaps left by the current view. */
680 for (i = 0; i < view->nr && int128_nz(remain); ++i) {
681 if (int128_ge(base, addrrange_end(view->ranges[i].addr))) {
682 continue;
683 }
684 if (int128_lt(base, view->ranges[i].addr.start)) {
685 now = int128_min(remain,
686 int128_sub(view->ranges[i].addr.start, base));
687 fr.offset_in_region = offset_in_region;
688 fr.addr = addrrange_make(base, now);
689 flatview_insert(view, i, &fr);
690 ++i;
691 int128_addto(&base, now);
692 offset_in_region += int128_get64(now);
693 int128_subfrom(&remain, now);
694 }
695 now = int128_sub(int128_min(int128_add(base, remain),
696 addrrange_end(view->ranges[i].addr)),
697 base);
698 int128_addto(&base, now);
699 offset_in_region += int128_get64(now);
700 int128_subfrom(&remain, now);
701 }
702 if (int128_nz(remain)) {
703 fr.offset_in_region = offset_in_region;
704 fr.addr = addrrange_make(base, remain);
705 flatview_insert(view, i, &fr);
706 }
707 }
708
709 /* Render a memory topology into a list of disjoint absolute ranges. */
710 static FlatView *generate_memory_topology(MemoryRegion *mr)
711 {
712 FlatView *view;
713
714 view = g_new(FlatView, 1);
715 flatview_init(view);
716
717 if (mr) {
718 render_memory_region(view, mr, int128_zero(),
719 addrrange_make(int128_zero(), int128_2_64()), false);
720 }
721 flatview_simplify(view);
722
723 return view;
724 }
725
726 static void address_space_add_del_ioeventfds(AddressSpace *as,
727 MemoryRegionIoeventfd *fds_new,
728 unsigned fds_new_nb,
729 MemoryRegionIoeventfd *fds_old,
730 unsigned fds_old_nb)
731 {
732 unsigned iold, inew;
733 MemoryRegionIoeventfd *fd;
734 MemoryRegionSection section;
735
736 /* Generate a symmetric difference of the old and new fd sets, adding
737 * and deleting as necessary.
738 */
739
740 iold = inew = 0;
741 while (iold < fds_old_nb || inew < fds_new_nb) {
742 if (iold < fds_old_nb
743 && (inew == fds_new_nb
744 || memory_region_ioeventfd_before(fds_old[iold],
745 fds_new[inew]))) {
746 fd = &fds_old[iold];
747 section = (MemoryRegionSection) {
748 .address_space = as,
749 .offset_within_address_space = int128_get64(fd->addr.start),
750 .size = fd->addr.size,
751 };
752 MEMORY_LISTENER_CALL(eventfd_del, Forward, &section,
753 fd->match_data, fd->data, fd->e);
754 ++iold;
755 } else if (inew < fds_new_nb
756 && (iold == fds_old_nb
757 || memory_region_ioeventfd_before(fds_new[inew],
758 fds_old[iold]))) {
759 fd = &fds_new[inew];
760 section = (MemoryRegionSection) {
761 .address_space = as,
762 .offset_within_address_space = int128_get64(fd->addr.start),
763 .size = fd->addr.size,
764 };
765 MEMORY_LISTENER_CALL(eventfd_add, Reverse, &section,
766 fd->match_data, fd->data, fd->e);
767 ++inew;
768 } else {
769 ++iold;
770 ++inew;
771 }
772 }
773 }
774
775 static FlatView *address_space_get_flatview(AddressSpace *as)
776 {
777 FlatView *view;
778
779 rcu_read_lock();
780 view = atomic_rcu_read(&as->current_map);
781 flatview_ref(view);
782 rcu_read_unlock();
783 return view;
784 }
785
786 static void address_space_update_ioeventfds(AddressSpace *as)
787 {
788 FlatView *view;
789 FlatRange *fr;
790 unsigned ioeventfd_nb = 0;
791 MemoryRegionIoeventfd *ioeventfds = NULL;
792 AddrRange tmp;
793 unsigned i;
794
795 view = address_space_get_flatview(as);
796 FOR_EACH_FLAT_RANGE(fr, view) {
797 for (i = 0; i < fr->mr->ioeventfd_nb; ++i) {
798 tmp = addrrange_shift(fr->mr->ioeventfds[i].addr,
799 int128_sub(fr->addr.start,
800 int128_make64(fr->offset_in_region)));
801 if (addrrange_intersects(fr->addr, tmp)) {
802 ++ioeventfd_nb;
803 ioeventfds = g_realloc(ioeventfds,
804 ioeventfd_nb * sizeof(*ioeventfds));
805 ioeventfds[ioeventfd_nb-1] = fr->mr->ioeventfds[i];
806 ioeventfds[ioeventfd_nb-1].addr = tmp;
807 }
808 }
809 }
810
811 address_space_add_del_ioeventfds(as, ioeventfds, ioeventfd_nb,
812 as->ioeventfds, as->ioeventfd_nb);
813
814 g_free(as->ioeventfds);
815 as->ioeventfds = ioeventfds;
816 as->ioeventfd_nb = ioeventfd_nb;
817 flatview_unref(view);
818 }
819
820 static void address_space_update_topology_pass(AddressSpace *as,
821 const FlatView *old_view,
822 const FlatView *new_view,
823 bool adding)
824 {
825 unsigned iold, inew;
826 FlatRange *frold, *frnew;
827
828 /* Generate a symmetric difference of the old and new memory maps.
829 * Kill ranges in the old map, and instantiate ranges in the new map.
830 */
831 iold = inew = 0;
832 while (iold < old_view->nr || inew < new_view->nr) {
833 if (iold < old_view->nr) {
834 frold = &old_view->ranges[iold];
835 } else {
836 frold = NULL;
837 }
838 if (inew < new_view->nr) {
839 frnew = &new_view->ranges[inew];
840 } else {
841 frnew = NULL;
842 }
843
844 if (frold
845 && (!frnew
846 || int128_lt(frold->addr.start, frnew->addr.start)
847 || (int128_eq(frold->addr.start, frnew->addr.start)
848 && !flatrange_equal(frold, frnew)))) {
849 /* In old but not in new, or in both but attributes changed. */
850
851 if (!adding) {
852 MEMORY_LISTENER_UPDATE_REGION(frold, as, Reverse, region_del);
853 }
854
855 ++iold;
856 } else if (frold && frnew && flatrange_equal(frold, frnew)) {
857 /* In both and unchanged (except logging may have changed) */
858
859 if (adding) {
860 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_nop);
861 if (frnew->dirty_log_mask & ~frold->dirty_log_mask) {
862 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, log_start,
863 frold->dirty_log_mask,
864 frnew->dirty_log_mask);
865 }
866 if (frold->dirty_log_mask & ~frnew->dirty_log_mask) {
867 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop,
868 frold->dirty_log_mask,
869 frnew->dirty_log_mask);
870 }
871 }
872
873 ++iold;
874 ++inew;
875 } else {
876 /* In new */
877
878 if (adding) {
879 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_add);
880 }
881
882 ++inew;
883 }
884 }
885 }
886
887
888 static void address_space_update_topology(AddressSpace *as)
889 {
890 FlatView *old_view = address_space_get_flatview(as);
891 FlatView *new_view = generate_memory_topology(as->root);
892
893 address_space_update_topology_pass(as, old_view, new_view, false);
894 address_space_update_topology_pass(as, old_view, new_view, true);
895
896 /* Writes are protected by the BQL. */
897 atomic_rcu_set(&as->current_map, new_view);
898 call_rcu(old_view, flatview_unref, rcu);
899
900 /* Note that all the old MemoryRegions are still alive up to this
901 * point. This relieves most MemoryListeners from the need to
902 * ref/unref the MemoryRegions they get---unless they use them
903 * outside the iothread mutex, in which case precise reference
904 * counting is necessary.
905 */
906 flatview_unref(old_view);
907
908 address_space_update_ioeventfds(as);
909 }
910
911 void memory_region_transaction_begin(void)
912 {
913 qemu_flush_coalesced_mmio_buffer();
914 ++memory_region_transaction_depth;
915 }
916
917 static void memory_region_clear_pending(void)
918 {
919 memory_region_update_pending = false;
920 ioeventfd_update_pending = false;
921 }
922
923 void memory_region_transaction_commit(void)
924 {
925 AddressSpace *as;
926
927 assert(memory_region_transaction_depth);
928 --memory_region_transaction_depth;
929 if (!memory_region_transaction_depth) {
930 if (memory_region_update_pending) {
931 MEMORY_LISTENER_CALL_GLOBAL(begin, Forward);
932
933 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
934 address_space_update_topology(as);
935 }
936
937 MEMORY_LISTENER_CALL_GLOBAL(commit, Forward);
938 } else if (ioeventfd_update_pending) {
939 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
940 address_space_update_ioeventfds(as);
941 }
942 }
943 memory_region_clear_pending();
944 }
945 }
946
947 static void memory_region_destructor_none(MemoryRegion *mr)
948 {
949 }
950
951 static void memory_region_destructor_ram(MemoryRegion *mr)
952 {
953 qemu_ram_free(mr->ram_block);
954 }
955
956 static bool memory_region_need_escape(char c)
957 {
958 return c == '/' || c == '[' || c == '\\' || c == ']';
959 }
960
961 static char *memory_region_escape_name(const char *name)
962 {
963 const char *p;
964 char *escaped, *q;
965 uint8_t c;
966 size_t bytes = 0;
967
968 for (p = name; *p; p++) {
969 bytes += memory_region_need_escape(*p) ? 4 : 1;
970 }
971 if (bytes == p - name) {
972 return g_memdup(name, bytes + 1);
973 }
974
975 escaped = g_malloc(bytes + 1);
976 for (p = name, q = escaped; *p; p++) {
977 c = *p;
978 if (unlikely(memory_region_need_escape(c))) {
979 *q++ = '\\';
980 *q++ = 'x';
981 *q++ = "0123456789abcdef"[c >> 4];
982 c = "0123456789abcdef"[c & 15];
983 }
984 *q++ = c;
985 }
986 *q = 0;
987 return escaped;
988 }
989
990 void memory_region_init(MemoryRegion *mr,
991 Object *owner,
992 const char *name,
993 uint64_t size)
994 {
995 object_initialize(mr, sizeof(*mr), TYPE_MEMORY_REGION);
996 mr->size = int128_make64(size);
997 if (size == UINT64_MAX) {
998 mr->size = int128_2_64();
999 }
1000 mr->name = g_strdup(name);
1001 mr->owner = owner;
1002 mr->ram_block = NULL;
1003
1004 if (name) {
1005 char *escaped_name = memory_region_escape_name(name);
1006 char *name_array = g_strdup_printf("%s[*]", escaped_name);
1007
1008 if (!owner) {
1009 owner = container_get(qdev_get_machine(), "/unattached");
1010 }
1011
1012 object_property_add_child(owner, name_array, OBJECT(mr), &error_abort);
1013 object_unref(OBJECT(mr));
1014 g_free(name_array);
1015 g_free(escaped_name);
1016 }
1017 }
1018
1019 static void memory_region_get_addr(Object *obj, Visitor *v, const char *name,
1020 void *opaque, Error **errp)
1021 {
1022 MemoryRegion *mr = MEMORY_REGION(obj);
1023 uint64_t value = mr->addr;
1024
1025 visit_type_uint64(v, name, &value, errp);
1026 }
1027
1028 static void memory_region_get_container(Object *obj, Visitor *v,
1029 const char *name, void *opaque,
1030 Error **errp)
1031 {
1032 MemoryRegion *mr = MEMORY_REGION(obj);
1033 gchar *path = (gchar *)"";
1034
1035 if (mr->container) {
1036 path = object_get_canonical_path(OBJECT(mr->container));
1037 }
1038 visit_type_str(v, name, &path, errp);
1039 if (mr->container) {
1040 g_free(path);
1041 }
1042 }
1043
1044 static Object *memory_region_resolve_container(Object *obj, void *opaque,
1045 const char *part)
1046 {
1047 MemoryRegion *mr = MEMORY_REGION(obj);
1048
1049 return OBJECT(mr->container);
1050 }
1051
1052 static void memory_region_get_priority(Object *obj, Visitor *v,
1053 const char *name, void *opaque,
1054 Error **errp)
1055 {
1056 MemoryRegion *mr = MEMORY_REGION(obj);
1057 int32_t value = mr->priority;
1058
1059 visit_type_int32(v, name, &value, errp);
1060 }
1061
1062 static void memory_region_get_size(Object *obj, Visitor *v, const char *name,
1063 void *opaque, Error **errp)
1064 {
1065 MemoryRegion *mr = MEMORY_REGION(obj);
1066 uint64_t value = memory_region_size(mr);
1067
1068 visit_type_uint64(v, name, &value, errp);
1069 }
1070
1071 static void memory_region_initfn(Object *obj)
1072 {
1073 MemoryRegion *mr = MEMORY_REGION(obj);
1074 ObjectProperty *op;
1075
1076 mr->ops = &unassigned_mem_ops;
1077 mr->enabled = true;
1078 mr->romd_mode = true;
1079 mr->global_locking = true;
1080 mr->destructor = memory_region_destructor_none;
1081 QTAILQ_INIT(&mr->subregions);
1082 QTAILQ_INIT(&mr->coalesced);
1083
1084 op = object_property_add(OBJECT(mr), "container",
1085 "link<" TYPE_MEMORY_REGION ">",
1086 memory_region_get_container,
1087 NULL, /* memory_region_set_container */
1088 NULL, NULL, &error_abort);
1089 op->resolve = memory_region_resolve_container;
1090
1091 object_property_add(OBJECT(mr), "addr", "uint64",
1092 memory_region_get_addr,
1093 NULL, /* memory_region_set_addr */
1094 NULL, NULL, &error_abort);
1095 object_property_add(OBJECT(mr), "priority", "uint32",
1096 memory_region_get_priority,
1097 NULL, /* memory_region_set_priority */
1098 NULL, NULL, &error_abort);
1099 object_property_add(OBJECT(mr), "size", "uint64",
1100 memory_region_get_size,
1101 NULL, /* memory_region_set_size, */
1102 NULL, NULL, &error_abort);
1103 }
1104
1105 static uint64_t unassigned_mem_read(void *opaque, hwaddr addr,
1106 unsigned size)
1107 {
1108 #ifdef DEBUG_UNASSIGNED
1109 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
1110 #endif
1111 if (current_cpu != NULL) {
1112 cpu_unassigned_access(current_cpu, addr, false, false, 0, size);
1113 }
1114 return 0;
1115 }
1116
1117 static void unassigned_mem_write(void *opaque, hwaddr addr,
1118 uint64_t val, unsigned size)
1119 {
1120 #ifdef DEBUG_UNASSIGNED
1121 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%"PRIx64"\n", addr, val);
1122 #endif
1123 if (current_cpu != NULL) {
1124 cpu_unassigned_access(current_cpu, addr, true, false, 0, size);
1125 }
1126 }
1127
1128 static bool unassigned_mem_accepts(void *opaque, hwaddr addr,
1129 unsigned size, bool is_write)
1130 {
1131 return false;
1132 }
1133
1134 const MemoryRegionOps unassigned_mem_ops = {
1135 .valid.accepts = unassigned_mem_accepts,
1136 .endianness = DEVICE_NATIVE_ENDIAN,
1137 };
1138
1139 bool memory_region_access_valid(MemoryRegion *mr,
1140 hwaddr addr,
1141 unsigned size,
1142 bool is_write)
1143 {
1144 int access_size_min, access_size_max;
1145 int access_size, i;
1146
1147 if (!mr->ops->valid.unaligned && (addr & (size - 1))) {
1148 return false;
1149 }
1150
1151 if (!mr->ops->valid.accepts) {
1152 return true;
1153 }
1154
1155 access_size_min = mr->ops->valid.min_access_size;
1156 if (!mr->ops->valid.min_access_size) {
1157 access_size_min = 1;
1158 }
1159
1160 access_size_max = mr->ops->valid.max_access_size;
1161 if (!mr->ops->valid.max_access_size) {
1162 access_size_max = 4;
1163 }
1164
1165 access_size = MAX(MIN(size, access_size_max), access_size_min);
1166 for (i = 0; i < size; i += access_size) {
1167 if (!mr->ops->valid.accepts(mr->opaque, addr + i, access_size,
1168 is_write)) {
1169 return false;
1170 }
1171 }
1172
1173 return true;
1174 }
1175
1176 static MemTxResult memory_region_dispatch_read1(MemoryRegion *mr,
1177 hwaddr addr,
1178 uint64_t *pval,
1179 unsigned size,
1180 MemTxAttrs attrs)
1181 {
1182 *pval = 0;
1183
1184 if (mr->ops->read) {
1185 return access_with_adjusted_size(addr, pval, size,
1186 mr->ops->impl.min_access_size,
1187 mr->ops->impl.max_access_size,
1188 memory_region_read_accessor,
1189 mr, attrs);
1190 } else if (mr->ops->read_with_attrs) {
1191 return access_with_adjusted_size(addr, pval, size,
1192 mr->ops->impl.min_access_size,
1193 mr->ops->impl.max_access_size,
1194 memory_region_read_with_attrs_accessor,
1195 mr, attrs);
1196 } else {
1197 return access_with_adjusted_size(addr, pval, size, 1, 4,
1198 memory_region_oldmmio_read_accessor,
1199 mr, attrs);
1200 }
1201 }
1202
1203 MemTxResult memory_region_dispatch_read(MemoryRegion *mr,
1204 hwaddr addr,
1205 uint64_t *pval,
1206 unsigned size,
1207 MemTxAttrs attrs)
1208 {
1209 MemTxResult r;
1210
1211 if (!memory_region_access_valid(mr, addr, size, false)) {
1212 *pval = unassigned_mem_read(mr, addr, size);
1213 return MEMTX_DECODE_ERROR;
1214 }
1215
1216 r = memory_region_dispatch_read1(mr, addr, pval, size, attrs);
1217 adjust_endianness(mr, pval, size);
1218 return r;
1219 }
1220
1221 /* Return true if an eventfd was signalled */
1222 static bool memory_region_dispatch_write_eventfds(MemoryRegion *mr,
1223 hwaddr addr,
1224 uint64_t data,
1225 unsigned size,
1226 MemTxAttrs attrs)
1227 {
1228 MemoryRegionIoeventfd ioeventfd = {
1229 .addr = addrrange_make(int128_make64(addr), int128_make64(size)),
1230 .data = data,
1231 };
1232 unsigned i;
1233
1234 for (i = 0; i < mr->ioeventfd_nb; i++) {
1235 ioeventfd.match_data = mr->ioeventfds[i].match_data;
1236 ioeventfd.e = mr->ioeventfds[i].e;
1237
1238 if (memory_region_ioeventfd_equal(ioeventfd, mr->ioeventfds[i])) {
1239 event_notifier_set(ioeventfd.e);
1240 return true;
1241 }
1242 }
1243
1244 return false;
1245 }
1246
1247 MemTxResult memory_region_dispatch_write(MemoryRegion *mr,
1248 hwaddr addr,
1249 uint64_t data,
1250 unsigned size,
1251 MemTxAttrs attrs)
1252 {
1253 if (!memory_region_access_valid(mr, addr, size, true)) {
1254 unassigned_mem_write(mr, addr, data, size);
1255 return MEMTX_DECODE_ERROR;
1256 }
1257
1258 adjust_endianness(mr, &data, size);
1259
1260 if ((!kvm_eventfds_enabled()) &&
1261 memory_region_dispatch_write_eventfds(mr, addr, data, size, attrs)) {
1262 return MEMTX_OK;
1263 }
1264
1265 if (mr->ops->write) {
1266 return access_with_adjusted_size(addr, &data, size,
1267 mr->ops->impl.min_access_size,
1268 mr->ops->impl.max_access_size,
1269 memory_region_write_accessor, mr,
1270 attrs);
1271 } else if (mr->ops->write_with_attrs) {
1272 return
1273 access_with_adjusted_size(addr, &data, size,
1274 mr->ops->impl.min_access_size,
1275 mr->ops->impl.max_access_size,
1276 memory_region_write_with_attrs_accessor,
1277 mr, attrs);
1278 } else {
1279 return access_with_adjusted_size(addr, &data, size, 1, 4,
1280 memory_region_oldmmio_write_accessor,
1281 mr, attrs);
1282 }
1283 }
1284
1285 void memory_region_init_io(MemoryRegion *mr,
1286 Object *owner,
1287 const MemoryRegionOps *ops,
1288 void *opaque,
1289 const char *name,
1290 uint64_t size)
1291 {
1292 memory_region_init(mr, owner, name, size);
1293 mr->ops = ops ? ops : &unassigned_mem_ops;
1294 mr->opaque = opaque;
1295 mr->terminates = true;
1296 }
1297
1298 void memory_region_init_ram(MemoryRegion *mr,
1299 Object *owner,
1300 const char *name,
1301 uint64_t size,
1302 Error **errp)
1303 {
1304 memory_region_init(mr, owner, name, size);
1305 mr->ram = true;
1306 mr->terminates = true;
1307 mr->destructor = memory_region_destructor_ram;
1308 mr->ram_block = qemu_ram_alloc(size, mr, errp);
1309 mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
1310 }
1311
1312 void memory_region_init_resizeable_ram(MemoryRegion *mr,
1313 Object *owner,
1314 const char *name,
1315 uint64_t size,
1316 uint64_t max_size,
1317 void (*resized)(const char*,
1318 uint64_t length,
1319 void *host),
1320 Error **errp)
1321 {
1322 memory_region_init(mr, owner, name, size);
1323 mr->ram = true;
1324 mr->terminates = true;
1325 mr->destructor = memory_region_destructor_ram;
1326 mr->ram_block = qemu_ram_alloc_resizeable(size, max_size, resized,
1327 mr, errp);
1328 mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
1329 }
1330
1331 #ifdef __linux__
1332 void memory_region_init_ram_from_file(MemoryRegion *mr,
1333 struct Object *owner,
1334 const char *name,
1335 uint64_t size,
1336 bool share,
1337 const char *path,
1338 Error **errp)
1339 {
1340 memory_region_init(mr, owner, name, size);
1341 mr->ram = true;
1342 mr->terminates = true;
1343 mr->destructor = memory_region_destructor_ram;
1344 mr->ram_block = qemu_ram_alloc_from_file(size, mr, share, path, errp);
1345 mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
1346 }
1347 #endif
1348
1349 void memory_region_init_ram_ptr(MemoryRegion *mr,
1350 Object *owner,
1351 const char *name,
1352 uint64_t size,
1353 void *ptr)
1354 {
1355 memory_region_init(mr, owner, name, size);
1356 mr->ram = true;
1357 mr->terminates = true;
1358 mr->destructor = memory_region_destructor_ram;
1359 mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
1360
1361 /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL. */
1362 assert(ptr != NULL);
1363 mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_fatal);
1364 }
1365
1366 void memory_region_set_skip_dump(MemoryRegion *mr)
1367 {
1368 mr->skip_dump = true;
1369 }
1370
1371 void memory_region_init_alias(MemoryRegion *mr,
1372 Object *owner,
1373 const char *name,
1374 MemoryRegion *orig,
1375 hwaddr offset,
1376 uint64_t size)
1377 {
1378 memory_region_init(mr, owner, name, size);
1379 mr->alias = orig;
1380 mr->alias_offset = offset;
1381 }
1382
1383 void memory_region_init_rom(MemoryRegion *mr,
1384 struct Object *owner,
1385 const char *name,
1386 uint64_t size,
1387 Error **errp)
1388 {
1389 memory_region_init(mr, owner, name, size);
1390 mr->ram = true;
1391 mr->readonly = true;
1392 mr->terminates = true;
1393 mr->destructor = memory_region_destructor_ram;
1394 mr->ram_block = qemu_ram_alloc(size, mr, errp);
1395 mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
1396 }
1397
1398 void memory_region_init_rom_device(MemoryRegion *mr,
1399 Object *owner,
1400 const MemoryRegionOps *ops,
1401 void *opaque,
1402 const char *name,
1403 uint64_t size,
1404 Error **errp)
1405 {
1406 assert(ops);
1407 memory_region_init(mr, owner, name, size);
1408 mr->ops = ops;
1409 mr->opaque = opaque;
1410 mr->terminates = true;
1411 mr->rom_device = true;
1412 mr->destructor = memory_region_destructor_ram;
1413 mr->ram_block = qemu_ram_alloc(size, mr, errp);
1414 }
1415
1416 void memory_region_init_iommu(MemoryRegion *mr,
1417 Object *owner,
1418 const MemoryRegionIOMMUOps *ops,
1419 const char *name,
1420 uint64_t size)
1421 {
1422 memory_region_init(mr, owner, name, size);
1423 mr->iommu_ops = ops,
1424 mr->terminates = true; /* then re-forwards */
1425 QLIST_INIT(&mr->iommu_notify);
1426 mr->iommu_notify_flags = IOMMU_NOTIFIER_NONE;
1427 }
1428
1429 static void memory_region_finalize(Object *obj)
1430 {
1431 MemoryRegion *mr = MEMORY_REGION(obj);
1432
1433 assert(!mr->container);
1434
1435 /* We know the region is not visible in any address space (it
1436 * does not have a container and cannot be a root either because
1437 * it has no references, so we can blindly clear mr->enabled.
1438 * memory_region_set_enabled instead could trigger a transaction
1439 * and cause an infinite loop.
1440 */
1441 mr->enabled = false;
1442 memory_region_transaction_begin();
1443 while (!QTAILQ_EMPTY(&mr->subregions)) {
1444 MemoryRegion *subregion = QTAILQ_FIRST(&mr->subregions);
1445 memory_region_del_subregion(mr, subregion);
1446 }
1447 memory_region_transaction_commit();
1448
1449 mr->destructor(mr);
1450 memory_region_clear_coalescing(mr);
1451 g_free((char *)mr->name);
1452 g_free(mr->ioeventfds);
1453 }
1454
1455 Object *memory_region_owner(MemoryRegion *mr)
1456 {
1457 Object *obj = OBJECT(mr);
1458 return obj->parent;
1459 }
1460
1461 void memory_region_ref(MemoryRegion *mr)
1462 {
1463 /* MMIO callbacks most likely will access data that belongs
1464 * to the owner, hence the need to ref/unref the owner whenever
1465 * the memory region is in use.
1466 *
1467 * The memory region is a child of its owner. As long as the
1468 * owner doesn't call unparent itself on the memory region,
1469 * ref-ing the owner will also keep the memory region alive.
1470 * Memory regions without an owner are supposed to never go away;
1471 * we do not ref/unref them because it slows down DMA sensibly.
1472 */
1473 if (mr && mr->owner) {
1474 object_ref(mr->owner);
1475 }
1476 }
1477
1478 void memory_region_unref(MemoryRegion *mr)
1479 {
1480 if (mr && mr->owner) {
1481 object_unref(mr->owner);
1482 }
1483 }
1484
1485 uint64_t memory_region_size(MemoryRegion *mr)
1486 {
1487 if (int128_eq(mr->size, int128_2_64())) {
1488 return UINT64_MAX;
1489 }
1490 return int128_get64(mr->size);
1491 }
1492
1493 const char *memory_region_name(const MemoryRegion *mr)
1494 {
1495 if (!mr->name) {
1496 ((MemoryRegion *)mr)->name =
1497 object_get_canonical_path_component(OBJECT(mr));
1498 }
1499 return mr->name;
1500 }
1501
1502 bool memory_region_is_skip_dump(MemoryRegion *mr)
1503 {
1504 return mr->skip_dump;
1505 }
1506
1507 uint8_t memory_region_get_dirty_log_mask(MemoryRegion *mr)
1508 {
1509 uint8_t mask = mr->dirty_log_mask;
1510 if (global_dirty_log) {
1511 mask |= (1 << DIRTY_MEMORY_MIGRATION);
1512 }
1513 return mask;
1514 }
1515
1516 bool memory_region_is_logging(MemoryRegion *mr, uint8_t client)
1517 {
1518 return memory_region_get_dirty_log_mask(mr) & (1 << client);
1519 }
1520
1521 static void memory_region_update_iommu_notify_flags(MemoryRegion *mr)
1522 {
1523 IOMMUNotifierFlag flags = IOMMU_NOTIFIER_NONE;
1524 IOMMUNotifier *iommu_notifier;
1525
1526 QLIST_FOREACH(iommu_notifier, &mr->iommu_notify, node) {
1527 flags |= iommu_notifier->notifier_flags;
1528 }
1529
1530 if (flags != mr->iommu_notify_flags &&
1531 mr->iommu_ops->notify_flag_changed) {
1532 mr->iommu_ops->notify_flag_changed(mr, mr->iommu_notify_flags,
1533 flags);
1534 }
1535
1536 mr->iommu_notify_flags = flags;
1537 }
1538
1539 void memory_region_register_iommu_notifier(MemoryRegion *mr,
1540 IOMMUNotifier *n)
1541 {
1542 /* We need to register for at least one bitfield */
1543 assert(n->notifier_flags != IOMMU_NOTIFIER_NONE);
1544 QLIST_INSERT_HEAD(&mr->iommu_notify, n, node);
1545 memory_region_update_iommu_notify_flags(mr);
1546 }
1547
1548 uint64_t memory_region_iommu_get_min_page_size(MemoryRegion *mr)
1549 {
1550 assert(memory_region_is_iommu(mr));
1551 if (mr->iommu_ops && mr->iommu_ops->get_min_page_size) {
1552 return mr->iommu_ops->get_min_page_size(mr);
1553 }
1554 return TARGET_PAGE_SIZE;
1555 }
1556
1557 void memory_region_iommu_replay(MemoryRegion *mr, IOMMUNotifier *n,
1558 bool is_write)
1559 {
1560 hwaddr addr, granularity;
1561 IOMMUTLBEntry iotlb;
1562
1563 granularity = memory_region_iommu_get_min_page_size(mr);
1564
1565 for (addr = 0; addr < memory_region_size(mr); addr += granularity) {
1566 iotlb = mr->iommu_ops->translate(mr, addr, is_write);
1567 if (iotlb.perm != IOMMU_NONE) {
1568 n->notify(n, &iotlb);
1569 }
1570
1571 /* if (2^64 - MR size) < granularity, it's possible to get an
1572 * infinite loop here. This should catch such a wraparound */
1573 if ((addr + granularity) < addr) {
1574 break;
1575 }
1576 }
1577 }
1578
1579 void memory_region_unregister_iommu_notifier(MemoryRegion *mr,
1580 IOMMUNotifier *n)
1581 {
1582 QLIST_REMOVE(n, node);
1583 memory_region_update_iommu_notify_flags(mr);
1584 }
1585
1586 void memory_region_notify_iommu(MemoryRegion *mr,
1587 IOMMUTLBEntry entry)
1588 {
1589 IOMMUNotifier *iommu_notifier;
1590 IOMMUNotifierFlag request_flags;
1591
1592 assert(memory_region_is_iommu(mr));
1593
1594 if (entry.perm & IOMMU_RW) {
1595 request_flags = IOMMU_NOTIFIER_MAP;
1596 } else {
1597 request_flags = IOMMU_NOTIFIER_UNMAP;
1598 }
1599
1600 QLIST_FOREACH(iommu_notifier, &mr->iommu_notify, node) {
1601 if (iommu_notifier->notifier_flags & request_flags) {
1602 iommu_notifier->notify(iommu_notifier, &entry);
1603 }
1604 }
1605 }
1606
1607 void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client)
1608 {
1609 uint8_t mask = 1 << client;
1610 uint8_t old_logging;
1611
1612 assert(client == DIRTY_MEMORY_VGA);
1613 old_logging = mr->vga_logging_count;
1614 mr->vga_logging_count += log ? 1 : -1;
1615 if (!!old_logging == !!mr->vga_logging_count) {
1616 return;
1617 }
1618
1619 memory_region_transaction_begin();
1620 mr->dirty_log_mask = (mr->dirty_log_mask & ~mask) | (log * mask);
1621 memory_region_update_pending |= mr->enabled;
1622 memory_region_transaction_commit();
1623 }
1624
1625 bool memory_region_get_dirty(MemoryRegion *mr, hwaddr addr,
1626 hwaddr size, unsigned client)
1627 {
1628 assert(mr->ram_block);
1629 return cpu_physical_memory_get_dirty(memory_region_get_ram_addr(mr) + addr,
1630 size, client);
1631 }
1632
1633 void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr,
1634 hwaddr size)
1635 {
1636 assert(mr->ram_block);
1637 cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr,
1638 size,
1639 memory_region_get_dirty_log_mask(mr));
1640 }
1641
1642 bool memory_region_test_and_clear_dirty(MemoryRegion *mr, hwaddr addr,
1643 hwaddr size, unsigned client)
1644 {
1645 assert(mr->ram_block);
1646 return cpu_physical_memory_test_and_clear_dirty(
1647 memory_region_get_ram_addr(mr) + addr, size, client);
1648 }
1649
1650
1651 void memory_region_sync_dirty_bitmap(MemoryRegion *mr)
1652 {
1653 AddressSpace *as;
1654 FlatRange *fr;
1655
1656 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
1657 FlatView *view = address_space_get_flatview(as);
1658 FOR_EACH_FLAT_RANGE(fr, view) {
1659 if (fr->mr == mr) {
1660 MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, log_sync);
1661 }
1662 }
1663 flatview_unref(view);
1664 }
1665 }
1666
1667 void memory_region_set_readonly(MemoryRegion *mr, bool readonly)
1668 {
1669 if (mr->readonly != readonly) {
1670 memory_region_transaction_begin();
1671 mr->readonly = readonly;
1672 memory_region_update_pending |= mr->enabled;
1673 memory_region_transaction_commit();
1674 }
1675 }
1676
1677 void memory_region_rom_device_set_romd(MemoryRegion *mr, bool romd_mode)
1678 {
1679 if (mr->romd_mode != romd_mode) {
1680 memory_region_transaction_begin();
1681 mr->romd_mode = romd_mode;
1682 memory_region_update_pending |= mr->enabled;
1683 memory_region_transaction_commit();
1684 }
1685 }
1686
1687 void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr,
1688 hwaddr size, unsigned client)
1689 {
1690 assert(mr->ram_block);
1691 cpu_physical_memory_test_and_clear_dirty(
1692 memory_region_get_ram_addr(mr) + addr, size, client);
1693 }
1694
1695 int memory_region_get_fd(MemoryRegion *mr)
1696 {
1697 int fd;
1698
1699 rcu_read_lock();
1700 while (mr->alias) {
1701 mr = mr->alias;
1702 }
1703 fd = mr->ram_block->fd;
1704 rcu_read_unlock();
1705
1706 return fd;
1707 }
1708
1709 void memory_region_set_fd(MemoryRegion *mr, int fd)
1710 {
1711 rcu_read_lock();
1712 while (mr->alias) {
1713 mr = mr->alias;
1714 }
1715 mr->ram_block->fd = fd;
1716 rcu_read_unlock();
1717 }
1718
1719 void *memory_region_get_ram_ptr(MemoryRegion *mr)
1720 {
1721 void *ptr;
1722 uint64_t offset = 0;
1723
1724 rcu_read_lock();
1725 while (mr->alias) {
1726 offset += mr->alias_offset;
1727 mr = mr->alias;
1728 }
1729 assert(mr->ram_block);
1730 ptr = qemu_map_ram_ptr(mr->ram_block, offset);
1731 rcu_read_unlock();
1732
1733 return ptr;
1734 }
1735
1736 MemoryRegion *memory_region_from_host(void *ptr, ram_addr_t *offset)
1737 {
1738 RAMBlock *block;
1739
1740 block = qemu_ram_block_from_host(ptr, false, offset);
1741 if (!block) {
1742 return NULL;
1743 }
1744
1745 return block->mr;
1746 }
1747
1748 ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr)
1749 {
1750 return mr->ram_block ? mr->ram_block->offset : RAM_ADDR_INVALID;
1751 }
1752
1753 void memory_region_ram_resize(MemoryRegion *mr, ram_addr_t newsize, Error **errp)
1754 {
1755 assert(mr->ram_block);
1756
1757 qemu_ram_resize(mr->ram_block, newsize, errp);
1758 }
1759
1760 static void memory_region_update_coalesced_range_as(MemoryRegion *mr, AddressSpace *as)
1761 {
1762 FlatView *view;
1763 FlatRange *fr;
1764 CoalescedMemoryRange *cmr;
1765 AddrRange tmp;
1766 MemoryRegionSection section;
1767
1768 view = address_space_get_flatview(as);
1769 FOR_EACH_FLAT_RANGE(fr, view) {
1770 if (fr->mr == mr) {
1771 section = (MemoryRegionSection) {
1772 .address_space = as,
1773 .offset_within_address_space = int128_get64(fr->addr.start),
1774 .size = fr->addr.size,
1775 };
1776
1777 MEMORY_LISTENER_CALL(coalesced_mmio_del, Reverse, &section,
1778 int128_get64(fr->addr.start),
1779 int128_get64(fr->addr.size));
1780 QTAILQ_FOREACH(cmr, &mr->coalesced, link) {
1781 tmp = addrrange_shift(cmr->addr,
1782 int128_sub(fr->addr.start,
1783 int128_make64(fr->offset_in_region)));
1784 if (!addrrange_intersects(tmp, fr->addr)) {
1785 continue;
1786 }
1787 tmp = addrrange_intersection(tmp, fr->addr);
1788 MEMORY_LISTENER_CALL(coalesced_mmio_add, Forward, &section,
1789 int128_get64(tmp.start),
1790 int128_get64(tmp.size));
1791 }
1792 }
1793 }
1794 flatview_unref(view);
1795 }
1796
1797 static void memory_region_update_coalesced_range(MemoryRegion *mr)
1798 {
1799 AddressSpace *as;
1800
1801 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
1802 memory_region_update_coalesced_range_as(mr, as);
1803 }
1804 }
1805
1806 void memory_region_set_coalescing(MemoryRegion *mr)
1807 {
1808 memory_region_clear_coalescing(mr);
1809 memory_region_add_coalescing(mr, 0, int128_get64(mr->size));
1810 }
1811
1812 void memory_region_add_coalescing(MemoryRegion *mr,
1813 hwaddr offset,
1814 uint64_t size)
1815 {
1816 CoalescedMemoryRange *cmr = g_malloc(sizeof(*cmr));
1817
1818 cmr->addr = addrrange_make(int128_make64(offset), int128_make64(size));
1819 QTAILQ_INSERT_TAIL(&mr->coalesced, cmr, link);
1820 memory_region_update_coalesced_range(mr);
1821 memory_region_set_flush_coalesced(mr);
1822 }
1823
1824 void memory_region_clear_coalescing(MemoryRegion *mr)
1825 {
1826 CoalescedMemoryRange *cmr;
1827 bool updated = false;
1828
1829 qemu_flush_coalesced_mmio_buffer();
1830 mr->flush_coalesced_mmio = false;
1831
1832 while (!QTAILQ_EMPTY(&mr->coalesced)) {
1833 cmr = QTAILQ_FIRST(&mr->coalesced);
1834 QTAILQ_REMOVE(&mr->coalesced, cmr, link);
1835 g_free(cmr);
1836 updated = true;
1837 }
1838
1839 if (updated) {
1840 memory_region_update_coalesced_range(mr);
1841 }
1842 }
1843
1844 void memory_region_set_flush_coalesced(MemoryRegion *mr)
1845 {
1846 mr->flush_coalesced_mmio = true;
1847 }
1848
1849 void memory_region_clear_flush_coalesced(MemoryRegion *mr)
1850 {
1851 qemu_flush_coalesced_mmio_buffer();
1852 if (QTAILQ_EMPTY(&mr->coalesced)) {
1853 mr->flush_coalesced_mmio = false;
1854 }
1855 }
1856
1857 void memory_region_set_global_locking(MemoryRegion *mr)
1858 {
1859 mr->global_locking = true;
1860 }
1861
1862 void memory_region_clear_global_locking(MemoryRegion *mr)
1863 {
1864 mr->global_locking = false;
1865 }
1866
1867 static bool userspace_eventfd_warning;
1868
1869 void memory_region_add_eventfd(MemoryRegion *mr,
1870 hwaddr addr,
1871 unsigned size,
1872 bool match_data,
1873 uint64_t data,
1874 EventNotifier *e)
1875 {
1876 MemoryRegionIoeventfd mrfd = {
1877 .addr.start = int128_make64(addr),
1878 .addr.size = int128_make64(size),
1879 .match_data = match_data,
1880 .data = data,
1881 .e = e,
1882 };
1883 unsigned i;
1884
1885 if (kvm_enabled() && (!(kvm_eventfds_enabled() ||
1886 userspace_eventfd_warning))) {
1887 userspace_eventfd_warning = true;
1888 error_report("Using eventfd without MMIO binding in KVM. "
1889 "Suboptimal performance expected");
1890 }
1891
1892 if (size) {
1893 adjust_endianness(mr, &mrfd.data, size);
1894 }
1895 memory_region_transaction_begin();
1896 for (i = 0; i < mr->ioeventfd_nb; ++i) {
1897 if (memory_region_ioeventfd_before(mrfd, mr->ioeventfds[i])) {
1898 break;
1899 }
1900 }
1901 ++mr->ioeventfd_nb;
1902 mr->ioeventfds = g_realloc(mr->ioeventfds,
1903 sizeof(*mr->ioeventfds) * mr->ioeventfd_nb);
1904 memmove(&mr->ioeventfds[i+1], &mr->ioeventfds[i],
1905 sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb-1 - i));
1906 mr->ioeventfds[i] = mrfd;
1907 ioeventfd_update_pending |= mr->enabled;
1908 memory_region_transaction_commit();
1909 }
1910
1911 void memory_region_del_eventfd(MemoryRegion *mr,
1912 hwaddr addr,
1913 unsigned size,
1914 bool match_data,
1915 uint64_t data,
1916 EventNotifier *e)
1917 {
1918 MemoryRegionIoeventfd mrfd = {
1919 .addr.start = int128_make64(addr),
1920 .addr.size = int128_make64(size),
1921 .match_data = match_data,
1922 .data = data,
1923 .e = e,
1924 };
1925 unsigned i;
1926
1927 if (size) {
1928 adjust_endianness(mr, &mrfd.data, size);
1929 }
1930 memory_region_transaction_begin();
1931 for (i = 0; i < mr->ioeventfd_nb; ++i) {
1932 if (memory_region_ioeventfd_equal(mrfd, mr->ioeventfds[i])) {
1933 break;
1934 }
1935 }
1936 assert(i != mr->ioeventfd_nb);
1937 memmove(&mr->ioeventfds[i], &mr->ioeventfds[i+1],
1938 sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb - (i+1)));
1939 --mr->ioeventfd_nb;
1940 mr->ioeventfds = g_realloc(mr->ioeventfds,
1941 sizeof(*mr->ioeventfds)*mr->ioeventfd_nb + 1);
1942 ioeventfd_update_pending |= mr->enabled;
1943 memory_region_transaction_commit();
1944 }
1945
1946 static void memory_region_update_container_subregions(MemoryRegion *subregion)
1947 {
1948 MemoryRegion *mr = subregion->container;
1949 MemoryRegion *other;
1950
1951 memory_region_transaction_begin();
1952
1953 memory_region_ref(subregion);
1954 QTAILQ_FOREACH(other, &mr->subregions, subregions_link) {
1955 if (subregion->priority >= other->priority) {
1956 QTAILQ_INSERT_BEFORE(other, subregion, subregions_link);
1957 goto done;
1958 }
1959 }
1960 QTAILQ_INSERT_TAIL(&mr->subregions, subregion, subregions_link);
1961 done:
1962 memory_region_update_pending |= mr->enabled && subregion->enabled;
1963 memory_region_transaction_commit();
1964 }
1965
1966 static void memory_region_add_subregion_common(MemoryRegion *mr,
1967 hwaddr offset,
1968 MemoryRegion *subregion)
1969 {
1970 assert(!subregion->container);
1971 subregion->container = mr;
1972 subregion->addr = offset;
1973 memory_region_update_container_subregions(subregion);
1974 }
1975
1976 void memory_region_add_subregion(MemoryRegion *mr,
1977 hwaddr offset,
1978 MemoryRegion *subregion)
1979 {
1980 subregion->priority = 0;
1981 memory_region_add_subregion_common(mr, offset, subregion);
1982 }
1983
1984 void memory_region_add_subregion_overlap(MemoryRegion *mr,
1985 hwaddr offset,
1986 MemoryRegion *subregion,
1987 int priority)
1988 {
1989 subregion->priority = priority;
1990 memory_region_add_subregion_common(mr, offset, subregion);
1991 }
1992
1993 void memory_region_del_subregion(MemoryRegion *mr,
1994 MemoryRegion *subregion)
1995 {
1996 memory_region_transaction_begin();
1997 assert(subregion->container == mr);
1998 subregion->container = NULL;
1999 QTAILQ_REMOVE(&mr->subregions, subregion, subregions_link);
2000 memory_region_unref(subregion);
2001 memory_region_update_pending |= mr->enabled && subregion->enabled;
2002 memory_region_transaction_commit();
2003 }
2004
2005 void memory_region_set_enabled(MemoryRegion *mr, bool enabled)
2006 {
2007 if (enabled == mr->enabled) {
2008 return;
2009 }
2010 memory_region_transaction_begin();
2011 mr->enabled = enabled;
2012 memory_region_update_pending = true;
2013 memory_region_transaction_commit();
2014 }
2015
2016 void memory_region_set_size(MemoryRegion *mr, uint64_t size)
2017 {
2018 Int128 s = int128_make64(size);
2019
2020 if (size == UINT64_MAX) {
2021 s = int128_2_64();
2022 }
2023 if (int128_eq(s, mr->size)) {
2024 return;
2025 }
2026 memory_region_transaction_begin();
2027 mr->size = s;
2028 memory_region_update_pending = true;
2029 memory_region_transaction_commit();
2030 }
2031
2032 static void memory_region_readd_subregion(MemoryRegion *mr)
2033 {
2034 MemoryRegion *container = mr->container;
2035
2036 if (container) {
2037 memory_region_transaction_begin();
2038 memory_region_ref(mr);
2039 memory_region_del_subregion(container, mr);
2040 mr->container = container;
2041 memory_region_update_container_subregions(mr);
2042 memory_region_unref(mr);
2043 memory_region_transaction_commit();
2044 }
2045 }
2046
2047 void memory_region_set_address(MemoryRegion *mr, hwaddr addr)
2048 {
2049 if (addr != mr->addr) {
2050 mr->addr = addr;
2051 memory_region_readd_subregion(mr);
2052 }
2053 }
2054
2055 void memory_region_set_alias_offset(MemoryRegion *mr, hwaddr offset)
2056 {
2057 assert(mr->alias);
2058
2059 if (offset == mr->alias_offset) {
2060 return;
2061 }
2062
2063 memory_region_transaction_begin();
2064 mr->alias_offset = offset;
2065 memory_region_update_pending |= mr->enabled;
2066 memory_region_transaction_commit();
2067 }
2068
2069 uint64_t memory_region_get_alignment(const MemoryRegion *mr)
2070 {
2071 return mr->align;
2072 }
2073
2074 static int cmp_flatrange_addr(const void *addr_, const void *fr_)
2075 {
2076 const AddrRange *addr = addr_;
2077 const FlatRange *fr = fr_;
2078
2079 if (int128_le(addrrange_end(*addr), fr->addr.start)) {
2080 return -1;
2081 } else if (int128_ge(addr->start, addrrange_end(fr->addr))) {
2082 return 1;
2083 }
2084 return 0;
2085 }
2086
2087 static FlatRange *flatview_lookup(FlatView *view, AddrRange addr)
2088 {
2089 return bsearch(&addr, view->ranges, view->nr,
2090 sizeof(FlatRange), cmp_flatrange_addr);
2091 }
2092
2093 bool memory_region_is_mapped(MemoryRegion *mr)
2094 {
2095 return mr->container ? true : false;
2096 }
2097
2098 /* Same as memory_region_find, but it does not add a reference to the
2099 * returned region. It must be called from an RCU critical section.
2100 */
2101 static MemoryRegionSection memory_region_find_rcu(MemoryRegion *mr,
2102 hwaddr addr, uint64_t size)
2103 {
2104 MemoryRegionSection ret = { .mr = NULL };
2105 MemoryRegion *root;
2106 AddressSpace *as;
2107 AddrRange range;
2108 FlatView *view;
2109 FlatRange *fr;
2110
2111 addr += mr->addr;
2112 for (root = mr; root->container; ) {
2113 root = root->container;
2114 addr += root->addr;
2115 }
2116
2117 as = memory_region_to_address_space(root);
2118 if (!as) {
2119 return ret;
2120 }
2121 range = addrrange_make(int128_make64(addr), int128_make64(size));
2122
2123 view = atomic_rcu_read(&as->current_map);
2124 fr = flatview_lookup(view, range);
2125 if (!fr) {
2126 return ret;
2127 }
2128
2129 while (fr > view->ranges && addrrange_intersects(fr[-1].addr, range)) {
2130 --fr;
2131 }
2132
2133 ret.mr = fr->mr;
2134 ret.address_space = as;
2135 range = addrrange_intersection(range, fr->addr);
2136 ret.offset_within_region = fr->offset_in_region;
2137 ret.offset_within_region += int128_get64(int128_sub(range.start,
2138 fr->addr.start));
2139 ret.size = range.size;
2140 ret.offset_within_address_space = int128_get64(range.start);
2141 ret.readonly = fr->readonly;
2142 return ret;
2143 }
2144
2145 MemoryRegionSection memory_region_find(MemoryRegion *mr,
2146 hwaddr addr, uint64_t size)
2147 {
2148 MemoryRegionSection ret;
2149 rcu_read_lock();
2150 ret = memory_region_find_rcu(mr, addr, size);
2151 if (ret.mr) {
2152 memory_region_ref(ret.mr);
2153 }
2154 rcu_read_unlock();
2155 return ret;
2156 }
2157
2158 bool memory_region_present(MemoryRegion *container, hwaddr addr)
2159 {
2160 MemoryRegion *mr;
2161
2162 rcu_read_lock();
2163 mr = memory_region_find_rcu(container, addr, 1).mr;
2164 rcu_read_unlock();
2165 return mr && mr != container;
2166 }
2167
2168 void memory_global_dirty_log_sync(void)
2169 {
2170 MemoryListener *listener;
2171 AddressSpace *as;
2172 FlatView *view;
2173 FlatRange *fr;
2174
2175 QTAILQ_FOREACH(listener, &memory_listeners, link) {
2176 if (!listener->log_sync) {
2177 continue;
2178 }
2179 /* Global listeners are being phased out. */
2180 assert(listener->address_space_filter);
2181 as = listener->address_space_filter;
2182 view = address_space_get_flatview(as);
2183 FOR_EACH_FLAT_RANGE(fr, view) {
2184 MemoryRegionSection mrs = section_from_flat_range(fr, as);
2185 listener->log_sync(listener, &mrs);
2186 }
2187 flatview_unref(view);
2188 }
2189 }
2190
2191 void memory_global_dirty_log_start(void)
2192 {
2193 global_dirty_log = true;
2194
2195 MEMORY_LISTENER_CALL_GLOBAL(log_global_start, Forward);
2196
2197 /* Refresh DIRTY_LOG_MIGRATION bit. */
2198 memory_region_transaction_begin();
2199 memory_region_update_pending = true;
2200 memory_region_transaction_commit();
2201 }
2202
2203 void memory_global_dirty_log_stop(void)
2204 {
2205 global_dirty_log = false;
2206
2207 /* Refresh DIRTY_LOG_MIGRATION bit. */
2208 memory_region_transaction_begin();
2209 memory_region_update_pending = true;
2210 memory_region_transaction_commit();
2211
2212 MEMORY_LISTENER_CALL_GLOBAL(log_global_stop, Reverse);
2213 }
2214
2215 static void listener_add_address_space(MemoryListener *listener,
2216 AddressSpace *as)
2217 {
2218 FlatView *view;
2219 FlatRange *fr;
2220
2221 if (listener->address_space_filter
2222 && listener->address_space_filter != as) {
2223 return;
2224 }
2225
2226 if (listener->begin) {
2227 listener->begin(listener);
2228 }
2229 if (global_dirty_log) {
2230 if (listener->log_global_start) {
2231 listener->log_global_start(listener);
2232 }
2233 }
2234
2235 view = address_space_get_flatview(as);
2236 FOR_EACH_FLAT_RANGE(fr, view) {
2237 MemoryRegionSection section = {
2238 .mr = fr->mr,
2239 .address_space = as,
2240 .offset_within_region = fr->offset_in_region,
2241 .size = fr->addr.size,
2242 .offset_within_address_space = int128_get64(fr->addr.start),
2243 .readonly = fr->readonly,
2244 };
2245 if (fr->dirty_log_mask && listener->log_start) {
2246 listener->log_start(listener, &section, 0, fr->dirty_log_mask);
2247 }
2248 if (listener->region_add) {
2249 listener->region_add(listener, &section);
2250 }
2251 }
2252 if (listener->commit) {
2253 listener->commit(listener);
2254 }
2255 flatview_unref(view);
2256 }
2257
2258 void memory_listener_register(MemoryListener *listener, AddressSpace *filter)
2259 {
2260 MemoryListener *other = NULL;
2261 AddressSpace *as;
2262
2263 listener->address_space_filter = filter;
2264 if (QTAILQ_EMPTY(&memory_listeners)
2265 || listener->priority >= QTAILQ_LAST(&memory_listeners,
2266 memory_listeners)->priority) {
2267 QTAILQ_INSERT_TAIL(&memory_listeners, listener, link);
2268 } else {
2269 QTAILQ_FOREACH(other, &memory_listeners, link) {
2270 if (listener->priority < other->priority) {
2271 break;
2272 }
2273 }
2274 QTAILQ_INSERT_BEFORE(other, listener, link);
2275 }
2276
2277 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
2278 listener_add_address_space(listener, as);
2279 }
2280 }
2281
2282 void memory_listener_unregister(MemoryListener *listener)
2283 {
2284 QTAILQ_REMOVE(&memory_listeners, listener, link);
2285 }
2286
2287 void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name)
2288 {
2289 memory_region_ref(root);
2290 memory_region_transaction_begin();
2291 as->ref_count = 1;
2292 as->root = root;
2293 as->malloced = false;
2294 as->current_map = g_new(FlatView, 1);
2295 flatview_init(as->current_map);
2296 as->ioeventfd_nb = 0;
2297 as->ioeventfds = NULL;
2298 QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link);
2299 as->name = g_strdup(name ? name : "anonymous");
2300 address_space_init_dispatch(as);
2301 memory_region_update_pending |= root->enabled;
2302 memory_region_transaction_commit();
2303 }
2304
2305 static void do_address_space_destroy(AddressSpace *as)
2306 {
2307 MemoryListener *listener;
2308 bool do_free = as->malloced;
2309
2310 address_space_destroy_dispatch(as);
2311
2312 QTAILQ_FOREACH(listener, &memory_listeners, link) {
2313 assert(listener->address_space_filter != as);
2314 }
2315
2316 flatview_unref(as->current_map);
2317 g_free(as->name);
2318 g_free(as->ioeventfds);
2319 memory_region_unref(as->root);
2320 if (do_free) {
2321 g_free(as);
2322 }
2323 }
2324
2325 AddressSpace *address_space_init_shareable(MemoryRegion *root, const char *name)
2326 {
2327 AddressSpace *as;
2328
2329 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
2330 if (root == as->root && as->malloced) {
2331 as->ref_count++;
2332 return as;
2333 }
2334 }
2335
2336 as = g_malloc0(sizeof *as);
2337 address_space_init(as, root, name);
2338 as->malloced = true;
2339 return as;
2340 }
2341
2342 void address_space_destroy(AddressSpace *as)
2343 {
2344 MemoryRegion *root = as->root;
2345
2346 as->ref_count--;
2347 if (as->ref_count) {
2348 return;
2349 }
2350 /* Flush out anything from MemoryListeners listening in on this */
2351 memory_region_transaction_begin();
2352 as->root = NULL;
2353 memory_region_transaction_commit();
2354 QTAILQ_REMOVE(&address_spaces, as, address_spaces_link);
2355 address_space_unregister(as);
2356
2357 /* At this point, as->dispatch and as->current_map are dummy
2358 * entries that the guest should never use. Wait for the old
2359 * values to expire before freeing the data.
2360 */
2361 as->root = root;
2362 call_rcu(as, do_address_space_destroy, rcu);
2363 }
2364
2365 typedef struct MemoryRegionList MemoryRegionList;
2366
2367 struct MemoryRegionList {
2368 const MemoryRegion *mr;
2369 QTAILQ_ENTRY(MemoryRegionList) queue;
2370 };
2371
2372 typedef QTAILQ_HEAD(queue, MemoryRegionList) MemoryRegionListHead;
2373
2374 static void mtree_print_mr(fprintf_function mon_printf, void *f,
2375 const MemoryRegion *mr, unsigned int level,
2376 hwaddr base,
2377 MemoryRegionListHead *alias_print_queue)
2378 {
2379 MemoryRegionList *new_ml, *ml, *next_ml;
2380 MemoryRegionListHead submr_print_queue;
2381 const MemoryRegion *submr;
2382 unsigned int i;
2383
2384 if (!mr) {
2385 return;
2386 }
2387
2388 for (i = 0; i < level; i++) {
2389 mon_printf(f, " ");
2390 }
2391
2392 if (mr->alias) {
2393 MemoryRegionList *ml;
2394 bool found = false;
2395
2396 /* check if the alias is already in the queue */
2397 QTAILQ_FOREACH(ml, alias_print_queue, queue) {
2398 if (ml->mr == mr->alias) {
2399 found = true;
2400 }
2401 }
2402
2403 if (!found) {
2404 ml = g_new(MemoryRegionList, 1);
2405 ml->mr = mr->alias;
2406 QTAILQ_INSERT_TAIL(alias_print_queue, ml, queue);
2407 }
2408 mon_printf(f, TARGET_FMT_plx "-" TARGET_FMT_plx
2409 " (prio %d, %c%c): alias %s @%s " TARGET_FMT_plx
2410 "-" TARGET_FMT_plx "%s\n",
2411 base + mr->addr,
2412 base + mr->addr
2413 + (int128_nz(mr->size) ?
2414 (hwaddr)int128_get64(int128_sub(mr->size,
2415 int128_one())) : 0),
2416 mr->priority,
2417 mr->romd_mode ? 'R' : '-',
2418 !mr->readonly && !(mr->rom_device && mr->romd_mode) ? 'W'
2419 : '-',
2420 memory_region_name(mr),
2421 memory_region_name(mr->alias),
2422 mr->alias_offset,
2423 mr->alias_offset
2424 + (int128_nz(mr->size) ?
2425 (hwaddr)int128_get64(int128_sub(mr->size,
2426 int128_one())) : 0),
2427 mr->enabled ? "" : " [disabled]");
2428 } else {
2429 mon_printf(f,
2430 TARGET_FMT_plx "-" TARGET_FMT_plx " (prio %d, %c%c): %s%s\n",
2431 base + mr->addr,
2432 base + mr->addr
2433 + (int128_nz(mr->size) ?
2434 (hwaddr)int128_get64(int128_sub(mr->size,
2435 int128_one())) : 0),
2436 mr->priority,
2437 mr->romd_mode ? 'R' : '-',
2438 !mr->readonly && !(mr->rom_device && mr->romd_mode) ? 'W'
2439 : '-',
2440 memory_region_name(mr),
2441 mr->enabled ? "" : " [disabled]");
2442 }
2443
2444 QTAILQ_INIT(&submr_print_queue);
2445
2446 QTAILQ_FOREACH(submr, &mr->subregions, subregions_link) {
2447 new_ml = g_new(MemoryRegionList, 1);
2448 new_ml->mr = submr;
2449 QTAILQ_FOREACH(ml, &submr_print_queue, queue) {
2450 if (new_ml->mr->addr < ml->mr->addr ||
2451 (new_ml->mr->addr == ml->mr->addr &&
2452 new_ml->mr->priority > ml->mr->priority)) {
2453 QTAILQ_INSERT_BEFORE(ml, new_ml, queue);
2454 new_ml = NULL;
2455 break;
2456 }
2457 }
2458 if (new_ml) {
2459 QTAILQ_INSERT_TAIL(&submr_print_queue, new_ml, queue);
2460 }
2461 }
2462
2463 QTAILQ_FOREACH(ml, &submr_print_queue, queue) {
2464 mtree_print_mr(mon_printf, f, ml->mr, level + 1, base + mr->addr,
2465 alias_print_queue);
2466 }
2467
2468 QTAILQ_FOREACH_SAFE(ml, &submr_print_queue, queue, next_ml) {
2469 g_free(ml);
2470 }
2471 }
2472
2473 void mtree_info(fprintf_function mon_printf, void *f)
2474 {
2475 MemoryRegionListHead ml_head;
2476 MemoryRegionList *ml, *ml2;
2477 AddressSpace *as;
2478
2479 QTAILQ_INIT(&ml_head);
2480
2481 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
2482 mon_printf(f, "address-space: %s\n", as->name);
2483 mtree_print_mr(mon_printf, f, as->root, 1, 0, &ml_head);
2484 mon_printf(f, "\n");
2485 }
2486
2487 /* print aliased regions */
2488 QTAILQ_FOREACH(ml, &ml_head, queue) {
2489 mon_printf(f, "memory-region: %s\n", memory_region_name(ml->mr));
2490 mtree_print_mr(mon_printf, f, ml->mr, 1, 0, &ml_head);
2491 mon_printf(f, "\n");
2492 }
2493
2494 QTAILQ_FOREACH_SAFE(ml, &ml_head, queue, ml2) {
2495 g_free(ml);
2496 }
2497 }
2498
2499 static const TypeInfo memory_region_info = {
2500 .parent = TYPE_OBJECT,
2501 .name = TYPE_MEMORY_REGION,
2502 .instance_size = sizeof(MemoryRegion),
2503 .instance_init = memory_region_initfn,
2504 .instance_finalize = memory_region_finalize,
2505 };
2506
2507 static void memory_register_types(void)
2508 {
2509 type_register_static(&memory_region_info);
2510 }
2511
2512 type_init(memory_register_types)
QEmu