?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
exec: Make memory_region_section_get_iotlb use section AS
[qemu.git] / exec.c
blob122e6e5f11bc1b7a0bb1f13b3a2843fab159d25f
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
51
52 #include "exec/memory-internal.h"
53 #include "exec/ram_addr.h"
54 #include "qemu/cache-utils.h"
55
56 #include "qemu/range.h"
57
58 //#define DEBUG_SUBPAGE
59
60 #if !defined(CONFIG_USER_ONLY)
61 static bool in_migration;
62
63 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
64
65 static MemoryRegion *system_memory;
66 static MemoryRegion *system_io;
67
68 AddressSpace address_space_io;
69 AddressSpace address_space_memory;
70
71 MemoryRegion io_mem_rom, io_mem_notdirty;
72 static MemoryRegion io_mem_unassigned;
73
74 #endif
75
76 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
77 /* current CPU in the current thread. It is only valid inside
78 cpu_exec() */
79 DEFINE_TLS(CPUState *, current_cpu);
80 /* 0 = Do not count executed instructions.
81 1 = Precise instruction counting.
82 2 = Adaptive rate instruction counting. */
83 int use_icount;
84
85 #if !defined(CONFIG_USER_ONLY)
86
87 typedef struct PhysPageEntry PhysPageEntry;
88
89 struct PhysPageEntry {
90 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
91 uint32_t skip : 6;
92 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
93 uint32_t ptr : 26;
94 };
95
96 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
97
98 /* Size of the L2 (and L3, etc) page tables. */
99 #define ADDR_SPACE_BITS 64
100
101 #define P_L2_BITS 9
102 #define P_L2_SIZE (1 << P_L2_BITS)
103
104 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
105
106 typedef PhysPageEntry Node[P_L2_SIZE];
107
108 typedef struct PhysPageMap {
109 unsigned sections_nb;
110 unsigned sections_nb_alloc;
111 unsigned nodes_nb;
112 unsigned nodes_nb_alloc;
113 Node *nodes;
114 MemoryRegionSection *sections;
115 } PhysPageMap;
116
117 struct AddressSpaceDispatch {
118 /* This is a multi-level map on the physical address space.
119 * The bottom level has pointers to MemoryRegionSections.
120 */
121 PhysPageEntry phys_map;
122 PhysPageMap map;
123 AddressSpace *as;
124 };
125
126 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
127 typedef struct subpage_t {
128 MemoryRegion iomem;
129 AddressSpace *as;
130 hwaddr base;
131 uint16_t sub_section[TARGET_PAGE_SIZE];
132 } subpage_t;
133
134 #define PHYS_SECTION_UNASSIGNED 0
135 #define PHYS_SECTION_NOTDIRTY 1
136 #define PHYS_SECTION_ROM 2
137 #define PHYS_SECTION_WATCH 3
138
139 static void io_mem_init(void);
140 static void memory_map_init(void);
141
142 static MemoryRegion io_mem_watch;
143 #endif
144
145 #if !defined(CONFIG_USER_ONLY)
146
147 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
148 {
149 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
150 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
151 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
152 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
153 }
154 }
155
156 static uint32_t phys_map_node_alloc(PhysPageMap *map)
157 {
158 unsigned i;
159 uint32_t ret;
160
161 ret = map->nodes_nb++;
162 assert(ret != PHYS_MAP_NODE_NIL);
163 assert(ret != map->nodes_nb_alloc);
164 for (i = 0; i < P_L2_SIZE; ++i) {
165 map->nodes[ret][i].skip = 1;
166 map->nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
167 }
168 return ret;
169 }
170
171 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
172 hwaddr *index, hwaddr *nb, uint16_t leaf,
173 int level)
174 {
175 PhysPageEntry *p;
176 int i;
177 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
178
179 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
180 lp->ptr = phys_map_node_alloc(map);
181 p = map->nodes[lp->ptr];
182 if (level == 0) {
183 for (i = 0; i < P_L2_SIZE; i++) {
184 p[i].skip = 0;
185 p[i].ptr = PHYS_SECTION_UNASSIGNED;
186 }
187 }
188 } else {
189 p = map->nodes[lp->ptr];
190 }
191 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
192
193 while (*nb && lp < &p[P_L2_SIZE]) {
194 if ((*index & (step - 1)) == 0 && *nb >= step) {
195 lp->skip = 0;
196 lp->ptr = leaf;
197 *index += step;
198 *nb -= step;
199 } else {
200 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
201 }
202 ++lp;
203 }
204 }
205
206 static void phys_page_set(AddressSpaceDispatch *d,
207 hwaddr index, hwaddr nb,
208 uint16_t leaf)
209 {
210 /* Wildly overreserve - it doesn't matter much. */
211 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
212
213 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
214 }
215
216 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
217 * and update our entry so we can skip it and go directly to the destination.
218 */
219 static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)
220 {
221 unsigned valid_ptr = P_L2_SIZE;
222 int valid = 0;
223 PhysPageEntry *p;
224 int i;
225
226 if (lp->ptr == PHYS_MAP_NODE_NIL) {
227 return;
228 }
229
230 p = nodes[lp->ptr];
231 for (i = 0; i < P_L2_SIZE; i++) {
232 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
233 continue;
234 }
235
236 valid_ptr = i;
237 valid++;
238 if (p[i].skip) {
239 phys_page_compact(&p[i], nodes, compacted);
240 }
241 }
242
243 /* We can only compress if there's only one child. */
244 if (valid != 1) {
245 return;
246 }
247
248 assert(valid_ptr < P_L2_SIZE);
249
250 /* Don't compress if it won't fit in the # of bits we have. */
251 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
252 return;
253 }
254
255 lp->ptr = p[valid_ptr].ptr;
256 if (!p[valid_ptr].skip) {
257 /* If our only child is a leaf, make this a leaf. */
258 /* By design, we should have made this node a leaf to begin with so we
259 * should never reach here.
260 * But since it's so simple to handle this, let's do it just in case we
261 * change this rule.
262 */
263 lp->skip = 0;
264 } else {
265 lp->skip += p[valid_ptr].skip;
266 }
267 }
268
269 static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
270 {
271 DECLARE_BITMAP(compacted, nodes_nb);
272
273 if (d->phys_map.skip) {
274 phys_page_compact(&d->phys_map, d->map.nodes, compacted);
275 }
276 }
277
278 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
279 Node *nodes, MemoryRegionSection *sections)
280 {
281 PhysPageEntry *p;
282 hwaddr index = addr >> TARGET_PAGE_BITS;
283 int i;
284
285 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
286 if (lp.ptr == PHYS_MAP_NODE_NIL) {
287 return &sections[PHYS_SECTION_UNASSIGNED];
288 }
289 p = nodes[lp.ptr];
290 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
291 }
292
293 if (sections[lp.ptr].size.hi ||
294 range_covers_byte(sections[lp.ptr].offset_within_address_space,
295 sections[lp.ptr].size.lo, addr)) {
296 return &sections[lp.ptr];
297 } else {
298 return &sections[PHYS_SECTION_UNASSIGNED];
299 }
300 }
301
302 bool memory_region_is_unassigned(MemoryRegion *mr)
303 {
304 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
305 && mr != &io_mem_watch;
306 }
307
308 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
309 hwaddr addr,
310 bool resolve_subpage)
311 {
312 MemoryRegionSection *section;
313 subpage_t *subpage;
314
315 section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);
316 if (resolve_subpage && section->mr->subpage) {
317 subpage = container_of(section->mr, subpage_t, iomem);
318 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
319 }
320 return section;
321 }
322
323 static MemoryRegionSection *
324 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
325 hwaddr *plen, bool resolve_subpage)
326 {
327 MemoryRegionSection *section;
328 Int128 diff;
329
330 section = address_space_lookup_region(d, addr, resolve_subpage);
331 /* Compute offset within MemoryRegionSection */
332 addr -= section->offset_within_address_space;
333
334 /* Compute offset within MemoryRegion */
335 *xlat = addr + section->offset_within_region;
336
337 diff = int128_sub(section->mr->size, int128_make64(addr));
338 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
339 return section;
340 }
341
342 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
343 {
344 if (memory_region_is_ram(mr)) {
345 return !(is_write && mr->readonly);
346 }
347 if (memory_region_is_romd(mr)) {
348 return !is_write;
349 }
350
351 return false;
352 }
353
354 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
355 hwaddr *xlat, hwaddr *plen,
356 bool is_write)
357 {
358 IOMMUTLBEntry iotlb;
359 MemoryRegionSection *section;
360 MemoryRegion *mr;
361 hwaddr len = *plen;
362
363 for (;;) {
364 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
365 mr = section->mr;
366
367 if (!mr->iommu_ops) {
368 break;
369 }
370
371 iotlb = mr->iommu_ops->translate(mr, addr);
372 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
373 | (addr & iotlb.addr_mask));
374 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
375 if (!(iotlb.perm & (1 << is_write))) {
376 mr = &io_mem_unassigned;
377 break;
378 }
379
380 as = iotlb.target_as;
381 }
382
383 if (memory_access_is_direct(mr, is_write)) {
384 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
385 len = MIN(page, len);
386 }
387
388 *plen = len;
389 *xlat = addr;
390 return mr;
391 }
392
393 MemoryRegionSection *
394 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
395 hwaddr *plen)
396 {
397 MemoryRegionSection *section;
398 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
399
400 assert(!section->mr->iommu_ops);
401 return section;
402 }
403 #endif
404
405 void cpu_exec_init_all(void)
406 {
407 #if !defined(CONFIG_USER_ONLY)
408 qemu_mutex_init(&ram_list.mutex);
409 memory_map_init();
410 io_mem_init();
411 #endif
412 }
413
414 #if !defined(CONFIG_USER_ONLY)
415
416 static int cpu_common_post_load(void *opaque, int version_id)
417 {
418 CPUState *cpu = opaque;
419
420 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
421 version_id is increased. */
422 cpu->interrupt_request &= ~0x01;
423 tlb_flush(cpu->env_ptr, 1);
424
425 return 0;
426 }
427
428 const VMStateDescription vmstate_cpu_common = {
429 .name = "cpu_common",
430 .version_id = 1,
431 .minimum_version_id = 1,
432 .minimum_version_id_old = 1,
433 .post_load = cpu_common_post_load,
434 .fields = (VMStateField []) {
435 VMSTATE_UINT32(halted, CPUState),
436 VMSTATE_UINT32(interrupt_request, CPUState),
437 VMSTATE_END_OF_LIST()
438 }
439 };
440
441 #endif
442
443 CPUState *qemu_get_cpu(int index)
444 {
445 CPUState *cpu;
446
447 CPU_FOREACH(cpu) {
448 if (cpu->cpu_index == index) {
449 return cpu;
450 }
451 }
452
453 return NULL;
454 }
455
456 void cpu_exec_init(CPUArchState *env)
457 {
458 CPUState *cpu = ENV_GET_CPU(env);
459 CPUClass *cc = CPU_GET_CLASS(cpu);
460 CPUState *some_cpu;
461 int cpu_index;
462
463 #if defined(CONFIG_USER_ONLY)
464 cpu_list_lock();
465 #endif
466 cpu_index = 0;
467 CPU_FOREACH(some_cpu) {
468 cpu_index++;
469 }
470 cpu->cpu_index = cpu_index;
471 cpu->numa_node = 0;
472 QTAILQ_INIT(&env->breakpoints);
473 QTAILQ_INIT(&env->watchpoints);
474 #ifndef CONFIG_USER_ONLY
475 cpu->thread_id = qemu_get_thread_id();
476 #endif
477 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
478 #if defined(CONFIG_USER_ONLY)
479 cpu_list_unlock();
480 #endif
481 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
482 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
483 }
484 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
485 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
486 cpu_save, cpu_load, env);
487 assert(cc->vmsd == NULL);
488 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
489 #endif
490 if (cc->vmsd != NULL) {
491 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
492 }
493 }
494
495 #if defined(TARGET_HAS_ICE)
496 #if defined(CONFIG_USER_ONLY)
497 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
498 {
499 tb_invalidate_phys_page_range(pc, pc + 1, 0);
500 }
501 #else
502 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
503 {
504 hwaddr phys = cpu_get_phys_page_debug(cpu, pc);
505 if (phys != -1) {
506 tb_invalidate_phys_addr(&address_space_memory,
507 phys | (pc & ~TARGET_PAGE_MASK));
508 }
509 }
510 #endif
511 #endif /* TARGET_HAS_ICE */
512
513 #if defined(CONFIG_USER_ONLY)
514 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
515
516 {
517 }
518
519 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
520 int flags, CPUWatchpoint **watchpoint)
521 {
522 return -ENOSYS;
523 }
524 #else
525 /* Add a watchpoint. */
526 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
527 int flags, CPUWatchpoint **watchpoint)
528 {
529 target_ulong len_mask = ~(len - 1);
530 CPUWatchpoint *wp;
531
532 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
533 if ((len & (len - 1)) || (addr & ~len_mask) ||
534 len == 0 || len > TARGET_PAGE_SIZE) {
535 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
536 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
537 return -EINVAL;
538 }
539 wp = g_malloc(sizeof(*wp));
540
541 wp->vaddr = addr;
542 wp->len_mask = len_mask;
543 wp->flags = flags;
544
545 /* keep all GDB-injected watchpoints in front */
546 if (flags & BP_GDB)
547 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
548 else
549 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
550
551 tlb_flush_page(env, addr);
552
553 if (watchpoint)
554 *watchpoint = wp;
555 return 0;
556 }
557
558 /* Remove a specific watchpoint. */
559 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
560 int flags)
561 {
562 target_ulong len_mask = ~(len - 1);
563 CPUWatchpoint *wp;
564
565 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
566 if (addr == wp->vaddr && len_mask == wp->len_mask
567 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
568 cpu_watchpoint_remove_by_ref(env, wp);
569 return 0;
570 }
571 }
572 return -ENOENT;
573 }
574
575 /* Remove a specific watchpoint by reference. */
576 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
577 {
578 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
579
580 tlb_flush_page(env, watchpoint->vaddr);
581
582 g_free(watchpoint);
583 }
584
585 /* Remove all matching watchpoints. */
586 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
587 {
588 CPUWatchpoint *wp, *next;
589
590 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
591 if (wp->flags & mask)
592 cpu_watchpoint_remove_by_ref(env, wp);
593 }
594 }
595 #endif
596
597 /* Add a breakpoint. */
598 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
599 CPUBreakpoint **breakpoint)
600 {
601 #if defined(TARGET_HAS_ICE)
602 CPUBreakpoint *bp;
603
604 bp = g_malloc(sizeof(*bp));
605
606 bp->pc = pc;
607 bp->flags = flags;
608
609 /* keep all GDB-injected breakpoints in front */
610 if (flags & BP_GDB) {
611 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
612 } else {
613 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
614 }
615
616 breakpoint_invalidate(ENV_GET_CPU(env), pc);
617
618 if (breakpoint) {
619 *breakpoint = bp;
620 }
621 return 0;
622 #else
623 return -ENOSYS;
624 #endif
625 }
626
627 /* Remove a specific breakpoint. */
628 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
629 {
630 #if defined(TARGET_HAS_ICE)
631 CPUBreakpoint *bp;
632
633 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
634 if (bp->pc == pc && bp->flags == flags) {
635 cpu_breakpoint_remove_by_ref(env, bp);
636 return 0;
637 }
638 }
639 return -ENOENT;
640 #else
641 return -ENOSYS;
642 #endif
643 }
644
645 /* Remove a specific breakpoint by reference. */
646 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
647 {
648 #if defined(TARGET_HAS_ICE)
649 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
650
651 breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
652
653 g_free(breakpoint);
654 #endif
655 }
656
657 /* Remove all matching breakpoints. */
658 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
659 {
660 #if defined(TARGET_HAS_ICE)
661 CPUBreakpoint *bp, *next;
662
663 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
664 if (bp->flags & mask)
665 cpu_breakpoint_remove_by_ref(env, bp);
666 }
667 #endif
668 }
669
670 /* enable or disable single step mode. EXCP_DEBUG is returned by the
671 CPU loop after each instruction */
672 void cpu_single_step(CPUState *cpu, int enabled)
673 {
674 #if defined(TARGET_HAS_ICE)
675 if (cpu->singlestep_enabled != enabled) {
676 cpu->singlestep_enabled = enabled;
677 if (kvm_enabled()) {
678 kvm_update_guest_debug(cpu, 0);
679 } else {
680 /* must flush all the translated code to avoid inconsistencies */
681 /* XXX: only flush what is necessary */
682 CPUArchState *env = cpu->env_ptr;
683 tb_flush(env);
684 }
685 }
686 #endif
687 }
688
689 void cpu_abort(CPUArchState *env, const char *fmt, ...)
690 {
691 CPUState *cpu = ENV_GET_CPU(env);
692 va_list ap;
693 va_list ap2;
694
695 va_start(ap, fmt);
696 va_copy(ap2, ap);
697 fprintf(stderr, "qemu: fatal: ");
698 vfprintf(stderr, fmt, ap);
699 fprintf(stderr, "\n");
700 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
701 if (qemu_log_enabled()) {
702 qemu_log("qemu: fatal: ");
703 qemu_log_vprintf(fmt, ap2);
704 qemu_log("\n");
705 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
706 qemu_log_flush();
707 qemu_log_close();
708 }
709 va_end(ap2);
710 va_end(ap);
711 #if defined(CONFIG_USER_ONLY)
712 {
713 struct sigaction act;
714 sigfillset(&act.sa_mask);
715 act.sa_handler = SIG_DFL;
716 sigaction(SIGABRT, &act, NULL);
717 }
718 #endif
719 abort();
720 }
721
722 #if !defined(CONFIG_USER_ONLY)
723 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
724 {
725 RAMBlock *block;
726
727 /* The list is protected by the iothread lock here. */
728 block = ram_list.mru_block;
729 if (block && addr - block->offset < block->length) {
730 goto found;
731 }
732 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
733 if (addr - block->offset < block->length) {
734 goto found;
735 }
736 }
737
738 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
739 abort();
740
741 found:
742 ram_list.mru_block = block;
743 return block;
744 }
745
746 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
747 {
748 ram_addr_t start1;
749 RAMBlock *block;
750 ram_addr_t end;
751
752 end = TARGET_PAGE_ALIGN(start + length);
753 start &= TARGET_PAGE_MASK;
754
755 block = qemu_get_ram_block(start);
756 assert(block == qemu_get_ram_block(end - 1));
757 start1 = (uintptr_t)block->host + (start - block->offset);
758 cpu_tlb_reset_dirty_all(start1, length);
759 }
760
761 /* Note: start and end must be within the same ram block. */
762 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length,
763 unsigned client)
764 {
765 if (length == 0)
766 return;
767 cpu_physical_memory_clear_dirty_range(start, length, client);
768
769 if (tcg_enabled()) {
770 tlb_reset_dirty_range_all(start, length);
771 }
772 }
773
774 static void cpu_physical_memory_set_dirty_tracking(bool enable)
775 {
776 in_migration = enable;
777 }
778
779 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
780 MemoryRegionSection *section,
781 target_ulong vaddr,
782 hwaddr paddr, hwaddr xlat,
783 int prot,
784 target_ulong *address)
785 {
786 hwaddr iotlb;
787 CPUWatchpoint *wp;
788
789 if (memory_region_is_ram(section->mr)) {
790 /* Normal RAM. */
791 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
792 + xlat;
793 if (!section->readonly) {
794 iotlb |= PHYS_SECTION_NOTDIRTY;
795 } else {
796 iotlb |= PHYS_SECTION_ROM;
797 }
798 } else {
799 iotlb = section - section->address_space->dispatch->map.sections;
800 iotlb += xlat;
801 }
802
803 /* Make accesses to pages with watchpoints go via the
804 watchpoint trap routines. */
805 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
806 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
807 /* Avoid trapping reads of pages with a write breakpoint. */
808 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
809 iotlb = PHYS_SECTION_WATCH + paddr;
810 *address |= TLB_MMIO;
811 break;
812 }
813 }
814 }
815
816 return iotlb;
817 }
818 #endif /* defined(CONFIG_USER_ONLY) */
819
820 #if !defined(CONFIG_USER_ONLY)
821
822 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
823 uint16_t section);
824 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
825
826 static void *(*phys_mem_alloc)(size_t size) = qemu_anon_ram_alloc;
827
828 /*
829 * Set a custom physical guest memory alloator.
830 * Accelerators with unusual needs may need this. Hopefully, we can
831 * get rid of it eventually.
832 */
833 void phys_mem_set_alloc(void *(*alloc)(size_t))
834 {
835 phys_mem_alloc = alloc;
836 }
837
838 static uint16_t phys_section_add(PhysPageMap *map,
839 MemoryRegionSection *section)
840 {
841 /* The physical section number is ORed with a page-aligned
842 * pointer to produce the iotlb entries. Thus it should
843 * never overflow into the page-aligned value.
844 */
845 assert(map->sections_nb < TARGET_PAGE_SIZE);
846
847 if (map->sections_nb == map->sections_nb_alloc) {
848 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
849 map->sections = g_renew(MemoryRegionSection, map->sections,
850 map->sections_nb_alloc);
851 }
852 map->sections[map->sections_nb] = *section;
853 memory_region_ref(section->mr);
854 return map->sections_nb++;
855 }
856
857 static void phys_section_destroy(MemoryRegion *mr)
858 {
859 memory_region_unref(mr);
860
861 if (mr->subpage) {
862 subpage_t *subpage = container_of(mr, subpage_t, iomem);
863 memory_region_destroy(&subpage->iomem);
864 g_free(subpage);
865 }
866 }
867
868 static void phys_sections_free(PhysPageMap *map)
869 {
870 while (map->sections_nb > 0) {
871 MemoryRegionSection *section = &map->sections[--map->sections_nb];
872 phys_section_destroy(section->mr);
873 }
874 g_free(map->sections);
875 g_free(map->nodes);
876 }
877
878 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
879 {
880 subpage_t *subpage;
881 hwaddr base = section->offset_within_address_space
882 & TARGET_PAGE_MASK;
883 MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
884 d->map.nodes, d->map.sections);
885 MemoryRegionSection subsection = {
886 .offset_within_address_space = base,
887 .size = int128_make64(TARGET_PAGE_SIZE),
888 };
889 hwaddr start, end;
890
891 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
892
893 if (!(existing->mr->subpage)) {
894 subpage = subpage_init(d->as, base);
895 subsection.address_space = d->as;
896 subsection.mr = &subpage->iomem;
897 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
898 phys_section_add(&d->map, &subsection));
899 } else {
900 subpage = container_of(existing->mr, subpage_t, iomem);
901 }
902 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
903 end = start + int128_get64(section->size) - 1;
904 subpage_register(subpage, start, end,
905 phys_section_add(&d->map, section));
906 }
907
908
909 static void register_multipage(AddressSpaceDispatch *d,
910 MemoryRegionSection *section)
911 {
912 hwaddr start_addr = section->offset_within_address_space;
913 uint16_t section_index = phys_section_add(&d->map, section);
914 uint64_t num_pages = int128_get64(int128_rshift(section->size,
915 TARGET_PAGE_BITS));
916
917 assert(num_pages);
918 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
919 }
920
921 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
922 {
923 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
924 AddressSpaceDispatch *d = as->next_dispatch;
925 MemoryRegionSection now = *section, remain = *section;
926 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
927
928 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
929 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
930 - now.offset_within_address_space;
931
932 now.size = int128_min(int128_make64(left), now.size);
933 register_subpage(d, &now);
934 } else {
935 now.size = int128_zero();
936 }
937 while (int128_ne(remain.size, now.size)) {
938 remain.size = int128_sub(remain.size, now.size);
939 remain.offset_within_address_space += int128_get64(now.size);
940 remain.offset_within_region += int128_get64(now.size);
941 now = remain;
942 if (int128_lt(remain.size, page_size)) {
943 register_subpage(d, &now);
944 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
945 now.size = page_size;
946 register_subpage(d, &now);
947 } else {
948 now.size = int128_and(now.size, int128_neg(page_size));
949 register_multipage(d, &now);
950 }
951 }
952 }
953
954 void qemu_flush_coalesced_mmio_buffer(void)
955 {
956 if (kvm_enabled())
957 kvm_flush_coalesced_mmio_buffer();
958 }
959
960 void qemu_mutex_lock_ramlist(void)
961 {
962 qemu_mutex_lock(&ram_list.mutex);
963 }
964
965 void qemu_mutex_unlock_ramlist(void)
966 {
967 qemu_mutex_unlock(&ram_list.mutex);
968 }
969
970 #ifdef __linux__
971
972 #include <sys/vfs.h>
973
974 #define HUGETLBFS_MAGIC 0x958458f6
975
976 static long gethugepagesize(const char *path)
977 {
978 struct statfs fs;
979 int ret;
980
981 do {
982 ret = statfs(path, &fs);
983 } while (ret != 0 && errno == EINTR);
984
985 if (ret != 0) {
986 perror(path);
987 return 0;
988 }
989
990 if (fs.f_type != HUGETLBFS_MAGIC)
991 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
992
993 return fs.f_bsize;
994 }
995
996 static sigjmp_buf sigjump;
997
998 static void sigbus_handler(int signal)
999 {
1000 siglongjmp(sigjump, 1);
1001 }
1002
1003 static void *file_ram_alloc(RAMBlock *block,
1004 ram_addr_t memory,
1005 const char *path)
1006 {
1007 char *filename;
1008 char *sanitized_name;
1009 char *c;
1010 void *area;
1011 int fd;
1012 unsigned long hpagesize;
1013
1014 hpagesize = gethugepagesize(path);
1015 if (!hpagesize) {
1016 return NULL;
1017 }
1018
1019 if (memory < hpagesize) {
1020 return NULL;
1021 }
1022
1023 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1024 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
1025 return NULL;
1026 }
1027
1028 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1029 sanitized_name = g_strdup(block->mr->name);
1030 for (c = sanitized_name; *c != '\0'; c++) {
1031 if (*c == '/')
1032 *c = '_';
1033 }
1034
1035 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1036 sanitized_name);
1037 g_free(sanitized_name);
1038
1039 fd = mkstemp(filename);
1040 if (fd < 0) {
1041 perror("unable to create backing store for hugepages");
1042 g_free(filename);
1043 return NULL;
1044 }
1045 unlink(filename);
1046 g_free(filename);
1047
1048 memory = (memory+hpagesize-1) & ~(hpagesize-1);
1049
1050 /*
1051 * ftruncate is not supported by hugetlbfs in older
1052 * hosts, so don't bother bailing out on errors.
1053 * If anything goes wrong with it under other filesystems,
1054 * mmap will fail.
1055 */
1056 if (ftruncate(fd, memory))
1057 perror("ftruncate");
1058
1059 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
1060 if (area == MAP_FAILED) {
1061 perror("file_ram_alloc: can't mmap RAM pages");
1062 close(fd);
1063 return (NULL);
1064 }
1065
1066 if (mem_prealloc) {
1067 int ret, i;
1068 struct sigaction act, oldact;
1069 sigset_t set, oldset;
1070
1071 memset(&act, 0, sizeof(act));
1072 act.sa_handler = &sigbus_handler;
1073 act.sa_flags = 0;
1074
1075 ret = sigaction(SIGBUS, &act, &oldact);
1076 if (ret) {
1077 perror("file_ram_alloc: failed to install signal handler");
1078 exit(1);
1079 }
1080
1081 /* unblock SIGBUS */
1082 sigemptyset(&set);
1083 sigaddset(&set, SIGBUS);
1084 pthread_sigmask(SIG_UNBLOCK, &set, &oldset);
1085
1086 if (sigsetjmp(sigjump, 1)) {
1087 fprintf(stderr, "file_ram_alloc: failed to preallocate pages\n");
1088 exit(1);
1089 }
1090
1091 /* MAP_POPULATE silently ignores failures */
1092 for (i = 0; i < (memory/hpagesize); i++) {
1093 memset(area + (hpagesize*i), 0, 1);
1094 }
1095
1096 ret = sigaction(SIGBUS, &oldact, NULL);
1097 if (ret) {
1098 perror("file_ram_alloc: failed to reinstall signal handler");
1099 exit(1);
1100 }
1101
1102 pthread_sigmask(SIG_SETMASK, &oldset, NULL);
1103 }
1104
1105 block->fd = fd;
1106 return area;
1107 }
1108 #else
1109 static void *file_ram_alloc(RAMBlock *block,
1110 ram_addr_t memory,
1111 const char *path)
1112 {
1113 fprintf(stderr, "-mem-path not supported on this host\n");
1114 exit(1);
1115 }
1116 #endif
1117
1118 static ram_addr_t find_ram_offset(ram_addr_t size)
1119 {
1120 RAMBlock *block, *next_block;
1121 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1122
1123 assert(size != 0); /* it would hand out same offset multiple times */
1124
1125 if (QTAILQ_EMPTY(&ram_list.blocks))
1126 return 0;
1127
1128 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1129 ram_addr_t end, next = RAM_ADDR_MAX;
1130
1131 end = block->offset + block->length;
1132
1133 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1134 if (next_block->offset >= end) {
1135 next = MIN(next, next_block->offset);
1136 }
1137 }
1138 if (next - end >= size && next - end < mingap) {
1139 offset = end;
1140 mingap = next - end;
1141 }
1142 }
1143
1144 if (offset == RAM_ADDR_MAX) {
1145 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1146 (uint64_t)size);
1147 abort();
1148 }
1149
1150 return offset;
1151 }
1152
1153 ram_addr_t last_ram_offset(void)
1154 {
1155 RAMBlock *block;
1156 ram_addr_t last = 0;
1157
1158 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1159 last = MAX(last, block->offset + block->length);
1160
1161 return last;
1162 }
1163
1164 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1165 {
1166 int ret;
1167
1168 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1169 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1170 "dump-guest-core", true)) {
1171 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1172 if (ret) {
1173 perror("qemu_madvise");
1174 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1175 "but dump_guest_core=off specified\n");
1176 }
1177 }
1178 }
1179
1180 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1181 {
1182 RAMBlock *new_block, *block;
1183
1184 new_block = NULL;
1185 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1186 if (block->offset == addr) {
1187 new_block = block;
1188 break;
1189 }
1190 }
1191 assert(new_block);
1192 assert(!new_block->idstr[0]);
1193
1194 if (dev) {
1195 char *id = qdev_get_dev_path(dev);
1196 if (id) {
1197 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1198 g_free(id);
1199 }
1200 }
1201 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1202
1203 /* This assumes the iothread lock is taken here too. */
1204 qemu_mutex_lock_ramlist();
1205 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1206 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1207 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1208 new_block->idstr);
1209 abort();
1210 }
1211 }
1212 qemu_mutex_unlock_ramlist();
1213 }
1214
1215 static int memory_try_enable_merging(void *addr, size_t len)
1216 {
1217 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1218 /* disabled by the user */
1219 return 0;
1220 }
1221
1222 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1223 }
1224
1225 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1226 MemoryRegion *mr)
1227 {
1228 RAMBlock *block, *new_block;
1229 ram_addr_t old_ram_size, new_ram_size;
1230
1231 old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1232
1233 size = TARGET_PAGE_ALIGN(size);
1234 new_block = g_malloc0(sizeof(*new_block));
1235 new_block->fd = -1;
1236
1237 /* This assumes the iothread lock is taken here too. */
1238 qemu_mutex_lock_ramlist();
1239 new_block->mr = mr;
1240 new_block->offset = find_ram_offset(size);
1241 if (host) {
1242 new_block->host = host;
1243 new_block->flags |= RAM_PREALLOC_MASK;
1244 } else if (xen_enabled()) {
1245 if (mem_path) {
1246 fprintf(stderr, "-mem-path not supported with Xen\n");
1247 exit(1);
1248 }
1249 xen_ram_alloc(new_block->offset, size, mr);
1250 } else {
1251 if (mem_path) {
1252 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1253 /*
1254 * file_ram_alloc() needs to allocate just like
1255 * phys_mem_alloc, but we haven't bothered to provide
1256 * a hook there.
1257 */
1258 fprintf(stderr,
1259 "-mem-path not supported with this accelerator\n");
1260 exit(1);
1261 }
1262 new_block->host = file_ram_alloc(new_block, size, mem_path);
1263 }
1264 if (!new_block->host) {
1265 new_block->host = phys_mem_alloc(size);
1266 if (!new_block->host) {
1267 fprintf(stderr, "Cannot set up guest memory '%s': %s\n",
1268 new_block->mr->name, strerror(errno));
1269 exit(1);
1270 }
1271 memory_try_enable_merging(new_block->host, size);
1272 }
1273 }
1274 new_block->length = size;
1275
1276 /* Keep the list sorted from biggest to smallest block. */
1277 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1278 if (block->length < new_block->length) {
1279 break;
1280 }
1281 }
1282 if (block) {
1283 QTAILQ_INSERT_BEFORE(block, new_block, next);
1284 } else {
1285 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1286 }
1287 ram_list.mru_block = NULL;
1288
1289 ram_list.version++;
1290 qemu_mutex_unlock_ramlist();
1291
1292 new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1293
1294 if (new_ram_size > old_ram_size) {
1295 int i;
1296 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1297 ram_list.dirty_memory[i] =
1298 bitmap_zero_extend(ram_list.dirty_memory[i],
1299 old_ram_size, new_ram_size);
1300 }
1301 }
1302 cpu_physical_memory_set_dirty_range(new_block->offset, size);
1303
1304 qemu_ram_setup_dump(new_block->host, size);
1305 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1306 qemu_madvise(new_block->host, size, QEMU_MADV_DONTFORK);
1307
1308 if (kvm_enabled())
1309 kvm_setup_guest_memory(new_block->host, size);
1310
1311 return new_block->offset;
1312 }
1313
1314 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1315 {
1316 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1317 }
1318
1319 void qemu_ram_free_from_ptr(ram_addr_t addr)
1320 {
1321 RAMBlock *block;
1322
1323 /* This assumes the iothread lock is taken here too. */
1324 qemu_mutex_lock_ramlist();
1325 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1326 if (addr == block->offset) {
1327 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1328 ram_list.mru_block = NULL;
1329 ram_list.version++;
1330 g_free(block);
1331 break;
1332 }
1333 }
1334 qemu_mutex_unlock_ramlist();
1335 }
1336
1337 void qemu_ram_free(ram_addr_t addr)
1338 {
1339 RAMBlock *block;
1340
1341 /* This assumes the iothread lock is taken here too. */
1342 qemu_mutex_lock_ramlist();
1343 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1344 if (addr == block->offset) {
1345 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1346 ram_list.mru_block = NULL;
1347 ram_list.version++;
1348 if (block->flags & RAM_PREALLOC_MASK) {
1349 ;
1350 } else if (xen_enabled()) {
1351 xen_invalidate_map_cache_entry(block->host);
1352 #ifndef _WIN32
1353 } else if (block->fd >= 0) {
1354 munmap(block->host, block->length);
1355 close(block->fd);
1356 #endif
1357 } else {
1358 qemu_anon_ram_free(block->host, block->length);
1359 }
1360 g_free(block);
1361 break;
1362 }
1363 }
1364 qemu_mutex_unlock_ramlist();
1365
1366 }
1367
1368 #ifndef _WIN32
1369 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1370 {
1371 RAMBlock *block;
1372 ram_addr_t offset;
1373 int flags;
1374 void *area, *vaddr;
1375
1376 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1377 offset = addr - block->offset;
1378 if (offset < block->length) {
1379 vaddr = block->host + offset;
1380 if (block->flags & RAM_PREALLOC_MASK) {
1381 ;
1382 } else if (xen_enabled()) {
1383 abort();
1384 } else {
1385 flags = MAP_FIXED;
1386 munmap(vaddr, length);
1387 if (block->fd >= 0) {
1388 #ifdef MAP_POPULATE
1389 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1390 MAP_PRIVATE;
1391 #else
1392 flags |= MAP_PRIVATE;
1393 #endif
1394 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1395 flags, block->fd, offset);
1396 } else {
1397 /*
1398 * Remap needs to match alloc. Accelerators that
1399 * set phys_mem_alloc never remap. If they did,
1400 * we'd need a remap hook here.
1401 */
1402 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1403
1404 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1405 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1406 flags, -1, 0);
1407 }
1408 if (area != vaddr) {
1409 fprintf(stderr, "Could not remap addr: "
1410 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1411 length, addr);
1412 exit(1);
1413 }
1414 memory_try_enable_merging(vaddr, length);
1415 qemu_ram_setup_dump(vaddr, length);
1416 }
1417 return;
1418 }
1419 }
1420 }
1421 #endif /* !_WIN32 */
1422
1423 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1424 With the exception of the softmmu code in this file, this should
1425 only be used for local memory (e.g. video ram) that the device owns,
1426 and knows it isn't going to access beyond the end of the block.
1427
1428 It should not be used for general purpose DMA.
1429 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1430 */
1431 void *qemu_get_ram_ptr(ram_addr_t addr)
1432 {
1433 RAMBlock *block = qemu_get_ram_block(addr);
1434
1435 if (xen_enabled()) {
1436 /* We need to check if the requested address is in the RAM
1437 * because we don't want to map the entire memory in QEMU.
1438 * In that case just map until the end of the page.
1439 */
1440 if (block->offset == 0) {
1441 return xen_map_cache(addr, 0, 0);
1442 } else if (block->host == NULL) {
1443 block->host =
1444 xen_map_cache(block->offset, block->length, 1);
1445 }
1446 }
1447 return block->host + (addr - block->offset);
1448 }
1449
1450 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1451 * but takes a size argument */
1452 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1453 {
1454 if (*size == 0) {
1455 return NULL;
1456 }
1457 if (xen_enabled()) {
1458 return xen_map_cache(addr, *size, 1);
1459 } else {
1460 RAMBlock *block;
1461
1462 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1463 if (addr - block->offset < block->length) {
1464 if (addr - block->offset + *size > block->length)
1465 *size = block->length - addr + block->offset;
1466 return block->host + (addr - block->offset);
1467 }
1468 }
1469
1470 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1471 abort();
1472 }
1473 }
1474
1475 /* Some of the softmmu routines need to translate from a host pointer
1476 (typically a TLB entry) back to a ram offset. */
1477 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1478 {
1479 RAMBlock *block;
1480 uint8_t *host = ptr;
1481
1482 if (xen_enabled()) {
1483 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1484 return qemu_get_ram_block(*ram_addr)->mr;
1485 }
1486
1487 block = ram_list.mru_block;
1488 if (block && block->host && host - block->host < block->length) {
1489 goto found;
1490 }
1491
1492 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1493 /* This case append when the block is not mapped. */
1494 if (block->host == NULL) {
1495 continue;
1496 }
1497 if (host - block->host < block->length) {
1498 goto found;
1499 }
1500 }
1501
1502 return NULL;
1503
1504 found:
1505 *ram_addr = block->offset + (host - block->host);
1506 return block->mr;
1507 }
1508
1509 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1510 uint64_t val, unsigned size)
1511 {
1512 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
1513 tb_invalidate_phys_page_fast(ram_addr, size);
1514 }
1515 switch (size) {
1516 case 1:
1517 stb_p(qemu_get_ram_ptr(ram_addr), val);
1518 break;
1519 case 2:
1520 stw_p(qemu_get_ram_ptr(ram_addr), val);
1521 break;
1522 case 4:
1523 stl_p(qemu_get_ram_ptr(ram_addr), val);
1524 break;
1525 default:
1526 abort();
1527 }
1528 cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_MIGRATION);
1529 cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_VGA);
1530 /* we remove the notdirty callback only if the code has been
1531 flushed */
1532 if (!cpu_physical_memory_is_clean(ram_addr)) {
1533 CPUArchState *env = current_cpu->env_ptr;
1534 tlb_set_dirty(env, env->mem_io_vaddr);
1535 }
1536 }
1537
1538 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1539 unsigned size, bool is_write)
1540 {
1541 return is_write;
1542 }
1543
1544 static const MemoryRegionOps notdirty_mem_ops = {
1545 .write = notdirty_mem_write,
1546 .valid.accepts = notdirty_mem_accepts,
1547 .endianness = DEVICE_NATIVE_ENDIAN,
1548 };
1549
1550 /* Generate a debug exception if a watchpoint has been hit. */
1551 static void check_watchpoint(int offset, int len_mask, int flags)
1552 {
1553 CPUArchState *env = current_cpu->env_ptr;
1554 target_ulong pc, cs_base;
1555 target_ulong vaddr;
1556 CPUWatchpoint *wp;
1557 int cpu_flags;
1558
1559 if (env->watchpoint_hit) {
1560 /* We re-entered the check after replacing the TB. Now raise
1561 * the debug interrupt so that is will trigger after the
1562 * current instruction. */
1563 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1564 return;
1565 }
1566 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1567 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1568 if ((vaddr == (wp->vaddr & len_mask) ||
1569 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1570 wp->flags |= BP_WATCHPOINT_HIT;
1571 if (!env->watchpoint_hit) {
1572 env->watchpoint_hit = wp;
1573 tb_check_watchpoint(env);
1574 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1575 env->exception_index = EXCP_DEBUG;
1576 cpu_loop_exit(env);
1577 } else {
1578 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1579 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1580 cpu_resume_from_signal(env, NULL);
1581 }
1582 }
1583 } else {
1584 wp->flags &= ~BP_WATCHPOINT_HIT;
1585 }
1586 }
1587 }
1588
1589 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1590 so these check for a hit then pass through to the normal out-of-line
1591 phys routines. */
1592 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1593 unsigned size)
1594 {
1595 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1596 switch (size) {
1597 case 1: return ldub_phys(addr);
1598 case 2: return lduw_phys(addr);
1599 case 4: return ldl_phys(addr);
1600 default: abort();
1601 }
1602 }
1603
1604 static void watch_mem_write(void *opaque, hwaddr addr,
1605 uint64_t val, unsigned size)
1606 {
1607 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1608 switch (size) {
1609 case 1:
1610 stb_phys(addr, val);
1611 break;
1612 case 2:
1613 stw_phys(addr, val);
1614 break;
1615 case 4:
1616 stl_phys(addr, val);
1617 break;
1618 default: abort();
1619 }
1620 }
1621
1622 static const MemoryRegionOps watch_mem_ops = {
1623 .read = watch_mem_read,
1624 .write = watch_mem_write,
1625 .endianness = DEVICE_NATIVE_ENDIAN,
1626 };
1627
1628 static uint64_t subpage_read(void *opaque, hwaddr addr,
1629 unsigned len)
1630 {
1631 subpage_t *subpage = opaque;
1632 uint8_t buf[4];
1633
1634 #if defined(DEBUG_SUBPAGE)
1635 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
1636 subpage, len, addr);
1637 #endif
1638 address_space_read(subpage->as, addr + subpage->base, buf, len);
1639 switch (len) {
1640 case 1:
1641 return ldub_p(buf);
1642 case 2:
1643 return lduw_p(buf);
1644 case 4:
1645 return ldl_p(buf);
1646 default:
1647 abort();
1648 }
1649 }
1650
1651 static void subpage_write(void *opaque, hwaddr addr,
1652 uint64_t value, unsigned len)
1653 {
1654 subpage_t *subpage = opaque;
1655 uint8_t buf[4];
1656
1657 #if defined(DEBUG_SUBPAGE)
1658 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
1659 " value %"PRIx64"\n",
1660 __func__, subpage, len, addr, value);
1661 #endif
1662 switch (len) {
1663 case 1:
1664 stb_p(buf, value);
1665 break;
1666 case 2:
1667 stw_p(buf, value);
1668 break;
1669 case 4:
1670 stl_p(buf, value);
1671 break;
1672 default:
1673 abort();
1674 }
1675 address_space_write(subpage->as, addr + subpage->base, buf, len);
1676 }
1677
1678 static bool subpage_accepts(void *opaque, hwaddr addr,
1679 unsigned len, bool is_write)
1680 {
1681 subpage_t *subpage = opaque;
1682 #if defined(DEBUG_SUBPAGE)
1683 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
1684 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1685 #endif
1686
1687 return address_space_access_valid(subpage->as, addr + subpage->base,
1688 len, is_write);
1689 }
1690
1691 static const MemoryRegionOps subpage_ops = {
1692 .read = subpage_read,
1693 .write = subpage_write,
1694 .valid.accepts = subpage_accepts,
1695 .endianness = DEVICE_NATIVE_ENDIAN,
1696 };
1697
1698 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1699 uint16_t section)
1700 {
1701 int idx, eidx;
1702
1703 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1704 return -1;
1705 idx = SUBPAGE_IDX(start);
1706 eidx = SUBPAGE_IDX(end);
1707 #if defined(DEBUG_SUBPAGE)
1708 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
1709 __func__, mmio, start, end, idx, eidx, section);
1710 #endif
1711 for (; idx <= eidx; idx++) {
1712 mmio->sub_section[idx] = section;
1713 }
1714
1715 return 0;
1716 }
1717
1718 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1719 {
1720 subpage_t *mmio;
1721
1722 mmio = g_malloc0(sizeof(subpage_t));
1723
1724 mmio->as = as;
1725 mmio->base = base;
1726 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1727 "subpage", TARGET_PAGE_SIZE);
1728 mmio->iomem.subpage = true;
1729 #if defined(DEBUG_SUBPAGE)
1730 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
1731 mmio, base, TARGET_PAGE_SIZE);
1732 #endif
1733 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1734
1735 return mmio;
1736 }
1737
1738 static uint16_t dummy_section(PhysPageMap *map, MemoryRegion *mr)
1739 {
1740 MemoryRegionSection section = {
1741 .address_space = &address_space_memory,
1742 .mr = mr,
1743 .offset_within_address_space = 0,
1744 .offset_within_region = 0,
1745 .size = int128_2_64(),
1746 };
1747
1748 return phys_section_add(map, &section);
1749 }
1750
1751 MemoryRegion *iotlb_to_region(AddressSpace *as, hwaddr index)
1752 {
1753 return as->dispatch->map.sections[index & ~TARGET_PAGE_MASK].mr;
1754 }
1755
1756 static void io_mem_init(void)
1757 {
1758 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1759 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1760 "unassigned", UINT64_MAX);
1761 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1762 "notdirty", UINT64_MAX);
1763 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1764 "watch", UINT64_MAX);
1765 }
1766
1767 static void mem_begin(MemoryListener *listener)
1768 {
1769 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1770 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
1771 uint16_t n;
1772
1773 n = dummy_section(&d->map, &io_mem_unassigned);
1774 assert(n == PHYS_SECTION_UNASSIGNED);
1775 n = dummy_section(&d->map, &io_mem_notdirty);
1776 assert(n == PHYS_SECTION_NOTDIRTY);
1777 n = dummy_section(&d->map, &io_mem_rom);
1778 assert(n == PHYS_SECTION_ROM);
1779 n = dummy_section(&d->map, &io_mem_watch);
1780 assert(n == PHYS_SECTION_WATCH);
1781
1782 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
1783 d->as = as;
1784 as->next_dispatch = d;
1785 }
1786
1787 static void mem_commit(MemoryListener *listener)
1788 {
1789 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1790 AddressSpaceDispatch *cur = as->dispatch;
1791 AddressSpaceDispatch *next = as->next_dispatch;
1792
1793 phys_page_compact_all(next, next->map.nodes_nb);
1794
1795 as->dispatch = next;
1796
1797 if (cur) {
1798 phys_sections_free(&cur->map);
1799 g_free(cur);
1800 }
1801 }
1802
1803 static void tcg_commit(MemoryListener *listener)
1804 {
1805 CPUState *cpu;
1806
1807 /* since each CPU stores ram addresses in its TLB cache, we must
1808 reset the modified entries */
1809 /* XXX: slow ! */
1810 CPU_FOREACH(cpu) {
1811 CPUArchState *env = cpu->env_ptr;
1812
1813 tlb_flush(env, 1);
1814 }
1815 }
1816
1817 static void core_log_global_start(MemoryListener *listener)
1818 {
1819 cpu_physical_memory_set_dirty_tracking(true);
1820 }
1821
1822 static void core_log_global_stop(MemoryListener *listener)
1823 {
1824 cpu_physical_memory_set_dirty_tracking(false);
1825 }
1826
1827 static MemoryListener core_memory_listener = {
1828 .log_global_start = core_log_global_start,
1829 .log_global_stop = core_log_global_stop,
1830 .priority = 1,
1831 };
1832
1833 static MemoryListener tcg_memory_listener = {
1834 .commit = tcg_commit,
1835 };
1836
1837 void address_space_init_dispatch(AddressSpace *as)
1838 {
1839 as->dispatch = NULL;
1840 as->dispatch_listener = (MemoryListener) {
1841 .begin = mem_begin,
1842 .commit = mem_commit,
1843 .region_add = mem_add,
1844 .region_nop = mem_add,
1845 .priority = 0,
1846 };
1847 memory_listener_register(&as->dispatch_listener, as);
1848 }
1849
1850 void address_space_destroy_dispatch(AddressSpace *as)
1851 {
1852 AddressSpaceDispatch *d = as->dispatch;
1853
1854 memory_listener_unregister(&as->dispatch_listener);
1855 g_free(d);
1856 as->dispatch = NULL;
1857 }
1858
1859 static void memory_map_init(void)
1860 {
1861 system_memory = g_malloc(sizeof(*system_memory));
1862
1863 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
1864 address_space_init(&address_space_memory, system_memory, "memory");
1865
1866 system_io = g_malloc(sizeof(*system_io));
1867 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
1868 65536);
1869 address_space_init(&address_space_io, system_io, "I/O");
1870
1871 memory_listener_register(&core_memory_listener, &address_space_memory);
1872 if (tcg_enabled()) {
1873 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1874 }
1875 }
1876
1877 MemoryRegion *get_system_memory(void)
1878 {
1879 return system_memory;
1880 }
1881
1882 MemoryRegion *get_system_io(void)
1883 {
1884 return system_io;
1885 }
1886
1887 #endif /* !defined(CONFIG_USER_ONLY) */
1888
1889 /* physical memory access (slow version, mainly for debug) */
1890 #if defined(CONFIG_USER_ONLY)
1891 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1892 uint8_t *buf, int len, int is_write)
1893 {
1894 int l, flags;
1895 target_ulong page;
1896 void * p;
1897
1898 while (len > 0) {
1899 page = addr & TARGET_PAGE_MASK;
1900 l = (page + TARGET_PAGE_SIZE) - addr;
1901 if (l > len)
1902 l = len;
1903 flags = page_get_flags(page);
1904 if (!(flags & PAGE_VALID))
1905 return -1;
1906 if (is_write) {
1907 if (!(flags & PAGE_WRITE))
1908 return -1;
1909 /* XXX: this code should not depend on lock_user */
1910 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1911 return -1;
1912 memcpy(p, buf, l);
1913 unlock_user(p, addr, l);
1914 } else {
1915 if (!(flags & PAGE_READ))
1916 return -1;
1917 /* XXX: this code should not depend on lock_user */
1918 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1919 return -1;
1920 memcpy(buf, p, l);
1921 unlock_user(p, addr, 0);
1922 }
1923 len -= l;
1924 buf += l;
1925 addr += l;
1926 }
1927 return 0;
1928 }
1929
1930 #else
1931
1932 static void invalidate_and_set_dirty(hwaddr addr,
1933 hwaddr length)
1934 {
1935 if (cpu_physical_memory_is_clean(addr)) {
1936 /* invalidate code */
1937 tb_invalidate_phys_page_range(addr, addr + length, 0);
1938 /* set dirty bit */
1939 cpu_physical_memory_set_dirty_flag(addr, DIRTY_MEMORY_VGA);
1940 cpu_physical_memory_set_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
1941 }
1942 xen_modified_memory(addr, length);
1943 }
1944
1945 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1946 {
1947 unsigned access_size_max = mr->ops->valid.max_access_size;
1948
1949 /* Regions are assumed to support 1-4 byte accesses unless
1950 otherwise specified. */
1951 if (access_size_max == 0) {
1952 access_size_max = 4;
1953 }
1954
1955 /* Bound the maximum access by the alignment of the address. */
1956 if (!mr->ops->impl.unaligned) {
1957 unsigned align_size_max = addr & -addr;
1958 if (align_size_max != 0 && align_size_max < access_size_max) {
1959 access_size_max = align_size_max;
1960 }
1961 }
1962
1963 /* Don't attempt accesses larger than the maximum. */
1964 if (l > access_size_max) {
1965 l = access_size_max;
1966 }
1967 if (l & (l - 1)) {
1968 l = 1 << (qemu_fls(l) - 1);
1969 }
1970
1971 return l;
1972 }
1973
1974 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1975 int len, bool is_write)
1976 {
1977 hwaddr l;
1978 uint8_t *ptr;
1979 uint64_t val;
1980 hwaddr addr1;
1981 MemoryRegion *mr;
1982 bool error = false;
1983
1984 while (len > 0) {
1985 l = len;
1986 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1987
1988 if (is_write) {
1989 if (!memory_access_is_direct(mr, is_write)) {
1990 l = memory_access_size(mr, l, addr1);
1991 /* XXX: could force current_cpu to NULL to avoid
1992 potential bugs */
1993 switch (l) {
1994 case 8:
1995 /* 64 bit write access */
1996 val = ldq_p(buf);
1997 error |= io_mem_write(mr, addr1, val, 8);
1998 break;
1999 case 4:
2000 /* 32 bit write access */
2001 val = ldl_p(buf);
2002 error |= io_mem_write(mr, addr1, val, 4);
2003 break;
2004 case 2:
2005 /* 16 bit write access */
2006 val = lduw_p(buf);
2007 error |= io_mem_write(mr, addr1, val, 2);
2008 break;
2009 case 1:
2010 /* 8 bit write access */
2011 val = ldub_p(buf);
2012 error |= io_mem_write(mr, addr1, val, 1);
2013 break;
2014 default:
2015 abort();
2016 }
2017 } else {
2018 addr1 += memory_region_get_ram_addr(mr);
2019 /* RAM case */
2020 ptr = qemu_get_ram_ptr(addr1);
2021 memcpy(ptr, buf, l);
2022 invalidate_and_set_dirty(addr1, l);
2023 }
2024 } else {
2025 if (!memory_access_is_direct(mr, is_write)) {
2026 /* I/O case */
2027 l = memory_access_size(mr, l, addr1);
2028 switch (l) {
2029 case 8:
2030 /* 64 bit read access */
2031 error |= io_mem_read(mr, addr1, &val, 8);
2032 stq_p(buf, val);
2033 break;
2034 case 4:
2035 /* 32 bit read access */
2036 error |= io_mem_read(mr, addr1, &val, 4);
2037 stl_p(buf, val);
2038 break;
2039 case 2:
2040 /* 16 bit read access */
2041 error |= io_mem_read(mr, addr1, &val, 2);
2042 stw_p(buf, val);
2043 break;
2044 case 1:
2045 /* 8 bit read access */
2046 error |= io_mem_read(mr, addr1, &val, 1);
2047 stb_p(buf, val);
2048 break;
2049 default:
2050 abort();
2051 }
2052 } else {
2053 /* RAM case */
2054 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2055 memcpy(buf, ptr, l);
2056 }
2057 }
2058 len -= l;
2059 buf += l;
2060 addr += l;
2061 }
2062
2063 return error;
2064 }
2065
2066 bool address_space_write(AddressSpace *as, hwaddr addr,
2067 const uint8_t *buf, int len)
2068 {
2069 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2070 }
2071
2072 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2073 {
2074 return address_space_rw(as, addr, buf, len, false);
2075 }
2076
2077
2078 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2079 int len, int is_write)
2080 {
2081 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2082 }
2083
2084 enum write_rom_type {
2085 WRITE_DATA,
2086 FLUSH_CACHE,
2087 };
2088
2089 static inline void cpu_physical_memory_write_rom_internal(
2090 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
2091 {
2092 hwaddr l;
2093 uint8_t *ptr;
2094 hwaddr addr1;
2095 MemoryRegion *mr;
2096
2097 while (len > 0) {
2098 l = len;
2099 mr = address_space_translate(&address_space_memory,
2100 addr, &addr1, &l, true);
2101
2102 if (!(memory_region_is_ram(mr) ||
2103 memory_region_is_romd(mr))) {
2104 /* do nothing */
2105 } else {
2106 addr1 += memory_region_get_ram_addr(mr);
2107 /* ROM/RAM case */
2108 ptr = qemu_get_ram_ptr(addr1);
2109 switch (type) {
2110 case WRITE_DATA:
2111 memcpy(ptr, buf, l);
2112 invalidate_and_set_dirty(addr1, l);
2113 break;
2114 case FLUSH_CACHE:
2115 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
2116 break;
2117 }
2118 }
2119 len -= l;
2120 buf += l;
2121 addr += l;
2122 }
2123 }
2124
2125 /* used for ROM loading : can write in RAM and ROM */
2126 void cpu_physical_memory_write_rom(hwaddr addr,
2127 const uint8_t *buf, int len)
2128 {
2129 cpu_physical_memory_write_rom_internal(addr, buf, len, WRITE_DATA);
2130 }
2131
2132 void cpu_flush_icache_range(hwaddr start, int len)
2133 {
2134 /*
2135 * This function should do the same thing as an icache flush that was
2136 * triggered from within the guest. For TCG we are always cache coherent,
2137 * so there is no need to flush anything. For KVM / Xen we need to flush
2138 * the host's instruction cache at least.
2139 */
2140 if (tcg_enabled()) {
2141 return;
2142 }
2143
2144 cpu_physical_memory_write_rom_internal(start, NULL, len, FLUSH_CACHE);
2145 }
2146
2147 typedef struct {
2148 MemoryRegion *mr;
2149 void *buffer;
2150 hwaddr addr;
2151 hwaddr len;
2152 } BounceBuffer;
2153
2154 static BounceBuffer bounce;
2155
2156 typedef struct MapClient {
2157 void *opaque;
2158 void (*callback)(void *opaque);
2159 QLIST_ENTRY(MapClient) link;
2160 } MapClient;
2161
2162 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2163 = QLIST_HEAD_INITIALIZER(map_client_list);
2164
2165 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2166 {
2167 MapClient *client = g_malloc(sizeof(*client));
2168
2169 client->opaque = opaque;
2170 client->callback = callback;
2171 QLIST_INSERT_HEAD(&map_client_list, client, link);
2172 return client;
2173 }
2174
2175 static void cpu_unregister_map_client(void *_client)
2176 {
2177 MapClient *client = (MapClient *)_client;
2178
2179 QLIST_REMOVE(client, link);
2180 g_free(client);
2181 }
2182
2183 static void cpu_notify_map_clients(void)
2184 {
2185 MapClient *client;
2186
2187 while (!QLIST_EMPTY(&map_client_list)) {
2188 client = QLIST_FIRST(&map_client_list);
2189 client->callback(client->opaque);
2190 cpu_unregister_map_client(client);
2191 }
2192 }
2193
2194 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2195 {
2196 MemoryRegion *mr;
2197 hwaddr l, xlat;
2198
2199 while (len > 0) {
2200 l = len;
2201 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2202 if (!memory_access_is_direct(mr, is_write)) {
2203 l = memory_access_size(mr, l, addr);
2204 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2205 return false;
2206 }
2207 }
2208
2209 len -= l;
2210 addr += l;
2211 }
2212 return true;
2213 }
2214
2215 /* Map a physical memory region into a host virtual address.
2216 * May map a subset of the requested range, given by and returned in *plen.
2217 * May return NULL if resources needed to perform the mapping are exhausted.
2218 * Use only for reads OR writes - not for read-modify-write operations.
2219 * Use cpu_register_map_client() to know when retrying the map operation is
2220 * likely to succeed.
2221 */
2222 void *address_space_map(AddressSpace *as,
2223 hwaddr addr,
2224 hwaddr *plen,
2225 bool is_write)
2226 {
2227 hwaddr len = *plen;
2228 hwaddr done = 0;
2229 hwaddr l, xlat, base;
2230 MemoryRegion *mr, *this_mr;
2231 ram_addr_t raddr;
2232
2233 if (len == 0) {
2234 return NULL;
2235 }
2236
2237 l = len;
2238 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2239 if (!memory_access_is_direct(mr, is_write)) {
2240 if (bounce.buffer) {
2241 return NULL;
2242 }
2243 /* Avoid unbounded allocations */
2244 l = MIN(l, TARGET_PAGE_SIZE);
2245 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
2246 bounce.addr = addr;
2247 bounce.len = l;
2248
2249 memory_region_ref(mr);
2250 bounce.mr = mr;
2251 if (!is_write) {
2252 address_space_read(as, addr, bounce.buffer, l);
2253 }
2254
2255 *plen = l;
2256 return bounce.buffer;
2257 }
2258
2259 base = xlat;
2260 raddr = memory_region_get_ram_addr(mr);
2261
2262 for (;;) {
2263 len -= l;
2264 addr += l;
2265 done += l;
2266 if (len == 0) {
2267 break;
2268 }
2269
2270 l = len;
2271 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2272 if (this_mr != mr || xlat != base + done) {
2273 break;
2274 }
2275 }
2276
2277 memory_region_ref(mr);
2278 *plen = done;
2279 return qemu_ram_ptr_length(raddr + base, plen);
2280 }
2281
2282 /* Unmaps a memory region previously mapped by address_space_map().
2283 * Will also mark the memory as dirty if is_write == 1. access_len gives
2284 * the amount of memory that was actually read or written by the caller.
2285 */
2286 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2287 int is_write, hwaddr access_len)
2288 {
2289 if (buffer != bounce.buffer) {
2290 MemoryRegion *mr;
2291 ram_addr_t addr1;
2292
2293 mr = qemu_ram_addr_from_host(buffer, &addr1);
2294 assert(mr != NULL);
2295 if (is_write) {
2296 while (access_len) {
2297 unsigned l;
2298 l = TARGET_PAGE_SIZE;
2299 if (l > access_len)
2300 l = access_len;
2301 invalidate_and_set_dirty(addr1, l);
2302 addr1 += l;
2303 access_len -= l;
2304 }
2305 }
2306 if (xen_enabled()) {
2307 xen_invalidate_map_cache_entry(buffer);
2308 }
2309 memory_region_unref(mr);
2310 return;
2311 }
2312 if (is_write) {
2313 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2314 }
2315 qemu_vfree(bounce.buffer);
2316 bounce.buffer = NULL;
2317 memory_region_unref(bounce.mr);
2318 cpu_notify_map_clients();
2319 }
2320
2321 void *cpu_physical_memory_map(hwaddr addr,
2322 hwaddr *plen,
2323 int is_write)
2324 {
2325 return address_space_map(&address_space_memory, addr, plen, is_write);
2326 }
2327
2328 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2329 int is_write, hwaddr access_len)
2330 {
2331 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2332 }
2333
2334 /* warning: addr must be aligned */
2335 static inline uint32_t ldl_phys_internal(hwaddr addr,
2336 enum device_endian endian)
2337 {
2338 uint8_t *ptr;
2339 uint64_t val;
2340 MemoryRegion *mr;
2341 hwaddr l = 4;
2342 hwaddr addr1;
2343
2344 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2345 false);
2346 if (l < 4 || !memory_access_is_direct(mr, false)) {
2347 /* I/O case */
2348 io_mem_read(mr, addr1, &val, 4);
2349 #if defined(TARGET_WORDS_BIGENDIAN)
2350 if (endian == DEVICE_LITTLE_ENDIAN) {
2351 val = bswap32(val);
2352 }
2353 #else
2354 if (endian == DEVICE_BIG_ENDIAN) {
2355 val = bswap32(val);
2356 }
2357 #endif
2358 } else {
2359 /* RAM case */
2360 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2361 & TARGET_PAGE_MASK)
2362 + addr1);
2363 switch (endian) {
2364 case DEVICE_LITTLE_ENDIAN:
2365 val = ldl_le_p(ptr);
2366 break;
2367 case DEVICE_BIG_ENDIAN:
2368 val = ldl_be_p(ptr);
2369 break;
2370 default:
2371 val = ldl_p(ptr);
2372 break;
2373 }
2374 }
2375 return val;
2376 }
2377
2378 uint32_t ldl_phys(hwaddr addr)
2379 {
2380 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2381 }
2382
2383 uint32_t ldl_le_phys(hwaddr addr)
2384 {
2385 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2386 }
2387
2388 uint32_t ldl_be_phys(hwaddr addr)
2389 {
2390 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2391 }
2392
2393 /* warning: addr must be aligned */
2394 static inline uint64_t ldq_phys_internal(hwaddr addr,
2395 enum device_endian endian)
2396 {
2397 uint8_t *ptr;
2398 uint64_t val;
2399 MemoryRegion *mr;
2400 hwaddr l = 8;
2401 hwaddr addr1;
2402
2403 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2404 false);
2405 if (l < 8 || !memory_access_is_direct(mr, false)) {
2406 /* I/O case */
2407 io_mem_read(mr, addr1, &val, 8);
2408 #if defined(TARGET_WORDS_BIGENDIAN)
2409 if (endian == DEVICE_LITTLE_ENDIAN) {
2410 val = bswap64(val);
2411 }
2412 #else
2413 if (endian == DEVICE_BIG_ENDIAN) {
2414 val = bswap64(val);
2415 }
2416 #endif
2417 } else {
2418 /* RAM case */
2419 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2420 & TARGET_PAGE_MASK)
2421 + addr1);
2422 switch (endian) {
2423 case DEVICE_LITTLE_ENDIAN:
2424 val = ldq_le_p(ptr);
2425 break;
2426 case DEVICE_BIG_ENDIAN:
2427 val = ldq_be_p(ptr);
2428 break;
2429 default:
2430 val = ldq_p(ptr);
2431 break;
2432 }
2433 }
2434 return val;
2435 }
2436
2437 uint64_t ldq_phys(hwaddr addr)
2438 {
2439 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2440 }
2441
2442 uint64_t ldq_le_phys(hwaddr addr)
2443 {
2444 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2445 }
2446
2447 uint64_t ldq_be_phys(hwaddr addr)
2448 {
2449 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2450 }
2451
2452 /* XXX: optimize */
2453 uint32_t ldub_phys(hwaddr addr)
2454 {
2455 uint8_t val;
2456 cpu_physical_memory_read(addr, &val, 1);
2457 return val;
2458 }
2459
2460 /* warning: addr must be aligned */
2461 static inline uint32_t lduw_phys_internal(hwaddr addr,
2462 enum device_endian endian)
2463 {
2464 uint8_t *ptr;
2465 uint64_t val;
2466 MemoryRegion *mr;
2467 hwaddr l = 2;
2468 hwaddr addr1;
2469
2470 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2471 false);
2472 if (l < 2 || !memory_access_is_direct(mr, false)) {
2473 /* I/O case */
2474 io_mem_read(mr, addr1, &val, 2);
2475 #if defined(TARGET_WORDS_BIGENDIAN)
2476 if (endian == DEVICE_LITTLE_ENDIAN) {
2477 val = bswap16(val);
2478 }
2479 #else
2480 if (endian == DEVICE_BIG_ENDIAN) {
2481 val = bswap16(val);
2482 }
2483 #endif
2484 } else {
2485 /* RAM case */
2486 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2487 & TARGET_PAGE_MASK)
2488 + addr1);
2489 switch (endian) {
2490 case DEVICE_LITTLE_ENDIAN:
2491 val = lduw_le_p(ptr);
2492 break;
2493 case DEVICE_BIG_ENDIAN:
2494 val = lduw_be_p(ptr);
2495 break;
2496 default:
2497 val = lduw_p(ptr);
2498 break;
2499 }
2500 }
2501 return val;
2502 }
2503
2504 uint32_t lduw_phys(hwaddr addr)
2505 {
2506 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2507 }
2508
2509 uint32_t lduw_le_phys(hwaddr addr)
2510 {
2511 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2512 }
2513
2514 uint32_t lduw_be_phys(hwaddr addr)
2515 {
2516 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2517 }
2518
2519 /* warning: addr must be aligned. The ram page is not masked as dirty
2520 and the code inside is not invalidated. It is useful if the dirty
2521 bits are used to track modified PTEs */
2522 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2523 {
2524 uint8_t *ptr;
2525 MemoryRegion *mr;
2526 hwaddr l = 4;
2527 hwaddr addr1;
2528
2529 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2530 true);
2531 if (l < 4 || !memory_access_is_direct(mr, true)) {
2532 io_mem_write(mr, addr1, val, 4);
2533 } else {
2534 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2535 ptr = qemu_get_ram_ptr(addr1);
2536 stl_p(ptr, val);
2537
2538 if (unlikely(in_migration)) {
2539 if (cpu_physical_memory_is_clean(addr1)) {
2540 /* invalidate code */
2541 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2542 /* set dirty bit */
2543 cpu_physical_memory_set_dirty_flag(addr1,
2544 DIRTY_MEMORY_MIGRATION);
2545 cpu_physical_memory_set_dirty_flag(addr1, DIRTY_MEMORY_VGA);
2546 }
2547 }
2548 }
2549 }
2550
2551 /* warning: addr must be aligned */
2552 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2553 enum device_endian endian)
2554 {
2555 uint8_t *ptr;
2556 MemoryRegion *mr;
2557 hwaddr l = 4;
2558 hwaddr addr1;
2559
2560 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2561 true);
2562 if (l < 4 || !memory_access_is_direct(mr, true)) {
2563 #if defined(TARGET_WORDS_BIGENDIAN)
2564 if (endian == DEVICE_LITTLE_ENDIAN) {
2565 val = bswap32(val);
2566 }
2567 #else
2568 if (endian == DEVICE_BIG_ENDIAN) {
2569 val = bswap32(val);
2570 }
2571 #endif
2572 io_mem_write(mr, addr1, val, 4);
2573 } else {
2574 /* RAM case */
2575 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2576 ptr = qemu_get_ram_ptr(addr1);
2577 switch (endian) {
2578 case DEVICE_LITTLE_ENDIAN:
2579 stl_le_p(ptr, val);
2580 break;
2581 case DEVICE_BIG_ENDIAN:
2582 stl_be_p(ptr, val);
2583 break;
2584 default:
2585 stl_p(ptr, val);
2586 break;
2587 }
2588 invalidate_and_set_dirty(addr1, 4);
2589 }
2590 }
2591
2592 void stl_phys(hwaddr addr, uint32_t val)
2593 {
2594 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2595 }
2596
2597 void stl_le_phys(hwaddr addr, uint32_t val)
2598 {
2599 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2600 }
2601
2602 void stl_be_phys(hwaddr addr, uint32_t val)
2603 {
2604 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2605 }
2606
2607 /* XXX: optimize */
2608 void stb_phys(hwaddr addr, uint32_t val)
2609 {
2610 uint8_t v = val;
2611 cpu_physical_memory_write(addr, &v, 1);
2612 }
2613
2614 /* warning: addr must be aligned */
2615 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2616 enum device_endian endian)
2617 {
2618 uint8_t *ptr;
2619 MemoryRegion *mr;
2620 hwaddr l = 2;
2621 hwaddr addr1;
2622
2623 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2624 true);
2625 if (l < 2 || !memory_access_is_direct(mr, true)) {
2626 #if defined(TARGET_WORDS_BIGENDIAN)
2627 if (endian == DEVICE_LITTLE_ENDIAN) {
2628 val = bswap16(val);
2629 }
2630 #else
2631 if (endian == DEVICE_BIG_ENDIAN) {
2632 val = bswap16(val);
2633 }
2634 #endif
2635 io_mem_write(mr, addr1, val, 2);
2636 } else {
2637 /* RAM case */
2638 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2639 ptr = qemu_get_ram_ptr(addr1);
2640 switch (endian) {
2641 case DEVICE_LITTLE_ENDIAN:
2642 stw_le_p(ptr, val);
2643 break;
2644 case DEVICE_BIG_ENDIAN:
2645 stw_be_p(ptr, val);
2646 break;
2647 default:
2648 stw_p(ptr, val);
2649 break;
2650 }
2651 invalidate_and_set_dirty(addr1, 2);
2652 }
2653 }
2654
2655 void stw_phys(hwaddr addr, uint32_t val)
2656 {
2657 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2658 }
2659
2660 void stw_le_phys(hwaddr addr, uint32_t val)
2661 {
2662 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2663 }
2664
2665 void stw_be_phys(hwaddr addr, uint32_t val)
2666 {
2667 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2668 }
2669
2670 /* XXX: optimize */
2671 void stq_phys(hwaddr addr, uint64_t val)
2672 {
2673 val = tswap64(val);
2674 cpu_physical_memory_write(addr, &val, 8);
2675 }
2676
2677 void stq_le_phys(hwaddr addr, uint64_t val)
2678 {
2679 val = cpu_to_le64(val);
2680 cpu_physical_memory_write(addr, &val, 8);
2681 }
2682
2683 void stq_be_phys(hwaddr addr, uint64_t val)
2684 {
2685 val = cpu_to_be64(val);
2686 cpu_physical_memory_write(addr, &val, 8);
2687 }
2688
2689 /* virtual memory access for debug (includes writing to ROM) */
2690 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2691 uint8_t *buf, int len, int is_write)
2692 {
2693 int l;
2694 hwaddr phys_addr;
2695 target_ulong page;
2696
2697 while (len > 0) {
2698 page = addr & TARGET_PAGE_MASK;
2699 phys_addr = cpu_get_phys_page_debug(cpu, page);
2700 /* if no physical page mapped, return an error */
2701 if (phys_addr == -1)
2702 return -1;
2703 l = (page + TARGET_PAGE_SIZE) - addr;
2704 if (l > len)
2705 l = len;
2706 phys_addr += (addr & ~TARGET_PAGE_MASK);
2707 if (is_write)
2708 cpu_physical_memory_write_rom(phys_addr, buf, l);
2709 else
2710 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2711 len -= l;
2712 buf += l;
2713 addr += l;
2714 }
2715 return 0;
2716 }
2717 #endif
2718
2719 #if !defined(CONFIG_USER_ONLY)
2720
2721 /*
2722 * A helper function for the _utterly broken_ virtio device model to find out if
2723 * it's running on a big endian machine. Don't do this at home kids!
2724 */
2725 bool virtio_is_big_endian(void);
2726 bool virtio_is_big_endian(void)
2727 {
2728 #if defined(TARGET_WORDS_BIGENDIAN)
2729 return true;
2730 #else
2731 return false;
2732 #endif
2733 }
2734
2735 #endif
2736
2737 #ifndef CONFIG_USER_ONLY
2738 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2739 {
2740 MemoryRegion*mr;
2741 hwaddr l = 1;
2742
2743 mr = address_space_translate(&address_space_memory,
2744 phys_addr, &phys_addr, &l, false);
2745
2746 return !(memory_region_is_ram(mr) ||
2747 memory_region_is_romd(mr));
2748 }
2749
2750 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2751 {
2752 RAMBlock *block;
2753
2754 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2755 func(block->host, block->offset, block->length, opaque);
2756 }
2757 }
2758 #endif
QEmu