?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Move hardcoded initialization of xen-platform device.
[qemu.git] / exec.c
blob0b0118bd4170710340b07e068835acf878761b89
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "hw/xen/xen.h"
35 #include "qemu/timer.h"
36 #include "qemu/config-file.h"
37 #include "exec/memory.h"
38 #include "sysemu/dma.h"
39 #include "exec/address-spaces.h"
40 #if defined(CONFIG_USER_ONLY)
41 #include <qemu.h>
42 #else /* !CONFIG_USER_ONLY */
43 #include "sysemu/xen-mapcache.h"
44 #include "trace.h"
45 #endif
46 #include "exec/cpu-all.h"
47
48 #include "exec/cputlb.h"
49 #include "translate-all.h"
50
51 #include "exec/memory-internal.h"
52
53 //#define DEBUG_SUBPAGE
54
55 #if !defined(CONFIG_USER_ONLY)
56 int phys_ram_fd;
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 CPUArchState *first_cpu;
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUArchState *,cpu_single_env);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 struct AddressSpaceDispatch {
92 /* This is a multi-level map on the physical address space.
93 * The bottom level has pointers to MemoryRegionSections.
94 */
95 PhysPageEntry phys_map;
96 MemoryListener listener;
97 AddressSpace *as;
98 };
99
100 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
101 typedef struct subpage_t {
102 MemoryRegion iomem;
103 AddressSpace *as;
104 hwaddr base;
105 uint16_t sub_section[TARGET_PAGE_SIZE];
106 } subpage_t;
107
108 static MemoryRegionSection *phys_sections;
109 static unsigned phys_sections_nb, phys_sections_nb_alloc;
110 static uint16_t phys_section_unassigned;
111 static uint16_t phys_section_notdirty;
112 static uint16_t phys_section_rom;
113 static uint16_t phys_section_watch;
114
115 /* Simple allocator for PhysPageEntry nodes */
116 static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
117 static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
118
119 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
120
121 static void io_mem_init(void);
122 static void memory_map_init(void);
123 static void *qemu_safe_ram_ptr(ram_addr_t addr);
124
125 static MemoryRegion io_mem_watch;
126 #endif
127
128 #if !defined(CONFIG_USER_ONLY)
129
130 static void phys_map_node_reserve(unsigned nodes)
131 {
132 if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) {
133 typedef PhysPageEntry Node[L2_SIZE];
134 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16);
135 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc,
136 phys_map_nodes_nb + nodes);
137 phys_map_nodes = g_renew(Node, phys_map_nodes,
138 phys_map_nodes_nb_alloc);
139 }
140 }
141
142 static uint16_t phys_map_node_alloc(void)
143 {
144 unsigned i;
145 uint16_t ret;
146
147 ret = phys_map_nodes_nb++;
148 assert(ret != PHYS_MAP_NODE_NIL);
149 assert(ret != phys_map_nodes_nb_alloc);
150 for (i = 0; i < L2_SIZE; ++i) {
151 phys_map_nodes[ret][i].is_leaf = 0;
152 phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
153 }
154 return ret;
155 }
156
157 static void phys_map_nodes_reset(void)
158 {
159 phys_map_nodes_nb = 0;
160 }
161
162
163 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
164 hwaddr *nb, uint16_t leaf,
165 int level)
166 {
167 PhysPageEntry *p;
168 int i;
169 hwaddr step = (hwaddr)1 << (level * L2_BITS);
170
171 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
172 lp->ptr = phys_map_node_alloc();
173 p = phys_map_nodes[lp->ptr];
174 if (level == 0) {
175 for (i = 0; i < L2_SIZE; i++) {
176 p[i].is_leaf = 1;
177 p[i].ptr = phys_section_unassigned;
178 }
179 }
180 } else {
181 p = phys_map_nodes[lp->ptr];
182 }
183 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
184
185 while (*nb && lp < &p[L2_SIZE]) {
186 if ((*index & (step - 1)) == 0 && *nb >= step) {
187 lp->is_leaf = true;
188 lp->ptr = leaf;
189 *index += step;
190 *nb -= step;
191 } else {
192 phys_page_set_level(lp, index, nb, leaf, level - 1);
193 }
194 ++lp;
195 }
196 }
197
198 static void phys_page_set(AddressSpaceDispatch *d,
199 hwaddr index, hwaddr nb,
200 uint16_t leaf)
201 {
202 /* Wildly overreserve - it doesn't matter much. */
203 phys_map_node_reserve(3 * P_L2_LEVELS);
204
205 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
206 }
207
208 static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr index)
209 {
210 PhysPageEntry lp = d->phys_map;
211 PhysPageEntry *p;
212 int i;
213
214 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
215 if (lp.ptr == PHYS_MAP_NODE_NIL) {
216 return &phys_sections[phys_section_unassigned];
217 }
218 p = phys_map_nodes[lp.ptr];
219 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
220 }
221 return &phys_sections[lp.ptr];
222 }
223
224 bool memory_region_is_unassigned(MemoryRegion *mr)
225 {
226 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
227 && mr != &io_mem_watch;
228 }
229
230 static MemoryRegionSection *address_space_lookup_region(AddressSpace *as,
231 hwaddr addr,
232 bool resolve_subpage)
233 {
234 MemoryRegionSection *section;
235 subpage_t *subpage;
236
237 section = phys_page_find(as->dispatch, addr >> TARGET_PAGE_BITS);
238 if (resolve_subpage && section->mr->subpage) {
239 subpage = container_of(section->mr, subpage_t, iomem);
240 section = &phys_sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
241 }
242 return section;
243 }
244
245 static MemoryRegionSection *
246 address_space_translate_internal(AddressSpace *as, hwaddr addr, hwaddr *xlat,
247 hwaddr *plen, bool resolve_subpage)
248 {
249 MemoryRegionSection *section;
250 Int128 diff;
251
252 section = address_space_lookup_region(as, addr, resolve_subpage);
253 /* Compute offset within MemoryRegionSection */
254 addr -= section->offset_within_address_space;
255
256 /* Compute offset within MemoryRegion */
257 *xlat = addr + section->offset_within_region;
258
259 diff = int128_sub(section->mr->size, int128_make64(addr));
260 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
261 return section;
262 }
263
264 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
265 hwaddr *xlat, hwaddr *plen,
266 bool is_write)
267 {
268 IOMMUTLBEntry iotlb;
269 MemoryRegionSection *section;
270 MemoryRegion *mr;
271 hwaddr len = *plen;
272
273 for (;;) {
274 section = address_space_translate_internal(as, addr, &addr, plen, true);
275 mr = section->mr;
276
277 if (!mr->iommu_ops) {
278 break;
279 }
280
281 iotlb = mr->iommu_ops->translate(mr, addr);
282 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
283 | (addr & iotlb.addr_mask));
284 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
285 if (!(iotlb.perm & (1 << is_write))) {
286 mr = &io_mem_unassigned;
287 break;
288 }
289
290 as = iotlb.target_as;
291 }
292
293 *plen = len;
294 *xlat = addr;
295 return mr;
296 }
297
298 MemoryRegionSection *
299 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
300 hwaddr *plen)
301 {
302 MemoryRegionSection *section;
303 section = address_space_translate_internal(as, addr, xlat, plen, false);
304
305 assert(!section->mr->iommu_ops);
306 return section;
307 }
308 #endif
309
310 void cpu_exec_init_all(void)
311 {
312 #if !defined(CONFIG_USER_ONLY)
313 qemu_mutex_init(&ram_list.mutex);
314 memory_map_init();
315 io_mem_init();
316 #endif
317 }
318
319 #if !defined(CONFIG_USER_ONLY)
320
321 static int cpu_common_post_load(void *opaque, int version_id)
322 {
323 CPUState *cpu = opaque;
324
325 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
326 version_id is increased. */
327 cpu->interrupt_request &= ~0x01;
328 tlb_flush(cpu->env_ptr, 1);
329
330 return 0;
331 }
332
333 static const VMStateDescription vmstate_cpu_common = {
334 .name = "cpu_common",
335 .version_id = 1,
336 .minimum_version_id = 1,
337 .minimum_version_id_old = 1,
338 .post_load = cpu_common_post_load,
339 .fields = (VMStateField []) {
340 VMSTATE_UINT32(halted, CPUState),
341 VMSTATE_UINT32(interrupt_request, CPUState),
342 VMSTATE_END_OF_LIST()
343 }
344 };
345 #else
346 #define vmstate_cpu_common vmstate_dummy
347 #endif
348
349 CPUState *qemu_get_cpu(int index)
350 {
351 CPUArchState *env = first_cpu;
352 CPUState *cpu = NULL;
353
354 while (env) {
355 cpu = ENV_GET_CPU(env);
356 if (cpu->cpu_index == index) {
357 break;
358 }
359 env = env->next_cpu;
360 }
361
362 return env ? cpu : NULL;
363 }
364
365 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
366 {
367 CPUArchState *env = first_cpu;
368
369 while (env) {
370 func(ENV_GET_CPU(env), data);
371 env = env->next_cpu;
372 }
373 }
374
375 void cpu_exec_init(CPUArchState *env)
376 {
377 CPUState *cpu = ENV_GET_CPU(env);
378 CPUClass *cc = CPU_GET_CLASS(cpu);
379 CPUArchState **penv;
380 int cpu_index;
381
382 #if defined(CONFIG_USER_ONLY)
383 cpu_list_lock();
384 #endif
385 env->next_cpu = NULL;
386 penv = &first_cpu;
387 cpu_index = 0;
388 while (*penv != NULL) {
389 penv = &(*penv)->next_cpu;
390 cpu_index++;
391 }
392 cpu->cpu_index = cpu_index;
393 cpu->numa_node = 0;
394 QTAILQ_INIT(&env->breakpoints);
395 QTAILQ_INIT(&env->watchpoints);
396 #ifndef CONFIG_USER_ONLY
397 cpu->thread_id = qemu_get_thread_id();
398 #endif
399 *penv = env;
400 #if defined(CONFIG_USER_ONLY)
401 cpu_list_unlock();
402 #endif
403 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
404 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
405 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
406 cpu_save, cpu_load, env);
407 assert(cc->vmsd == NULL);
408 #endif
409 if (cc->vmsd != NULL) {
410 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
411 }
412 }
413
414 #if defined(TARGET_HAS_ICE)
415 #if defined(CONFIG_USER_ONLY)
416 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
417 {
418 tb_invalidate_phys_page_range(pc, pc + 1, 0);
419 }
420 #else
421 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
422 {
423 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
424 (pc & ~TARGET_PAGE_MASK));
425 }
426 #endif
427 #endif /* TARGET_HAS_ICE */
428
429 #if defined(CONFIG_USER_ONLY)
430 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
431
432 {
433 }
434
435 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
436 int flags, CPUWatchpoint **watchpoint)
437 {
438 return -ENOSYS;
439 }
440 #else
441 /* Add a watchpoint. */
442 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
443 int flags, CPUWatchpoint **watchpoint)
444 {
445 target_ulong len_mask = ~(len - 1);
446 CPUWatchpoint *wp;
447
448 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
449 if ((len & (len - 1)) || (addr & ~len_mask) ||
450 len == 0 || len > TARGET_PAGE_SIZE) {
451 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
452 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
453 return -EINVAL;
454 }
455 wp = g_malloc(sizeof(*wp));
456
457 wp->vaddr = addr;
458 wp->len_mask = len_mask;
459 wp->flags = flags;
460
461 /* keep all GDB-injected watchpoints in front */
462 if (flags & BP_GDB)
463 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
464 else
465 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
466
467 tlb_flush_page(env, addr);
468
469 if (watchpoint)
470 *watchpoint = wp;
471 return 0;
472 }
473
474 /* Remove a specific watchpoint. */
475 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
476 int flags)
477 {
478 target_ulong len_mask = ~(len - 1);
479 CPUWatchpoint *wp;
480
481 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
482 if (addr == wp->vaddr && len_mask == wp->len_mask
483 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
484 cpu_watchpoint_remove_by_ref(env, wp);
485 return 0;
486 }
487 }
488 return -ENOENT;
489 }
490
491 /* Remove a specific watchpoint by reference. */
492 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
493 {
494 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
495
496 tlb_flush_page(env, watchpoint->vaddr);
497
498 g_free(watchpoint);
499 }
500
501 /* Remove all matching watchpoints. */
502 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
503 {
504 CPUWatchpoint *wp, *next;
505
506 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
507 if (wp->flags & mask)
508 cpu_watchpoint_remove_by_ref(env, wp);
509 }
510 }
511 #endif
512
513 /* Add a breakpoint. */
514 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
515 CPUBreakpoint **breakpoint)
516 {
517 #if defined(TARGET_HAS_ICE)
518 CPUBreakpoint *bp;
519
520 bp = g_malloc(sizeof(*bp));
521
522 bp->pc = pc;
523 bp->flags = flags;
524
525 /* keep all GDB-injected breakpoints in front */
526 if (flags & BP_GDB)
527 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
528 else
529 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
530
531 breakpoint_invalidate(env, pc);
532
533 if (breakpoint)
534 *breakpoint = bp;
535 return 0;
536 #else
537 return -ENOSYS;
538 #endif
539 }
540
541 /* Remove a specific breakpoint. */
542 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
543 {
544 #if defined(TARGET_HAS_ICE)
545 CPUBreakpoint *bp;
546
547 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
548 if (bp->pc == pc && bp->flags == flags) {
549 cpu_breakpoint_remove_by_ref(env, bp);
550 return 0;
551 }
552 }
553 return -ENOENT;
554 #else
555 return -ENOSYS;
556 #endif
557 }
558
559 /* Remove a specific breakpoint by reference. */
560 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
561 {
562 #if defined(TARGET_HAS_ICE)
563 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
564
565 breakpoint_invalidate(env, breakpoint->pc);
566
567 g_free(breakpoint);
568 #endif
569 }
570
571 /* Remove all matching breakpoints. */
572 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
573 {
574 #if defined(TARGET_HAS_ICE)
575 CPUBreakpoint *bp, *next;
576
577 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
578 if (bp->flags & mask)
579 cpu_breakpoint_remove_by_ref(env, bp);
580 }
581 #endif
582 }
583
584 /* enable or disable single step mode. EXCP_DEBUG is returned by the
585 CPU loop after each instruction */
586 void cpu_single_step(CPUArchState *env, int enabled)
587 {
588 #if defined(TARGET_HAS_ICE)
589 if (env->singlestep_enabled != enabled) {
590 env->singlestep_enabled = enabled;
591 if (kvm_enabled())
592 kvm_update_guest_debug(env, 0);
593 else {
594 /* must flush all the translated code to avoid inconsistencies */
595 /* XXX: only flush what is necessary */
596 tb_flush(env);
597 }
598 }
599 #endif
600 }
601
602 void cpu_exit(CPUArchState *env)
603 {
604 CPUState *cpu = ENV_GET_CPU(env);
605
606 cpu->exit_request = 1;
607 cpu->tcg_exit_req = 1;
608 }
609
610 void cpu_abort(CPUArchState *env, const char *fmt, ...)
611 {
612 va_list ap;
613 va_list ap2;
614
615 va_start(ap, fmt);
616 va_copy(ap2, ap);
617 fprintf(stderr, "qemu: fatal: ");
618 vfprintf(stderr, fmt, ap);
619 fprintf(stderr, "\n");
620 cpu_dump_state(env, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
621 if (qemu_log_enabled()) {
622 qemu_log("qemu: fatal: ");
623 qemu_log_vprintf(fmt, ap2);
624 qemu_log("\n");
625 log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
626 qemu_log_flush();
627 qemu_log_close();
628 }
629 va_end(ap2);
630 va_end(ap);
631 #if defined(CONFIG_USER_ONLY)
632 {
633 struct sigaction act;
634 sigfillset(&act.sa_mask);
635 act.sa_handler = SIG_DFL;
636 sigaction(SIGABRT, &act, NULL);
637 }
638 #endif
639 abort();
640 }
641
642 CPUArchState *cpu_copy(CPUArchState *env)
643 {
644 CPUArchState *new_env = cpu_init(env->cpu_model_str);
645 CPUArchState *next_cpu = new_env->next_cpu;
646 #if defined(TARGET_HAS_ICE)
647 CPUBreakpoint *bp;
648 CPUWatchpoint *wp;
649 #endif
650
651 memcpy(new_env, env, sizeof(CPUArchState));
652
653 /* Preserve chaining. */
654 new_env->next_cpu = next_cpu;
655
656 /* Clone all break/watchpoints.
657 Note: Once we support ptrace with hw-debug register access, make sure
658 BP_CPU break/watchpoints are handled correctly on clone. */
659 QTAILQ_INIT(&env->breakpoints);
660 QTAILQ_INIT(&env->watchpoints);
661 #if defined(TARGET_HAS_ICE)
662 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
663 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
664 }
665 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
666 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
667 wp->flags, NULL);
668 }
669 #endif
670
671 return new_env;
672 }
673
674 #if !defined(CONFIG_USER_ONLY)
675 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
676 uintptr_t length)
677 {
678 uintptr_t start1;
679
680 /* we modify the TLB cache so that the dirty bit will be set again
681 when accessing the range */
682 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
683 /* Check that we don't span multiple blocks - this breaks the
684 address comparisons below. */
685 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
686 != (end - 1) - start) {
687 abort();
688 }
689 cpu_tlb_reset_dirty_all(start1, length);
690
691 }
692
693 /* Note: start and end must be within the same ram block. */
694 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
695 int dirty_flags)
696 {
697 uintptr_t length;
698
699 start &= TARGET_PAGE_MASK;
700 end = TARGET_PAGE_ALIGN(end);
701
702 length = end - start;
703 if (length == 0)
704 return;
705 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
706
707 if (tcg_enabled()) {
708 tlb_reset_dirty_range_all(start, end, length);
709 }
710 }
711
712 static int cpu_physical_memory_set_dirty_tracking(int enable)
713 {
714 int ret = 0;
715 in_migration = enable;
716 return ret;
717 }
718
719 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
720 MemoryRegionSection *section,
721 target_ulong vaddr,
722 hwaddr paddr, hwaddr xlat,
723 int prot,
724 target_ulong *address)
725 {
726 hwaddr iotlb;
727 CPUWatchpoint *wp;
728
729 if (memory_region_is_ram(section->mr)) {
730 /* Normal RAM. */
731 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
732 + xlat;
733 if (!section->readonly) {
734 iotlb |= phys_section_notdirty;
735 } else {
736 iotlb |= phys_section_rom;
737 }
738 } else {
739 iotlb = section - phys_sections;
740 iotlb += xlat;
741 }
742
743 /* Make accesses to pages with watchpoints go via the
744 watchpoint trap routines. */
745 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
746 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
747 /* Avoid trapping reads of pages with a write breakpoint. */
748 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
749 iotlb = phys_section_watch + paddr;
750 *address |= TLB_MMIO;
751 break;
752 }
753 }
754 }
755
756 return iotlb;
757 }
758 #endif /* defined(CONFIG_USER_ONLY) */
759
760 #if !defined(CONFIG_USER_ONLY)
761
762 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
763 uint16_t section);
764 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
765 static void destroy_page_desc(uint16_t section_index)
766 {
767 MemoryRegionSection *section = &phys_sections[section_index];
768 MemoryRegion *mr = section->mr;
769
770 if (mr->subpage) {
771 subpage_t *subpage = container_of(mr, subpage_t, iomem);
772 memory_region_destroy(&subpage->iomem);
773 g_free(subpage);
774 }
775 }
776
777 static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)
778 {
779 unsigned i;
780 PhysPageEntry *p;
781
782 if (lp->ptr == PHYS_MAP_NODE_NIL) {
783 return;
784 }
785
786 p = phys_map_nodes[lp->ptr];
787 for (i = 0; i < L2_SIZE; ++i) {
788 if (!p[i].is_leaf) {
789 destroy_l2_mapping(&p[i], level - 1);
790 } else {
791 destroy_page_desc(p[i].ptr);
792 }
793 }
794 lp->is_leaf = 0;
795 lp->ptr = PHYS_MAP_NODE_NIL;
796 }
797
798 static void destroy_all_mappings(AddressSpaceDispatch *d)
799 {
800 destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
801 phys_map_nodes_reset();
802 }
803
804 static uint16_t phys_section_add(MemoryRegionSection *section)
805 {
806 /* The physical section number is ORed with a page-aligned
807 * pointer to produce the iotlb entries. Thus it should
808 * never overflow into the page-aligned value.
809 */
810 assert(phys_sections_nb < TARGET_PAGE_SIZE);
811
812 if (phys_sections_nb == phys_sections_nb_alloc) {
813 phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16);
814 phys_sections = g_renew(MemoryRegionSection, phys_sections,
815 phys_sections_nb_alloc);
816 }
817 phys_sections[phys_sections_nb] = *section;
818 return phys_sections_nb++;
819 }
820
821 static void phys_sections_clear(void)
822 {
823 phys_sections_nb = 0;
824 }
825
826 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
827 {
828 subpage_t *subpage;
829 hwaddr base = section->offset_within_address_space
830 & TARGET_PAGE_MASK;
831 MemoryRegionSection *existing = phys_page_find(d, base >> TARGET_PAGE_BITS);
832 MemoryRegionSection subsection = {
833 .offset_within_address_space = base,
834 .size = int128_make64(TARGET_PAGE_SIZE),
835 };
836 hwaddr start, end;
837
838 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
839
840 if (!(existing->mr->subpage)) {
841 subpage = subpage_init(d->as, base);
842 subsection.mr = &subpage->iomem;
843 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
844 phys_section_add(&subsection));
845 } else {
846 subpage = container_of(existing->mr, subpage_t, iomem);
847 }
848 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
849 end = start + int128_get64(section->size) - 1;
850 subpage_register(subpage, start, end, phys_section_add(section));
851 }
852
853
854 static void register_multipage(AddressSpaceDispatch *d,
855 MemoryRegionSection *section)
856 {
857 hwaddr start_addr = section->offset_within_address_space;
858 uint16_t section_index = phys_section_add(section);
859 uint64_t num_pages = int128_get64(int128_rshift(section->size,
860 TARGET_PAGE_BITS));
861
862 assert(num_pages);
863 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
864 }
865
866 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
867 {
868 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
869 MemoryRegionSection now = *section, remain = *section;
870 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
871
872 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
873 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
874 - now.offset_within_address_space;
875
876 now.size = int128_min(int128_make64(left), now.size);
877 register_subpage(d, &now);
878 } else {
879 now.size = int128_zero();
880 }
881 while (int128_ne(remain.size, now.size)) {
882 remain.size = int128_sub(remain.size, now.size);
883 remain.offset_within_address_space += int128_get64(now.size);
884 remain.offset_within_region += int128_get64(now.size);
885 now = remain;
886 if (int128_lt(remain.size, page_size)) {
887 register_subpage(d, &now);
888 } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
889 now.size = page_size;
890 register_subpage(d, &now);
891 } else {
892 now.size = int128_and(now.size, int128_neg(page_size));
893 register_multipage(d, &now);
894 }
895 }
896 }
897
898 void qemu_flush_coalesced_mmio_buffer(void)
899 {
900 if (kvm_enabled())
901 kvm_flush_coalesced_mmio_buffer();
902 }
903
904 void qemu_mutex_lock_ramlist(void)
905 {
906 qemu_mutex_lock(&ram_list.mutex);
907 }
908
909 void qemu_mutex_unlock_ramlist(void)
910 {
911 qemu_mutex_unlock(&ram_list.mutex);
912 }
913
914 #if defined(__linux__) && !defined(TARGET_S390X)
915
916 #include <sys/vfs.h>
917
918 #define HUGETLBFS_MAGIC 0x958458f6
919
920 static long gethugepagesize(const char *path)
921 {
922 struct statfs fs;
923 int ret;
924
925 do {
926 ret = statfs(path, &fs);
927 } while (ret != 0 && errno == EINTR);
928
929 if (ret != 0) {
930 perror(path);
931 return 0;
932 }
933
934 if (fs.f_type != HUGETLBFS_MAGIC)
935 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
936
937 return fs.f_bsize;
938 }
939
940 static void *file_ram_alloc(RAMBlock *block,
941 ram_addr_t memory,
942 const char *path)
943 {
944 char *filename;
945 char *sanitized_name;
946 char *c;
947 void *area;
948 int fd;
949 #ifdef MAP_POPULATE
950 int flags;
951 #endif
952 unsigned long hpagesize;
953
954 hpagesize = gethugepagesize(path);
955 if (!hpagesize) {
956 return NULL;
957 }
958
959 if (memory < hpagesize) {
960 return NULL;
961 }
962
963 if (kvm_enabled() && !kvm_has_sync_mmu()) {
964 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
965 return NULL;
966 }
967
968 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
969 sanitized_name = g_strdup(block->mr->name);
970 for (c = sanitized_name; *c != '\0'; c++) {
971 if (*c == '/')
972 *c = '_';
973 }
974
975 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
976 sanitized_name);
977 g_free(sanitized_name);
978
979 fd = mkstemp(filename);
980 if (fd < 0) {
981 perror("unable to create backing store for hugepages");
982 g_free(filename);
983 return NULL;
984 }
985 unlink(filename);
986 g_free(filename);
987
988 memory = (memory+hpagesize-1) & ~(hpagesize-1);
989
990 /*
991 * ftruncate is not supported by hugetlbfs in older
992 * hosts, so don't bother bailing out on errors.
993 * If anything goes wrong with it under other filesystems,
994 * mmap will fail.
995 */
996 if (ftruncate(fd, memory))
997 perror("ftruncate");
998
999 #ifdef MAP_POPULATE
1000 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
1001 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
1002 * to sidestep this quirk.
1003 */
1004 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
1005 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
1006 #else
1007 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
1008 #endif
1009 if (area == MAP_FAILED) {
1010 perror("file_ram_alloc: can't mmap RAM pages");
1011 close(fd);
1012 return (NULL);
1013 }
1014 block->fd = fd;
1015 return area;
1016 }
1017 #endif
1018
1019 static ram_addr_t find_ram_offset(ram_addr_t size)
1020 {
1021 RAMBlock *block, *next_block;
1022 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1023
1024 assert(size != 0); /* it would hand out same offset multiple times */
1025
1026 if (QTAILQ_EMPTY(&ram_list.blocks))
1027 return 0;
1028
1029 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1030 ram_addr_t end, next = RAM_ADDR_MAX;
1031
1032 end = block->offset + block->length;
1033
1034 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1035 if (next_block->offset >= end) {
1036 next = MIN(next, next_block->offset);
1037 }
1038 }
1039 if (next - end >= size && next - end < mingap) {
1040 offset = end;
1041 mingap = next - end;
1042 }
1043 }
1044
1045 if (offset == RAM_ADDR_MAX) {
1046 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1047 (uint64_t)size);
1048 abort();
1049 }
1050
1051 return offset;
1052 }
1053
1054 ram_addr_t last_ram_offset(void)
1055 {
1056 RAMBlock *block;
1057 ram_addr_t last = 0;
1058
1059 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1060 last = MAX(last, block->offset + block->length);
1061
1062 return last;
1063 }
1064
1065 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1066 {
1067 int ret;
1068 QemuOpts *machine_opts;
1069
1070 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1071 machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1072 if (machine_opts &&
1073 !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
1074 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1075 if (ret) {
1076 perror("qemu_madvise");
1077 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1078 "but dump_guest_core=off specified\n");
1079 }
1080 }
1081 }
1082
1083 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1084 {
1085 RAMBlock *new_block, *block;
1086
1087 new_block = NULL;
1088 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1089 if (block->offset == addr) {
1090 new_block = block;
1091 break;
1092 }
1093 }
1094 assert(new_block);
1095 assert(!new_block->idstr[0]);
1096
1097 if (dev) {
1098 char *id = qdev_get_dev_path(dev);
1099 if (id) {
1100 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1101 g_free(id);
1102 }
1103 }
1104 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1105
1106 /* This assumes the iothread lock is taken here too. */
1107 qemu_mutex_lock_ramlist();
1108 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1109 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1110 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1111 new_block->idstr);
1112 abort();
1113 }
1114 }
1115 qemu_mutex_unlock_ramlist();
1116 }
1117
1118 static int memory_try_enable_merging(void *addr, size_t len)
1119 {
1120 QemuOpts *opts;
1121
1122 opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1123 if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
1124 /* disabled by the user */
1125 return 0;
1126 }
1127
1128 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1129 }
1130
1131 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1132 MemoryRegion *mr)
1133 {
1134 RAMBlock *block, *new_block;
1135
1136 size = TARGET_PAGE_ALIGN(size);
1137 new_block = g_malloc0(sizeof(*new_block));
1138
1139 /* This assumes the iothread lock is taken here too. */
1140 qemu_mutex_lock_ramlist();
1141 new_block->mr = mr;
1142 new_block->offset = find_ram_offset(size);
1143 if (host) {
1144 new_block->host = host;
1145 new_block->flags |= RAM_PREALLOC_MASK;
1146 } else {
1147 if (mem_path) {
1148 #if defined (__linux__) && !defined(TARGET_S390X)
1149 new_block->host = file_ram_alloc(new_block, size, mem_path);
1150 if (!new_block->host) {
1151 new_block->host = qemu_anon_ram_alloc(size);
1152 memory_try_enable_merging(new_block->host, size);
1153 }
1154 #else
1155 fprintf(stderr, "-mem-path option unsupported\n");
1156 exit(1);
1157 #endif
1158 } else {
1159 if (xen_enabled()) {
1160 xen_ram_alloc(new_block->offset, size, mr);
1161 } else if (kvm_enabled()) {
1162 /* some s390/kvm configurations have special constraints */
1163 new_block->host = kvm_ram_alloc(size);
1164 } else {
1165 new_block->host = qemu_anon_ram_alloc(size);
1166 }
1167 memory_try_enable_merging(new_block->host, size);
1168 }
1169 }
1170 new_block->length = size;
1171
1172 /* Keep the list sorted from biggest to smallest block. */
1173 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1174 if (block->length < new_block->length) {
1175 break;
1176 }
1177 }
1178 if (block) {
1179 QTAILQ_INSERT_BEFORE(block, new_block, next);
1180 } else {
1181 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1182 }
1183 ram_list.mru_block = NULL;
1184
1185 ram_list.version++;
1186 qemu_mutex_unlock_ramlist();
1187
1188 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1189 last_ram_offset() >> TARGET_PAGE_BITS);
1190 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1191 0, size >> TARGET_PAGE_BITS);
1192 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1193
1194 qemu_ram_setup_dump(new_block->host, size);
1195 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1196
1197 if (kvm_enabled())
1198 kvm_setup_guest_memory(new_block->host, size);
1199
1200 return new_block->offset;
1201 }
1202
1203 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1204 {
1205 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1206 }
1207
1208 void qemu_ram_free_from_ptr(ram_addr_t addr)
1209 {
1210 RAMBlock *block;
1211
1212 /* This assumes the iothread lock is taken here too. */
1213 qemu_mutex_lock_ramlist();
1214 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1215 if (addr == block->offset) {
1216 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1217 ram_list.mru_block = NULL;
1218 ram_list.version++;
1219 g_free(block);
1220 break;
1221 }
1222 }
1223 qemu_mutex_unlock_ramlist();
1224 }
1225
1226 void qemu_ram_free(ram_addr_t addr)
1227 {
1228 RAMBlock *block;
1229
1230 /* This assumes the iothread lock is taken here too. */
1231 qemu_mutex_lock_ramlist();
1232 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1233 if (addr == block->offset) {
1234 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1235 ram_list.mru_block = NULL;
1236 ram_list.version++;
1237 if (block->flags & RAM_PREALLOC_MASK) {
1238 ;
1239 } else if (mem_path) {
1240 #if defined (__linux__) && !defined(TARGET_S390X)
1241 if (block->fd) {
1242 munmap(block->host, block->length);
1243 close(block->fd);
1244 } else {
1245 qemu_anon_ram_free(block->host, block->length);
1246 }
1247 #else
1248 abort();
1249 #endif
1250 } else {
1251 if (xen_enabled()) {
1252 xen_invalidate_map_cache_entry(block->host);
1253 } else {
1254 qemu_anon_ram_free(block->host, block->length);
1255 }
1256 }
1257 g_free(block);
1258 break;
1259 }
1260 }
1261 qemu_mutex_unlock_ramlist();
1262
1263 }
1264
1265 #ifndef _WIN32
1266 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1267 {
1268 RAMBlock *block;
1269 ram_addr_t offset;
1270 int flags;
1271 void *area, *vaddr;
1272
1273 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1274 offset = addr - block->offset;
1275 if (offset < block->length) {
1276 vaddr = block->host + offset;
1277 if (block->flags & RAM_PREALLOC_MASK) {
1278 ;
1279 } else {
1280 flags = MAP_FIXED;
1281 munmap(vaddr, length);
1282 if (mem_path) {
1283 #if defined(__linux__) && !defined(TARGET_S390X)
1284 if (block->fd) {
1285 #ifdef MAP_POPULATE
1286 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1287 MAP_PRIVATE;
1288 #else
1289 flags |= MAP_PRIVATE;
1290 #endif
1291 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1292 flags, block->fd, offset);
1293 } else {
1294 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1295 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1296 flags, -1, 0);
1297 }
1298 #else
1299 abort();
1300 #endif
1301 } else {
1302 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1303 flags |= MAP_SHARED | MAP_ANONYMOUS;
1304 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1305 flags, -1, 0);
1306 #else
1307 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1308 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1309 flags, -1, 0);
1310 #endif
1311 }
1312 if (area != vaddr) {
1313 fprintf(stderr, "Could not remap addr: "
1314 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1315 length, addr);
1316 exit(1);
1317 }
1318 memory_try_enable_merging(vaddr, length);
1319 qemu_ram_setup_dump(vaddr, length);
1320 }
1321 return;
1322 }
1323 }
1324 }
1325 #endif /* !_WIN32 */
1326
1327 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1328 With the exception of the softmmu code in this file, this should
1329 only be used for local memory (e.g. video ram) that the device owns,
1330 and knows it isn't going to access beyond the end of the block.
1331
1332 It should not be used for general purpose DMA.
1333 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1334 */
1335 void *qemu_get_ram_ptr(ram_addr_t addr)
1336 {
1337 RAMBlock *block;
1338
1339 /* The list is protected by the iothread lock here. */
1340 block = ram_list.mru_block;
1341 if (block && addr - block->offset < block->length) {
1342 goto found;
1343 }
1344 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1345 if (addr - block->offset < block->length) {
1346 goto found;
1347 }
1348 }
1349
1350 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1351 abort();
1352
1353 found:
1354 ram_list.mru_block = block;
1355 if (xen_enabled()) {
1356 /* We need to check if the requested address is in the RAM
1357 * because we don't want to map the entire memory in QEMU.
1358 * In that case just map until the end of the page.
1359 */
1360 if (block->offset == 0) {
1361 return xen_map_cache(addr, 0, 0);
1362 } else if (block->host == NULL) {
1363 block->host =
1364 xen_map_cache(block->offset, block->length, 1);
1365 }
1366 }
1367 return block->host + (addr - block->offset);
1368 }
1369
1370 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1371 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1372 *
1373 * ??? Is this still necessary?
1374 */
1375 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1376 {
1377 RAMBlock *block;
1378
1379 /* The list is protected by the iothread lock here. */
1380 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1381 if (addr - block->offset < block->length) {
1382 if (xen_enabled()) {
1383 /* We need to check if the requested address is in the RAM
1384 * because we don't want to map the entire memory in QEMU.
1385 * In that case just map until the end of the page.
1386 */
1387 if (block->offset == 0) {
1388 return xen_map_cache(addr, 0, 0);
1389 } else if (block->host == NULL) {
1390 block->host =
1391 xen_map_cache(block->offset, block->length, 1);
1392 }
1393 }
1394 return block->host + (addr - block->offset);
1395 }
1396 }
1397
1398 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1399 abort();
1400
1401 return NULL;
1402 }
1403
1404 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1405 * but takes a size argument */
1406 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1407 {
1408 if (*size == 0) {
1409 return NULL;
1410 }
1411 if (xen_enabled()) {
1412 return xen_map_cache(addr, *size, 1);
1413 } else {
1414 RAMBlock *block;
1415
1416 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1417 if (addr - block->offset < block->length) {
1418 if (addr - block->offset + *size > block->length)
1419 *size = block->length - addr + block->offset;
1420 return block->host + (addr - block->offset);
1421 }
1422 }
1423
1424 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1425 abort();
1426 }
1427 }
1428
1429 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1430 {
1431 RAMBlock *block;
1432 uint8_t *host = ptr;
1433
1434 if (xen_enabled()) {
1435 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1436 return 0;
1437 }
1438
1439 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1440 /* This case append when the block is not mapped. */
1441 if (block->host == NULL) {
1442 continue;
1443 }
1444 if (host - block->host < block->length) {
1445 *ram_addr = block->offset + (host - block->host);
1446 return 0;
1447 }
1448 }
1449
1450 return -1;
1451 }
1452
1453 /* Some of the softmmu routines need to translate from a host pointer
1454 (typically a TLB entry) back to a ram offset. */
1455 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
1456 {
1457 ram_addr_t ram_addr;
1458
1459 if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
1460 fprintf(stderr, "Bad ram pointer %p\n", ptr);
1461 abort();
1462 }
1463 return ram_addr;
1464 }
1465
1466 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1467 uint64_t val, unsigned size)
1468 {
1469 int dirty_flags;
1470 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1471 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1472 tb_invalidate_phys_page_fast(ram_addr, size);
1473 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1474 }
1475 switch (size) {
1476 case 1:
1477 stb_p(qemu_get_ram_ptr(ram_addr), val);
1478 break;
1479 case 2:
1480 stw_p(qemu_get_ram_ptr(ram_addr), val);
1481 break;
1482 case 4:
1483 stl_p(qemu_get_ram_ptr(ram_addr), val);
1484 break;
1485 default:
1486 abort();
1487 }
1488 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1489 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1490 /* we remove the notdirty callback only if the code has been
1491 flushed */
1492 if (dirty_flags == 0xff)
1493 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1494 }
1495
1496 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1497 unsigned size, bool is_write)
1498 {
1499 return is_write;
1500 }
1501
1502 static const MemoryRegionOps notdirty_mem_ops = {
1503 .write = notdirty_mem_write,
1504 .valid.accepts = notdirty_mem_accepts,
1505 .endianness = DEVICE_NATIVE_ENDIAN,
1506 };
1507
1508 /* Generate a debug exception if a watchpoint has been hit. */
1509 static void check_watchpoint(int offset, int len_mask, int flags)
1510 {
1511 CPUArchState *env = cpu_single_env;
1512 target_ulong pc, cs_base;
1513 target_ulong vaddr;
1514 CPUWatchpoint *wp;
1515 int cpu_flags;
1516
1517 if (env->watchpoint_hit) {
1518 /* We re-entered the check after replacing the TB. Now raise
1519 * the debug interrupt so that is will trigger after the
1520 * current instruction. */
1521 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1522 return;
1523 }
1524 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1525 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1526 if ((vaddr == (wp->vaddr & len_mask) ||
1527 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1528 wp->flags |= BP_WATCHPOINT_HIT;
1529 if (!env->watchpoint_hit) {
1530 env->watchpoint_hit = wp;
1531 tb_check_watchpoint(env);
1532 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1533 env->exception_index = EXCP_DEBUG;
1534 cpu_loop_exit(env);
1535 } else {
1536 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1537 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1538 cpu_resume_from_signal(env, NULL);
1539 }
1540 }
1541 } else {
1542 wp->flags &= ~BP_WATCHPOINT_HIT;
1543 }
1544 }
1545 }
1546
1547 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1548 so these check for a hit then pass through to the normal out-of-line
1549 phys routines. */
1550 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1551 unsigned size)
1552 {
1553 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1554 switch (size) {
1555 case 1: return ldub_phys(addr);
1556 case 2: return lduw_phys(addr);
1557 case 4: return ldl_phys(addr);
1558 default: abort();
1559 }
1560 }
1561
1562 static void watch_mem_write(void *opaque, hwaddr addr,
1563 uint64_t val, unsigned size)
1564 {
1565 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1566 switch (size) {
1567 case 1:
1568 stb_phys(addr, val);
1569 break;
1570 case 2:
1571 stw_phys(addr, val);
1572 break;
1573 case 4:
1574 stl_phys(addr, val);
1575 break;
1576 default: abort();
1577 }
1578 }
1579
1580 static const MemoryRegionOps watch_mem_ops = {
1581 .read = watch_mem_read,
1582 .write = watch_mem_write,
1583 .endianness = DEVICE_NATIVE_ENDIAN,
1584 };
1585
1586 static uint64_t subpage_read(void *opaque, hwaddr addr,
1587 unsigned len)
1588 {
1589 subpage_t *subpage = opaque;
1590 uint8_t buf[4];
1591
1592 #if defined(DEBUG_SUBPAGE)
1593 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1594 subpage, len, addr);
1595 #endif
1596 address_space_read(subpage->as, addr + subpage->base, buf, len);
1597 switch (len) {
1598 case 1:
1599 return ldub_p(buf);
1600 case 2:
1601 return lduw_p(buf);
1602 case 4:
1603 return ldl_p(buf);
1604 default:
1605 abort();
1606 }
1607 }
1608
1609 static void subpage_write(void *opaque, hwaddr addr,
1610 uint64_t value, unsigned len)
1611 {
1612 subpage_t *subpage = opaque;
1613 uint8_t buf[4];
1614
1615 #if defined(DEBUG_SUBPAGE)
1616 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1617 " value %"PRIx64"\n",
1618 __func__, subpage, len, addr, value);
1619 #endif
1620 switch (len) {
1621 case 1:
1622 stb_p(buf, value);
1623 break;
1624 case 2:
1625 stw_p(buf, value);
1626 break;
1627 case 4:
1628 stl_p(buf, value);
1629 break;
1630 default:
1631 abort();
1632 }
1633 address_space_write(subpage->as, addr + subpage->base, buf, len);
1634 }
1635
1636 static bool subpage_accepts(void *opaque, hwaddr addr,
1637 unsigned size, bool is_write)
1638 {
1639 subpage_t *subpage = opaque;
1640 #if defined(DEBUG_SUBPAGE)
1641 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1642 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1643 #endif
1644
1645 return address_space_access_valid(subpage->as, addr + subpage->base,
1646 size, is_write);
1647 }
1648
1649 static const MemoryRegionOps subpage_ops = {
1650 .read = subpage_read,
1651 .write = subpage_write,
1652 .valid.accepts = subpage_accepts,
1653 .endianness = DEVICE_NATIVE_ENDIAN,
1654 };
1655
1656 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1657 uint16_t section)
1658 {
1659 int idx, eidx;
1660
1661 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1662 return -1;
1663 idx = SUBPAGE_IDX(start);
1664 eidx = SUBPAGE_IDX(end);
1665 #if defined(DEBUG_SUBPAGE)
1666 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1667 mmio, start, end, idx, eidx, memory);
1668 #endif
1669 for (; idx <= eidx; idx++) {
1670 mmio->sub_section[idx] = section;
1671 }
1672
1673 return 0;
1674 }
1675
1676 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1677 {
1678 subpage_t *mmio;
1679
1680 mmio = g_malloc0(sizeof(subpage_t));
1681
1682 mmio->as = as;
1683 mmio->base = base;
1684 memory_region_init_io(&mmio->iomem, &subpage_ops, mmio,
1685 "subpage", TARGET_PAGE_SIZE);
1686 mmio->iomem.subpage = true;
1687 #if defined(DEBUG_SUBPAGE)
1688 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1689 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1690 #endif
1691 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, phys_section_unassigned);
1692
1693 return mmio;
1694 }
1695
1696 static uint16_t dummy_section(MemoryRegion *mr)
1697 {
1698 MemoryRegionSection section = {
1699 .mr = mr,
1700 .offset_within_address_space = 0,
1701 .offset_within_region = 0,
1702 .size = int128_2_64(),
1703 };
1704
1705 return phys_section_add(&section);
1706 }
1707
1708 MemoryRegion *iotlb_to_region(hwaddr index)
1709 {
1710 return phys_sections[index & ~TARGET_PAGE_MASK].mr;
1711 }
1712
1713 static void io_mem_init(void)
1714 {
1715 memory_region_init_io(&io_mem_rom, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1716 memory_region_init_io(&io_mem_unassigned, &unassigned_mem_ops, NULL,
1717 "unassigned", UINT64_MAX);
1718 memory_region_init_io(&io_mem_notdirty, &notdirty_mem_ops, NULL,
1719 "notdirty", UINT64_MAX);
1720 memory_region_init_io(&io_mem_watch, &watch_mem_ops, NULL,
1721 "watch", UINT64_MAX);
1722 }
1723
1724 static void mem_begin(MemoryListener *listener)
1725 {
1726 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
1727
1728 destroy_all_mappings(d);
1729 d->phys_map.ptr = PHYS_MAP_NODE_NIL;
1730 }
1731
1732 static void core_begin(MemoryListener *listener)
1733 {
1734 phys_sections_clear();
1735 phys_section_unassigned = dummy_section(&io_mem_unassigned);
1736 phys_section_notdirty = dummy_section(&io_mem_notdirty);
1737 phys_section_rom = dummy_section(&io_mem_rom);
1738 phys_section_watch = dummy_section(&io_mem_watch);
1739 }
1740
1741 static void tcg_commit(MemoryListener *listener)
1742 {
1743 CPUArchState *env;
1744
1745 /* since each CPU stores ram addresses in its TLB cache, we must
1746 reset the modified entries */
1747 /* XXX: slow ! */
1748 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1749 tlb_flush(env, 1);
1750 }
1751 }
1752
1753 static void core_log_global_start(MemoryListener *listener)
1754 {
1755 cpu_physical_memory_set_dirty_tracking(1);
1756 }
1757
1758 static void core_log_global_stop(MemoryListener *listener)
1759 {
1760 cpu_physical_memory_set_dirty_tracking(0);
1761 }
1762
1763 static void io_region_add(MemoryListener *listener,
1764 MemoryRegionSection *section)
1765 {
1766 MemoryRegionIORange *mrio = g_new(MemoryRegionIORange, 1);
1767
1768 mrio->mr = section->mr;
1769 mrio->offset = section->offset_within_region;
1770 iorange_init(&mrio->iorange, &memory_region_iorange_ops,
1771 section->offset_within_address_space,
1772 int128_get64(section->size));
1773 ioport_register(&mrio->iorange);
1774 }
1775
1776 static void io_region_del(MemoryListener *listener,
1777 MemoryRegionSection *section)
1778 {
1779 isa_unassign_ioport(section->offset_within_address_space,
1780 int128_get64(section->size));
1781 }
1782
1783 static MemoryListener core_memory_listener = {
1784 .begin = core_begin,
1785 .log_global_start = core_log_global_start,
1786 .log_global_stop = core_log_global_stop,
1787 .priority = 1,
1788 };
1789
1790 static MemoryListener io_memory_listener = {
1791 .region_add = io_region_add,
1792 .region_del = io_region_del,
1793 .priority = 0,
1794 };
1795
1796 static MemoryListener tcg_memory_listener = {
1797 .commit = tcg_commit,
1798 };
1799
1800 void address_space_init_dispatch(AddressSpace *as)
1801 {
1802 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1803
1804 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1805 d->listener = (MemoryListener) {
1806 .begin = mem_begin,
1807 .region_add = mem_add,
1808 .region_nop = mem_add,
1809 .priority = 0,
1810 };
1811 d->as = as;
1812 as->dispatch = d;
1813 memory_listener_register(&d->listener, as);
1814 }
1815
1816 void address_space_destroy_dispatch(AddressSpace *as)
1817 {
1818 AddressSpaceDispatch *d = as->dispatch;
1819
1820 memory_listener_unregister(&d->listener);
1821 destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
1822 g_free(d);
1823 as->dispatch = NULL;
1824 }
1825
1826 static void memory_map_init(void)
1827 {
1828 system_memory = g_malloc(sizeof(*system_memory));
1829 memory_region_init(system_memory, "system", INT64_MAX);
1830 address_space_init(&address_space_memory, system_memory, "memory");
1831
1832 system_io = g_malloc(sizeof(*system_io));
1833 memory_region_init(system_io, "io", 65536);
1834 address_space_init(&address_space_io, system_io, "I/O");
1835
1836 memory_listener_register(&core_memory_listener, &address_space_memory);
1837 memory_listener_register(&io_memory_listener, &address_space_io);
1838 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1839 }
1840
1841 MemoryRegion *get_system_memory(void)
1842 {
1843 return system_memory;
1844 }
1845
1846 MemoryRegion *get_system_io(void)
1847 {
1848 return system_io;
1849 }
1850
1851 #endif /* !defined(CONFIG_USER_ONLY) */
1852
1853 /* physical memory access (slow version, mainly for debug) */
1854 #if defined(CONFIG_USER_ONLY)
1855 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1856 uint8_t *buf, int len, int is_write)
1857 {
1858 int l, flags;
1859 target_ulong page;
1860 void * p;
1861
1862 while (len > 0) {
1863 page = addr & TARGET_PAGE_MASK;
1864 l = (page + TARGET_PAGE_SIZE) - addr;
1865 if (l > len)
1866 l = len;
1867 flags = page_get_flags(page);
1868 if (!(flags & PAGE_VALID))
1869 return -1;
1870 if (is_write) {
1871 if (!(flags & PAGE_WRITE))
1872 return -1;
1873 /* XXX: this code should not depend on lock_user */
1874 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1875 return -1;
1876 memcpy(p, buf, l);
1877 unlock_user(p, addr, l);
1878 } else {
1879 if (!(flags & PAGE_READ))
1880 return -1;
1881 /* XXX: this code should not depend on lock_user */
1882 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1883 return -1;
1884 memcpy(buf, p, l);
1885 unlock_user(p, addr, 0);
1886 }
1887 len -= l;
1888 buf += l;
1889 addr += l;
1890 }
1891 return 0;
1892 }
1893
1894 #else
1895
1896 static void invalidate_and_set_dirty(hwaddr addr,
1897 hwaddr length)
1898 {
1899 if (!cpu_physical_memory_is_dirty(addr)) {
1900 /* invalidate code */
1901 tb_invalidate_phys_page_range(addr, addr + length, 0);
1902 /* set dirty bit */
1903 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1904 }
1905 xen_modified_memory(addr, length);
1906 }
1907
1908 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1909 {
1910 if (memory_region_is_ram(mr)) {
1911 return !(is_write && mr->readonly);
1912 }
1913 if (memory_region_is_romd(mr)) {
1914 return !is_write;
1915 }
1916
1917 return false;
1918 }
1919
1920 static inline int memory_access_size(MemoryRegion *mr, int l, hwaddr addr)
1921 {
1922 if (l >= 4 && (((addr & 3) == 0 || mr->ops->impl.unaligned))) {
1923 return 4;
1924 }
1925 if (l >= 2 && (((addr & 1) == 0) || mr->ops->impl.unaligned)) {
1926 return 2;
1927 }
1928 return 1;
1929 }
1930
1931 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1932 int len, bool is_write)
1933 {
1934 hwaddr l;
1935 uint8_t *ptr;
1936 uint64_t val;
1937 hwaddr addr1;
1938 MemoryRegion *mr;
1939 bool error = false;
1940
1941 while (len > 0) {
1942 l = len;
1943 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1944
1945 if (is_write) {
1946 if (!memory_access_is_direct(mr, is_write)) {
1947 l = memory_access_size(mr, l, addr1);
1948 /* XXX: could force cpu_single_env to NULL to avoid
1949 potential bugs */
1950 if (l == 4) {
1951 /* 32 bit write access */
1952 val = ldl_p(buf);
1953 error |= io_mem_write(mr, addr1, val, 4);
1954 } else if (l == 2) {
1955 /* 16 bit write access */
1956 val = lduw_p(buf);
1957 error |= io_mem_write(mr, addr1, val, 2);
1958 } else {
1959 /* 8 bit write access */
1960 val = ldub_p(buf);
1961 error |= io_mem_write(mr, addr1, val, 1);
1962 }
1963 } else {
1964 addr1 += memory_region_get_ram_addr(mr);
1965 /* RAM case */
1966 ptr = qemu_get_ram_ptr(addr1);
1967 memcpy(ptr, buf, l);
1968 invalidate_and_set_dirty(addr1, l);
1969 }
1970 } else {
1971 if (!memory_access_is_direct(mr, is_write)) {
1972 /* I/O case */
1973 l = memory_access_size(mr, l, addr1);
1974 if (l == 4) {
1975 /* 32 bit read access */
1976 error |= io_mem_read(mr, addr1, &val, 4);
1977 stl_p(buf, val);
1978 } else if (l == 2) {
1979 /* 16 bit read access */
1980 error |= io_mem_read(mr, addr1, &val, 2);
1981 stw_p(buf, val);
1982 } else {
1983 /* 8 bit read access */
1984 error |= io_mem_read(mr, addr1, &val, 1);
1985 stb_p(buf, val);
1986 }
1987 } else {
1988 /* RAM case */
1989 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1990 memcpy(buf, ptr, l);
1991 }
1992 }
1993 len -= l;
1994 buf += l;
1995 addr += l;
1996 }
1997
1998 return error;
1999 }
2000
2001 bool address_space_write(AddressSpace *as, hwaddr addr,
2002 const uint8_t *buf, int len)
2003 {
2004 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2005 }
2006
2007 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2008 {
2009 return address_space_rw(as, addr, buf, len, false);
2010 }
2011
2012
2013 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2014 int len, int is_write)
2015 {
2016 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2017 }
2018
2019 /* used for ROM loading : can write in RAM and ROM */
2020 void cpu_physical_memory_write_rom(hwaddr addr,
2021 const uint8_t *buf, int len)
2022 {
2023 hwaddr l;
2024 uint8_t *ptr;
2025 hwaddr addr1;
2026 MemoryRegion *mr;
2027
2028 while (len > 0) {
2029 l = len;
2030 mr = address_space_translate(&address_space_memory,
2031 addr, &addr1, &l, true);
2032
2033 if (!(memory_region_is_ram(mr) ||
2034 memory_region_is_romd(mr))) {
2035 /* do nothing */
2036 } else {
2037 addr1 += memory_region_get_ram_addr(mr);
2038 /* ROM/RAM case */
2039 ptr = qemu_get_ram_ptr(addr1);
2040 memcpy(ptr, buf, l);
2041 invalidate_and_set_dirty(addr1, l);
2042 }
2043 len -= l;
2044 buf += l;
2045 addr += l;
2046 }
2047 }
2048
2049 typedef struct {
2050 void *buffer;
2051 hwaddr addr;
2052 hwaddr len;
2053 } BounceBuffer;
2054
2055 static BounceBuffer bounce;
2056
2057 typedef struct MapClient {
2058 void *opaque;
2059 void (*callback)(void *opaque);
2060 QLIST_ENTRY(MapClient) link;
2061 } MapClient;
2062
2063 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2064 = QLIST_HEAD_INITIALIZER(map_client_list);
2065
2066 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2067 {
2068 MapClient *client = g_malloc(sizeof(*client));
2069
2070 client->opaque = opaque;
2071 client->callback = callback;
2072 QLIST_INSERT_HEAD(&map_client_list, client, link);
2073 return client;
2074 }
2075
2076 static void cpu_unregister_map_client(void *_client)
2077 {
2078 MapClient *client = (MapClient *)_client;
2079
2080 QLIST_REMOVE(client, link);
2081 g_free(client);
2082 }
2083
2084 static void cpu_notify_map_clients(void)
2085 {
2086 MapClient *client;
2087
2088 while (!QLIST_EMPTY(&map_client_list)) {
2089 client = QLIST_FIRST(&map_client_list);
2090 client->callback(client->opaque);
2091 cpu_unregister_map_client(client);
2092 }
2093 }
2094
2095 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2096 {
2097 MemoryRegion *mr;
2098 hwaddr l, xlat;
2099
2100 while (len > 0) {
2101 l = len;
2102 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2103 if (!memory_access_is_direct(mr, is_write)) {
2104 l = memory_access_size(mr, l, addr);
2105 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2106 return false;
2107 }
2108 }
2109
2110 len -= l;
2111 addr += l;
2112 }
2113 return true;
2114 }
2115
2116 /* Map a physical memory region into a host virtual address.
2117 * May map a subset of the requested range, given by and returned in *plen.
2118 * May return NULL if resources needed to perform the mapping are exhausted.
2119 * Use only for reads OR writes - not for read-modify-write operations.
2120 * Use cpu_register_map_client() to know when retrying the map operation is
2121 * likely to succeed.
2122 */
2123 void *address_space_map(AddressSpace *as,
2124 hwaddr addr,
2125 hwaddr *plen,
2126 bool is_write)
2127 {
2128 hwaddr len = *plen;
2129 hwaddr todo = 0;
2130 hwaddr l, xlat;
2131 MemoryRegion *mr;
2132 ram_addr_t raddr = RAM_ADDR_MAX;
2133 ram_addr_t rlen;
2134 void *ret;
2135
2136 while (len > 0) {
2137 l = len;
2138 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2139
2140 if (!memory_access_is_direct(mr, is_write)) {
2141 if (todo || bounce.buffer) {
2142 break;
2143 }
2144 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2145 bounce.addr = addr;
2146 bounce.len = l;
2147 if (!is_write) {
2148 address_space_read(as, addr, bounce.buffer, l);
2149 }
2150
2151 *plen = l;
2152 return bounce.buffer;
2153 }
2154 if (!todo) {
2155 raddr = memory_region_get_ram_addr(mr) + xlat;
2156 } else {
2157 if (memory_region_get_ram_addr(mr) + xlat != raddr + todo) {
2158 break;
2159 }
2160 }
2161
2162 len -= l;
2163 addr += l;
2164 todo += l;
2165 }
2166 rlen = todo;
2167 ret = qemu_ram_ptr_length(raddr, &rlen);
2168 *plen = rlen;
2169 return ret;
2170 }
2171
2172 /* Unmaps a memory region previously mapped by address_space_map().
2173 * Will also mark the memory as dirty if is_write == 1. access_len gives
2174 * the amount of memory that was actually read or written by the caller.
2175 */
2176 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2177 int is_write, hwaddr access_len)
2178 {
2179 if (buffer != bounce.buffer) {
2180 if (is_write) {
2181 ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
2182 while (access_len) {
2183 unsigned l;
2184 l = TARGET_PAGE_SIZE;
2185 if (l > access_len)
2186 l = access_len;
2187 invalidate_and_set_dirty(addr1, l);
2188 addr1 += l;
2189 access_len -= l;
2190 }
2191 }
2192 if (xen_enabled()) {
2193 xen_invalidate_map_cache_entry(buffer);
2194 }
2195 return;
2196 }
2197 if (is_write) {
2198 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2199 }
2200 qemu_vfree(bounce.buffer);
2201 bounce.buffer = NULL;
2202 cpu_notify_map_clients();
2203 }
2204
2205 void *cpu_physical_memory_map(hwaddr addr,
2206 hwaddr *plen,
2207 int is_write)
2208 {
2209 return address_space_map(&address_space_memory, addr, plen, is_write);
2210 }
2211
2212 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2213 int is_write, hwaddr access_len)
2214 {
2215 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2216 }
2217
2218 /* warning: addr must be aligned */
2219 static inline uint32_t ldl_phys_internal(hwaddr addr,
2220 enum device_endian endian)
2221 {
2222 uint8_t *ptr;
2223 uint64_t val;
2224 MemoryRegion *mr;
2225 hwaddr l = 4;
2226 hwaddr addr1;
2227
2228 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2229 false);
2230 if (l < 4 || !memory_access_is_direct(mr, false)) {
2231 /* I/O case */
2232 io_mem_read(mr, addr1, &val, 4);
2233 #if defined(TARGET_WORDS_BIGENDIAN)
2234 if (endian == DEVICE_LITTLE_ENDIAN) {
2235 val = bswap32(val);
2236 }
2237 #else
2238 if (endian == DEVICE_BIG_ENDIAN) {
2239 val = bswap32(val);
2240 }
2241 #endif
2242 } else {
2243 /* RAM case */
2244 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2245 & TARGET_PAGE_MASK)
2246 + addr1);
2247 switch (endian) {
2248 case DEVICE_LITTLE_ENDIAN:
2249 val = ldl_le_p(ptr);
2250 break;
2251 case DEVICE_BIG_ENDIAN:
2252 val = ldl_be_p(ptr);
2253 break;
2254 default:
2255 val = ldl_p(ptr);
2256 break;
2257 }
2258 }
2259 return val;
2260 }
2261
2262 uint32_t ldl_phys(hwaddr addr)
2263 {
2264 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2265 }
2266
2267 uint32_t ldl_le_phys(hwaddr addr)
2268 {
2269 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2270 }
2271
2272 uint32_t ldl_be_phys(hwaddr addr)
2273 {
2274 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2275 }
2276
2277 /* warning: addr must be aligned */
2278 static inline uint64_t ldq_phys_internal(hwaddr addr,
2279 enum device_endian endian)
2280 {
2281 uint8_t *ptr;
2282 uint64_t val;
2283 MemoryRegion *mr;
2284 hwaddr l = 8;
2285 hwaddr addr1;
2286
2287 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2288 false);
2289 if (l < 8 || !memory_access_is_direct(mr, false)) {
2290 /* I/O case */
2291 io_mem_read(mr, addr1, &val, 8);
2292 #if defined(TARGET_WORDS_BIGENDIAN)
2293 if (endian == DEVICE_LITTLE_ENDIAN) {
2294 val = bswap64(val);
2295 }
2296 #else
2297 if (endian == DEVICE_BIG_ENDIAN) {
2298 val = bswap64(val);
2299 }
2300 #endif
2301 } else {
2302 /* RAM case */
2303 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2304 & TARGET_PAGE_MASK)
2305 + addr1);
2306 switch (endian) {
2307 case DEVICE_LITTLE_ENDIAN:
2308 val = ldq_le_p(ptr);
2309 break;
2310 case DEVICE_BIG_ENDIAN:
2311 val = ldq_be_p(ptr);
2312 break;
2313 default:
2314 val = ldq_p(ptr);
2315 break;
2316 }
2317 }
2318 return val;
2319 }
2320
2321 uint64_t ldq_phys(hwaddr addr)
2322 {
2323 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2324 }
2325
2326 uint64_t ldq_le_phys(hwaddr addr)
2327 {
2328 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2329 }
2330
2331 uint64_t ldq_be_phys(hwaddr addr)
2332 {
2333 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2334 }
2335
2336 /* XXX: optimize */
2337 uint32_t ldub_phys(hwaddr addr)
2338 {
2339 uint8_t val;
2340 cpu_physical_memory_read(addr, &val, 1);
2341 return val;
2342 }
2343
2344 /* warning: addr must be aligned */
2345 static inline uint32_t lduw_phys_internal(hwaddr addr,
2346 enum device_endian endian)
2347 {
2348 uint8_t *ptr;
2349 uint64_t val;
2350 MemoryRegion *mr;
2351 hwaddr l = 2;
2352 hwaddr addr1;
2353
2354 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2355 false);
2356 if (l < 2 || !memory_access_is_direct(mr, false)) {
2357 /* I/O case */
2358 io_mem_read(mr, addr1, &val, 2);
2359 #if defined(TARGET_WORDS_BIGENDIAN)
2360 if (endian == DEVICE_LITTLE_ENDIAN) {
2361 val = bswap16(val);
2362 }
2363 #else
2364 if (endian == DEVICE_BIG_ENDIAN) {
2365 val = bswap16(val);
2366 }
2367 #endif
2368 } else {
2369 /* RAM case */
2370 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2371 & TARGET_PAGE_MASK)
2372 + addr1);
2373 switch (endian) {
2374 case DEVICE_LITTLE_ENDIAN:
2375 val = lduw_le_p(ptr);
2376 break;
2377 case DEVICE_BIG_ENDIAN:
2378 val = lduw_be_p(ptr);
2379 break;
2380 default:
2381 val = lduw_p(ptr);
2382 break;
2383 }
2384 }
2385 return val;
2386 }
2387
2388 uint32_t lduw_phys(hwaddr addr)
2389 {
2390 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2391 }
2392
2393 uint32_t lduw_le_phys(hwaddr addr)
2394 {
2395 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2396 }
2397
2398 uint32_t lduw_be_phys(hwaddr addr)
2399 {
2400 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2401 }
2402
2403 /* warning: addr must be aligned. The ram page is not masked as dirty
2404 and the code inside is not invalidated. It is useful if the dirty
2405 bits are used to track modified PTEs */
2406 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2407 {
2408 uint8_t *ptr;
2409 MemoryRegion *mr;
2410 hwaddr l = 4;
2411 hwaddr addr1;
2412
2413 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2414 true);
2415 if (l < 4 || !memory_access_is_direct(mr, true)) {
2416 io_mem_write(mr, addr1, val, 4);
2417 } else {
2418 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2419 ptr = qemu_get_ram_ptr(addr1);
2420 stl_p(ptr, val);
2421
2422 if (unlikely(in_migration)) {
2423 if (!cpu_physical_memory_is_dirty(addr1)) {
2424 /* invalidate code */
2425 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2426 /* set dirty bit */
2427 cpu_physical_memory_set_dirty_flags(
2428 addr1, (0xff & ~CODE_DIRTY_FLAG));
2429 }
2430 }
2431 }
2432 }
2433
2434 /* warning: addr must be aligned */
2435 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2436 enum device_endian endian)
2437 {
2438 uint8_t *ptr;
2439 MemoryRegion *mr;
2440 hwaddr l = 4;
2441 hwaddr addr1;
2442
2443 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2444 true);
2445 if (l < 4 || !memory_access_is_direct(mr, true)) {
2446 #if defined(TARGET_WORDS_BIGENDIAN)
2447 if (endian == DEVICE_LITTLE_ENDIAN) {
2448 val = bswap32(val);
2449 }
2450 #else
2451 if (endian == DEVICE_BIG_ENDIAN) {
2452 val = bswap32(val);
2453 }
2454 #endif
2455 io_mem_write(mr, addr1, val, 4);
2456 } else {
2457 /* RAM case */
2458 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2459 ptr = qemu_get_ram_ptr(addr1);
2460 switch (endian) {
2461 case DEVICE_LITTLE_ENDIAN:
2462 stl_le_p(ptr, val);
2463 break;
2464 case DEVICE_BIG_ENDIAN:
2465 stl_be_p(ptr, val);
2466 break;
2467 default:
2468 stl_p(ptr, val);
2469 break;
2470 }
2471 invalidate_and_set_dirty(addr1, 4);
2472 }
2473 }
2474
2475 void stl_phys(hwaddr addr, uint32_t val)
2476 {
2477 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2478 }
2479
2480 void stl_le_phys(hwaddr addr, uint32_t val)
2481 {
2482 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2483 }
2484
2485 void stl_be_phys(hwaddr addr, uint32_t val)
2486 {
2487 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2488 }
2489
2490 /* XXX: optimize */
2491 void stb_phys(hwaddr addr, uint32_t val)
2492 {
2493 uint8_t v = val;
2494 cpu_physical_memory_write(addr, &v, 1);
2495 }
2496
2497 /* warning: addr must be aligned */
2498 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2499 enum device_endian endian)
2500 {
2501 uint8_t *ptr;
2502 MemoryRegion *mr;
2503 hwaddr l = 2;
2504 hwaddr addr1;
2505
2506 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2507 true);
2508 if (l < 2 || !memory_access_is_direct(mr, true)) {
2509 #if defined(TARGET_WORDS_BIGENDIAN)
2510 if (endian == DEVICE_LITTLE_ENDIAN) {
2511 val = bswap16(val);
2512 }
2513 #else
2514 if (endian == DEVICE_BIG_ENDIAN) {
2515 val = bswap16(val);
2516 }
2517 #endif
2518 io_mem_write(mr, addr1, val, 2);
2519 } else {
2520 /* RAM case */
2521 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2522 ptr = qemu_get_ram_ptr(addr1);
2523 switch (endian) {
2524 case DEVICE_LITTLE_ENDIAN:
2525 stw_le_p(ptr, val);
2526 break;
2527 case DEVICE_BIG_ENDIAN:
2528 stw_be_p(ptr, val);
2529 break;
2530 default:
2531 stw_p(ptr, val);
2532 break;
2533 }
2534 invalidate_and_set_dirty(addr1, 2);
2535 }
2536 }
2537
2538 void stw_phys(hwaddr addr, uint32_t val)
2539 {
2540 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2541 }
2542
2543 void stw_le_phys(hwaddr addr, uint32_t val)
2544 {
2545 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2546 }
2547
2548 void stw_be_phys(hwaddr addr, uint32_t val)
2549 {
2550 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2551 }
2552
2553 /* XXX: optimize */
2554 void stq_phys(hwaddr addr, uint64_t val)
2555 {
2556 val = tswap64(val);
2557 cpu_physical_memory_write(addr, &val, 8);
2558 }
2559
2560 void stq_le_phys(hwaddr addr, uint64_t val)
2561 {
2562 val = cpu_to_le64(val);
2563 cpu_physical_memory_write(addr, &val, 8);
2564 }
2565
2566 void stq_be_phys(hwaddr addr, uint64_t val)
2567 {
2568 val = cpu_to_be64(val);
2569 cpu_physical_memory_write(addr, &val, 8);
2570 }
2571
2572 /* virtual memory access for debug (includes writing to ROM) */
2573 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2574 uint8_t *buf, int len, int is_write)
2575 {
2576 int l;
2577 hwaddr phys_addr;
2578 target_ulong page;
2579
2580 while (len > 0) {
2581 page = addr & TARGET_PAGE_MASK;
2582 phys_addr = cpu_get_phys_page_debug(env, page);
2583 /* if no physical page mapped, return an error */
2584 if (phys_addr == -1)
2585 return -1;
2586 l = (page + TARGET_PAGE_SIZE) - addr;
2587 if (l > len)
2588 l = len;
2589 phys_addr += (addr & ~TARGET_PAGE_MASK);
2590 if (is_write)
2591 cpu_physical_memory_write_rom(phys_addr, buf, l);
2592 else
2593 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2594 len -= l;
2595 buf += l;
2596 addr += l;
2597 }
2598 return 0;
2599 }
2600 #endif
2601
2602 #if !defined(CONFIG_USER_ONLY)
2603
2604 /*
2605 * A helper function for the _utterly broken_ virtio device model to find out if
2606 * it's running on a big endian machine. Don't do this at home kids!
2607 */
2608 bool virtio_is_big_endian(void);
2609 bool virtio_is_big_endian(void)
2610 {
2611 #if defined(TARGET_WORDS_BIGENDIAN)
2612 return true;
2613 #else
2614 return false;
2615 #endif
2616 }
2617
2618 #endif
2619
2620 #ifndef CONFIG_USER_ONLY
2621 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2622 {
2623 MemoryRegion*mr;
2624 hwaddr l = 1;
2625
2626 mr = address_space_translate(&address_space_memory,
2627 phys_addr, &phys_addr, &l, false);
2628
2629 return !(memory_region_is_ram(mr) ||
2630 memory_region_is_romd(mr));
2631 }
2632 #endif
QEmu