?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
exec: Fix broken build for MinGW (regression)
[qemu/ar7.git] / exec.c
blobefeca147e77873e3f4c227332a28b11516247637
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
51
52 #include "exec/memory-internal.h"
53
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 typedef PhysPageEntry Node[L2_SIZE];
92
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101 };
102
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
110
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
115
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
124
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
127
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
133
134 static MemoryRegion io_mem_watch;
135 #endif
136
137 #if !defined(CONFIG_USER_ONLY)
138
139 static void phys_map_node_reserve(unsigned nodes)
140 {
141 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
142 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
143 16);
144 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
145 next_map.nodes_nb + nodes);
146 next_map.nodes = g_renew(Node, next_map.nodes,
147 next_map.nodes_nb_alloc);
148 }
149 }
150
151 static uint16_t phys_map_node_alloc(void)
152 {
153 unsigned i;
154 uint16_t ret;
155
156 ret = next_map.nodes_nb++;
157 assert(ret != PHYS_MAP_NODE_NIL);
158 assert(ret != next_map.nodes_nb_alloc);
159 for (i = 0; i < L2_SIZE; ++i) {
160 next_map.nodes[ret][i].is_leaf = 0;
161 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
162 }
163 return ret;
164 }
165
166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
167 hwaddr *nb, uint16_t leaf,
168 int level)
169 {
170 PhysPageEntry *p;
171 int i;
172 hwaddr step = (hwaddr)1 << (level * L2_BITS);
173
174 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
175 lp->ptr = phys_map_node_alloc();
176 p = next_map.nodes[lp->ptr];
177 if (level == 0) {
178 for (i = 0; i < L2_SIZE; i++) {
179 p[i].is_leaf = 1;
180 p[i].ptr = PHYS_SECTION_UNASSIGNED;
181 }
182 }
183 } else {
184 p = next_map.nodes[lp->ptr];
185 }
186 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
187
188 while (*nb && lp < &p[L2_SIZE]) {
189 if ((*index & (step - 1)) == 0 && *nb >= step) {
190 lp->is_leaf = true;
191 lp->ptr = leaf;
192 *index += step;
193 *nb -= step;
194 } else {
195 phys_page_set_level(lp, index, nb, leaf, level - 1);
196 }
197 ++lp;
198 }
199 }
200
201 static void phys_page_set(AddressSpaceDispatch *d,
202 hwaddr index, hwaddr nb,
203 uint16_t leaf)
204 {
205 /* Wildly overreserve - it doesn't matter much. */
206 phys_map_node_reserve(3 * P_L2_LEVELS);
207
208 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
209 }
210
211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
212 Node *nodes, MemoryRegionSection *sections)
213 {
214 PhysPageEntry *p;
215 int i;
216
217 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
218 if (lp.ptr == PHYS_MAP_NODE_NIL) {
219 return &sections[PHYS_SECTION_UNASSIGNED];
220 }
221 p = nodes[lp.ptr];
222 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
223 }
224 return &sections[lp.ptr];
225 }
226
227 bool memory_region_is_unassigned(MemoryRegion *mr)
228 {
229 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
230 && mr != &io_mem_watch;
231 }
232
233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
234 hwaddr addr,
235 bool resolve_subpage)
236 {
237 MemoryRegionSection *section;
238 subpage_t *subpage;
239
240 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
241 d->nodes, d->sections);
242 if (resolve_subpage && section->mr->subpage) {
243 subpage = container_of(section->mr, subpage_t, iomem);
244 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
245 }
246 return section;
247 }
248
249 static MemoryRegionSection *
250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
251 hwaddr *plen, bool resolve_subpage)
252 {
253 MemoryRegionSection *section;
254 Int128 diff;
255
256 section = address_space_lookup_region(d, addr, resolve_subpage);
257 /* Compute offset within MemoryRegionSection */
258 addr -= section->offset_within_address_space;
259
260 /* Compute offset within MemoryRegion */
261 *xlat = addr + section->offset_within_region;
262
263 diff = int128_sub(section->mr->size, int128_make64(addr));
264 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
265 return section;
266 }
267
268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
269 hwaddr *xlat, hwaddr *plen,
270 bool is_write)
271 {
272 IOMMUTLBEntry iotlb;
273 MemoryRegionSection *section;
274 MemoryRegion *mr;
275 hwaddr len = *plen;
276
277 for (;;) {
278 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
279 mr = section->mr;
280
281 if (!mr->iommu_ops) {
282 break;
283 }
284
285 iotlb = mr->iommu_ops->translate(mr, addr);
286 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
287 | (addr & iotlb.addr_mask));
288 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
289 if (!(iotlb.perm & (1 << is_write))) {
290 mr = &io_mem_unassigned;
291 break;
292 }
293
294 as = iotlb.target_as;
295 }
296
297 *plen = len;
298 *xlat = addr;
299 return mr;
300 }
301
302 MemoryRegionSection *
303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
304 hwaddr *plen)
305 {
306 MemoryRegionSection *section;
307 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
308
309 assert(!section->mr->iommu_ops);
310 return section;
311 }
312 #endif
313
314 void cpu_exec_init_all(void)
315 {
316 #if !defined(CONFIG_USER_ONLY)
317 qemu_mutex_init(&ram_list.mutex);
318 memory_map_init();
319 io_mem_init();
320 #endif
321 }
322
323 #if !defined(CONFIG_USER_ONLY)
324
325 static int cpu_common_post_load(void *opaque, int version_id)
326 {
327 CPUState *cpu = opaque;
328
329 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
330 version_id is increased. */
331 cpu->interrupt_request &= ~0x01;
332 tlb_flush(cpu->env_ptr, 1);
333
334 return 0;
335 }
336
337 const VMStateDescription vmstate_cpu_common = {
338 .name = "cpu_common",
339 .version_id = 1,
340 .minimum_version_id = 1,
341 .minimum_version_id_old = 1,
342 .post_load = cpu_common_post_load,
343 .fields = (VMStateField []) {
344 VMSTATE_UINT32(halted, CPUState),
345 VMSTATE_UINT32(interrupt_request, CPUState),
346 VMSTATE_END_OF_LIST()
347 }
348 };
349
350 #endif
351
352 CPUState *qemu_get_cpu(int index)
353 {
354 CPUState *cpu;
355
356 CPU_FOREACH(cpu) {
357 if (cpu->cpu_index == index) {
358 return cpu;
359 }
360 }
361
362 return NULL;
363 }
364
365 void cpu_exec_init(CPUArchState *env)
366 {
367 CPUState *cpu = ENV_GET_CPU(env);
368 CPUClass *cc = CPU_GET_CLASS(cpu);
369 CPUState *some_cpu;
370 int cpu_index;
371
372 #if defined(CONFIG_USER_ONLY)
373 cpu_list_lock();
374 #endif
375 cpu_index = 0;
376 CPU_FOREACH(some_cpu) {
377 cpu_index++;
378 }
379 cpu->cpu_index = cpu_index;
380 cpu->numa_node = 0;
381 QTAILQ_INIT(&env->breakpoints);
382 QTAILQ_INIT(&env->watchpoints);
383 #ifndef CONFIG_USER_ONLY
384 cpu->thread_id = qemu_get_thread_id();
385 #endif
386 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
387 #if defined(CONFIG_USER_ONLY)
388 cpu_list_unlock();
389 #endif
390 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
391 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
392 }
393 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
394 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
395 cpu_save, cpu_load, env);
396 assert(cc->vmsd == NULL);
397 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
398 #endif
399 if (cc->vmsd != NULL) {
400 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
401 }
402 }
403
404 #if defined(TARGET_HAS_ICE)
405 #if defined(CONFIG_USER_ONLY)
406 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
407 {
408 tb_invalidate_phys_page_range(pc, pc + 1, 0);
409 }
410 #else
411 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
412 {
413 tb_invalidate_phys_addr(cpu_get_phys_page_debug(cpu, pc) |
414 (pc & ~TARGET_PAGE_MASK));
415 }
416 #endif
417 #endif /* TARGET_HAS_ICE */
418
419 #if defined(CONFIG_USER_ONLY)
420 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
421
422 {
423 }
424
425 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
426 int flags, CPUWatchpoint **watchpoint)
427 {
428 return -ENOSYS;
429 }
430 #else
431 /* Add a watchpoint. */
432 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
433 int flags, CPUWatchpoint **watchpoint)
434 {
435 target_ulong len_mask = ~(len - 1);
436 CPUWatchpoint *wp;
437
438 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
439 if ((len & (len - 1)) || (addr & ~len_mask) ||
440 len == 0 || len > TARGET_PAGE_SIZE) {
441 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
442 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
443 return -EINVAL;
444 }
445 wp = g_malloc(sizeof(*wp));
446
447 wp->vaddr = addr;
448 wp->len_mask = len_mask;
449 wp->flags = flags;
450
451 /* keep all GDB-injected watchpoints in front */
452 if (flags & BP_GDB)
453 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
454 else
455 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
456
457 tlb_flush_page(env, addr);
458
459 if (watchpoint)
460 *watchpoint = wp;
461 return 0;
462 }
463
464 /* Remove a specific watchpoint. */
465 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
466 int flags)
467 {
468 target_ulong len_mask = ~(len - 1);
469 CPUWatchpoint *wp;
470
471 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
472 if (addr == wp->vaddr && len_mask == wp->len_mask
473 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
474 cpu_watchpoint_remove_by_ref(env, wp);
475 return 0;
476 }
477 }
478 return -ENOENT;
479 }
480
481 /* Remove a specific watchpoint by reference. */
482 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
483 {
484 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
485
486 tlb_flush_page(env, watchpoint->vaddr);
487
488 g_free(watchpoint);
489 }
490
491 /* Remove all matching watchpoints. */
492 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
493 {
494 CPUWatchpoint *wp, *next;
495
496 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
497 if (wp->flags & mask)
498 cpu_watchpoint_remove_by_ref(env, wp);
499 }
500 }
501 #endif
502
503 /* Add a breakpoint. */
504 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
505 CPUBreakpoint **breakpoint)
506 {
507 #if defined(TARGET_HAS_ICE)
508 CPUBreakpoint *bp;
509
510 bp = g_malloc(sizeof(*bp));
511
512 bp->pc = pc;
513 bp->flags = flags;
514
515 /* keep all GDB-injected breakpoints in front */
516 if (flags & BP_GDB) {
517 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
518 } else {
519 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
520 }
521
522 breakpoint_invalidate(ENV_GET_CPU(env), pc);
523
524 if (breakpoint) {
525 *breakpoint = bp;
526 }
527 return 0;
528 #else
529 return -ENOSYS;
530 #endif
531 }
532
533 /* Remove a specific breakpoint. */
534 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
535 {
536 #if defined(TARGET_HAS_ICE)
537 CPUBreakpoint *bp;
538
539 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
540 if (bp->pc == pc && bp->flags == flags) {
541 cpu_breakpoint_remove_by_ref(env, bp);
542 return 0;
543 }
544 }
545 return -ENOENT;
546 #else
547 return -ENOSYS;
548 #endif
549 }
550
551 /* Remove a specific breakpoint by reference. */
552 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
553 {
554 #if defined(TARGET_HAS_ICE)
555 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
556
557 breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
558
559 g_free(breakpoint);
560 #endif
561 }
562
563 /* Remove all matching breakpoints. */
564 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
565 {
566 #if defined(TARGET_HAS_ICE)
567 CPUBreakpoint *bp, *next;
568
569 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
570 if (bp->flags & mask)
571 cpu_breakpoint_remove_by_ref(env, bp);
572 }
573 #endif
574 }
575
576 /* enable or disable single step mode. EXCP_DEBUG is returned by the
577 CPU loop after each instruction */
578 void cpu_single_step(CPUState *cpu, int enabled)
579 {
580 #if defined(TARGET_HAS_ICE)
581 if (cpu->singlestep_enabled != enabled) {
582 cpu->singlestep_enabled = enabled;
583 if (kvm_enabled()) {
584 kvm_update_guest_debug(cpu, 0);
585 } else {
586 /* must flush all the translated code to avoid inconsistencies */
587 /* XXX: only flush what is necessary */
588 CPUArchState *env = cpu->env_ptr;
589 tb_flush(env);
590 }
591 }
592 #endif
593 }
594
595 void cpu_abort(CPUArchState *env, const char *fmt, ...)
596 {
597 CPUState *cpu = ENV_GET_CPU(env);
598 va_list ap;
599 va_list ap2;
600
601 va_start(ap, fmt);
602 va_copy(ap2, ap);
603 fprintf(stderr, "qemu: fatal: ");
604 vfprintf(stderr, fmt, ap);
605 fprintf(stderr, "\n");
606 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
607 if (qemu_log_enabled()) {
608 qemu_log("qemu: fatal: ");
609 qemu_log_vprintf(fmt, ap2);
610 qemu_log("\n");
611 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
612 qemu_log_flush();
613 qemu_log_close();
614 }
615 va_end(ap2);
616 va_end(ap);
617 #if defined(CONFIG_USER_ONLY)
618 {
619 struct sigaction act;
620 sigfillset(&act.sa_mask);
621 act.sa_handler = SIG_DFL;
622 sigaction(SIGABRT, &act, NULL);
623 }
624 #endif
625 abort();
626 }
627
628 CPUArchState *cpu_copy(CPUArchState *env)
629 {
630 CPUArchState *new_env = cpu_init(env->cpu_model_str);
631 #if defined(TARGET_HAS_ICE)
632 CPUBreakpoint *bp;
633 CPUWatchpoint *wp;
634 #endif
635
636 /* Reset non arch specific state */
637 cpu_reset(ENV_GET_CPU(new_env));
638
639 /* Copy arch specific state into the new CPU */
640 memcpy(new_env, env, sizeof(CPUArchState));
641
642 /* Clone all break/watchpoints.
643 Note: Once we support ptrace with hw-debug register access, make sure
644 BP_CPU break/watchpoints are handled correctly on clone. */
645 QTAILQ_INIT(&env->breakpoints);
646 QTAILQ_INIT(&env->watchpoints);
647 #if defined(TARGET_HAS_ICE)
648 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
649 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
650 }
651 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
652 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
653 wp->flags, NULL);
654 }
655 #endif
656
657 return new_env;
658 }
659
660 #if !defined(CONFIG_USER_ONLY)
661 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
662 uintptr_t length)
663 {
664 uintptr_t start1;
665
666 /* we modify the TLB cache so that the dirty bit will be set again
667 when accessing the range */
668 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
669 /* Check that we don't span multiple blocks - this breaks the
670 address comparisons below. */
671 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
672 != (end - 1) - start) {
673 abort();
674 }
675 cpu_tlb_reset_dirty_all(start1, length);
676
677 }
678
679 /* Note: start and end must be within the same ram block. */
680 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
681 int dirty_flags)
682 {
683 uintptr_t length;
684
685 start &= TARGET_PAGE_MASK;
686 end = TARGET_PAGE_ALIGN(end);
687
688 length = end - start;
689 if (length == 0)
690 return;
691 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
692
693 if (tcg_enabled()) {
694 tlb_reset_dirty_range_all(start, end, length);
695 }
696 }
697
698 static int cpu_physical_memory_set_dirty_tracking(int enable)
699 {
700 int ret = 0;
701 in_migration = enable;
702 return ret;
703 }
704
705 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
706 MemoryRegionSection *section,
707 target_ulong vaddr,
708 hwaddr paddr, hwaddr xlat,
709 int prot,
710 target_ulong *address)
711 {
712 hwaddr iotlb;
713 CPUWatchpoint *wp;
714
715 if (memory_region_is_ram(section->mr)) {
716 /* Normal RAM. */
717 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
718 + xlat;
719 if (!section->readonly) {
720 iotlb |= PHYS_SECTION_NOTDIRTY;
721 } else {
722 iotlb |= PHYS_SECTION_ROM;
723 }
724 } else {
725 iotlb = section - address_space_memory.dispatch->sections;
726 iotlb += xlat;
727 }
728
729 /* Make accesses to pages with watchpoints go via the
730 watchpoint trap routines. */
731 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
732 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
733 /* Avoid trapping reads of pages with a write breakpoint. */
734 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
735 iotlb = PHYS_SECTION_WATCH + paddr;
736 *address |= TLB_MMIO;
737 break;
738 }
739 }
740 }
741
742 return iotlb;
743 }
744 #endif /* defined(CONFIG_USER_ONLY) */
745
746 #if !defined(CONFIG_USER_ONLY)
747
748 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
749 uint16_t section);
750 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
751
752 static void *(*phys_mem_alloc)(ram_addr_t size) = qemu_anon_ram_alloc;
753
754 /*
755 * Set a custom physical guest memory alloator.
756 * Accelerators with unusual needs may need this. Hopefully, we can
757 * get rid of it eventually.
758 */
759 void phys_mem_set_alloc(void *(*alloc)(ram_addr_t))
760 {
761 phys_mem_alloc = alloc;
762 }
763
764 static uint16_t phys_section_add(MemoryRegionSection *section)
765 {
766 /* The physical section number is ORed with a page-aligned
767 * pointer to produce the iotlb entries. Thus it should
768 * never overflow into the page-aligned value.
769 */
770 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
771
772 if (next_map.sections_nb == next_map.sections_nb_alloc) {
773 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
774 16);
775 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
776 next_map.sections_nb_alloc);
777 }
778 next_map.sections[next_map.sections_nb] = *section;
779 memory_region_ref(section->mr);
780 return next_map.sections_nb++;
781 }
782
783 static void phys_section_destroy(MemoryRegion *mr)
784 {
785 memory_region_unref(mr);
786
787 if (mr->subpage) {
788 subpage_t *subpage = container_of(mr, subpage_t, iomem);
789 memory_region_destroy(&subpage->iomem);
790 g_free(subpage);
791 }
792 }
793
794 static void phys_sections_free(PhysPageMap *map)
795 {
796 while (map->sections_nb > 0) {
797 MemoryRegionSection *section = &map->sections[--map->sections_nb];
798 phys_section_destroy(section->mr);
799 }
800 g_free(map->sections);
801 g_free(map->nodes);
802 g_free(map);
803 }
804
805 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
806 {
807 subpage_t *subpage;
808 hwaddr base = section->offset_within_address_space
809 & TARGET_PAGE_MASK;
810 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
811 next_map.nodes, next_map.sections);
812 MemoryRegionSection subsection = {
813 .offset_within_address_space = base,
814 .size = int128_make64(TARGET_PAGE_SIZE),
815 };
816 hwaddr start, end;
817
818 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
819
820 if (!(existing->mr->subpage)) {
821 subpage = subpage_init(d->as, base);
822 subsection.mr = &subpage->iomem;
823 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
824 phys_section_add(&subsection));
825 } else {
826 subpage = container_of(existing->mr, subpage_t, iomem);
827 }
828 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
829 end = start + int128_get64(section->size) - 1;
830 subpage_register(subpage, start, end, phys_section_add(section));
831 }
832
833
834 static void register_multipage(AddressSpaceDispatch *d,
835 MemoryRegionSection *section)
836 {
837 hwaddr start_addr = section->offset_within_address_space;
838 uint16_t section_index = phys_section_add(section);
839 uint64_t num_pages = int128_get64(int128_rshift(section->size,
840 TARGET_PAGE_BITS));
841
842 assert(num_pages);
843 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
844 }
845
846 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
847 {
848 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
849 AddressSpaceDispatch *d = as->next_dispatch;
850 MemoryRegionSection now = *section, remain = *section;
851 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
852
853 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
854 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
855 - now.offset_within_address_space;
856
857 now.size = int128_min(int128_make64(left), now.size);
858 register_subpage(d, &now);
859 } else {
860 now.size = int128_zero();
861 }
862 while (int128_ne(remain.size, now.size)) {
863 remain.size = int128_sub(remain.size, now.size);
864 remain.offset_within_address_space += int128_get64(now.size);
865 remain.offset_within_region += int128_get64(now.size);
866 now = remain;
867 if (int128_lt(remain.size, page_size)) {
868 register_subpage(d, &now);
869 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
870 now.size = page_size;
871 register_subpage(d, &now);
872 } else {
873 now.size = int128_and(now.size, int128_neg(page_size));
874 register_multipage(d, &now);
875 }
876 }
877 }
878
879 void qemu_flush_coalesced_mmio_buffer(void)
880 {
881 if (kvm_enabled())
882 kvm_flush_coalesced_mmio_buffer();
883 }
884
885 void qemu_mutex_lock_ramlist(void)
886 {
887 qemu_mutex_lock(&ram_list.mutex);
888 }
889
890 void qemu_mutex_unlock_ramlist(void)
891 {
892 qemu_mutex_unlock(&ram_list.mutex);
893 }
894
895 #ifdef __linux__
896
897 #include <sys/vfs.h>
898
899 #define HUGETLBFS_MAGIC 0x958458f6
900
901 static long gethugepagesize(const char *path)
902 {
903 struct statfs fs;
904 int ret;
905
906 do {
907 ret = statfs(path, &fs);
908 } while (ret != 0 && errno == EINTR);
909
910 if (ret != 0) {
911 perror(path);
912 return 0;
913 }
914
915 if (fs.f_type != HUGETLBFS_MAGIC)
916 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
917
918 return fs.f_bsize;
919 }
920
921 static void *file_ram_alloc(RAMBlock *block,
922 ram_addr_t memory,
923 const char *path)
924 {
925 char *filename;
926 char *sanitized_name;
927 char *c;
928 void *area;
929 int fd;
930 #ifdef MAP_POPULATE
931 int flags;
932 #endif
933 unsigned long hpagesize;
934
935 hpagesize = gethugepagesize(path);
936 if (!hpagesize) {
937 return NULL;
938 }
939
940 if (memory < hpagesize) {
941 return NULL;
942 }
943
944 if (kvm_enabled() && !kvm_has_sync_mmu()) {
945 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
946 return NULL;
947 }
948
949 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
950 sanitized_name = g_strdup(block->mr->name);
951 for (c = sanitized_name; *c != '\0'; c++) {
952 if (*c == '/')
953 *c = '_';
954 }
955
956 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
957 sanitized_name);
958 g_free(sanitized_name);
959
960 fd = mkstemp(filename);
961 if (fd < 0) {
962 perror("unable to create backing store for hugepages");
963 g_free(filename);
964 return NULL;
965 }
966 unlink(filename);
967 g_free(filename);
968
969 memory = (memory+hpagesize-1) & ~(hpagesize-1);
970
971 /*
972 * ftruncate is not supported by hugetlbfs in older
973 * hosts, so don't bother bailing out on errors.
974 * If anything goes wrong with it under other filesystems,
975 * mmap will fail.
976 */
977 if (ftruncate(fd, memory))
978 perror("ftruncate");
979
980 #ifdef MAP_POPULATE
981 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
982 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
983 * to sidestep this quirk.
984 */
985 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
986 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
987 #else
988 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
989 #endif
990 if (area == MAP_FAILED) {
991 perror("file_ram_alloc: can't mmap RAM pages");
992 close(fd);
993 return (NULL);
994 }
995 block->fd = fd;
996 return area;
997 }
998 #else
999 static void *file_ram_alloc(RAMBlock *block,
1000 ram_addr_t memory,
1001 const char *path)
1002 {
1003 fprintf(stderr, "-mem-path not supported on this host\n");
1004 exit(1);
1005 }
1006 #endif
1007
1008 static ram_addr_t find_ram_offset(ram_addr_t size)
1009 {
1010 RAMBlock *block, *next_block;
1011 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1012
1013 assert(size != 0); /* it would hand out same offset multiple times */
1014
1015 if (QTAILQ_EMPTY(&ram_list.blocks))
1016 return 0;
1017
1018 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1019 ram_addr_t end, next = RAM_ADDR_MAX;
1020
1021 end = block->offset + block->length;
1022
1023 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1024 if (next_block->offset >= end) {
1025 next = MIN(next, next_block->offset);
1026 }
1027 }
1028 if (next - end >= size && next - end < mingap) {
1029 offset = end;
1030 mingap = next - end;
1031 }
1032 }
1033
1034 if (offset == RAM_ADDR_MAX) {
1035 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1036 (uint64_t)size);
1037 abort();
1038 }
1039
1040 return offset;
1041 }
1042
1043 ram_addr_t last_ram_offset(void)
1044 {
1045 RAMBlock *block;
1046 ram_addr_t last = 0;
1047
1048 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1049 last = MAX(last, block->offset + block->length);
1050
1051 return last;
1052 }
1053
1054 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1055 {
1056 int ret;
1057
1058 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1059 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1060 "dump-guest-core", true)) {
1061 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1062 if (ret) {
1063 perror("qemu_madvise");
1064 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1065 "but dump_guest_core=off specified\n");
1066 }
1067 }
1068 }
1069
1070 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1071 {
1072 RAMBlock *new_block, *block;
1073
1074 new_block = NULL;
1075 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1076 if (block->offset == addr) {
1077 new_block = block;
1078 break;
1079 }
1080 }
1081 assert(new_block);
1082 assert(!new_block->idstr[0]);
1083
1084 if (dev) {
1085 char *id = qdev_get_dev_path(dev);
1086 if (id) {
1087 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1088 g_free(id);
1089 }
1090 }
1091 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1092
1093 /* This assumes the iothread lock is taken here too. */
1094 qemu_mutex_lock_ramlist();
1095 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1096 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1097 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1098 new_block->idstr);
1099 abort();
1100 }
1101 }
1102 qemu_mutex_unlock_ramlist();
1103 }
1104
1105 static int memory_try_enable_merging(void *addr, size_t len)
1106 {
1107 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1108 /* disabled by the user */
1109 return 0;
1110 }
1111
1112 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1113 }
1114
1115 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1116 MemoryRegion *mr)
1117 {
1118 RAMBlock *block, *new_block;
1119
1120 size = TARGET_PAGE_ALIGN(size);
1121 new_block = g_malloc0(sizeof(*new_block));
1122 new_block->fd = -1;
1123
1124 /* This assumes the iothread lock is taken here too. */
1125 qemu_mutex_lock_ramlist();
1126 new_block->mr = mr;
1127 new_block->offset = find_ram_offset(size);
1128 if (host) {
1129 new_block->host = host;
1130 new_block->flags |= RAM_PREALLOC_MASK;
1131 } else if (xen_enabled()) {
1132 if (mem_path) {
1133 fprintf(stderr, "-mem-path not supported with Xen\n");
1134 exit(1);
1135 }
1136 xen_ram_alloc(new_block->offset, size, mr);
1137 } else {
1138 if (mem_path) {
1139 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1140 /*
1141 * file_ram_alloc() needs to allocate just like
1142 * phys_mem_alloc, but we haven't bothered to provide
1143 * a hook there.
1144 */
1145 fprintf(stderr,
1146 "-mem-path not supported with this accelerator\n");
1147 exit(1);
1148 }
1149 new_block->host = file_ram_alloc(new_block, size, mem_path);
1150 }
1151 if (!new_block->host) {
1152 new_block->host = phys_mem_alloc(size);
1153 if (!new_block->host) {
1154 fprintf(stderr, "Cannot set up guest memory '%s': %s\n",
1155 new_block->mr->name, strerror(errno));
1156 exit(1);
1157 }
1158 memory_try_enable_merging(new_block->host, size);
1159 }
1160 }
1161 new_block->length = size;
1162
1163 /* Keep the list sorted from biggest to smallest block. */
1164 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1165 if (block->length < new_block->length) {
1166 break;
1167 }
1168 }
1169 if (block) {
1170 QTAILQ_INSERT_BEFORE(block, new_block, next);
1171 } else {
1172 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1173 }
1174 ram_list.mru_block = NULL;
1175
1176 ram_list.version++;
1177 qemu_mutex_unlock_ramlist();
1178
1179 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1180 last_ram_offset() >> TARGET_PAGE_BITS);
1181 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1182 0, size >> TARGET_PAGE_BITS);
1183 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1184
1185 qemu_ram_setup_dump(new_block->host, size);
1186 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1187
1188 if (kvm_enabled())
1189 kvm_setup_guest_memory(new_block->host, size);
1190
1191 return new_block->offset;
1192 }
1193
1194 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1195 {
1196 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1197 }
1198
1199 void qemu_ram_free_from_ptr(ram_addr_t addr)
1200 {
1201 RAMBlock *block;
1202
1203 /* This assumes the iothread lock is taken here too. */
1204 qemu_mutex_lock_ramlist();
1205 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1206 if (addr == block->offset) {
1207 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1208 ram_list.mru_block = NULL;
1209 ram_list.version++;
1210 g_free(block);
1211 break;
1212 }
1213 }
1214 qemu_mutex_unlock_ramlist();
1215 }
1216
1217 void qemu_ram_free(ram_addr_t addr)
1218 {
1219 RAMBlock *block;
1220
1221 /* This assumes the iothread lock is taken here too. */
1222 qemu_mutex_lock_ramlist();
1223 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1224 if (addr == block->offset) {
1225 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1226 ram_list.mru_block = NULL;
1227 ram_list.version++;
1228 if (block->flags & RAM_PREALLOC_MASK) {
1229 ;
1230 } else if (xen_enabled()) {
1231 xen_invalidate_map_cache_entry(block->host);
1232 #ifndef _WIN32
1233 } else if (block->fd >= 0) {
1234 munmap(block->host, block->length);
1235 close(block->fd);
1236 #endif
1237 } else {
1238 qemu_anon_ram_free(block->host, block->length);
1239 }
1240 g_free(block);
1241 break;
1242 }
1243 }
1244 qemu_mutex_unlock_ramlist();
1245
1246 }
1247
1248 #ifndef _WIN32
1249 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1250 {
1251 RAMBlock *block;
1252 ram_addr_t offset;
1253 int flags;
1254 void *area, *vaddr;
1255
1256 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1257 offset = addr - block->offset;
1258 if (offset < block->length) {
1259 vaddr = block->host + offset;
1260 if (block->flags & RAM_PREALLOC_MASK) {
1261 ;
1262 } else if (xen_enabled()) {
1263 abort();
1264 } else {
1265 flags = MAP_FIXED;
1266 munmap(vaddr, length);
1267 if (block->fd >= 0) {
1268 #ifdef MAP_POPULATE
1269 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1270 MAP_PRIVATE;
1271 #else
1272 flags |= MAP_PRIVATE;
1273 #endif
1274 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1275 flags, block->fd, offset);
1276 } else {
1277 /*
1278 * Remap needs to match alloc. Accelerators that
1279 * set phys_mem_alloc never remap. If they did,
1280 * we'd need a remap hook here.
1281 */
1282 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1283
1284 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1285 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1286 flags, -1, 0);
1287 }
1288 if (area != vaddr) {
1289 fprintf(stderr, "Could not remap addr: "
1290 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1291 length, addr);
1292 exit(1);
1293 }
1294 memory_try_enable_merging(vaddr, length);
1295 qemu_ram_setup_dump(vaddr, length);
1296 }
1297 return;
1298 }
1299 }
1300 }
1301 #endif /* !_WIN32 */
1302
1303 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1304 {
1305 RAMBlock *block;
1306
1307 /* The list is protected by the iothread lock here. */
1308 block = ram_list.mru_block;
1309 if (block && addr - block->offset < block->length) {
1310 goto found;
1311 }
1312 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1313 if (addr - block->offset < block->length) {
1314 goto found;
1315 }
1316 }
1317
1318 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1319 abort();
1320
1321 found:
1322 ram_list.mru_block = block;
1323 return block;
1324 }
1325
1326 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1327 With the exception of the softmmu code in this file, this should
1328 only be used for local memory (e.g. video ram) that the device owns,
1329 and knows it isn't going to access beyond the end of the block.
1330
1331 It should not be used for general purpose DMA.
1332 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1333 */
1334 void *qemu_get_ram_ptr(ram_addr_t addr)
1335 {
1336 RAMBlock *block = qemu_get_ram_block(addr);
1337
1338 if (xen_enabled()) {
1339 /* We need to check if the requested address is in the RAM
1340 * because we don't want to map the entire memory in QEMU.
1341 * In that case just map until the end of the page.
1342 */
1343 if (block->offset == 0) {
1344 return xen_map_cache(addr, 0, 0);
1345 } else if (block->host == NULL) {
1346 block->host =
1347 xen_map_cache(block->offset, block->length, 1);
1348 }
1349 }
1350 return block->host + (addr - block->offset);
1351 }
1352
1353 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1354 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1355 *
1356 * ??? Is this still necessary?
1357 */
1358 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1359 {
1360 RAMBlock *block;
1361
1362 /* The list is protected by the iothread lock here. */
1363 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1364 if (addr - block->offset < block->length) {
1365 if (xen_enabled()) {
1366 /* We need to check if the requested address is in the RAM
1367 * because we don't want to map the entire memory in QEMU.
1368 * In that case just map until the end of the page.
1369 */
1370 if (block->offset == 0) {
1371 return xen_map_cache(addr, 0, 0);
1372 } else if (block->host == NULL) {
1373 block->host =
1374 xen_map_cache(block->offset, block->length, 1);
1375 }
1376 }
1377 return block->host + (addr - block->offset);
1378 }
1379 }
1380
1381 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1382 abort();
1383
1384 return NULL;
1385 }
1386
1387 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1388 * but takes a size argument */
1389 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1390 {
1391 if (*size == 0) {
1392 return NULL;
1393 }
1394 if (xen_enabled()) {
1395 return xen_map_cache(addr, *size, 1);
1396 } else {
1397 RAMBlock *block;
1398
1399 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1400 if (addr - block->offset < block->length) {
1401 if (addr - block->offset + *size > block->length)
1402 *size = block->length - addr + block->offset;
1403 return block->host + (addr - block->offset);
1404 }
1405 }
1406
1407 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1408 abort();
1409 }
1410 }
1411
1412 /* Some of the softmmu routines need to translate from a host pointer
1413 (typically a TLB entry) back to a ram offset. */
1414 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1415 {
1416 RAMBlock *block;
1417 uint8_t *host = ptr;
1418
1419 if (xen_enabled()) {
1420 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1421 return qemu_get_ram_block(*ram_addr)->mr;
1422 }
1423
1424 block = ram_list.mru_block;
1425 if (block && block->host && host - block->host < block->length) {
1426 goto found;
1427 }
1428
1429 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1430 /* This case append when the block is not mapped. */
1431 if (block->host == NULL) {
1432 continue;
1433 }
1434 if (host - block->host < block->length) {
1435 goto found;
1436 }
1437 }
1438
1439 return NULL;
1440
1441 found:
1442 *ram_addr = block->offset + (host - block->host);
1443 return block->mr;
1444 }
1445
1446 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1447 uint64_t val, unsigned size)
1448 {
1449 int dirty_flags;
1450 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1451 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1452 tb_invalidate_phys_page_fast(ram_addr, size);
1453 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1454 }
1455 switch (size) {
1456 case 1:
1457 stb_p(qemu_get_ram_ptr(ram_addr), val);
1458 break;
1459 case 2:
1460 stw_p(qemu_get_ram_ptr(ram_addr), val);
1461 break;
1462 case 4:
1463 stl_p(qemu_get_ram_ptr(ram_addr), val);
1464 break;
1465 default:
1466 abort();
1467 }
1468 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1469 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1470 /* we remove the notdirty callback only if the code has been
1471 flushed */
1472 if (dirty_flags == 0xff) {
1473 CPUArchState *env = current_cpu->env_ptr;
1474 tlb_set_dirty(env, env->mem_io_vaddr);
1475 }
1476 }
1477
1478 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1479 unsigned size, bool is_write)
1480 {
1481 return is_write;
1482 }
1483
1484 static const MemoryRegionOps notdirty_mem_ops = {
1485 .write = notdirty_mem_write,
1486 .valid.accepts = notdirty_mem_accepts,
1487 .endianness = DEVICE_NATIVE_ENDIAN,
1488 };
1489
1490 /* Generate a debug exception if a watchpoint has been hit. */
1491 static void check_watchpoint(int offset, int len_mask, int flags)
1492 {
1493 CPUArchState *env = current_cpu->env_ptr;
1494 target_ulong pc, cs_base;
1495 target_ulong vaddr;
1496 CPUWatchpoint *wp;
1497 int cpu_flags;
1498
1499 if (env->watchpoint_hit) {
1500 /* We re-entered the check after replacing the TB. Now raise
1501 * the debug interrupt so that is will trigger after the
1502 * current instruction. */
1503 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1504 return;
1505 }
1506 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1507 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1508 if ((vaddr == (wp->vaddr & len_mask) ||
1509 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1510 wp->flags |= BP_WATCHPOINT_HIT;
1511 if (!env->watchpoint_hit) {
1512 env->watchpoint_hit = wp;
1513 tb_check_watchpoint(env);
1514 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1515 env->exception_index = EXCP_DEBUG;
1516 cpu_loop_exit(env);
1517 } else {
1518 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1519 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1520 cpu_resume_from_signal(env, NULL);
1521 }
1522 }
1523 } else {
1524 wp->flags &= ~BP_WATCHPOINT_HIT;
1525 }
1526 }
1527 }
1528
1529 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1530 so these check for a hit then pass through to the normal out-of-line
1531 phys routines. */
1532 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1533 unsigned size)
1534 {
1535 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1536 switch (size) {
1537 case 1: return ldub_phys(addr);
1538 case 2: return lduw_phys(addr);
1539 case 4: return ldl_phys(addr);
1540 default: abort();
1541 }
1542 }
1543
1544 static void watch_mem_write(void *opaque, hwaddr addr,
1545 uint64_t val, unsigned size)
1546 {
1547 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1548 switch (size) {
1549 case 1:
1550 stb_phys(addr, val);
1551 break;
1552 case 2:
1553 stw_phys(addr, val);
1554 break;
1555 case 4:
1556 stl_phys(addr, val);
1557 break;
1558 default: abort();
1559 }
1560 }
1561
1562 static const MemoryRegionOps watch_mem_ops = {
1563 .read = watch_mem_read,
1564 .write = watch_mem_write,
1565 .endianness = DEVICE_NATIVE_ENDIAN,
1566 };
1567
1568 static uint64_t subpage_read(void *opaque, hwaddr addr,
1569 unsigned len)
1570 {
1571 subpage_t *subpage = opaque;
1572 uint8_t buf[4];
1573
1574 #if defined(DEBUG_SUBPAGE)
1575 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1576 subpage, len, addr);
1577 #endif
1578 address_space_read(subpage->as, addr + subpage->base, buf, len);
1579 switch (len) {
1580 case 1:
1581 return ldub_p(buf);
1582 case 2:
1583 return lduw_p(buf);
1584 case 4:
1585 return ldl_p(buf);
1586 default:
1587 abort();
1588 }
1589 }
1590
1591 static void subpage_write(void *opaque, hwaddr addr,
1592 uint64_t value, unsigned len)
1593 {
1594 subpage_t *subpage = opaque;
1595 uint8_t buf[4];
1596
1597 #if defined(DEBUG_SUBPAGE)
1598 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1599 " value %"PRIx64"\n",
1600 __func__, subpage, len, addr, value);
1601 #endif
1602 switch (len) {
1603 case 1:
1604 stb_p(buf, value);
1605 break;
1606 case 2:
1607 stw_p(buf, value);
1608 break;
1609 case 4:
1610 stl_p(buf, value);
1611 break;
1612 default:
1613 abort();
1614 }
1615 address_space_write(subpage->as, addr + subpage->base, buf, len);
1616 }
1617
1618 static bool subpage_accepts(void *opaque, hwaddr addr,
1619 unsigned size, bool is_write)
1620 {
1621 subpage_t *subpage = opaque;
1622 #if defined(DEBUG_SUBPAGE)
1623 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1624 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1625 #endif
1626
1627 return address_space_access_valid(subpage->as, addr + subpage->base,
1628 size, is_write);
1629 }
1630
1631 static const MemoryRegionOps subpage_ops = {
1632 .read = subpage_read,
1633 .write = subpage_write,
1634 .valid.accepts = subpage_accepts,
1635 .endianness = DEVICE_NATIVE_ENDIAN,
1636 };
1637
1638 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1639 uint16_t section)
1640 {
1641 int idx, eidx;
1642
1643 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1644 return -1;
1645 idx = SUBPAGE_IDX(start);
1646 eidx = SUBPAGE_IDX(end);
1647 #if defined(DEBUG_SUBPAGE)
1648 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1649 mmio, start, end, idx, eidx, memory);
1650 #endif
1651 for (; idx <= eidx; idx++) {
1652 mmio->sub_section[idx] = section;
1653 }
1654
1655 return 0;
1656 }
1657
1658 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1659 {
1660 subpage_t *mmio;
1661
1662 mmio = g_malloc0(sizeof(subpage_t));
1663
1664 mmio->as = as;
1665 mmio->base = base;
1666 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1667 "subpage", TARGET_PAGE_SIZE);
1668 mmio->iomem.subpage = true;
1669 #if defined(DEBUG_SUBPAGE)
1670 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1671 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1672 #endif
1673 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1674
1675 return mmio;
1676 }
1677
1678 static uint16_t dummy_section(MemoryRegion *mr)
1679 {
1680 MemoryRegionSection section = {
1681 .mr = mr,
1682 .offset_within_address_space = 0,
1683 .offset_within_region = 0,
1684 .size = int128_2_64(),
1685 };
1686
1687 return phys_section_add(&section);
1688 }
1689
1690 MemoryRegion *iotlb_to_region(hwaddr index)
1691 {
1692 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1693 }
1694
1695 static void io_mem_init(void)
1696 {
1697 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1698 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1699 "unassigned", UINT64_MAX);
1700 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1701 "notdirty", UINT64_MAX);
1702 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1703 "watch", UINT64_MAX);
1704 }
1705
1706 static void mem_begin(MemoryListener *listener)
1707 {
1708 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1709 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1710
1711 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1712 d->as = as;
1713 as->next_dispatch = d;
1714 }
1715
1716 static void mem_commit(MemoryListener *listener)
1717 {
1718 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1719 AddressSpaceDispatch *cur = as->dispatch;
1720 AddressSpaceDispatch *next = as->next_dispatch;
1721
1722 next->nodes = next_map.nodes;
1723 next->sections = next_map.sections;
1724
1725 as->dispatch = next;
1726 g_free(cur);
1727 }
1728
1729 static void core_begin(MemoryListener *listener)
1730 {
1731 uint16_t n;
1732
1733 prev_map = g_new(PhysPageMap, 1);
1734 *prev_map = next_map;
1735
1736 memset(&next_map, 0, sizeof(next_map));
1737 n = dummy_section(&io_mem_unassigned);
1738 assert(n == PHYS_SECTION_UNASSIGNED);
1739 n = dummy_section(&io_mem_notdirty);
1740 assert(n == PHYS_SECTION_NOTDIRTY);
1741 n = dummy_section(&io_mem_rom);
1742 assert(n == PHYS_SECTION_ROM);
1743 n = dummy_section(&io_mem_watch);
1744 assert(n == PHYS_SECTION_WATCH);
1745 }
1746
1747 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1748 * All AddressSpaceDispatch instances have switched to the next map.
1749 */
1750 static void core_commit(MemoryListener *listener)
1751 {
1752 phys_sections_free(prev_map);
1753 }
1754
1755 static void tcg_commit(MemoryListener *listener)
1756 {
1757 CPUState *cpu;
1758
1759 /* since each CPU stores ram addresses in its TLB cache, we must
1760 reset the modified entries */
1761 /* XXX: slow ! */
1762 CPU_FOREACH(cpu) {
1763 CPUArchState *env = cpu->env_ptr;
1764
1765 tlb_flush(env, 1);
1766 }
1767 }
1768
1769 static void core_log_global_start(MemoryListener *listener)
1770 {
1771 cpu_physical_memory_set_dirty_tracking(1);
1772 }
1773
1774 static void core_log_global_stop(MemoryListener *listener)
1775 {
1776 cpu_physical_memory_set_dirty_tracking(0);
1777 }
1778
1779 static MemoryListener core_memory_listener = {
1780 .begin = core_begin,
1781 .commit = core_commit,
1782 .log_global_start = core_log_global_start,
1783 .log_global_stop = core_log_global_stop,
1784 .priority = 1,
1785 };
1786
1787 static MemoryListener tcg_memory_listener = {
1788 .commit = tcg_commit,
1789 };
1790
1791 void address_space_init_dispatch(AddressSpace *as)
1792 {
1793 as->dispatch = NULL;
1794 as->dispatch_listener = (MemoryListener) {
1795 .begin = mem_begin,
1796 .commit = mem_commit,
1797 .region_add = mem_add,
1798 .region_nop = mem_add,
1799 .priority = 0,
1800 };
1801 memory_listener_register(&as->dispatch_listener, as);
1802 }
1803
1804 void address_space_destroy_dispatch(AddressSpace *as)
1805 {
1806 AddressSpaceDispatch *d = as->dispatch;
1807
1808 memory_listener_unregister(&as->dispatch_listener);
1809 g_free(d);
1810 as->dispatch = NULL;
1811 }
1812
1813 static void memory_map_init(void)
1814 {
1815 system_memory = g_malloc(sizeof(*system_memory));
1816 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1817 address_space_init(&address_space_memory, system_memory, "memory");
1818
1819 system_io = g_malloc(sizeof(*system_io));
1820 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
1821 65536);
1822 address_space_init(&address_space_io, system_io, "I/O");
1823
1824 memory_listener_register(&core_memory_listener, &address_space_memory);
1825 if (tcg_enabled()) {
1826 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1827 }
1828 }
1829
1830 MemoryRegion *get_system_memory(void)
1831 {
1832 return system_memory;
1833 }
1834
1835 MemoryRegion *get_system_io(void)
1836 {
1837 return system_io;
1838 }
1839
1840 #endif /* !defined(CONFIG_USER_ONLY) */
1841
1842 /* physical memory access (slow version, mainly for debug) */
1843 #if defined(CONFIG_USER_ONLY)
1844 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1845 uint8_t *buf, int len, int is_write)
1846 {
1847 int l, flags;
1848 target_ulong page;
1849 void * p;
1850
1851 while (len > 0) {
1852 page = addr & TARGET_PAGE_MASK;
1853 l = (page + TARGET_PAGE_SIZE) - addr;
1854 if (l > len)
1855 l = len;
1856 flags = page_get_flags(page);
1857 if (!(flags & PAGE_VALID))
1858 return -1;
1859 if (is_write) {
1860 if (!(flags & PAGE_WRITE))
1861 return -1;
1862 /* XXX: this code should not depend on lock_user */
1863 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1864 return -1;
1865 memcpy(p, buf, l);
1866 unlock_user(p, addr, l);
1867 } else {
1868 if (!(flags & PAGE_READ))
1869 return -1;
1870 /* XXX: this code should not depend on lock_user */
1871 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1872 return -1;
1873 memcpy(buf, p, l);
1874 unlock_user(p, addr, 0);
1875 }
1876 len -= l;
1877 buf += l;
1878 addr += l;
1879 }
1880 return 0;
1881 }
1882
1883 #else
1884
1885 static void invalidate_and_set_dirty(hwaddr addr,
1886 hwaddr length)
1887 {
1888 if (!cpu_physical_memory_is_dirty(addr)) {
1889 /* invalidate code */
1890 tb_invalidate_phys_page_range(addr, addr + length, 0);
1891 /* set dirty bit */
1892 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1893 }
1894 xen_modified_memory(addr, length);
1895 }
1896
1897 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1898 {
1899 if (memory_region_is_ram(mr)) {
1900 return !(is_write && mr->readonly);
1901 }
1902 if (memory_region_is_romd(mr)) {
1903 return !is_write;
1904 }
1905
1906 return false;
1907 }
1908
1909 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1910 {
1911 unsigned access_size_max = mr->ops->valid.max_access_size;
1912
1913 /* Regions are assumed to support 1-4 byte accesses unless
1914 otherwise specified. */
1915 if (access_size_max == 0) {
1916 access_size_max = 4;
1917 }
1918
1919 /* Bound the maximum access by the alignment of the address. */
1920 if (!mr->ops->impl.unaligned) {
1921 unsigned align_size_max = addr & -addr;
1922 if (align_size_max != 0 && align_size_max < access_size_max) {
1923 access_size_max = align_size_max;
1924 }
1925 }
1926
1927 /* Don't attempt accesses larger than the maximum. */
1928 if (l > access_size_max) {
1929 l = access_size_max;
1930 }
1931 if (l & (l - 1)) {
1932 l = 1 << (qemu_fls(l) - 1);
1933 }
1934
1935 return l;
1936 }
1937
1938 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1939 int len, bool is_write)
1940 {
1941 hwaddr l;
1942 uint8_t *ptr;
1943 uint64_t val;
1944 hwaddr addr1;
1945 MemoryRegion *mr;
1946 bool error = false;
1947
1948 while (len > 0) {
1949 l = len;
1950 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1951
1952 if (is_write) {
1953 if (!memory_access_is_direct(mr, is_write)) {
1954 l = memory_access_size(mr, l, addr1);
1955 /* XXX: could force current_cpu to NULL to avoid
1956 potential bugs */
1957 switch (l) {
1958 case 8:
1959 /* 64 bit write access */
1960 val = ldq_p(buf);
1961 error |= io_mem_write(mr, addr1, val, 8);
1962 break;
1963 case 4:
1964 /* 32 bit write access */
1965 val = ldl_p(buf);
1966 error |= io_mem_write(mr, addr1, val, 4);
1967 break;
1968 case 2:
1969 /* 16 bit write access */
1970 val = lduw_p(buf);
1971 error |= io_mem_write(mr, addr1, val, 2);
1972 break;
1973 case 1:
1974 /* 8 bit write access */
1975 val = ldub_p(buf);
1976 error |= io_mem_write(mr, addr1, val, 1);
1977 break;
1978 default:
1979 abort();
1980 }
1981 } else {
1982 addr1 += memory_region_get_ram_addr(mr);
1983 /* RAM case */
1984 ptr = qemu_get_ram_ptr(addr1);
1985 memcpy(ptr, buf, l);
1986 invalidate_and_set_dirty(addr1, l);
1987 }
1988 } else {
1989 if (!memory_access_is_direct(mr, is_write)) {
1990 /* I/O case */
1991 l = memory_access_size(mr, l, addr1);
1992 switch (l) {
1993 case 8:
1994 /* 64 bit read access */
1995 error |= io_mem_read(mr, addr1, &val, 8);
1996 stq_p(buf, val);
1997 break;
1998 case 4:
1999 /* 32 bit read access */
2000 error |= io_mem_read(mr, addr1, &val, 4);
2001 stl_p(buf, val);
2002 break;
2003 case 2:
2004 /* 16 bit read access */
2005 error |= io_mem_read(mr, addr1, &val, 2);
2006 stw_p(buf, val);
2007 break;
2008 case 1:
2009 /* 8 bit read access */
2010 error |= io_mem_read(mr, addr1, &val, 1);
2011 stb_p(buf, val);
2012 break;
2013 default:
2014 abort();
2015 }
2016 } else {
2017 /* RAM case */
2018 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2019 memcpy(buf, ptr, l);
2020 }
2021 }
2022 len -= l;
2023 buf += l;
2024 addr += l;
2025 }
2026
2027 return error;
2028 }
2029
2030 bool address_space_write(AddressSpace *as, hwaddr addr,
2031 const uint8_t *buf, int len)
2032 {
2033 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2034 }
2035
2036 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2037 {
2038 return address_space_rw(as, addr, buf, len, false);
2039 }
2040
2041
2042 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2043 int len, int is_write)
2044 {
2045 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2046 }
2047
2048 /* used for ROM loading : can write in RAM and ROM */
2049 void cpu_physical_memory_write_rom(hwaddr addr,
2050 const uint8_t *buf, int len)
2051 {
2052 hwaddr l;
2053 uint8_t *ptr;
2054 hwaddr addr1;
2055 MemoryRegion *mr;
2056
2057 while (len > 0) {
2058 l = len;
2059 mr = address_space_translate(&address_space_memory,
2060 addr, &addr1, &l, true);
2061
2062 if (!(memory_region_is_ram(mr) ||
2063 memory_region_is_romd(mr))) {
2064 /* do nothing */
2065 } else {
2066 addr1 += memory_region_get_ram_addr(mr);
2067 /* ROM/RAM case */
2068 ptr = qemu_get_ram_ptr(addr1);
2069 memcpy(ptr, buf, l);
2070 invalidate_and_set_dirty(addr1, l);
2071 }
2072 len -= l;
2073 buf += l;
2074 addr += l;
2075 }
2076 }
2077
2078 typedef struct {
2079 MemoryRegion *mr;
2080 void *buffer;
2081 hwaddr addr;
2082 hwaddr len;
2083 } BounceBuffer;
2084
2085 static BounceBuffer bounce;
2086
2087 typedef struct MapClient {
2088 void *opaque;
2089 void (*callback)(void *opaque);
2090 QLIST_ENTRY(MapClient) link;
2091 } MapClient;
2092
2093 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2094 = QLIST_HEAD_INITIALIZER(map_client_list);
2095
2096 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2097 {
2098 MapClient *client = g_malloc(sizeof(*client));
2099
2100 client->opaque = opaque;
2101 client->callback = callback;
2102 QLIST_INSERT_HEAD(&map_client_list, client, link);
2103 return client;
2104 }
2105
2106 static void cpu_unregister_map_client(void *_client)
2107 {
2108 MapClient *client = (MapClient *)_client;
2109
2110 QLIST_REMOVE(client, link);
2111 g_free(client);
2112 }
2113
2114 static void cpu_notify_map_clients(void)
2115 {
2116 MapClient *client;
2117
2118 while (!QLIST_EMPTY(&map_client_list)) {
2119 client = QLIST_FIRST(&map_client_list);
2120 client->callback(client->opaque);
2121 cpu_unregister_map_client(client);
2122 }
2123 }
2124
2125 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2126 {
2127 MemoryRegion *mr;
2128 hwaddr l, xlat;
2129
2130 while (len > 0) {
2131 l = len;
2132 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2133 if (!memory_access_is_direct(mr, is_write)) {
2134 l = memory_access_size(mr, l, addr);
2135 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2136 return false;
2137 }
2138 }
2139
2140 len -= l;
2141 addr += l;
2142 }
2143 return true;
2144 }
2145
2146 /* Map a physical memory region into a host virtual address.
2147 * May map a subset of the requested range, given by and returned in *plen.
2148 * May return NULL if resources needed to perform the mapping are exhausted.
2149 * Use only for reads OR writes - not for read-modify-write operations.
2150 * Use cpu_register_map_client() to know when retrying the map operation is
2151 * likely to succeed.
2152 */
2153 void *address_space_map(AddressSpace *as,
2154 hwaddr addr,
2155 hwaddr *plen,
2156 bool is_write)
2157 {
2158 hwaddr len = *plen;
2159 hwaddr done = 0;
2160 hwaddr l, xlat, base;
2161 MemoryRegion *mr, *this_mr;
2162 ram_addr_t raddr;
2163
2164 if (len == 0) {
2165 return NULL;
2166 }
2167
2168 l = len;
2169 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2170 if (!memory_access_is_direct(mr, is_write)) {
2171 if (bounce.buffer) {
2172 return NULL;
2173 }
2174 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2175 bounce.addr = addr;
2176 bounce.len = l;
2177
2178 memory_region_ref(mr);
2179 bounce.mr = mr;
2180 if (!is_write) {
2181 address_space_read(as, addr, bounce.buffer, l);
2182 }
2183
2184 *plen = l;
2185 return bounce.buffer;
2186 }
2187
2188 base = xlat;
2189 raddr = memory_region_get_ram_addr(mr);
2190
2191 for (;;) {
2192 len -= l;
2193 addr += l;
2194 done += l;
2195 if (len == 0) {
2196 break;
2197 }
2198
2199 l = len;
2200 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2201 if (this_mr != mr || xlat != base + done) {
2202 break;
2203 }
2204 }
2205
2206 memory_region_ref(mr);
2207 *plen = done;
2208 return qemu_ram_ptr_length(raddr + base, plen);
2209 }
2210
2211 /* Unmaps a memory region previously mapped by address_space_map().
2212 * Will also mark the memory as dirty if is_write == 1. access_len gives
2213 * the amount of memory that was actually read or written by the caller.
2214 */
2215 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2216 int is_write, hwaddr access_len)
2217 {
2218 if (buffer != bounce.buffer) {
2219 MemoryRegion *mr;
2220 ram_addr_t addr1;
2221
2222 mr = qemu_ram_addr_from_host(buffer, &addr1);
2223 assert(mr != NULL);
2224 if (is_write) {
2225 while (access_len) {
2226 unsigned l;
2227 l = TARGET_PAGE_SIZE;
2228 if (l > access_len)
2229 l = access_len;
2230 invalidate_and_set_dirty(addr1, l);
2231 addr1 += l;
2232 access_len -= l;
2233 }
2234 }
2235 if (xen_enabled()) {
2236 xen_invalidate_map_cache_entry(buffer);
2237 }
2238 memory_region_unref(mr);
2239 return;
2240 }
2241 if (is_write) {
2242 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2243 }
2244 qemu_vfree(bounce.buffer);
2245 bounce.buffer = NULL;
2246 memory_region_unref(bounce.mr);
2247 cpu_notify_map_clients();
2248 }
2249
2250 void *cpu_physical_memory_map(hwaddr addr,
2251 hwaddr *plen,
2252 int is_write)
2253 {
2254 return address_space_map(&address_space_memory, addr, plen, is_write);
2255 }
2256
2257 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2258 int is_write, hwaddr access_len)
2259 {
2260 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2261 }
2262
2263 /* warning: addr must be aligned */
2264 static inline uint32_t ldl_phys_internal(hwaddr addr,
2265 enum device_endian endian)
2266 {
2267 uint8_t *ptr;
2268 uint64_t val;
2269 MemoryRegion *mr;
2270 hwaddr l = 4;
2271 hwaddr addr1;
2272
2273 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2274 false);
2275 if (l < 4 || !memory_access_is_direct(mr, false)) {
2276 /* I/O case */
2277 io_mem_read(mr, addr1, &val, 4);
2278 #if defined(TARGET_WORDS_BIGENDIAN)
2279 if (endian == DEVICE_LITTLE_ENDIAN) {
2280 val = bswap32(val);
2281 }
2282 #else
2283 if (endian == DEVICE_BIG_ENDIAN) {
2284 val = bswap32(val);
2285 }
2286 #endif
2287 } else {
2288 /* RAM case */
2289 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2290 & TARGET_PAGE_MASK)
2291 + addr1);
2292 switch (endian) {
2293 case DEVICE_LITTLE_ENDIAN:
2294 val = ldl_le_p(ptr);
2295 break;
2296 case DEVICE_BIG_ENDIAN:
2297 val = ldl_be_p(ptr);
2298 break;
2299 default:
2300 val = ldl_p(ptr);
2301 break;
2302 }
2303 }
2304 return val;
2305 }
2306
2307 uint32_t ldl_phys(hwaddr addr)
2308 {
2309 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2310 }
2311
2312 uint32_t ldl_le_phys(hwaddr addr)
2313 {
2314 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2315 }
2316
2317 uint32_t ldl_be_phys(hwaddr addr)
2318 {
2319 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2320 }
2321
2322 /* warning: addr must be aligned */
2323 static inline uint64_t ldq_phys_internal(hwaddr addr,
2324 enum device_endian endian)
2325 {
2326 uint8_t *ptr;
2327 uint64_t val;
2328 MemoryRegion *mr;
2329 hwaddr l = 8;
2330 hwaddr addr1;
2331
2332 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2333 false);
2334 if (l < 8 || !memory_access_is_direct(mr, false)) {
2335 /* I/O case */
2336 io_mem_read(mr, addr1, &val, 8);
2337 #if defined(TARGET_WORDS_BIGENDIAN)
2338 if (endian == DEVICE_LITTLE_ENDIAN) {
2339 val = bswap64(val);
2340 }
2341 #else
2342 if (endian == DEVICE_BIG_ENDIAN) {
2343 val = bswap64(val);
2344 }
2345 #endif
2346 } else {
2347 /* RAM case */
2348 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2349 & TARGET_PAGE_MASK)
2350 + addr1);
2351 switch (endian) {
2352 case DEVICE_LITTLE_ENDIAN:
2353 val = ldq_le_p(ptr);
2354 break;
2355 case DEVICE_BIG_ENDIAN:
2356 val = ldq_be_p(ptr);
2357 break;
2358 default:
2359 val = ldq_p(ptr);
2360 break;
2361 }
2362 }
2363 return val;
2364 }
2365
2366 uint64_t ldq_phys(hwaddr addr)
2367 {
2368 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2369 }
2370
2371 uint64_t ldq_le_phys(hwaddr addr)
2372 {
2373 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2374 }
2375
2376 uint64_t ldq_be_phys(hwaddr addr)
2377 {
2378 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2379 }
2380
2381 /* XXX: optimize */
2382 uint32_t ldub_phys(hwaddr addr)
2383 {
2384 uint8_t val;
2385 cpu_physical_memory_read(addr, &val, 1);
2386 return val;
2387 }
2388
2389 /* warning: addr must be aligned */
2390 static inline uint32_t lduw_phys_internal(hwaddr addr,
2391 enum device_endian endian)
2392 {
2393 uint8_t *ptr;
2394 uint64_t val;
2395 MemoryRegion *mr;
2396 hwaddr l = 2;
2397 hwaddr addr1;
2398
2399 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2400 false);
2401 if (l < 2 || !memory_access_is_direct(mr, false)) {
2402 /* I/O case */
2403 io_mem_read(mr, addr1, &val, 2);
2404 #if defined(TARGET_WORDS_BIGENDIAN)
2405 if (endian == DEVICE_LITTLE_ENDIAN) {
2406 val = bswap16(val);
2407 }
2408 #else
2409 if (endian == DEVICE_BIG_ENDIAN) {
2410 val = bswap16(val);
2411 }
2412 #endif
2413 } else {
2414 /* RAM case */
2415 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2416 & TARGET_PAGE_MASK)
2417 + addr1);
2418 switch (endian) {
2419 case DEVICE_LITTLE_ENDIAN:
2420 val = lduw_le_p(ptr);
2421 break;
2422 case DEVICE_BIG_ENDIAN:
2423 val = lduw_be_p(ptr);
2424 break;
2425 default:
2426 val = lduw_p(ptr);
2427 break;
2428 }
2429 }
2430 return val;
2431 }
2432
2433 uint32_t lduw_phys(hwaddr addr)
2434 {
2435 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2436 }
2437
2438 uint32_t lduw_le_phys(hwaddr addr)
2439 {
2440 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2441 }
2442
2443 uint32_t lduw_be_phys(hwaddr addr)
2444 {
2445 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2446 }
2447
2448 /* warning: addr must be aligned. The ram page is not masked as dirty
2449 and the code inside is not invalidated. It is useful if the dirty
2450 bits are used to track modified PTEs */
2451 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2452 {
2453 uint8_t *ptr;
2454 MemoryRegion *mr;
2455 hwaddr l = 4;
2456 hwaddr addr1;
2457
2458 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2459 true);
2460 if (l < 4 || !memory_access_is_direct(mr, true)) {
2461 io_mem_write(mr, addr1, val, 4);
2462 } else {
2463 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2464 ptr = qemu_get_ram_ptr(addr1);
2465 stl_p(ptr, val);
2466
2467 if (unlikely(in_migration)) {
2468 if (!cpu_physical_memory_is_dirty(addr1)) {
2469 /* invalidate code */
2470 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2471 /* set dirty bit */
2472 cpu_physical_memory_set_dirty_flags(
2473 addr1, (0xff & ~CODE_DIRTY_FLAG));
2474 }
2475 }
2476 }
2477 }
2478
2479 /* warning: addr must be aligned */
2480 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2481 enum device_endian endian)
2482 {
2483 uint8_t *ptr;
2484 MemoryRegion *mr;
2485 hwaddr l = 4;
2486 hwaddr addr1;
2487
2488 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2489 true);
2490 if (l < 4 || !memory_access_is_direct(mr, true)) {
2491 #if defined(TARGET_WORDS_BIGENDIAN)
2492 if (endian == DEVICE_LITTLE_ENDIAN) {
2493 val = bswap32(val);
2494 }
2495 #else
2496 if (endian == DEVICE_BIG_ENDIAN) {
2497 val = bswap32(val);
2498 }
2499 #endif
2500 io_mem_write(mr, addr1, val, 4);
2501 } else {
2502 /* RAM case */
2503 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2504 ptr = qemu_get_ram_ptr(addr1);
2505 switch (endian) {
2506 case DEVICE_LITTLE_ENDIAN:
2507 stl_le_p(ptr, val);
2508 break;
2509 case DEVICE_BIG_ENDIAN:
2510 stl_be_p(ptr, val);
2511 break;
2512 default:
2513 stl_p(ptr, val);
2514 break;
2515 }
2516 invalidate_and_set_dirty(addr1, 4);
2517 }
2518 }
2519
2520 void stl_phys(hwaddr addr, uint32_t val)
2521 {
2522 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2523 }
2524
2525 void stl_le_phys(hwaddr addr, uint32_t val)
2526 {
2527 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2528 }
2529
2530 void stl_be_phys(hwaddr addr, uint32_t val)
2531 {
2532 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2533 }
2534
2535 /* XXX: optimize */
2536 void stb_phys(hwaddr addr, uint32_t val)
2537 {
2538 uint8_t v = val;
2539 cpu_physical_memory_write(addr, &v, 1);
2540 }
2541
2542 /* warning: addr must be aligned */
2543 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2544 enum device_endian endian)
2545 {
2546 uint8_t *ptr;
2547 MemoryRegion *mr;
2548 hwaddr l = 2;
2549 hwaddr addr1;
2550
2551 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2552 true);
2553 if (l < 2 || !memory_access_is_direct(mr, true)) {
2554 #if defined(TARGET_WORDS_BIGENDIAN)
2555 if (endian == DEVICE_LITTLE_ENDIAN) {
2556 val = bswap16(val);
2557 }
2558 #else
2559 if (endian == DEVICE_BIG_ENDIAN) {
2560 val = bswap16(val);
2561 }
2562 #endif
2563 io_mem_write(mr, addr1, val, 2);
2564 } else {
2565 /* RAM case */
2566 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2567 ptr = qemu_get_ram_ptr(addr1);
2568 switch (endian) {
2569 case DEVICE_LITTLE_ENDIAN:
2570 stw_le_p(ptr, val);
2571 break;
2572 case DEVICE_BIG_ENDIAN:
2573 stw_be_p(ptr, val);
2574 break;
2575 default:
2576 stw_p(ptr, val);
2577 break;
2578 }
2579 invalidate_and_set_dirty(addr1, 2);
2580 }
2581 }
2582
2583 void stw_phys(hwaddr addr, uint32_t val)
2584 {
2585 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2586 }
2587
2588 void stw_le_phys(hwaddr addr, uint32_t val)
2589 {
2590 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2591 }
2592
2593 void stw_be_phys(hwaddr addr, uint32_t val)
2594 {
2595 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2596 }
2597
2598 /* XXX: optimize */
2599 void stq_phys(hwaddr addr, uint64_t val)
2600 {
2601 val = tswap64(val);
2602 cpu_physical_memory_write(addr, &val, 8);
2603 }
2604
2605 void stq_le_phys(hwaddr addr, uint64_t val)
2606 {
2607 val = cpu_to_le64(val);
2608 cpu_physical_memory_write(addr, &val, 8);
2609 }
2610
2611 void stq_be_phys(hwaddr addr, uint64_t val)
2612 {
2613 val = cpu_to_be64(val);
2614 cpu_physical_memory_write(addr, &val, 8);
2615 }
2616
2617 /* virtual memory access for debug (includes writing to ROM) */
2618 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2619 uint8_t *buf, int len, int is_write)
2620 {
2621 int l;
2622 hwaddr phys_addr;
2623 target_ulong page;
2624
2625 while (len > 0) {
2626 page = addr & TARGET_PAGE_MASK;
2627 phys_addr = cpu_get_phys_page_debug(cpu, page);
2628 /* if no physical page mapped, return an error */
2629 if (phys_addr == -1)
2630 return -1;
2631 l = (page + TARGET_PAGE_SIZE) - addr;
2632 if (l > len)
2633 l = len;
2634 phys_addr += (addr & ~TARGET_PAGE_MASK);
2635 if (is_write)
2636 cpu_physical_memory_write_rom(phys_addr, buf, l);
2637 else
2638 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2639 len -= l;
2640 buf += l;
2641 addr += l;
2642 }
2643 return 0;
2644 }
2645 #endif
2646
2647 #if !defined(CONFIG_USER_ONLY)
2648
2649 /*
2650 * A helper function for the _utterly broken_ virtio device model to find out if
2651 * it's running on a big endian machine. Don't do this at home kids!
2652 */
2653 bool virtio_is_big_endian(void);
2654 bool virtio_is_big_endian(void)
2655 {
2656 #if defined(TARGET_WORDS_BIGENDIAN)
2657 return true;
2658 #else
2659 return false;
2660 #endif
2661 }
2662
2663 #endif
2664
2665 #ifndef CONFIG_USER_ONLY
2666 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2667 {
2668 MemoryRegion*mr;
2669 hwaddr l = 1;
2670
2671 mr = address_space_translate(&address_space_memory,
2672 phys_addr, &phys_addr, &l, false);
2673
2674 return !(memory_region_is_ram(mr) ||
2675 memory_region_is_romd(mr));
2676 }
2677
2678 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2679 {
2680 RAMBlock *block;
2681
2682 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2683 func(block->host, block->offset, block->length, opaque);
2684 }
2685 }
2686 #endif
AR7 emulation for QEMU