?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
virtio: Implement MMIO based virtio transport
[qemu/ar7.git] / exec.c
blobc99a8839c5704a13292fd7325f2f688ace4adfcb
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
51
52 #include "exec/memory-internal.h"
53
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 CPUState *first_cpu;
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 typedef PhysPageEntry Node[L2_SIZE];
92
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101 };
102
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
110
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
115
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
124
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
127
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
133
134 static MemoryRegion io_mem_watch;
135 #endif
136
137 #if !defined(CONFIG_USER_ONLY)
138
139 static void phys_map_node_reserve(unsigned nodes)
140 {
141 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
142 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
143 16);
144 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
145 next_map.nodes_nb + nodes);
146 next_map.nodes = g_renew(Node, next_map.nodes,
147 next_map.nodes_nb_alloc);
148 }
149 }
150
151 static uint16_t phys_map_node_alloc(void)
152 {
153 unsigned i;
154 uint16_t ret;
155
156 ret = next_map.nodes_nb++;
157 assert(ret != PHYS_MAP_NODE_NIL);
158 assert(ret != next_map.nodes_nb_alloc);
159 for (i = 0; i < L2_SIZE; ++i) {
160 next_map.nodes[ret][i].is_leaf = 0;
161 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
162 }
163 return ret;
164 }
165
166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
167 hwaddr *nb, uint16_t leaf,
168 int level)
169 {
170 PhysPageEntry *p;
171 int i;
172 hwaddr step = (hwaddr)1 << (level * L2_BITS);
173
174 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
175 lp->ptr = phys_map_node_alloc();
176 p = next_map.nodes[lp->ptr];
177 if (level == 0) {
178 for (i = 0; i < L2_SIZE; i++) {
179 p[i].is_leaf = 1;
180 p[i].ptr = PHYS_SECTION_UNASSIGNED;
181 }
182 }
183 } else {
184 p = next_map.nodes[lp->ptr];
185 }
186 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
187
188 while (*nb && lp < &p[L2_SIZE]) {
189 if ((*index & (step - 1)) == 0 && *nb >= step) {
190 lp->is_leaf = true;
191 lp->ptr = leaf;
192 *index += step;
193 *nb -= step;
194 } else {
195 phys_page_set_level(lp, index, nb, leaf, level - 1);
196 }
197 ++lp;
198 }
199 }
200
201 static void phys_page_set(AddressSpaceDispatch *d,
202 hwaddr index, hwaddr nb,
203 uint16_t leaf)
204 {
205 /* Wildly overreserve - it doesn't matter much. */
206 phys_map_node_reserve(3 * P_L2_LEVELS);
207
208 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
209 }
210
211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
212 Node *nodes, MemoryRegionSection *sections)
213 {
214 PhysPageEntry *p;
215 int i;
216
217 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
218 if (lp.ptr == PHYS_MAP_NODE_NIL) {
219 return &sections[PHYS_SECTION_UNASSIGNED];
220 }
221 p = nodes[lp.ptr];
222 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
223 }
224 return &sections[lp.ptr];
225 }
226
227 bool memory_region_is_unassigned(MemoryRegion *mr)
228 {
229 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
230 && mr != &io_mem_watch;
231 }
232
233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
234 hwaddr addr,
235 bool resolve_subpage)
236 {
237 MemoryRegionSection *section;
238 subpage_t *subpage;
239
240 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
241 d->nodes, d->sections);
242 if (resolve_subpage && section->mr->subpage) {
243 subpage = container_of(section->mr, subpage_t, iomem);
244 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
245 }
246 return section;
247 }
248
249 static MemoryRegionSection *
250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
251 hwaddr *plen, bool resolve_subpage)
252 {
253 MemoryRegionSection *section;
254 Int128 diff;
255
256 section = address_space_lookup_region(d, addr, resolve_subpage);
257 /* Compute offset within MemoryRegionSection */
258 addr -= section->offset_within_address_space;
259
260 /* Compute offset within MemoryRegion */
261 *xlat = addr + section->offset_within_region;
262
263 diff = int128_sub(section->mr->size, int128_make64(addr));
264 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
265 return section;
266 }
267
268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
269 hwaddr *xlat, hwaddr *plen,
270 bool is_write)
271 {
272 IOMMUTLBEntry iotlb;
273 MemoryRegionSection *section;
274 MemoryRegion *mr;
275 hwaddr len = *plen;
276
277 for (;;) {
278 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
279 mr = section->mr;
280
281 if (!mr->iommu_ops) {
282 break;
283 }
284
285 iotlb = mr->iommu_ops->translate(mr, addr);
286 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
287 | (addr & iotlb.addr_mask));
288 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
289 if (!(iotlb.perm & (1 << is_write))) {
290 mr = &io_mem_unassigned;
291 break;
292 }
293
294 as = iotlb.target_as;
295 }
296
297 *plen = len;
298 *xlat = addr;
299 return mr;
300 }
301
302 MemoryRegionSection *
303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
304 hwaddr *plen)
305 {
306 MemoryRegionSection *section;
307 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
308
309 assert(!section->mr->iommu_ops);
310 return section;
311 }
312 #endif
313
314 void cpu_exec_init_all(void)
315 {
316 #if !defined(CONFIG_USER_ONLY)
317 qemu_mutex_init(&ram_list.mutex);
318 memory_map_init();
319 io_mem_init();
320 #endif
321 }
322
323 #if !defined(CONFIG_USER_ONLY)
324
325 static int cpu_common_post_load(void *opaque, int version_id)
326 {
327 CPUState *cpu = opaque;
328
329 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
330 version_id is increased. */
331 cpu->interrupt_request &= ~0x01;
332 tlb_flush(cpu->env_ptr, 1);
333
334 return 0;
335 }
336
337 const VMStateDescription vmstate_cpu_common = {
338 .name = "cpu_common",
339 .version_id = 1,
340 .minimum_version_id = 1,
341 .minimum_version_id_old = 1,
342 .post_load = cpu_common_post_load,
343 .fields = (VMStateField []) {
344 VMSTATE_UINT32(halted, CPUState),
345 VMSTATE_UINT32(interrupt_request, CPUState),
346 VMSTATE_END_OF_LIST()
347 }
348 };
349
350 #endif
351
352 CPUState *qemu_get_cpu(int index)
353 {
354 CPUState *cpu = first_cpu;
355
356 while (cpu) {
357 if (cpu->cpu_index == index) {
358 break;
359 }
360 cpu = cpu->next_cpu;
361 }
362
363 return cpu;
364 }
365
366 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
367 {
368 CPUState *cpu;
369
370 cpu = first_cpu;
371 while (cpu) {
372 func(cpu, data);
373 cpu = cpu->next_cpu;
374 }
375 }
376
377 void cpu_exec_init(CPUArchState *env)
378 {
379 CPUState *cpu = ENV_GET_CPU(env);
380 CPUClass *cc = CPU_GET_CLASS(cpu);
381 CPUState **pcpu;
382 int cpu_index;
383
384 #if defined(CONFIG_USER_ONLY)
385 cpu_list_lock();
386 #endif
387 cpu->next_cpu = NULL;
388 pcpu = &first_cpu;
389 cpu_index = 0;
390 while (*pcpu != NULL) {
391 pcpu = &(*pcpu)->next_cpu;
392 cpu_index++;
393 }
394 cpu->cpu_index = cpu_index;
395 cpu->numa_node = 0;
396 QTAILQ_INIT(&env->breakpoints);
397 QTAILQ_INIT(&env->watchpoints);
398 #ifndef CONFIG_USER_ONLY
399 cpu->thread_id = qemu_get_thread_id();
400 #endif
401 *pcpu = cpu;
402 #if defined(CONFIG_USER_ONLY)
403 cpu_list_unlock();
404 #endif
405 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
406 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
407 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
408 cpu_save, cpu_load, env);
409 assert(cc->vmsd == NULL);
410 #endif
411 if (cc->vmsd != NULL) {
412 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
413 }
414 }
415
416 #if defined(TARGET_HAS_ICE)
417 #if defined(CONFIG_USER_ONLY)
418 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
419 {
420 tb_invalidate_phys_page_range(pc, pc + 1, 0);
421 }
422 #else
423 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
424 {
425 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
426 (pc & ~TARGET_PAGE_MASK));
427 }
428 #endif
429 #endif /* TARGET_HAS_ICE */
430
431 #if defined(CONFIG_USER_ONLY)
432 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
433
434 {
435 }
436
437 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
438 int flags, CPUWatchpoint **watchpoint)
439 {
440 return -ENOSYS;
441 }
442 #else
443 /* Add a watchpoint. */
444 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
445 int flags, CPUWatchpoint **watchpoint)
446 {
447 target_ulong len_mask = ~(len - 1);
448 CPUWatchpoint *wp;
449
450 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
451 if ((len & (len - 1)) || (addr & ~len_mask) ||
452 len == 0 || len > TARGET_PAGE_SIZE) {
453 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
454 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
455 return -EINVAL;
456 }
457 wp = g_malloc(sizeof(*wp));
458
459 wp->vaddr = addr;
460 wp->len_mask = len_mask;
461 wp->flags = flags;
462
463 /* keep all GDB-injected watchpoints in front */
464 if (flags & BP_GDB)
465 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
466 else
467 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
468
469 tlb_flush_page(env, addr);
470
471 if (watchpoint)
472 *watchpoint = wp;
473 return 0;
474 }
475
476 /* Remove a specific watchpoint. */
477 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
478 int flags)
479 {
480 target_ulong len_mask = ~(len - 1);
481 CPUWatchpoint *wp;
482
483 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
484 if (addr == wp->vaddr && len_mask == wp->len_mask
485 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
486 cpu_watchpoint_remove_by_ref(env, wp);
487 return 0;
488 }
489 }
490 return -ENOENT;
491 }
492
493 /* Remove a specific watchpoint by reference. */
494 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
495 {
496 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
497
498 tlb_flush_page(env, watchpoint->vaddr);
499
500 g_free(watchpoint);
501 }
502
503 /* Remove all matching watchpoints. */
504 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
505 {
506 CPUWatchpoint *wp, *next;
507
508 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
509 if (wp->flags & mask)
510 cpu_watchpoint_remove_by_ref(env, wp);
511 }
512 }
513 #endif
514
515 /* Add a breakpoint. */
516 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
517 CPUBreakpoint **breakpoint)
518 {
519 #if defined(TARGET_HAS_ICE)
520 CPUBreakpoint *bp;
521
522 bp = g_malloc(sizeof(*bp));
523
524 bp->pc = pc;
525 bp->flags = flags;
526
527 /* keep all GDB-injected breakpoints in front */
528 if (flags & BP_GDB)
529 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
530 else
531 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
532
533 breakpoint_invalidate(env, pc);
534
535 if (breakpoint)
536 *breakpoint = bp;
537 return 0;
538 #else
539 return -ENOSYS;
540 #endif
541 }
542
543 /* Remove a specific breakpoint. */
544 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
545 {
546 #if defined(TARGET_HAS_ICE)
547 CPUBreakpoint *bp;
548
549 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
550 if (bp->pc == pc && bp->flags == flags) {
551 cpu_breakpoint_remove_by_ref(env, bp);
552 return 0;
553 }
554 }
555 return -ENOENT;
556 #else
557 return -ENOSYS;
558 #endif
559 }
560
561 /* Remove a specific breakpoint by reference. */
562 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
563 {
564 #if defined(TARGET_HAS_ICE)
565 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
566
567 breakpoint_invalidate(env, breakpoint->pc);
568
569 g_free(breakpoint);
570 #endif
571 }
572
573 /* Remove all matching breakpoints. */
574 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
575 {
576 #if defined(TARGET_HAS_ICE)
577 CPUBreakpoint *bp, *next;
578
579 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
580 if (bp->flags & mask)
581 cpu_breakpoint_remove_by_ref(env, bp);
582 }
583 #endif
584 }
585
586 /* enable or disable single step mode. EXCP_DEBUG is returned by the
587 CPU loop after each instruction */
588 void cpu_single_step(CPUArchState *env, int enabled)
589 {
590 #if defined(TARGET_HAS_ICE)
591 if (env->singlestep_enabled != enabled) {
592 env->singlestep_enabled = enabled;
593 if (kvm_enabled())
594 kvm_update_guest_debug(env, 0);
595 else {
596 /* must flush all the translated code to avoid inconsistencies */
597 /* XXX: only flush what is necessary */
598 tb_flush(env);
599 }
600 }
601 #endif
602 }
603
604 void cpu_abort(CPUArchState *env, const char *fmt, ...)
605 {
606 CPUState *cpu = ENV_GET_CPU(env);
607 va_list ap;
608 va_list ap2;
609
610 va_start(ap, fmt);
611 va_copy(ap2, ap);
612 fprintf(stderr, "qemu: fatal: ");
613 vfprintf(stderr, fmt, ap);
614 fprintf(stderr, "\n");
615 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
616 if (qemu_log_enabled()) {
617 qemu_log("qemu: fatal: ");
618 qemu_log_vprintf(fmt, ap2);
619 qemu_log("\n");
620 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
621 qemu_log_flush();
622 qemu_log_close();
623 }
624 va_end(ap2);
625 va_end(ap);
626 #if defined(CONFIG_USER_ONLY)
627 {
628 struct sigaction act;
629 sigfillset(&act.sa_mask);
630 act.sa_handler = SIG_DFL;
631 sigaction(SIGABRT, &act, NULL);
632 }
633 #endif
634 abort();
635 }
636
637 CPUArchState *cpu_copy(CPUArchState *env)
638 {
639 CPUArchState *new_env = cpu_init(env->cpu_model_str);
640 #if defined(TARGET_HAS_ICE)
641 CPUBreakpoint *bp;
642 CPUWatchpoint *wp;
643 #endif
644
645 memcpy(new_env, env, sizeof(CPUArchState));
646
647 /* Clone all break/watchpoints.
648 Note: Once we support ptrace with hw-debug register access, make sure
649 BP_CPU break/watchpoints are handled correctly on clone. */
650 QTAILQ_INIT(&env->breakpoints);
651 QTAILQ_INIT(&env->watchpoints);
652 #if defined(TARGET_HAS_ICE)
653 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
654 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
655 }
656 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
657 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
658 wp->flags, NULL);
659 }
660 #endif
661
662 return new_env;
663 }
664
665 #if !defined(CONFIG_USER_ONLY)
666 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
667 uintptr_t length)
668 {
669 uintptr_t start1;
670
671 /* we modify the TLB cache so that the dirty bit will be set again
672 when accessing the range */
673 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
674 /* Check that we don't span multiple blocks - this breaks the
675 address comparisons below. */
676 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
677 != (end - 1) - start) {
678 abort();
679 }
680 cpu_tlb_reset_dirty_all(start1, length);
681
682 }
683
684 /* Note: start and end must be within the same ram block. */
685 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
686 int dirty_flags)
687 {
688 uintptr_t length;
689
690 start &= TARGET_PAGE_MASK;
691 end = TARGET_PAGE_ALIGN(end);
692
693 length = end - start;
694 if (length == 0)
695 return;
696 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
697
698 if (tcg_enabled()) {
699 tlb_reset_dirty_range_all(start, end, length);
700 }
701 }
702
703 static int cpu_physical_memory_set_dirty_tracking(int enable)
704 {
705 int ret = 0;
706 in_migration = enable;
707 return ret;
708 }
709
710 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
711 MemoryRegionSection *section,
712 target_ulong vaddr,
713 hwaddr paddr, hwaddr xlat,
714 int prot,
715 target_ulong *address)
716 {
717 hwaddr iotlb;
718 CPUWatchpoint *wp;
719
720 if (memory_region_is_ram(section->mr)) {
721 /* Normal RAM. */
722 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
723 + xlat;
724 if (!section->readonly) {
725 iotlb |= PHYS_SECTION_NOTDIRTY;
726 } else {
727 iotlb |= PHYS_SECTION_ROM;
728 }
729 } else {
730 iotlb = section - address_space_memory.dispatch->sections;
731 iotlb += xlat;
732 }
733
734 /* Make accesses to pages with watchpoints go via the
735 watchpoint trap routines. */
736 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
737 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
738 /* Avoid trapping reads of pages with a write breakpoint. */
739 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
740 iotlb = PHYS_SECTION_WATCH + paddr;
741 *address |= TLB_MMIO;
742 break;
743 }
744 }
745 }
746
747 return iotlb;
748 }
749 #endif /* defined(CONFIG_USER_ONLY) */
750
751 #if !defined(CONFIG_USER_ONLY)
752
753 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
754 uint16_t section);
755 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
756
757 static uint16_t phys_section_add(MemoryRegionSection *section)
758 {
759 /* The physical section number is ORed with a page-aligned
760 * pointer to produce the iotlb entries. Thus it should
761 * never overflow into the page-aligned value.
762 */
763 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
764
765 if (next_map.sections_nb == next_map.sections_nb_alloc) {
766 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
767 16);
768 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
769 next_map.sections_nb_alloc);
770 }
771 next_map.sections[next_map.sections_nb] = *section;
772 memory_region_ref(section->mr);
773 return next_map.sections_nb++;
774 }
775
776 static void phys_section_destroy(MemoryRegion *mr)
777 {
778 memory_region_unref(mr);
779
780 if (mr->subpage) {
781 subpage_t *subpage = container_of(mr, subpage_t, iomem);
782 memory_region_destroy(&subpage->iomem);
783 g_free(subpage);
784 }
785 }
786
787 static void phys_sections_free(PhysPageMap *map)
788 {
789 while (map->sections_nb > 0) {
790 MemoryRegionSection *section = &map->sections[--map->sections_nb];
791 phys_section_destroy(section->mr);
792 }
793 g_free(map->sections);
794 g_free(map->nodes);
795 g_free(map);
796 }
797
798 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
799 {
800 subpage_t *subpage;
801 hwaddr base = section->offset_within_address_space
802 & TARGET_PAGE_MASK;
803 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
804 next_map.nodes, next_map.sections);
805 MemoryRegionSection subsection = {
806 .offset_within_address_space = base,
807 .size = int128_make64(TARGET_PAGE_SIZE),
808 };
809 hwaddr start, end;
810
811 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
812
813 if (!(existing->mr->subpage)) {
814 subpage = subpage_init(d->as, base);
815 subsection.mr = &subpage->iomem;
816 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
817 phys_section_add(&subsection));
818 } else {
819 subpage = container_of(existing->mr, subpage_t, iomem);
820 }
821 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
822 end = start + int128_get64(section->size) - 1;
823 subpage_register(subpage, start, end, phys_section_add(section));
824 }
825
826
827 static void register_multipage(AddressSpaceDispatch *d,
828 MemoryRegionSection *section)
829 {
830 hwaddr start_addr = section->offset_within_address_space;
831 uint16_t section_index = phys_section_add(section);
832 uint64_t num_pages = int128_get64(int128_rshift(section->size,
833 TARGET_PAGE_BITS));
834
835 assert(num_pages);
836 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
837 }
838
839 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
840 {
841 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
842 AddressSpaceDispatch *d = as->next_dispatch;
843 MemoryRegionSection now = *section, remain = *section;
844 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
845
846 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
847 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
848 - now.offset_within_address_space;
849
850 now.size = int128_min(int128_make64(left), now.size);
851 register_subpage(d, &now);
852 } else {
853 now.size = int128_zero();
854 }
855 while (int128_ne(remain.size, now.size)) {
856 remain.size = int128_sub(remain.size, now.size);
857 remain.offset_within_address_space += int128_get64(now.size);
858 remain.offset_within_region += int128_get64(now.size);
859 now = remain;
860 if (int128_lt(remain.size, page_size)) {
861 register_subpage(d, &now);
862 } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
863 now.size = page_size;
864 register_subpage(d, &now);
865 } else {
866 now.size = int128_and(now.size, int128_neg(page_size));
867 register_multipage(d, &now);
868 }
869 }
870 }
871
872 void qemu_flush_coalesced_mmio_buffer(void)
873 {
874 if (kvm_enabled())
875 kvm_flush_coalesced_mmio_buffer();
876 }
877
878 void qemu_mutex_lock_ramlist(void)
879 {
880 qemu_mutex_lock(&ram_list.mutex);
881 }
882
883 void qemu_mutex_unlock_ramlist(void)
884 {
885 qemu_mutex_unlock(&ram_list.mutex);
886 }
887
888 #if defined(__linux__) && !defined(TARGET_S390X)
889
890 #include <sys/vfs.h>
891
892 #define HUGETLBFS_MAGIC 0x958458f6
893
894 static long gethugepagesize(const char *path)
895 {
896 struct statfs fs;
897 int ret;
898
899 do {
900 ret = statfs(path, &fs);
901 } while (ret != 0 && errno == EINTR);
902
903 if (ret != 0) {
904 perror(path);
905 return 0;
906 }
907
908 if (fs.f_type != HUGETLBFS_MAGIC)
909 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
910
911 return fs.f_bsize;
912 }
913
914 static void *file_ram_alloc(RAMBlock *block,
915 ram_addr_t memory,
916 const char *path)
917 {
918 char *filename;
919 char *sanitized_name;
920 char *c;
921 void *area;
922 int fd;
923 #ifdef MAP_POPULATE
924 int flags;
925 #endif
926 unsigned long hpagesize;
927
928 hpagesize = gethugepagesize(path);
929 if (!hpagesize) {
930 return NULL;
931 }
932
933 if (memory < hpagesize) {
934 return NULL;
935 }
936
937 if (kvm_enabled() && !kvm_has_sync_mmu()) {
938 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
939 return NULL;
940 }
941
942 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
943 sanitized_name = g_strdup(block->mr->name);
944 for (c = sanitized_name; *c != '\0'; c++) {
945 if (*c == '/')
946 *c = '_';
947 }
948
949 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
950 sanitized_name);
951 g_free(sanitized_name);
952
953 fd = mkstemp(filename);
954 if (fd < 0) {
955 perror("unable to create backing store for hugepages");
956 g_free(filename);
957 return NULL;
958 }
959 unlink(filename);
960 g_free(filename);
961
962 memory = (memory+hpagesize-1) & ~(hpagesize-1);
963
964 /*
965 * ftruncate is not supported by hugetlbfs in older
966 * hosts, so don't bother bailing out on errors.
967 * If anything goes wrong with it under other filesystems,
968 * mmap will fail.
969 */
970 if (ftruncate(fd, memory))
971 perror("ftruncate");
972
973 #ifdef MAP_POPULATE
974 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
975 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
976 * to sidestep this quirk.
977 */
978 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
979 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
980 #else
981 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
982 #endif
983 if (area == MAP_FAILED) {
984 perror("file_ram_alloc: can't mmap RAM pages");
985 close(fd);
986 return (NULL);
987 }
988 block->fd = fd;
989 return area;
990 }
991 #endif
992
993 static ram_addr_t find_ram_offset(ram_addr_t size)
994 {
995 RAMBlock *block, *next_block;
996 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
997
998 assert(size != 0); /* it would hand out same offset multiple times */
999
1000 if (QTAILQ_EMPTY(&ram_list.blocks))
1001 return 0;
1002
1003 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1004 ram_addr_t end, next = RAM_ADDR_MAX;
1005
1006 end = block->offset + block->length;
1007
1008 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1009 if (next_block->offset >= end) {
1010 next = MIN(next, next_block->offset);
1011 }
1012 }
1013 if (next - end >= size && next - end < mingap) {
1014 offset = end;
1015 mingap = next - end;
1016 }
1017 }
1018
1019 if (offset == RAM_ADDR_MAX) {
1020 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1021 (uint64_t)size);
1022 abort();
1023 }
1024
1025 return offset;
1026 }
1027
1028 ram_addr_t last_ram_offset(void)
1029 {
1030 RAMBlock *block;
1031 ram_addr_t last = 0;
1032
1033 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1034 last = MAX(last, block->offset + block->length);
1035
1036 return last;
1037 }
1038
1039 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1040 {
1041 int ret;
1042
1043 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1044 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1045 "dump-guest-core", true)) {
1046 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1047 if (ret) {
1048 perror("qemu_madvise");
1049 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1050 "but dump_guest_core=off specified\n");
1051 }
1052 }
1053 }
1054
1055 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1056 {
1057 RAMBlock *new_block, *block;
1058
1059 new_block = NULL;
1060 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1061 if (block->offset == addr) {
1062 new_block = block;
1063 break;
1064 }
1065 }
1066 assert(new_block);
1067 assert(!new_block->idstr[0]);
1068
1069 if (dev) {
1070 char *id = qdev_get_dev_path(dev);
1071 if (id) {
1072 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1073 g_free(id);
1074 }
1075 }
1076 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1077
1078 /* This assumes the iothread lock is taken here too. */
1079 qemu_mutex_lock_ramlist();
1080 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1081 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1082 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1083 new_block->idstr);
1084 abort();
1085 }
1086 }
1087 qemu_mutex_unlock_ramlist();
1088 }
1089
1090 static int memory_try_enable_merging(void *addr, size_t len)
1091 {
1092 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1093 /* disabled by the user */
1094 return 0;
1095 }
1096
1097 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1098 }
1099
1100 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1101 MemoryRegion *mr)
1102 {
1103 RAMBlock *block, *new_block;
1104
1105 size = TARGET_PAGE_ALIGN(size);
1106 new_block = g_malloc0(sizeof(*new_block));
1107
1108 /* This assumes the iothread lock is taken here too. */
1109 qemu_mutex_lock_ramlist();
1110 new_block->mr = mr;
1111 new_block->offset = find_ram_offset(size);
1112 if (host) {
1113 new_block->host = host;
1114 new_block->flags |= RAM_PREALLOC_MASK;
1115 } else {
1116 if (mem_path) {
1117 #if defined (__linux__) && !defined(TARGET_S390X)
1118 new_block->host = file_ram_alloc(new_block, size, mem_path);
1119 if (!new_block->host) {
1120 new_block->host = qemu_anon_ram_alloc(size);
1121 memory_try_enable_merging(new_block->host, size);
1122 }
1123 #else
1124 fprintf(stderr, "-mem-path option unsupported\n");
1125 exit(1);
1126 #endif
1127 } else {
1128 if (xen_enabled()) {
1129 xen_ram_alloc(new_block->offset, size, mr);
1130 } else if (kvm_enabled()) {
1131 /* some s390/kvm configurations have special constraints */
1132 new_block->host = kvm_ram_alloc(size);
1133 } else {
1134 new_block->host = qemu_anon_ram_alloc(size);
1135 }
1136 memory_try_enable_merging(new_block->host, size);
1137 }
1138 }
1139 new_block->length = size;
1140
1141 /* Keep the list sorted from biggest to smallest block. */
1142 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1143 if (block->length < new_block->length) {
1144 break;
1145 }
1146 }
1147 if (block) {
1148 QTAILQ_INSERT_BEFORE(block, new_block, next);
1149 } else {
1150 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1151 }
1152 ram_list.mru_block = NULL;
1153
1154 ram_list.version++;
1155 qemu_mutex_unlock_ramlist();
1156
1157 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1158 last_ram_offset() >> TARGET_PAGE_BITS);
1159 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1160 0, size >> TARGET_PAGE_BITS);
1161 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1162
1163 qemu_ram_setup_dump(new_block->host, size);
1164 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1165
1166 if (kvm_enabled())
1167 kvm_setup_guest_memory(new_block->host, size);
1168
1169 return new_block->offset;
1170 }
1171
1172 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1173 {
1174 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1175 }
1176
1177 void qemu_ram_free_from_ptr(ram_addr_t addr)
1178 {
1179 RAMBlock *block;
1180
1181 /* This assumes the iothread lock is taken here too. */
1182 qemu_mutex_lock_ramlist();
1183 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1184 if (addr == block->offset) {
1185 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1186 ram_list.mru_block = NULL;
1187 ram_list.version++;
1188 g_free(block);
1189 break;
1190 }
1191 }
1192 qemu_mutex_unlock_ramlist();
1193 }
1194
1195 void qemu_ram_free(ram_addr_t addr)
1196 {
1197 RAMBlock *block;
1198
1199 /* This assumes the iothread lock is taken here too. */
1200 qemu_mutex_lock_ramlist();
1201 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1202 if (addr == block->offset) {
1203 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1204 ram_list.mru_block = NULL;
1205 ram_list.version++;
1206 if (block->flags & RAM_PREALLOC_MASK) {
1207 ;
1208 } else if (mem_path) {
1209 #if defined (__linux__) && !defined(TARGET_S390X)
1210 if (block->fd) {
1211 munmap(block->host, block->length);
1212 close(block->fd);
1213 } else {
1214 qemu_anon_ram_free(block->host, block->length);
1215 }
1216 #else
1217 abort();
1218 #endif
1219 } else {
1220 if (xen_enabled()) {
1221 xen_invalidate_map_cache_entry(block->host);
1222 } else {
1223 qemu_anon_ram_free(block->host, block->length);
1224 }
1225 }
1226 g_free(block);
1227 break;
1228 }
1229 }
1230 qemu_mutex_unlock_ramlist();
1231
1232 }
1233
1234 #ifndef _WIN32
1235 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1236 {
1237 RAMBlock *block;
1238 ram_addr_t offset;
1239 int flags;
1240 void *area, *vaddr;
1241
1242 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1243 offset = addr - block->offset;
1244 if (offset < block->length) {
1245 vaddr = block->host + offset;
1246 if (block->flags & RAM_PREALLOC_MASK) {
1247 ;
1248 } else {
1249 flags = MAP_FIXED;
1250 munmap(vaddr, length);
1251 if (mem_path) {
1252 #if defined(__linux__) && !defined(TARGET_S390X)
1253 if (block->fd) {
1254 #ifdef MAP_POPULATE
1255 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1256 MAP_PRIVATE;
1257 #else
1258 flags |= MAP_PRIVATE;
1259 #endif
1260 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1261 flags, block->fd, offset);
1262 } else {
1263 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1264 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1265 flags, -1, 0);
1266 }
1267 #else
1268 abort();
1269 #endif
1270 } else {
1271 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1272 flags |= MAP_SHARED | MAP_ANONYMOUS;
1273 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1274 flags, -1, 0);
1275 #else
1276 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1277 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1278 flags, -1, 0);
1279 #endif
1280 }
1281 if (area != vaddr) {
1282 fprintf(stderr, "Could not remap addr: "
1283 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1284 length, addr);
1285 exit(1);
1286 }
1287 memory_try_enable_merging(vaddr, length);
1288 qemu_ram_setup_dump(vaddr, length);
1289 }
1290 return;
1291 }
1292 }
1293 }
1294 #endif /* !_WIN32 */
1295
1296 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1297 {
1298 RAMBlock *block;
1299
1300 /* The list is protected by the iothread lock here. */
1301 block = ram_list.mru_block;
1302 if (block && addr - block->offset < block->length) {
1303 goto found;
1304 }
1305 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1306 if (addr - block->offset < block->length) {
1307 goto found;
1308 }
1309 }
1310
1311 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1312 abort();
1313
1314 found:
1315 ram_list.mru_block = block;
1316 return block;
1317 }
1318
1319 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1320 With the exception of the softmmu code in this file, this should
1321 only be used for local memory (e.g. video ram) that the device owns,
1322 and knows it isn't going to access beyond the end of the block.
1323
1324 It should not be used for general purpose DMA.
1325 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1326 */
1327 void *qemu_get_ram_ptr(ram_addr_t addr)
1328 {
1329 RAMBlock *block = qemu_get_ram_block(addr);
1330
1331 if (xen_enabled()) {
1332 /* We need to check if the requested address is in the RAM
1333 * because we don't want to map the entire memory in QEMU.
1334 * In that case just map until the end of the page.
1335 */
1336 if (block->offset == 0) {
1337 return xen_map_cache(addr, 0, 0);
1338 } else if (block->host == NULL) {
1339 block->host =
1340 xen_map_cache(block->offset, block->length, 1);
1341 }
1342 }
1343 return block->host + (addr - block->offset);
1344 }
1345
1346 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1347 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1348 *
1349 * ??? Is this still necessary?
1350 */
1351 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1352 {
1353 RAMBlock *block;
1354
1355 /* The list is protected by the iothread lock here. */
1356 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1357 if (addr - block->offset < block->length) {
1358 if (xen_enabled()) {
1359 /* We need to check if the requested address is in the RAM
1360 * because we don't want to map the entire memory in QEMU.
1361 * In that case just map until the end of the page.
1362 */
1363 if (block->offset == 0) {
1364 return xen_map_cache(addr, 0, 0);
1365 } else if (block->host == NULL) {
1366 block->host =
1367 xen_map_cache(block->offset, block->length, 1);
1368 }
1369 }
1370 return block->host + (addr - block->offset);
1371 }
1372 }
1373
1374 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1375 abort();
1376
1377 return NULL;
1378 }
1379
1380 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1381 * but takes a size argument */
1382 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1383 {
1384 if (*size == 0) {
1385 return NULL;
1386 }
1387 if (xen_enabled()) {
1388 return xen_map_cache(addr, *size, 1);
1389 } else {
1390 RAMBlock *block;
1391
1392 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1393 if (addr - block->offset < block->length) {
1394 if (addr - block->offset + *size > block->length)
1395 *size = block->length - addr + block->offset;
1396 return block->host + (addr - block->offset);
1397 }
1398 }
1399
1400 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1401 abort();
1402 }
1403 }
1404
1405 /* Some of the softmmu routines need to translate from a host pointer
1406 (typically a TLB entry) back to a ram offset. */
1407 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1408 {
1409 RAMBlock *block;
1410 uint8_t *host = ptr;
1411
1412 if (xen_enabled()) {
1413 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1414 return qemu_get_ram_block(*ram_addr)->mr;
1415 }
1416
1417 block = ram_list.mru_block;
1418 if (block && block->host && host - block->host < block->length) {
1419 goto found;
1420 }
1421
1422 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1423 /* This case append when the block is not mapped. */
1424 if (block->host == NULL) {
1425 continue;
1426 }
1427 if (host - block->host < block->length) {
1428 goto found;
1429 }
1430 }
1431
1432 return NULL;
1433
1434 found:
1435 *ram_addr = block->offset + (host - block->host);
1436 return block->mr;
1437 }
1438
1439 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1440 uint64_t val, unsigned size)
1441 {
1442 int dirty_flags;
1443 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1444 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1445 tb_invalidate_phys_page_fast(ram_addr, size);
1446 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1447 }
1448 switch (size) {
1449 case 1:
1450 stb_p(qemu_get_ram_ptr(ram_addr), val);
1451 break;
1452 case 2:
1453 stw_p(qemu_get_ram_ptr(ram_addr), val);
1454 break;
1455 case 4:
1456 stl_p(qemu_get_ram_ptr(ram_addr), val);
1457 break;
1458 default:
1459 abort();
1460 }
1461 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1462 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1463 /* we remove the notdirty callback only if the code has been
1464 flushed */
1465 if (dirty_flags == 0xff) {
1466 CPUArchState *env = current_cpu->env_ptr;
1467 tlb_set_dirty(env, env->mem_io_vaddr);
1468 }
1469 }
1470
1471 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1472 unsigned size, bool is_write)
1473 {
1474 return is_write;
1475 }
1476
1477 static const MemoryRegionOps notdirty_mem_ops = {
1478 .write = notdirty_mem_write,
1479 .valid.accepts = notdirty_mem_accepts,
1480 .endianness = DEVICE_NATIVE_ENDIAN,
1481 };
1482
1483 /* Generate a debug exception if a watchpoint has been hit. */
1484 static void check_watchpoint(int offset, int len_mask, int flags)
1485 {
1486 CPUArchState *env = current_cpu->env_ptr;
1487 target_ulong pc, cs_base;
1488 target_ulong vaddr;
1489 CPUWatchpoint *wp;
1490 int cpu_flags;
1491
1492 if (env->watchpoint_hit) {
1493 /* We re-entered the check after replacing the TB. Now raise
1494 * the debug interrupt so that is will trigger after the
1495 * current instruction. */
1496 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1497 return;
1498 }
1499 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1500 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1501 if ((vaddr == (wp->vaddr & len_mask) ||
1502 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1503 wp->flags |= BP_WATCHPOINT_HIT;
1504 if (!env->watchpoint_hit) {
1505 env->watchpoint_hit = wp;
1506 tb_check_watchpoint(env);
1507 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1508 env->exception_index = EXCP_DEBUG;
1509 cpu_loop_exit(env);
1510 } else {
1511 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1512 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1513 cpu_resume_from_signal(env, NULL);
1514 }
1515 }
1516 } else {
1517 wp->flags &= ~BP_WATCHPOINT_HIT;
1518 }
1519 }
1520 }
1521
1522 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1523 so these check for a hit then pass through to the normal out-of-line
1524 phys routines. */
1525 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1526 unsigned size)
1527 {
1528 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1529 switch (size) {
1530 case 1: return ldub_phys(addr);
1531 case 2: return lduw_phys(addr);
1532 case 4: return ldl_phys(addr);
1533 default: abort();
1534 }
1535 }
1536
1537 static void watch_mem_write(void *opaque, hwaddr addr,
1538 uint64_t val, unsigned size)
1539 {
1540 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1541 switch (size) {
1542 case 1:
1543 stb_phys(addr, val);
1544 break;
1545 case 2:
1546 stw_phys(addr, val);
1547 break;
1548 case 4:
1549 stl_phys(addr, val);
1550 break;
1551 default: abort();
1552 }
1553 }
1554
1555 static const MemoryRegionOps watch_mem_ops = {
1556 .read = watch_mem_read,
1557 .write = watch_mem_write,
1558 .endianness = DEVICE_NATIVE_ENDIAN,
1559 };
1560
1561 static uint64_t subpage_read(void *opaque, hwaddr addr,
1562 unsigned len)
1563 {
1564 subpage_t *subpage = opaque;
1565 uint8_t buf[4];
1566
1567 #if defined(DEBUG_SUBPAGE)
1568 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1569 subpage, len, addr);
1570 #endif
1571 address_space_read(subpage->as, addr + subpage->base, buf, len);
1572 switch (len) {
1573 case 1:
1574 return ldub_p(buf);
1575 case 2:
1576 return lduw_p(buf);
1577 case 4:
1578 return ldl_p(buf);
1579 default:
1580 abort();
1581 }
1582 }
1583
1584 static void subpage_write(void *opaque, hwaddr addr,
1585 uint64_t value, unsigned len)
1586 {
1587 subpage_t *subpage = opaque;
1588 uint8_t buf[4];
1589
1590 #if defined(DEBUG_SUBPAGE)
1591 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1592 " value %"PRIx64"\n",
1593 __func__, subpage, len, addr, value);
1594 #endif
1595 switch (len) {
1596 case 1:
1597 stb_p(buf, value);
1598 break;
1599 case 2:
1600 stw_p(buf, value);
1601 break;
1602 case 4:
1603 stl_p(buf, value);
1604 break;
1605 default:
1606 abort();
1607 }
1608 address_space_write(subpage->as, addr + subpage->base, buf, len);
1609 }
1610
1611 static bool subpage_accepts(void *opaque, hwaddr addr,
1612 unsigned size, bool is_write)
1613 {
1614 subpage_t *subpage = opaque;
1615 #if defined(DEBUG_SUBPAGE)
1616 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1617 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1618 #endif
1619
1620 return address_space_access_valid(subpage->as, addr + subpage->base,
1621 size, is_write);
1622 }
1623
1624 static const MemoryRegionOps subpage_ops = {
1625 .read = subpage_read,
1626 .write = subpage_write,
1627 .valid.accepts = subpage_accepts,
1628 .endianness = DEVICE_NATIVE_ENDIAN,
1629 };
1630
1631 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1632 uint16_t section)
1633 {
1634 int idx, eidx;
1635
1636 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1637 return -1;
1638 idx = SUBPAGE_IDX(start);
1639 eidx = SUBPAGE_IDX(end);
1640 #if defined(DEBUG_SUBPAGE)
1641 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1642 mmio, start, end, idx, eidx, memory);
1643 #endif
1644 for (; idx <= eidx; idx++) {
1645 mmio->sub_section[idx] = section;
1646 }
1647
1648 return 0;
1649 }
1650
1651 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1652 {
1653 subpage_t *mmio;
1654
1655 mmio = g_malloc0(sizeof(subpage_t));
1656
1657 mmio->as = as;
1658 mmio->base = base;
1659 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1660 "subpage", TARGET_PAGE_SIZE);
1661 mmio->iomem.subpage = true;
1662 #if defined(DEBUG_SUBPAGE)
1663 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1664 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1665 #endif
1666 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1667
1668 return mmio;
1669 }
1670
1671 static uint16_t dummy_section(MemoryRegion *mr)
1672 {
1673 MemoryRegionSection section = {
1674 .mr = mr,
1675 .offset_within_address_space = 0,
1676 .offset_within_region = 0,
1677 .size = int128_2_64(),
1678 };
1679
1680 return phys_section_add(&section);
1681 }
1682
1683 MemoryRegion *iotlb_to_region(hwaddr index)
1684 {
1685 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1686 }
1687
1688 static void io_mem_init(void)
1689 {
1690 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1691 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1692 "unassigned", UINT64_MAX);
1693 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1694 "notdirty", UINT64_MAX);
1695 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1696 "watch", UINT64_MAX);
1697 }
1698
1699 static void mem_begin(MemoryListener *listener)
1700 {
1701 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1702 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1703
1704 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1705 d->as = as;
1706 as->next_dispatch = d;
1707 }
1708
1709 static void mem_commit(MemoryListener *listener)
1710 {
1711 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1712 AddressSpaceDispatch *cur = as->dispatch;
1713 AddressSpaceDispatch *next = as->next_dispatch;
1714
1715 next->nodes = next_map.nodes;
1716 next->sections = next_map.sections;
1717
1718 as->dispatch = next;
1719 g_free(cur);
1720 }
1721
1722 static void core_begin(MemoryListener *listener)
1723 {
1724 uint16_t n;
1725
1726 prev_map = g_new(PhysPageMap, 1);
1727 *prev_map = next_map;
1728
1729 memset(&next_map, 0, sizeof(next_map));
1730 n = dummy_section(&io_mem_unassigned);
1731 assert(n == PHYS_SECTION_UNASSIGNED);
1732 n = dummy_section(&io_mem_notdirty);
1733 assert(n == PHYS_SECTION_NOTDIRTY);
1734 n = dummy_section(&io_mem_rom);
1735 assert(n == PHYS_SECTION_ROM);
1736 n = dummy_section(&io_mem_watch);
1737 assert(n == PHYS_SECTION_WATCH);
1738 }
1739
1740 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1741 * All AddressSpaceDispatch instances have switched to the next map.
1742 */
1743 static void core_commit(MemoryListener *listener)
1744 {
1745 phys_sections_free(prev_map);
1746 }
1747
1748 static void tcg_commit(MemoryListener *listener)
1749 {
1750 CPUState *cpu;
1751
1752 /* since each CPU stores ram addresses in its TLB cache, we must
1753 reset the modified entries */
1754 /* XXX: slow ! */
1755 for (cpu = first_cpu; cpu != NULL; cpu = cpu->next_cpu) {
1756 CPUArchState *env = cpu->env_ptr;
1757
1758 tlb_flush(env, 1);
1759 }
1760 }
1761
1762 static void core_log_global_start(MemoryListener *listener)
1763 {
1764 cpu_physical_memory_set_dirty_tracking(1);
1765 }
1766
1767 static void core_log_global_stop(MemoryListener *listener)
1768 {
1769 cpu_physical_memory_set_dirty_tracking(0);
1770 }
1771
1772 static MemoryListener core_memory_listener = {
1773 .begin = core_begin,
1774 .commit = core_commit,
1775 .log_global_start = core_log_global_start,
1776 .log_global_stop = core_log_global_stop,
1777 .priority = 1,
1778 };
1779
1780 static MemoryListener tcg_memory_listener = {
1781 .commit = tcg_commit,
1782 };
1783
1784 void address_space_init_dispatch(AddressSpace *as)
1785 {
1786 as->dispatch = NULL;
1787 as->dispatch_listener = (MemoryListener) {
1788 .begin = mem_begin,
1789 .commit = mem_commit,
1790 .region_add = mem_add,
1791 .region_nop = mem_add,
1792 .priority = 0,
1793 };
1794 memory_listener_register(&as->dispatch_listener, as);
1795 }
1796
1797 void address_space_destroy_dispatch(AddressSpace *as)
1798 {
1799 AddressSpaceDispatch *d = as->dispatch;
1800
1801 memory_listener_unregister(&as->dispatch_listener);
1802 g_free(d);
1803 as->dispatch = NULL;
1804 }
1805
1806 static void memory_map_init(void)
1807 {
1808 system_memory = g_malloc(sizeof(*system_memory));
1809 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1810 address_space_init(&address_space_memory, system_memory, "memory");
1811
1812 system_io = g_malloc(sizeof(*system_io));
1813 memory_region_init(system_io, NULL, "io", 65536);
1814 address_space_init(&address_space_io, system_io, "I/O");
1815
1816 memory_listener_register(&core_memory_listener, &address_space_memory);
1817 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1818 }
1819
1820 MemoryRegion *get_system_memory(void)
1821 {
1822 return system_memory;
1823 }
1824
1825 MemoryRegion *get_system_io(void)
1826 {
1827 return system_io;
1828 }
1829
1830 #endif /* !defined(CONFIG_USER_ONLY) */
1831
1832 /* physical memory access (slow version, mainly for debug) */
1833 #if defined(CONFIG_USER_ONLY)
1834 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1835 uint8_t *buf, int len, int is_write)
1836 {
1837 int l, flags;
1838 target_ulong page;
1839 void * p;
1840
1841 while (len > 0) {
1842 page = addr & TARGET_PAGE_MASK;
1843 l = (page + TARGET_PAGE_SIZE) - addr;
1844 if (l > len)
1845 l = len;
1846 flags = page_get_flags(page);
1847 if (!(flags & PAGE_VALID))
1848 return -1;
1849 if (is_write) {
1850 if (!(flags & PAGE_WRITE))
1851 return -1;
1852 /* XXX: this code should not depend on lock_user */
1853 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1854 return -1;
1855 memcpy(p, buf, l);
1856 unlock_user(p, addr, l);
1857 } else {
1858 if (!(flags & PAGE_READ))
1859 return -1;
1860 /* XXX: this code should not depend on lock_user */
1861 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1862 return -1;
1863 memcpy(buf, p, l);
1864 unlock_user(p, addr, 0);
1865 }
1866 len -= l;
1867 buf += l;
1868 addr += l;
1869 }
1870 return 0;
1871 }
1872
1873 #else
1874
1875 static void invalidate_and_set_dirty(hwaddr addr,
1876 hwaddr length)
1877 {
1878 if (!cpu_physical_memory_is_dirty(addr)) {
1879 /* invalidate code */
1880 tb_invalidate_phys_page_range(addr, addr + length, 0);
1881 /* set dirty bit */
1882 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1883 }
1884 xen_modified_memory(addr, length);
1885 }
1886
1887 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1888 {
1889 if (memory_region_is_ram(mr)) {
1890 return !(is_write && mr->readonly);
1891 }
1892 if (memory_region_is_romd(mr)) {
1893 return !is_write;
1894 }
1895
1896 return false;
1897 }
1898
1899 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1900 {
1901 unsigned access_size_min = mr->ops->impl.min_access_size;
1902 unsigned access_size_max = mr->ops->impl.max_access_size;
1903
1904 /* Regions are assumed to support 1-4 byte accesses unless
1905 otherwise specified. */
1906 if (access_size_min == 0) {
1907 access_size_min = 1;
1908 }
1909 if (access_size_max == 0) {
1910 access_size_max = 4;
1911 }
1912
1913 /* Bound the maximum access by the alignment of the address. */
1914 if (!mr->ops->impl.unaligned) {
1915 unsigned align_size_max = addr & -addr;
1916 if (align_size_max != 0 && align_size_max < access_size_max) {
1917 access_size_max = align_size_max;
1918 }
1919 }
1920
1921 /* Don't attempt accesses larger than the maximum. */
1922 if (l > access_size_max) {
1923 l = access_size_max;
1924 }
1925 /* ??? The users of this function are wrong, not supporting minimums larger
1926 than the remaining length. C.f. memory.c:access_with_adjusted_size. */
1927 assert(l >= access_size_min);
1928
1929 return l;
1930 }
1931
1932 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1933 int len, bool is_write)
1934 {
1935 hwaddr l;
1936 uint8_t *ptr;
1937 uint64_t val;
1938 hwaddr addr1;
1939 MemoryRegion *mr;
1940 bool error = false;
1941
1942 while (len > 0) {
1943 l = len;
1944 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1945
1946 if (is_write) {
1947 if (!memory_access_is_direct(mr, is_write)) {
1948 l = memory_access_size(mr, l, addr1);
1949 /* XXX: could force current_cpu to NULL to avoid
1950 potential bugs */
1951 switch (l) {
1952 case 8:
1953 /* 64 bit write access */
1954 val = ldq_p(buf);
1955 error |= io_mem_write(mr, addr1, val, 8);
1956 break;
1957 case 4:
1958 /* 32 bit write access */
1959 val = ldl_p(buf);
1960 error |= io_mem_write(mr, addr1, val, 4);
1961 break;
1962 case 2:
1963 /* 16 bit write access */
1964 val = lduw_p(buf);
1965 error |= io_mem_write(mr, addr1, val, 2);
1966 break;
1967 case 1:
1968 /* 8 bit write access */
1969 val = ldub_p(buf);
1970 error |= io_mem_write(mr, addr1, val, 1);
1971 break;
1972 default:
1973 abort();
1974 }
1975 } else {
1976 addr1 += memory_region_get_ram_addr(mr);
1977 /* RAM case */
1978 ptr = qemu_get_ram_ptr(addr1);
1979 memcpy(ptr, buf, l);
1980 invalidate_and_set_dirty(addr1, l);
1981 }
1982 } else {
1983 if (!memory_access_is_direct(mr, is_write)) {
1984 /* I/O case */
1985 l = memory_access_size(mr, l, addr1);
1986 switch (l) {
1987 case 8:
1988 /* 64 bit read access */
1989 error |= io_mem_read(mr, addr1, &val, 8);
1990 stq_p(buf, val);
1991 break;
1992 case 4:
1993 /* 32 bit read access */
1994 error |= io_mem_read(mr, addr1, &val, 4);
1995 stl_p(buf, val);
1996 break;
1997 case 2:
1998 /* 16 bit read access */
1999 error |= io_mem_read(mr, addr1, &val, 2);
2000 stw_p(buf, val);
2001 break;
2002 case 1:
2003 /* 8 bit read access */
2004 error |= io_mem_read(mr, addr1, &val, 1);
2005 stb_p(buf, val);
2006 break;
2007 default:
2008 abort();
2009 }
2010 } else {
2011 /* RAM case */
2012 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2013 memcpy(buf, ptr, l);
2014 }
2015 }
2016 len -= l;
2017 buf += l;
2018 addr += l;
2019 }
2020
2021 return error;
2022 }
2023
2024 bool address_space_write(AddressSpace *as, hwaddr addr,
2025 const uint8_t *buf, int len)
2026 {
2027 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2028 }
2029
2030 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2031 {
2032 return address_space_rw(as, addr, buf, len, false);
2033 }
2034
2035
2036 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2037 int len, int is_write)
2038 {
2039 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2040 }
2041
2042 /* used for ROM loading : can write in RAM and ROM */
2043 void cpu_physical_memory_write_rom(hwaddr addr,
2044 const uint8_t *buf, int len)
2045 {
2046 hwaddr l;
2047 uint8_t *ptr;
2048 hwaddr addr1;
2049 MemoryRegion *mr;
2050
2051 while (len > 0) {
2052 l = len;
2053 mr = address_space_translate(&address_space_memory,
2054 addr, &addr1, &l, true);
2055
2056 if (!(memory_region_is_ram(mr) ||
2057 memory_region_is_romd(mr))) {
2058 /* do nothing */
2059 } else {
2060 addr1 += memory_region_get_ram_addr(mr);
2061 /* ROM/RAM case */
2062 ptr = qemu_get_ram_ptr(addr1);
2063 memcpy(ptr, buf, l);
2064 invalidate_and_set_dirty(addr1, l);
2065 }
2066 len -= l;
2067 buf += l;
2068 addr += l;
2069 }
2070 }
2071
2072 typedef struct {
2073 MemoryRegion *mr;
2074 void *buffer;
2075 hwaddr addr;
2076 hwaddr len;
2077 } BounceBuffer;
2078
2079 static BounceBuffer bounce;
2080
2081 typedef struct MapClient {
2082 void *opaque;
2083 void (*callback)(void *opaque);
2084 QLIST_ENTRY(MapClient) link;
2085 } MapClient;
2086
2087 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2088 = QLIST_HEAD_INITIALIZER(map_client_list);
2089
2090 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2091 {
2092 MapClient *client = g_malloc(sizeof(*client));
2093
2094 client->opaque = opaque;
2095 client->callback = callback;
2096 QLIST_INSERT_HEAD(&map_client_list, client, link);
2097 return client;
2098 }
2099
2100 static void cpu_unregister_map_client(void *_client)
2101 {
2102 MapClient *client = (MapClient *)_client;
2103
2104 QLIST_REMOVE(client, link);
2105 g_free(client);
2106 }
2107
2108 static void cpu_notify_map_clients(void)
2109 {
2110 MapClient *client;
2111
2112 while (!QLIST_EMPTY(&map_client_list)) {
2113 client = QLIST_FIRST(&map_client_list);
2114 client->callback(client->opaque);
2115 cpu_unregister_map_client(client);
2116 }
2117 }
2118
2119 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2120 {
2121 MemoryRegion *mr;
2122 hwaddr l, xlat;
2123
2124 while (len > 0) {
2125 l = len;
2126 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2127 if (!memory_access_is_direct(mr, is_write)) {
2128 l = memory_access_size(mr, l, addr);
2129 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2130 return false;
2131 }
2132 }
2133
2134 len -= l;
2135 addr += l;
2136 }
2137 return true;
2138 }
2139
2140 /* Map a physical memory region into a host virtual address.
2141 * May map a subset of the requested range, given by and returned in *plen.
2142 * May return NULL if resources needed to perform the mapping are exhausted.
2143 * Use only for reads OR writes - not for read-modify-write operations.
2144 * Use cpu_register_map_client() to know when retrying the map operation is
2145 * likely to succeed.
2146 */
2147 void *address_space_map(AddressSpace *as,
2148 hwaddr addr,
2149 hwaddr *plen,
2150 bool is_write)
2151 {
2152 hwaddr len = *plen;
2153 hwaddr done = 0;
2154 hwaddr l, xlat, base;
2155 MemoryRegion *mr, *this_mr;
2156 ram_addr_t raddr;
2157
2158 if (len == 0) {
2159 return NULL;
2160 }
2161
2162 l = len;
2163 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2164 if (!memory_access_is_direct(mr, is_write)) {
2165 if (bounce.buffer) {
2166 return NULL;
2167 }
2168 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2169 bounce.addr = addr;
2170 bounce.len = l;
2171
2172 memory_region_ref(mr);
2173 bounce.mr = mr;
2174 if (!is_write) {
2175 address_space_read(as, addr, bounce.buffer, l);
2176 }
2177
2178 *plen = l;
2179 return bounce.buffer;
2180 }
2181
2182 base = xlat;
2183 raddr = memory_region_get_ram_addr(mr);
2184
2185 for (;;) {
2186 len -= l;
2187 addr += l;
2188 done += l;
2189 if (len == 0) {
2190 break;
2191 }
2192
2193 l = len;
2194 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2195 if (this_mr != mr || xlat != base + done) {
2196 break;
2197 }
2198 }
2199
2200 memory_region_ref(mr);
2201 *plen = done;
2202 return qemu_ram_ptr_length(raddr + base, plen);
2203 }
2204
2205 /* Unmaps a memory region previously mapped by address_space_map().
2206 * Will also mark the memory as dirty if is_write == 1. access_len gives
2207 * the amount of memory that was actually read or written by the caller.
2208 */
2209 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2210 int is_write, hwaddr access_len)
2211 {
2212 if (buffer != bounce.buffer) {
2213 MemoryRegion *mr;
2214 ram_addr_t addr1;
2215
2216 mr = qemu_ram_addr_from_host(buffer, &addr1);
2217 assert(mr != NULL);
2218 if (is_write) {
2219 while (access_len) {
2220 unsigned l;
2221 l = TARGET_PAGE_SIZE;
2222 if (l > access_len)
2223 l = access_len;
2224 invalidate_and_set_dirty(addr1, l);
2225 addr1 += l;
2226 access_len -= l;
2227 }
2228 }
2229 if (xen_enabled()) {
2230 xen_invalidate_map_cache_entry(buffer);
2231 }
2232 memory_region_unref(mr);
2233 return;
2234 }
2235 if (is_write) {
2236 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2237 }
2238 qemu_vfree(bounce.buffer);
2239 bounce.buffer = NULL;
2240 memory_region_unref(bounce.mr);
2241 cpu_notify_map_clients();
2242 }
2243
2244 void *cpu_physical_memory_map(hwaddr addr,
2245 hwaddr *plen,
2246 int is_write)
2247 {
2248 return address_space_map(&address_space_memory, addr, plen, is_write);
2249 }
2250
2251 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2252 int is_write, hwaddr access_len)
2253 {
2254 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2255 }
2256
2257 /* warning: addr must be aligned */
2258 static inline uint32_t ldl_phys_internal(hwaddr addr,
2259 enum device_endian endian)
2260 {
2261 uint8_t *ptr;
2262 uint64_t val;
2263 MemoryRegion *mr;
2264 hwaddr l = 4;
2265 hwaddr addr1;
2266
2267 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2268 false);
2269 if (l < 4 || !memory_access_is_direct(mr, false)) {
2270 /* I/O case */
2271 io_mem_read(mr, addr1, &val, 4);
2272 #if defined(TARGET_WORDS_BIGENDIAN)
2273 if (endian == DEVICE_LITTLE_ENDIAN) {
2274 val = bswap32(val);
2275 }
2276 #else
2277 if (endian == DEVICE_BIG_ENDIAN) {
2278 val = bswap32(val);
2279 }
2280 #endif
2281 } else {
2282 /* RAM case */
2283 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2284 & TARGET_PAGE_MASK)
2285 + addr1);
2286 switch (endian) {
2287 case DEVICE_LITTLE_ENDIAN:
2288 val = ldl_le_p(ptr);
2289 break;
2290 case DEVICE_BIG_ENDIAN:
2291 val = ldl_be_p(ptr);
2292 break;
2293 default:
2294 val = ldl_p(ptr);
2295 break;
2296 }
2297 }
2298 return val;
2299 }
2300
2301 uint32_t ldl_phys(hwaddr addr)
2302 {
2303 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2304 }
2305
2306 uint32_t ldl_le_phys(hwaddr addr)
2307 {
2308 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2309 }
2310
2311 uint32_t ldl_be_phys(hwaddr addr)
2312 {
2313 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2314 }
2315
2316 /* warning: addr must be aligned */
2317 static inline uint64_t ldq_phys_internal(hwaddr addr,
2318 enum device_endian endian)
2319 {
2320 uint8_t *ptr;
2321 uint64_t val;
2322 MemoryRegion *mr;
2323 hwaddr l = 8;
2324 hwaddr addr1;
2325
2326 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2327 false);
2328 if (l < 8 || !memory_access_is_direct(mr, false)) {
2329 /* I/O case */
2330 io_mem_read(mr, addr1, &val, 8);
2331 #if defined(TARGET_WORDS_BIGENDIAN)
2332 if (endian == DEVICE_LITTLE_ENDIAN) {
2333 val = bswap64(val);
2334 }
2335 #else
2336 if (endian == DEVICE_BIG_ENDIAN) {
2337 val = bswap64(val);
2338 }
2339 #endif
2340 } else {
2341 /* RAM case */
2342 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2343 & TARGET_PAGE_MASK)
2344 + addr1);
2345 switch (endian) {
2346 case DEVICE_LITTLE_ENDIAN:
2347 val = ldq_le_p(ptr);
2348 break;
2349 case DEVICE_BIG_ENDIAN:
2350 val = ldq_be_p(ptr);
2351 break;
2352 default:
2353 val = ldq_p(ptr);
2354 break;
2355 }
2356 }
2357 return val;
2358 }
2359
2360 uint64_t ldq_phys(hwaddr addr)
2361 {
2362 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2363 }
2364
2365 uint64_t ldq_le_phys(hwaddr addr)
2366 {
2367 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2368 }
2369
2370 uint64_t ldq_be_phys(hwaddr addr)
2371 {
2372 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2373 }
2374
2375 /* XXX: optimize */
2376 uint32_t ldub_phys(hwaddr addr)
2377 {
2378 uint8_t val;
2379 cpu_physical_memory_read(addr, &val, 1);
2380 return val;
2381 }
2382
2383 /* warning: addr must be aligned */
2384 static inline uint32_t lduw_phys_internal(hwaddr addr,
2385 enum device_endian endian)
2386 {
2387 uint8_t *ptr;
2388 uint64_t val;
2389 MemoryRegion *mr;
2390 hwaddr l = 2;
2391 hwaddr addr1;
2392
2393 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2394 false);
2395 if (l < 2 || !memory_access_is_direct(mr, false)) {
2396 /* I/O case */
2397 io_mem_read(mr, addr1, &val, 2);
2398 #if defined(TARGET_WORDS_BIGENDIAN)
2399 if (endian == DEVICE_LITTLE_ENDIAN) {
2400 val = bswap16(val);
2401 }
2402 #else
2403 if (endian == DEVICE_BIG_ENDIAN) {
2404 val = bswap16(val);
2405 }
2406 #endif
2407 } else {
2408 /* RAM case */
2409 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2410 & TARGET_PAGE_MASK)
2411 + addr1);
2412 switch (endian) {
2413 case DEVICE_LITTLE_ENDIAN:
2414 val = lduw_le_p(ptr);
2415 break;
2416 case DEVICE_BIG_ENDIAN:
2417 val = lduw_be_p(ptr);
2418 break;
2419 default:
2420 val = lduw_p(ptr);
2421 break;
2422 }
2423 }
2424 return val;
2425 }
2426
2427 uint32_t lduw_phys(hwaddr addr)
2428 {
2429 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2430 }
2431
2432 uint32_t lduw_le_phys(hwaddr addr)
2433 {
2434 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2435 }
2436
2437 uint32_t lduw_be_phys(hwaddr addr)
2438 {
2439 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2440 }
2441
2442 /* warning: addr must be aligned. The ram page is not masked as dirty
2443 and the code inside is not invalidated. It is useful if the dirty
2444 bits are used to track modified PTEs */
2445 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2446 {
2447 uint8_t *ptr;
2448 MemoryRegion *mr;
2449 hwaddr l = 4;
2450 hwaddr addr1;
2451
2452 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2453 true);
2454 if (l < 4 || !memory_access_is_direct(mr, true)) {
2455 io_mem_write(mr, addr1, val, 4);
2456 } else {
2457 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2458 ptr = qemu_get_ram_ptr(addr1);
2459 stl_p(ptr, val);
2460
2461 if (unlikely(in_migration)) {
2462 if (!cpu_physical_memory_is_dirty(addr1)) {
2463 /* invalidate code */
2464 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2465 /* set dirty bit */
2466 cpu_physical_memory_set_dirty_flags(
2467 addr1, (0xff & ~CODE_DIRTY_FLAG));
2468 }
2469 }
2470 }
2471 }
2472
2473 /* warning: addr must be aligned */
2474 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2475 enum device_endian endian)
2476 {
2477 uint8_t *ptr;
2478 MemoryRegion *mr;
2479 hwaddr l = 4;
2480 hwaddr addr1;
2481
2482 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2483 true);
2484 if (l < 4 || !memory_access_is_direct(mr, true)) {
2485 #if defined(TARGET_WORDS_BIGENDIAN)
2486 if (endian == DEVICE_LITTLE_ENDIAN) {
2487 val = bswap32(val);
2488 }
2489 #else
2490 if (endian == DEVICE_BIG_ENDIAN) {
2491 val = bswap32(val);
2492 }
2493 #endif
2494 io_mem_write(mr, addr1, val, 4);
2495 } else {
2496 /* RAM case */
2497 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2498 ptr = qemu_get_ram_ptr(addr1);
2499 switch (endian) {
2500 case DEVICE_LITTLE_ENDIAN:
2501 stl_le_p(ptr, val);
2502 break;
2503 case DEVICE_BIG_ENDIAN:
2504 stl_be_p(ptr, val);
2505 break;
2506 default:
2507 stl_p(ptr, val);
2508 break;
2509 }
2510 invalidate_and_set_dirty(addr1, 4);
2511 }
2512 }
2513
2514 void stl_phys(hwaddr addr, uint32_t val)
2515 {
2516 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2517 }
2518
2519 void stl_le_phys(hwaddr addr, uint32_t val)
2520 {
2521 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2522 }
2523
2524 void stl_be_phys(hwaddr addr, uint32_t val)
2525 {
2526 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2527 }
2528
2529 /* XXX: optimize */
2530 void stb_phys(hwaddr addr, uint32_t val)
2531 {
2532 uint8_t v = val;
2533 cpu_physical_memory_write(addr, &v, 1);
2534 }
2535
2536 /* warning: addr must be aligned */
2537 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2538 enum device_endian endian)
2539 {
2540 uint8_t *ptr;
2541 MemoryRegion *mr;
2542 hwaddr l = 2;
2543 hwaddr addr1;
2544
2545 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2546 true);
2547 if (l < 2 || !memory_access_is_direct(mr, true)) {
2548 #if defined(TARGET_WORDS_BIGENDIAN)
2549 if (endian == DEVICE_LITTLE_ENDIAN) {
2550 val = bswap16(val);
2551 }
2552 #else
2553 if (endian == DEVICE_BIG_ENDIAN) {
2554 val = bswap16(val);
2555 }
2556 #endif
2557 io_mem_write(mr, addr1, val, 2);
2558 } else {
2559 /* RAM case */
2560 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2561 ptr = qemu_get_ram_ptr(addr1);
2562 switch (endian) {
2563 case DEVICE_LITTLE_ENDIAN:
2564 stw_le_p(ptr, val);
2565 break;
2566 case DEVICE_BIG_ENDIAN:
2567 stw_be_p(ptr, val);
2568 break;
2569 default:
2570 stw_p(ptr, val);
2571 break;
2572 }
2573 invalidate_and_set_dirty(addr1, 2);
2574 }
2575 }
2576
2577 void stw_phys(hwaddr addr, uint32_t val)
2578 {
2579 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2580 }
2581
2582 void stw_le_phys(hwaddr addr, uint32_t val)
2583 {
2584 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2585 }
2586
2587 void stw_be_phys(hwaddr addr, uint32_t val)
2588 {
2589 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2590 }
2591
2592 /* XXX: optimize */
2593 void stq_phys(hwaddr addr, uint64_t val)
2594 {
2595 val = tswap64(val);
2596 cpu_physical_memory_write(addr, &val, 8);
2597 }
2598
2599 void stq_le_phys(hwaddr addr, uint64_t val)
2600 {
2601 val = cpu_to_le64(val);
2602 cpu_physical_memory_write(addr, &val, 8);
2603 }
2604
2605 void stq_be_phys(hwaddr addr, uint64_t val)
2606 {
2607 val = cpu_to_be64(val);
2608 cpu_physical_memory_write(addr, &val, 8);
2609 }
2610
2611 /* virtual memory access for debug (includes writing to ROM) */
2612 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2613 uint8_t *buf, int len, int is_write)
2614 {
2615 int l;
2616 hwaddr phys_addr;
2617 target_ulong page;
2618
2619 while (len > 0) {
2620 page = addr & TARGET_PAGE_MASK;
2621 phys_addr = cpu_get_phys_page_debug(env, page);
2622 /* if no physical page mapped, return an error */
2623 if (phys_addr == -1)
2624 return -1;
2625 l = (page + TARGET_PAGE_SIZE) - addr;
2626 if (l > len)
2627 l = len;
2628 phys_addr += (addr & ~TARGET_PAGE_MASK);
2629 if (is_write)
2630 cpu_physical_memory_write_rom(phys_addr, buf, l);
2631 else
2632 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2633 len -= l;
2634 buf += l;
2635 addr += l;
2636 }
2637 return 0;
2638 }
2639 #endif
2640
2641 #if !defined(CONFIG_USER_ONLY)
2642
2643 /*
2644 * A helper function for the _utterly broken_ virtio device model to find out if
2645 * it's running on a big endian machine. Don't do this at home kids!
2646 */
2647 bool virtio_is_big_endian(void);
2648 bool virtio_is_big_endian(void)
2649 {
2650 #if defined(TARGET_WORDS_BIGENDIAN)
2651 return true;
2652 #else
2653 return false;
2654 #endif
2655 }
2656
2657 #endif
2658
2659 #ifndef CONFIG_USER_ONLY
2660 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2661 {
2662 MemoryRegion*mr;
2663 hwaddr l = 1;
2664
2665 mr = address_space_translate(&address_space_memory,
2666 phys_addr, &phys_addr, &l, false);
2667
2668 return !(memory_region_is_ram(mr) ||
2669 memory_region_is_romd(mr));
2670 }
2671
2672 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2673 {
2674 RAMBlock *block;
2675
2676 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2677 func(block->host, block->offset, block->length, opaque);
2678 }
2679 }
2680 #endif
AR7 emulation for QEMU