?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
vga: Mark relevant portio lists regions as coalesced MMIO flushing
[qemu-kvm.git] / exec.c
blobbea2cffd94d28eadf3c27a9ce47bcf3b6ea09f6a
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
51
52 #include "exec/memory-internal.h"
53
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 typedef PhysPageEntry Node[L2_SIZE];
92
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101 };
102
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
110
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
115
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
124
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
127
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
133
134 static MemoryRegion io_mem_watch;
135 #endif
136
137 #if !defined(CONFIG_USER_ONLY)
138
139 static void phys_map_node_reserve(unsigned nodes)
140 {
141 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
142 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
143 16);
144 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
145 next_map.nodes_nb + nodes);
146 next_map.nodes = g_renew(Node, next_map.nodes,
147 next_map.nodes_nb_alloc);
148 }
149 }
150
151 static uint16_t phys_map_node_alloc(void)
152 {
153 unsigned i;
154 uint16_t ret;
155
156 ret = next_map.nodes_nb++;
157 assert(ret != PHYS_MAP_NODE_NIL);
158 assert(ret != next_map.nodes_nb_alloc);
159 for (i = 0; i < L2_SIZE; ++i) {
160 next_map.nodes[ret][i].is_leaf = 0;
161 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
162 }
163 return ret;
164 }
165
166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
167 hwaddr *nb, uint16_t leaf,
168 int level)
169 {
170 PhysPageEntry *p;
171 int i;
172 hwaddr step = (hwaddr)1 << (level * L2_BITS);
173
174 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
175 lp->ptr = phys_map_node_alloc();
176 p = next_map.nodes[lp->ptr];
177 if (level == 0) {
178 for (i = 0; i < L2_SIZE; i++) {
179 p[i].is_leaf = 1;
180 p[i].ptr = PHYS_SECTION_UNASSIGNED;
181 }
182 }
183 } else {
184 p = next_map.nodes[lp->ptr];
185 }
186 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
187
188 while (*nb && lp < &p[L2_SIZE]) {
189 if ((*index & (step - 1)) == 0 && *nb >= step) {
190 lp->is_leaf = true;
191 lp->ptr = leaf;
192 *index += step;
193 *nb -= step;
194 } else {
195 phys_page_set_level(lp, index, nb, leaf, level - 1);
196 }
197 ++lp;
198 }
199 }
200
201 static void phys_page_set(AddressSpaceDispatch *d,
202 hwaddr index, hwaddr nb,
203 uint16_t leaf)
204 {
205 /* Wildly overreserve - it doesn't matter much. */
206 phys_map_node_reserve(3 * P_L2_LEVELS);
207
208 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
209 }
210
211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
212 Node *nodes, MemoryRegionSection *sections)
213 {
214 PhysPageEntry *p;
215 int i;
216
217 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
218 if (lp.ptr == PHYS_MAP_NODE_NIL) {
219 return &sections[PHYS_SECTION_UNASSIGNED];
220 }
221 p = nodes[lp.ptr];
222 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
223 }
224 return &sections[lp.ptr];
225 }
226
227 bool memory_region_is_unassigned(MemoryRegion *mr)
228 {
229 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
230 && mr != &io_mem_watch;
231 }
232
233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
234 hwaddr addr,
235 bool resolve_subpage)
236 {
237 MemoryRegionSection *section;
238 subpage_t *subpage;
239
240 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
241 d->nodes, d->sections);
242 if (resolve_subpage && section->mr->subpage) {
243 subpage = container_of(section->mr, subpage_t, iomem);
244 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
245 }
246 return section;
247 }
248
249 static MemoryRegionSection *
250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
251 hwaddr *plen, bool resolve_subpage)
252 {
253 MemoryRegionSection *section;
254 Int128 diff;
255
256 section = address_space_lookup_region(d, addr, resolve_subpage);
257 /* Compute offset within MemoryRegionSection */
258 addr -= section->offset_within_address_space;
259
260 /* Compute offset within MemoryRegion */
261 *xlat = addr + section->offset_within_region;
262
263 diff = int128_sub(section->mr->size, int128_make64(addr));
264 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
265 return section;
266 }
267
268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
269 hwaddr *xlat, hwaddr *plen,
270 bool is_write)
271 {
272 IOMMUTLBEntry iotlb;
273 MemoryRegionSection *section;
274 MemoryRegion *mr;
275 hwaddr len = *plen;
276
277 for (;;) {
278 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
279 mr = section->mr;
280
281 if (!mr->iommu_ops) {
282 break;
283 }
284
285 iotlb = mr->iommu_ops->translate(mr, addr);
286 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
287 | (addr & iotlb.addr_mask));
288 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
289 if (!(iotlb.perm & (1 << is_write))) {
290 mr = &io_mem_unassigned;
291 break;
292 }
293
294 as = iotlb.target_as;
295 }
296
297 *plen = len;
298 *xlat = addr;
299 return mr;
300 }
301
302 MemoryRegionSection *
303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
304 hwaddr *plen)
305 {
306 MemoryRegionSection *section;
307 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
308
309 assert(!section->mr->iommu_ops);
310 return section;
311 }
312 #endif
313
314 void cpu_exec_init_all(void)
315 {
316 #if !defined(CONFIG_USER_ONLY)
317 qemu_mutex_init(&ram_list.mutex);
318 memory_map_init();
319 io_mem_init();
320 #endif
321 }
322
323 #if !defined(CONFIG_USER_ONLY)
324
325 static int cpu_common_post_load(void *opaque, int version_id)
326 {
327 CPUState *cpu = opaque;
328
329 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
330 version_id is increased. */
331 cpu->interrupt_request &= ~0x01;
332 tlb_flush(cpu->env_ptr, 1);
333
334 return 0;
335 }
336
337 const VMStateDescription vmstate_cpu_common = {
338 .name = "cpu_common",
339 .version_id = 1,
340 .minimum_version_id = 1,
341 .minimum_version_id_old = 1,
342 .post_load = cpu_common_post_load,
343 .fields = (VMStateField []) {
344 VMSTATE_UINT32(halted, CPUState),
345 VMSTATE_UINT32(interrupt_request, CPUState),
346 VMSTATE_END_OF_LIST()
347 }
348 };
349
350 #endif
351
352 CPUState *qemu_get_cpu(int index)
353 {
354 CPUState *cpu;
355
356 CPU_FOREACH(cpu) {
357 if (cpu->cpu_index == index) {
358 return cpu;
359 }
360 }
361
362 return NULL;
363 }
364
365 void cpu_exec_init(CPUArchState *env)
366 {
367 CPUState *cpu = ENV_GET_CPU(env);
368 CPUClass *cc = CPU_GET_CLASS(cpu);
369 CPUState *some_cpu;
370 int cpu_index;
371
372 #if defined(CONFIG_USER_ONLY)
373 cpu_list_lock();
374 #endif
375 cpu_index = 0;
376 CPU_FOREACH(some_cpu) {
377 cpu_index++;
378 }
379 cpu->cpu_index = cpu_index;
380 cpu->numa_node = 0;
381 QTAILQ_INIT(&env->breakpoints);
382 QTAILQ_INIT(&env->watchpoints);
383 #ifndef CONFIG_USER_ONLY
384 cpu->thread_id = qemu_get_thread_id();
385 #endif
386 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
387 #if defined(CONFIG_USER_ONLY)
388 cpu_list_unlock();
389 #endif
390 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
391 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
392 }
393 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
394 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
395 cpu_save, cpu_load, env);
396 assert(cc->vmsd == NULL);
397 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
398 #endif
399 if (cc->vmsd != NULL) {
400 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
401 }
402 }
403
404 #if defined(TARGET_HAS_ICE)
405 #if defined(CONFIG_USER_ONLY)
406 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
407 {
408 tb_invalidate_phys_page_range(pc, pc + 1, 0);
409 }
410 #else
411 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
412 {
413 tb_invalidate_phys_addr(cpu_get_phys_page_debug(cpu, pc) |
414 (pc & ~TARGET_PAGE_MASK));
415 }
416 #endif
417 #endif /* TARGET_HAS_ICE */
418
419 #if defined(CONFIG_USER_ONLY)
420 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
421
422 {
423 }
424
425 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
426 int flags, CPUWatchpoint **watchpoint)
427 {
428 return -ENOSYS;
429 }
430 #else
431 /* Add a watchpoint. */
432 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
433 int flags, CPUWatchpoint **watchpoint)
434 {
435 target_ulong len_mask = ~(len - 1);
436 CPUWatchpoint *wp;
437
438 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
439 if ((len & (len - 1)) || (addr & ~len_mask) ||
440 len == 0 || len > TARGET_PAGE_SIZE) {
441 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
442 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
443 return -EINVAL;
444 }
445 wp = g_malloc(sizeof(*wp));
446
447 wp->vaddr = addr;
448 wp->len_mask = len_mask;
449 wp->flags = flags;
450
451 /* keep all GDB-injected watchpoints in front */
452 if (flags & BP_GDB)
453 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
454 else
455 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
456
457 tlb_flush_page(env, addr);
458
459 if (watchpoint)
460 *watchpoint = wp;
461 return 0;
462 }
463
464 /* Remove a specific watchpoint. */
465 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
466 int flags)
467 {
468 target_ulong len_mask = ~(len - 1);
469 CPUWatchpoint *wp;
470
471 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
472 if (addr == wp->vaddr && len_mask == wp->len_mask
473 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
474 cpu_watchpoint_remove_by_ref(env, wp);
475 return 0;
476 }
477 }
478 return -ENOENT;
479 }
480
481 /* Remove a specific watchpoint by reference. */
482 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
483 {
484 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
485
486 tlb_flush_page(env, watchpoint->vaddr);
487
488 g_free(watchpoint);
489 }
490
491 /* Remove all matching watchpoints. */
492 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
493 {
494 CPUWatchpoint *wp, *next;
495
496 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
497 if (wp->flags & mask)
498 cpu_watchpoint_remove_by_ref(env, wp);
499 }
500 }
501 #endif
502
503 /* Add a breakpoint. */
504 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
505 CPUBreakpoint **breakpoint)
506 {
507 #if defined(TARGET_HAS_ICE)
508 CPUBreakpoint *bp;
509
510 bp = g_malloc(sizeof(*bp));
511
512 bp->pc = pc;
513 bp->flags = flags;
514
515 /* keep all GDB-injected breakpoints in front */
516 if (flags & BP_GDB) {
517 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
518 } else {
519 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
520 }
521
522 breakpoint_invalidate(ENV_GET_CPU(env), pc);
523
524 if (breakpoint) {
525 *breakpoint = bp;
526 }
527 return 0;
528 #else
529 return -ENOSYS;
530 #endif
531 }
532
533 /* Remove a specific breakpoint. */
534 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
535 {
536 #if defined(TARGET_HAS_ICE)
537 CPUBreakpoint *bp;
538
539 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
540 if (bp->pc == pc && bp->flags == flags) {
541 cpu_breakpoint_remove_by_ref(env, bp);
542 return 0;
543 }
544 }
545 return -ENOENT;
546 #else
547 return -ENOSYS;
548 #endif
549 }
550
551 /* Remove a specific breakpoint by reference. */
552 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
553 {
554 #if defined(TARGET_HAS_ICE)
555 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
556
557 breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
558
559 g_free(breakpoint);
560 #endif
561 }
562
563 /* Remove all matching breakpoints. */
564 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
565 {
566 #if defined(TARGET_HAS_ICE)
567 CPUBreakpoint *bp, *next;
568
569 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
570 if (bp->flags & mask)
571 cpu_breakpoint_remove_by_ref(env, bp);
572 }
573 #endif
574 }
575
576 /* enable or disable single step mode. EXCP_DEBUG is returned by the
577 CPU loop after each instruction */
578 void cpu_single_step(CPUState *cpu, int enabled)
579 {
580 #if defined(TARGET_HAS_ICE)
581 if (cpu->singlestep_enabled != enabled) {
582 cpu->singlestep_enabled = enabled;
583 if (kvm_enabled()) {
584 kvm_update_guest_debug(cpu, 0);
585 } else {
586 /* must flush all the translated code to avoid inconsistencies */
587 /* XXX: only flush what is necessary */
588 CPUArchState *env = cpu->env_ptr;
589 tb_flush(env);
590 }
591 }
592 #endif
593 }
594
595 void cpu_abort(CPUArchState *env, const char *fmt, ...)
596 {
597 CPUState *cpu = ENV_GET_CPU(env);
598 va_list ap;
599 va_list ap2;
600
601 va_start(ap, fmt);
602 va_copy(ap2, ap);
603 fprintf(stderr, "qemu: fatal: ");
604 vfprintf(stderr, fmt, ap);
605 fprintf(stderr, "\n");
606 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
607 if (qemu_log_enabled()) {
608 qemu_log("qemu: fatal: ");
609 qemu_log_vprintf(fmt, ap2);
610 qemu_log("\n");
611 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
612 qemu_log_flush();
613 qemu_log_close();
614 }
615 va_end(ap2);
616 va_end(ap);
617 #if defined(CONFIG_USER_ONLY)
618 {
619 struct sigaction act;
620 sigfillset(&act.sa_mask);
621 act.sa_handler = SIG_DFL;
622 sigaction(SIGABRT, &act, NULL);
623 }
624 #endif
625 abort();
626 }
627
628 #if !defined(CONFIG_USER_ONLY)
629 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
630 uintptr_t length)
631 {
632 uintptr_t start1;
633
634 /* we modify the TLB cache so that the dirty bit will be set again
635 when accessing the range */
636 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
637 /* Check that we don't span multiple blocks - this breaks the
638 address comparisons below. */
639 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
640 != (end - 1) - start) {
641 abort();
642 }
643 cpu_tlb_reset_dirty_all(start1, length);
644
645 }
646
647 /* Note: start and end must be within the same ram block. */
648 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
649 int dirty_flags)
650 {
651 uintptr_t length;
652
653 start &= TARGET_PAGE_MASK;
654 end = TARGET_PAGE_ALIGN(end);
655
656 length = end - start;
657 if (length == 0)
658 return;
659 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
660
661 if (tcg_enabled()) {
662 tlb_reset_dirty_range_all(start, end, length);
663 }
664 }
665
666 static int cpu_physical_memory_set_dirty_tracking(int enable)
667 {
668 int ret = 0;
669 in_migration = enable;
670 return ret;
671 }
672
673 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
674 MemoryRegionSection *section,
675 target_ulong vaddr,
676 hwaddr paddr, hwaddr xlat,
677 int prot,
678 target_ulong *address)
679 {
680 hwaddr iotlb;
681 CPUWatchpoint *wp;
682
683 if (memory_region_is_ram(section->mr)) {
684 /* Normal RAM. */
685 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
686 + xlat;
687 if (!section->readonly) {
688 iotlb |= PHYS_SECTION_NOTDIRTY;
689 } else {
690 iotlb |= PHYS_SECTION_ROM;
691 }
692 } else {
693 iotlb = section - address_space_memory.dispatch->sections;
694 iotlb += xlat;
695 }
696
697 /* Make accesses to pages with watchpoints go via the
698 watchpoint trap routines. */
699 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
700 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
701 /* Avoid trapping reads of pages with a write breakpoint. */
702 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
703 iotlb = PHYS_SECTION_WATCH + paddr;
704 *address |= TLB_MMIO;
705 break;
706 }
707 }
708 }
709
710 return iotlb;
711 }
712 #endif /* defined(CONFIG_USER_ONLY) */
713
714 #if !defined(CONFIG_USER_ONLY)
715
716 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
717 uint16_t section);
718 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
719
720 static void *(*phys_mem_alloc)(size_t size) = qemu_anon_ram_alloc;
721
722 /*
723 * Set a custom physical guest memory alloator.
724 * Accelerators with unusual needs may need this. Hopefully, we can
725 * get rid of it eventually.
726 */
727 void phys_mem_set_alloc(void *(*alloc)(size_t))
728 {
729 phys_mem_alloc = alloc;
730 }
731
732 static uint16_t phys_section_add(MemoryRegionSection *section)
733 {
734 /* The physical section number is ORed with a page-aligned
735 * pointer to produce the iotlb entries. Thus it should
736 * never overflow into the page-aligned value.
737 */
738 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
739
740 if (next_map.sections_nb == next_map.sections_nb_alloc) {
741 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
742 16);
743 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
744 next_map.sections_nb_alloc);
745 }
746 next_map.sections[next_map.sections_nb] = *section;
747 memory_region_ref(section->mr);
748 return next_map.sections_nb++;
749 }
750
751 static void phys_section_destroy(MemoryRegion *mr)
752 {
753 memory_region_unref(mr);
754
755 if (mr->subpage) {
756 subpage_t *subpage = container_of(mr, subpage_t, iomem);
757 memory_region_destroy(&subpage->iomem);
758 g_free(subpage);
759 }
760 }
761
762 static void phys_sections_free(PhysPageMap *map)
763 {
764 while (map->sections_nb > 0) {
765 MemoryRegionSection *section = &map->sections[--map->sections_nb];
766 phys_section_destroy(section->mr);
767 }
768 g_free(map->sections);
769 g_free(map->nodes);
770 g_free(map);
771 }
772
773 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
774 {
775 subpage_t *subpage;
776 hwaddr base = section->offset_within_address_space
777 & TARGET_PAGE_MASK;
778 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
779 next_map.nodes, next_map.sections);
780 MemoryRegionSection subsection = {
781 .offset_within_address_space = base,
782 .size = int128_make64(TARGET_PAGE_SIZE),
783 };
784 hwaddr start, end;
785
786 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
787
788 if (!(existing->mr->subpage)) {
789 subpage = subpage_init(d->as, base);
790 subsection.mr = &subpage->iomem;
791 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
792 phys_section_add(&subsection));
793 } else {
794 subpage = container_of(existing->mr, subpage_t, iomem);
795 }
796 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
797 end = start + int128_get64(section->size) - 1;
798 subpage_register(subpage, start, end, phys_section_add(section));
799 }
800
801
802 static void register_multipage(AddressSpaceDispatch *d,
803 MemoryRegionSection *section)
804 {
805 hwaddr start_addr = section->offset_within_address_space;
806 uint16_t section_index = phys_section_add(section);
807 uint64_t num_pages = int128_get64(int128_rshift(section->size,
808 TARGET_PAGE_BITS));
809
810 assert(num_pages);
811 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
812 }
813
814 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
815 {
816 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
817 AddressSpaceDispatch *d = as->next_dispatch;
818 MemoryRegionSection now = *section, remain = *section;
819 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
820
821 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
822 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
823 - now.offset_within_address_space;
824
825 now.size = int128_min(int128_make64(left), now.size);
826 register_subpage(d, &now);
827 } else {
828 now.size = int128_zero();
829 }
830 while (int128_ne(remain.size, now.size)) {
831 remain.size = int128_sub(remain.size, now.size);
832 remain.offset_within_address_space += int128_get64(now.size);
833 remain.offset_within_region += int128_get64(now.size);
834 now = remain;
835 if (int128_lt(remain.size, page_size)) {
836 register_subpage(d, &now);
837 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
838 now.size = page_size;
839 register_subpage(d, &now);
840 } else {
841 now.size = int128_and(now.size, int128_neg(page_size));
842 register_multipage(d, &now);
843 }
844 }
845 }
846
847 void qemu_flush_coalesced_mmio_buffer(void)
848 {
849 if (kvm_enabled())
850 kvm_flush_coalesced_mmio_buffer();
851 }
852
853 void qemu_mutex_lock_ramlist(void)
854 {
855 qemu_mutex_lock(&ram_list.mutex);
856 }
857
858 void qemu_mutex_unlock_ramlist(void)
859 {
860 qemu_mutex_unlock(&ram_list.mutex);
861 }
862
863 #ifdef __linux__
864
865 #include <sys/vfs.h>
866
867 #define HUGETLBFS_MAGIC 0x958458f6
868
869 static long gethugepagesize(const char *path)
870 {
871 struct statfs fs;
872 int ret;
873
874 do {
875 ret = statfs(path, &fs);
876 } while (ret != 0 && errno == EINTR);
877
878 if (ret != 0) {
879 perror(path);
880 return 0;
881 }
882
883 if (fs.f_type != HUGETLBFS_MAGIC)
884 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
885
886 return fs.f_bsize;
887 }
888
889 static void *file_ram_alloc(RAMBlock *block,
890 ram_addr_t memory,
891 const char *path)
892 {
893 char *filename;
894 char *sanitized_name;
895 char *c;
896 void *area;
897 int fd;
898 #ifdef MAP_POPULATE
899 int flags;
900 #endif
901 unsigned long hpagesize;
902
903 hpagesize = gethugepagesize(path);
904 if (!hpagesize) {
905 return NULL;
906 }
907
908 if (memory < hpagesize) {
909 return NULL;
910 }
911
912 if (kvm_enabled() && !kvm_has_sync_mmu()) {
913 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
914 return NULL;
915 }
916
917 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
918 sanitized_name = g_strdup(block->mr->name);
919 for (c = sanitized_name; *c != '\0'; c++) {
920 if (*c == '/')
921 *c = '_';
922 }
923
924 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
925 sanitized_name);
926 g_free(sanitized_name);
927
928 fd = mkstemp(filename);
929 if (fd < 0) {
930 perror("unable to create backing store for hugepages");
931 g_free(filename);
932 return NULL;
933 }
934 unlink(filename);
935 g_free(filename);
936
937 memory = (memory+hpagesize-1) & ~(hpagesize-1);
938
939 /*
940 * ftruncate is not supported by hugetlbfs in older
941 * hosts, so don't bother bailing out on errors.
942 * If anything goes wrong with it under other filesystems,
943 * mmap will fail.
944 */
945 if (ftruncate(fd, memory))
946 perror("ftruncate");
947
948 #ifdef MAP_POPULATE
949 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
950 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
951 * to sidestep this quirk.
952 */
953 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
954 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
955 #else
956 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
957 #endif
958 if (area == MAP_FAILED) {
959 perror("file_ram_alloc: can't mmap RAM pages");
960 close(fd);
961 return (NULL);
962 }
963 block->fd = fd;
964 return area;
965 }
966 #else
967 static void *file_ram_alloc(RAMBlock *block,
968 ram_addr_t memory,
969 const char *path)
970 {
971 fprintf(stderr, "-mem-path not supported on this host\n");
972 exit(1);
973 }
974 #endif
975
976 static ram_addr_t find_ram_offset(ram_addr_t size)
977 {
978 RAMBlock *block, *next_block;
979 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
980
981 assert(size != 0); /* it would hand out same offset multiple times */
982
983 if (QTAILQ_EMPTY(&ram_list.blocks))
984 return 0;
985
986 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
987 ram_addr_t end, next = RAM_ADDR_MAX;
988
989 end = block->offset + block->length;
990
991 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
992 if (next_block->offset >= end) {
993 next = MIN(next, next_block->offset);
994 }
995 }
996 if (next - end >= size && next - end < mingap) {
997 offset = end;
998 mingap = next - end;
999 }
1000 }
1001
1002 if (offset == RAM_ADDR_MAX) {
1003 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1004 (uint64_t)size);
1005 abort();
1006 }
1007
1008 return offset;
1009 }
1010
1011 ram_addr_t last_ram_offset(void)
1012 {
1013 RAMBlock *block;
1014 ram_addr_t last = 0;
1015
1016 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1017 last = MAX(last, block->offset + block->length);
1018
1019 return last;
1020 }
1021
1022 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1023 {
1024 int ret;
1025
1026 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1027 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1028 "dump-guest-core", true)) {
1029 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1030 if (ret) {
1031 perror("qemu_madvise");
1032 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1033 "but dump_guest_core=off specified\n");
1034 }
1035 }
1036 }
1037
1038 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1039 {
1040 RAMBlock *new_block, *block;
1041
1042 new_block = NULL;
1043 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1044 if (block->offset == addr) {
1045 new_block = block;
1046 break;
1047 }
1048 }
1049 assert(new_block);
1050 assert(!new_block->idstr[0]);
1051
1052 if (dev) {
1053 char *id = qdev_get_dev_path(dev);
1054 if (id) {
1055 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1056 g_free(id);
1057 }
1058 }
1059 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1060
1061 /* This assumes the iothread lock is taken here too. */
1062 qemu_mutex_lock_ramlist();
1063 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1064 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1065 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1066 new_block->idstr);
1067 abort();
1068 }
1069 }
1070 qemu_mutex_unlock_ramlist();
1071 }
1072
1073 static int memory_try_enable_merging(void *addr, size_t len)
1074 {
1075 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1076 /* disabled by the user */
1077 return 0;
1078 }
1079
1080 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1081 }
1082
1083 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1084 MemoryRegion *mr)
1085 {
1086 RAMBlock *block, *new_block;
1087
1088 size = TARGET_PAGE_ALIGN(size);
1089 new_block = g_malloc0(sizeof(*new_block));
1090 new_block->fd = -1;
1091
1092 /* This assumes the iothread lock is taken here too. */
1093 qemu_mutex_lock_ramlist();
1094 new_block->mr = mr;
1095 new_block->offset = find_ram_offset(size);
1096 if (host) {
1097 new_block->host = host;
1098 new_block->flags |= RAM_PREALLOC_MASK;
1099 } else if (xen_enabled()) {
1100 if (mem_path) {
1101 fprintf(stderr, "-mem-path not supported with Xen\n");
1102 exit(1);
1103 }
1104 xen_ram_alloc(new_block->offset, size, mr);
1105 } else {
1106 if (mem_path) {
1107 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1108 /*
1109 * file_ram_alloc() needs to allocate just like
1110 * phys_mem_alloc, but we haven't bothered to provide
1111 * a hook there.
1112 */
1113 fprintf(stderr,
1114 "-mem-path not supported with this accelerator\n");
1115 exit(1);
1116 }
1117 new_block->host = file_ram_alloc(new_block, size, mem_path);
1118 }
1119 if (!new_block->host) {
1120 new_block->host = phys_mem_alloc(size);
1121 if (!new_block->host) {
1122 fprintf(stderr, "Cannot set up guest memory '%s': %s\n",
1123 new_block->mr->name, strerror(errno));
1124 exit(1);
1125 }
1126 memory_try_enable_merging(new_block->host, size);
1127 }
1128 }
1129 new_block->length = size;
1130
1131 /* Keep the list sorted from biggest to smallest block. */
1132 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1133 if (block->length < new_block->length) {
1134 break;
1135 }
1136 }
1137 if (block) {
1138 QTAILQ_INSERT_BEFORE(block, new_block, next);
1139 } else {
1140 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1141 }
1142 ram_list.mru_block = NULL;
1143
1144 ram_list.version++;
1145 qemu_mutex_unlock_ramlist();
1146
1147 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1148 last_ram_offset() >> TARGET_PAGE_BITS);
1149 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1150 0, size >> TARGET_PAGE_BITS);
1151 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1152
1153 qemu_ram_setup_dump(new_block->host, size);
1154 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1155 qemu_madvise(new_block->host, size, QEMU_MADV_DONTFORK);
1156
1157 if (kvm_enabled())
1158 kvm_setup_guest_memory(new_block->host, size);
1159
1160 return new_block->offset;
1161 }
1162
1163 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1164 {
1165 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1166 }
1167
1168 void qemu_ram_free_from_ptr(ram_addr_t addr)
1169 {
1170 RAMBlock *block;
1171
1172 /* This assumes the iothread lock is taken here too. */
1173 qemu_mutex_lock_ramlist();
1174 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1175 if (addr == block->offset) {
1176 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1177 ram_list.mru_block = NULL;
1178 ram_list.version++;
1179 g_free(block);
1180 break;
1181 }
1182 }
1183 qemu_mutex_unlock_ramlist();
1184 }
1185
1186 void qemu_ram_free(ram_addr_t addr)
1187 {
1188 RAMBlock *block;
1189
1190 /* This assumes the iothread lock is taken here too. */
1191 qemu_mutex_lock_ramlist();
1192 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1193 if (addr == block->offset) {
1194 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1195 ram_list.mru_block = NULL;
1196 ram_list.version++;
1197 if (block->flags & RAM_PREALLOC_MASK) {
1198 ;
1199 } else if (xen_enabled()) {
1200 xen_invalidate_map_cache_entry(block->host);
1201 #ifndef _WIN32
1202 } else if (block->fd >= 0) {
1203 munmap(block->host, block->length);
1204 close(block->fd);
1205 #endif
1206 } else {
1207 qemu_anon_ram_free(block->host, block->length);
1208 }
1209 g_free(block);
1210 break;
1211 }
1212 }
1213 qemu_mutex_unlock_ramlist();
1214
1215 }
1216
1217 #ifndef _WIN32
1218 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1219 {
1220 RAMBlock *block;
1221 ram_addr_t offset;
1222 int flags;
1223 void *area, *vaddr;
1224
1225 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1226 offset = addr - block->offset;
1227 if (offset < block->length) {
1228 vaddr = block->host + offset;
1229 if (block->flags & RAM_PREALLOC_MASK) {
1230 ;
1231 } else if (xen_enabled()) {
1232 abort();
1233 } else {
1234 flags = MAP_FIXED;
1235 munmap(vaddr, length);
1236 if (block->fd >= 0) {
1237 #ifdef MAP_POPULATE
1238 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1239 MAP_PRIVATE;
1240 #else
1241 flags |= MAP_PRIVATE;
1242 #endif
1243 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1244 flags, block->fd, offset);
1245 } else {
1246 /*
1247 * Remap needs to match alloc. Accelerators that
1248 * set phys_mem_alloc never remap. If they did,
1249 * we'd need a remap hook here.
1250 */
1251 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1252
1253 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1254 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1255 flags, -1, 0);
1256 }
1257 if (area != vaddr) {
1258 fprintf(stderr, "Could not remap addr: "
1259 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1260 length, addr);
1261 exit(1);
1262 }
1263 memory_try_enable_merging(vaddr, length);
1264 qemu_ram_setup_dump(vaddr, length);
1265 }
1266 return;
1267 }
1268 }
1269 }
1270 #endif /* !_WIN32 */
1271
1272 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1273 {
1274 RAMBlock *block;
1275
1276 /* The list is protected by the iothread lock here. */
1277 block = ram_list.mru_block;
1278 if (block && addr - block->offset < block->length) {
1279 goto found;
1280 }
1281 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1282 if (addr - block->offset < block->length) {
1283 goto found;
1284 }
1285 }
1286
1287 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1288 abort();
1289
1290 found:
1291 ram_list.mru_block = block;
1292 return block;
1293 }
1294
1295 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1296 With the exception of the softmmu code in this file, this should
1297 only be used for local memory (e.g. video ram) that the device owns,
1298 and knows it isn't going to access beyond the end of the block.
1299
1300 It should not be used for general purpose DMA.
1301 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1302 */
1303 void *qemu_get_ram_ptr(ram_addr_t addr)
1304 {
1305 RAMBlock *block = qemu_get_ram_block(addr);
1306
1307 if (xen_enabled()) {
1308 /* We need to check if the requested address is in the RAM
1309 * because we don't want to map the entire memory in QEMU.
1310 * In that case just map until the end of the page.
1311 */
1312 if (block->offset == 0) {
1313 return xen_map_cache(addr, 0, 0);
1314 } else if (block->host == NULL) {
1315 block->host =
1316 xen_map_cache(block->offset, block->length, 1);
1317 }
1318 }
1319 return block->host + (addr - block->offset);
1320 }
1321
1322 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1323 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1324 *
1325 * ??? Is this still necessary?
1326 */
1327 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1328 {
1329 RAMBlock *block;
1330
1331 /* The list is protected by the iothread lock here. */
1332 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1333 if (addr - block->offset < block->length) {
1334 if (xen_enabled()) {
1335 /* We need to check if the requested address is in the RAM
1336 * because we don't want to map the entire memory in QEMU.
1337 * In that case just map until the end of the page.
1338 */
1339 if (block->offset == 0) {
1340 return xen_map_cache(addr, 0, 0);
1341 } else if (block->host == NULL) {
1342 block->host =
1343 xen_map_cache(block->offset, block->length, 1);
1344 }
1345 }
1346 return block->host + (addr - block->offset);
1347 }
1348 }
1349
1350 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1351 abort();
1352
1353 return NULL;
1354 }
1355
1356 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1357 * but takes a size argument */
1358 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1359 {
1360 if (*size == 0) {
1361 return NULL;
1362 }
1363 if (xen_enabled()) {
1364 return xen_map_cache(addr, *size, 1);
1365 } else {
1366 RAMBlock *block;
1367
1368 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1369 if (addr - block->offset < block->length) {
1370 if (addr - block->offset + *size > block->length)
1371 *size = block->length - addr + block->offset;
1372 return block->host + (addr - block->offset);
1373 }
1374 }
1375
1376 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1377 abort();
1378 }
1379 }
1380
1381 /* Some of the softmmu routines need to translate from a host pointer
1382 (typically a TLB entry) back to a ram offset. */
1383 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1384 {
1385 RAMBlock *block;
1386 uint8_t *host = ptr;
1387
1388 if (xen_enabled()) {
1389 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1390 return qemu_get_ram_block(*ram_addr)->mr;
1391 }
1392
1393 block = ram_list.mru_block;
1394 if (block && block->host && host - block->host < block->length) {
1395 goto found;
1396 }
1397
1398 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1399 /* This case append when the block is not mapped. */
1400 if (block->host == NULL) {
1401 continue;
1402 }
1403 if (host - block->host < block->length) {
1404 goto found;
1405 }
1406 }
1407
1408 return NULL;
1409
1410 found:
1411 *ram_addr = block->offset + (host - block->host);
1412 return block->mr;
1413 }
1414
1415 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1416 uint64_t val, unsigned size)
1417 {
1418 int dirty_flags;
1419 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1420 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1421 tb_invalidate_phys_page_fast(ram_addr, size);
1422 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1423 }
1424 switch (size) {
1425 case 1:
1426 stb_p(qemu_get_ram_ptr(ram_addr), val);
1427 break;
1428 case 2:
1429 stw_p(qemu_get_ram_ptr(ram_addr), val);
1430 break;
1431 case 4:
1432 stl_p(qemu_get_ram_ptr(ram_addr), val);
1433 break;
1434 default:
1435 abort();
1436 }
1437 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1438 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1439 /* we remove the notdirty callback only if the code has been
1440 flushed */
1441 if (dirty_flags == 0xff) {
1442 CPUArchState *env = current_cpu->env_ptr;
1443 tlb_set_dirty(env, env->mem_io_vaddr);
1444 }
1445 }
1446
1447 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1448 unsigned size, bool is_write)
1449 {
1450 return is_write;
1451 }
1452
1453 static const MemoryRegionOps notdirty_mem_ops = {
1454 .write = notdirty_mem_write,
1455 .valid.accepts = notdirty_mem_accepts,
1456 .endianness = DEVICE_NATIVE_ENDIAN,
1457 };
1458
1459 /* Generate a debug exception if a watchpoint has been hit. */
1460 static void check_watchpoint(int offset, int len_mask, int flags)
1461 {
1462 CPUArchState *env = current_cpu->env_ptr;
1463 target_ulong pc, cs_base;
1464 target_ulong vaddr;
1465 CPUWatchpoint *wp;
1466 int cpu_flags;
1467
1468 if (env->watchpoint_hit) {
1469 /* We re-entered the check after replacing the TB. Now raise
1470 * the debug interrupt so that is will trigger after the
1471 * current instruction. */
1472 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1473 return;
1474 }
1475 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1476 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1477 if ((vaddr == (wp->vaddr & len_mask) ||
1478 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1479 wp->flags |= BP_WATCHPOINT_HIT;
1480 if (!env->watchpoint_hit) {
1481 env->watchpoint_hit = wp;
1482 tb_check_watchpoint(env);
1483 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1484 env->exception_index = EXCP_DEBUG;
1485 cpu_loop_exit(env);
1486 } else {
1487 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1488 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1489 cpu_resume_from_signal(env, NULL);
1490 }
1491 }
1492 } else {
1493 wp->flags &= ~BP_WATCHPOINT_HIT;
1494 }
1495 }
1496 }
1497
1498 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1499 so these check for a hit then pass through to the normal out-of-line
1500 phys routines. */
1501 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1502 unsigned size)
1503 {
1504 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1505 switch (size) {
1506 case 1: return ldub_phys(addr);
1507 case 2: return lduw_phys(addr);
1508 case 4: return ldl_phys(addr);
1509 default: abort();
1510 }
1511 }
1512
1513 static void watch_mem_write(void *opaque, hwaddr addr,
1514 uint64_t val, unsigned size)
1515 {
1516 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1517 switch (size) {
1518 case 1:
1519 stb_phys(addr, val);
1520 break;
1521 case 2:
1522 stw_phys(addr, val);
1523 break;
1524 case 4:
1525 stl_phys(addr, val);
1526 break;
1527 default: abort();
1528 }
1529 }
1530
1531 static const MemoryRegionOps watch_mem_ops = {
1532 .read = watch_mem_read,
1533 .write = watch_mem_write,
1534 .endianness = DEVICE_NATIVE_ENDIAN,
1535 };
1536
1537 static uint64_t subpage_read(void *opaque, hwaddr addr,
1538 unsigned len)
1539 {
1540 subpage_t *subpage = opaque;
1541 uint8_t buf[4];
1542
1543 #if defined(DEBUG_SUBPAGE)
1544 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
1545 subpage, len, addr);
1546 #endif
1547 address_space_read(subpage->as, addr + subpage->base, buf, len);
1548 switch (len) {
1549 case 1:
1550 return ldub_p(buf);
1551 case 2:
1552 return lduw_p(buf);
1553 case 4:
1554 return ldl_p(buf);
1555 default:
1556 abort();
1557 }
1558 }
1559
1560 static void subpage_write(void *opaque, hwaddr addr,
1561 uint64_t value, unsigned len)
1562 {
1563 subpage_t *subpage = opaque;
1564 uint8_t buf[4];
1565
1566 #if defined(DEBUG_SUBPAGE)
1567 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
1568 " value %"PRIx64"\n",
1569 __func__, subpage, len, addr, value);
1570 #endif
1571 switch (len) {
1572 case 1:
1573 stb_p(buf, value);
1574 break;
1575 case 2:
1576 stw_p(buf, value);
1577 break;
1578 case 4:
1579 stl_p(buf, value);
1580 break;
1581 default:
1582 abort();
1583 }
1584 address_space_write(subpage->as, addr + subpage->base, buf, len);
1585 }
1586
1587 static bool subpage_accepts(void *opaque, hwaddr addr,
1588 unsigned len, bool is_write)
1589 {
1590 subpage_t *subpage = opaque;
1591 #if defined(DEBUG_SUBPAGE)
1592 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
1593 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1594 #endif
1595
1596 return address_space_access_valid(subpage->as, addr + subpage->base,
1597 len, is_write);
1598 }
1599
1600 static const MemoryRegionOps subpage_ops = {
1601 .read = subpage_read,
1602 .write = subpage_write,
1603 .valid.accepts = subpage_accepts,
1604 .endianness = DEVICE_NATIVE_ENDIAN,
1605 };
1606
1607 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1608 uint16_t section)
1609 {
1610 int idx, eidx;
1611
1612 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1613 return -1;
1614 idx = SUBPAGE_IDX(start);
1615 eidx = SUBPAGE_IDX(end);
1616 #if defined(DEBUG_SUBPAGE)
1617 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
1618 __func__, mmio, start, end, idx, eidx, section);
1619 #endif
1620 for (; idx <= eidx; idx++) {
1621 mmio->sub_section[idx] = section;
1622 }
1623
1624 return 0;
1625 }
1626
1627 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1628 {
1629 subpage_t *mmio;
1630
1631 mmio = g_malloc0(sizeof(subpage_t));
1632
1633 mmio->as = as;
1634 mmio->base = base;
1635 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1636 "subpage", TARGET_PAGE_SIZE);
1637 mmio->iomem.subpage = true;
1638 #if defined(DEBUG_SUBPAGE)
1639 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
1640 mmio, base, TARGET_PAGE_SIZE);
1641 #endif
1642 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1643
1644 return mmio;
1645 }
1646
1647 static uint16_t dummy_section(MemoryRegion *mr)
1648 {
1649 MemoryRegionSection section = {
1650 .mr = mr,
1651 .offset_within_address_space = 0,
1652 .offset_within_region = 0,
1653 .size = int128_2_64(),
1654 };
1655
1656 return phys_section_add(&section);
1657 }
1658
1659 MemoryRegion *iotlb_to_region(hwaddr index)
1660 {
1661 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1662 }
1663
1664 static void io_mem_init(void)
1665 {
1666 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1667 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1668 "unassigned", UINT64_MAX);
1669 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1670 "notdirty", UINT64_MAX);
1671 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1672 "watch", UINT64_MAX);
1673 }
1674
1675 static void mem_begin(MemoryListener *listener)
1676 {
1677 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1678 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1679
1680 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1681 d->as = as;
1682 as->next_dispatch = d;
1683 }
1684
1685 static void mem_commit(MemoryListener *listener)
1686 {
1687 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1688 AddressSpaceDispatch *cur = as->dispatch;
1689 AddressSpaceDispatch *next = as->next_dispatch;
1690
1691 next->nodes = next_map.nodes;
1692 next->sections = next_map.sections;
1693
1694 as->dispatch = next;
1695 g_free(cur);
1696 }
1697
1698 static void core_begin(MemoryListener *listener)
1699 {
1700 uint16_t n;
1701
1702 prev_map = g_new(PhysPageMap, 1);
1703 *prev_map = next_map;
1704
1705 memset(&next_map, 0, sizeof(next_map));
1706 n = dummy_section(&io_mem_unassigned);
1707 assert(n == PHYS_SECTION_UNASSIGNED);
1708 n = dummy_section(&io_mem_notdirty);
1709 assert(n == PHYS_SECTION_NOTDIRTY);
1710 n = dummy_section(&io_mem_rom);
1711 assert(n == PHYS_SECTION_ROM);
1712 n = dummy_section(&io_mem_watch);
1713 assert(n == PHYS_SECTION_WATCH);
1714 }
1715
1716 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1717 * All AddressSpaceDispatch instances have switched to the next map.
1718 */
1719 static void core_commit(MemoryListener *listener)
1720 {
1721 phys_sections_free(prev_map);
1722 }
1723
1724 static void tcg_commit(MemoryListener *listener)
1725 {
1726 CPUState *cpu;
1727
1728 /* since each CPU stores ram addresses in its TLB cache, we must
1729 reset the modified entries */
1730 /* XXX: slow ! */
1731 CPU_FOREACH(cpu) {
1732 CPUArchState *env = cpu->env_ptr;
1733
1734 tlb_flush(env, 1);
1735 }
1736 }
1737
1738 static void core_log_global_start(MemoryListener *listener)
1739 {
1740 cpu_physical_memory_set_dirty_tracking(1);
1741 }
1742
1743 static void core_log_global_stop(MemoryListener *listener)
1744 {
1745 cpu_physical_memory_set_dirty_tracking(0);
1746 }
1747
1748 static MemoryListener core_memory_listener = {
1749 .begin = core_begin,
1750 .commit = core_commit,
1751 .log_global_start = core_log_global_start,
1752 .log_global_stop = core_log_global_stop,
1753 .priority = 1,
1754 };
1755
1756 static MemoryListener tcg_memory_listener = {
1757 .commit = tcg_commit,
1758 };
1759
1760 void address_space_init_dispatch(AddressSpace *as)
1761 {
1762 as->dispatch = NULL;
1763 as->dispatch_listener = (MemoryListener) {
1764 .begin = mem_begin,
1765 .commit = mem_commit,
1766 .region_add = mem_add,
1767 .region_nop = mem_add,
1768 .priority = 0,
1769 };
1770 memory_listener_register(&as->dispatch_listener, as);
1771 }
1772
1773 void address_space_destroy_dispatch(AddressSpace *as)
1774 {
1775 AddressSpaceDispatch *d = as->dispatch;
1776
1777 memory_listener_unregister(&as->dispatch_listener);
1778 g_free(d);
1779 as->dispatch = NULL;
1780 }
1781
1782 static void memory_map_init(void)
1783 {
1784 system_memory = g_malloc(sizeof(*system_memory));
1785 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1786 address_space_init(&address_space_memory, system_memory, "memory");
1787
1788 system_io = g_malloc(sizeof(*system_io));
1789 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
1790 65536);
1791 address_space_init(&address_space_io, system_io, "I/O");
1792
1793 memory_listener_register(&core_memory_listener, &address_space_memory);
1794 if (tcg_enabled()) {
1795 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1796 }
1797 }
1798
1799 MemoryRegion *get_system_memory(void)
1800 {
1801 return system_memory;
1802 }
1803
1804 MemoryRegion *get_system_io(void)
1805 {
1806 return system_io;
1807 }
1808
1809 #endif /* !defined(CONFIG_USER_ONLY) */
1810
1811 /* physical memory access (slow version, mainly for debug) */
1812 #if defined(CONFIG_USER_ONLY)
1813 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1814 uint8_t *buf, int len, int is_write)
1815 {
1816 int l, flags;
1817 target_ulong page;
1818 void * p;
1819
1820 while (len > 0) {
1821 page = addr & TARGET_PAGE_MASK;
1822 l = (page + TARGET_PAGE_SIZE) - addr;
1823 if (l > len)
1824 l = len;
1825 flags = page_get_flags(page);
1826 if (!(flags & PAGE_VALID))
1827 return -1;
1828 if (is_write) {
1829 if (!(flags & PAGE_WRITE))
1830 return -1;
1831 /* XXX: this code should not depend on lock_user */
1832 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1833 return -1;
1834 memcpy(p, buf, l);
1835 unlock_user(p, addr, l);
1836 } else {
1837 if (!(flags & PAGE_READ))
1838 return -1;
1839 /* XXX: this code should not depend on lock_user */
1840 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1841 return -1;
1842 memcpy(buf, p, l);
1843 unlock_user(p, addr, 0);
1844 }
1845 len -= l;
1846 buf += l;
1847 addr += l;
1848 }
1849 return 0;
1850 }
1851
1852 #else
1853
1854 static void invalidate_and_set_dirty(hwaddr addr,
1855 hwaddr length)
1856 {
1857 if (!cpu_physical_memory_is_dirty(addr)) {
1858 /* invalidate code */
1859 tb_invalidate_phys_page_range(addr, addr + length, 0);
1860 /* set dirty bit */
1861 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1862 }
1863 xen_modified_memory(addr, length);
1864 }
1865
1866 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1867 {
1868 if (memory_region_is_ram(mr)) {
1869 return !(is_write && mr->readonly);
1870 }
1871 if (memory_region_is_romd(mr)) {
1872 return !is_write;
1873 }
1874
1875 return false;
1876 }
1877
1878 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1879 {
1880 unsigned access_size_max = mr->ops->valid.max_access_size;
1881
1882 /* Regions are assumed to support 1-4 byte accesses unless
1883 otherwise specified. */
1884 if (access_size_max == 0) {
1885 access_size_max = 4;
1886 }
1887
1888 /* Bound the maximum access by the alignment of the address. */
1889 if (!mr->ops->impl.unaligned) {
1890 unsigned align_size_max = addr & -addr;
1891 if (align_size_max != 0 && align_size_max < access_size_max) {
1892 access_size_max = align_size_max;
1893 }
1894 }
1895
1896 /* Don't attempt accesses larger than the maximum. */
1897 if (l > access_size_max) {
1898 l = access_size_max;
1899 }
1900 if (l & (l - 1)) {
1901 l = 1 << (qemu_fls(l) - 1);
1902 }
1903
1904 return l;
1905 }
1906
1907 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1908 int len, bool is_write)
1909 {
1910 hwaddr l;
1911 uint8_t *ptr;
1912 uint64_t val;
1913 hwaddr addr1;
1914 MemoryRegion *mr;
1915 bool error = false;
1916
1917 while (len > 0) {
1918 l = len;
1919 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1920
1921 if (is_write) {
1922 if (!memory_access_is_direct(mr, is_write)) {
1923 l = memory_access_size(mr, l, addr1);
1924 /* XXX: could force current_cpu to NULL to avoid
1925 potential bugs */
1926 switch (l) {
1927 case 8:
1928 /* 64 bit write access */
1929 val = ldq_p(buf);
1930 error |= io_mem_write(mr, addr1, val, 8);
1931 break;
1932 case 4:
1933 /* 32 bit write access */
1934 val = ldl_p(buf);
1935 error |= io_mem_write(mr, addr1, val, 4);
1936 break;
1937 case 2:
1938 /* 16 bit write access */
1939 val = lduw_p(buf);
1940 error |= io_mem_write(mr, addr1, val, 2);
1941 break;
1942 case 1:
1943 /* 8 bit write access */
1944 val = ldub_p(buf);
1945 error |= io_mem_write(mr, addr1, val, 1);
1946 break;
1947 default:
1948 abort();
1949 }
1950 } else {
1951 addr1 += memory_region_get_ram_addr(mr);
1952 /* RAM case */
1953 ptr = qemu_get_ram_ptr(addr1);
1954 memcpy(ptr, buf, l);
1955 invalidate_and_set_dirty(addr1, l);
1956 }
1957 } else {
1958 if (!memory_access_is_direct(mr, is_write)) {
1959 /* I/O case */
1960 l = memory_access_size(mr, l, addr1);
1961 switch (l) {
1962 case 8:
1963 /* 64 bit read access */
1964 error |= io_mem_read(mr, addr1, &val, 8);
1965 stq_p(buf, val);
1966 break;
1967 case 4:
1968 /* 32 bit read access */
1969 error |= io_mem_read(mr, addr1, &val, 4);
1970 stl_p(buf, val);
1971 break;
1972 case 2:
1973 /* 16 bit read access */
1974 error |= io_mem_read(mr, addr1, &val, 2);
1975 stw_p(buf, val);
1976 break;
1977 case 1:
1978 /* 8 bit read access */
1979 error |= io_mem_read(mr, addr1, &val, 1);
1980 stb_p(buf, val);
1981 break;
1982 default:
1983 abort();
1984 }
1985 } else {
1986 /* RAM case */
1987 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1988 memcpy(buf, ptr, l);
1989 }
1990 }
1991 len -= l;
1992 buf += l;
1993 addr += l;
1994 }
1995
1996 return error;
1997 }
1998
1999 bool address_space_write(AddressSpace *as, hwaddr addr,
2000 const uint8_t *buf, int len)
2001 {
2002 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2003 }
2004
2005 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2006 {
2007 return address_space_rw(as, addr, buf, len, false);
2008 }
2009
2010
2011 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2012 int len, int is_write)
2013 {
2014 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2015 }
2016
2017 /* used for ROM loading : can write in RAM and ROM */
2018 void cpu_physical_memory_write_rom(hwaddr addr,
2019 const uint8_t *buf, int len)
2020 {
2021 hwaddr l;
2022 uint8_t *ptr;
2023 hwaddr addr1;
2024 MemoryRegion *mr;
2025
2026 while (len > 0) {
2027 l = len;
2028 mr = address_space_translate(&address_space_memory,
2029 addr, &addr1, &l, true);
2030
2031 if (!(memory_region_is_ram(mr) ||
2032 memory_region_is_romd(mr))) {
2033 /* do nothing */
2034 } else {
2035 addr1 += memory_region_get_ram_addr(mr);
2036 /* ROM/RAM case */
2037 ptr = qemu_get_ram_ptr(addr1);
2038 memcpy(ptr, buf, l);
2039 invalidate_and_set_dirty(addr1, l);
2040 }
2041 len -= l;
2042 buf += l;
2043 addr += l;
2044 }
2045 }
2046
2047 typedef struct {
2048 MemoryRegion *mr;
2049 void *buffer;
2050 hwaddr addr;
2051 hwaddr len;
2052 } BounceBuffer;
2053
2054 static BounceBuffer bounce;
2055
2056 typedef struct MapClient {
2057 void *opaque;
2058 void (*callback)(void *opaque);
2059 QLIST_ENTRY(MapClient) link;
2060 } MapClient;
2061
2062 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2063 = QLIST_HEAD_INITIALIZER(map_client_list);
2064
2065 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2066 {
2067 MapClient *client = g_malloc(sizeof(*client));
2068
2069 client->opaque = opaque;
2070 client->callback = callback;
2071 QLIST_INSERT_HEAD(&map_client_list, client, link);
2072 return client;
2073 }
2074
2075 static void cpu_unregister_map_client(void *_client)
2076 {
2077 MapClient *client = (MapClient *)_client;
2078
2079 QLIST_REMOVE(client, link);
2080 g_free(client);
2081 }
2082
2083 static void cpu_notify_map_clients(void)
2084 {
2085 MapClient *client;
2086
2087 while (!QLIST_EMPTY(&map_client_list)) {
2088 client = QLIST_FIRST(&map_client_list);
2089 client->callback(client->opaque);
2090 cpu_unregister_map_client(client);
2091 }
2092 }
2093
2094 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2095 {
2096 MemoryRegion *mr;
2097 hwaddr l, xlat;
2098
2099 while (len > 0) {
2100 l = len;
2101 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2102 if (!memory_access_is_direct(mr, is_write)) {
2103 l = memory_access_size(mr, l, addr);
2104 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2105 return false;
2106 }
2107 }
2108
2109 len -= l;
2110 addr += l;
2111 }
2112 return true;
2113 }
2114
2115 /* Map a physical memory region into a host virtual address.
2116 * May map a subset of the requested range, given by and returned in *plen.
2117 * May return NULL if resources needed to perform the mapping are exhausted.
2118 * Use only for reads OR writes - not for read-modify-write operations.
2119 * Use cpu_register_map_client() to know when retrying the map operation is
2120 * likely to succeed.
2121 */
2122 void *address_space_map(AddressSpace *as,
2123 hwaddr addr,
2124 hwaddr *plen,
2125 bool is_write)
2126 {
2127 hwaddr len = *plen;
2128 hwaddr done = 0;
2129 hwaddr l, xlat, base;
2130 MemoryRegion *mr, *this_mr;
2131 ram_addr_t raddr;
2132
2133 if (len == 0) {
2134 return NULL;
2135 }
2136
2137 l = len;
2138 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2139 if (!memory_access_is_direct(mr, is_write)) {
2140 if (bounce.buffer) {
2141 return NULL;
2142 }
2143 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2144 bounce.addr = addr;
2145 bounce.len = l;
2146
2147 memory_region_ref(mr);
2148 bounce.mr = mr;
2149 if (!is_write) {
2150 address_space_read(as, addr, bounce.buffer, l);
2151 }
2152
2153 *plen = l;
2154 return bounce.buffer;
2155 }
2156
2157 base = xlat;
2158 raddr = memory_region_get_ram_addr(mr);
2159
2160 for (;;) {
2161 len -= l;
2162 addr += l;
2163 done += l;
2164 if (len == 0) {
2165 break;
2166 }
2167
2168 l = len;
2169 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2170 if (this_mr != mr || xlat != base + done) {
2171 break;
2172 }
2173 }
2174
2175 memory_region_ref(mr);
2176 *plen = done;
2177 return qemu_ram_ptr_length(raddr + base, plen);
2178 }
2179
2180 /* Unmaps a memory region previously mapped by address_space_map().
2181 * Will also mark the memory as dirty if is_write == 1. access_len gives
2182 * the amount of memory that was actually read or written by the caller.
2183 */
2184 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2185 int is_write, hwaddr access_len)
2186 {
2187 if (buffer != bounce.buffer) {
2188 MemoryRegion *mr;
2189 ram_addr_t addr1;
2190
2191 mr = qemu_ram_addr_from_host(buffer, &addr1);
2192 assert(mr != NULL);
2193 if (is_write) {
2194 while (access_len) {
2195 unsigned l;
2196 l = TARGET_PAGE_SIZE;
2197 if (l > access_len)
2198 l = access_len;
2199 invalidate_and_set_dirty(addr1, l);
2200 addr1 += l;
2201 access_len -= l;
2202 }
2203 }
2204 if (xen_enabled()) {
2205 xen_invalidate_map_cache_entry(buffer);
2206 }
2207 memory_region_unref(mr);
2208 return;
2209 }
2210 if (is_write) {
2211 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2212 }
2213 qemu_vfree(bounce.buffer);
2214 bounce.buffer = NULL;
2215 memory_region_unref(bounce.mr);
2216 cpu_notify_map_clients();
2217 }
2218
2219 void *cpu_physical_memory_map(hwaddr addr,
2220 hwaddr *plen,
2221 int is_write)
2222 {
2223 return address_space_map(&address_space_memory, addr, plen, is_write);
2224 }
2225
2226 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2227 int is_write, hwaddr access_len)
2228 {
2229 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2230 }
2231
2232 /* warning: addr must be aligned */
2233 static inline uint32_t ldl_phys_internal(hwaddr addr,
2234 enum device_endian endian)
2235 {
2236 uint8_t *ptr;
2237 uint64_t val;
2238 MemoryRegion *mr;
2239 hwaddr l = 4;
2240 hwaddr addr1;
2241
2242 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2243 false);
2244 if (l < 4 || !memory_access_is_direct(mr, false)) {
2245 /* I/O case */
2246 io_mem_read(mr, addr1, &val, 4);
2247 #if defined(TARGET_WORDS_BIGENDIAN)
2248 if (endian == DEVICE_LITTLE_ENDIAN) {
2249 val = bswap32(val);
2250 }
2251 #else
2252 if (endian == DEVICE_BIG_ENDIAN) {
2253 val = bswap32(val);
2254 }
2255 #endif
2256 } else {
2257 /* RAM case */
2258 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2259 & TARGET_PAGE_MASK)
2260 + addr1);
2261 switch (endian) {
2262 case DEVICE_LITTLE_ENDIAN:
2263 val = ldl_le_p(ptr);
2264 break;
2265 case DEVICE_BIG_ENDIAN:
2266 val = ldl_be_p(ptr);
2267 break;
2268 default:
2269 val = ldl_p(ptr);
2270 break;
2271 }
2272 }
2273 return val;
2274 }
2275
2276 uint32_t ldl_phys(hwaddr addr)
2277 {
2278 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2279 }
2280
2281 uint32_t ldl_le_phys(hwaddr addr)
2282 {
2283 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2284 }
2285
2286 uint32_t ldl_be_phys(hwaddr addr)
2287 {
2288 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2289 }
2290
2291 /* warning: addr must be aligned */
2292 static inline uint64_t ldq_phys_internal(hwaddr addr,
2293 enum device_endian endian)
2294 {
2295 uint8_t *ptr;
2296 uint64_t val;
2297 MemoryRegion *mr;
2298 hwaddr l = 8;
2299 hwaddr addr1;
2300
2301 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2302 false);
2303 if (l < 8 || !memory_access_is_direct(mr, false)) {
2304 /* I/O case */
2305 io_mem_read(mr, addr1, &val, 8);
2306 #if defined(TARGET_WORDS_BIGENDIAN)
2307 if (endian == DEVICE_LITTLE_ENDIAN) {
2308 val = bswap64(val);
2309 }
2310 #else
2311 if (endian == DEVICE_BIG_ENDIAN) {
2312 val = bswap64(val);
2313 }
2314 #endif
2315 } else {
2316 /* RAM case */
2317 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2318 & TARGET_PAGE_MASK)
2319 + addr1);
2320 switch (endian) {
2321 case DEVICE_LITTLE_ENDIAN:
2322 val = ldq_le_p(ptr);
2323 break;
2324 case DEVICE_BIG_ENDIAN:
2325 val = ldq_be_p(ptr);
2326 break;
2327 default:
2328 val = ldq_p(ptr);
2329 break;
2330 }
2331 }
2332 return val;
2333 }
2334
2335 uint64_t ldq_phys(hwaddr addr)
2336 {
2337 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2338 }
2339
2340 uint64_t ldq_le_phys(hwaddr addr)
2341 {
2342 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2343 }
2344
2345 uint64_t ldq_be_phys(hwaddr addr)
2346 {
2347 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2348 }
2349
2350 /* XXX: optimize */
2351 uint32_t ldub_phys(hwaddr addr)
2352 {
2353 uint8_t val;
2354 cpu_physical_memory_read(addr, &val, 1);
2355 return val;
2356 }
2357
2358 /* warning: addr must be aligned */
2359 static inline uint32_t lduw_phys_internal(hwaddr addr,
2360 enum device_endian endian)
2361 {
2362 uint8_t *ptr;
2363 uint64_t val;
2364 MemoryRegion *mr;
2365 hwaddr l = 2;
2366 hwaddr addr1;
2367
2368 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2369 false);
2370 if (l < 2 || !memory_access_is_direct(mr, false)) {
2371 /* I/O case */
2372 io_mem_read(mr, addr1, &val, 2);
2373 #if defined(TARGET_WORDS_BIGENDIAN)
2374 if (endian == DEVICE_LITTLE_ENDIAN) {
2375 val = bswap16(val);
2376 }
2377 #else
2378 if (endian == DEVICE_BIG_ENDIAN) {
2379 val = bswap16(val);
2380 }
2381 #endif
2382 } else {
2383 /* RAM case */
2384 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2385 & TARGET_PAGE_MASK)
2386 + addr1);
2387 switch (endian) {
2388 case DEVICE_LITTLE_ENDIAN:
2389 val = lduw_le_p(ptr);
2390 break;
2391 case DEVICE_BIG_ENDIAN:
2392 val = lduw_be_p(ptr);
2393 break;
2394 default:
2395 val = lduw_p(ptr);
2396 break;
2397 }
2398 }
2399 return val;
2400 }
2401
2402 uint32_t lduw_phys(hwaddr addr)
2403 {
2404 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2405 }
2406
2407 uint32_t lduw_le_phys(hwaddr addr)
2408 {
2409 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2410 }
2411
2412 uint32_t lduw_be_phys(hwaddr addr)
2413 {
2414 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2415 }
2416
2417 /* warning: addr must be aligned. The ram page is not masked as dirty
2418 and the code inside is not invalidated. It is useful if the dirty
2419 bits are used to track modified PTEs */
2420 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2421 {
2422 uint8_t *ptr;
2423 MemoryRegion *mr;
2424 hwaddr l = 4;
2425 hwaddr addr1;
2426
2427 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2428 true);
2429 if (l < 4 || !memory_access_is_direct(mr, true)) {
2430 io_mem_write(mr, addr1, val, 4);
2431 } else {
2432 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2433 ptr = qemu_get_ram_ptr(addr1);
2434 stl_p(ptr, val);
2435
2436 if (unlikely(in_migration)) {
2437 if (!cpu_physical_memory_is_dirty(addr1)) {
2438 /* invalidate code */
2439 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2440 /* set dirty bit */
2441 cpu_physical_memory_set_dirty_flags(
2442 addr1, (0xff & ~CODE_DIRTY_FLAG));
2443 }
2444 }
2445 }
2446 }
2447
2448 /* warning: addr must be aligned */
2449 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2450 enum device_endian endian)
2451 {
2452 uint8_t *ptr;
2453 MemoryRegion *mr;
2454 hwaddr l = 4;
2455 hwaddr addr1;
2456
2457 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2458 true);
2459 if (l < 4 || !memory_access_is_direct(mr, true)) {
2460 #if defined(TARGET_WORDS_BIGENDIAN)
2461 if (endian == DEVICE_LITTLE_ENDIAN) {
2462 val = bswap32(val);
2463 }
2464 #else
2465 if (endian == DEVICE_BIG_ENDIAN) {
2466 val = bswap32(val);
2467 }
2468 #endif
2469 io_mem_write(mr, addr1, val, 4);
2470 } else {
2471 /* RAM case */
2472 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2473 ptr = qemu_get_ram_ptr(addr1);
2474 switch (endian) {
2475 case DEVICE_LITTLE_ENDIAN:
2476 stl_le_p(ptr, val);
2477 break;
2478 case DEVICE_BIG_ENDIAN:
2479 stl_be_p(ptr, val);
2480 break;
2481 default:
2482 stl_p(ptr, val);
2483 break;
2484 }
2485 invalidate_and_set_dirty(addr1, 4);
2486 }
2487 }
2488
2489 void stl_phys(hwaddr addr, uint32_t val)
2490 {
2491 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2492 }
2493
2494 void stl_le_phys(hwaddr addr, uint32_t val)
2495 {
2496 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2497 }
2498
2499 void stl_be_phys(hwaddr addr, uint32_t val)
2500 {
2501 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2502 }
2503
2504 /* XXX: optimize */
2505 void stb_phys(hwaddr addr, uint32_t val)
2506 {
2507 uint8_t v = val;
2508 cpu_physical_memory_write(addr, &v, 1);
2509 }
2510
2511 /* warning: addr must be aligned */
2512 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2513 enum device_endian endian)
2514 {
2515 uint8_t *ptr;
2516 MemoryRegion *mr;
2517 hwaddr l = 2;
2518 hwaddr addr1;
2519
2520 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2521 true);
2522 if (l < 2 || !memory_access_is_direct(mr, true)) {
2523 #if defined(TARGET_WORDS_BIGENDIAN)
2524 if (endian == DEVICE_LITTLE_ENDIAN) {
2525 val = bswap16(val);
2526 }
2527 #else
2528 if (endian == DEVICE_BIG_ENDIAN) {
2529 val = bswap16(val);
2530 }
2531 #endif
2532 io_mem_write(mr, addr1, val, 2);
2533 } else {
2534 /* RAM case */
2535 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2536 ptr = qemu_get_ram_ptr(addr1);
2537 switch (endian) {
2538 case DEVICE_LITTLE_ENDIAN:
2539 stw_le_p(ptr, val);
2540 break;
2541 case DEVICE_BIG_ENDIAN:
2542 stw_be_p(ptr, val);
2543 break;
2544 default:
2545 stw_p(ptr, val);
2546 break;
2547 }
2548 invalidate_and_set_dirty(addr1, 2);
2549 }
2550 }
2551
2552 void stw_phys(hwaddr addr, uint32_t val)
2553 {
2554 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2555 }
2556
2557 void stw_le_phys(hwaddr addr, uint32_t val)
2558 {
2559 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2560 }
2561
2562 void stw_be_phys(hwaddr addr, uint32_t val)
2563 {
2564 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2565 }
2566
2567 /* XXX: optimize */
2568 void stq_phys(hwaddr addr, uint64_t val)
2569 {
2570 val = tswap64(val);
2571 cpu_physical_memory_write(addr, &val, 8);
2572 }
2573
2574 void stq_le_phys(hwaddr addr, uint64_t val)
2575 {
2576 val = cpu_to_le64(val);
2577 cpu_physical_memory_write(addr, &val, 8);
2578 }
2579
2580 void stq_be_phys(hwaddr addr, uint64_t val)
2581 {
2582 val = cpu_to_be64(val);
2583 cpu_physical_memory_write(addr, &val, 8);
2584 }
2585
2586 /* virtual memory access for debug (includes writing to ROM) */
2587 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2588 uint8_t *buf, int len, int is_write)
2589 {
2590 int l;
2591 hwaddr phys_addr;
2592 target_ulong page;
2593
2594 while (len > 0) {
2595 page = addr & TARGET_PAGE_MASK;
2596 phys_addr = cpu_get_phys_page_debug(cpu, page);
2597 /* if no physical page mapped, return an error */
2598 if (phys_addr == -1)
2599 return -1;
2600 l = (page + TARGET_PAGE_SIZE) - addr;
2601 if (l > len)
2602 l = len;
2603 phys_addr += (addr & ~TARGET_PAGE_MASK);
2604 if (is_write)
2605 cpu_physical_memory_write_rom(phys_addr, buf, l);
2606 else
2607 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2608 len -= l;
2609 buf += l;
2610 addr += l;
2611 }
2612 return 0;
2613 }
2614 #endif
2615
2616 #if !defined(CONFIG_USER_ONLY)
2617
2618 /*
2619 * A helper function for the _utterly broken_ virtio device model to find out if
2620 * it's running on a big endian machine. Don't do this at home kids!
2621 */
2622 bool virtio_is_big_endian(void);
2623 bool virtio_is_big_endian(void)
2624 {
2625 #if defined(TARGET_WORDS_BIGENDIAN)
2626 return true;
2627 #else
2628 return false;
2629 #endif
2630 }
2631
2632 #endif
2633
2634 #ifndef CONFIG_USER_ONLY
2635 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2636 {
2637 MemoryRegion*mr;
2638 hwaddr l = 1;
2639
2640 mr = address_space_translate(&address_space_memory,
2641 phys_addr, &phys_addr, &l, false);
2642
2643 return !(memory_region_is_ram(mr) ||
2644 memory_region_is_romd(mr));
2645 }
2646
2647 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2648 {
2649 RAMBlock *block;
2650
2651 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2652 func(block->host, block->offset, block->length, opaque);
2653 }
2654 }
2655 #endif
QEMU modified to work with kvm