?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge remote-tracking branch 'kraxel/chardev.6' into staging
[qemu.git] / exec.c
blob3a9ddcb41f4c477c8247ad9ebfa7ebc061935bae
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "hw/xen/xen.h"
35 #include "qemu/timer.h"
36 #include "qemu/config-file.h"
37 #include "exec/memory.h"
38 #include "sysemu/dma.h"
39 #include "exec/address-spaces.h"
40 #if defined(CONFIG_USER_ONLY)
41 #include <qemu.h>
42 #else /* !CONFIG_USER_ONLY */
43 #include "sysemu/xen-mapcache.h"
44 #include "trace.h"
45 #endif
46 #include "exec/cpu-all.h"
47
48 #include "exec/cputlb.h"
49 #include "translate-all.h"
50
51 #include "exec/memory-internal.h"
52
53 //#define DEBUG_UNASSIGNED
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 int phys_ram_fd;
58 static int in_migration;
59
60 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
61
62 static MemoryRegion *system_memory;
63 static MemoryRegion *system_io;
64
65 AddressSpace address_space_io;
66 AddressSpace address_space_memory;
67 DMAContext dma_context_memory;
68
69 MemoryRegion io_mem_ram, io_mem_rom, io_mem_unassigned, io_mem_notdirty;
70 static MemoryRegion io_mem_subpage_ram;
71
72 #endif
73
74 CPUArchState *first_cpu;
75 /* current CPU in the current thread. It is only valid inside
76 cpu_exec() */
77 DEFINE_TLS(CPUArchState *,cpu_single_env);
78 /* 0 = Do not count executed instructions.
79 1 = Precise instruction counting.
80 2 = Adaptive rate instruction counting. */
81 int use_icount;
82
83 #if !defined(CONFIG_USER_ONLY)
84
85 static MemoryRegionSection *phys_sections;
86 static unsigned phys_sections_nb, phys_sections_nb_alloc;
87 static uint16_t phys_section_unassigned;
88 static uint16_t phys_section_notdirty;
89 static uint16_t phys_section_rom;
90 static uint16_t phys_section_watch;
91
92 /* Simple allocator for PhysPageEntry nodes */
93 static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
94 static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
95
96 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
97
98 static void io_mem_init(void);
99 static void memory_map_init(void);
100 static void *qemu_safe_ram_ptr(ram_addr_t addr);
101
102 static MemoryRegion io_mem_watch;
103 #endif
104
105 #if !defined(CONFIG_USER_ONLY)
106
107 static void phys_map_node_reserve(unsigned nodes)
108 {
109 if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) {
110 typedef PhysPageEntry Node[L2_SIZE];
111 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16);
112 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc,
113 phys_map_nodes_nb + nodes);
114 phys_map_nodes = g_renew(Node, phys_map_nodes,
115 phys_map_nodes_nb_alloc);
116 }
117 }
118
119 static uint16_t phys_map_node_alloc(void)
120 {
121 unsigned i;
122 uint16_t ret;
123
124 ret = phys_map_nodes_nb++;
125 assert(ret != PHYS_MAP_NODE_NIL);
126 assert(ret != phys_map_nodes_nb_alloc);
127 for (i = 0; i < L2_SIZE; ++i) {
128 phys_map_nodes[ret][i].is_leaf = 0;
129 phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
130 }
131 return ret;
132 }
133
134 static void phys_map_nodes_reset(void)
135 {
136 phys_map_nodes_nb = 0;
137 }
138
139
140 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
141 hwaddr *nb, uint16_t leaf,
142 int level)
143 {
144 PhysPageEntry *p;
145 int i;
146 hwaddr step = (hwaddr)1 << (level * L2_BITS);
147
148 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
149 lp->ptr = phys_map_node_alloc();
150 p = phys_map_nodes[lp->ptr];
151 if (level == 0) {
152 for (i = 0; i < L2_SIZE; i++) {
153 p[i].is_leaf = 1;
154 p[i].ptr = phys_section_unassigned;
155 }
156 }
157 } else {
158 p = phys_map_nodes[lp->ptr];
159 }
160 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
161
162 while (*nb && lp < &p[L2_SIZE]) {
163 if ((*index & (step - 1)) == 0 && *nb >= step) {
164 lp->is_leaf = true;
165 lp->ptr = leaf;
166 *index += step;
167 *nb -= step;
168 } else {
169 phys_page_set_level(lp, index, nb, leaf, level - 1);
170 }
171 ++lp;
172 }
173 }
174
175 static void phys_page_set(AddressSpaceDispatch *d,
176 hwaddr index, hwaddr nb,
177 uint16_t leaf)
178 {
179 /* Wildly overreserve - it doesn't matter much. */
180 phys_map_node_reserve(3 * P_L2_LEVELS);
181
182 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
183 }
184
185 MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr index)
186 {
187 PhysPageEntry lp = d->phys_map;
188 PhysPageEntry *p;
189 int i;
190
191 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
192 if (lp.ptr == PHYS_MAP_NODE_NIL) {
193 return &phys_sections[phys_section_unassigned];
194 }
195 p = phys_map_nodes[lp.ptr];
196 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
197 }
198 return &phys_sections[lp.ptr];
199 }
200
201 bool memory_region_is_unassigned(MemoryRegion *mr)
202 {
203 return mr != &io_mem_ram && mr != &io_mem_rom
204 && mr != &io_mem_notdirty && !mr->rom_device
205 && mr != &io_mem_watch;
206 }
207 #endif
208
209 void cpu_exec_init_all(void)
210 {
211 #if !defined(CONFIG_USER_ONLY)
212 qemu_mutex_init(&ram_list.mutex);
213 memory_map_init();
214 io_mem_init();
215 #endif
216 }
217
218 #if !defined(CONFIG_USER_ONLY)
219
220 static int cpu_common_post_load(void *opaque, int version_id)
221 {
222 CPUState *cpu = opaque;
223
224 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
225 version_id is increased. */
226 cpu->interrupt_request &= ~0x01;
227 tlb_flush(cpu->env_ptr, 1);
228
229 return 0;
230 }
231
232 static const VMStateDescription vmstate_cpu_common = {
233 .name = "cpu_common",
234 .version_id = 1,
235 .minimum_version_id = 1,
236 .minimum_version_id_old = 1,
237 .post_load = cpu_common_post_load,
238 .fields = (VMStateField []) {
239 VMSTATE_UINT32(halted, CPUState),
240 VMSTATE_UINT32(interrupt_request, CPUState),
241 VMSTATE_END_OF_LIST()
242 }
243 };
244 #else
245 #define vmstate_cpu_common vmstate_dummy
246 #endif
247
248 CPUState *qemu_get_cpu(int index)
249 {
250 CPUArchState *env = first_cpu;
251 CPUState *cpu = NULL;
252
253 while (env) {
254 cpu = ENV_GET_CPU(env);
255 if (cpu->cpu_index == index) {
256 break;
257 }
258 env = env->next_cpu;
259 }
260
261 return env ? cpu : NULL;
262 }
263
264 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
265 {
266 CPUArchState *env = first_cpu;
267
268 while (env) {
269 func(ENV_GET_CPU(env), data);
270 env = env->next_cpu;
271 }
272 }
273
274 void cpu_exec_init(CPUArchState *env)
275 {
276 CPUState *cpu = ENV_GET_CPU(env);
277 CPUClass *cc = CPU_GET_CLASS(cpu);
278 CPUArchState **penv;
279 int cpu_index;
280
281 #if defined(CONFIG_USER_ONLY)
282 cpu_list_lock();
283 #endif
284 env->next_cpu = NULL;
285 penv = &first_cpu;
286 cpu_index = 0;
287 while (*penv != NULL) {
288 penv = &(*penv)->next_cpu;
289 cpu_index++;
290 }
291 cpu->cpu_index = cpu_index;
292 cpu->numa_node = 0;
293 QTAILQ_INIT(&env->breakpoints);
294 QTAILQ_INIT(&env->watchpoints);
295 #ifndef CONFIG_USER_ONLY
296 cpu->thread_id = qemu_get_thread_id();
297 #endif
298 *penv = env;
299 #if defined(CONFIG_USER_ONLY)
300 cpu_list_unlock();
301 #endif
302 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
303 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
304 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
305 cpu_save, cpu_load, env);
306 assert(cc->vmsd == NULL);
307 #endif
308 if (cc->vmsd != NULL) {
309 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
310 }
311 }
312
313 #if defined(TARGET_HAS_ICE)
314 #if defined(CONFIG_USER_ONLY)
315 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
316 {
317 tb_invalidate_phys_page_range(pc, pc + 1, 0);
318 }
319 #else
320 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
321 {
322 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
323 (pc & ~TARGET_PAGE_MASK));
324 }
325 #endif
326 #endif /* TARGET_HAS_ICE */
327
328 #if defined(CONFIG_USER_ONLY)
329 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
330
331 {
332 }
333
334 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
335 int flags, CPUWatchpoint **watchpoint)
336 {
337 return -ENOSYS;
338 }
339 #else
340 /* Add a watchpoint. */
341 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
342 int flags, CPUWatchpoint **watchpoint)
343 {
344 target_ulong len_mask = ~(len - 1);
345 CPUWatchpoint *wp;
346
347 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
348 if ((len & (len - 1)) || (addr & ~len_mask) ||
349 len == 0 || len > TARGET_PAGE_SIZE) {
350 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
351 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
352 return -EINVAL;
353 }
354 wp = g_malloc(sizeof(*wp));
355
356 wp->vaddr = addr;
357 wp->len_mask = len_mask;
358 wp->flags = flags;
359
360 /* keep all GDB-injected watchpoints in front */
361 if (flags & BP_GDB)
362 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
363 else
364 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
365
366 tlb_flush_page(env, addr);
367
368 if (watchpoint)
369 *watchpoint = wp;
370 return 0;
371 }
372
373 /* Remove a specific watchpoint. */
374 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
375 int flags)
376 {
377 target_ulong len_mask = ~(len - 1);
378 CPUWatchpoint *wp;
379
380 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
381 if (addr == wp->vaddr && len_mask == wp->len_mask
382 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
383 cpu_watchpoint_remove_by_ref(env, wp);
384 return 0;
385 }
386 }
387 return -ENOENT;
388 }
389
390 /* Remove a specific watchpoint by reference. */
391 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
392 {
393 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
394
395 tlb_flush_page(env, watchpoint->vaddr);
396
397 g_free(watchpoint);
398 }
399
400 /* Remove all matching watchpoints. */
401 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
402 {
403 CPUWatchpoint *wp, *next;
404
405 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
406 if (wp->flags & mask)
407 cpu_watchpoint_remove_by_ref(env, wp);
408 }
409 }
410 #endif
411
412 /* Add a breakpoint. */
413 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
414 CPUBreakpoint **breakpoint)
415 {
416 #if defined(TARGET_HAS_ICE)
417 CPUBreakpoint *bp;
418
419 bp = g_malloc(sizeof(*bp));
420
421 bp->pc = pc;
422 bp->flags = flags;
423
424 /* keep all GDB-injected breakpoints in front */
425 if (flags & BP_GDB)
426 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
427 else
428 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
429
430 breakpoint_invalidate(env, pc);
431
432 if (breakpoint)
433 *breakpoint = bp;
434 return 0;
435 #else
436 return -ENOSYS;
437 #endif
438 }
439
440 /* Remove a specific breakpoint. */
441 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
442 {
443 #if defined(TARGET_HAS_ICE)
444 CPUBreakpoint *bp;
445
446 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
447 if (bp->pc == pc && bp->flags == flags) {
448 cpu_breakpoint_remove_by_ref(env, bp);
449 return 0;
450 }
451 }
452 return -ENOENT;
453 #else
454 return -ENOSYS;
455 #endif
456 }
457
458 /* Remove a specific breakpoint by reference. */
459 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
460 {
461 #if defined(TARGET_HAS_ICE)
462 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
463
464 breakpoint_invalidate(env, breakpoint->pc);
465
466 g_free(breakpoint);
467 #endif
468 }
469
470 /* Remove all matching breakpoints. */
471 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
472 {
473 #if defined(TARGET_HAS_ICE)
474 CPUBreakpoint *bp, *next;
475
476 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
477 if (bp->flags & mask)
478 cpu_breakpoint_remove_by_ref(env, bp);
479 }
480 #endif
481 }
482
483 /* enable or disable single step mode. EXCP_DEBUG is returned by the
484 CPU loop after each instruction */
485 void cpu_single_step(CPUArchState *env, int enabled)
486 {
487 #if defined(TARGET_HAS_ICE)
488 if (env->singlestep_enabled != enabled) {
489 env->singlestep_enabled = enabled;
490 if (kvm_enabled())
491 kvm_update_guest_debug(env, 0);
492 else {
493 /* must flush all the translated code to avoid inconsistencies */
494 /* XXX: only flush what is necessary */
495 tb_flush(env);
496 }
497 }
498 #endif
499 }
500
501 void cpu_exit(CPUArchState *env)
502 {
503 CPUState *cpu = ENV_GET_CPU(env);
504
505 cpu->exit_request = 1;
506 cpu->tcg_exit_req = 1;
507 }
508
509 void cpu_abort(CPUArchState *env, const char *fmt, ...)
510 {
511 va_list ap;
512 va_list ap2;
513
514 va_start(ap, fmt);
515 va_copy(ap2, ap);
516 fprintf(stderr, "qemu: fatal: ");
517 vfprintf(stderr, fmt, ap);
518 fprintf(stderr, "\n");
519 cpu_dump_state(env, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
520 if (qemu_log_enabled()) {
521 qemu_log("qemu: fatal: ");
522 qemu_log_vprintf(fmt, ap2);
523 qemu_log("\n");
524 log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
525 qemu_log_flush();
526 qemu_log_close();
527 }
528 va_end(ap2);
529 va_end(ap);
530 #if defined(CONFIG_USER_ONLY)
531 {
532 struct sigaction act;
533 sigfillset(&act.sa_mask);
534 act.sa_handler = SIG_DFL;
535 sigaction(SIGABRT, &act, NULL);
536 }
537 #endif
538 abort();
539 }
540
541 CPUArchState *cpu_copy(CPUArchState *env)
542 {
543 CPUArchState *new_env = cpu_init(env->cpu_model_str);
544 CPUArchState *next_cpu = new_env->next_cpu;
545 #if defined(TARGET_HAS_ICE)
546 CPUBreakpoint *bp;
547 CPUWatchpoint *wp;
548 #endif
549
550 memcpy(new_env, env, sizeof(CPUArchState));
551
552 /* Preserve chaining. */
553 new_env->next_cpu = next_cpu;
554
555 /* Clone all break/watchpoints.
556 Note: Once we support ptrace with hw-debug register access, make sure
557 BP_CPU break/watchpoints are handled correctly on clone. */
558 QTAILQ_INIT(&env->breakpoints);
559 QTAILQ_INIT(&env->watchpoints);
560 #if defined(TARGET_HAS_ICE)
561 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
562 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
563 }
564 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
565 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
566 wp->flags, NULL);
567 }
568 #endif
569
570 return new_env;
571 }
572
573 #if !defined(CONFIG_USER_ONLY)
574 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
575 uintptr_t length)
576 {
577 uintptr_t start1;
578
579 /* we modify the TLB cache so that the dirty bit will be set again
580 when accessing the range */
581 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
582 /* Check that we don't span multiple blocks - this breaks the
583 address comparisons below. */
584 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
585 != (end - 1) - start) {
586 abort();
587 }
588 cpu_tlb_reset_dirty_all(start1, length);
589
590 }
591
592 /* Note: start and end must be within the same ram block. */
593 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
594 int dirty_flags)
595 {
596 uintptr_t length;
597
598 start &= TARGET_PAGE_MASK;
599 end = TARGET_PAGE_ALIGN(end);
600
601 length = end - start;
602 if (length == 0)
603 return;
604 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
605
606 if (tcg_enabled()) {
607 tlb_reset_dirty_range_all(start, end, length);
608 }
609 }
610
611 static int cpu_physical_memory_set_dirty_tracking(int enable)
612 {
613 int ret = 0;
614 in_migration = enable;
615 return ret;
616 }
617
618 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
619 MemoryRegionSection *section,
620 target_ulong vaddr,
621 hwaddr paddr,
622 int prot,
623 target_ulong *address)
624 {
625 hwaddr iotlb;
626 CPUWatchpoint *wp;
627
628 if (memory_region_is_ram(section->mr)) {
629 /* Normal RAM. */
630 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
631 + memory_region_section_addr(section, paddr);
632 if (!section->readonly) {
633 iotlb |= phys_section_notdirty;
634 } else {
635 iotlb |= phys_section_rom;
636 }
637 } else {
638 iotlb = section - phys_sections;
639 iotlb += memory_region_section_addr(section, paddr);
640 }
641
642 /* Make accesses to pages with watchpoints go via the
643 watchpoint trap routines. */
644 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
645 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
646 /* Avoid trapping reads of pages with a write breakpoint. */
647 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
648 iotlb = phys_section_watch + paddr;
649 *address |= TLB_MMIO;
650 break;
651 }
652 }
653 }
654
655 return iotlb;
656 }
657 #endif /* defined(CONFIG_USER_ONLY) */
658
659 #if !defined(CONFIG_USER_ONLY)
660
661 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
662 typedef struct subpage_t {
663 MemoryRegion iomem;
664 hwaddr base;
665 uint16_t sub_section[TARGET_PAGE_SIZE];
666 } subpage_t;
667
668 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
669 uint16_t section);
670 static subpage_t *subpage_init(hwaddr base);
671 static void destroy_page_desc(uint16_t section_index)
672 {
673 MemoryRegionSection *section = &phys_sections[section_index];
674 MemoryRegion *mr = section->mr;
675
676 if (mr->subpage) {
677 subpage_t *subpage = container_of(mr, subpage_t, iomem);
678 memory_region_destroy(&subpage->iomem);
679 g_free(subpage);
680 }
681 }
682
683 static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)
684 {
685 unsigned i;
686 PhysPageEntry *p;
687
688 if (lp->ptr == PHYS_MAP_NODE_NIL) {
689 return;
690 }
691
692 p = phys_map_nodes[lp->ptr];
693 for (i = 0; i < L2_SIZE; ++i) {
694 if (!p[i].is_leaf) {
695 destroy_l2_mapping(&p[i], level - 1);
696 } else {
697 destroy_page_desc(p[i].ptr);
698 }
699 }
700 lp->is_leaf = 0;
701 lp->ptr = PHYS_MAP_NODE_NIL;
702 }
703
704 static void destroy_all_mappings(AddressSpaceDispatch *d)
705 {
706 destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
707 phys_map_nodes_reset();
708 }
709
710 static uint16_t phys_section_add(MemoryRegionSection *section)
711 {
712 /* The physical section number is ORed with a page-aligned
713 * pointer to produce the iotlb entries. Thus it should
714 * never overflow into the page-aligned value.
715 */
716 assert(phys_sections_nb < TARGET_PAGE_SIZE);
717
718 if (phys_sections_nb == phys_sections_nb_alloc) {
719 phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16);
720 phys_sections = g_renew(MemoryRegionSection, phys_sections,
721 phys_sections_nb_alloc);
722 }
723 phys_sections[phys_sections_nb] = *section;
724 return phys_sections_nb++;
725 }
726
727 static void phys_sections_clear(void)
728 {
729 phys_sections_nb = 0;
730 }
731
732 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
733 {
734 subpage_t *subpage;
735 hwaddr base = section->offset_within_address_space
736 & TARGET_PAGE_MASK;
737 MemoryRegionSection *existing = phys_page_find(d, base >> TARGET_PAGE_BITS);
738 MemoryRegionSection subsection = {
739 .offset_within_address_space = base,
740 .size = TARGET_PAGE_SIZE,
741 };
742 hwaddr start, end;
743
744 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
745
746 if (!(existing->mr->subpage)) {
747 subpage = subpage_init(base);
748 subsection.mr = &subpage->iomem;
749 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
750 phys_section_add(&subsection));
751 } else {
752 subpage = container_of(existing->mr, subpage_t, iomem);
753 }
754 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
755 end = start + section->size - 1;
756 subpage_register(subpage, start, end, phys_section_add(section));
757 }
758
759
760 static void register_multipage(AddressSpaceDispatch *d, MemoryRegionSection *section)
761 {
762 hwaddr start_addr = section->offset_within_address_space;
763 ram_addr_t size = section->size;
764 hwaddr addr;
765 uint16_t section_index = phys_section_add(section);
766
767 assert(size);
768
769 addr = start_addr;
770 phys_page_set(d, addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,
771 section_index);
772 }
773
774 QEMU_BUILD_BUG_ON(TARGET_PHYS_ADDR_SPACE_BITS > MAX_PHYS_ADDR_SPACE_BITS)
775
776 static MemoryRegionSection limit(MemoryRegionSection section)
777 {
778 section.size = MIN(section.offset_within_address_space + section.size,
779 MAX_PHYS_ADDR + 1)
780 - section.offset_within_address_space;
781
782 return section;
783 }
784
785 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
786 {
787 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
788 MemoryRegionSection now = limit(*section), remain = limit(*section);
789
790 if ((now.offset_within_address_space & ~TARGET_PAGE_MASK)
791 || (now.size < TARGET_PAGE_SIZE)) {
792 now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space)
793 - now.offset_within_address_space,
794 now.size);
795 register_subpage(d, &now);
796 remain.size -= now.size;
797 remain.offset_within_address_space += now.size;
798 remain.offset_within_region += now.size;
799 }
800 while (remain.size >= TARGET_PAGE_SIZE) {
801 now = remain;
802 if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
803 now.size = TARGET_PAGE_SIZE;
804 register_subpage(d, &now);
805 } else {
806 now.size &= TARGET_PAGE_MASK;
807 register_multipage(d, &now);
808 }
809 remain.size -= now.size;
810 remain.offset_within_address_space += now.size;
811 remain.offset_within_region += now.size;
812 }
813 now = remain;
814 if (now.size) {
815 register_subpage(d, &now);
816 }
817 }
818
819 void qemu_flush_coalesced_mmio_buffer(void)
820 {
821 if (kvm_enabled())
822 kvm_flush_coalesced_mmio_buffer();
823 }
824
825 void qemu_mutex_lock_ramlist(void)
826 {
827 qemu_mutex_lock(&ram_list.mutex);
828 }
829
830 void qemu_mutex_unlock_ramlist(void)
831 {
832 qemu_mutex_unlock(&ram_list.mutex);
833 }
834
835 #if defined(__linux__) && !defined(TARGET_S390X)
836
837 #include <sys/vfs.h>
838
839 #define HUGETLBFS_MAGIC 0x958458f6
840
841 static long gethugepagesize(const char *path)
842 {
843 struct statfs fs;
844 int ret;
845
846 do {
847 ret = statfs(path, &fs);
848 } while (ret != 0 && errno == EINTR);
849
850 if (ret != 0) {
851 perror(path);
852 return 0;
853 }
854
855 if (fs.f_type != HUGETLBFS_MAGIC)
856 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
857
858 return fs.f_bsize;
859 }
860
861 static void *file_ram_alloc(RAMBlock *block,
862 ram_addr_t memory,
863 const char *path)
864 {
865 char *filename;
866 char *sanitized_name;
867 char *c;
868 void *area;
869 int fd;
870 #ifdef MAP_POPULATE
871 int flags;
872 #endif
873 unsigned long hpagesize;
874
875 hpagesize = gethugepagesize(path);
876 if (!hpagesize) {
877 return NULL;
878 }
879
880 if (memory < hpagesize) {
881 return NULL;
882 }
883
884 if (kvm_enabled() && !kvm_has_sync_mmu()) {
885 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
886 return NULL;
887 }
888
889 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
890 sanitized_name = g_strdup(block->mr->name);
891 for (c = sanitized_name; *c != '\0'; c++) {
892 if (*c == '/')
893 *c = '_';
894 }
895
896 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
897 sanitized_name);
898 g_free(sanitized_name);
899
900 fd = mkstemp(filename);
901 if (fd < 0) {
902 perror("unable to create backing store for hugepages");
903 g_free(filename);
904 return NULL;
905 }
906 unlink(filename);
907 g_free(filename);
908
909 memory = (memory+hpagesize-1) & ~(hpagesize-1);
910
911 /*
912 * ftruncate is not supported by hugetlbfs in older
913 * hosts, so don't bother bailing out on errors.
914 * If anything goes wrong with it under other filesystems,
915 * mmap will fail.
916 */
917 if (ftruncate(fd, memory))
918 perror("ftruncate");
919
920 #ifdef MAP_POPULATE
921 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
922 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
923 * to sidestep this quirk.
924 */
925 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
926 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
927 #else
928 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
929 #endif
930 if (area == MAP_FAILED) {
931 perror("file_ram_alloc: can't mmap RAM pages");
932 close(fd);
933 return (NULL);
934 }
935 block->fd = fd;
936 return area;
937 }
938 #endif
939
940 static ram_addr_t find_ram_offset(ram_addr_t size)
941 {
942 RAMBlock *block, *next_block;
943 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
944
945 assert(size != 0); /* it would hand out same offset multiple times */
946
947 if (QTAILQ_EMPTY(&ram_list.blocks))
948 return 0;
949
950 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
951 ram_addr_t end, next = RAM_ADDR_MAX;
952
953 end = block->offset + block->length;
954
955 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
956 if (next_block->offset >= end) {
957 next = MIN(next, next_block->offset);
958 }
959 }
960 if (next - end >= size && next - end < mingap) {
961 offset = end;
962 mingap = next - end;
963 }
964 }
965
966 if (offset == RAM_ADDR_MAX) {
967 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
968 (uint64_t)size);
969 abort();
970 }
971
972 return offset;
973 }
974
975 ram_addr_t last_ram_offset(void)
976 {
977 RAMBlock *block;
978 ram_addr_t last = 0;
979
980 QTAILQ_FOREACH(block, &ram_list.blocks, next)
981 last = MAX(last, block->offset + block->length);
982
983 return last;
984 }
985
986 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
987 {
988 int ret;
989 QemuOpts *machine_opts;
990
991 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
992 machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
993 if (machine_opts &&
994 !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
995 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
996 if (ret) {
997 perror("qemu_madvise");
998 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
999 "but dump_guest_core=off specified\n");
1000 }
1001 }
1002 }
1003
1004 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1005 {
1006 RAMBlock *new_block, *block;
1007
1008 new_block = NULL;
1009 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1010 if (block->offset == addr) {
1011 new_block = block;
1012 break;
1013 }
1014 }
1015 assert(new_block);
1016 assert(!new_block->idstr[0]);
1017
1018 if (dev) {
1019 char *id = qdev_get_dev_path(dev);
1020 if (id) {
1021 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1022 g_free(id);
1023 }
1024 }
1025 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1026
1027 /* This assumes the iothread lock is taken here too. */
1028 qemu_mutex_lock_ramlist();
1029 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1030 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1031 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1032 new_block->idstr);
1033 abort();
1034 }
1035 }
1036 qemu_mutex_unlock_ramlist();
1037 }
1038
1039 static int memory_try_enable_merging(void *addr, size_t len)
1040 {
1041 QemuOpts *opts;
1042
1043 opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1044 if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
1045 /* disabled by the user */
1046 return 0;
1047 }
1048
1049 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1050 }
1051
1052 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1053 MemoryRegion *mr)
1054 {
1055 RAMBlock *block, *new_block;
1056
1057 size = TARGET_PAGE_ALIGN(size);
1058 new_block = g_malloc0(sizeof(*new_block));
1059
1060 /* This assumes the iothread lock is taken here too. */
1061 qemu_mutex_lock_ramlist();
1062 new_block->mr = mr;
1063 new_block->offset = find_ram_offset(size);
1064 if (host) {
1065 new_block->host = host;
1066 new_block->flags |= RAM_PREALLOC_MASK;
1067 } else {
1068 if (mem_path) {
1069 #if defined (__linux__) && !defined(TARGET_S390X)
1070 new_block->host = file_ram_alloc(new_block, size, mem_path);
1071 if (!new_block->host) {
1072 new_block->host = qemu_anon_ram_alloc(size);
1073 memory_try_enable_merging(new_block->host, size);
1074 }
1075 #else
1076 fprintf(stderr, "-mem-path option unsupported\n");
1077 exit(1);
1078 #endif
1079 } else {
1080 if (xen_enabled()) {
1081 xen_ram_alloc(new_block->offset, size, mr);
1082 } else if (kvm_enabled()) {
1083 /* some s390/kvm configurations have special constraints */
1084 new_block->host = kvm_ram_alloc(size);
1085 } else {
1086 new_block->host = qemu_anon_ram_alloc(size);
1087 }
1088 memory_try_enable_merging(new_block->host, size);
1089 }
1090 }
1091 new_block->length = size;
1092
1093 /* Keep the list sorted from biggest to smallest block. */
1094 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1095 if (block->length < new_block->length) {
1096 break;
1097 }
1098 }
1099 if (block) {
1100 QTAILQ_INSERT_BEFORE(block, new_block, next);
1101 } else {
1102 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1103 }
1104 ram_list.mru_block = NULL;
1105
1106 ram_list.version++;
1107 qemu_mutex_unlock_ramlist();
1108
1109 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1110 last_ram_offset() >> TARGET_PAGE_BITS);
1111 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1112 0, size >> TARGET_PAGE_BITS);
1113 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1114
1115 qemu_ram_setup_dump(new_block->host, size);
1116 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1117
1118 if (kvm_enabled())
1119 kvm_setup_guest_memory(new_block->host, size);
1120
1121 return new_block->offset;
1122 }
1123
1124 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1125 {
1126 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1127 }
1128
1129 void qemu_ram_free_from_ptr(ram_addr_t addr)
1130 {
1131 RAMBlock *block;
1132
1133 /* This assumes the iothread lock is taken here too. */
1134 qemu_mutex_lock_ramlist();
1135 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1136 if (addr == block->offset) {
1137 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1138 ram_list.mru_block = NULL;
1139 ram_list.version++;
1140 g_free(block);
1141 break;
1142 }
1143 }
1144 qemu_mutex_unlock_ramlist();
1145 }
1146
1147 void qemu_ram_free(ram_addr_t addr)
1148 {
1149 RAMBlock *block;
1150
1151 /* This assumes the iothread lock is taken here too. */
1152 qemu_mutex_lock_ramlist();
1153 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1154 if (addr == block->offset) {
1155 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1156 ram_list.mru_block = NULL;
1157 ram_list.version++;
1158 if (block->flags & RAM_PREALLOC_MASK) {
1159 ;
1160 } else if (mem_path) {
1161 #if defined (__linux__) && !defined(TARGET_S390X)
1162 if (block->fd) {
1163 munmap(block->host, block->length);
1164 close(block->fd);
1165 } else {
1166 qemu_anon_ram_free(block->host, block->length);
1167 }
1168 #else
1169 abort();
1170 #endif
1171 } else {
1172 if (xen_enabled()) {
1173 xen_invalidate_map_cache_entry(block->host);
1174 } else {
1175 qemu_anon_ram_free(block->host, block->length);
1176 }
1177 }
1178 g_free(block);
1179 break;
1180 }
1181 }
1182 qemu_mutex_unlock_ramlist();
1183
1184 }
1185
1186 #ifndef _WIN32
1187 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1188 {
1189 RAMBlock *block;
1190 ram_addr_t offset;
1191 int flags;
1192 void *area, *vaddr;
1193
1194 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1195 offset = addr - block->offset;
1196 if (offset < block->length) {
1197 vaddr = block->host + offset;
1198 if (block->flags & RAM_PREALLOC_MASK) {
1199 ;
1200 } else {
1201 flags = MAP_FIXED;
1202 munmap(vaddr, length);
1203 if (mem_path) {
1204 #if defined(__linux__) && !defined(TARGET_S390X)
1205 if (block->fd) {
1206 #ifdef MAP_POPULATE
1207 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1208 MAP_PRIVATE;
1209 #else
1210 flags |= MAP_PRIVATE;
1211 #endif
1212 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1213 flags, block->fd, offset);
1214 } else {
1215 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1216 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1217 flags, -1, 0);
1218 }
1219 #else
1220 abort();
1221 #endif
1222 } else {
1223 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1224 flags |= MAP_SHARED | MAP_ANONYMOUS;
1225 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1226 flags, -1, 0);
1227 #else
1228 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1229 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1230 flags, -1, 0);
1231 #endif
1232 }
1233 if (area != vaddr) {
1234 fprintf(stderr, "Could not remap addr: "
1235 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1236 length, addr);
1237 exit(1);
1238 }
1239 memory_try_enable_merging(vaddr, length);
1240 qemu_ram_setup_dump(vaddr, length);
1241 }
1242 return;
1243 }
1244 }
1245 }
1246 #endif /* !_WIN32 */
1247
1248 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1249 With the exception of the softmmu code in this file, this should
1250 only be used for local memory (e.g. video ram) that the device owns,
1251 and knows it isn't going to access beyond the end of the block.
1252
1253 It should not be used for general purpose DMA.
1254 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1255 */
1256 void *qemu_get_ram_ptr(ram_addr_t addr)
1257 {
1258 RAMBlock *block;
1259
1260 /* The list is protected by the iothread lock here. */
1261 block = ram_list.mru_block;
1262 if (block && addr - block->offset < block->length) {
1263 goto found;
1264 }
1265 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1266 if (addr - block->offset < block->length) {
1267 goto found;
1268 }
1269 }
1270
1271 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1272 abort();
1273
1274 found:
1275 ram_list.mru_block = block;
1276 if (xen_enabled()) {
1277 /* We need to check if the requested address is in the RAM
1278 * because we don't want to map the entire memory in QEMU.
1279 * In that case just map until the end of the page.
1280 */
1281 if (block->offset == 0) {
1282 return xen_map_cache(addr, 0, 0);
1283 } else if (block->host == NULL) {
1284 block->host =
1285 xen_map_cache(block->offset, block->length, 1);
1286 }
1287 }
1288 return block->host + (addr - block->offset);
1289 }
1290
1291 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1292 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1293 *
1294 * ??? Is this still necessary?
1295 */
1296 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1297 {
1298 RAMBlock *block;
1299
1300 /* The list is protected by the iothread lock here. */
1301 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1302 if (addr - block->offset < block->length) {
1303 if (xen_enabled()) {
1304 /* We need to check if the requested address is in the RAM
1305 * because we don't want to map the entire memory in QEMU.
1306 * In that case just map until the end of the page.
1307 */
1308 if (block->offset == 0) {
1309 return xen_map_cache(addr, 0, 0);
1310 } else if (block->host == NULL) {
1311 block->host =
1312 xen_map_cache(block->offset, block->length, 1);
1313 }
1314 }
1315 return block->host + (addr - block->offset);
1316 }
1317 }
1318
1319 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1320 abort();
1321
1322 return NULL;
1323 }
1324
1325 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1326 * but takes a size argument */
1327 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1328 {
1329 if (*size == 0) {
1330 return NULL;
1331 }
1332 if (xen_enabled()) {
1333 return xen_map_cache(addr, *size, 1);
1334 } else {
1335 RAMBlock *block;
1336
1337 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1338 if (addr - block->offset < block->length) {
1339 if (addr - block->offset + *size > block->length)
1340 *size = block->length - addr + block->offset;
1341 return block->host + (addr - block->offset);
1342 }
1343 }
1344
1345 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1346 abort();
1347 }
1348 }
1349
1350 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1351 {
1352 RAMBlock *block;
1353 uint8_t *host = ptr;
1354
1355 if (xen_enabled()) {
1356 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1357 return 0;
1358 }
1359
1360 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1361 /* This case append when the block is not mapped. */
1362 if (block->host == NULL) {
1363 continue;
1364 }
1365 if (host - block->host < block->length) {
1366 *ram_addr = block->offset + (host - block->host);
1367 return 0;
1368 }
1369 }
1370
1371 return -1;
1372 }
1373
1374 /* Some of the softmmu routines need to translate from a host pointer
1375 (typically a TLB entry) back to a ram offset. */
1376 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
1377 {
1378 ram_addr_t ram_addr;
1379
1380 if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
1381 fprintf(stderr, "Bad ram pointer %p\n", ptr);
1382 abort();
1383 }
1384 return ram_addr;
1385 }
1386
1387 static uint64_t unassigned_mem_read(void *opaque, hwaddr addr,
1388 unsigned size)
1389 {
1390 #ifdef DEBUG_UNASSIGNED
1391 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
1392 #endif
1393 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1394 cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, size);
1395 #endif
1396 return 0;
1397 }
1398
1399 static void unassigned_mem_write(void *opaque, hwaddr addr,
1400 uint64_t val, unsigned size)
1401 {
1402 #ifdef DEBUG_UNASSIGNED
1403 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%"PRIx64"\n", addr, val);
1404 #endif
1405 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1406 cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, size);
1407 #endif
1408 }
1409
1410 static const MemoryRegionOps unassigned_mem_ops = {
1411 .read = unassigned_mem_read,
1412 .write = unassigned_mem_write,
1413 .endianness = DEVICE_NATIVE_ENDIAN,
1414 };
1415
1416 static uint64_t error_mem_read(void *opaque, hwaddr addr,
1417 unsigned size)
1418 {
1419 abort();
1420 }
1421
1422 static void error_mem_write(void *opaque, hwaddr addr,
1423 uint64_t value, unsigned size)
1424 {
1425 abort();
1426 }
1427
1428 static const MemoryRegionOps error_mem_ops = {
1429 .read = error_mem_read,
1430 .write = error_mem_write,
1431 .endianness = DEVICE_NATIVE_ENDIAN,
1432 };
1433
1434 static const MemoryRegionOps rom_mem_ops = {
1435 .read = error_mem_read,
1436 .write = unassigned_mem_write,
1437 .endianness = DEVICE_NATIVE_ENDIAN,
1438 };
1439
1440 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1441 uint64_t val, unsigned size)
1442 {
1443 int dirty_flags;
1444 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1445 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1446 #if !defined(CONFIG_USER_ONLY)
1447 tb_invalidate_phys_page_fast(ram_addr, size);
1448 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1449 #endif
1450 }
1451 switch (size) {
1452 case 1:
1453 stb_p(qemu_get_ram_ptr(ram_addr), val);
1454 break;
1455 case 2:
1456 stw_p(qemu_get_ram_ptr(ram_addr), val);
1457 break;
1458 case 4:
1459 stl_p(qemu_get_ram_ptr(ram_addr), val);
1460 break;
1461 default:
1462 abort();
1463 }
1464 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1465 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1466 /* we remove the notdirty callback only if the code has been
1467 flushed */
1468 if (dirty_flags == 0xff)
1469 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1470 }
1471
1472 static const MemoryRegionOps notdirty_mem_ops = {
1473 .read = error_mem_read,
1474 .write = notdirty_mem_write,
1475 .endianness = DEVICE_NATIVE_ENDIAN,
1476 };
1477
1478 /* Generate a debug exception if a watchpoint has been hit. */
1479 static void check_watchpoint(int offset, int len_mask, int flags)
1480 {
1481 CPUArchState *env = cpu_single_env;
1482 target_ulong pc, cs_base;
1483 target_ulong vaddr;
1484 CPUWatchpoint *wp;
1485 int cpu_flags;
1486
1487 if (env->watchpoint_hit) {
1488 /* We re-entered the check after replacing the TB. Now raise
1489 * the debug interrupt so that is will trigger after the
1490 * current instruction. */
1491 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1492 return;
1493 }
1494 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1495 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1496 if ((vaddr == (wp->vaddr & len_mask) ||
1497 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1498 wp->flags |= BP_WATCHPOINT_HIT;
1499 if (!env->watchpoint_hit) {
1500 env->watchpoint_hit = wp;
1501 tb_check_watchpoint(env);
1502 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1503 env->exception_index = EXCP_DEBUG;
1504 cpu_loop_exit(env);
1505 } else {
1506 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1507 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1508 cpu_resume_from_signal(env, NULL);
1509 }
1510 }
1511 } else {
1512 wp->flags &= ~BP_WATCHPOINT_HIT;
1513 }
1514 }
1515 }
1516
1517 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1518 so these check for a hit then pass through to the normal out-of-line
1519 phys routines. */
1520 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1521 unsigned size)
1522 {
1523 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1524 switch (size) {
1525 case 1: return ldub_phys(addr);
1526 case 2: return lduw_phys(addr);
1527 case 4: return ldl_phys(addr);
1528 default: abort();
1529 }
1530 }
1531
1532 static void watch_mem_write(void *opaque, hwaddr addr,
1533 uint64_t val, unsigned size)
1534 {
1535 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1536 switch (size) {
1537 case 1:
1538 stb_phys(addr, val);
1539 break;
1540 case 2:
1541 stw_phys(addr, val);
1542 break;
1543 case 4:
1544 stl_phys(addr, val);
1545 break;
1546 default: abort();
1547 }
1548 }
1549
1550 static const MemoryRegionOps watch_mem_ops = {
1551 .read = watch_mem_read,
1552 .write = watch_mem_write,
1553 .endianness = DEVICE_NATIVE_ENDIAN,
1554 };
1555
1556 static uint64_t subpage_read(void *opaque, hwaddr addr,
1557 unsigned len)
1558 {
1559 subpage_t *mmio = opaque;
1560 unsigned int idx = SUBPAGE_IDX(addr);
1561 MemoryRegionSection *section;
1562 #if defined(DEBUG_SUBPAGE)
1563 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
1564 mmio, len, addr, idx);
1565 #endif
1566
1567 section = &phys_sections[mmio->sub_section[idx]];
1568 addr += mmio->base;
1569 addr -= section->offset_within_address_space;
1570 addr += section->offset_within_region;
1571 return io_mem_read(section->mr, addr, len);
1572 }
1573
1574 static void subpage_write(void *opaque, hwaddr addr,
1575 uint64_t value, unsigned len)
1576 {
1577 subpage_t *mmio = opaque;
1578 unsigned int idx = SUBPAGE_IDX(addr);
1579 MemoryRegionSection *section;
1580 #if defined(DEBUG_SUBPAGE)
1581 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1582 " idx %d value %"PRIx64"\n",
1583 __func__, mmio, len, addr, idx, value);
1584 #endif
1585
1586 section = &phys_sections[mmio->sub_section[idx]];
1587 addr += mmio->base;
1588 addr -= section->offset_within_address_space;
1589 addr += section->offset_within_region;
1590 io_mem_write(section->mr, addr, value, len);
1591 }
1592
1593 static const MemoryRegionOps subpage_ops = {
1594 .read = subpage_read,
1595 .write = subpage_write,
1596 .endianness = DEVICE_NATIVE_ENDIAN,
1597 };
1598
1599 static uint64_t subpage_ram_read(void *opaque, hwaddr addr,
1600 unsigned size)
1601 {
1602 ram_addr_t raddr = addr;
1603 void *ptr = qemu_get_ram_ptr(raddr);
1604 switch (size) {
1605 case 1: return ldub_p(ptr);
1606 case 2: return lduw_p(ptr);
1607 case 4: return ldl_p(ptr);
1608 default: abort();
1609 }
1610 }
1611
1612 static void subpage_ram_write(void *opaque, hwaddr addr,
1613 uint64_t value, unsigned size)
1614 {
1615 ram_addr_t raddr = addr;
1616 void *ptr = qemu_get_ram_ptr(raddr);
1617 switch (size) {
1618 case 1: return stb_p(ptr, value);
1619 case 2: return stw_p(ptr, value);
1620 case 4: return stl_p(ptr, value);
1621 default: abort();
1622 }
1623 }
1624
1625 static const MemoryRegionOps subpage_ram_ops = {
1626 .read = subpage_ram_read,
1627 .write = subpage_ram_write,
1628 .endianness = DEVICE_NATIVE_ENDIAN,
1629 };
1630
1631 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1632 uint16_t section)
1633 {
1634 int idx, eidx;
1635
1636 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1637 return -1;
1638 idx = SUBPAGE_IDX(start);
1639 eidx = SUBPAGE_IDX(end);
1640 #if defined(DEBUG_SUBPAGE)
1641 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1642 mmio, start, end, idx, eidx, memory);
1643 #endif
1644 if (memory_region_is_ram(phys_sections[section].mr)) {
1645 MemoryRegionSection new_section = phys_sections[section];
1646 new_section.mr = &io_mem_subpage_ram;
1647 section = phys_section_add(&new_section);
1648 }
1649 for (; idx <= eidx; idx++) {
1650 mmio->sub_section[idx] = section;
1651 }
1652
1653 return 0;
1654 }
1655
1656 static subpage_t *subpage_init(hwaddr base)
1657 {
1658 subpage_t *mmio;
1659
1660 mmio = g_malloc0(sizeof(subpage_t));
1661
1662 mmio->base = base;
1663 memory_region_init_io(&mmio->iomem, &subpage_ops, mmio,
1664 "subpage", TARGET_PAGE_SIZE);
1665 mmio->iomem.subpage = true;
1666 #if defined(DEBUG_SUBPAGE)
1667 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1668 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1669 #endif
1670 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, phys_section_unassigned);
1671
1672 return mmio;
1673 }
1674
1675 static uint16_t dummy_section(MemoryRegion *mr)
1676 {
1677 MemoryRegionSection section = {
1678 .mr = mr,
1679 .offset_within_address_space = 0,
1680 .offset_within_region = 0,
1681 .size = UINT64_MAX,
1682 };
1683
1684 return phys_section_add(&section);
1685 }
1686
1687 MemoryRegion *iotlb_to_region(hwaddr index)
1688 {
1689 return phys_sections[index & ~TARGET_PAGE_MASK].mr;
1690 }
1691
1692 static void io_mem_init(void)
1693 {
1694 memory_region_init_io(&io_mem_ram, &error_mem_ops, NULL, "ram", UINT64_MAX);
1695 memory_region_init_io(&io_mem_rom, &rom_mem_ops, NULL, "rom", UINT64_MAX);
1696 memory_region_init_io(&io_mem_unassigned, &unassigned_mem_ops, NULL,
1697 "unassigned", UINT64_MAX);
1698 memory_region_init_io(&io_mem_notdirty, &notdirty_mem_ops, NULL,
1699 "notdirty", UINT64_MAX);
1700 memory_region_init_io(&io_mem_subpage_ram, &subpage_ram_ops, NULL,
1701 "subpage-ram", UINT64_MAX);
1702 memory_region_init_io(&io_mem_watch, &watch_mem_ops, NULL,
1703 "watch", UINT64_MAX);
1704 }
1705
1706 static void mem_begin(MemoryListener *listener)
1707 {
1708 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
1709
1710 destroy_all_mappings(d);
1711 d->phys_map.ptr = PHYS_MAP_NODE_NIL;
1712 }
1713
1714 static void core_begin(MemoryListener *listener)
1715 {
1716 phys_sections_clear();
1717 phys_section_unassigned = dummy_section(&io_mem_unassigned);
1718 phys_section_notdirty = dummy_section(&io_mem_notdirty);
1719 phys_section_rom = dummy_section(&io_mem_rom);
1720 phys_section_watch = dummy_section(&io_mem_watch);
1721 }
1722
1723 static void tcg_commit(MemoryListener *listener)
1724 {
1725 CPUArchState *env;
1726
1727 /* since each CPU stores ram addresses in its TLB cache, we must
1728 reset the modified entries */
1729 /* XXX: slow ! */
1730 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1731 tlb_flush(env, 1);
1732 }
1733 }
1734
1735 static void core_log_global_start(MemoryListener *listener)
1736 {
1737 cpu_physical_memory_set_dirty_tracking(1);
1738 }
1739
1740 static void core_log_global_stop(MemoryListener *listener)
1741 {
1742 cpu_physical_memory_set_dirty_tracking(0);
1743 }
1744
1745 static void io_region_add(MemoryListener *listener,
1746 MemoryRegionSection *section)
1747 {
1748 MemoryRegionIORange *mrio = g_new(MemoryRegionIORange, 1);
1749
1750 mrio->mr = section->mr;
1751 mrio->offset = section->offset_within_region;
1752 iorange_init(&mrio->iorange, &memory_region_iorange_ops,
1753 section->offset_within_address_space, section->size);
1754 ioport_register(&mrio->iorange);
1755 }
1756
1757 static void io_region_del(MemoryListener *listener,
1758 MemoryRegionSection *section)
1759 {
1760 isa_unassign_ioport(section->offset_within_address_space, section->size);
1761 }
1762
1763 static MemoryListener core_memory_listener = {
1764 .begin = core_begin,
1765 .log_global_start = core_log_global_start,
1766 .log_global_stop = core_log_global_stop,
1767 .priority = 1,
1768 };
1769
1770 static MemoryListener io_memory_listener = {
1771 .region_add = io_region_add,
1772 .region_del = io_region_del,
1773 .priority = 0,
1774 };
1775
1776 static MemoryListener tcg_memory_listener = {
1777 .commit = tcg_commit,
1778 };
1779
1780 void address_space_init_dispatch(AddressSpace *as)
1781 {
1782 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1783
1784 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1785 d->listener = (MemoryListener) {
1786 .begin = mem_begin,
1787 .region_add = mem_add,
1788 .region_nop = mem_add,
1789 .priority = 0,
1790 };
1791 as->dispatch = d;
1792 memory_listener_register(&d->listener, as);
1793 }
1794
1795 void address_space_destroy_dispatch(AddressSpace *as)
1796 {
1797 AddressSpaceDispatch *d = as->dispatch;
1798
1799 memory_listener_unregister(&d->listener);
1800 destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
1801 g_free(d);
1802 as->dispatch = NULL;
1803 }
1804
1805 static void memory_map_init(void)
1806 {
1807 system_memory = g_malloc(sizeof(*system_memory));
1808 memory_region_init(system_memory, "system", INT64_MAX);
1809 address_space_init(&address_space_memory, system_memory);
1810 address_space_memory.name = "memory";
1811
1812 system_io = g_malloc(sizeof(*system_io));
1813 memory_region_init(system_io, "io", 65536);
1814 address_space_init(&address_space_io, system_io);
1815 address_space_io.name = "I/O";
1816
1817 memory_listener_register(&core_memory_listener, &address_space_memory);
1818 memory_listener_register(&io_memory_listener, &address_space_io);
1819 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1820
1821 dma_context_init(&dma_context_memory, &address_space_memory,
1822 NULL, NULL, NULL);
1823 }
1824
1825 MemoryRegion *get_system_memory(void)
1826 {
1827 return system_memory;
1828 }
1829
1830 MemoryRegion *get_system_io(void)
1831 {
1832 return system_io;
1833 }
1834
1835 #endif /* !defined(CONFIG_USER_ONLY) */
1836
1837 /* physical memory access (slow version, mainly for debug) */
1838 #if defined(CONFIG_USER_ONLY)
1839 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1840 uint8_t *buf, int len, int is_write)
1841 {
1842 int l, flags;
1843 target_ulong page;
1844 void * p;
1845
1846 while (len > 0) {
1847 page = addr & TARGET_PAGE_MASK;
1848 l = (page + TARGET_PAGE_SIZE) - addr;
1849 if (l > len)
1850 l = len;
1851 flags = page_get_flags(page);
1852 if (!(flags & PAGE_VALID))
1853 return -1;
1854 if (is_write) {
1855 if (!(flags & PAGE_WRITE))
1856 return -1;
1857 /* XXX: this code should not depend on lock_user */
1858 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1859 return -1;
1860 memcpy(p, buf, l);
1861 unlock_user(p, addr, l);
1862 } else {
1863 if (!(flags & PAGE_READ))
1864 return -1;
1865 /* XXX: this code should not depend on lock_user */
1866 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1867 return -1;
1868 memcpy(buf, p, l);
1869 unlock_user(p, addr, 0);
1870 }
1871 len -= l;
1872 buf += l;
1873 addr += l;
1874 }
1875 return 0;
1876 }
1877
1878 #else
1879
1880 static void invalidate_and_set_dirty(hwaddr addr,
1881 hwaddr length)
1882 {
1883 if (!cpu_physical_memory_is_dirty(addr)) {
1884 /* invalidate code */
1885 tb_invalidate_phys_page_range(addr, addr + length, 0);
1886 /* set dirty bit */
1887 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1888 }
1889 xen_modified_memory(addr, length);
1890 }
1891
1892 void address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1893 int len, bool is_write)
1894 {
1895 AddressSpaceDispatch *d = as->dispatch;
1896 int l;
1897 uint8_t *ptr;
1898 uint32_t val;
1899 hwaddr page;
1900 MemoryRegionSection *section;
1901
1902 while (len > 0) {
1903 page = addr & TARGET_PAGE_MASK;
1904 l = (page + TARGET_PAGE_SIZE) - addr;
1905 if (l > len)
1906 l = len;
1907 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
1908
1909 if (is_write) {
1910 if (!memory_region_is_ram(section->mr)) {
1911 hwaddr addr1;
1912 addr1 = memory_region_section_addr(section, addr);
1913 /* XXX: could force cpu_single_env to NULL to avoid
1914 potential bugs */
1915 if (l >= 4 && ((addr1 & 3) == 0)) {
1916 /* 32 bit write access */
1917 val = ldl_p(buf);
1918 io_mem_write(section->mr, addr1, val, 4);
1919 l = 4;
1920 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1921 /* 16 bit write access */
1922 val = lduw_p(buf);
1923 io_mem_write(section->mr, addr1, val, 2);
1924 l = 2;
1925 } else {
1926 /* 8 bit write access */
1927 val = ldub_p(buf);
1928 io_mem_write(section->mr, addr1, val, 1);
1929 l = 1;
1930 }
1931 } else if (!section->readonly) {
1932 ram_addr_t addr1;
1933 addr1 = memory_region_get_ram_addr(section->mr)
1934 + memory_region_section_addr(section, addr);
1935 /* RAM case */
1936 ptr = qemu_get_ram_ptr(addr1);
1937 memcpy(ptr, buf, l);
1938 invalidate_and_set_dirty(addr1, l);
1939 }
1940 } else {
1941 if (!(memory_region_is_ram(section->mr) ||
1942 memory_region_is_romd(section->mr))) {
1943 hwaddr addr1;
1944 /* I/O case */
1945 addr1 = memory_region_section_addr(section, addr);
1946 if (l >= 4 && ((addr1 & 3) == 0)) {
1947 /* 32 bit read access */
1948 val = io_mem_read(section->mr, addr1, 4);
1949 stl_p(buf, val);
1950 l = 4;
1951 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1952 /* 16 bit read access */
1953 val = io_mem_read(section->mr, addr1, 2);
1954 stw_p(buf, val);
1955 l = 2;
1956 } else {
1957 /* 8 bit read access */
1958 val = io_mem_read(section->mr, addr1, 1);
1959 stb_p(buf, val);
1960 l = 1;
1961 }
1962 } else {
1963 /* RAM case */
1964 ptr = qemu_get_ram_ptr(section->mr->ram_addr
1965 + memory_region_section_addr(section,
1966 addr));
1967 memcpy(buf, ptr, l);
1968 }
1969 }
1970 len -= l;
1971 buf += l;
1972 addr += l;
1973 }
1974 }
1975
1976 void address_space_write(AddressSpace *as, hwaddr addr,
1977 const uint8_t *buf, int len)
1978 {
1979 address_space_rw(as, addr, (uint8_t *)buf, len, true);
1980 }
1981
1982 /**
1983 * address_space_read: read from an address space.
1984 *
1985 * @as: #AddressSpace to be accessed
1986 * @addr: address within that address space
1987 * @buf: buffer with the data transferred
1988 */
1989 void address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1990 {
1991 address_space_rw(as, addr, buf, len, false);
1992 }
1993
1994
1995 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1996 int len, int is_write)
1997 {
1998 return address_space_rw(&address_space_memory, addr, buf, len, is_write);
1999 }
2000
2001 /* used for ROM loading : can write in RAM and ROM */
2002 void cpu_physical_memory_write_rom(hwaddr addr,
2003 const uint8_t *buf, int len)
2004 {
2005 AddressSpaceDispatch *d = address_space_memory.dispatch;
2006 int l;
2007 uint8_t *ptr;
2008 hwaddr page;
2009 MemoryRegionSection *section;
2010
2011 while (len > 0) {
2012 page = addr & TARGET_PAGE_MASK;
2013 l = (page + TARGET_PAGE_SIZE) - addr;
2014 if (l > len)
2015 l = len;
2016 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
2017
2018 if (!(memory_region_is_ram(section->mr) ||
2019 memory_region_is_romd(section->mr))) {
2020 /* do nothing */
2021 } else {
2022 unsigned long addr1;
2023 addr1 = memory_region_get_ram_addr(section->mr)
2024 + memory_region_section_addr(section, addr);
2025 /* ROM/RAM case */
2026 ptr = qemu_get_ram_ptr(addr1);
2027 memcpy(ptr, buf, l);
2028 invalidate_and_set_dirty(addr1, l);
2029 }
2030 len -= l;
2031 buf += l;
2032 addr += l;
2033 }
2034 }
2035
2036 typedef struct {
2037 void *buffer;
2038 hwaddr addr;
2039 hwaddr len;
2040 } BounceBuffer;
2041
2042 static BounceBuffer bounce;
2043
2044 typedef struct MapClient {
2045 void *opaque;
2046 void (*callback)(void *opaque);
2047 QLIST_ENTRY(MapClient) link;
2048 } MapClient;
2049
2050 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2051 = QLIST_HEAD_INITIALIZER(map_client_list);
2052
2053 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2054 {
2055 MapClient *client = g_malloc(sizeof(*client));
2056
2057 client->opaque = opaque;
2058 client->callback = callback;
2059 QLIST_INSERT_HEAD(&map_client_list, client, link);
2060 return client;
2061 }
2062
2063 static void cpu_unregister_map_client(void *_client)
2064 {
2065 MapClient *client = (MapClient *)_client;
2066
2067 QLIST_REMOVE(client, link);
2068 g_free(client);
2069 }
2070
2071 static void cpu_notify_map_clients(void)
2072 {
2073 MapClient *client;
2074
2075 while (!QLIST_EMPTY(&map_client_list)) {
2076 client = QLIST_FIRST(&map_client_list);
2077 client->callback(client->opaque);
2078 cpu_unregister_map_client(client);
2079 }
2080 }
2081
2082 /* Map a physical memory region into a host virtual address.
2083 * May map a subset of the requested range, given by and returned in *plen.
2084 * May return NULL if resources needed to perform the mapping are exhausted.
2085 * Use only for reads OR writes - not for read-modify-write operations.
2086 * Use cpu_register_map_client() to know when retrying the map operation is
2087 * likely to succeed.
2088 */
2089 void *address_space_map(AddressSpace *as,
2090 hwaddr addr,
2091 hwaddr *plen,
2092 bool is_write)
2093 {
2094 AddressSpaceDispatch *d = as->dispatch;
2095 hwaddr len = *plen;
2096 hwaddr todo = 0;
2097 int l;
2098 hwaddr page;
2099 MemoryRegionSection *section;
2100 ram_addr_t raddr = RAM_ADDR_MAX;
2101 ram_addr_t rlen;
2102 void *ret;
2103
2104 while (len > 0) {
2105 page = addr & TARGET_PAGE_MASK;
2106 l = (page + TARGET_PAGE_SIZE) - addr;
2107 if (l > len)
2108 l = len;
2109 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
2110
2111 if (!(memory_region_is_ram(section->mr) && !section->readonly)) {
2112 if (todo || bounce.buffer) {
2113 break;
2114 }
2115 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2116 bounce.addr = addr;
2117 bounce.len = l;
2118 if (!is_write) {
2119 address_space_read(as, addr, bounce.buffer, l);
2120 }
2121
2122 *plen = l;
2123 return bounce.buffer;
2124 }
2125 if (!todo) {
2126 raddr = memory_region_get_ram_addr(section->mr)
2127 + memory_region_section_addr(section, addr);
2128 }
2129
2130 len -= l;
2131 addr += l;
2132 todo += l;
2133 }
2134 rlen = todo;
2135 ret = qemu_ram_ptr_length(raddr, &rlen);
2136 *plen = rlen;
2137 return ret;
2138 }
2139
2140 /* Unmaps a memory region previously mapped by address_space_map().
2141 * Will also mark the memory as dirty if is_write == 1. access_len gives
2142 * the amount of memory that was actually read or written by the caller.
2143 */
2144 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2145 int is_write, hwaddr access_len)
2146 {
2147 if (buffer != bounce.buffer) {
2148 if (is_write) {
2149 ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
2150 while (access_len) {
2151 unsigned l;
2152 l = TARGET_PAGE_SIZE;
2153 if (l > access_len)
2154 l = access_len;
2155 invalidate_and_set_dirty(addr1, l);
2156 addr1 += l;
2157 access_len -= l;
2158 }
2159 }
2160 if (xen_enabled()) {
2161 xen_invalidate_map_cache_entry(buffer);
2162 }
2163 return;
2164 }
2165 if (is_write) {
2166 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2167 }
2168 qemu_vfree(bounce.buffer);
2169 bounce.buffer = NULL;
2170 cpu_notify_map_clients();
2171 }
2172
2173 void *cpu_physical_memory_map(hwaddr addr,
2174 hwaddr *plen,
2175 int is_write)
2176 {
2177 return address_space_map(&address_space_memory, addr, plen, is_write);
2178 }
2179
2180 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2181 int is_write, hwaddr access_len)
2182 {
2183 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2184 }
2185
2186 /* warning: addr must be aligned */
2187 static inline uint32_t ldl_phys_internal(hwaddr addr,
2188 enum device_endian endian)
2189 {
2190 uint8_t *ptr;
2191 uint32_t val;
2192 MemoryRegionSection *section;
2193
2194 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2195
2196 if (!(memory_region_is_ram(section->mr) ||
2197 memory_region_is_romd(section->mr))) {
2198 /* I/O case */
2199 addr = memory_region_section_addr(section, addr);
2200 val = io_mem_read(section->mr, addr, 4);
2201 #if defined(TARGET_WORDS_BIGENDIAN)
2202 if (endian == DEVICE_LITTLE_ENDIAN) {
2203 val = bswap32(val);
2204 }
2205 #else
2206 if (endian == DEVICE_BIG_ENDIAN) {
2207 val = bswap32(val);
2208 }
2209 #endif
2210 } else {
2211 /* RAM case */
2212 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2213 & TARGET_PAGE_MASK)
2214 + memory_region_section_addr(section, addr));
2215 switch (endian) {
2216 case DEVICE_LITTLE_ENDIAN:
2217 val = ldl_le_p(ptr);
2218 break;
2219 case DEVICE_BIG_ENDIAN:
2220 val = ldl_be_p(ptr);
2221 break;
2222 default:
2223 val = ldl_p(ptr);
2224 break;
2225 }
2226 }
2227 return val;
2228 }
2229
2230 uint32_t ldl_phys(hwaddr addr)
2231 {
2232 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2233 }
2234
2235 uint32_t ldl_le_phys(hwaddr addr)
2236 {
2237 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2238 }
2239
2240 uint32_t ldl_be_phys(hwaddr addr)
2241 {
2242 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2243 }
2244
2245 /* warning: addr must be aligned */
2246 static inline uint64_t ldq_phys_internal(hwaddr addr,
2247 enum device_endian endian)
2248 {
2249 uint8_t *ptr;
2250 uint64_t val;
2251 MemoryRegionSection *section;
2252
2253 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2254
2255 if (!(memory_region_is_ram(section->mr) ||
2256 memory_region_is_romd(section->mr))) {
2257 /* I/O case */
2258 addr = memory_region_section_addr(section, addr);
2259
2260 /* XXX This is broken when device endian != cpu endian.
2261 Fix and add "endian" variable check */
2262 #ifdef TARGET_WORDS_BIGENDIAN
2263 val = io_mem_read(section->mr, addr, 4) << 32;
2264 val |= io_mem_read(section->mr, addr + 4, 4);
2265 #else
2266 val = io_mem_read(section->mr, addr, 4);
2267 val |= io_mem_read(section->mr, addr + 4, 4) << 32;
2268 #endif
2269 } else {
2270 /* RAM case */
2271 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2272 & TARGET_PAGE_MASK)
2273 + memory_region_section_addr(section, addr));
2274 switch (endian) {
2275 case DEVICE_LITTLE_ENDIAN:
2276 val = ldq_le_p(ptr);
2277 break;
2278 case DEVICE_BIG_ENDIAN:
2279 val = ldq_be_p(ptr);
2280 break;
2281 default:
2282 val = ldq_p(ptr);
2283 break;
2284 }
2285 }
2286 return val;
2287 }
2288
2289 uint64_t ldq_phys(hwaddr addr)
2290 {
2291 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2292 }
2293
2294 uint64_t ldq_le_phys(hwaddr addr)
2295 {
2296 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2297 }
2298
2299 uint64_t ldq_be_phys(hwaddr addr)
2300 {
2301 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2302 }
2303
2304 /* XXX: optimize */
2305 uint32_t ldub_phys(hwaddr addr)
2306 {
2307 uint8_t val;
2308 cpu_physical_memory_read(addr, &val, 1);
2309 return val;
2310 }
2311
2312 /* warning: addr must be aligned */
2313 static inline uint32_t lduw_phys_internal(hwaddr addr,
2314 enum device_endian endian)
2315 {
2316 uint8_t *ptr;
2317 uint64_t val;
2318 MemoryRegionSection *section;
2319
2320 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2321
2322 if (!(memory_region_is_ram(section->mr) ||
2323 memory_region_is_romd(section->mr))) {
2324 /* I/O case */
2325 addr = memory_region_section_addr(section, addr);
2326 val = io_mem_read(section->mr, addr, 2);
2327 #if defined(TARGET_WORDS_BIGENDIAN)
2328 if (endian == DEVICE_LITTLE_ENDIAN) {
2329 val = bswap16(val);
2330 }
2331 #else
2332 if (endian == DEVICE_BIG_ENDIAN) {
2333 val = bswap16(val);
2334 }
2335 #endif
2336 } else {
2337 /* RAM case */
2338 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2339 & TARGET_PAGE_MASK)
2340 + memory_region_section_addr(section, addr));
2341 switch (endian) {
2342 case DEVICE_LITTLE_ENDIAN:
2343 val = lduw_le_p(ptr);
2344 break;
2345 case DEVICE_BIG_ENDIAN:
2346 val = lduw_be_p(ptr);
2347 break;
2348 default:
2349 val = lduw_p(ptr);
2350 break;
2351 }
2352 }
2353 return val;
2354 }
2355
2356 uint32_t lduw_phys(hwaddr addr)
2357 {
2358 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2359 }
2360
2361 uint32_t lduw_le_phys(hwaddr addr)
2362 {
2363 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2364 }
2365
2366 uint32_t lduw_be_phys(hwaddr addr)
2367 {
2368 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2369 }
2370
2371 /* warning: addr must be aligned. The ram page is not masked as dirty
2372 and the code inside is not invalidated. It is useful if the dirty
2373 bits are used to track modified PTEs */
2374 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2375 {
2376 uint8_t *ptr;
2377 MemoryRegionSection *section;
2378
2379 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2380
2381 if (!memory_region_is_ram(section->mr) || section->readonly) {
2382 addr = memory_region_section_addr(section, addr);
2383 if (memory_region_is_ram(section->mr)) {
2384 section = &phys_sections[phys_section_rom];
2385 }
2386 io_mem_write(section->mr, addr, val, 4);
2387 } else {
2388 unsigned long addr1 = (memory_region_get_ram_addr(section->mr)
2389 & TARGET_PAGE_MASK)
2390 + memory_region_section_addr(section, addr);
2391 ptr = qemu_get_ram_ptr(addr1);
2392 stl_p(ptr, val);
2393
2394 if (unlikely(in_migration)) {
2395 if (!cpu_physical_memory_is_dirty(addr1)) {
2396 /* invalidate code */
2397 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2398 /* set dirty bit */
2399 cpu_physical_memory_set_dirty_flags(
2400 addr1, (0xff & ~CODE_DIRTY_FLAG));
2401 }
2402 }
2403 }
2404 }
2405
2406 /* warning: addr must be aligned */
2407 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2408 enum device_endian endian)
2409 {
2410 uint8_t *ptr;
2411 MemoryRegionSection *section;
2412
2413 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2414
2415 if (!memory_region_is_ram(section->mr) || section->readonly) {
2416 addr = memory_region_section_addr(section, addr);
2417 if (memory_region_is_ram(section->mr)) {
2418 section = &phys_sections[phys_section_rom];
2419 }
2420 #if defined(TARGET_WORDS_BIGENDIAN)
2421 if (endian == DEVICE_LITTLE_ENDIAN) {
2422 val = bswap32(val);
2423 }
2424 #else
2425 if (endian == DEVICE_BIG_ENDIAN) {
2426 val = bswap32(val);
2427 }
2428 #endif
2429 io_mem_write(section->mr, addr, val, 4);
2430 } else {
2431 unsigned long addr1;
2432 addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2433 + memory_region_section_addr(section, addr);
2434 /* RAM case */
2435 ptr = qemu_get_ram_ptr(addr1);
2436 switch (endian) {
2437 case DEVICE_LITTLE_ENDIAN:
2438 stl_le_p(ptr, val);
2439 break;
2440 case DEVICE_BIG_ENDIAN:
2441 stl_be_p(ptr, val);
2442 break;
2443 default:
2444 stl_p(ptr, val);
2445 break;
2446 }
2447 invalidate_and_set_dirty(addr1, 4);
2448 }
2449 }
2450
2451 void stl_phys(hwaddr addr, uint32_t val)
2452 {
2453 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2454 }
2455
2456 void stl_le_phys(hwaddr addr, uint32_t val)
2457 {
2458 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2459 }
2460
2461 void stl_be_phys(hwaddr addr, uint32_t val)
2462 {
2463 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2464 }
2465
2466 /* XXX: optimize */
2467 void stb_phys(hwaddr addr, uint32_t val)
2468 {
2469 uint8_t v = val;
2470 cpu_physical_memory_write(addr, &v, 1);
2471 }
2472
2473 /* warning: addr must be aligned */
2474 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2475 enum device_endian endian)
2476 {
2477 uint8_t *ptr;
2478 MemoryRegionSection *section;
2479
2480 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2481
2482 if (!memory_region_is_ram(section->mr) || section->readonly) {
2483 addr = memory_region_section_addr(section, addr);
2484 if (memory_region_is_ram(section->mr)) {
2485 section = &phys_sections[phys_section_rom];
2486 }
2487 #if defined(TARGET_WORDS_BIGENDIAN)
2488 if (endian == DEVICE_LITTLE_ENDIAN) {
2489 val = bswap16(val);
2490 }
2491 #else
2492 if (endian == DEVICE_BIG_ENDIAN) {
2493 val = bswap16(val);
2494 }
2495 #endif
2496 io_mem_write(section->mr, addr, val, 2);
2497 } else {
2498 unsigned long addr1;
2499 addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2500 + memory_region_section_addr(section, addr);
2501 /* RAM case */
2502 ptr = qemu_get_ram_ptr(addr1);
2503 switch (endian) {
2504 case DEVICE_LITTLE_ENDIAN:
2505 stw_le_p(ptr, val);
2506 break;
2507 case DEVICE_BIG_ENDIAN:
2508 stw_be_p(ptr, val);
2509 break;
2510 default:
2511 stw_p(ptr, val);
2512 break;
2513 }
2514 invalidate_and_set_dirty(addr1, 2);
2515 }
2516 }
2517
2518 void stw_phys(hwaddr addr, uint32_t val)
2519 {
2520 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2521 }
2522
2523 void stw_le_phys(hwaddr addr, uint32_t val)
2524 {
2525 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2526 }
2527
2528 void stw_be_phys(hwaddr addr, uint32_t val)
2529 {
2530 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2531 }
2532
2533 /* XXX: optimize */
2534 void stq_phys(hwaddr addr, uint64_t val)
2535 {
2536 val = tswap64(val);
2537 cpu_physical_memory_write(addr, &val, 8);
2538 }
2539
2540 void stq_le_phys(hwaddr addr, uint64_t val)
2541 {
2542 val = cpu_to_le64(val);
2543 cpu_physical_memory_write(addr, &val, 8);
2544 }
2545
2546 void stq_be_phys(hwaddr addr, uint64_t val)
2547 {
2548 val = cpu_to_be64(val);
2549 cpu_physical_memory_write(addr, &val, 8);
2550 }
2551
2552 /* virtual memory access for debug (includes writing to ROM) */
2553 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2554 uint8_t *buf, int len, int is_write)
2555 {
2556 int l;
2557 hwaddr phys_addr;
2558 target_ulong page;
2559
2560 while (len > 0) {
2561 page = addr & TARGET_PAGE_MASK;
2562 phys_addr = cpu_get_phys_page_debug(env, page);
2563 /* if no physical page mapped, return an error */
2564 if (phys_addr == -1)
2565 return -1;
2566 l = (page + TARGET_PAGE_SIZE) - addr;
2567 if (l > len)
2568 l = len;
2569 phys_addr += (addr & ~TARGET_PAGE_MASK);
2570 if (is_write)
2571 cpu_physical_memory_write_rom(phys_addr, buf, l);
2572 else
2573 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2574 len -= l;
2575 buf += l;
2576 addr += l;
2577 }
2578 return 0;
2579 }
2580 #endif
2581
2582 #if !defined(CONFIG_USER_ONLY)
2583
2584 /*
2585 * A helper function for the _utterly broken_ virtio device model to find out if
2586 * it's running on a big endian machine. Don't do this at home kids!
2587 */
2588 bool virtio_is_big_endian(void);
2589 bool virtio_is_big_endian(void)
2590 {
2591 #if defined(TARGET_WORDS_BIGENDIAN)
2592 return true;
2593 #else
2594 return false;
2595 #endif
2596 }
2597
2598 #endif
2599
2600 #ifndef CONFIG_USER_ONLY
2601 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2602 {
2603 MemoryRegionSection *section;
2604
2605 section = phys_page_find(address_space_memory.dispatch,
2606 phys_addr >> TARGET_PAGE_BITS);
2607
2608 return !(memory_region_is_ram(section->mr) ||
2609 memory_region_is_romd(section->mr));
2610 }
2611 #endif
QEmu