?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
pcie: Fix bug in pcie_ext_cap_set_next
[qemu/ar7.git] / exec.c
blob28abd7e7102013d9ac04d6d026689c7b37d0c5bc
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "hw/xen.h"
35 #include "qemu/timer.h"
36 #include "qemu/config-file.h"
37 #include "exec/memory.h"
38 #include "sysemu/dma.h"
39 #include "exec/address-spaces.h"
40 #if defined(CONFIG_USER_ONLY)
41 #include <qemu.h>
42 #else /* !CONFIG_USER_ONLY */
43 #include "sysemu/xen-mapcache.h"
44 #include "trace.h"
45 #endif
46
47 #include "exec/cputlb.h"
48 #include "translate-all.h"
49
50 #include "exec/memory-internal.h"
51
52 //#define DEBUG_UNASSIGNED
53 //#define DEBUG_SUBPAGE
54
55 #if !defined(CONFIG_USER_ONLY)
56 int phys_ram_fd;
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66 DMAContext dma_context_memory;
67
68 MemoryRegion io_mem_ram, io_mem_rom, io_mem_unassigned, io_mem_notdirty;
69 static MemoryRegion io_mem_subpage_ram;
70
71 #endif
72
73 CPUArchState *first_cpu;
74 /* current CPU in the current thread. It is only valid inside
75 cpu_exec() */
76 DEFINE_TLS(CPUArchState *,cpu_single_env);
77 /* 0 = Do not count executed instructions.
78 1 = Precise instruction counting.
79 2 = Adaptive rate instruction counting. */
80 int use_icount = 0;
81
82 #if !defined(CONFIG_USER_ONLY)
83
84 static MemoryRegionSection *phys_sections;
85 static unsigned phys_sections_nb, phys_sections_nb_alloc;
86 static uint16_t phys_section_unassigned;
87 static uint16_t phys_section_notdirty;
88 static uint16_t phys_section_rom;
89 static uint16_t phys_section_watch;
90
91 /* Simple allocator for PhysPageEntry nodes */
92 static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
93 static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
94
95 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
96
97 static void io_mem_init(void);
98 static void memory_map_init(void);
99 static void *qemu_safe_ram_ptr(ram_addr_t addr);
100
101 static MemoryRegion io_mem_watch;
102 #endif
103
104 #if !defined(CONFIG_USER_ONLY)
105
106 static void phys_map_node_reserve(unsigned nodes)
107 {
108 if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) {
109 typedef PhysPageEntry Node[L2_SIZE];
110 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16);
111 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc,
112 phys_map_nodes_nb + nodes);
113 phys_map_nodes = g_renew(Node, phys_map_nodes,
114 phys_map_nodes_nb_alloc);
115 }
116 }
117
118 static uint16_t phys_map_node_alloc(void)
119 {
120 unsigned i;
121 uint16_t ret;
122
123 ret = phys_map_nodes_nb++;
124 assert(ret != PHYS_MAP_NODE_NIL);
125 assert(ret != phys_map_nodes_nb_alloc);
126 for (i = 0; i < L2_SIZE; ++i) {
127 phys_map_nodes[ret][i].is_leaf = 0;
128 phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
129 }
130 return ret;
131 }
132
133 static void phys_map_nodes_reset(void)
134 {
135 phys_map_nodes_nb = 0;
136 }
137
138
139 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
140 hwaddr *nb, uint16_t leaf,
141 int level)
142 {
143 PhysPageEntry *p;
144 int i;
145 hwaddr step = (hwaddr)1 << (level * L2_BITS);
146
147 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
148 lp->ptr = phys_map_node_alloc();
149 p = phys_map_nodes[lp->ptr];
150 if (level == 0) {
151 for (i = 0; i < L2_SIZE; i++) {
152 p[i].is_leaf = 1;
153 p[i].ptr = phys_section_unassigned;
154 }
155 }
156 } else {
157 p = phys_map_nodes[lp->ptr];
158 }
159 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
160
161 while (*nb && lp < &p[L2_SIZE]) {
162 if ((*index & (step - 1)) == 0 && *nb >= step) {
163 lp->is_leaf = true;
164 lp->ptr = leaf;
165 *index += step;
166 *nb -= step;
167 } else {
168 phys_page_set_level(lp, index, nb, leaf, level - 1);
169 }
170 ++lp;
171 }
172 }
173
174 static void phys_page_set(AddressSpaceDispatch *d,
175 hwaddr index, hwaddr nb,
176 uint16_t leaf)
177 {
178 /* Wildly overreserve - it doesn't matter much. */
179 phys_map_node_reserve(3 * P_L2_LEVELS);
180
181 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
182 }
183
184 MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr index)
185 {
186 PhysPageEntry lp = d->phys_map;
187 PhysPageEntry *p;
188 int i;
189 uint16_t s_index = phys_section_unassigned;
190
191 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
192 if (lp.ptr == PHYS_MAP_NODE_NIL) {
193 goto not_found;
194 }
195 p = phys_map_nodes[lp.ptr];
196 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
197 }
198
199 s_index = lp.ptr;
200 not_found:
201 return &phys_sections[s_index];
202 }
203
204 bool memory_region_is_unassigned(MemoryRegion *mr)
205 {
206 return mr != &io_mem_ram && mr != &io_mem_rom
207 && mr != &io_mem_notdirty && !mr->rom_device
208 && mr != &io_mem_watch;
209 }
210 #endif
211
212 void cpu_exec_init_all(void)
213 {
214 #if !defined(CONFIG_USER_ONLY)
215 memory_map_init();
216 io_mem_init();
217 #endif
218 }
219
220 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
221
222 static int cpu_common_post_load(void *opaque, int version_id)
223 {
224 CPUArchState *env = opaque;
225
226 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
227 version_id is increased. */
228 env->interrupt_request &= ~0x01;
229 tlb_flush(env, 1);
230
231 return 0;
232 }
233
234 static const VMStateDescription vmstate_cpu_common = {
235 .name = "cpu_common",
236 .version_id = 1,
237 .minimum_version_id = 1,
238 .minimum_version_id_old = 1,
239 .post_load = cpu_common_post_load,
240 .fields = (VMStateField []) {
241 VMSTATE_UINT32(halted, CPUArchState),
242 VMSTATE_UINT32(interrupt_request, CPUArchState),
243 VMSTATE_END_OF_LIST()
244 }
245 };
246 #endif
247
248 CPUArchState *qemu_get_cpu(int cpu)
249 {
250 CPUArchState *env = first_cpu;
251
252 while (env) {
253 if (env->cpu_index == cpu)
254 break;
255 env = env->next_cpu;
256 }
257
258 return env;
259 }
260
261 void cpu_exec_init(CPUArchState *env)
262 {
263 #ifndef CONFIG_USER_ONLY
264 CPUState *cpu = ENV_GET_CPU(env);
265 #endif
266 CPUArchState **penv;
267 int cpu_index;
268
269 #if defined(CONFIG_USER_ONLY)
270 cpu_list_lock();
271 #endif
272 env->next_cpu = NULL;
273 penv = &first_cpu;
274 cpu_index = 0;
275 while (*penv != NULL) {
276 penv = &(*penv)->next_cpu;
277 cpu_index++;
278 }
279 env->cpu_index = cpu_index;
280 env->numa_node = 0;
281 QTAILQ_INIT(&env->breakpoints);
282 QTAILQ_INIT(&env->watchpoints);
283 #ifndef CONFIG_USER_ONLY
284 cpu->thread_id = qemu_get_thread_id();
285 #endif
286 *penv = env;
287 #if defined(CONFIG_USER_ONLY)
288 cpu_list_unlock();
289 #endif
290 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
291 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
292 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
293 cpu_save, cpu_load, env);
294 #endif
295 }
296
297 #if defined(TARGET_HAS_ICE)
298 #if defined(CONFIG_USER_ONLY)
299 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
300 {
301 tb_invalidate_phys_page_range(pc, pc + 1, 0);
302 }
303 #else
304 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
305 {
306 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
307 (pc & ~TARGET_PAGE_MASK));
308 }
309 #endif
310 #endif /* TARGET_HAS_ICE */
311
312 #if defined(CONFIG_USER_ONLY)
313 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
314
315 {
316 }
317
318 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
319 int flags, CPUWatchpoint **watchpoint)
320 {
321 return -ENOSYS;
322 }
323 #else
324 /* Add a watchpoint. */
325 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
326 int flags, CPUWatchpoint **watchpoint)
327 {
328 target_ulong len_mask = ~(len - 1);
329 CPUWatchpoint *wp;
330
331 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
332 if ((len & (len - 1)) || (addr & ~len_mask) ||
333 len == 0 || len > TARGET_PAGE_SIZE) {
334 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
335 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
336 return -EINVAL;
337 }
338 wp = g_malloc(sizeof(*wp));
339
340 wp->vaddr = addr;
341 wp->len_mask = len_mask;
342 wp->flags = flags;
343
344 /* keep all GDB-injected watchpoints in front */
345 if (flags & BP_GDB)
346 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
347 else
348 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
349
350 tlb_flush_page(env, addr);
351
352 if (watchpoint)
353 *watchpoint = wp;
354 return 0;
355 }
356
357 /* Remove a specific watchpoint. */
358 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
359 int flags)
360 {
361 target_ulong len_mask = ~(len - 1);
362 CPUWatchpoint *wp;
363
364 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
365 if (addr == wp->vaddr && len_mask == wp->len_mask
366 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
367 cpu_watchpoint_remove_by_ref(env, wp);
368 return 0;
369 }
370 }
371 return -ENOENT;
372 }
373
374 /* Remove a specific watchpoint by reference. */
375 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
376 {
377 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
378
379 tlb_flush_page(env, watchpoint->vaddr);
380
381 g_free(watchpoint);
382 }
383
384 /* Remove all matching watchpoints. */
385 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
386 {
387 CPUWatchpoint *wp, *next;
388
389 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
390 if (wp->flags & mask)
391 cpu_watchpoint_remove_by_ref(env, wp);
392 }
393 }
394 #endif
395
396 /* Add a breakpoint. */
397 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
398 CPUBreakpoint **breakpoint)
399 {
400 #if defined(TARGET_HAS_ICE)
401 CPUBreakpoint *bp;
402
403 bp = g_malloc(sizeof(*bp));
404
405 bp->pc = pc;
406 bp->flags = flags;
407
408 /* keep all GDB-injected breakpoints in front */
409 if (flags & BP_GDB)
410 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
411 else
412 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
413
414 breakpoint_invalidate(env, pc);
415
416 if (breakpoint)
417 *breakpoint = bp;
418 return 0;
419 #else
420 return -ENOSYS;
421 #endif
422 }
423
424 /* Remove a specific breakpoint. */
425 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
426 {
427 #if defined(TARGET_HAS_ICE)
428 CPUBreakpoint *bp;
429
430 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
431 if (bp->pc == pc && bp->flags == flags) {
432 cpu_breakpoint_remove_by_ref(env, bp);
433 return 0;
434 }
435 }
436 return -ENOENT;
437 #else
438 return -ENOSYS;
439 #endif
440 }
441
442 /* Remove a specific breakpoint by reference. */
443 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
444 {
445 #if defined(TARGET_HAS_ICE)
446 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
447
448 breakpoint_invalidate(env, breakpoint->pc);
449
450 g_free(breakpoint);
451 #endif
452 }
453
454 /* Remove all matching breakpoints. */
455 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
456 {
457 #if defined(TARGET_HAS_ICE)
458 CPUBreakpoint *bp, *next;
459
460 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
461 if (bp->flags & mask)
462 cpu_breakpoint_remove_by_ref(env, bp);
463 }
464 #endif
465 }
466
467 /* enable or disable single step mode. EXCP_DEBUG is returned by the
468 CPU loop after each instruction */
469 void cpu_single_step(CPUArchState *env, int enabled)
470 {
471 #if defined(TARGET_HAS_ICE)
472 if (env->singlestep_enabled != enabled) {
473 env->singlestep_enabled = enabled;
474 if (kvm_enabled())
475 kvm_update_guest_debug(env, 0);
476 else {
477 /* must flush all the translated code to avoid inconsistencies */
478 /* XXX: only flush what is necessary */
479 tb_flush(env);
480 }
481 }
482 #endif
483 }
484
485 void cpu_reset_interrupt(CPUArchState *env, int mask)
486 {
487 env->interrupt_request &= ~mask;
488 }
489
490 void cpu_exit(CPUArchState *env)
491 {
492 env->exit_request = 1;
493 cpu_unlink_tb(env);
494 }
495
496 void cpu_abort(CPUArchState *env, const char *fmt, ...)
497 {
498 va_list ap;
499 va_list ap2;
500
501 va_start(ap, fmt);
502 va_copy(ap2, ap);
503 fprintf(stderr, "qemu: fatal: ");
504 vfprintf(stderr, fmt, ap);
505 fprintf(stderr, "\n");
506 cpu_dump_state(env, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
507 if (qemu_log_enabled()) {
508 qemu_log("qemu: fatal: ");
509 qemu_log_vprintf(fmt, ap2);
510 qemu_log("\n");
511 log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
512 qemu_log_flush();
513 qemu_log_close();
514 }
515 va_end(ap2);
516 va_end(ap);
517 #if defined(CONFIG_USER_ONLY)
518 {
519 struct sigaction act;
520 sigfillset(&act.sa_mask);
521 act.sa_handler = SIG_DFL;
522 sigaction(SIGABRT, &act, NULL);
523 }
524 #endif
525 abort();
526 }
527
528 CPUArchState *cpu_copy(CPUArchState *env)
529 {
530 CPUArchState *new_env = cpu_init(env->cpu_model_str);
531 CPUArchState *next_cpu = new_env->next_cpu;
532 int cpu_index = new_env->cpu_index;
533 #if defined(TARGET_HAS_ICE)
534 CPUBreakpoint *bp;
535 CPUWatchpoint *wp;
536 #endif
537
538 memcpy(new_env, env, sizeof(CPUArchState));
539
540 /* Preserve chaining and index. */
541 new_env->next_cpu = next_cpu;
542 new_env->cpu_index = cpu_index;
543
544 /* Clone all break/watchpoints.
545 Note: Once we support ptrace with hw-debug register access, make sure
546 BP_CPU break/watchpoints are handled correctly on clone. */
547 QTAILQ_INIT(&env->breakpoints);
548 QTAILQ_INIT(&env->watchpoints);
549 #if defined(TARGET_HAS_ICE)
550 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
551 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
552 }
553 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
554 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
555 wp->flags, NULL);
556 }
557 #endif
558
559 return new_env;
560 }
561
562 #if !defined(CONFIG_USER_ONLY)
563 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
564 uintptr_t length)
565 {
566 uintptr_t start1;
567
568 /* we modify the TLB cache so that the dirty bit will be set again
569 when accessing the range */
570 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
571 /* Check that we don't span multiple blocks - this breaks the
572 address comparisons below. */
573 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
574 != (end - 1) - start) {
575 abort();
576 }
577 cpu_tlb_reset_dirty_all(start1, length);
578
579 }
580
581 /* Note: start and end must be within the same ram block. */
582 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
583 int dirty_flags)
584 {
585 uintptr_t length;
586
587 start &= TARGET_PAGE_MASK;
588 end = TARGET_PAGE_ALIGN(end);
589
590 length = end - start;
591 if (length == 0)
592 return;
593 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
594
595 if (tcg_enabled()) {
596 tlb_reset_dirty_range_all(start, end, length);
597 }
598 }
599
600 static int cpu_physical_memory_set_dirty_tracking(int enable)
601 {
602 int ret = 0;
603 in_migration = enable;
604 return ret;
605 }
606
607 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
608 MemoryRegionSection *section,
609 target_ulong vaddr,
610 hwaddr paddr,
611 int prot,
612 target_ulong *address)
613 {
614 hwaddr iotlb;
615 CPUWatchpoint *wp;
616
617 if (memory_region_is_ram(section->mr)) {
618 /* Normal RAM. */
619 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
620 + memory_region_section_addr(section, paddr);
621 if (!section->readonly) {
622 iotlb |= phys_section_notdirty;
623 } else {
624 iotlb |= phys_section_rom;
625 }
626 } else {
627 /* IO handlers are currently passed a physical address.
628 It would be nice to pass an offset from the base address
629 of that region. This would avoid having to special case RAM,
630 and avoid full address decoding in every device.
631 We can't use the high bits of pd for this because
632 IO_MEM_ROMD uses these as a ram address. */
633 iotlb = section - phys_sections;
634 iotlb += memory_region_section_addr(section, paddr);
635 }
636
637 /* Make accesses to pages with watchpoints go via the
638 watchpoint trap routines. */
639 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
640 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
641 /* Avoid trapping reads of pages with a write breakpoint. */
642 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
643 iotlb = phys_section_watch + paddr;
644 *address |= TLB_MMIO;
645 break;
646 }
647 }
648 }
649
650 return iotlb;
651 }
652 #endif /* defined(CONFIG_USER_ONLY) */
653
654 #if !defined(CONFIG_USER_ONLY)
655
656 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
657 typedef struct subpage_t {
658 MemoryRegion iomem;
659 hwaddr base;
660 uint16_t sub_section[TARGET_PAGE_SIZE];
661 } subpage_t;
662
663 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
664 uint16_t section);
665 static subpage_t *subpage_init(hwaddr base);
666 static void destroy_page_desc(uint16_t section_index)
667 {
668 MemoryRegionSection *section = &phys_sections[section_index];
669 MemoryRegion *mr = section->mr;
670
671 if (mr->subpage) {
672 subpage_t *subpage = container_of(mr, subpage_t, iomem);
673 memory_region_destroy(&subpage->iomem);
674 g_free(subpage);
675 }
676 }
677
678 static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)
679 {
680 unsigned i;
681 PhysPageEntry *p;
682
683 if (lp->ptr == PHYS_MAP_NODE_NIL) {
684 return;
685 }
686
687 p = phys_map_nodes[lp->ptr];
688 for (i = 0; i < L2_SIZE; ++i) {
689 if (!p[i].is_leaf) {
690 destroy_l2_mapping(&p[i], level - 1);
691 } else {
692 destroy_page_desc(p[i].ptr);
693 }
694 }
695 lp->is_leaf = 0;
696 lp->ptr = PHYS_MAP_NODE_NIL;
697 }
698
699 static void destroy_all_mappings(AddressSpaceDispatch *d)
700 {
701 destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
702 phys_map_nodes_reset();
703 }
704
705 static uint16_t phys_section_add(MemoryRegionSection *section)
706 {
707 if (phys_sections_nb == phys_sections_nb_alloc) {
708 phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16);
709 phys_sections = g_renew(MemoryRegionSection, phys_sections,
710 phys_sections_nb_alloc);
711 }
712 phys_sections[phys_sections_nb] = *section;
713 return phys_sections_nb++;
714 }
715
716 static void phys_sections_clear(void)
717 {
718 phys_sections_nb = 0;
719 }
720
721 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
722 {
723 subpage_t *subpage;
724 hwaddr base = section->offset_within_address_space
725 & TARGET_PAGE_MASK;
726 MemoryRegionSection *existing = phys_page_find(d, base >> TARGET_PAGE_BITS);
727 MemoryRegionSection subsection = {
728 .offset_within_address_space = base,
729 .size = TARGET_PAGE_SIZE,
730 };
731 hwaddr start, end;
732
733 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
734
735 if (!(existing->mr->subpage)) {
736 subpage = subpage_init(base);
737 subsection.mr = &subpage->iomem;
738 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
739 phys_section_add(&subsection));
740 } else {
741 subpage = container_of(existing->mr, subpage_t, iomem);
742 }
743 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
744 end = start + section->size - 1;
745 subpage_register(subpage, start, end, phys_section_add(section));
746 }
747
748
749 static void register_multipage(AddressSpaceDispatch *d, MemoryRegionSection *section)
750 {
751 hwaddr start_addr = section->offset_within_address_space;
752 ram_addr_t size = section->size;
753 hwaddr addr;
754 uint16_t section_index = phys_section_add(section);
755
756 assert(size);
757
758 addr = start_addr;
759 phys_page_set(d, addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,
760 section_index);
761 }
762
763 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
764 {
765 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
766 MemoryRegionSection now = *section, remain = *section;
767
768 if ((now.offset_within_address_space & ~TARGET_PAGE_MASK)
769 || (now.size < TARGET_PAGE_SIZE)) {
770 now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space)
771 - now.offset_within_address_space,
772 now.size);
773 register_subpage(d, &now);
774 remain.size -= now.size;
775 remain.offset_within_address_space += now.size;
776 remain.offset_within_region += now.size;
777 }
778 while (remain.size >= TARGET_PAGE_SIZE) {
779 now = remain;
780 if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
781 now.size = TARGET_PAGE_SIZE;
782 register_subpage(d, &now);
783 } else {
784 now.size &= TARGET_PAGE_MASK;
785 register_multipage(d, &now);
786 }
787 remain.size -= now.size;
788 remain.offset_within_address_space += now.size;
789 remain.offset_within_region += now.size;
790 }
791 now = remain;
792 if (now.size) {
793 register_subpage(d, &now);
794 }
795 }
796
797 void qemu_flush_coalesced_mmio_buffer(void)
798 {
799 if (kvm_enabled())
800 kvm_flush_coalesced_mmio_buffer();
801 }
802
803 #if defined(__linux__) && !defined(TARGET_S390X)
804
805 #include <sys/vfs.h>
806
807 #define HUGETLBFS_MAGIC 0x958458f6
808
809 static long gethugepagesize(const char *path)
810 {
811 struct statfs fs;
812 int ret;
813
814 do {
815 ret = statfs(path, &fs);
816 } while (ret != 0 && errno == EINTR);
817
818 if (ret != 0) {
819 perror(path);
820 return 0;
821 }
822
823 if (fs.f_type != HUGETLBFS_MAGIC)
824 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
825
826 return fs.f_bsize;
827 }
828
829 static void *file_ram_alloc(RAMBlock *block,
830 ram_addr_t memory,
831 const char *path)
832 {
833 char *filename;
834 void *area;
835 int fd;
836 #ifdef MAP_POPULATE
837 int flags;
838 #endif
839 unsigned long hpagesize;
840
841 hpagesize = gethugepagesize(path);
842 if (!hpagesize) {
843 return NULL;
844 }
845
846 if (memory < hpagesize) {
847 return NULL;
848 }
849
850 if (kvm_enabled() && !kvm_has_sync_mmu()) {
851 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
852 return NULL;
853 }
854
855 if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
856 return NULL;
857 }
858
859 fd = mkstemp(filename);
860 if (fd < 0) {
861 perror("unable to create backing store for hugepages");
862 free(filename);
863 return NULL;
864 }
865 unlink(filename);
866 free(filename);
867
868 memory = (memory+hpagesize-1) & ~(hpagesize-1);
869
870 /*
871 * ftruncate is not supported by hugetlbfs in older
872 * hosts, so don't bother bailing out on errors.
873 * If anything goes wrong with it under other filesystems,
874 * mmap will fail.
875 */
876 if (ftruncate(fd, memory))
877 perror("ftruncate");
878
879 #ifdef MAP_POPULATE
880 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
881 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
882 * to sidestep this quirk.
883 */
884 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
885 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
886 #else
887 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
888 #endif
889 if (area == MAP_FAILED) {
890 perror("file_ram_alloc: can't mmap RAM pages");
891 close(fd);
892 return (NULL);
893 }
894 block->fd = fd;
895 return area;
896 }
897 #endif
898
899 static ram_addr_t find_ram_offset(ram_addr_t size)
900 {
901 RAMBlock *block, *next_block;
902 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
903
904 if (QLIST_EMPTY(&ram_list.blocks))
905 return 0;
906
907 QLIST_FOREACH(block, &ram_list.blocks, next) {
908 ram_addr_t end, next = RAM_ADDR_MAX;
909
910 end = block->offset + block->length;
911
912 QLIST_FOREACH(next_block, &ram_list.blocks, next) {
913 if (next_block->offset >= end) {
914 next = MIN(next, next_block->offset);
915 }
916 }
917 if (next - end >= size && next - end < mingap) {
918 offset = end;
919 mingap = next - end;
920 }
921 }
922
923 if (offset == RAM_ADDR_MAX) {
924 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
925 (uint64_t)size);
926 abort();
927 }
928
929 return offset;
930 }
931
932 ram_addr_t last_ram_offset(void)
933 {
934 RAMBlock *block;
935 ram_addr_t last = 0;
936
937 QLIST_FOREACH(block, &ram_list.blocks, next)
938 last = MAX(last, block->offset + block->length);
939
940 return last;
941 }
942
943 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
944 {
945 int ret;
946 QemuOpts *machine_opts;
947
948 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
949 machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
950 if (machine_opts &&
951 !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
952 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
953 if (ret) {
954 perror("qemu_madvise");
955 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
956 "but dump_guest_core=off specified\n");
957 }
958 }
959 }
960
961 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
962 {
963 RAMBlock *new_block, *block;
964
965 new_block = NULL;
966 QLIST_FOREACH(block, &ram_list.blocks, next) {
967 if (block->offset == addr) {
968 new_block = block;
969 break;
970 }
971 }
972 assert(new_block);
973 assert(!new_block->idstr[0]);
974
975 if (dev) {
976 char *id = qdev_get_dev_path(dev);
977 if (id) {
978 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
979 g_free(id);
980 }
981 }
982 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
983
984 QLIST_FOREACH(block, &ram_list.blocks, next) {
985 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
986 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
987 new_block->idstr);
988 abort();
989 }
990 }
991 }
992
993 static int memory_try_enable_merging(void *addr, size_t len)
994 {
995 QemuOpts *opts;
996
997 opts = qemu_opts_find(qemu_find_opts("machine"), 0);
998 if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
999 /* disabled by the user */
1000 return 0;
1001 }
1002
1003 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1004 }
1005
1006 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1007 MemoryRegion *mr)
1008 {
1009 RAMBlock *new_block;
1010
1011 size = TARGET_PAGE_ALIGN(size);
1012 new_block = g_malloc0(sizeof(*new_block));
1013
1014 new_block->mr = mr;
1015 new_block->offset = find_ram_offset(size);
1016 if (host) {
1017 new_block->host = host;
1018 new_block->flags |= RAM_PREALLOC_MASK;
1019 } else {
1020 if (mem_path) {
1021 #if defined (__linux__) && !defined(TARGET_S390X)
1022 new_block->host = file_ram_alloc(new_block, size, mem_path);
1023 if (!new_block->host) {
1024 new_block->host = qemu_vmalloc(size);
1025 memory_try_enable_merging(new_block->host, size);
1026 }
1027 #else
1028 fprintf(stderr, "-mem-path option unsupported\n");
1029 exit(1);
1030 #endif
1031 } else {
1032 if (xen_enabled()) {
1033 xen_ram_alloc(new_block->offset, size, mr);
1034 } else if (kvm_enabled()) {
1035 /* some s390/kvm configurations have special constraints */
1036 new_block->host = kvm_vmalloc(size);
1037 } else {
1038 new_block->host = qemu_vmalloc(size);
1039 }
1040 memory_try_enable_merging(new_block->host, size);
1041 }
1042 }
1043 new_block->length = size;
1044
1045 QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
1046
1047 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1048 last_ram_offset() >> TARGET_PAGE_BITS);
1049 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1050 0, size >> TARGET_PAGE_BITS);
1051 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1052
1053 qemu_ram_setup_dump(new_block->host, size);
1054 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1055
1056 if (kvm_enabled())
1057 kvm_setup_guest_memory(new_block->host, size);
1058
1059 return new_block->offset;
1060 }
1061
1062 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1063 {
1064 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1065 }
1066
1067 void qemu_ram_free_from_ptr(ram_addr_t addr)
1068 {
1069 RAMBlock *block;
1070
1071 QLIST_FOREACH(block, &ram_list.blocks, next) {
1072 if (addr == block->offset) {
1073 QLIST_REMOVE(block, next);
1074 g_free(block);
1075 return;
1076 }
1077 }
1078 }
1079
1080 void qemu_ram_free(ram_addr_t addr)
1081 {
1082 RAMBlock *block;
1083
1084 QLIST_FOREACH(block, &ram_list.blocks, next) {
1085 if (addr == block->offset) {
1086 QLIST_REMOVE(block, next);
1087 if (block->flags & RAM_PREALLOC_MASK) {
1088 ;
1089 } else if (mem_path) {
1090 #if defined (__linux__) && !defined(TARGET_S390X)
1091 if (block->fd) {
1092 munmap(block->host, block->length);
1093 close(block->fd);
1094 } else {
1095 qemu_vfree(block->host);
1096 }
1097 #else
1098 abort();
1099 #endif
1100 } else {
1101 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1102 munmap(block->host, block->length);
1103 #else
1104 if (xen_enabled()) {
1105 xen_invalidate_map_cache_entry(block->host);
1106 } else {
1107 qemu_vfree(block->host);
1108 }
1109 #endif
1110 }
1111 g_free(block);
1112 return;
1113 }
1114 }
1115
1116 }
1117
1118 #ifndef _WIN32
1119 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1120 {
1121 RAMBlock *block;
1122 ram_addr_t offset;
1123 int flags;
1124 void *area, *vaddr;
1125
1126 QLIST_FOREACH(block, &ram_list.blocks, next) {
1127 offset = addr - block->offset;
1128 if (offset < block->length) {
1129 vaddr = block->host + offset;
1130 if (block->flags & RAM_PREALLOC_MASK) {
1131 ;
1132 } else {
1133 flags = MAP_FIXED;
1134 munmap(vaddr, length);
1135 if (mem_path) {
1136 #if defined(__linux__) && !defined(TARGET_S390X)
1137 if (block->fd) {
1138 #ifdef MAP_POPULATE
1139 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1140 MAP_PRIVATE;
1141 #else
1142 flags |= MAP_PRIVATE;
1143 #endif
1144 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1145 flags, block->fd, offset);
1146 } else {
1147 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1148 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1149 flags, -1, 0);
1150 }
1151 #else
1152 abort();
1153 #endif
1154 } else {
1155 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1156 flags |= MAP_SHARED | MAP_ANONYMOUS;
1157 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1158 flags, -1, 0);
1159 #else
1160 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1161 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1162 flags, -1, 0);
1163 #endif
1164 }
1165 if (area != vaddr) {
1166 fprintf(stderr, "Could not remap addr: "
1167 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1168 length, addr);
1169 exit(1);
1170 }
1171 memory_try_enable_merging(vaddr, length);
1172 qemu_ram_setup_dump(vaddr, length);
1173 }
1174 return;
1175 }
1176 }
1177 }
1178 #endif /* !_WIN32 */
1179
1180 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1181 With the exception of the softmmu code in this file, this should
1182 only be used for local memory (e.g. video ram) that the device owns,
1183 and knows it isn't going to access beyond the end of the block.
1184
1185 It should not be used for general purpose DMA.
1186 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1187 */
1188 void *qemu_get_ram_ptr(ram_addr_t addr)
1189 {
1190 RAMBlock *block;
1191
1192 QLIST_FOREACH(block, &ram_list.blocks, next) {
1193 if (addr - block->offset < block->length) {
1194 /* Move this entry to to start of the list. */
1195 if (block != QLIST_FIRST(&ram_list.blocks)) {
1196 QLIST_REMOVE(block, next);
1197 QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
1198 }
1199 if (xen_enabled()) {
1200 /* We need to check if the requested address is in the RAM
1201 * because we don't want to map the entire memory in QEMU.
1202 * In that case just map until the end of the page.
1203 */
1204 if (block->offset == 0) {
1205 return xen_map_cache(addr, 0, 0);
1206 } else if (block->host == NULL) {
1207 block->host =
1208 xen_map_cache(block->offset, block->length, 1);
1209 }
1210 }
1211 return block->host + (addr - block->offset);
1212 }
1213 }
1214
1215 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1216 abort();
1217
1218 return NULL;
1219 }
1220
1221 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1222 * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
1223 */
1224 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1225 {
1226 RAMBlock *block;
1227
1228 QLIST_FOREACH(block, &ram_list.blocks, next) {
1229 if (addr - block->offset < block->length) {
1230 if (xen_enabled()) {
1231 /* We need to check if the requested address is in the RAM
1232 * because we don't want to map the entire memory in QEMU.
1233 * In that case just map until the end of the page.
1234 */
1235 if (block->offset == 0) {
1236 return xen_map_cache(addr, 0, 0);
1237 } else if (block->host == NULL) {
1238 block->host =
1239 xen_map_cache(block->offset, block->length, 1);
1240 }
1241 }
1242 return block->host + (addr - block->offset);
1243 }
1244 }
1245
1246 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1247 abort();
1248
1249 return NULL;
1250 }
1251
1252 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1253 * but takes a size argument */
1254 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1255 {
1256 if (*size == 0) {
1257 return NULL;
1258 }
1259 if (xen_enabled()) {
1260 return xen_map_cache(addr, *size, 1);
1261 } else {
1262 RAMBlock *block;
1263
1264 QLIST_FOREACH(block, &ram_list.blocks, next) {
1265 if (addr - block->offset < block->length) {
1266 if (addr - block->offset + *size > block->length)
1267 *size = block->length - addr + block->offset;
1268 return block->host + (addr - block->offset);
1269 }
1270 }
1271
1272 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1273 abort();
1274 }
1275 }
1276
1277 void qemu_put_ram_ptr(void *addr)
1278 {
1279 trace_qemu_put_ram_ptr(addr);
1280 }
1281
1282 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1283 {
1284 RAMBlock *block;
1285 uint8_t *host = ptr;
1286
1287 if (xen_enabled()) {
1288 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1289 return 0;
1290 }
1291
1292 QLIST_FOREACH(block, &ram_list.blocks, next) {
1293 /* This case append when the block is not mapped. */
1294 if (block->host == NULL) {
1295 continue;
1296 }
1297 if (host - block->host < block->length) {
1298 *ram_addr = block->offset + (host - block->host);
1299 return 0;
1300 }
1301 }
1302
1303 return -1;
1304 }
1305
1306 /* Some of the softmmu routines need to translate from a host pointer
1307 (typically a TLB entry) back to a ram offset. */
1308 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
1309 {
1310 ram_addr_t ram_addr;
1311
1312 if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
1313 fprintf(stderr, "Bad ram pointer %p\n", ptr);
1314 abort();
1315 }
1316 return ram_addr;
1317 }
1318
1319 static uint64_t unassigned_mem_read(void *opaque, hwaddr addr,
1320 unsigned size)
1321 {
1322 #ifdef DEBUG_UNASSIGNED
1323 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
1324 #endif
1325 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1326 cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, size);
1327 #endif
1328 return 0;
1329 }
1330
1331 static void unassigned_mem_write(void *opaque, hwaddr addr,
1332 uint64_t val, unsigned size)
1333 {
1334 #ifdef DEBUG_UNASSIGNED
1335 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%"PRIx64"\n", addr, val);
1336 #endif
1337 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1338 cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, size);
1339 #endif
1340 }
1341
1342 static const MemoryRegionOps unassigned_mem_ops = {
1343 .read = unassigned_mem_read,
1344 .write = unassigned_mem_write,
1345 .endianness = DEVICE_NATIVE_ENDIAN,
1346 };
1347
1348 static uint64_t error_mem_read(void *opaque, hwaddr addr,
1349 unsigned size)
1350 {
1351 abort();
1352 }
1353
1354 static void error_mem_write(void *opaque, hwaddr addr,
1355 uint64_t value, unsigned size)
1356 {
1357 abort();
1358 }
1359
1360 static const MemoryRegionOps error_mem_ops = {
1361 .read = error_mem_read,
1362 .write = error_mem_write,
1363 .endianness = DEVICE_NATIVE_ENDIAN,
1364 };
1365
1366 static const MemoryRegionOps rom_mem_ops = {
1367 .read = error_mem_read,
1368 .write = unassigned_mem_write,
1369 .endianness = DEVICE_NATIVE_ENDIAN,
1370 };
1371
1372 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1373 uint64_t val, unsigned size)
1374 {
1375 int dirty_flags;
1376 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1377 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1378 #if !defined(CONFIG_USER_ONLY)
1379 tb_invalidate_phys_page_fast(ram_addr, size);
1380 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1381 #endif
1382 }
1383 switch (size) {
1384 case 1:
1385 stb_p(qemu_get_ram_ptr(ram_addr), val);
1386 break;
1387 case 2:
1388 stw_p(qemu_get_ram_ptr(ram_addr), val);
1389 break;
1390 case 4:
1391 stl_p(qemu_get_ram_ptr(ram_addr), val);
1392 break;
1393 default:
1394 abort();
1395 }
1396 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1397 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1398 /* we remove the notdirty callback only if the code has been
1399 flushed */
1400 if (dirty_flags == 0xff)
1401 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1402 }
1403
1404 static const MemoryRegionOps notdirty_mem_ops = {
1405 .read = error_mem_read,
1406 .write = notdirty_mem_write,
1407 .endianness = DEVICE_NATIVE_ENDIAN,
1408 };
1409
1410 /* Generate a debug exception if a watchpoint has been hit. */
1411 static void check_watchpoint(int offset, int len_mask, int flags)
1412 {
1413 CPUArchState *env = cpu_single_env;
1414 target_ulong pc, cs_base;
1415 target_ulong vaddr;
1416 CPUWatchpoint *wp;
1417 int cpu_flags;
1418
1419 if (env->watchpoint_hit) {
1420 /* We re-entered the check after replacing the TB. Now raise
1421 * the debug interrupt so that is will trigger after the
1422 * current instruction. */
1423 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
1424 return;
1425 }
1426 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1427 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1428 if ((vaddr == (wp->vaddr & len_mask) ||
1429 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1430 wp->flags |= BP_WATCHPOINT_HIT;
1431 if (!env->watchpoint_hit) {
1432 env->watchpoint_hit = wp;
1433 tb_check_watchpoint(env);
1434 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1435 env->exception_index = EXCP_DEBUG;
1436 cpu_loop_exit(env);
1437 } else {
1438 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1439 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1440 cpu_resume_from_signal(env, NULL);
1441 }
1442 }
1443 } else {
1444 wp->flags &= ~BP_WATCHPOINT_HIT;
1445 }
1446 }
1447 }
1448
1449 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1450 so these check for a hit then pass through to the normal out-of-line
1451 phys routines. */
1452 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1453 unsigned size)
1454 {
1455 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1456 switch (size) {
1457 case 1: return ldub_phys(addr);
1458 case 2: return lduw_phys(addr);
1459 case 4: return ldl_phys(addr);
1460 default: abort();
1461 }
1462 }
1463
1464 static void watch_mem_write(void *opaque, hwaddr addr,
1465 uint64_t val, unsigned size)
1466 {
1467 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1468 switch (size) {
1469 case 1:
1470 stb_phys(addr, val);
1471 break;
1472 case 2:
1473 stw_phys(addr, val);
1474 break;
1475 case 4:
1476 stl_phys(addr, val);
1477 break;
1478 default: abort();
1479 }
1480 }
1481
1482 static const MemoryRegionOps watch_mem_ops = {
1483 .read = watch_mem_read,
1484 .write = watch_mem_write,
1485 .endianness = DEVICE_NATIVE_ENDIAN,
1486 };
1487
1488 static uint64_t subpage_read(void *opaque, hwaddr addr,
1489 unsigned len)
1490 {
1491 subpage_t *mmio = opaque;
1492 unsigned int idx = SUBPAGE_IDX(addr);
1493 MemoryRegionSection *section;
1494 #if defined(DEBUG_SUBPAGE)
1495 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
1496 mmio, len, addr, idx);
1497 #endif
1498
1499 section = &phys_sections[mmio->sub_section[idx]];
1500 addr += mmio->base;
1501 addr -= section->offset_within_address_space;
1502 addr += section->offset_within_region;
1503 return io_mem_read(section->mr, addr, len);
1504 }
1505
1506 static void subpage_write(void *opaque, hwaddr addr,
1507 uint64_t value, unsigned len)
1508 {
1509 subpage_t *mmio = opaque;
1510 unsigned int idx = SUBPAGE_IDX(addr);
1511 MemoryRegionSection *section;
1512 #if defined(DEBUG_SUBPAGE)
1513 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1514 " idx %d value %"PRIx64"\n",
1515 __func__, mmio, len, addr, idx, value);
1516 #endif
1517
1518 section = &phys_sections[mmio->sub_section[idx]];
1519 addr += mmio->base;
1520 addr -= section->offset_within_address_space;
1521 addr += section->offset_within_region;
1522 io_mem_write(section->mr, addr, value, len);
1523 }
1524
1525 static const MemoryRegionOps subpage_ops = {
1526 .read = subpage_read,
1527 .write = subpage_write,
1528 .endianness = DEVICE_NATIVE_ENDIAN,
1529 };
1530
1531 static uint64_t subpage_ram_read(void *opaque, hwaddr addr,
1532 unsigned size)
1533 {
1534 ram_addr_t raddr = addr;
1535 void *ptr = qemu_get_ram_ptr(raddr);
1536 switch (size) {
1537 case 1: return ldub_p(ptr);
1538 case 2: return lduw_p(ptr);
1539 case 4: return ldl_p(ptr);
1540 default: abort();
1541 }
1542 }
1543
1544 static void subpage_ram_write(void *opaque, hwaddr addr,
1545 uint64_t value, unsigned size)
1546 {
1547 ram_addr_t raddr = addr;
1548 void *ptr = qemu_get_ram_ptr(raddr);
1549 switch (size) {
1550 case 1: return stb_p(ptr, value);
1551 case 2: return stw_p(ptr, value);
1552 case 4: return stl_p(ptr, value);
1553 default: abort();
1554 }
1555 }
1556
1557 static const MemoryRegionOps subpage_ram_ops = {
1558 .read = subpage_ram_read,
1559 .write = subpage_ram_write,
1560 .endianness = DEVICE_NATIVE_ENDIAN,
1561 };
1562
1563 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1564 uint16_t section)
1565 {
1566 int idx, eidx;
1567
1568 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1569 return -1;
1570 idx = SUBPAGE_IDX(start);
1571 eidx = SUBPAGE_IDX(end);
1572 #if defined(DEBUG_SUBPAGE)
1573 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1574 mmio, start, end, idx, eidx, memory);
1575 #endif
1576 if (memory_region_is_ram(phys_sections[section].mr)) {
1577 MemoryRegionSection new_section = phys_sections[section];
1578 new_section.mr = &io_mem_subpage_ram;
1579 section = phys_section_add(&new_section);
1580 }
1581 for (; idx <= eidx; idx++) {
1582 mmio->sub_section[idx] = section;
1583 }
1584
1585 return 0;
1586 }
1587
1588 static subpage_t *subpage_init(hwaddr base)
1589 {
1590 subpage_t *mmio;
1591
1592 mmio = g_malloc0(sizeof(subpage_t));
1593
1594 mmio->base = base;
1595 memory_region_init_io(&mmio->iomem, &subpage_ops, mmio,
1596 "subpage", TARGET_PAGE_SIZE);
1597 mmio->iomem.subpage = true;
1598 #if defined(DEBUG_SUBPAGE)
1599 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1600 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1601 #endif
1602 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, phys_section_unassigned);
1603
1604 return mmio;
1605 }
1606
1607 static uint16_t dummy_section(MemoryRegion *mr)
1608 {
1609 MemoryRegionSection section = {
1610 .mr = mr,
1611 .offset_within_address_space = 0,
1612 .offset_within_region = 0,
1613 .size = UINT64_MAX,
1614 };
1615
1616 return phys_section_add(&section);
1617 }
1618
1619 MemoryRegion *iotlb_to_region(hwaddr index)
1620 {
1621 return phys_sections[index & ~TARGET_PAGE_MASK].mr;
1622 }
1623
1624 static void io_mem_init(void)
1625 {
1626 memory_region_init_io(&io_mem_ram, &error_mem_ops, NULL, "ram", UINT64_MAX);
1627 memory_region_init_io(&io_mem_rom, &rom_mem_ops, NULL, "rom", UINT64_MAX);
1628 memory_region_init_io(&io_mem_unassigned, &unassigned_mem_ops, NULL,
1629 "unassigned", UINT64_MAX);
1630 memory_region_init_io(&io_mem_notdirty, &notdirty_mem_ops, NULL,
1631 "notdirty", UINT64_MAX);
1632 memory_region_init_io(&io_mem_subpage_ram, &subpage_ram_ops, NULL,
1633 "subpage-ram", UINT64_MAX);
1634 memory_region_init_io(&io_mem_watch, &watch_mem_ops, NULL,
1635 "watch", UINT64_MAX);
1636 }
1637
1638 static void mem_begin(MemoryListener *listener)
1639 {
1640 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
1641
1642 destroy_all_mappings(d);
1643 d->phys_map.ptr = PHYS_MAP_NODE_NIL;
1644 }
1645
1646 static void core_begin(MemoryListener *listener)
1647 {
1648 phys_sections_clear();
1649 phys_section_unassigned = dummy_section(&io_mem_unassigned);
1650 phys_section_notdirty = dummy_section(&io_mem_notdirty);
1651 phys_section_rom = dummy_section(&io_mem_rom);
1652 phys_section_watch = dummy_section(&io_mem_watch);
1653 }
1654
1655 static void tcg_commit(MemoryListener *listener)
1656 {
1657 CPUArchState *env;
1658
1659 /* since each CPU stores ram addresses in its TLB cache, we must
1660 reset the modified entries */
1661 /* XXX: slow ! */
1662 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1663 tlb_flush(env, 1);
1664 }
1665 }
1666
1667 static void core_log_global_start(MemoryListener *listener)
1668 {
1669 cpu_physical_memory_set_dirty_tracking(1);
1670 }
1671
1672 static void core_log_global_stop(MemoryListener *listener)
1673 {
1674 cpu_physical_memory_set_dirty_tracking(0);
1675 }
1676
1677 static void io_region_add(MemoryListener *listener,
1678 MemoryRegionSection *section)
1679 {
1680 MemoryRegionIORange *mrio = g_new(MemoryRegionIORange, 1);
1681
1682 mrio->mr = section->mr;
1683 mrio->offset = section->offset_within_region;
1684 iorange_init(&mrio->iorange, &memory_region_iorange_ops,
1685 section->offset_within_address_space, section->size);
1686 ioport_register(&mrio->iorange);
1687 }
1688
1689 static void io_region_del(MemoryListener *listener,
1690 MemoryRegionSection *section)
1691 {
1692 isa_unassign_ioport(section->offset_within_address_space, section->size);
1693 }
1694
1695 static MemoryListener core_memory_listener = {
1696 .begin = core_begin,
1697 .log_global_start = core_log_global_start,
1698 .log_global_stop = core_log_global_stop,
1699 .priority = 1,
1700 };
1701
1702 static MemoryListener io_memory_listener = {
1703 .region_add = io_region_add,
1704 .region_del = io_region_del,
1705 .priority = 0,
1706 };
1707
1708 static MemoryListener tcg_memory_listener = {
1709 .commit = tcg_commit,
1710 };
1711
1712 void address_space_init_dispatch(AddressSpace *as)
1713 {
1714 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1715
1716 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1717 d->listener = (MemoryListener) {
1718 .begin = mem_begin,
1719 .region_add = mem_add,
1720 .region_nop = mem_add,
1721 .priority = 0,
1722 };
1723 as->dispatch = d;
1724 memory_listener_register(&d->listener, as);
1725 }
1726
1727 void address_space_destroy_dispatch(AddressSpace *as)
1728 {
1729 AddressSpaceDispatch *d = as->dispatch;
1730
1731 memory_listener_unregister(&d->listener);
1732 destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
1733 g_free(d);
1734 as->dispatch = NULL;
1735 }
1736
1737 static void memory_map_init(void)
1738 {
1739 system_memory = g_malloc(sizeof(*system_memory));
1740 memory_region_init(system_memory, "system", INT64_MAX);
1741 address_space_init(&address_space_memory, system_memory);
1742 address_space_memory.name = "memory";
1743
1744 system_io = g_malloc(sizeof(*system_io));
1745 memory_region_init(system_io, "io", 65536);
1746 address_space_init(&address_space_io, system_io);
1747 address_space_io.name = "I/O";
1748
1749 memory_listener_register(&core_memory_listener, &address_space_memory);
1750 memory_listener_register(&io_memory_listener, &address_space_io);
1751 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1752
1753 dma_context_init(&dma_context_memory, &address_space_memory,
1754 NULL, NULL, NULL);
1755 }
1756
1757 MemoryRegion *get_system_memory(void)
1758 {
1759 return system_memory;
1760 }
1761
1762 MemoryRegion *get_system_io(void)
1763 {
1764 return system_io;
1765 }
1766
1767 #endif /* !defined(CONFIG_USER_ONLY) */
1768
1769 /* physical memory access (slow version, mainly for debug) */
1770 #if defined(CONFIG_USER_ONLY)
1771 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1772 uint8_t *buf, int len, int is_write)
1773 {
1774 int l, flags;
1775 target_ulong page;
1776 void * p;
1777
1778 while (len > 0) {
1779 page = addr & TARGET_PAGE_MASK;
1780 l = (page + TARGET_PAGE_SIZE) - addr;
1781 if (l > len)
1782 l = len;
1783 flags = page_get_flags(page);
1784 if (!(flags & PAGE_VALID))
1785 return -1;
1786 if (is_write) {
1787 if (!(flags & PAGE_WRITE))
1788 return -1;
1789 /* XXX: this code should not depend on lock_user */
1790 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1791 return -1;
1792 memcpy(p, buf, l);
1793 unlock_user(p, addr, l);
1794 } else {
1795 if (!(flags & PAGE_READ))
1796 return -1;
1797 /* XXX: this code should not depend on lock_user */
1798 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1799 return -1;
1800 memcpy(buf, p, l);
1801 unlock_user(p, addr, 0);
1802 }
1803 len -= l;
1804 buf += l;
1805 addr += l;
1806 }
1807 return 0;
1808 }
1809
1810 #else
1811
1812 static void invalidate_and_set_dirty(hwaddr addr,
1813 hwaddr length)
1814 {
1815 if (!cpu_physical_memory_is_dirty(addr)) {
1816 /* invalidate code */
1817 tb_invalidate_phys_page_range(addr, addr + length, 0);
1818 /* set dirty bit */
1819 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1820 }
1821 xen_modified_memory(addr, length);
1822 }
1823
1824 void address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1825 int len, bool is_write)
1826 {
1827 AddressSpaceDispatch *d = as->dispatch;
1828 int l;
1829 uint8_t *ptr;
1830 uint32_t val;
1831 hwaddr page;
1832 MemoryRegionSection *section;
1833
1834 while (len > 0) {
1835 page = addr & TARGET_PAGE_MASK;
1836 l = (page + TARGET_PAGE_SIZE) - addr;
1837 if (l > len)
1838 l = len;
1839 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
1840
1841 if (is_write) {
1842 if (!memory_region_is_ram(section->mr)) {
1843 hwaddr addr1;
1844 addr1 = memory_region_section_addr(section, addr);
1845 /* XXX: could force cpu_single_env to NULL to avoid
1846 potential bugs */
1847 if (l >= 4 && ((addr1 & 3) == 0)) {
1848 /* 32 bit write access */
1849 val = ldl_p(buf);
1850 io_mem_write(section->mr, addr1, val, 4);
1851 l = 4;
1852 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1853 /* 16 bit write access */
1854 val = lduw_p(buf);
1855 io_mem_write(section->mr, addr1, val, 2);
1856 l = 2;
1857 } else {
1858 /* 8 bit write access */
1859 val = ldub_p(buf);
1860 io_mem_write(section->mr, addr1, val, 1);
1861 l = 1;
1862 }
1863 } else if (!section->readonly) {
1864 ram_addr_t addr1;
1865 addr1 = memory_region_get_ram_addr(section->mr)
1866 + memory_region_section_addr(section, addr);
1867 /* RAM case */
1868 ptr = qemu_get_ram_ptr(addr1);
1869 memcpy(ptr, buf, l);
1870 invalidate_and_set_dirty(addr1, l);
1871 qemu_put_ram_ptr(ptr);
1872 }
1873 } else {
1874 if (!(memory_region_is_ram(section->mr) ||
1875 memory_region_is_romd(section->mr))) {
1876 hwaddr addr1;
1877 /* I/O case */
1878 addr1 = memory_region_section_addr(section, addr);
1879 if (l >= 4 && ((addr1 & 3) == 0)) {
1880 /* 32 bit read access */
1881 val = io_mem_read(section->mr, addr1, 4);
1882 stl_p(buf, val);
1883 l = 4;
1884 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1885 /* 16 bit read access */
1886 val = io_mem_read(section->mr, addr1, 2);
1887 stw_p(buf, val);
1888 l = 2;
1889 } else {
1890 /* 8 bit read access */
1891 val = io_mem_read(section->mr, addr1, 1);
1892 stb_p(buf, val);
1893 l = 1;
1894 }
1895 } else {
1896 /* RAM case */
1897 ptr = qemu_get_ram_ptr(section->mr->ram_addr
1898 + memory_region_section_addr(section,
1899 addr));
1900 memcpy(buf, ptr, l);
1901 qemu_put_ram_ptr(ptr);
1902 }
1903 }
1904 len -= l;
1905 buf += l;
1906 addr += l;
1907 }
1908 }
1909
1910 void address_space_write(AddressSpace *as, hwaddr addr,
1911 const uint8_t *buf, int len)
1912 {
1913 address_space_rw(as, addr, (uint8_t *)buf, len, true);
1914 }
1915
1916 /**
1917 * address_space_read: read from an address space.
1918 *
1919 * @as: #AddressSpace to be accessed
1920 * @addr: address within that address space
1921 * @buf: buffer with the data transferred
1922 */
1923 void address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1924 {
1925 address_space_rw(as, addr, buf, len, false);
1926 }
1927
1928
1929 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1930 int len, int is_write)
1931 {
1932 return address_space_rw(&address_space_memory, addr, buf, len, is_write);
1933 }
1934
1935 /* used for ROM loading : can write in RAM and ROM */
1936 void cpu_physical_memory_write_rom(hwaddr addr,
1937 const uint8_t *buf, int len)
1938 {
1939 AddressSpaceDispatch *d = address_space_memory.dispatch;
1940 int l;
1941 uint8_t *ptr;
1942 hwaddr page;
1943 MemoryRegionSection *section;
1944
1945 while (len > 0) {
1946 page = addr & TARGET_PAGE_MASK;
1947 l = (page + TARGET_PAGE_SIZE) - addr;
1948 if (l > len)
1949 l = len;
1950 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
1951
1952 if (!(memory_region_is_ram(section->mr) ||
1953 memory_region_is_romd(section->mr))) {
1954 /* do nothing */
1955 } else {
1956 unsigned long addr1;
1957 addr1 = memory_region_get_ram_addr(section->mr)
1958 + memory_region_section_addr(section, addr);
1959 /* ROM/RAM case */
1960 ptr = qemu_get_ram_ptr(addr1);
1961 memcpy(ptr, buf, l);
1962 invalidate_and_set_dirty(addr1, l);
1963 qemu_put_ram_ptr(ptr);
1964 }
1965 len -= l;
1966 buf += l;
1967 addr += l;
1968 }
1969 }
1970
1971 typedef struct {
1972 void *buffer;
1973 hwaddr addr;
1974 hwaddr len;
1975 } BounceBuffer;
1976
1977 static BounceBuffer bounce;
1978
1979 typedef struct MapClient {
1980 void *opaque;
1981 void (*callback)(void *opaque);
1982 QLIST_ENTRY(MapClient) link;
1983 } MapClient;
1984
1985 static QLIST_HEAD(map_client_list, MapClient) map_client_list
1986 = QLIST_HEAD_INITIALIZER(map_client_list);
1987
1988 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
1989 {
1990 MapClient *client = g_malloc(sizeof(*client));
1991
1992 client->opaque = opaque;
1993 client->callback = callback;
1994 QLIST_INSERT_HEAD(&map_client_list, client, link);
1995 return client;
1996 }
1997
1998 static void cpu_unregister_map_client(void *_client)
1999 {
2000 MapClient *client = (MapClient *)_client;
2001
2002 QLIST_REMOVE(client, link);
2003 g_free(client);
2004 }
2005
2006 static void cpu_notify_map_clients(void)
2007 {
2008 MapClient *client;
2009
2010 while (!QLIST_EMPTY(&map_client_list)) {
2011 client = QLIST_FIRST(&map_client_list);
2012 client->callback(client->opaque);
2013 cpu_unregister_map_client(client);
2014 }
2015 }
2016
2017 /* Map a physical memory region into a host virtual address.
2018 * May map a subset of the requested range, given by and returned in *plen.
2019 * May return NULL if resources needed to perform the mapping are exhausted.
2020 * Use only for reads OR writes - not for read-modify-write operations.
2021 * Use cpu_register_map_client() to know when retrying the map operation is
2022 * likely to succeed.
2023 */
2024 void *address_space_map(AddressSpace *as,
2025 hwaddr addr,
2026 hwaddr *plen,
2027 bool is_write)
2028 {
2029 AddressSpaceDispatch *d = as->dispatch;
2030 hwaddr len = *plen;
2031 hwaddr todo = 0;
2032 int l;
2033 hwaddr page;
2034 MemoryRegionSection *section;
2035 ram_addr_t raddr = RAM_ADDR_MAX;
2036 ram_addr_t rlen;
2037 void *ret;
2038
2039 while (len > 0) {
2040 page = addr & TARGET_PAGE_MASK;
2041 l = (page + TARGET_PAGE_SIZE) - addr;
2042 if (l > len)
2043 l = len;
2044 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
2045
2046 if (!(memory_region_is_ram(section->mr) && !section->readonly)) {
2047 if (todo || bounce.buffer) {
2048 break;
2049 }
2050 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2051 bounce.addr = addr;
2052 bounce.len = l;
2053 if (!is_write) {
2054 address_space_read(as, addr, bounce.buffer, l);
2055 }
2056
2057 *plen = l;
2058 return bounce.buffer;
2059 }
2060 if (!todo) {
2061 raddr = memory_region_get_ram_addr(section->mr)
2062 + memory_region_section_addr(section, addr);
2063 }
2064
2065 len -= l;
2066 addr += l;
2067 todo += l;
2068 }
2069 rlen = todo;
2070 ret = qemu_ram_ptr_length(raddr, &rlen);
2071 *plen = rlen;
2072 return ret;
2073 }
2074
2075 /* Unmaps a memory region previously mapped by address_space_map().
2076 * Will also mark the memory as dirty if is_write == 1. access_len gives
2077 * the amount of memory that was actually read or written by the caller.
2078 */
2079 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2080 int is_write, hwaddr access_len)
2081 {
2082 if (buffer != bounce.buffer) {
2083 if (is_write) {
2084 ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
2085 while (access_len) {
2086 unsigned l;
2087 l = TARGET_PAGE_SIZE;
2088 if (l > access_len)
2089 l = access_len;
2090 invalidate_and_set_dirty(addr1, l);
2091 addr1 += l;
2092 access_len -= l;
2093 }
2094 }
2095 if (xen_enabled()) {
2096 xen_invalidate_map_cache_entry(buffer);
2097 }
2098 return;
2099 }
2100 if (is_write) {
2101 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2102 }
2103 qemu_vfree(bounce.buffer);
2104 bounce.buffer = NULL;
2105 cpu_notify_map_clients();
2106 }
2107
2108 void *cpu_physical_memory_map(hwaddr addr,
2109 hwaddr *plen,
2110 int is_write)
2111 {
2112 return address_space_map(&address_space_memory, addr, plen, is_write);
2113 }
2114
2115 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2116 int is_write, hwaddr access_len)
2117 {
2118 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2119 }
2120
2121 /* warning: addr must be aligned */
2122 static inline uint32_t ldl_phys_internal(hwaddr addr,
2123 enum device_endian endian)
2124 {
2125 uint8_t *ptr;
2126 uint32_t val;
2127 MemoryRegionSection *section;
2128
2129 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2130
2131 if (!(memory_region_is_ram(section->mr) ||
2132 memory_region_is_romd(section->mr))) {
2133 /* I/O case */
2134 addr = memory_region_section_addr(section, addr);
2135 val = io_mem_read(section->mr, addr, 4);
2136 #if defined(TARGET_WORDS_BIGENDIAN)
2137 if (endian == DEVICE_LITTLE_ENDIAN) {
2138 val = bswap32(val);
2139 }
2140 #else
2141 if (endian == DEVICE_BIG_ENDIAN) {
2142 val = bswap32(val);
2143 }
2144 #endif
2145 } else {
2146 /* RAM case */
2147 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2148 & TARGET_PAGE_MASK)
2149 + memory_region_section_addr(section, addr));
2150 switch (endian) {
2151 case DEVICE_LITTLE_ENDIAN:
2152 val = ldl_le_p(ptr);
2153 break;
2154 case DEVICE_BIG_ENDIAN:
2155 val = ldl_be_p(ptr);
2156 break;
2157 default:
2158 val = ldl_p(ptr);
2159 break;
2160 }
2161 }
2162 return val;
2163 }
2164
2165 uint32_t ldl_phys(hwaddr addr)
2166 {
2167 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2168 }
2169
2170 uint32_t ldl_le_phys(hwaddr addr)
2171 {
2172 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2173 }
2174
2175 uint32_t ldl_be_phys(hwaddr addr)
2176 {
2177 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2178 }
2179
2180 /* warning: addr must be aligned */
2181 static inline uint64_t ldq_phys_internal(hwaddr addr,
2182 enum device_endian endian)
2183 {
2184 uint8_t *ptr;
2185 uint64_t val;
2186 MemoryRegionSection *section;
2187
2188 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2189
2190 if (!(memory_region_is_ram(section->mr) ||
2191 memory_region_is_romd(section->mr))) {
2192 /* I/O case */
2193 addr = memory_region_section_addr(section, addr);
2194
2195 /* XXX This is broken when device endian != cpu endian.
2196 Fix and add "endian" variable check */
2197 #ifdef TARGET_WORDS_BIGENDIAN
2198 val = io_mem_read(section->mr, addr, 4) << 32;
2199 val |= io_mem_read(section->mr, addr + 4, 4);
2200 #else
2201 val = io_mem_read(section->mr, addr, 4);
2202 val |= io_mem_read(section->mr, addr + 4, 4) << 32;
2203 #endif
2204 } else {
2205 /* RAM case */
2206 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2207 & TARGET_PAGE_MASK)
2208 + memory_region_section_addr(section, addr));
2209 switch (endian) {
2210 case DEVICE_LITTLE_ENDIAN:
2211 val = ldq_le_p(ptr);
2212 break;
2213 case DEVICE_BIG_ENDIAN:
2214 val = ldq_be_p(ptr);
2215 break;
2216 default:
2217 val = ldq_p(ptr);
2218 break;
2219 }
2220 }
2221 return val;
2222 }
2223
2224 uint64_t ldq_phys(hwaddr addr)
2225 {
2226 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2227 }
2228
2229 uint64_t ldq_le_phys(hwaddr addr)
2230 {
2231 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2232 }
2233
2234 uint64_t ldq_be_phys(hwaddr addr)
2235 {
2236 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2237 }
2238
2239 /* XXX: optimize */
2240 uint32_t ldub_phys(hwaddr addr)
2241 {
2242 uint8_t val;
2243 cpu_physical_memory_read(addr, &val, 1);
2244 return val;
2245 }
2246
2247 /* warning: addr must be aligned */
2248 static inline uint32_t lduw_phys_internal(hwaddr addr,
2249 enum device_endian endian)
2250 {
2251 uint8_t *ptr;
2252 uint64_t val;
2253 MemoryRegionSection *section;
2254
2255 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2256
2257 if (!(memory_region_is_ram(section->mr) ||
2258 memory_region_is_romd(section->mr))) {
2259 /* I/O case */
2260 addr = memory_region_section_addr(section, addr);
2261 val = io_mem_read(section->mr, addr, 2);
2262 #if defined(TARGET_WORDS_BIGENDIAN)
2263 if (endian == DEVICE_LITTLE_ENDIAN) {
2264 val = bswap16(val);
2265 }
2266 #else
2267 if (endian == DEVICE_BIG_ENDIAN) {
2268 val = bswap16(val);
2269 }
2270 #endif
2271 } else {
2272 /* RAM case */
2273 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2274 & TARGET_PAGE_MASK)
2275 + memory_region_section_addr(section, addr));
2276 switch (endian) {
2277 case DEVICE_LITTLE_ENDIAN:
2278 val = lduw_le_p(ptr);
2279 break;
2280 case DEVICE_BIG_ENDIAN:
2281 val = lduw_be_p(ptr);
2282 break;
2283 default:
2284 val = lduw_p(ptr);
2285 break;
2286 }
2287 }
2288 return val;
2289 }
2290
2291 uint32_t lduw_phys(hwaddr addr)
2292 {
2293 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2294 }
2295
2296 uint32_t lduw_le_phys(hwaddr addr)
2297 {
2298 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2299 }
2300
2301 uint32_t lduw_be_phys(hwaddr addr)
2302 {
2303 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2304 }
2305
2306 /* warning: addr must be aligned. The ram page is not masked as dirty
2307 and the code inside is not invalidated. It is useful if the dirty
2308 bits are used to track modified PTEs */
2309 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2310 {
2311 uint8_t *ptr;
2312 MemoryRegionSection *section;
2313
2314 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2315
2316 if (!memory_region_is_ram(section->mr) || section->readonly) {
2317 addr = memory_region_section_addr(section, addr);
2318 if (memory_region_is_ram(section->mr)) {
2319 section = &phys_sections[phys_section_rom];
2320 }
2321 io_mem_write(section->mr, addr, val, 4);
2322 } else {
2323 unsigned long addr1 = (memory_region_get_ram_addr(section->mr)
2324 & TARGET_PAGE_MASK)
2325 + memory_region_section_addr(section, addr);
2326 ptr = qemu_get_ram_ptr(addr1);
2327 stl_p(ptr, val);
2328
2329 if (unlikely(in_migration)) {
2330 if (!cpu_physical_memory_is_dirty(addr1)) {
2331 /* invalidate code */
2332 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2333 /* set dirty bit */
2334 cpu_physical_memory_set_dirty_flags(
2335 addr1, (0xff & ~CODE_DIRTY_FLAG));
2336 }
2337 }
2338 }
2339 }
2340
2341 void stq_phys_notdirty(hwaddr addr, uint64_t val)
2342 {
2343 uint8_t *ptr;
2344 MemoryRegionSection *section;
2345
2346 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2347
2348 if (!memory_region_is_ram(section->mr) || section->readonly) {
2349 addr = memory_region_section_addr(section, addr);
2350 if (memory_region_is_ram(section->mr)) {
2351 section = &phys_sections[phys_section_rom];
2352 }
2353 #ifdef TARGET_WORDS_BIGENDIAN
2354 io_mem_write(section->mr, addr, val >> 32, 4);
2355 io_mem_write(section->mr, addr + 4, (uint32_t)val, 4);
2356 #else
2357 io_mem_write(section->mr, addr, (uint32_t)val, 4);
2358 io_mem_write(section->mr, addr + 4, val >> 32, 4);
2359 #endif
2360 } else {
2361 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2362 & TARGET_PAGE_MASK)
2363 + memory_region_section_addr(section, addr));
2364 stq_p(ptr, val);
2365 }
2366 }
2367
2368 /* warning: addr must be aligned */
2369 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2370 enum device_endian endian)
2371 {
2372 uint8_t *ptr;
2373 MemoryRegionSection *section;
2374
2375 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2376
2377 if (!memory_region_is_ram(section->mr) || section->readonly) {
2378 addr = memory_region_section_addr(section, addr);
2379 if (memory_region_is_ram(section->mr)) {
2380 section = &phys_sections[phys_section_rom];
2381 }
2382 #if defined(TARGET_WORDS_BIGENDIAN)
2383 if (endian == DEVICE_LITTLE_ENDIAN) {
2384 val = bswap32(val);
2385 }
2386 #else
2387 if (endian == DEVICE_BIG_ENDIAN) {
2388 val = bswap32(val);
2389 }
2390 #endif
2391 io_mem_write(section->mr, addr, val, 4);
2392 } else {
2393 unsigned long addr1;
2394 addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2395 + memory_region_section_addr(section, addr);
2396 /* RAM case */
2397 ptr = qemu_get_ram_ptr(addr1);
2398 switch (endian) {
2399 case DEVICE_LITTLE_ENDIAN:
2400 stl_le_p(ptr, val);
2401 break;
2402 case DEVICE_BIG_ENDIAN:
2403 stl_be_p(ptr, val);
2404 break;
2405 default:
2406 stl_p(ptr, val);
2407 break;
2408 }
2409 invalidate_and_set_dirty(addr1, 4);
2410 }
2411 }
2412
2413 void stl_phys(hwaddr addr, uint32_t val)
2414 {
2415 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2416 }
2417
2418 void stl_le_phys(hwaddr addr, uint32_t val)
2419 {
2420 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2421 }
2422
2423 void stl_be_phys(hwaddr addr, uint32_t val)
2424 {
2425 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2426 }
2427
2428 /* XXX: optimize */
2429 void stb_phys(hwaddr addr, uint32_t val)
2430 {
2431 uint8_t v = val;
2432 cpu_physical_memory_write(addr, &v, 1);
2433 }
2434
2435 /* warning: addr must be aligned */
2436 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2437 enum device_endian endian)
2438 {
2439 uint8_t *ptr;
2440 MemoryRegionSection *section;
2441
2442 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2443
2444 if (!memory_region_is_ram(section->mr) || section->readonly) {
2445 addr = memory_region_section_addr(section, addr);
2446 if (memory_region_is_ram(section->mr)) {
2447 section = &phys_sections[phys_section_rom];
2448 }
2449 #if defined(TARGET_WORDS_BIGENDIAN)
2450 if (endian == DEVICE_LITTLE_ENDIAN) {
2451 val = bswap16(val);
2452 }
2453 #else
2454 if (endian == DEVICE_BIG_ENDIAN) {
2455 val = bswap16(val);
2456 }
2457 #endif
2458 io_mem_write(section->mr, addr, val, 2);
2459 } else {
2460 unsigned long addr1;
2461 addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2462 + memory_region_section_addr(section, addr);
2463 /* RAM case */
2464 ptr = qemu_get_ram_ptr(addr1);
2465 switch (endian) {
2466 case DEVICE_LITTLE_ENDIAN:
2467 stw_le_p(ptr, val);
2468 break;
2469 case DEVICE_BIG_ENDIAN:
2470 stw_be_p(ptr, val);
2471 break;
2472 default:
2473 stw_p(ptr, val);
2474 break;
2475 }
2476 invalidate_and_set_dirty(addr1, 2);
2477 }
2478 }
2479
2480 void stw_phys(hwaddr addr, uint32_t val)
2481 {
2482 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2483 }
2484
2485 void stw_le_phys(hwaddr addr, uint32_t val)
2486 {
2487 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2488 }
2489
2490 void stw_be_phys(hwaddr addr, uint32_t val)
2491 {
2492 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2493 }
2494
2495 /* XXX: optimize */
2496 void stq_phys(hwaddr addr, uint64_t val)
2497 {
2498 val = tswap64(val);
2499 cpu_physical_memory_write(addr, &val, 8);
2500 }
2501
2502 void stq_le_phys(hwaddr addr, uint64_t val)
2503 {
2504 val = cpu_to_le64(val);
2505 cpu_physical_memory_write(addr, &val, 8);
2506 }
2507
2508 void stq_be_phys(hwaddr addr, uint64_t val)
2509 {
2510 val = cpu_to_be64(val);
2511 cpu_physical_memory_write(addr, &val, 8);
2512 }
2513
2514 /* virtual memory access for debug (includes writing to ROM) */
2515 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2516 uint8_t *buf, int len, int is_write)
2517 {
2518 int l;
2519 hwaddr phys_addr;
2520 target_ulong page;
2521
2522 while (len > 0) {
2523 page = addr & TARGET_PAGE_MASK;
2524 phys_addr = cpu_get_phys_page_debug(env, page);
2525 /* if no physical page mapped, return an error */
2526 if (phys_addr == -1)
2527 return -1;
2528 l = (page + TARGET_PAGE_SIZE) - addr;
2529 if (l > len)
2530 l = len;
2531 phys_addr += (addr & ~TARGET_PAGE_MASK);
2532 if (is_write)
2533 cpu_physical_memory_write_rom(phys_addr, buf, l);
2534 else
2535 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2536 len -= l;
2537 buf += l;
2538 addr += l;
2539 }
2540 return 0;
2541 }
2542 #endif
2543
2544 #if !defined(CONFIG_USER_ONLY)
2545
2546 /*
2547 * A helper function for the _utterly broken_ virtio device model to find out if
2548 * it's running on a big endian machine. Don't do this at home kids!
2549 */
2550 bool virtio_is_big_endian(void);
2551 bool virtio_is_big_endian(void)
2552 {
2553 #if defined(TARGET_WORDS_BIGENDIAN)
2554 return true;
2555 #else
2556 return false;
2557 #endif
2558 }
2559
2560 #endif
2561
2562 #ifndef CONFIG_USER_ONLY
2563 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2564 {
2565 MemoryRegionSection *section;
2566
2567 section = phys_page_find(address_space_memory.dispatch,
2568 phys_addr >> TARGET_PAGE_BITS);
2569
2570 return !(memory_region_is_ram(section->mr) ||
2571 memory_region_is_romd(section->mr));
2572 }
2573 #endif
AR7 emulation for QEMU