?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
xen: fix trivial PCI passthrough MSI-X bug
[qemu/cris-port.git] / exec.c
blob4c1246a9f94a4ee4536f3290f7c910c713fa14df
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "osdep.h"
33 #include "kvm.h"
34 #include "hw/xen.h"
35 #include "qemu-timer.h"
36 #include "memory.h"
37 #include "dma.h"
38 #include "exec-memory.h"
39 #if defined(CONFIG_USER_ONLY)
40 #include <qemu.h>
41 #else /* !CONFIG_USER_ONLY */
42 #include "xen-mapcache.h"
43 #include "trace.h"
44 #endif
45
46 #include "cputlb.h"
47 #include "translate-all.h"
48
49 #include "memory-internal.h"
50
51 //#define DEBUG_UNASSIGNED
52 //#define DEBUG_SUBPAGE
53
54 #if !defined(CONFIG_USER_ONLY)
55 int phys_ram_fd;
56 static int in_migration;
57
58 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
59
60 static MemoryRegion *system_memory;
61 static MemoryRegion *system_io;
62
63 AddressSpace address_space_io;
64 AddressSpace address_space_memory;
65 DMAContext dma_context_memory;
66
67 MemoryRegion io_mem_ram, io_mem_rom, io_mem_unassigned, io_mem_notdirty;
68 static MemoryRegion io_mem_subpage_ram;
69
70 #endif
71
72 CPUArchState *first_cpu;
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUArchState *,cpu_single_env);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount = 0;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 static MemoryRegionSection *phys_sections;
84 static unsigned phys_sections_nb, phys_sections_nb_alloc;
85 static uint16_t phys_section_unassigned;
86 static uint16_t phys_section_notdirty;
87 static uint16_t phys_section_rom;
88 static uint16_t phys_section_watch;
89
90 /* Simple allocator for PhysPageEntry nodes */
91 static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
92 static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
93
94 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
95
96 static void io_mem_init(void);
97 static void memory_map_init(void);
98 static void *qemu_safe_ram_ptr(ram_addr_t addr);
99
100 static MemoryRegion io_mem_watch;
101 #endif
102
103 #if !defined(CONFIG_USER_ONLY)
104
105 static void phys_map_node_reserve(unsigned nodes)
106 {
107 if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) {
108 typedef PhysPageEntry Node[L2_SIZE];
109 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16);
110 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc,
111 phys_map_nodes_nb + nodes);
112 phys_map_nodes = g_renew(Node, phys_map_nodes,
113 phys_map_nodes_nb_alloc);
114 }
115 }
116
117 static uint16_t phys_map_node_alloc(void)
118 {
119 unsigned i;
120 uint16_t ret;
121
122 ret = phys_map_nodes_nb++;
123 assert(ret != PHYS_MAP_NODE_NIL);
124 assert(ret != phys_map_nodes_nb_alloc);
125 for (i = 0; i < L2_SIZE; ++i) {
126 phys_map_nodes[ret][i].is_leaf = 0;
127 phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
128 }
129 return ret;
130 }
131
132 static void phys_map_nodes_reset(void)
133 {
134 phys_map_nodes_nb = 0;
135 }
136
137
138 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
139 hwaddr *nb, uint16_t leaf,
140 int level)
141 {
142 PhysPageEntry *p;
143 int i;
144 hwaddr step = (hwaddr)1 << (level * L2_BITS);
145
146 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
147 lp->ptr = phys_map_node_alloc();
148 p = phys_map_nodes[lp->ptr];
149 if (level == 0) {
150 for (i = 0; i < L2_SIZE; i++) {
151 p[i].is_leaf = 1;
152 p[i].ptr = phys_section_unassigned;
153 }
154 }
155 } else {
156 p = phys_map_nodes[lp->ptr];
157 }
158 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
159
160 while (*nb && lp < &p[L2_SIZE]) {
161 if ((*index & (step - 1)) == 0 && *nb >= step) {
162 lp->is_leaf = true;
163 lp->ptr = leaf;
164 *index += step;
165 *nb -= step;
166 } else {
167 phys_page_set_level(lp, index, nb, leaf, level - 1);
168 }
169 ++lp;
170 }
171 }
172
173 static void phys_page_set(AddressSpaceDispatch *d,
174 hwaddr index, hwaddr nb,
175 uint16_t leaf)
176 {
177 /* Wildly overreserve - it doesn't matter much. */
178 phys_map_node_reserve(3 * P_L2_LEVELS);
179
180 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
181 }
182
183 MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr index)
184 {
185 PhysPageEntry lp = d->phys_map;
186 PhysPageEntry *p;
187 int i;
188 uint16_t s_index = phys_section_unassigned;
189
190 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
191 if (lp.ptr == PHYS_MAP_NODE_NIL) {
192 goto not_found;
193 }
194 p = phys_map_nodes[lp.ptr];
195 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
196 }
197
198 s_index = lp.ptr;
199 not_found:
200 return &phys_sections[s_index];
201 }
202
203 bool memory_region_is_unassigned(MemoryRegion *mr)
204 {
205 return mr != &io_mem_ram && mr != &io_mem_rom
206 && mr != &io_mem_notdirty && !mr->rom_device
207 && mr != &io_mem_watch;
208 }
209 #endif
210
211 void cpu_exec_init_all(void)
212 {
213 #if !defined(CONFIG_USER_ONLY)
214 memory_map_init();
215 io_mem_init();
216 #endif
217 }
218
219 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
220
221 static int cpu_common_post_load(void *opaque, int version_id)
222 {
223 CPUArchState *env = opaque;
224
225 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
226 version_id is increased. */
227 env->interrupt_request &= ~0x01;
228 tlb_flush(env, 1);
229
230 return 0;
231 }
232
233 static const VMStateDescription vmstate_cpu_common = {
234 .name = "cpu_common",
235 .version_id = 1,
236 .minimum_version_id = 1,
237 .minimum_version_id_old = 1,
238 .post_load = cpu_common_post_load,
239 .fields = (VMStateField []) {
240 VMSTATE_UINT32(halted, CPUArchState),
241 VMSTATE_UINT32(interrupt_request, CPUArchState),
242 VMSTATE_END_OF_LIST()
243 }
244 };
245 #endif
246
247 CPUArchState *qemu_get_cpu(int cpu)
248 {
249 CPUArchState *env = first_cpu;
250
251 while (env) {
252 if (env->cpu_index == cpu)
253 break;
254 env = env->next_cpu;
255 }
256
257 return env;
258 }
259
260 void cpu_exec_init(CPUArchState *env)
261 {
262 #ifndef CONFIG_USER_ONLY
263 CPUState *cpu = ENV_GET_CPU(env);
264 #endif
265 CPUArchState **penv;
266 int cpu_index;
267
268 #if defined(CONFIG_USER_ONLY)
269 cpu_list_lock();
270 #endif
271 env->next_cpu = NULL;
272 penv = &first_cpu;
273 cpu_index = 0;
274 while (*penv != NULL) {
275 penv = &(*penv)->next_cpu;
276 cpu_index++;
277 }
278 env->cpu_index = cpu_index;
279 env->numa_node = 0;
280 QTAILQ_INIT(&env->breakpoints);
281 QTAILQ_INIT(&env->watchpoints);
282 #ifndef CONFIG_USER_ONLY
283 cpu->thread_id = qemu_get_thread_id();
284 #endif
285 *penv = env;
286 #if defined(CONFIG_USER_ONLY)
287 cpu_list_unlock();
288 #endif
289 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
290 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
291 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
292 cpu_save, cpu_load, env);
293 #endif
294 }
295
296 #if defined(TARGET_HAS_ICE)
297 #if defined(CONFIG_USER_ONLY)
298 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
299 {
300 tb_invalidate_phys_page_range(pc, pc + 1, 0);
301 }
302 #else
303 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
304 {
305 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
306 (pc & ~TARGET_PAGE_MASK));
307 }
308 #endif
309 #endif /* TARGET_HAS_ICE */
310
311 #if defined(CONFIG_USER_ONLY)
312 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
313
314 {
315 }
316
317 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
318 int flags, CPUWatchpoint **watchpoint)
319 {
320 return -ENOSYS;
321 }
322 #else
323 /* Add a watchpoint. */
324 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
325 int flags, CPUWatchpoint **watchpoint)
326 {
327 target_ulong len_mask = ~(len - 1);
328 CPUWatchpoint *wp;
329
330 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
331 if ((len & (len - 1)) || (addr & ~len_mask) ||
332 len == 0 || len > TARGET_PAGE_SIZE) {
333 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
334 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
335 return -EINVAL;
336 }
337 wp = g_malloc(sizeof(*wp));
338
339 wp->vaddr = addr;
340 wp->len_mask = len_mask;
341 wp->flags = flags;
342
343 /* keep all GDB-injected watchpoints in front */
344 if (flags & BP_GDB)
345 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
346 else
347 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
348
349 tlb_flush_page(env, addr);
350
351 if (watchpoint)
352 *watchpoint = wp;
353 return 0;
354 }
355
356 /* Remove a specific watchpoint. */
357 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
358 int flags)
359 {
360 target_ulong len_mask = ~(len - 1);
361 CPUWatchpoint *wp;
362
363 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
364 if (addr == wp->vaddr && len_mask == wp->len_mask
365 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
366 cpu_watchpoint_remove_by_ref(env, wp);
367 return 0;
368 }
369 }
370 return -ENOENT;
371 }
372
373 /* Remove a specific watchpoint by reference. */
374 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
375 {
376 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
377
378 tlb_flush_page(env, watchpoint->vaddr);
379
380 g_free(watchpoint);
381 }
382
383 /* Remove all matching watchpoints. */
384 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
385 {
386 CPUWatchpoint *wp, *next;
387
388 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
389 if (wp->flags & mask)
390 cpu_watchpoint_remove_by_ref(env, wp);
391 }
392 }
393 #endif
394
395 /* Add a breakpoint. */
396 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
397 CPUBreakpoint **breakpoint)
398 {
399 #if defined(TARGET_HAS_ICE)
400 CPUBreakpoint *bp;
401
402 bp = g_malloc(sizeof(*bp));
403
404 bp->pc = pc;
405 bp->flags = flags;
406
407 /* keep all GDB-injected breakpoints in front */
408 if (flags & BP_GDB)
409 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
410 else
411 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
412
413 breakpoint_invalidate(env, pc);
414
415 if (breakpoint)
416 *breakpoint = bp;
417 return 0;
418 #else
419 return -ENOSYS;
420 #endif
421 }
422
423 /* Remove a specific breakpoint. */
424 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
425 {
426 #if defined(TARGET_HAS_ICE)
427 CPUBreakpoint *bp;
428
429 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
430 if (bp->pc == pc && bp->flags == flags) {
431 cpu_breakpoint_remove_by_ref(env, bp);
432 return 0;
433 }
434 }
435 return -ENOENT;
436 #else
437 return -ENOSYS;
438 #endif
439 }
440
441 /* Remove a specific breakpoint by reference. */
442 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
443 {
444 #if defined(TARGET_HAS_ICE)
445 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
446
447 breakpoint_invalidate(env, breakpoint->pc);
448
449 g_free(breakpoint);
450 #endif
451 }
452
453 /* Remove all matching breakpoints. */
454 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
455 {
456 #if defined(TARGET_HAS_ICE)
457 CPUBreakpoint *bp, *next;
458
459 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
460 if (bp->flags & mask)
461 cpu_breakpoint_remove_by_ref(env, bp);
462 }
463 #endif
464 }
465
466 /* enable or disable single step mode. EXCP_DEBUG is returned by the
467 CPU loop after each instruction */
468 void cpu_single_step(CPUArchState *env, int enabled)
469 {
470 #if defined(TARGET_HAS_ICE)
471 if (env->singlestep_enabled != enabled) {
472 env->singlestep_enabled = enabled;
473 if (kvm_enabled())
474 kvm_update_guest_debug(env, 0);
475 else {
476 /* must flush all the translated code to avoid inconsistencies */
477 /* XXX: only flush what is necessary */
478 tb_flush(env);
479 }
480 }
481 #endif
482 }
483
484 void cpu_reset_interrupt(CPUArchState *env, int mask)
485 {
486 env->interrupt_request &= ~mask;
487 }
488
489 void cpu_exit(CPUArchState *env)
490 {
491 env->exit_request = 1;
492 cpu_unlink_tb(env);
493 }
494
495 void cpu_abort(CPUArchState *env, const char *fmt, ...)
496 {
497 va_list ap;
498 va_list ap2;
499
500 va_start(ap, fmt);
501 va_copy(ap2, ap);
502 fprintf(stderr, "qemu: fatal: ");
503 vfprintf(stderr, fmt, ap);
504 fprintf(stderr, "\n");
505 cpu_dump_state(env, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
506 if (qemu_log_enabled()) {
507 qemu_log("qemu: fatal: ");
508 qemu_log_vprintf(fmt, ap2);
509 qemu_log("\n");
510 log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
511 qemu_log_flush();
512 qemu_log_close();
513 }
514 va_end(ap2);
515 va_end(ap);
516 #if defined(CONFIG_USER_ONLY)
517 {
518 struct sigaction act;
519 sigfillset(&act.sa_mask);
520 act.sa_handler = SIG_DFL;
521 sigaction(SIGABRT, &act, NULL);
522 }
523 #endif
524 abort();
525 }
526
527 CPUArchState *cpu_copy(CPUArchState *env)
528 {
529 CPUArchState *new_env = cpu_init(env->cpu_model_str);
530 CPUArchState *next_cpu = new_env->next_cpu;
531 int cpu_index = new_env->cpu_index;
532 #if defined(TARGET_HAS_ICE)
533 CPUBreakpoint *bp;
534 CPUWatchpoint *wp;
535 #endif
536
537 memcpy(new_env, env, sizeof(CPUArchState));
538
539 /* Preserve chaining and index. */
540 new_env->next_cpu = next_cpu;
541 new_env->cpu_index = cpu_index;
542
543 /* Clone all break/watchpoints.
544 Note: Once we support ptrace with hw-debug register access, make sure
545 BP_CPU break/watchpoints are handled correctly on clone. */
546 QTAILQ_INIT(&env->breakpoints);
547 QTAILQ_INIT(&env->watchpoints);
548 #if defined(TARGET_HAS_ICE)
549 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
550 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
551 }
552 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
553 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
554 wp->flags, NULL);
555 }
556 #endif
557
558 return new_env;
559 }
560
561 #if !defined(CONFIG_USER_ONLY)
562 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
563 uintptr_t length)
564 {
565 uintptr_t start1;
566
567 /* we modify the TLB cache so that the dirty bit will be set again
568 when accessing the range */
569 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
570 /* Check that we don't span multiple blocks - this breaks the
571 address comparisons below. */
572 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
573 != (end - 1) - start) {
574 abort();
575 }
576 cpu_tlb_reset_dirty_all(start1, length);
577
578 }
579
580 /* Note: start and end must be within the same ram block. */
581 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
582 int dirty_flags)
583 {
584 uintptr_t length;
585
586 start &= TARGET_PAGE_MASK;
587 end = TARGET_PAGE_ALIGN(end);
588
589 length = end - start;
590 if (length == 0)
591 return;
592 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
593
594 if (tcg_enabled()) {
595 tlb_reset_dirty_range_all(start, end, length);
596 }
597 }
598
599 static int cpu_physical_memory_set_dirty_tracking(int enable)
600 {
601 int ret = 0;
602 in_migration = enable;
603 return ret;
604 }
605
606 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
607 MemoryRegionSection *section,
608 target_ulong vaddr,
609 hwaddr paddr,
610 int prot,
611 target_ulong *address)
612 {
613 hwaddr iotlb;
614 CPUWatchpoint *wp;
615
616 if (memory_region_is_ram(section->mr)) {
617 /* Normal RAM. */
618 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
619 + memory_region_section_addr(section, paddr);
620 if (!section->readonly) {
621 iotlb |= phys_section_notdirty;
622 } else {
623 iotlb |= phys_section_rom;
624 }
625 } else {
626 /* IO handlers are currently passed a physical address.
627 It would be nice to pass an offset from the base address
628 of that region. This would avoid having to special case RAM,
629 and avoid full address decoding in every device.
630 We can't use the high bits of pd for this because
631 IO_MEM_ROMD uses these as a ram address. */
632 iotlb = section - phys_sections;
633 iotlb += memory_region_section_addr(section, paddr);
634 }
635
636 /* Make accesses to pages with watchpoints go via the
637 watchpoint trap routines. */
638 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
639 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
640 /* Avoid trapping reads of pages with a write breakpoint. */
641 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
642 iotlb = phys_section_watch + paddr;
643 *address |= TLB_MMIO;
644 break;
645 }
646 }
647 }
648
649 return iotlb;
650 }
651 #endif /* defined(CONFIG_USER_ONLY) */
652
653 #if !defined(CONFIG_USER_ONLY)
654
655 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
656 typedef struct subpage_t {
657 MemoryRegion iomem;
658 hwaddr base;
659 uint16_t sub_section[TARGET_PAGE_SIZE];
660 } subpage_t;
661
662 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
663 uint16_t section);
664 static subpage_t *subpage_init(hwaddr base);
665 static void destroy_page_desc(uint16_t section_index)
666 {
667 MemoryRegionSection *section = &phys_sections[section_index];
668 MemoryRegion *mr = section->mr;
669
670 if (mr->subpage) {
671 subpage_t *subpage = container_of(mr, subpage_t, iomem);
672 memory_region_destroy(&subpage->iomem);
673 g_free(subpage);
674 }
675 }
676
677 static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)
678 {
679 unsigned i;
680 PhysPageEntry *p;
681
682 if (lp->ptr == PHYS_MAP_NODE_NIL) {
683 return;
684 }
685
686 p = phys_map_nodes[lp->ptr];
687 for (i = 0; i < L2_SIZE; ++i) {
688 if (!p[i].is_leaf) {
689 destroy_l2_mapping(&p[i], level - 1);
690 } else {
691 destroy_page_desc(p[i].ptr);
692 }
693 }
694 lp->is_leaf = 0;
695 lp->ptr = PHYS_MAP_NODE_NIL;
696 }
697
698 static void destroy_all_mappings(AddressSpaceDispatch *d)
699 {
700 destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
701 phys_map_nodes_reset();
702 }
703
704 static uint16_t phys_section_add(MemoryRegionSection *section)
705 {
706 if (phys_sections_nb == phys_sections_nb_alloc) {
707 phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16);
708 phys_sections = g_renew(MemoryRegionSection, phys_sections,
709 phys_sections_nb_alloc);
710 }
711 phys_sections[phys_sections_nb] = *section;
712 return phys_sections_nb++;
713 }
714
715 static void phys_sections_clear(void)
716 {
717 phys_sections_nb = 0;
718 }
719
720 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
721 {
722 subpage_t *subpage;
723 hwaddr base = section->offset_within_address_space
724 & TARGET_PAGE_MASK;
725 MemoryRegionSection *existing = phys_page_find(d, base >> TARGET_PAGE_BITS);
726 MemoryRegionSection subsection = {
727 .offset_within_address_space = base,
728 .size = TARGET_PAGE_SIZE,
729 };
730 hwaddr start, end;
731
732 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
733
734 if (!(existing->mr->subpage)) {
735 subpage = subpage_init(base);
736 subsection.mr = &subpage->iomem;
737 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
738 phys_section_add(&subsection));
739 } else {
740 subpage = container_of(existing->mr, subpage_t, iomem);
741 }
742 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
743 end = start + section->size - 1;
744 subpage_register(subpage, start, end, phys_section_add(section));
745 }
746
747
748 static void register_multipage(AddressSpaceDispatch *d, MemoryRegionSection *section)
749 {
750 hwaddr start_addr = section->offset_within_address_space;
751 ram_addr_t size = section->size;
752 hwaddr addr;
753 uint16_t section_index = phys_section_add(section);
754
755 assert(size);
756
757 addr = start_addr;
758 phys_page_set(d, addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,
759 section_index);
760 }
761
762 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
763 {
764 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
765 MemoryRegionSection now = *section, remain = *section;
766
767 if ((now.offset_within_address_space & ~TARGET_PAGE_MASK)
768 || (now.size < TARGET_PAGE_SIZE)) {
769 now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space)
770 - now.offset_within_address_space,
771 now.size);
772 register_subpage(d, &now);
773 remain.size -= now.size;
774 remain.offset_within_address_space += now.size;
775 remain.offset_within_region += now.size;
776 }
777 while (remain.size >= TARGET_PAGE_SIZE) {
778 now = remain;
779 if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
780 now.size = TARGET_PAGE_SIZE;
781 register_subpage(d, &now);
782 } else {
783 now.size &= TARGET_PAGE_MASK;
784 register_multipage(d, &now);
785 }
786 remain.size -= now.size;
787 remain.offset_within_address_space += now.size;
788 remain.offset_within_region += now.size;
789 }
790 now = remain;
791 if (now.size) {
792 register_subpage(d, &now);
793 }
794 }
795
796 void qemu_flush_coalesced_mmio_buffer(void)
797 {
798 if (kvm_enabled())
799 kvm_flush_coalesced_mmio_buffer();
800 }
801
802 #if defined(__linux__) && !defined(TARGET_S390X)
803
804 #include <sys/vfs.h>
805
806 #define HUGETLBFS_MAGIC 0x958458f6
807
808 static long gethugepagesize(const char *path)
809 {
810 struct statfs fs;
811 int ret;
812
813 do {
814 ret = statfs(path, &fs);
815 } while (ret != 0 && errno == EINTR);
816
817 if (ret != 0) {
818 perror(path);
819 return 0;
820 }
821
822 if (fs.f_type != HUGETLBFS_MAGIC)
823 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
824
825 return fs.f_bsize;
826 }
827
828 static void *file_ram_alloc(RAMBlock *block,
829 ram_addr_t memory,
830 const char *path)
831 {
832 char *filename;
833 void *area;
834 int fd;
835 #ifdef MAP_POPULATE
836 int flags;
837 #endif
838 unsigned long hpagesize;
839
840 hpagesize = gethugepagesize(path);
841 if (!hpagesize) {
842 return NULL;
843 }
844
845 if (memory < hpagesize) {
846 return NULL;
847 }
848
849 if (kvm_enabled() && !kvm_has_sync_mmu()) {
850 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
851 return NULL;
852 }
853
854 if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
855 return NULL;
856 }
857
858 fd = mkstemp(filename);
859 if (fd < 0) {
860 perror("unable to create backing store for hugepages");
861 free(filename);
862 return NULL;
863 }
864 unlink(filename);
865 free(filename);
866
867 memory = (memory+hpagesize-1) & ~(hpagesize-1);
868
869 /*
870 * ftruncate is not supported by hugetlbfs in older
871 * hosts, so don't bother bailing out on errors.
872 * If anything goes wrong with it under other filesystems,
873 * mmap will fail.
874 */
875 if (ftruncate(fd, memory))
876 perror("ftruncate");
877
878 #ifdef MAP_POPULATE
879 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
880 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
881 * to sidestep this quirk.
882 */
883 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
884 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
885 #else
886 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
887 #endif
888 if (area == MAP_FAILED) {
889 perror("file_ram_alloc: can't mmap RAM pages");
890 close(fd);
891 return (NULL);
892 }
893 block->fd = fd;
894 return area;
895 }
896 #endif
897
898 static ram_addr_t find_ram_offset(ram_addr_t size)
899 {
900 RAMBlock *block, *next_block;
901 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
902
903 if (QLIST_EMPTY(&ram_list.blocks))
904 return 0;
905
906 QLIST_FOREACH(block, &ram_list.blocks, next) {
907 ram_addr_t end, next = RAM_ADDR_MAX;
908
909 end = block->offset + block->length;
910
911 QLIST_FOREACH(next_block, &ram_list.blocks, next) {
912 if (next_block->offset >= end) {
913 next = MIN(next, next_block->offset);
914 }
915 }
916 if (next - end >= size && next - end < mingap) {
917 offset = end;
918 mingap = next - end;
919 }
920 }
921
922 if (offset == RAM_ADDR_MAX) {
923 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
924 (uint64_t)size);
925 abort();
926 }
927
928 return offset;
929 }
930
931 ram_addr_t last_ram_offset(void)
932 {
933 RAMBlock *block;
934 ram_addr_t last = 0;
935
936 QLIST_FOREACH(block, &ram_list.blocks, next)
937 last = MAX(last, block->offset + block->length);
938
939 return last;
940 }
941
942 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
943 {
944 int ret;
945 QemuOpts *machine_opts;
946
947 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
948 machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
949 if (machine_opts &&
950 !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
951 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
952 if (ret) {
953 perror("qemu_madvise");
954 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
955 "but dump_guest_core=off specified\n");
956 }
957 }
958 }
959
960 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
961 {
962 RAMBlock *new_block, *block;
963
964 new_block = NULL;
965 QLIST_FOREACH(block, &ram_list.blocks, next) {
966 if (block->offset == addr) {
967 new_block = block;
968 break;
969 }
970 }
971 assert(new_block);
972 assert(!new_block->idstr[0]);
973
974 if (dev) {
975 char *id = qdev_get_dev_path(dev);
976 if (id) {
977 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
978 g_free(id);
979 }
980 }
981 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
982
983 QLIST_FOREACH(block, &ram_list.blocks, next) {
984 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
985 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
986 new_block->idstr);
987 abort();
988 }
989 }
990 }
991
992 static int memory_try_enable_merging(void *addr, size_t len)
993 {
994 QemuOpts *opts;
995
996 opts = qemu_opts_find(qemu_find_opts("machine"), 0);
997 if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
998 /* disabled by the user */
999 return 0;
1000 }
1001
1002 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1003 }
1004
1005 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1006 MemoryRegion *mr)
1007 {
1008 RAMBlock *new_block;
1009
1010 size = TARGET_PAGE_ALIGN(size);
1011 new_block = g_malloc0(sizeof(*new_block));
1012
1013 new_block->mr = mr;
1014 new_block->offset = find_ram_offset(size);
1015 if (host) {
1016 new_block->host = host;
1017 new_block->flags |= RAM_PREALLOC_MASK;
1018 } else {
1019 if (mem_path) {
1020 #if defined (__linux__) && !defined(TARGET_S390X)
1021 new_block->host = file_ram_alloc(new_block, size, mem_path);
1022 if (!new_block->host) {
1023 new_block->host = qemu_vmalloc(size);
1024 memory_try_enable_merging(new_block->host, size);
1025 }
1026 #else
1027 fprintf(stderr, "-mem-path option unsupported\n");
1028 exit(1);
1029 #endif
1030 } else {
1031 if (xen_enabled()) {
1032 xen_ram_alloc(new_block->offset, size, mr);
1033 } else if (kvm_enabled()) {
1034 /* some s390/kvm configurations have special constraints */
1035 new_block->host = kvm_vmalloc(size);
1036 } else {
1037 new_block->host = qemu_vmalloc(size);
1038 }
1039 memory_try_enable_merging(new_block->host, size);
1040 }
1041 }
1042 new_block->length = size;
1043
1044 QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
1045
1046 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1047 last_ram_offset() >> TARGET_PAGE_BITS);
1048 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1049 0, size >> TARGET_PAGE_BITS);
1050 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1051
1052 qemu_ram_setup_dump(new_block->host, size);
1053 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1054
1055 if (kvm_enabled())
1056 kvm_setup_guest_memory(new_block->host, size);
1057
1058 return new_block->offset;
1059 }
1060
1061 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1062 {
1063 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1064 }
1065
1066 void qemu_ram_free_from_ptr(ram_addr_t addr)
1067 {
1068 RAMBlock *block;
1069
1070 QLIST_FOREACH(block, &ram_list.blocks, next) {
1071 if (addr == block->offset) {
1072 QLIST_REMOVE(block, next);
1073 g_free(block);
1074 return;
1075 }
1076 }
1077 }
1078
1079 void qemu_ram_free(ram_addr_t addr)
1080 {
1081 RAMBlock *block;
1082
1083 QLIST_FOREACH(block, &ram_list.blocks, next) {
1084 if (addr == block->offset) {
1085 QLIST_REMOVE(block, next);
1086 if (block->flags & RAM_PREALLOC_MASK) {
1087 ;
1088 } else if (mem_path) {
1089 #if defined (__linux__) && !defined(TARGET_S390X)
1090 if (block->fd) {
1091 munmap(block->host, block->length);
1092 close(block->fd);
1093 } else {
1094 qemu_vfree(block->host);
1095 }
1096 #else
1097 abort();
1098 #endif
1099 } else {
1100 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1101 munmap(block->host, block->length);
1102 #else
1103 if (xen_enabled()) {
1104 xen_invalidate_map_cache_entry(block->host);
1105 } else {
1106 qemu_vfree(block->host);
1107 }
1108 #endif
1109 }
1110 g_free(block);
1111 return;
1112 }
1113 }
1114
1115 }
1116
1117 #ifndef _WIN32
1118 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1119 {
1120 RAMBlock *block;
1121 ram_addr_t offset;
1122 int flags;
1123 void *area, *vaddr;
1124
1125 QLIST_FOREACH(block, &ram_list.blocks, next) {
1126 offset = addr - block->offset;
1127 if (offset < block->length) {
1128 vaddr = block->host + offset;
1129 if (block->flags & RAM_PREALLOC_MASK) {
1130 ;
1131 } else {
1132 flags = MAP_FIXED;
1133 munmap(vaddr, length);
1134 if (mem_path) {
1135 #if defined(__linux__) && !defined(TARGET_S390X)
1136 if (block->fd) {
1137 #ifdef MAP_POPULATE
1138 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1139 MAP_PRIVATE;
1140 #else
1141 flags |= MAP_PRIVATE;
1142 #endif
1143 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1144 flags, block->fd, offset);
1145 } else {
1146 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1147 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1148 flags, -1, 0);
1149 }
1150 #else
1151 abort();
1152 #endif
1153 } else {
1154 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1155 flags |= MAP_SHARED | MAP_ANONYMOUS;
1156 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1157 flags, -1, 0);
1158 #else
1159 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1160 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1161 flags, -1, 0);
1162 #endif
1163 }
1164 if (area != vaddr) {
1165 fprintf(stderr, "Could not remap addr: "
1166 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1167 length, addr);
1168 exit(1);
1169 }
1170 memory_try_enable_merging(vaddr, length);
1171 qemu_ram_setup_dump(vaddr, length);
1172 }
1173 return;
1174 }
1175 }
1176 }
1177 #endif /* !_WIN32 */
1178
1179 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1180 With the exception of the softmmu code in this file, this should
1181 only be used for local memory (e.g. video ram) that the device owns,
1182 and knows it isn't going to access beyond the end of the block.
1183
1184 It should not be used for general purpose DMA.
1185 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1186 */
1187 void *qemu_get_ram_ptr(ram_addr_t addr)
1188 {
1189 RAMBlock *block;
1190
1191 QLIST_FOREACH(block, &ram_list.blocks, next) {
1192 if (addr - block->offset < block->length) {
1193 /* Move this entry to to start of the list. */
1194 if (block != QLIST_FIRST(&ram_list.blocks)) {
1195 QLIST_REMOVE(block, next);
1196 QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
1197 }
1198 if (xen_enabled()) {
1199 /* We need to check if the requested address is in the RAM
1200 * because we don't want to map the entire memory in QEMU.
1201 * In that case just map until the end of the page.
1202 */
1203 if (block->offset == 0) {
1204 return xen_map_cache(addr, 0, 0);
1205 } else if (block->host == NULL) {
1206 block->host =
1207 xen_map_cache(block->offset, block->length, 1);
1208 }
1209 }
1210 return block->host + (addr - block->offset);
1211 }
1212 }
1213
1214 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1215 abort();
1216
1217 return NULL;
1218 }
1219
1220 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1221 * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
1222 */
1223 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1224 {
1225 RAMBlock *block;
1226
1227 QLIST_FOREACH(block, &ram_list.blocks, next) {
1228 if (addr - block->offset < block->length) {
1229 if (xen_enabled()) {
1230 /* We need to check if the requested address is in the RAM
1231 * because we don't want to map the entire memory in QEMU.
1232 * In that case just map until the end of the page.
1233 */
1234 if (block->offset == 0) {
1235 return xen_map_cache(addr, 0, 0);
1236 } else if (block->host == NULL) {
1237 block->host =
1238 xen_map_cache(block->offset, block->length, 1);
1239 }
1240 }
1241 return block->host + (addr - block->offset);
1242 }
1243 }
1244
1245 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1246 abort();
1247
1248 return NULL;
1249 }
1250
1251 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1252 * but takes a size argument */
1253 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1254 {
1255 if (*size == 0) {
1256 return NULL;
1257 }
1258 if (xen_enabled()) {
1259 return xen_map_cache(addr, *size, 1);
1260 } else {
1261 RAMBlock *block;
1262
1263 QLIST_FOREACH(block, &ram_list.blocks, next) {
1264 if (addr - block->offset < block->length) {
1265 if (addr - block->offset + *size > block->length)
1266 *size = block->length - addr + block->offset;
1267 return block->host + (addr - block->offset);
1268 }
1269 }
1270
1271 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1272 abort();
1273 }
1274 }
1275
1276 void qemu_put_ram_ptr(void *addr)
1277 {
1278 trace_qemu_put_ram_ptr(addr);
1279 }
1280
1281 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1282 {
1283 RAMBlock *block;
1284 uint8_t *host = ptr;
1285
1286 if (xen_enabled()) {
1287 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1288 return 0;
1289 }
1290
1291 QLIST_FOREACH(block, &ram_list.blocks, next) {
1292 /* This case append when the block is not mapped. */
1293 if (block->host == NULL) {
1294 continue;
1295 }
1296 if (host - block->host < block->length) {
1297 *ram_addr = block->offset + (host - block->host);
1298 return 0;
1299 }
1300 }
1301
1302 return -1;
1303 }
1304
1305 /* Some of the softmmu routines need to translate from a host pointer
1306 (typically a TLB entry) back to a ram offset. */
1307 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
1308 {
1309 ram_addr_t ram_addr;
1310
1311 if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
1312 fprintf(stderr, "Bad ram pointer %p\n", ptr);
1313 abort();
1314 }
1315 return ram_addr;
1316 }
1317
1318 static uint64_t unassigned_mem_read(void *opaque, hwaddr addr,
1319 unsigned size)
1320 {
1321 #ifdef DEBUG_UNASSIGNED
1322 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
1323 #endif
1324 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1325 cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, size);
1326 #endif
1327 return 0;
1328 }
1329
1330 static void unassigned_mem_write(void *opaque, hwaddr addr,
1331 uint64_t val, unsigned size)
1332 {
1333 #ifdef DEBUG_UNASSIGNED
1334 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%"PRIx64"\n", addr, val);
1335 #endif
1336 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1337 cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, size);
1338 #endif
1339 }
1340
1341 static const MemoryRegionOps unassigned_mem_ops = {
1342 .read = unassigned_mem_read,
1343 .write = unassigned_mem_write,
1344 .endianness = DEVICE_NATIVE_ENDIAN,
1345 };
1346
1347 static uint64_t error_mem_read(void *opaque, hwaddr addr,
1348 unsigned size)
1349 {
1350 abort();
1351 }
1352
1353 static void error_mem_write(void *opaque, hwaddr addr,
1354 uint64_t value, unsigned size)
1355 {
1356 abort();
1357 }
1358
1359 static const MemoryRegionOps error_mem_ops = {
1360 .read = error_mem_read,
1361 .write = error_mem_write,
1362 .endianness = DEVICE_NATIVE_ENDIAN,
1363 };
1364
1365 static const MemoryRegionOps rom_mem_ops = {
1366 .read = error_mem_read,
1367 .write = unassigned_mem_write,
1368 .endianness = DEVICE_NATIVE_ENDIAN,
1369 };
1370
1371 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1372 uint64_t val, unsigned size)
1373 {
1374 int dirty_flags;
1375 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1376 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1377 #if !defined(CONFIG_USER_ONLY)
1378 tb_invalidate_phys_page_fast(ram_addr, size);
1379 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1380 #endif
1381 }
1382 switch (size) {
1383 case 1:
1384 stb_p(qemu_get_ram_ptr(ram_addr), val);
1385 break;
1386 case 2:
1387 stw_p(qemu_get_ram_ptr(ram_addr), val);
1388 break;
1389 case 4:
1390 stl_p(qemu_get_ram_ptr(ram_addr), val);
1391 break;
1392 default:
1393 abort();
1394 }
1395 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1396 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1397 /* we remove the notdirty callback only if the code has been
1398 flushed */
1399 if (dirty_flags == 0xff)
1400 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1401 }
1402
1403 static const MemoryRegionOps notdirty_mem_ops = {
1404 .read = error_mem_read,
1405 .write = notdirty_mem_write,
1406 .endianness = DEVICE_NATIVE_ENDIAN,
1407 };
1408
1409 /* Generate a debug exception if a watchpoint has been hit. */
1410 static void check_watchpoint(int offset, int len_mask, int flags)
1411 {
1412 CPUArchState *env = cpu_single_env;
1413 target_ulong pc, cs_base;
1414 target_ulong vaddr;
1415 CPUWatchpoint *wp;
1416 int cpu_flags;
1417
1418 if (env->watchpoint_hit) {
1419 /* We re-entered the check after replacing the TB. Now raise
1420 * the debug interrupt so that is will trigger after the
1421 * current instruction. */
1422 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
1423 return;
1424 }
1425 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1426 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1427 if ((vaddr == (wp->vaddr & len_mask) ||
1428 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1429 wp->flags |= BP_WATCHPOINT_HIT;
1430 if (!env->watchpoint_hit) {
1431 env->watchpoint_hit = wp;
1432 tb_check_watchpoint(env);
1433 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1434 env->exception_index = EXCP_DEBUG;
1435 cpu_loop_exit(env);
1436 } else {
1437 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1438 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1439 cpu_resume_from_signal(env, NULL);
1440 }
1441 }
1442 } else {
1443 wp->flags &= ~BP_WATCHPOINT_HIT;
1444 }
1445 }
1446 }
1447
1448 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1449 so these check for a hit then pass through to the normal out-of-line
1450 phys routines. */
1451 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1452 unsigned size)
1453 {
1454 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1455 switch (size) {
1456 case 1: return ldub_phys(addr);
1457 case 2: return lduw_phys(addr);
1458 case 4: return ldl_phys(addr);
1459 default: abort();
1460 }
1461 }
1462
1463 static void watch_mem_write(void *opaque, hwaddr addr,
1464 uint64_t val, unsigned size)
1465 {
1466 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1467 switch (size) {
1468 case 1:
1469 stb_phys(addr, val);
1470 break;
1471 case 2:
1472 stw_phys(addr, val);
1473 break;
1474 case 4:
1475 stl_phys(addr, val);
1476 break;
1477 default: abort();
1478 }
1479 }
1480
1481 static const MemoryRegionOps watch_mem_ops = {
1482 .read = watch_mem_read,
1483 .write = watch_mem_write,
1484 .endianness = DEVICE_NATIVE_ENDIAN,
1485 };
1486
1487 static uint64_t subpage_read(void *opaque, hwaddr addr,
1488 unsigned len)
1489 {
1490 subpage_t *mmio = opaque;
1491 unsigned int idx = SUBPAGE_IDX(addr);
1492 MemoryRegionSection *section;
1493 #if defined(DEBUG_SUBPAGE)
1494 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
1495 mmio, len, addr, idx);
1496 #endif
1497
1498 section = &phys_sections[mmio->sub_section[idx]];
1499 addr += mmio->base;
1500 addr -= section->offset_within_address_space;
1501 addr += section->offset_within_region;
1502 return io_mem_read(section->mr, addr, len);
1503 }
1504
1505 static void subpage_write(void *opaque, hwaddr addr,
1506 uint64_t value, unsigned len)
1507 {
1508 subpage_t *mmio = opaque;
1509 unsigned int idx = SUBPAGE_IDX(addr);
1510 MemoryRegionSection *section;
1511 #if defined(DEBUG_SUBPAGE)
1512 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1513 " idx %d value %"PRIx64"\n",
1514 __func__, mmio, len, addr, idx, value);
1515 #endif
1516
1517 section = &phys_sections[mmio->sub_section[idx]];
1518 addr += mmio->base;
1519 addr -= section->offset_within_address_space;
1520 addr += section->offset_within_region;
1521 io_mem_write(section->mr, addr, value, len);
1522 }
1523
1524 static const MemoryRegionOps subpage_ops = {
1525 .read = subpage_read,
1526 .write = subpage_write,
1527 .endianness = DEVICE_NATIVE_ENDIAN,
1528 };
1529
1530 static uint64_t subpage_ram_read(void *opaque, hwaddr addr,
1531 unsigned size)
1532 {
1533 ram_addr_t raddr = addr;
1534 void *ptr = qemu_get_ram_ptr(raddr);
1535 switch (size) {
1536 case 1: return ldub_p(ptr);
1537 case 2: return lduw_p(ptr);
1538 case 4: return ldl_p(ptr);
1539 default: abort();
1540 }
1541 }
1542
1543 static void subpage_ram_write(void *opaque, hwaddr addr,
1544 uint64_t value, unsigned size)
1545 {
1546 ram_addr_t raddr = addr;
1547 void *ptr = qemu_get_ram_ptr(raddr);
1548 switch (size) {
1549 case 1: return stb_p(ptr, value);
1550 case 2: return stw_p(ptr, value);
1551 case 4: return stl_p(ptr, value);
1552 default: abort();
1553 }
1554 }
1555
1556 static const MemoryRegionOps subpage_ram_ops = {
1557 .read = subpage_ram_read,
1558 .write = subpage_ram_write,
1559 .endianness = DEVICE_NATIVE_ENDIAN,
1560 };
1561
1562 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1563 uint16_t section)
1564 {
1565 int idx, eidx;
1566
1567 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1568 return -1;
1569 idx = SUBPAGE_IDX(start);
1570 eidx = SUBPAGE_IDX(end);
1571 #if defined(DEBUG_SUBPAGE)
1572 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1573 mmio, start, end, idx, eidx, memory);
1574 #endif
1575 if (memory_region_is_ram(phys_sections[section].mr)) {
1576 MemoryRegionSection new_section = phys_sections[section];
1577 new_section.mr = &io_mem_subpage_ram;
1578 section = phys_section_add(&new_section);
1579 }
1580 for (; idx <= eidx; idx++) {
1581 mmio->sub_section[idx] = section;
1582 }
1583
1584 return 0;
1585 }
1586
1587 static subpage_t *subpage_init(hwaddr base)
1588 {
1589 subpage_t *mmio;
1590
1591 mmio = g_malloc0(sizeof(subpage_t));
1592
1593 mmio->base = base;
1594 memory_region_init_io(&mmio->iomem, &subpage_ops, mmio,
1595 "subpage", TARGET_PAGE_SIZE);
1596 mmio->iomem.subpage = true;
1597 #if defined(DEBUG_SUBPAGE)
1598 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1599 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1600 #endif
1601 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, phys_section_unassigned);
1602
1603 return mmio;
1604 }
1605
1606 static uint16_t dummy_section(MemoryRegion *mr)
1607 {
1608 MemoryRegionSection section = {
1609 .mr = mr,
1610 .offset_within_address_space = 0,
1611 .offset_within_region = 0,
1612 .size = UINT64_MAX,
1613 };
1614
1615 return phys_section_add(&section);
1616 }
1617
1618 MemoryRegion *iotlb_to_region(hwaddr index)
1619 {
1620 return phys_sections[index & ~TARGET_PAGE_MASK].mr;
1621 }
1622
1623 static void io_mem_init(void)
1624 {
1625 memory_region_init_io(&io_mem_ram, &error_mem_ops, NULL, "ram", UINT64_MAX);
1626 memory_region_init_io(&io_mem_rom, &rom_mem_ops, NULL, "rom", UINT64_MAX);
1627 memory_region_init_io(&io_mem_unassigned, &unassigned_mem_ops, NULL,
1628 "unassigned", UINT64_MAX);
1629 memory_region_init_io(&io_mem_notdirty, &notdirty_mem_ops, NULL,
1630 "notdirty", UINT64_MAX);
1631 memory_region_init_io(&io_mem_subpage_ram, &subpage_ram_ops, NULL,
1632 "subpage-ram", UINT64_MAX);
1633 memory_region_init_io(&io_mem_watch, &watch_mem_ops, NULL,
1634 "watch", UINT64_MAX);
1635 }
1636
1637 static void mem_begin(MemoryListener *listener)
1638 {
1639 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
1640
1641 destroy_all_mappings(d);
1642 d->phys_map.ptr = PHYS_MAP_NODE_NIL;
1643 }
1644
1645 static void core_begin(MemoryListener *listener)
1646 {
1647 phys_sections_clear();
1648 phys_section_unassigned = dummy_section(&io_mem_unassigned);
1649 phys_section_notdirty = dummy_section(&io_mem_notdirty);
1650 phys_section_rom = dummy_section(&io_mem_rom);
1651 phys_section_watch = dummy_section(&io_mem_watch);
1652 }
1653
1654 static void tcg_commit(MemoryListener *listener)
1655 {
1656 CPUArchState *env;
1657
1658 /* since each CPU stores ram addresses in its TLB cache, we must
1659 reset the modified entries */
1660 /* XXX: slow ! */
1661 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1662 tlb_flush(env, 1);
1663 }
1664 }
1665
1666 static void core_log_global_start(MemoryListener *listener)
1667 {
1668 cpu_physical_memory_set_dirty_tracking(1);
1669 }
1670
1671 static void core_log_global_stop(MemoryListener *listener)
1672 {
1673 cpu_physical_memory_set_dirty_tracking(0);
1674 }
1675
1676 static void io_region_add(MemoryListener *listener,
1677 MemoryRegionSection *section)
1678 {
1679 MemoryRegionIORange *mrio = g_new(MemoryRegionIORange, 1);
1680
1681 mrio->mr = section->mr;
1682 mrio->offset = section->offset_within_region;
1683 iorange_init(&mrio->iorange, &memory_region_iorange_ops,
1684 section->offset_within_address_space, section->size);
1685 ioport_register(&mrio->iorange);
1686 }
1687
1688 static void io_region_del(MemoryListener *listener,
1689 MemoryRegionSection *section)
1690 {
1691 isa_unassign_ioport(section->offset_within_address_space, section->size);
1692 }
1693
1694 static MemoryListener core_memory_listener = {
1695 .begin = core_begin,
1696 .log_global_start = core_log_global_start,
1697 .log_global_stop = core_log_global_stop,
1698 .priority = 1,
1699 };
1700
1701 static MemoryListener io_memory_listener = {
1702 .region_add = io_region_add,
1703 .region_del = io_region_del,
1704 .priority = 0,
1705 };
1706
1707 static MemoryListener tcg_memory_listener = {
1708 .commit = tcg_commit,
1709 };
1710
1711 void address_space_init_dispatch(AddressSpace *as)
1712 {
1713 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1714
1715 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1716 d->listener = (MemoryListener) {
1717 .begin = mem_begin,
1718 .region_add = mem_add,
1719 .region_nop = mem_add,
1720 .priority = 0,
1721 };
1722 as->dispatch = d;
1723 memory_listener_register(&d->listener, as);
1724 }
1725
1726 void address_space_destroy_dispatch(AddressSpace *as)
1727 {
1728 AddressSpaceDispatch *d = as->dispatch;
1729
1730 memory_listener_unregister(&d->listener);
1731 destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
1732 g_free(d);
1733 as->dispatch = NULL;
1734 }
1735
1736 static void memory_map_init(void)
1737 {
1738 system_memory = g_malloc(sizeof(*system_memory));
1739 memory_region_init(system_memory, "system", INT64_MAX);
1740 address_space_init(&address_space_memory, system_memory);
1741 address_space_memory.name = "memory";
1742
1743 system_io = g_malloc(sizeof(*system_io));
1744 memory_region_init(system_io, "io", 65536);
1745 address_space_init(&address_space_io, system_io);
1746 address_space_io.name = "I/O";
1747
1748 memory_listener_register(&core_memory_listener, &address_space_memory);
1749 memory_listener_register(&io_memory_listener, &address_space_io);
1750 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1751
1752 dma_context_init(&dma_context_memory, &address_space_memory,
1753 NULL, NULL, NULL);
1754 }
1755
1756 MemoryRegion *get_system_memory(void)
1757 {
1758 return system_memory;
1759 }
1760
1761 MemoryRegion *get_system_io(void)
1762 {
1763 return system_io;
1764 }
1765
1766 #endif /* !defined(CONFIG_USER_ONLY) */
1767
1768 /* physical memory access (slow version, mainly for debug) */
1769 #if defined(CONFIG_USER_ONLY)
1770 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1771 uint8_t *buf, int len, int is_write)
1772 {
1773 int l, flags;
1774 target_ulong page;
1775 void * p;
1776
1777 while (len > 0) {
1778 page = addr & TARGET_PAGE_MASK;
1779 l = (page + TARGET_PAGE_SIZE) - addr;
1780 if (l > len)
1781 l = len;
1782 flags = page_get_flags(page);
1783 if (!(flags & PAGE_VALID))
1784 return -1;
1785 if (is_write) {
1786 if (!(flags & PAGE_WRITE))
1787 return -1;
1788 /* XXX: this code should not depend on lock_user */
1789 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1790 return -1;
1791 memcpy(p, buf, l);
1792 unlock_user(p, addr, l);
1793 } else {
1794 if (!(flags & PAGE_READ))
1795 return -1;
1796 /* XXX: this code should not depend on lock_user */
1797 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1798 return -1;
1799 memcpy(buf, p, l);
1800 unlock_user(p, addr, 0);
1801 }
1802 len -= l;
1803 buf += l;
1804 addr += l;
1805 }
1806 return 0;
1807 }
1808
1809 #else
1810
1811 static void invalidate_and_set_dirty(hwaddr addr,
1812 hwaddr length)
1813 {
1814 if (!cpu_physical_memory_is_dirty(addr)) {
1815 /* invalidate code */
1816 tb_invalidate_phys_page_range(addr, addr + length, 0);
1817 /* set dirty bit */
1818 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1819 }
1820 xen_modified_memory(addr, length);
1821 }
1822
1823 void address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1824 int len, bool is_write)
1825 {
1826 AddressSpaceDispatch *d = as->dispatch;
1827 int l;
1828 uint8_t *ptr;
1829 uint32_t val;
1830 hwaddr page;
1831 MemoryRegionSection *section;
1832
1833 while (len > 0) {
1834 page = addr & TARGET_PAGE_MASK;
1835 l = (page + TARGET_PAGE_SIZE) - addr;
1836 if (l > len)
1837 l = len;
1838 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
1839
1840 if (is_write) {
1841 if (!memory_region_is_ram(section->mr)) {
1842 hwaddr addr1;
1843 addr1 = memory_region_section_addr(section, addr);
1844 /* XXX: could force cpu_single_env to NULL to avoid
1845 potential bugs */
1846 if (l >= 4 && ((addr1 & 3) == 0)) {
1847 /* 32 bit write access */
1848 val = ldl_p(buf);
1849 io_mem_write(section->mr, addr1, val, 4);
1850 l = 4;
1851 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1852 /* 16 bit write access */
1853 val = lduw_p(buf);
1854 io_mem_write(section->mr, addr1, val, 2);
1855 l = 2;
1856 } else {
1857 /* 8 bit write access */
1858 val = ldub_p(buf);
1859 io_mem_write(section->mr, addr1, val, 1);
1860 l = 1;
1861 }
1862 } else if (!section->readonly) {
1863 ram_addr_t addr1;
1864 addr1 = memory_region_get_ram_addr(section->mr)
1865 + memory_region_section_addr(section, addr);
1866 /* RAM case */
1867 ptr = qemu_get_ram_ptr(addr1);
1868 memcpy(ptr, buf, l);
1869 invalidate_and_set_dirty(addr1, l);
1870 qemu_put_ram_ptr(ptr);
1871 }
1872 } else {
1873 if (!(memory_region_is_ram(section->mr) ||
1874 memory_region_is_romd(section->mr))) {
1875 hwaddr addr1;
1876 /* I/O case */
1877 addr1 = memory_region_section_addr(section, addr);
1878 if (l >= 4 && ((addr1 & 3) == 0)) {
1879 /* 32 bit read access */
1880 val = io_mem_read(section->mr, addr1, 4);
1881 stl_p(buf, val);
1882 l = 4;
1883 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1884 /* 16 bit read access */
1885 val = io_mem_read(section->mr, addr1, 2);
1886 stw_p(buf, val);
1887 l = 2;
1888 } else {
1889 /* 8 bit read access */
1890 val = io_mem_read(section->mr, addr1, 1);
1891 stb_p(buf, val);
1892 l = 1;
1893 }
1894 } else {
1895 /* RAM case */
1896 ptr = qemu_get_ram_ptr(section->mr->ram_addr
1897 + memory_region_section_addr(section,
1898 addr));
1899 memcpy(buf, ptr, l);
1900 qemu_put_ram_ptr(ptr);
1901 }
1902 }
1903 len -= l;
1904 buf += l;
1905 addr += l;
1906 }
1907 }
1908
1909 void address_space_write(AddressSpace *as, hwaddr addr,
1910 const uint8_t *buf, int len)
1911 {
1912 address_space_rw(as, addr, (uint8_t *)buf, len, true);
1913 }
1914
1915 /**
1916 * address_space_read: read from an address space.
1917 *
1918 * @as: #AddressSpace to be accessed
1919 * @addr: address within that address space
1920 * @buf: buffer with the data transferred
1921 */
1922 void address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1923 {
1924 address_space_rw(as, addr, buf, len, false);
1925 }
1926
1927
1928 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1929 int len, int is_write)
1930 {
1931 return address_space_rw(&address_space_memory, addr, buf, len, is_write);
1932 }
1933
1934 /* used for ROM loading : can write in RAM and ROM */
1935 void cpu_physical_memory_write_rom(hwaddr addr,
1936 const uint8_t *buf, int len)
1937 {
1938 AddressSpaceDispatch *d = address_space_memory.dispatch;
1939 int l;
1940 uint8_t *ptr;
1941 hwaddr page;
1942 MemoryRegionSection *section;
1943
1944 while (len > 0) {
1945 page = addr & TARGET_PAGE_MASK;
1946 l = (page + TARGET_PAGE_SIZE) - addr;
1947 if (l > len)
1948 l = len;
1949 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
1950
1951 if (!(memory_region_is_ram(section->mr) ||
1952 memory_region_is_romd(section->mr))) {
1953 /* do nothing */
1954 } else {
1955 unsigned long addr1;
1956 addr1 = memory_region_get_ram_addr(section->mr)
1957 + memory_region_section_addr(section, addr);
1958 /* ROM/RAM case */
1959 ptr = qemu_get_ram_ptr(addr1);
1960 memcpy(ptr, buf, l);
1961 invalidate_and_set_dirty(addr1, l);
1962 qemu_put_ram_ptr(ptr);
1963 }
1964 len -= l;
1965 buf += l;
1966 addr += l;
1967 }
1968 }
1969
1970 typedef struct {
1971 void *buffer;
1972 hwaddr addr;
1973 hwaddr len;
1974 } BounceBuffer;
1975
1976 static BounceBuffer bounce;
1977
1978 typedef struct MapClient {
1979 void *opaque;
1980 void (*callback)(void *opaque);
1981 QLIST_ENTRY(MapClient) link;
1982 } MapClient;
1983
1984 static QLIST_HEAD(map_client_list, MapClient) map_client_list
1985 = QLIST_HEAD_INITIALIZER(map_client_list);
1986
1987 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
1988 {
1989 MapClient *client = g_malloc(sizeof(*client));
1990
1991 client->opaque = opaque;
1992 client->callback = callback;
1993 QLIST_INSERT_HEAD(&map_client_list, client, link);
1994 return client;
1995 }
1996
1997 static void cpu_unregister_map_client(void *_client)
1998 {
1999 MapClient *client = (MapClient *)_client;
2000
2001 QLIST_REMOVE(client, link);
2002 g_free(client);
2003 }
2004
2005 static void cpu_notify_map_clients(void)
2006 {
2007 MapClient *client;
2008
2009 while (!QLIST_EMPTY(&map_client_list)) {
2010 client = QLIST_FIRST(&map_client_list);
2011 client->callback(client->opaque);
2012 cpu_unregister_map_client(client);
2013 }
2014 }
2015
2016 /* Map a physical memory region into a host virtual address.
2017 * May map a subset of the requested range, given by and returned in *plen.
2018 * May return NULL if resources needed to perform the mapping are exhausted.
2019 * Use only for reads OR writes - not for read-modify-write operations.
2020 * Use cpu_register_map_client() to know when retrying the map operation is
2021 * likely to succeed.
2022 */
2023 void *address_space_map(AddressSpace *as,
2024 hwaddr addr,
2025 hwaddr *plen,
2026 bool is_write)
2027 {
2028 AddressSpaceDispatch *d = as->dispatch;
2029 hwaddr len = *plen;
2030 hwaddr todo = 0;
2031 int l;
2032 hwaddr page;
2033 MemoryRegionSection *section;
2034 ram_addr_t raddr = RAM_ADDR_MAX;
2035 ram_addr_t rlen;
2036 void *ret;
2037
2038 while (len > 0) {
2039 page = addr & TARGET_PAGE_MASK;
2040 l = (page + TARGET_PAGE_SIZE) - addr;
2041 if (l > len)
2042 l = len;
2043 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
2044
2045 if (!(memory_region_is_ram(section->mr) && !section->readonly)) {
2046 if (todo || bounce.buffer) {
2047 break;
2048 }
2049 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2050 bounce.addr = addr;
2051 bounce.len = l;
2052 if (!is_write) {
2053 address_space_read(as, addr, bounce.buffer, l);
2054 }
2055
2056 *plen = l;
2057 return bounce.buffer;
2058 }
2059 if (!todo) {
2060 raddr = memory_region_get_ram_addr(section->mr)
2061 + memory_region_section_addr(section, addr);
2062 }
2063
2064 len -= l;
2065 addr += l;
2066 todo += l;
2067 }
2068 rlen = todo;
2069 ret = qemu_ram_ptr_length(raddr, &rlen);
2070 *plen = rlen;
2071 return ret;
2072 }
2073
2074 /* Unmaps a memory region previously mapped by address_space_map().
2075 * Will also mark the memory as dirty if is_write == 1. access_len gives
2076 * the amount of memory that was actually read or written by the caller.
2077 */
2078 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2079 int is_write, hwaddr access_len)
2080 {
2081 if (buffer != bounce.buffer) {
2082 if (is_write) {
2083 ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
2084 while (access_len) {
2085 unsigned l;
2086 l = TARGET_PAGE_SIZE;
2087 if (l > access_len)
2088 l = access_len;
2089 invalidate_and_set_dirty(addr1, l);
2090 addr1 += l;
2091 access_len -= l;
2092 }
2093 }
2094 if (xen_enabled()) {
2095 xen_invalidate_map_cache_entry(buffer);
2096 }
2097 return;
2098 }
2099 if (is_write) {
2100 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2101 }
2102 qemu_vfree(bounce.buffer);
2103 bounce.buffer = NULL;
2104 cpu_notify_map_clients();
2105 }
2106
2107 void *cpu_physical_memory_map(hwaddr addr,
2108 hwaddr *plen,
2109 int is_write)
2110 {
2111 return address_space_map(&address_space_memory, addr, plen, is_write);
2112 }
2113
2114 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2115 int is_write, hwaddr access_len)
2116 {
2117 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2118 }
2119
2120 /* warning: addr must be aligned */
2121 static inline uint32_t ldl_phys_internal(hwaddr addr,
2122 enum device_endian endian)
2123 {
2124 uint8_t *ptr;
2125 uint32_t val;
2126 MemoryRegionSection *section;
2127
2128 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2129
2130 if (!(memory_region_is_ram(section->mr) ||
2131 memory_region_is_romd(section->mr))) {
2132 /* I/O case */
2133 addr = memory_region_section_addr(section, addr);
2134 val = io_mem_read(section->mr, addr, 4);
2135 #if defined(TARGET_WORDS_BIGENDIAN)
2136 if (endian == DEVICE_LITTLE_ENDIAN) {
2137 val = bswap32(val);
2138 }
2139 #else
2140 if (endian == DEVICE_BIG_ENDIAN) {
2141 val = bswap32(val);
2142 }
2143 #endif
2144 } else {
2145 /* RAM case */
2146 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2147 & TARGET_PAGE_MASK)
2148 + memory_region_section_addr(section, addr));
2149 switch (endian) {
2150 case DEVICE_LITTLE_ENDIAN:
2151 val = ldl_le_p(ptr);
2152 break;
2153 case DEVICE_BIG_ENDIAN:
2154 val = ldl_be_p(ptr);
2155 break;
2156 default:
2157 val = ldl_p(ptr);
2158 break;
2159 }
2160 }
2161 return val;
2162 }
2163
2164 uint32_t ldl_phys(hwaddr addr)
2165 {
2166 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2167 }
2168
2169 uint32_t ldl_le_phys(hwaddr addr)
2170 {
2171 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2172 }
2173
2174 uint32_t ldl_be_phys(hwaddr addr)
2175 {
2176 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2177 }
2178
2179 /* warning: addr must be aligned */
2180 static inline uint64_t ldq_phys_internal(hwaddr addr,
2181 enum device_endian endian)
2182 {
2183 uint8_t *ptr;
2184 uint64_t val;
2185 MemoryRegionSection *section;
2186
2187 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2188
2189 if (!(memory_region_is_ram(section->mr) ||
2190 memory_region_is_romd(section->mr))) {
2191 /* I/O case */
2192 addr = memory_region_section_addr(section, addr);
2193
2194 /* XXX This is broken when device endian != cpu endian.
2195 Fix and add "endian" variable check */
2196 #ifdef TARGET_WORDS_BIGENDIAN
2197 val = io_mem_read(section->mr, addr, 4) << 32;
2198 val |= io_mem_read(section->mr, addr + 4, 4);
2199 #else
2200 val = io_mem_read(section->mr, addr, 4);
2201 val |= io_mem_read(section->mr, addr + 4, 4) << 32;
2202 #endif
2203 } else {
2204 /* RAM case */
2205 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2206 & TARGET_PAGE_MASK)
2207 + memory_region_section_addr(section, addr));
2208 switch (endian) {
2209 case DEVICE_LITTLE_ENDIAN:
2210 val = ldq_le_p(ptr);
2211 break;
2212 case DEVICE_BIG_ENDIAN:
2213 val = ldq_be_p(ptr);
2214 break;
2215 default:
2216 val = ldq_p(ptr);
2217 break;
2218 }
2219 }
2220 return val;
2221 }
2222
2223 uint64_t ldq_phys(hwaddr addr)
2224 {
2225 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2226 }
2227
2228 uint64_t ldq_le_phys(hwaddr addr)
2229 {
2230 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2231 }
2232
2233 uint64_t ldq_be_phys(hwaddr addr)
2234 {
2235 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2236 }
2237
2238 /* XXX: optimize */
2239 uint32_t ldub_phys(hwaddr addr)
2240 {
2241 uint8_t val;
2242 cpu_physical_memory_read(addr, &val, 1);
2243 return val;
2244 }
2245
2246 /* warning: addr must be aligned */
2247 static inline uint32_t lduw_phys_internal(hwaddr addr,
2248 enum device_endian endian)
2249 {
2250 uint8_t *ptr;
2251 uint64_t val;
2252 MemoryRegionSection *section;
2253
2254 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2255
2256 if (!(memory_region_is_ram(section->mr) ||
2257 memory_region_is_romd(section->mr))) {
2258 /* I/O case */
2259 addr = memory_region_section_addr(section, addr);
2260 val = io_mem_read(section->mr, addr, 2);
2261 #if defined(TARGET_WORDS_BIGENDIAN)
2262 if (endian == DEVICE_LITTLE_ENDIAN) {
2263 val = bswap16(val);
2264 }
2265 #else
2266 if (endian == DEVICE_BIG_ENDIAN) {
2267 val = bswap16(val);
2268 }
2269 #endif
2270 } else {
2271 /* RAM case */
2272 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2273 & TARGET_PAGE_MASK)
2274 + memory_region_section_addr(section, addr));
2275 switch (endian) {
2276 case DEVICE_LITTLE_ENDIAN:
2277 val = lduw_le_p(ptr);
2278 break;
2279 case DEVICE_BIG_ENDIAN:
2280 val = lduw_be_p(ptr);
2281 break;
2282 default:
2283 val = lduw_p(ptr);
2284 break;
2285 }
2286 }
2287 return val;
2288 }
2289
2290 uint32_t lduw_phys(hwaddr addr)
2291 {
2292 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2293 }
2294
2295 uint32_t lduw_le_phys(hwaddr addr)
2296 {
2297 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2298 }
2299
2300 uint32_t lduw_be_phys(hwaddr addr)
2301 {
2302 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2303 }
2304
2305 /* warning: addr must be aligned. The ram page is not masked as dirty
2306 and the code inside is not invalidated. It is useful if the dirty
2307 bits are used to track modified PTEs */
2308 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2309 {
2310 uint8_t *ptr;
2311 MemoryRegionSection *section;
2312
2313 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2314
2315 if (!memory_region_is_ram(section->mr) || section->readonly) {
2316 addr = memory_region_section_addr(section, addr);
2317 if (memory_region_is_ram(section->mr)) {
2318 section = &phys_sections[phys_section_rom];
2319 }
2320 io_mem_write(section->mr, addr, val, 4);
2321 } else {
2322 unsigned long addr1 = (memory_region_get_ram_addr(section->mr)
2323 & TARGET_PAGE_MASK)
2324 + memory_region_section_addr(section, addr);
2325 ptr = qemu_get_ram_ptr(addr1);
2326 stl_p(ptr, val);
2327
2328 if (unlikely(in_migration)) {
2329 if (!cpu_physical_memory_is_dirty(addr1)) {
2330 /* invalidate code */
2331 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2332 /* set dirty bit */
2333 cpu_physical_memory_set_dirty_flags(
2334 addr1, (0xff & ~CODE_DIRTY_FLAG));
2335 }
2336 }
2337 }
2338 }
2339
2340 void stq_phys_notdirty(hwaddr addr, uint64_t val)
2341 {
2342 uint8_t *ptr;
2343 MemoryRegionSection *section;
2344
2345 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2346
2347 if (!memory_region_is_ram(section->mr) || section->readonly) {
2348 addr = memory_region_section_addr(section, addr);
2349 if (memory_region_is_ram(section->mr)) {
2350 section = &phys_sections[phys_section_rom];
2351 }
2352 #ifdef TARGET_WORDS_BIGENDIAN
2353 io_mem_write(section->mr, addr, val >> 32, 4);
2354 io_mem_write(section->mr, addr + 4, (uint32_t)val, 4);
2355 #else
2356 io_mem_write(section->mr, addr, (uint32_t)val, 4);
2357 io_mem_write(section->mr, addr + 4, val >> 32, 4);
2358 #endif
2359 } else {
2360 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2361 & TARGET_PAGE_MASK)
2362 + memory_region_section_addr(section, addr));
2363 stq_p(ptr, val);
2364 }
2365 }
2366
2367 /* warning: addr must be aligned */
2368 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2369 enum device_endian endian)
2370 {
2371 uint8_t *ptr;
2372 MemoryRegionSection *section;
2373
2374 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2375
2376 if (!memory_region_is_ram(section->mr) || section->readonly) {
2377 addr = memory_region_section_addr(section, addr);
2378 if (memory_region_is_ram(section->mr)) {
2379 section = &phys_sections[phys_section_rom];
2380 }
2381 #if defined(TARGET_WORDS_BIGENDIAN)
2382 if (endian == DEVICE_LITTLE_ENDIAN) {
2383 val = bswap32(val);
2384 }
2385 #else
2386 if (endian == DEVICE_BIG_ENDIAN) {
2387 val = bswap32(val);
2388 }
2389 #endif
2390 io_mem_write(section->mr, addr, val, 4);
2391 } else {
2392 unsigned long addr1;
2393 addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2394 + memory_region_section_addr(section, addr);
2395 /* RAM case */
2396 ptr = qemu_get_ram_ptr(addr1);
2397 switch (endian) {
2398 case DEVICE_LITTLE_ENDIAN:
2399 stl_le_p(ptr, val);
2400 break;
2401 case DEVICE_BIG_ENDIAN:
2402 stl_be_p(ptr, val);
2403 break;
2404 default:
2405 stl_p(ptr, val);
2406 break;
2407 }
2408 invalidate_and_set_dirty(addr1, 4);
2409 }
2410 }
2411
2412 void stl_phys(hwaddr addr, uint32_t val)
2413 {
2414 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2415 }
2416
2417 void stl_le_phys(hwaddr addr, uint32_t val)
2418 {
2419 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2420 }
2421
2422 void stl_be_phys(hwaddr addr, uint32_t val)
2423 {
2424 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2425 }
2426
2427 /* XXX: optimize */
2428 void stb_phys(hwaddr addr, uint32_t val)
2429 {
2430 uint8_t v = val;
2431 cpu_physical_memory_write(addr, &v, 1);
2432 }
2433
2434 /* warning: addr must be aligned */
2435 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2436 enum device_endian endian)
2437 {
2438 uint8_t *ptr;
2439 MemoryRegionSection *section;
2440
2441 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2442
2443 if (!memory_region_is_ram(section->mr) || section->readonly) {
2444 addr = memory_region_section_addr(section, addr);
2445 if (memory_region_is_ram(section->mr)) {
2446 section = &phys_sections[phys_section_rom];
2447 }
2448 #if defined(TARGET_WORDS_BIGENDIAN)
2449 if (endian == DEVICE_LITTLE_ENDIAN) {
2450 val = bswap16(val);
2451 }
2452 #else
2453 if (endian == DEVICE_BIG_ENDIAN) {
2454 val = bswap16(val);
2455 }
2456 #endif
2457 io_mem_write(section->mr, addr, val, 2);
2458 } else {
2459 unsigned long addr1;
2460 addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2461 + memory_region_section_addr(section, addr);
2462 /* RAM case */
2463 ptr = qemu_get_ram_ptr(addr1);
2464 switch (endian) {
2465 case DEVICE_LITTLE_ENDIAN:
2466 stw_le_p(ptr, val);
2467 break;
2468 case DEVICE_BIG_ENDIAN:
2469 stw_be_p(ptr, val);
2470 break;
2471 default:
2472 stw_p(ptr, val);
2473 break;
2474 }
2475 invalidate_and_set_dirty(addr1, 2);
2476 }
2477 }
2478
2479 void stw_phys(hwaddr addr, uint32_t val)
2480 {
2481 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2482 }
2483
2484 void stw_le_phys(hwaddr addr, uint32_t val)
2485 {
2486 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2487 }
2488
2489 void stw_be_phys(hwaddr addr, uint32_t val)
2490 {
2491 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2492 }
2493
2494 /* XXX: optimize */
2495 void stq_phys(hwaddr addr, uint64_t val)
2496 {
2497 val = tswap64(val);
2498 cpu_physical_memory_write(addr, &val, 8);
2499 }
2500
2501 void stq_le_phys(hwaddr addr, uint64_t val)
2502 {
2503 val = cpu_to_le64(val);
2504 cpu_physical_memory_write(addr, &val, 8);
2505 }
2506
2507 void stq_be_phys(hwaddr addr, uint64_t val)
2508 {
2509 val = cpu_to_be64(val);
2510 cpu_physical_memory_write(addr, &val, 8);
2511 }
2512
2513 /* virtual memory access for debug (includes writing to ROM) */
2514 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2515 uint8_t *buf, int len, int is_write)
2516 {
2517 int l;
2518 hwaddr phys_addr;
2519 target_ulong page;
2520
2521 while (len > 0) {
2522 page = addr & TARGET_PAGE_MASK;
2523 phys_addr = cpu_get_phys_page_debug(env, page);
2524 /* if no physical page mapped, return an error */
2525 if (phys_addr == -1)
2526 return -1;
2527 l = (page + TARGET_PAGE_SIZE) - addr;
2528 if (l > len)
2529 l = len;
2530 phys_addr += (addr & ~TARGET_PAGE_MASK);
2531 if (is_write)
2532 cpu_physical_memory_write_rom(phys_addr, buf, l);
2533 else
2534 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2535 len -= l;
2536 buf += l;
2537 addr += l;
2538 }
2539 return 0;
2540 }
2541 #endif
2542
2543 #if !defined(CONFIG_USER_ONLY)
2544
2545 /*
2546 * A helper function for the _utterly broken_ virtio device model to find out if
2547 * it's running on a big endian machine. Don't do this at home kids!
2548 */
2549 bool virtio_is_big_endian(void);
2550 bool virtio_is_big_endian(void)
2551 {
2552 #if defined(TARGET_WORDS_BIGENDIAN)
2553 return true;
2554 #else
2555 return false;
2556 #endif
2557 }
2558
2559 #endif
2560
2561 #ifndef CONFIG_USER_ONLY
2562 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2563 {
2564 MemoryRegionSection *section;
2565
2566 section = phys_page_find(address_space_memory.dispatch,
2567 phys_addr >> TARGET_PAGE_BITS);
2568
2569 return !(memory_region_is_ram(section->mr) ||
2570 memory_region_is_romd(section->mr));
2571 }
2572 #endif
CRIS target port of Qemu