?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
target-unicore32: Implement CPUClass::set_pc()
[qemu/rayw.git] / exec.c
blobc8658c6f9d5b84a2320e02d091102b96d42cc2b0
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
51
52 #include "exec/memory-internal.h"
53
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 CPUState *first_cpu;
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 typedef PhysPageEntry Node[L2_SIZE];
92
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101 };
102
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
110
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
115
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
124
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
127
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
133
134 static MemoryRegion io_mem_watch;
135 #endif
136
137 #if !defined(CONFIG_USER_ONLY)
138
139 static void phys_map_node_reserve(unsigned nodes)
140 {
141 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
142 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
143 16);
144 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
145 next_map.nodes_nb + nodes);
146 next_map.nodes = g_renew(Node, next_map.nodes,
147 next_map.nodes_nb_alloc);
148 }
149 }
150
151 static uint16_t phys_map_node_alloc(void)
152 {
153 unsigned i;
154 uint16_t ret;
155
156 ret = next_map.nodes_nb++;
157 assert(ret != PHYS_MAP_NODE_NIL);
158 assert(ret != next_map.nodes_nb_alloc);
159 for (i = 0; i < L2_SIZE; ++i) {
160 next_map.nodes[ret][i].is_leaf = 0;
161 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
162 }
163 return ret;
164 }
165
166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
167 hwaddr *nb, uint16_t leaf,
168 int level)
169 {
170 PhysPageEntry *p;
171 int i;
172 hwaddr step = (hwaddr)1 << (level * L2_BITS);
173
174 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
175 lp->ptr = phys_map_node_alloc();
176 p = next_map.nodes[lp->ptr];
177 if (level == 0) {
178 for (i = 0; i < L2_SIZE; i++) {
179 p[i].is_leaf = 1;
180 p[i].ptr = PHYS_SECTION_UNASSIGNED;
181 }
182 }
183 } else {
184 p = next_map.nodes[lp->ptr];
185 }
186 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
187
188 while (*nb && lp < &p[L2_SIZE]) {
189 if ((*index & (step - 1)) == 0 && *nb >= step) {
190 lp->is_leaf = true;
191 lp->ptr = leaf;
192 *index += step;
193 *nb -= step;
194 } else {
195 phys_page_set_level(lp, index, nb, leaf, level - 1);
196 }
197 ++lp;
198 }
199 }
200
201 static void phys_page_set(AddressSpaceDispatch *d,
202 hwaddr index, hwaddr nb,
203 uint16_t leaf)
204 {
205 /* Wildly overreserve - it doesn't matter much. */
206 phys_map_node_reserve(3 * P_L2_LEVELS);
207
208 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
209 }
210
211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
212 Node *nodes, MemoryRegionSection *sections)
213 {
214 PhysPageEntry *p;
215 int i;
216
217 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
218 if (lp.ptr == PHYS_MAP_NODE_NIL) {
219 return &sections[PHYS_SECTION_UNASSIGNED];
220 }
221 p = nodes[lp.ptr];
222 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
223 }
224 return &sections[lp.ptr];
225 }
226
227 bool memory_region_is_unassigned(MemoryRegion *mr)
228 {
229 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
230 && mr != &io_mem_watch;
231 }
232
233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
234 hwaddr addr,
235 bool resolve_subpage)
236 {
237 MemoryRegionSection *section;
238 subpage_t *subpage;
239
240 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
241 d->nodes, d->sections);
242 if (resolve_subpage && section->mr->subpage) {
243 subpage = container_of(section->mr, subpage_t, iomem);
244 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
245 }
246 return section;
247 }
248
249 static MemoryRegionSection *
250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
251 hwaddr *plen, bool resolve_subpage)
252 {
253 MemoryRegionSection *section;
254 Int128 diff;
255
256 section = address_space_lookup_region(d, addr, resolve_subpage);
257 /* Compute offset within MemoryRegionSection */
258 addr -= section->offset_within_address_space;
259
260 /* Compute offset within MemoryRegion */
261 *xlat = addr + section->offset_within_region;
262
263 diff = int128_sub(section->mr->size, int128_make64(addr));
264 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
265 return section;
266 }
267
268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
269 hwaddr *xlat, hwaddr *plen,
270 bool is_write)
271 {
272 IOMMUTLBEntry iotlb;
273 MemoryRegionSection *section;
274 MemoryRegion *mr;
275 hwaddr len = *plen;
276
277 for (;;) {
278 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
279 mr = section->mr;
280
281 if (!mr->iommu_ops) {
282 break;
283 }
284
285 iotlb = mr->iommu_ops->translate(mr, addr);
286 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
287 | (addr & iotlb.addr_mask));
288 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
289 if (!(iotlb.perm & (1 << is_write))) {
290 mr = &io_mem_unassigned;
291 break;
292 }
293
294 as = iotlb.target_as;
295 }
296
297 *plen = len;
298 *xlat = addr;
299 return mr;
300 }
301
302 MemoryRegionSection *
303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
304 hwaddr *plen)
305 {
306 MemoryRegionSection *section;
307 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
308
309 assert(!section->mr->iommu_ops);
310 return section;
311 }
312 #endif
313
314 void cpu_exec_init_all(void)
315 {
316 #if !defined(CONFIG_USER_ONLY)
317 qemu_mutex_init(&ram_list.mutex);
318 memory_map_init();
319 io_mem_init();
320 #endif
321 }
322
323 #if !defined(CONFIG_USER_ONLY)
324
325 static int cpu_common_post_load(void *opaque, int version_id)
326 {
327 CPUState *cpu = opaque;
328
329 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
330 version_id is increased. */
331 cpu->interrupt_request &= ~0x01;
332 tlb_flush(cpu->env_ptr, 1);
333
334 return 0;
335 }
336
337 const VMStateDescription vmstate_cpu_common = {
338 .name = "cpu_common",
339 .version_id = 1,
340 .minimum_version_id = 1,
341 .minimum_version_id_old = 1,
342 .post_load = cpu_common_post_load,
343 .fields = (VMStateField []) {
344 VMSTATE_UINT32(halted, CPUState),
345 VMSTATE_UINT32(interrupt_request, CPUState),
346 VMSTATE_END_OF_LIST()
347 }
348 };
349
350 #endif
351
352 CPUState *qemu_get_cpu(int index)
353 {
354 CPUState *cpu = first_cpu;
355
356 while (cpu) {
357 if (cpu->cpu_index == index) {
358 break;
359 }
360 cpu = cpu->next_cpu;
361 }
362
363 return cpu;
364 }
365
366 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
367 {
368 CPUState *cpu;
369
370 cpu = first_cpu;
371 while (cpu) {
372 func(cpu, data);
373 cpu = cpu->next_cpu;
374 }
375 }
376
377 void cpu_exec_init(CPUArchState *env)
378 {
379 CPUState *cpu = ENV_GET_CPU(env);
380 CPUClass *cc = CPU_GET_CLASS(cpu);
381 CPUState **pcpu;
382 int cpu_index;
383
384 #if defined(CONFIG_USER_ONLY)
385 cpu_list_lock();
386 #endif
387 cpu->next_cpu = NULL;
388 pcpu = &first_cpu;
389 cpu_index = 0;
390 while (*pcpu != NULL) {
391 pcpu = &(*pcpu)->next_cpu;
392 cpu_index++;
393 }
394 cpu->cpu_index = cpu_index;
395 cpu->numa_node = 0;
396 QTAILQ_INIT(&env->breakpoints);
397 QTAILQ_INIT(&env->watchpoints);
398 #ifndef CONFIG_USER_ONLY
399 cpu->thread_id = qemu_get_thread_id();
400 #endif
401 *pcpu = cpu;
402 #if defined(CONFIG_USER_ONLY)
403 cpu_list_unlock();
404 #endif
405 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
406 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
407 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
408 cpu_save, cpu_load, env);
409 assert(cc->vmsd == NULL);
410 #endif
411 if (cc->vmsd != NULL) {
412 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
413 }
414 }
415
416 #if defined(TARGET_HAS_ICE)
417 #if defined(CONFIG_USER_ONLY)
418 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
419 {
420 tb_invalidate_phys_page_range(pc, pc + 1, 0);
421 }
422 #else
423 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
424 {
425 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
426 (pc & ~TARGET_PAGE_MASK));
427 }
428 #endif
429 #endif /* TARGET_HAS_ICE */
430
431 #if defined(CONFIG_USER_ONLY)
432 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
433
434 {
435 }
436
437 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
438 int flags, CPUWatchpoint **watchpoint)
439 {
440 return -ENOSYS;
441 }
442 #else
443 /* Add a watchpoint. */
444 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
445 int flags, CPUWatchpoint **watchpoint)
446 {
447 target_ulong len_mask = ~(len - 1);
448 CPUWatchpoint *wp;
449
450 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
451 if ((len & (len - 1)) || (addr & ~len_mask) ||
452 len == 0 || len > TARGET_PAGE_SIZE) {
453 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
454 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
455 return -EINVAL;
456 }
457 wp = g_malloc(sizeof(*wp));
458
459 wp->vaddr = addr;
460 wp->len_mask = len_mask;
461 wp->flags = flags;
462
463 /* keep all GDB-injected watchpoints in front */
464 if (flags & BP_GDB)
465 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
466 else
467 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
468
469 tlb_flush_page(env, addr);
470
471 if (watchpoint)
472 *watchpoint = wp;
473 return 0;
474 }
475
476 /* Remove a specific watchpoint. */
477 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
478 int flags)
479 {
480 target_ulong len_mask = ~(len - 1);
481 CPUWatchpoint *wp;
482
483 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
484 if (addr == wp->vaddr && len_mask == wp->len_mask
485 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
486 cpu_watchpoint_remove_by_ref(env, wp);
487 return 0;
488 }
489 }
490 return -ENOENT;
491 }
492
493 /* Remove a specific watchpoint by reference. */
494 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
495 {
496 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
497
498 tlb_flush_page(env, watchpoint->vaddr);
499
500 g_free(watchpoint);
501 }
502
503 /* Remove all matching watchpoints. */
504 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
505 {
506 CPUWatchpoint *wp, *next;
507
508 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
509 if (wp->flags & mask)
510 cpu_watchpoint_remove_by_ref(env, wp);
511 }
512 }
513 #endif
514
515 /* Add a breakpoint. */
516 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
517 CPUBreakpoint **breakpoint)
518 {
519 #if defined(TARGET_HAS_ICE)
520 CPUBreakpoint *bp;
521
522 bp = g_malloc(sizeof(*bp));
523
524 bp->pc = pc;
525 bp->flags = flags;
526
527 /* keep all GDB-injected breakpoints in front */
528 if (flags & BP_GDB)
529 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
530 else
531 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
532
533 breakpoint_invalidate(env, pc);
534
535 if (breakpoint)
536 *breakpoint = bp;
537 return 0;
538 #else
539 return -ENOSYS;
540 #endif
541 }
542
543 /* Remove a specific breakpoint. */
544 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
545 {
546 #if defined(TARGET_HAS_ICE)
547 CPUBreakpoint *bp;
548
549 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
550 if (bp->pc == pc && bp->flags == flags) {
551 cpu_breakpoint_remove_by_ref(env, bp);
552 return 0;
553 }
554 }
555 return -ENOENT;
556 #else
557 return -ENOSYS;
558 #endif
559 }
560
561 /* Remove a specific breakpoint by reference. */
562 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
563 {
564 #if defined(TARGET_HAS_ICE)
565 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
566
567 breakpoint_invalidate(env, breakpoint->pc);
568
569 g_free(breakpoint);
570 #endif
571 }
572
573 /* Remove all matching breakpoints. */
574 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
575 {
576 #if defined(TARGET_HAS_ICE)
577 CPUBreakpoint *bp, *next;
578
579 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
580 if (bp->flags & mask)
581 cpu_breakpoint_remove_by_ref(env, bp);
582 }
583 #endif
584 }
585
586 /* enable or disable single step mode. EXCP_DEBUG is returned by the
587 CPU loop after each instruction */
588 void cpu_single_step(CPUArchState *env, int enabled)
589 {
590 #if defined(TARGET_HAS_ICE)
591 if (env->singlestep_enabled != enabled) {
592 env->singlestep_enabled = enabled;
593 if (kvm_enabled())
594 kvm_update_guest_debug(env, 0);
595 else {
596 /* must flush all the translated code to avoid inconsistencies */
597 /* XXX: only flush what is necessary */
598 tb_flush(env);
599 }
600 }
601 #endif
602 }
603
604 void cpu_abort(CPUArchState *env, const char *fmt, ...)
605 {
606 CPUState *cpu = ENV_GET_CPU(env);
607 va_list ap;
608 va_list ap2;
609
610 va_start(ap, fmt);
611 va_copy(ap2, ap);
612 fprintf(stderr, "qemu: fatal: ");
613 vfprintf(stderr, fmt, ap);
614 fprintf(stderr, "\n");
615 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
616 if (qemu_log_enabled()) {
617 qemu_log("qemu: fatal: ");
618 qemu_log_vprintf(fmt, ap2);
619 qemu_log("\n");
620 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
621 qemu_log_flush();
622 qemu_log_close();
623 }
624 va_end(ap2);
625 va_end(ap);
626 #if defined(CONFIG_USER_ONLY)
627 {
628 struct sigaction act;
629 sigfillset(&act.sa_mask);
630 act.sa_handler = SIG_DFL;
631 sigaction(SIGABRT, &act, NULL);
632 }
633 #endif
634 abort();
635 }
636
637 CPUArchState *cpu_copy(CPUArchState *env)
638 {
639 CPUArchState *new_env = cpu_init(env->cpu_model_str);
640 #if defined(TARGET_HAS_ICE)
641 CPUBreakpoint *bp;
642 CPUWatchpoint *wp;
643 #endif
644
645 memcpy(new_env, env, sizeof(CPUArchState));
646
647 /* Clone all break/watchpoints.
648 Note: Once we support ptrace with hw-debug register access, make sure
649 BP_CPU break/watchpoints are handled correctly on clone. */
650 QTAILQ_INIT(&env->breakpoints);
651 QTAILQ_INIT(&env->watchpoints);
652 #if defined(TARGET_HAS_ICE)
653 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
654 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
655 }
656 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
657 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
658 wp->flags, NULL);
659 }
660 #endif
661
662 return new_env;
663 }
664
665 #if !defined(CONFIG_USER_ONLY)
666 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
667 uintptr_t length)
668 {
669 uintptr_t start1;
670
671 /* we modify the TLB cache so that the dirty bit will be set again
672 when accessing the range */
673 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
674 /* Check that we don't span multiple blocks - this breaks the
675 address comparisons below. */
676 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
677 != (end - 1) - start) {
678 abort();
679 }
680 cpu_tlb_reset_dirty_all(start1, length);
681
682 }
683
684 /* Note: start and end must be within the same ram block. */
685 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
686 int dirty_flags)
687 {
688 uintptr_t length;
689
690 start &= TARGET_PAGE_MASK;
691 end = TARGET_PAGE_ALIGN(end);
692
693 length = end - start;
694 if (length == 0)
695 return;
696 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
697
698 if (tcg_enabled()) {
699 tlb_reset_dirty_range_all(start, end, length);
700 }
701 }
702
703 static int cpu_physical_memory_set_dirty_tracking(int enable)
704 {
705 int ret = 0;
706 in_migration = enable;
707 return ret;
708 }
709
710 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
711 MemoryRegionSection *section,
712 target_ulong vaddr,
713 hwaddr paddr, hwaddr xlat,
714 int prot,
715 target_ulong *address)
716 {
717 hwaddr iotlb;
718 CPUWatchpoint *wp;
719
720 if (memory_region_is_ram(section->mr)) {
721 /* Normal RAM. */
722 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
723 + xlat;
724 if (!section->readonly) {
725 iotlb |= PHYS_SECTION_NOTDIRTY;
726 } else {
727 iotlb |= PHYS_SECTION_ROM;
728 }
729 } else {
730 iotlb = section - address_space_memory.dispatch->sections;
731 iotlb += xlat;
732 }
733
734 /* Make accesses to pages with watchpoints go via the
735 watchpoint trap routines. */
736 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
737 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
738 /* Avoid trapping reads of pages with a write breakpoint. */
739 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
740 iotlb = PHYS_SECTION_WATCH + paddr;
741 *address |= TLB_MMIO;
742 break;
743 }
744 }
745 }
746
747 return iotlb;
748 }
749 #endif /* defined(CONFIG_USER_ONLY) */
750
751 #if !defined(CONFIG_USER_ONLY)
752
753 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
754 uint16_t section);
755 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
756
757 static uint16_t phys_section_add(MemoryRegionSection *section)
758 {
759 /* The physical section number is ORed with a page-aligned
760 * pointer to produce the iotlb entries. Thus it should
761 * never overflow into the page-aligned value.
762 */
763 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
764
765 if (next_map.sections_nb == next_map.sections_nb_alloc) {
766 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
767 16);
768 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
769 next_map.sections_nb_alloc);
770 }
771 next_map.sections[next_map.sections_nb] = *section;
772 memory_region_ref(section->mr);
773 return next_map.sections_nb++;
774 }
775
776 static void phys_section_destroy(MemoryRegion *mr)
777 {
778 memory_region_unref(mr);
779
780 if (mr->subpage) {
781 subpage_t *subpage = container_of(mr, subpage_t, iomem);
782 memory_region_destroy(&subpage->iomem);
783 g_free(subpage);
784 }
785 }
786
787 static void phys_sections_free(PhysPageMap *map)
788 {
789 while (map->sections_nb > 0) {
790 MemoryRegionSection *section = &map->sections[--map->sections_nb];
791 phys_section_destroy(section->mr);
792 }
793 g_free(map->sections);
794 g_free(map->nodes);
795 g_free(map);
796 }
797
798 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
799 {
800 subpage_t *subpage;
801 hwaddr base = section->offset_within_address_space
802 & TARGET_PAGE_MASK;
803 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
804 next_map.nodes, next_map.sections);
805 MemoryRegionSection subsection = {
806 .offset_within_address_space = base,
807 .size = int128_make64(TARGET_PAGE_SIZE),
808 };
809 hwaddr start, end;
810
811 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
812
813 if (!(existing->mr->subpage)) {
814 subpage = subpage_init(d->as, base);
815 subsection.mr = &subpage->iomem;
816 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
817 phys_section_add(&subsection));
818 } else {
819 subpage = container_of(existing->mr, subpage_t, iomem);
820 }
821 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
822 end = start + int128_get64(section->size) - 1;
823 subpage_register(subpage, start, end, phys_section_add(section));
824 }
825
826
827 static void register_multipage(AddressSpaceDispatch *d,
828 MemoryRegionSection *section)
829 {
830 hwaddr start_addr = section->offset_within_address_space;
831 uint16_t section_index = phys_section_add(section);
832 uint64_t num_pages = int128_get64(int128_rshift(section->size,
833 TARGET_PAGE_BITS));
834
835 assert(num_pages);
836 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
837 }
838
839 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
840 {
841 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
842 AddressSpaceDispatch *d = as->next_dispatch;
843 MemoryRegionSection now = *section, remain = *section;
844 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
845
846 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
847 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
848 - now.offset_within_address_space;
849
850 now.size = int128_min(int128_make64(left), now.size);
851 register_subpage(d, &now);
852 } else {
853 now.size = int128_zero();
854 }
855 while (int128_ne(remain.size, now.size)) {
856 remain.size = int128_sub(remain.size, now.size);
857 remain.offset_within_address_space += int128_get64(now.size);
858 remain.offset_within_region += int128_get64(now.size);
859 now = remain;
860 if (int128_lt(remain.size, page_size)) {
861 register_subpage(d, &now);
862 } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
863 now.size = page_size;
864 register_subpage(d, &now);
865 } else {
866 now.size = int128_and(now.size, int128_neg(page_size));
867 register_multipage(d, &now);
868 }
869 }
870 }
871
872 void qemu_flush_coalesced_mmio_buffer(void)
873 {
874 if (kvm_enabled())
875 kvm_flush_coalesced_mmio_buffer();
876 }
877
878 void qemu_mutex_lock_ramlist(void)
879 {
880 qemu_mutex_lock(&ram_list.mutex);
881 }
882
883 void qemu_mutex_unlock_ramlist(void)
884 {
885 qemu_mutex_unlock(&ram_list.mutex);
886 }
887
888 #if defined(__linux__) && !defined(TARGET_S390X)
889
890 #include <sys/vfs.h>
891
892 #define HUGETLBFS_MAGIC 0x958458f6
893
894 static long gethugepagesize(const char *path)
895 {
896 struct statfs fs;
897 int ret;
898
899 do {
900 ret = statfs(path, &fs);
901 } while (ret != 0 && errno == EINTR);
902
903 if (ret != 0) {
904 perror(path);
905 return 0;
906 }
907
908 if (fs.f_type != HUGETLBFS_MAGIC)
909 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
910
911 return fs.f_bsize;
912 }
913
914 static void *file_ram_alloc(RAMBlock *block,
915 ram_addr_t memory,
916 const char *path)
917 {
918 char *filename;
919 char *sanitized_name;
920 char *c;
921 void *area;
922 int fd;
923 #ifdef MAP_POPULATE
924 int flags;
925 #endif
926 unsigned long hpagesize;
927
928 hpagesize = gethugepagesize(path);
929 if (!hpagesize) {
930 return NULL;
931 }
932
933 if (memory < hpagesize) {
934 return NULL;
935 }
936
937 if (kvm_enabled() && !kvm_has_sync_mmu()) {
938 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
939 return NULL;
940 }
941
942 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
943 sanitized_name = g_strdup(block->mr->name);
944 for (c = sanitized_name; *c != '\0'; c++) {
945 if (*c == '/')
946 *c = '_';
947 }
948
949 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
950 sanitized_name);
951 g_free(sanitized_name);
952
953 fd = mkstemp(filename);
954 if (fd < 0) {
955 perror("unable to create backing store for hugepages");
956 g_free(filename);
957 return NULL;
958 }
959 unlink(filename);
960 g_free(filename);
961
962 memory = (memory+hpagesize-1) & ~(hpagesize-1);
963
964 /*
965 * ftruncate is not supported by hugetlbfs in older
966 * hosts, so don't bother bailing out on errors.
967 * If anything goes wrong with it under other filesystems,
968 * mmap will fail.
969 */
970 if (ftruncate(fd, memory))
971 perror("ftruncate");
972
973 #ifdef MAP_POPULATE
974 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
975 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
976 * to sidestep this quirk.
977 */
978 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
979 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
980 #else
981 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
982 #endif
983 if (area == MAP_FAILED) {
984 perror("file_ram_alloc: can't mmap RAM pages");
985 close(fd);
986 return (NULL);
987 }
988 block->fd = fd;
989 return area;
990 }
991 #endif
992
993 static ram_addr_t find_ram_offset(ram_addr_t size)
994 {
995 RAMBlock *block, *next_block;
996 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
997
998 assert(size != 0); /* it would hand out same offset multiple times */
999
1000 if (QTAILQ_EMPTY(&ram_list.blocks))
1001 return 0;
1002
1003 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1004 ram_addr_t end, next = RAM_ADDR_MAX;
1005
1006 end = block->offset + block->length;
1007
1008 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1009 if (next_block->offset >= end) {
1010 next = MIN(next, next_block->offset);
1011 }
1012 }
1013 if (next - end >= size && next - end < mingap) {
1014 offset = end;
1015 mingap = next - end;
1016 }
1017 }
1018
1019 if (offset == RAM_ADDR_MAX) {
1020 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1021 (uint64_t)size);
1022 abort();
1023 }
1024
1025 return offset;
1026 }
1027
1028 ram_addr_t last_ram_offset(void)
1029 {
1030 RAMBlock *block;
1031 ram_addr_t last = 0;
1032
1033 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1034 last = MAX(last, block->offset + block->length);
1035
1036 return last;
1037 }
1038
1039 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1040 {
1041 int ret;
1042
1043 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1044 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1045 "dump-guest-core", true)) {
1046 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1047 if (ret) {
1048 perror("qemu_madvise");
1049 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1050 "but dump_guest_core=off specified\n");
1051 }
1052 }
1053 }
1054
1055 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1056 {
1057 RAMBlock *new_block, *block;
1058
1059 new_block = NULL;
1060 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1061 if (block->offset == addr) {
1062 new_block = block;
1063 break;
1064 }
1065 }
1066 assert(new_block);
1067 assert(!new_block->idstr[0]);
1068
1069 if (dev) {
1070 char *id = qdev_get_dev_path(dev);
1071 if (id) {
1072 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1073 g_free(id);
1074 }
1075 }
1076 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1077
1078 /* This assumes the iothread lock is taken here too. */
1079 qemu_mutex_lock_ramlist();
1080 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1081 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1082 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1083 new_block->idstr);
1084 abort();
1085 }
1086 }
1087 qemu_mutex_unlock_ramlist();
1088 }
1089
1090 static int memory_try_enable_merging(void *addr, size_t len)
1091 {
1092 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1093 /* disabled by the user */
1094 return 0;
1095 }
1096
1097 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1098 }
1099
1100 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1101 MemoryRegion *mr)
1102 {
1103 RAMBlock *block, *new_block;
1104
1105 size = TARGET_PAGE_ALIGN(size);
1106 new_block = g_malloc0(sizeof(*new_block));
1107
1108 /* This assumes the iothread lock is taken here too. */
1109 qemu_mutex_lock_ramlist();
1110 new_block->mr = mr;
1111 new_block->offset = find_ram_offset(size);
1112 if (host) {
1113 new_block->host = host;
1114 new_block->flags |= RAM_PREALLOC_MASK;
1115 } else {
1116 if (mem_path) {
1117 #if defined (__linux__) && !defined(TARGET_S390X)
1118 new_block->host = file_ram_alloc(new_block, size, mem_path);
1119 if (!new_block->host) {
1120 new_block->host = qemu_anon_ram_alloc(size);
1121 memory_try_enable_merging(new_block->host, size);
1122 }
1123 #else
1124 fprintf(stderr, "-mem-path option unsupported\n");
1125 exit(1);
1126 #endif
1127 } else {
1128 if (xen_enabled()) {
1129 xen_ram_alloc(new_block->offset, size, mr);
1130 } else if (kvm_enabled()) {
1131 /* some s390/kvm configurations have special constraints */
1132 new_block->host = kvm_ram_alloc(size);
1133 } else {
1134 new_block->host = qemu_anon_ram_alloc(size);
1135 }
1136 memory_try_enable_merging(new_block->host, size);
1137 }
1138 }
1139 new_block->length = size;
1140
1141 /* Keep the list sorted from biggest to smallest block. */
1142 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1143 if (block->length < new_block->length) {
1144 break;
1145 }
1146 }
1147 if (block) {
1148 QTAILQ_INSERT_BEFORE(block, new_block, next);
1149 } else {
1150 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1151 }
1152 ram_list.mru_block = NULL;
1153
1154 ram_list.version++;
1155 qemu_mutex_unlock_ramlist();
1156
1157 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1158 last_ram_offset() >> TARGET_PAGE_BITS);
1159 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1160 0, size >> TARGET_PAGE_BITS);
1161 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1162
1163 qemu_ram_setup_dump(new_block->host, size);
1164 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1165
1166 if (kvm_enabled())
1167 kvm_setup_guest_memory(new_block->host, size);
1168
1169 return new_block->offset;
1170 }
1171
1172 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1173 {
1174 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1175 }
1176
1177 void qemu_ram_free_from_ptr(ram_addr_t addr)
1178 {
1179 RAMBlock *block;
1180
1181 /* This assumes the iothread lock is taken here too. */
1182 qemu_mutex_lock_ramlist();
1183 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1184 if (addr == block->offset) {
1185 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1186 ram_list.mru_block = NULL;
1187 ram_list.version++;
1188 g_free(block);
1189 break;
1190 }
1191 }
1192 qemu_mutex_unlock_ramlist();
1193 }
1194
1195 void qemu_ram_free(ram_addr_t addr)
1196 {
1197 RAMBlock *block;
1198
1199 /* This assumes the iothread lock is taken here too. */
1200 qemu_mutex_lock_ramlist();
1201 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1202 if (addr == block->offset) {
1203 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1204 ram_list.mru_block = NULL;
1205 ram_list.version++;
1206 if (block->flags & RAM_PREALLOC_MASK) {
1207 ;
1208 } else if (mem_path) {
1209 #if defined (__linux__) && !defined(TARGET_S390X)
1210 if (block->fd) {
1211 munmap(block->host, block->length);
1212 close(block->fd);
1213 } else {
1214 qemu_anon_ram_free(block->host, block->length);
1215 }
1216 #else
1217 abort();
1218 #endif
1219 } else {
1220 if (xen_enabled()) {
1221 xen_invalidate_map_cache_entry(block->host);
1222 } else {
1223 qemu_anon_ram_free(block->host, block->length);
1224 }
1225 }
1226 g_free(block);
1227 break;
1228 }
1229 }
1230 qemu_mutex_unlock_ramlist();
1231
1232 }
1233
1234 #ifndef _WIN32
1235 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1236 {
1237 RAMBlock *block;
1238 ram_addr_t offset;
1239 int flags;
1240 void *area, *vaddr;
1241
1242 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1243 offset = addr - block->offset;
1244 if (offset < block->length) {
1245 vaddr = block->host + offset;
1246 if (block->flags & RAM_PREALLOC_MASK) {
1247 ;
1248 } else {
1249 flags = MAP_FIXED;
1250 munmap(vaddr, length);
1251 if (mem_path) {
1252 #if defined(__linux__) && !defined(TARGET_S390X)
1253 if (block->fd) {
1254 #ifdef MAP_POPULATE
1255 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1256 MAP_PRIVATE;
1257 #else
1258 flags |= MAP_PRIVATE;
1259 #endif
1260 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1261 flags, block->fd, offset);
1262 } else {
1263 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1264 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1265 flags, -1, 0);
1266 }
1267 #else
1268 abort();
1269 #endif
1270 } else {
1271 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1272 flags |= MAP_SHARED | MAP_ANONYMOUS;
1273 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1274 flags, -1, 0);
1275 #else
1276 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1277 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1278 flags, -1, 0);
1279 #endif
1280 }
1281 if (area != vaddr) {
1282 fprintf(stderr, "Could not remap addr: "
1283 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1284 length, addr);
1285 exit(1);
1286 }
1287 memory_try_enable_merging(vaddr, length);
1288 qemu_ram_setup_dump(vaddr, length);
1289 }
1290 return;
1291 }
1292 }
1293 }
1294 #endif /* !_WIN32 */
1295
1296 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1297 {
1298 RAMBlock *block;
1299
1300 /* The list is protected by the iothread lock here. */
1301 block = ram_list.mru_block;
1302 if (block && addr - block->offset < block->length) {
1303 goto found;
1304 }
1305 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1306 if (addr - block->offset < block->length) {
1307 goto found;
1308 }
1309 }
1310
1311 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1312 abort();
1313
1314 found:
1315 ram_list.mru_block = block;
1316 return block;
1317 }
1318
1319 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1320 With the exception of the softmmu code in this file, this should
1321 only be used for local memory (e.g. video ram) that the device owns,
1322 and knows it isn't going to access beyond the end of the block.
1323
1324 It should not be used for general purpose DMA.
1325 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1326 */
1327 void *qemu_get_ram_ptr(ram_addr_t addr)
1328 {
1329 RAMBlock *block = qemu_get_ram_block(addr);
1330
1331 if (xen_enabled()) {
1332 /* We need to check if the requested address is in the RAM
1333 * because we don't want to map the entire memory in QEMU.
1334 * In that case just map until the end of the page.
1335 */
1336 if (block->offset == 0) {
1337 return xen_map_cache(addr, 0, 0);
1338 } else if (block->host == NULL) {
1339 block->host =
1340 xen_map_cache(block->offset, block->length, 1);
1341 }
1342 }
1343 return block->host + (addr - block->offset);
1344 }
1345
1346 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1347 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1348 *
1349 * ??? Is this still necessary?
1350 */
1351 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1352 {
1353 RAMBlock *block;
1354
1355 /* The list is protected by the iothread lock here. */
1356 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1357 if (addr - block->offset < block->length) {
1358 if (xen_enabled()) {
1359 /* We need to check if the requested address is in the RAM
1360 * because we don't want to map the entire memory in QEMU.
1361 * In that case just map until the end of the page.
1362 */
1363 if (block->offset == 0) {
1364 return xen_map_cache(addr, 0, 0);
1365 } else if (block->host == NULL) {
1366 block->host =
1367 xen_map_cache(block->offset, block->length, 1);
1368 }
1369 }
1370 return block->host + (addr - block->offset);
1371 }
1372 }
1373
1374 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1375 abort();
1376
1377 return NULL;
1378 }
1379
1380 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1381 * but takes a size argument */
1382 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1383 {
1384 if (*size == 0) {
1385 return NULL;
1386 }
1387 if (xen_enabled()) {
1388 return xen_map_cache(addr, *size, 1);
1389 } else {
1390 RAMBlock *block;
1391
1392 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1393 if (addr - block->offset < block->length) {
1394 if (addr - block->offset + *size > block->length)
1395 *size = block->length - addr + block->offset;
1396 return block->host + (addr - block->offset);
1397 }
1398 }
1399
1400 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1401 abort();
1402 }
1403 }
1404
1405 /* Some of the softmmu routines need to translate from a host pointer
1406 (typically a TLB entry) back to a ram offset. */
1407 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1408 {
1409 RAMBlock *block;
1410 uint8_t *host = ptr;
1411
1412 if (xen_enabled()) {
1413 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1414 return qemu_get_ram_block(*ram_addr)->mr;
1415 }
1416
1417 block = ram_list.mru_block;
1418 if (block && block->host && host - block->host < block->length) {
1419 goto found;
1420 }
1421
1422 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1423 /* This case append when the block is not mapped. */
1424 if (block->host == NULL) {
1425 continue;
1426 }
1427 if (host - block->host < block->length) {
1428 goto found;
1429 }
1430 }
1431
1432 return NULL;
1433
1434 found:
1435 *ram_addr = block->offset + (host - block->host);
1436 return block->mr;
1437 }
1438
1439 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1440 uint64_t val, unsigned size)
1441 {
1442 int dirty_flags;
1443 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1444 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1445 tb_invalidate_phys_page_fast(ram_addr, size);
1446 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1447 }
1448 switch (size) {
1449 case 1:
1450 stb_p(qemu_get_ram_ptr(ram_addr), val);
1451 break;
1452 case 2:
1453 stw_p(qemu_get_ram_ptr(ram_addr), val);
1454 break;
1455 case 4:
1456 stl_p(qemu_get_ram_ptr(ram_addr), val);
1457 break;
1458 default:
1459 abort();
1460 }
1461 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1462 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1463 /* we remove the notdirty callback only if the code has been
1464 flushed */
1465 if (dirty_flags == 0xff) {
1466 CPUArchState *env = current_cpu->env_ptr;
1467 tlb_set_dirty(env, env->mem_io_vaddr);
1468 }
1469 }
1470
1471 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1472 unsigned size, bool is_write)
1473 {
1474 return is_write;
1475 }
1476
1477 static const MemoryRegionOps notdirty_mem_ops = {
1478 .write = notdirty_mem_write,
1479 .valid.accepts = notdirty_mem_accepts,
1480 .endianness = DEVICE_NATIVE_ENDIAN,
1481 };
1482
1483 /* Generate a debug exception if a watchpoint has been hit. */
1484 static void check_watchpoint(int offset, int len_mask, int flags)
1485 {
1486 CPUArchState *env = current_cpu->env_ptr;
1487 target_ulong pc, cs_base;
1488 target_ulong vaddr;
1489 CPUWatchpoint *wp;
1490 int cpu_flags;
1491
1492 if (env->watchpoint_hit) {
1493 /* We re-entered the check after replacing the TB. Now raise
1494 * the debug interrupt so that is will trigger after the
1495 * current instruction. */
1496 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1497 return;
1498 }
1499 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1500 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1501 if ((vaddr == (wp->vaddr & len_mask) ||
1502 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1503 wp->flags |= BP_WATCHPOINT_HIT;
1504 if (!env->watchpoint_hit) {
1505 env->watchpoint_hit = wp;
1506 tb_check_watchpoint(env);
1507 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1508 env->exception_index = EXCP_DEBUG;
1509 cpu_loop_exit(env);
1510 } else {
1511 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1512 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1513 cpu_resume_from_signal(env, NULL);
1514 }
1515 }
1516 } else {
1517 wp->flags &= ~BP_WATCHPOINT_HIT;
1518 }
1519 }
1520 }
1521
1522 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1523 so these check for a hit then pass through to the normal out-of-line
1524 phys routines. */
1525 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1526 unsigned size)
1527 {
1528 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1529 switch (size) {
1530 case 1: return ldub_phys(addr);
1531 case 2: return lduw_phys(addr);
1532 case 4: return ldl_phys(addr);
1533 default: abort();
1534 }
1535 }
1536
1537 static void watch_mem_write(void *opaque, hwaddr addr,
1538 uint64_t val, unsigned size)
1539 {
1540 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1541 switch (size) {
1542 case 1:
1543 stb_phys(addr, val);
1544 break;
1545 case 2:
1546 stw_phys(addr, val);
1547 break;
1548 case 4:
1549 stl_phys(addr, val);
1550 break;
1551 default: abort();
1552 }
1553 }
1554
1555 static const MemoryRegionOps watch_mem_ops = {
1556 .read = watch_mem_read,
1557 .write = watch_mem_write,
1558 .endianness = DEVICE_NATIVE_ENDIAN,
1559 };
1560
1561 static uint64_t subpage_read(void *opaque, hwaddr addr,
1562 unsigned len)
1563 {
1564 subpage_t *subpage = opaque;
1565 uint8_t buf[4];
1566
1567 #if defined(DEBUG_SUBPAGE)
1568 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1569 subpage, len, addr);
1570 #endif
1571 address_space_read(subpage->as, addr + subpage->base, buf, len);
1572 switch (len) {
1573 case 1:
1574 return ldub_p(buf);
1575 case 2:
1576 return lduw_p(buf);
1577 case 4:
1578 return ldl_p(buf);
1579 default:
1580 abort();
1581 }
1582 }
1583
1584 static void subpage_write(void *opaque, hwaddr addr,
1585 uint64_t value, unsigned len)
1586 {
1587 subpage_t *subpage = opaque;
1588 uint8_t buf[4];
1589
1590 #if defined(DEBUG_SUBPAGE)
1591 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1592 " value %"PRIx64"\n",
1593 __func__, subpage, len, addr, value);
1594 #endif
1595 switch (len) {
1596 case 1:
1597 stb_p(buf, value);
1598 break;
1599 case 2:
1600 stw_p(buf, value);
1601 break;
1602 case 4:
1603 stl_p(buf, value);
1604 break;
1605 default:
1606 abort();
1607 }
1608 address_space_write(subpage->as, addr + subpage->base, buf, len);
1609 }
1610
1611 static bool subpage_accepts(void *opaque, hwaddr addr,
1612 unsigned size, bool is_write)
1613 {
1614 subpage_t *subpage = opaque;
1615 #if defined(DEBUG_SUBPAGE)
1616 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1617 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1618 #endif
1619
1620 return address_space_access_valid(subpage->as, addr + subpage->base,
1621 size, is_write);
1622 }
1623
1624 static const MemoryRegionOps subpage_ops = {
1625 .read = subpage_read,
1626 .write = subpage_write,
1627 .valid.accepts = subpage_accepts,
1628 .endianness = DEVICE_NATIVE_ENDIAN,
1629 };
1630
1631 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1632 uint16_t section)
1633 {
1634 int idx, eidx;
1635
1636 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1637 return -1;
1638 idx = SUBPAGE_IDX(start);
1639 eidx = SUBPAGE_IDX(end);
1640 #if defined(DEBUG_SUBPAGE)
1641 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1642 mmio, start, end, idx, eidx, memory);
1643 #endif
1644 for (; idx <= eidx; idx++) {
1645 mmio->sub_section[idx] = section;
1646 }
1647
1648 return 0;
1649 }
1650
1651 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1652 {
1653 subpage_t *mmio;
1654
1655 mmio = g_malloc0(sizeof(subpage_t));
1656
1657 mmio->as = as;
1658 mmio->base = base;
1659 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1660 "subpage", TARGET_PAGE_SIZE);
1661 mmio->iomem.subpage = true;
1662 #if defined(DEBUG_SUBPAGE)
1663 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1664 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1665 #endif
1666 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1667
1668 return mmio;
1669 }
1670
1671 static uint16_t dummy_section(MemoryRegion *mr)
1672 {
1673 MemoryRegionSection section = {
1674 .mr = mr,
1675 .offset_within_address_space = 0,
1676 .offset_within_region = 0,
1677 .size = int128_2_64(),
1678 };
1679
1680 return phys_section_add(&section);
1681 }
1682
1683 MemoryRegion *iotlb_to_region(hwaddr index)
1684 {
1685 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1686 }
1687
1688 static void io_mem_init(void)
1689 {
1690 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1691 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1692 "unassigned", UINT64_MAX);
1693 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1694 "notdirty", UINT64_MAX);
1695 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1696 "watch", UINT64_MAX);
1697 }
1698
1699 static void mem_begin(MemoryListener *listener)
1700 {
1701 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1702 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1703
1704 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1705 d->as = as;
1706 as->next_dispatch = d;
1707 }
1708
1709 static void mem_commit(MemoryListener *listener)
1710 {
1711 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1712 AddressSpaceDispatch *cur = as->dispatch;
1713 AddressSpaceDispatch *next = as->next_dispatch;
1714
1715 next->nodes = next_map.nodes;
1716 next->sections = next_map.sections;
1717
1718 as->dispatch = next;
1719 g_free(cur);
1720 }
1721
1722 static void core_begin(MemoryListener *listener)
1723 {
1724 uint16_t n;
1725
1726 prev_map = g_new(PhysPageMap, 1);
1727 *prev_map = next_map;
1728
1729 memset(&next_map, 0, sizeof(next_map));
1730 n = dummy_section(&io_mem_unassigned);
1731 assert(n == PHYS_SECTION_UNASSIGNED);
1732 n = dummy_section(&io_mem_notdirty);
1733 assert(n == PHYS_SECTION_NOTDIRTY);
1734 n = dummy_section(&io_mem_rom);
1735 assert(n == PHYS_SECTION_ROM);
1736 n = dummy_section(&io_mem_watch);
1737 assert(n == PHYS_SECTION_WATCH);
1738 }
1739
1740 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1741 * All AddressSpaceDispatch instances have switched to the next map.
1742 */
1743 static void core_commit(MemoryListener *listener)
1744 {
1745 phys_sections_free(prev_map);
1746 }
1747
1748 static void tcg_commit(MemoryListener *listener)
1749 {
1750 CPUState *cpu;
1751
1752 /* since each CPU stores ram addresses in its TLB cache, we must
1753 reset the modified entries */
1754 /* XXX: slow ! */
1755 for (cpu = first_cpu; cpu != NULL; cpu = cpu->next_cpu) {
1756 CPUArchState *env = cpu->env_ptr;
1757
1758 tlb_flush(env, 1);
1759 }
1760 }
1761
1762 static void core_log_global_start(MemoryListener *listener)
1763 {
1764 cpu_physical_memory_set_dirty_tracking(1);
1765 }
1766
1767 static void core_log_global_stop(MemoryListener *listener)
1768 {
1769 cpu_physical_memory_set_dirty_tracking(0);
1770 }
1771
1772 static MemoryListener core_memory_listener = {
1773 .begin = core_begin,
1774 .commit = core_commit,
1775 .log_global_start = core_log_global_start,
1776 .log_global_stop = core_log_global_stop,
1777 .priority = 1,
1778 };
1779
1780 static MemoryListener tcg_memory_listener = {
1781 .commit = tcg_commit,
1782 };
1783
1784 void address_space_init_dispatch(AddressSpace *as)
1785 {
1786 as->dispatch = NULL;
1787 as->dispatch_listener = (MemoryListener) {
1788 .begin = mem_begin,
1789 .commit = mem_commit,
1790 .region_add = mem_add,
1791 .region_nop = mem_add,
1792 .priority = 0,
1793 };
1794 memory_listener_register(&as->dispatch_listener, as);
1795 }
1796
1797 void address_space_destroy_dispatch(AddressSpace *as)
1798 {
1799 AddressSpaceDispatch *d = as->dispatch;
1800
1801 memory_listener_unregister(&as->dispatch_listener);
1802 g_free(d);
1803 as->dispatch = NULL;
1804 }
1805
1806 static void memory_map_init(void)
1807 {
1808 system_memory = g_malloc(sizeof(*system_memory));
1809 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1810 address_space_init(&address_space_memory, system_memory, "memory");
1811
1812 system_io = g_malloc(sizeof(*system_io));
1813 memory_region_init(system_io, NULL, "io", 65536);
1814 address_space_init(&address_space_io, system_io, "I/O");
1815
1816 memory_listener_register(&core_memory_listener, &address_space_memory);
1817 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1818 }
1819
1820 MemoryRegion *get_system_memory(void)
1821 {
1822 return system_memory;
1823 }
1824
1825 MemoryRegion *get_system_io(void)
1826 {
1827 return system_io;
1828 }
1829
1830 #endif /* !defined(CONFIG_USER_ONLY) */
1831
1832 /* physical memory access (slow version, mainly for debug) */
1833 #if defined(CONFIG_USER_ONLY)
1834 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1835 uint8_t *buf, int len, int is_write)
1836 {
1837 int l, flags;
1838 target_ulong page;
1839 void * p;
1840
1841 while (len > 0) {
1842 page = addr & TARGET_PAGE_MASK;
1843 l = (page + TARGET_PAGE_SIZE) - addr;
1844 if (l > len)
1845 l = len;
1846 flags = page_get_flags(page);
1847 if (!(flags & PAGE_VALID))
1848 return -1;
1849 if (is_write) {
1850 if (!(flags & PAGE_WRITE))
1851 return -1;
1852 /* XXX: this code should not depend on lock_user */
1853 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1854 return -1;
1855 memcpy(p, buf, l);
1856 unlock_user(p, addr, l);
1857 } else {
1858 if (!(flags & PAGE_READ))
1859 return -1;
1860 /* XXX: this code should not depend on lock_user */
1861 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1862 return -1;
1863 memcpy(buf, p, l);
1864 unlock_user(p, addr, 0);
1865 }
1866 len -= l;
1867 buf += l;
1868 addr += l;
1869 }
1870 return 0;
1871 }
1872
1873 #else
1874
1875 static void invalidate_and_set_dirty(hwaddr addr,
1876 hwaddr length)
1877 {
1878 if (!cpu_physical_memory_is_dirty(addr)) {
1879 /* invalidate code */
1880 tb_invalidate_phys_page_range(addr, addr + length, 0);
1881 /* set dirty bit */
1882 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1883 }
1884 xen_modified_memory(addr, length);
1885 }
1886
1887 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1888 {
1889 if (memory_region_is_ram(mr)) {
1890 return !(is_write && mr->readonly);
1891 }
1892 if (memory_region_is_romd(mr)) {
1893 return !is_write;
1894 }
1895
1896 return false;
1897 }
1898
1899 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1900 {
1901 unsigned access_size_max = mr->ops->valid.max_access_size;
1902
1903 /* Regions are assumed to support 1-4 byte accesses unless
1904 otherwise specified. */
1905 if (access_size_max == 0) {
1906 access_size_max = 4;
1907 }
1908
1909 /* Bound the maximum access by the alignment of the address. */
1910 if (!mr->ops->impl.unaligned) {
1911 unsigned align_size_max = addr & -addr;
1912 if (align_size_max != 0 && align_size_max < access_size_max) {
1913 access_size_max = align_size_max;
1914 }
1915 }
1916
1917 /* Don't attempt accesses larger than the maximum. */
1918 if (l > access_size_max) {
1919 l = access_size_max;
1920 }
1921
1922 return l;
1923 }
1924
1925 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1926 int len, bool is_write)
1927 {
1928 hwaddr l;
1929 uint8_t *ptr;
1930 uint64_t val;
1931 hwaddr addr1;
1932 MemoryRegion *mr;
1933 bool error = false;
1934
1935 while (len > 0) {
1936 l = len;
1937 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1938
1939 if (is_write) {
1940 if (!memory_access_is_direct(mr, is_write)) {
1941 l = memory_access_size(mr, l, addr1);
1942 /* XXX: could force current_cpu to NULL to avoid
1943 potential bugs */
1944 switch (l) {
1945 case 8:
1946 /* 64 bit write access */
1947 val = ldq_p(buf);
1948 error |= io_mem_write(mr, addr1, val, 8);
1949 break;
1950 case 4:
1951 /* 32 bit write access */
1952 val = ldl_p(buf);
1953 error |= io_mem_write(mr, addr1, val, 4);
1954 break;
1955 case 2:
1956 /* 16 bit write access */
1957 val = lduw_p(buf);
1958 error |= io_mem_write(mr, addr1, val, 2);
1959 break;
1960 case 1:
1961 /* 8 bit write access */
1962 val = ldub_p(buf);
1963 error |= io_mem_write(mr, addr1, val, 1);
1964 break;
1965 default:
1966 abort();
1967 }
1968 } else {
1969 addr1 += memory_region_get_ram_addr(mr);
1970 /* RAM case */
1971 ptr = qemu_get_ram_ptr(addr1);
1972 memcpy(ptr, buf, l);
1973 invalidate_and_set_dirty(addr1, l);
1974 }
1975 } else {
1976 if (!memory_access_is_direct(mr, is_write)) {
1977 /* I/O case */
1978 l = memory_access_size(mr, l, addr1);
1979 switch (l) {
1980 case 8:
1981 /* 64 bit read access */
1982 error |= io_mem_read(mr, addr1, &val, 8);
1983 stq_p(buf, val);
1984 break;
1985 case 4:
1986 /* 32 bit read access */
1987 error |= io_mem_read(mr, addr1, &val, 4);
1988 stl_p(buf, val);
1989 break;
1990 case 2:
1991 /* 16 bit read access */
1992 error |= io_mem_read(mr, addr1, &val, 2);
1993 stw_p(buf, val);
1994 break;
1995 case 1:
1996 /* 8 bit read access */
1997 error |= io_mem_read(mr, addr1, &val, 1);
1998 stb_p(buf, val);
1999 break;
2000 default:
2001 abort();
2002 }
2003 } else {
2004 /* RAM case */
2005 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2006 memcpy(buf, ptr, l);
2007 }
2008 }
2009 len -= l;
2010 buf += l;
2011 addr += l;
2012 }
2013
2014 return error;
2015 }
2016
2017 bool address_space_write(AddressSpace *as, hwaddr addr,
2018 const uint8_t *buf, int len)
2019 {
2020 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2021 }
2022
2023 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2024 {
2025 return address_space_rw(as, addr, buf, len, false);
2026 }
2027
2028
2029 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2030 int len, int is_write)
2031 {
2032 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2033 }
2034
2035 /* used for ROM loading : can write in RAM and ROM */
2036 void cpu_physical_memory_write_rom(hwaddr addr,
2037 const uint8_t *buf, int len)
2038 {
2039 hwaddr l;
2040 uint8_t *ptr;
2041 hwaddr addr1;
2042 MemoryRegion *mr;
2043
2044 while (len > 0) {
2045 l = len;
2046 mr = address_space_translate(&address_space_memory,
2047 addr, &addr1, &l, true);
2048
2049 if (!(memory_region_is_ram(mr) ||
2050 memory_region_is_romd(mr))) {
2051 /* do nothing */
2052 } else {
2053 addr1 += memory_region_get_ram_addr(mr);
2054 /* ROM/RAM case */
2055 ptr = qemu_get_ram_ptr(addr1);
2056 memcpy(ptr, buf, l);
2057 invalidate_and_set_dirty(addr1, l);
2058 }
2059 len -= l;
2060 buf += l;
2061 addr += l;
2062 }
2063 }
2064
2065 typedef struct {
2066 MemoryRegion *mr;
2067 void *buffer;
2068 hwaddr addr;
2069 hwaddr len;
2070 } BounceBuffer;
2071
2072 static BounceBuffer bounce;
2073
2074 typedef struct MapClient {
2075 void *opaque;
2076 void (*callback)(void *opaque);
2077 QLIST_ENTRY(MapClient) link;
2078 } MapClient;
2079
2080 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2081 = QLIST_HEAD_INITIALIZER(map_client_list);
2082
2083 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2084 {
2085 MapClient *client = g_malloc(sizeof(*client));
2086
2087 client->opaque = opaque;
2088 client->callback = callback;
2089 QLIST_INSERT_HEAD(&map_client_list, client, link);
2090 return client;
2091 }
2092
2093 static void cpu_unregister_map_client(void *_client)
2094 {
2095 MapClient *client = (MapClient *)_client;
2096
2097 QLIST_REMOVE(client, link);
2098 g_free(client);
2099 }
2100
2101 static void cpu_notify_map_clients(void)
2102 {
2103 MapClient *client;
2104
2105 while (!QLIST_EMPTY(&map_client_list)) {
2106 client = QLIST_FIRST(&map_client_list);
2107 client->callback(client->opaque);
2108 cpu_unregister_map_client(client);
2109 }
2110 }
2111
2112 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2113 {
2114 MemoryRegion *mr;
2115 hwaddr l, xlat;
2116
2117 while (len > 0) {
2118 l = len;
2119 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2120 if (!memory_access_is_direct(mr, is_write)) {
2121 l = memory_access_size(mr, l, addr);
2122 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2123 return false;
2124 }
2125 }
2126
2127 len -= l;
2128 addr += l;
2129 }
2130 return true;
2131 }
2132
2133 /* Map a physical memory region into a host virtual address.
2134 * May map a subset of the requested range, given by and returned in *plen.
2135 * May return NULL if resources needed to perform the mapping are exhausted.
2136 * Use only for reads OR writes - not for read-modify-write operations.
2137 * Use cpu_register_map_client() to know when retrying the map operation is
2138 * likely to succeed.
2139 */
2140 void *address_space_map(AddressSpace *as,
2141 hwaddr addr,
2142 hwaddr *plen,
2143 bool is_write)
2144 {
2145 hwaddr len = *plen;
2146 hwaddr done = 0;
2147 hwaddr l, xlat, base;
2148 MemoryRegion *mr, *this_mr;
2149 ram_addr_t raddr;
2150
2151 if (len == 0) {
2152 return NULL;
2153 }
2154
2155 l = len;
2156 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2157 if (!memory_access_is_direct(mr, is_write)) {
2158 if (bounce.buffer) {
2159 return NULL;
2160 }
2161 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2162 bounce.addr = addr;
2163 bounce.len = l;
2164
2165 memory_region_ref(mr);
2166 bounce.mr = mr;
2167 if (!is_write) {
2168 address_space_read(as, addr, bounce.buffer, l);
2169 }
2170
2171 *plen = l;
2172 return bounce.buffer;
2173 }
2174
2175 base = xlat;
2176 raddr = memory_region_get_ram_addr(mr);
2177
2178 for (;;) {
2179 len -= l;
2180 addr += l;
2181 done += l;
2182 if (len == 0) {
2183 break;
2184 }
2185
2186 l = len;
2187 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2188 if (this_mr != mr || xlat != base + done) {
2189 break;
2190 }
2191 }
2192
2193 memory_region_ref(mr);
2194 *plen = done;
2195 return qemu_ram_ptr_length(raddr + base, plen);
2196 }
2197
2198 /* Unmaps a memory region previously mapped by address_space_map().
2199 * Will also mark the memory as dirty if is_write == 1. access_len gives
2200 * the amount of memory that was actually read or written by the caller.
2201 */
2202 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2203 int is_write, hwaddr access_len)
2204 {
2205 if (buffer != bounce.buffer) {
2206 MemoryRegion *mr;
2207 ram_addr_t addr1;
2208
2209 mr = qemu_ram_addr_from_host(buffer, &addr1);
2210 assert(mr != NULL);
2211 if (is_write) {
2212 while (access_len) {
2213 unsigned l;
2214 l = TARGET_PAGE_SIZE;
2215 if (l > access_len)
2216 l = access_len;
2217 invalidate_and_set_dirty(addr1, l);
2218 addr1 += l;
2219 access_len -= l;
2220 }
2221 }
2222 if (xen_enabled()) {
2223 xen_invalidate_map_cache_entry(buffer);
2224 }
2225 memory_region_unref(mr);
2226 return;
2227 }
2228 if (is_write) {
2229 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2230 }
2231 qemu_vfree(bounce.buffer);
2232 bounce.buffer = NULL;
2233 memory_region_unref(bounce.mr);
2234 cpu_notify_map_clients();
2235 }
2236
2237 void *cpu_physical_memory_map(hwaddr addr,
2238 hwaddr *plen,
2239 int is_write)
2240 {
2241 return address_space_map(&address_space_memory, addr, plen, is_write);
2242 }
2243
2244 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2245 int is_write, hwaddr access_len)
2246 {
2247 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2248 }
2249
2250 /* warning: addr must be aligned */
2251 static inline uint32_t ldl_phys_internal(hwaddr addr,
2252 enum device_endian endian)
2253 {
2254 uint8_t *ptr;
2255 uint64_t val;
2256 MemoryRegion *mr;
2257 hwaddr l = 4;
2258 hwaddr addr1;
2259
2260 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2261 false);
2262 if (l < 4 || !memory_access_is_direct(mr, false)) {
2263 /* I/O case */
2264 io_mem_read(mr, addr1, &val, 4);
2265 #if defined(TARGET_WORDS_BIGENDIAN)
2266 if (endian == DEVICE_LITTLE_ENDIAN) {
2267 val = bswap32(val);
2268 }
2269 #else
2270 if (endian == DEVICE_BIG_ENDIAN) {
2271 val = bswap32(val);
2272 }
2273 #endif
2274 } else {
2275 /* RAM case */
2276 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2277 & TARGET_PAGE_MASK)
2278 + addr1);
2279 switch (endian) {
2280 case DEVICE_LITTLE_ENDIAN:
2281 val = ldl_le_p(ptr);
2282 break;
2283 case DEVICE_BIG_ENDIAN:
2284 val = ldl_be_p(ptr);
2285 break;
2286 default:
2287 val = ldl_p(ptr);
2288 break;
2289 }
2290 }
2291 return val;
2292 }
2293
2294 uint32_t ldl_phys(hwaddr addr)
2295 {
2296 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2297 }
2298
2299 uint32_t ldl_le_phys(hwaddr addr)
2300 {
2301 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2302 }
2303
2304 uint32_t ldl_be_phys(hwaddr addr)
2305 {
2306 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2307 }
2308
2309 /* warning: addr must be aligned */
2310 static inline uint64_t ldq_phys_internal(hwaddr addr,
2311 enum device_endian endian)
2312 {
2313 uint8_t *ptr;
2314 uint64_t val;
2315 MemoryRegion *mr;
2316 hwaddr l = 8;
2317 hwaddr addr1;
2318
2319 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2320 false);
2321 if (l < 8 || !memory_access_is_direct(mr, false)) {
2322 /* I/O case */
2323 io_mem_read(mr, addr1, &val, 8);
2324 #if defined(TARGET_WORDS_BIGENDIAN)
2325 if (endian == DEVICE_LITTLE_ENDIAN) {
2326 val = bswap64(val);
2327 }
2328 #else
2329 if (endian == DEVICE_BIG_ENDIAN) {
2330 val = bswap64(val);
2331 }
2332 #endif
2333 } else {
2334 /* RAM case */
2335 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2336 & TARGET_PAGE_MASK)
2337 + addr1);
2338 switch (endian) {
2339 case DEVICE_LITTLE_ENDIAN:
2340 val = ldq_le_p(ptr);
2341 break;
2342 case DEVICE_BIG_ENDIAN:
2343 val = ldq_be_p(ptr);
2344 break;
2345 default:
2346 val = ldq_p(ptr);
2347 break;
2348 }
2349 }
2350 return val;
2351 }
2352
2353 uint64_t ldq_phys(hwaddr addr)
2354 {
2355 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2356 }
2357
2358 uint64_t ldq_le_phys(hwaddr addr)
2359 {
2360 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2361 }
2362
2363 uint64_t ldq_be_phys(hwaddr addr)
2364 {
2365 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2366 }
2367
2368 /* XXX: optimize */
2369 uint32_t ldub_phys(hwaddr addr)
2370 {
2371 uint8_t val;
2372 cpu_physical_memory_read(addr, &val, 1);
2373 return val;
2374 }
2375
2376 /* warning: addr must be aligned */
2377 static inline uint32_t lduw_phys_internal(hwaddr addr,
2378 enum device_endian endian)
2379 {
2380 uint8_t *ptr;
2381 uint64_t val;
2382 MemoryRegion *mr;
2383 hwaddr l = 2;
2384 hwaddr addr1;
2385
2386 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2387 false);
2388 if (l < 2 || !memory_access_is_direct(mr, false)) {
2389 /* I/O case */
2390 io_mem_read(mr, addr1, &val, 2);
2391 #if defined(TARGET_WORDS_BIGENDIAN)
2392 if (endian == DEVICE_LITTLE_ENDIAN) {
2393 val = bswap16(val);
2394 }
2395 #else
2396 if (endian == DEVICE_BIG_ENDIAN) {
2397 val = bswap16(val);
2398 }
2399 #endif
2400 } else {
2401 /* RAM case */
2402 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2403 & TARGET_PAGE_MASK)
2404 + addr1);
2405 switch (endian) {
2406 case DEVICE_LITTLE_ENDIAN:
2407 val = lduw_le_p(ptr);
2408 break;
2409 case DEVICE_BIG_ENDIAN:
2410 val = lduw_be_p(ptr);
2411 break;
2412 default:
2413 val = lduw_p(ptr);
2414 break;
2415 }
2416 }
2417 return val;
2418 }
2419
2420 uint32_t lduw_phys(hwaddr addr)
2421 {
2422 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2423 }
2424
2425 uint32_t lduw_le_phys(hwaddr addr)
2426 {
2427 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2428 }
2429
2430 uint32_t lduw_be_phys(hwaddr addr)
2431 {
2432 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2433 }
2434
2435 /* warning: addr must be aligned. The ram page is not masked as dirty
2436 and the code inside is not invalidated. It is useful if the dirty
2437 bits are used to track modified PTEs */
2438 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2439 {
2440 uint8_t *ptr;
2441 MemoryRegion *mr;
2442 hwaddr l = 4;
2443 hwaddr addr1;
2444
2445 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2446 true);
2447 if (l < 4 || !memory_access_is_direct(mr, true)) {
2448 io_mem_write(mr, addr1, val, 4);
2449 } else {
2450 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2451 ptr = qemu_get_ram_ptr(addr1);
2452 stl_p(ptr, val);
2453
2454 if (unlikely(in_migration)) {
2455 if (!cpu_physical_memory_is_dirty(addr1)) {
2456 /* invalidate code */
2457 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2458 /* set dirty bit */
2459 cpu_physical_memory_set_dirty_flags(
2460 addr1, (0xff & ~CODE_DIRTY_FLAG));
2461 }
2462 }
2463 }
2464 }
2465
2466 /* warning: addr must be aligned */
2467 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2468 enum device_endian endian)
2469 {
2470 uint8_t *ptr;
2471 MemoryRegion *mr;
2472 hwaddr l = 4;
2473 hwaddr addr1;
2474
2475 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2476 true);
2477 if (l < 4 || !memory_access_is_direct(mr, true)) {
2478 #if defined(TARGET_WORDS_BIGENDIAN)
2479 if (endian == DEVICE_LITTLE_ENDIAN) {
2480 val = bswap32(val);
2481 }
2482 #else
2483 if (endian == DEVICE_BIG_ENDIAN) {
2484 val = bswap32(val);
2485 }
2486 #endif
2487 io_mem_write(mr, addr1, val, 4);
2488 } else {
2489 /* RAM case */
2490 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2491 ptr = qemu_get_ram_ptr(addr1);
2492 switch (endian) {
2493 case DEVICE_LITTLE_ENDIAN:
2494 stl_le_p(ptr, val);
2495 break;
2496 case DEVICE_BIG_ENDIAN:
2497 stl_be_p(ptr, val);
2498 break;
2499 default:
2500 stl_p(ptr, val);
2501 break;
2502 }
2503 invalidate_and_set_dirty(addr1, 4);
2504 }
2505 }
2506
2507 void stl_phys(hwaddr addr, uint32_t val)
2508 {
2509 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2510 }
2511
2512 void stl_le_phys(hwaddr addr, uint32_t val)
2513 {
2514 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2515 }
2516
2517 void stl_be_phys(hwaddr addr, uint32_t val)
2518 {
2519 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2520 }
2521
2522 /* XXX: optimize */
2523 void stb_phys(hwaddr addr, uint32_t val)
2524 {
2525 uint8_t v = val;
2526 cpu_physical_memory_write(addr, &v, 1);
2527 }
2528
2529 /* warning: addr must be aligned */
2530 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2531 enum device_endian endian)
2532 {
2533 uint8_t *ptr;
2534 MemoryRegion *mr;
2535 hwaddr l = 2;
2536 hwaddr addr1;
2537
2538 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2539 true);
2540 if (l < 2 || !memory_access_is_direct(mr, true)) {
2541 #if defined(TARGET_WORDS_BIGENDIAN)
2542 if (endian == DEVICE_LITTLE_ENDIAN) {
2543 val = bswap16(val);
2544 }
2545 #else
2546 if (endian == DEVICE_BIG_ENDIAN) {
2547 val = bswap16(val);
2548 }
2549 #endif
2550 io_mem_write(mr, addr1, val, 2);
2551 } else {
2552 /* RAM case */
2553 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2554 ptr = qemu_get_ram_ptr(addr1);
2555 switch (endian) {
2556 case DEVICE_LITTLE_ENDIAN:
2557 stw_le_p(ptr, val);
2558 break;
2559 case DEVICE_BIG_ENDIAN:
2560 stw_be_p(ptr, val);
2561 break;
2562 default:
2563 stw_p(ptr, val);
2564 break;
2565 }
2566 invalidate_and_set_dirty(addr1, 2);
2567 }
2568 }
2569
2570 void stw_phys(hwaddr addr, uint32_t val)
2571 {
2572 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2573 }
2574
2575 void stw_le_phys(hwaddr addr, uint32_t val)
2576 {
2577 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2578 }
2579
2580 void stw_be_phys(hwaddr addr, uint32_t val)
2581 {
2582 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2583 }
2584
2585 /* XXX: optimize */
2586 void stq_phys(hwaddr addr, uint64_t val)
2587 {
2588 val = tswap64(val);
2589 cpu_physical_memory_write(addr, &val, 8);
2590 }
2591
2592 void stq_le_phys(hwaddr addr, uint64_t val)
2593 {
2594 val = cpu_to_le64(val);
2595 cpu_physical_memory_write(addr, &val, 8);
2596 }
2597
2598 void stq_be_phys(hwaddr addr, uint64_t val)
2599 {
2600 val = cpu_to_be64(val);
2601 cpu_physical_memory_write(addr, &val, 8);
2602 }
2603
2604 /* virtual memory access for debug (includes writing to ROM) */
2605 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2606 uint8_t *buf, int len, int is_write)
2607 {
2608 int l;
2609 hwaddr phys_addr;
2610 target_ulong page;
2611
2612 while (len > 0) {
2613 page = addr & TARGET_PAGE_MASK;
2614 phys_addr = cpu_get_phys_page_debug(env, page);
2615 /* if no physical page mapped, return an error */
2616 if (phys_addr == -1)
2617 return -1;
2618 l = (page + TARGET_PAGE_SIZE) - addr;
2619 if (l > len)
2620 l = len;
2621 phys_addr += (addr & ~TARGET_PAGE_MASK);
2622 if (is_write)
2623 cpu_physical_memory_write_rom(phys_addr, buf, l);
2624 else
2625 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2626 len -= l;
2627 buf += l;
2628 addr += l;
2629 }
2630 return 0;
2631 }
2632 #endif
2633
2634 #if !defined(CONFIG_USER_ONLY)
2635
2636 /*
2637 * A helper function for the _utterly broken_ virtio device model to find out if
2638 * it's running on a big endian machine. Don't do this at home kids!
2639 */
2640 bool virtio_is_big_endian(void);
2641 bool virtio_is_big_endian(void)
2642 {
2643 #if defined(TARGET_WORDS_BIGENDIAN)
2644 return true;
2645 #else
2646 return false;
2647 #endif
2648 }
2649
2650 #endif
2651
2652 #ifndef CONFIG_USER_ONLY
2653 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2654 {
2655 MemoryRegion*mr;
2656 hwaddr l = 1;
2657
2658 mr = address_space_translate(&address_space_memory,
2659 phys_addr, &phys_addr, &l, false);
2660
2661 return !(memory_region_is_ram(mr) ||
2662 memory_region_is_romd(mr));
2663 }
2664
2665 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2666 {
2667 RAMBlock *block;
2668
2669 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2670 func(block->host, block->offset, block->length, opaque);
2671 }
2672 }
2673 #endif
Ray Wang's QEMU Repository