?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
exec: separate current memory map from the one being built
[qemu/ar7.git] / exec.c
blob598ac3a012882ff00813fb5190b4ec0648b440be
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "hw/xen/xen.h"
35 #include "qemu/timer.h"
36 #include "qemu/config-file.h"
37 #include "exec/memory.h"
38 #include "sysemu/dma.h"
39 #include "exec/address-spaces.h"
40 #if defined(CONFIG_USER_ONLY)
41 #include <qemu.h>
42 #else /* !CONFIG_USER_ONLY */
43 #include "sysemu/xen-mapcache.h"
44 #include "trace.h"
45 #endif
46 #include "exec/cpu-all.h"
47
48 #include "exec/cputlb.h"
49 #include "translate-all.h"
50
51 #include "exec/memory-internal.h"
52
53 //#define DEBUG_SUBPAGE
54
55 #if !defined(CONFIG_USER_ONLY)
56 int phys_ram_fd;
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 CPUArchState *first_cpu;
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUArchState *,cpu_single_env);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 struct AddressSpaceDispatch {
92 /* This is a multi-level map on the physical address space.
93 * The bottom level has pointers to MemoryRegionSections.
94 */
95 PhysPageEntry phys_map;
96 MemoryListener listener;
97 AddressSpace *as;
98 };
99
100 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
101 typedef struct subpage_t {
102 MemoryRegion iomem;
103 AddressSpace *as;
104 hwaddr base;
105 uint16_t sub_section[TARGET_PAGE_SIZE];
106 } subpage_t;
107
108 #define PHYS_SECTION_UNASSIGNED 0
109 #define PHYS_SECTION_NOTDIRTY 1
110 #define PHYS_SECTION_ROM 2
111 #define PHYS_SECTION_WATCH 3
112
113 typedef PhysPageEntry Node[L2_SIZE];
114
115 typedef struct PhysPageMap {
116 unsigned sections_nb;
117 unsigned sections_nb_alloc;
118 unsigned nodes_nb;
119 unsigned nodes_nb_alloc;
120 Node *nodes;
121 MemoryRegionSection *sections;
122 } PhysPageMap;
123
124 static PhysPageMap cur_map;
125 static PhysPageMap next_map;
126
127 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
128
129 static void io_mem_init(void);
130 static void memory_map_init(void);
131 static void *qemu_safe_ram_ptr(ram_addr_t addr);
132
133 static MemoryRegion io_mem_watch;
134 #endif
135
136 #if !defined(CONFIG_USER_ONLY)
137
138 static void phys_map_node_reserve(unsigned nodes)
139 {
140 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
141 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
142 16);
143 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
144 next_map.nodes_nb + nodes);
145 next_map.nodes = g_renew(Node, next_map.nodes,
146 next_map.nodes_nb_alloc);
147 }
148 }
149
150 static uint16_t phys_map_node_alloc(void)
151 {
152 unsigned i;
153 uint16_t ret;
154
155 ret = next_map.nodes_nb++;
156 assert(ret != PHYS_MAP_NODE_NIL);
157 assert(ret != next_map.nodes_nb_alloc);
158 for (i = 0; i < L2_SIZE; ++i) {
159 next_map.nodes[ret][i].is_leaf = 0;
160 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
161 }
162 return ret;
163 }
164
165 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
166 hwaddr *nb, uint16_t leaf,
167 int level)
168 {
169 PhysPageEntry *p;
170 int i;
171 hwaddr step = (hwaddr)1 << (level * L2_BITS);
172
173 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
174 lp->ptr = phys_map_node_alloc();
175 p = next_map.nodes[lp->ptr];
176 if (level == 0) {
177 for (i = 0; i < L2_SIZE; i++) {
178 p[i].is_leaf = 1;
179 p[i].ptr = PHYS_SECTION_UNASSIGNED;
180 }
181 }
182 } else {
183 p = next_map.nodes[lp->ptr];
184 }
185 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
186
187 while (*nb && lp < &p[L2_SIZE]) {
188 if ((*index & (step - 1)) == 0 && *nb >= step) {
189 lp->is_leaf = true;
190 lp->ptr = leaf;
191 *index += step;
192 *nb -= step;
193 } else {
194 phys_page_set_level(lp, index, nb, leaf, level - 1);
195 }
196 ++lp;
197 }
198 }
199
200 static void phys_page_set(AddressSpaceDispatch *d,
201 hwaddr index, hwaddr nb,
202 uint16_t leaf)
203 {
204 /* Wildly overreserve - it doesn't matter much. */
205 phys_map_node_reserve(3 * P_L2_LEVELS);
206
207 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
208 }
209
210 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
211 Node *nodes, MemoryRegionSection *sections)
212 {
213 PhysPageEntry *p;
214 int i;
215
216 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
217 if (lp.ptr == PHYS_MAP_NODE_NIL) {
218 return &sections[PHYS_SECTION_UNASSIGNED];
219 }
220 p = nodes[lp.ptr];
221 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
222 }
223 return &sections[lp.ptr];
224 }
225
226 bool memory_region_is_unassigned(MemoryRegion *mr)
227 {
228 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
229 && mr != &io_mem_watch;
230 }
231
232 static MemoryRegionSection *address_space_lookup_region(AddressSpace *as,
233 hwaddr addr,
234 bool resolve_subpage)
235 {
236 MemoryRegionSection *section;
237 subpage_t *subpage;
238
239 section = phys_page_find(as->dispatch->phys_map, addr >> TARGET_PAGE_BITS,
240 cur_map.nodes, cur_map.sections);
241 if (resolve_subpage && section->mr->subpage) {
242 subpage = container_of(section->mr, subpage_t, iomem);
243 section = &cur_map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
244 }
245 return section;
246 }
247
248 static MemoryRegionSection *
249 address_space_translate_internal(AddressSpace *as, hwaddr addr, hwaddr *xlat,
250 hwaddr *plen, bool resolve_subpage)
251 {
252 MemoryRegionSection *section;
253 Int128 diff;
254
255 section = address_space_lookup_region(as, addr, resolve_subpage);
256 /* Compute offset within MemoryRegionSection */
257 addr -= section->offset_within_address_space;
258
259 /* Compute offset within MemoryRegion */
260 *xlat = addr + section->offset_within_region;
261
262 diff = int128_sub(section->mr->size, int128_make64(addr));
263 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
264 return section;
265 }
266
267 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
268 hwaddr *xlat, hwaddr *plen,
269 bool is_write)
270 {
271 IOMMUTLBEntry iotlb;
272 MemoryRegionSection *section;
273 MemoryRegion *mr;
274 hwaddr len = *plen;
275
276 for (;;) {
277 section = address_space_translate_internal(as, addr, &addr, plen, true);
278 mr = section->mr;
279
280 if (!mr->iommu_ops) {
281 break;
282 }
283
284 iotlb = mr->iommu_ops->translate(mr, addr);
285 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
286 | (addr & iotlb.addr_mask));
287 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
288 if (!(iotlb.perm & (1 << is_write))) {
289 mr = &io_mem_unassigned;
290 break;
291 }
292
293 as = iotlb.target_as;
294 }
295
296 *plen = len;
297 *xlat = addr;
298 return mr;
299 }
300
301 MemoryRegionSection *
302 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
303 hwaddr *plen)
304 {
305 MemoryRegionSection *section;
306 section = address_space_translate_internal(as, addr, xlat, plen, false);
307
308 assert(!section->mr->iommu_ops);
309 return section;
310 }
311 #endif
312
313 void cpu_exec_init_all(void)
314 {
315 #if !defined(CONFIG_USER_ONLY)
316 qemu_mutex_init(&ram_list.mutex);
317 memory_map_init();
318 io_mem_init();
319 #endif
320 }
321
322 #if !defined(CONFIG_USER_ONLY)
323
324 static int cpu_common_post_load(void *opaque, int version_id)
325 {
326 CPUState *cpu = opaque;
327
328 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
329 version_id is increased. */
330 cpu->interrupt_request &= ~0x01;
331 tlb_flush(cpu->env_ptr, 1);
332
333 return 0;
334 }
335
336 const VMStateDescription vmstate_cpu_common = {
337 .name = "cpu_common",
338 .version_id = 1,
339 .minimum_version_id = 1,
340 .minimum_version_id_old = 1,
341 .post_load = cpu_common_post_load,
342 .fields = (VMStateField []) {
343 VMSTATE_UINT32(halted, CPUState),
344 VMSTATE_UINT32(interrupt_request, CPUState),
345 VMSTATE_END_OF_LIST()
346 }
347 };
348
349 #endif
350
351 CPUState *qemu_get_cpu(int index)
352 {
353 CPUArchState *env = first_cpu;
354 CPUState *cpu = NULL;
355
356 while (env) {
357 cpu = ENV_GET_CPU(env);
358 if (cpu->cpu_index == index) {
359 break;
360 }
361 env = env->next_cpu;
362 }
363
364 return env ? cpu : NULL;
365 }
366
367 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
368 {
369 CPUArchState *env = first_cpu;
370
371 while (env) {
372 func(ENV_GET_CPU(env), data);
373 env = env->next_cpu;
374 }
375 }
376
377 void cpu_exec_init(CPUArchState *env)
378 {
379 CPUState *cpu = ENV_GET_CPU(env);
380 CPUClass *cc = CPU_GET_CLASS(cpu);
381 CPUArchState **penv;
382 int cpu_index;
383
384 #if defined(CONFIG_USER_ONLY)
385 cpu_list_lock();
386 #endif
387 env->next_cpu = NULL;
388 penv = &first_cpu;
389 cpu_index = 0;
390 while (*penv != NULL) {
391 penv = &(*penv)->next_cpu;
392 cpu_index++;
393 }
394 cpu->cpu_index = cpu_index;
395 cpu->numa_node = 0;
396 QTAILQ_INIT(&env->breakpoints);
397 QTAILQ_INIT(&env->watchpoints);
398 #ifndef CONFIG_USER_ONLY
399 cpu->thread_id = qemu_get_thread_id();
400 #endif
401 *penv = env;
402 #if defined(CONFIG_USER_ONLY)
403 cpu_list_unlock();
404 #endif
405 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
406 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
407 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
408 cpu_save, cpu_load, env);
409 assert(cc->vmsd == NULL);
410 #endif
411 if (cc->vmsd != NULL) {
412 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
413 }
414 }
415
416 #if defined(TARGET_HAS_ICE)
417 #if defined(CONFIG_USER_ONLY)
418 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
419 {
420 tb_invalidate_phys_page_range(pc, pc + 1, 0);
421 }
422 #else
423 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
424 {
425 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
426 (pc & ~TARGET_PAGE_MASK));
427 }
428 #endif
429 #endif /* TARGET_HAS_ICE */
430
431 #if defined(CONFIG_USER_ONLY)
432 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
433
434 {
435 }
436
437 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
438 int flags, CPUWatchpoint **watchpoint)
439 {
440 return -ENOSYS;
441 }
442 #else
443 /* Add a watchpoint. */
444 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
445 int flags, CPUWatchpoint **watchpoint)
446 {
447 target_ulong len_mask = ~(len - 1);
448 CPUWatchpoint *wp;
449
450 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
451 if ((len & (len - 1)) || (addr & ~len_mask) ||
452 len == 0 || len > TARGET_PAGE_SIZE) {
453 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
454 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
455 return -EINVAL;
456 }
457 wp = g_malloc(sizeof(*wp));
458
459 wp->vaddr = addr;
460 wp->len_mask = len_mask;
461 wp->flags = flags;
462
463 /* keep all GDB-injected watchpoints in front */
464 if (flags & BP_GDB)
465 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
466 else
467 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
468
469 tlb_flush_page(env, addr);
470
471 if (watchpoint)
472 *watchpoint = wp;
473 return 0;
474 }
475
476 /* Remove a specific watchpoint. */
477 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
478 int flags)
479 {
480 target_ulong len_mask = ~(len - 1);
481 CPUWatchpoint *wp;
482
483 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
484 if (addr == wp->vaddr && len_mask == wp->len_mask
485 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
486 cpu_watchpoint_remove_by_ref(env, wp);
487 return 0;
488 }
489 }
490 return -ENOENT;
491 }
492
493 /* Remove a specific watchpoint by reference. */
494 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
495 {
496 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
497
498 tlb_flush_page(env, watchpoint->vaddr);
499
500 g_free(watchpoint);
501 }
502
503 /* Remove all matching watchpoints. */
504 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
505 {
506 CPUWatchpoint *wp, *next;
507
508 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
509 if (wp->flags & mask)
510 cpu_watchpoint_remove_by_ref(env, wp);
511 }
512 }
513 #endif
514
515 /* Add a breakpoint. */
516 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
517 CPUBreakpoint **breakpoint)
518 {
519 #if defined(TARGET_HAS_ICE)
520 CPUBreakpoint *bp;
521
522 bp = g_malloc(sizeof(*bp));
523
524 bp->pc = pc;
525 bp->flags = flags;
526
527 /* keep all GDB-injected breakpoints in front */
528 if (flags & BP_GDB)
529 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
530 else
531 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
532
533 breakpoint_invalidate(env, pc);
534
535 if (breakpoint)
536 *breakpoint = bp;
537 return 0;
538 #else
539 return -ENOSYS;
540 #endif
541 }
542
543 /* Remove a specific breakpoint. */
544 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
545 {
546 #if defined(TARGET_HAS_ICE)
547 CPUBreakpoint *bp;
548
549 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
550 if (bp->pc == pc && bp->flags == flags) {
551 cpu_breakpoint_remove_by_ref(env, bp);
552 return 0;
553 }
554 }
555 return -ENOENT;
556 #else
557 return -ENOSYS;
558 #endif
559 }
560
561 /* Remove a specific breakpoint by reference. */
562 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
563 {
564 #if defined(TARGET_HAS_ICE)
565 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
566
567 breakpoint_invalidate(env, breakpoint->pc);
568
569 g_free(breakpoint);
570 #endif
571 }
572
573 /* Remove all matching breakpoints. */
574 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
575 {
576 #if defined(TARGET_HAS_ICE)
577 CPUBreakpoint *bp, *next;
578
579 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
580 if (bp->flags & mask)
581 cpu_breakpoint_remove_by_ref(env, bp);
582 }
583 #endif
584 }
585
586 /* enable or disable single step mode. EXCP_DEBUG is returned by the
587 CPU loop after each instruction */
588 void cpu_single_step(CPUArchState *env, int enabled)
589 {
590 #if defined(TARGET_HAS_ICE)
591 if (env->singlestep_enabled != enabled) {
592 env->singlestep_enabled = enabled;
593 if (kvm_enabled())
594 kvm_update_guest_debug(env, 0);
595 else {
596 /* must flush all the translated code to avoid inconsistencies */
597 /* XXX: only flush what is necessary */
598 tb_flush(env);
599 }
600 }
601 #endif
602 }
603
604 void cpu_abort(CPUArchState *env, const char *fmt, ...)
605 {
606 CPUState *cpu = ENV_GET_CPU(env);
607 va_list ap;
608 va_list ap2;
609
610 va_start(ap, fmt);
611 va_copy(ap2, ap);
612 fprintf(stderr, "qemu: fatal: ");
613 vfprintf(stderr, fmt, ap);
614 fprintf(stderr, "\n");
615 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
616 if (qemu_log_enabled()) {
617 qemu_log("qemu: fatal: ");
618 qemu_log_vprintf(fmt, ap2);
619 qemu_log("\n");
620 log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
621 qemu_log_flush();
622 qemu_log_close();
623 }
624 va_end(ap2);
625 va_end(ap);
626 #if defined(CONFIG_USER_ONLY)
627 {
628 struct sigaction act;
629 sigfillset(&act.sa_mask);
630 act.sa_handler = SIG_DFL;
631 sigaction(SIGABRT, &act, NULL);
632 }
633 #endif
634 abort();
635 }
636
637 CPUArchState *cpu_copy(CPUArchState *env)
638 {
639 CPUArchState *new_env = cpu_init(env->cpu_model_str);
640 CPUArchState *next_cpu = new_env->next_cpu;
641 #if defined(TARGET_HAS_ICE)
642 CPUBreakpoint *bp;
643 CPUWatchpoint *wp;
644 #endif
645
646 memcpy(new_env, env, sizeof(CPUArchState));
647
648 /* Preserve chaining. */
649 new_env->next_cpu = next_cpu;
650
651 /* Clone all break/watchpoints.
652 Note: Once we support ptrace with hw-debug register access, make sure
653 BP_CPU break/watchpoints are handled correctly on clone. */
654 QTAILQ_INIT(&env->breakpoints);
655 QTAILQ_INIT(&env->watchpoints);
656 #if defined(TARGET_HAS_ICE)
657 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
658 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
659 }
660 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
661 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
662 wp->flags, NULL);
663 }
664 #endif
665
666 return new_env;
667 }
668
669 #if !defined(CONFIG_USER_ONLY)
670 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
671 uintptr_t length)
672 {
673 uintptr_t start1;
674
675 /* we modify the TLB cache so that the dirty bit will be set again
676 when accessing the range */
677 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
678 /* Check that we don't span multiple blocks - this breaks the
679 address comparisons below. */
680 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
681 != (end - 1) - start) {
682 abort();
683 }
684 cpu_tlb_reset_dirty_all(start1, length);
685
686 }
687
688 /* Note: start and end must be within the same ram block. */
689 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
690 int dirty_flags)
691 {
692 uintptr_t length;
693
694 start &= TARGET_PAGE_MASK;
695 end = TARGET_PAGE_ALIGN(end);
696
697 length = end - start;
698 if (length == 0)
699 return;
700 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
701
702 if (tcg_enabled()) {
703 tlb_reset_dirty_range_all(start, end, length);
704 }
705 }
706
707 static int cpu_physical_memory_set_dirty_tracking(int enable)
708 {
709 int ret = 0;
710 in_migration = enable;
711 return ret;
712 }
713
714 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
715 MemoryRegionSection *section,
716 target_ulong vaddr,
717 hwaddr paddr, hwaddr xlat,
718 int prot,
719 target_ulong *address)
720 {
721 hwaddr iotlb;
722 CPUWatchpoint *wp;
723
724 if (memory_region_is_ram(section->mr)) {
725 /* Normal RAM. */
726 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
727 + xlat;
728 if (!section->readonly) {
729 iotlb |= PHYS_SECTION_NOTDIRTY;
730 } else {
731 iotlb |= PHYS_SECTION_ROM;
732 }
733 } else {
734 iotlb = section - cur_map.sections;
735 iotlb += xlat;
736 }
737
738 /* Make accesses to pages with watchpoints go via the
739 watchpoint trap routines. */
740 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
741 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
742 /* Avoid trapping reads of pages with a write breakpoint. */
743 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
744 iotlb = PHYS_SECTION_WATCH + paddr;
745 *address |= TLB_MMIO;
746 break;
747 }
748 }
749 }
750
751 return iotlb;
752 }
753 #endif /* defined(CONFIG_USER_ONLY) */
754
755 #if !defined(CONFIG_USER_ONLY)
756
757 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
758 uint16_t section);
759 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
760
761 static uint16_t phys_section_add(MemoryRegionSection *section)
762 {
763 /* The physical section number is ORed with a page-aligned
764 * pointer to produce the iotlb entries. Thus it should
765 * never overflow into the page-aligned value.
766 */
767 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
768
769 if (next_map.sections_nb == next_map.sections_nb_alloc) {
770 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
771 16);
772 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
773 next_map.sections_nb_alloc);
774 }
775 next_map.sections[next_map.sections_nb] = *section;
776 memory_region_ref(section->mr);
777 return next_map.sections_nb++;
778 }
779
780 static void phys_section_destroy(MemoryRegion *mr)
781 {
782 memory_region_unref(mr);
783
784 if (mr->subpage) {
785 subpage_t *subpage = container_of(mr, subpage_t, iomem);
786 memory_region_destroy(&subpage->iomem);
787 g_free(subpage);
788 }
789 }
790
791 static void phys_sections_clear(PhysPageMap *map)
792 {
793 while (map->sections_nb > 0) {
794 MemoryRegionSection *section = &map->sections[--map->sections_nb];
795 phys_section_destroy(section->mr);
796 }
797 g_free(map->sections);
798 g_free(map->nodes);
799 }
800
801 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
802 {
803 subpage_t *subpage;
804 hwaddr base = section->offset_within_address_space
805 & TARGET_PAGE_MASK;
806 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
807 next_map.nodes, next_map.sections);
808 MemoryRegionSection subsection = {
809 .offset_within_address_space = base,
810 .size = int128_make64(TARGET_PAGE_SIZE),
811 };
812 hwaddr start, end;
813
814 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
815
816 if (!(existing->mr->subpage)) {
817 subpage = subpage_init(d->as, base);
818 subsection.mr = &subpage->iomem;
819 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
820 phys_section_add(&subsection));
821 } else {
822 subpage = container_of(existing->mr, subpage_t, iomem);
823 }
824 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
825 end = start + int128_get64(section->size) - 1;
826 subpage_register(subpage, start, end, phys_section_add(section));
827 }
828
829
830 static void register_multipage(AddressSpaceDispatch *d,
831 MemoryRegionSection *section)
832 {
833 hwaddr start_addr = section->offset_within_address_space;
834 uint16_t section_index = phys_section_add(section);
835 uint64_t num_pages = int128_get64(int128_rshift(section->size,
836 TARGET_PAGE_BITS));
837
838 assert(num_pages);
839 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
840 }
841
842 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
843 {
844 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
845 MemoryRegionSection now = *section, remain = *section;
846 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
847
848 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
849 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
850 - now.offset_within_address_space;
851
852 now.size = int128_min(int128_make64(left), now.size);
853 register_subpage(d, &now);
854 } else {
855 now.size = int128_zero();
856 }
857 while (int128_ne(remain.size, now.size)) {
858 remain.size = int128_sub(remain.size, now.size);
859 remain.offset_within_address_space += int128_get64(now.size);
860 remain.offset_within_region += int128_get64(now.size);
861 now = remain;
862 if (int128_lt(remain.size, page_size)) {
863 register_subpage(d, &now);
864 } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
865 now.size = page_size;
866 register_subpage(d, &now);
867 } else {
868 now.size = int128_and(now.size, int128_neg(page_size));
869 register_multipage(d, &now);
870 }
871 }
872 }
873
874 void qemu_flush_coalesced_mmio_buffer(void)
875 {
876 if (kvm_enabled())
877 kvm_flush_coalesced_mmio_buffer();
878 }
879
880 void qemu_mutex_lock_ramlist(void)
881 {
882 qemu_mutex_lock(&ram_list.mutex);
883 }
884
885 void qemu_mutex_unlock_ramlist(void)
886 {
887 qemu_mutex_unlock(&ram_list.mutex);
888 }
889
890 #if defined(__linux__) && !defined(TARGET_S390X)
891
892 #include <sys/vfs.h>
893
894 #define HUGETLBFS_MAGIC 0x958458f6
895
896 static long gethugepagesize(const char *path)
897 {
898 struct statfs fs;
899 int ret;
900
901 do {
902 ret = statfs(path, &fs);
903 } while (ret != 0 && errno == EINTR);
904
905 if (ret != 0) {
906 perror(path);
907 return 0;
908 }
909
910 if (fs.f_type != HUGETLBFS_MAGIC)
911 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
912
913 return fs.f_bsize;
914 }
915
916 static void *file_ram_alloc(RAMBlock *block,
917 ram_addr_t memory,
918 const char *path)
919 {
920 char *filename;
921 char *sanitized_name;
922 char *c;
923 void *area;
924 int fd;
925 #ifdef MAP_POPULATE
926 int flags;
927 #endif
928 unsigned long hpagesize;
929
930 hpagesize = gethugepagesize(path);
931 if (!hpagesize) {
932 return NULL;
933 }
934
935 if (memory < hpagesize) {
936 return NULL;
937 }
938
939 if (kvm_enabled() && !kvm_has_sync_mmu()) {
940 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
941 return NULL;
942 }
943
944 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
945 sanitized_name = g_strdup(block->mr->name);
946 for (c = sanitized_name; *c != '\0'; c++) {
947 if (*c == '/')
948 *c = '_';
949 }
950
951 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
952 sanitized_name);
953 g_free(sanitized_name);
954
955 fd = mkstemp(filename);
956 if (fd < 0) {
957 perror("unable to create backing store for hugepages");
958 g_free(filename);
959 return NULL;
960 }
961 unlink(filename);
962 g_free(filename);
963
964 memory = (memory+hpagesize-1) & ~(hpagesize-1);
965
966 /*
967 * ftruncate is not supported by hugetlbfs in older
968 * hosts, so don't bother bailing out on errors.
969 * If anything goes wrong with it under other filesystems,
970 * mmap will fail.
971 */
972 if (ftruncate(fd, memory))
973 perror("ftruncate");
974
975 #ifdef MAP_POPULATE
976 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
977 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
978 * to sidestep this quirk.
979 */
980 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
981 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
982 #else
983 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
984 #endif
985 if (area == MAP_FAILED) {
986 perror("file_ram_alloc: can't mmap RAM pages");
987 close(fd);
988 return (NULL);
989 }
990 block->fd = fd;
991 return area;
992 }
993 #endif
994
995 static ram_addr_t find_ram_offset(ram_addr_t size)
996 {
997 RAMBlock *block, *next_block;
998 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
999
1000 assert(size != 0); /* it would hand out same offset multiple times */
1001
1002 if (QTAILQ_EMPTY(&ram_list.blocks))
1003 return 0;
1004
1005 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1006 ram_addr_t end, next = RAM_ADDR_MAX;
1007
1008 end = block->offset + block->length;
1009
1010 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1011 if (next_block->offset >= end) {
1012 next = MIN(next, next_block->offset);
1013 }
1014 }
1015 if (next - end >= size && next - end < mingap) {
1016 offset = end;
1017 mingap = next - end;
1018 }
1019 }
1020
1021 if (offset == RAM_ADDR_MAX) {
1022 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1023 (uint64_t)size);
1024 abort();
1025 }
1026
1027 return offset;
1028 }
1029
1030 ram_addr_t last_ram_offset(void)
1031 {
1032 RAMBlock *block;
1033 ram_addr_t last = 0;
1034
1035 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1036 last = MAX(last, block->offset + block->length);
1037
1038 return last;
1039 }
1040
1041 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1042 {
1043 int ret;
1044 QemuOpts *machine_opts;
1045
1046 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1047 machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1048 if (machine_opts &&
1049 !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
1050 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1051 if (ret) {
1052 perror("qemu_madvise");
1053 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1054 "but dump_guest_core=off specified\n");
1055 }
1056 }
1057 }
1058
1059 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1060 {
1061 RAMBlock *new_block, *block;
1062
1063 new_block = NULL;
1064 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1065 if (block->offset == addr) {
1066 new_block = block;
1067 break;
1068 }
1069 }
1070 assert(new_block);
1071 assert(!new_block->idstr[0]);
1072
1073 if (dev) {
1074 char *id = qdev_get_dev_path(dev);
1075 if (id) {
1076 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1077 g_free(id);
1078 }
1079 }
1080 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1081
1082 /* This assumes the iothread lock is taken here too. */
1083 qemu_mutex_lock_ramlist();
1084 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1085 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1086 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1087 new_block->idstr);
1088 abort();
1089 }
1090 }
1091 qemu_mutex_unlock_ramlist();
1092 }
1093
1094 static int memory_try_enable_merging(void *addr, size_t len)
1095 {
1096 QemuOpts *opts;
1097
1098 opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1099 if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
1100 /* disabled by the user */
1101 return 0;
1102 }
1103
1104 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1105 }
1106
1107 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1108 MemoryRegion *mr)
1109 {
1110 RAMBlock *block, *new_block;
1111
1112 size = TARGET_PAGE_ALIGN(size);
1113 new_block = g_malloc0(sizeof(*new_block));
1114
1115 /* This assumes the iothread lock is taken here too. */
1116 qemu_mutex_lock_ramlist();
1117 new_block->mr = mr;
1118 new_block->offset = find_ram_offset(size);
1119 if (host) {
1120 new_block->host = host;
1121 new_block->flags |= RAM_PREALLOC_MASK;
1122 } else {
1123 if (mem_path) {
1124 #if defined (__linux__) && !defined(TARGET_S390X)
1125 new_block->host = file_ram_alloc(new_block, size, mem_path);
1126 if (!new_block->host) {
1127 new_block->host = qemu_anon_ram_alloc(size);
1128 memory_try_enable_merging(new_block->host, size);
1129 }
1130 #else
1131 fprintf(stderr, "-mem-path option unsupported\n");
1132 exit(1);
1133 #endif
1134 } else {
1135 if (xen_enabled()) {
1136 xen_ram_alloc(new_block->offset, size, mr);
1137 } else if (kvm_enabled()) {
1138 /* some s390/kvm configurations have special constraints */
1139 new_block->host = kvm_ram_alloc(size);
1140 } else {
1141 new_block->host = qemu_anon_ram_alloc(size);
1142 }
1143 memory_try_enable_merging(new_block->host, size);
1144 }
1145 }
1146 new_block->length = size;
1147
1148 /* Keep the list sorted from biggest to smallest block. */
1149 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1150 if (block->length < new_block->length) {
1151 break;
1152 }
1153 }
1154 if (block) {
1155 QTAILQ_INSERT_BEFORE(block, new_block, next);
1156 } else {
1157 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1158 }
1159 ram_list.mru_block = NULL;
1160
1161 ram_list.version++;
1162 qemu_mutex_unlock_ramlist();
1163
1164 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1165 last_ram_offset() >> TARGET_PAGE_BITS);
1166 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1167 0, size >> TARGET_PAGE_BITS);
1168 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1169
1170 qemu_ram_setup_dump(new_block->host, size);
1171 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1172
1173 if (kvm_enabled())
1174 kvm_setup_guest_memory(new_block->host, size);
1175
1176 return new_block->offset;
1177 }
1178
1179 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1180 {
1181 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1182 }
1183
1184 void qemu_ram_free_from_ptr(ram_addr_t addr)
1185 {
1186 RAMBlock *block;
1187
1188 /* This assumes the iothread lock is taken here too. */
1189 qemu_mutex_lock_ramlist();
1190 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1191 if (addr == block->offset) {
1192 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1193 ram_list.mru_block = NULL;
1194 ram_list.version++;
1195 g_free(block);
1196 break;
1197 }
1198 }
1199 qemu_mutex_unlock_ramlist();
1200 }
1201
1202 void qemu_ram_free(ram_addr_t addr)
1203 {
1204 RAMBlock *block;
1205
1206 /* This assumes the iothread lock is taken here too. */
1207 qemu_mutex_lock_ramlist();
1208 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1209 if (addr == block->offset) {
1210 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1211 ram_list.mru_block = NULL;
1212 ram_list.version++;
1213 if (block->flags & RAM_PREALLOC_MASK) {
1214 ;
1215 } else if (mem_path) {
1216 #if defined (__linux__) && !defined(TARGET_S390X)
1217 if (block->fd) {
1218 munmap(block->host, block->length);
1219 close(block->fd);
1220 } else {
1221 qemu_anon_ram_free(block->host, block->length);
1222 }
1223 #else
1224 abort();
1225 #endif
1226 } else {
1227 if (xen_enabled()) {
1228 xen_invalidate_map_cache_entry(block->host);
1229 } else {
1230 qemu_anon_ram_free(block->host, block->length);
1231 }
1232 }
1233 g_free(block);
1234 break;
1235 }
1236 }
1237 qemu_mutex_unlock_ramlist();
1238
1239 }
1240
1241 #ifndef _WIN32
1242 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1243 {
1244 RAMBlock *block;
1245 ram_addr_t offset;
1246 int flags;
1247 void *area, *vaddr;
1248
1249 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1250 offset = addr - block->offset;
1251 if (offset < block->length) {
1252 vaddr = block->host + offset;
1253 if (block->flags & RAM_PREALLOC_MASK) {
1254 ;
1255 } else {
1256 flags = MAP_FIXED;
1257 munmap(vaddr, length);
1258 if (mem_path) {
1259 #if defined(__linux__) && !defined(TARGET_S390X)
1260 if (block->fd) {
1261 #ifdef MAP_POPULATE
1262 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1263 MAP_PRIVATE;
1264 #else
1265 flags |= MAP_PRIVATE;
1266 #endif
1267 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1268 flags, block->fd, offset);
1269 } else {
1270 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1271 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1272 flags, -1, 0);
1273 }
1274 #else
1275 abort();
1276 #endif
1277 } else {
1278 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1279 flags |= MAP_SHARED | MAP_ANONYMOUS;
1280 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1281 flags, -1, 0);
1282 #else
1283 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1284 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1285 flags, -1, 0);
1286 #endif
1287 }
1288 if (area != vaddr) {
1289 fprintf(stderr, "Could not remap addr: "
1290 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1291 length, addr);
1292 exit(1);
1293 }
1294 memory_try_enable_merging(vaddr, length);
1295 qemu_ram_setup_dump(vaddr, length);
1296 }
1297 return;
1298 }
1299 }
1300 }
1301 #endif /* !_WIN32 */
1302
1303 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1304 {
1305 RAMBlock *block;
1306
1307 /* The list is protected by the iothread lock here. */
1308 block = ram_list.mru_block;
1309 if (block && addr - block->offset < block->length) {
1310 goto found;
1311 }
1312 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1313 if (addr - block->offset < block->length) {
1314 goto found;
1315 }
1316 }
1317
1318 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1319 abort();
1320
1321 found:
1322 ram_list.mru_block = block;
1323 return block;
1324 }
1325
1326 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1327 With the exception of the softmmu code in this file, this should
1328 only be used for local memory (e.g. video ram) that the device owns,
1329 and knows it isn't going to access beyond the end of the block.
1330
1331 It should not be used for general purpose DMA.
1332 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1333 */
1334 void *qemu_get_ram_ptr(ram_addr_t addr)
1335 {
1336 RAMBlock *block = qemu_get_ram_block(addr);
1337
1338 if (xen_enabled()) {
1339 /* We need to check if the requested address is in the RAM
1340 * because we don't want to map the entire memory in QEMU.
1341 * In that case just map until the end of the page.
1342 */
1343 if (block->offset == 0) {
1344 return xen_map_cache(addr, 0, 0);
1345 } else if (block->host == NULL) {
1346 block->host =
1347 xen_map_cache(block->offset, block->length, 1);
1348 }
1349 }
1350 return block->host + (addr - block->offset);
1351 }
1352
1353 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1354 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1355 *
1356 * ??? Is this still necessary?
1357 */
1358 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1359 {
1360 RAMBlock *block;
1361
1362 /* The list is protected by the iothread lock here. */
1363 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1364 if (addr - block->offset < block->length) {
1365 if (xen_enabled()) {
1366 /* We need to check if the requested address is in the RAM
1367 * because we don't want to map the entire memory in QEMU.
1368 * In that case just map until the end of the page.
1369 */
1370 if (block->offset == 0) {
1371 return xen_map_cache(addr, 0, 0);
1372 } else if (block->host == NULL) {
1373 block->host =
1374 xen_map_cache(block->offset, block->length, 1);
1375 }
1376 }
1377 return block->host + (addr - block->offset);
1378 }
1379 }
1380
1381 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1382 abort();
1383
1384 return NULL;
1385 }
1386
1387 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1388 * but takes a size argument */
1389 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1390 {
1391 if (*size == 0) {
1392 return NULL;
1393 }
1394 if (xen_enabled()) {
1395 return xen_map_cache(addr, *size, 1);
1396 } else {
1397 RAMBlock *block;
1398
1399 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1400 if (addr - block->offset < block->length) {
1401 if (addr - block->offset + *size > block->length)
1402 *size = block->length - addr + block->offset;
1403 return block->host + (addr - block->offset);
1404 }
1405 }
1406
1407 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1408 abort();
1409 }
1410 }
1411
1412 /* Some of the softmmu routines need to translate from a host pointer
1413 (typically a TLB entry) back to a ram offset. */
1414 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1415 {
1416 RAMBlock *block;
1417 uint8_t *host = ptr;
1418
1419 if (xen_enabled()) {
1420 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1421 return qemu_get_ram_block(*ram_addr)->mr;
1422 }
1423
1424 block = ram_list.mru_block;
1425 if (block && block->host && host - block->host < block->length) {
1426 goto found;
1427 }
1428
1429 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1430 /* This case append when the block is not mapped. */
1431 if (block->host == NULL) {
1432 continue;
1433 }
1434 if (host - block->host < block->length) {
1435 goto found;
1436 }
1437 }
1438
1439 return NULL;
1440
1441 found:
1442 *ram_addr = block->offset + (host - block->host);
1443 return block->mr;
1444 }
1445
1446 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1447 uint64_t val, unsigned size)
1448 {
1449 int dirty_flags;
1450 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1451 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1452 tb_invalidate_phys_page_fast(ram_addr, size);
1453 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1454 }
1455 switch (size) {
1456 case 1:
1457 stb_p(qemu_get_ram_ptr(ram_addr), val);
1458 break;
1459 case 2:
1460 stw_p(qemu_get_ram_ptr(ram_addr), val);
1461 break;
1462 case 4:
1463 stl_p(qemu_get_ram_ptr(ram_addr), val);
1464 break;
1465 default:
1466 abort();
1467 }
1468 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1469 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1470 /* we remove the notdirty callback only if the code has been
1471 flushed */
1472 if (dirty_flags == 0xff)
1473 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1474 }
1475
1476 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1477 unsigned size, bool is_write)
1478 {
1479 return is_write;
1480 }
1481
1482 static const MemoryRegionOps notdirty_mem_ops = {
1483 .write = notdirty_mem_write,
1484 .valid.accepts = notdirty_mem_accepts,
1485 .endianness = DEVICE_NATIVE_ENDIAN,
1486 };
1487
1488 /* Generate a debug exception if a watchpoint has been hit. */
1489 static void check_watchpoint(int offset, int len_mask, int flags)
1490 {
1491 CPUArchState *env = cpu_single_env;
1492 target_ulong pc, cs_base;
1493 target_ulong vaddr;
1494 CPUWatchpoint *wp;
1495 int cpu_flags;
1496
1497 if (env->watchpoint_hit) {
1498 /* We re-entered the check after replacing the TB. Now raise
1499 * the debug interrupt so that is will trigger after the
1500 * current instruction. */
1501 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1502 return;
1503 }
1504 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1505 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1506 if ((vaddr == (wp->vaddr & len_mask) ||
1507 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1508 wp->flags |= BP_WATCHPOINT_HIT;
1509 if (!env->watchpoint_hit) {
1510 env->watchpoint_hit = wp;
1511 tb_check_watchpoint(env);
1512 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1513 env->exception_index = EXCP_DEBUG;
1514 cpu_loop_exit(env);
1515 } else {
1516 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1517 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1518 cpu_resume_from_signal(env, NULL);
1519 }
1520 }
1521 } else {
1522 wp->flags &= ~BP_WATCHPOINT_HIT;
1523 }
1524 }
1525 }
1526
1527 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1528 so these check for a hit then pass through to the normal out-of-line
1529 phys routines. */
1530 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1531 unsigned size)
1532 {
1533 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1534 switch (size) {
1535 case 1: return ldub_phys(addr);
1536 case 2: return lduw_phys(addr);
1537 case 4: return ldl_phys(addr);
1538 default: abort();
1539 }
1540 }
1541
1542 static void watch_mem_write(void *opaque, hwaddr addr,
1543 uint64_t val, unsigned size)
1544 {
1545 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1546 switch (size) {
1547 case 1:
1548 stb_phys(addr, val);
1549 break;
1550 case 2:
1551 stw_phys(addr, val);
1552 break;
1553 case 4:
1554 stl_phys(addr, val);
1555 break;
1556 default: abort();
1557 }
1558 }
1559
1560 static const MemoryRegionOps watch_mem_ops = {
1561 .read = watch_mem_read,
1562 .write = watch_mem_write,
1563 .endianness = DEVICE_NATIVE_ENDIAN,
1564 };
1565
1566 static uint64_t subpage_read(void *opaque, hwaddr addr,
1567 unsigned len)
1568 {
1569 subpage_t *subpage = opaque;
1570 uint8_t buf[4];
1571
1572 #if defined(DEBUG_SUBPAGE)
1573 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1574 subpage, len, addr);
1575 #endif
1576 address_space_read(subpage->as, addr + subpage->base, buf, len);
1577 switch (len) {
1578 case 1:
1579 return ldub_p(buf);
1580 case 2:
1581 return lduw_p(buf);
1582 case 4:
1583 return ldl_p(buf);
1584 default:
1585 abort();
1586 }
1587 }
1588
1589 static void subpage_write(void *opaque, hwaddr addr,
1590 uint64_t value, unsigned len)
1591 {
1592 subpage_t *subpage = opaque;
1593 uint8_t buf[4];
1594
1595 #if defined(DEBUG_SUBPAGE)
1596 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1597 " value %"PRIx64"\n",
1598 __func__, subpage, len, addr, value);
1599 #endif
1600 switch (len) {
1601 case 1:
1602 stb_p(buf, value);
1603 break;
1604 case 2:
1605 stw_p(buf, value);
1606 break;
1607 case 4:
1608 stl_p(buf, value);
1609 break;
1610 default:
1611 abort();
1612 }
1613 address_space_write(subpage->as, addr + subpage->base, buf, len);
1614 }
1615
1616 static bool subpage_accepts(void *opaque, hwaddr addr,
1617 unsigned size, bool is_write)
1618 {
1619 subpage_t *subpage = opaque;
1620 #if defined(DEBUG_SUBPAGE)
1621 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1622 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1623 #endif
1624
1625 return address_space_access_valid(subpage->as, addr + subpage->base,
1626 size, is_write);
1627 }
1628
1629 static const MemoryRegionOps subpage_ops = {
1630 .read = subpage_read,
1631 .write = subpage_write,
1632 .valid.accepts = subpage_accepts,
1633 .endianness = DEVICE_NATIVE_ENDIAN,
1634 };
1635
1636 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1637 uint16_t section)
1638 {
1639 int idx, eidx;
1640
1641 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1642 return -1;
1643 idx = SUBPAGE_IDX(start);
1644 eidx = SUBPAGE_IDX(end);
1645 #if defined(DEBUG_SUBPAGE)
1646 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1647 mmio, start, end, idx, eidx, memory);
1648 #endif
1649 for (; idx <= eidx; idx++) {
1650 mmio->sub_section[idx] = section;
1651 }
1652
1653 return 0;
1654 }
1655
1656 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1657 {
1658 subpage_t *mmio;
1659
1660 mmio = g_malloc0(sizeof(subpage_t));
1661
1662 mmio->as = as;
1663 mmio->base = base;
1664 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1665 "subpage", TARGET_PAGE_SIZE);
1666 mmio->iomem.subpage = true;
1667 #if defined(DEBUG_SUBPAGE)
1668 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1669 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1670 #endif
1671 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1672
1673 return mmio;
1674 }
1675
1676 static uint16_t dummy_section(MemoryRegion *mr)
1677 {
1678 MemoryRegionSection section = {
1679 .mr = mr,
1680 .offset_within_address_space = 0,
1681 .offset_within_region = 0,
1682 .size = int128_2_64(),
1683 };
1684
1685 return phys_section_add(&section);
1686 }
1687
1688 MemoryRegion *iotlb_to_region(hwaddr index)
1689 {
1690 return cur_map.sections[index & ~TARGET_PAGE_MASK].mr;
1691 }
1692
1693 static void io_mem_init(void)
1694 {
1695 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1696 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1697 "unassigned", UINT64_MAX);
1698 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1699 "notdirty", UINT64_MAX);
1700 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1701 "watch", UINT64_MAX);
1702 }
1703
1704 static void mem_begin(MemoryListener *listener)
1705 {
1706 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
1707
1708 d->phys_map.ptr = PHYS_MAP_NODE_NIL;
1709 }
1710
1711 static void core_begin(MemoryListener *listener)
1712 {
1713 uint16_t n;
1714
1715 memset(&next_map, 0, sizeof(next_map));
1716 n = dummy_section(&io_mem_unassigned);
1717 assert(n == PHYS_SECTION_UNASSIGNED);
1718 n = dummy_section(&io_mem_notdirty);
1719 assert(n == PHYS_SECTION_NOTDIRTY);
1720 n = dummy_section(&io_mem_rom);
1721 assert(n == PHYS_SECTION_ROM);
1722 n = dummy_section(&io_mem_watch);
1723 assert(n == PHYS_SECTION_WATCH);
1724 }
1725
1726 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1727 * All AddressSpaceDispatch instances have switched to the next map.
1728 */
1729 static void core_commit(MemoryListener *listener)
1730 {
1731 PhysPageMap info = cur_map;
1732 cur_map = next_map;
1733 phys_sections_clear(&info);
1734 }
1735
1736 static void tcg_commit(MemoryListener *listener)
1737 {
1738 CPUArchState *env;
1739
1740 /* since each CPU stores ram addresses in its TLB cache, we must
1741 reset the modified entries */
1742 /* XXX: slow ! */
1743 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1744 tlb_flush(env, 1);
1745 }
1746 }
1747
1748 static void core_log_global_start(MemoryListener *listener)
1749 {
1750 cpu_physical_memory_set_dirty_tracking(1);
1751 }
1752
1753 static void core_log_global_stop(MemoryListener *listener)
1754 {
1755 cpu_physical_memory_set_dirty_tracking(0);
1756 }
1757
1758 static MemoryListener core_memory_listener = {
1759 .begin = core_begin,
1760 .commit = core_commit,
1761 .log_global_start = core_log_global_start,
1762 .log_global_stop = core_log_global_stop,
1763 .priority = 1,
1764 };
1765
1766 static MemoryListener tcg_memory_listener = {
1767 .commit = tcg_commit,
1768 };
1769
1770 void address_space_init_dispatch(AddressSpace *as)
1771 {
1772 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1773
1774 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1775 d->listener = (MemoryListener) {
1776 .begin = mem_begin,
1777 .region_add = mem_add,
1778 .region_nop = mem_add,
1779 .priority = 0,
1780 };
1781 d->as = as;
1782 as->dispatch = d;
1783 memory_listener_register(&d->listener, as);
1784 }
1785
1786 void address_space_destroy_dispatch(AddressSpace *as)
1787 {
1788 AddressSpaceDispatch *d = as->dispatch;
1789
1790 memory_listener_unregister(&d->listener);
1791 g_free(d);
1792 as->dispatch = NULL;
1793 }
1794
1795 static void memory_map_init(void)
1796 {
1797 system_memory = g_malloc(sizeof(*system_memory));
1798 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1799 address_space_init(&address_space_memory, system_memory, "memory");
1800
1801 system_io = g_malloc(sizeof(*system_io));
1802 memory_region_init(system_io, NULL, "io", 65536);
1803 address_space_init(&address_space_io, system_io, "I/O");
1804
1805 memory_listener_register(&core_memory_listener, &address_space_memory);
1806 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1807 }
1808
1809 MemoryRegion *get_system_memory(void)
1810 {
1811 return system_memory;
1812 }
1813
1814 MemoryRegion *get_system_io(void)
1815 {
1816 return system_io;
1817 }
1818
1819 #endif /* !defined(CONFIG_USER_ONLY) */
1820
1821 /* physical memory access (slow version, mainly for debug) */
1822 #if defined(CONFIG_USER_ONLY)
1823 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1824 uint8_t *buf, int len, int is_write)
1825 {
1826 int l, flags;
1827 target_ulong page;
1828 void * p;
1829
1830 while (len > 0) {
1831 page = addr & TARGET_PAGE_MASK;
1832 l = (page + TARGET_PAGE_SIZE) - addr;
1833 if (l > len)
1834 l = len;
1835 flags = page_get_flags(page);
1836 if (!(flags & PAGE_VALID))
1837 return -1;
1838 if (is_write) {
1839 if (!(flags & PAGE_WRITE))
1840 return -1;
1841 /* XXX: this code should not depend on lock_user */
1842 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1843 return -1;
1844 memcpy(p, buf, l);
1845 unlock_user(p, addr, l);
1846 } else {
1847 if (!(flags & PAGE_READ))
1848 return -1;
1849 /* XXX: this code should not depend on lock_user */
1850 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1851 return -1;
1852 memcpy(buf, p, l);
1853 unlock_user(p, addr, 0);
1854 }
1855 len -= l;
1856 buf += l;
1857 addr += l;
1858 }
1859 return 0;
1860 }
1861
1862 #else
1863
1864 static void invalidate_and_set_dirty(hwaddr addr,
1865 hwaddr length)
1866 {
1867 if (!cpu_physical_memory_is_dirty(addr)) {
1868 /* invalidate code */
1869 tb_invalidate_phys_page_range(addr, addr + length, 0);
1870 /* set dirty bit */
1871 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1872 }
1873 xen_modified_memory(addr, length);
1874 }
1875
1876 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1877 {
1878 if (memory_region_is_ram(mr)) {
1879 return !(is_write && mr->readonly);
1880 }
1881 if (memory_region_is_romd(mr)) {
1882 return !is_write;
1883 }
1884
1885 return false;
1886 }
1887
1888 static inline int memory_access_size(MemoryRegion *mr, int l, hwaddr addr)
1889 {
1890 if (l >= 4 && (((addr & 3) == 0 || mr->ops->impl.unaligned))) {
1891 return 4;
1892 }
1893 if (l >= 2 && (((addr & 1) == 0) || mr->ops->impl.unaligned)) {
1894 return 2;
1895 }
1896 return 1;
1897 }
1898
1899 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1900 int len, bool is_write)
1901 {
1902 hwaddr l;
1903 uint8_t *ptr;
1904 uint64_t val;
1905 hwaddr addr1;
1906 MemoryRegion *mr;
1907 bool error = false;
1908
1909 while (len > 0) {
1910 l = len;
1911 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1912
1913 if (is_write) {
1914 if (!memory_access_is_direct(mr, is_write)) {
1915 l = memory_access_size(mr, l, addr1);
1916 /* XXX: could force cpu_single_env to NULL to avoid
1917 potential bugs */
1918 if (l == 4) {
1919 /* 32 bit write access */
1920 val = ldl_p(buf);
1921 error |= io_mem_write(mr, addr1, val, 4);
1922 } else if (l == 2) {
1923 /* 16 bit write access */
1924 val = lduw_p(buf);
1925 error |= io_mem_write(mr, addr1, val, 2);
1926 } else {
1927 /* 8 bit write access */
1928 val = ldub_p(buf);
1929 error |= io_mem_write(mr, addr1, val, 1);
1930 }
1931 } else {
1932 addr1 += memory_region_get_ram_addr(mr);
1933 /* RAM case */
1934 ptr = qemu_get_ram_ptr(addr1);
1935 memcpy(ptr, buf, l);
1936 invalidate_and_set_dirty(addr1, l);
1937 }
1938 } else {
1939 if (!memory_access_is_direct(mr, is_write)) {
1940 /* I/O case */
1941 l = memory_access_size(mr, l, addr1);
1942 if (l == 4) {
1943 /* 32 bit read access */
1944 error |= io_mem_read(mr, addr1, &val, 4);
1945 stl_p(buf, val);
1946 } else if (l == 2) {
1947 /* 16 bit read access */
1948 error |= io_mem_read(mr, addr1, &val, 2);
1949 stw_p(buf, val);
1950 } else {
1951 /* 8 bit read access */
1952 error |= io_mem_read(mr, addr1, &val, 1);
1953 stb_p(buf, val);
1954 }
1955 } else {
1956 /* RAM case */
1957 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1958 memcpy(buf, ptr, l);
1959 }
1960 }
1961 len -= l;
1962 buf += l;
1963 addr += l;
1964 }
1965
1966 return error;
1967 }
1968
1969 bool address_space_write(AddressSpace *as, hwaddr addr,
1970 const uint8_t *buf, int len)
1971 {
1972 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
1973 }
1974
1975 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1976 {
1977 return address_space_rw(as, addr, buf, len, false);
1978 }
1979
1980
1981 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1982 int len, int is_write)
1983 {
1984 address_space_rw(&address_space_memory, addr, buf, len, is_write);
1985 }
1986
1987 /* used for ROM loading : can write in RAM and ROM */
1988 void cpu_physical_memory_write_rom(hwaddr addr,
1989 const uint8_t *buf, int len)
1990 {
1991 hwaddr l;
1992 uint8_t *ptr;
1993 hwaddr addr1;
1994 MemoryRegion *mr;
1995
1996 while (len > 0) {
1997 l = len;
1998 mr = address_space_translate(&address_space_memory,
1999 addr, &addr1, &l, true);
2000
2001 if (!(memory_region_is_ram(mr) ||
2002 memory_region_is_romd(mr))) {
2003 /* do nothing */
2004 } else {
2005 addr1 += memory_region_get_ram_addr(mr);
2006 /* ROM/RAM case */
2007 ptr = qemu_get_ram_ptr(addr1);
2008 memcpy(ptr, buf, l);
2009 invalidate_and_set_dirty(addr1, l);
2010 }
2011 len -= l;
2012 buf += l;
2013 addr += l;
2014 }
2015 }
2016
2017 typedef struct {
2018 MemoryRegion *mr;
2019 void *buffer;
2020 hwaddr addr;
2021 hwaddr len;
2022 } BounceBuffer;
2023
2024 static BounceBuffer bounce;
2025
2026 typedef struct MapClient {
2027 void *opaque;
2028 void (*callback)(void *opaque);
2029 QLIST_ENTRY(MapClient) link;
2030 } MapClient;
2031
2032 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2033 = QLIST_HEAD_INITIALIZER(map_client_list);
2034
2035 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2036 {
2037 MapClient *client = g_malloc(sizeof(*client));
2038
2039 client->opaque = opaque;
2040 client->callback = callback;
2041 QLIST_INSERT_HEAD(&map_client_list, client, link);
2042 return client;
2043 }
2044
2045 static void cpu_unregister_map_client(void *_client)
2046 {
2047 MapClient *client = (MapClient *)_client;
2048
2049 QLIST_REMOVE(client, link);
2050 g_free(client);
2051 }
2052
2053 static void cpu_notify_map_clients(void)
2054 {
2055 MapClient *client;
2056
2057 while (!QLIST_EMPTY(&map_client_list)) {
2058 client = QLIST_FIRST(&map_client_list);
2059 client->callback(client->opaque);
2060 cpu_unregister_map_client(client);
2061 }
2062 }
2063
2064 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2065 {
2066 MemoryRegion *mr;
2067 hwaddr l, xlat;
2068
2069 while (len > 0) {
2070 l = len;
2071 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2072 if (!memory_access_is_direct(mr, is_write)) {
2073 l = memory_access_size(mr, l, addr);
2074 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2075 return false;
2076 }
2077 }
2078
2079 len -= l;
2080 addr += l;
2081 }
2082 return true;
2083 }
2084
2085 /* Map a physical memory region into a host virtual address.
2086 * May map a subset of the requested range, given by and returned in *plen.
2087 * May return NULL if resources needed to perform the mapping are exhausted.
2088 * Use only for reads OR writes - not for read-modify-write operations.
2089 * Use cpu_register_map_client() to know when retrying the map operation is
2090 * likely to succeed.
2091 */
2092 void *address_space_map(AddressSpace *as,
2093 hwaddr addr,
2094 hwaddr *plen,
2095 bool is_write)
2096 {
2097 hwaddr len = *plen;
2098 hwaddr done = 0;
2099 hwaddr l, xlat, base;
2100 MemoryRegion *mr, *this_mr;
2101 ram_addr_t raddr;
2102
2103 if (len == 0) {
2104 return NULL;
2105 }
2106
2107 l = len;
2108 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2109 if (!memory_access_is_direct(mr, is_write)) {
2110 if (bounce.buffer) {
2111 return NULL;
2112 }
2113 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2114 bounce.addr = addr;
2115 bounce.len = l;
2116
2117 memory_region_ref(mr);
2118 bounce.mr = mr;
2119 if (!is_write) {
2120 address_space_read(as, addr, bounce.buffer, l);
2121 }
2122
2123 *plen = l;
2124 return bounce.buffer;
2125 }
2126
2127 base = xlat;
2128 raddr = memory_region_get_ram_addr(mr);
2129
2130 for (;;) {
2131 len -= l;
2132 addr += l;
2133 done += l;
2134 if (len == 0) {
2135 break;
2136 }
2137
2138 l = len;
2139 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2140 if (this_mr != mr || xlat != base + done) {
2141 break;
2142 }
2143 }
2144
2145 memory_region_ref(mr);
2146 *plen = done;
2147 return qemu_ram_ptr_length(raddr + base, plen);
2148 }
2149
2150 /* Unmaps a memory region previously mapped by address_space_map().
2151 * Will also mark the memory as dirty if is_write == 1. access_len gives
2152 * the amount of memory that was actually read or written by the caller.
2153 */
2154 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2155 int is_write, hwaddr access_len)
2156 {
2157 if (buffer != bounce.buffer) {
2158 MemoryRegion *mr;
2159 ram_addr_t addr1;
2160
2161 mr = qemu_ram_addr_from_host(buffer, &addr1);
2162 assert(mr != NULL);
2163 if (is_write) {
2164 while (access_len) {
2165 unsigned l;
2166 l = TARGET_PAGE_SIZE;
2167 if (l > access_len)
2168 l = access_len;
2169 invalidate_and_set_dirty(addr1, l);
2170 addr1 += l;
2171 access_len -= l;
2172 }
2173 }
2174 if (xen_enabled()) {
2175 xen_invalidate_map_cache_entry(buffer);
2176 }
2177 memory_region_unref(mr);
2178 return;
2179 }
2180 if (is_write) {
2181 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2182 }
2183 qemu_vfree(bounce.buffer);
2184 bounce.buffer = NULL;
2185 memory_region_unref(bounce.mr);
2186 cpu_notify_map_clients();
2187 }
2188
2189 void *cpu_physical_memory_map(hwaddr addr,
2190 hwaddr *plen,
2191 int is_write)
2192 {
2193 return address_space_map(&address_space_memory, addr, plen, is_write);
2194 }
2195
2196 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2197 int is_write, hwaddr access_len)
2198 {
2199 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2200 }
2201
2202 /* warning: addr must be aligned */
2203 static inline uint32_t ldl_phys_internal(hwaddr addr,
2204 enum device_endian endian)
2205 {
2206 uint8_t *ptr;
2207 uint64_t val;
2208 MemoryRegion *mr;
2209 hwaddr l = 4;
2210 hwaddr addr1;
2211
2212 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2213 false);
2214 if (l < 4 || !memory_access_is_direct(mr, false)) {
2215 /* I/O case */
2216 io_mem_read(mr, addr1, &val, 4);
2217 #if defined(TARGET_WORDS_BIGENDIAN)
2218 if (endian == DEVICE_LITTLE_ENDIAN) {
2219 val = bswap32(val);
2220 }
2221 #else
2222 if (endian == DEVICE_BIG_ENDIAN) {
2223 val = bswap32(val);
2224 }
2225 #endif
2226 } else {
2227 /* RAM case */
2228 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2229 & TARGET_PAGE_MASK)
2230 + addr1);
2231 switch (endian) {
2232 case DEVICE_LITTLE_ENDIAN:
2233 val = ldl_le_p(ptr);
2234 break;
2235 case DEVICE_BIG_ENDIAN:
2236 val = ldl_be_p(ptr);
2237 break;
2238 default:
2239 val = ldl_p(ptr);
2240 break;
2241 }
2242 }
2243 return val;
2244 }
2245
2246 uint32_t ldl_phys(hwaddr addr)
2247 {
2248 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2249 }
2250
2251 uint32_t ldl_le_phys(hwaddr addr)
2252 {
2253 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2254 }
2255
2256 uint32_t ldl_be_phys(hwaddr addr)
2257 {
2258 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2259 }
2260
2261 /* warning: addr must be aligned */
2262 static inline uint64_t ldq_phys_internal(hwaddr addr,
2263 enum device_endian endian)
2264 {
2265 uint8_t *ptr;
2266 uint64_t val;
2267 MemoryRegion *mr;
2268 hwaddr l = 8;
2269 hwaddr addr1;
2270
2271 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2272 false);
2273 if (l < 8 || !memory_access_is_direct(mr, false)) {
2274 /* I/O case */
2275 io_mem_read(mr, addr1, &val, 8);
2276 #if defined(TARGET_WORDS_BIGENDIAN)
2277 if (endian == DEVICE_LITTLE_ENDIAN) {
2278 val = bswap64(val);
2279 }
2280 #else
2281 if (endian == DEVICE_BIG_ENDIAN) {
2282 val = bswap64(val);
2283 }
2284 #endif
2285 } else {
2286 /* RAM case */
2287 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2288 & TARGET_PAGE_MASK)
2289 + addr1);
2290 switch (endian) {
2291 case DEVICE_LITTLE_ENDIAN:
2292 val = ldq_le_p(ptr);
2293 break;
2294 case DEVICE_BIG_ENDIAN:
2295 val = ldq_be_p(ptr);
2296 break;
2297 default:
2298 val = ldq_p(ptr);
2299 break;
2300 }
2301 }
2302 return val;
2303 }
2304
2305 uint64_t ldq_phys(hwaddr addr)
2306 {
2307 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2308 }
2309
2310 uint64_t ldq_le_phys(hwaddr addr)
2311 {
2312 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2313 }
2314
2315 uint64_t ldq_be_phys(hwaddr addr)
2316 {
2317 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2318 }
2319
2320 /* XXX: optimize */
2321 uint32_t ldub_phys(hwaddr addr)
2322 {
2323 uint8_t val;
2324 cpu_physical_memory_read(addr, &val, 1);
2325 return val;
2326 }
2327
2328 /* warning: addr must be aligned */
2329 static inline uint32_t lduw_phys_internal(hwaddr addr,
2330 enum device_endian endian)
2331 {
2332 uint8_t *ptr;
2333 uint64_t val;
2334 MemoryRegion *mr;
2335 hwaddr l = 2;
2336 hwaddr addr1;
2337
2338 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2339 false);
2340 if (l < 2 || !memory_access_is_direct(mr, false)) {
2341 /* I/O case */
2342 io_mem_read(mr, addr1, &val, 2);
2343 #if defined(TARGET_WORDS_BIGENDIAN)
2344 if (endian == DEVICE_LITTLE_ENDIAN) {
2345 val = bswap16(val);
2346 }
2347 #else
2348 if (endian == DEVICE_BIG_ENDIAN) {
2349 val = bswap16(val);
2350 }
2351 #endif
2352 } else {
2353 /* RAM case */
2354 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2355 & TARGET_PAGE_MASK)
2356 + addr1);
2357 switch (endian) {
2358 case DEVICE_LITTLE_ENDIAN:
2359 val = lduw_le_p(ptr);
2360 break;
2361 case DEVICE_BIG_ENDIAN:
2362 val = lduw_be_p(ptr);
2363 break;
2364 default:
2365 val = lduw_p(ptr);
2366 break;
2367 }
2368 }
2369 return val;
2370 }
2371
2372 uint32_t lduw_phys(hwaddr addr)
2373 {
2374 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2375 }
2376
2377 uint32_t lduw_le_phys(hwaddr addr)
2378 {
2379 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2380 }
2381
2382 uint32_t lduw_be_phys(hwaddr addr)
2383 {
2384 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2385 }
2386
2387 /* warning: addr must be aligned. The ram page is not masked as dirty
2388 and the code inside is not invalidated. It is useful if the dirty
2389 bits are used to track modified PTEs */
2390 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2391 {
2392 uint8_t *ptr;
2393 MemoryRegion *mr;
2394 hwaddr l = 4;
2395 hwaddr addr1;
2396
2397 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2398 true);
2399 if (l < 4 || !memory_access_is_direct(mr, true)) {
2400 io_mem_write(mr, addr1, val, 4);
2401 } else {
2402 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2403 ptr = qemu_get_ram_ptr(addr1);
2404 stl_p(ptr, val);
2405
2406 if (unlikely(in_migration)) {
2407 if (!cpu_physical_memory_is_dirty(addr1)) {
2408 /* invalidate code */
2409 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2410 /* set dirty bit */
2411 cpu_physical_memory_set_dirty_flags(
2412 addr1, (0xff & ~CODE_DIRTY_FLAG));
2413 }
2414 }
2415 }
2416 }
2417
2418 /* warning: addr must be aligned */
2419 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2420 enum device_endian endian)
2421 {
2422 uint8_t *ptr;
2423 MemoryRegion *mr;
2424 hwaddr l = 4;
2425 hwaddr addr1;
2426
2427 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2428 true);
2429 if (l < 4 || !memory_access_is_direct(mr, true)) {
2430 #if defined(TARGET_WORDS_BIGENDIAN)
2431 if (endian == DEVICE_LITTLE_ENDIAN) {
2432 val = bswap32(val);
2433 }
2434 #else
2435 if (endian == DEVICE_BIG_ENDIAN) {
2436 val = bswap32(val);
2437 }
2438 #endif
2439 io_mem_write(mr, addr1, val, 4);
2440 } else {
2441 /* RAM case */
2442 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2443 ptr = qemu_get_ram_ptr(addr1);
2444 switch (endian) {
2445 case DEVICE_LITTLE_ENDIAN:
2446 stl_le_p(ptr, val);
2447 break;
2448 case DEVICE_BIG_ENDIAN:
2449 stl_be_p(ptr, val);
2450 break;
2451 default:
2452 stl_p(ptr, val);
2453 break;
2454 }
2455 invalidate_and_set_dirty(addr1, 4);
2456 }
2457 }
2458
2459 void stl_phys(hwaddr addr, uint32_t val)
2460 {
2461 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2462 }
2463
2464 void stl_le_phys(hwaddr addr, uint32_t val)
2465 {
2466 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2467 }
2468
2469 void stl_be_phys(hwaddr addr, uint32_t val)
2470 {
2471 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2472 }
2473
2474 /* XXX: optimize */
2475 void stb_phys(hwaddr addr, uint32_t val)
2476 {
2477 uint8_t v = val;
2478 cpu_physical_memory_write(addr, &v, 1);
2479 }
2480
2481 /* warning: addr must be aligned */
2482 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2483 enum device_endian endian)
2484 {
2485 uint8_t *ptr;
2486 MemoryRegion *mr;
2487 hwaddr l = 2;
2488 hwaddr addr1;
2489
2490 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2491 true);
2492 if (l < 2 || !memory_access_is_direct(mr, true)) {
2493 #if defined(TARGET_WORDS_BIGENDIAN)
2494 if (endian == DEVICE_LITTLE_ENDIAN) {
2495 val = bswap16(val);
2496 }
2497 #else
2498 if (endian == DEVICE_BIG_ENDIAN) {
2499 val = bswap16(val);
2500 }
2501 #endif
2502 io_mem_write(mr, addr1, val, 2);
2503 } else {
2504 /* RAM case */
2505 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2506 ptr = qemu_get_ram_ptr(addr1);
2507 switch (endian) {
2508 case DEVICE_LITTLE_ENDIAN:
2509 stw_le_p(ptr, val);
2510 break;
2511 case DEVICE_BIG_ENDIAN:
2512 stw_be_p(ptr, val);
2513 break;
2514 default:
2515 stw_p(ptr, val);
2516 break;
2517 }
2518 invalidate_and_set_dirty(addr1, 2);
2519 }
2520 }
2521
2522 void stw_phys(hwaddr addr, uint32_t val)
2523 {
2524 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2525 }
2526
2527 void stw_le_phys(hwaddr addr, uint32_t val)
2528 {
2529 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2530 }
2531
2532 void stw_be_phys(hwaddr addr, uint32_t val)
2533 {
2534 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2535 }
2536
2537 /* XXX: optimize */
2538 void stq_phys(hwaddr addr, uint64_t val)
2539 {
2540 val = tswap64(val);
2541 cpu_physical_memory_write(addr, &val, 8);
2542 }
2543
2544 void stq_le_phys(hwaddr addr, uint64_t val)
2545 {
2546 val = cpu_to_le64(val);
2547 cpu_physical_memory_write(addr, &val, 8);
2548 }
2549
2550 void stq_be_phys(hwaddr addr, uint64_t val)
2551 {
2552 val = cpu_to_be64(val);
2553 cpu_physical_memory_write(addr, &val, 8);
2554 }
2555
2556 /* virtual memory access for debug (includes writing to ROM) */
2557 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2558 uint8_t *buf, int len, int is_write)
2559 {
2560 int l;
2561 hwaddr phys_addr;
2562 target_ulong page;
2563
2564 while (len > 0) {
2565 page = addr & TARGET_PAGE_MASK;
2566 phys_addr = cpu_get_phys_page_debug(env, page);
2567 /* if no physical page mapped, return an error */
2568 if (phys_addr == -1)
2569 return -1;
2570 l = (page + TARGET_PAGE_SIZE) - addr;
2571 if (l > len)
2572 l = len;
2573 phys_addr += (addr & ~TARGET_PAGE_MASK);
2574 if (is_write)
2575 cpu_physical_memory_write_rom(phys_addr, buf, l);
2576 else
2577 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2578 len -= l;
2579 buf += l;
2580 addr += l;
2581 }
2582 return 0;
2583 }
2584 #endif
2585
2586 #if !defined(CONFIG_USER_ONLY)
2587
2588 /*
2589 * A helper function for the _utterly broken_ virtio device model to find out if
2590 * it's running on a big endian machine. Don't do this at home kids!
2591 */
2592 bool virtio_is_big_endian(void);
2593 bool virtio_is_big_endian(void)
2594 {
2595 #if defined(TARGET_WORDS_BIGENDIAN)
2596 return true;
2597 #else
2598 return false;
2599 #endif
2600 }
2601
2602 #endif
2603
2604 #ifndef CONFIG_USER_ONLY
2605 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2606 {
2607 MemoryRegion*mr;
2608 hwaddr l = 1;
2609
2610 mr = address_space_translate(&address_space_memory,
2611 phys_addr, &phys_addr, &l, false);
2612
2613 return !(memory_region_is_ram(mr) ||
2614 memory_region_is_romd(mr));
2615 }
2616
2617 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2618 {
2619 RAMBlock *block;
2620
2621 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2622 func(block->host, block->offset, block->length, opaque);
2623 }
2624 }
2625 #endif
AR7 emulation for QEMU