?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
exec: change well-known physical sections to macros
[qemu/ar7.git] / exec.c
blobada2ff217a0dcb47ba8d8484314627efe215819e
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "hw/xen/xen.h"
35 #include "qemu/timer.h"
36 #include "qemu/config-file.h"
37 #include "exec/memory.h"
38 #include "sysemu/dma.h"
39 #include "exec/address-spaces.h"
40 #if defined(CONFIG_USER_ONLY)
41 #include <qemu.h>
42 #else /* !CONFIG_USER_ONLY */
43 #include "sysemu/xen-mapcache.h"
44 #include "trace.h"
45 #endif
46 #include "exec/cpu-all.h"
47
48 #include "exec/cputlb.h"
49 #include "translate-all.h"
50
51 #include "exec/memory-internal.h"
52
53 //#define DEBUG_SUBPAGE
54
55 #if !defined(CONFIG_USER_ONLY)
56 int phys_ram_fd;
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 CPUArchState *first_cpu;
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUArchState *,cpu_single_env);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 struct AddressSpaceDispatch {
92 /* This is a multi-level map on the physical address space.
93 * The bottom level has pointers to MemoryRegionSections.
94 */
95 PhysPageEntry phys_map;
96 MemoryListener listener;
97 AddressSpace *as;
98 };
99
100 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
101 typedef struct subpage_t {
102 MemoryRegion iomem;
103 AddressSpace *as;
104 hwaddr base;
105 uint16_t sub_section[TARGET_PAGE_SIZE];
106 } subpage_t;
107
108 static MemoryRegionSection *phys_sections;
109 static unsigned phys_sections_nb, phys_sections_nb_alloc;
110 #define PHYS_SECTION_UNASSIGNED 0
111 #define PHYS_SECTION_NOTDIRTY 1
112 #define PHYS_SECTION_ROM 2
113 #define PHYS_SECTION_WATCH 3
114
115 /* Simple allocator for PhysPageEntry nodes */
116 static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
117 static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
118
119 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
120
121 static void io_mem_init(void);
122 static void memory_map_init(void);
123 static void *qemu_safe_ram_ptr(ram_addr_t addr);
124
125 static MemoryRegion io_mem_watch;
126 #endif
127
128 #if !defined(CONFIG_USER_ONLY)
129
130 static void phys_map_node_reserve(unsigned nodes)
131 {
132 if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) {
133 typedef PhysPageEntry Node[L2_SIZE];
134 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16);
135 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc,
136 phys_map_nodes_nb + nodes);
137 phys_map_nodes = g_renew(Node, phys_map_nodes,
138 phys_map_nodes_nb_alloc);
139 }
140 }
141
142 static uint16_t phys_map_node_alloc(void)
143 {
144 unsigned i;
145 uint16_t ret;
146
147 ret = phys_map_nodes_nb++;
148 assert(ret != PHYS_MAP_NODE_NIL);
149 assert(ret != phys_map_nodes_nb_alloc);
150 for (i = 0; i < L2_SIZE; ++i) {
151 phys_map_nodes[ret][i].is_leaf = 0;
152 phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
153 }
154 return ret;
155 }
156
157 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
158 hwaddr *nb, uint16_t leaf,
159 int level)
160 {
161 PhysPageEntry *p;
162 int i;
163 hwaddr step = (hwaddr)1 << (level * L2_BITS);
164
165 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
166 lp->ptr = phys_map_node_alloc();
167 p = phys_map_nodes[lp->ptr];
168 if (level == 0) {
169 for (i = 0; i < L2_SIZE; i++) {
170 p[i].is_leaf = 1;
171 p[i].ptr = PHYS_SECTION_UNASSIGNED;
172 }
173 }
174 } else {
175 p = phys_map_nodes[lp->ptr];
176 }
177 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
178
179 while (*nb && lp < &p[L2_SIZE]) {
180 if ((*index & (step - 1)) == 0 && *nb >= step) {
181 lp->is_leaf = true;
182 lp->ptr = leaf;
183 *index += step;
184 *nb -= step;
185 } else {
186 phys_page_set_level(lp, index, nb, leaf, level - 1);
187 }
188 ++lp;
189 }
190 }
191
192 static void phys_page_set(AddressSpaceDispatch *d,
193 hwaddr index, hwaddr nb,
194 uint16_t leaf)
195 {
196 /* Wildly overreserve - it doesn't matter much. */
197 phys_map_node_reserve(3 * P_L2_LEVELS);
198
199 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
200 }
201
202 static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr index)
203 {
204 PhysPageEntry lp = d->phys_map;
205 PhysPageEntry *p;
206 int i;
207
208 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
209 if (lp.ptr == PHYS_MAP_NODE_NIL) {
210 return &phys_sections[PHYS_SECTION_UNASSIGNED];
211 }
212 p = phys_map_nodes[lp.ptr];
213 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
214 }
215 return &phys_sections[lp.ptr];
216 }
217
218 bool memory_region_is_unassigned(MemoryRegion *mr)
219 {
220 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
221 && mr != &io_mem_watch;
222 }
223
224 static MemoryRegionSection *address_space_lookup_region(AddressSpace *as,
225 hwaddr addr,
226 bool resolve_subpage)
227 {
228 MemoryRegionSection *section;
229 subpage_t *subpage;
230
231 section = phys_page_find(as->dispatch, addr >> TARGET_PAGE_BITS);
232 if (resolve_subpage && section->mr->subpage) {
233 subpage = container_of(section->mr, subpage_t, iomem);
234 section = &phys_sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
235 }
236 return section;
237 }
238
239 static MemoryRegionSection *
240 address_space_translate_internal(AddressSpace *as, hwaddr addr, hwaddr *xlat,
241 hwaddr *plen, bool resolve_subpage)
242 {
243 MemoryRegionSection *section;
244 Int128 diff;
245
246 section = address_space_lookup_region(as, addr, resolve_subpage);
247 /* Compute offset within MemoryRegionSection */
248 addr -= section->offset_within_address_space;
249
250 /* Compute offset within MemoryRegion */
251 *xlat = addr + section->offset_within_region;
252
253 diff = int128_sub(section->mr->size, int128_make64(addr));
254 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
255 return section;
256 }
257
258 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
259 hwaddr *xlat, hwaddr *plen,
260 bool is_write)
261 {
262 IOMMUTLBEntry iotlb;
263 MemoryRegionSection *section;
264 MemoryRegion *mr;
265 hwaddr len = *plen;
266
267 for (;;) {
268 section = address_space_translate_internal(as, addr, &addr, plen, true);
269 mr = section->mr;
270
271 if (!mr->iommu_ops) {
272 break;
273 }
274
275 iotlb = mr->iommu_ops->translate(mr, addr);
276 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
277 | (addr & iotlb.addr_mask));
278 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
279 if (!(iotlb.perm & (1 << is_write))) {
280 mr = &io_mem_unassigned;
281 break;
282 }
283
284 as = iotlb.target_as;
285 }
286
287 *plen = len;
288 *xlat = addr;
289 return mr;
290 }
291
292 MemoryRegionSection *
293 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
294 hwaddr *plen)
295 {
296 MemoryRegionSection *section;
297 section = address_space_translate_internal(as, addr, xlat, plen, false);
298
299 assert(!section->mr->iommu_ops);
300 return section;
301 }
302 #endif
303
304 void cpu_exec_init_all(void)
305 {
306 #if !defined(CONFIG_USER_ONLY)
307 qemu_mutex_init(&ram_list.mutex);
308 memory_map_init();
309 io_mem_init();
310 #endif
311 }
312
313 #if !defined(CONFIG_USER_ONLY)
314
315 static int cpu_common_post_load(void *opaque, int version_id)
316 {
317 CPUState *cpu = opaque;
318
319 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
320 version_id is increased. */
321 cpu->interrupt_request &= ~0x01;
322 tlb_flush(cpu->env_ptr, 1);
323
324 return 0;
325 }
326
327 const VMStateDescription vmstate_cpu_common = {
328 .name = "cpu_common",
329 .version_id = 1,
330 .minimum_version_id = 1,
331 .minimum_version_id_old = 1,
332 .post_load = cpu_common_post_load,
333 .fields = (VMStateField []) {
334 VMSTATE_UINT32(halted, CPUState),
335 VMSTATE_UINT32(interrupt_request, CPUState),
336 VMSTATE_END_OF_LIST()
337 }
338 };
339
340 #endif
341
342 CPUState *qemu_get_cpu(int index)
343 {
344 CPUArchState *env = first_cpu;
345 CPUState *cpu = NULL;
346
347 while (env) {
348 cpu = ENV_GET_CPU(env);
349 if (cpu->cpu_index == index) {
350 break;
351 }
352 env = env->next_cpu;
353 }
354
355 return env ? cpu : NULL;
356 }
357
358 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
359 {
360 CPUArchState *env = first_cpu;
361
362 while (env) {
363 func(ENV_GET_CPU(env), data);
364 env = env->next_cpu;
365 }
366 }
367
368 void cpu_exec_init(CPUArchState *env)
369 {
370 CPUState *cpu = ENV_GET_CPU(env);
371 CPUClass *cc = CPU_GET_CLASS(cpu);
372 CPUArchState **penv;
373 int cpu_index;
374
375 #if defined(CONFIG_USER_ONLY)
376 cpu_list_lock();
377 #endif
378 env->next_cpu = NULL;
379 penv = &first_cpu;
380 cpu_index = 0;
381 while (*penv != NULL) {
382 penv = &(*penv)->next_cpu;
383 cpu_index++;
384 }
385 cpu->cpu_index = cpu_index;
386 cpu->numa_node = 0;
387 QTAILQ_INIT(&env->breakpoints);
388 QTAILQ_INIT(&env->watchpoints);
389 #ifndef CONFIG_USER_ONLY
390 cpu->thread_id = qemu_get_thread_id();
391 #endif
392 *penv = env;
393 #if defined(CONFIG_USER_ONLY)
394 cpu_list_unlock();
395 #endif
396 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
397 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
398 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
399 cpu_save, cpu_load, env);
400 assert(cc->vmsd == NULL);
401 #endif
402 if (cc->vmsd != NULL) {
403 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
404 }
405 }
406
407 #if defined(TARGET_HAS_ICE)
408 #if defined(CONFIG_USER_ONLY)
409 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
410 {
411 tb_invalidate_phys_page_range(pc, pc + 1, 0);
412 }
413 #else
414 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
415 {
416 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
417 (pc & ~TARGET_PAGE_MASK));
418 }
419 #endif
420 #endif /* TARGET_HAS_ICE */
421
422 #if defined(CONFIG_USER_ONLY)
423 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
424
425 {
426 }
427
428 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
429 int flags, CPUWatchpoint **watchpoint)
430 {
431 return -ENOSYS;
432 }
433 #else
434 /* Add a watchpoint. */
435 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
436 int flags, CPUWatchpoint **watchpoint)
437 {
438 target_ulong len_mask = ~(len - 1);
439 CPUWatchpoint *wp;
440
441 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
442 if ((len & (len - 1)) || (addr & ~len_mask) ||
443 len == 0 || len > TARGET_PAGE_SIZE) {
444 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
445 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
446 return -EINVAL;
447 }
448 wp = g_malloc(sizeof(*wp));
449
450 wp->vaddr = addr;
451 wp->len_mask = len_mask;
452 wp->flags = flags;
453
454 /* keep all GDB-injected watchpoints in front */
455 if (flags & BP_GDB)
456 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
457 else
458 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
459
460 tlb_flush_page(env, addr);
461
462 if (watchpoint)
463 *watchpoint = wp;
464 return 0;
465 }
466
467 /* Remove a specific watchpoint. */
468 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
469 int flags)
470 {
471 target_ulong len_mask = ~(len - 1);
472 CPUWatchpoint *wp;
473
474 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
475 if (addr == wp->vaddr && len_mask == wp->len_mask
476 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
477 cpu_watchpoint_remove_by_ref(env, wp);
478 return 0;
479 }
480 }
481 return -ENOENT;
482 }
483
484 /* Remove a specific watchpoint by reference. */
485 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
486 {
487 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
488
489 tlb_flush_page(env, watchpoint->vaddr);
490
491 g_free(watchpoint);
492 }
493
494 /* Remove all matching watchpoints. */
495 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
496 {
497 CPUWatchpoint *wp, *next;
498
499 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
500 if (wp->flags & mask)
501 cpu_watchpoint_remove_by_ref(env, wp);
502 }
503 }
504 #endif
505
506 /* Add a breakpoint. */
507 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
508 CPUBreakpoint **breakpoint)
509 {
510 #if defined(TARGET_HAS_ICE)
511 CPUBreakpoint *bp;
512
513 bp = g_malloc(sizeof(*bp));
514
515 bp->pc = pc;
516 bp->flags = flags;
517
518 /* keep all GDB-injected breakpoints in front */
519 if (flags & BP_GDB)
520 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
521 else
522 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
523
524 breakpoint_invalidate(env, pc);
525
526 if (breakpoint)
527 *breakpoint = bp;
528 return 0;
529 #else
530 return -ENOSYS;
531 #endif
532 }
533
534 /* Remove a specific breakpoint. */
535 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
536 {
537 #if defined(TARGET_HAS_ICE)
538 CPUBreakpoint *bp;
539
540 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
541 if (bp->pc == pc && bp->flags == flags) {
542 cpu_breakpoint_remove_by_ref(env, bp);
543 return 0;
544 }
545 }
546 return -ENOENT;
547 #else
548 return -ENOSYS;
549 #endif
550 }
551
552 /* Remove a specific breakpoint by reference. */
553 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
554 {
555 #if defined(TARGET_HAS_ICE)
556 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
557
558 breakpoint_invalidate(env, breakpoint->pc);
559
560 g_free(breakpoint);
561 #endif
562 }
563
564 /* Remove all matching breakpoints. */
565 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
566 {
567 #if defined(TARGET_HAS_ICE)
568 CPUBreakpoint *bp, *next;
569
570 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
571 if (bp->flags & mask)
572 cpu_breakpoint_remove_by_ref(env, bp);
573 }
574 #endif
575 }
576
577 /* enable or disable single step mode. EXCP_DEBUG is returned by the
578 CPU loop after each instruction */
579 void cpu_single_step(CPUArchState *env, int enabled)
580 {
581 #if defined(TARGET_HAS_ICE)
582 if (env->singlestep_enabled != enabled) {
583 env->singlestep_enabled = enabled;
584 if (kvm_enabled())
585 kvm_update_guest_debug(env, 0);
586 else {
587 /* must flush all the translated code to avoid inconsistencies */
588 /* XXX: only flush what is necessary */
589 tb_flush(env);
590 }
591 }
592 #endif
593 }
594
595 void cpu_abort(CPUArchState *env, const char *fmt, ...)
596 {
597 CPUState *cpu = ENV_GET_CPU(env);
598 va_list ap;
599 va_list ap2;
600
601 va_start(ap, fmt);
602 va_copy(ap2, ap);
603 fprintf(stderr, "qemu: fatal: ");
604 vfprintf(stderr, fmt, ap);
605 fprintf(stderr, "\n");
606 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
607 if (qemu_log_enabled()) {
608 qemu_log("qemu: fatal: ");
609 qemu_log_vprintf(fmt, ap2);
610 qemu_log("\n");
611 log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
612 qemu_log_flush();
613 qemu_log_close();
614 }
615 va_end(ap2);
616 va_end(ap);
617 #if defined(CONFIG_USER_ONLY)
618 {
619 struct sigaction act;
620 sigfillset(&act.sa_mask);
621 act.sa_handler = SIG_DFL;
622 sigaction(SIGABRT, &act, NULL);
623 }
624 #endif
625 abort();
626 }
627
628 CPUArchState *cpu_copy(CPUArchState *env)
629 {
630 CPUArchState *new_env = cpu_init(env->cpu_model_str);
631 CPUArchState *next_cpu = new_env->next_cpu;
632 #if defined(TARGET_HAS_ICE)
633 CPUBreakpoint *bp;
634 CPUWatchpoint *wp;
635 #endif
636
637 memcpy(new_env, env, sizeof(CPUArchState));
638
639 /* Preserve chaining. */
640 new_env->next_cpu = next_cpu;
641
642 /* Clone all break/watchpoints.
643 Note: Once we support ptrace with hw-debug register access, make sure
644 BP_CPU break/watchpoints are handled correctly on clone. */
645 QTAILQ_INIT(&env->breakpoints);
646 QTAILQ_INIT(&env->watchpoints);
647 #if defined(TARGET_HAS_ICE)
648 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
649 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
650 }
651 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
652 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
653 wp->flags, NULL);
654 }
655 #endif
656
657 return new_env;
658 }
659
660 #if !defined(CONFIG_USER_ONLY)
661 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
662 uintptr_t length)
663 {
664 uintptr_t start1;
665
666 /* we modify the TLB cache so that the dirty bit will be set again
667 when accessing the range */
668 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
669 /* Check that we don't span multiple blocks - this breaks the
670 address comparisons below. */
671 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
672 != (end - 1) - start) {
673 abort();
674 }
675 cpu_tlb_reset_dirty_all(start1, length);
676
677 }
678
679 /* Note: start and end must be within the same ram block. */
680 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
681 int dirty_flags)
682 {
683 uintptr_t length;
684
685 start &= TARGET_PAGE_MASK;
686 end = TARGET_PAGE_ALIGN(end);
687
688 length = end - start;
689 if (length == 0)
690 return;
691 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
692
693 if (tcg_enabled()) {
694 tlb_reset_dirty_range_all(start, end, length);
695 }
696 }
697
698 static int cpu_physical_memory_set_dirty_tracking(int enable)
699 {
700 int ret = 0;
701 in_migration = enable;
702 return ret;
703 }
704
705 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
706 MemoryRegionSection *section,
707 target_ulong vaddr,
708 hwaddr paddr, hwaddr xlat,
709 int prot,
710 target_ulong *address)
711 {
712 hwaddr iotlb;
713 CPUWatchpoint *wp;
714
715 if (memory_region_is_ram(section->mr)) {
716 /* Normal RAM. */
717 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
718 + xlat;
719 if (!section->readonly) {
720 iotlb |= PHYS_SECTION_NOTDIRTY;
721 } else {
722 iotlb |= PHYS_SECTION_ROM;
723 }
724 } else {
725 iotlb = section - phys_sections;
726 iotlb += xlat;
727 }
728
729 /* Make accesses to pages with watchpoints go via the
730 watchpoint trap routines. */
731 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
732 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
733 /* Avoid trapping reads of pages with a write breakpoint. */
734 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
735 iotlb = PHYS_SECTION_WATCH + paddr;
736 *address |= TLB_MMIO;
737 break;
738 }
739 }
740 }
741
742 return iotlb;
743 }
744 #endif /* defined(CONFIG_USER_ONLY) */
745
746 #if !defined(CONFIG_USER_ONLY)
747
748 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
749 uint16_t section);
750 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
751
752 static uint16_t phys_section_add(MemoryRegionSection *section)
753 {
754 /* The physical section number is ORed with a page-aligned
755 * pointer to produce the iotlb entries. Thus it should
756 * never overflow into the page-aligned value.
757 */
758 assert(phys_sections_nb < TARGET_PAGE_SIZE);
759
760 if (phys_sections_nb == phys_sections_nb_alloc) {
761 phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16);
762 phys_sections = g_renew(MemoryRegionSection, phys_sections,
763 phys_sections_nb_alloc);
764 }
765 phys_sections[phys_sections_nb] = *section;
766 memory_region_ref(section->mr);
767 return phys_sections_nb++;
768 }
769
770 static void phys_section_destroy(MemoryRegion *mr)
771 {
772 memory_region_unref(mr);
773
774 if (mr->subpage) {
775 subpage_t *subpage = container_of(mr, subpage_t, iomem);
776 memory_region_destroy(&subpage->iomem);
777 g_free(subpage);
778 }
779 }
780
781 static void phys_sections_clear(void)
782 {
783 while (phys_sections_nb > 0) {
784 MemoryRegionSection *section = &phys_sections[--phys_sections_nb];
785 phys_section_destroy(section->mr);
786 }
787 phys_map_nodes_nb = 0;
788 }
789
790 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
791 {
792 subpage_t *subpage;
793 hwaddr base = section->offset_within_address_space
794 & TARGET_PAGE_MASK;
795 MemoryRegionSection *existing = phys_page_find(d, base >> TARGET_PAGE_BITS);
796 MemoryRegionSection subsection = {
797 .offset_within_address_space = base,
798 .size = int128_make64(TARGET_PAGE_SIZE),
799 };
800 hwaddr start, end;
801
802 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
803
804 if (!(existing->mr->subpage)) {
805 subpage = subpage_init(d->as, base);
806 subsection.mr = &subpage->iomem;
807 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
808 phys_section_add(&subsection));
809 } else {
810 subpage = container_of(existing->mr, subpage_t, iomem);
811 }
812 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
813 end = start + int128_get64(section->size) - 1;
814 subpage_register(subpage, start, end, phys_section_add(section));
815 }
816
817
818 static void register_multipage(AddressSpaceDispatch *d,
819 MemoryRegionSection *section)
820 {
821 hwaddr start_addr = section->offset_within_address_space;
822 uint16_t section_index = phys_section_add(section);
823 uint64_t num_pages = int128_get64(int128_rshift(section->size,
824 TARGET_PAGE_BITS));
825
826 assert(num_pages);
827 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
828 }
829
830 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
831 {
832 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
833 MemoryRegionSection now = *section, remain = *section;
834 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
835
836 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
837 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
838 - now.offset_within_address_space;
839
840 now.size = int128_min(int128_make64(left), now.size);
841 register_subpage(d, &now);
842 } else {
843 now.size = int128_zero();
844 }
845 while (int128_ne(remain.size, now.size)) {
846 remain.size = int128_sub(remain.size, now.size);
847 remain.offset_within_address_space += int128_get64(now.size);
848 remain.offset_within_region += int128_get64(now.size);
849 now = remain;
850 if (int128_lt(remain.size, page_size)) {
851 register_subpage(d, &now);
852 } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
853 now.size = page_size;
854 register_subpage(d, &now);
855 } else {
856 now.size = int128_and(now.size, int128_neg(page_size));
857 register_multipage(d, &now);
858 }
859 }
860 }
861
862 void qemu_flush_coalesced_mmio_buffer(void)
863 {
864 if (kvm_enabled())
865 kvm_flush_coalesced_mmio_buffer();
866 }
867
868 void qemu_mutex_lock_ramlist(void)
869 {
870 qemu_mutex_lock(&ram_list.mutex);
871 }
872
873 void qemu_mutex_unlock_ramlist(void)
874 {
875 qemu_mutex_unlock(&ram_list.mutex);
876 }
877
878 #if defined(__linux__) && !defined(TARGET_S390X)
879
880 #include <sys/vfs.h>
881
882 #define HUGETLBFS_MAGIC 0x958458f6
883
884 static long gethugepagesize(const char *path)
885 {
886 struct statfs fs;
887 int ret;
888
889 do {
890 ret = statfs(path, &fs);
891 } while (ret != 0 && errno == EINTR);
892
893 if (ret != 0) {
894 perror(path);
895 return 0;
896 }
897
898 if (fs.f_type != HUGETLBFS_MAGIC)
899 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
900
901 return fs.f_bsize;
902 }
903
904 static void *file_ram_alloc(RAMBlock *block,
905 ram_addr_t memory,
906 const char *path)
907 {
908 char *filename;
909 char *sanitized_name;
910 char *c;
911 void *area;
912 int fd;
913 #ifdef MAP_POPULATE
914 int flags;
915 #endif
916 unsigned long hpagesize;
917
918 hpagesize = gethugepagesize(path);
919 if (!hpagesize) {
920 return NULL;
921 }
922
923 if (memory < hpagesize) {
924 return NULL;
925 }
926
927 if (kvm_enabled() && !kvm_has_sync_mmu()) {
928 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
929 return NULL;
930 }
931
932 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
933 sanitized_name = g_strdup(block->mr->name);
934 for (c = sanitized_name; *c != '\0'; c++) {
935 if (*c == '/')
936 *c = '_';
937 }
938
939 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
940 sanitized_name);
941 g_free(sanitized_name);
942
943 fd = mkstemp(filename);
944 if (fd < 0) {
945 perror("unable to create backing store for hugepages");
946 g_free(filename);
947 return NULL;
948 }
949 unlink(filename);
950 g_free(filename);
951
952 memory = (memory+hpagesize-1) & ~(hpagesize-1);
953
954 /*
955 * ftruncate is not supported by hugetlbfs in older
956 * hosts, so don't bother bailing out on errors.
957 * If anything goes wrong with it under other filesystems,
958 * mmap will fail.
959 */
960 if (ftruncate(fd, memory))
961 perror("ftruncate");
962
963 #ifdef MAP_POPULATE
964 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
965 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
966 * to sidestep this quirk.
967 */
968 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
969 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
970 #else
971 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
972 #endif
973 if (area == MAP_FAILED) {
974 perror("file_ram_alloc: can't mmap RAM pages");
975 close(fd);
976 return (NULL);
977 }
978 block->fd = fd;
979 return area;
980 }
981 #endif
982
983 static ram_addr_t find_ram_offset(ram_addr_t size)
984 {
985 RAMBlock *block, *next_block;
986 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
987
988 assert(size != 0); /* it would hand out same offset multiple times */
989
990 if (QTAILQ_EMPTY(&ram_list.blocks))
991 return 0;
992
993 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
994 ram_addr_t end, next = RAM_ADDR_MAX;
995
996 end = block->offset + block->length;
997
998 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
999 if (next_block->offset >= end) {
1000 next = MIN(next, next_block->offset);
1001 }
1002 }
1003 if (next - end >= size && next - end < mingap) {
1004 offset = end;
1005 mingap = next - end;
1006 }
1007 }
1008
1009 if (offset == RAM_ADDR_MAX) {
1010 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1011 (uint64_t)size);
1012 abort();
1013 }
1014
1015 return offset;
1016 }
1017
1018 ram_addr_t last_ram_offset(void)
1019 {
1020 RAMBlock *block;
1021 ram_addr_t last = 0;
1022
1023 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1024 last = MAX(last, block->offset + block->length);
1025
1026 return last;
1027 }
1028
1029 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1030 {
1031 int ret;
1032 QemuOpts *machine_opts;
1033
1034 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1035 machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1036 if (machine_opts &&
1037 !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
1038 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1039 if (ret) {
1040 perror("qemu_madvise");
1041 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1042 "but dump_guest_core=off specified\n");
1043 }
1044 }
1045 }
1046
1047 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1048 {
1049 RAMBlock *new_block, *block;
1050
1051 new_block = NULL;
1052 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1053 if (block->offset == addr) {
1054 new_block = block;
1055 break;
1056 }
1057 }
1058 assert(new_block);
1059 assert(!new_block->idstr[0]);
1060
1061 if (dev) {
1062 char *id = qdev_get_dev_path(dev);
1063 if (id) {
1064 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1065 g_free(id);
1066 }
1067 }
1068 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1069
1070 /* This assumes the iothread lock is taken here too. */
1071 qemu_mutex_lock_ramlist();
1072 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1073 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1074 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1075 new_block->idstr);
1076 abort();
1077 }
1078 }
1079 qemu_mutex_unlock_ramlist();
1080 }
1081
1082 static int memory_try_enable_merging(void *addr, size_t len)
1083 {
1084 QemuOpts *opts;
1085
1086 opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1087 if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
1088 /* disabled by the user */
1089 return 0;
1090 }
1091
1092 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1093 }
1094
1095 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1096 MemoryRegion *mr)
1097 {
1098 RAMBlock *block, *new_block;
1099
1100 size = TARGET_PAGE_ALIGN(size);
1101 new_block = g_malloc0(sizeof(*new_block));
1102
1103 /* This assumes the iothread lock is taken here too. */
1104 qemu_mutex_lock_ramlist();
1105 new_block->mr = mr;
1106 new_block->offset = find_ram_offset(size);
1107 if (host) {
1108 new_block->host = host;
1109 new_block->flags |= RAM_PREALLOC_MASK;
1110 } else {
1111 if (mem_path) {
1112 #if defined (__linux__) && !defined(TARGET_S390X)
1113 new_block->host = file_ram_alloc(new_block, size, mem_path);
1114 if (!new_block->host) {
1115 new_block->host = qemu_anon_ram_alloc(size);
1116 memory_try_enable_merging(new_block->host, size);
1117 }
1118 #else
1119 fprintf(stderr, "-mem-path option unsupported\n");
1120 exit(1);
1121 #endif
1122 } else {
1123 if (xen_enabled()) {
1124 xen_ram_alloc(new_block->offset, size, mr);
1125 } else if (kvm_enabled()) {
1126 /* some s390/kvm configurations have special constraints */
1127 new_block->host = kvm_ram_alloc(size);
1128 } else {
1129 new_block->host = qemu_anon_ram_alloc(size);
1130 }
1131 memory_try_enable_merging(new_block->host, size);
1132 }
1133 }
1134 new_block->length = size;
1135
1136 /* Keep the list sorted from biggest to smallest block. */
1137 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1138 if (block->length < new_block->length) {
1139 break;
1140 }
1141 }
1142 if (block) {
1143 QTAILQ_INSERT_BEFORE(block, new_block, next);
1144 } else {
1145 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1146 }
1147 ram_list.mru_block = NULL;
1148
1149 ram_list.version++;
1150 qemu_mutex_unlock_ramlist();
1151
1152 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1153 last_ram_offset() >> TARGET_PAGE_BITS);
1154 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1155 0, size >> TARGET_PAGE_BITS);
1156 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1157
1158 qemu_ram_setup_dump(new_block->host, size);
1159 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1160
1161 if (kvm_enabled())
1162 kvm_setup_guest_memory(new_block->host, size);
1163
1164 return new_block->offset;
1165 }
1166
1167 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1168 {
1169 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1170 }
1171
1172 void qemu_ram_free_from_ptr(ram_addr_t addr)
1173 {
1174 RAMBlock *block;
1175
1176 /* This assumes the iothread lock is taken here too. */
1177 qemu_mutex_lock_ramlist();
1178 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1179 if (addr == block->offset) {
1180 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1181 ram_list.mru_block = NULL;
1182 ram_list.version++;
1183 g_free(block);
1184 break;
1185 }
1186 }
1187 qemu_mutex_unlock_ramlist();
1188 }
1189
1190 void qemu_ram_free(ram_addr_t addr)
1191 {
1192 RAMBlock *block;
1193
1194 /* This assumes the iothread lock is taken here too. */
1195 qemu_mutex_lock_ramlist();
1196 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1197 if (addr == block->offset) {
1198 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1199 ram_list.mru_block = NULL;
1200 ram_list.version++;
1201 if (block->flags & RAM_PREALLOC_MASK) {
1202 ;
1203 } else if (mem_path) {
1204 #if defined (__linux__) && !defined(TARGET_S390X)
1205 if (block->fd) {
1206 munmap(block->host, block->length);
1207 close(block->fd);
1208 } else {
1209 qemu_anon_ram_free(block->host, block->length);
1210 }
1211 #else
1212 abort();
1213 #endif
1214 } else {
1215 if (xen_enabled()) {
1216 xen_invalidate_map_cache_entry(block->host);
1217 } else {
1218 qemu_anon_ram_free(block->host, block->length);
1219 }
1220 }
1221 g_free(block);
1222 break;
1223 }
1224 }
1225 qemu_mutex_unlock_ramlist();
1226
1227 }
1228
1229 #ifndef _WIN32
1230 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1231 {
1232 RAMBlock *block;
1233 ram_addr_t offset;
1234 int flags;
1235 void *area, *vaddr;
1236
1237 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1238 offset = addr - block->offset;
1239 if (offset < block->length) {
1240 vaddr = block->host + offset;
1241 if (block->flags & RAM_PREALLOC_MASK) {
1242 ;
1243 } else {
1244 flags = MAP_FIXED;
1245 munmap(vaddr, length);
1246 if (mem_path) {
1247 #if defined(__linux__) && !defined(TARGET_S390X)
1248 if (block->fd) {
1249 #ifdef MAP_POPULATE
1250 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1251 MAP_PRIVATE;
1252 #else
1253 flags |= MAP_PRIVATE;
1254 #endif
1255 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1256 flags, block->fd, offset);
1257 } else {
1258 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1259 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1260 flags, -1, 0);
1261 }
1262 #else
1263 abort();
1264 #endif
1265 } else {
1266 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1267 flags |= MAP_SHARED | MAP_ANONYMOUS;
1268 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1269 flags, -1, 0);
1270 #else
1271 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1272 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1273 flags, -1, 0);
1274 #endif
1275 }
1276 if (area != vaddr) {
1277 fprintf(stderr, "Could not remap addr: "
1278 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1279 length, addr);
1280 exit(1);
1281 }
1282 memory_try_enable_merging(vaddr, length);
1283 qemu_ram_setup_dump(vaddr, length);
1284 }
1285 return;
1286 }
1287 }
1288 }
1289 #endif /* !_WIN32 */
1290
1291 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1292 {
1293 RAMBlock *block;
1294
1295 /* The list is protected by the iothread lock here. */
1296 block = ram_list.mru_block;
1297 if (block && addr - block->offset < block->length) {
1298 goto found;
1299 }
1300 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1301 if (addr - block->offset < block->length) {
1302 goto found;
1303 }
1304 }
1305
1306 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1307 abort();
1308
1309 found:
1310 ram_list.mru_block = block;
1311 return block;
1312 }
1313
1314 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1315 With the exception of the softmmu code in this file, this should
1316 only be used for local memory (e.g. video ram) that the device owns,
1317 and knows it isn't going to access beyond the end of the block.
1318
1319 It should not be used for general purpose DMA.
1320 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1321 */
1322 void *qemu_get_ram_ptr(ram_addr_t addr)
1323 {
1324 RAMBlock *block = qemu_get_ram_block(addr);
1325
1326 if (xen_enabled()) {
1327 /* We need to check if the requested address is in the RAM
1328 * because we don't want to map the entire memory in QEMU.
1329 * In that case just map until the end of the page.
1330 */
1331 if (block->offset == 0) {
1332 return xen_map_cache(addr, 0, 0);
1333 } else if (block->host == NULL) {
1334 block->host =
1335 xen_map_cache(block->offset, block->length, 1);
1336 }
1337 }
1338 return block->host + (addr - block->offset);
1339 }
1340
1341 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1342 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1343 *
1344 * ??? Is this still necessary?
1345 */
1346 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1347 {
1348 RAMBlock *block;
1349
1350 /* The list is protected by the iothread lock here. */
1351 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1352 if (addr - block->offset < block->length) {
1353 if (xen_enabled()) {
1354 /* We need to check if the requested address is in the RAM
1355 * because we don't want to map the entire memory in QEMU.
1356 * In that case just map until the end of the page.
1357 */
1358 if (block->offset == 0) {
1359 return xen_map_cache(addr, 0, 0);
1360 } else if (block->host == NULL) {
1361 block->host =
1362 xen_map_cache(block->offset, block->length, 1);
1363 }
1364 }
1365 return block->host + (addr - block->offset);
1366 }
1367 }
1368
1369 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1370 abort();
1371
1372 return NULL;
1373 }
1374
1375 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1376 * but takes a size argument */
1377 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1378 {
1379 if (*size == 0) {
1380 return NULL;
1381 }
1382 if (xen_enabled()) {
1383 return xen_map_cache(addr, *size, 1);
1384 } else {
1385 RAMBlock *block;
1386
1387 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1388 if (addr - block->offset < block->length) {
1389 if (addr - block->offset + *size > block->length)
1390 *size = block->length - addr + block->offset;
1391 return block->host + (addr - block->offset);
1392 }
1393 }
1394
1395 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1396 abort();
1397 }
1398 }
1399
1400 /* Some of the softmmu routines need to translate from a host pointer
1401 (typically a TLB entry) back to a ram offset. */
1402 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1403 {
1404 RAMBlock *block;
1405 uint8_t *host = ptr;
1406
1407 if (xen_enabled()) {
1408 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1409 return qemu_get_ram_block(*ram_addr)->mr;
1410 }
1411
1412 block = ram_list.mru_block;
1413 if (block && block->host && host - block->host < block->length) {
1414 goto found;
1415 }
1416
1417 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1418 /* This case append when the block is not mapped. */
1419 if (block->host == NULL) {
1420 continue;
1421 }
1422 if (host - block->host < block->length) {
1423 goto found;
1424 }
1425 }
1426
1427 return NULL;
1428
1429 found:
1430 *ram_addr = block->offset + (host - block->host);
1431 return block->mr;
1432 }
1433
1434 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1435 uint64_t val, unsigned size)
1436 {
1437 int dirty_flags;
1438 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1439 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1440 tb_invalidate_phys_page_fast(ram_addr, size);
1441 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1442 }
1443 switch (size) {
1444 case 1:
1445 stb_p(qemu_get_ram_ptr(ram_addr), val);
1446 break;
1447 case 2:
1448 stw_p(qemu_get_ram_ptr(ram_addr), val);
1449 break;
1450 case 4:
1451 stl_p(qemu_get_ram_ptr(ram_addr), val);
1452 break;
1453 default:
1454 abort();
1455 }
1456 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1457 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1458 /* we remove the notdirty callback only if the code has been
1459 flushed */
1460 if (dirty_flags == 0xff)
1461 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1462 }
1463
1464 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1465 unsigned size, bool is_write)
1466 {
1467 return is_write;
1468 }
1469
1470 static const MemoryRegionOps notdirty_mem_ops = {
1471 .write = notdirty_mem_write,
1472 .valid.accepts = notdirty_mem_accepts,
1473 .endianness = DEVICE_NATIVE_ENDIAN,
1474 };
1475
1476 /* Generate a debug exception if a watchpoint has been hit. */
1477 static void check_watchpoint(int offset, int len_mask, int flags)
1478 {
1479 CPUArchState *env = cpu_single_env;
1480 target_ulong pc, cs_base;
1481 target_ulong vaddr;
1482 CPUWatchpoint *wp;
1483 int cpu_flags;
1484
1485 if (env->watchpoint_hit) {
1486 /* We re-entered the check after replacing the TB. Now raise
1487 * the debug interrupt so that is will trigger after the
1488 * current instruction. */
1489 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1490 return;
1491 }
1492 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1493 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1494 if ((vaddr == (wp->vaddr & len_mask) ||
1495 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1496 wp->flags |= BP_WATCHPOINT_HIT;
1497 if (!env->watchpoint_hit) {
1498 env->watchpoint_hit = wp;
1499 tb_check_watchpoint(env);
1500 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1501 env->exception_index = EXCP_DEBUG;
1502 cpu_loop_exit(env);
1503 } else {
1504 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1505 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1506 cpu_resume_from_signal(env, NULL);
1507 }
1508 }
1509 } else {
1510 wp->flags &= ~BP_WATCHPOINT_HIT;
1511 }
1512 }
1513 }
1514
1515 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1516 so these check for a hit then pass through to the normal out-of-line
1517 phys routines. */
1518 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1519 unsigned size)
1520 {
1521 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1522 switch (size) {
1523 case 1: return ldub_phys(addr);
1524 case 2: return lduw_phys(addr);
1525 case 4: return ldl_phys(addr);
1526 default: abort();
1527 }
1528 }
1529
1530 static void watch_mem_write(void *opaque, hwaddr addr,
1531 uint64_t val, unsigned size)
1532 {
1533 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1534 switch (size) {
1535 case 1:
1536 stb_phys(addr, val);
1537 break;
1538 case 2:
1539 stw_phys(addr, val);
1540 break;
1541 case 4:
1542 stl_phys(addr, val);
1543 break;
1544 default: abort();
1545 }
1546 }
1547
1548 static const MemoryRegionOps watch_mem_ops = {
1549 .read = watch_mem_read,
1550 .write = watch_mem_write,
1551 .endianness = DEVICE_NATIVE_ENDIAN,
1552 };
1553
1554 static uint64_t subpage_read(void *opaque, hwaddr addr,
1555 unsigned len)
1556 {
1557 subpage_t *subpage = opaque;
1558 uint8_t buf[4];
1559
1560 #if defined(DEBUG_SUBPAGE)
1561 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1562 subpage, len, addr);
1563 #endif
1564 address_space_read(subpage->as, addr + subpage->base, buf, len);
1565 switch (len) {
1566 case 1:
1567 return ldub_p(buf);
1568 case 2:
1569 return lduw_p(buf);
1570 case 4:
1571 return ldl_p(buf);
1572 default:
1573 abort();
1574 }
1575 }
1576
1577 static void subpage_write(void *opaque, hwaddr addr,
1578 uint64_t value, unsigned len)
1579 {
1580 subpage_t *subpage = opaque;
1581 uint8_t buf[4];
1582
1583 #if defined(DEBUG_SUBPAGE)
1584 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1585 " value %"PRIx64"\n",
1586 __func__, subpage, len, addr, value);
1587 #endif
1588 switch (len) {
1589 case 1:
1590 stb_p(buf, value);
1591 break;
1592 case 2:
1593 stw_p(buf, value);
1594 break;
1595 case 4:
1596 stl_p(buf, value);
1597 break;
1598 default:
1599 abort();
1600 }
1601 address_space_write(subpage->as, addr + subpage->base, buf, len);
1602 }
1603
1604 static bool subpage_accepts(void *opaque, hwaddr addr,
1605 unsigned size, bool is_write)
1606 {
1607 subpage_t *subpage = opaque;
1608 #if defined(DEBUG_SUBPAGE)
1609 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1610 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1611 #endif
1612
1613 return address_space_access_valid(subpage->as, addr + subpage->base,
1614 size, is_write);
1615 }
1616
1617 static const MemoryRegionOps subpage_ops = {
1618 .read = subpage_read,
1619 .write = subpage_write,
1620 .valid.accepts = subpage_accepts,
1621 .endianness = DEVICE_NATIVE_ENDIAN,
1622 };
1623
1624 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1625 uint16_t section)
1626 {
1627 int idx, eidx;
1628
1629 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1630 return -1;
1631 idx = SUBPAGE_IDX(start);
1632 eidx = SUBPAGE_IDX(end);
1633 #if defined(DEBUG_SUBPAGE)
1634 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1635 mmio, start, end, idx, eidx, memory);
1636 #endif
1637 for (; idx <= eidx; idx++) {
1638 mmio->sub_section[idx] = section;
1639 }
1640
1641 return 0;
1642 }
1643
1644 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1645 {
1646 subpage_t *mmio;
1647
1648 mmio = g_malloc0(sizeof(subpage_t));
1649
1650 mmio->as = as;
1651 mmio->base = base;
1652 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1653 "subpage", TARGET_PAGE_SIZE);
1654 mmio->iomem.subpage = true;
1655 #if defined(DEBUG_SUBPAGE)
1656 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1657 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1658 #endif
1659 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1660
1661 return mmio;
1662 }
1663
1664 static uint16_t dummy_section(MemoryRegion *mr)
1665 {
1666 MemoryRegionSection section = {
1667 .mr = mr,
1668 .offset_within_address_space = 0,
1669 .offset_within_region = 0,
1670 .size = int128_2_64(),
1671 };
1672
1673 return phys_section_add(&section);
1674 }
1675
1676 MemoryRegion *iotlb_to_region(hwaddr index)
1677 {
1678 return phys_sections[index & ~TARGET_PAGE_MASK].mr;
1679 }
1680
1681 static void io_mem_init(void)
1682 {
1683 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1684 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1685 "unassigned", UINT64_MAX);
1686 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1687 "notdirty", UINT64_MAX);
1688 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1689 "watch", UINT64_MAX);
1690 }
1691
1692 static void mem_begin(MemoryListener *listener)
1693 {
1694 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
1695
1696 d->phys_map.ptr = PHYS_MAP_NODE_NIL;
1697 }
1698
1699 static void core_begin(MemoryListener *listener)
1700 {
1701 uint16_t n;
1702
1703 phys_sections_clear();
1704 n = dummy_section(&io_mem_unassigned);
1705 assert(n == PHYS_SECTION_UNASSIGNED);
1706 n = dummy_section(&io_mem_notdirty);
1707 assert(n == PHYS_SECTION_NOTDIRTY);
1708 n = dummy_section(&io_mem_rom);
1709 assert(n == PHYS_SECTION_ROM);
1710 n = dummy_section(&io_mem_watch);
1711 assert(n == PHYS_SECTION_WATCH);
1712 }
1713
1714 static void tcg_commit(MemoryListener *listener)
1715 {
1716 CPUArchState *env;
1717
1718 /* since each CPU stores ram addresses in its TLB cache, we must
1719 reset the modified entries */
1720 /* XXX: slow ! */
1721 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1722 tlb_flush(env, 1);
1723 }
1724 }
1725
1726 static void core_log_global_start(MemoryListener *listener)
1727 {
1728 cpu_physical_memory_set_dirty_tracking(1);
1729 }
1730
1731 static void core_log_global_stop(MemoryListener *listener)
1732 {
1733 cpu_physical_memory_set_dirty_tracking(0);
1734 }
1735
1736 static MemoryListener core_memory_listener = {
1737 .begin = core_begin,
1738 .log_global_start = core_log_global_start,
1739 .log_global_stop = core_log_global_stop,
1740 .priority = 1,
1741 };
1742
1743 static MemoryListener tcg_memory_listener = {
1744 .commit = tcg_commit,
1745 };
1746
1747 void address_space_init_dispatch(AddressSpace *as)
1748 {
1749 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1750
1751 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1752 d->listener = (MemoryListener) {
1753 .begin = mem_begin,
1754 .region_add = mem_add,
1755 .region_nop = mem_add,
1756 .priority = 0,
1757 };
1758 d->as = as;
1759 as->dispatch = d;
1760 memory_listener_register(&d->listener, as);
1761 }
1762
1763 void address_space_destroy_dispatch(AddressSpace *as)
1764 {
1765 AddressSpaceDispatch *d = as->dispatch;
1766
1767 memory_listener_unregister(&d->listener);
1768 g_free(d);
1769 as->dispatch = NULL;
1770 }
1771
1772 static void memory_map_init(void)
1773 {
1774 system_memory = g_malloc(sizeof(*system_memory));
1775 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1776 address_space_init(&address_space_memory, system_memory, "memory");
1777
1778 system_io = g_malloc(sizeof(*system_io));
1779 memory_region_init(system_io, NULL, "io", 65536);
1780 address_space_init(&address_space_io, system_io, "I/O");
1781
1782 memory_listener_register(&core_memory_listener, &address_space_memory);
1783 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1784 }
1785
1786 MemoryRegion *get_system_memory(void)
1787 {
1788 return system_memory;
1789 }
1790
1791 MemoryRegion *get_system_io(void)
1792 {
1793 return system_io;
1794 }
1795
1796 #endif /* !defined(CONFIG_USER_ONLY) */
1797
1798 /* physical memory access (slow version, mainly for debug) */
1799 #if defined(CONFIG_USER_ONLY)
1800 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1801 uint8_t *buf, int len, int is_write)
1802 {
1803 int l, flags;
1804 target_ulong page;
1805 void * p;
1806
1807 while (len > 0) {
1808 page = addr & TARGET_PAGE_MASK;
1809 l = (page + TARGET_PAGE_SIZE) - addr;
1810 if (l > len)
1811 l = len;
1812 flags = page_get_flags(page);
1813 if (!(flags & PAGE_VALID))
1814 return -1;
1815 if (is_write) {
1816 if (!(flags & PAGE_WRITE))
1817 return -1;
1818 /* XXX: this code should not depend on lock_user */
1819 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1820 return -1;
1821 memcpy(p, buf, l);
1822 unlock_user(p, addr, l);
1823 } else {
1824 if (!(flags & PAGE_READ))
1825 return -1;
1826 /* XXX: this code should not depend on lock_user */
1827 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1828 return -1;
1829 memcpy(buf, p, l);
1830 unlock_user(p, addr, 0);
1831 }
1832 len -= l;
1833 buf += l;
1834 addr += l;
1835 }
1836 return 0;
1837 }
1838
1839 #else
1840
1841 static void invalidate_and_set_dirty(hwaddr addr,
1842 hwaddr length)
1843 {
1844 if (!cpu_physical_memory_is_dirty(addr)) {
1845 /* invalidate code */
1846 tb_invalidate_phys_page_range(addr, addr + length, 0);
1847 /* set dirty bit */
1848 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1849 }
1850 xen_modified_memory(addr, length);
1851 }
1852
1853 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1854 {
1855 if (memory_region_is_ram(mr)) {
1856 return !(is_write && mr->readonly);
1857 }
1858 if (memory_region_is_romd(mr)) {
1859 return !is_write;
1860 }
1861
1862 return false;
1863 }
1864
1865 static inline int memory_access_size(MemoryRegion *mr, int l, hwaddr addr)
1866 {
1867 if (l >= 4 && (((addr & 3) == 0 || mr->ops->impl.unaligned))) {
1868 return 4;
1869 }
1870 if (l >= 2 && (((addr & 1) == 0) || mr->ops->impl.unaligned)) {
1871 return 2;
1872 }
1873 return 1;
1874 }
1875
1876 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1877 int len, bool is_write)
1878 {
1879 hwaddr l;
1880 uint8_t *ptr;
1881 uint64_t val;
1882 hwaddr addr1;
1883 MemoryRegion *mr;
1884 bool error = false;
1885
1886 while (len > 0) {
1887 l = len;
1888 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1889
1890 if (is_write) {
1891 if (!memory_access_is_direct(mr, is_write)) {
1892 l = memory_access_size(mr, l, addr1);
1893 /* XXX: could force cpu_single_env to NULL to avoid
1894 potential bugs */
1895 if (l == 4) {
1896 /* 32 bit write access */
1897 val = ldl_p(buf);
1898 error |= io_mem_write(mr, addr1, val, 4);
1899 } else if (l == 2) {
1900 /* 16 bit write access */
1901 val = lduw_p(buf);
1902 error |= io_mem_write(mr, addr1, val, 2);
1903 } else {
1904 /* 8 bit write access */
1905 val = ldub_p(buf);
1906 error |= io_mem_write(mr, addr1, val, 1);
1907 }
1908 } else {
1909 addr1 += memory_region_get_ram_addr(mr);
1910 /* RAM case */
1911 ptr = qemu_get_ram_ptr(addr1);
1912 memcpy(ptr, buf, l);
1913 invalidate_and_set_dirty(addr1, l);
1914 }
1915 } else {
1916 if (!memory_access_is_direct(mr, is_write)) {
1917 /* I/O case */
1918 l = memory_access_size(mr, l, addr1);
1919 if (l == 4) {
1920 /* 32 bit read access */
1921 error |= io_mem_read(mr, addr1, &val, 4);
1922 stl_p(buf, val);
1923 } else if (l == 2) {
1924 /* 16 bit read access */
1925 error |= io_mem_read(mr, addr1, &val, 2);
1926 stw_p(buf, val);
1927 } else {
1928 /* 8 bit read access */
1929 error |= io_mem_read(mr, addr1, &val, 1);
1930 stb_p(buf, val);
1931 }
1932 } else {
1933 /* RAM case */
1934 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1935 memcpy(buf, ptr, l);
1936 }
1937 }
1938 len -= l;
1939 buf += l;
1940 addr += l;
1941 }
1942
1943 return error;
1944 }
1945
1946 bool address_space_write(AddressSpace *as, hwaddr addr,
1947 const uint8_t *buf, int len)
1948 {
1949 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
1950 }
1951
1952 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1953 {
1954 return address_space_rw(as, addr, buf, len, false);
1955 }
1956
1957
1958 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1959 int len, int is_write)
1960 {
1961 address_space_rw(&address_space_memory, addr, buf, len, is_write);
1962 }
1963
1964 /* used for ROM loading : can write in RAM and ROM */
1965 void cpu_physical_memory_write_rom(hwaddr addr,
1966 const uint8_t *buf, int len)
1967 {
1968 hwaddr l;
1969 uint8_t *ptr;
1970 hwaddr addr1;
1971 MemoryRegion *mr;
1972
1973 while (len > 0) {
1974 l = len;
1975 mr = address_space_translate(&address_space_memory,
1976 addr, &addr1, &l, true);
1977
1978 if (!(memory_region_is_ram(mr) ||
1979 memory_region_is_romd(mr))) {
1980 /* do nothing */
1981 } else {
1982 addr1 += memory_region_get_ram_addr(mr);
1983 /* ROM/RAM case */
1984 ptr = qemu_get_ram_ptr(addr1);
1985 memcpy(ptr, buf, l);
1986 invalidate_and_set_dirty(addr1, l);
1987 }
1988 len -= l;
1989 buf += l;
1990 addr += l;
1991 }
1992 }
1993
1994 typedef struct {
1995 MemoryRegion *mr;
1996 void *buffer;
1997 hwaddr addr;
1998 hwaddr len;
1999 } BounceBuffer;
2000
2001 static BounceBuffer bounce;
2002
2003 typedef struct MapClient {
2004 void *opaque;
2005 void (*callback)(void *opaque);
2006 QLIST_ENTRY(MapClient) link;
2007 } MapClient;
2008
2009 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2010 = QLIST_HEAD_INITIALIZER(map_client_list);
2011
2012 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2013 {
2014 MapClient *client = g_malloc(sizeof(*client));
2015
2016 client->opaque = opaque;
2017 client->callback = callback;
2018 QLIST_INSERT_HEAD(&map_client_list, client, link);
2019 return client;
2020 }
2021
2022 static void cpu_unregister_map_client(void *_client)
2023 {
2024 MapClient *client = (MapClient *)_client;
2025
2026 QLIST_REMOVE(client, link);
2027 g_free(client);
2028 }
2029
2030 static void cpu_notify_map_clients(void)
2031 {
2032 MapClient *client;
2033
2034 while (!QLIST_EMPTY(&map_client_list)) {
2035 client = QLIST_FIRST(&map_client_list);
2036 client->callback(client->opaque);
2037 cpu_unregister_map_client(client);
2038 }
2039 }
2040
2041 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2042 {
2043 MemoryRegion *mr;
2044 hwaddr l, xlat;
2045
2046 while (len > 0) {
2047 l = len;
2048 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2049 if (!memory_access_is_direct(mr, is_write)) {
2050 l = memory_access_size(mr, l, addr);
2051 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2052 return false;
2053 }
2054 }
2055
2056 len -= l;
2057 addr += l;
2058 }
2059 return true;
2060 }
2061
2062 /* Map a physical memory region into a host virtual address.
2063 * May map a subset of the requested range, given by and returned in *plen.
2064 * May return NULL if resources needed to perform the mapping are exhausted.
2065 * Use only for reads OR writes - not for read-modify-write operations.
2066 * Use cpu_register_map_client() to know when retrying the map operation is
2067 * likely to succeed.
2068 */
2069 void *address_space_map(AddressSpace *as,
2070 hwaddr addr,
2071 hwaddr *plen,
2072 bool is_write)
2073 {
2074 hwaddr len = *plen;
2075 hwaddr done = 0;
2076 hwaddr l, xlat, base;
2077 MemoryRegion *mr, *this_mr;
2078 ram_addr_t raddr;
2079
2080 if (len == 0) {
2081 return NULL;
2082 }
2083
2084 l = len;
2085 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2086 if (!memory_access_is_direct(mr, is_write)) {
2087 if (bounce.buffer) {
2088 return NULL;
2089 }
2090 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2091 bounce.addr = addr;
2092 bounce.len = l;
2093
2094 memory_region_ref(mr);
2095 bounce.mr = mr;
2096 if (!is_write) {
2097 address_space_read(as, addr, bounce.buffer, l);
2098 }
2099
2100 *plen = l;
2101 return bounce.buffer;
2102 }
2103
2104 base = xlat;
2105 raddr = memory_region_get_ram_addr(mr);
2106
2107 for (;;) {
2108 len -= l;
2109 addr += l;
2110 done += l;
2111 if (len == 0) {
2112 break;
2113 }
2114
2115 l = len;
2116 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2117 if (this_mr != mr || xlat != base + done) {
2118 break;
2119 }
2120 }
2121
2122 memory_region_ref(mr);
2123 *plen = done;
2124 return qemu_ram_ptr_length(raddr + base, plen);
2125 }
2126
2127 /* Unmaps a memory region previously mapped by address_space_map().
2128 * Will also mark the memory as dirty if is_write == 1. access_len gives
2129 * the amount of memory that was actually read or written by the caller.
2130 */
2131 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2132 int is_write, hwaddr access_len)
2133 {
2134 if (buffer != bounce.buffer) {
2135 MemoryRegion *mr;
2136 ram_addr_t addr1;
2137
2138 mr = qemu_ram_addr_from_host(buffer, &addr1);
2139 assert(mr != NULL);
2140 if (is_write) {
2141 while (access_len) {
2142 unsigned l;
2143 l = TARGET_PAGE_SIZE;
2144 if (l > access_len)
2145 l = access_len;
2146 invalidate_and_set_dirty(addr1, l);
2147 addr1 += l;
2148 access_len -= l;
2149 }
2150 }
2151 if (xen_enabled()) {
2152 xen_invalidate_map_cache_entry(buffer);
2153 }
2154 memory_region_unref(mr);
2155 return;
2156 }
2157 if (is_write) {
2158 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2159 }
2160 qemu_vfree(bounce.buffer);
2161 bounce.buffer = NULL;
2162 memory_region_unref(bounce.mr);
2163 cpu_notify_map_clients();
2164 }
2165
2166 void *cpu_physical_memory_map(hwaddr addr,
2167 hwaddr *plen,
2168 int is_write)
2169 {
2170 return address_space_map(&address_space_memory, addr, plen, is_write);
2171 }
2172
2173 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2174 int is_write, hwaddr access_len)
2175 {
2176 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2177 }
2178
2179 /* warning: addr must be aligned */
2180 static inline uint32_t ldl_phys_internal(hwaddr addr,
2181 enum device_endian endian)
2182 {
2183 uint8_t *ptr;
2184 uint64_t val;
2185 MemoryRegion *mr;
2186 hwaddr l = 4;
2187 hwaddr addr1;
2188
2189 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2190 false);
2191 if (l < 4 || !memory_access_is_direct(mr, false)) {
2192 /* I/O case */
2193 io_mem_read(mr, addr1, &val, 4);
2194 #if defined(TARGET_WORDS_BIGENDIAN)
2195 if (endian == DEVICE_LITTLE_ENDIAN) {
2196 val = bswap32(val);
2197 }
2198 #else
2199 if (endian == DEVICE_BIG_ENDIAN) {
2200 val = bswap32(val);
2201 }
2202 #endif
2203 } else {
2204 /* RAM case */
2205 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2206 & TARGET_PAGE_MASK)
2207 + addr1);
2208 switch (endian) {
2209 case DEVICE_LITTLE_ENDIAN:
2210 val = ldl_le_p(ptr);
2211 break;
2212 case DEVICE_BIG_ENDIAN:
2213 val = ldl_be_p(ptr);
2214 break;
2215 default:
2216 val = ldl_p(ptr);
2217 break;
2218 }
2219 }
2220 return val;
2221 }
2222
2223 uint32_t ldl_phys(hwaddr addr)
2224 {
2225 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2226 }
2227
2228 uint32_t ldl_le_phys(hwaddr addr)
2229 {
2230 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2231 }
2232
2233 uint32_t ldl_be_phys(hwaddr addr)
2234 {
2235 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2236 }
2237
2238 /* warning: addr must be aligned */
2239 static inline uint64_t ldq_phys_internal(hwaddr addr,
2240 enum device_endian endian)
2241 {
2242 uint8_t *ptr;
2243 uint64_t val;
2244 MemoryRegion *mr;
2245 hwaddr l = 8;
2246 hwaddr addr1;
2247
2248 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2249 false);
2250 if (l < 8 || !memory_access_is_direct(mr, false)) {
2251 /* I/O case */
2252 io_mem_read(mr, addr1, &val, 8);
2253 #if defined(TARGET_WORDS_BIGENDIAN)
2254 if (endian == DEVICE_LITTLE_ENDIAN) {
2255 val = bswap64(val);
2256 }
2257 #else
2258 if (endian == DEVICE_BIG_ENDIAN) {
2259 val = bswap64(val);
2260 }
2261 #endif
2262 } else {
2263 /* RAM case */
2264 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2265 & TARGET_PAGE_MASK)
2266 + addr1);
2267 switch (endian) {
2268 case DEVICE_LITTLE_ENDIAN:
2269 val = ldq_le_p(ptr);
2270 break;
2271 case DEVICE_BIG_ENDIAN:
2272 val = ldq_be_p(ptr);
2273 break;
2274 default:
2275 val = ldq_p(ptr);
2276 break;
2277 }
2278 }
2279 return val;
2280 }
2281
2282 uint64_t ldq_phys(hwaddr addr)
2283 {
2284 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2285 }
2286
2287 uint64_t ldq_le_phys(hwaddr addr)
2288 {
2289 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2290 }
2291
2292 uint64_t ldq_be_phys(hwaddr addr)
2293 {
2294 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2295 }
2296
2297 /* XXX: optimize */
2298 uint32_t ldub_phys(hwaddr addr)
2299 {
2300 uint8_t val;
2301 cpu_physical_memory_read(addr, &val, 1);
2302 return val;
2303 }
2304
2305 /* warning: addr must be aligned */
2306 static inline uint32_t lduw_phys_internal(hwaddr addr,
2307 enum device_endian endian)
2308 {
2309 uint8_t *ptr;
2310 uint64_t val;
2311 MemoryRegion *mr;
2312 hwaddr l = 2;
2313 hwaddr addr1;
2314
2315 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2316 false);
2317 if (l < 2 || !memory_access_is_direct(mr, false)) {
2318 /* I/O case */
2319 io_mem_read(mr, addr1, &val, 2);
2320 #if defined(TARGET_WORDS_BIGENDIAN)
2321 if (endian == DEVICE_LITTLE_ENDIAN) {
2322 val = bswap16(val);
2323 }
2324 #else
2325 if (endian == DEVICE_BIG_ENDIAN) {
2326 val = bswap16(val);
2327 }
2328 #endif
2329 } else {
2330 /* RAM case */
2331 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2332 & TARGET_PAGE_MASK)
2333 + addr1);
2334 switch (endian) {
2335 case DEVICE_LITTLE_ENDIAN:
2336 val = lduw_le_p(ptr);
2337 break;
2338 case DEVICE_BIG_ENDIAN:
2339 val = lduw_be_p(ptr);
2340 break;
2341 default:
2342 val = lduw_p(ptr);
2343 break;
2344 }
2345 }
2346 return val;
2347 }
2348
2349 uint32_t lduw_phys(hwaddr addr)
2350 {
2351 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2352 }
2353
2354 uint32_t lduw_le_phys(hwaddr addr)
2355 {
2356 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2357 }
2358
2359 uint32_t lduw_be_phys(hwaddr addr)
2360 {
2361 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2362 }
2363
2364 /* warning: addr must be aligned. The ram page is not masked as dirty
2365 and the code inside is not invalidated. It is useful if the dirty
2366 bits are used to track modified PTEs */
2367 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2368 {
2369 uint8_t *ptr;
2370 MemoryRegion *mr;
2371 hwaddr l = 4;
2372 hwaddr addr1;
2373
2374 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2375 true);
2376 if (l < 4 || !memory_access_is_direct(mr, true)) {
2377 io_mem_write(mr, addr1, val, 4);
2378 } else {
2379 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2380 ptr = qemu_get_ram_ptr(addr1);
2381 stl_p(ptr, val);
2382
2383 if (unlikely(in_migration)) {
2384 if (!cpu_physical_memory_is_dirty(addr1)) {
2385 /* invalidate code */
2386 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2387 /* set dirty bit */
2388 cpu_physical_memory_set_dirty_flags(
2389 addr1, (0xff & ~CODE_DIRTY_FLAG));
2390 }
2391 }
2392 }
2393 }
2394
2395 /* warning: addr must be aligned */
2396 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2397 enum device_endian endian)
2398 {
2399 uint8_t *ptr;
2400 MemoryRegion *mr;
2401 hwaddr l = 4;
2402 hwaddr addr1;
2403
2404 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2405 true);
2406 if (l < 4 || !memory_access_is_direct(mr, true)) {
2407 #if defined(TARGET_WORDS_BIGENDIAN)
2408 if (endian == DEVICE_LITTLE_ENDIAN) {
2409 val = bswap32(val);
2410 }
2411 #else
2412 if (endian == DEVICE_BIG_ENDIAN) {
2413 val = bswap32(val);
2414 }
2415 #endif
2416 io_mem_write(mr, addr1, val, 4);
2417 } else {
2418 /* RAM case */
2419 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2420 ptr = qemu_get_ram_ptr(addr1);
2421 switch (endian) {
2422 case DEVICE_LITTLE_ENDIAN:
2423 stl_le_p(ptr, val);
2424 break;
2425 case DEVICE_BIG_ENDIAN:
2426 stl_be_p(ptr, val);
2427 break;
2428 default:
2429 stl_p(ptr, val);
2430 break;
2431 }
2432 invalidate_and_set_dirty(addr1, 4);
2433 }
2434 }
2435
2436 void stl_phys(hwaddr addr, uint32_t val)
2437 {
2438 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2439 }
2440
2441 void stl_le_phys(hwaddr addr, uint32_t val)
2442 {
2443 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2444 }
2445
2446 void stl_be_phys(hwaddr addr, uint32_t val)
2447 {
2448 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2449 }
2450
2451 /* XXX: optimize */
2452 void stb_phys(hwaddr addr, uint32_t val)
2453 {
2454 uint8_t v = val;
2455 cpu_physical_memory_write(addr, &v, 1);
2456 }
2457
2458 /* warning: addr must be aligned */
2459 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2460 enum device_endian endian)
2461 {
2462 uint8_t *ptr;
2463 MemoryRegion *mr;
2464 hwaddr l = 2;
2465 hwaddr addr1;
2466
2467 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2468 true);
2469 if (l < 2 || !memory_access_is_direct(mr, true)) {
2470 #if defined(TARGET_WORDS_BIGENDIAN)
2471 if (endian == DEVICE_LITTLE_ENDIAN) {
2472 val = bswap16(val);
2473 }
2474 #else
2475 if (endian == DEVICE_BIG_ENDIAN) {
2476 val = bswap16(val);
2477 }
2478 #endif
2479 io_mem_write(mr, addr1, val, 2);
2480 } else {
2481 /* RAM case */
2482 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2483 ptr = qemu_get_ram_ptr(addr1);
2484 switch (endian) {
2485 case DEVICE_LITTLE_ENDIAN:
2486 stw_le_p(ptr, val);
2487 break;
2488 case DEVICE_BIG_ENDIAN:
2489 stw_be_p(ptr, val);
2490 break;
2491 default:
2492 stw_p(ptr, val);
2493 break;
2494 }
2495 invalidate_and_set_dirty(addr1, 2);
2496 }
2497 }
2498
2499 void stw_phys(hwaddr addr, uint32_t val)
2500 {
2501 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2502 }
2503
2504 void stw_le_phys(hwaddr addr, uint32_t val)
2505 {
2506 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2507 }
2508
2509 void stw_be_phys(hwaddr addr, uint32_t val)
2510 {
2511 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2512 }
2513
2514 /* XXX: optimize */
2515 void stq_phys(hwaddr addr, uint64_t val)
2516 {
2517 val = tswap64(val);
2518 cpu_physical_memory_write(addr, &val, 8);
2519 }
2520
2521 void stq_le_phys(hwaddr addr, uint64_t val)
2522 {
2523 val = cpu_to_le64(val);
2524 cpu_physical_memory_write(addr, &val, 8);
2525 }
2526
2527 void stq_be_phys(hwaddr addr, uint64_t val)
2528 {
2529 val = cpu_to_be64(val);
2530 cpu_physical_memory_write(addr, &val, 8);
2531 }
2532
2533 /* virtual memory access for debug (includes writing to ROM) */
2534 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2535 uint8_t *buf, int len, int is_write)
2536 {
2537 int l;
2538 hwaddr phys_addr;
2539 target_ulong page;
2540
2541 while (len > 0) {
2542 page = addr & TARGET_PAGE_MASK;
2543 phys_addr = cpu_get_phys_page_debug(env, page);
2544 /* if no physical page mapped, return an error */
2545 if (phys_addr == -1)
2546 return -1;
2547 l = (page + TARGET_PAGE_SIZE) - addr;
2548 if (l > len)
2549 l = len;
2550 phys_addr += (addr & ~TARGET_PAGE_MASK);
2551 if (is_write)
2552 cpu_physical_memory_write_rom(phys_addr, buf, l);
2553 else
2554 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2555 len -= l;
2556 buf += l;
2557 addr += l;
2558 }
2559 return 0;
2560 }
2561 #endif
2562
2563 #if !defined(CONFIG_USER_ONLY)
2564
2565 /*
2566 * A helper function for the _utterly broken_ virtio device model to find out if
2567 * it's running on a big endian machine. Don't do this at home kids!
2568 */
2569 bool virtio_is_big_endian(void);
2570 bool virtio_is_big_endian(void)
2571 {
2572 #if defined(TARGET_WORDS_BIGENDIAN)
2573 return true;
2574 #else
2575 return false;
2576 #endif
2577 }
2578
2579 #endif
2580
2581 #ifndef CONFIG_USER_ONLY
2582 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2583 {
2584 MemoryRegion*mr;
2585 hwaddr l = 1;
2586
2587 mr = address_space_translate(&address_space_memory,
2588 phys_addr, &phys_addr, &l, false);
2589
2590 return !(memory_region_is_ram(mr) ||
2591 memory_region_is_romd(mr));
2592 }
2593
2594 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2595 {
2596 RAMBlock *block;
2597
2598 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2599 func(block->host, block->offset, block->length, opaque);
2600 }
2601 }
2602 #endif
AR7 emulation for QEMU