?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
tcg-ppc64: Implement CONFIG_QEMU_LDST_OPTIMIZATION
[qemu/kevin.git] / exec.c
blob26681ce02179ad36f6b194ba250db8cd0ace396f
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
51
52 #include "exec/memory-internal.h"
53
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 typedef PhysPageEntry Node[L2_SIZE];
92
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101 };
102
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
110
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
115
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
124
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
127
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
133
134 static MemoryRegion io_mem_watch;
135 #endif
136
137 #if !defined(CONFIG_USER_ONLY)
138
139 static void phys_map_node_reserve(unsigned nodes)
140 {
141 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
142 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
143 16);
144 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
145 next_map.nodes_nb + nodes);
146 next_map.nodes = g_renew(Node, next_map.nodes,
147 next_map.nodes_nb_alloc);
148 }
149 }
150
151 static uint16_t phys_map_node_alloc(void)
152 {
153 unsigned i;
154 uint16_t ret;
155
156 ret = next_map.nodes_nb++;
157 assert(ret != PHYS_MAP_NODE_NIL);
158 assert(ret != next_map.nodes_nb_alloc);
159 for (i = 0; i < L2_SIZE; ++i) {
160 next_map.nodes[ret][i].is_leaf = 0;
161 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
162 }
163 return ret;
164 }
165
166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
167 hwaddr *nb, uint16_t leaf,
168 int level)
169 {
170 PhysPageEntry *p;
171 int i;
172 hwaddr step = (hwaddr)1 << (level * L2_BITS);
173
174 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
175 lp->ptr = phys_map_node_alloc();
176 p = next_map.nodes[lp->ptr];
177 if (level == 0) {
178 for (i = 0; i < L2_SIZE; i++) {
179 p[i].is_leaf = 1;
180 p[i].ptr = PHYS_SECTION_UNASSIGNED;
181 }
182 }
183 } else {
184 p = next_map.nodes[lp->ptr];
185 }
186 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
187
188 while (*nb && lp < &p[L2_SIZE]) {
189 if ((*index & (step - 1)) == 0 && *nb >= step) {
190 lp->is_leaf = true;
191 lp->ptr = leaf;
192 *index += step;
193 *nb -= step;
194 } else {
195 phys_page_set_level(lp, index, nb, leaf, level - 1);
196 }
197 ++lp;
198 }
199 }
200
201 static void phys_page_set(AddressSpaceDispatch *d,
202 hwaddr index, hwaddr nb,
203 uint16_t leaf)
204 {
205 /* Wildly overreserve - it doesn't matter much. */
206 phys_map_node_reserve(3 * P_L2_LEVELS);
207
208 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
209 }
210
211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
212 Node *nodes, MemoryRegionSection *sections)
213 {
214 PhysPageEntry *p;
215 int i;
216
217 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
218 if (lp.ptr == PHYS_MAP_NODE_NIL) {
219 return &sections[PHYS_SECTION_UNASSIGNED];
220 }
221 p = nodes[lp.ptr];
222 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
223 }
224 return &sections[lp.ptr];
225 }
226
227 bool memory_region_is_unassigned(MemoryRegion *mr)
228 {
229 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
230 && mr != &io_mem_watch;
231 }
232
233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
234 hwaddr addr,
235 bool resolve_subpage)
236 {
237 MemoryRegionSection *section;
238 subpage_t *subpage;
239
240 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
241 d->nodes, d->sections);
242 if (resolve_subpage && section->mr->subpage) {
243 subpage = container_of(section->mr, subpage_t, iomem);
244 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
245 }
246 return section;
247 }
248
249 static MemoryRegionSection *
250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
251 hwaddr *plen, bool resolve_subpage)
252 {
253 MemoryRegionSection *section;
254 Int128 diff;
255
256 section = address_space_lookup_region(d, addr, resolve_subpage);
257 /* Compute offset within MemoryRegionSection */
258 addr -= section->offset_within_address_space;
259
260 /* Compute offset within MemoryRegion */
261 *xlat = addr + section->offset_within_region;
262
263 diff = int128_sub(section->mr->size, int128_make64(addr));
264 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
265 return section;
266 }
267
268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
269 hwaddr *xlat, hwaddr *plen,
270 bool is_write)
271 {
272 IOMMUTLBEntry iotlb;
273 MemoryRegionSection *section;
274 MemoryRegion *mr;
275 hwaddr len = *plen;
276
277 for (;;) {
278 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
279 mr = section->mr;
280
281 if (!mr->iommu_ops) {
282 break;
283 }
284
285 iotlb = mr->iommu_ops->translate(mr, addr);
286 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
287 | (addr & iotlb.addr_mask));
288 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
289 if (!(iotlb.perm & (1 << is_write))) {
290 mr = &io_mem_unassigned;
291 break;
292 }
293
294 as = iotlb.target_as;
295 }
296
297 *plen = len;
298 *xlat = addr;
299 return mr;
300 }
301
302 MemoryRegionSection *
303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
304 hwaddr *plen)
305 {
306 MemoryRegionSection *section;
307 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
308
309 assert(!section->mr->iommu_ops);
310 return section;
311 }
312 #endif
313
314 void cpu_exec_init_all(void)
315 {
316 #if !defined(CONFIG_USER_ONLY)
317 qemu_mutex_init(&ram_list.mutex);
318 memory_map_init();
319 io_mem_init();
320 #endif
321 }
322
323 #if !defined(CONFIG_USER_ONLY)
324
325 static int cpu_common_post_load(void *opaque, int version_id)
326 {
327 CPUState *cpu = opaque;
328
329 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
330 version_id is increased. */
331 cpu->interrupt_request &= ~0x01;
332 tlb_flush(cpu->env_ptr, 1);
333
334 return 0;
335 }
336
337 const VMStateDescription vmstate_cpu_common = {
338 .name = "cpu_common",
339 .version_id = 1,
340 .minimum_version_id = 1,
341 .minimum_version_id_old = 1,
342 .post_load = cpu_common_post_load,
343 .fields = (VMStateField []) {
344 VMSTATE_UINT32(halted, CPUState),
345 VMSTATE_UINT32(interrupt_request, CPUState),
346 VMSTATE_END_OF_LIST()
347 }
348 };
349
350 #endif
351
352 CPUState *qemu_get_cpu(int index)
353 {
354 CPUState *cpu;
355
356 CPU_FOREACH(cpu) {
357 if (cpu->cpu_index == index) {
358 return cpu;
359 }
360 }
361
362 return NULL;
363 }
364
365 void cpu_exec_init(CPUArchState *env)
366 {
367 CPUState *cpu = ENV_GET_CPU(env);
368 CPUClass *cc = CPU_GET_CLASS(cpu);
369 CPUState *some_cpu;
370 int cpu_index;
371
372 #if defined(CONFIG_USER_ONLY)
373 cpu_list_lock();
374 #endif
375 cpu_index = 0;
376 CPU_FOREACH(some_cpu) {
377 cpu_index++;
378 }
379 cpu->cpu_index = cpu_index;
380 cpu->numa_node = 0;
381 QTAILQ_INIT(&env->breakpoints);
382 QTAILQ_INIT(&env->watchpoints);
383 #ifndef CONFIG_USER_ONLY
384 cpu->thread_id = qemu_get_thread_id();
385 #endif
386 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
387 #if defined(CONFIG_USER_ONLY)
388 cpu_list_unlock();
389 #endif
390 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
391 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
392 }
393 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
394 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
395 cpu_save, cpu_load, env);
396 assert(cc->vmsd == NULL);
397 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
398 #endif
399 if (cc->vmsd != NULL) {
400 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
401 }
402 }
403
404 #if defined(TARGET_HAS_ICE)
405 #if defined(CONFIG_USER_ONLY)
406 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
407 {
408 tb_invalidate_phys_page_range(pc, pc + 1, 0);
409 }
410 #else
411 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
412 {
413 tb_invalidate_phys_addr(cpu_get_phys_page_debug(cpu, pc) |
414 (pc & ~TARGET_PAGE_MASK));
415 }
416 #endif
417 #endif /* TARGET_HAS_ICE */
418
419 #if defined(CONFIG_USER_ONLY)
420 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
421
422 {
423 }
424
425 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
426 int flags, CPUWatchpoint **watchpoint)
427 {
428 return -ENOSYS;
429 }
430 #else
431 /* Add a watchpoint. */
432 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
433 int flags, CPUWatchpoint **watchpoint)
434 {
435 target_ulong len_mask = ~(len - 1);
436 CPUWatchpoint *wp;
437
438 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
439 if ((len & (len - 1)) || (addr & ~len_mask) ||
440 len == 0 || len > TARGET_PAGE_SIZE) {
441 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
442 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
443 return -EINVAL;
444 }
445 wp = g_malloc(sizeof(*wp));
446
447 wp->vaddr = addr;
448 wp->len_mask = len_mask;
449 wp->flags = flags;
450
451 /* keep all GDB-injected watchpoints in front */
452 if (flags & BP_GDB)
453 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
454 else
455 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
456
457 tlb_flush_page(env, addr);
458
459 if (watchpoint)
460 *watchpoint = wp;
461 return 0;
462 }
463
464 /* Remove a specific watchpoint. */
465 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
466 int flags)
467 {
468 target_ulong len_mask = ~(len - 1);
469 CPUWatchpoint *wp;
470
471 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
472 if (addr == wp->vaddr && len_mask == wp->len_mask
473 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
474 cpu_watchpoint_remove_by_ref(env, wp);
475 return 0;
476 }
477 }
478 return -ENOENT;
479 }
480
481 /* Remove a specific watchpoint by reference. */
482 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
483 {
484 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
485
486 tlb_flush_page(env, watchpoint->vaddr);
487
488 g_free(watchpoint);
489 }
490
491 /* Remove all matching watchpoints. */
492 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
493 {
494 CPUWatchpoint *wp, *next;
495
496 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
497 if (wp->flags & mask)
498 cpu_watchpoint_remove_by_ref(env, wp);
499 }
500 }
501 #endif
502
503 /* Add a breakpoint. */
504 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
505 CPUBreakpoint **breakpoint)
506 {
507 #if defined(TARGET_HAS_ICE)
508 CPUBreakpoint *bp;
509
510 bp = g_malloc(sizeof(*bp));
511
512 bp->pc = pc;
513 bp->flags = flags;
514
515 /* keep all GDB-injected breakpoints in front */
516 if (flags & BP_GDB) {
517 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
518 } else {
519 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
520 }
521
522 breakpoint_invalidate(ENV_GET_CPU(env), pc);
523
524 if (breakpoint) {
525 *breakpoint = bp;
526 }
527 return 0;
528 #else
529 return -ENOSYS;
530 #endif
531 }
532
533 /* Remove a specific breakpoint. */
534 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
535 {
536 #if defined(TARGET_HAS_ICE)
537 CPUBreakpoint *bp;
538
539 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
540 if (bp->pc == pc && bp->flags == flags) {
541 cpu_breakpoint_remove_by_ref(env, bp);
542 return 0;
543 }
544 }
545 return -ENOENT;
546 #else
547 return -ENOSYS;
548 #endif
549 }
550
551 /* Remove a specific breakpoint by reference. */
552 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
553 {
554 #if defined(TARGET_HAS_ICE)
555 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
556
557 breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
558
559 g_free(breakpoint);
560 #endif
561 }
562
563 /* Remove all matching breakpoints. */
564 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
565 {
566 #if defined(TARGET_HAS_ICE)
567 CPUBreakpoint *bp, *next;
568
569 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
570 if (bp->flags & mask)
571 cpu_breakpoint_remove_by_ref(env, bp);
572 }
573 #endif
574 }
575
576 /* enable or disable single step mode. EXCP_DEBUG is returned by the
577 CPU loop after each instruction */
578 void cpu_single_step(CPUState *cpu, int enabled)
579 {
580 #if defined(TARGET_HAS_ICE)
581 if (cpu->singlestep_enabled != enabled) {
582 cpu->singlestep_enabled = enabled;
583 if (kvm_enabled()) {
584 kvm_update_guest_debug(cpu, 0);
585 } else {
586 /* must flush all the translated code to avoid inconsistencies */
587 /* XXX: only flush what is necessary */
588 CPUArchState *env = cpu->env_ptr;
589 tb_flush(env);
590 }
591 }
592 #endif
593 }
594
595 void cpu_abort(CPUArchState *env, const char *fmt, ...)
596 {
597 CPUState *cpu = ENV_GET_CPU(env);
598 va_list ap;
599 va_list ap2;
600
601 va_start(ap, fmt);
602 va_copy(ap2, ap);
603 fprintf(stderr, "qemu: fatal: ");
604 vfprintf(stderr, fmt, ap);
605 fprintf(stderr, "\n");
606 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
607 if (qemu_log_enabled()) {
608 qemu_log("qemu: fatal: ");
609 qemu_log_vprintf(fmt, ap2);
610 qemu_log("\n");
611 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
612 qemu_log_flush();
613 qemu_log_close();
614 }
615 va_end(ap2);
616 va_end(ap);
617 #if defined(CONFIG_USER_ONLY)
618 {
619 struct sigaction act;
620 sigfillset(&act.sa_mask);
621 act.sa_handler = SIG_DFL;
622 sigaction(SIGABRT, &act, NULL);
623 }
624 #endif
625 abort();
626 }
627
628 CPUArchState *cpu_copy(CPUArchState *env)
629 {
630 CPUArchState *new_env = cpu_init(env->cpu_model_str);
631 #if defined(TARGET_HAS_ICE)
632 CPUBreakpoint *bp;
633 CPUWatchpoint *wp;
634 #endif
635
636 /* Reset non arch specific state */
637 cpu_reset(ENV_GET_CPU(new_env));
638
639 /* Copy arch specific state into the new CPU */
640 memcpy(new_env, env, sizeof(CPUArchState));
641
642 /* Clone all break/watchpoints.
643 Note: Once we support ptrace with hw-debug register access, make sure
644 BP_CPU break/watchpoints are handled correctly on clone. */
645 QTAILQ_INIT(&env->breakpoints);
646 QTAILQ_INIT(&env->watchpoints);
647 #if defined(TARGET_HAS_ICE)
648 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
649 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
650 }
651 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
652 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
653 wp->flags, NULL);
654 }
655 #endif
656
657 return new_env;
658 }
659
660 #if !defined(CONFIG_USER_ONLY)
661 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
662 uintptr_t length)
663 {
664 uintptr_t start1;
665
666 /* we modify the TLB cache so that the dirty bit will be set again
667 when accessing the range */
668 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
669 /* Check that we don't span multiple blocks - this breaks the
670 address comparisons below. */
671 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
672 != (end - 1) - start) {
673 abort();
674 }
675 cpu_tlb_reset_dirty_all(start1, length);
676
677 }
678
679 /* Note: start and end must be within the same ram block. */
680 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
681 int dirty_flags)
682 {
683 uintptr_t length;
684
685 start &= TARGET_PAGE_MASK;
686 end = TARGET_PAGE_ALIGN(end);
687
688 length = end - start;
689 if (length == 0)
690 return;
691 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
692
693 if (tcg_enabled()) {
694 tlb_reset_dirty_range_all(start, end, length);
695 }
696 }
697
698 static int cpu_physical_memory_set_dirty_tracking(int enable)
699 {
700 int ret = 0;
701 in_migration = enable;
702 return ret;
703 }
704
705 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
706 MemoryRegionSection *section,
707 target_ulong vaddr,
708 hwaddr paddr, hwaddr xlat,
709 int prot,
710 target_ulong *address)
711 {
712 hwaddr iotlb;
713 CPUWatchpoint *wp;
714
715 if (memory_region_is_ram(section->mr)) {
716 /* Normal RAM. */
717 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
718 + xlat;
719 if (!section->readonly) {
720 iotlb |= PHYS_SECTION_NOTDIRTY;
721 } else {
722 iotlb |= PHYS_SECTION_ROM;
723 }
724 } else {
725 iotlb = section - address_space_memory.dispatch->sections;
726 iotlb += xlat;
727 }
728
729 /* Make accesses to pages with watchpoints go via the
730 watchpoint trap routines. */
731 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
732 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
733 /* Avoid trapping reads of pages with a write breakpoint. */
734 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
735 iotlb = PHYS_SECTION_WATCH + paddr;
736 *address |= TLB_MMIO;
737 break;
738 }
739 }
740 }
741
742 return iotlb;
743 }
744 #endif /* defined(CONFIG_USER_ONLY) */
745
746 #if !defined(CONFIG_USER_ONLY)
747
748 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
749 uint16_t section);
750 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
751
752 static void *(*phys_mem_alloc)(ram_addr_t size) = qemu_anon_ram_alloc;
753
754 /*
755 * Set a custom physical guest memory alloator.
756 * Accelerators with unusual needs may need this. Hopefully, we can
757 * get rid of it eventually.
758 */
759 void phys_mem_set_alloc(void *(*alloc)(ram_addr_t))
760 {
761 phys_mem_alloc = alloc;
762 }
763
764 static uint16_t phys_section_add(MemoryRegionSection *section)
765 {
766 /* The physical section number is ORed with a page-aligned
767 * pointer to produce the iotlb entries. Thus it should
768 * never overflow into the page-aligned value.
769 */
770 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
771
772 if (next_map.sections_nb == next_map.sections_nb_alloc) {
773 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
774 16);
775 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
776 next_map.sections_nb_alloc);
777 }
778 next_map.sections[next_map.sections_nb] = *section;
779 memory_region_ref(section->mr);
780 return next_map.sections_nb++;
781 }
782
783 static void phys_section_destroy(MemoryRegion *mr)
784 {
785 memory_region_unref(mr);
786
787 if (mr->subpage) {
788 subpage_t *subpage = container_of(mr, subpage_t, iomem);
789 memory_region_destroy(&subpage->iomem);
790 g_free(subpage);
791 }
792 }
793
794 static void phys_sections_free(PhysPageMap *map)
795 {
796 while (map->sections_nb > 0) {
797 MemoryRegionSection *section = &map->sections[--map->sections_nb];
798 phys_section_destroy(section->mr);
799 }
800 g_free(map->sections);
801 g_free(map->nodes);
802 g_free(map);
803 }
804
805 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
806 {
807 subpage_t *subpage;
808 hwaddr base = section->offset_within_address_space
809 & TARGET_PAGE_MASK;
810 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
811 next_map.nodes, next_map.sections);
812 MemoryRegionSection subsection = {
813 .offset_within_address_space = base,
814 .size = int128_make64(TARGET_PAGE_SIZE),
815 };
816 hwaddr start, end;
817
818 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
819
820 if (!(existing->mr->subpage)) {
821 subpage = subpage_init(d->as, base);
822 subsection.mr = &subpage->iomem;
823 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
824 phys_section_add(&subsection));
825 } else {
826 subpage = container_of(existing->mr, subpage_t, iomem);
827 }
828 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
829 end = start + int128_get64(section->size) - 1;
830 subpage_register(subpage, start, end, phys_section_add(section));
831 }
832
833
834 static void register_multipage(AddressSpaceDispatch *d,
835 MemoryRegionSection *section)
836 {
837 hwaddr start_addr = section->offset_within_address_space;
838 uint16_t section_index = phys_section_add(section);
839 uint64_t num_pages = int128_get64(int128_rshift(section->size,
840 TARGET_PAGE_BITS));
841
842 assert(num_pages);
843 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
844 }
845
846 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
847 {
848 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
849 AddressSpaceDispatch *d = as->next_dispatch;
850 MemoryRegionSection now = *section, remain = *section;
851 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
852
853 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
854 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
855 - now.offset_within_address_space;
856
857 now.size = int128_min(int128_make64(left), now.size);
858 register_subpage(d, &now);
859 } else {
860 now.size = int128_zero();
861 }
862 while (int128_ne(remain.size, now.size)) {
863 remain.size = int128_sub(remain.size, now.size);
864 remain.offset_within_address_space += int128_get64(now.size);
865 remain.offset_within_region += int128_get64(now.size);
866 now = remain;
867 if (int128_lt(remain.size, page_size)) {
868 register_subpage(d, &now);
869 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
870 now.size = page_size;
871 register_subpage(d, &now);
872 } else {
873 now.size = int128_and(now.size, int128_neg(page_size));
874 register_multipage(d, &now);
875 }
876 }
877 }
878
879 void qemu_flush_coalesced_mmio_buffer(void)
880 {
881 if (kvm_enabled())
882 kvm_flush_coalesced_mmio_buffer();
883 }
884
885 void qemu_mutex_lock_ramlist(void)
886 {
887 qemu_mutex_lock(&ram_list.mutex);
888 }
889
890 void qemu_mutex_unlock_ramlist(void)
891 {
892 qemu_mutex_unlock(&ram_list.mutex);
893 }
894
895 #ifdef __linux__
896
897 #include <sys/vfs.h>
898
899 #define HUGETLBFS_MAGIC 0x958458f6
900
901 static long gethugepagesize(const char *path)
902 {
903 struct statfs fs;
904 int ret;
905
906 do {
907 ret = statfs(path, &fs);
908 } while (ret != 0 && errno == EINTR);
909
910 if (ret != 0) {
911 perror(path);
912 return 0;
913 }
914
915 if (fs.f_type != HUGETLBFS_MAGIC)
916 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
917
918 return fs.f_bsize;
919 }
920
921 static void *file_ram_alloc(RAMBlock *block,
922 ram_addr_t memory,
923 const char *path)
924 {
925 char *filename;
926 char *sanitized_name;
927 char *c;
928 void *area;
929 int fd;
930 #ifdef MAP_POPULATE
931 int flags;
932 #endif
933 unsigned long hpagesize;
934
935 hpagesize = gethugepagesize(path);
936 if (!hpagesize) {
937 return NULL;
938 }
939
940 if (memory < hpagesize) {
941 return NULL;
942 }
943
944 if (kvm_enabled() && !kvm_has_sync_mmu()) {
945 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
946 return NULL;
947 }
948
949 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
950 sanitized_name = g_strdup(block->mr->name);
951 for (c = sanitized_name; *c != '\0'; c++) {
952 if (*c == '/')
953 *c = '_';
954 }
955
956 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
957 sanitized_name);
958 g_free(sanitized_name);
959
960 fd = mkstemp(filename);
961 if (fd < 0) {
962 perror("unable to create backing store for hugepages");
963 g_free(filename);
964 return NULL;
965 }
966 unlink(filename);
967 g_free(filename);
968
969 memory = (memory+hpagesize-1) & ~(hpagesize-1);
970
971 /*
972 * ftruncate is not supported by hugetlbfs in older
973 * hosts, so don't bother bailing out on errors.
974 * If anything goes wrong with it under other filesystems,
975 * mmap will fail.
976 */
977 if (ftruncate(fd, memory))
978 perror("ftruncate");
979
980 #ifdef MAP_POPULATE
981 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
982 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
983 * to sidestep this quirk.
984 */
985 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
986 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
987 #else
988 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
989 #endif
990 if (area == MAP_FAILED) {
991 perror("file_ram_alloc: can't mmap RAM pages");
992 close(fd);
993 return (NULL);
994 }
995 block->fd = fd;
996 return area;
997 }
998 #else
999 static void *file_ram_alloc(RAMBlock *block,
1000 ram_addr_t memory,
1001 const char *path)
1002 {
1003 fprintf(stderr, "-mem-path not supported on this host\n");
1004 exit(1);
1005 }
1006 #endif
1007
1008 static ram_addr_t find_ram_offset(ram_addr_t size)
1009 {
1010 RAMBlock *block, *next_block;
1011 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1012
1013 assert(size != 0); /* it would hand out same offset multiple times */
1014
1015 if (QTAILQ_EMPTY(&ram_list.blocks))
1016 return 0;
1017
1018 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1019 ram_addr_t end, next = RAM_ADDR_MAX;
1020
1021 end = block->offset + block->length;
1022
1023 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1024 if (next_block->offset >= end) {
1025 next = MIN(next, next_block->offset);
1026 }
1027 }
1028 if (next - end >= size && next - end < mingap) {
1029 offset = end;
1030 mingap = next - end;
1031 }
1032 }
1033
1034 if (offset == RAM_ADDR_MAX) {
1035 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1036 (uint64_t)size);
1037 abort();
1038 }
1039
1040 return offset;
1041 }
1042
1043 ram_addr_t last_ram_offset(void)
1044 {
1045 RAMBlock *block;
1046 ram_addr_t last = 0;
1047
1048 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1049 last = MAX(last, block->offset + block->length);
1050
1051 return last;
1052 }
1053
1054 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1055 {
1056 int ret;
1057
1058 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1059 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1060 "dump-guest-core", true)) {
1061 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1062 if (ret) {
1063 perror("qemu_madvise");
1064 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1065 "but dump_guest_core=off specified\n");
1066 }
1067 }
1068 }
1069
1070 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1071 {
1072 RAMBlock *new_block, *block;
1073
1074 new_block = NULL;
1075 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1076 if (block->offset == addr) {
1077 new_block = block;
1078 break;
1079 }
1080 }
1081 assert(new_block);
1082 assert(!new_block->idstr[0]);
1083
1084 if (dev) {
1085 char *id = qdev_get_dev_path(dev);
1086 if (id) {
1087 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1088 g_free(id);
1089 }
1090 }
1091 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1092
1093 /* This assumes the iothread lock is taken here too. */
1094 qemu_mutex_lock_ramlist();
1095 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1096 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1097 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1098 new_block->idstr);
1099 abort();
1100 }
1101 }
1102 qemu_mutex_unlock_ramlist();
1103 }
1104
1105 static int memory_try_enable_merging(void *addr, size_t len)
1106 {
1107 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1108 /* disabled by the user */
1109 return 0;
1110 }
1111
1112 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1113 }
1114
1115 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1116 MemoryRegion *mr)
1117 {
1118 RAMBlock *block, *new_block;
1119
1120 size = TARGET_PAGE_ALIGN(size);
1121 new_block = g_malloc0(sizeof(*new_block));
1122 new_block->fd = -1;
1123
1124 /* This assumes the iothread lock is taken here too. */
1125 qemu_mutex_lock_ramlist();
1126 new_block->mr = mr;
1127 new_block->offset = find_ram_offset(size);
1128 if (host) {
1129 new_block->host = host;
1130 new_block->flags |= RAM_PREALLOC_MASK;
1131 } else if (xen_enabled()) {
1132 if (mem_path) {
1133 fprintf(stderr, "-mem-path not supported with Xen\n");
1134 exit(1);
1135 }
1136 xen_ram_alloc(new_block->offset, size, mr);
1137 } else {
1138 if (mem_path) {
1139 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1140 /*
1141 * file_ram_alloc() needs to allocate just like
1142 * phys_mem_alloc, but we haven't bothered to provide
1143 * a hook there.
1144 */
1145 fprintf(stderr,
1146 "-mem-path not supported with this accelerator\n");
1147 exit(1);
1148 }
1149 new_block->host = file_ram_alloc(new_block, size, mem_path);
1150 }
1151 if (!new_block->host) {
1152 new_block->host = phys_mem_alloc(size);
1153 if (!new_block->host) {
1154 fprintf(stderr, "Cannot set up guest memory '%s': %s\n",
1155 new_block->mr->name, strerror(errno));
1156 exit(1);
1157 }
1158 memory_try_enable_merging(new_block->host, size);
1159 }
1160 }
1161 new_block->length = size;
1162
1163 /* Keep the list sorted from biggest to smallest block. */
1164 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1165 if (block->length < new_block->length) {
1166 break;
1167 }
1168 }
1169 if (block) {
1170 QTAILQ_INSERT_BEFORE(block, new_block, next);
1171 } else {
1172 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1173 }
1174 ram_list.mru_block = NULL;
1175
1176 ram_list.version++;
1177 qemu_mutex_unlock_ramlist();
1178
1179 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1180 last_ram_offset() >> TARGET_PAGE_BITS);
1181 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1182 0, size >> TARGET_PAGE_BITS);
1183 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1184
1185 qemu_ram_setup_dump(new_block->host, size);
1186 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1187 qemu_madvise(new_block->host, size, QEMU_MADV_DONTFORK);
1188
1189 if (kvm_enabled())
1190 kvm_setup_guest_memory(new_block->host, size);
1191
1192 return new_block->offset;
1193 }
1194
1195 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1196 {
1197 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1198 }
1199
1200 void qemu_ram_free_from_ptr(ram_addr_t addr)
1201 {
1202 RAMBlock *block;
1203
1204 /* This assumes the iothread lock is taken here too. */
1205 qemu_mutex_lock_ramlist();
1206 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1207 if (addr == block->offset) {
1208 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1209 ram_list.mru_block = NULL;
1210 ram_list.version++;
1211 g_free(block);
1212 break;
1213 }
1214 }
1215 qemu_mutex_unlock_ramlist();
1216 }
1217
1218 void qemu_ram_free(ram_addr_t addr)
1219 {
1220 RAMBlock *block;
1221
1222 /* This assumes the iothread lock is taken here too. */
1223 qemu_mutex_lock_ramlist();
1224 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1225 if (addr == block->offset) {
1226 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1227 ram_list.mru_block = NULL;
1228 ram_list.version++;
1229 if (block->flags & RAM_PREALLOC_MASK) {
1230 ;
1231 } else if (xen_enabled()) {
1232 xen_invalidate_map_cache_entry(block->host);
1233 #ifndef _WIN32
1234 } else if (block->fd >= 0) {
1235 munmap(block->host, block->length);
1236 close(block->fd);
1237 #endif
1238 } else {
1239 qemu_anon_ram_free(block->host, block->length);
1240 }
1241 g_free(block);
1242 break;
1243 }
1244 }
1245 qemu_mutex_unlock_ramlist();
1246
1247 }
1248
1249 #ifndef _WIN32
1250 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1251 {
1252 RAMBlock *block;
1253 ram_addr_t offset;
1254 int flags;
1255 void *area, *vaddr;
1256
1257 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1258 offset = addr - block->offset;
1259 if (offset < block->length) {
1260 vaddr = block->host + offset;
1261 if (block->flags & RAM_PREALLOC_MASK) {
1262 ;
1263 } else if (xen_enabled()) {
1264 abort();
1265 } else {
1266 flags = MAP_FIXED;
1267 munmap(vaddr, length);
1268 if (block->fd >= 0) {
1269 #ifdef MAP_POPULATE
1270 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1271 MAP_PRIVATE;
1272 #else
1273 flags |= MAP_PRIVATE;
1274 #endif
1275 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1276 flags, block->fd, offset);
1277 } else {
1278 /*
1279 * Remap needs to match alloc. Accelerators that
1280 * set phys_mem_alloc never remap. If they did,
1281 * we'd need a remap hook here.
1282 */
1283 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1284
1285 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1286 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1287 flags, -1, 0);
1288 }
1289 if (area != vaddr) {
1290 fprintf(stderr, "Could not remap addr: "
1291 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1292 length, addr);
1293 exit(1);
1294 }
1295 memory_try_enable_merging(vaddr, length);
1296 qemu_ram_setup_dump(vaddr, length);
1297 }
1298 return;
1299 }
1300 }
1301 }
1302 #endif /* !_WIN32 */
1303
1304 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1305 {
1306 RAMBlock *block;
1307
1308 /* The list is protected by the iothread lock here. */
1309 block = ram_list.mru_block;
1310 if (block && addr - block->offset < block->length) {
1311 goto found;
1312 }
1313 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1314 if (addr - block->offset < block->length) {
1315 goto found;
1316 }
1317 }
1318
1319 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1320 abort();
1321
1322 found:
1323 ram_list.mru_block = block;
1324 return block;
1325 }
1326
1327 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1328 With the exception of the softmmu code in this file, this should
1329 only be used for local memory (e.g. video ram) that the device owns,
1330 and knows it isn't going to access beyond the end of the block.
1331
1332 It should not be used for general purpose DMA.
1333 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1334 */
1335 void *qemu_get_ram_ptr(ram_addr_t addr)
1336 {
1337 RAMBlock *block = qemu_get_ram_block(addr);
1338
1339 if (xen_enabled()) {
1340 /* We need to check if the requested address is in the RAM
1341 * because we don't want to map the entire memory in QEMU.
1342 * In that case just map until the end of the page.
1343 */
1344 if (block->offset == 0) {
1345 return xen_map_cache(addr, 0, 0);
1346 } else if (block->host == NULL) {
1347 block->host =
1348 xen_map_cache(block->offset, block->length, 1);
1349 }
1350 }
1351 return block->host + (addr - block->offset);
1352 }
1353
1354 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1355 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1356 *
1357 * ??? Is this still necessary?
1358 */
1359 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1360 {
1361 RAMBlock *block;
1362
1363 /* The list is protected by the iothread lock here. */
1364 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1365 if (addr - block->offset < block->length) {
1366 if (xen_enabled()) {
1367 /* We need to check if the requested address is in the RAM
1368 * because we don't want to map the entire memory in QEMU.
1369 * In that case just map until the end of the page.
1370 */
1371 if (block->offset == 0) {
1372 return xen_map_cache(addr, 0, 0);
1373 } else if (block->host == NULL) {
1374 block->host =
1375 xen_map_cache(block->offset, block->length, 1);
1376 }
1377 }
1378 return block->host + (addr - block->offset);
1379 }
1380 }
1381
1382 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1383 abort();
1384
1385 return NULL;
1386 }
1387
1388 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1389 * but takes a size argument */
1390 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1391 {
1392 if (*size == 0) {
1393 return NULL;
1394 }
1395 if (xen_enabled()) {
1396 return xen_map_cache(addr, *size, 1);
1397 } else {
1398 RAMBlock *block;
1399
1400 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1401 if (addr - block->offset < block->length) {
1402 if (addr - block->offset + *size > block->length)
1403 *size = block->length - addr + block->offset;
1404 return block->host + (addr - block->offset);
1405 }
1406 }
1407
1408 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1409 abort();
1410 }
1411 }
1412
1413 /* Some of the softmmu routines need to translate from a host pointer
1414 (typically a TLB entry) back to a ram offset. */
1415 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1416 {
1417 RAMBlock *block;
1418 uint8_t *host = ptr;
1419
1420 if (xen_enabled()) {
1421 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1422 return qemu_get_ram_block(*ram_addr)->mr;
1423 }
1424
1425 block = ram_list.mru_block;
1426 if (block && block->host && host - block->host < block->length) {
1427 goto found;
1428 }
1429
1430 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1431 /* This case append when the block is not mapped. */
1432 if (block->host == NULL) {
1433 continue;
1434 }
1435 if (host - block->host < block->length) {
1436 goto found;
1437 }
1438 }
1439
1440 return NULL;
1441
1442 found:
1443 *ram_addr = block->offset + (host - block->host);
1444 return block->mr;
1445 }
1446
1447 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1448 uint64_t val, unsigned size)
1449 {
1450 int dirty_flags;
1451 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1452 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1453 tb_invalidate_phys_page_fast(ram_addr, size);
1454 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1455 }
1456 switch (size) {
1457 case 1:
1458 stb_p(qemu_get_ram_ptr(ram_addr), val);
1459 break;
1460 case 2:
1461 stw_p(qemu_get_ram_ptr(ram_addr), val);
1462 break;
1463 case 4:
1464 stl_p(qemu_get_ram_ptr(ram_addr), val);
1465 break;
1466 default:
1467 abort();
1468 }
1469 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1470 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1471 /* we remove the notdirty callback only if the code has been
1472 flushed */
1473 if (dirty_flags == 0xff) {
1474 CPUArchState *env = current_cpu->env_ptr;
1475 tlb_set_dirty(env, env->mem_io_vaddr);
1476 }
1477 }
1478
1479 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1480 unsigned size, bool is_write)
1481 {
1482 return is_write;
1483 }
1484
1485 static const MemoryRegionOps notdirty_mem_ops = {
1486 .write = notdirty_mem_write,
1487 .valid.accepts = notdirty_mem_accepts,
1488 .endianness = DEVICE_NATIVE_ENDIAN,
1489 };
1490
1491 /* Generate a debug exception if a watchpoint has been hit. */
1492 static void check_watchpoint(int offset, int len_mask, int flags)
1493 {
1494 CPUArchState *env = current_cpu->env_ptr;
1495 target_ulong pc, cs_base;
1496 target_ulong vaddr;
1497 CPUWatchpoint *wp;
1498 int cpu_flags;
1499
1500 if (env->watchpoint_hit) {
1501 /* We re-entered the check after replacing the TB. Now raise
1502 * the debug interrupt so that is will trigger after the
1503 * current instruction. */
1504 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1505 return;
1506 }
1507 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1508 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1509 if ((vaddr == (wp->vaddr & len_mask) ||
1510 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1511 wp->flags |= BP_WATCHPOINT_HIT;
1512 if (!env->watchpoint_hit) {
1513 env->watchpoint_hit = wp;
1514 tb_check_watchpoint(env);
1515 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1516 env->exception_index = EXCP_DEBUG;
1517 cpu_loop_exit(env);
1518 } else {
1519 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1520 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1521 cpu_resume_from_signal(env, NULL);
1522 }
1523 }
1524 } else {
1525 wp->flags &= ~BP_WATCHPOINT_HIT;
1526 }
1527 }
1528 }
1529
1530 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1531 so these check for a hit then pass through to the normal out-of-line
1532 phys routines. */
1533 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1534 unsigned size)
1535 {
1536 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1537 switch (size) {
1538 case 1: return ldub_phys(addr);
1539 case 2: return lduw_phys(addr);
1540 case 4: return ldl_phys(addr);
1541 default: abort();
1542 }
1543 }
1544
1545 static void watch_mem_write(void *opaque, hwaddr addr,
1546 uint64_t val, unsigned size)
1547 {
1548 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1549 switch (size) {
1550 case 1:
1551 stb_phys(addr, val);
1552 break;
1553 case 2:
1554 stw_phys(addr, val);
1555 break;
1556 case 4:
1557 stl_phys(addr, val);
1558 break;
1559 default: abort();
1560 }
1561 }
1562
1563 static const MemoryRegionOps watch_mem_ops = {
1564 .read = watch_mem_read,
1565 .write = watch_mem_write,
1566 .endianness = DEVICE_NATIVE_ENDIAN,
1567 };
1568
1569 static uint64_t subpage_read(void *opaque, hwaddr addr,
1570 unsigned len)
1571 {
1572 subpage_t *subpage = opaque;
1573 uint8_t buf[4];
1574
1575 #if defined(DEBUG_SUBPAGE)
1576 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1577 subpage, len, addr);
1578 #endif
1579 address_space_read(subpage->as, addr + subpage->base, buf, len);
1580 switch (len) {
1581 case 1:
1582 return ldub_p(buf);
1583 case 2:
1584 return lduw_p(buf);
1585 case 4:
1586 return ldl_p(buf);
1587 default:
1588 abort();
1589 }
1590 }
1591
1592 static void subpage_write(void *opaque, hwaddr addr,
1593 uint64_t value, unsigned len)
1594 {
1595 subpage_t *subpage = opaque;
1596 uint8_t buf[4];
1597
1598 #if defined(DEBUG_SUBPAGE)
1599 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1600 " value %"PRIx64"\n",
1601 __func__, subpage, len, addr, value);
1602 #endif
1603 switch (len) {
1604 case 1:
1605 stb_p(buf, value);
1606 break;
1607 case 2:
1608 stw_p(buf, value);
1609 break;
1610 case 4:
1611 stl_p(buf, value);
1612 break;
1613 default:
1614 abort();
1615 }
1616 address_space_write(subpage->as, addr + subpage->base, buf, len);
1617 }
1618
1619 static bool subpage_accepts(void *opaque, hwaddr addr,
1620 unsigned size, bool is_write)
1621 {
1622 subpage_t *subpage = opaque;
1623 #if defined(DEBUG_SUBPAGE)
1624 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1625 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1626 #endif
1627
1628 return address_space_access_valid(subpage->as, addr + subpage->base,
1629 size, is_write);
1630 }
1631
1632 static const MemoryRegionOps subpage_ops = {
1633 .read = subpage_read,
1634 .write = subpage_write,
1635 .valid.accepts = subpage_accepts,
1636 .endianness = DEVICE_NATIVE_ENDIAN,
1637 };
1638
1639 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1640 uint16_t section)
1641 {
1642 int idx, eidx;
1643
1644 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1645 return -1;
1646 idx = SUBPAGE_IDX(start);
1647 eidx = SUBPAGE_IDX(end);
1648 #if defined(DEBUG_SUBPAGE)
1649 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1650 mmio, start, end, idx, eidx, memory);
1651 #endif
1652 for (; idx <= eidx; idx++) {
1653 mmio->sub_section[idx] = section;
1654 }
1655
1656 return 0;
1657 }
1658
1659 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1660 {
1661 subpage_t *mmio;
1662
1663 mmio = g_malloc0(sizeof(subpage_t));
1664
1665 mmio->as = as;
1666 mmio->base = base;
1667 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1668 "subpage", TARGET_PAGE_SIZE);
1669 mmio->iomem.subpage = true;
1670 #if defined(DEBUG_SUBPAGE)
1671 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1672 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1673 #endif
1674 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1675
1676 return mmio;
1677 }
1678
1679 static uint16_t dummy_section(MemoryRegion *mr)
1680 {
1681 MemoryRegionSection section = {
1682 .mr = mr,
1683 .offset_within_address_space = 0,
1684 .offset_within_region = 0,
1685 .size = int128_2_64(),
1686 };
1687
1688 return phys_section_add(&section);
1689 }
1690
1691 MemoryRegion *iotlb_to_region(hwaddr index)
1692 {
1693 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1694 }
1695
1696 static void io_mem_init(void)
1697 {
1698 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1699 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1700 "unassigned", UINT64_MAX);
1701 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1702 "notdirty", UINT64_MAX);
1703 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1704 "watch", UINT64_MAX);
1705 }
1706
1707 static void mem_begin(MemoryListener *listener)
1708 {
1709 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1710 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1711
1712 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1713 d->as = as;
1714 as->next_dispatch = d;
1715 }
1716
1717 static void mem_commit(MemoryListener *listener)
1718 {
1719 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1720 AddressSpaceDispatch *cur = as->dispatch;
1721 AddressSpaceDispatch *next = as->next_dispatch;
1722
1723 next->nodes = next_map.nodes;
1724 next->sections = next_map.sections;
1725
1726 as->dispatch = next;
1727 g_free(cur);
1728 }
1729
1730 static void core_begin(MemoryListener *listener)
1731 {
1732 uint16_t n;
1733
1734 prev_map = g_new(PhysPageMap, 1);
1735 *prev_map = next_map;
1736
1737 memset(&next_map, 0, sizeof(next_map));
1738 n = dummy_section(&io_mem_unassigned);
1739 assert(n == PHYS_SECTION_UNASSIGNED);
1740 n = dummy_section(&io_mem_notdirty);
1741 assert(n == PHYS_SECTION_NOTDIRTY);
1742 n = dummy_section(&io_mem_rom);
1743 assert(n == PHYS_SECTION_ROM);
1744 n = dummy_section(&io_mem_watch);
1745 assert(n == PHYS_SECTION_WATCH);
1746 }
1747
1748 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1749 * All AddressSpaceDispatch instances have switched to the next map.
1750 */
1751 static void core_commit(MemoryListener *listener)
1752 {
1753 phys_sections_free(prev_map);
1754 }
1755
1756 static void tcg_commit(MemoryListener *listener)
1757 {
1758 CPUState *cpu;
1759
1760 /* since each CPU stores ram addresses in its TLB cache, we must
1761 reset the modified entries */
1762 /* XXX: slow ! */
1763 CPU_FOREACH(cpu) {
1764 CPUArchState *env = cpu->env_ptr;
1765
1766 tlb_flush(env, 1);
1767 }
1768 }
1769
1770 static void core_log_global_start(MemoryListener *listener)
1771 {
1772 cpu_physical_memory_set_dirty_tracking(1);
1773 }
1774
1775 static void core_log_global_stop(MemoryListener *listener)
1776 {
1777 cpu_physical_memory_set_dirty_tracking(0);
1778 }
1779
1780 static MemoryListener core_memory_listener = {
1781 .begin = core_begin,
1782 .commit = core_commit,
1783 .log_global_start = core_log_global_start,
1784 .log_global_stop = core_log_global_stop,
1785 .priority = 1,
1786 };
1787
1788 static MemoryListener tcg_memory_listener = {
1789 .commit = tcg_commit,
1790 };
1791
1792 void address_space_init_dispatch(AddressSpace *as)
1793 {
1794 as->dispatch = NULL;
1795 as->dispatch_listener = (MemoryListener) {
1796 .begin = mem_begin,
1797 .commit = mem_commit,
1798 .region_add = mem_add,
1799 .region_nop = mem_add,
1800 .priority = 0,
1801 };
1802 memory_listener_register(&as->dispatch_listener, as);
1803 }
1804
1805 void address_space_destroy_dispatch(AddressSpace *as)
1806 {
1807 AddressSpaceDispatch *d = as->dispatch;
1808
1809 memory_listener_unregister(&as->dispatch_listener);
1810 g_free(d);
1811 as->dispatch = NULL;
1812 }
1813
1814 static void memory_map_init(void)
1815 {
1816 system_memory = g_malloc(sizeof(*system_memory));
1817 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1818 address_space_init(&address_space_memory, system_memory, "memory");
1819
1820 system_io = g_malloc(sizeof(*system_io));
1821 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
1822 65536);
1823 address_space_init(&address_space_io, system_io, "I/O");
1824
1825 memory_listener_register(&core_memory_listener, &address_space_memory);
1826 if (tcg_enabled()) {
1827 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1828 }
1829 }
1830
1831 MemoryRegion *get_system_memory(void)
1832 {
1833 return system_memory;
1834 }
1835
1836 MemoryRegion *get_system_io(void)
1837 {
1838 return system_io;
1839 }
1840
1841 #endif /* !defined(CONFIG_USER_ONLY) */
1842
1843 /* physical memory access (slow version, mainly for debug) */
1844 #if defined(CONFIG_USER_ONLY)
1845 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1846 uint8_t *buf, int len, int is_write)
1847 {
1848 int l, flags;
1849 target_ulong page;
1850 void * p;
1851
1852 while (len > 0) {
1853 page = addr & TARGET_PAGE_MASK;
1854 l = (page + TARGET_PAGE_SIZE) - addr;
1855 if (l > len)
1856 l = len;
1857 flags = page_get_flags(page);
1858 if (!(flags & PAGE_VALID))
1859 return -1;
1860 if (is_write) {
1861 if (!(flags & PAGE_WRITE))
1862 return -1;
1863 /* XXX: this code should not depend on lock_user */
1864 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1865 return -1;
1866 memcpy(p, buf, l);
1867 unlock_user(p, addr, l);
1868 } else {
1869 if (!(flags & PAGE_READ))
1870 return -1;
1871 /* XXX: this code should not depend on lock_user */
1872 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1873 return -1;
1874 memcpy(buf, p, l);
1875 unlock_user(p, addr, 0);
1876 }
1877 len -= l;
1878 buf += l;
1879 addr += l;
1880 }
1881 return 0;
1882 }
1883
1884 #else
1885
1886 static void invalidate_and_set_dirty(hwaddr addr,
1887 hwaddr length)
1888 {
1889 if (!cpu_physical_memory_is_dirty(addr)) {
1890 /* invalidate code */
1891 tb_invalidate_phys_page_range(addr, addr + length, 0);
1892 /* set dirty bit */
1893 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1894 }
1895 xen_modified_memory(addr, length);
1896 }
1897
1898 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1899 {
1900 if (memory_region_is_ram(mr)) {
1901 return !(is_write && mr->readonly);
1902 }
1903 if (memory_region_is_romd(mr)) {
1904 return !is_write;
1905 }
1906
1907 return false;
1908 }
1909
1910 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1911 {
1912 unsigned access_size_max = mr->ops->valid.max_access_size;
1913
1914 /* Regions are assumed to support 1-4 byte accesses unless
1915 otherwise specified. */
1916 if (access_size_max == 0) {
1917 access_size_max = 4;
1918 }
1919
1920 /* Bound the maximum access by the alignment of the address. */
1921 if (!mr->ops->impl.unaligned) {
1922 unsigned align_size_max = addr & -addr;
1923 if (align_size_max != 0 && align_size_max < access_size_max) {
1924 access_size_max = align_size_max;
1925 }
1926 }
1927
1928 /* Don't attempt accesses larger than the maximum. */
1929 if (l > access_size_max) {
1930 l = access_size_max;
1931 }
1932 if (l & (l - 1)) {
1933 l = 1 << (qemu_fls(l) - 1);
1934 }
1935
1936 return l;
1937 }
1938
1939 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1940 int len, bool is_write)
1941 {
1942 hwaddr l;
1943 uint8_t *ptr;
1944 uint64_t val;
1945 hwaddr addr1;
1946 MemoryRegion *mr;
1947 bool error = false;
1948
1949 while (len > 0) {
1950 l = len;
1951 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1952
1953 if (is_write) {
1954 if (!memory_access_is_direct(mr, is_write)) {
1955 l = memory_access_size(mr, l, addr1);
1956 /* XXX: could force current_cpu to NULL to avoid
1957 potential bugs */
1958 switch (l) {
1959 case 8:
1960 /* 64 bit write access */
1961 val = ldq_p(buf);
1962 error |= io_mem_write(mr, addr1, val, 8);
1963 break;
1964 case 4:
1965 /* 32 bit write access */
1966 val = ldl_p(buf);
1967 error |= io_mem_write(mr, addr1, val, 4);
1968 break;
1969 case 2:
1970 /* 16 bit write access */
1971 val = lduw_p(buf);
1972 error |= io_mem_write(mr, addr1, val, 2);
1973 break;
1974 case 1:
1975 /* 8 bit write access */
1976 val = ldub_p(buf);
1977 error |= io_mem_write(mr, addr1, val, 1);
1978 break;
1979 default:
1980 abort();
1981 }
1982 } else {
1983 addr1 += memory_region_get_ram_addr(mr);
1984 /* RAM case */
1985 ptr = qemu_get_ram_ptr(addr1);
1986 memcpy(ptr, buf, l);
1987 invalidate_and_set_dirty(addr1, l);
1988 }
1989 } else {
1990 if (!memory_access_is_direct(mr, is_write)) {
1991 /* I/O case */
1992 l = memory_access_size(mr, l, addr1);
1993 switch (l) {
1994 case 8:
1995 /* 64 bit read access */
1996 error |= io_mem_read(mr, addr1, &val, 8);
1997 stq_p(buf, val);
1998 break;
1999 case 4:
2000 /* 32 bit read access */
2001 error |= io_mem_read(mr, addr1, &val, 4);
2002 stl_p(buf, val);
2003 break;
2004 case 2:
2005 /* 16 bit read access */
2006 error |= io_mem_read(mr, addr1, &val, 2);
2007 stw_p(buf, val);
2008 break;
2009 case 1:
2010 /* 8 bit read access */
2011 error |= io_mem_read(mr, addr1, &val, 1);
2012 stb_p(buf, val);
2013 break;
2014 default:
2015 abort();
2016 }
2017 } else {
2018 /* RAM case */
2019 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2020 memcpy(buf, ptr, l);
2021 }
2022 }
2023 len -= l;
2024 buf += l;
2025 addr += l;
2026 }
2027
2028 return error;
2029 }
2030
2031 bool address_space_write(AddressSpace *as, hwaddr addr,
2032 const uint8_t *buf, int len)
2033 {
2034 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2035 }
2036
2037 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2038 {
2039 return address_space_rw(as, addr, buf, len, false);
2040 }
2041
2042
2043 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2044 int len, int is_write)
2045 {
2046 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2047 }
2048
2049 /* used for ROM loading : can write in RAM and ROM */
2050 void cpu_physical_memory_write_rom(hwaddr addr,
2051 const uint8_t *buf, int len)
2052 {
2053 hwaddr l;
2054 uint8_t *ptr;
2055 hwaddr addr1;
2056 MemoryRegion *mr;
2057
2058 while (len > 0) {
2059 l = len;
2060 mr = address_space_translate(&address_space_memory,
2061 addr, &addr1, &l, true);
2062
2063 if (!(memory_region_is_ram(mr) ||
2064 memory_region_is_romd(mr))) {
2065 /* do nothing */
2066 } else {
2067 addr1 += memory_region_get_ram_addr(mr);
2068 /* ROM/RAM case */
2069 ptr = qemu_get_ram_ptr(addr1);
2070 memcpy(ptr, buf, l);
2071 invalidate_and_set_dirty(addr1, l);
2072 }
2073 len -= l;
2074 buf += l;
2075 addr += l;
2076 }
2077 }
2078
2079 typedef struct {
2080 MemoryRegion *mr;
2081 void *buffer;
2082 hwaddr addr;
2083 hwaddr len;
2084 } BounceBuffer;
2085
2086 static BounceBuffer bounce;
2087
2088 typedef struct MapClient {
2089 void *opaque;
2090 void (*callback)(void *opaque);
2091 QLIST_ENTRY(MapClient) link;
2092 } MapClient;
2093
2094 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2095 = QLIST_HEAD_INITIALIZER(map_client_list);
2096
2097 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2098 {
2099 MapClient *client = g_malloc(sizeof(*client));
2100
2101 client->opaque = opaque;
2102 client->callback = callback;
2103 QLIST_INSERT_HEAD(&map_client_list, client, link);
2104 return client;
2105 }
2106
2107 static void cpu_unregister_map_client(void *_client)
2108 {
2109 MapClient *client = (MapClient *)_client;
2110
2111 QLIST_REMOVE(client, link);
2112 g_free(client);
2113 }
2114
2115 static void cpu_notify_map_clients(void)
2116 {
2117 MapClient *client;
2118
2119 while (!QLIST_EMPTY(&map_client_list)) {
2120 client = QLIST_FIRST(&map_client_list);
2121 client->callback(client->opaque);
2122 cpu_unregister_map_client(client);
2123 }
2124 }
2125
2126 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2127 {
2128 MemoryRegion *mr;
2129 hwaddr l, xlat;
2130
2131 while (len > 0) {
2132 l = len;
2133 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2134 if (!memory_access_is_direct(mr, is_write)) {
2135 l = memory_access_size(mr, l, addr);
2136 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2137 return false;
2138 }
2139 }
2140
2141 len -= l;
2142 addr += l;
2143 }
2144 return true;
2145 }
2146
2147 /* Map a physical memory region into a host virtual address.
2148 * May map a subset of the requested range, given by and returned in *plen.
2149 * May return NULL if resources needed to perform the mapping are exhausted.
2150 * Use only for reads OR writes - not for read-modify-write operations.
2151 * Use cpu_register_map_client() to know when retrying the map operation is
2152 * likely to succeed.
2153 */
2154 void *address_space_map(AddressSpace *as,
2155 hwaddr addr,
2156 hwaddr *plen,
2157 bool is_write)
2158 {
2159 hwaddr len = *plen;
2160 hwaddr done = 0;
2161 hwaddr l, xlat, base;
2162 MemoryRegion *mr, *this_mr;
2163 ram_addr_t raddr;
2164
2165 if (len == 0) {
2166 return NULL;
2167 }
2168
2169 l = len;
2170 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2171 if (!memory_access_is_direct(mr, is_write)) {
2172 if (bounce.buffer) {
2173 return NULL;
2174 }
2175 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2176 bounce.addr = addr;
2177 bounce.len = l;
2178
2179 memory_region_ref(mr);
2180 bounce.mr = mr;
2181 if (!is_write) {
2182 address_space_read(as, addr, bounce.buffer, l);
2183 }
2184
2185 *plen = l;
2186 return bounce.buffer;
2187 }
2188
2189 base = xlat;
2190 raddr = memory_region_get_ram_addr(mr);
2191
2192 for (;;) {
2193 len -= l;
2194 addr += l;
2195 done += l;
2196 if (len == 0) {
2197 break;
2198 }
2199
2200 l = len;
2201 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2202 if (this_mr != mr || xlat != base + done) {
2203 break;
2204 }
2205 }
2206
2207 memory_region_ref(mr);
2208 *plen = done;
2209 return qemu_ram_ptr_length(raddr + base, plen);
2210 }
2211
2212 /* Unmaps a memory region previously mapped by address_space_map().
2213 * Will also mark the memory as dirty if is_write == 1. access_len gives
2214 * the amount of memory that was actually read or written by the caller.
2215 */
2216 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2217 int is_write, hwaddr access_len)
2218 {
2219 if (buffer != bounce.buffer) {
2220 MemoryRegion *mr;
2221 ram_addr_t addr1;
2222
2223 mr = qemu_ram_addr_from_host(buffer, &addr1);
2224 assert(mr != NULL);
2225 if (is_write) {
2226 while (access_len) {
2227 unsigned l;
2228 l = TARGET_PAGE_SIZE;
2229 if (l > access_len)
2230 l = access_len;
2231 invalidate_and_set_dirty(addr1, l);
2232 addr1 += l;
2233 access_len -= l;
2234 }
2235 }
2236 if (xen_enabled()) {
2237 xen_invalidate_map_cache_entry(buffer);
2238 }
2239 memory_region_unref(mr);
2240 return;
2241 }
2242 if (is_write) {
2243 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2244 }
2245 qemu_vfree(bounce.buffer);
2246 bounce.buffer = NULL;
2247 memory_region_unref(bounce.mr);
2248 cpu_notify_map_clients();
2249 }
2250
2251 void *cpu_physical_memory_map(hwaddr addr,
2252 hwaddr *plen,
2253 int is_write)
2254 {
2255 return address_space_map(&address_space_memory, addr, plen, is_write);
2256 }
2257
2258 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2259 int is_write, hwaddr access_len)
2260 {
2261 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2262 }
2263
2264 /* warning: addr must be aligned */
2265 static inline uint32_t ldl_phys_internal(hwaddr addr,
2266 enum device_endian endian)
2267 {
2268 uint8_t *ptr;
2269 uint64_t val;
2270 MemoryRegion *mr;
2271 hwaddr l = 4;
2272 hwaddr addr1;
2273
2274 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2275 false);
2276 if (l < 4 || !memory_access_is_direct(mr, false)) {
2277 /* I/O case */
2278 io_mem_read(mr, addr1, &val, 4);
2279 #if defined(TARGET_WORDS_BIGENDIAN)
2280 if (endian == DEVICE_LITTLE_ENDIAN) {
2281 val = bswap32(val);
2282 }
2283 #else
2284 if (endian == DEVICE_BIG_ENDIAN) {
2285 val = bswap32(val);
2286 }
2287 #endif
2288 } else {
2289 /* RAM case */
2290 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2291 & TARGET_PAGE_MASK)
2292 + addr1);
2293 switch (endian) {
2294 case DEVICE_LITTLE_ENDIAN:
2295 val = ldl_le_p(ptr);
2296 break;
2297 case DEVICE_BIG_ENDIAN:
2298 val = ldl_be_p(ptr);
2299 break;
2300 default:
2301 val = ldl_p(ptr);
2302 break;
2303 }
2304 }
2305 return val;
2306 }
2307
2308 uint32_t ldl_phys(hwaddr addr)
2309 {
2310 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2311 }
2312
2313 uint32_t ldl_le_phys(hwaddr addr)
2314 {
2315 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2316 }
2317
2318 uint32_t ldl_be_phys(hwaddr addr)
2319 {
2320 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2321 }
2322
2323 /* warning: addr must be aligned */
2324 static inline uint64_t ldq_phys_internal(hwaddr addr,
2325 enum device_endian endian)
2326 {
2327 uint8_t *ptr;
2328 uint64_t val;
2329 MemoryRegion *mr;
2330 hwaddr l = 8;
2331 hwaddr addr1;
2332
2333 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2334 false);
2335 if (l < 8 || !memory_access_is_direct(mr, false)) {
2336 /* I/O case */
2337 io_mem_read(mr, addr1, &val, 8);
2338 #if defined(TARGET_WORDS_BIGENDIAN)
2339 if (endian == DEVICE_LITTLE_ENDIAN) {
2340 val = bswap64(val);
2341 }
2342 #else
2343 if (endian == DEVICE_BIG_ENDIAN) {
2344 val = bswap64(val);
2345 }
2346 #endif
2347 } else {
2348 /* RAM case */
2349 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2350 & TARGET_PAGE_MASK)
2351 + addr1);
2352 switch (endian) {
2353 case DEVICE_LITTLE_ENDIAN:
2354 val = ldq_le_p(ptr);
2355 break;
2356 case DEVICE_BIG_ENDIAN:
2357 val = ldq_be_p(ptr);
2358 break;
2359 default:
2360 val = ldq_p(ptr);
2361 break;
2362 }
2363 }
2364 return val;
2365 }
2366
2367 uint64_t ldq_phys(hwaddr addr)
2368 {
2369 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2370 }
2371
2372 uint64_t ldq_le_phys(hwaddr addr)
2373 {
2374 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2375 }
2376
2377 uint64_t ldq_be_phys(hwaddr addr)
2378 {
2379 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2380 }
2381
2382 /* XXX: optimize */
2383 uint32_t ldub_phys(hwaddr addr)
2384 {
2385 uint8_t val;
2386 cpu_physical_memory_read(addr, &val, 1);
2387 return val;
2388 }
2389
2390 /* warning: addr must be aligned */
2391 static inline uint32_t lduw_phys_internal(hwaddr addr,
2392 enum device_endian endian)
2393 {
2394 uint8_t *ptr;
2395 uint64_t val;
2396 MemoryRegion *mr;
2397 hwaddr l = 2;
2398 hwaddr addr1;
2399
2400 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2401 false);
2402 if (l < 2 || !memory_access_is_direct(mr, false)) {
2403 /* I/O case */
2404 io_mem_read(mr, addr1, &val, 2);
2405 #if defined(TARGET_WORDS_BIGENDIAN)
2406 if (endian == DEVICE_LITTLE_ENDIAN) {
2407 val = bswap16(val);
2408 }
2409 #else
2410 if (endian == DEVICE_BIG_ENDIAN) {
2411 val = bswap16(val);
2412 }
2413 #endif
2414 } else {
2415 /* RAM case */
2416 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2417 & TARGET_PAGE_MASK)
2418 + addr1);
2419 switch (endian) {
2420 case DEVICE_LITTLE_ENDIAN:
2421 val = lduw_le_p(ptr);
2422 break;
2423 case DEVICE_BIG_ENDIAN:
2424 val = lduw_be_p(ptr);
2425 break;
2426 default:
2427 val = lduw_p(ptr);
2428 break;
2429 }
2430 }
2431 return val;
2432 }
2433
2434 uint32_t lduw_phys(hwaddr addr)
2435 {
2436 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2437 }
2438
2439 uint32_t lduw_le_phys(hwaddr addr)
2440 {
2441 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2442 }
2443
2444 uint32_t lduw_be_phys(hwaddr addr)
2445 {
2446 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2447 }
2448
2449 /* warning: addr must be aligned. The ram page is not masked as dirty
2450 and the code inside is not invalidated. It is useful if the dirty
2451 bits are used to track modified PTEs */
2452 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2453 {
2454 uint8_t *ptr;
2455 MemoryRegion *mr;
2456 hwaddr l = 4;
2457 hwaddr addr1;
2458
2459 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2460 true);
2461 if (l < 4 || !memory_access_is_direct(mr, true)) {
2462 io_mem_write(mr, addr1, val, 4);
2463 } else {
2464 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2465 ptr = qemu_get_ram_ptr(addr1);
2466 stl_p(ptr, val);
2467
2468 if (unlikely(in_migration)) {
2469 if (!cpu_physical_memory_is_dirty(addr1)) {
2470 /* invalidate code */
2471 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2472 /* set dirty bit */
2473 cpu_physical_memory_set_dirty_flags(
2474 addr1, (0xff & ~CODE_DIRTY_FLAG));
2475 }
2476 }
2477 }
2478 }
2479
2480 /* warning: addr must be aligned */
2481 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2482 enum device_endian endian)
2483 {
2484 uint8_t *ptr;
2485 MemoryRegion *mr;
2486 hwaddr l = 4;
2487 hwaddr addr1;
2488
2489 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2490 true);
2491 if (l < 4 || !memory_access_is_direct(mr, true)) {
2492 #if defined(TARGET_WORDS_BIGENDIAN)
2493 if (endian == DEVICE_LITTLE_ENDIAN) {
2494 val = bswap32(val);
2495 }
2496 #else
2497 if (endian == DEVICE_BIG_ENDIAN) {
2498 val = bswap32(val);
2499 }
2500 #endif
2501 io_mem_write(mr, addr1, val, 4);
2502 } else {
2503 /* RAM case */
2504 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2505 ptr = qemu_get_ram_ptr(addr1);
2506 switch (endian) {
2507 case DEVICE_LITTLE_ENDIAN:
2508 stl_le_p(ptr, val);
2509 break;
2510 case DEVICE_BIG_ENDIAN:
2511 stl_be_p(ptr, val);
2512 break;
2513 default:
2514 stl_p(ptr, val);
2515 break;
2516 }
2517 invalidate_and_set_dirty(addr1, 4);
2518 }
2519 }
2520
2521 void stl_phys(hwaddr addr, uint32_t val)
2522 {
2523 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2524 }
2525
2526 void stl_le_phys(hwaddr addr, uint32_t val)
2527 {
2528 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2529 }
2530
2531 void stl_be_phys(hwaddr addr, uint32_t val)
2532 {
2533 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2534 }
2535
2536 /* XXX: optimize */
2537 void stb_phys(hwaddr addr, uint32_t val)
2538 {
2539 uint8_t v = val;
2540 cpu_physical_memory_write(addr, &v, 1);
2541 }
2542
2543 /* warning: addr must be aligned */
2544 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2545 enum device_endian endian)
2546 {
2547 uint8_t *ptr;
2548 MemoryRegion *mr;
2549 hwaddr l = 2;
2550 hwaddr addr1;
2551
2552 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2553 true);
2554 if (l < 2 || !memory_access_is_direct(mr, true)) {
2555 #if defined(TARGET_WORDS_BIGENDIAN)
2556 if (endian == DEVICE_LITTLE_ENDIAN) {
2557 val = bswap16(val);
2558 }
2559 #else
2560 if (endian == DEVICE_BIG_ENDIAN) {
2561 val = bswap16(val);
2562 }
2563 #endif
2564 io_mem_write(mr, addr1, val, 2);
2565 } else {
2566 /* RAM case */
2567 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2568 ptr = qemu_get_ram_ptr(addr1);
2569 switch (endian) {
2570 case DEVICE_LITTLE_ENDIAN:
2571 stw_le_p(ptr, val);
2572 break;
2573 case DEVICE_BIG_ENDIAN:
2574 stw_be_p(ptr, val);
2575 break;
2576 default:
2577 stw_p(ptr, val);
2578 break;
2579 }
2580 invalidate_and_set_dirty(addr1, 2);
2581 }
2582 }
2583
2584 void stw_phys(hwaddr addr, uint32_t val)
2585 {
2586 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2587 }
2588
2589 void stw_le_phys(hwaddr addr, uint32_t val)
2590 {
2591 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2592 }
2593
2594 void stw_be_phys(hwaddr addr, uint32_t val)
2595 {
2596 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2597 }
2598
2599 /* XXX: optimize */
2600 void stq_phys(hwaddr addr, uint64_t val)
2601 {
2602 val = tswap64(val);
2603 cpu_physical_memory_write(addr, &val, 8);
2604 }
2605
2606 void stq_le_phys(hwaddr addr, uint64_t val)
2607 {
2608 val = cpu_to_le64(val);
2609 cpu_physical_memory_write(addr, &val, 8);
2610 }
2611
2612 void stq_be_phys(hwaddr addr, uint64_t val)
2613 {
2614 val = cpu_to_be64(val);
2615 cpu_physical_memory_write(addr, &val, 8);
2616 }
2617
2618 /* virtual memory access for debug (includes writing to ROM) */
2619 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2620 uint8_t *buf, int len, int is_write)
2621 {
2622 int l;
2623 hwaddr phys_addr;
2624 target_ulong page;
2625
2626 while (len > 0) {
2627 page = addr & TARGET_PAGE_MASK;
2628 phys_addr = cpu_get_phys_page_debug(cpu, page);
2629 /* if no physical page mapped, return an error */
2630 if (phys_addr == -1)
2631 return -1;
2632 l = (page + TARGET_PAGE_SIZE) - addr;
2633 if (l > len)
2634 l = len;
2635 phys_addr += (addr & ~TARGET_PAGE_MASK);
2636 if (is_write)
2637 cpu_physical_memory_write_rom(phys_addr, buf, l);
2638 else
2639 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2640 len -= l;
2641 buf += l;
2642 addr += l;
2643 }
2644 return 0;
2645 }
2646 #endif
2647
2648 #if !defined(CONFIG_USER_ONLY)
2649
2650 /*
2651 * A helper function for the _utterly broken_ virtio device model to find out if
2652 * it's running on a big endian machine. Don't do this at home kids!
2653 */
2654 bool virtio_is_big_endian(void);
2655 bool virtio_is_big_endian(void)
2656 {
2657 #if defined(TARGET_WORDS_BIGENDIAN)
2658 return true;
2659 #else
2660 return false;
2661 #endif
2662 }
2663
2664 #endif
2665
2666 #ifndef CONFIG_USER_ONLY
2667 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2668 {
2669 MemoryRegion*mr;
2670 hwaddr l = 1;
2671
2672 mr = address_space_translate(&address_space_memory,
2673 phys_addr, &phys_addr, &l, false);
2674
2675 return !(memory_region_is_ram(mr) ||
2676 memory_region_is_romd(mr));
2677 }
2678
2679 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2680 {
2681 RAMBlock *block;
2682
2683 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2684 func(block->host, block->offset, block->length, opaque);
2685 }
2686 }
2687 #endif
Random patches