?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
qemu-help: Sort devices by logical functionality
[qemu.git] / exec.c
blobc4f2894ea12b40a356ea37ddbdf116ce491e8735
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
51
52 #include "exec/memory-internal.h"
53
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 CPUState *first_cpu;
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 typedef PhysPageEntry Node[L2_SIZE];
92
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101 };
102
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
110
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
115
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
124
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
127
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
133
134 static MemoryRegion io_mem_watch;
135 #endif
136
137 #if !defined(CONFIG_USER_ONLY)
138
139 static void phys_map_node_reserve(unsigned nodes)
140 {
141 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
142 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
143 16);
144 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
145 next_map.nodes_nb + nodes);
146 next_map.nodes = g_renew(Node, next_map.nodes,
147 next_map.nodes_nb_alloc);
148 }
149 }
150
151 static uint16_t phys_map_node_alloc(void)
152 {
153 unsigned i;
154 uint16_t ret;
155
156 ret = next_map.nodes_nb++;
157 assert(ret != PHYS_MAP_NODE_NIL);
158 assert(ret != next_map.nodes_nb_alloc);
159 for (i = 0; i < L2_SIZE; ++i) {
160 next_map.nodes[ret][i].is_leaf = 0;
161 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
162 }
163 return ret;
164 }
165
166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
167 hwaddr *nb, uint16_t leaf,
168 int level)
169 {
170 PhysPageEntry *p;
171 int i;
172 hwaddr step = (hwaddr)1 << (level * L2_BITS);
173
174 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
175 lp->ptr = phys_map_node_alloc();
176 p = next_map.nodes[lp->ptr];
177 if (level == 0) {
178 for (i = 0; i < L2_SIZE; i++) {
179 p[i].is_leaf = 1;
180 p[i].ptr = PHYS_SECTION_UNASSIGNED;
181 }
182 }
183 } else {
184 p = next_map.nodes[lp->ptr];
185 }
186 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
187
188 while (*nb && lp < &p[L2_SIZE]) {
189 if ((*index & (step - 1)) == 0 && *nb >= step) {
190 lp->is_leaf = true;
191 lp->ptr = leaf;
192 *index += step;
193 *nb -= step;
194 } else {
195 phys_page_set_level(lp, index, nb, leaf, level - 1);
196 }
197 ++lp;
198 }
199 }
200
201 static void phys_page_set(AddressSpaceDispatch *d,
202 hwaddr index, hwaddr nb,
203 uint16_t leaf)
204 {
205 /* Wildly overreserve - it doesn't matter much. */
206 phys_map_node_reserve(3 * P_L2_LEVELS);
207
208 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
209 }
210
211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
212 Node *nodes, MemoryRegionSection *sections)
213 {
214 PhysPageEntry *p;
215 int i;
216
217 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
218 if (lp.ptr == PHYS_MAP_NODE_NIL) {
219 return &sections[PHYS_SECTION_UNASSIGNED];
220 }
221 p = nodes[lp.ptr];
222 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
223 }
224 return &sections[lp.ptr];
225 }
226
227 bool memory_region_is_unassigned(MemoryRegion *mr)
228 {
229 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
230 && mr != &io_mem_watch;
231 }
232
233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
234 hwaddr addr,
235 bool resolve_subpage)
236 {
237 MemoryRegionSection *section;
238 subpage_t *subpage;
239
240 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
241 d->nodes, d->sections);
242 if (resolve_subpage && section->mr->subpage) {
243 subpage = container_of(section->mr, subpage_t, iomem);
244 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
245 }
246 return section;
247 }
248
249 static MemoryRegionSection *
250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
251 hwaddr *plen, bool resolve_subpage)
252 {
253 MemoryRegionSection *section;
254 Int128 diff;
255
256 section = address_space_lookup_region(d, addr, resolve_subpage);
257 /* Compute offset within MemoryRegionSection */
258 addr -= section->offset_within_address_space;
259
260 /* Compute offset within MemoryRegion */
261 *xlat = addr + section->offset_within_region;
262
263 diff = int128_sub(section->mr->size, int128_make64(addr));
264 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
265 return section;
266 }
267
268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
269 hwaddr *xlat, hwaddr *plen,
270 bool is_write)
271 {
272 IOMMUTLBEntry iotlb;
273 MemoryRegionSection *section;
274 MemoryRegion *mr;
275 hwaddr len = *plen;
276
277 for (;;) {
278 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
279 mr = section->mr;
280
281 if (!mr->iommu_ops) {
282 break;
283 }
284
285 iotlb = mr->iommu_ops->translate(mr, addr);
286 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
287 | (addr & iotlb.addr_mask));
288 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
289 if (!(iotlb.perm & (1 << is_write))) {
290 mr = &io_mem_unassigned;
291 break;
292 }
293
294 as = iotlb.target_as;
295 }
296
297 *plen = len;
298 *xlat = addr;
299 return mr;
300 }
301
302 MemoryRegionSection *
303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
304 hwaddr *plen)
305 {
306 MemoryRegionSection *section;
307 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
308
309 assert(!section->mr->iommu_ops);
310 return section;
311 }
312 #endif
313
314 void cpu_exec_init_all(void)
315 {
316 #if !defined(CONFIG_USER_ONLY)
317 qemu_mutex_init(&ram_list.mutex);
318 memory_map_init();
319 io_mem_init();
320 #endif
321 }
322
323 #if !defined(CONFIG_USER_ONLY)
324
325 static int cpu_common_post_load(void *opaque, int version_id)
326 {
327 CPUState *cpu = opaque;
328
329 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
330 version_id is increased. */
331 cpu->interrupt_request &= ~0x01;
332 tlb_flush(cpu->env_ptr, 1);
333
334 return 0;
335 }
336
337 const VMStateDescription vmstate_cpu_common = {
338 .name = "cpu_common",
339 .version_id = 1,
340 .minimum_version_id = 1,
341 .minimum_version_id_old = 1,
342 .post_load = cpu_common_post_load,
343 .fields = (VMStateField []) {
344 VMSTATE_UINT32(halted, CPUState),
345 VMSTATE_UINT32(interrupt_request, CPUState),
346 VMSTATE_END_OF_LIST()
347 }
348 };
349
350 #endif
351
352 CPUState *qemu_get_cpu(int index)
353 {
354 CPUState *cpu = first_cpu;
355
356 while (cpu) {
357 if (cpu->cpu_index == index) {
358 break;
359 }
360 cpu = cpu->next_cpu;
361 }
362
363 return cpu;
364 }
365
366 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
367 {
368 CPUState *cpu;
369
370 cpu = first_cpu;
371 while (cpu) {
372 func(cpu, data);
373 cpu = cpu->next_cpu;
374 }
375 }
376
377 void cpu_exec_init(CPUArchState *env)
378 {
379 CPUState *cpu = ENV_GET_CPU(env);
380 CPUClass *cc = CPU_GET_CLASS(cpu);
381 CPUState **pcpu;
382 int cpu_index;
383
384 #if defined(CONFIG_USER_ONLY)
385 cpu_list_lock();
386 #endif
387 cpu->next_cpu = NULL;
388 pcpu = &first_cpu;
389 cpu_index = 0;
390 while (*pcpu != NULL) {
391 pcpu = &(*pcpu)->next_cpu;
392 cpu_index++;
393 }
394 cpu->cpu_index = cpu_index;
395 cpu->numa_node = 0;
396 QTAILQ_INIT(&env->breakpoints);
397 QTAILQ_INIT(&env->watchpoints);
398 #ifndef CONFIG_USER_ONLY
399 cpu->thread_id = qemu_get_thread_id();
400 #endif
401 *pcpu = cpu;
402 #if defined(CONFIG_USER_ONLY)
403 cpu_list_unlock();
404 #endif
405 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
406 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
407 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
408 cpu_save, cpu_load, env);
409 assert(cc->vmsd == NULL);
410 #endif
411 if (cc->vmsd != NULL) {
412 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
413 }
414 }
415
416 #if defined(TARGET_HAS_ICE)
417 #if defined(CONFIG_USER_ONLY)
418 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
419 {
420 tb_invalidate_phys_page_range(pc, pc + 1, 0);
421 }
422 #else
423 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
424 {
425 tb_invalidate_phys_addr(cpu_get_phys_page_debug(cpu, pc) |
426 (pc & ~TARGET_PAGE_MASK));
427 }
428 #endif
429 #endif /* TARGET_HAS_ICE */
430
431 #if defined(CONFIG_USER_ONLY)
432 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
433
434 {
435 }
436
437 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
438 int flags, CPUWatchpoint **watchpoint)
439 {
440 return -ENOSYS;
441 }
442 #else
443 /* Add a watchpoint. */
444 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
445 int flags, CPUWatchpoint **watchpoint)
446 {
447 target_ulong len_mask = ~(len - 1);
448 CPUWatchpoint *wp;
449
450 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
451 if ((len & (len - 1)) || (addr & ~len_mask) ||
452 len == 0 || len > TARGET_PAGE_SIZE) {
453 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
454 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
455 return -EINVAL;
456 }
457 wp = g_malloc(sizeof(*wp));
458
459 wp->vaddr = addr;
460 wp->len_mask = len_mask;
461 wp->flags = flags;
462
463 /* keep all GDB-injected watchpoints in front */
464 if (flags & BP_GDB)
465 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
466 else
467 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
468
469 tlb_flush_page(env, addr);
470
471 if (watchpoint)
472 *watchpoint = wp;
473 return 0;
474 }
475
476 /* Remove a specific watchpoint. */
477 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
478 int flags)
479 {
480 target_ulong len_mask = ~(len - 1);
481 CPUWatchpoint *wp;
482
483 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
484 if (addr == wp->vaddr && len_mask == wp->len_mask
485 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
486 cpu_watchpoint_remove_by_ref(env, wp);
487 return 0;
488 }
489 }
490 return -ENOENT;
491 }
492
493 /* Remove a specific watchpoint by reference. */
494 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
495 {
496 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
497
498 tlb_flush_page(env, watchpoint->vaddr);
499
500 g_free(watchpoint);
501 }
502
503 /* Remove all matching watchpoints. */
504 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
505 {
506 CPUWatchpoint *wp, *next;
507
508 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
509 if (wp->flags & mask)
510 cpu_watchpoint_remove_by_ref(env, wp);
511 }
512 }
513 #endif
514
515 /* Add a breakpoint. */
516 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
517 CPUBreakpoint **breakpoint)
518 {
519 #if defined(TARGET_HAS_ICE)
520 CPUBreakpoint *bp;
521
522 bp = g_malloc(sizeof(*bp));
523
524 bp->pc = pc;
525 bp->flags = flags;
526
527 /* keep all GDB-injected breakpoints in front */
528 if (flags & BP_GDB) {
529 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
530 } else {
531 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
532 }
533
534 breakpoint_invalidate(ENV_GET_CPU(env), pc);
535
536 if (breakpoint) {
537 *breakpoint = bp;
538 }
539 return 0;
540 #else
541 return -ENOSYS;
542 #endif
543 }
544
545 /* Remove a specific breakpoint. */
546 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
547 {
548 #if defined(TARGET_HAS_ICE)
549 CPUBreakpoint *bp;
550
551 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
552 if (bp->pc == pc && bp->flags == flags) {
553 cpu_breakpoint_remove_by_ref(env, bp);
554 return 0;
555 }
556 }
557 return -ENOENT;
558 #else
559 return -ENOSYS;
560 #endif
561 }
562
563 /* Remove a specific breakpoint by reference. */
564 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
565 {
566 #if defined(TARGET_HAS_ICE)
567 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
568
569 breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
570
571 g_free(breakpoint);
572 #endif
573 }
574
575 /* Remove all matching breakpoints. */
576 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
577 {
578 #if defined(TARGET_HAS_ICE)
579 CPUBreakpoint *bp, *next;
580
581 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
582 if (bp->flags & mask)
583 cpu_breakpoint_remove_by_ref(env, bp);
584 }
585 #endif
586 }
587
588 /* enable or disable single step mode. EXCP_DEBUG is returned by the
589 CPU loop after each instruction */
590 void cpu_single_step(CPUState *cpu, int enabled)
591 {
592 #if defined(TARGET_HAS_ICE)
593 if (cpu->singlestep_enabled != enabled) {
594 cpu->singlestep_enabled = enabled;
595 if (kvm_enabled()) {
596 kvm_update_guest_debug(cpu, 0);
597 } else {
598 /* must flush all the translated code to avoid inconsistencies */
599 /* XXX: only flush what is necessary */
600 CPUArchState *env = cpu->env_ptr;
601 tb_flush(env);
602 }
603 }
604 #endif
605 }
606
607 void cpu_abort(CPUArchState *env, const char *fmt, ...)
608 {
609 CPUState *cpu = ENV_GET_CPU(env);
610 va_list ap;
611 va_list ap2;
612
613 va_start(ap, fmt);
614 va_copy(ap2, ap);
615 fprintf(stderr, "qemu: fatal: ");
616 vfprintf(stderr, fmt, ap);
617 fprintf(stderr, "\n");
618 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
619 if (qemu_log_enabled()) {
620 qemu_log("qemu: fatal: ");
621 qemu_log_vprintf(fmt, ap2);
622 qemu_log("\n");
623 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
624 qemu_log_flush();
625 qemu_log_close();
626 }
627 va_end(ap2);
628 va_end(ap);
629 #if defined(CONFIG_USER_ONLY)
630 {
631 struct sigaction act;
632 sigfillset(&act.sa_mask);
633 act.sa_handler = SIG_DFL;
634 sigaction(SIGABRT, &act, NULL);
635 }
636 #endif
637 abort();
638 }
639
640 CPUArchState *cpu_copy(CPUArchState *env)
641 {
642 CPUArchState *new_env = cpu_init(env->cpu_model_str);
643 #if defined(TARGET_HAS_ICE)
644 CPUBreakpoint *bp;
645 CPUWatchpoint *wp;
646 #endif
647
648 /* Reset non arch specific state */
649 cpu_reset(ENV_GET_CPU(new_env));
650
651 /* Copy arch specific state into the new CPU */
652 memcpy(new_env, env, sizeof(CPUArchState));
653
654 /* Clone all break/watchpoints.
655 Note: Once we support ptrace with hw-debug register access, make sure
656 BP_CPU break/watchpoints are handled correctly on clone. */
657 QTAILQ_INIT(&env->breakpoints);
658 QTAILQ_INIT(&env->watchpoints);
659 #if defined(TARGET_HAS_ICE)
660 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
661 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
662 }
663 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
664 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
665 wp->flags, NULL);
666 }
667 #endif
668
669 return new_env;
670 }
671
672 #if !defined(CONFIG_USER_ONLY)
673 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
674 uintptr_t length)
675 {
676 uintptr_t start1;
677
678 /* we modify the TLB cache so that the dirty bit will be set again
679 when accessing the range */
680 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
681 /* Check that we don't span multiple blocks - this breaks the
682 address comparisons below. */
683 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
684 != (end - 1) - start) {
685 abort();
686 }
687 cpu_tlb_reset_dirty_all(start1, length);
688
689 }
690
691 /* Note: start and end must be within the same ram block. */
692 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
693 int dirty_flags)
694 {
695 uintptr_t length;
696
697 start &= TARGET_PAGE_MASK;
698 end = TARGET_PAGE_ALIGN(end);
699
700 length = end - start;
701 if (length == 0)
702 return;
703 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
704
705 if (tcg_enabled()) {
706 tlb_reset_dirty_range_all(start, end, length);
707 }
708 }
709
710 static int cpu_physical_memory_set_dirty_tracking(int enable)
711 {
712 int ret = 0;
713 in_migration = enable;
714 return ret;
715 }
716
717 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
718 MemoryRegionSection *section,
719 target_ulong vaddr,
720 hwaddr paddr, hwaddr xlat,
721 int prot,
722 target_ulong *address)
723 {
724 hwaddr iotlb;
725 CPUWatchpoint *wp;
726
727 if (memory_region_is_ram(section->mr)) {
728 /* Normal RAM. */
729 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
730 + xlat;
731 if (!section->readonly) {
732 iotlb |= PHYS_SECTION_NOTDIRTY;
733 } else {
734 iotlb |= PHYS_SECTION_ROM;
735 }
736 } else {
737 iotlb = section - address_space_memory.dispatch->sections;
738 iotlb += xlat;
739 }
740
741 /* Make accesses to pages with watchpoints go via the
742 watchpoint trap routines. */
743 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
744 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
745 /* Avoid trapping reads of pages with a write breakpoint. */
746 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
747 iotlb = PHYS_SECTION_WATCH + paddr;
748 *address |= TLB_MMIO;
749 break;
750 }
751 }
752 }
753
754 return iotlb;
755 }
756 #endif /* defined(CONFIG_USER_ONLY) */
757
758 #if !defined(CONFIG_USER_ONLY)
759
760 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
761 uint16_t section);
762 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
763
764 static uint16_t phys_section_add(MemoryRegionSection *section)
765 {
766 /* The physical section number is ORed with a page-aligned
767 * pointer to produce the iotlb entries. Thus it should
768 * never overflow into the page-aligned value.
769 */
770 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
771
772 if (next_map.sections_nb == next_map.sections_nb_alloc) {
773 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
774 16);
775 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
776 next_map.sections_nb_alloc);
777 }
778 next_map.sections[next_map.sections_nb] = *section;
779 memory_region_ref(section->mr);
780 return next_map.sections_nb++;
781 }
782
783 static void phys_section_destroy(MemoryRegion *mr)
784 {
785 memory_region_unref(mr);
786
787 if (mr->subpage) {
788 subpage_t *subpage = container_of(mr, subpage_t, iomem);
789 memory_region_destroy(&subpage->iomem);
790 g_free(subpage);
791 }
792 }
793
794 static void phys_sections_free(PhysPageMap *map)
795 {
796 while (map->sections_nb > 0) {
797 MemoryRegionSection *section = &map->sections[--map->sections_nb];
798 phys_section_destroy(section->mr);
799 }
800 g_free(map->sections);
801 g_free(map->nodes);
802 g_free(map);
803 }
804
805 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
806 {
807 subpage_t *subpage;
808 hwaddr base = section->offset_within_address_space
809 & TARGET_PAGE_MASK;
810 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
811 next_map.nodes, next_map.sections);
812 MemoryRegionSection subsection = {
813 .offset_within_address_space = base,
814 .size = int128_make64(TARGET_PAGE_SIZE),
815 };
816 hwaddr start, end;
817
818 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
819
820 if (!(existing->mr->subpage)) {
821 subpage = subpage_init(d->as, base);
822 subsection.mr = &subpage->iomem;
823 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
824 phys_section_add(&subsection));
825 } else {
826 subpage = container_of(existing->mr, subpage_t, iomem);
827 }
828 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
829 end = start + int128_get64(section->size) - 1;
830 subpage_register(subpage, start, end, phys_section_add(section));
831 }
832
833
834 static void register_multipage(AddressSpaceDispatch *d,
835 MemoryRegionSection *section)
836 {
837 hwaddr start_addr = section->offset_within_address_space;
838 uint16_t section_index = phys_section_add(section);
839 uint64_t num_pages = int128_get64(int128_rshift(section->size,
840 TARGET_PAGE_BITS));
841
842 assert(num_pages);
843 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
844 }
845
846 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
847 {
848 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
849 AddressSpaceDispatch *d = as->next_dispatch;
850 MemoryRegionSection now = *section, remain = *section;
851 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
852
853 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
854 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
855 - now.offset_within_address_space;
856
857 now.size = int128_min(int128_make64(left), now.size);
858 register_subpage(d, &now);
859 } else {
860 now.size = int128_zero();
861 }
862 while (int128_ne(remain.size, now.size)) {
863 remain.size = int128_sub(remain.size, now.size);
864 remain.offset_within_address_space += int128_get64(now.size);
865 remain.offset_within_region += int128_get64(now.size);
866 now = remain;
867 if (int128_lt(remain.size, page_size)) {
868 register_subpage(d, &now);
869 } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
870 now.size = page_size;
871 register_subpage(d, &now);
872 } else {
873 now.size = int128_and(now.size, int128_neg(page_size));
874 register_multipage(d, &now);
875 }
876 }
877 }
878
879 void qemu_flush_coalesced_mmio_buffer(void)
880 {
881 if (kvm_enabled())
882 kvm_flush_coalesced_mmio_buffer();
883 }
884
885 void qemu_mutex_lock_ramlist(void)
886 {
887 qemu_mutex_lock(&ram_list.mutex);
888 }
889
890 void qemu_mutex_unlock_ramlist(void)
891 {
892 qemu_mutex_unlock(&ram_list.mutex);
893 }
894
895 #if defined(__linux__) && !defined(TARGET_S390X)
896
897 #include <sys/vfs.h>
898
899 #define HUGETLBFS_MAGIC 0x958458f6
900
901 static long gethugepagesize(const char *path)
902 {
903 struct statfs fs;
904 int ret;
905
906 do {
907 ret = statfs(path, &fs);
908 } while (ret != 0 && errno == EINTR);
909
910 if (ret != 0) {
911 perror(path);
912 return 0;
913 }
914
915 if (fs.f_type != HUGETLBFS_MAGIC)
916 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
917
918 return fs.f_bsize;
919 }
920
921 static void *file_ram_alloc(RAMBlock *block,
922 ram_addr_t memory,
923 const char *path)
924 {
925 char *filename;
926 char *sanitized_name;
927 char *c;
928 void *area;
929 int fd;
930 #ifdef MAP_POPULATE
931 int flags;
932 #endif
933 unsigned long hpagesize;
934
935 hpagesize = gethugepagesize(path);
936 if (!hpagesize) {
937 return NULL;
938 }
939
940 if (memory < hpagesize) {
941 return NULL;
942 }
943
944 if (kvm_enabled() && !kvm_has_sync_mmu()) {
945 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
946 return NULL;
947 }
948
949 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
950 sanitized_name = g_strdup(block->mr->name);
951 for (c = sanitized_name; *c != '\0'; c++) {
952 if (*c == '/')
953 *c = '_';
954 }
955
956 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
957 sanitized_name);
958 g_free(sanitized_name);
959
960 fd = mkstemp(filename);
961 if (fd < 0) {
962 perror("unable to create backing store for hugepages");
963 g_free(filename);
964 return NULL;
965 }
966 unlink(filename);
967 g_free(filename);
968
969 memory = (memory+hpagesize-1) & ~(hpagesize-1);
970
971 /*
972 * ftruncate is not supported by hugetlbfs in older
973 * hosts, so don't bother bailing out on errors.
974 * If anything goes wrong with it under other filesystems,
975 * mmap will fail.
976 */
977 if (ftruncate(fd, memory))
978 perror("ftruncate");
979
980 #ifdef MAP_POPULATE
981 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
982 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
983 * to sidestep this quirk.
984 */
985 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
986 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
987 #else
988 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
989 #endif
990 if (area == MAP_FAILED) {
991 perror("file_ram_alloc: can't mmap RAM pages");
992 close(fd);
993 return (NULL);
994 }
995 block->fd = fd;
996 return area;
997 }
998 #endif
999
1000 static ram_addr_t find_ram_offset(ram_addr_t size)
1001 {
1002 RAMBlock *block, *next_block;
1003 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1004
1005 assert(size != 0); /* it would hand out same offset multiple times */
1006
1007 if (QTAILQ_EMPTY(&ram_list.blocks))
1008 return 0;
1009
1010 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1011 ram_addr_t end, next = RAM_ADDR_MAX;
1012
1013 end = block->offset + block->length;
1014
1015 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1016 if (next_block->offset >= end) {
1017 next = MIN(next, next_block->offset);
1018 }
1019 }
1020 if (next - end >= size && next - end < mingap) {
1021 offset = end;
1022 mingap = next - end;
1023 }
1024 }
1025
1026 if (offset == RAM_ADDR_MAX) {
1027 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1028 (uint64_t)size);
1029 abort();
1030 }
1031
1032 return offset;
1033 }
1034
1035 ram_addr_t last_ram_offset(void)
1036 {
1037 RAMBlock *block;
1038 ram_addr_t last = 0;
1039
1040 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1041 last = MAX(last, block->offset + block->length);
1042
1043 return last;
1044 }
1045
1046 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1047 {
1048 int ret;
1049
1050 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1051 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1052 "dump-guest-core", true)) {
1053 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1054 if (ret) {
1055 perror("qemu_madvise");
1056 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1057 "but dump_guest_core=off specified\n");
1058 }
1059 }
1060 }
1061
1062 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1063 {
1064 RAMBlock *new_block, *block;
1065
1066 new_block = NULL;
1067 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1068 if (block->offset == addr) {
1069 new_block = block;
1070 break;
1071 }
1072 }
1073 assert(new_block);
1074 assert(!new_block->idstr[0]);
1075
1076 if (dev) {
1077 char *id = qdev_get_dev_path(dev);
1078 if (id) {
1079 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1080 g_free(id);
1081 }
1082 }
1083 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1084
1085 /* This assumes the iothread lock is taken here too. */
1086 qemu_mutex_lock_ramlist();
1087 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1088 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1089 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1090 new_block->idstr);
1091 abort();
1092 }
1093 }
1094 qemu_mutex_unlock_ramlist();
1095 }
1096
1097 static int memory_try_enable_merging(void *addr, size_t len)
1098 {
1099 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1100 /* disabled by the user */
1101 return 0;
1102 }
1103
1104 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1105 }
1106
1107 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1108 MemoryRegion *mr)
1109 {
1110 RAMBlock *block, *new_block;
1111
1112 size = TARGET_PAGE_ALIGN(size);
1113 new_block = g_malloc0(sizeof(*new_block));
1114
1115 /* This assumes the iothread lock is taken here too. */
1116 qemu_mutex_lock_ramlist();
1117 new_block->mr = mr;
1118 new_block->offset = find_ram_offset(size);
1119 if (host) {
1120 new_block->host = host;
1121 new_block->flags |= RAM_PREALLOC_MASK;
1122 } else {
1123 if (mem_path) {
1124 #if defined (__linux__) && !defined(TARGET_S390X)
1125 new_block->host = file_ram_alloc(new_block, size, mem_path);
1126 if (!new_block->host) {
1127 new_block->host = qemu_anon_ram_alloc(size);
1128 memory_try_enable_merging(new_block->host, size);
1129 }
1130 #else
1131 fprintf(stderr, "-mem-path option unsupported\n");
1132 exit(1);
1133 #endif
1134 } else {
1135 if (xen_enabled()) {
1136 xen_ram_alloc(new_block->offset, size, mr);
1137 } else if (kvm_enabled()) {
1138 /* some s390/kvm configurations have special constraints */
1139 new_block->host = kvm_ram_alloc(size);
1140 } else {
1141 new_block->host = qemu_anon_ram_alloc(size);
1142 }
1143 memory_try_enable_merging(new_block->host, size);
1144 }
1145 }
1146 new_block->length = size;
1147
1148 /* Keep the list sorted from biggest to smallest block. */
1149 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1150 if (block->length < new_block->length) {
1151 break;
1152 }
1153 }
1154 if (block) {
1155 QTAILQ_INSERT_BEFORE(block, new_block, next);
1156 } else {
1157 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1158 }
1159 ram_list.mru_block = NULL;
1160
1161 ram_list.version++;
1162 qemu_mutex_unlock_ramlist();
1163
1164 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1165 last_ram_offset() >> TARGET_PAGE_BITS);
1166 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1167 0, size >> TARGET_PAGE_BITS);
1168 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1169
1170 qemu_ram_setup_dump(new_block->host, size);
1171 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1172
1173 if (kvm_enabled())
1174 kvm_setup_guest_memory(new_block->host, size);
1175
1176 return new_block->offset;
1177 }
1178
1179 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1180 {
1181 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1182 }
1183
1184 void qemu_ram_free_from_ptr(ram_addr_t addr)
1185 {
1186 RAMBlock *block;
1187
1188 /* This assumes the iothread lock is taken here too. */
1189 qemu_mutex_lock_ramlist();
1190 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1191 if (addr == block->offset) {
1192 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1193 ram_list.mru_block = NULL;
1194 ram_list.version++;
1195 g_free(block);
1196 break;
1197 }
1198 }
1199 qemu_mutex_unlock_ramlist();
1200 }
1201
1202 void qemu_ram_free(ram_addr_t addr)
1203 {
1204 RAMBlock *block;
1205
1206 /* This assumes the iothread lock is taken here too. */
1207 qemu_mutex_lock_ramlist();
1208 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1209 if (addr == block->offset) {
1210 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1211 ram_list.mru_block = NULL;
1212 ram_list.version++;
1213 if (block->flags & RAM_PREALLOC_MASK) {
1214 ;
1215 } else if (mem_path) {
1216 #if defined (__linux__) && !defined(TARGET_S390X)
1217 if (block->fd) {
1218 munmap(block->host, block->length);
1219 close(block->fd);
1220 } else {
1221 qemu_anon_ram_free(block->host, block->length);
1222 }
1223 #else
1224 abort();
1225 #endif
1226 } else {
1227 if (xen_enabled()) {
1228 xen_invalidate_map_cache_entry(block->host);
1229 } else {
1230 qemu_anon_ram_free(block->host, block->length);
1231 }
1232 }
1233 g_free(block);
1234 break;
1235 }
1236 }
1237 qemu_mutex_unlock_ramlist();
1238
1239 }
1240
1241 #ifndef _WIN32
1242 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1243 {
1244 RAMBlock *block;
1245 ram_addr_t offset;
1246 int flags;
1247 void *area, *vaddr;
1248
1249 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1250 offset = addr - block->offset;
1251 if (offset < block->length) {
1252 vaddr = block->host + offset;
1253 if (block->flags & RAM_PREALLOC_MASK) {
1254 ;
1255 } else {
1256 flags = MAP_FIXED;
1257 munmap(vaddr, length);
1258 if (mem_path) {
1259 #if defined(__linux__) && !defined(TARGET_S390X)
1260 if (block->fd) {
1261 #ifdef MAP_POPULATE
1262 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1263 MAP_PRIVATE;
1264 #else
1265 flags |= MAP_PRIVATE;
1266 #endif
1267 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1268 flags, block->fd, offset);
1269 } else {
1270 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1271 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1272 flags, -1, 0);
1273 }
1274 #else
1275 abort();
1276 #endif
1277 } else {
1278 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1279 flags |= MAP_SHARED | MAP_ANONYMOUS;
1280 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1281 flags, -1, 0);
1282 #else
1283 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1284 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1285 flags, -1, 0);
1286 #endif
1287 }
1288 if (area != vaddr) {
1289 fprintf(stderr, "Could not remap addr: "
1290 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1291 length, addr);
1292 exit(1);
1293 }
1294 memory_try_enable_merging(vaddr, length);
1295 qemu_ram_setup_dump(vaddr, length);
1296 }
1297 return;
1298 }
1299 }
1300 }
1301 #endif /* !_WIN32 */
1302
1303 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1304 {
1305 RAMBlock *block;
1306
1307 /* The list is protected by the iothread lock here. */
1308 block = ram_list.mru_block;
1309 if (block && addr - block->offset < block->length) {
1310 goto found;
1311 }
1312 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1313 if (addr - block->offset < block->length) {
1314 goto found;
1315 }
1316 }
1317
1318 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1319 abort();
1320
1321 found:
1322 ram_list.mru_block = block;
1323 return block;
1324 }
1325
1326 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1327 With the exception of the softmmu code in this file, this should
1328 only be used for local memory (e.g. video ram) that the device owns,
1329 and knows it isn't going to access beyond the end of the block.
1330
1331 It should not be used for general purpose DMA.
1332 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1333 */
1334 void *qemu_get_ram_ptr(ram_addr_t addr)
1335 {
1336 RAMBlock *block = qemu_get_ram_block(addr);
1337
1338 if (xen_enabled()) {
1339 /* We need to check if the requested address is in the RAM
1340 * because we don't want to map the entire memory in QEMU.
1341 * In that case just map until the end of the page.
1342 */
1343 if (block->offset == 0) {
1344 return xen_map_cache(addr, 0, 0);
1345 } else if (block->host == NULL) {
1346 block->host =
1347 xen_map_cache(block->offset, block->length, 1);
1348 }
1349 }
1350 return block->host + (addr - block->offset);
1351 }
1352
1353 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1354 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1355 *
1356 * ??? Is this still necessary?
1357 */
1358 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1359 {
1360 RAMBlock *block;
1361
1362 /* The list is protected by the iothread lock here. */
1363 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1364 if (addr - block->offset < block->length) {
1365 if (xen_enabled()) {
1366 /* We need to check if the requested address is in the RAM
1367 * because we don't want to map the entire memory in QEMU.
1368 * In that case just map until the end of the page.
1369 */
1370 if (block->offset == 0) {
1371 return xen_map_cache(addr, 0, 0);
1372 } else if (block->host == NULL) {
1373 block->host =
1374 xen_map_cache(block->offset, block->length, 1);
1375 }
1376 }
1377 return block->host + (addr - block->offset);
1378 }
1379 }
1380
1381 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1382 abort();
1383
1384 return NULL;
1385 }
1386
1387 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1388 * but takes a size argument */
1389 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1390 {
1391 if (*size == 0) {
1392 return NULL;
1393 }
1394 if (xen_enabled()) {
1395 return xen_map_cache(addr, *size, 1);
1396 } else {
1397 RAMBlock *block;
1398
1399 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1400 if (addr - block->offset < block->length) {
1401 if (addr - block->offset + *size > block->length)
1402 *size = block->length - addr + block->offset;
1403 return block->host + (addr - block->offset);
1404 }
1405 }
1406
1407 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1408 abort();
1409 }
1410 }
1411
1412 /* Some of the softmmu routines need to translate from a host pointer
1413 (typically a TLB entry) back to a ram offset. */
1414 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1415 {
1416 RAMBlock *block;
1417 uint8_t *host = ptr;
1418
1419 if (xen_enabled()) {
1420 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1421 return qemu_get_ram_block(*ram_addr)->mr;
1422 }
1423
1424 block = ram_list.mru_block;
1425 if (block && block->host && host - block->host < block->length) {
1426 goto found;
1427 }
1428
1429 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1430 /* This case append when the block is not mapped. */
1431 if (block->host == NULL) {
1432 continue;
1433 }
1434 if (host - block->host < block->length) {
1435 goto found;
1436 }
1437 }
1438
1439 return NULL;
1440
1441 found:
1442 *ram_addr = block->offset + (host - block->host);
1443 return block->mr;
1444 }
1445
1446 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1447 uint64_t val, unsigned size)
1448 {
1449 int dirty_flags;
1450 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1451 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1452 tb_invalidate_phys_page_fast(ram_addr, size);
1453 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1454 }
1455 switch (size) {
1456 case 1:
1457 stb_p(qemu_get_ram_ptr(ram_addr), val);
1458 break;
1459 case 2:
1460 stw_p(qemu_get_ram_ptr(ram_addr), val);
1461 break;
1462 case 4:
1463 stl_p(qemu_get_ram_ptr(ram_addr), val);
1464 break;
1465 default:
1466 abort();
1467 }
1468 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1469 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1470 /* we remove the notdirty callback only if the code has been
1471 flushed */
1472 if (dirty_flags == 0xff) {
1473 CPUArchState *env = current_cpu->env_ptr;
1474 tlb_set_dirty(env, env->mem_io_vaddr);
1475 }
1476 }
1477
1478 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1479 unsigned size, bool is_write)
1480 {
1481 return is_write;
1482 }
1483
1484 static const MemoryRegionOps notdirty_mem_ops = {
1485 .write = notdirty_mem_write,
1486 .valid.accepts = notdirty_mem_accepts,
1487 .endianness = DEVICE_NATIVE_ENDIAN,
1488 };
1489
1490 /* Generate a debug exception if a watchpoint has been hit. */
1491 static void check_watchpoint(int offset, int len_mask, int flags)
1492 {
1493 CPUArchState *env = current_cpu->env_ptr;
1494 target_ulong pc, cs_base;
1495 target_ulong vaddr;
1496 CPUWatchpoint *wp;
1497 int cpu_flags;
1498
1499 if (env->watchpoint_hit) {
1500 /* We re-entered the check after replacing the TB. Now raise
1501 * the debug interrupt so that is will trigger after the
1502 * current instruction. */
1503 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1504 return;
1505 }
1506 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1507 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1508 if ((vaddr == (wp->vaddr & len_mask) ||
1509 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1510 wp->flags |= BP_WATCHPOINT_HIT;
1511 if (!env->watchpoint_hit) {
1512 env->watchpoint_hit = wp;
1513 tb_check_watchpoint(env);
1514 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1515 env->exception_index = EXCP_DEBUG;
1516 cpu_loop_exit(env);
1517 } else {
1518 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1519 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1520 cpu_resume_from_signal(env, NULL);
1521 }
1522 }
1523 } else {
1524 wp->flags &= ~BP_WATCHPOINT_HIT;
1525 }
1526 }
1527 }
1528
1529 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1530 so these check for a hit then pass through to the normal out-of-line
1531 phys routines. */
1532 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1533 unsigned size)
1534 {
1535 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1536 switch (size) {
1537 case 1: return ldub_phys(addr);
1538 case 2: return lduw_phys(addr);
1539 case 4: return ldl_phys(addr);
1540 default: abort();
1541 }
1542 }
1543
1544 static void watch_mem_write(void *opaque, hwaddr addr,
1545 uint64_t val, unsigned size)
1546 {
1547 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1548 switch (size) {
1549 case 1:
1550 stb_phys(addr, val);
1551 break;
1552 case 2:
1553 stw_phys(addr, val);
1554 break;
1555 case 4:
1556 stl_phys(addr, val);
1557 break;
1558 default: abort();
1559 }
1560 }
1561
1562 static const MemoryRegionOps watch_mem_ops = {
1563 .read = watch_mem_read,
1564 .write = watch_mem_write,
1565 .endianness = DEVICE_NATIVE_ENDIAN,
1566 };
1567
1568 static uint64_t subpage_read(void *opaque, hwaddr addr,
1569 unsigned len)
1570 {
1571 subpage_t *subpage = opaque;
1572 uint8_t buf[4];
1573
1574 #if defined(DEBUG_SUBPAGE)
1575 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1576 subpage, len, addr);
1577 #endif
1578 address_space_read(subpage->as, addr + subpage->base, buf, len);
1579 switch (len) {
1580 case 1:
1581 return ldub_p(buf);
1582 case 2:
1583 return lduw_p(buf);
1584 case 4:
1585 return ldl_p(buf);
1586 default:
1587 abort();
1588 }
1589 }
1590
1591 static void subpage_write(void *opaque, hwaddr addr,
1592 uint64_t value, unsigned len)
1593 {
1594 subpage_t *subpage = opaque;
1595 uint8_t buf[4];
1596
1597 #if defined(DEBUG_SUBPAGE)
1598 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1599 " value %"PRIx64"\n",
1600 __func__, subpage, len, addr, value);
1601 #endif
1602 switch (len) {
1603 case 1:
1604 stb_p(buf, value);
1605 break;
1606 case 2:
1607 stw_p(buf, value);
1608 break;
1609 case 4:
1610 stl_p(buf, value);
1611 break;
1612 default:
1613 abort();
1614 }
1615 address_space_write(subpage->as, addr + subpage->base, buf, len);
1616 }
1617
1618 static bool subpage_accepts(void *opaque, hwaddr addr,
1619 unsigned size, bool is_write)
1620 {
1621 subpage_t *subpage = opaque;
1622 #if defined(DEBUG_SUBPAGE)
1623 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1624 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1625 #endif
1626
1627 return address_space_access_valid(subpage->as, addr + subpage->base,
1628 size, is_write);
1629 }
1630
1631 static const MemoryRegionOps subpage_ops = {
1632 .read = subpage_read,
1633 .write = subpage_write,
1634 .valid.accepts = subpage_accepts,
1635 .endianness = DEVICE_NATIVE_ENDIAN,
1636 };
1637
1638 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1639 uint16_t section)
1640 {
1641 int idx, eidx;
1642
1643 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1644 return -1;
1645 idx = SUBPAGE_IDX(start);
1646 eidx = SUBPAGE_IDX(end);
1647 #if defined(DEBUG_SUBPAGE)
1648 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1649 mmio, start, end, idx, eidx, memory);
1650 #endif
1651 for (; idx <= eidx; idx++) {
1652 mmio->sub_section[idx] = section;
1653 }
1654
1655 return 0;
1656 }
1657
1658 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1659 {
1660 subpage_t *mmio;
1661
1662 mmio = g_malloc0(sizeof(subpage_t));
1663
1664 mmio->as = as;
1665 mmio->base = base;
1666 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1667 "subpage", TARGET_PAGE_SIZE);
1668 mmio->iomem.subpage = true;
1669 #if defined(DEBUG_SUBPAGE)
1670 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1671 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1672 #endif
1673 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1674
1675 return mmio;
1676 }
1677
1678 static uint16_t dummy_section(MemoryRegion *mr)
1679 {
1680 MemoryRegionSection section = {
1681 .mr = mr,
1682 .offset_within_address_space = 0,
1683 .offset_within_region = 0,
1684 .size = int128_2_64(),
1685 };
1686
1687 return phys_section_add(&section);
1688 }
1689
1690 MemoryRegion *iotlb_to_region(hwaddr index)
1691 {
1692 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1693 }
1694
1695 static void io_mem_init(void)
1696 {
1697 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1698 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1699 "unassigned", UINT64_MAX);
1700 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1701 "notdirty", UINT64_MAX);
1702 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1703 "watch", UINT64_MAX);
1704 }
1705
1706 static void mem_begin(MemoryListener *listener)
1707 {
1708 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1709 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1710
1711 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1712 d->as = as;
1713 as->next_dispatch = d;
1714 }
1715
1716 static void mem_commit(MemoryListener *listener)
1717 {
1718 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1719 AddressSpaceDispatch *cur = as->dispatch;
1720 AddressSpaceDispatch *next = as->next_dispatch;
1721
1722 next->nodes = next_map.nodes;
1723 next->sections = next_map.sections;
1724
1725 as->dispatch = next;
1726 g_free(cur);
1727 }
1728
1729 static void core_begin(MemoryListener *listener)
1730 {
1731 uint16_t n;
1732
1733 prev_map = g_new(PhysPageMap, 1);
1734 *prev_map = next_map;
1735
1736 memset(&next_map, 0, sizeof(next_map));
1737 n = dummy_section(&io_mem_unassigned);
1738 assert(n == PHYS_SECTION_UNASSIGNED);
1739 n = dummy_section(&io_mem_notdirty);
1740 assert(n == PHYS_SECTION_NOTDIRTY);
1741 n = dummy_section(&io_mem_rom);
1742 assert(n == PHYS_SECTION_ROM);
1743 n = dummy_section(&io_mem_watch);
1744 assert(n == PHYS_SECTION_WATCH);
1745 }
1746
1747 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1748 * All AddressSpaceDispatch instances have switched to the next map.
1749 */
1750 static void core_commit(MemoryListener *listener)
1751 {
1752 phys_sections_free(prev_map);
1753 }
1754
1755 static void tcg_commit(MemoryListener *listener)
1756 {
1757 CPUState *cpu;
1758
1759 /* since each CPU stores ram addresses in its TLB cache, we must
1760 reset the modified entries */
1761 /* XXX: slow ! */
1762 for (cpu = first_cpu; cpu != NULL; cpu = cpu->next_cpu) {
1763 CPUArchState *env = cpu->env_ptr;
1764
1765 tlb_flush(env, 1);
1766 }
1767 }
1768
1769 static void core_log_global_start(MemoryListener *listener)
1770 {
1771 cpu_physical_memory_set_dirty_tracking(1);
1772 }
1773
1774 static void core_log_global_stop(MemoryListener *listener)
1775 {
1776 cpu_physical_memory_set_dirty_tracking(0);
1777 }
1778
1779 static MemoryListener core_memory_listener = {
1780 .begin = core_begin,
1781 .commit = core_commit,
1782 .log_global_start = core_log_global_start,
1783 .log_global_stop = core_log_global_stop,
1784 .priority = 1,
1785 };
1786
1787 static MemoryListener tcg_memory_listener = {
1788 .commit = tcg_commit,
1789 };
1790
1791 void address_space_init_dispatch(AddressSpace *as)
1792 {
1793 as->dispatch = NULL;
1794 as->dispatch_listener = (MemoryListener) {
1795 .begin = mem_begin,
1796 .commit = mem_commit,
1797 .region_add = mem_add,
1798 .region_nop = mem_add,
1799 .priority = 0,
1800 };
1801 memory_listener_register(&as->dispatch_listener, as);
1802 }
1803
1804 void address_space_destroy_dispatch(AddressSpace *as)
1805 {
1806 AddressSpaceDispatch *d = as->dispatch;
1807
1808 memory_listener_unregister(&as->dispatch_listener);
1809 g_free(d);
1810 as->dispatch = NULL;
1811 }
1812
1813 static void memory_map_init(void)
1814 {
1815 system_memory = g_malloc(sizeof(*system_memory));
1816 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1817 address_space_init(&address_space_memory, system_memory, "memory");
1818
1819 system_io = g_malloc(sizeof(*system_io));
1820 memory_region_init(system_io, NULL, "io", 65536);
1821 address_space_init(&address_space_io, system_io, "I/O");
1822
1823 memory_listener_register(&core_memory_listener, &address_space_memory);
1824 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1825 }
1826
1827 MemoryRegion *get_system_memory(void)
1828 {
1829 return system_memory;
1830 }
1831
1832 MemoryRegion *get_system_io(void)
1833 {
1834 return system_io;
1835 }
1836
1837 #endif /* !defined(CONFIG_USER_ONLY) */
1838
1839 /* physical memory access (slow version, mainly for debug) */
1840 #if defined(CONFIG_USER_ONLY)
1841 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1842 uint8_t *buf, int len, int is_write)
1843 {
1844 int l, flags;
1845 target_ulong page;
1846 void * p;
1847
1848 while (len > 0) {
1849 page = addr & TARGET_PAGE_MASK;
1850 l = (page + TARGET_PAGE_SIZE) - addr;
1851 if (l > len)
1852 l = len;
1853 flags = page_get_flags(page);
1854 if (!(flags & PAGE_VALID))
1855 return -1;
1856 if (is_write) {
1857 if (!(flags & PAGE_WRITE))
1858 return -1;
1859 /* XXX: this code should not depend on lock_user */
1860 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1861 return -1;
1862 memcpy(p, buf, l);
1863 unlock_user(p, addr, l);
1864 } else {
1865 if (!(flags & PAGE_READ))
1866 return -1;
1867 /* XXX: this code should not depend on lock_user */
1868 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1869 return -1;
1870 memcpy(buf, p, l);
1871 unlock_user(p, addr, 0);
1872 }
1873 len -= l;
1874 buf += l;
1875 addr += l;
1876 }
1877 return 0;
1878 }
1879
1880 #else
1881
1882 static void invalidate_and_set_dirty(hwaddr addr,
1883 hwaddr length)
1884 {
1885 if (!cpu_physical_memory_is_dirty(addr)) {
1886 /* invalidate code */
1887 tb_invalidate_phys_page_range(addr, addr + length, 0);
1888 /* set dirty bit */
1889 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1890 }
1891 xen_modified_memory(addr, length);
1892 }
1893
1894 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1895 {
1896 if (memory_region_is_ram(mr)) {
1897 return !(is_write && mr->readonly);
1898 }
1899 if (memory_region_is_romd(mr)) {
1900 return !is_write;
1901 }
1902
1903 return false;
1904 }
1905
1906 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1907 {
1908 unsigned access_size_max = mr->ops->valid.max_access_size;
1909
1910 /* Regions are assumed to support 1-4 byte accesses unless
1911 otherwise specified. */
1912 if (access_size_max == 0) {
1913 access_size_max = 4;
1914 }
1915
1916 /* Bound the maximum access by the alignment of the address. */
1917 if (!mr->ops->impl.unaligned) {
1918 unsigned align_size_max = addr & -addr;
1919 if (align_size_max != 0 && align_size_max < access_size_max) {
1920 access_size_max = align_size_max;
1921 }
1922 }
1923
1924 /* Don't attempt accesses larger than the maximum. */
1925 if (l > access_size_max) {
1926 l = access_size_max;
1927 }
1928
1929 return l;
1930 }
1931
1932 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1933 int len, bool is_write)
1934 {
1935 hwaddr l;
1936 uint8_t *ptr;
1937 uint64_t val;
1938 hwaddr addr1;
1939 MemoryRegion *mr;
1940 bool error = false;
1941
1942 while (len > 0) {
1943 l = len;
1944 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1945
1946 if (is_write) {
1947 if (!memory_access_is_direct(mr, is_write)) {
1948 l = memory_access_size(mr, l, addr1);
1949 /* XXX: could force current_cpu to NULL to avoid
1950 potential bugs */
1951 switch (l) {
1952 case 8:
1953 /* 64 bit write access */
1954 val = ldq_p(buf);
1955 error |= io_mem_write(mr, addr1, val, 8);
1956 break;
1957 case 4:
1958 /* 32 bit write access */
1959 val = ldl_p(buf);
1960 error |= io_mem_write(mr, addr1, val, 4);
1961 break;
1962 case 2:
1963 /* 16 bit write access */
1964 val = lduw_p(buf);
1965 error |= io_mem_write(mr, addr1, val, 2);
1966 break;
1967 case 1:
1968 /* 8 bit write access */
1969 val = ldub_p(buf);
1970 error |= io_mem_write(mr, addr1, val, 1);
1971 break;
1972 default:
1973 abort();
1974 }
1975 } else {
1976 addr1 += memory_region_get_ram_addr(mr);
1977 /* RAM case */
1978 ptr = qemu_get_ram_ptr(addr1);
1979 memcpy(ptr, buf, l);
1980 invalidate_and_set_dirty(addr1, l);
1981 }
1982 } else {
1983 if (!memory_access_is_direct(mr, is_write)) {
1984 /* I/O case */
1985 l = memory_access_size(mr, l, addr1);
1986 switch (l) {
1987 case 8:
1988 /* 64 bit read access */
1989 error |= io_mem_read(mr, addr1, &val, 8);
1990 stq_p(buf, val);
1991 break;
1992 case 4:
1993 /* 32 bit read access */
1994 error |= io_mem_read(mr, addr1, &val, 4);
1995 stl_p(buf, val);
1996 break;
1997 case 2:
1998 /* 16 bit read access */
1999 error |= io_mem_read(mr, addr1, &val, 2);
2000 stw_p(buf, val);
2001 break;
2002 case 1:
2003 /* 8 bit read access */
2004 error |= io_mem_read(mr, addr1, &val, 1);
2005 stb_p(buf, val);
2006 break;
2007 default:
2008 abort();
2009 }
2010 } else {
2011 /* RAM case */
2012 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2013 memcpy(buf, ptr, l);
2014 }
2015 }
2016 len -= l;
2017 buf += l;
2018 addr += l;
2019 }
2020
2021 return error;
2022 }
2023
2024 bool address_space_write(AddressSpace *as, hwaddr addr,
2025 const uint8_t *buf, int len)
2026 {
2027 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2028 }
2029
2030 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2031 {
2032 return address_space_rw(as, addr, buf, len, false);
2033 }
2034
2035
2036 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2037 int len, int is_write)
2038 {
2039 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2040 }
2041
2042 /* used for ROM loading : can write in RAM and ROM */
2043 void cpu_physical_memory_write_rom(hwaddr addr,
2044 const uint8_t *buf, int len)
2045 {
2046 hwaddr l;
2047 uint8_t *ptr;
2048 hwaddr addr1;
2049 MemoryRegion *mr;
2050
2051 while (len > 0) {
2052 l = len;
2053 mr = address_space_translate(&address_space_memory,
2054 addr, &addr1, &l, true);
2055
2056 if (!(memory_region_is_ram(mr) ||
2057 memory_region_is_romd(mr))) {
2058 /* do nothing */
2059 } else {
2060 addr1 += memory_region_get_ram_addr(mr);
2061 /* ROM/RAM case */
2062 ptr = qemu_get_ram_ptr(addr1);
2063 memcpy(ptr, buf, l);
2064 invalidate_and_set_dirty(addr1, l);
2065 }
2066 len -= l;
2067 buf += l;
2068 addr += l;
2069 }
2070 }
2071
2072 typedef struct {
2073 MemoryRegion *mr;
2074 void *buffer;
2075 hwaddr addr;
2076 hwaddr len;
2077 } BounceBuffer;
2078
2079 static BounceBuffer bounce;
2080
2081 typedef struct MapClient {
2082 void *opaque;
2083 void (*callback)(void *opaque);
2084 QLIST_ENTRY(MapClient) link;
2085 } MapClient;
2086
2087 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2088 = QLIST_HEAD_INITIALIZER(map_client_list);
2089
2090 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2091 {
2092 MapClient *client = g_malloc(sizeof(*client));
2093
2094 client->opaque = opaque;
2095 client->callback = callback;
2096 QLIST_INSERT_HEAD(&map_client_list, client, link);
2097 return client;
2098 }
2099
2100 static void cpu_unregister_map_client(void *_client)
2101 {
2102 MapClient *client = (MapClient *)_client;
2103
2104 QLIST_REMOVE(client, link);
2105 g_free(client);
2106 }
2107
2108 static void cpu_notify_map_clients(void)
2109 {
2110 MapClient *client;
2111
2112 while (!QLIST_EMPTY(&map_client_list)) {
2113 client = QLIST_FIRST(&map_client_list);
2114 client->callback(client->opaque);
2115 cpu_unregister_map_client(client);
2116 }
2117 }
2118
2119 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2120 {
2121 MemoryRegion *mr;
2122 hwaddr l, xlat;
2123
2124 while (len > 0) {
2125 l = len;
2126 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2127 if (!memory_access_is_direct(mr, is_write)) {
2128 l = memory_access_size(mr, l, addr);
2129 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2130 return false;
2131 }
2132 }
2133
2134 len -= l;
2135 addr += l;
2136 }
2137 return true;
2138 }
2139
2140 /* Map a physical memory region into a host virtual address.
2141 * May map a subset of the requested range, given by and returned in *plen.
2142 * May return NULL if resources needed to perform the mapping are exhausted.
2143 * Use only for reads OR writes - not for read-modify-write operations.
2144 * Use cpu_register_map_client() to know when retrying the map operation is
2145 * likely to succeed.
2146 */
2147 void *address_space_map(AddressSpace *as,
2148 hwaddr addr,
2149 hwaddr *plen,
2150 bool is_write)
2151 {
2152 hwaddr len = *plen;
2153 hwaddr done = 0;
2154 hwaddr l, xlat, base;
2155 MemoryRegion *mr, *this_mr;
2156 ram_addr_t raddr;
2157
2158 if (len == 0) {
2159 return NULL;
2160 }
2161
2162 l = len;
2163 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2164 if (!memory_access_is_direct(mr, is_write)) {
2165 if (bounce.buffer) {
2166 return NULL;
2167 }
2168 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2169 bounce.addr = addr;
2170 bounce.len = l;
2171
2172 memory_region_ref(mr);
2173 bounce.mr = mr;
2174 if (!is_write) {
2175 address_space_read(as, addr, bounce.buffer, l);
2176 }
2177
2178 *plen = l;
2179 return bounce.buffer;
2180 }
2181
2182 base = xlat;
2183 raddr = memory_region_get_ram_addr(mr);
2184
2185 for (;;) {
2186 len -= l;
2187 addr += l;
2188 done += l;
2189 if (len == 0) {
2190 break;
2191 }
2192
2193 l = len;
2194 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2195 if (this_mr != mr || xlat != base + done) {
2196 break;
2197 }
2198 }
2199
2200 memory_region_ref(mr);
2201 *plen = done;
2202 return qemu_ram_ptr_length(raddr + base, plen);
2203 }
2204
2205 /* Unmaps a memory region previously mapped by address_space_map().
2206 * Will also mark the memory as dirty if is_write == 1. access_len gives
2207 * the amount of memory that was actually read or written by the caller.
2208 */
2209 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2210 int is_write, hwaddr access_len)
2211 {
2212 if (buffer != bounce.buffer) {
2213 MemoryRegion *mr;
2214 ram_addr_t addr1;
2215
2216 mr = qemu_ram_addr_from_host(buffer, &addr1);
2217 assert(mr != NULL);
2218 if (is_write) {
2219 while (access_len) {
2220 unsigned l;
2221 l = TARGET_PAGE_SIZE;
2222 if (l > access_len)
2223 l = access_len;
2224 invalidate_and_set_dirty(addr1, l);
2225 addr1 += l;
2226 access_len -= l;
2227 }
2228 }
2229 if (xen_enabled()) {
2230 xen_invalidate_map_cache_entry(buffer);
2231 }
2232 memory_region_unref(mr);
2233 return;
2234 }
2235 if (is_write) {
2236 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2237 }
2238 qemu_vfree(bounce.buffer);
2239 bounce.buffer = NULL;
2240 memory_region_unref(bounce.mr);
2241 cpu_notify_map_clients();
2242 }
2243
2244 void *cpu_physical_memory_map(hwaddr addr,
2245 hwaddr *plen,
2246 int is_write)
2247 {
2248 return address_space_map(&address_space_memory, addr, plen, is_write);
2249 }
2250
2251 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2252 int is_write, hwaddr access_len)
2253 {
2254 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2255 }
2256
2257 /* warning: addr must be aligned */
2258 static inline uint32_t ldl_phys_internal(hwaddr addr,
2259 enum device_endian endian)
2260 {
2261 uint8_t *ptr;
2262 uint64_t val;
2263 MemoryRegion *mr;
2264 hwaddr l = 4;
2265 hwaddr addr1;
2266
2267 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2268 false);
2269 if (l < 4 || !memory_access_is_direct(mr, false)) {
2270 /* I/O case */
2271 io_mem_read(mr, addr1, &val, 4);
2272 #if defined(TARGET_WORDS_BIGENDIAN)
2273 if (endian == DEVICE_LITTLE_ENDIAN) {
2274 val = bswap32(val);
2275 }
2276 #else
2277 if (endian == DEVICE_BIG_ENDIAN) {
2278 val = bswap32(val);
2279 }
2280 #endif
2281 } else {
2282 /* RAM case */
2283 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2284 & TARGET_PAGE_MASK)
2285 + addr1);
2286 switch (endian) {
2287 case DEVICE_LITTLE_ENDIAN:
2288 val = ldl_le_p(ptr);
2289 break;
2290 case DEVICE_BIG_ENDIAN:
2291 val = ldl_be_p(ptr);
2292 break;
2293 default:
2294 val = ldl_p(ptr);
2295 break;
2296 }
2297 }
2298 return val;
2299 }
2300
2301 uint32_t ldl_phys(hwaddr addr)
2302 {
2303 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2304 }
2305
2306 uint32_t ldl_le_phys(hwaddr addr)
2307 {
2308 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2309 }
2310
2311 uint32_t ldl_be_phys(hwaddr addr)
2312 {
2313 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2314 }
2315
2316 /* warning: addr must be aligned */
2317 static inline uint64_t ldq_phys_internal(hwaddr addr,
2318 enum device_endian endian)
2319 {
2320 uint8_t *ptr;
2321 uint64_t val;
2322 MemoryRegion *mr;
2323 hwaddr l = 8;
2324 hwaddr addr1;
2325
2326 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2327 false);
2328 if (l < 8 || !memory_access_is_direct(mr, false)) {
2329 /* I/O case */
2330 io_mem_read(mr, addr1, &val, 8);
2331 #if defined(TARGET_WORDS_BIGENDIAN)
2332 if (endian == DEVICE_LITTLE_ENDIAN) {
2333 val = bswap64(val);
2334 }
2335 #else
2336 if (endian == DEVICE_BIG_ENDIAN) {
2337 val = bswap64(val);
2338 }
2339 #endif
2340 } else {
2341 /* RAM case */
2342 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2343 & TARGET_PAGE_MASK)
2344 + addr1);
2345 switch (endian) {
2346 case DEVICE_LITTLE_ENDIAN:
2347 val = ldq_le_p(ptr);
2348 break;
2349 case DEVICE_BIG_ENDIAN:
2350 val = ldq_be_p(ptr);
2351 break;
2352 default:
2353 val = ldq_p(ptr);
2354 break;
2355 }
2356 }
2357 return val;
2358 }
2359
2360 uint64_t ldq_phys(hwaddr addr)
2361 {
2362 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2363 }
2364
2365 uint64_t ldq_le_phys(hwaddr addr)
2366 {
2367 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2368 }
2369
2370 uint64_t ldq_be_phys(hwaddr addr)
2371 {
2372 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2373 }
2374
2375 /* XXX: optimize */
2376 uint32_t ldub_phys(hwaddr addr)
2377 {
2378 uint8_t val;
2379 cpu_physical_memory_read(addr, &val, 1);
2380 return val;
2381 }
2382
2383 /* warning: addr must be aligned */
2384 static inline uint32_t lduw_phys_internal(hwaddr addr,
2385 enum device_endian endian)
2386 {
2387 uint8_t *ptr;
2388 uint64_t val;
2389 MemoryRegion *mr;
2390 hwaddr l = 2;
2391 hwaddr addr1;
2392
2393 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2394 false);
2395 if (l < 2 || !memory_access_is_direct(mr, false)) {
2396 /* I/O case */
2397 io_mem_read(mr, addr1, &val, 2);
2398 #if defined(TARGET_WORDS_BIGENDIAN)
2399 if (endian == DEVICE_LITTLE_ENDIAN) {
2400 val = bswap16(val);
2401 }
2402 #else
2403 if (endian == DEVICE_BIG_ENDIAN) {
2404 val = bswap16(val);
2405 }
2406 #endif
2407 } else {
2408 /* RAM case */
2409 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2410 & TARGET_PAGE_MASK)
2411 + addr1);
2412 switch (endian) {
2413 case DEVICE_LITTLE_ENDIAN:
2414 val = lduw_le_p(ptr);
2415 break;
2416 case DEVICE_BIG_ENDIAN:
2417 val = lduw_be_p(ptr);
2418 break;
2419 default:
2420 val = lduw_p(ptr);
2421 break;
2422 }
2423 }
2424 return val;
2425 }
2426
2427 uint32_t lduw_phys(hwaddr addr)
2428 {
2429 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2430 }
2431
2432 uint32_t lduw_le_phys(hwaddr addr)
2433 {
2434 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2435 }
2436
2437 uint32_t lduw_be_phys(hwaddr addr)
2438 {
2439 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2440 }
2441
2442 /* warning: addr must be aligned. The ram page is not masked as dirty
2443 and the code inside is not invalidated. It is useful if the dirty
2444 bits are used to track modified PTEs */
2445 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2446 {
2447 uint8_t *ptr;
2448 MemoryRegion *mr;
2449 hwaddr l = 4;
2450 hwaddr addr1;
2451
2452 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2453 true);
2454 if (l < 4 || !memory_access_is_direct(mr, true)) {
2455 io_mem_write(mr, addr1, val, 4);
2456 } else {
2457 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2458 ptr = qemu_get_ram_ptr(addr1);
2459 stl_p(ptr, val);
2460
2461 if (unlikely(in_migration)) {
2462 if (!cpu_physical_memory_is_dirty(addr1)) {
2463 /* invalidate code */
2464 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2465 /* set dirty bit */
2466 cpu_physical_memory_set_dirty_flags(
2467 addr1, (0xff & ~CODE_DIRTY_FLAG));
2468 }
2469 }
2470 }
2471 }
2472
2473 /* warning: addr must be aligned */
2474 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2475 enum device_endian endian)
2476 {
2477 uint8_t *ptr;
2478 MemoryRegion *mr;
2479 hwaddr l = 4;
2480 hwaddr addr1;
2481
2482 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2483 true);
2484 if (l < 4 || !memory_access_is_direct(mr, true)) {
2485 #if defined(TARGET_WORDS_BIGENDIAN)
2486 if (endian == DEVICE_LITTLE_ENDIAN) {
2487 val = bswap32(val);
2488 }
2489 #else
2490 if (endian == DEVICE_BIG_ENDIAN) {
2491 val = bswap32(val);
2492 }
2493 #endif
2494 io_mem_write(mr, addr1, val, 4);
2495 } else {
2496 /* RAM case */
2497 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2498 ptr = qemu_get_ram_ptr(addr1);
2499 switch (endian) {
2500 case DEVICE_LITTLE_ENDIAN:
2501 stl_le_p(ptr, val);
2502 break;
2503 case DEVICE_BIG_ENDIAN:
2504 stl_be_p(ptr, val);
2505 break;
2506 default:
2507 stl_p(ptr, val);
2508 break;
2509 }
2510 invalidate_and_set_dirty(addr1, 4);
2511 }
2512 }
2513
2514 void stl_phys(hwaddr addr, uint32_t val)
2515 {
2516 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2517 }
2518
2519 void stl_le_phys(hwaddr addr, uint32_t val)
2520 {
2521 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2522 }
2523
2524 void stl_be_phys(hwaddr addr, uint32_t val)
2525 {
2526 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2527 }
2528
2529 /* XXX: optimize */
2530 void stb_phys(hwaddr addr, uint32_t val)
2531 {
2532 uint8_t v = val;
2533 cpu_physical_memory_write(addr, &v, 1);
2534 }
2535
2536 /* warning: addr must be aligned */
2537 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2538 enum device_endian endian)
2539 {
2540 uint8_t *ptr;
2541 MemoryRegion *mr;
2542 hwaddr l = 2;
2543 hwaddr addr1;
2544
2545 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2546 true);
2547 if (l < 2 || !memory_access_is_direct(mr, true)) {
2548 #if defined(TARGET_WORDS_BIGENDIAN)
2549 if (endian == DEVICE_LITTLE_ENDIAN) {
2550 val = bswap16(val);
2551 }
2552 #else
2553 if (endian == DEVICE_BIG_ENDIAN) {
2554 val = bswap16(val);
2555 }
2556 #endif
2557 io_mem_write(mr, addr1, val, 2);
2558 } else {
2559 /* RAM case */
2560 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2561 ptr = qemu_get_ram_ptr(addr1);
2562 switch (endian) {
2563 case DEVICE_LITTLE_ENDIAN:
2564 stw_le_p(ptr, val);
2565 break;
2566 case DEVICE_BIG_ENDIAN:
2567 stw_be_p(ptr, val);
2568 break;
2569 default:
2570 stw_p(ptr, val);
2571 break;
2572 }
2573 invalidate_and_set_dirty(addr1, 2);
2574 }
2575 }
2576
2577 void stw_phys(hwaddr addr, uint32_t val)
2578 {
2579 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2580 }
2581
2582 void stw_le_phys(hwaddr addr, uint32_t val)
2583 {
2584 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2585 }
2586
2587 void stw_be_phys(hwaddr addr, uint32_t val)
2588 {
2589 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2590 }
2591
2592 /* XXX: optimize */
2593 void stq_phys(hwaddr addr, uint64_t val)
2594 {
2595 val = tswap64(val);
2596 cpu_physical_memory_write(addr, &val, 8);
2597 }
2598
2599 void stq_le_phys(hwaddr addr, uint64_t val)
2600 {
2601 val = cpu_to_le64(val);
2602 cpu_physical_memory_write(addr, &val, 8);
2603 }
2604
2605 void stq_be_phys(hwaddr addr, uint64_t val)
2606 {
2607 val = cpu_to_be64(val);
2608 cpu_physical_memory_write(addr, &val, 8);
2609 }
2610
2611 /* virtual memory access for debug (includes writing to ROM) */
2612 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2613 uint8_t *buf, int len, int is_write)
2614 {
2615 int l;
2616 hwaddr phys_addr;
2617 target_ulong page;
2618
2619 while (len > 0) {
2620 page = addr & TARGET_PAGE_MASK;
2621 phys_addr = cpu_get_phys_page_debug(cpu, page);
2622 /* if no physical page mapped, return an error */
2623 if (phys_addr == -1)
2624 return -1;
2625 l = (page + TARGET_PAGE_SIZE) - addr;
2626 if (l > len)
2627 l = len;
2628 phys_addr += (addr & ~TARGET_PAGE_MASK);
2629 if (is_write)
2630 cpu_physical_memory_write_rom(phys_addr, buf, l);
2631 else
2632 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2633 len -= l;
2634 buf += l;
2635 addr += l;
2636 }
2637 return 0;
2638 }
2639 #endif
2640
2641 #if !defined(CONFIG_USER_ONLY)
2642
2643 /*
2644 * A helper function for the _utterly broken_ virtio device model to find out if
2645 * it's running on a big endian machine. Don't do this at home kids!
2646 */
2647 bool virtio_is_big_endian(void);
2648 bool virtio_is_big_endian(void)
2649 {
2650 #if defined(TARGET_WORDS_BIGENDIAN)
2651 return true;
2652 #else
2653 return false;
2654 #endif
2655 }
2656
2657 #endif
2658
2659 #ifndef CONFIG_USER_ONLY
2660 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2661 {
2662 MemoryRegion*mr;
2663 hwaddr l = 1;
2664
2665 mr = address_space_translate(&address_space_memory,
2666 phys_addr, &phys_addr, &l, false);
2667
2668 return !(memory_region_is_ram(mr) ||
2669 memory_region_is_romd(mr));
2670 }
2671
2672 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2673 {
2674 RAMBlock *block;
2675
2676 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2677 func(block->host, block->offset, block->length, opaque);
2678 }
2679 }
2680 #endif
QEmu