?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
cpu: Replace cpu_single_env with CPUState current_cpu
[qemu.git] / exec.c
blob9ca80cc8dc0da276258934d5e67a29dcbf04dc76
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "hw/xen/xen.h"
35 #include "qemu/timer.h"
36 #include "qemu/config-file.h"
37 #include "exec/memory.h"
38 #include "sysemu/dma.h"
39 #include "exec/address-spaces.h"
40 #if defined(CONFIG_USER_ONLY)
41 #include <qemu.h>
42 #else /* !CONFIG_USER_ONLY */
43 #include "sysemu/xen-mapcache.h"
44 #include "trace.h"
45 #endif
46 #include "exec/cpu-all.h"
47
48 #include "exec/cputlb.h"
49 #include "translate-all.h"
50
51 #include "exec/memory-internal.h"
52
53 //#define DEBUG_SUBPAGE
54
55 #if !defined(CONFIG_USER_ONLY)
56 int phys_ram_fd;
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 CPUArchState *first_cpu;
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 typedef PhysPageEntry Node[L2_SIZE];
92
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101 };
102
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
110
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
115
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
124
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
127
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
133
134 static MemoryRegion io_mem_watch;
135 #endif
136
137 #if !defined(CONFIG_USER_ONLY)
138
139 static void phys_map_node_reserve(unsigned nodes)
140 {
141 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
142 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
143 16);
144 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
145 next_map.nodes_nb + nodes);
146 next_map.nodes = g_renew(Node, next_map.nodes,
147 next_map.nodes_nb_alloc);
148 }
149 }
150
151 static uint16_t phys_map_node_alloc(void)
152 {
153 unsigned i;
154 uint16_t ret;
155
156 ret = next_map.nodes_nb++;
157 assert(ret != PHYS_MAP_NODE_NIL);
158 assert(ret != next_map.nodes_nb_alloc);
159 for (i = 0; i < L2_SIZE; ++i) {
160 next_map.nodes[ret][i].is_leaf = 0;
161 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
162 }
163 return ret;
164 }
165
166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
167 hwaddr *nb, uint16_t leaf,
168 int level)
169 {
170 PhysPageEntry *p;
171 int i;
172 hwaddr step = (hwaddr)1 << (level * L2_BITS);
173
174 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
175 lp->ptr = phys_map_node_alloc();
176 p = next_map.nodes[lp->ptr];
177 if (level == 0) {
178 for (i = 0; i < L2_SIZE; i++) {
179 p[i].is_leaf = 1;
180 p[i].ptr = PHYS_SECTION_UNASSIGNED;
181 }
182 }
183 } else {
184 p = next_map.nodes[lp->ptr];
185 }
186 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
187
188 while (*nb && lp < &p[L2_SIZE]) {
189 if ((*index & (step - 1)) == 0 && *nb >= step) {
190 lp->is_leaf = true;
191 lp->ptr = leaf;
192 *index += step;
193 *nb -= step;
194 } else {
195 phys_page_set_level(lp, index, nb, leaf, level - 1);
196 }
197 ++lp;
198 }
199 }
200
201 static void phys_page_set(AddressSpaceDispatch *d,
202 hwaddr index, hwaddr nb,
203 uint16_t leaf)
204 {
205 /* Wildly overreserve - it doesn't matter much. */
206 phys_map_node_reserve(3 * P_L2_LEVELS);
207
208 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
209 }
210
211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
212 Node *nodes, MemoryRegionSection *sections)
213 {
214 PhysPageEntry *p;
215 int i;
216
217 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
218 if (lp.ptr == PHYS_MAP_NODE_NIL) {
219 return &sections[PHYS_SECTION_UNASSIGNED];
220 }
221 p = nodes[lp.ptr];
222 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
223 }
224 return &sections[lp.ptr];
225 }
226
227 bool memory_region_is_unassigned(MemoryRegion *mr)
228 {
229 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
230 && mr != &io_mem_watch;
231 }
232
233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
234 hwaddr addr,
235 bool resolve_subpage)
236 {
237 MemoryRegionSection *section;
238 subpage_t *subpage;
239
240 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
241 d->nodes, d->sections);
242 if (resolve_subpage && section->mr->subpage) {
243 subpage = container_of(section->mr, subpage_t, iomem);
244 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
245 }
246 return section;
247 }
248
249 static MemoryRegionSection *
250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
251 hwaddr *plen, bool resolve_subpage)
252 {
253 MemoryRegionSection *section;
254 Int128 diff;
255
256 section = address_space_lookup_region(d, addr, resolve_subpage);
257 /* Compute offset within MemoryRegionSection */
258 addr -= section->offset_within_address_space;
259
260 /* Compute offset within MemoryRegion */
261 *xlat = addr + section->offset_within_region;
262
263 diff = int128_sub(section->mr->size, int128_make64(addr));
264 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
265 return section;
266 }
267
268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
269 hwaddr *xlat, hwaddr *plen,
270 bool is_write)
271 {
272 IOMMUTLBEntry iotlb;
273 MemoryRegionSection *section;
274 MemoryRegion *mr;
275 hwaddr len = *plen;
276
277 for (;;) {
278 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
279 mr = section->mr;
280
281 if (!mr->iommu_ops) {
282 break;
283 }
284
285 iotlb = mr->iommu_ops->translate(mr, addr);
286 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
287 | (addr & iotlb.addr_mask));
288 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
289 if (!(iotlb.perm & (1 << is_write))) {
290 mr = &io_mem_unassigned;
291 break;
292 }
293
294 as = iotlb.target_as;
295 }
296
297 *plen = len;
298 *xlat = addr;
299 return mr;
300 }
301
302 MemoryRegionSection *
303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
304 hwaddr *plen)
305 {
306 MemoryRegionSection *section;
307 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
308
309 assert(!section->mr->iommu_ops);
310 return section;
311 }
312 #endif
313
314 void cpu_exec_init_all(void)
315 {
316 #if !defined(CONFIG_USER_ONLY)
317 qemu_mutex_init(&ram_list.mutex);
318 memory_map_init();
319 io_mem_init();
320 #endif
321 }
322
323 #if !defined(CONFIG_USER_ONLY)
324
325 static int cpu_common_post_load(void *opaque, int version_id)
326 {
327 CPUState *cpu = opaque;
328
329 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
330 version_id is increased. */
331 cpu->interrupt_request &= ~0x01;
332 tlb_flush(cpu->env_ptr, 1);
333
334 return 0;
335 }
336
337 const VMStateDescription vmstate_cpu_common = {
338 .name = "cpu_common",
339 .version_id = 1,
340 .minimum_version_id = 1,
341 .minimum_version_id_old = 1,
342 .post_load = cpu_common_post_load,
343 .fields = (VMStateField []) {
344 VMSTATE_UINT32(halted, CPUState),
345 VMSTATE_UINT32(interrupt_request, CPUState),
346 VMSTATE_END_OF_LIST()
347 }
348 };
349
350 #endif
351
352 CPUState *qemu_get_cpu(int index)
353 {
354 CPUArchState *env = first_cpu;
355 CPUState *cpu = NULL;
356
357 while (env) {
358 cpu = ENV_GET_CPU(env);
359 if (cpu->cpu_index == index) {
360 break;
361 }
362 env = env->next_cpu;
363 }
364
365 return env ? cpu : NULL;
366 }
367
368 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
369 {
370 CPUArchState *env = first_cpu;
371
372 while (env) {
373 func(ENV_GET_CPU(env), data);
374 env = env->next_cpu;
375 }
376 }
377
378 void cpu_exec_init(CPUArchState *env)
379 {
380 CPUState *cpu = ENV_GET_CPU(env);
381 CPUClass *cc = CPU_GET_CLASS(cpu);
382 CPUArchState **penv;
383 int cpu_index;
384
385 #if defined(CONFIG_USER_ONLY)
386 cpu_list_lock();
387 #endif
388 env->next_cpu = NULL;
389 penv = &first_cpu;
390 cpu_index = 0;
391 while (*penv != NULL) {
392 penv = &(*penv)->next_cpu;
393 cpu_index++;
394 }
395 cpu->cpu_index = cpu_index;
396 cpu->numa_node = 0;
397 QTAILQ_INIT(&env->breakpoints);
398 QTAILQ_INIT(&env->watchpoints);
399 #ifndef CONFIG_USER_ONLY
400 cpu->thread_id = qemu_get_thread_id();
401 #endif
402 *penv = env;
403 #if defined(CONFIG_USER_ONLY)
404 cpu_list_unlock();
405 #endif
406 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
407 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
408 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
409 cpu_save, cpu_load, env);
410 assert(cc->vmsd == NULL);
411 #endif
412 if (cc->vmsd != NULL) {
413 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
414 }
415 }
416
417 #if defined(TARGET_HAS_ICE)
418 #if defined(CONFIG_USER_ONLY)
419 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
420 {
421 tb_invalidate_phys_page_range(pc, pc + 1, 0);
422 }
423 #else
424 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
425 {
426 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
427 (pc & ~TARGET_PAGE_MASK));
428 }
429 #endif
430 #endif /* TARGET_HAS_ICE */
431
432 #if defined(CONFIG_USER_ONLY)
433 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
434
435 {
436 }
437
438 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
439 int flags, CPUWatchpoint **watchpoint)
440 {
441 return -ENOSYS;
442 }
443 #else
444 /* Add a watchpoint. */
445 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
446 int flags, CPUWatchpoint **watchpoint)
447 {
448 target_ulong len_mask = ~(len - 1);
449 CPUWatchpoint *wp;
450
451 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
452 if ((len & (len - 1)) || (addr & ~len_mask) ||
453 len == 0 || len > TARGET_PAGE_SIZE) {
454 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
455 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
456 return -EINVAL;
457 }
458 wp = g_malloc(sizeof(*wp));
459
460 wp->vaddr = addr;
461 wp->len_mask = len_mask;
462 wp->flags = flags;
463
464 /* keep all GDB-injected watchpoints in front */
465 if (flags & BP_GDB)
466 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
467 else
468 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
469
470 tlb_flush_page(env, addr);
471
472 if (watchpoint)
473 *watchpoint = wp;
474 return 0;
475 }
476
477 /* Remove a specific watchpoint. */
478 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
479 int flags)
480 {
481 target_ulong len_mask = ~(len - 1);
482 CPUWatchpoint *wp;
483
484 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
485 if (addr == wp->vaddr && len_mask == wp->len_mask
486 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
487 cpu_watchpoint_remove_by_ref(env, wp);
488 return 0;
489 }
490 }
491 return -ENOENT;
492 }
493
494 /* Remove a specific watchpoint by reference. */
495 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
496 {
497 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
498
499 tlb_flush_page(env, watchpoint->vaddr);
500
501 g_free(watchpoint);
502 }
503
504 /* Remove all matching watchpoints. */
505 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
506 {
507 CPUWatchpoint *wp, *next;
508
509 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
510 if (wp->flags & mask)
511 cpu_watchpoint_remove_by_ref(env, wp);
512 }
513 }
514 #endif
515
516 /* Add a breakpoint. */
517 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
518 CPUBreakpoint **breakpoint)
519 {
520 #if defined(TARGET_HAS_ICE)
521 CPUBreakpoint *bp;
522
523 bp = g_malloc(sizeof(*bp));
524
525 bp->pc = pc;
526 bp->flags = flags;
527
528 /* keep all GDB-injected breakpoints in front */
529 if (flags & BP_GDB)
530 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
531 else
532 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
533
534 breakpoint_invalidate(env, pc);
535
536 if (breakpoint)
537 *breakpoint = bp;
538 return 0;
539 #else
540 return -ENOSYS;
541 #endif
542 }
543
544 /* Remove a specific breakpoint. */
545 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
546 {
547 #if defined(TARGET_HAS_ICE)
548 CPUBreakpoint *bp;
549
550 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
551 if (bp->pc == pc && bp->flags == flags) {
552 cpu_breakpoint_remove_by_ref(env, bp);
553 return 0;
554 }
555 }
556 return -ENOENT;
557 #else
558 return -ENOSYS;
559 #endif
560 }
561
562 /* Remove a specific breakpoint by reference. */
563 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
564 {
565 #if defined(TARGET_HAS_ICE)
566 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
567
568 breakpoint_invalidate(env, breakpoint->pc);
569
570 g_free(breakpoint);
571 #endif
572 }
573
574 /* Remove all matching breakpoints. */
575 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
576 {
577 #if defined(TARGET_HAS_ICE)
578 CPUBreakpoint *bp, *next;
579
580 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
581 if (bp->flags & mask)
582 cpu_breakpoint_remove_by_ref(env, bp);
583 }
584 #endif
585 }
586
587 /* enable or disable single step mode. EXCP_DEBUG is returned by the
588 CPU loop after each instruction */
589 void cpu_single_step(CPUArchState *env, int enabled)
590 {
591 #if defined(TARGET_HAS_ICE)
592 if (env->singlestep_enabled != enabled) {
593 env->singlestep_enabled = enabled;
594 if (kvm_enabled())
595 kvm_update_guest_debug(env, 0);
596 else {
597 /* must flush all the translated code to avoid inconsistencies */
598 /* XXX: only flush what is necessary */
599 tb_flush(env);
600 }
601 }
602 #endif
603 }
604
605 void cpu_abort(CPUArchState *env, const char *fmt, ...)
606 {
607 CPUState *cpu = ENV_GET_CPU(env);
608 va_list ap;
609 va_list ap2;
610
611 va_start(ap, fmt);
612 va_copy(ap2, ap);
613 fprintf(stderr, "qemu: fatal: ");
614 vfprintf(stderr, fmt, ap);
615 fprintf(stderr, "\n");
616 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
617 if (qemu_log_enabled()) {
618 qemu_log("qemu: fatal: ");
619 qemu_log_vprintf(fmt, ap2);
620 qemu_log("\n");
621 log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
622 qemu_log_flush();
623 qemu_log_close();
624 }
625 va_end(ap2);
626 va_end(ap);
627 #if defined(CONFIG_USER_ONLY)
628 {
629 struct sigaction act;
630 sigfillset(&act.sa_mask);
631 act.sa_handler = SIG_DFL;
632 sigaction(SIGABRT, &act, NULL);
633 }
634 #endif
635 abort();
636 }
637
638 CPUArchState *cpu_copy(CPUArchState *env)
639 {
640 CPUArchState *new_env = cpu_init(env->cpu_model_str);
641 CPUArchState *next_cpu = new_env->next_cpu;
642 #if defined(TARGET_HAS_ICE)
643 CPUBreakpoint *bp;
644 CPUWatchpoint *wp;
645 #endif
646
647 memcpy(new_env, env, sizeof(CPUArchState));
648
649 /* Preserve chaining. */
650 new_env->next_cpu = next_cpu;
651
652 /* Clone all break/watchpoints.
653 Note: Once we support ptrace with hw-debug register access, make sure
654 BP_CPU break/watchpoints are handled correctly on clone. */
655 QTAILQ_INIT(&env->breakpoints);
656 QTAILQ_INIT(&env->watchpoints);
657 #if defined(TARGET_HAS_ICE)
658 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
659 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
660 }
661 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
662 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
663 wp->flags, NULL);
664 }
665 #endif
666
667 return new_env;
668 }
669
670 #if !defined(CONFIG_USER_ONLY)
671 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
672 uintptr_t length)
673 {
674 uintptr_t start1;
675
676 /* we modify the TLB cache so that the dirty bit will be set again
677 when accessing the range */
678 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
679 /* Check that we don't span multiple blocks - this breaks the
680 address comparisons below. */
681 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
682 != (end - 1) - start) {
683 abort();
684 }
685 cpu_tlb_reset_dirty_all(start1, length);
686
687 }
688
689 /* Note: start and end must be within the same ram block. */
690 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
691 int dirty_flags)
692 {
693 uintptr_t length;
694
695 start &= TARGET_PAGE_MASK;
696 end = TARGET_PAGE_ALIGN(end);
697
698 length = end - start;
699 if (length == 0)
700 return;
701 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
702
703 if (tcg_enabled()) {
704 tlb_reset_dirty_range_all(start, end, length);
705 }
706 }
707
708 static int cpu_physical_memory_set_dirty_tracking(int enable)
709 {
710 int ret = 0;
711 in_migration = enable;
712 return ret;
713 }
714
715 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
716 MemoryRegionSection *section,
717 target_ulong vaddr,
718 hwaddr paddr, hwaddr xlat,
719 int prot,
720 target_ulong *address)
721 {
722 hwaddr iotlb;
723 CPUWatchpoint *wp;
724
725 if (memory_region_is_ram(section->mr)) {
726 /* Normal RAM. */
727 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
728 + xlat;
729 if (!section->readonly) {
730 iotlb |= PHYS_SECTION_NOTDIRTY;
731 } else {
732 iotlb |= PHYS_SECTION_ROM;
733 }
734 } else {
735 iotlb = section - address_space_memory.dispatch->sections;
736 iotlb += xlat;
737 }
738
739 /* Make accesses to pages with watchpoints go via the
740 watchpoint trap routines. */
741 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
742 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
743 /* Avoid trapping reads of pages with a write breakpoint. */
744 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
745 iotlb = PHYS_SECTION_WATCH + paddr;
746 *address |= TLB_MMIO;
747 break;
748 }
749 }
750 }
751
752 return iotlb;
753 }
754 #endif /* defined(CONFIG_USER_ONLY) */
755
756 #if !defined(CONFIG_USER_ONLY)
757
758 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
759 uint16_t section);
760 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
761
762 static uint16_t phys_section_add(MemoryRegionSection *section)
763 {
764 /* The physical section number is ORed with a page-aligned
765 * pointer to produce the iotlb entries. Thus it should
766 * never overflow into the page-aligned value.
767 */
768 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
769
770 if (next_map.sections_nb == next_map.sections_nb_alloc) {
771 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
772 16);
773 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
774 next_map.sections_nb_alloc);
775 }
776 next_map.sections[next_map.sections_nb] = *section;
777 memory_region_ref(section->mr);
778 return next_map.sections_nb++;
779 }
780
781 static void phys_section_destroy(MemoryRegion *mr)
782 {
783 memory_region_unref(mr);
784
785 if (mr->subpage) {
786 subpage_t *subpage = container_of(mr, subpage_t, iomem);
787 memory_region_destroy(&subpage->iomem);
788 g_free(subpage);
789 }
790 }
791
792 static void phys_sections_free(PhysPageMap *map)
793 {
794 while (map->sections_nb > 0) {
795 MemoryRegionSection *section = &map->sections[--map->sections_nb];
796 phys_section_destroy(section->mr);
797 }
798 g_free(map->sections);
799 g_free(map->nodes);
800 g_free(map);
801 }
802
803 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
804 {
805 subpage_t *subpage;
806 hwaddr base = section->offset_within_address_space
807 & TARGET_PAGE_MASK;
808 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
809 next_map.nodes, next_map.sections);
810 MemoryRegionSection subsection = {
811 .offset_within_address_space = base,
812 .size = int128_make64(TARGET_PAGE_SIZE),
813 };
814 hwaddr start, end;
815
816 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
817
818 if (!(existing->mr->subpage)) {
819 subpage = subpage_init(d->as, base);
820 subsection.mr = &subpage->iomem;
821 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
822 phys_section_add(&subsection));
823 } else {
824 subpage = container_of(existing->mr, subpage_t, iomem);
825 }
826 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
827 end = start + int128_get64(section->size) - 1;
828 subpage_register(subpage, start, end, phys_section_add(section));
829 }
830
831
832 static void register_multipage(AddressSpaceDispatch *d,
833 MemoryRegionSection *section)
834 {
835 hwaddr start_addr = section->offset_within_address_space;
836 uint16_t section_index = phys_section_add(section);
837 uint64_t num_pages = int128_get64(int128_rshift(section->size,
838 TARGET_PAGE_BITS));
839
840 assert(num_pages);
841 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
842 }
843
844 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
845 {
846 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
847 AddressSpaceDispatch *d = as->next_dispatch;
848 MemoryRegionSection now = *section, remain = *section;
849 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
850
851 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
852 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
853 - now.offset_within_address_space;
854
855 now.size = int128_min(int128_make64(left), now.size);
856 register_subpage(d, &now);
857 } else {
858 now.size = int128_zero();
859 }
860 while (int128_ne(remain.size, now.size)) {
861 remain.size = int128_sub(remain.size, now.size);
862 remain.offset_within_address_space += int128_get64(now.size);
863 remain.offset_within_region += int128_get64(now.size);
864 now = remain;
865 if (int128_lt(remain.size, page_size)) {
866 register_subpage(d, &now);
867 } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
868 now.size = page_size;
869 register_subpage(d, &now);
870 } else {
871 now.size = int128_and(now.size, int128_neg(page_size));
872 register_multipage(d, &now);
873 }
874 }
875 }
876
877 void qemu_flush_coalesced_mmio_buffer(void)
878 {
879 if (kvm_enabled())
880 kvm_flush_coalesced_mmio_buffer();
881 }
882
883 void qemu_mutex_lock_ramlist(void)
884 {
885 qemu_mutex_lock(&ram_list.mutex);
886 }
887
888 void qemu_mutex_unlock_ramlist(void)
889 {
890 qemu_mutex_unlock(&ram_list.mutex);
891 }
892
893 #if defined(__linux__) && !defined(TARGET_S390X)
894
895 #include <sys/vfs.h>
896
897 #define HUGETLBFS_MAGIC 0x958458f6
898
899 static long gethugepagesize(const char *path)
900 {
901 struct statfs fs;
902 int ret;
903
904 do {
905 ret = statfs(path, &fs);
906 } while (ret != 0 && errno == EINTR);
907
908 if (ret != 0) {
909 perror(path);
910 return 0;
911 }
912
913 if (fs.f_type != HUGETLBFS_MAGIC)
914 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
915
916 return fs.f_bsize;
917 }
918
919 static void *file_ram_alloc(RAMBlock *block,
920 ram_addr_t memory,
921 const char *path)
922 {
923 char *filename;
924 char *sanitized_name;
925 char *c;
926 void *area;
927 int fd;
928 #ifdef MAP_POPULATE
929 int flags;
930 #endif
931 unsigned long hpagesize;
932
933 hpagesize = gethugepagesize(path);
934 if (!hpagesize) {
935 return NULL;
936 }
937
938 if (memory < hpagesize) {
939 return NULL;
940 }
941
942 if (kvm_enabled() && !kvm_has_sync_mmu()) {
943 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
944 return NULL;
945 }
946
947 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
948 sanitized_name = g_strdup(block->mr->name);
949 for (c = sanitized_name; *c != '\0'; c++) {
950 if (*c == '/')
951 *c = '_';
952 }
953
954 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
955 sanitized_name);
956 g_free(sanitized_name);
957
958 fd = mkstemp(filename);
959 if (fd < 0) {
960 perror("unable to create backing store for hugepages");
961 g_free(filename);
962 return NULL;
963 }
964 unlink(filename);
965 g_free(filename);
966
967 memory = (memory+hpagesize-1) & ~(hpagesize-1);
968
969 /*
970 * ftruncate is not supported by hugetlbfs in older
971 * hosts, so don't bother bailing out on errors.
972 * If anything goes wrong with it under other filesystems,
973 * mmap will fail.
974 */
975 if (ftruncate(fd, memory))
976 perror("ftruncate");
977
978 #ifdef MAP_POPULATE
979 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
980 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
981 * to sidestep this quirk.
982 */
983 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
984 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
985 #else
986 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
987 #endif
988 if (area == MAP_FAILED) {
989 perror("file_ram_alloc: can't mmap RAM pages");
990 close(fd);
991 return (NULL);
992 }
993 block->fd = fd;
994 return area;
995 }
996 #endif
997
998 static ram_addr_t find_ram_offset(ram_addr_t size)
999 {
1000 RAMBlock *block, *next_block;
1001 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1002
1003 assert(size != 0); /* it would hand out same offset multiple times */
1004
1005 if (QTAILQ_EMPTY(&ram_list.blocks))
1006 return 0;
1007
1008 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1009 ram_addr_t end, next = RAM_ADDR_MAX;
1010
1011 end = block->offset + block->length;
1012
1013 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1014 if (next_block->offset >= end) {
1015 next = MIN(next, next_block->offset);
1016 }
1017 }
1018 if (next - end >= size && next - end < mingap) {
1019 offset = end;
1020 mingap = next - end;
1021 }
1022 }
1023
1024 if (offset == RAM_ADDR_MAX) {
1025 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1026 (uint64_t)size);
1027 abort();
1028 }
1029
1030 return offset;
1031 }
1032
1033 ram_addr_t last_ram_offset(void)
1034 {
1035 RAMBlock *block;
1036 ram_addr_t last = 0;
1037
1038 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1039 last = MAX(last, block->offset + block->length);
1040
1041 return last;
1042 }
1043
1044 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1045 {
1046 int ret;
1047 QemuOpts *machine_opts;
1048
1049 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1050 machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1051 if (machine_opts &&
1052 !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
1053 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1054 if (ret) {
1055 perror("qemu_madvise");
1056 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1057 "but dump_guest_core=off specified\n");
1058 }
1059 }
1060 }
1061
1062 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1063 {
1064 RAMBlock *new_block, *block;
1065
1066 new_block = NULL;
1067 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1068 if (block->offset == addr) {
1069 new_block = block;
1070 break;
1071 }
1072 }
1073 assert(new_block);
1074 assert(!new_block->idstr[0]);
1075
1076 if (dev) {
1077 char *id = qdev_get_dev_path(dev);
1078 if (id) {
1079 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1080 g_free(id);
1081 }
1082 }
1083 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1084
1085 /* This assumes the iothread lock is taken here too. */
1086 qemu_mutex_lock_ramlist();
1087 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1088 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1089 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1090 new_block->idstr);
1091 abort();
1092 }
1093 }
1094 qemu_mutex_unlock_ramlist();
1095 }
1096
1097 static int memory_try_enable_merging(void *addr, size_t len)
1098 {
1099 QemuOpts *opts;
1100
1101 opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1102 if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
1103 /* disabled by the user */
1104 return 0;
1105 }
1106
1107 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1108 }
1109
1110 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1111 MemoryRegion *mr)
1112 {
1113 RAMBlock *block, *new_block;
1114
1115 size = TARGET_PAGE_ALIGN(size);
1116 new_block = g_malloc0(sizeof(*new_block));
1117
1118 /* This assumes the iothread lock is taken here too. */
1119 qemu_mutex_lock_ramlist();
1120 new_block->mr = mr;
1121 new_block->offset = find_ram_offset(size);
1122 if (host) {
1123 new_block->host = host;
1124 new_block->flags |= RAM_PREALLOC_MASK;
1125 } else {
1126 if (mem_path) {
1127 #if defined (__linux__) && !defined(TARGET_S390X)
1128 new_block->host = file_ram_alloc(new_block, size, mem_path);
1129 if (!new_block->host) {
1130 new_block->host = qemu_anon_ram_alloc(size);
1131 memory_try_enable_merging(new_block->host, size);
1132 }
1133 #else
1134 fprintf(stderr, "-mem-path option unsupported\n");
1135 exit(1);
1136 #endif
1137 } else {
1138 if (xen_enabled()) {
1139 xen_ram_alloc(new_block->offset, size, mr);
1140 } else if (kvm_enabled()) {
1141 /* some s390/kvm configurations have special constraints */
1142 new_block->host = kvm_ram_alloc(size);
1143 } else {
1144 new_block->host = qemu_anon_ram_alloc(size);
1145 }
1146 memory_try_enable_merging(new_block->host, size);
1147 }
1148 }
1149 new_block->length = size;
1150
1151 /* Keep the list sorted from biggest to smallest block. */
1152 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1153 if (block->length < new_block->length) {
1154 break;
1155 }
1156 }
1157 if (block) {
1158 QTAILQ_INSERT_BEFORE(block, new_block, next);
1159 } else {
1160 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1161 }
1162 ram_list.mru_block = NULL;
1163
1164 ram_list.version++;
1165 qemu_mutex_unlock_ramlist();
1166
1167 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1168 last_ram_offset() >> TARGET_PAGE_BITS);
1169 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1170 0, size >> TARGET_PAGE_BITS);
1171 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1172
1173 qemu_ram_setup_dump(new_block->host, size);
1174 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1175
1176 if (kvm_enabled())
1177 kvm_setup_guest_memory(new_block->host, size);
1178
1179 return new_block->offset;
1180 }
1181
1182 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1183 {
1184 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1185 }
1186
1187 void qemu_ram_free_from_ptr(ram_addr_t addr)
1188 {
1189 RAMBlock *block;
1190
1191 /* This assumes the iothread lock is taken here too. */
1192 qemu_mutex_lock_ramlist();
1193 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1194 if (addr == block->offset) {
1195 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1196 ram_list.mru_block = NULL;
1197 ram_list.version++;
1198 g_free(block);
1199 break;
1200 }
1201 }
1202 qemu_mutex_unlock_ramlist();
1203 }
1204
1205 void qemu_ram_free(ram_addr_t addr)
1206 {
1207 RAMBlock *block;
1208
1209 /* This assumes the iothread lock is taken here too. */
1210 qemu_mutex_lock_ramlist();
1211 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1212 if (addr == block->offset) {
1213 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1214 ram_list.mru_block = NULL;
1215 ram_list.version++;
1216 if (block->flags & RAM_PREALLOC_MASK) {
1217 ;
1218 } else if (mem_path) {
1219 #if defined (__linux__) && !defined(TARGET_S390X)
1220 if (block->fd) {
1221 munmap(block->host, block->length);
1222 close(block->fd);
1223 } else {
1224 qemu_anon_ram_free(block->host, block->length);
1225 }
1226 #else
1227 abort();
1228 #endif
1229 } else {
1230 if (xen_enabled()) {
1231 xen_invalidate_map_cache_entry(block->host);
1232 } else {
1233 qemu_anon_ram_free(block->host, block->length);
1234 }
1235 }
1236 g_free(block);
1237 break;
1238 }
1239 }
1240 qemu_mutex_unlock_ramlist();
1241
1242 }
1243
1244 #ifndef _WIN32
1245 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1246 {
1247 RAMBlock *block;
1248 ram_addr_t offset;
1249 int flags;
1250 void *area, *vaddr;
1251
1252 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1253 offset = addr - block->offset;
1254 if (offset < block->length) {
1255 vaddr = block->host + offset;
1256 if (block->flags & RAM_PREALLOC_MASK) {
1257 ;
1258 } else {
1259 flags = MAP_FIXED;
1260 munmap(vaddr, length);
1261 if (mem_path) {
1262 #if defined(__linux__) && !defined(TARGET_S390X)
1263 if (block->fd) {
1264 #ifdef MAP_POPULATE
1265 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1266 MAP_PRIVATE;
1267 #else
1268 flags |= MAP_PRIVATE;
1269 #endif
1270 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1271 flags, block->fd, offset);
1272 } else {
1273 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1274 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1275 flags, -1, 0);
1276 }
1277 #else
1278 abort();
1279 #endif
1280 } else {
1281 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1282 flags |= MAP_SHARED | MAP_ANONYMOUS;
1283 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1284 flags, -1, 0);
1285 #else
1286 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1287 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1288 flags, -1, 0);
1289 #endif
1290 }
1291 if (area != vaddr) {
1292 fprintf(stderr, "Could not remap addr: "
1293 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1294 length, addr);
1295 exit(1);
1296 }
1297 memory_try_enable_merging(vaddr, length);
1298 qemu_ram_setup_dump(vaddr, length);
1299 }
1300 return;
1301 }
1302 }
1303 }
1304 #endif /* !_WIN32 */
1305
1306 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1307 {
1308 RAMBlock *block;
1309
1310 /* The list is protected by the iothread lock here. */
1311 block = ram_list.mru_block;
1312 if (block && addr - block->offset < block->length) {
1313 goto found;
1314 }
1315 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1316 if (addr - block->offset < block->length) {
1317 goto found;
1318 }
1319 }
1320
1321 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1322 abort();
1323
1324 found:
1325 ram_list.mru_block = block;
1326 return block;
1327 }
1328
1329 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1330 With the exception of the softmmu code in this file, this should
1331 only be used for local memory (e.g. video ram) that the device owns,
1332 and knows it isn't going to access beyond the end of the block.
1333
1334 It should not be used for general purpose DMA.
1335 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1336 */
1337 void *qemu_get_ram_ptr(ram_addr_t addr)
1338 {
1339 RAMBlock *block = qemu_get_ram_block(addr);
1340
1341 if (xen_enabled()) {
1342 /* We need to check if the requested address is in the RAM
1343 * because we don't want to map the entire memory in QEMU.
1344 * In that case just map until the end of the page.
1345 */
1346 if (block->offset == 0) {
1347 return xen_map_cache(addr, 0, 0);
1348 } else if (block->host == NULL) {
1349 block->host =
1350 xen_map_cache(block->offset, block->length, 1);
1351 }
1352 }
1353 return block->host + (addr - block->offset);
1354 }
1355
1356 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1357 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1358 *
1359 * ??? Is this still necessary?
1360 */
1361 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1362 {
1363 RAMBlock *block;
1364
1365 /* The list is protected by the iothread lock here. */
1366 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1367 if (addr - block->offset < block->length) {
1368 if (xen_enabled()) {
1369 /* We need to check if the requested address is in the RAM
1370 * because we don't want to map the entire memory in QEMU.
1371 * In that case just map until the end of the page.
1372 */
1373 if (block->offset == 0) {
1374 return xen_map_cache(addr, 0, 0);
1375 } else if (block->host == NULL) {
1376 block->host =
1377 xen_map_cache(block->offset, block->length, 1);
1378 }
1379 }
1380 return block->host + (addr - block->offset);
1381 }
1382 }
1383
1384 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1385 abort();
1386
1387 return NULL;
1388 }
1389
1390 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1391 * but takes a size argument */
1392 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1393 {
1394 if (*size == 0) {
1395 return NULL;
1396 }
1397 if (xen_enabled()) {
1398 return xen_map_cache(addr, *size, 1);
1399 } else {
1400 RAMBlock *block;
1401
1402 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1403 if (addr - block->offset < block->length) {
1404 if (addr - block->offset + *size > block->length)
1405 *size = block->length - addr + block->offset;
1406 return block->host + (addr - block->offset);
1407 }
1408 }
1409
1410 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1411 abort();
1412 }
1413 }
1414
1415 /* Some of the softmmu routines need to translate from a host pointer
1416 (typically a TLB entry) back to a ram offset. */
1417 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1418 {
1419 RAMBlock *block;
1420 uint8_t *host = ptr;
1421
1422 if (xen_enabled()) {
1423 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1424 return qemu_get_ram_block(*ram_addr)->mr;
1425 }
1426
1427 block = ram_list.mru_block;
1428 if (block && block->host && host - block->host < block->length) {
1429 goto found;
1430 }
1431
1432 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1433 /* This case append when the block is not mapped. */
1434 if (block->host == NULL) {
1435 continue;
1436 }
1437 if (host - block->host < block->length) {
1438 goto found;
1439 }
1440 }
1441
1442 return NULL;
1443
1444 found:
1445 *ram_addr = block->offset + (host - block->host);
1446 return block->mr;
1447 }
1448
1449 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1450 uint64_t val, unsigned size)
1451 {
1452 int dirty_flags;
1453 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1454 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1455 tb_invalidate_phys_page_fast(ram_addr, size);
1456 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1457 }
1458 switch (size) {
1459 case 1:
1460 stb_p(qemu_get_ram_ptr(ram_addr), val);
1461 break;
1462 case 2:
1463 stw_p(qemu_get_ram_ptr(ram_addr), val);
1464 break;
1465 case 4:
1466 stl_p(qemu_get_ram_ptr(ram_addr), val);
1467 break;
1468 default:
1469 abort();
1470 }
1471 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1472 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1473 /* we remove the notdirty callback only if the code has been
1474 flushed */
1475 if (dirty_flags == 0xff) {
1476 CPUArchState *env = current_cpu->env_ptr;
1477 tlb_set_dirty(env, env->mem_io_vaddr);
1478 }
1479 }
1480
1481 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1482 unsigned size, bool is_write)
1483 {
1484 return is_write;
1485 }
1486
1487 static const MemoryRegionOps notdirty_mem_ops = {
1488 .write = notdirty_mem_write,
1489 .valid.accepts = notdirty_mem_accepts,
1490 .endianness = DEVICE_NATIVE_ENDIAN,
1491 };
1492
1493 /* Generate a debug exception if a watchpoint has been hit. */
1494 static void check_watchpoint(int offset, int len_mask, int flags)
1495 {
1496 CPUArchState *env = current_cpu->env_ptr;
1497 target_ulong pc, cs_base;
1498 target_ulong vaddr;
1499 CPUWatchpoint *wp;
1500 int cpu_flags;
1501
1502 if (env->watchpoint_hit) {
1503 /* We re-entered the check after replacing the TB. Now raise
1504 * the debug interrupt so that is will trigger after the
1505 * current instruction. */
1506 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1507 return;
1508 }
1509 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1510 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1511 if ((vaddr == (wp->vaddr & len_mask) ||
1512 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1513 wp->flags |= BP_WATCHPOINT_HIT;
1514 if (!env->watchpoint_hit) {
1515 env->watchpoint_hit = wp;
1516 tb_check_watchpoint(env);
1517 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1518 env->exception_index = EXCP_DEBUG;
1519 cpu_loop_exit(env);
1520 } else {
1521 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1522 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1523 cpu_resume_from_signal(env, NULL);
1524 }
1525 }
1526 } else {
1527 wp->flags &= ~BP_WATCHPOINT_HIT;
1528 }
1529 }
1530 }
1531
1532 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1533 so these check for a hit then pass through to the normal out-of-line
1534 phys routines. */
1535 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1536 unsigned size)
1537 {
1538 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1539 switch (size) {
1540 case 1: return ldub_phys(addr);
1541 case 2: return lduw_phys(addr);
1542 case 4: return ldl_phys(addr);
1543 default: abort();
1544 }
1545 }
1546
1547 static void watch_mem_write(void *opaque, hwaddr addr,
1548 uint64_t val, unsigned size)
1549 {
1550 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1551 switch (size) {
1552 case 1:
1553 stb_phys(addr, val);
1554 break;
1555 case 2:
1556 stw_phys(addr, val);
1557 break;
1558 case 4:
1559 stl_phys(addr, val);
1560 break;
1561 default: abort();
1562 }
1563 }
1564
1565 static const MemoryRegionOps watch_mem_ops = {
1566 .read = watch_mem_read,
1567 .write = watch_mem_write,
1568 .endianness = DEVICE_NATIVE_ENDIAN,
1569 };
1570
1571 static uint64_t subpage_read(void *opaque, hwaddr addr,
1572 unsigned len)
1573 {
1574 subpage_t *subpage = opaque;
1575 uint8_t buf[4];
1576
1577 #if defined(DEBUG_SUBPAGE)
1578 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1579 subpage, len, addr);
1580 #endif
1581 address_space_read(subpage->as, addr + subpage->base, buf, len);
1582 switch (len) {
1583 case 1:
1584 return ldub_p(buf);
1585 case 2:
1586 return lduw_p(buf);
1587 case 4:
1588 return ldl_p(buf);
1589 default:
1590 abort();
1591 }
1592 }
1593
1594 static void subpage_write(void *opaque, hwaddr addr,
1595 uint64_t value, unsigned len)
1596 {
1597 subpage_t *subpage = opaque;
1598 uint8_t buf[4];
1599
1600 #if defined(DEBUG_SUBPAGE)
1601 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1602 " value %"PRIx64"\n",
1603 __func__, subpage, len, addr, value);
1604 #endif
1605 switch (len) {
1606 case 1:
1607 stb_p(buf, value);
1608 break;
1609 case 2:
1610 stw_p(buf, value);
1611 break;
1612 case 4:
1613 stl_p(buf, value);
1614 break;
1615 default:
1616 abort();
1617 }
1618 address_space_write(subpage->as, addr + subpage->base, buf, len);
1619 }
1620
1621 static bool subpage_accepts(void *opaque, hwaddr addr,
1622 unsigned size, bool is_write)
1623 {
1624 subpage_t *subpage = opaque;
1625 #if defined(DEBUG_SUBPAGE)
1626 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1627 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1628 #endif
1629
1630 return address_space_access_valid(subpage->as, addr + subpage->base,
1631 size, is_write);
1632 }
1633
1634 static const MemoryRegionOps subpage_ops = {
1635 .read = subpage_read,
1636 .write = subpage_write,
1637 .valid.accepts = subpage_accepts,
1638 .endianness = DEVICE_NATIVE_ENDIAN,
1639 };
1640
1641 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1642 uint16_t section)
1643 {
1644 int idx, eidx;
1645
1646 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1647 return -1;
1648 idx = SUBPAGE_IDX(start);
1649 eidx = SUBPAGE_IDX(end);
1650 #if defined(DEBUG_SUBPAGE)
1651 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1652 mmio, start, end, idx, eidx, memory);
1653 #endif
1654 for (; idx <= eidx; idx++) {
1655 mmio->sub_section[idx] = section;
1656 }
1657
1658 return 0;
1659 }
1660
1661 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1662 {
1663 subpage_t *mmio;
1664
1665 mmio = g_malloc0(sizeof(subpage_t));
1666
1667 mmio->as = as;
1668 mmio->base = base;
1669 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1670 "subpage", TARGET_PAGE_SIZE);
1671 mmio->iomem.subpage = true;
1672 #if defined(DEBUG_SUBPAGE)
1673 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1674 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1675 #endif
1676 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1677
1678 return mmio;
1679 }
1680
1681 static uint16_t dummy_section(MemoryRegion *mr)
1682 {
1683 MemoryRegionSection section = {
1684 .mr = mr,
1685 .offset_within_address_space = 0,
1686 .offset_within_region = 0,
1687 .size = int128_2_64(),
1688 };
1689
1690 return phys_section_add(&section);
1691 }
1692
1693 MemoryRegion *iotlb_to_region(hwaddr index)
1694 {
1695 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1696 }
1697
1698 static void io_mem_init(void)
1699 {
1700 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1701 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1702 "unassigned", UINT64_MAX);
1703 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1704 "notdirty", UINT64_MAX);
1705 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1706 "watch", UINT64_MAX);
1707 }
1708
1709 static void mem_begin(MemoryListener *listener)
1710 {
1711 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1712 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1713
1714 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1715 d->as = as;
1716 as->next_dispatch = d;
1717 }
1718
1719 static void mem_commit(MemoryListener *listener)
1720 {
1721 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1722 AddressSpaceDispatch *cur = as->dispatch;
1723 AddressSpaceDispatch *next = as->next_dispatch;
1724
1725 next->nodes = next_map.nodes;
1726 next->sections = next_map.sections;
1727
1728 as->dispatch = next;
1729 g_free(cur);
1730 }
1731
1732 static void core_begin(MemoryListener *listener)
1733 {
1734 uint16_t n;
1735
1736 prev_map = g_new(PhysPageMap, 1);
1737 *prev_map = next_map;
1738
1739 memset(&next_map, 0, sizeof(next_map));
1740 n = dummy_section(&io_mem_unassigned);
1741 assert(n == PHYS_SECTION_UNASSIGNED);
1742 n = dummy_section(&io_mem_notdirty);
1743 assert(n == PHYS_SECTION_NOTDIRTY);
1744 n = dummy_section(&io_mem_rom);
1745 assert(n == PHYS_SECTION_ROM);
1746 n = dummy_section(&io_mem_watch);
1747 assert(n == PHYS_SECTION_WATCH);
1748 }
1749
1750 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1751 * All AddressSpaceDispatch instances have switched to the next map.
1752 */
1753 static void core_commit(MemoryListener *listener)
1754 {
1755 phys_sections_free(prev_map);
1756 }
1757
1758 static void tcg_commit(MemoryListener *listener)
1759 {
1760 CPUArchState *env;
1761
1762 /* since each CPU stores ram addresses in its TLB cache, we must
1763 reset the modified entries */
1764 /* XXX: slow ! */
1765 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1766 tlb_flush(env, 1);
1767 }
1768 }
1769
1770 static void core_log_global_start(MemoryListener *listener)
1771 {
1772 cpu_physical_memory_set_dirty_tracking(1);
1773 }
1774
1775 static void core_log_global_stop(MemoryListener *listener)
1776 {
1777 cpu_physical_memory_set_dirty_tracking(0);
1778 }
1779
1780 static MemoryListener core_memory_listener = {
1781 .begin = core_begin,
1782 .commit = core_commit,
1783 .log_global_start = core_log_global_start,
1784 .log_global_stop = core_log_global_stop,
1785 .priority = 1,
1786 };
1787
1788 static MemoryListener tcg_memory_listener = {
1789 .commit = tcg_commit,
1790 };
1791
1792 void address_space_init_dispatch(AddressSpace *as)
1793 {
1794 as->dispatch = NULL;
1795 as->dispatch_listener = (MemoryListener) {
1796 .begin = mem_begin,
1797 .commit = mem_commit,
1798 .region_add = mem_add,
1799 .region_nop = mem_add,
1800 .priority = 0,
1801 };
1802 memory_listener_register(&as->dispatch_listener, as);
1803 }
1804
1805 void address_space_destroy_dispatch(AddressSpace *as)
1806 {
1807 AddressSpaceDispatch *d = as->dispatch;
1808
1809 memory_listener_unregister(&as->dispatch_listener);
1810 g_free(d);
1811 as->dispatch = NULL;
1812 }
1813
1814 static void memory_map_init(void)
1815 {
1816 system_memory = g_malloc(sizeof(*system_memory));
1817 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1818 address_space_init(&address_space_memory, system_memory, "memory");
1819
1820 system_io = g_malloc(sizeof(*system_io));
1821 memory_region_init(system_io, NULL, "io", 65536);
1822 address_space_init(&address_space_io, system_io, "I/O");
1823
1824 memory_listener_register(&core_memory_listener, &address_space_memory);
1825 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1826 }
1827
1828 MemoryRegion *get_system_memory(void)
1829 {
1830 return system_memory;
1831 }
1832
1833 MemoryRegion *get_system_io(void)
1834 {
1835 return system_io;
1836 }
1837
1838 #endif /* !defined(CONFIG_USER_ONLY) */
1839
1840 /* physical memory access (slow version, mainly for debug) */
1841 #if defined(CONFIG_USER_ONLY)
1842 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1843 uint8_t *buf, int len, int is_write)
1844 {
1845 int l, flags;
1846 target_ulong page;
1847 void * p;
1848
1849 while (len > 0) {
1850 page = addr & TARGET_PAGE_MASK;
1851 l = (page + TARGET_PAGE_SIZE) - addr;
1852 if (l > len)
1853 l = len;
1854 flags = page_get_flags(page);
1855 if (!(flags & PAGE_VALID))
1856 return -1;
1857 if (is_write) {
1858 if (!(flags & PAGE_WRITE))
1859 return -1;
1860 /* XXX: this code should not depend on lock_user */
1861 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1862 return -1;
1863 memcpy(p, buf, l);
1864 unlock_user(p, addr, l);
1865 } else {
1866 if (!(flags & PAGE_READ))
1867 return -1;
1868 /* XXX: this code should not depend on lock_user */
1869 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1870 return -1;
1871 memcpy(buf, p, l);
1872 unlock_user(p, addr, 0);
1873 }
1874 len -= l;
1875 buf += l;
1876 addr += l;
1877 }
1878 return 0;
1879 }
1880
1881 #else
1882
1883 static void invalidate_and_set_dirty(hwaddr addr,
1884 hwaddr length)
1885 {
1886 if (!cpu_physical_memory_is_dirty(addr)) {
1887 /* invalidate code */
1888 tb_invalidate_phys_page_range(addr, addr + length, 0);
1889 /* set dirty bit */
1890 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1891 }
1892 xen_modified_memory(addr, length);
1893 }
1894
1895 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1896 {
1897 if (memory_region_is_ram(mr)) {
1898 return !(is_write && mr->readonly);
1899 }
1900 if (memory_region_is_romd(mr)) {
1901 return !is_write;
1902 }
1903
1904 return false;
1905 }
1906
1907 static inline int memory_access_size(MemoryRegion *mr, int l, hwaddr addr)
1908 {
1909 if (l >= 4 && (((addr & 3) == 0 || mr->ops->impl.unaligned))) {
1910 return 4;
1911 }
1912 if (l >= 2 && (((addr & 1) == 0) || mr->ops->impl.unaligned)) {
1913 return 2;
1914 }
1915 return 1;
1916 }
1917
1918 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1919 int len, bool is_write)
1920 {
1921 hwaddr l;
1922 uint8_t *ptr;
1923 uint64_t val;
1924 hwaddr addr1;
1925 MemoryRegion *mr;
1926 bool error = false;
1927
1928 while (len > 0) {
1929 l = len;
1930 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1931
1932 if (is_write) {
1933 if (!memory_access_is_direct(mr, is_write)) {
1934 l = memory_access_size(mr, l, addr1);
1935 /* XXX: could force current_cpu to NULL to avoid
1936 potential bugs */
1937 if (l == 4) {
1938 /* 32 bit write access */
1939 val = ldl_p(buf);
1940 error |= io_mem_write(mr, addr1, val, 4);
1941 } else if (l == 2) {
1942 /* 16 bit write access */
1943 val = lduw_p(buf);
1944 error |= io_mem_write(mr, addr1, val, 2);
1945 } else {
1946 /* 8 bit write access */
1947 val = ldub_p(buf);
1948 error |= io_mem_write(mr, addr1, val, 1);
1949 }
1950 } else {
1951 addr1 += memory_region_get_ram_addr(mr);
1952 /* RAM case */
1953 ptr = qemu_get_ram_ptr(addr1);
1954 memcpy(ptr, buf, l);
1955 invalidate_and_set_dirty(addr1, l);
1956 }
1957 } else {
1958 if (!memory_access_is_direct(mr, is_write)) {
1959 /* I/O case */
1960 l = memory_access_size(mr, l, addr1);
1961 if (l == 4) {
1962 /* 32 bit read access */
1963 error |= io_mem_read(mr, addr1, &val, 4);
1964 stl_p(buf, val);
1965 } else if (l == 2) {
1966 /* 16 bit read access */
1967 error |= io_mem_read(mr, addr1, &val, 2);
1968 stw_p(buf, val);
1969 } else {
1970 /* 8 bit read access */
1971 error |= io_mem_read(mr, addr1, &val, 1);
1972 stb_p(buf, val);
1973 }
1974 } else {
1975 /* RAM case */
1976 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1977 memcpy(buf, ptr, l);
1978 }
1979 }
1980 len -= l;
1981 buf += l;
1982 addr += l;
1983 }
1984
1985 return error;
1986 }
1987
1988 bool address_space_write(AddressSpace *as, hwaddr addr,
1989 const uint8_t *buf, int len)
1990 {
1991 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
1992 }
1993
1994 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1995 {
1996 return address_space_rw(as, addr, buf, len, false);
1997 }
1998
1999
2000 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2001 int len, int is_write)
2002 {
2003 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2004 }
2005
2006 /* used for ROM loading : can write in RAM and ROM */
2007 void cpu_physical_memory_write_rom(hwaddr addr,
2008 const uint8_t *buf, int len)
2009 {
2010 hwaddr l;
2011 uint8_t *ptr;
2012 hwaddr addr1;
2013 MemoryRegion *mr;
2014
2015 while (len > 0) {
2016 l = len;
2017 mr = address_space_translate(&address_space_memory,
2018 addr, &addr1, &l, true);
2019
2020 if (!(memory_region_is_ram(mr) ||
2021 memory_region_is_romd(mr))) {
2022 /* do nothing */
2023 } else {
2024 addr1 += memory_region_get_ram_addr(mr);
2025 /* ROM/RAM case */
2026 ptr = qemu_get_ram_ptr(addr1);
2027 memcpy(ptr, buf, l);
2028 invalidate_and_set_dirty(addr1, l);
2029 }
2030 len -= l;
2031 buf += l;
2032 addr += l;
2033 }
2034 }
2035
2036 typedef struct {
2037 MemoryRegion *mr;
2038 void *buffer;
2039 hwaddr addr;
2040 hwaddr len;
2041 } BounceBuffer;
2042
2043 static BounceBuffer bounce;
2044
2045 typedef struct MapClient {
2046 void *opaque;
2047 void (*callback)(void *opaque);
2048 QLIST_ENTRY(MapClient) link;
2049 } MapClient;
2050
2051 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2052 = QLIST_HEAD_INITIALIZER(map_client_list);
2053
2054 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2055 {
2056 MapClient *client = g_malloc(sizeof(*client));
2057
2058 client->opaque = opaque;
2059 client->callback = callback;
2060 QLIST_INSERT_HEAD(&map_client_list, client, link);
2061 return client;
2062 }
2063
2064 static void cpu_unregister_map_client(void *_client)
2065 {
2066 MapClient *client = (MapClient *)_client;
2067
2068 QLIST_REMOVE(client, link);
2069 g_free(client);
2070 }
2071
2072 static void cpu_notify_map_clients(void)
2073 {
2074 MapClient *client;
2075
2076 while (!QLIST_EMPTY(&map_client_list)) {
2077 client = QLIST_FIRST(&map_client_list);
2078 client->callback(client->opaque);
2079 cpu_unregister_map_client(client);
2080 }
2081 }
2082
2083 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2084 {
2085 MemoryRegion *mr;
2086 hwaddr l, xlat;
2087
2088 while (len > 0) {
2089 l = len;
2090 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2091 if (!memory_access_is_direct(mr, is_write)) {
2092 l = memory_access_size(mr, l, addr);
2093 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2094 return false;
2095 }
2096 }
2097
2098 len -= l;
2099 addr += l;
2100 }
2101 return true;
2102 }
2103
2104 /* Map a physical memory region into a host virtual address.
2105 * May map a subset of the requested range, given by and returned in *plen.
2106 * May return NULL if resources needed to perform the mapping are exhausted.
2107 * Use only for reads OR writes - not for read-modify-write operations.
2108 * Use cpu_register_map_client() to know when retrying the map operation is
2109 * likely to succeed.
2110 */
2111 void *address_space_map(AddressSpace *as,
2112 hwaddr addr,
2113 hwaddr *plen,
2114 bool is_write)
2115 {
2116 hwaddr len = *plen;
2117 hwaddr done = 0;
2118 hwaddr l, xlat, base;
2119 MemoryRegion *mr, *this_mr;
2120 ram_addr_t raddr;
2121
2122 if (len == 0) {
2123 return NULL;
2124 }
2125
2126 l = len;
2127 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2128 if (!memory_access_is_direct(mr, is_write)) {
2129 if (bounce.buffer) {
2130 return NULL;
2131 }
2132 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2133 bounce.addr = addr;
2134 bounce.len = l;
2135
2136 memory_region_ref(mr);
2137 bounce.mr = mr;
2138 if (!is_write) {
2139 address_space_read(as, addr, bounce.buffer, l);
2140 }
2141
2142 *plen = l;
2143 return bounce.buffer;
2144 }
2145
2146 base = xlat;
2147 raddr = memory_region_get_ram_addr(mr);
2148
2149 for (;;) {
2150 len -= l;
2151 addr += l;
2152 done += l;
2153 if (len == 0) {
2154 break;
2155 }
2156
2157 l = len;
2158 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2159 if (this_mr != mr || xlat != base + done) {
2160 break;
2161 }
2162 }
2163
2164 memory_region_ref(mr);
2165 *plen = done;
2166 return qemu_ram_ptr_length(raddr + base, plen);
2167 }
2168
2169 /* Unmaps a memory region previously mapped by address_space_map().
2170 * Will also mark the memory as dirty if is_write == 1. access_len gives
2171 * the amount of memory that was actually read or written by the caller.
2172 */
2173 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2174 int is_write, hwaddr access_len)
2175 {
2176 if (buffer != bounce.buffer) {
2177 MemoryRegion *mr;
2178 ram_addr_t addr1;
2179
2180 mr = qemu_ram_addr_from_host(buffer, &addr1);
2181 assert(mr != NULL);
2182 if (is_write) {
2183 while (access_len) {
2184 unsigned l;
2185 l = TARGET_PAGE_SIZE;
2186 if (l > access_len)
2187 l = access_len;
2188 invalidate_and_set_dirty(addr1, l);
2189 addr1 += l;
2190 access_len -= l;
2191 }
2192 }
2193 if (xen_enabled()) {
2194 xen_invalidate_map_cache_entry(buffer);
2195 }
2196 memory_region_unref(mr);
2197 return;
2198 }
2199 if (is_write) {
2200 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2201 }
2202 qemu_vfree(bounce.buffer);
2203 bounce.buffer = NULL;
2204 memory_region_unref(bounce.mr);
2205 cpu_notify_map_clients();
2206 }
2207
2208 void *cpu_physical_memory_map(hwaddr addr,
2209 hwaddr *plen,
2210 int is_write)
2211 {
2212 return address_space_map(&address_space_memory, addr, plen, is_write);
2213 }
2214
2215 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2216 int is_write, hwaddr access_len)
2217 {
2218 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2219 }
2220
2221 /* warning: addr must be aligned */
2222 static inline uint32_t ldl_phys_internal(hwaddr addr,
2223 enum device_endian endian)
2224 {
2225 uint8_t *ptr;
2226 uint64_t val;
2227 MemoryRegion *mr;
2228 hwaddr l = 4;
2229 hwaddr addr1;
2230
2231 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2232 false);
2233 if (l < 4 || !memory_access_is_direct(mr, false)) {
2234 /* I/O case */
2235 io_mem_read(mr, addr1, &val, 4);
2236 #if defined(TARGET_WORDS_BIGENDIAN)
2237 if (endian == DEVICE_LITTLE_ENDIAN) {
2238 val = bswap32(val);
2239 }
2240 #else
2241 if (endian == DEVICE_BIG_ENDIAN) {
2242 val = bswap32(val);
2243 }
2244 #endif
2245 } else {
2246 /* RAM case */
2247 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2248 & TARGET_PAGE_MASK)
2249 + addr1);
2250 switch (endian) {
2251 case DEVICE_LITTLE_ENDIAN:
2252 val = ldl_le_p(ptr);
2253 break;
2254 case DEVICE_BIG_ENDIAN:
2255 val = ldl_be_p(ptr);
2256 break;
2257 default:
2258 val = ldl_p(ptr);
2259 break;
2260 }
2261 }
2262 return val;
2263 }
2264
2265 uint32_t ldl_phys(hwaddr addr)
2266 {
2267 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2268 }
2269
2270 uint32_t ldl_le_phys(hwaddr addr)
2271 {
2272 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2273 }
2274
2275 uint32_t ldl_be_phys(hwaddr addr)
2276 {
2277 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2278 }
2279
2280 /* warning: addr must be aligned */
2281 static inline uint64_t ldq_phys_internal(hwaddr addr,
2282 enum device_endian endian)
2283 {
2284 uint8_t *ptr;
2285 uint64_t val;
2286 MemoryRegion *mr;
2287 hwaddr l = 8;
2288 hwaddr addr1;
2289
2290 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2291 false);
2292 if (l < 8 || !memory_access_is_direct(mr, false)) {
2293 /* I/O case */
2294 io_mem_read(mr, addr1, &val, 8);
2295 #if defined(TARGET_WORDS_BIGENDIAN)
2296 if (endian == DEVICE_LITTLE_ENDIAN) {
2297 val = bswap64(val);
2298 }
2299 #else
2300 if (endian == DEVICE_BIG_ENDIAN) {
2301 val = bswap64(val);
2302 }
2303 #endif
2304 } else {
2305 /* RAM case */
2306 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2307 & TARGET_PAGE_MASK)
2308 + addr1);
2309 switch (endian) {
2310 case DEVICE_LITTLE_ENDIAN:
2311 val = ldq_le_p(ptr);
2312 break;
2313 case DEVICE_BIG_ENDIAN:
2314 val = ldq_be_p(ptr);
2315 break;
2316 default:
2317 val = ldq_p(ptr);
2318 break;
2319 }
2320 }
2321 return val;
2322 }
2323
2324 uint64_t ldq_phys(hwaddr addr)
2325 {
2326 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2327 }
2328
2329 uint64_t ldq_le_phys(hwaddr addr)
2330 {
2331 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2332 }
2333
2334 uint64_t ldq_be_phys(hwaddr addr)
2335 {
2336 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2337 }
2338
2339 /* XXX: optimize */
2340 uint32_t ldub_phys(hwaddr addr)
2341 {
2342 uint8_t val;
2343 cpu_physical_memory_read(addr, &val, 1);
2344 return val;
2345 }
2346
2347 /* warning: addr must be aligned */
2348 static inline uint32_t lduw_phys_internal(hwaddr addr,
2349 enum device_endian endian)
2350 {
2351 uint8_t *ptr;
2352 uint64_t val;
2353 MemoryRegion *mr;
2354 hwaddr l = 2;
2355 hwaddr addr1;
2356
2357 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2358 false);
2359 if (l < 2 || !memory_access_is_direct(mr, false)) {
2360 /* I/O case */
2361 io_mem_read(mr, addr1, &val, 2);
2362 #if defined(TARGET_WORDS_BIGENDIAN)
2363 if (endian == DEVICE_LITTLE_ENDIAN) {
2364 val = bswap16(val);
2365 }
2366 #else
2367 if (endian == DEVICE_BIG_ENDIAN) {
2368 val = bswap16(val);
2369 }
2370 #endif
2371 } else {
2372 /* RAM case */
2373 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2374 & TARGET_PAGE_MASK)
2375 + addr1);
2376 switch (endian) {
2377 case DEVICE_LITTLE_ENDIAN:
2378 val = lduw_le_p(ptr);
2379 break;
2380 case DEVICE_BIG_ENDIAN:
2381 val = lduw_be_p(ptr);
2382 break;
2383 default:
2384 val = lduw_p(ptr);
2385 break;
2386 }
2387 }
2388 return val;
2389 }
2390
2391 uint32_t lduw_phys(hwaddr addr)
2392 {
2393 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2394 }
2395
2396 uint32_t lduw_le_phys(hwaddr addr)
2397 {
2398 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2399 }
2400
2401 uint32_t lduw_be_phys(hwaddr addr)
2402 {
2403 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2404 }
2405
2406 /* warning: addr must be aligned. The ram page is not masked as dirty
2407 and the code inside is not invalidated. It is useful if the dirty
2408 bits are used to track modified PTEs */
2409 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2410 {
2411 uint8_t *ptr;
2412 MemoryRegion *mr;
2413 hwaddr l = 4;
2414 hwaddr addr1;
2415
2416 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2417 true);
2418 if (l < 4 || !memory_access_is_direct(mr, true)) {
2419 io_mem_write(mr, addr1, val, 4);
2420 } else {
2421 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2422 ptr = qemu_get_ram_ptr(addr1);
2423 stl_p(ptr, val);
2424
2425 if (unlikely(in_migration)) {
2426 if (!cpu_physical_memory_is_dirty(addr1)) {
2427 /* invalidate code */
2428 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2429 /* set dirty bit */
2430 cpu_physical_memory_set_dirty_flags(
2431 addr1, (0xff & ~CODE_DIRTY_FLAG));
2432 }
2433 }
2434 }
2435 }
2436
2437 /* warning: addr must be aligned */
2438 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2439 enum device_endian endian)
2440 {
2441 uint8_t *ptr;
2442 MemoryRegion *mr;
2443 hwaddr l = 4;
2444 hwaddr addr1;
2445
2446 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2447 true);
2448 if (l < 4 || !memory_access_is_direct(mr, true)) {
2449 #if defined(TARGET_WORDS_BIGENDIAN)
2450 if (endian == DEVICE_LITTLE_ENDIAN) {
2451 val = bswap32(val);
2452 }
2453 #else
2454 if (endian == DEVICE_BIG_ENDIAN) {
2455 val = bswap32(val);
2456 }
2457 #endif
2458 io_mem_write(mr, addr1, val, 4);
2459 } else {
2460 /* RAM case */
2461 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2462 ptr = qemu_get_ram_ptr(addr1);
2463 switch (endian) {
2464 case DEVICE_LITTLE_ENDIAN:
2465 stl_le_p(ptr, val);
2466 break;
2467 case DEVICE_BIG_ENDIAN:
2468 stl_be_p(ptr, val);
2469 break;
2470 default:
2471 stl_p(ptr, val);
2472 break;
2473 }
2474 invalidate_and_set_dirty(addr1, 4);
2475 }
2476 }
2477
2478 void stl_phys(hwaddr addr, uint32_t val)
2479 {
2480 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2481 }
2482
2483 void stl_le_phys(hwaddr addr, uint32_t val)
2484 {
2485 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2486 }
2487
2488 void stl_be_phys(hwaddr addr, uint32_t val)
2489 {
2490 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2491 }
2492
2493 /* XXX: optimize */
2494 void stb_phys(hwaddr addr, uint32_t val)
2495 {
2496 uint8_t v = val;
2497 cpu_physical_memory_write(addr, &v, 1);
2498 }
2499
2500 /* warning: addr must be aligned */
2501 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2502 enum device_endian endian)
2503 {
2504 uint8_t *ptr;
2505 MemoryRegion *mr;
2506 hwaddr l = 2;
2507 hwaddr addr1;
2508
2509 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2510 true);
2511 if (l < 2 || !memory_access_is_direct(mr, true)) {
2512 #if defined(TARGET_WORDS_BIGENDIAN)
2513 if (endian == DEVICE_LITTLE_ENDIAN) {
2514 val = bswap16(val);
2515 }
2516 #else
2517 if (endian == DEVICE_BIG_ENDIAN) {
2518 val = bswap16(val);
2519 }
2520 #endif
2521 io_mem_write(mr, addr1, val, 2);
2522 } else {
2523 /* RAM case */
2524 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2525 ptr = qemu_get_ram_ptr(addr1);
2526 switch (endian) {
2527 case DEVICE_LITTLE_ENDIAN:
2528 stw_le_p(ptr, val);
2529 break;
2530 case DEVICE_BIG_ENDIAN:
2531 stw_be_p(ptr, val);
2532 break;
2533 default:
2534 stw_p(ptr, val);
2535 break;
2536 }
2537 invalidate_and_set_dirty(addr1, 2);
2538 }
2539 }
2540
2541 void stw_phys(hwaddr addr, uint32_t val)
2542 {
2543 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2544 }
2545
2546 void stw_le_phys(hwaddr addr, uint32_t val)
2547 {
2548 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2549 }
2550
2551 void stw_be_phys(hwaddr addr, uint32_t val)
2552 {
2553 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2554 }
2555
2556 /* XXX: optimize */
2557 void stq_phys(hwaddr addr, uint64_t val)
2558 {
2559 val = tswap64(val);
2560 cpu_physical_memory_write(addr, &val, 8);
2561 }
2562
2563 void stq_le_phys(hwaddr addr, uint64_t val)
2564 {
2565 val = cpu_to_le64(val);
2566 cpu_physical_memory_write(addr, &val, 8);
2567 }
2568
2569 void stq_be_phys(hwaddr addr, uint64_t val)
2570 {
2571 val = cpu_to_be64(val);
2572 cpu_physical_memory_write(addr, &val, 8);
2573 }
2574
2575 /* virtual memory access for debug (includes writing to ROM) */
2576 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2577 uint8_t *buf, int len, int is_write)
2578 {
2579 int l;
2580 hwaddr phys_addr;
2581 target_ulong page;
2582
2583 while (len > 0) {
2584 page = addr & TARGET_PAGE_MASK;
2585 phys_addr = cpu_get_phys_page_debug(env, page);
2586 /* if no physical page mapped, return an error */
2587 if (phys_addr == -1)
2588 return -1;
2589 l = (page + TARGET_PAGE_SIZE) - addr;
2590 if (l > len)
2591 l = len;
2592 phys_addr += (addr & ~TARGET_PAGE_MASK);
2593 if (is_write)
2594 cpu_physical_memory_write_rom(phys_addr, buf, l);
2595 else
2596 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2597 len -= l;
2598 buf += l;
2599 addr += l;
2600 }
2601 return 0;
2602 }
2603 #endif
2604
2605 #if !defined(CONFIG_USER_ONLY)
2606
2607 /*
2608 * A helper function for the _utterly broken_ virtio device model to find out if
2609 * it's running on a big endian machine. Don't do this at home kids!
2610 */
2611 bool virtio_is_big_endian(void);
2612 bool virtio_is_big_endian(void)
2613 {
2614 #if defined(TARGET_WORDS_BIGENDIAN)
2615 return true;
2616 #else
2617 return false;
2618 #endif
2619 }
2620
2621 #endif
2622
2623 #ifndef CONFIG_USER_ONLY
2624 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2625 {
2626 MemoryRegion*mr;
2627 hwaddr l = 1;
2628
2629 mr = address_space_translate(&address_space_memory,
2630 phys_addr, &phys_addr, &l, false);
2631
2632 return !(memory_region_is_ram(mr) ||
2633 memory_region_is_romd(mr));
2634 }
2635
2636 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2637 {
2638 RAMBlock *block;
2639
2640 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2641 func(block->host, block->offset, block->length, opaque);
2642 }
2643 }
2644 #endif
QEmu