?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
audio/mixeng_template.h: fix inline declaration
[qemu/rayw.git] / exec.c
blob2e31ffcb2c17e798073bfe561480a2301a2fee8e
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
51
52 #include "exec/memory-internal.h"
53
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 typedef PhysPageEntry Node[L2_SIZE];
92
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101 };
102
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
110
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
115
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
124
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
127
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132
133 static MemoryRegion io_mem_watch;
134 #endif
135
136 #if !defined(CONFIG_USER_ONLY)
137
138 static void phys_map_node_reserve(unsigned nodes)
139 {
140 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
141 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
142 16);
143 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
144 next_map.nodes_nb + nodes);
145 next_map.nodes = g_renew(Node, next_map.nodes,
146 next_map.nodes_nb_alloc);
147 }
148 }
149
150 static uint16_t phys_map_node_alloc(void)
151 {
152 unsigned i;
153 uint16_t ret;
154
155 ret = next_map.nodes_nb++;
156 assert(ret != PHYS_MAP_NODE_NIL);
157 assert(ret != next_map.nodes_nb_alloc);
158 for (i = 0; i < L2_SIZE; ++i) {
159 next_map.nodes[ret][i].is_leaf = 0;
160 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
161 }
162 return ret;
163 }
164
165 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
166 hwaddr *nb, uint16_t leaf,
167 int level)
168 {
169 PhysPageEntry *p;
170 int i;
171 hwaddr step = (hwaddr)1 << (level * L2_BITS);
172
173 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
174 lp->ptr = phys_map_node_alloc();
175 p = next_map.nodes[lp->ptr];
176 if (level == 0) {
177 for (i = 0; i < L2_SIZE; i++) {
178 p[i].is_leaf = 1;
179 p[i].ptr = PHYS_SECTION_UNASSIGNED;
180 }
181 }
182 } else {
183 p = next_map.nodes[lp->ptr];
184 }
185 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
186
187 while (*nb && lp < &p[L2_SIZE]) {
188 if ((*index & (step - 1)) == 0 && *nb >= step) {
189 lp->is_leaf = true;
190 lp->ptr = leaf;
191 *index += step;
192 *nb -= step;
193 } else {
194 phys_page_set_level(lp, index, nb, leaf, level - 1);
195 }
196 ++lp;
197 }
198 }
199
200 static void phys_page_set(AddressSpaceDispatch *d,
201 hwaddr index, hwaddr nb,
202 uint16_t leaf)
203 {
204 /* Wildly overreserve - it doesn't matter much. */
205 phys_map_node_reserve(3 * P_L2_LEVELS);
206
207 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
208 }
209
210 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
211 Node *nodes, MemoryRegionSection *sections)
212 {
213 PhysPageEntry *p;
214 int i;
215
216 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
217 if (lp.ptr == PHYS_MAP_NODE_NIL) {
218 return &sections[PHYS_SECTION_UNASSIGNED];
219 }
220 p = nodes[lp.ptr];
221 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
222 }
223 return &sections[lp.ptr];
224 }
225
226 bool memory_region_is_unassigned(MemoryRegion *mr)
227 {
228 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
229 && mr != &io_mem_watch;
230 }
231
232 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
233 hwaddr addr,
234 bool resolve_subpage)
235 {
236 MemoryRegionSection *section;
237 subpage_t *subpage;
238
239 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
240 d->nodes, d->sections);
241 if (resolve_subpage && section->mr->subpage) {
242 subpage = container_of(section->mr, subpage_t, iomem);
243 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
244 }
245 return section;
246 }
247
248 static MemoryRegionSection *
249 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
250 hwaddr *plen, bool resolve_subpage)
251 {
252 MemoryRegionSection *section;
253 Int128 diff;
254
255 section = address_space_lookup_region(d, addr, resolve_subpage);
256 /* Compute offset within MemoryRegionSection */
257 addr -= section->offset_within_address_space;
258
259 /* Compute offset within MemoryRegion */
260 *xlat = addr + section->offset_within_region;
261
262 diff = int128_sub(section->mr->size, int128_make64(addr));
263 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
264 return section;
265 }
266
267 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
268 hwaddr *xlat, hwaddr *plen,
269 bool is_write)
270 {
271 IOMMUTLBEntry iotlb;
272 MemoryRegionSection *section;
273 MemoryRegion *mr;
274 hwaddr len = *plen;
275
276 for (;;) {
277 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
278 mr = section->mr;
279
280 if (!mr->iommu_ops) {
281 break;
282 }
283
284 iotlb = mr->iommu_ops->translate(mr, addr);
285 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
286 | (addr & iotlb.addr_mask));
287 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
288 if (!(iotlb.perm & (1 << is_write))) {
289 mr = &io_mem_unassigned;
290 break;
291 }
292
293 as = iotlb.target_as;
294 }
295
296 *plen = len;
297 *xlat = addr;
298 return mr;
299 }
300
301 MemoryRegionSection *
302 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
303 hwaddr *plen)
304 {
305 MemoryRegionSection *section;
306 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
307
308 assert(!section->mr->iommu_ops);
309 return section;
310 }
311 #endif
312
313 void cpu_exec_init_all(void)
314 {
315 #if !defined(CONFIG_USER_ONLY)
316 qemu_mutex_init(&ram_list.mutex);
317 memory_map_init();
318 io_mem_init();
319 #endif
320 }
321
322 #if !defined(CONFIG_USER_ONLY)
323
324 static int cpu_common_post_load(void *opaque, int version_id)
325 {
326 CPUState *cpu = opaque;
327
328 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
329 version_id is increased. */
330 cpu->interrupt_request &= ~0x01;
331 tlb_flush(cpu->env_ptr, 1);
332
333 return 0;
334 }
335
336 const VMStateDescription vmstate_cpu_common = {
337 .name = "cpu_common",
338 .version_id = 1,
339 .minimum_version_id = 1,
340 .minimum_version_id_old = 1,
341 .post_load = cpu_common_post_load,
342 .fields = (VMStateField []) {
343 VMSTATE_UINT32(halted, CPUState),
344 VMSTATE_UINT32(interrupt_request, CPUState),
345 VMSTATE_END_OF_LIST()
346 }
347 };
348
349 #endif
350
351 CPUState *qemu_get_cpu(int index)
352 {
353 CPUState *cpu;
354
355 CPU_FOREACH(cpu) {
356 if (cpu->cpu_index == index) {
357 return cpu;
358 }
359 }
360
361 return NULL;
362 }
363
364 void cpu_exec_init(CPUArchState *env)
365 {
366 CPUState *cpu = ENV_GET_CPU(env);
367 CPUClass *cc = CPU_GET_CLASS(cpu);
368 CPUState *some_cpu;
369 int cpu_index;
370
371 #if defined(CONFIG_USER_ONLY)
372 cpu_list_lock();
373 #endif
374 cpu_index = 0;
375 CPU_FOREACH(some_cpu) {
376 cpu_index++;
377 }
378 cpu->cpu_index = cpu_index;
379 cpu->numa_node = 0;
380 QTAILQ_INIT(&env->breakpoints);
381 QTAILQ_INIT(&env->watchpoints);
382 #ifndef CONFIG_USER_ONLY
383 cpu->thread_id = qemu_get_thread_id();
384 #endif
385 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
386 #if defined(CONFIG_USER_ONLY)
387 cpu_list_unlock();
388 #endif
389 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
390 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
391 }
392 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
393 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
394 cpu_save, cpu_load, env);
395 assert(cc->vmsd == NULL);
396 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
397 #endif
398 if (cc->vmsd != NULL) {
399 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
400 }
401 }
402
403 #if defined(TARGET_HAS_ICE)
404 #if defined(CONFIG_USER_ONLY)
405 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
406 {
407 tb_invalidate_phys_page_range(pc, pc + 1, 0);
408 }
409 #else
410 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
411 {
412 tb_invalidate_phys_addr(cpu_get_phys_page_debug(cpu, pc) |
413 (pc & ~TARGET_PAGE_MASK));
414 }
415 #endif
416 #endif /* TARGET_HAS_ICE */
417
418 #if defined(CONFIG_USER_ONLY)
419 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
420
421 {
422 }
423
424 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
425 int flags, CPUWatchpoint **watchpoint)
426 {
427 return -ENOSYS;
428 }
429 #else
430 /* Add a watchpoint. */
431 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
432 int flags, CPUWatchpoint **watchpoint)
433 {
434 target_ulong len_mask = ~(len - 1);
435 CPUWatchpoint *wp;
436
437 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
438 if ((len & (len - 1)) || (addr & ~len_mask) ||
439 len == 0 || len > TARGET_PAGE_SIZE) {
440 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
441 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
442 return -EINVAL;
443 }
444 wp = g_malloc(sizeof(*wp));
445
446 wp->vaddr = addr;
447 wp->len_mask = len_mask;
448 wp->flags = flags;
449
450 /* keep all GDB-injected watchpoints in front */
451 if (flags & BP_GDB)
452 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
453 else
454 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
455
456 tlb_flush_page(env, addr);
457
458 if (watchpoint)
459 *watchpoint = wp;
460 return 0;
461 }
462
463 /* Remove a specific watchpoint. */
464 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
465 int flags)
466 {
467 target_ulong len_mask = ~(len - 1);
468 CPUWatchpoint *wp;
469
470 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
471 if (addr == wp->vaddr && len_mask == wp->len_mask
472 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
473 cpu_watchpoint_remove_by_ref(env, wp);
474 return 0;
475 }
476 }
477 return -ENOENT;
478 }
479
480 /* Remove a specific watchpoint by reference. */
481 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
482 {
483 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
484
485 tlb_flush_page(env, watchpoint->vaddr);
486
487 g_free(watchpoint);
488 }
489
490 /* Remove all matching watchpoints. */
491 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
492 {
493 CPUWatchpoint *wp, *next;
494
495 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
496 if (wp->flags & mask)
497 cpu_watchpoint_remove_by_ref(env, wp);
498 }
499 }
500 #endif
501
502 /* Add a breakpoint. */
503 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
504 CPUBreakpoint **breakpoint)
505 {
506 #if defined(TARGET_HAS_ICE)
507 CPUBreakpoint *bp;
508
509 bp = g_malloc(sizeof(*bp));
510
511 bp->pc = pc;
512 bp->flags = flags;
513
514 /* keep all GDB-injected breakpoints in front */
515 if (flags & BP_GDB) {
516 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
517 } else {
518 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
519 }
520
521 breakpoint_invalidate(ENV_GET_CPU(env), pc);
522
523 if (breakpoint) {
524 *breakpoint = bp;
525 }
526 return 0;
527 #else
528 return -ENOSYS;
529 #endif
530 }
531
532 /* Remove a specific breakpoint. */
533 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
534 {
535 #if defined(TARGET_HAS_ICE)
536 CPUBreakpoint *bp;
537
538 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
539 if (bp->pc == pc && bp->flags == flags) {
540 cpu_breakpoint_remove_by_ref(env, bp);
541 return 0;
542 }
543 }
544 return -ENOENT;
545 #else
546 return -ENOSYS;
547 #endif
548 }
549
550 /* Remove a specific breakpoint by reference. */
551 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
552 {
553 #if defined(TARGET_HAS_ICE)
554 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
555
556 breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
557
558 g_free(breakpoint);
559 #endif
560 }
561
562 /* Remove all matching breakpoints. */
563 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
564 {
565 #if defined(TARGET_HAS_ICE)
566 CPUBreakpoint *bp, *next;
567
568 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
569 if (bp->flags & mask)
570 cpu_breakpoint_remove_by_ref(env, bp);
571 }
572 #endif
573 }
574
575 /* enable or disable single step mode. EXCP_DEBUG is returned by the
576 CPU loop after each instruction */
577 void cpu_single_step(CPUState *cpu, int enabled)
578 {
579 #if defined(TARGET_HAS_ICE)
580 if (cpu->singlestep_enabled != enabled) {
581 cpu->singlestep_enabled = enabled;
582 if (kvm_enabled()) {
583 kvm_update_guest_debug(cpu, 0);
584 } else {
585 /* must flush all the translated code to avoid inconsistencies */
586 /* XXX: only flush what is necessary */
587 CPUArchState *env = cpu->env_ptr;
588 tb_flush(env);
589 }
590 }
591 #endif
592 }
593
594 void cpu_abort(CPUArchState *env, const char *fmt, ...)
595 {
596 CPUState *cpu = ENV_GET_CPU(env);
597 va_list ap;
598 va_list ap2;
599
600 va_start(ap, fmt);
601 va_copy(ap2, ap);
602 fprintf(stderr, "qemu: fatal: ");
603 vfprintf(stderr, fmt, ap);
604 fprintf(stderr, "\n");
605 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
606 if (qemu_log_enabled()) {
607 qemu_log("qemu: fatal: ");
608 qemu_log_vprintf(fmt, ap2);
609 qemu_log("\n");
610 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
611 qemu_log_flush();
612 qemu_log_close();
613 }
614 va_end(ap2);
615 va_end(ap);
616 #if defined(CONFIG_USER_ONLY)
617 {
618 struct sigaction act;
619 sigfillset(&act.sa_mask);
620 act.sa_handler = SIG_DFL;
621 sigaction(SIGABRT, &act, NULL);
622 }
623 #endif
624 abort();
625 }
626
627 #if !defined(CONFIG_USER_ONLY)
628 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
629 {
630 RAMBlock *block;
631
632 /* The list is protected by the iothread lock here. */
633 block = ram_list.mru_block;
634 if (block && addr - block->offset < block->length) {
635 goto found;
636 }
637 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
638 if (addr - block->offset < block->length) {
639 goto found;
640 }
641 }
642
643 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
644 abort();
645
646 found:
647 ram_list.mru_block = block;
648 return block;
649 }
650
651 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
652 uintptr_t length)
653 {
654 RAMBlock *block;
655 ram_addr_t start1;
656
657 block = qemu_get_ram_block(start);
658 assert(block == qemu_get_ram_block(end - 1));
659 start1 = (uintptr_t)block->host + (start - block->offset);
660 cpu_tlb_reset_dirty_all(start1, length);
661 }
662
663 /* Note: start and end must be within the same ram block. */
664 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
665 int dirty_flags)
666 {
667 uintptr_t length;
668
669 start &= TARGET_PAGE_MASK;
670 end = TARGET_PAGE_ALIGN(end);
671
672 length = end - start;
673 if (length == 0)
674 return;
675 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
676
677 if (tcg_enabled()) {
678 tlb_reset_dirty_range_all(start, end, length);
679 }
680 }
681
682 static int cpu_physical_memory_set_dirty_tracking(int enable)
683 {
684 int ret = 0;
685 in_migration = enable;
686 return ret;
687 }
688
689 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
690 MemoryRegionSection *section,
691 target_ulong vaddr,
692 hwaddr paddr, hwaddr xlat,
693 int prot,
694 target_ulong *address)
695 {
696 hwaddr iotlb;
697 CPUWatchpoint *wp;
698
699 if (memory_region_is_ram(section->mr)) {
700 /* Normal RAM. */
701 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
702 + xlat;
703 if (!section->readonly) {
704 iotlb |= PHYS_SECTION_NOTDIRTY;
705 } else {
706 iotlb |= PHYS_SECTION_ROM;
707 }
708 } else {
709 iotlb = section - address_space_memory.dispatch->sections;
710 iotlb += xlat;
711 }
712
713 /* Make accesses to pages with watchpoints go via the
714 watchpoint trap routines. */
715 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
716 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
717 /* Avoid trapping reads of pages with a write breakpoint. */
718 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
719 iotlb = PHYS_SECTION_WATCH + paddr;
720 *address |= TLB_MMIO;
721 break;
722 }
723 }
724 }
725
726 return iotlb;
727 }
728 #endif /* defined(CONFIG_USER_ONLY) */
729
730 #if !defined(CONFIG_USER_ONLY)
731
732 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
733 uint16_t section);
734 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
735
736 static void *(*phys_mem_alloc)(size_t size) = qemu_anon_ram_alloc;
737
738 /*
739 * Set a custom physical guest memory alloator.
740 * Accelerators with unusual needs may need this. Hopefully, we can
741 * get rid of it eventually.
742 */
743 void phys_mem_set_alloc(void *(*alloc)(size_t))
744 {
745 phys_mem_alloc = alloc;
746 }
747
748 static uint16_t phys_section_add(MemoryRegionSection *section)
749 {
750 /* The physical section number is ORed with a page-aligned
751 * pointer to produce the iotlb entries. Thus it should
752 * never overflow into the page-aligned value.
753 */
754 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
755
756 if (next_map.sections_nb == next_map.sections_nb_alloc) {
757 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
758 16);
759 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
760 next_map.sections_nb_alloc);
761 }
762 next_map.sections[next_map.sections_nb] = *section;
763 memory_region_ref(section->mr);
764 return next_map.sections_nb++;
765 }
766
767 static void phys_section_destroy(MemoryRegion *mr)
768 {
769 memory_region_unref(mr);
770
771 if (mr->subpage) {
772 subpage_t *subpage = container_of(mr, subpage_t, iomem);
773 memory_region_destroy(&subpage->iomem);
774 g_free(subpage);
775 }
776 }
777
778 static void phys_sections_free(PhysPageMap *map)
779 {
780 while (map->sections_nb > 0) {
781 MemoryRegionSection *section = &map->sections[--map->sections_nb];
782 phys_section_destroy(section->mr);
783 }
784 g_free(map->sections);
785 g_free(map->nodes);
786 g_free(map);
787 }
788
789 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
790 {
791 subpage_t *subpage;
792 hwaddr base = section->offset_within_address_space
793 & TARGET_PAGE_MASK;
794 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
795 next_map.nodes, next_map.sections);
796 MemoryRegionSection subsection = {
797 .offset_within_address_space = base,
798 .size = int128_make64(TARGET_PAGE_SIZE),
799 };
800 hwaddr start, end;
801
802 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
803
804 if (!(existing->mr->subpage)) {
805 subpage = subpage_init(d->as, base);
806 subsection.mr = &subpage->iomem;
807 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
808 phys_section_add(&subsection));
809 } else {
810 subpage = container_of(existing->mr, subpage_t, iomem);
811 }
812 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
813 end = start + int128_get64(section->size) - 1;
814 subpage_register(subpage, start, end, phys_section_add(section));
815 }
816
817
818 static void register_multipage(AddressSpaceDispatch *d,
819 MemoryRegionSection *section)
820 {
821 hwaddr start_addr = section->offset_within_address_space;
822 uint16_t section_index = phys_section_add(section);
823 uint64_t num_pages = int128_get64(int128_rshift(section->size,
824 TARGET_PAGE_BITS));
825
826 assert(num_pages);
827 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
828 }
829
830 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
831 {
832 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
833 AddressSpaceDispatch *d = as->next_dispatch;
834 MemoryRegionSection now = *section, remain = *section;
835 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
836
837 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
838 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
839 - now.offset_within_address_space;
840
841 now.size = int128_min(int128_make64(left), now.size);
842 register_subpage(d, &now);
843 } else {
844 now.size = int128_zero();
845 }
846 while (int128_ne(remain.size, now.size)) {
847 remain.size = int128_sub(remain.size, now.size);
848 remain.offset_within_address_space += int128_get64(now.size);
849 remain.offset_within_region += int128_get64(now.size);
850 now = remain;
851 if (int128_lt(remain.size, page_size)) {
852 register_subpage(d, &now);
853 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
854 now.size = page_size;
855 register_subpage(d, &now);
856 } else {
857 now.size = int128_and(now.size, int128_neg(page_size));
858 register_multipage(d, &now);
859 }
860 }
861 }
862
863 void qemu_flush_coalesced_mmio_buffer(void)
864 {
865 if (kvm_enabled())
866 kvm_flush_coalesced_mmio_buffer();
867 }
868
869 void qemu_mutex_lock_ramlist(void)
870 {
871 qemu_mutex_lock(&ram_list.mutex);
872 }
873
874 void qemu_mutex_unlock_ramlist(void)
875 {
876 qemu_mutex_unlock(&ram_list.mutex);
877 }
878
879 #ifdef __linux__
880
881 #include <sys/vfs.h>
882
883 #define HUGETLBFS_MAGIC 0x958458f6
884
885 static long gethugepagesize(const char *path)
886 {
887 struct statfs fs;
888 int ret;
889
890 do {
891 ret = statfs(path, &fs);
892 } while (ret != 0 && errno == EINTR);
893
894 if (ret != 0) {
895 perror(path);
896 return 0;
897 }
898
899 if (fs.f_type != HUGETLBFS_MAGIC)
900 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
901
902 return fs.f_bsize;
903 }
904
905 static void *file_ram_alloc(RAMBlock *block,
906 ram_addr_t memory,
907 const char *path)
908 {
909 char *filename;
910 char *sanitized_name;
911 char *c;
912 void *area;
913 int fd;
914 #ifdef MAP_POPULATE
915 int flags;
916 #endif
917 unsigned long hpagesize;
918
919 hpagesize = gethugepagesize(path);
920 if (!hpagesize) {
921 return NULL;
922 }
923
924 if (memory < hpagesize) {
925 return NULL;
926 }
927
928 if (kvm_enabled() && !kvm_has_sync_mmu()) {
929 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
930 return NULL;
931 }
932
933 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
934 sanitized_name = g_strdup(block->mr->name);
935 for (c = sanitized_name; *c != '\0'; c++) {
936 if (*c == '/')
937 *c = '_';
938 }
939
940 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
941 sanitized_name);
942 g_free(sanitized_name);
943
944 fd = mkstemp(filename);
945 if (fd < 0) {
946 perror("unable to create backing store for hugepages");
947 g_free(filename);
948 return NULL;
949 }
950 unlink(filename);
951 g_free(filename);
952
953 memory = (memory+hpagesize-1) & ~(hpagesize-1);
954
955 /*
956 * ftruncate is not supported by hugetlbfs in older
957 * hosts, so don't bother bailing out on errors.
958 * If anything goes wrong with it under other filesystems,
959 * mmap will fail.
960 */
961 if (ftruncate(fd, memory))
962 perror("ftruncate");
963
964 #ifdef MAP_POPULATE
965 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
966 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
967 * to sidestep this quirk.
968 */
969 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
970 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
971 #else
972 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
973 #endif
974 if (area == MAP_FAILED) {
975 perror("file_ram_alloc: can't mmap RAM pages");
976 close(fd);
977 return (NULL);
978 }
979 block->fd = fd;
980 return area;
981 }
982 #else
983 static void *file_ram_alloc(RAMBlock *block,
984 ram_addr_t memory,
985 const char *path)
986 {
987 fprintf(stderr, "-mem-path not supported on this host\n");
988 exit(1);
989 }
990 #endif
991
992 static ram_addr_t find_ram_offset(ram_addr_t size)
993 {
994 RAMBlock *block, *next_block;
995 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
996
997 assert(size != 0); /* it would hand out same offset multiple times */
998
999 if (QTAILQ_EMPTY(&ram_list.blocks))
1000 return 0;
1001
1002 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1003 ram_addr_t end, next = RAM_ADDR_MAX;
1004
1005 end = block->offset + block->length;
1006
1007 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1008 if (next_block->offset >= end) {
1009 next = MIN(next, next_block->offset);
1010 }
1011 }
1012 if (next - end >= size && next - end < mingap) {
1013 offset = end;
1014 mingap = next - end;
1015 }
1016 }
1017
1018 if (offset == RAM_ADDR_MAX) {
1019 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1020 (uint64_t)size);
1021 abort();
1022 }
1023
1024 return offset;
1025 }
1026
1027 ram_addr_t last_ram_offset(void)
1028 {
1029 RAMBlock *block;
1030 ram_addr_t last = 0;
1031
1032 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1033 last = MAX(last, block->offset + block->length);
1034
1035 return last;
1036 }
1037
1038 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1039 {
1040 int ret;
1041
1042 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1043 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1044 "dump-guest-core", true)) {
1045 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1046 if (ret) {
1047 perror("qemu_madvise");
1048 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1049 "but dump_guest_core=off specified\n");
1050 }
1051 }
1052 }
1053
1054 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1055 {
1056 RAMBlock *new_block, *block;
1057
1058 new_block = NULL;
1059 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1060 if (block->offset == addr) {
1061 new_block = block;
1062 break;
1063 }
1064 }
1065 assert(new_block);
1066 assert(!new_block->idstr[0]);
1067
1068 if (dev) {
1069 char *id = qdev_get_dev_path(dev);
1070 if (id) {
1071 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1072 g_free(id);
1073 }
1074 }
1075 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1076
1077 /* This assumes the iothread lock is taken here too. */
1078 qemu_mutex_lock_ramlist();
1079 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1080 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1081 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1082 new_block->idstr);
1083 abort();
1084 }
1085 }
1086 qemu_mutex_unlock_ramlist();
1087 }
1088
1089 static int memory_try_enable_merging(void *addr, size_t len)
1090 {
1091 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1092 /* disabled by the user */
1093 return 0;
1094 }
1095
1096 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1097 }
1098
1099 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1100 MemoryRegion *mr)
1101 {
1102 RAMBlock *block, *new_block;
1103
1104 size = TARGET_PAGE_ALIGN(size);
1105 new_block = g_malloc0(sizeof(*new_block));
1106 new_block->fd = -1;
1107
1108 /* This assumes the iothread lock is taken here too. */
1109 qemu_mutex_lock_ramlist();
1110 new_block->mr = mr;
1111 new_block->offset = find_ram_offset(size);
1112 if (host) {
1113 new_block->host = host;
1114 new_block->flags |= RAM_PREALLOC_MASK;
1115 } else if (xen_enabled()) {
1116 if (mem_path) {
1117 fprintf(stderr, "-mem-path not supported with Xen\n");
1118 exit(1);
1119 }
1120 xen_ram_alloc(new_block->offset, size, mr);
1121 } else {
1122 if (mem_path) {
1123 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1124 /*
1125 * file_ram_alloc() needs to allocate just like
1126 * phys_mem_alloc, but we haven't bothered to provide
1127 * a hook there.
1128 */
1129 fprintf(stderr,
1130 "-mem-path not supported with this accelerator\n");
1131 exit(1);
1132 }
1133 new_block->host = file_ram_alloc(new_block, size, mem_path);
1134 }
1135 if (!new_block->host) {
1136 new_block->host = phys_mem_alloc(size);
1137 if (!new_block->host) {
1138 fprintf(stderr, "Cannot set up guest memory '%s': %s\n",
1139 new_block->mr->name, strerror(errno));
1140 exit(1);
1141 }
1142 memory_try_enable_merging(new_block->host, size);
1143 }
1144 }
1145 new_block->length = size;
1146
1147 /* Keep the list sorted from biggest to smallest block. */
1148 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1149 if (block->length < new_block->length) {
1150 break;
1151 }
1152 }
1153 if (block) {
1154 QTAILQ_INSERT_BEFORE(block, new_block, next);
1155 } else {
1156 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1157 }
1158 ram_list.mru_block = NULL;
1159
1160 ram_list.version++;
1161 qemu_mutex_unlock_ramlist();
1162
1163 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1164 last_ram_offset() >> TARGET_PAGE_BITS);
1165 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1166 0, size >> TARGET_PAGE_BITS);
1167 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1168
1169 qemu_ram_setup_dump(new_block->host, size);
1170 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1171 qemu_madvise(new_block->host, size, QEMU_MADV_DONTFORK);
1172
1173 if (kvm_enabled())
1174 kvm_setup_guest_memory(new_block->host, size);
1175
1176 return new_block->offset;
1177 }
1178
1179 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1180 {
1181 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1182 }
1183
1184 void qemu_ram_free_from_ptr(ram_addr_t addr)
1185 {
1186 RAMBlock *block;
1187
1188 /* This assumes the iothread lock is taken here too. */
1189 qemu_mutex_lock_ramlist();
1190 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1191 if (addr == block->offset) {
1192 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1193 ram_list.mru_block = NULL;
1194 ram_list.version++;
1195 g_free(block);
1196 break;
1197 }
1198 }
1199 qemu_mutex_unlock_ramlist();
1200 }
1201
1202 void qemu_ram_free(ram_addr_t addr)
1203 {
1204 RAMBlock *block;
1205
1206 /* This assumes the iothread lock is taken here too. */
1207 qemu_mutex_lock_ramlist();
1208 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1209 if (addr == block->offset) {
1210 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1211 ram_list.mru_block = NULL;
1212 ram_list.version++;
1213 if (block->flags & RAM_PREALLOC_MASK) {
1214 ;
1215 } else if (xen_enabled()) {
1216 xen_invalidate_map_cache_entry(block->host);
1217 #ifndef _WIN32
1218 } else if (block->fd >= 0) {
1219 munmap(block->host, block->length);
1220 close(block->fd);
1221 #endif
1222 } else {
1223 qemu_anon_ram_free(block->host, block->length);
1224 }
1225 g_free(block);
1226 break;
1227 }
1228 }
1229 qemu_mutex_unlock_ramlist();
1230
1231 }
1232
1233 #ifndef _WIN32
1234 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1235 {
1236 RAMBlock *block;
1237 ram_addr_t offset;
1238 int flags;
1239 void *area, *vaddr;
1240
1241 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1242 offset = addr - block->offset;
1243 if (offset < block->length) {
1244 vaddr = block->host + offset;
1245 if (block->flags & RAM_PREALLOC_MASK) {
1246 ;
1247 } else if (xen_enabled()) {
1248 abort();
1249 } else {
1250 flags = MAP_FIXED;
1251 munmap(vaddr, length);
1252 if (block->fd >= 0) {
1253 #ifdef MAP_POPULATE
1254 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1255 MAP_PRIVATE;
1256 #else
1257 flags |= MAP_PRIVATE;
1258 #endif
1259 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1260 flags, block->fd, offset);
1261 } else {
1262 /*
1263 * Remap needs to match alloc. Accelerators that
1264 * set phys_mem_alloc never remap. If they did,
1265 * we'd need a remap hook here.
1266 */
1267 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1268
1269 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1270 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1271 flags, -1, 0);
1272 }
1273 if (area != vaddr) {
1274 fprintf(stderr, "Could not remap addr: "
1275 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1276 length, addr);
1277 exit(1);
1278 }
1279 memory_try_enable_merging(vaddr, length);
1280 qemu_ram_setup_dump(vaddr, length);
1281 }
1282 return;
1283 }
1284 }
1285 }
1286 #endif /* !_WIN32 */
1287
1288 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1289 With the exception of the softmmu code in this file, this should
1290 only be used for local memory (e.g. video ram) that the device owns,
1291 and knows it isn't going to access beyond the end of the block.
1292
1293 It should not be used for general purpose DMA.
1294 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1295 */
1296 void *qemu_get_ram_ptr(ram_addr_t addr)
1297 {
1298 RAMBlock *block = qemu_get_ram_block(addr);
1299
1300 if (xen_enabled()) {
1301 /* We need to check if the requested address is in the RAM
1302 * because we don't want to map the entire memory in QEMU.
1303 * In that case just map until the end of the page.
1304 */
1305 if (block->offset == 0) {
1306 return xen_map_cache(addr, 0, 0);
1307 } else if (block->host == NULL) {
1308 block->host =
1309 xen_map_cache(block->offset, block->length, 1);
1310 }
1311 }
1312 return block->host + (addr - block->offset);
1313 }
1314
1315 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1316 * but takes a size argument */
1317 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1318 {
1319 if (*size == 0) {
1320 return NULL;
1321 }
1322 if (xen_enabled()) {
1323 return xen_map_cache(addr, *size, 1);
1324 } else {
1325 RAMBlock *block;
1326
1327 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1328 if (addr - block->offset < block->length) {
1329 if (addr - block->offset + *size > block->length)
1330 *size = block->length - addr + block->offset;
1331 return block->host + (addr - block->offset);
1332 }
1333 }
1334
1335 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1336 abort();
1337 }
1338 }
1339
1340 /* Some of the softmmu routines need to translate from a host pointer
1341 (typically a TLB entry) back to a ram offset. */
1342 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1343 {
1344 RAMBlock *block;
1345 uint8_t *host = ptr;
1346
1347 if (xen_enabled()) {
1348 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1349 return qemu_get_ram_block(*ram_addr)->mr;
1350 }
1351
1352 block = ram_list.mru_block;
1353 if (block && block->host && host - block->host < block->length) {
1354 goto found;
1355 }
1356
1357 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1358 /* This case append when the block is not mapped. */
1359 if (block->host == NULL) {
1360 continue;
1361 }
1362 if (host - block->host < block->length) {
1363 goto found;
1364 }
1365 }
1366
1367 return NULL;
1368
1369 found:
1370 *ram_addr = block->offset + (host - block->host);
1371 return block->mr;
1372 }
1373
1374 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1375 uint64_t val, unsigned size)
1376 {
1377 int dirty_flags;
1378 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1379 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1380 tb_invalidate_phys_page_fast(ram_addr, size);
1381 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1382 }
1383 switch (size) {
1384 case 1:
1385 stb_p(qemu_get_ram_ptr(ram_addr), val);
1386 break;
1387 case 2:
1388 stw_p(qemu_get_ram_ptr(ram_addr), val);
1389 break;
1390 case 4:
1391 stl_p(qemu_get_ram_ptr(ram_addr), val);
1392 break;
1393 default:
1394 abort();
1395 }
1396 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1397 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1398 /* we remove the notdirty callback only if the code has been
1399 flushed */
1400 if (dirty_flags == 0xff) {
1401 CPUArchState *env = current_cpu->env_ptr;
1402 tlb_set_dirty(env, env->mem_io_vaddr);
1403 }
1404 }
1405
1406 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1407 unsigned size, bool is_write)
1408 {
1409 return is_write;
1410 }
1411
1412 static const MemoryRegionOps notdirty_mem_ops = {
1413 .write = notdirty_mem_write,
1414 .valid.accepts = notdirty_mem_accepts,
1415 .endianness = DEVICE_NATIVE_ENDIAN,
1416 };
1417
1418 /* Generate a debug exception if a watchpoint has been hit. */
1419 static void check_watchpoint(int offset, int len_mask, int flags)
1420 {
1421 CPUArchState *env = current_cpu->env_ptr;
1422 target_ulong pc, cs_base;
1423 target_ulong vaddr;
1424 CPUWatchpoint *wp;
1425 int cpu_flags;
1426
1427 if (env->watchpoint_hit) {
1428 /* We re-entered the check after replacing the TB. Now raise
1429 * the debug interrupt so that is will trigger after the
1430 * current instruction. */
1431 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1432 return;
1433 }
1434 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1435 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1436 if ((vaddr == (wp->vaddr & len_mask) ||
1437 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1438 wp->flags |= BP_WATCHPOINT_HIT;
1439 if (!env->watchpoint_hit) {
1440 env->watchpoint_hit = wp;
1441 tb_check_watchpoint(env);
1442 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1443 env->exception_index = EXCP_DEBUG;
1444 cpu_loop_exit(env);
1445 } else {
1446 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1447 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1448 cpu_resume_from_signal(env, NULL);
1449 }
1450 }
1451 } else {
1452 wp->flags &= ~BP_WATCHPOINT_HIT;
1453 }
1454 }
1455 }
1456
1457 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1458 so these check for a hit then pass through to the normal out-of-line
1459 phys routines. */
1460 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1461 unsigned size)
1462 {
1463 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1464 switch (size) {
1465 case 1: return ldub_phys(addr);
1466 case 2: return lduw_phys(addr);
1467 case 4: return ldl_phys(addr);
1468 default: abort();
1469 }
1470 }
1471
1472 static void watch_mem_write(void *opaque, hwaddr addr,
1473 uint64_t val, unsigned size)
1474 {
1475 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1476 switch (size) {
1477 case 1:
1478 stb_phys(addr, val);
1479 break;
1480 case 2:
1481 stw_phys(addr, val);
1482 break;
1483 case 4:
1484 stl_phys(addr, val);
1485 break;
1486 default: abort();
1487 }
1488 }
1489
1490 static const MemoryRegionOps watch_mem_ops = {
1491 .read = watch_mem_read,
1492 .write = watch_mem_write,
1493 .endianness = DEVICE_NATIVE_ENDIAN,
1494 };
1495
1496 static uint64_t subpage_read(void *opaque, hwaddr addr,
1497 unsigned len)
1498 {
1499 subpage_t *subpage = opaque;
1500 uint8_t buf[4];
1501
1502 #if defined(DEBUG_SUBPAGE)
1503 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
1504 subpage, len, addr);
1505 #endif
1506 address_space_read(subpage->as, addr + subpage->base, buf, len);
1507 switch (len) {
1508 case 1:
1509 return ldub_p(buf);
1510 case 2:
1511 return lduw_p(buf);
1512 case 4:
1513 return ldl_p(buf);
1514 default:
1515 abort();
1516 }
1517 }
1518
1519 static void subpage_write(void *opaque, hwaddr addr,
1520 uint64_t value, unsigned len)
1521 {
1522 subpage_t *subpage = opaque;
1523 uint8_t buf[4];
1524
1525 #if defined(DEBUG_SUBPAGE)
1526 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
1527 " value %"PRIx64"\n",
1528 __func__, subpage, len, addr, value);
1529 #endif
1530 switch (len) {
1531 case 1:
1532 stb_p(buf, value);
1533 break;
1534 case 2:
1535 stw_p(buf, value);
1536 break;
1537 case 4:
1538 stl_p(buf, value);
1539 break;
1540 default:
1541 abort();
1542 }
1543 address_space_write(subpage->as, addr + subpage->base, buf, len);
1544 }
1545
1546 static bool subpage_accepts(void *opaque, hwaddr addr,
1547 unsigned len, bool is_write)
1548 {
1549 subpage_t *subpage = opaque;
1550 #if defined(DEBUG_SUBPAGE)
1551 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
1552 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1553 #endif
1554
1555 return address_space_access_valid(subpage->as, addr + subpage->base,
1556 len, is_write);
1557 }
1558
1559 static const MemoryRegionOps subpage_ops = {
1560 .read = subpage_read,
1561 .write = subpage_write,
1562 .valid.accepts = subpage_accepts,
1563 .endianness = DEVICE_NATIVE_ENDIAN,
1564 };
1565
1566 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1567 uint16_t section)
1568 {
1569 int idx, eidx;
1570
1571 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1572 return -1;
1573 idx = SUBPAGE_IDX(start);
1574 eidx = SUBPAGE_IDX(end);
1575 #if defined(DEBUG_SUBPAGE)
1576 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
1577 __func__, mmio, start, end, idx, eidx, section);
1578 #endif
1579 for (; idx <= eidx; idx++) {
1580 mmio->sub_section[idx] = section;
1581 }
1582
1583 return 0;
1584 }
1585
1586 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1587 {
1588 subpage_t *mmio;
1589
1590 mmio = g_malloc0(sizeof(subpage_t));
1591
1592 mmio->as = as;
1593 mmio->base = base;
1594 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1595 "subpage", TARGET_PAGE_SIZE);
1596 mmio->iomem.subpage = true;
1597 #if defined(DEBUG_SUBPAGE)
1598 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
1599 mmio, base, TARGET_PAGE_SIZE);
1600 #endif
1601 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1602
1603 return mmio;
1604 }
1605
1606 static uint16_t dummy_section(MemoryRegion *mr)
1607 {
1608 MemoryRegionSection section = {
1609 .mr = mr,
1610 .offset_within_address_space = 0,
1611 .offset_within_region = 0,
1612 .size = int128_2_64(),
1613 };
1614
1615 return phys_section_add(&section);
1616 }
1617
1618 MemoryRegion *iotlb_to_region(hwaddr index)
1619 {
1620 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1621 }
1622
1623 static void io_mem_init(void)
1624 {
1625 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1626 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1627 "unassigned", UINT64_MAX);
1628 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1629 "notdirty", UINT64_MAX);
1630 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1631 "watch", UINT64_MAX);
1632 }
1633
1634 static void mem_begin(MemoryListener *listener)
1635 {
1636 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1637 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1638
1639 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1640 d->as = as;
1641 as->next_dispatch = d;
1642 }
1643
1644 static void mem_commit(MemoryListener *listener)
1645 {
1646 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1647 AddressSpaceDispatch *cur = as->dispatch;
1648 AddressSpaceDispatch *next = as->next_dispatch;
1649
1650 next->nodes = next_map.nodes;
1651 next->sections = next_map.sections;
1652
1653 as->dispatch = next;
1654 g_free(cur);
1655 }
1656
1657 static void core_begin(MemoryListener *listener)
1658 {
1659 uint16_t n;
1660
1661 prev_map = g_new(PhysPageMap, 1);
1662 *prev_map = next_map;
1663
1664 memset(&next_map, 0, sizeof(next_map));
1665 n = dummy_section(&io_mem_unassigned);
1666 assert(n == PHYS_SECTION_UNASSIGNED);
1667 n = dummy_section(&io_mem_notdirty);
1668 assert(n == PHYS_SECTION_NOTDIRTY);
1669 n = dummy_section(&io_mem_rom);
1670 assert(n == PHYS_SECTION_ROM);
1671 n = dummy_section(&io_mem_watch);
1672 assert(n == PHYS_SECTION_WATCH);
1673 }
1674
1675 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1676 * All AddressSpaceDispatch instances have switched to the next map.
1677 */
1678 static void core_commit(MemoryListener *listener)
1679 {
1680 phys_sections_free(prev_map);
1681 }
1682
1683 static void tcg_commit(MemoryListener *listener)
1684 {
1685 CPUState *cpu;
1686
1687 /* since each CPU stores ram addresses in its TLB cache, we must
1688 reset the modified entries */
1689 /* XXX: slow ! */
1690 CPU_FOREACH(cpu) {
1691 CPUArchState *env = cpu->env_ptr;
1692
1693 tlb_flush(env, 1);
1694 }
1695 }
1696
1697 static void core_log_global_start(MemoryListener *listener)
1698 {
1699 cpu_physical_memory_set_dirty_tracking(1);
1700 }
1701
1702 static void core_log_global_stop(MemoryListener *listener)
1703 {
1704 cpu_physical_memory_set_dirty_tracking(0);
1705 }
1706
1707 static MemoryListener core_memory_listener = {
1708 .begin = core_begin,
1709 .commit = core_commit,
1710 .log_global_start = core_log_global_start,
1711 .log_global_stop = core_log_global_stop,
1712 .priority = 1,
1713 };
1714
1715 static MemoryListener tcg_memory_listener = {
1716 .commit = tcg_commit,
1717 };
1718
1719 void address_space_init_dispatch(AddressSpace *as)
1720 {
1721 as->dispatch = NULL;
1722 as->dispatch_listener = (MemoryListener) {
1723 .begin = mem_begin,
1724 .commit = mem_commit,
1725 .region_add = mem_add,
1726 .region_nop = mem_add,
1727 .priority = 0,
1728 };
1729 memory_listener_register(&as->dispatch_listener, as);
1730 }
1731
1732 void address_space_destroy_dispatch(AddressSpace *as)
1733 {
1734 AddressSpaceDispatch *d = as->dispatch;
1735
1736 memory_listener_unregister(&as->dispatch_listener);
1737 g_free(d);
1738 as->dispatch = NULL;
1739 }
1740
1741 static void memory_map_init(void)
1742 {
1743 system_memory = g_malloc(sizeof(*system_memory));
1744 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1745 address_space_init(&address_space_memory, system_memory, "memory");
1746
1747 system_io = g_malloc(sizeof(*system_io));
1748 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
1749 65536);
1750 address_space_init(&address_space_io, system_io, "I/O");
1751
1752 memory_listener_register(&core_memory_listener, &address_space_memory);
1753 if (tcg_enabled()) {
1754 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1755 }
1756 }
1757
1758 MemoryRegion *get_system_memory(void)
1759 {
1760 return system_memory;
1761 }
1762
1763 MemoryRegion *get_system_io(void)
1764 {
1765 return system_io;
1766 }
1767
1768 #endif /* !defined(CONFIG_USER_ONLY) */
1769
1770 /* physical memory access (slow version, mainly for debug) */
1771 #if defined(CONFIG_USER_ONLY)
1772 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1773 uint8_t *buf, int len, int is_write)
1774 {
1775 int l, flags;
1776 target_ulong page;
1777 void * p;
1778
1779 while (len > 0) {
1780 page = addr & TARGET_PAGE_MASK;
1781 l = (page + TARGET_PAGE_SIZE) - addr;
1782 if (l > len)
1783 l = len;
1784 flags = page_get_flags(page);
1785 if (!(flags & PAGE_VALID))
1786 return -1;
1787 if (is_write) {
1788 if (!(flags & PAGE_WRITE))
1789 return -1;
1790 /* XXX: this code should not depend on lock_user */
1791 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1792 return -1;
1793 memcpy(p, buf, l);
1794 unlock_user(p, addr, l);
1795 } else {
1796 if (!(flags & PAGE_READ))
1797 return -1;
1798 /* XXX: this code should not depend on lock_user */
1799 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1800 return -1;
1801 memcpy(buf, p, l);
1802 unlock_user(p, addr, 0);
1803 }
1804 len -= l;
1805 buf += l;
1806 addr += l;
1807 }
1808 return 0;
1809 }
1810
1811 #else
1812
1813 static void invalidate_and_set_dirty(hwaddr addr,
1814 hwaddr length)
1815 {
1816 if (!cpu_physical_memory_is_dirty(addr)) {
1817 /* invalidate code */
1818 tb_invalidate_phys_page_range(addr, addr + length, 0);
1819 /* set dirty bit */
1820 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1821 }
1822 xen_modified_memory(addr, length);
1823 }
1824
1825 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1826 {
1827 if (memory_region_is_ram(mr)) {
1828 return !(is_write && mr->readonly);
1829 }
1830 if (memory_region_is_romd(mr)) {
1831 return !is_write;
1832 }
1833
1834 return false;
1835 }
1836
1837 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1838 {
1839 unsigned access_size_max = mr->ops->valid.max_access_size;
1840
1841 /* Regions are assumed to support 1-4 byte accesses unless
1842 otherwise specified. */
1843 if (access_size_max == 0) {
1844 access_size_max = 4;
1845 }
1846
1847 /* Bound the maximum access by the alignment of the address. */
1848 if (!mr->ops->impl.unaligned) {
1849 unsigned align_size_max = addr & -addr;
1850 if (align_size_max != 0 && align_size_max < access_size_max) {
1851 access_size_max = align_size_max;
1852 }
1853 }
1854
1855 /* Don't attempt accesses larger than the maximum. */
1856 if (l > access_size_max) {
1857 l = access_size_max;
1858 }
1859 if (l & (l - 1)) {
1860 l = 1 << (qemu_fls(l) - 1);
1861 }
1862
1863 return l;
1864 }
1865
1866 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1867 int len, bool is_write)
1868 {
1869 hwaddr l;
1870 uint8_t *ptr;
1871 uint64_t val;
1872 hwaddr addr1;
1873 MemoryRegion *mr;
1874 bool error = false;
1875
1876 while (len > 0) {
1877 l = len;
1878 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1879
1880 if (is_write) {
1881 if (!memory_access_is_direct(mr, is_write)) {
1882 l = memory_access_size(mr, l, addr1);
1883 /* XXX: could force current_cpu to NULL to avoid
1884 potential bugs */
1885 switch (l) {
1886 case 8:
1887 /* 64 bit write access */
1888 val = ldq_p(buf);
1889 error |= io_mem_write(mr, addr1, val, 8);
1890 break;
1891 case 4:
1892 /* 32 bit write access */
1893 val = ldl_p(buf);
1894 error |= io_mem_write(mr, addr1, val, 4);
1895 break;
1896 case 2:
1897 /* 16 bit write access */
1898 val = lduw_p(buf);
1899 error |= io_mem_write(mr, addr1, val, 2);
1900 break;
1901 case 1:
1902 /* 8 bit write access */
1903 val = ldub_p(buf);
1904 error |= io_mem_write(mr, addr1, val, 1);
1905 break;
1906 default:
1907 abort();
1908 }
1909 } else {
1910 addr1 += memory_region_get_ram_addr(mr);
1911 /* RAM case */
1912 ptr = qemu_get_ram_ptr(addr1);
1913 memcpy(ptr, buf, l);
1914 invalidate_and_set_dirty(addr1, l);
1915 }
1916 } else {
1917 if (!memory_access_is_direct(mr, is_write)) {
1918 /* I/O case */
1919 l = memory_access_size(mr, l, addr1);
1920 switch (l) {
1921 case 8:
1922 /* 64 bit read access */
1923 error |= io_mem_read(mr, addr1, &val, 8);
1924 stq_p(buf, val);
1925 break;
1926 case 4:
1927 /* 32 bit read access */
1928 error |= io_mem_read(mr, addr1, &val, 4);
1929 stl_p(buf, val);
1930 break;
1931 case 2:
1932 /* 16 bit read access */
1933 error |= io_mem_read(mr, addr1, &val, 2);
1934 stw_p(buf, val);
1935 break;
1936 case 1:
1937 /* 8 bit read access */
1938 error |= io_mem_read(mr, addr1, &val, 1);
1939 stb_p(buf, val);
1940 break;
1941 default:
1942 abort();
1943 }
1944 } else {
1945 /* RAM case */
1946 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1947 memcpy(buf, ptr, l);
1948 }
1949 }
1950 len -= l;
1951 buf += l;
1952 addr += l;
1953 }
1954
1955 return error;
1956 }
1957
1958 bool address_space_write(AddressSpace *as, hwaddr addr,
1959 const uint8_t *buf, int len)
1960 {
1961 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
1962 }
1963
1964 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1965 {
1966 return address_space_rw(as, addr, buf, len, false);
1967 }
1968
1969
1970 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1971 int len, int is_write)
1972 {
1973 address_space_rw(&address_space_memory, addr, buf, len, is_write);
1974 }
1975
1976 /* used for ROM loading : can write in RAM and ROM */
1977 void cpu_physical_memory_write_rom(hwaddr addr,
1978 const uint8_t *buf, int len)
1979 {
1980 hwaddr l;
1981 uint8_t *ptr;
1982 hwaddr addr1;
1983 MemoryRegion *mr;
1984
1985 while (len > 0) {
1986 l = len;
1987 mr = address_space_translate(&address_space_memory,
1988 addr, &addr1, &l, true);
1989
1990 if (!(memory_region_is_ram(mr) ||
1991 memory_region_is_romd(mr))) {
1992 /* do nothing */
1993 } else {
1994 addr1 += memory_region_get_ram_addr(mr);
1995 /* ROM/RAM case */
1996 ptr = qemu_get_ram_ptr(addr1);
1997 memcpy(ptr, buf, l);
1998 invalidate_and_set_dirty(addr1, l);
1999 }
2000 len -= l;
2001 buf += l;
2002 addr += l;
2003 }
2004 }
2005
2006 typedef struct {
2007 MemoryRegion *mr;
2008 void *buffer;
2009 hwaddr addr;
2010 hwaddr len;
2011 } BounceBuffer;
2012
2013 static BounceBuffer bounce;
2014
2015 typedef struct MapClient {
2016 void *opaque;
2017 void (*callback)(void *opaque);
2018 QLIST_ENTRY(MapClient) link;
2019 } MapClient;
2020
2021 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2022 = QLIST_HEAD_INITIALIZER(map_client_list);
2023
2024 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2025 {
2026 MapClient *client = g_malloc(sizeof(*client));
2027
2028 client->opaque = opaque;
2029 client->callback = callback;
2030 QLIST_INSERT_HEAD(&map_client_list, client, link);
2031 return client;
2032 }
2033
2034 static void cpu_unregister_map_client(void *_client)
2035 {
2036 MapClient *client = (MapClient *)_client;
2037
2038 QLIST_REMOVE(client, link);
2039 g_free(client);
2040 }
2041
2042 static void cpu_notify_map_clients(void)
2043 {
2044 MapClient *client;
2045
2046 while (!QLIST_EMPTY(&map_client_list)) {
2047 client = QLIST_FIRST(&map_client_list);
2048 client->callback(client->opaque);
2049 cpu_unregister_map_client(client);
2050 }
2051 }
2052
2053 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2054 {
2055 MemoryRegion *mr;
2056 hwaddr l, xlat;
2057
2058 while (len > 0) {
2059 l = len;
2060 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2061 if (!memory_access_is_direct(mr, is_write)) {
2062 l = memory_access_size(mr, l, addr);
2063 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2064 return false;
2065 }
2066 }
2067
2068 len -= l;
2069 addr += l;
2070 }
2071 return true;
2072 }
2073
2074 /* Map a physical memory region into a host virtual address.
2075 * May map a subset of the requested range, given by and returned in *plen.
2076 * May return NULL if resources needed to perform the mapping are exhausted.
2077 * Use only for reads OR writes - not for read-modify-write operations.
2078 * Use cpu_register_map_client() to know when retrying the map operation is
2079 * likely to succeed.
2080 */
2081 void *address_space_map(AddressSpace *as,
2082 hwaddr addr,
2083 hwaddr *plen,
2084 bool is_write)
2085 {
2086 hwaddr len = *plen;
2087 hwaddr done = 0;
2088 hwaddr l, xlat, base;
2089 MemoryRegion *mr, *this_mr;
2090 ram_addr_t raddr;
2091
2092 if (len == 0) {
2093 return NULL;
2094 }
2095
2096 l = len;
2097 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2098 if (!memory_access_is_direct(mr, is_write)) {
2099 if (bounce.buffer) {
2100 return NULL;
2101 }
2102 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2103 bounce.addr = addr;
2104 bounce.len = l;
2105
2106 memory_region_ref(mr);
2107 bounce.mr = mr;
2108 if (!is_write) {
2109 address_space_read(as, addr, bounce.buffer, l);
2110 }
2111
2112 *plen = l;
2113 return bounce.buffer;
2114 }
2115
2116 base = xlat;
2117 raddr = memory_region_get_ram_addr(mr);
2118
2119 for (;;) {
2120 len -= l;
2121 addr += l;
2122 done += l;
2123 if (len == 0) {
2124 break;
2125 }
2126
2127 l = len;
2128 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2129 if (this_mr != mr || xlat != base + done) {
2130 break;
2131 }
2132 }
2133
2134 memory_region_ref(mr);
2135 *plen = done;
2136 return qemu_ram_ptr_length(raddr + base, plen);
2137 }
2138
2139 /* Unmaps a memory region previously mapped by address_space_map().
2140 * Will also mark the memory as dirty if is_write == 1. access_len gives
2141 * the amount of memory that was actually read or written by the caller.
2142 */
2143 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2144 int is_write, hwaddr access_len)
2145 {
2146 if (buffer != bounce.buffer) {
2147 MemoryRegion *mr;
2148 ram_addr_t addr1;
2149
2150 mr = qemu_ram_addr_from_host(buffer, &addr1);
2151 assert(mr != NULL);
2152 if (is_write) {
2153 while (access_len) {
2154 unsigned l;
2155 l = TARGET_PAGE_SIZE;
2156 if (l > access_len)
2157 l = access_len;
2158 invalidate_and_set_dirty(addr1, l);
2159 addr1 += l;
2160 access_len -= l;
2161 }
2162 }
2163 if (xen_enabled()) {
2164 xen_invalidate_map_cache_entry(buffer);
2165 }
2166 memory_region_unref(mr);
2167 return;
2168 }
2169 if (is_write) {
2170 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2171 }
2172 qemu_vfree(bounce.buffer);
2173 bounce.buffer = NULL;
2174 memory_region_unref(bounce.mr);
2175 cpu_notify_map_clients();
2176 }
2177
2178 void *cpu_physical_memory_map(hwaddr addr,
2179 hwaddr *plen,
2180 int is_write)
2181 {
2182 return address_space_map(&address_space_memory, addr, plen, is_write);
2183 }
2184
2185 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2186 int is_write, hwaddr access_len)
2187 {
2188 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2189 }
2190
2191 /* warning: addr must be aligned */
2192 static inline uint32_t ldl_phys_internal(hwaddr addr,
2193 enum device_endian endian)
2194 {
2195 uint8_t *ptr;
2196 uint64_t val;
2197 MemoryRegion *mr;
2198 hwaddr l = 4;
2199 hwaddr addr1;
2200
2201 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2202 false);
2203 if (l < 4 || !memory_access_is_direct(mr, false)) {
2204 /* I/O case */
2205 io_mem_read(mr, addr1, &val, 4);
2206 #if defined(TARGET_WORDS_BIGENDIAN)
2207 if (endian == DEVICE_LITTLE_ENDIAN) {
2208 val = bswap32(val);
2209 }
2210 #else
2211 if (endian == DEVICE_BIG_ENDIAN) {
2212 val = bswap32(val);
2213 }
2214 #endif
2215 } else {
2216 /* RAM case */
2217 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2218 & TARGET_PAGE_MASK)
2219 + addr1);
2220 switch (endian) {
2221 case DEVICE_LITTLE_ENDIAN:
2222 val = ldl_le_p(ptr);
2223 break;
2224 case DEVICE_BIG_ENDIAN:
2225 val = ldl_be_p(ptr);
2226 break;
2227 default:
2228 val = ldl_p(ptr);
2229 break;
2230 }
2231 }
2232 return val;
2233 }
2234
2235 uint32_t ldl_phys(hwaddr addr)
2236 {
2237 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2238 }
2239
2240 uint32_t ldl_le_phys(hwaddr addr)
2241 {
2242 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2243 }
2244
2245 uint32_t ldl_be_phys(hwaddr addr)
2246 {
2247 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2248 }
2249
2250 /* warning: addr must be aligned */
2251 static inline uint64_t ldq_phys_internal(hwaddr addr,
2252 enum device_endian endian)
2253 {
2254 uint8_t *ptr;
2255 uint64_t val;
2256 MemoryRegion *mr;
2257 hwaddr l = 8;
2258 hwaddr addr1;
2259
2260 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2261 false);
2262 if (l < 8 || !memory_access_is_direct(mr, false)) {
2263 /* I/O case */
2264 io_mem_read(mr, addr1, &val, 8);
2265 #if defined(TARGET_WORDS_BIGENDIAN)
2266 if (endian == DEVICE_LITTLE_ENDIAN) {
2267 val = bswap64(val);
2268 }
2269 #else
2270 if (endian == DEVICE_BIG_ENDIAN) {
2271 val = bswap64(val);
2272 }
2273 #endif
2274 } else {
2275 /* RAM case */
2276 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2277 & TARGET_PAGE_MASK)
2278 + addr1);
2279 switch (endian) {
2280 case DEVICE_LITTLE_ENDIAN:
2281 val = ldq_le_p(ptr);
2282 break;
2283 case DEVICE_BIG_ENDIAN:
2284 val = ldq_be_p(ptr);
2285 break;
2286 default:
2287 val = ldq_p(ptr);
2288 break;
2289 }
2290 }
2291 return val;
2292 }
2293
2294 uint64_t ldq_phys(hwaddr addr)
2295 {
2296 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2297 }
2298
2299 uint64_t ldq_le_phys(hwaddr addr)
2300 {
2301 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2302 }
2303
2304 uint64_t ldq_be_phys(hwaddr addr)
2305 {
2306 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2307 }
2308
2309 /* XXX: optimize */
2310 uint32_t ldub_phys(hwaddr addr)
2311 {
2312 uint8_t val;
2313 cpu_physical_memory_read(addr, &val, 1);
2314 return val;
2315 }
2316
2317 /* warning: addr must be aligned */
2318 static inline uint32_t lduw_phys_internal(hwaddr addr,
2319 enum device_endian endian)
2320 {
2321 uint8_t *ptr;
2322 uint64_t val;
2323 MemoryRegion *mr;
2324 hwaddr l = 2;
2325 hwaddr addr1;
2326
2327 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2328 false);
2329 if (l < 2 || !memory_access_is_direct(mr, false)) {
2330 /* I/O case */
2331 io_mem_read(mr, addr1, &val, 2);
2332 #if defined(TARGET_WORDS_BIGENDIAN)
2333 if (endian == DEVICE_LITTLE_ENDIAN) {
2334 val = bswap16(val);
2335 }
2336 #else
2337 if (endian == DEVICE_BIG_ENDIAN) {
2338 val = bswap16(val);
2339 }
2340 #endif
2341 } else {
2342 /* RAM case */
2343 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2344 & TARGET_PAGE_MASK)
2345 + addr1);
2346 switch (endian) {
2347 case DEVICE_LITTLE_ENDIAN:
2348 val = lduw_le_p(ptr);
2349 break;
2350 case DEVICE_BIG_ENDIAN:
2351 val = lduw_be_p(ptr);
2352 break;
2353 default:
2354 val = lduw_p(ptr);
2355 break;
2356 }
2357 }
2358 return val;
2359 }
2360
2361 uint32_t lduw_phys(hwaddr addr)
2362 {
2363 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2364 }
2365
2366 uint32_t lduw_le_phys(hwaddr addr)
2367 {
2368 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2369 }
2370
2371 uint32_t lduw_be_phys(hwaddr addr)
2372 {
2373 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2374 }
2375
2376 /* warning: addr must be aligned. The ram page is not masked as dirty
2377 and the code inside is not invalidated. It is useful if the dirty
2378 bits are used to track modified PTEs */
2379 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2380 {
2381 uint8_t *ptr;
2382 MemoryRegion *mr;
2383 hwaddr l = 4;
2384 hwaddr addr1;
2385
2386 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2387 true);
2388 if (l < 4 || !memory_access_is_direct(mr, true)) {
2389 io_mem_write(mr, addr1, val, 4);
2390 } else {
2391 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2392 ptr = qemu_get_ram_ptr(addr1);
2393 stl_p(ptr, val);
2394
2395 if (unlikely(in_migration)) {
2396 if (!cpu_physical_memory_is_dirty(addr1)) {
2397 /* invalidate code */
2398 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2399 /* set dirty bit */
2400 cpu_physical_memory_set_dirty_flags(
2401 addr1, (0xff & ~CODE_DIRTY_FLAG));
2402 }
2403 }
2404 }
2405 }
2406
2407 /* warning: addr must be aligned */
2408 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2409 enum device_endian endian)
2410 {
2411 uint8_t *ptr;
2412 MemoryRegion *mr;
2413 hwaddr l = 4;
2414 hwaddr addr1;
2415
2416 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2417 true);
2418 if (l < 4 || !memory_access_is_direct(mr, true)) {
2419 #if defined(TARGET_WORDS_BIGENDIAN)
2420 if (endian == DEVICE_LITTLE_ENDIAN) {
2421 val = bswap32(val);
2422 }
2423 #else
2424 if (endian == DEVICE_BIG_ENDIAN) {
2425 val = bswap32(val);
2426 }
2427 #endif
2428 io_mem_write(mr, addr1, val, 4);
2429 } else {
2430 /* RAM case */
2431 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2432 ptr = qemu_get_ram_ptr(addr1);
2433 switch (endian) {
2434 case DEVICE_LITTLE_ENDIAN:
2435 stl_le_p(ptr, val);
2436 break;
2437 case DEVICE_BIG_ENDIAN:
2438 stl_be_p(ptr, val);
2439 break;
2440 default:
2441 stl_p(ptr, val);
2442 break;
2443 }
2444 invalidate_and_set_dirty(addr1, 4);
2445 }
2446 }
2447
2448 void stl_phys(hwaddr addr, uint32_t val)
2449 {
2450 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2451 }
2452
2453 void stl_le_phys(hwaddr addr, uint32_t val)
2454 {
2455 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2456 }
2457
2458 void stl_be_phys(hwaddr addr, uint32_t val)
2459 {
2460 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2461 }
2462
2463 /* XXX: optimize */
2464 void stb_phys(hwaddr addr, uint32_t val)
2465 {
2466 uint8_t v = val;
2467 cpu_physical_memory_write(addr, &v, 1);
2468 }
2469
2470 /* warning: addr must be aligned */
2471 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2472 enum device_endian endian)
2473 {
2474 uint8_t *ptr;
2475 MemoryRegion *mr;
2476 hwaddr l = 2;
2477 hwaddr addr1;
2478
2479 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2480 true);
2481 if (l < 2 || !memory_access_is_direct(mr, true)) {
2482 #if defined(TARGET_WORDS_BIGENDIAN)
2483 if (endian == DEVICE_LITTLE_ENDIAN) {
2484 val = bswap16(val);
2485 }
2486 #else
2487 if (endian == DEVICE_BIG_ENDIAN) {
2488 val = bswap16(val);
2489 }
2490 #endif
2491 io_mem_write(mr, addr1, val, 2);
2492 } else {
2493 /* RAM case */
2494 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2495 ptr = qemu_get_ram_ptr(addr1);
2496 switch (endian) {
2497 case DEVICE_LITTLE_ENDIAN:
2498 stw_le_p(ptr, val);
2499 break;
2500 case DEVICE_BIG_ENDIAN:
2501 stw_be_p(ptr, val);
2502 break;
2503 default:
2504 stw_p(ptr, val);
2505 break;
2506 }
2507 invalidate_and_set_dirty(addr1, 2);
2508 }
2509 }
2510
2511 void stw_phys(hwaddr addr, uint32_t val)
2512 {
2513 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2514 }
2515
2516 void stw_le_phys(hwaddr addr, uint32_t val)
2517 {
2518 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2519 }
2520
2521 void stw_be_phys(hwaddr addr, uint32_t val)
2522 {
2523 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2524 }
2525
2526 /* XXX: optimize */
2527 void stq_phys(hwaddr addr, uint64_t val)
2528 {
2529 val = tswap64(val);
2530 cpu_physical_memory_write(addr, &val, 8);
2531 }
2532
2533 void stq_le_phys(hwaddr addr, uint64_t val)
2534 {
2535 val = cpu_to_le64(val);
2536 cpu_physical_memory_write(addr, &val, 8);
2537 }
2538
2539 void stq_be_phys(hwaddr addr, uint64_t val)
2540 {
2541 val = cpu_to_be64(val);
2542 cpu_physical_memory_write(addr, &val, 8);
2543 }
2544
2545 /* virtual memory access for debug (includes writing to ROM) */
2546 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2547 uint8_t *buf, int len, int is_write)
2548 {
2549 int l;
2550 hwaddr phys_addr;
2551 target_ulong page;
2552
2553 while (len > 0) {
2554 page = addr & TARGET_PAGE_MASK;
2555 phys_addr = cpu_get_phys_page_debug(cpu, page);
2556 /* if no physical page mapped, return an error */
2557 if (phys_addr == -1)
2558 return -1;
2559 l = (page + TARGET_PAGE_SIZE) - addr;
2560 if (l > len)
2561 l = len;
2562 phys_addr += (addr & ~TARGET_PAGE_MASK);
2563 if (is_write)
2564 cpu_physical_memory_write_rom(phys_addr, buf, l);
2565 else
2566 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2567 len -= l;
2568 buf += l;
2569 addr += l;
2570 }
2571 return 0;
2572 }
2573 #endif
2574
2575 #if !defined(CONFIG_USER_ONLY)
2576
2577 /*
2578 * A helper function for the _utterly broken_ virtio device model to find out if
2579 * it's running on a big endian machine. Don't do this at home kids!
2580 */
2581 bool virtio_is_big_endian(void);
2582 bool virtio_is_big_endian(void)
2583 {
2584 #if defined(TARGET_WORDS_BIGENDIAN)
2585 return true;
2586 #else
2587 return false;
2588 #endif
2589 }
2590
2591 #endif
2592
2593 #ifndef CONFIG_USER_ONLY
2594 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2595 {
2596 MemoryRegion*mr;
2597 hwaddr l = 1;
2598
2599 mr = address_space_translate(&address_space_memory,
2600 phys_addr, &phys_addr, &l, false);
2601
2602 return !(memory_region_is_ram(mr) ||
2603 memory_region_is_romd(mr));
2604 }
2605
2606 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2607 {
2608 RAMBlock *block;
2609
2610 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2611 func(block->host, block->offset, block->length, opaque);
2612 }
2613 }
2614 #endif
Ray Wang's QEMU Repository