?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge remote-tracking branch 'mst/tags/for_anthony' into staging
[qemu/ar7.git] / exec.c
blob79610ce37a67b5b1b4dda30790c526a37c87dc00
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
51
52 #include "exec/memory-internal.h"
53
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 typedef PhysPageEntry Node[L2_SIZE];
92
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101 };
102
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
110
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
115
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
124
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
127
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132
133 static MemoryRegion io_mem_watch;
134 #endif
135
136 #if !defined(CONFIG_USER_ONLY)
137
138 static void phys_map_node_reserve(unsigned nodes)
139 {
140 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
141 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
142 16);
143 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
144 next_map.nodes_nb + nodes);
145 next_map.nodes = g_renew(Node, next_map.nodes,
146 next_map.nodes_nb_alloc);
147 }
148 }
149
150 static uint16_t phys_map_node_alloc(void)
151 {
152 unsigned i;
153 uint16_t ret;
154
155 ret = next_map.nodes_nb++;
156 assert(ret != PHYS_MAP_NODE_NIL);
157 assert(ret != next_map.nodes_nb_alloc);
158 for (i = 0; i < L2_SIZE; ++i) {
159 next_map.nodes[ret][i].is_leaf = 0;
160 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
161 }
162 return ret;
163 }
164
165 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
166 hwaddr *nb, uint16_t leaf,
167 int level)
168 {
169 PhysPageEntry *p;
170 int i;
171 hwaddr step = (hwaddr)1 << (level * L2_BITS);
172
173 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
174 lp->ptr = phys_map_node_alloc();
175 p = next_map.nodes[lp->ptr];
176 if (level == 0) {
177 for (i = 0; i < L2_SIZE; i++) {
178 p[i].is_leaf = 1;
179 p[i].ptr = PHYS_SECTION_UNASSIGNED;
180 }
181 }
182 } else {
183 p = next_map.nodes[lp->ptr];
184 }
185 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
186
187 while (*nb && lp < &p[L2_SIZE]) {
188 if ((*index & (step - 1)) == 0 && *nb >= step) {
189 lp->is_leaf = true;
190 lp->ptr = leaf;
191 *index += step;
192 *nb -= step;
193 } else {
194 phys_page_set_level(lp, index, nb, leaf, level - 1);
195 }
196 ++lp;
197 }
198 }
199
200 static void phys_page_set(AddressSpaceDispatch *d,
201 hwaddr index, hwaddr nb,
202 uint16_t leaf)
203 {
204 /* Wildly overreserve - it doesn't matter much. */
205 phys_map_node_reserve(3 * P_L2_LEVELS);
206
207 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
208 }
209
210 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
211 Node *nodes, MemoryRegionSection *sections)
212 {
213 PhysPageEntry *p;
214 int i;
215
216 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
217 if (lp.ptr == PHYS_MAP_NODE_NIL) {
218 return &sections[PHYS_SECTION_UNASSIGNED];
219 }
220 p = nodes[lp.ptr];
221 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
222 }
223 return &sections[lp.ptr];
224 }
225
226 bool memory_region_is_unassigned(MemoryRegion *mr)
227 {
228 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
229 && mr != &io_mem_watch;
230 }
231
232 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
233 hwaddr addr,
234 bool resolve_subpage)
235 {
236 MemoryRegionSection *section;
237 subpage_t *subpage;
238
239 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
240 d->nodes, d->sections);
241 if (resolve_subpage && section->mr->subpage) {
242 subpage = container_of(section->mr, subpage_t, iomem);
243 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
244 }
245 return section;
246 }
247
248 static MemoryRegionSection *
249 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
250 hwaddr *plen, bool resolve_subpage)
251 {
252 MemoryRegionSection *section;
253 Int128 diff;
254
255 section = address_space_lookup_region(d, addr, resolve_subpage);
256 /* Compute offset within MemoryRegionSection */
257 addr -= section->offset_within_address_space;
258
259 /* Compute offset within MemoryRegion */
260 *xlat = addr + section->offset_within_region;
261
262 diff = int128_sub(section->mr->size, int128_make64(addr));
263 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
264 return section;
265 }
266
267 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
268 hwaddr *xlat, hwaddr *plen,
269 bool is_write)
270 {
271 IOMMUTLBEntry iotlb;
272 MemoryRegionSection *section;
273 MemoryRegion *mr;
274 hwaddr len = *plen;
275
276 for (;;) {
277 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
278 mr = section->mr;
279
280 if (!mr->iommu_ops) {
281 break;
282 }
283
284 iotlb = mr->iommu_ops->translate(mr, addr);
285 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
286 | (addr & iotlb.addr_mask));
287 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
288 if (!(iotlb.perm & (1 << is_write))) {
289 mr = &io_mem_unassigned;
290 break;
291 }
292
293 as = iotlb.target_as;
294 }
295
296 *plen = len;
297 *xlat = addr;
298 return mr;
299 }
300
301 MemoryRegionSection *
302 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
303 hwaddr *plen)
304 {
305 MemoryRegionSection *section;
306 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
307
308 assert(!section->mr->iommu_ops);
309 return section;
310 }
311 #endif
312
313 void cpu_exec_init_all(void)
314 {
315 #if !defined(CONFIG_USER_ONLY)
316 qemu_mutex_init(&ram_list.mutex);
317 memory_map_init();
318 io_mem_init();
319 #endif
320 }
321
322 #if !defined(CONFIG_USER_ONLY)
323
324 static int cpu_common_post_load(void *opaque, int version_id)
325 {
326 CPUState *cpu = opaque;
327
328 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
329 version_id is increased. */
330 cpu->interrupt_request &= ~0x01;
331 tlb_flush(cpu->env_ptr, 1);
332
333 return 0;
334 }
335
336 const VMStateDescription vmstate_cpu_common = {
337 .name = "cpu_common",
338 .version_id = 1,
339 .minimum_version_id = 1,
340 .minimum_version_id_old = 1,
341 .post_load = cpu_common_post_load,
342 .fields = (VMStateField []) {
343 VMSTATE_UINT32(halted, CPUState),
344 VMSTATE_UINT32(interrupt_request, CPUState),
345 VMSTATE_END_OF_LIST()
346 }
347 };
348
349 #endif
350
351 CPUState *qemu_get_cpu(int index)
352 {
353 CPUState *cpu;
354
355 CPU_FOREACH(cpu) {
356 if (cpu->cpu_index == index) {
357 return cpu;
358 }
359 }
360
361 return NULL;
362 }
363
364 void cpu_exec_init(CPUArchState *env)
365 {
366 CPUState *cpu = ENV_GET_CPU(env);
367 CPUClass *cc = CPU_GET_CLASS(cpu);
368 CPUState *some_cpu;
369 int cpu_index;
370
371 #if defined(CONFIG_USER_ONLY)
372 cpu_list_lock();
373 #endif
374 cpu_index = 0;
375 CPU_FOREACH(some_cpu) {
376 cpu_index++;
377 }
378 cpu->cpu_index = cpu_index;
379 cpu->numa_node = 0;
380 QTAILQ_INIT(&env->breakpoints);
381 QTAILQ_INIT(&env->watchpoints);
382 #ifndef CONFIG_USER_ONLY
383 cpu->thread_id = qemu_get_thread_id();
384 #endif
385 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
386 #if defined(CONFIG_USER_ONLY)
387 cpu_list_unlock();
388 #endif
389 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
390 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
391 }
392 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
393 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
394 cpu_save, cpu_load, env);
395 assert(cc->vmsd == NULL);
396 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
397 #endif
398 if (cc->vmsd != NULL) {
399 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
400 }
401 }
402
403 #if defined(TARGET_HAS_ICE)
404 #if defined(CONFIG_USER_ONLY)
405 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
406 {
407 tb_invalidate_phys_page_range(pc, pc + 1, 0);
408 }
409 #else
410 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
411 {
412 tb_invalidate_phys_addr(cpu_get_phys_page_debug(cpu, pc) |
413 (pc & ~TARGET_PAGE_MASK));
414 }
415 #endif
416 #endif /* TARGET_HAS_ICE */
417
418 #if defined(CONFIG_USER_ONLY)
419 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
420
421 {
422 }
423
424 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
425 int flags, CPUWatchpoint **watchpoint)
426 {
427 return -ENOSYS;
428 }
429 #else
430 /* Add a watchpoint. */
431 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
432 int flags, CPUWatchpoint **watchpoint)
433 {
434 target_ulong len_mask = ~(len - 1);
435 CPUWatchpoint *wp;
436
437 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
438 if ((len & (len - 1)) || (addr & ~len_mask) ||
439 len == 0 || len > TARGET_PAGE_SIZE) {
440 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
441 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
442 return -EINVAL;
443 }
444 wp = g_malloc(sizeof(*wp));
445
446 wp->vaddr = addr;
447 wp->len_mask = len_mask;
448 wp->flags = flags;
449
450 /* keep all GDB-injected watchpoints in front */
451 if (flags & BP_GDB)
452 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
453 else
454 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
455
456 tlb_flush_page(env, addr);
457
458 if (watchpoint)
459 *watchpoint = wp;
460 return 0;
461 }
462
463 /* Remove a specific watchpoint. */
464 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
465 int flags)
466 {
467 target_ulong len_mask = ~(len - 1);
468 CPUWatchpoint *wp;
469
470 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
471 if (addr == wp->vaddr && len_mask == wp->len_mask
472 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
473 cpu_watchpoint_remove_by_ref(env, wp);
474 return 0;
475 }
476 }
477 return -ENOENT;
478 }
479
480 /* Remove a specific watchpoint by reference. */
481 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
482 {
483 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
484
485 tlb_flush_page(env, watchpoint->vaddr);
486
487 g_free(watchpoint);
488 }
489
490 /* Remove all matching watchpoints. */
491 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
492 {
493 CPUWatchpoint *wp, *next;
494
495 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
496 if (wp->flags & mask)
497 cpu_watchpoint_remove_by_ref(env, wp);
498 }
499 }
500 #endif
501
502 /* Add a breakpoint. */
503 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
504 CPUBreakpoint **breakpoint)
505 {
506 #if defined(TARGET_HAS_ICE)
507 CPUBreakpoint *bp;
508
509 bp = g_malloc(sizeof(*bp));
510
511 bp->pc = pc;
512 bp->flags = flags;
513
514 /* keep all GDB-injected breakpoints in front */
515 if (flags & BP_GDB) {
516 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
517 } else {
518 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
519 }
520
521 breakpoint_invalidate(ENV_GET_CPU(env), pc);
522
523 if (breakpoint) {
524 *breakpoint = bp;
525 }
526 return 0;
527 #else
528 return -ENOSYS;
529 #endif
530 }
531
532 /* Remove a specific breakpoint. */
533 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
534 {
535 #if defined(TARGET_HAS_ICE)
536 CPUBreakpoint *bp;
537
538 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
539 if (bp->pc == pc && bp->flags == flags) {
540 cpu_breakpoint_remove_by_ref(env, bp);
541 return 0;
542 }
543 }
544 return -ENOENT;
545 #else
546 return -ENOSYS;
547 #endif
548 }
549
550 /* Remove a specific breakpoint by reference. */
551 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
552 {
553 #if defined(TARGET_HAS_ICE)
554 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
555
556 breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
557
558 g_free(breakpoint);
559 #endif
560 }
561
562 /* Remove all matching breakpoints. */
563 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
564 {
565 #if defined(TARGET_HAS_ICE)
566 CPUBreakpoint *bp, *next;
567
568 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
569 if (bp->flags & mask)
570 cpu_breakpoint_remove_by_ref(env, bp);
571 }
572 #endif
573 }
574
575 /* enable or disable single step mode. EXCP_DEBUG is returned by the
576 CPU loop after each instruction */
577 void cpu_single_step(CPUState *cpu, int enabled)
578 {
579 #if defined(TARGET_HAS_ICE)
580 if (cpu->singlestep_enabled != enabled) {
581 cpu->singlestep_enabled = enabled;
582 if (kvm_enabled()) {
583 kvm_update_guest_debug(cpu, 0);
584 } else {
585 /* must flush all the translated code to avoid inconsistencies */
586 /* XXX: only flush what is necessary */
587 CPUArchState *env = cpu->env_ptr;
588 tb_flush(env);
589 }
590 }
591 #endif
592 }
593
594 void cpu_abort(CPUArchState *env, const char *fmt, ...)
595 {
596 CPUState *cpu = ENV_GET_CPU(env);
597 va_list ap;
598 va_list ap2;
599
600 va_start(ap, fmt);
601 va_copy(ap2, ap);
602 fprintf(stderr, "qemu: fatal: ");
603 vfprintf(stderr, fmt, ap);
604 fprintf(stderr, "\n");
605 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
606 if (qemu_log_enabled()) {
607 qemu_log("qemu: fatal: ");
608 qemu_log_vprintf(fmt, ap2);
609 qemu_log("\n");
610 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
611 qemu_log_flush();
612 qemu_log_close();
613 }
614 va_end(ap2);
615 va_end(ap);
616 #if defined(CONFIG_USER_ONLY)
617 {
618 struct sigaction act;
619 sigfillset(&act.sa_mask);
620 act.sa_handler = SIG_DFL;
621 sigaction(SIGABRT, &act, NULL);
622 }
623 #endif
624 abort();
625 }
626
627 #if !defined(CONFIG_USER_ONLY)
628 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
629 {
630 RAMBlock *block;
631
632 /* The list is protected by the iothread lock here. */
633 block = ram_list.mru_block;
634 if (block && addr - block->offset < block->length) {
635 goto found;
636 }
637 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
638 if (addr - block->offset < block->length) {
639 goto found;
640 }
641 }
642
643 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
644 abort();
645
646 found:
647 ram_list.mru_block = block;
648 return block;
649 }
650
651 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
652 uintptr_t length)
653 {
654 RAMBlock *block;
655 ram_addr_t start1;
656
657 block = qemu_get_ram_block(start);
658 assert(block == qemu_get_ram_block(end - 1));
659 start1 = (uintptr_t)block->host + (start - block->offset);
660 cpu_tlb_reset_dirty_all(start1, length);
661 }
662
663 /* Note: start and end must be within the same ram block. */
664 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
665 int dirty_flags)
666 {
667 uintptr_t length;
668
669 start &= TARGET_PAGE_MASK;
670 end = TARGET_PAGE_ALIGN(end);
671
672 length = end - start;
673 if (length == 0)
674 return;
675 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
676
677 if (tcg_enabled()) {
678 tlb_reset_dirty_range_all(start, end, length);
679 }
680 }
681
682 static int cpu_physical_memory_set_dirty_tracking(int enable)
683 {
684 int ret = 0;
685 in_migration = enable;
686 return ret;
687 }
688
689 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
690 MemoryRegionSection *section,
691 target_ulong vaddr,
692 hwaddr paddr, hwaddr xlat,
693 int prot,
694 target_ulong *address)
695 {
696 hwaddr iotlb;
697 CPUWatchpoint *wp;
698
699 if (memory_region_is_ram(section->mr)) {
700 /* Normal RAM. */
701 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
702 + xlat;
703 if (!section->readonly) {
704 iotlb |= PHYS_SECTION_NOTDIRTY;
705 } else {
706 iotlb |= PHYS_SECTION_ROM;
707 }
708 } else {
709 iotlb = section - address_space_memory.dispatch->sections;
710 iotlb += xlat;
711 }
712
713 /* Make accesses to pages with watchpoints go via the
714 watchpoint trap routines. */
715 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
716 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
717 /* Avoid trapping reads of pages with a write breakpoint. */
718 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
719 iotlb = PHYS_SECTION_WATCH + paddr;
720 *address |= TLB_MMIO;
721 break;
722 }
723 }
724 }
725
726 return iotlb;
727 }
728 #endif /* defined(CONFIG_USER_ONLY) */
729
730 #if !defined(CONFIG_USER_ONLY)
731
732 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
733 uint16_t section);
734 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
735
736 static void *(*phys_mem_alloc)(size_t size) = qemu_anon_ram_alloc;
737
738 /*
739 * Set a custom physical guest memory alloator.
740 * Accelerators with unusual needs may need this. Hopefully, we can
741 * get rid of it eventually.
742 */
743 void phys_mem_set_alloc(void *(*alloc)(size_t))
744 {
745 phys_mem_alloc = alloc;
746 }
747
748 static uint16_t phys_section_add(MemoryRegionSection *section)
749 {
750 /* The physical section number is ORed with a page-aligned
751 * pointer to produce the iotlb entries. Thus it should
752 * never overflow into the page-aligned value.
753 */
754 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
755
756 if (next_map.sections_nb == next_map.sections_nb_alloc) {
757 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
758 16);
759 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
760 next_map.sections_nb_alloc);
761 }
762 next_map.sections[next_map.sections_nb] = *section;
763 memory_region_ref(section->mr);
764 return next_map.sections_nb++;
765 }
766
767 static void phys_section_destroy(MemoryRegion *mr)
768 {
769 memory_region_unref(mr);
770
771 if (mr->subpage) {
772 subpage_t *subpage = container_of(mr, subpage_t, iomem);
773 memory_region_destroy(&subpage->iomem);
774 g_free(subpage);
775 }
776 }
777
778 static void phys_sections_free(PhysPageMap *map)
779 {
780 while (map->sections_nb > 0) {
781 MemoryRegionSection *section = &map->sections[--map->sections_nb];
782 phys_section_destroy(section->mr);
783 }
784 g_free(map->sections);
785 g_free(map->nodes);
786 g_free(map);
787 }
788
789 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
790 {
791 subpage_t *subpage;
792 hwaddr base = section->offset_within_address_space
793 & TARGET_PAGE_MASK;
794 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
795 next_map.nodes, next_map.sections);
796 MemoryRegionSection subsection = {
797 .offset_within_address_space = base,
798 .size = int128_make64(TARGET_PAGE_SIZE),
799 };
800 hwaddr start, end;
801
802 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
803
804 if (!(existing->mr->subpage)) {
805 subpage = subpage_init(d->as, base);
806 subsection.mr = &subpage->iomem;
807 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
808 phys_section_add(&subsection));
809 } else {
810 subpage = container_of(existing->mr, subpage_t, iomem);
811 }
812 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
813 end = start + int128_get64(section->size) - 1;
814 subpage_register(subpage, start, end, phys_section_add(section));
815 }
816
817
818 static void register_multipage(AddressSpaceDispatch *d,
819 MemoryRegionSection *section)
820 {
821 hwaddr start_addr = section->offset_within_address_space;
822 uint16_t section_index = phys_section_add(section);
823 uint64_t num_pages = int128_get64(int128_rshift(section->size,
824 TARGET_PAGE_BITS));
825
826 assert(num_pages);
827 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
828 }
829
830 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
831 {
832 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
833 AddressSpaceDispatch *d = as->next_dispatch;
834 MemoryRegionSection now = *section, remain = *section;
835 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
836
837 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
838 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
839 - now.offset_within_address_space;
840
841 now.size = int128_min(int128_make64(left), now.size);
842 register_subpage(d, &now);
843 } else {
844 now.size = int128_zero();
845 }
846 while (int128_ne(remain.size, now.size)) {
847 remain.size = int128_sub(remain.size, now.size);
848 remain.offset_within_address_space += int128_get64(now.size);
849 remain.offset_within_region += int128_get64(now.size);
850 now = remain;
851 if (int128_lt(remain.size, page_size)) {
852 register_subpage(d, &now);
853 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
854 now.size = page_size;
855 register_subpage(d, &now);
856 } else {
857 now.size = int128_and(now.size, int128_neg(page_size));
858 register_multipage(d, &now);
859 }
860 }
861 }
862
863 void qemu_flush_coalesced_mmio_buffer(void)
864 {
865 if (kvm_enabled())
866 kvm_flush_coalesced_mmio_buffer();
867 }
868
869 void qemu_mutex_lock_ramlist(void)
870 {
871 qemu_mutex_lock(&ram_list.mutex);
872 }
873
874 void qemu_mutex_unlock_ramlist(void)
875 {
876 qemu_mutex_unlock(&ram_list.mutex);
877 }
878
879 #ifdef __linux__
880
881 #include <sys/vfs.h>
882
883 #define HUGETLBFS_MAGIC 0x958458f6
884
885 static long gethugepagesize(const char *path)
886 {
887 struct statfs fs;
888 int ret;
889
890 do {
891 ret = statfs(path, &fs);
892 } while (ret != 0 && errno == EINTR);
893
894 if (ret != 0) {
895 perror(path);
896 return 0;
897 }
898
899 if (fs.f_type != HUGETLBFS_MAGIC)
900 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
901
902 return fs.f_bsize;
903 }
904
905 static void *file_ram_alloc(RAMBlock *block,
906 ram_addr_t memory,
907 const char *path)
908 {
909 char *filename;
910 char *sanitized_name;
911 char *c;
912 void *area;
913 int fd;
914 #ifdef MAP_POPULATE
915 int flags;
916 #endif
917 unsigned long hpagesize;
918
919 hpagesize = gethugepagesize(path);
920 if (!hpagesize) {
921 return NULL;
922 }
923
924 if (memory < hpagesize) {
925 return NULL;
926 }
927
928 if (kvm_enabled() && !kvm_has_sync_mmu()) {
929 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
930 return NULL;
931 }
932
933 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
934 sanitized_name = g_strdup(block->mr->name);
935 for (c = sanitized_name; *c != '\0'; c++) {
936 if (*c == '/')
937 *c = '_';
938 }
939
940 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
941 sanitized_name);
942 g_free(sanitized_name);
943
944 fd = mkstemp(filename);
945 if (fd < 0) {
946 perror("unable to create backing store for hugepages");
947 g_free(filename);
948 return NULL;
949 }
950 unlink(filename);
951 g_free(filename);
952
953 memory = (memory+hpagesize-1) & ~(hpagesize-1);
954
955 /*
956 * ftruncate is not supported by hugetlbfs in older
957 * hosts, so don't bother bailing out on errors.
958 * If anything goes wrong with it under other filesystems,
959 * mmap will fail.
960 */
961 if (ftruncate(fd, memory))
962 perror("ftruncate");
963
964 #ifdef MAP_POPULATE
965 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
966 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
967 * to sidestep this quirk.
968 */
969 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
970 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
971 #else
972 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
973 #endif
974 if (area == MAP_FAILED) {
975 perror("file_ram_alloc: can't mmap RAM pages");
976 close(fd);
977 return (NULL);
978 }
979 block->fd = fd;
980 return area;
981 }
982 #else
983 static void *file_ram_alloc(RAMBlock *block,
984 ram_addr_t memory,
985 const char *path)
986 {
987 fprintf(stderr, "-mem-path not supported on this host\n");
988 exit(1);
989 }
990 #endif
991
992 static ram_addr_t find_ram_offset(ram_addr_t size)
993 {
994 RAMBlock *block, *next_block;
995 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
996
997 assert(size != 0); /* it would hand out same offset multiple times */
998
999 if (QTAILQ_EMPTY(&ram_list.blocks))
1000 return 0;
1001
1002 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1003 ram_addr_t end, next = RAM_ADDR_MAX;
1004
1005 end = block->offset + block->length;
1006
1007 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1008 if (next_block->offset >= end) {
1009 next = MIN(next, next_block->offset);
1010 }
1011 }
1012 if (next - end >= size && next - end < mingap) {
1013 offset = end;
1014 mingap = next - end;
1015 }
1016 }
1017
1018 if (offset == RAM_ADDR_MAX) {
1019 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1020 (uint64_t)size);
1021 abort();
1022 }
1023
1024 return offset;
1025 }
1026
1027 ram_addr_t last_ram_offset(void)
1028 {
1029 RAMBlock *block;
1030 ram_addr_t last = 0;
1031
1032 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1033 last = MAX(last, block->offset + block->length);
1034
1035 return last;
1036 }
1037
1038 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1039 {
1040 int ret;
1041
1042 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1043 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1044 "dump-guest-core", true)) {
1045 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1046 if (ret) {
1047 perror("qemu_madvise");
1048 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1049 "but dump_guest_core=off specified\n");
1050 }
1051 }
1052 }
1053
1054 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1055 {
1056 RAMBlock *new_block, *block;
1057
1058 new_block = NULL;
1059 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1060 if (block->offset == addr) {
1061 new_block = block;
1062 break;
1063 }
1064 }
1065 assert(new_block);
1066 assert(!new_block->idstr[0]);
1067
1068 if (dev) {
1069 char *id = qdev_get_dev_path(dev);
1070 if (id) {
1071 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1072 g_free(id);
1073 }
1074 }
1075 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1076
1077 /* This assumes the iothread lock is taken here too. */
1078 qemu_mutex_lock_ramlist();
1079 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1080 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1081 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1082 new_block->idstr);
1083 abort();
1084 }
1085 }
1086 qemu_mutex_unlock_ramlist();
1087 }
1088
1089 static int memory_try_enable_merging(void *addr, size_t len)
1090 {
1091 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1092 /* disabled by the user */
1093 return 0;
1094 }
1095
1096 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1097 }
1098
1099 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1100 MemoryRegion *mr)
1101 {
1102 RAMBlock *block, *new_block;
1103
1104 size = TARGET_PAGE_ALIGN(size);
1105 new_block = g_malloc0(sizeof(*new_block));
1106 new_block->fd = -1;
1107
1108 /* This assumes the iothread lock is taken here too. */
1109 qemu_mutex_lock_ramlist();
1110 new_block->mr = mr;
1111 new_block->offset = find_ram_offset(size);
1112 if (host) {
1113 new_block->host = host;
1114 new_block->flags |= RAM_PREALLOC_MASK;
1115 } else if (xen_enabled()) {
1116 if (mem_path) {
1117 fprintf(stderr, "-mem-path not supported with Xen\n");
1118 exit(1);
1119 }
1120 xen_ram_alloc(new_block->offset, size, mr);
1121 } else {
1122 if (mem_path) {
1123 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1124 /*
1125 * file_ram_alloc() needs to allocate just like
1126 * phys_mem_alloc, but we haven't bothered to provide
1127 * a hook there.
1128 */
1129 fprintf(stderr,
1130 "-mem-path not supported with this accelerator\n");
1131 exit(1);
1132 }
1133 new_block->host = file_ram_alloc(new_block, size, mem_path);
1134 }
1135 if (!new_block->host) {
1136 new_block->host = phys_mem_alloc(size);
1137 if (!new_block->host) {
1138 fprintf(stderr, "Cannot set up guest memory '%s': %s\n",
1139 new_block->mr->name, strerror(errno));
1140 exit(1);
1141 }
1142 memory_try_enable_merging(new_block->host, size);
1143 }
1144 }
1145 new_block->length = size;
1146
1147 /* Keep the list sorted from biggest to smallest block. */
1148 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1149 if (block->length < new_block->length) {
1150 break;
1151 }
1152 }
1153 if (block) {
1154 QTAILQ_INSERT_BEFORE(block, new_block, next);
1155 } else {
1156 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1157 }
1158 ram_list.mru_block = NULL;
1159
1160 ram_list.version++;
1161 qemu_mutex_unlock_ramlist();
1162
1163 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1164 last_ram_offset() >> TARGET_PAGE_BITS);
1165 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1166 0, size >> TARGET_PAGE_BITS);
1167 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1168
1169 qemu_ram_setup_dump(new_block->host, size);
1170 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1171 qemu_madvise(new_block->host, size, QEMU_MADV_DONTFORK);
1172
1173 if (kvm_enabled())
1174 kvm_setup_guest_memory(new_block->host, size);
1175
1176 return new_block->offset;
1177 }
1178
1179 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1180 {
1181 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1182 }
1183
1184 void qemu_ram_free_from_ptr(ram_addr_t addr)
1185 {
1186 RAMBlock *block;
1187
1188 /* This assumes the iothread lock is taken here too. */
1189 qemu_mutex_lock_ramlist();
1190 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1191 if (addr == block->offset) {
1192 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1193 ram_list.mru_block = NULL;
1194 ram_list.version++;
1195 g_free(block);
1196 break;
1197 }
1198 }
1199 qemu_mutex_unlock_ramlist();
1200 }
1201
1202 void qemu_ram_free(ram_addr_t addr)
1203 {
1204 RAMBlock *block;
1205
1206 /* This assumes the iothread lock is taken here too. */
1207 qemu_mutex_lock_ramlist();
1208 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1209 if (addr == block->offset) {
1210 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1211 ram_list.mru_block = NULL;
1212 ram_list.version++;
1213 if (block->flags & RAM_PREALLOC_MASK) {
1214 ;
1215 } else if (xen_enabled()) {
1216 xen_invalidate_map_cache_entry(block->host);
1217 #ifndef _WIN32
1218 } else if (block->fd >= 0) {
1219 munmap(block->host, block->length);
1220 close(block->fd);
1221 #endif
1222 } else {
1223 qemu_anon_ram_free(block->host, block->length);
1224 }
1225 g_free(block);
1226 break;
1227 }
1228 }
1229 qemu_mutex_unlock_ramlist();
1230
1231 }
1232
1233 #ifndef _WIN32
1234 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1235 {
1236 RAMBlock *block;
1237 ram_addr_t offset;
1238 int flags;
1239 void *area, *vaddr;
1240
1241 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1242 offset = addr - block->offset;
1243 if (offset < block->length) {
1244 vaddr = block->host + offset;
1245 if (block->flags & RAM_PREALLOC_MASK) {
1246 ;
1247 } else if (xen_enabled()) {
1248 abort();
1249 } else {
1250 flags = MAP_FIXED;
1251 munmap(vaddr, length);
1252 if (block->fd >= 0) {
1253 #ifdef MAP_POPULATE
1254 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1255 MAP_PRIVATE;
1256 #else
1257 flags |= MAP_PRIVATE;
1258 #endif
1259 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1260 flags, block->fd, offset);
1261 } else {
1262 /*
1263 * Remap needs to match alloc. Accelerators that
1264 * set phys_mem_alloc never remap. If they did,
1265 * we'd need a remap hook here.
1266 */
1267 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1268
1269 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1270 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1271 flags, -1, 0);
1272 }
1273 if (area != vaddr) {
1274 fprintf(stderr, "Could not remap addr: "
1275 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1276 length, addr);
1277 exit(1);
1278 }
1279 memory_try_enable_merging(vaddr, length);
1280 qemu_ram_setup_dump(vaddr, length);
1281 }
1282 return;
1283 }
1284 }
1285 }
1286 #endif /* !_WIN32 */
1287
1288 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1289 With the exception of the softmmu code in this file, this should
1290 only be used for local memory (e.g. video ram) that the device owns,
1291 and knows it isn't going to access beyond the end of the block.
1292
1293 It should not be used for general purpose DMA.
1294 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1295 */
1296 void *qemu_get_ram_ptr(ram_addr_t addr)
1297 {
1298 RAMBlock *block = qemu_get_ram_block(addr);
1299
1300 if (xen_enabled()) {
1301 /* We need to check if the requested address is in the RAM
1302 * because we don't want to map the entire memory in QEMU.
1303 * In that case just map until the end of the page.
1304 */
1305 if (block->offset == 0) {
1306 return xen_map_cache(addr, 0, 0);
1307 } else if (block->host == NULL) {
1308 block->host =
1309 xen_map_cache(block->offset, block->length, 1);
1310 }
1311 }
1312 return block->host + (addr - block->offset);
1313 }
1314
1315 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1316 * but takes a size argument */
1317 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1318 {
1319 if (*size == 0) {
1320 return NULL;
1321 }
1322 if (xen_enabled()) {
1323 return xen_map_cache(addr, *size, 1);
1324 } else {
1325 RAMBlock *block;
1326
1327 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1328 if (addr - block->offset < block->length) {
1329 if (addr - block->offset + *size > block->length)
1330 *size = block->length - addr + block->offset;
1331 return block->host + (addr - block->offset);
1332 }
1333 }
1334
1335 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1336 abort();
1337 }
1338 }
1339
1340 /* Some of the softmmu routines need to translate from a host pointer
1341 (typically a TLB entry) back to a ram offset. */
1342 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1343 {
1344 RAMBlock *block;
1345 uint8_t *host = ptr;
1346
1347 if (xen_enabled()) {
1348 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1349 return qemu_get_ram_block(*ram_addr)->mr;
1350 }
1351
1352 block = ram_list.mru_block;
1353 if (block && block->host && host - block->host < block->length) {
1354 goto found;
1355 }
1356
1357 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1358 /* This case append when the block is not mapped. */
1359 if (block->host == NULL) {
1360 continue;
1361 }
1362 if (host - block->host < block->length) {
1363 goto found;
1364 }
1365 }
1366
1367 return NULL;
1368
1369 found:
1370 *ram_addr = block->offset + (host - block->host);
1371 return block->mr;
1372 }
1373
1374 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1375 uint64_t val, unsigned size)
1376 {
1377 int dirty_flags;
1378 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1379 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1380 tb_invalidate_phys_page_fast(ram_addr, size);
1381 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1382 }
1383 switch (size) {
1384 case 1:
1385 stb_p(qemu_get_ram_ptr(ram_addr), val);
1386 break;
1387 case 2:
1388 stw_p(qemu_get_ram_ptr(ram_addr), val);
1389 break;
1390 case 4:
1391 stl_p(qemu_get_ram_ptr(ram_addr), val);
1392 break;
1393 default:
1394 abort();
1395 }
1396 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1397 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1398 /* we remove the notdirty callback only if the code has been
1399 flushed */
1400 if (dirty_flags == 0xff) {
1401 CPUArchState *env = current_cpu->env_ptr;
1402 tlb_set_dirty(env, env->mem_io_vaddr);
1403 }
1404 }
1405
1406 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1407 unsigned size, bool is_write)
1408 {
1409 return is_write;
1410 }
1411
1412 static const MemoryRegionOps notdirty_mem_ops = {
1413 .write = notdirty_mem_write,
1414 .valid.accepts = notdirty_mem_accepts,
1415 .endianness = DEVICE_NATIVE_ENDIAN,
1416 };
1417
1418 /* Generate a debug exception if a watchpoint has been hit. */
1419 static void check_watchpoint(int offset, int len_mask, int flags)
1420 {
1421 CPUArchState *env = current_cpu->env_ptr;
1422 target_ulong pc, cs_base;
1423 target_ulong vaddr;
1424 CPUWatchpoint *wp;
1425 int cpu_flags;
1426
1427 if (env->watchpoint_hit) {
1428 /* We re-entered the check after replacing the TB. Now raise
1429 * the debug interrupt so that is will trigger after the
1430 * current instruction. */
1431 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1432 return;
1433 }
1434 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1435 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1436 if ((vaddr == (wp->vaddr & len_mask) ||
1437 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1438 wp->flags |= BP_WATCHPOINT_HIT;
1439 if (!env->watchpoint_hit) {
1440 env->watchpoint_hit = wp;
1441 tb_check_watchpoint(env);
1442 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1443 env->exception_index = EXCP_DEBUG;
1444 cpu_loop_exit(env);
1445 } else {
1446 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1447 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1448 cpu_resume_from_signal(env, NULL);
1449 }
1450 }
1451 } else {
1452 wp->flags &= ~BP_WATCHPOINT_HIT;
1453 }
1454 }
1455 }
1456
1457 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1458 so these check for a hit then pass through to the normal out-of-line
1459 phys routines. */
1460 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1461 unsigned size)
1462 {
1463 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1464 switch (size) {
1465 case 1: return ldub_phys(addr);
1466 case 2: return lduw_phys(addr);
1467 case 4: return ldl_phys(addr);
1468 default: abort();
1469 }
1470 }
1471
1472 static void watch_mem_write(void *opaque, hwaddr addr,
1473 uint64_t val, unsigned size)
1474 {
1475 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1476 switch (size) {
1477 case 1:
1478 stb_phys(addr, val);
1479 break;
1480 case 2:
1481 stw_phys(addr, val);
1482 break;
1483 case 4:
1484 stl_phys(addr, val);
1485 break;
1486 default: abort();
1487 }
1488 }
1489
1490 static const MemoryRegionOps watch_mem_ops = {
1491 .read = watch_mem_read,
1492 .write = watch_mem_write,
1493 .endianness = DEVICE_NATIVE_ENDIAN,
1494 };
1495
1496 static uint64_t subpage_read(void *opaque, hwaddr addr,
1497 unsigned len)
1498 {
1499 subpage_t *subpage = opaque;
1500 uint8_t buf[4];
1501
1502 #if defined(DEBUG_SUBPAGE)
1503 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
1504 subpage, len, addr);
1505 #endif
1506 address_space_read(subpage->as, addr + subpage->base, buf, len);
1507 switch (len) {
1508 case 1:
1509 return ldub_p(buf);
1510 case 2:
1511 return lduw_p(buf);
1512 case 4:
1513 return ldl_p(buf);
1514 default:
1515 abort();
1516 }
1517 }
1518
1519 static void subpage_write(void *opaque, hwaddr addr,
1520 uint64_t value, unsigned len)
1521 {
1522 subpage_t *subpage = opaque;
1523 uint8_t buf[4];
1524
1525 #if defined(DEBUG_SUBPAGE)
1526 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
1527 " value %"PRIx64"\n",
1528 __func__, subpage, len, addr, value);
1529 #endif
1530 switch (len) {
1531 case 1:
1532 stb_p(buf, value);
1533 break;
1534 case 2:
1535 stw_p(buf, value);
1536 break;
1537 case 4:
1538 stl_p(buf, value);
1539 break;
1540 default:
1541 abort();
1542 }
1543 address_space_write(subpage->as, addr + subpage->base, buf, len);
1544 }
1545
1546 static bool subpage_accepts(void *opaque, hwaddr addr,
1547 unsigned len, bool is_write)
1548 {
1549 subpage_t *subpage = opaque;
1550 #if defined(DEBUG_SUBPAGE)
1551 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
1552 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1553 #endif
1554
1555 return address_space_access_valid(subpage->as, addr + subpage->base,
1556 len, is_write);
1557 }
1558
1559 static const MemoryRegionOps subpage_ops = {
1560 .read = subpage_read,
1561 .write = subpage_write,
1562 .valid.accepts = subpage_accepts,
1563 .endianness = DEVICE_NATIVE_ENDIAN,
1564 };
1565
1566 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1567 uint16_t section)
1568 {
1569 int idx, eidx;
1570
1571 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1572 return -1;
1573 idx = SUBPAGE_IDX(start);
1574 eidx = SUBPAGE_IDX(end);
1575 #if defined(DEBUG_SUBPAGE)
1576 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
1577 __func__, mmio, start, end, idx, eidx, section);
1578 #endif
1579 for (; idx <= eidx; idx++) {
1580 mmio->sub_section[idx] = section;
1581 }
1582
1583 return 0;
1584 }
1585
1586 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1587 {
1588 subpage_t *mmio;
1589
1590 mmio = g_malloc0(sizeof(subpage_t));
1591
1592 mmio->as = as;
1593 mmio->base = base;
1594 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1595 "subpage", TARGET_PAGE_SIZE);
1596 mmio->iomem.subpage = true;
1597 #if defined(DEBUG_SUBPAGE)
1598 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
1599 mmio, base, TARGET_PAGE_SIZE);
1600 #endif
1601 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1602
1603 return mmio;
1604 }
1605
1606 static uint16_t dummy_section(MemoryRegion *mr)
1607 {
1608 MemoryRegionSection section = {
1609 .mr = mr,
1610 .offset_within_address_space = 0,
1611 .offset_within_region = 0,
1612 .size = int128_2_64(),
1613 };
1614
1615 return phys_section_add(&section);
1616 }
1617
1618 MemoryRegion *iotlb_to_region(hwaddr index)
1619 {
1620 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1621 }
1622
1623 static void io_mem_init(void)
1624 {
1625 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1626 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1627 "unassigned", UINT64_MAX);
1628 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1629 "notdirty", UINT64_MAX);
1630 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1631 "watch", UINT64_MAX);
1632 }
1633
1634 static void mem_begin(MemoryListener *listener)
1635 {
1636 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1637 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1638
1639 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1640 d->as = as;
1641 as->next_dispatch = d;
1642 }
1643
1644 static void mem_commit(MemoryListener *listener)
1645 {
1646 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1647 AddressSpaceDispatch *cur = as->dispatch;
1648 AddressSpaceDispatch *next = as->next_dispatch;
1649
1650 next->nodes = next_map.nodes;
1651 next->sections = next_map.sections;
1652
1653 as->dispatch = next;
1654 g_free(cur);
1655 }
1656
1657 static void core_begin(MemoryListener *listener)
1658 {
1659 uint16_t n;
1660
1661 prev_map = g_new(PhysPageMap, 1);
1662 *prev_map = next_map;
1663
1664 memset(&next_map, 0, sizeof(next_map));
1665 n = dummy_section(&io_mem_unassigned);
1666 assert(n == PHYS_SECTION_UNASSIGNED);
1667 n = dummy_section(&io_mem_notdirty);
1668 assert(n == PHYS_SECTION_NOTDIRTY);
1669 n = dummy_section(&io_mem_rom);
1670 assert(n == PHYS_SECTION_ROM);
1671 n = dummy_section(&io_mem_watch);
1672 assert(n == PHYS_SECTION_WATCH);
1673 }
1674
1675 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1676 * All AddressSpaceDispatch instances have switched to the next map.
1677 */
1678 static void core_commit(MemoryListener *listener)
1679 {
1680 phys_sections_free(prev_map);
1681 }
1682
1683 static void tcg_commit(MemoryListener *listener)
1684 {
1685 CPUState *cpu;
1686
1687 /* since each CPU stores ram addresses in its TLB cache, we must
1688 reset the modified entries */
1689 /* XXX: slow ! */
1690 CPU_FOREACH(cpu) {
1691 CPUArchState *env = cpu->env_ptr;
1692
1693 tlb_flush(env, 1);
1694 }
1695 }
1696
1697 static void core_log_global_start(MemoryListener *listener)
1698 {
1699 cpu_physical_memory_set_dirty_tracking(1);
1700 }
1701
1702 static void core_log_global_stop(MemoryListener *listener)
1703 {
1704 cpu_physical_memory_set_dirty_tracking(0);
1705 }
1706
1707 static MemoryListener core_memory_listener = {
1708 .begin = core_begin,
1709 .commit = core_commit,
1710 .log_global_start = core_log_global_start,
1711 .log_global_stop = core_log_global_stop,
1712 .priority = 1,
1713 };
1714
1715 static MemoryListener tcg_memory_listener = {
1716 .commit = tcg_commit,
1717 };
1718
1719 void address_space_init_dispatch(AddressSpace *as)
1720 {
1721 as->dispatch = NULL;
1722 as->dispatch_listener = (MemoryListener) {
1723 .begin = mem_begin,
1724 .commit = mem_commit,
1725 .region_add = mem_add,
1726 .region_nop = mem_add,
1727 .priority = 0,
1728 };
1729 memory_listener_register(&as->dispatch_listener, as);
1730 }
1731
1732 void address_space_destroy_dispatch(AddressSpace *as)
1733 {
1734 AddressSpaceDispatch *d = as->dispatch;
1735
1736 memory_listener_unregister(&as->dispatch_listener);
1737 g_free(d);
1738 as->dispatch = NULL;
1739 }
1740
1741 static void memory_map_init(void)
1742 {
1743 system_memory = g_malloc(sizeof(*system_memory));
1744
1745 assert(TARGET_PHYS_ADDR_SPACE_BITS <= 64);
1746
1747 memory_region_init(system_memory, NULL, "system",
1748 TARGET_PHYS_ADDR_SPACE_BITS == 64 ?
1749 UINT64_MAX : (0x1ULL << TARGET_PHYS_ADDR_SPACE_BITS));
1750 address_space_init(&address_space_memory, system_memory, "memory");
1751
1752 system_io = g_malloc(sizeof(*system_io));
1753 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
1754 65536);
1755 address_space_init(&address_space_io, system_io, "I/O");
1756
1757 memory_listener_register(&core_memory_listener, &address_space_memory);
1758 if (tcg_enabled()) {
1759 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1760 }
1761 }
1762
1763 MemoryRegion *get_system_memory(void)
1764 {
1765 return system_memory;
1766 }
1767
1768 MemoryRegion *get_system_io(void)
1769 {
1770 return system_io;
1771 }
1772
1773 #endif /* !defined(CONFIG_USER_ONLY) */
1774
1775 /* physical memory access (slow version, mainly for debug) */
1776 #if defined(CONFIG_USER_ONLY)
1777 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1778 uint8_t *buf, int len, int is_write)
1779 {
1780 int l, flags;
1781 target_ulong page;
1782 void * p;
1783
1784 while (len > 0) {
1785 page = addr & TARGET_PAGE_MASK;
1786 l = (page + TARGET_PAGE_SIZE) - addr;
1787 if (l > len)
1788 l = len;
1789 flags = page_get_flags(page);
1790 if (!(flags & PAGE_VALID))
1791 return -1;
1792 if (is_write) {
1793 if (!(flags & PAGE_WRITE))
1794 return -1;
1795 /* XXX: this code should not depend on lock_user */
1796 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1797 return -1;
1798 memcpy(p, buf, l);
1799 unlock_user(p, addr, l);
1800 } else {
1801 if (!(flags & PAGE_READ))
1802 return -1;
1803 /* XXX: this code should not depend on lock_user */
1804 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1805 return -1;
1806 memcpy(buf, p, l);
1807 unlock_user(p, addr, 0);
1808 }
1809 len -= l;
1810 buf += l;
1811 addr += l;
1812 }
1813 return 0;
1814 }
1815
1816 #else
1817
1818 static void invalidate_and_set_dirty(hwaddr addr,
1819 hwaddr length)
1820 {
1821 if (!cpu_physical_memory_is_dirty(addr)) {
1822 /* invalidate code */
1823 tb_invalidate_phys_page_range(addr, addr + length, 0);
1824 /* set dirty bit */
1825 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1826 }
1827 xen_modified_memory(addr, length);
1828 }
1829
1830 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1831 {
1832 if (memory_region_is_ram(mr)) {
1833 return !(is_write && mr->readonly);
1834 }
1835 if (memory_region_is_romd(mr)) {
1836 return !is_write;
1837 }
1838
1839 return false;
1840 }
1841
1842 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1843 {
1844 unsigned access_size_max = mr->ops->valid.max_access_size;
1845
1846 /* Regions are assumed to support 1-4 byte accesses unless
1847 otherwise specified. */
1848 if (access_size_max == 0) {
1849 access_size_max = 4;
1850 }
1851
1852 /* Bound the maximum access by the alignment of the address. */
1853 if (!mr->ops->impl.unaligned) {
1854 unsigned align_size_max = addr & -addr;
1855 if (align_size_max != 0 && align_size_max < access_size_max) {
1856 access_size_max = align_size_max;
1857 }
1858 }
1859
1860 /* Don't attempt accesses larger than the maximum. */
1861 if (l > access_size_max) {
1862 l = access_size_max;
1863 }
1864 if (l & (l - 1)) {
1865 l = 1 << (qemu_fls(l) - 1);
1866 }
1867
1868 return l;
1869 }
1870
1871 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1872 int len, bool is_write)
1873 {
1874 hwaddr l;
1875 uint8_t *ptr;
1876 uint64_t val;
1877 hwaddr addr1;
1878 MemoryRegion *mr;
1879 bool error = false;
1880
1881 while (len > 0) {
1882 l = len;
1883 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1884
1885 if (is_write) {
1886 if (!memory_access_is_direct(mr, is_write)) {
1887 l = memory_access_size(mr, l, addr1);
1888 /* XXX: could force current_cpu to NULL to avoid
1889 potential bugs */
1890 switch (l) {
1891 case 8:
1892 /* 64 bit write access */
1893 val = ldq_p(buf);
1894 error |= io_mem_write(mr, addr1, val, 8);
1895 break;
1896 case 4:
1897 /* 32 bit write access */
1898 val = ldl_p(buf);
1899 error |= io_mem_write(mr, addr1, val, 4);
1900 break;
1901 case 2:
1902 /* 16 bit write access */
1903 val = lduw_p(buf);
1904 error |= io_mem_write(mr, addr1, val, 2);
1905 break;
1906 case 1:
1907 /* 8 bit write access */
1908 val = ldub_p(buf);
1909 error |= io_mem_write(mr, addr1, val, 1);
1910 break;
1911 default:
1912 abort();
1913 }
1914 } else {
1915 addr1 += memory_region_get_ram_addr(mr);
1916 /* RAM case */
1917 ptr = qemu_get_ram_ptr(addr1);
1918 memcpy(ptr, buf, l);
1919 invalidate_and_set_dirty(addr1, l);
1920 }
1921 } else {
1922 if (!memory_access_is_direct(mr, is_write)) {
1923 /* I/O case */
1924 l = memory_access_size(mr, l, addr1);
1925 switch (l) {
1926 case 8:
1927 /* 64 bit read access */
1928 error |= io_mem_read(mr, addr1, &val, 8);
1929 stq_p(buf, val);
1930 break;
1931 case 4:
1932 /* 32 bit read access */
1933 error |= io_mem_read(mr, addr1, &val, 4);
1934 stl_p(buf, val);
1935 break;
1936 case 2:
1937 /* 16 bit read access */
1938 error |= io_mem_read(mr, addr1, &val, 2);
1939 stw_p(buf, val);
1940 break;
1941 case 1:
1942 /* 8 bit read access */
1943 error |= io_mem_read(mr, addr1, &val, 1);
1944 stb_p(buf, val);
1945 break;
1946 default:
1947 abort();
1948 }
1949 } else {
1950 /* RAM case */
1951 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1952 memcpy(buf, ptr, l);
1953 }
1954 }
1955 len -= l;
1956 buf += l;
1957 addr += l;
1958 }
1959
1960 return error;
1961 }
1962
1963 bool address_space_write(AddressSpace *as, hwaddr addr,
1964 const uint8_t *buf, int len)
1965 {
1966 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
1967 }
1968
1969 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1970 {
1971 return address_space_rw(as, addr, buf, len, false);
1972 }
1973
1974
1975 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1976 int len, int is_write)
1977 {
1978 address_space_rw(&address_space_memory, addr, buf, len, is_write);
1979 }
1980
1981 /* used for ROM loading : can write in RAM and ROM */
1982 void cpu_physical_memory_write_rom(hwaddr addr,
1983 const uint8_t *buf, int len)
1984 {
1985 hwaddr l;
1986 uint8_t *ptr;
1987 hwaddr addr1;
1988 MemoryRegion *mr;
1989
1990 while (len > 0) {
1991 l = len;
1992 mr = address_space_translate(&address_space_memory,
1993 addr, &addr1, &l, true);
1994
1995 if (!(memory_region_is_ram(mr) ||
1996 memory_region_is_romd(mr))) {
1997 /* do nothing */
1998 } else {
1999 addr1 += memory_region_get_ram_addr(mr);
2000 /* ROM/RAM case */
2001 ptr = qemu_get_ram_ptr(addr1);
2002 memcpy(ptr, buf, l);
2003 invalidate_and_set_dirty(addr1, l);
2004 }
2005 len -= l;
2006 buf += l;
2007 addr += l;
2008 }
2009 }
2010
2011 typedef struct {
2012 MemoryRegion *mr;
2013 void *buffer;
2014 hwaddr addr;
2015 hwaddr len;
2016 } BounceBuffer;
2017
2018 static BounceBuffer bounce;
2019
2020 typedef struct MapClient {
2021 void *opaque;
2022 void (*callback)(void *opaque);
2023 QLIST_ENTRY(MapClient) link;
2024 } MapClient;
2025
2026 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2027 = QLIST_HEAD_INITIALIZER(map_client_list);
2028
2029 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2030 {
2031 MapClient *client = g_malloc(sizeof(*client));
2032
2033 client->opaque = opaque;
2034 client->callback = callback;
2035 QLIST_INSERT_HEAD(&map_client_list, client, link);
2036 return client;
2037 }
2038
2039 static void cpu_unregister_map_client(void *_client)
2040 {
2041 MapClient *client = (MapClient *)_client;
2042
2043 QLIST_REMOVE(client, link);
2044 g_free(client);
2045 }
2046
2047 static void cpu_notify_map_clients(void)
2048 {
2049 MapClient *client;
2050
2051 while (!QLIST_EMPTY(&map_client_list)) {
2052 client = QLIST_FIRST(&map_client_list);
2053 client->callback(client->opaque);
2054 cpu_unregister_map_client(client);
2055 }
2056 }
2057
2058 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2059 {
2060 MemoryRegion *mr;
2061 hwaddr l, xlat;
2062
2063 while (len > 0) {
2064 l = len;
2065 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2066 if (!memory_access_is_direct(mr, is_write)) {
2067 l = memory_access_size(mr, l, addr);
2068 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2069 return false;
2070 }
2071 }
2072
2073 len -= l;
2074 addr += l;
2075 }
2076 return true;
2077 }
2078
2079 /* Map a physical memory region into a host virtual address.
2080 * May map a subset of the requested range, given by and returned in *plen.
2081 * May return NULL if resources needed to perform the mapping are exhausted.
2082 * Use only for reads OR writes - not for read-modify-write operations.
2083 * Use cpu_register_map_client() to know when retrying the map operation is
2084 * likely to succeed.
2085 */
2086 void *address_space_map(AddressSpace *as,
2087 hwaddr addr,
2088 hwaddr *plen,
2089 bool is_write)
2090 {
2091 hwaddr len = *plen;
2092 hwaddr done = 0;
2093 hwaddr l, xlat, base;
2094 MemoryRegion *mr, *this_mr;
2095 ram_addr_t raddr;
2096
2097 if (len == 0) {
2098 return NULL;
2099 }
2100
2101 l = len;
2102 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2103 if (!memory_access_is_direct(mr, is_write)) {
2104 if (bounce.buffer) {
2105 return NULL;
2106 }
2107 /* Avoid unbounded allocations */
2108 l = MIN(l, TARGET_PAGE_SIZE);
2109 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
2110 bounce.addr = addr;
2111 bounce.len = l;
2112
2113 memory_region_ref(mr);
2114 bounce.mr = mr;
2115 if (!is_write) {
2116 address_space_read(as, addr, bounce.buffer, l);
2117 }
2118
2119 *plen = l;
2120 return bounce.buffer;
2121 }
2122
2123 base = xlat;
2124 raddr = memory_region_get_ram_addr(mr);
2125
2126 for (;;) {
2127 len -= l;
2128 addr += l;
2129 done += l;
2130 if (len == 0) {
2131 break;
2132 }
2133
2134 l = len;
2135 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2136 if (this_mr != mr || xlat != base + done) {
2137 break;
2138 }
2139 }
2140
2141 memory_region_ref(mr);
2142 *plen = done;
2143 return qemu_ram_ptr_length(raddr + base, plen);
2144 }
2145
2146 /* Unmaps a memory region previously mapped by address_space_map().
2147 * Will also mark the memory as dirty if is_write == 1. access_len gives
2148 * the amount of memory that was actually read or written by the caller.
2149 */
2150 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2151 int is_write, hwaddr access_len)
2152 {
2153 if (buffer != bounce.buffer) {
2154 MemoryRegion *mr;
2155 ram_addr_t addr1;
2156
2157 mr = qemu_ram_addr_from_host(buffer, &addr1);
2158 assert(mr != NULL);
2159 if (is_write) {
2160 while (access_len) {
2161 unsigned l;
2162 l = TARGET_PAGE_SIZE;
2163 if (l > access_len)
2164 l = access_len;
2165 invalidate_and_set_dirty(addr1, l);
2166 addr1 += l;
2167 access_len -= l;
2168 }
2169 }
2170 if (xen_enabled()) {
2171 xen_invalidate_map_cache_entry(buffer);
2172 }
2173 memory_region_unref(mr);
2174 return;
2175 }
2176 if (is_write) {
2177 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2178 }
2179 qemu_vfree(bounce.buffer);
2180 bounce.buffer = NULL;
2181 memory_region_unref(bounce.mr);
2182 cpu_notify_map_clients();
2183 }
2184
2185 void *cpu_physical_memory_map(hwaddr addr,
2186 hwaddr *plen,
2187 int is_write)
2188 {
2189 return address_space_map(&address_space_memory, addr, plen, is_write);
2190 }
2191
2192 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2193 int is_write, hwaddr access_len)
2194 {
2195 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2196 }
2197
2198 /* warning: addr must be aligned */
2199 static inline uint32_t ldl_phys_internal(hwaddr addr,
2200 enum device_endian endian)
2201 {
2202 uint8_t *ptr;
2203 uint64_t val;
2204 MemoryRegion *mr;
2205 hwaddr l = 4;
2206 hwaddr addr1;
2207
2208 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2209 false);
2210 if (l < 4 || !memory_access_is_direct(mr, false)) {
2211 /* I/O case */
2212 io_mem_read(mr, addr1, &val, 4);
2213 #if defined(TARGET_WORDS_BIGENDIAN)
2214 if (endian == DEVICE_LITTLE_ENDIAN) {
2215 val = bswap32(val);
2216 }
2217 #else
2218 if (endian == DEVICE_BIG_ENDIAN) {
2219 val = bswap32(val);
2220 }
2221 #endif
2222 } else {
2223 /* RAM case */
2224 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2225 & TARGET_PAGE_MASK)
2226 + addr1);
2227 switch (endian) {
2228 case DEVICE_LITTLE_ENDIAN:
2229 val = ldl_le_p(ptr);
2230 break;
2231 case DEVICE_BIG_ENDIAN:
2232 val = ldl_be_p(ptr);
2233 break;
2234 default:
2235 val = ldl_p(ptr);
2236 break;
2237 }
2238 }
2239 return val;
2240 }
2241
2242 uint32_t ldl_phys(hwaddr addr)
2243 {
2244 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2245 }
2246
2247 uint32_t ldl_le_phys(hwaddr addr)
2248 {
2249 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2250 }
2251
2252 uint32_t ldl_be_phys(hwaddr addr)
2253 {
2254 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2255 }
2256
2257 /* warning: addr must be aligned */
2258 static inline uint64_t ldq_phys_internal(hwaddr addr,
2259 enum device_endian endian)
2260 {
2261 uint8_t *ptr;
2262 uint64_t val;
2263 MemoryRegion *mr;
2264 hwaddr l = 8;
2265 hwaddr addr1;
2266
2267 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2268 false);
2269 if (l < 8 || !memory_access_is_direct(mr, false)) {
2270 /* I/O case */
2271 io_mem_read(mr, addr1, &val, 8);
2272 #if defined(TARGET_WORDS_BIGENDIAN)
2273 if (endian == DEVICE_LITTLE_ENDIAN) {
2274 val = bswap64(val);
2275 }
2276 #else
2277 if (endian == DEVICE_BIG_ENDIAN) {
2278 val = bswap64(val);
2279 }
2280 #endif
2281 } else {
2282 /* RAM case */
2283 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2284 & TARGET_PAGE_MASK)
2285 + addr1);
2286 switch (endian) {
2287 case DEVICE_LITTLE_ENDIAN:
2288 val = ldq_le_p(ptr);
2289 break;
2290 case DEVICE_BIG_ENDIAN:
2291 val = ldq_be_p(ptr);
2292 break;
2293 default:
2294 val = ldq_p(ptr);
2295 break;
2296 }
2297 }
2298 return val;
2299 }
2300
2301 uint64_t ldq_phys(hwaddr addr)
2302 {
2303 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2304 }
2305
2306 uint64_t ldq_le_phys(hwaddr addr)
2307 {
2308 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2309 }
2310
2311 uint64_t ldq_be_phys(hwaddr addr)
2312 {
2313 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2314 }
2315
2316 /* XXX: optimize */
2317 uint32_t ldub_phys(hwaddr addr)
2318 {
2319 uint8_t val;
2320 cpu_physical_memory_read(addr, &val, 1);
2321 return val;
2322 }
2323
2324 /* warning: addr must be aligned */
2325 static inline uint32_t lduw_phys_internal(hwaddr addr,
2326 enum device_endian endian)
2327 {
2328 uint8_t *ptr;
2329 uint64_t val;
2330 MemoryRegion *mr;
2331 hwaddr l = 2;
2332 hwaddr addr1;
2333
2334 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2335 false);
2336 if (l < 2 || !memory_access_is_direct(mr, false)) {
2337 /* I/O case */
2338 io_mem_read(mr, addr1, &val, 2);
2339 #if defined(TARGET_WORDS_BIGENDIAN)
2340 if (endian == DEVICE_LITTLE_ENDIAN) {
2341 val = bswap16(val);
2342 }
2343 #else
2344 if (endian == DEVICE_BIG_ENDIAN) {
2345 val = bswap16(val);
2346 }
2347 #endif
2348 } else {
2349 /* RAM case */
2350 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2351 & TARGET_PAGE_MASK)
2352 + addr1);
2353 switch (endian) {
2354 case DEVICE_LITTLE_ENDIAN:
2355 val = lduw_le_p(ptr);
2356 break;
2357 case DEVICE_BIG_ENDIAN:
2358 val = lduw_be_p(ptr);
2359 break;
2360 default:
2361 val = lduw_p(ptr);
2362 break;
2363 }
2364 }
2365 return val;
2366 }
2367
2368 uint32_t lduw_phys(hwaddr addr)
2369 {
2370 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2371 }
2372
2373 uint32_t lduw_le_phys(hwaddr addr)
2374 {
2375 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2376 }
2377
2378 uint32_t lduw_be_phys(hwaddr addr)
2379 {
2380 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2381 }
2382
2383 /* warning: addr must be aligned. The ram page is not masked as dirty
2384 and the code inside is not invalidated. It is useful if the dirty
2385 bits are used to track modified PTEs */
2386 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2387 {
2388 uint8_t *ptr;
2389 MemoryRegion *mr;
2390 hwaddr l = 4;
2391 hwaddr addr1;
2392
2393 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2394 true);
2395 if (l < 4 || !memory_access_is_direct(mr, true)) {
2396 io_mem_write(mr, addr1, val, 4);
2397 } else {
2398 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2399 ptr = qemu_get_ram_ptr(addr1);
2400 stl_p(ptr, val);
2401
2402 if (unlikely(in_migration)) {
2403 if (!cpu_physical_memory_is_dirty(addr1)) {
2404 /* invalidate code */
2405 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2406 /* set dirty bit */
2407 cpu_physical_memory_set_dirty_flags(
2408 addr1, (0xff & ~CODE_DIRTY_FLAG));
2409 }
2410 }
2411 }
2412 }
2413
2414 /* warning: addr must be aligned */
2415 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2416 enum device_endian endian)
2417 {
2418 uint8_t *ptr;
2419 MemoryRegion *mr;
2420 hwaddr l = 4;
2421 hwaddr addr1;
2422
2423 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2424 true);
2425 if (l < 4 || !memory_access_is_direct(mr, true)) {
2426 #if defined(TARGET_WORDS_BIGENDIAN)
2427 if (endian == DEVICE_LITTLE_ENDIAN) {
2428 val = bswap32(val);
2429 }
2430 #else
2431 if (endian == DEVICE_BIG_ENDIAN) {
2432 val = bswap32(val);
2433 }
2434 #endif
2435 io_mem_write(mr, addr1, val, 4);
2436 } else {
2437 /* RAM case */
2438 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2439 ptr = qemu_get_ram_ptr(addr1);
2440 switch (endian) {
2441 case DEVICE_LITTLE_ENDIAN:
2442 stl_le_p(ptr, val);
2443 break;
2444 case DEVICE_BIG_ENDIAN:
2445 stl_be_p(ptr, val);
2446 break;
2447 default:
2448 stl_p(ptr, val);
2449 break;
2450 }
2451 invalidate_and_set_dirty(addr1, 4);
2452 }
2453 }
2454
2455 void stl_phys(hwaddr addr, uint32_t val)
2456 {
2457 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2458 }
2459
2460 void stl_le_phys(hwaddr addr, uint32_t val)
2461 {
2462 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2463 }
2464
2465 void stl_be_phys(hwaddr addr, uint32_t val)
2466 {
2467 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2468 }
2469
2470 /* XXX: optimize */
2471 void stb_phys(hwaddr addr, uint32_t val)
2472 {
2473 uint8_t v = val;
2474 cpu_physical_memory_write(addr, &v, 1);
2475 }
2476
2477 /* warning: addr must be aligned */
2478 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2479 enum device_endian endian)
2480 {
2481 uint8_t *ptr;
2482 MemoryRegion *mr;
2483 hwaddr l = 2;
2484 hwaddr addr1;
2485
2486 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2487 true);
2488 if (l < 2 || !memory_access_is_direct(mr, true)) {
2489 #if defined(TARGET_WORDS_BIGENDIAN)
2490 if (endian == DEVICE_LITTLE_ENDIAN) {
2491 val = bswap16(val);
2492 }
2493 #else
2494 if (endian == DEVICE_BIG_ENDIAN) {
2495 val = bswap16(val);
2496 }
2497 #endif
2498 io_mem_write(mr, addr1, val, 2);
2499 } else {
2500 /* RAM case */
2501 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2502 ptr = qemu_get_ram_ptr(addr1);
2503 switch (endian) {
2504 case DEVICE_LITTLE_ENDIAN:
2505 stw_le_p(ptr, val);
2506 break;
2507 case DEVICE_BIG_ENDIAN:
2508 stw_be_p(ptr, val);
2509 break;
2510 default:
2511 stw_p(ptr, val);
2512 break;
2513 }
2514 invalidate_and_set_dirty(addr1, 2);
2515 }
2516 }
2517
2518 void stw_phys(hwaddr addr, uint32_t val)
2519 {
2520 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2521 }
2522
2523 void stw_le_phys(hwaddr addr, uint32_t val)
2524 {
2525 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2526 }
2527
2528 void stw_be_phys(hwaddr addr, uint32_t val)
2529 {
2530 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2531 }
2532
2533 /* XXX: optimize */
2534 void stq_phys(hwaddr addr, uint64_t val)
2535 {
2536 val = tswap64(val);
2537 cpu_physical_memory_write(addr, &val, 8);
2538 }
2539
2540 void stq_le_phys(hwaddr addr, uint64_t val)
2541 {
2542 val = cpu_to_le64(val);
2543 cpu_physical_memory_write(addr, &val, 8);
2544 }
2545
2546 void stq_be_phys(hwaddr addr, uint64_t val)
2547 {
2548 val = cpu_to_be64(val);
2549 cpu_physical_memory_write(addr, &val, 8);
2550 }
2551
2552 /* virtual memory access for debug (includes writing to ROM) */
2553 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2554 uint8_t *buf, int len, int is_write)
2555 {
2556 int l;
2557 hwaddr phys_addr;
2558 target_ulong page;
2559
2560 while (len > 0) {
2561 page = addr & TARGET_PAGE_MASK;
2562 phys_addr = cpu_get_phys_page_debug(cpu, page);
2563 /* if no physical page mapped, return an error */
2564 if (phys_addr == -1)
2565 return -1;
2566 l = (page + TARGET_PAGE_SIZE) - addr;
2567 if (l > len)
2568 l = len;
2569 phys_addr += (addr & ~TARGET_PAGE_MASK);
2570 if (is_write)
2571 cpu_physical_memory_write_rom(phys_addr, buf, l);
2572 else
2573 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2574 len -= l;
2575 buf += l;
2576 addr += l;
2577 }
2578 return 0;
2579 }
2580 #endif
2581
2582 #if !defined(CONFIG_USER_ONLY)
2583
2584 /*
2585 * A helper function for the _utterly broken_ virtio device model to find out if
2586 * it's running on a big endian machine. Don't do this at home kids!
2587 */
2588 bool virtio_is_big_endian(void);
2589 bool virtio_is_big_endian(void)
2590 {
2591 #if defined(TARGET_WORDS_BIGENDIAN)
2592 return true;
2593 #else
2594 return false;
2595 #endif
2596 }
2597
2598 #endif
2599
2600 #ifndef CONFIG_USER_ONLY
2601 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2602 {
2603 MemoryRegion*mr;
2604 hwaddr l = 1;
2605
2606 mr = address_space_translate(&address_space_memory,
2607 phys_addr, &phys_addr, &l, false);
2608
2609 return !(memory_region_is_ram(mr) ||
2610 memory_region_is_romd(mr));
2611 }
2612
2613 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2614 {
2615 RAMBlock *block;
2616
2617 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2618 func(block->host, block->offset, block->length, opaque);
2619 }
2620 }
2621 #endif
AR7 emulation for QEMU