?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
vhost: multiqueue support
[qemu/kevin.git] / exec.c
blobb85508ba30d4d5d3714e0eae5f38c2d16e570f25
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "hw/xen.h"
35 #include "qemu/timer.h"
36 #include "qemu/config-file.h"
37 #include "exec/memory.h"
38 #include "sysemu/dma.h"
39 #include "exec/address-spaces.h"
40 #if defined(CONFIG_USER_ONLY)
41 #include <qemu.h>
42 #else /* !CONFIG_USER_ONLY */
43 #include "sysemu/xen-mapcache.h"
44 #include "trace.h"
45 #endif
46 #include "exec/cpu-all.h"
47
48 #include "exec/cputlb.h"
49 #include "translate-all.h"
50
51 #include "exec/memory-internal.h"
52
53 //#define DEBUG_UNASSIGNED
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 int phys_ram_fd;
58 static int in_migration;
59
60 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
61
62 static MemoryRegion *system_memory;
63 static MemoryRegion *system_io;
64
65 AddressSpace address_space_io;
66 AddressSpace address_space_memory;
67 DMAContext dma_context_memory;
68
69 MemoryRegion io_mem_ram, io_mem_rom, io_mem_unassigned, io_mem_notdirty;
70 static MemoryRegion io_mem_subpage_ram;
71
72 #endif
73
74 CPUArchState *first_cpu;
75 /* current CPU in the current thread. It is only valid inside
76 cpu_exec() */
77 DEFINE_TLS(CPUArchState *,cpu_single_env);
78 /* 0 = Do not count executed instructions.
79 1 = Precise instruction counting.
80 2 = Adaptive rate instruction counting. */
81 int use_icount;
82
83 #if !defined(CONFIG_USER_ONLY)
84
85 static MemoryRegionSection *phys_sections;
86 static unsigned phys_sections_nb, phys_sections_nb_alloc;
87 static uint16_t phys_section_unassigned;
88 static uint16_t phys_section_notdirty;
89 static uint16_t phys_section_rom;
90 static uint16_t phys_section_watch;
91
92 /* Simple allocator for PhysPageEntry nodes */
93 static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
94 static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
95
96 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
97
98 static void io_mem_init(void);
99 static void memory_map_init(void);
100 static void *qemu_safe_ram_ptr(ram_addr_t addr);
101
102 static MemoryRegion io_mem_watch;
103 #endif
104
105 #if !defined(CONFIG_USER_ONLY)
106
107 static void phys_map_node_reserve(unsigned nodes)
108 {
109 if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) {
110 typedef PhysPageEntry Node[L2_SIZE];
111 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16);
112 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc,
113 phys_map_nodes_nb + nodes);
114 phys_map_nodes = g_renew(Node, phys_map_nodes,
115 phys_map_nodes_nb_alloc);
116 }
117 }
118
119 static uint16_t phys_map_node_alloc(void)
120 {
121 unsigned i;
122 uint16_t ret;
123
124 ret = phys_map_nodes_nb++;
125 assert(ret != PHYS_MAP_NODE_NIL);
126 assert(ret != phys_map_nodes_nb_alloc);
127 for (i = 0; i < L2_SIZE; ++i) {
128 phys_map_nodes[ret][i].is_leaf = 0;
129 phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
130 }
131 return ret;
132 }
133
134 static void phys_map_nodes_reset(void)
135 {
136 phys_map_nodes_nb = 0;
137 }
138
139
140 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
141 hwaddr *nb, uint16_t leaf,
142 int level)
143 {
144 PhysPageEntry *p;
145 int i;
146 hwaddr step = (hwaddr)1 << (level * L2_BITS);
147
148 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
149 lp->ptr = phys_map_node_alloc();
150 p = phys_map_nodes[lp->ptr];
151 if (level == 0) {
152 for (i = 0; i < L2_SIZE; i++) {
153 p[i].is_leaf = 1;
154 p[i].ptr = phys_section_unassigned;
155 }
156 }
157 } else {
158 p = phys_map_nodes[lp->ptr];
159 }
160 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
161
162 while (*nb && lp < &p[L2_SIZE]) {
163 if ((*index & (step - 1)) == 0 && *nb >= step) {
164 lp->is_leaf = true;
165 lp->ptr = leaf;
166 *index += step;
167 *nb -= step;
168 } else {
169 phys_page_set_level(lp, index, nb, leaf, level - 1);
170 }
171 ++lp;
172 }
173 }
174
175 static void phys_page_set(AddressSpaceDispatch *d,
176 hwaddr index, hwaddr nb,
177 uint16_t leaf)
178 {
179 /* Wildly overreserve - it doesn't matter much. */
180 phys_map_node_reserve(3 * P_L2_LEVELS);
181
182 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
183 }
184
185 MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr index)
186 {
187 PhysPageEntry lp = d->phys_map;
188 PhysPageEntry *p;
189 int i;
190 uint16_t s_index = phys_section_unassigned;
191
192 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
193 if (lp.ptr == PHYS_MAP_NODE_NIL) {
194 goto not_found;
195 }
196 p = phys_map_nodes[lp.ptr];
197 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
198 }
199
200 s_index = lp.ptr;
201 not_found:
202 return &phys_sections[s_index];
203 }
204
205 bool memory_region_is_unassigned(MemoryRegion *mr)
206 {
207 return mr != &io_mem_ram && mr != &io_mem_rom
208 && mr != &io_mem_notdirty && !mr->rom_device
209 && mr != &io_mem_watch;
210 }
211 #endif
212
213 void cpu_exec_init_all(void)
214 {
215 #if !defined(CONFIG_USER_ONLY)
216 qemu_mutex_init(&ram_list.mutex);
217 memory_map_init();
218 io_mem_init();
219 #endif
220 }
221
222 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
223
224 static int cpu_common_post_load(void *opaque, int version_id)
225 {
226 CPUArchState *env = opaque;
227
228 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
229 version_id is increased. */
230 env->interrupt_request &= ~0x01;
231 tlb_flush(env, 1);
232
233 return 0;
234 }
235
236 static const VMStateDescription vmstate_cpu_common = {
237 .name = "cpu_common",
238 .version_id = 1,
239 .minimum_version_id = 1,
240 .minimum_version_id_old = 1,
241 .post_load = cpu_common_post_load,
242 .fields = (VMStateField []) {
243 VMSTATE_UINT32(halted, CPUArchState),
244 VMSTATE_UINT32(interrupt_request, CPUArchState),
245 VMSTATE_END_OF_LIST()
246 }
247 };
248 #endif
249
250 CPUState *qemu_get_cpu(int index)
251 {
252 CPUArchState *env = first_cpu;
253 CPUState *cpu = NULL;
254
255 while (env) {
256 cpu = ENV_GET_CPU(env);
257 if (cpu->cpu_index == index) {
258 break;
259 }
260 env = env->next_cpu;
261 }
262
263 return cpu;
264 }
265
266 void cpu_exec_init(CPUArchState *env)
267 {
268 CPUState *cpu = ENV_GET_CPU(env);
269 CPUArchState **penv;
270 int cpu_index;
271
272 #if defined(CONFIG_USER_ONLY)
273 cpu_list_lock();
274 #endif
275 env->next_cpu = NULL;
276 penv = &first_cpu;
277 cpu_index = 0;
278 while (*penv != NULL) {
279 penv = &(*penv)->next_cpu;
280 cpu_index++;
281 }
282 cpu->cpu_index = cpu_index;
283 cpu->numa_node = 0;
284 QTAILQ_INIT(&env->breakpoints);
285 QTAILQ_INIT(&env->watchpoints);
286 #ifndef CONFIG_USER_ONLY
287 cpu->thread_id = qemu_get_thread_id();
288 #endif
289 *penv = env;
290 #if defined(CONFIG_USER_ONLY)
291 cpu_list_unlock();
292 #endif
293 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
294 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
295 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
296 cpu_save, cpu_load, env);
297 #endif
298 }
299
300 #if defined(TARGET_HAS_ICE)
301 #if defined(CONFIG_USER_ONLY)
302 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
303 {
304 tb_invalidate_phys_page_range(pc, pc + 1, 0);
305 }
306 #else
307 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
308 {
309 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
310 (pc & ~TARGET_PAGE_MASK));
311 }
312 #endif
313 #endif /* TARGET_HAS_ICE */
314
315 #if defined(CONFIG_USER_ONLY)
316 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
317
318 {
319 }
320
321 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
322 int flags, CPUWatchpoint **watchpoint)
323 {
324 return -ENOSYS;
325 }
326 #else
327 /* Add a watchpoint. */
328 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
329 int flags, CPUWatchpoint **watchpoint)
330 {
331 target_ulong len_mask = ~(len - 1);
332 CPUWatchpoint *wp;
333
334 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
335 if ((len & (len - 1)) || (addr & ~len_mask) ||
336 len == 0 || len > TARGET_PAGE_SIZE) {
337 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
338 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
339 return -EINVAL;
340 }
341 wp = g_malloc(sizeof(*wp));
342
343 wp->vaddr = addr;
344 wp->len_mask = len_mask;
345 wp->flags = flags;
346
347 /* keep all GDB-injected watchpoints in front */
348 if (flags & BP_GDB)
349 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
350 else
351 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
352
353 tlb_flush_page(env, addr);
354
355 if (watchpoint)
356 *watchpoint = wp;
357 return 0;
358 }
359
360 /* Remove a specific watchpoint. */
361 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
362 int flags)
363 {
364 target_ulong len_mask = ~(len - 1);
365 CPUWatchpoint *wp;
366
367 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
368 if (addr == wp->vaddr && len_mask == wp->len_mask
369 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
370 cpu_watchpoint_remove_by_ref(env, wp);
371 return 0;
372 }
373 }
374 return -ENOENT;
375 }
376
377 /* Remove a specific watchpoint by reference. */
378 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
379 {
380 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
381
382 tlb_flush_page(env, watchpoint->vaddr);
383
384 g_free(watchpoint);
385 }
386
387 /* Remove all matching watchpoints. */
388 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
389 {
390 CPUWatchpoint *wp, *next;
391
392 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
393 if (wp->flags & mask)
394 cpu_watchpoint_remove_by_ref(env, wp);
395 }
396 }
397 #endif
398
399 /* Add a breakpoint. */
400 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
401 CPUBreakpoint **breakpoint)
402 {
403 #if defined(TARGET_HAS_ICE)
404 CPUBreakpoint *bp;
405
406 bp = g_malloc(sizeof(*bp));
407
408 bp->pc = pc;
409 bp->flags = flags;
410
411 /* keep all GDB-injected breakpoints in front */
412 if (flags & BP_GDB)
413 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
414 else
415 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
416
417 breakpoint_invalidate(env, pc);
418
419 if (breakpoint)
420 *breakpoint = bp;
421 return 0;
422 #else
423 return -ENOSYS;
424 #endif
425 }
426
427 /* Remove a specific breakpoint. */
428 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
429 {
430 #if defined(TARGET_HAS_ICE)
431 CPUBreakpoint *bp;
432
433 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
434 if (bp->pc == pc && bp->flags == flags) {
435 cpu_breakpoint_remove_by_ref(env, bp);
436 return 0;
437 }
438 }
439 return -ENOENT;
440 #else
441 return -ENOSYS;
442 #endif
443 }
444
445 /* Remove a specific breakpoint by reference. */
446 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
447 {
448 #if defined(TARGET_HAS_ICE)
449 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
450
451 breakpoint_invalidate(env, breakpoint->pc);
452
453 g_free(breakpoint);
454 #endif
455 }
456
457 /* Remove all matching breakpoints. */
458 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
459 {
460 #if defined(TARGET_HAS_ICE)
461 CPUBreakpoint *bp, *next;
462
463 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
464 if (bp->flags & mask)
465 cpu_breakpoint_remove_by_ref(env, bp);
466 }
467 #endif
468 }
469
470 /* enable or disable single step mode. EXCP_DEBUG is returned by the
471 CPU loop after each instruction */
472 void cpu_single_step(CPUArchState *env, int enabled)
473 {
474 #if defined(TARGET_HAS_ICE)
475 if (env->singlestep_enabled != enabled) {
476 env->singlestep_enabled = enabled;
477 if (kvm_enabled())
478 kvm_update_guest_debug(env, 0);
479 else {
480 /* must flush all the translated code to avoid inconsistencies */
481 /* XXX: only flush what is necessary */
482 tb_flush(env);
483 }
484 }
485 #endif
486 }
487
488 void cpu_reset_interrupt(CPUArchState *env, int mask)
489 {
490 env->interrupt_request &= ~mask;
491 }
492
493 void cpu_exit(CPUArchState *env)
494 {
495 env->exit_request = 1;
496 cpu_unlink_tb(env);
497 }
498
499 void cpu_abort(CPUArchState *env, const char *fmt, ...)
500 {
501 va_list ap;
502 va_list ap2;
503
504 va_start(ap, fmt);
505 va_copy(ap2, ap);
506 fprintf(stderr, "qemu: fatal: ");
507 vfprintf(stderr, fmt, ap);
508 fprintf(stderr, "\n");
509 cpu_dump_state(env, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
510 if (qemu_log_enabled()) {
511 qemu_log("qemu: fatal: ");
512 qemu_log_vprintf(fmt, ap2);
513 qemu_log("\n");
514 log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
515 qemu_log_flush();
516 qemu_log_close();
517 }
518 va_end(ap2);
519 va_end(ap);
520 #if defined(CONFIG_USER_ONLY)
521 {
522 struct sigaction act;
523 sigfillset(&act.sa_mask);
524 act.sa_handler = SIG_DFL;
525 sigaction(SIGABRT, &act, NULL);
526 }
527 #endif
528 abort();
529 }
530
531 CPUArchState *cpu_copy(CPUArchState *env)
532 {
533 CPUArchState *new_env = cpu_init(env->cpu_model_str);
534 CPUArchState *next_cpu = new_env->next_cpu;
535 #if defined(TARGET_HAS_ICE)
536 CPUBreakpoint *bp;
537 CPUWatchpoint *wp;
538 #endif
539
540 memcpy(new_env, env, sizeof(CPUArchState));
541
542 /* Preserve chaining. */
543 new_env->next_cpu = next_cpu;
544
545 /* Clone all break/watchpoints.
546 Note: Once we support ptrace with hw-debug register access, make sure
547 BP_CPU break/watchpoints are handled correctly on clone. */
548 QTAILQ_INIT(&env->breakpoints);
549 QTAILQ_INIT(&env->watchpoints);
550 #if defined(TARGET_HAS_ICE)
551 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
552 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
553 }
554 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
555 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
556 wp->flags, NULL);
557 }
558 #endif
559
560 return new_env;
561 }
562
563 #if !defined(CONFIG_USER_ONLY)
564 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
565 uintptr_t length)
566 {
567 uintptr_t start1;
568
569 /* we modify the TLB cache so that the dirty bit will be set again
570 when accessing the range */
571 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
572 /* Check that we don't span multiple blocks - this breaks the
573 address comparisons below. */
574 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
575 != (end - 1) - start) {
576 abort();
577 }
578 cpu_tlb_reset_dirty_all(start1, length);
579
580 }
581
582 /* Note: start and end must be within the same ram block. */
583 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
584 int dirty_flags)
585 {
586 uintptr_t length;
587
588 start &= TARGET_PAGE_MASK;
589 end = TARGET_PAGE_ALIGN(end);
590
591 length = end - start;
592 if (length == 0)
593 return;
594 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
595
596 if (tcg_enabled()) {
597 tlb_reset_dirty_range_all(start, end, length);
598 }
599 }
600
601 static int cpu_physical_memory_set_dirty_tracking(int enable)
602 {
603 int ret = 0;
604 in_migration = enable;
605 return ret;
606 }
607
608 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
609 MemoryRegionSection *section,
610 target_ulong vaddr,
611 hwaddr paddr,
612 int prot,
613 target_ulong *address)
614 {
615 hwaddr iotlb;
616 CPUWatchpoint *wp;
617
618 if (memory_region_is_ram(section->mr)) {
619 /* Normal RAM. */
620 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
621 + memory_region_section_addr(section, paddr);
622 if (!section->readonly) {
623 iotlb |= phys_section_notdirty;
624 } else {
625 iotlb |= phys_section_rom;
626 }
627 } else {
628 /* IO handlers are currently passed a physical address.
629 It would be nice to pass an offset from the base address
630 of that region. This would avoid having to special case RAM,
631 and avoid full address decoding in every device.
632 We can't use the high bits of pd for this because
633 IO_MEM_ROMD uses these as a ram address. */
634 iotlb = section - phys_sections;
635 iotlb += memory_region_section_addr(section, paddr);
636 }
637
638 /* Make accesses to pages with watchpoints go via the
639 watchpoint trap routines. */
640 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
641 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
642 /* Avoid trapping reads of pages with a write breakpoint. */
643 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
644 iotlb = phys_section_watch + paddr;
645 *address |= TLB_MMIO;
646 break;
647 }
648 }
649 }
650
651 return iotlb;
652 }
653 #endif /* defined(CONFIG_USER_ONLY) */
654
655 #if !defined(CONFIG_USER_ONLY)
656
657 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
658 typedef struct subpage_t {
659 MemoryRegion iomem;
660 hwaddr base;
661 uint16_t sub_section[TARGET_PAGE_SIZE];
662 } subpage_t;
663
664 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
665 uint16_t section);
666 static subpage_t *subpage_init(hwaddr base);
667 static void destroy_page_desc(uint16_t section_index)
668 {
669 MemoryRegionSection *section = &phys_sections[section_index];
670 MemoryRegion *mr = section->mr;
671
672 if (mr->subpage) {
673 subpage_t *subpage = container_of(mr, subpage_t, iomem);
674 memory_region_destroy(&subpage->iomem);
675 g_free(subpage);
676 }
677 }
678
679 static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)
680 {
681 unsigned i;
682 PhysPageEntry *p;
683
684 if (lp->ptr == PHYS_MAP_NODE_NIL) {
685 return;
686 }
687
688 p = phys_map_nodes[lp->ptr];
689 for (i = 0; i < L2_SIZE; ++i) {
690 if (!p[i].is_leaf) {
691 destroy_l2_mapping(&p[i], level - 1);
692 } else {
693 destroy_page_desc(p[i].ptr);
694 }
695 }
696 lp->is_leaf = 0;
697 lp->ptr = PHYS_MAP_NODE_NIL;
698 }
699
700 static void destroy_all_mappings(AddressSpaceDispatch *d)
701 {
702 destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
703 phys_map_nodes_reset();
704 }
705
706 static uint16_t phys_section_add(MemoryRegionSection *section)
707 {
708 if (phys_sections_nb == phys_sections_nb_alloc) {
709 phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16);
710 phys_sections = g_renew(MemoryRegionSection, phys_sections,
711 phys_sections_nb_alloc);
712 }
713 phys_sections[phys_sections_nb] = *section;
714 return phys_sections_nb++;
715 }
716
717 static void phys_sections_clear(void)
718 {
719 phys_sections_nb = 0;
720 }
721
722 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
723 {
724 subpage_t *subpage;
725 hwaddr base = section->offset_within_address_space
726 & TARGET_PAGE_MASK;
727 MemoryRegionSection *existing = phys_page_find(d, base >> TARGET_PAGE_BITS);
728 MemoryRegionSection subsection = {
729 .offset_within_address_space = base,
730 .size = TARGET_PAGE_SIZE,
731 };
732 hwaddr start, end;
733
734 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
735
736 if (!(existing->mr->subpage)) {
737 subpage = subpage_init(base);
738 subsection.mr = &subpage->iomem;
739 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
740 phys_section_add(&subsection));
741 } else {
742 subpage = container_of(existing->mr, subpage_t, iomem);
743 }
744 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
745 end = start + section->size - 1;
746 subpage_register(subpage, start, end, phys_section_add(section));
747 }
748
749
750 static void register_multipage(AddressSpaceDispatch *d, MemoryRegionSection *section)
751 {
752 hwaddr start_addr = section->offset_within_address_space;
753 ram_addr_t size = section->size;
754 hwaddr addr;
755 uint16_t section_index = phys_section_add(section);
756
757 assert(size);
758
759 addr = start_addr;
760 phys_page_set(d, addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,
761 section_index);
762 }
763
764 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
765 {
766 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
767 MemoryRegionSection now = *section, remain = *section;
768
769 if ((now.offset_within_address_space & ~TARGET_PAGE_MASK)
770 || (now.size < TARGET_PAGE_SIZE)) {
771 now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space)
772 - now.offset_within_address_space,
773 now.size);
774 register_subpage(d, &now);
775 remain.size -= now.size;
776 remain.offset_within_address_space += now.size;
777 remain.offset_within_region += now.size;
778 }
779 while (remain.size >= TARGET_PAGE_SIZE) {
780 now = remain;
781 if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
782 now.size = TARGET_PAGE_SIZE;
783 register_subpage(d, &now);
784 } else {
785 now.size &= TARGET_PAGE_MASK;
786 register_multipage(d, &now);
787 }
788 remain.size -= now.size;
789 remain.offset_within_address_space += now.size;
790 remain.offset_within_region += now.size;
791 }
792 now = remain;
793 if (now.size) {
794 register_subpage(d, &now);
795 }
796 }
797
798 void qemu_flush_coalesced_mmio_buffer(void)
799 {
800 if (kvm_enabled())
801 kvm_flush_coalesced_mmio_buffer();
802 }
803
804 void qemu_mutex_lock_ramlist(void)
805 {
806 qemu_mutex_lock(&ram_list.mutex);
807 }
808
809 void qemu_mutex_unlock_ramlist(void)
810 {
811 qemu_mutex_unlock(&ram_list.mutex);
812 }
813
814 #if defined(__linux__) && !defined(TARGET_S390X)
815
816 #include <sys/vfs.h>
817
818 #define HUGETLBFS_MAGIC 0x958458f6
819
820 static long gethugepagesize(const char *path)
821 {
822 struct statfs fs;
823 int ret;
824
825 do {
826 ret = statfs(path, &fs);
827 } while (ret != 0 && errno == EINTR);
828
829 if (ret != 0) {
830 perror(path);
831 return 0;
832 }
833
834 if (fs.f_type != HUGETLBFS_MAGIC)
835 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
836
837 return fs.f_bsize;
838 }
839
840 static void *file_ram_alloc(RAMBlock *block,
841 ram_addr_t memory,
842 const char *path)
843 {
844 char *filename;
845 void *area;
846 int fd;
847 #ifdef MAP_POPULATE
848 int flags;
849 #endif
850 unsigned long hpagesize;
851
852 hpagesize = gethugepagesize(path);
853 if (!hpagesize) {
854 return NULL;
855 }
856
857 if (memory < hpagesize) {
858 return NULL;
859 }
860
861 if (kvm_enabled() && !kvm_has_sync_mmu()) {
862 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
863 return NULL;
864 }
865
866 filename = g_strdup_printf("%s/qemu_back_mem.XXXXXX", path);
867
868 fd = mkstemp(filename);
869 if (fd < 0) {
870 perror("unable to create backing store for hugepages");
871 g_free(filename);
872 return NULL;
873 }
874 unlink(filename);
875 g_free(filename);
876
877 memory = (memory+hpagesize-1) & ~(hpagesize-1);
878
879 /*
880 * ftruncate is not supported by hugetlbfs in older
881 * hosts, so don't bother bailing out on errors.
882 * If anything goes wrong with it under other filesystems,
883 * mmap will fail.
884 */
885 if (ftruncate(fd, memory))
886 perror("ftruncate");
887
888 #ifdef MAP_POPULATE
889 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
890 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
891 * to sidestep this quirk.
892 */
893 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
894 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
895 #else
896 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
897 #endif
898 if (area == MAP_FAILED) {
899 perror("file_ram_alloc: can't mmap RAM pages");
900 close(fd);
901 return (NULL);
902 }
903 block->fd = fd;
904 return area;
905 }
906 #endif
907
908 static ram_addr_t find_ram_offset(ram_addr_t size)
909 {
910 RAMBlock *block, *next_block;
911 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
912
913 if (QTAILQ_EMPTY(&ram_list.blocks))
914 return 0;
915
916 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
917 ram_addr_t end, next = RAM_ADDR_MAX;
918
919 end = block->offset + block->length;
920
921 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
922 if (next_block->offset >= end) {
923 next = MIN(next, next_block->offset);
924 }
925 }
926 if (next - end >= size && next - end < mingap) {
927 offset = end;
928 mingap = next - end;
929 }
930 }
931
932 if (offset == RAM_ADDR_MAX) {
933 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
934 (uint64_t)size);
935 abort();
936 }
937
938 return offset;
939 }
940
941 ram_addr_t last_ram_offset(void)
942 {
943 RAMBlock *block;
944 ram_addr_t last = 0;
945
946 QTAILQ_FOREACH(block, &ram_list.blocks, next)
947 last = MAX(last, block->offset + block->length);
948
949 return last;
950 }
951
952 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
953 {
954 int ret;
955 QemuOpts *machine_opts;
956
957 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
958 machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
959 if (machine_opts &&
960 !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
961 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
962 if (ret) {
963 perror("qemu_madvise");
964 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
965 "but dump_guest_core=off specified\n");
966 }
967 }
968 }
969
970 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
971 {
972 RAMBlock *new_block, *block;
973
974 new_block = NULL;
975 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
976 if (block->offset == addr) {
977 new_block = block;
978 break;
979 }
980 }
981 assert(new_block);
982 assert(!new_block->idstr[0]);
983
984 if (dev) {
985 char *id = qdev_get_dev_path(dev);
986 if (id) {
987 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
988 g_free(id);
989 }
990 }
991 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
992
993 /* This assumes the iothread lock is taken here too. */
994 qemu_mutex_lock_ramlist();
995 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
996 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
997 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
998 new_block->idstr);
999 abort();
1000 }
1001 }
1002 qemu_mutex_unlock_ramlist();
1003 }
1004
1005 static int memory_try_enable_merging(void *addr, size_t len)
1006 {
1007 QemuOpts *opts;
1008
1009 opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1010 if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
1011 /* disabled by the user */
1012 return 0;
1013 }
1014
1015 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1016 }
1017
1018 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1019 MemoryRegion *mr)
1020 {
1021 RAMBlock *block, *new_block;
1022
1023 size = TARGET_PAGE_ALIGN(size);
1024 new_block = g_malloc0(sizeof(*new_block));
1025
1026 /* This assumes the iothread lock is taken here too. */
1027 qemu_mutex_lock_ramlist();
1028 new_block->mr = mr;
1029 new_block->offset = find_ram_offset(size);
1030 if (host) {
1031 new_block->host = host;
1032 new_block->flags |= RAM_PREALLOC_MASK;
1033 } else {
1034 if (mem_path) {
1035 #if defined (__linux__) && !defined(TARGET_S390X)
1036 new_block->host = file_ram_alloc(new_block, size, mem_path);
1037 if (!new_block->host) {
1038 new_block->host = qemu_vmalloc(size);
1039 memory_try_enable_merging(new_block->host, size);
1040 }
1041 #else
1042 fprintf(stderr, "-mem-path option unsupported\n");
1043 exit(1);
1044 #endif
1045 } else {
1046 if (xen_enabled()) {
1047 xen_ram_alloc(new_block->offset, size, mr);
1048 } else if (kvm_enabled()) {
1049 /* some s390/kvm configurations have special constraints */
1050 new_block->host = kvm_vmalloc(size);
1051 } else {
1052 new_block->host = qemu_vmalloc(size);
1053 }
1054 memory_try_enable_merging(new_block->host, size);
1055 }
1056 }
1057 new_block->length = size;
1058
1059 /* Keep the list sorted from biggest to smallest block. */
1060 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1061 if (block->length < new_block->length) {
1062 break;
1063 }
1064 }
1065 if (block) {
1066 QTAILQ_INSERT_BEFORE(block, new_block, next);
1067 } else {
1068 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1069 }
1070 ram_list.mru_block = NULL;
1071
1072 ram_list.version++;
1073 qemu_mutex_unlock_ramlist();
1074
1075 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1076 last_ram_offset() >> TARGET_PAGE_BITS);
1077 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1078 0, size >> TARGET_PAGE_BITS);
1079 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1080
1081 qemu_ram_setup_dump(new_block->host, size);
1082 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1083
1084 if (kvm_enabled())
1085 kvm_setup_guest_memory(new_block->host, size);
1086
1087 return new_block->offset;
1088 }
1089
1090 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1091 {
1092 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1093 }
1094
1095 void qemu_ram_free_from_ptr(ram_addr_t addr)
1096 {
1097 RAMBlock *block;
1098
1099 /* This assumes the iothread lock is taken here too. */
1100 qemu_mutex_lock_ramlist();
1101 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1102 if (addr == block->offset) {
1103 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1104 ram_list.mru_block = NULL;
1105 ram_list.version++;
1106 g_free(block);
1107 break;
1108 }
1109 }
1110 qemu_mutex_unlock_ramlist();
1111 }
1112
1113 void qemu_ram_free(ram_addr_t addr)
1114 {
1115 RAMBlock *block;
1116
1117 /* This assumes the iothread lock is taken here too. */
1118 qemu_mutex_lock_ramlist();
1119 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1120 if (addr == block->offset) {
1121 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1122 ram_list.mru_block = NULL;
1123 ram_list.version++;
1124 if (block->flags & RAM_PREALLOC_MASK) {
1125 ;
1126 } else if (mem_path) {
1127 #if defined (__linux__) && !defined(TARGET_S390X)
1128 if (block->fd) {
1129 munmap(block->host, block->length);
1130 close(block->fd);
1131 } else {
1132 qemu_vfree(block->host);
1133 }
1134 #else
1135 abort();
1136 #endif
1137 } else {
1138 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1139 munmap(block->host, block->length);
1140 #else
1141 if (xen_enabled()) {
1142 xen_invalidate_map_cache_entry(block->host);
1143 } else {
1144 qemu_vfree(block->host);
1145 }
1146 #endif
1147 }
1148 g_free(block);
1149 break;
1150 }
1151 }
1152 qemu_mutex_unlock_ramlist();
1153
1154 }
1155
1156 #ifndef _WIN32
1157 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1158 {
1159 RAMBlock *block;
1160 ram_addr_t offset;
1161 int flags;
1162 void *area, *vaddr;
1163
1164 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1165 offset = addr - block->offset;
1166 if (offset < block->length) {
1167 vaddr = block->host + offset;
1168 if (block->flags & RAM_PREALLOC_MASK) {
1169 ;
1170 } else {
1171 flags = MAP_FIXED;
1172 munmap(vaddr, length);
1173 if (mem_path) {
1174 #if defined(__linux__) && !defined(TARGET_S390X)
1175 if (block->fd) {
1176 #ifdef MAP_POPULATE
1177 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1178 MAP_PRIVATE;
1179 #else
1180 flags |= MAP_PRIVATE;
1181 #endif
1182 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1183 flags, block->fd, offset);
1184 } else {
1185 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1186 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1187 flags, -1, 0);
1188 }
1189 #else
1190 abort();
1191 #endif
1192 } else {
1193 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1194 flags |= MAP_SHARED | MAP_ANONYMOUS;
1195 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1196 flags, -1, 0);
1197 #else
1198 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1199 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1200 flags, -1, 0);
1201 #endif
1202 }
1203 if (area != vaddr) {
1204 fprintf(stderr, "Could not remap addr: "
1205 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1206 length, addr);
1207 exit(1);
1208 }
1209 memory_try_enable_merging(vaddr, length);
1210 qemu_ram_setup_dump(vaddr, length);
1211 }
1212 return;
1213 }
1214 }
1215 }
1216 #endif /* !_WIN32 */
1217
1218 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1219 With the exception of the softmmu code in this file, this should
1220 only be used for local memory (e.g. video ram) that the device owns,
1221 and knows it isn't going to access beyond the end of the block.
1222
1223 It should not be used for general purpose DMA.
1224 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1225 */
1226 void *qemu_get_ram_ptr(ram_addr_t addr)
1227 {
1228 RAMBlock *block;
1229
1230 /* The list is protected by the iothread lock here. */
1231 block = ram_list.mru_block;
1232 if (block && addr - block->offset < block->length) {
1233 goto found;
1234 }
1235 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1236 if (addr - block->offset < block->length) {
1237 goto found;
1238 }
1239 }
1240
1241 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1242 abort();
1243
1244 found:
1245 ram_list.mru_block = block;
1246 if (xen_enabled()) {
1247 /* We need to check if the requested address is in the RAM
1248 * because we don't want to map the entire memory in QEMU.
1249 * In that case just map until the end of the page.
1250 */
1251 if (block->offset == 0) {
1252 return xen_map_cache(addr, 0, 0);
1253 } else if (block->host == NULL) {
1254 block->host =
1255 xen_map_cache(block->offset, block->length, 1);
1256 }
1257 }
1258 return block->host + (addr - block->offset);
1259 }
1260
1261 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1262 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1263 *
1264 * ??? Is this still necessary?
1265 */
1266 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1267 {
1268 RAMBlock *block;
1269
1270 /* The list is protected by the iothread lock here. */
1271 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1272 if (addr - block->offset < block->length) {
1273 if (xen_enabled()) {
1274 /* We need to check if the requested address is in the RAM
1275 * because we don't want to map the entire memory in QEMU.
1276 * In that case just map until the end of the page.
1277 */
1278 if (block->offset == 0) {
1279 return xen_map_cache(addr, 0, 0);
1280 } else if (block->host == NULL) {
1281 block->host =
1282 xen_map_cache(block->offset, block->length, 1);
1283 }
1284 }
1285 return block->host + (addr - block->offset);
1286 }
1287 }
1288
1289 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1290 abort();
1291
1292 return NULL;
1293 }
1294
1295 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1296 * but takes a size argument */
1297 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1298 {
1299 if (*size == 0) {
1300 return NULL;
1301 }
1302 if (xen_enabled()) {
1303 return xen_map_cache(addr, *size, 1);
1304 } else {
1305 RAMBlock *block;
1306
1307 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1308 if (addr - block->offset < block->length) {
1309 if (addr - block->offset + *size > block->length)
1310 *size = block->length - addr + block->offset;
1311 return block->host + (addr - block->offset);
1312 }
1313 }
1314
1315 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1316 abort();
1317 }
1318 }
1319
1320 void qemu_put_ram_ptr(void *addr)
1321 {
1322 trace_qemu_put_ram_ptr(addr);
1323 }
1324
1325 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1326 {
1327 RAMBlock *block;
1328 uint8_t *host = ptr;
1329
1330 if (xen_enabled()) {
1331 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1332 return 0;
1333 }
1334
1335 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1336 /* This case append when the block is not mapped. */
1337 if (block->host == NULL) {
1338 continue;
1339 }
1340 if (host - block->host < block->length) {
1341 *ram_addr = block->offset + (host - block->host);
1342 return 0;
1343 }
1344 }
1345
1346 return -1;
1347 }
1348
1349 /* Some of the softmmu routines need to translate from a host pointer
1350 (typically a TLB entry) back to a ram offset. */
1351 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
1352 {
1353 ram_addr_t ram_addr;
1354
1355 if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
1356 fprintf(stderr, "Bad ram pointer %p\n", ptr);
1357 abort();
1358 }
1359 return ram_addr;
1360 }
1361
1362 static uint64_t unassigned_mem_read(void *opaque, hwaddr addr,
1363 unsigned size)
1364 {
1365 #ifdef DEBUG_UNASSIGNED
1366 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
1367 #endif
1368 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1369 cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, size);
1370 #endif
1371 return 0;
1372 }
1373
1374 static void unassigned_mem_write(void *opaque, hwaddr addr,
1375 uint64_t val, unsigned size)
1376 {
1377 #ifdef DEBUG_UNASSIGNED
1378 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%"PRIx64"\n", addr, val);
1379 #endif
1380 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1381 cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, size);
1382 #endif
1383 }
1384
1385 static const MemoryRegionOps unassigned_mem_ops = {
1386 .read = unassigned_mem_read,
1387 .write = unassigned_mem_write,
1388 .endianness = DEVICE_NATIVE_ENDIAN,
1389 };
1390
1391 static uint64_t error_mem_read(void *opaque, hwaddr addr,
1392 unsigned size)
1393 {
1394 abort();
1395 }
1396
1397 static void error_mem_write(void *opaque, hwaddr addr,
1398 uint64_t value, unsigned size)
1399 {
1400 abort();
1401 }
1402
1403 static const MemoryRegionOps error_mem_ops = {
1404 .read = error_mem_read,
1405 .write = error_mem_write,
1406 .endianness = DEVICE_NATIVE_ENDIAN,
1407 };
1408
1409 static const MemoryRegionOps rom_mem_ops = {
1410 .read = error_mem_read,
1411 .write = unassigned_mem_write,
1412 .endianness = DEVICE_NATIVE_ENDIAN,
1413 };
1414
1415 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1416 uint64_t val, unsigned size)
1417 {
1418 int dirty_flags;
1419 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1420 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1421 #if !defined(CONFIG_USER_ONLY)
1422 tb_invalidate_phys_page_fast(ram_addr, size);
1423 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1424 #endif
1425 }
1426 switch (size) {
1427 case 1:
1428 stb_p(qemu_get_ram_ptr(ram_addr), val);
1429 break;
1430 case 2:
1431 stw_p(qemu_get_ram_ptr(ram_addr), val);
1432 break;
1433 case 4:
1434 stl_p(qemu_get_ram_ptr(ram_addr), val);
1435 break;
1436 default:
1437 abort();
1438 }
1439 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1440 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1441 /* we remove the notdirty callback only if the code has been
1442 flushed */
1443 if (dirty_flags == 0xff)
1444 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1445 }
1446
1447 static const MemoryRegionOps notdirty_mem_ops = {
1448 .read = error_mem_read,
1449 .write = notdirty_mem_write,
1450 .endianness = DEVICE_NATIVE_ENDIAN,
1451 };
1452
1453 /* Generate a debug exception if a watchpoint has been hit. */
1454 static void check_watchpoint(int offset, int len_mask, int flags)
1455 {
1456 CPUArchState *env = cpu_single_env;
1457 target_ulong pc, cs_base;
1458 target_ulong vaddr;
1459 CPUWatchpoint *wp;
1460 int cpu_flags;
1461
1462 if (env->watchpoint_hit) {
1463 /* We re-entered the check after replacing the TB. Now raise
1464 * the debug interrupt so that is will trigger after the
1465 * current instruction. */
1466 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
1467 return;
1468 }
1469 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1470 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1471 if ((vaddr == (wp->vaddr & len_mask) ||
1472 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1473 wp->flags |= BP_WATCHPOINT_HIT;
1474 if (!env->watchpoint_hit) {
1475 env->watchpoint_hit = wp;
1476 tb_check_watchpoint(env);
1477 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1478 env->exception_index = EXCP_DEBUG;
1479 cpu_loop_exit(env);
1480 } else {
1481 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1482 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1483 cpu_resume_from_signal(env, NULL);
1484 }
1485 }
1486 } else {
1487 wp->flags &= ~BP_WATCHPOINT_HIT;
1488 }
1489 }
1490 }
1491
1492 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1493 so these check for a hit then pass through to the normal out-of-line
1494 phys routines. */
1495 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1496 unsigned size)
1497 {
1498 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1499 switch (size) {
1500 case 1: return ldub_phys(addr);
1501 case 2: return lduw_phys(addr);
1502 case 4: return ldl_phys(addr);
1503 default: abort();
1504 }
1505 }
1506
1507 static void watch_mem_write(void *opaque, hwaddr addr,
1508 uint64_t val, unsigned size)
1509 {
1510 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1511 switch (size) {
1512 case 1:
1513 stb_phys(addr, val);
1514 break;
1515 case 2:
1516 stw_phys(addr, val);
1517 break;
1518 case 4:
1519 stl_phys(addr, val);
1520 break;
1521 default: abort();
1522 }
1523 }
1524
1525 static const MemoryRegionOps watch_mem_ops = {
1526 .read = watch_mem_read,
1527 .write = watch_mem_write,
1528 .endianness = DEVICE_NATIVE_ENDIAN,
1529 };
1530
1531 static uint64_t subpage_read(void *opaque, hwaddr addr,
1532 unsigned len)
1533 {
1534 subpage_t *mmio = opaque;
1535 unsigned int idx = SUBPAGE_IDX(addr);
1536 MemoryRegionSection *section;
1537 #if defined(DEBUG_SUBPAGE)
1538 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
1539 mmio, len, addr, idx);
1540 #endif
1541
1542 section = &phys_sections[mmio->sub_section[idx]];
1543 addr += mmio->base;
1544 addr -= section->offset_within_address_space;
1545 addr += section->offset_within_region;
1546 return io_mem_read(section->mr, addr, len);
1547 }
1548
1549 static void subpage_write(void *opaque, hwaddr addr,
1550 uint64_t value, unsigned len)
1551 {
1552 subpage_t *mmio = opaque;
1553 unsigned int idx = SUBPAGE_IDX(addr);
1554 MemoryRegionSection *section;
1555 #if defined(DEBUG_SUBPAGE)
1556 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1557 " idx %d value %"PRIx64"\n",
1558 __func__, mmio, len, addr, idx, value);
1559 #endif
1560
1561 section = &phys_sections[mmio->sub_section[idx]];
1562 addr += mmio->base;
1563 addr -= section->offset_within_address_space;
1564 addr += section->offset_within_region;
1565 io_mem_write(section->mr, addr, value, len);
1566 }
1567
1568 static const MemoryRegionOps subpage_ops = {
1569 .read = subpage_read,
1570 .write = subpage_write,
1571 .endianness = DEVICE_NATIVE_ENDIAN,
1572 };
1573
1574 static uint64_t subpage_ram_read(void *opaque, hwaddr addr,
1575 unsigned size)
1576 {
1577 ram_addr_t raddr = addr;
1578 void *ptr = qemu_get_ram_ptr(raddr);
1579 switch (size) {
1580 case 1: return ldub_p(ptr);
1581 case 2: return lduw_p(ptr);
1582 case 4: return ldl_p(ptr);
1583 default: abort();
1584 }
1585 }
1586
1587 static void subpage_ram_write(void *opaque, hwaddr addr,
1588 uint64_t value, unsigned size)
1589 {
1590 ram_addr_t raddr = addr;
1591 void *ptr = qemu_get_ram_ptr(raddr);
1592 switch (size) {
1593 case 1: return stb_p(ptr, value);
1594 case 2: return stw_p(ptr, value);
1595 case 4: return stl_p(ptr, value);
1596 default: abort();
1597 }
1598 }
1599
1600 static const MemoryRegionOps subpage_ram_ops = {
1601 .read = subpage_ram_read,
1602 .write = subpage_ram_write,
1603 .endianness = DEVICE_NATIVE_ENDIAN,
1604 };
1605
1606 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1607 uint16_t section)
1608 {
1609 int idx, eidx;
1610
1611 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1612 return -1;
1613 idx = SUBPAGE_IDX(start);
1614 eidx = SUBPAGE_IDX(end);
1615 #if defined(DEBUG_SUBPAGE)
1616 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1617 mmio, start, end, idx, eidx, memory);
1618 #endif
1619 if (memory_region_is_ram(phys_sections[section].mr)) {
1620 MemoryRegionSection new_section = phys_sections[section];
1621 new_section.mr = &io_mem_subpage_ram;
1622 section = phys_section_add(&new_section);
1623 }
1624 for (; idx <= eidx; idx++) {
1625 mmio->sub_section[idx] = section;
1626 }
1627
1628 return 0;
1629 }
1630
1631 static subpage_t *subpage_init(hwaddr base)
1632 {
1633 subpage_t *mmio;
1634
1635 mmio = g_malloc0(sizeof(subpage_t));
1636
1637 mmio->base = base;
1638 memory_region_init_io(&mmio->iomem, &subpage_ops, mmio,
1639 "subpage", TARGET_PAGE_SIZE);
1640 mmio->iomem.subpage = true;
1641 #if defined(DEBUG_SUBPAGE)
1642 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1643 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1644 #endif
1645 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, phys_section_unassigned);
1646
1647 return mmio;
1648 }
1649
1650 static uint16_t dummy_section(MemoryRegion *mr)
1651 {
1652 MemoryRegionSection section = {
1653 .mr = mr,
1654 .offset_within_address_space = 0,
1655 .offset_within_region = 0,
1656 .size = UINT64_MAX,
1657 };
1658
1659 return phys_section_add(&section);
1660 }
1661
1662 MemoryRegion *iotlb_to_region(hwaddr index)
1663 {
1664 return phys_sections[index & ~TARGET_PAGE_MASK].mr;
1665 }
1666
1667 static void io_mem_init(void)
1668 {
1669 memory_region_init_io(&io_mem_ram, &error_mem_ops, NULL, "ram", UINT64_MAX);
1670 memory_region_init_io(&io_mem_rom, &rom_mem_ops, NULL, "rom", UINT64_MAX);
1671 memory_region_init_io(&io_mem_unassigned, &unassigned_mem_ops, NULL,
1672 "unassigned", UINT64_MAX);
1673 memory_region_init_io(&io_mem_notdirty, &notdirty_mem_ops, NULL,
1674 "notdirty", UINT64_MAX);
1675 memory_region_init_io(&io_mem_subpage_ram, &subpage_ram_ops, NULL,
1676 "subpage-ram", UINT64_MAX);
1677 memory_region_init_io(&io_mem_watch, &watch_mem_ops, NULL,
1678 "watch", UINT64_MAX);
1679 }
1680
1681 static void mem_begin(MemoryListener *listener)
1682 {
1683 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
1684
1685 destroy_all_mappings(d);
1686 d->phys_map.ptr = PHYS_MAP_NODE_NIL;
1687 }
1688
1689 static void core_begin(MemoryListener *listener)
1690 {
1691 phys_sections_clear();
1692 phys_section_unassigned = dummy_section(&io_mem_unassigned);
1693 phys_section_notdirty = dummy_section(&io_mem_notdirty);
1694 phys_section_rom = dummy_section(&io_mem_rom);
1695 phys_section_watch = dummy_section(&io_mem_watch);
1696 }
1697
1698 static void tcg_commit(MemoryListener *listener)
1699 {
1700 CPUArchState *env;
1701
1702 /* since each CPU stores ram addresses in its TLB cache, we must
1703 reset the modified entries */
1704 /* XXX: slow ! */
1705 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1706 tlb_flush(env, 1);
1707 }
1708 }
1709
1710 static void core_log_global_start(MemoryListener *listener)
1711 {
1712 cpu_physical_memory_set_dirty_tracking(1);
1713 }
1714
1715 static void core_log_global_stop(MemoryListener *listener)
1716 {
1717 cpu_physical_memory_set_dirty_tracking(0);
1718 }
1719
1720 static void io_region_add(MemoryListener *listener,
1721 MemoryRegionSection *section)
1722 {
1723 MemoryRegionIORange *mrio = g_new(MemoryRegionIORange, 1);
1724
1725 mrio->mr = section->mr;
1726 mrio->offset = section->offset_within_region;
1727 iorange_init(&mrio->iorange, &memory_region_iorange_ops,
1728 section->offset_within_address_space, section->size);
1729 ioport_register(&mrio->iorange);
1730 }
1731
1732 static void io_region_del(MemoryListener *listener,
1733 MemoryRegionSection *section)
1734 {
1735 isa_unassign_ioport(section->offset_within_address_space, section->size);
1736 }
1737
1738 static MemoryListener core_memory_listener = {
1739 .begin = core_begin,
1740 .log_global_start = core_log_global_start,
1741 .log_global_stop = core_log_global_stop,
1742 .priority = 1,
1743 };
1744
1745 static MemoryListener io_memory_listener = {
1746 .region_add = io_region_add,
1747 .region_del = io_region_del,
1748 .priority = 0,
1749 };
1750
1751 static MemoryListener tcg_memory_listener = {
1752 .commit = tcg_commit,
1753 };
1754
1755 void address_space_init_dispatch(AddressSpace *as)
1756 {
1757 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1758
1759 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1760 d->listener = (MemoryListener) {
1761 .begin = mem_begin,
1762 .region_add = mem_add,
1763 .region_nop = mem_add,
1764 .priority = 0,
1765 };
1766 as->dispatch = d;
1767 memory_listener_register(&d->listener, as);
1768 }
1769
1770 void address_space_destroy_dispatch(AddressSpace *as)
1771 {
1772 AddressSpaceDispatch *d = as->dispatch;
1773
1774 memory_listener_unregister(&d->listener);
1775 destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
1776 g_free(d);
1777 as->dispatch = NULL;
1778 }
1779
1780 static void memory_map_init(void)
1781 {
1782 system_memory = g_malloc(sizeof(*system_memory));
1783 memory_region_init(system_memory, "system", INT64_MAX);
1784 address_space_init(&address_space_memory, system_memory);
1785 address_space_memory.name = "memory";
1786
1787 system_io = g_malloc(sizeof(*system_io));
1788 memory_region_init(system_io, "io", 65536);
1789 address_space_init(&address_space_io, system_io);
1790 address_space_io.name = "I/O";
1791
1792 memory_listener_register(&core_memory_listener, &address_space_memory);
1793 memory_listener_register(&io_memory_listener, &address_space_io);
1794 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1795
1796 dma_context_init(&dma_context_memory, &address_space_memory,
1797 NULL, NULL, NULL);
1798 }
1799
1800 MemoryRegion *get_system_memory(void)
1801 {
1802 return system_memory;
1803 }
1804
1805 MemoryRegion *get_system_io(void)
1806 {
1807 return system_io;
1808 }
1809
1810 #endif /* !defined(CONFIG_USER_ONLY) */
1811
1812 /* physical memory access (slow version, mainly for debug) */
1813 #if defined(CONFIG_USER_ONLY)
1814 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1815 uint8_t *buf, int len, int is_write)
1816 {
1817 int l, flags;
1818 target_ulong page;
1819 void * p;
1820
1821 while (len > 0) {
1822 page = addr & TARGET_PAGE_MASK;
1823 l = (page + TARGET_PAGE_SIZE) - addr;
1824 if (l > len)
1825 l = len;
1826 flags = page_get_flags(page);
1827 if (!(flags & PAGE_VALID))
1828 return -1;
1829 if (is_write) {
1830 if (!(flags & PAGE_WRITE))
1831 return -1;
1832 /* XXX: this code should not depend on lock_user */
1833 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1834 return -1;
1835 memcpy(p, buf, l);
1836 unlock_user(p, addr, l);
1837 } else {
1838 if (!(flags & PAGE_READ))
1839 return -1;
1840 /* XXX: this code should not depend on lock_user */
1841 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1842 return -1;
1843 memcpy(buf, p, l);
1844 unlock_user(p, addr, 0);
1845 }
1846 len -= l;
1847 buf += l;
1848 addr += l;
1849 }
1850 return 0;
1851 }
1852
1853 #else
1854
1855 static void invalidate_and_set_dirty(hwaddr addr,
1856 hwaddr length)
1857 {
1858 if (!cpu_physical_memory_is_dirty(addr)) {
1859 /* invalidate code */
1860 tb_invalidate_phys_page_range(addr, addr + length, 0);
1861 /* set dirty bit */
1862 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1863 }
1864 xen_modified_memory(addr, length);
1865 }
1866
1867 void address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1868 int len, bool is_write)
1869 {
1870 AddressSpaceDispatch *d = as->dispatch;
1871 int l;
1872 uint8_t *ptr;
1873 uint32_t val;
1874 hwaddr page;
1875 MemoryRegionSection *section;
1876
1877 while (len > 0) {
1878 page = addr & TARGET_PAGE_MASK;
1879 l = (page + TARGET_PAGE_SIZE) - addr;
1880 if (l > len)
1881 l = len;
1882 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
1883
1884 if (is_write) {
1885 if (!memory_region_is_ram(section->mr)) {
1886 hwaddr addr1;
1887 addr1 = memory_region_section_addr(section, addr);
1888 /* XXX: could force cpu_single_env to NULL to avoid
1889 potential bugs */
1890 if (l >= 4 && ((addr1 & 3) == 0)) {
1891 /* 32 bit write access */
1892 val = ldl_p(buf);
1893 io_mem_write(section->mr, addr1, val, 4);
1894 l = 4;
1895 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1896 /* 16 bit write access */
1897 val = lduw_p(buf);
1898 io_mem_write(section->mr, addr1, val, 2);
1899 l = 2;
1900 } else {
1901 /* 8 bit write access */
1902 val = ldub_p(buf);
1903 io_mem_write(section->mr, addr1, val, 1);
1904 l = 1;
1905 }
1906 } else if (!section->readonly) {
1907 ram_addr_t addr1;
1908 addr1 = memory_region_get_ram_addr(section->mr)
1909 + memory_region_section_addr(section, addr);
1910 /* RAM case */
1911 ptr = qemu_get_ram_ptr(addr1);
1912 memcpy(ptr, buf, l);
1913 invalidate_and_set_dirty(addr1, l);
1914 qemu_put_ram_ptr(ptr);
1915 }
1916 } else {
1917 if (!(memory_region_is_ram(section->mr) ||
1918 memory_region_is_romd(section->mr))) {
1919 hwaddr addr1;
1920 /* I/O case */
1921 addr1 = memory_region_section_addr(section, addr);
1922 if (l >= 4 && ((addr1 & 3) == 0)) {
1923 /* 32 bit read access */
1924 val = io_mem_read(section->mr, addr1, 4);
1925 stl_p(buf, val);
1926 l = 4;
1927 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1928 /* 16 bit read access */
1929 val = io_mem_read(section->mr, addr1, 2);
1930 stw_p(buf, val);
1931 l = 2;
1932 } else {
1933 /* 8 bit read access */
1934 val = io_mem_read(section->mr, addr1, 1);
1935 stb_p(buf, val);
1936 l = 1;
1937 }
1938 } else {
1939 /* RAM case */
1940 ptr = qemu_get_ram_ptr(section->mr->ram_addr
1941 + memory_region_section_addr(section,
1942 addr));
1943 memcpy(buf, ptr, l);
1944 qemu_put_ram_ptr(ptr);
1945 }
1946 }
1947 len -= l;
1948 buf += l;
1949 addr += l;
1950 }
1951 }
1952
1953 void address_space_write(AddressSpace *as, hwaddr addr,
1954 const uint8_t *buf, int len)
1955 {
1956 address_space_rw(as, addr, (uint8_t *)buf, len, true);
1957 }
1958
1959 /**
1960 * address_space_read: read from an address space.
1961 *
1962 * @as: #AddressSpace to be accessed
1963 * @addr: address within that address space
1964 * @buf: buffer with the data transferred
1965 */
1966 void address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1967 {
1968 address_space_rw(as, addr, buf, len, false);
1969 }
1970
1971
1972 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1973 int len, int is_write)
1974 {
1975 return address_space_rw(&address_space_memory, addr, buf, len, is_write);
1976 }
1977
1978 /* used for ROM loading : can write in RAM and ROM */
1979 void cpu_physical_memory_write_rom(hwaddr addr,
1980 const uint8_t *buf, int len)
1981 {
1982 AddressSpaceDispatch *d = address_space_memory.dispatch;
1983 int l;
1984 uint8_t *ptr;
1985 hwaddr page;
1986 MemoryRegionSection *section;
1987
1988 while (len > 0) {
1989 page = addr & TARGET_PAGE_MASK;
1990 l = (page + TARGET_PAGE_SIZE) - addr;
1991 if (l > len)
1992 l = len;
1993 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
1994
1995 if (!(memory_region_is_ram(section->mr) ||
1996 memory_region_is_romd(section->mr))) {
1997 /* do nothing */
1998 } else {
1999 unsigned long addr1;
2000 addr1 = memory_region_get_ram_addr(section->mr)
2001 + memory_region_section_addr(section, addr);
2002 /* ROM/RAM case */
2003 ptr = qemu_get_ram_ptr(addr1);
2004 memcpy(ptr, buf, l);
2005 invalidate_and_set_dirty(addr1, l);
2006 qemu_put_ram_ptr(ptr);
2007 }
2008 len -= l;
2009 buf += l;
2010 addr += l;
2011 }
2012 }
2013
2014 typedef struct {
2015 void *buffer;
2016 hwaddr addr;
2017 hwaddr len;
2018 } BounceBuffer;
2019
2020 static BounceBuffer bounce;
2021
2022 typedef struct MapClient {
2023 void *opaque;
2024 void (*callback)(void *opaque);
2025 QLIST_ENTRY(MapClient) link;
2026 } MapClient;
2027
2028 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2029 = QLIST_HEAD_INITIALIZER(map_client_list);
2030
2031 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2032 {
2033 MapClient *client = g_malloc(sizeof(*client));
2034
2035 client->opaque = opaque;
2036 client->callback = callback;
2037 QLIST_INSERT_HEAD(&map_client_list, client, link);
2038 return client;
2039 }
2040
2041 static void cpu_unregister_map_client(void *_client)
2042 {
2043 MapClient *client = (MapClient *)_client;
2044
2045 QLIST_REMOVE(client, link);
2046 g_free(client);
2047 }
2048
2049 static void cpu_notify_map_clients(void)
2050 {
2051 MapClient *client;
2052
2053 while (!QLIST_EMPTY(&map_client_list)) {
2054 client = QLIST_FIRST(&map_client_list);
2055 client->callback(client->opaque);
2056 cpu_unregister_map_client(client);
2057 }
2058 }
2059
2060 /* Map a physical memory region into a host virtual address.
2061 * May map a subset of the requested range, given by and returned in *plen.
2062 * May return NULL if resources needed to perform the mapping are exhausted.
2063 * Use only for reads OR writes - not for read-modify-write operations.
2064 * Use cpu_register_map_client() to know when retrying the map operation is
2065 * likely to succeed.
2066 */
2067 void *address_space_map(AddressSpace *as,
2068 hwaddr addr,
2069 hwaddr *plen,
2070 bool is_write)
2071 {
2072 AddressSpaceDispatch *d = as->dispatch;
2073 hwaddr len = *plen;
2074 hwaddr todo = 0;
2075 int l;
2076 hwaddr page;
2077 MemoryRegionSection *section;
2078 ram_addr_t raddr = RAM_ADDR_MAX;
2079 ram_addr_t rlen;
2080 void *ret;
2081
2082 while (len > 0) {
2083 page = addr & TARGET_PAGE_MASK;
2084 l = (page + TARGET_PAGE_SIZE) - addr;
2085 if (l > len)
2086 l = len;
2087 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
2088
2089 if (!(memory_region_is_ram(section->mr) && !section->readonly)) {
2090 if (todo || bounce.buffer) {
2091 break;
2092 }
2093 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2094 bounce.addr = addr;
2095 bounce.len = l;
2096 if (!is_write) {
2097 address_space_read(as, addr, bounce.buffer, l);
2098 }
2099
2100 *plen = l;
2101 return bounce.buffer;
2102 }
2103 if (!todo) {
2104 raddr = memory_region_get_ram_addr(section->mr)
2105 + memory_region_section_addr(section, addr);
2106 }
2107
2108 len -= l;
2109 addr += l;
2110 todo += l;
2111 }
2112 rlen = todo;
2113 ret = qemu_ram_ptr_length(raddr, &rlen);
2114 *plen = rlen;
2115 return ret;
2116 }
2117
2118 /* Unmaps a memory region previously mapped by address_space_map().
2119 * Will also mark the memory as dirty if is_write == 1. access_len gives
2120 * the amount of memory that was actually read or written by the caller.
2121 */
2122 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2123 int is_write, hwaddr access_len)
2124 {
2125 if (buffer != bounce.buffer) {
2126 if (is_write) {
2127 ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
2128 while (access_len) {
2129 unsigned l;
2130 l = TARGET_PAGE_SIZE;
2131 if (l > access_len)
2132 l = access_len;
2133 invalidate_and_set_dirty(addr1, l);
2134 addr1 += l;
2135 access_len -= l;
2136 }
2137 }
2138 if (xen_enabled()) {
2139 xen_invalidate_map_cache_entry(buffer);
2140 }
2141 return;
2142 }
2143 if (is_write) {
2144 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2145 }
2146 qemu_vfree(bounce.buffer);
2147 bounce.buffer = NULL;
2148 cpu_notify_map_clients();
2149 }
2150
2151 void *cpu_physical_memory_map(hwaddr addr,
2152 hwaddr *plen,
2153 int is_write)
2154 {
2155 return address_space_map(&address_space_memory, addr, plen, is_write);
2156 }
2157
2158 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2159 int is_write, hwaddr access_len)
2160 {
2161 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2162 }
2163
2164 /* warning: addr must be aligned */
2165 static inline uint32_t ldl_phys_internal(hwaddr addr,
2166 enum device_endian endian)
2167 {
2168 uint8_t *ptr;
2169 uint32_t val;
2170 MemoryRegionSection *section;
2171
2172 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2173
2174 if (!(memory_region_is_ram(section->mr) ||
2175 memory_region_is_romd(section->mr))) {
2176 /* I/O case */
2177 addr = memory_region_section_addr(section, addr);
2178 val = io_mem_read(section->mr, addr, 4);
2179 #if defined(TARGET_WORDS_BIGENDIAN)
2180 if (endian == DEVICE_LITTLE_ENDIAN) {
2181 val = bswap32(val);
2182 }
2183 #else
2184 if (endian == DEVICE_BIG_ENDIAN) {
2185 val = bswap32(val);
2186 }
2187 #endif
2188 } else {
2189 /* RAM case */
2190 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2191 & TARGET_PAGE_MASK)
2192 + memory_region_section_addr(section, addr));
2193 switch (endian) {
2194 case DEVICE_LITTLE_ENDIAN:
2195 val = ldl_le_p(ptr);
2196 break;
2197 case DEVICE_BIG_ENDIAN:
2198 val = ldl_be_p(ptr);
2199 break;
2200 default:
2201 val = ldl_p(ptr);
2202 break;
2203 }
2204 }
2205 return val;
2206 }
2207
2208 uint32_t ldl_phys(hwaddr addr)
2209 {
2210 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2211 }
2212
2213 uint32_t ldl_le_phys(hwaddr addr)
2214 {
2215 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2216 }
2217
2218 uint32_t ldl_be_phys(hwaddr addr)
2219 {
2220 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2221 }
2222
2223 /* warning: addr must be aligned */
2224 static inline uint64_t ldq_phys_internal(hwaddr addr,
2225 enum device_endian endian)
2226 {
2227 uint8_t *ptr;
2228 uint64_t val;
2229 MemoryRegionSection *section;
2230
2231 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2232
2233 if (!(memory_region_is_ram(section->mr) ||
2234 memory_region_is_romd(section->mr))) {
2235 /* I/O case */
2236 addr = memory_region_section_addr(section, addr);
2237
2238 /* XXX This is broken when device endian != cpu endian.
2239 Fix and add "endian" variable check */
2240 #ifdef TARGET_WORDS_BIGENDIAN
2241 val = io_mem_read(section->mr, addr, 4) << 32;
2242 val |= io_mem_read(section->mr, addr + 4, 4);
2243 #else
2244 val = io_mem_read(section->mr, addr, 4);
2245 val |= io_mem_read(section->mr, addr + 4, 4) << 32;
2246 #endif
2247 } else {
2248 /* RAM case */
2249 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2250 & TARGET_PAGE_MASK)
2251 + memory_region_section_addr(section, addr));
2252 switch (endian) {
2253 case DEVICE_LITTLE_ENDIAN:
2254 val = ldq_le_p(ptr);
2255 break;
2256 case DEVICE_BIG_ENDIAN:
2257 val = ldq_be_p(ptr);
2258 break;
2259 default:
2260 val = ldq_p(ptr);
2261 break;
2262 }
2263 }
2264 return val;
2265 }
2266
2267 uint64_t ldq_phys(hwaddr addr)
2268 {
2269 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2270 }
2271
2272 uint64_t ldq_le_phys(hwaddr addr)
2273 {
2274 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2275 }
2276
2277 uint64_t ldq_be_phys(hwaddr addr)
2278 {
2279 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2280 }
2281
2282 /* XXX: optimize */
2283 uint32_t ldub_phys(hwaddr addr)
2284 {
2285 uint8_t val;
2286 cpu_physical_memory_read(addr, &val, 1);
2287 return val;
2288 }
2289
2290 /* warning: addr must be aligned */
2291 static inline uint32_t lduw_phys_internal(hwaddr addr,
2292 enum device_endian endian)
2293 {
2294 uint8_t *ptr;
2295 uint64_t val;
2296 MemoryRegionSection *section;
2297
2298 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2299
2300 if (!(memory_region_is_ram(section->mr) ||
2301 memory_region_is_romd(section->mr))) {
2302 /* I/O case */
2303 addr = memory_region_section_addr(section, addr);
2304 val = io_mem_read(section->mr, addr, 2);
2305 #if defined(TARGET_WORDS_BIGENDIAN)
2306 if (endian == DEVICE_LITTLE_ENDIAN) {
2307 val = bswap16(val);
2308 }
2309 #else
2310 if (endian == DEVICE_BIG_ENDIAN) {
2311 val = bswap16(val);
2312 }
2313 #endif
2314 } else {
2315 /* RAM case */
2316 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2317 & TARGET_PAGE_MASK)
2318 + memory_region_section_addr(section, addr));
2319 switch (endian) {
2320 case DEVICE_LITTLE_ENDIAN:
2321 val = lduw_le_p(ptr);
2322 break;
2323 case DEVICE_BIG_ENDIAN:
2324 val = lduw_be_p(ptr);
2325 break;
2326 default:
2327 val = lduw_p(ptr);
2328 break;
2329 }
2330 }
2331 return val;
2332 }
2333
2334 uint32_t lduw_phys(hwaddr addr)
2335 {
2336 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2337 }
2338
2339 uint32_t lduw_le_phys(hwaddr addr)
2340 {
2341 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2342 }
2343
2344 uint32_t lduw_be_phys(hwaddr addr)
2345 {
2346 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2347 }
2348
2349 /* warning: addr must be aligned. The ram page is not masked as dirty
2350 and the code inside is not invalidated. It is useful if the dirty
2351 bits are used to track modified PTEs */
2352 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2353 {
2354 uint8_t *ptr;
2355 MemoryRegionSection *section;
2356
2357 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2358
2359 if (!memory_region_is_ram(section->mr) || section->readonly) {
2360 addr = memory_region_section_addr(section, addr);
2361 if (memory_region_is_ram(section->mr)) {
2362 section = &phys_sections[phys_section_rom];
2363 }
2364 io_mem_write(section->mr, addr, val, 4);
2365 } else {
2366 unsigned long addr1 = (memory_region_get_ram_addr(section->mr)
2367 & TARGET_PAGE_MASK)
2368 + memory_region_section_addr(section, addr);
2369 ptr = qemu_get_ram_ptr(addr1);
2370 stl_p(ptr, val);
2371
2372 if (unlikely(in_migration)) {
2373 if (!cpu_physical_memory_is_dirty(addr1)) {
2374 /* invalidate code */
2375 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2376 /* set dirty bit */
2377 cpu_physical_memory_set_dirty_flags(
2378 addr1, (0xff & ~CODE_DIRTY_FLAG));
2379 }
2380 }
2381 }
2382 }
2383
2384 void stq_phys_notdirty(hwaddr addr, uint64_t val)
2385 {
2386 uint8_t *ptr;
2387 MemoryRegionSection *section;
2388
2389 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2390
2391 if (!memory_region_is_ram(section->mr) || section->readonly) {
2392 addr = memory_region_section_addr(section, addr);
2393 if (memory_region_is_ram(section->mr)) {
2394 section = &phys_sections[phys_section_rom];
2395 }
2396 #ifdef TARGET_WORDS_BIGENDIAN
2397 io_mem_write(section->mr, addr, val >> 32, 4);
2398 io_mem_write(section->mr, addr + 4, (uint32_t)val, 4);
2399 #else
2400 io_mem_write(section->mr, addr, (uint32_t)val, 4);
2401 io_mem_write(section->mr, addr + 4, val >> 32, 4);
2402 #endif
2403 } else {
2404 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2405 & TARGET_PAGE_MASK)
2406 + memory_region_section_addr(section, addr));
2407 stq_p(ptr, val);
2408 }
2409 }
2410
2411 /* warning: addr must be aligned */
2412 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2413 enum device_endian endian)
2414 {
2415 uint8_t *ptr;
2416 MemoryRegionSection *section;
2417
2418 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2419
2420 if (!memory_region_is_ram(section->mr) || section->readonly) {
2421 addr = memory_region_section_addr(section, addr);
2422 if (memory_region_is_ram(section->mr)) {
2423 section = &phys_sections[phys_section_rom];
2424 }
2425 #if defined(TARGET_WORDS_BIGENDIAN)
2426 if (endian == DEVICE_LITTLE_ENDIAN) {
2427 val = bswap32(val);
2428 }
2429 #else
2430 if (endian == DEVICE_BIG_ENDIAN) {
2431 val = bswap32(val);
2432 }
2433 #endif
2434 io_mem_write(section->mr, addr, val, 4);
2435 } else {
2436 unsigned long addr1;
2437 addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2438 + memory_region_section_addr(section, addr);
2439 /* RAM case */
2440 ptr = qemu_get_ram_ptr(addr1);
2441 switch (endian) {
2442 case DEVICE_LITTLE_ENDIAN:
2443 stl_le_p(ptr, val);
2444 break;
2445 case DEVICE_BIG_ENDIAN:
2446 stl_be_p(ptr, val);
2447 break;
2448 default:
2449 stl_p(ptr, val);
2450 break;
2451 }
2452 invalidate_and_set_dirty(addr1, 4);
2453 }
2454 }
2455
2456 void stl_phys(hwaddr addr, uint32_t val)
2457 {
2458 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2459 }
2460
2461 void stl_le_phys(hwaddr addr, uint32_t val)
2462 {
2463 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2464 }
2465
2466 void stl_be_phys(hwaddr addr, uint32_t val)
2467 {
2468 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2469 }
2470
2471 /* XXX: optimize */
2472 void stb_phys(hwaddr addr, uint32_t val)
2473 {
2474 uint8_t v = val;
2475 cpu_physical_memory_write(addr, &v, 1);
2476 }
2477
2478 /* warning: addr must be aligned */
2479 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2480 enum device_endian endian)
2481 {
2482 uint8_t *ptr;
2483 MemoryRegionSection *section;
2484
2485 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2486
2487 if (!memory_region_is_ram(section->mr) || section->readonly) {
2488 addr = memory_region_section_addr(section, addr);
2489 if (memory_region_is_ram(section->mr)) {
2490 section = &phys_sections[phys_section_rom];
2491 }
2492 #if defined(TARGET_WORDS_BIGENDIAN)
2493 if (endian == DEVICE_LITTLE_ENDIAN) {
2494 val = bswap16(val);
2495 }
2496 #else
2497 if (endian == DEVICE_BIG_ENDIAN) {
2498 val = bswap16(val);
2499 }
2500 #endif
2501 io_mem_write(section->mr, addr, val, 2);
2502 } else {
2503 unsigned long addr1;
2504 addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2505 + memory_region_section_addr(section, addr);
2506 /* RAM case */
2507 ptr = qemu_get_ram_ptr(addr1);
2508 switch (endian) {
2509 case DEVICE_LITTLE_ENDIAN:
2510 stw_le_p(ptr, val);
2511 break;
2512 case DEVICE_BIG_ENDIAN:
2513 stw_be_p(ptr, val);
2514 break;
2515 default:
2516 stw_p(ptr, val);
2517 break;
2518 }
2519 invalidate_and_set_dirty(addr1, 2);
2520 }
2521 }
2522
2523 void stw_phys(hwaddr addr, uint32_t val)
2524 {
2525 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2526 }
2527
2528 void stw_le_phys(hwaddr addr, uint32_t val)
2529 {
2530 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2531 }
2532
2533 void stw_be_phys(hwaddr addr, uint32_t val)
2534 {
2535 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2536 }
2537
2538 /* XXX: optimize */
2539 void stq_phys(hwaddr addr, uint64_t val)
2540 {
2541 val = tswap64(val);
2542 cpu_physical_memory_write(addr, &val, 8);
2543 }
2544
2545 void stq_le_phys(hwaddr addr, uint64_t val)
2546 {
2547 val = cpu_to_le64(val);
2548 cpu_physical_memory_write(addr, &val, 8);
2549 }
2550
2551 void stq_be_phys(hwaddr addr, uint64_t val)
2552 {
2553 val = cpu_to_be64(val);
2554 cpu_physical_memory_write(addr, &val, 8);
2555 }
2556
2557 /* virtual memory access for debug (includes writing to ROM) */
2558 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2559 uint8_t *buf, int len, int is_write)
2560 {
2561 int l;
2562 hwaddr phys_addr;
2563 target_ulong page;
2564
2565 while (len > 0) {
2566 page = addr & TARGET_PAGE_MASK;
2567 phys_addr = cpu_get_phys_page_debug(env, page);
2568 /* if no physical page mapped, return an error */
2569 if (phys_addr == -1)
2570 return -1;
2571 l = (page + TARGET_PAGE_SIZE) - addr;
2572 if (l > len)
2573 l = len;
2574 phys_addr += (addr & ~TARGET_PAGE_MASK);
2575 if (is_write)
2576 cpu_physical_memory_write_rom(phys_addr, buf, l);
2577 else
2578 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2579 len -= l;
2580 buf += l;
2581 addr += l;
2582 }
2583 return 0;
2584 }
2585 #endif
2586
2587 #if !defined(CONFIG_USER_ONLY)
2588
2589 /*
2590 * A helper function for the _utterly broken_ virtio device model to find out if
2591 * it's running on a big endian machine. Don't do this at home kids!
2592 */
2593 bool virtio_is_big_endian(void);
2594 bool virtio_is_big_endian(void)
2595 {
2596 #if defined(TARGET_WORDS_BIGENDIAN)
2597 return true;
2598 #else
2599 return false;
2600 #endif
2601 }
2602
2603 #endif
2604
2605 #ifndef CONFIG_USER_ONLY
2606 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2607 {
2608 MemoryRegionSection *section;
2609
2610 section = phys_page_find(address_space_memory.dispatch,
2611 phys_addr >> TARGET_PAGE_BITS);
2612
2613 return !(memory_region_is_ram(section->mr) ||
2614 memory_region_is_romd(section->mr));
2615 }
2616 #endif
Random patches